enigma-bbs/core/file_base_area.js

1057 lines
38 KiB
JavaScript
Raw Normal View History

/* jslint node: true */
'use strict';
// ENiGMA½
const Config = require('./config.js').get;
const Errors = require('./enig_error.js').Errors;
const sortAreasOrConfs = require('./conf_area_util.js').sortAreasOrConfs;
const FileEntry = require('./file_entry.js');
const FileDb = require('./database.js').dbs.file;
const ArchiveUtil = require('./archive_util.js');
const CRC32 = require('./crc.js').CRC32;
const Log = require('./logger.js').log;
const resolveMimeType = require('./mime_util.js').resolveMimeType;
const stringFormat = require('./string_format.js');
const wordWrapText = require('./word_wrap.js').wordWrapText;
const StatLog = require('./stat_log.js');
// deps
const _ = require('lodash');
const async = require('async');
const fs = require('graceful-fs');
const crypto = require('crypto');
const paths = require('path');
const temptmp = require('temptmp').createTrackedSession('file_area');
const iconv = require('iconv-lite');
const execFile = require('child_process').execFile;
const moment = require('moment');
exports.startup = startup;
exports.isInternalArea = isInternalArea;
exports.getAvailableFileAreas = getAvailableFileAreas;
exports.getAvailableFileAreaTags = getAvailableFileAreaTags;
exports.getSortedAvailableFileAreas = getSortedAvailableFileAreas;
exports.isValidStorageTag = isValidStorageTag;
exports.getAreaStorageDirectoryByTag = getAreaStorageDirectoryByTag;
exports.getAreaDefaultStorageDirectory = getAreaDefaultStorageDirectory;
exports.getAreaStorageLocations = getAreaStorageLocations;
exports.getDefaultFileAreaTag = getDefaultFileAreaTag;
exports.getFileAreaByTag = getFileAreaByTag;
exports.getFileEntryPath = getFileEntryPath;
exports.changeFileAreaWithOptions = changeFileAreaWithOptions;
exports.scanFile = scanFile;
exports.scanFileAreaForChanges = scanFileAreaForChanges;
exports.getDescFromFileName = getDescFromFileName;
exports.getAreaStats = getAreaStats;
exports.cleanUpTempSessionItems = cleanUpTempSessionItems;
// for scheduler:
exports.updateAreaStatsScheduledEvent = updateAreaStatsScheduledEvent;
const WellKnownAreaTags = exports.WellKnownAreaTags = {
Invalid : '',
MessageAreaAttach : 'system_message_attachment',
TempDownloads : 'system_temporary_download',
2016-10-01 19:25:32 +00:00
};
function startup(cb) {
return cleanUpTempSessionItems(cb);
}
function isInternalArea(areaTag) {
return [ WellKnownAreaTags.MessageAreaAttach, WellKnownAreaTags.TempDownloads ].includes(areaTag);
}
function getAvailableFileAreas(client, options) {
options = options || { };
// perform ACS check per conf & omit internal if desired
const allAreas = _.map(Config().fileBase.areas, (areaInfo, areaTag) => Object.assign(areaInfo, { areaTag : areaTag } ));
return _.omitBy(allAreas, areaInfo => {
if(!options.includeSystemInternal && isInternalArea(areaInfo.areaTag)) {
return true;
}
if(options.skipAcsCheck) {
return false; // no ACS checks (below)
}
if(options.writeAcs && !client.acs.hasFileAreaWrite(areaInfo)) {
return true; // omit
}
return !client.acs.hasFileAreaRead(areaInfo);
});
}
function getAvailableFileAreaTags(client, options) {
return _.map(getAvailableFileAreas(client, options), area => area.areaTag);
}
2016-10-01 19:25:32 +00:00
function getSortedAvailableFileAreas(client, options) {
const areas = _.map(getAvailableFileAreas(client, options), v => v);
sortAreasOrConfs(areas);
return areas;
2016-10-01 19:25:32 +00:00
}
function getDefaultFileAreaTag(client, disableAcsCheck) {
const config = Config();
let defaultArea = _.findKey(config.fileBase, o => o.default);
if(defaultArea) {
const area = config.fileBase.areas[defaultArea];
if(true === disableAcsCheck || client.acs.hasFileAreaRead(area)) {
return defaultArea;
}
}
// just use anything we can
defaultArea = _.findKey(config.fileBase.areas, (area, areaTag) => {
return WellKnownAreaTags.MessageAreaAttach !== areaTag && (true === disableAcsCheck || client.acs.hasFileAreaRead(area));
});
return defaultArea;
2016-10-01 19:25:32 +00:00
}
function getFileAreaByTag(areaTag) {
const areaInfo = Config().fileBase.areas[areaTag];
if(areaInfo) {
areaInfo.areaTag = areaTag; // convienence!
areaInfo.storage = getAreaStorageLocations(areaInfo);
return areaInfo;
}
2016-10-01 19:25:32 +00:00
}
function changeFileAreaWithOptions(client, areaTag, options, cb) {
async.waterfall(
[
function getArea(callback) {
const area = getFileAreaByTag(areaTag);
return callback(area ? null : Errors.Invalid('Invalid file areaTag'), area);
},
function validateAccess(area, callback) {
if(!client.acs.hasFileAreaRead(area)) {
return callback(Errors.AccessDenied('No access to this area'));
}
},
function changeArea(area, callback) {
if(true === options.persist) {
client.user.persistProperty('file_area_tag', areaTag, err => {
return callback(err, area);
});
} else {
client.user.properties['file_area_tag'] = areaTag;
return callback(null, area);
}
}
],
(err, area) => {
if(!err) {
client.log.info( { areaTag : areaTag, area : area }, 'Current file area changed');
} else {
client.log.warn( { areaTag : areaTag, area : area, error : err.message }, 'Could not change file area');
}
return cb(err);
}
);
2016-10-01 19:25:32 +00:00
}
function isValidStorageTag(storageTag) {
return storageTag in Config().fileBase.storageTags;
}
function getAreaStorageDirectoryByTag(storageTag) {
const config = Config();
const storageLocation = (storageTag && config.fileBase.storageTags[storageTag]);
return paths.resolve(config.fileBase.areaStoragePrefix, storageLocation || '');
}
function getAreaDefaultStorageDirectory(areaInfo) {
return getAreaStorageDirectoryByTag(areaInfo.storageTags[0]);
}
function getAreaStorageLocations(areaInfo) {
const storageTags = Array.isArray(areaInfo.storageTags) ?
areaInfo.storageTags :
[ areaInfo.storageTags || '' ];
const avail = Config().fileBase.storageTags;
return _.compact(storageTags.map(storageTag => {
if(avail[storageTag]) {
return {
storageTag : storageTag,
dir : getAreaStorageDirectoryByTag(storageTag),
};
}
}));
}
function getFileEntryPath(fileEntry) {
const areaInfo = getFileAreaByTag(fileEntry.areaTag);
if(areaInfo) {
return paths.join(areaInfo.storageDirectory, fileEntry.fileName);
}
}
function getExistingFileEntriesBySha256(sha256, cb) {
const entries = [];
FileDb.each(
`SELECT file_id, area_tag
FROM file
WHERE file_sha256=?;`,
[ sha256 ],
(err, fileRow) => {
if(fileRow) {
entries.push({
fileId : fileRow.file_id,
areaTag : fileRow.area_tag,
});
}
},
err => {
return cb(err, entries);
}
);
}
// :TODO: This is bascially sliceAtEOF() from art.js .... DRY!
function sliceAtSauceMarker(data) {
let eof = data.length;
const stopPos = Math.max(data.length - (256), 0); // 256 = 2 * sizeof(SAUCE)
for(let i = eof - 1; i > stopPos; i--) {
if(0x1a === data[i]) {
eof = i;
break;
}
}
return data.slice(0, eof);
}
function attemptSetEstimatedReleaseDate(fileEntry) {
// :TODO: yearEstPatterns RegExp's should be cached - we can do this @ Config (re)load time
const patterns = Config().fileBase.yearEstPatterns.map( p => new RegExp(p, 'gmi'));
function getMatch(input) {
if(input) {
let m;
for(let i = 0; i < patterns.length; ++i) {
m = patterns[i].exec(input);
if(m) {
return m;
}
}
}
}
//
// We attempt detection in short -> long order
//
// Throw out anything that is current_year + 2 (we give some leway)
// with the assumption that must be wrong.
//
const maxYear = moment().add(2, 'year').year();
const match = getMatch(fileEntry.desc) || getMatch(fileEntry.descLong);
if(match && match[1]) {
let year;
if(2 === match[1].length) {
year = parseInt(match[1]);
if(year) {
if(year > 70) {
year += 1900;
} else {
year += 2000;
}
}
} else {
year = parseInt(match[1]);
}
if(year && year <= maxYear) {
fileEntry.meta.est_release_year = year;
}
}
}
// a simple log proxy for when we call from oputil.js
2018-09-11 20:44:22 +00:00
const maybeLog = (obj, msg, level) => {
if(Log) {
2018-09-11 20:44:22 +00:00
Log[level](obj, msg);
} else if ('error' === level) {
console.error(`${msg}: ${JSON.stringify(obj)}`); // eslint-disable-line no-console
}
2018-09-11 20:44:22 +00:00
};
2018-09-11 20:44:22 +00:00
const logDebug = (obj, msg) => maybeLog(obj, msg, 'debug');
const logTrace = (obj, msg) => maybeLog(obj, msg, 'trace');
const logError = (obj, msg) => maybeLog(obj, msg, 'error');
2018-06-26 01:09:08 +00:00
function extractAndProcessDescFiles(fileEntry, filePath, archiveEntries, cb) {
async.waterfall(
[
function extractDescFiles(callback) {
// :TODO: would be nice if these RegExp's were cached
// :TODO: this is long winded...
const config = Config();
const extractList = [];
const shortDescFile = archiveEntries.find( e => {
return config.fileBase.fileNamePatterns.desc.find( pat => new RegExp(pat, 'i').test(e.fileName) );
});
if(shortDescFile) {
extractList.push(shortDescFile.fileName);
}
const longDescFile = archiveEntries.find( e => {
return config.fileBase.fileNamePatterns.descLong.find( pat => new RegExp(pat, 'i').test(e.fileName) );
});
if(longDescFile) {
extractList.push(longDescFile.fileName);
}
if(0 === extractList.length) {
return callback(null, [] );
}
temptmp.mkdir( { prefix : 'enigextract-' }, (err, tempDir) => {
if(err) {
return callback(err);
}
const archiveUtil = ArchiveUtil.getInstance();
archiveUtil.extractTo(filePath, tempDir, fileEntry.meta.archive_type, extractList, err => {
if(err) {
return callback(err);
}
const descFiles = {
desc : shortDescFile ? paths.join(tempDir, shortDescFile.fileName) : null,
descLong : longDescFile ? paths.join(tempDir, longDescFile.fileName) : null,
};
return callback(null, descFiles);
});
});
},
function readDescFiles(descFiles, callback) {
const config = Config();
async.each(Object.keys(descFiles), (descType, next) => {
const path = descFiles[descType];
if(!path) {
return next(null);
}
fs.stat(path, (err, stats) => {
if(err) {
return next(null);
}
// skip entries that are too large
const maxFileSizeKey = `max${_.upperFirst(descType)}FileByteSize`;
if(config.fileBase[maxFileSizeKey] && stats.size > config.fileBase[maxFileSizeKey]) {
logDebug( { byteSize : stats.size, maxByteSize : config.fileBase[maxFileSizeKey] }, `Skipping "${descType}"; Too large` );
return next(null);
}
fs.readFile(path, (err, data) => {
if(err || !data) {
return next(null);
}
//
// Assume FILE_ID.DIZ, NFO files, etc. are CP437.
//
// :TODO: This isn't really always the case - how to handle this? We could do a quick detection...
fileEntry[descType] = iconv.decode(sliceAtSauceMarker(data, 0x1a), 'cp437');
fileEntry[`${descType}Src`] = 'descFile';
return next(null);
});
});
}, () => {
// cleanup but don't wait
temptmp.cleanup( paths => {
// note: don't use client logger here - may not be avail
2018-09-11 20:44:22 +00:00
logTrace( { paths : paths, sessionId : temptmp.sessionId }, 'Cleaned up temporary files' );
});
return callback(null);
});
},
],
err => {
return cb(err);
}
);
}
function extractAndProcessSingleArchiveEntry(fileEntry, filePath, archiveEntries, cb) {
async.waterfall(
[
function extractToTemp(callback) {
// :TODO: we may want to skip this if the compressed file is too large...
temptmp.mkdir( { prefix : 'enigextract-' }, (err, tempDir) => {
if(err) {
return callback(err);
}
const archiveUtil = ArchiveUtil.getInstance();
// ensure we only extract one - there should only be one anyway -- we also just need the fileName
const extractList = archiveEntries.slice(0, 1).map(entry => entry.fileName);
archiveUtil.extractTo(filePath, tempDir, fileEntry.meta.archive_type, extractList, err => {
if(err) {
return callback(err);
}
return callback(null, paths.join(tempDir, extractList[0]));
});
});
},
function processSingleExtractedFile(extractedFile, callback) {
populateFileEntryInfoFromFile(fileEntry, extractedFile, err => {
if(!fileEntry.desc) {
fileEntry.desc = getDescFromFileName(filePath);
fileEntry.descSrc = 'fileName';
}
return callback(err);
});
}
],
err => {
return cb(err);
}
);
}
function populateFileEntryWithArchive(fileEntry, filePath, stepInfo, iterator, cb) {
const archiveUtil = ArchiveUtil.getInstance();
const archiveType = fileEntry.meta.archive_type; // we set this previous to populateFileEntryWithArchive()
async.waterfall(
[
function getArchiveFileList(callback) {
stepInfo.step = 'archive_list_start';
iterator(err => {
if(err) {
return callback(err);
}
archiveUtil.listEntries(filePath, archiveType, (err, entries) => {
if(err) {
stepInfo.step = 'archive_list_failed';
} else {
stepInfo.step = 'archive_list_finish';
stepInfo.archiveEntries = entries || [];
}
iterator(iterErr => {
return callback( iterErr, entries || [] ); // ignore original |err| here
});
});
});
},
function processDescFilesStart(entries, callback) {
stepInfo.step = 'desc_files_start';
iterator(err => {
return callback(err, entries);
});
},
function extractDescFromArchive(entries, callback) {
//
// If we have a -single- entry in the archive, extract that file
// and try retrieving info in the non-archive manor. This should
// work for things like zipped up .pdf files.
//
// Otherwise, try to find particular desc files such as FILE_ID.DIZ
// and README.1ST
//
const archDescHandler = (1 === entries.length) ? extractAndProcessSingleArchiveEntry : extractAndProcessDescFiles;
archDescHandler(fileEntry, filePath, entries, err => {
return callback(err);
});
},
function attemptReleaseYearEstimation(callback) {
attemptSetEstimatedReleaseDate(fileEntry);
return callback(null);
},
function processDescFilesFinish(callback) {
stepInfo.step = 'desc_files_finish';
return iterator(callback);
},
],
err => {
return cb(err);
}
);
}
function getInfoExtractUtilForDesc(mimeType, filePath, descType) {
const config = Config();
let fileType = _.get(config, [ 'fileTypes', mimeType ] );
if(Array.isArray(fileType)) {
// further refine by extention
fileType = fileType.find(ft => paths.extname(filePath) === ft.ext);
}
if(!_.isObject(fileType)) {
return;
}
let util = _.get(fileType, `${descType}DescUtil`);
if(!_.isString(util)) {
return;
}
util = _.get(config, [ 'infoExtractUtils', util ]);
if(!util || !_.isString(util.cmd)) {
return;
}
return util;
}
function populateFileEntryInfoFromFile(fileEntry, filePath, cb) {
const mimeType = resolveMimeType(filePath);
if(!mimeType) {
return cb(null);
}
async.eachSeries( [ 'short', 'long' ], (descType, nextDesc) => {
const util = getInfoExtractUtilForDesc(mimeType, filePath, descType);
if(!util) {
return nextDesc(null);
}
const args = (util.args || [ '{filePath}'] ).map( arg => stringFormat(arg, { filePath : filePath } ) );
execFile(util.cmd, args, { timeout : 1000 * 30 }, (err, stdout) => {
if(err || !stdout) {
const reason = err ? err.message : 'No description produced';
logDebug(
{ reason : reason, cmd : util.cmd, args : args },
`${_.upperFirst(descType)} description command failed`
);
} else {
stdout = (stdout || '').trim();
if(stdout.length > 0) {
const key = 'short' === descType ? 'desc' : 'descLong';
if('desc' === key) {
//
// Word wrap short descriptions to FILE_ID.DIZ spec
//
// "...no more than 45 characters long"
//
// See http://www.textfiles.com/computers/fileid.txt
//
stdout = (wordWrapText( stdout, { width : 45 } ).wrapped || []).join('\n');
}
fileEntry[key] = stdout;
fileEntry[`${key}Src`] = 'infoTool';
}
}
return nextDesc(null);
});
}, () => {
return cb(null);
});
}
function populateFileEntryNonArchive(fileEntry, filePath, stepInfo, iterator, cb) {
2017-05-14 02:59:42 +00:00
async.series(
[
function processDescFilesStart(callback) {
stepInfo.step = 'desc_files_start';
return iterator(callback);
},
function getDescriptions(callback) {
populateFileEntryInfoFromFile(fileEntry, filePath, err => {
if(!fileEntry.desc) {
fileEntry.desc = getDescFromFileName(filePath);
fileEntry.descSrc = 'fileName';
}
return callback(err);
});
},
function processDescFilesFinish(callback) {
stepInfo.step = 'desc_files_finish';
return iterator(callback);
},
],
err => {
return cb(err);
}
);
}
function addNewFileEntry(fileEntry, filePath, cb) {
// :TODO: Use detectTypeWithBuf() once avail - we *just* read some file data
async.series(
[
function addNewDbRecord(callback) {
return fileEntry.persist(callback);
}
],
err => {
return cb(err);
}
);
}
const HASH_NAMES = [ 'sha1', 'sha256', 'md5', 'crc32' ];
function scanFile(filePath, options, iterator, cb) {
if(3 === arguments.length && _.isFunction(iterator)) {
cb = iterator;
iterator = null;
} else if(2 === arguments.length && _.isFunction(options)) {
cb = options;
iterator = null;
options = {};
}
const fileEntry = new FileEntry({
areaTag : options.areaTag,
meta : options.meta,
hashTags : options.hashTags, // Set() or Array
fileName : paths.basename(filePath),
storageTag : options.storageTag,
fileSha256 : options.sha256, // caller may know this already
});
const stepInfo = {
filePath : filePath,
fileName : paths.basename(filePath),
};
const callIter = (next) => {
return iterator ? iterator(stepInfo, next) : next(null);
};
const readErrorCallIter = (origError, next) => {
stepInfo.step = 'read_error';
stepInfo.error = origError.message;
callIter( () => {
return next(origError);
});
};
let lastCalcHashPercent;
// don't re-calc hashes for any we already have in |options|
const hashesToCalc = HASH_NAMES.filter(hn => {
if('sha256' === hn && fileEntry.fileSha256) {
return false;
}
if(`file_${hn}` in fileEntry.meta) {
return false;
}
return true;
});
async.waterfall(
[
function startScan(callback) {
fs.stat(filePath, (err, stats) => {
if(err) {
return readErrorCallIter(err, callback);
}
stepInfo.step = 'start';
stepInfo.byteSize = fileEntry.meta.byte_size = stats.size;
return callIter(callback);
});
},
function processPhysicalFileGeneric(callback) {
stepInfo.bytesProcessed = 0;
const hashes = {};
hashesToCalc.forEach(hashName => {
if('crc32' === hashName) {
hashes.crc32 = new CRC32;
} else {
hashes[hashName] = crypto.createHash(hashName);
}
});
const updateHashes = (data) => {
for(let i = 0; i < hashesToCalc.length; ++i) {
hashes[hashesToCalc[i]].update(data);
}
};
//
// Note that we are not using fs.createReadStream() here:
// While convenient, it is quite a bit slower -- which adds
// up to many seconds in time for larger files.
//
const chunkSize = 1024 * 64;
const buffer = new Buffer(chunkSize);
fs.open(filePath, 'r', (err, fd) => {
if(err) {
return readErrorCallIter(err, callback);
}
const nextChunk = () => {
fs.read(fd, buffer, 0, chunkSize, null, (err, bytesRead) => {
if(err) {
2018-06-26 01:09:08 +00:00
return fs.close(fd, closeErr => {
if(closeErr) {
logError( { filePath, error : err.message }, 'Failed to close file');
}
return readErrorCallIter(err, callback);
});
}
if(0 === bytesRead) {
// done - finalize
fileEntry.meta.byte_size = stepInfo.bytesProcessed;
for(let i = 0; i < hashesToCalc.length; ++i) {
const hashName = hashesToCalc[i];
if('sha256' === hashName) {
stepInfo.sha256 = fileEntry.fileSha256 = hashes.sha256.digest('hex');
} else if('sha1' === hashName || 'md5' === hashName) {
stepInfo[hashName] = fileEntry.meta[`file_${hashName}`] = hashes[hashName].digest('hex');
} else if('crc32' === hashName) {
stepInfo.crc32 = fileEntry.meta.file_crc32 = hashes.crc32.finalize().toString(16);
}
}
stepInfo.step = 'hash_finish';
2018-06-26 01:09:08 +00:00
return fs.close(fd, closeErr => {
if(closeErr) {
logError( { filePath, error : err.message }, 'Failed to close file');
}
return callIter(callback);
});
}
stepInfo.bytesProcessed += bytesRead;
stepInfo.calcHashPercent = Math.round(((stepInfo.bytesProcessed / stepInfo.byteSize) * 100));
//
// Only send 'hash_update' step update if we have a noticable percentage change in progress
//
const data = bytesRead < chunkSize ? buffer.slice(0, bytesRead) : buffer;
if(!iterator || stepInfo.calcHashPercent === lastCalcHashPercent) {
updateHashes(data);
return nextChunk();
} else {
lastCalcHashPercent = stepInfo.calcHashPercent;
stepInfo.step = 'hash_update';
callIter(err => {
if(err) {
return callback(err);
}
updateHashes(data);
return nextChunk();
});
}
});
};
nextChunk();
});
},
function processPhysicalFileByType(callback) {
const archiveUtil = ArchiveUtil.getInstance();
archiveUtil.detectType(filePath, (err, archiveType) => {
if(archiveType) {
// save this off
fileEntry.meta.archive_type = archiveType;
populateFileEntryWithArchive(fileEntry, filePath, stepInfo, callIter, err => {
if(err) {
populateFileEntryNonArchive(fileEntry, filePath, stepInfo, callIter, err => {
if(err) {
logDebug( { error : err.message }, 'Non-archive file entry population failed');
}
return callback(null); // ignore err
});
} else {
return callback(null);
}
});
} else {
populateFileEntryNonArchive(fileEntry, filePath, stepInfo, callIter, err => {
if(err) {
logDebug( { error : err.message }, 'Non-archive file entry population failed');
}
return callback(null); // ignore err
});
}
});
},
function fetchExistingEntry(callback) {
getExistingFileEntriesBySha256(fileEntry.fileSha256, (err, dupeEntries) => {
return callback(err, dupeEntries);
});
},
function finished(dupeEntries, callback) {
stepInfo.step = 'finished';
callIter( () => {
return callback(null, dupeEntries);
});
}
],
(err, dupeEntries) => {
if(err) {
return cb(err);
}
return cb(null, fileEntry, dupeEntries);
}
);
}
function scanFileAreaForChanges(areaInfo, options, iterator, cb) {
if(3 === arguments.length && _.isFunction(iterator)) {
cb = iterator;
iterator = null;
} else if(2 === arguments.length && _.isFunction(options)) {
cb = options;
iterator = null;
options = {};
}
const storageLocations = getAreaStorageLocations(areaInfo);
async.eachSeries(storageLocations, (storageLoc, nextLocation) => {
async.series(
[
function scanPhysFiles(callback) {
const physDir = storageLoc.dir;
fs.readdir(physDir, (err, files) => {
if(err) {
return callback(err);
}
async.eachSeries(files, (fileName, nextFile) => {
const fullPath = paths.join(physDir, fileName);
fs.stat(fullPath, (err, stats) => {
if(err) {
// :TODO: Log me!
return nextFile(null); // always try next file
}
if(!stats.isFile()) {
return nextFile(null);
}
scanFile(
fullPath,
{
areaTag : areaInfo.areaTag,
storageTag : storageLoc.storageTag
},
iterator,
(err, fileEntry, dupeEntries) => {
if(err) {
// :TODO: Log me!!!
return nextFile(null); // try next anyway
}
if(dupeEntries.length > 0) {
// :TODO: Handle duplidates -- what to do here???
} else {
if(Array.isArray(options.tags)) {
options.tags.forEach(tag => {
fileEntry.hashTags.add(tag);
});
}
addNewFileEntry(fileEntry, fullPath, err => {
// pass along error; we failed to insert a record in our DB or something else bad
return nextFile(err);
});
}
}
);
});
}, err => {
return callback(err);
});
});
},
function scanDbEntries(callback) {
// :TODO: Look @ db entries for area that were *not* processed above
return callback(null);
}
],
err => {
return nextLocation(err);
}
);
},
err => {
return cb(err);
});
}
function getDescFromFileName(fileName) {
2018-06-30 19:03:08 +00:00
//
// Example filenames:
//
// input desired output
// -----------------------------------------------------------------------------------------
// Nintendo_Power_Issue_011_March-April_1990.cbr Nintendo Power Issue 011 March-April 1990
// Atari User Issue 3 (July 1985).pdf Atari User Issue 3 (July 1985)
// Out_Of_The_Shadows_010__1953_.cbz Out Of The Shadows 010 1953
// ABC A Basic Compiler 1.03 [pro].atr ABC A Basic Compiler 1.03 [pro]
// 221B Baker Street v1.0 (1987)(Datasoft)(Side B)[cr The Bounty].zip 221B Baker Street v1.0 (1987)(Datasoft)(Side B)[cr the Bounty]
//
// See also:
// * https://scenerules.org/
//
const ext = paths.extname(fileName);
const name = paths.basename(fileName, ext);
const asIsRe = /([vV]?(?:[0-9]{1,4})(?:\.[0-9]{1,4})+[-+]?(?:[a-z]{1,4})?)|(Incl\.)|(READ\.NFO)/g;
const normalize = (s) => {
return _.upperFirst(s.replace(/[-_.+]/g, ' ').replace(/\s+/g, ' '));
};
let out = '';
let m;
let pos;
do {
pos = asIsRe.lastIndex;
m = asIsRe.exec(name);
if(m) {
if(m.index > pos) {
out += normalize(name.slice(pos, m.index));
}
out += m[0]; // as-is
}
} while(0 != asIsRe.lastIndex);
if(pos < name.length) {
out += normalize(name.slice(pos));
}
2018-06-30 19:03:08 +00:00
return out;
}
//
// Return an object of stats about an area(s)
//
// {
//
// totalFiles : <totalFileCount>,
// totalBytes : <totalByteSize>,
// areas : {
// <areaTag> : {
// files : <fileCount>,
// bytes : <byteSize>
// }
// }
// }
//
function getAreaStats(cb) {
FileDb.all(
`SELECT DISTINCT f.area_tag, COUNT(f.file_id) AS total_files, SUM(m.meta_value) AS total_byte_size
FROM file f, file_meta m
WHERE f.file_id = m.file_id AND m.meta_name='byte_size'
GROUP BY f.area_tag;`,
(err, statRows) => {
if(err) {
return cb(err);
}
if(!statRows || 0 === statRows.length) {
return cb(Errors.DoesNotExist('No file areas to acquire stats from'));
}
return cb(
null,
statRows.reduce( (stats, v) => {
stats.totalFiles = (stats.totalFiles || 0) + v.total_files;
stats.totalBytes = (stats.totalBytes || 0) + v.total_byte_size;
stats.areas = stats.areas || {};
stats.areas[v.area_tag] = {
files : v.total_files,
bytes : v.total_byte_size,
};
return stats;
}, {})
);
}
);
}
// method exposed for event scheduler
function updateAreaStatsScheduledEvent(args, cb) {
getAreaStats( (err, stats) => {
if(!err) {
StatLog.setNonPeristentSystemStat('file_base_area_stats', stats);
}
return cb(err);
});
}
function cleanUpTempSessionItems(cb) {
// find (old) temporary session items and nuke 'em
const filter = {
areaTag : WellKnownAreaTags.TempDownloads,
metaPairs : [
{
name : 'session_temp_dl',
value : 1
}
]
};
FileEntry.findFiles(filter, (err, fileIds) => {
if(err) {
return cb(err);
}
async.each(fileIds, (fileId, nextFileId) => {
const fileEntry = new FileEntry();
fileEntry.load(fileId, err => {
if(err) {
Log.warn( { fileId }, 'Failed loading temporary session download item for cleanup');
return nextFileId(null);
}
FileEntry.removeEntry(fileEntry, { removePhysFile : true }, err => {
if(err) {
Log.warn( { fileId : fileEntry.fileId, filePath : fileEntry.filePath }, 'Failed to clean up temporary session download item');
}
return nextFileId(null);
});
});
}, () => {
return cb(null);
});
});
}