moon
/
enigma-bbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Split up code a bit in prep for DESCRIPT.ION generation

This commit is contained in:
Bryan Ashby 2018-03-11 21:23:35 -06:00
parent edc0bf5e06
commit 7bf49d973d
1 changed files with 164 additions and 352 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

308
core/file_base_list_export.js
View File

@ -2,7 +2,6 @@
'use strict'; 'use strict';
// ENiGMA½ // ENiGMA½
const { MenuModule } = require('./menu_module.js');
const stringFormat = require('./string_format.js'); const stringFormat = require('./string_format.js');
const FileEntry = require('./file_entry.js'); const FileEntry = require('./file_entry.js');
const FileArea = require('./file_base_area.js'); const FileArea = require('./file_base_area.js');
@ -11,142 +10,54 @@ const { Errors } = require('./enig_error.js');
const { const {
splitTextAtTerms, splitTextAtTerms,
isAnsi, isAnsi,
renderSubstr
} = require('./string_util.js'); } = require('./string_util.js');
const AnsiPrep = require('./ansi_prep.js'); const AnsiPrep = require('./ansi_prep.js');
const Events = require('./events.js');
const Log = require('./logger.js').log;
const DownloadQueue = require('./download_queue.js');
// deps // deps
const _ = require('lodash'); const _ = require('lodash');
const async = require('async'); const async = require('async');
const fs = require('graceful-fs'); const fs = require('graceful-fs');
const fse = require('fs-extra');
const paths = require('path'); const paths = require('path');
const iconv = require('iconv-lite'); const iconv = require('iconv-lite');
const moment = require('moment'); const moment = require('moment');
const uuidv4 = require('uuid/v4');
const yazl = require('yazl');
/* module.exports = function exportFileList(filterCriteria, options, cb) {
Module config block can contain the following: options.templateEncoding = options.templateEncoding || 'utf8';
templateEncoding - encoding of template files (utf8) options.headerTemplate = options.headerTemplate || 'description_export_header_template.asc';
tsFormat - timestamp format (theme 'short') options.entryTemplate = options.entryTemplate || 'descripion_export_entry_template.asc';
descWidth - max desc width (45) options.tsFormat = options.tsFormat || 'YYYY-MM-DD';
progBarChar - progress bar character (▒) options.descWidth = options.descWidth || 45; // FILE_ID.DIZ spec
compressThreshold - threshold to kick in comrpession for lists (1.44 MiB)
templates - object containing:
header - filename of header template (misc/file_list_header.asc)
entry - filename of entry template (misc/file_list_entry.asc)
Header template variables: const state = {
nowTs, boardName, totalFileCount, totalFileSize, total : 0,
filterAreaTag, filterAreaName, filterAreaDesc, current : 0,
filterTerms, filterHashTags step : 'preparing',
status : 'Preparing',
Entry template variables:
fileId, areaName, areaDesc, userRating, fileName,
fileSize, fileDesc, fileDescShort, fileSha256, fileCrc32,
fileMd5, fileSha1, uploadBy, fileUploadTs, fileHashTags,
currentFile, progress,
*/
exports.moduleInfo = {
name : 'File Base List Export',
desc : 'Exports file base listings for download',
author : 'NuSkooler',
};
const FormIds = {
main : 0,
};
const MciViewIds = {
main : {
status : 1,
progressBar : 2,
customRangeStart : 10,
}
};
const TEMPLATE_KEYS = [ // config.templates.*
'header', 'entry',
];
exports.getModule = class FileBaseListExport extends MenuModule {
constructor(options) {
super(options);
this.config = Object.assign({}, _.get(options, 'menuConfig.config'), options.extraArgs);
this.config.templateEncoding = this.config.templateEncoding || 'utf8';
this.config.tsFormat = this.config.tsFormat || this.client.currentTheme.helpers.getDateTimeFormat('short');
this.config.descWidth = this.config.descWidth || 45; // ie FILE_ID.DIZ
this.config.progBarChar = renderSubstr( (this.config.progBarChar || '▒'), 0, 1);
this.config.compressThreshold = this.config.compressThreshold || (1440000); // >= 1.44M by default :)
}
mciReady(mciData, cb) {
super.mciReady(mciData, err => {
if(err) {
return cb(err);
}
async.series(
[
(callback) => this.prepViewController('main', FormIds.main, mciData.menu, callback),
(callback) => this.prepareList(callback),
],
err => {
if(err) {
if('NORESULTS' === err.reasonCode) {
return this.gotoMenu(this.menuConfig.config.noResultsMenu || 'fileBaseExportListNoResults');
}
return this.prevMenu();
}
return cb(err);
}
);
});
}
finishedLoading() {
this.prevMenu();
}
prepareList(cb) {
const self = this;
const statusView = self.viewControllers.main.getView(MciViewIds.main.status);
const updateStatus = (status) => {
if(statusView) {
statusView.setText(status);
}
}; };
const progBarView = self.viewControllers.main.getView(MciViewIds.main.progressBar); const updateProgress = _.isFunction(options.progress) ?
const updateProgressBar = (curr, total) => { progCb => {
if(progBarView) { return options.progress(state, progCb);
const prog = Math.floor( (curr / total) * progBarView.dimens.width ); } :
progBarView.setText(self.config.progBarChar.repeat(prog)); progCb => {
return progCb(null);
} }
}; ;
async.waterfall( async.waterfall(
[ [
function readTemplateFiles(callback) { function readTemplateFiles(callback) {
updateStatus('Preparing'); updateProgress(err => {
if(err) {
return callback(err);
}
async.map(TEMPLATE_KEYS, (templateKey, nextKey) => { const templateFiles = [ options.headerTemplate, options.entryTemplate ];
let templatePath = _.get(self.config, [ 'templates', templateKey ]); async.map(templateFiles, (template, nextTemplate) => {
templatePath = templatePath || `file_list_${templateKey}.asc`; template = paths.isAbsolute(template) ? template : paths.join(Config.paths.misc, template);
templatePath = paths.isAbsolute(templatePath) ? templatePath : paths.join(Config.paths.misc, templatePath);
fs.readFile(templatePath, (err, data) => { fs.readFile(template, (err, data) => {
return nextKey(err, data); return nextTemplate(err, data);
}); });
}, (err, templates) => { }, (err, templates) => {
if(err) { if(err) {
@ -154,7 +65,7 @@ exports.getModule = class FileBaseListExport extends MenuModule {
} }
// decode + ensure DOS style CRLF // decode + ensure DOS style CRLF
templates = templates.map(tmp => iconv.decode(tmp, self.config.templateEncoding).replace(/\r?\n/g, '\r\n') ); templates = templates.map(tmp => iconv.decode(tmp, options.templateEncoding).replace(/\r?\n/g, '\r\n') );
// Look for the first {fileDesc} (if any) in 'entry' template & find indentation requirements // Look for the first {fileDesc} (if any) in 'entry' template & find indentation requirements
let descIndent = 0; let descIndent = 0;
@ -169,15 +80,16 @@ exports.getModule = class FileBaseListExport extends MenuModule {
return callback(null, templates[0], templates[1], descIndent); return callback(null, templates[0], templates[1], descIndent);
}); });
});
}, },
function findFiles(headerTemplate, entryTemplate, descIndent, callback) { function findFiles(headerTemplate, entryTemplate, descIndent, callback) {
const filterCriteria = Object.assign({}, self.config.filterCriteria); state.step = 'gathering';
if(!filterCriteria.areaTag) { state.status = 'Gathering files for supplied criteria';
filterCriteria.areaTag = FileArea.getAvailableFileAreaTags(self.client); updateProgress(err => {
if(err) {
return callback(err);
} }
updateStatus('Gathering files for supplied criteria');
FileEntry.findFiles(filterCriteria, (err, fileIds) => { FileEntry.findFiles(filterCriteria, (err, fileIds) => {
if(0 === fileIds.length) { if(0 === fileIds.length) {
return callback(Errors.General('No results for criteria', 'NORESULTS')); return callback(Errors.General('No results for criteria', 'NORESULTS'));
@ -185,6 +97,7 @@ exports.getModule = class FileBaseListExport extends MenuModule {
return callback(err, headerTemplate, entryTemplate, descIndent, fileIds); return callback(err, headerTemplate, entryTemplate, descIndent, fileIds);
}); });
});
}, },
function buildListEntries(headerTemplate, entryTemplate, descIndent, fileIds, callback) { function buildListEntries(headerTemplate, entryTemplate, descIndent, fileIds, callback) {
const formatObj = { const formatObj = {
@ -194,9 +107,9 @@ exports.getModule = class FileBaseListExport extends MenuModule {
let current = 0; let current = 0;
let listBody = ''; let listBody = '';
const totals = { fileCount : fileIds.length, bytes : 0 }; const totals = { fileCount : fileIds.length, bytes : 0 };
state.total = fileIds.length;
// this may take quite a while; temp disable of idle monitor state.step = 'file';
self.client.stopIdleMonitor();
async.eachSeries(fileIds, (fileId, nextFileId) => { async.eachSeries(fileIds, (fileId, nextFileId) => {
const fileInfo = new FileEntry(); const fileInfo = new FileEntry();
@ -207,18 +120,18 @@ exports.getModule = class FileBaseListExport extends MenuModule {
return nextFileId(null); // failed, but try the next return nextFileId(null); // failed, but try the next
} }
updateStatus(`Processing ${fileInfo.fileName}`);
totals.bytes += fileInfo.meta.byte_size; totals.bytes += fileInfo.meta.byte_size;
updateProgressBar(current, fileIds.length);
const appendFileInfo = () => { const appendFileInfo = () => {
listBody += stringFormat(entryTemplate, formatObj); listBody += stringFormat(entryTemplate, formatObj);
self.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, formatObj); state.current = current;
state.status = `Processing ${fileInfo.fileName}`;
state.fileInfo = formatObj;
return nextFileId(null); updateProgress(err => {
return nextFileId(err);
});
}; };
const area = FileArea.getFileAreaByTag(fileInfo.areaTag); const area = FileArea.getFileAreaByTag(fileInfo.areaTag);
@ -230,13 +143,13 @@ exports.getModule = class FileBaseListExport extends MenuModule {
formatObj.fileName = fileInfo.fileName; formatObj.fileName = fileInfo.fileName;
formatObj.fileSize = fileInfo.meta.byte_size; formatObj.fileSize = fileInfo.meta.byte_size;
formatObj.fileDesc = fileInfo.desc || ''; formatObj.fileDesc = fileInfo.desc || '';
formatObj.fileDescShort = formatObj.fileDesc.slice(0, self.config.descWidth); formatObj.fileDescShort = formatObj.fileDesc.slice(0, options.descWidth);
formatObj.fileSha256 = fileInfo.fileSha256; formatObj.fileSha256 = fileInfo.fileSha256;
formatObj.fileCrc32 = fileInfo.meta.file_crc32; formatObj.fileCrc32 = fileInfo.meta.file_crc32;
formatObj.fileMd5 = fileInfo.meta.file_md5; formatObj.fileMd5 = fileInfo.meta.file_md5;
formatObj.fileSha1 = fileInfo.meta.file_sha1; formatObj.fileSha1 = fileInfo.meta.file_sha1;
formatObj.uploadBy = fileInfo.meta.upload_by_username || 'N/A'; formatObj.uploadBy = fileInfo.meta.upload_by_username || 'N/A';
formatObj.fileUploadTs = moment(fileInfo.uploadTimestamp).format(self.config.tsFormat); formatObj.fileUploadTs = moment(fileInfo.uploadTimestamp).format(options.tsFormat);
formatObj.fileHashTags = fileInfo.hashTags.size > 0 ? Array.from(fileInfo.hashTags).join(', ') : 'N/A'; formatObj.fileHashTags = fileInfo.hashTags.size > 0 ? Array.from(fileInfo.hashTags).join(', ') : 'N/A';
formatObj.currentFile = current; formatObj.currentFile = current;
formatObj.progress = Math.floor( (current / fileIds.length) * 100 ); formatObj.progress = Math.floor( (current / fileIds.length) * 100 );
@ -245,7 +158,7 @@ exports.getModule = class FileBaseListExport extends MenuModule {
AnsiPrep( AnsiPrep(
fileInfo.desc, fileInfo.desc,
{ {
cols : Math.min(self.config.descWidth, 79 - descIndent), cols : Math.min(options.descWidth, 79 - descIndent),
forceLineTerm : true, // ensure each line is term'd forceLineTerm : true, // ensure each line is term'd
asciiMode : true, // export to ASCII asciiMode : true, // export to ASCII
fillLines : false, // don't fill up to |cols| fillLines : false, // don't fill up to |cols|
@ -265,9 +178,6 @@ exports.getModule = class FileBaseListExport extends MenuModule {
} }
}); });
}, err => { }, err => {
// re-enable idle monitor
self.client.startIdleMonitor();
return callback(err, listBody, headerTemplate, totals); return callback(err, listBody, headerTemplate, totals);
}); });
}, },
@ -276,8 +186,8 @@ exports.getModule = class FileBaseListExport extends MenuModule {
let filterAreaName; let filterAreaName;
let filterAreaDesc; let filterAreaDesc;
if(self.config.filterCriteria.areaTag) { if(filterCriteria.areaTag) {
const area = FileArea.getFileAreaByTag(self.config.filterCriteria.areaTag); const area = FileArea.getFileAreaByTag(filterCriteria.areaTag);
filterAreaName = _.get(area, 'name') || 'N/A'; filterAreaName = _.get(area, 'name') || 'N/A';
filterAreaDesc = _.get(area, 'desc') || 'N/A'; filterAreaDesc = _.get(area, 'desc') || 'N/A';
} else { } else {
@ -286,128 +196,30 @@ exports.getModule = class FileBaseListExport extends MenuModule {
} }
const headerFormatObj = { const headerFormatObj = {
nowTs : moment().format(self.config.tsFormat), nowTs : moment().format(options.tsFormat),
boardName : Config.general.boardName, boardName : Config.general.boardName,
totalFileCount : totals.fileCount, totalFileCount : totals.fileCount,
totalFileSize : totals.bytes, totalFileSize : totals.bytes,
filterAreaTag : self.config.filterCriteria.areaTag || '-ALL-', filterAreaTag : filterCriteria.areaTag || '-ALL-',
filterAreaName : filterAreaName, filterAreaName : filterAreaName,
filterAreaDesc : filterAreaDesc, filterAreaDesc : filterAreaDesc,
filterTerms : self.config.filterCriteria.terms || '(none)', filterTerms : filterCriteria.terms || '(none)',
filterHashTags : self.config.filterCriteria.tags || '(none)', filterHashTags : filterCriteria.tags || '(none)',
}; };
listBody = stringFormat(headerTemplate, headerFormatObj) + listBody; listBody = stringFormat(headerTemplate, headerFormatObj) + listBody;
return callback(null, listBody); return callback(null, listBody);
}, },
function persistList(listBody, callback) { function done(listBody, callback) {
delete state.fileInfo;
updateStatus('Persisting list'); state.step = 'finished';
state.status = 'Finished processing';
const sysTempDownloadArea = FileArea.getFileAreaByTag(FileArea.WellKnownAreaTags.TempDownloads); updateProgress( () => {
const sysTempDownloadDir = FileArea.getAreaDefaultStorageDirectory(sysTempDownloadArea); return callback(null, listBody);
fse.mkdirs(sysTempDownloadDir, err => {
if(err) {
return callback(err);
}
const outputFileName = paths.join(
sysTempDownloadDir,
`file_list_${uuidv4().substr(-8)}_${moment().format('YYYY-MM-DD')}.txt`
);
fs.writeFile(outputFileName, listBody, 'utf8', err => {
if(err) {
return callback(err);
}
self.getSizeAndCompressIfMeetsSizeThreshold(outputFileName, (err, finalOutputFileName, fileSize) => {
return callback(err, finalOutputFileName, fileSize, sysTempDownloadArea);
});
});
});
},
function persistFileEntry(outputFileName, fileSize, sysTempDownloadArea, callback) {
const newEntry = new FileEntry({
areaTag : sysTempDownloadArea.areaTag,
fileName : paths.basename(outputFileName),
storageTag : sysTempDownloadArea.storageTags[0],
meta : {
upload_by_username : self.client.user.username,
upload_by_user_id : self.client.user.userId,
byte_size : fileSize,
session_temp_dl : 1, // download is valid until session is over
}
});
newEntry.desc = 'File List Export';
newEntry.persist(err => {
if(!err) {
// queue it!
const dlQueue = new DownloadQueue(self.client);
dlQueue.add(newEntry);
// clean up after ourselves when the session ends
const thisClientId = self.client.session.id;
Events.once(Events.getSystemEvents().ClientDisconnected, evt => {
if(thisClientId === _.get(evt, 'client.session.id')) {
FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => {
if(err) {
Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' );
} else {
Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' );
}
}); });
} }
}); ], (err, listBody) => {
} return cb(err, listBody);
return callback(err);
});
},
function done(callback) {
updateStatus('Exported list has been added to your download queue');
return callback(null);
}
], err => {
return cb(err);
} }
); );
}
getSizeAndCompressIfMeetsSizeThreshold(filePath, cb) {
fse.stat(filePath, (err, stats) => {
if(err) {
return cb(err);
}
if(stats.size < this.config.compressThreshold) {
// small enough, keep orig
return cb(null, filePath, stats.size);
}
const zipFilePath = `${filePath}.zip`;
const zipFile = new yazl.ZipFile();
zipFile.addFile(filePath, paths.basename(filePath));
zipFile.end( () => {
const outZipFile = fs.createWriteStream(zipFilePath);
zipFile.outputStream.pipe(outZipFile);
zipFile.outputStream.on('finish', () => {
// delete the original
fse.unlink(filePath, err => {
if(err) {
return cb(err);
}
// finally stat the new output
fse.stat(zipFilePath, (err, stats) => {
return cb(err, zipFilePath, stats ? stats.size : 0);
});
});
});
});
});
}
}; };