moon
/
enigma-bbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Split up code a bit in prep for DESCRIPT.ION generation

This commit is contained in:
Bryan Ashby 2018-03-11 21:23:35 -06:00
parent edc0bf5e06
commit 7bf49d973d
1 changed files with 164 additions and 352 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

514
core/file_base_list_export.js
View File

@ -2,7 +2,6 @@
'use strict';
// ENiGMA½
const { MenuModule } = require('./menu_module.js');
const stringFormat = require('./string_format.js');
const FileEntry = require('./file_entry.js');
const FileArea = require('./file_base_area.js');
@ -11,142 +10,54 @@ const { Errors } = require('./enig_error.js');
const {
splitTextAtTerms,
isAnsi,
renderSubstr
} = require('./string_util.js');
const AnsiPrep = require('./ansi_prep.js');
const Events = require('./events.js');
const Log = require('./logger.js').log;
const DownloadQueue = require('./download_queue.js');
// deps
const _ = require('lodash');
const async = require('async');
const fs = require('graceful-fs');
const fse = require('fs-extra');
const paths = require('path');
const iconv = require('iconv-lite');
const moment = require('moment');
const uuidv4 = require('uuid/v4');
const yazl = require('yazl');
/*
Module config block can contain the following:
templateEncoding - encoding of template files (utf8)
tsFormat - timestamp format (theme 'short')
descWidth - max desc width (45)
progBarChar - progress bar character (▒)
compressThreshold - threshold to kick in comrpession for lists (1.44 MiB)
templates - object containing:
header - filename of header template (misc/file_list_header.asc)
entry - filename of entry template (misc/file_list_entry.asc)
module.exports = function exportFileList(filterCriteria, options, cb) {
options.templateEncoding = options.templateEncoding || 'utf8';
options.headerTemplate = options.headerTemplate || 'description_export_header_template.asc';
options.entryTemplate = options.entryTemplate || 'descripion_export_entry_template.asc';
options.tsFormat = options.tsFormat || 'YYYY-MM-DD';
options.descWidth = options.descWidth || 45; // FILE_ID.DIZ spec
Header template variables:
nowTs, boardName, totalFileCount, totalFileSize,
filterAreaTag, filterAreaName, filterAreaDesc,
filterTerms, filterHashTags
const state = {
total : 0,
current : 0,
step : 'preparing',
status : 'Preparing',
};
Entry template variables:
fileId, areaName, areaDesc, userRating, fileName,
fileSize, fileDesc, fileDescShort, fileSha256, fileCrc32,
fileMd5, fileSha1, uploadBy, fileUploadTs, fileHashTags,
currentFile, progress,
*/
const updateProgress = _.isFunction(options.progress) ?
progCb => {
return options.progress(state, progCb);
} :
progCb => {
return progCb(null);
}
;
exports.moduleInfo = {
name : 'File Base List Export',
desc : 'Exports file base listings for download',
author : 'NuSkooler',
};
const FormIds = {
main : 0,
};
const MciViewIds = {
main : {
status : 1,
progressBar : 2,
customRangeStart : 10,
}
};
const TEMPLATE_KEYS = [ // config.templates.*
'header', 'entry',
];
exports.getModule = class FileBaseListExport extends MenuModule {
constructor(options) {
super(options);
this.config = Object.assign({}, _.get(options, 'menuConfig.config'), options.extraArgs);
this.config.templateEncoding = this.config.templateEncoding || 'utf8';
this.config.tsFormat = this.config.tsFormat || this.client.currentTheme.helpers.getDateTimeFormat('short');
this.config.descWidth = this.config.descWidth || 45; // ie FILE_ID.DIZ
this.config.progBarChar = renderSubstr( (this.config.progBarChar || '▒'), 0, 1);
this.config.compressThreshold = this.config.compressThreshold || (1440000); // >= 1.44M by default :)
}
mciReady(mciData, cb) {
super.mciReady(mciData, err => {
if(err) {
return cb(err);
}
async.series(
[
(callback) => this.prepViewController('main', FormIds.main, mciData.menu, callback),
(callback) => this.prepareList(callback),
],
err => {
async.waterfall(
[
function readTemplateFiles(callback) {
updateProgress(err => {
if(err) {
if('NORESULTS' === err.reasonCode) {
return this.gotoMenu(this.menuConfig.config.noResultsMenu || 'fileBaseExportListNoResults');
}
return this.prevMenu();
return callback(err);
}
return cb(err);
}
);
});
}
finishedLoading() {
this.prevMenu();
}
const templateFiles = [ options.headerTemplate, options.entryTemplate ];
async.map(templateFiles, (template, nextTemplate) => {
template = paths.isAbsolute(template) ? template : paths.join(Config.paths.misc, template);
prepareList(cb) {
const self = this;
const statusView = self.viewControllers.main.getView(MciViewIds.main.status);
const updateStatus = (status) => {
if(statusView) {
statusView.setText(status);
}
};
const progBarView = self.viewControllers.main.getView(MciViewIds.main.progressBar);
const updateProgressBar = (curr, total) => {
if(progBarView) {
const prog = Math.floor( (curr / total) * progBarView.dimens.width );
progBarView.setText(self.config.progBarChar.repeat(prog));
}
};
async.waterfall(
[
function readTemplateFiles(callback) {
updateStatus('Preparing');
async.map(TEMPLATE_KEYS, (templateKey, nextKey) => {
let templatePath = _.get(self.config, [ 'templates', templateKey ]);
templatePath = templatePath || `file_list_${templateKey}.asc`;
templatePath = paths.isAbsolute(templatePath) ? templatePath : paths.join(Config.paths.misc, templatePath);
fs.readFile(templatePath, (err, data) => {
return nextKey(err, data);
fs.readFile(template, (err, data) => {
return nextTemplate(err, data);
});
}, (err, templates) => {
if(err) {
@ -154,7 +65,7 @@ exports.getModule = class FileBaseListExport extends MenuModule {
}
// decode + ensure DOS style CRLF
templates = templates.map(tmp => iconv.decode(tmp, self.config.templateEncoding).replace(/\r?\n/g, '\r\n') );
templates = templates.map(tmp => iconv.decode(tmp, options.templateEncoding).replace(/\r?\n/g, '\r\n') );
// Look for the first {fileDesc} (if any) in 'entry' template & find indentation requirements
let descIndent = 0;
@ -169,15 +80,16 @@ exports.getModule = class FileBaseListExport extends MenuModule {
return callback(null, templates[0], templates[1], descIndent);
});
},
function findFiles(headerTemplate, entryTemplate, descIndent, callback) {
const filterCriteria = Object.assign({}, self.config.filterCriteria);
if(!filterCriteria.areaTag) {
filterCriteria.areaTag = FileArea.getAvailableFileAreaTags(self.client);
});
},
function findFiles(headerTemplate, entryTemplate, descIndent, callback) {
state.step = 'gathering';
state.status = 'Gathering files for supplied criteria';
updateProgress(err => {
if(err) {
return callback(err);
}
updateStatus('Gathering files for supplied criteria');
FileEntry.findFiles(filterCriteria, (err, fileIds) => {
if(0 === fileIds.length) {
return callback(Errors.General('No results for criteria', 'NORESULTS'));
@ -185,229 +97,129 @@ exports.getModule = class FileBaseListExport extends MenuModule {
return callback(err, headerTemplate, entryTemplate, descIndent, fileIds);
});
},
function buildListEntries(headerTemplate, entryTemplate, descIndent, fileIds, callback) {
const formatObj = {
totalFileCount : fileIds.length,
};
let current = 0;
let listBody = '';
const totals = { fileCount : fileIds.length, bytes : 0 };
// this may take quite a while; temp disable of idle monitor
self.client.stopIdleMonitor();
async.eachSeries(fileIds, (fileId, nextFileId) => {
const fileInfo = new FileEntry();
current += 1;
fileInfo.load(fileId, err => {
if(err) {
return nextFileId(null); // failed, but try the next
}
updateStatus(`Processing ${fileInfo.fileName}`);
totals.bytes += fileInfo.meta.byte_size;
updateProgressBar(current, fileIds.length);
const appendFileInfo = () => {
listBody += stringFormat(entryTemplate, formatObj);
self.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, formatObj);
return nextFileId(null);
};
const area = FileArea.getFileAreaByTag(fileInfo.areaTag);
formatObj.fileId = fileId;
formatObj.areaName = _.get(area, 'name') || 'N/A';
formatObj.areaDesc = _.get(area, 'desc') || 'N/A';
formatObj.userRating = fileInfo.userRating || 0;
formatObj.fileName = fileInfo.fileName;
formatObj.fileSize = fileInfo.meta.byte_size;
formatObj.fileDesc = fileInfo.desc || '';
formatObj.fileDescShort = formatObj.fileDesc.slice(0, self.config.descWidth);
formatObj.fileSha256 = fileInfo.fileSha256;
formatObj.fileCrc32 = fileInfo.meta.file_crc32;
formatObj.fileMd5 = fileInfo.meta.file_md5;
formatObj.fileSha1 = fileInfo.meta.file_sha1;
formatObj.uploadBy = fileInfo.meta.upload_by_username || 'N/A';
formatObj.fileUploadTs = moment(fileInfo.uploadTimestamp).format(self.config.tsFormat);
formatObj.fileHashTags = fileInfo.hashTags.size > 0 ? Array.from(fileInfo.hashTags).join(', ') : 'N/A';
formatObj.currentFile = current;
formatObj.progress = Math.floor( (current / fileIds.length) * 100 );
if(isAnsi(fileInfo.desc)) {
AnsiPrep(
fileInfo.desc,
{
cols : Math.min(self.config.descWidth, 79 - descIndent),
forceLineTerm : true, // ensure each line is term'd
asciiMode : true, // export to ASCII
fillLines : false, // don't fill up to |cols|
indent : descIndent,
},
(err, desc) => {
if(desc) {
formatObj.fileDesc = desc;
}
return appendFileInfo();
}
);
} else {
const indentSpc = descIndent > 0 ? ' '.repeat(descIndent) : '';
formatObj.fileDesc = splitTextAtTerms(formatObj.fileDesc).join(`\r\n${indentSpc}`) + '\r\n';
return appendFileInfo();
}
});
}, err => {
// re-enable idle monitor
self.client.startIdleMonitor();
return callback(err, listBody, headerTemplate, totals);
});
},
function buildHeader(listBody, headerTemplate, totals, callback) {
// header is built last such that we can have totals/etc.
let filterAreaName;
let filterAreaDesc;
if(self.config.filterCriteria.areaTag) {
const area = FileArea.getFileAreaByTag(self.config.filterCriteria.areaTag);
filterAreaName = _.get(area, 'name') || 'N/A';
filterAreaDesc = _.get(area, 'desc') || 'N/A';
} else {
filterAreaName = '-ALL-';
filterAreaDesc = 'All areas';
}
const headerFormatObj = {
nowTs : moment().format(self.config.tsFormat),
boardName : Config.general.boardName,
totalFileCount : totals.fileCount,
totalFileSize : totals.bytes,
filterAreaTag : self.config.filterCriteria.areaTag || '-ALL-',
filterAreaName : filterAreaName,
filterAreaDesc : filterAreaDesc,
filterTerms : self.config.filterCriteria.terms || '(none)',
filterHashTags : self.config.filterCriteria.tags || '(none)',
};
listBody = stringFormat(headerTemplate, headerFormatObj) + listBody;
return callback(null, listBody);
},
function persistList(listBody, callback) {
updateStatus('Persisting list');
const sysTempDownloadArea = FileArea.getFileAreaByTag(FileArea.WellKnownAreaTags.TempDownloads);
const sysTempDownloadDir = FileArea.getAreaDefaultStorageDirectory(sysTempDownloadArea);
fse.mkdirs(sysTempDownloadDir, err => {
if(err) {
return callback(err);
}
const outputFileName = paths.join(
sysTempDownloadDir,
`file_list_${uuidv4().substr(-8)}_${moment().format('YYYY-MM-DD')}.txt`
);
fs.writeFile(outputFileName, listBody, 'utf8', err => {
if(err) {
return callback(err);
}
self.getSizeAndCompressIfMeetsSizeThreshold(outputFileName, (err, finalOutputFileName, fileSize) => {
return callback(err, finalOutputFileName, fileSize, sysTempDownloadArea);
});
});
});
},
function persistFileEntry(outputFileName, fileSize, sysTempDownloadArea, callback) {
const newEntry = new FileEntry({
areaTag : sysTempDownloadArea.areaTag,
fileName : paths.basename(outputFileName),
storageTag : sysTempDownloadArea.storageTags[0],
meta : {
upload_by_username : self.client.user.username,
upload_by_user_id : self.client.user.userId,
byte_size : fileSize,
session_temp_dl : 1, // download is valid until session is over
}
});
newEntry.desc = 'File List Export';
newEntry.persist(err => {
if(!err) {
// queue it!
const dlQueue = new DownloadQueue(self.client);
dlQueue.add(newEntry);
// clean up after ourselves when the session ends
const thisClientId = self.client.session.id;
Events.once(Events.getSystemEvents().ClientDisconnected, evt => {
if(thisClientId === _.get(evt, 'client.session.id')) {
FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => {
if(err) {
Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' );
} else {
Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' );
}
});
}
});
}
return callback(err);
});
},
function done(callback) {
updateStatus('Exported list has been added to your download queue');
return callback(null);
}
], err => {
return cb(err);
}
);
}
getSizeAndCompressIfMeetsSizeThreshold(filePath, cb) {
fse.stat(filePath, (err, stats) => {
if(err) {
return cb(err);
}
if(stats.size < this.config.compressThreshold) {
// small enough, keep orig
return cb(null, filePath, stats.size);
}
const zipFilePath = `${filePath}.zip`;
const zipFile = new yazl.ZipFile();
zipFile.addFile(filePath, paths.basename(filePath));
zipFile.end( () => {
const outZipFile = fs.createWriteStream(zipFilePath);
zipFile.outputStream.pipe(outZipFile);
zipFile.outputStream.on('finish', () => {
// delete the original
fse.unlink(filePath, err => {
if(err) {
return cb(err);
}
// finally stat the new output
fse.stat(zipFilePath, (err, stats) => {
return cb(err, zipFilePath, stats ? stats.size : 0);
});
});
});
});
});
}
},
function buildListEntries(headerTemplate, entryTemplate, descIndent, fileIds, callback) {
const formatObj = {
totalFileCount : fileIds.length,
};
let current = 0;
let listBody = '';
const totals = { fileCount : fileIds.length, bytes : 0 };
state.total = fileIds.length;
state.step = 'file';
async.eachSeries(fileIds, (fileId, nextFileId) => {
const fileInfo = new FileEntry();
current += 1;
fileInfo.load(fileId, err => {
if(err) {
return nextFileId(null); // failed, but try the next
}
totals.bytes += fileInfo.meta.byte_size;
const appendFileInfo = () => {
listBody += stringFormat(entryTemplate, formatObj);
state.current = current;
state.status = `Processing ${fileInfo.fileName}`;
state.fileInfo = formatObj;
updateProgress(err => {
return nextFileId(err);
});
};
const area = FileArea.getFileAreaByTag(fileInfo.areaTag);
formatObj.fileId = fileId;
formatObj.areaName = _.get(area, 'name') || 'N/A';
formatObj.areaDesc = _.get(area, 'desc') || 'N/A';
formatObj.userRating = fileInfo.userRating || 0;
formatObj.fileName = fileInfo.fileName;
formatObj.fileSize = fileInfo.meta.byte_size;
formatObj.fileDesc = fileInfo.desc || '';
formatObj.fileDescShort = formatObj.fileDesc.slice(0, options.descWidth);
formatObj.fileSha256 = fileInfo.fileSha256;
formatObj.fileCrc32 = fileInfo.meta.file_crc32;
formatObj.fileMd5 = fileInfo.meta.file_md5;
formatObj.fileSha1 = fileInfo.meta.file_sha1;
formatObj.uploadBy = fileInfo.meta.upload_by_username || 'N/A';
formatObj.fileUploadTs = moment(fileInfo.uploadTimestamp).format(options.tsFormat);
formatObj.fileHashTags = fileInfo.hashTags.size > 0 ? Array.from(fileInfo.hashTags).join(', ') : 'N/A';
formatObj.currentFile = current;
formatObj.progress = Math.floor( (current / fileIds.length) * 100 );
if(isAnsi(fileInfo.desc)) {
AnsiPrep(
fileInfo.desc,
{
cols : Math.min(options.descWidth, 79 - descIndent),
forceLineTerm : true, // ensure each line is term'd
asciiMode : true, // export to ASCII
fillLines : false, // don't fill up to |cols|
indent : descIndent,
},
(err, desc) => {
if(desc) {
formatObj.fileDesc = desc;
}
return appendFileInfo();
}
);
} else {
const indentSpc = descIndent > 0 ? ' '.repeat(descIndent) : '';
formatObj.fileDesc = splitTextAtTerms(formatObj.fileDesc).join(`\r\n${indentSpc}`) + '\r\n';
return appendFileInfo();
}
});
}, err => {
return callback(err, listBody, headerTemplate, totals);
});
},
function buildHeader(listBody, headerTemplate, totals, callback) {
// header is built last such that we can have totals/etc.
let filterAreaName;
let filterAreaDesc;
if(filterCriteria.areaTag) {
const area = FileArea.getFileAreaByTag(filterCriteria.areaTag);
filterAreaName = _.get(area, 'name') || 'N/A';
filterAreaDesc = _.get(area, 'desc') || 'N/A';
} else {
filterAreaName = '-ALL-';
filterAreaDesc = 'All areas';
}
const headerFormatObj = {
nowTs : moment().format(options.tsFormat),
boardName : Config.general.boardName,
totalFileCount : totals.fileCount,
totalFileSize : totals.bytes,
filterAreaTag : filterCriteria.areaTag || '-ALL-',
filterAreaName : filterAreaName,
filterAreaDesc : filterAreaDesc,
filterTerms : filterCriteria.terms || '(none)',
filterHashTags : filterCriteria.tags || '(none)',
};
listBody = stringFormat(headerTemplate, headerFormatObj) + listBody;
return callback(null, listBody);
},
function done(listBody, callback) {
delete state.fileInfo;
state.step = 'finished';
state.status = 'Finished processing';
updateProgress( () => {
return callback(null, listBody);
});
}
], (err, listBody) => {
return cb(err, listBody);
}
);
};