From 7bf49d973d1ff36a161b8a2cce59fd21e54d976a Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 11 Mar 2018 21:23:35 -0600 Subject: [PATCH] Split up code a bit in prep for DESCRIPT.ION generation --- core/file_base_list_export.js | 516 +++++++++++----------------------- 1 file changed, 164 insertions(+), 352 deletions(-) diff --git a/core/file_base_list_export.js b/core/file_base_list_export.js index 9c63ab2d..2ae4699d 100644 --- a/core/file_base_list_export.js +++ b/core/file_base_list_export.js @@ -2,7 +2,6 @@ 'use strict'; // ENiGMA½ -const { MenuModule } = require('./menu_module.js'); const stringFormat = require('./string_format.js'); const FileEntry = require('./file_entry.js'); const FileArea = require('./file_base_area.js'); @@ -11,142 +10,54 @@ const { Errors } = require('./enig_error.js'); const { splitTextAtTerms, isAnsi, - renderSubstr } = require('./string_util.js'); const AnsiPrep = require('./ansi_prep.js'); -const Events = require('./events.js'); -const Log = require('./logger.js').log; -const DownloadQueue = require('./download_queue.js'); // deps const _ = require('lodash'); const async = require('async'); const fs = require('graceful-fs'); -const fse = require('fs-extra'); const paths = require('path'); const iconv = require('iconv-lite'); const moment = require('moment'); -const uuidv4 = require('uuid/v4'); -const yazl = require('yazl'); -/* - Module config block can contain the following: - templateEncoding - encoding of template files (utf8) - tsFormat - timestamp format (theme 'short') - descWidth - max desc width (45) - progBarChar - progress bar character (▒) - compressThreshold - threshold to kick in comrpession for lists (1.44 MiB) - templates - object containing: - header - filename of header template (misc/file_list_header.asc) - entry - filename of entry template (misc/file_list_entry.asc) +module.exports = function exportFileList(filterCriteria, options, cb) { + options.templateEncoding = options.templateEncoding || 'utf8'; + options.headerTemplate = options.headerTemplate || 'description_export_header_template.asc'; + options.entryTemplate = options.entryTemplate || 'descripion_export_entry_template.asc'; + options.tsFormat = options.tsFormat || 'YYYY-MM-DD'; + options.descWidth = options.descWidth || 45; // FILE_ID.DIZ spec - Header template variables: - nowTs, boardName, totalFileCount, totalFileSize, - filterAreaTag, filterAreaName, filterAreaDesc, - filterTerms, filterHashTags + const state = { + total : 0, + current : 0, + step : 'preparing', + status : 'Preparing', + }; - Entry template variables: - fileId, areaName, areaDesc, userRating, fileName, - fileSize, fileDesc, fileDescShort, fileSha256, fileCrc32, - fileMd5, fileSha1, uploadBy, fileUploadTs, fileHashTags, - currentFile, progress, -*/ + const updateProgress = _.isFunction(options.progress) ? + progCb => { + return options.progress(state, progCb); + } : + progCb => { + return progCb(null); + } + ; -exports.moduleInfo = { - name : 'File Base List Export', - desc : 'Exports file base listings for download', - author : 'NuSkooler', -}; - -const FormIds = { - main : 0, -}; - -const MciViewIds = { - main : { - status : 1, - progressBar : 2, - - customRangeStart : 10, - } -}; - -const TEMPLATE_KEYS = [ // config.templates.* - 'header', 'entry', -]; - -exports.getModule = class FileBaseListExport extends MenuModule { - - constructor(options) { - super(options); - this.config = Object.assign({}, _.get(options, 'menuConfig.config'), options.extraArgs); - - this.config.templateEncoding = this.config.templateEncoding || 'utf8'; - this.config.tsFormat = this.config.tsFormat || this.client.currentTheme.helpers.getDateTimeFormat('short'); - this.config.descWidth = this.config.descWidth || 45; // ie FILE_ID.DIZ - this.config.progBarChar = renderSubstr( (this.config.progBarChar || '▒'), 0, 1); - this.config.compressThreshold = this.config.compressThreshold || (1440000); // >= 1.44M by default :) - } - - mciReady(mciData, cb) { - super.mciReady(mciData, err => { - if(err) { - return cb(err); - } - - async.series( - [ - (callback) => this.prepViewController('main', FormIds.main, mciData.menu, callback), - (callback) => this.prepareList(callback), - ], - err => { + async.waterfall( + [ + function readTemplateFiles(callback) { + updateProgress(err => { if(err) { - if('NORESULTS' === err.reasonCode) { - return this.gotoMenu(this.menuConfig.config.noResultsMenu || 'fileBaseExportListNoResults'); - } - - return this.prevMenu(); + return callback(err); } - return cb(err); - } - ); - }); - } - finishedLoading() { - this.prevMenu(); - } + const templateFiles = [ options.headerTemplate, options.entryTemplate ]; + async.map(templateFiles, (template, nextTemplate) => { + template = paths.isAbsolute(template) ? template : paths.join(Config.paths.misc, template); - prepareList(cb) { - const self = this; - - const statusView = self.viewControllers.main.getView(MciViewIds.main.status); - const updateStatus = (status) => { - if(statusView) { - statusView.setText(status); - } - }; - - const progBarView = self.viewControllers.main.getView(MciViewIds.main.progressBar); - const updateProgressBar = (curr, total) => { - if(progBarView) { - const prog = Math.floor( (curr / total) * progBarView.dimens.width ); - progBarView.setText(self.config.progBarChar.repeat(prog)); - } - }; - - async.waterfall( - [ - function readTemplateFiles(callback) { - updateStatus('Preparing'); - - async.map(TEMPLATE_KEYS, (templateKey, nextKey) => { - let templatePath = _.get(self.config, [ 'templates', templateKey ]); - templatePath = templatePath || `file_list_${templateKey}.asc`; - templatePath = paths.isAbsolute(templatePath) ? templatePath : paths.join(Config.paths.misc, templatePath); - - fs.readFile(templatePath, (err, data) => { - return nextKey(err, data); + fs.readFile(template, (err, data) => { + return nextTemplate(err, data); }); }, (err, templates) => { if(err) { @@ -154,7 +65,7 @@ exports.getModule = class FileBaseListExport extends MenuModule { } // decode + ensure DOS style CRLF - templates = templates.map(tmp => iconv.decode(tmp, self.config.templateEncoding).replace(/\r?\n/g, '\r\n') ); + templates = templates.map(tmp => iconv.decode(tmp, options.templateEncoding).replace(/\r?\n/g, '\r\n') ); // Look for the first {fileDesc} (if any) in 'entry' template & find indentation requirements let descIndent = 0; @@ -169,15 +80,16 @@ exports.getModule = class FileBaseListExport extends MenuModule { return callback(null, templates[0], templates[1], descIndent); }); - }, - function findFiles(headerTemplate, entryTemplate, descIndent, callback) { - const filterCriteria = Object.assign({}, self.config.filterCriteria); - if(!filterCriteria.areaTag) { - filterCriteria.areaTag = FileArea.getAvailableFileAreaTags(self.client); + }); + }, + function findFiles(headerTemplate, entryTemplate, descIndent, callback) { + state.step = 'gathering'; + state.status = 'Gathering files for supplied criteria'; + updateProgress(err => { + if(err) { + return callback(err); } - updateStatus('Gathering files for supplied criteria'); - FileEntry.findFiles(filterCriteria, (err, fileIds) => { if(0 === fileIds.length) { return callback(Errors.General('No results for criteria', 'NORESULTS')); @@ -185,229 +97,129 @@ exports.getModule = class FileBaseListExport extends MenuModule { return callback(err, headerTemplate, entryTemplate, descIndent, fileIds); }); - }, - function buildListEntries(headerTemplate, entryTemplate, descIndent, fileIds, callback) { - const formatObj = { - totalFileCount : fileIds.length, - }; - - let current = 0; - let listBody = ''; - const totals = { fileCount : fileIds.length, bytes : 0 }; - - // this may take quite a while; temp disable of idle monitor - self.client.stopIdleMonitor(); - - async.eachSeries(fileIds, (fileId, nextFileId) => { - const fileInfo = new FileEntry(); - current += 1; - - fileInfo.load(fileId, err => { - if(err) { - return nextFileId(null); // failed, but try the next - } - - updateStatus(`Processing ${fileInfo.fileName}`); - - totals.bytes += fileInfo.meta.byte_size; - - updateProgressBar(current, fileIds.length); - - const appendFileInfo = () => { - listBody += stringFormat(entryTemplate, formatObj); - - self.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, formatObj); - - return nextFileId(null); - }; - - const area = FileArea.getFileAreaByTag(fileInfo.areaTag); - - formatObj.fileId = fileId; - formatObj.areaName = _.get(area, 'name') || 'N/A'; - formatObj.areaDesc = _.get(area, 'desc') || 'N/A'; - formatObj.userRating = fileInfo.userRating || 0; - formatObj.fileName = fileInfo.fileName; - formatObj.fileSize = fileInfo.meta.byte_size; - formatObj.fileDesc = fileInfo.desc || ''; - formatObj.fileDescShort = formatObj.fileDesc.slice(0, self.config.descWidth); - formatObj.fileSha256 = fileInfo.fileSha256; - formatObj.fileCrc32 = fileInfo.meta.file_crc32; - formatObj.fileMd5 = fileInfo.meta.file_md5; - formatObj.fileSha1 = fileInfo.meta.file_sha1; - formatObj.uploadBy = fileInfo.meta.upload_by_username || 'N/A'; - formatObj.fileUploadTs = moment(fileInfo.uploadTimestamp).format(self.config.tsFormat); - formatObj.fileHashTags = fileInfo.hashTags.size > 0 ? Array.from(fileInfo.hashTags).join(', ') : 'N/A'; - formatObj.currentFile = current; - formatObj.progress = Math.floor( (current / fileIds.length) * 100 ); - - if(isAnsi(fileInfo.desc)) { - AnsiPrep( - fileInfo.desc, - { - cols : Math.min(self.config.descWidth, 79 - descIndent), - forceLineTerm : true, // ensure each line is term'd - asciiMode : true, // export to ASCII - fillLines : false, // don't fill up to |cols| - indent : descIndent, - }, - (err, desc) => { - if(desc) { - formatObj.fileDesc = desc; - } - return appendFileInfo(); - } - ); - } else { - const indentSpc = descIndent > 0 ? ' '.repeat(descIndent) : ''; - formatObj.fileDesc = splitTextAtTerms(formatObj.fileDesc).join(`\r\n${indentSpc}`) + '\r\n'; - return appendFileInfo(); - } - }); - }, err => { - // re-enable idle monitor - self.client.startIdleMonitor(); - - return callback(err, listBody, headerTemplate, totals); - }); - }, - function buildHeader(listBody, headerTemplate, totals, callback) { - // header is built last such that we can have totals/etc. - - let filterAreaName; - let filterAreaDesc; - if(self.config.filterCriteria.areaTag) { - const area = FileArea.getFileAreaByTag(self.config.filterCriteria.areaTag); - filterAreaName = _.get(area, 'name') || 'N/A'; - filterAreaDesc = _.get(area, 'desc') || 'N/A'; - } else { - filterAreaName = '-ALL-'; - filterAreaDesc = 'All areas'; - } - - const headerFormatObj = { - nowTs : moment().format(self.config.tsFormat), - boardName : Config.general.boardName, - totalFileCount : totals.fileCount, - totalFileSize : totals.bytes, - filterAreaTag : self.config.filterCriteria.areaTag || '-ALL-', - filterAreaName : filterAreaName, - filterAreaDesc : filterAreaDesc, - filterTerms : self.config.filterCriteria.terms || '(none)', - filterHashTags : self.config.filterCriteria.tags || '(none)', - }; - - listBody = stringFormat(headerTemplate, headerFormatObj) + listBody; - return callback(null, listBody); - }, - function persistList(listBody, callback) { - - updateStatus('Persisting list'); - - const sysTempDownloadArea = FileArea.getFileAreaByTag(FileArea.WellKnownAreaTags.TempDownloads); - const sysTempDownloadDir = FileArea.getAreaDefaultStorageDirectory(sysTempDownloadArea); - - fse.mkdirs(sysTempDownloadDir, err => { - if(err) { - return callback(err); - } - - const outputFileName = paths.join( - sysTempDownloadDir, - `file_list_${uuidv4().substr(-8)}_${moment().format('YYYY-MM-DD')}.txt` - ); - - fs.writeFile(outputFileName, listBody, 'utf8', err => { - if(err) { - return callback(err); - } - - self.getSizeAndCompressIfMeetsSizeThreshold(outputFileName, (err, finalOutputFileName, fileSize) => { - return callback(err, finalOutputFileName, fileSize, sysTempDownloadArea); - }); - }); - }); - }, - function persistFileEntry(outputFileName, fileSize, sysTempDownloadArea, callback) { - const newEntry = new FileEntry({ - areaTag : sysTempDownloadArea.areaTag, - fileName : paths.basename(outputFileName), - storageTag : sysTempDownloadArea.storageTags[0], - meta : { - upload_by_username : self.client.user.username, - upload_by_user_id : self.client.user.userId, - byte_size : fileSize, - session_temp_dl : 1, // download is valid until session is over - } - }); - - newEntry.desc = 'File List Export'; - - newEntry.persist(err => { - if(!err) { - // queue it! - const dlQueue = new DownloadQueue(self.client); - dlQueue.add(newEntry); - - // clean up after ourselves when the session ends - const thisClientId = self.client.session.id; - Events.once(Events.getSystemEvents().ClientDisconnected, evt => { - if(thisClientId === _.get(evt, 'client.session.id')) { - FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => { - if(err) { - Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' ); - } else { - Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' ); - } - }); - } - }); - } - return callback(err); - }); - }, - function done(callback) { - updateStatus('Exported list has been added to your download queue'); - return callback(null); - } - ], err => { - return cb(err); - } - ); - } - - getSizeAndCompressIfMeetsSizeThreshold(filePath, cb) { - fse.stat(filePath, (err, stats) => { - if(err) { - return cb(err); - } - - if(stats.size < this.config.compressThreshold) { - // small enough, keep orig - return cb(null, filePath, stats.size); - } - - const zipFilePath = `${filePath}.zip`; - - const zipFile = new yazl.ZipFile(); - zipFile.addFile(filePath, paths.basename(filePath)); - zipFile.end( () => { - const outZipFile = fs.createWriteStream(zipFilePath); - zipFile.outputStream.pipe(outZipFile); - zipFile.outputStream.on('finish', () => { - // delete the original - fse.unlink(filePath, err => { - if(err) { - return cb(err); - } - - // finally stat the new output - fse.stat(zipFilePath, (err, stats) => { - return cb(err, zipFilePath, stats ? stats.size : 0); - }); - }); }); - }); - }); - } -}; \ No newline at end of file + }, + function buildListEntries(headerTemplate, entryTemplate, descIndent, fileIds, callback) { + const formatObj = { + totalFileCount : fileIds.length, + }; + + let current = 0; + let listBody = ''; + const totals = { fileCount : fileIds.length, bytes : 0 }; + state.total = fileIds.length; + + state.step = 'file'; + + async.eachSeries(fileIds, (fileId, nextFileId) => { + const fileInfo = new FileEntry(); + current += 1; + + fileInfo.load(fileId, err => { + if(err) { + return nextFileId(null); // failed, but try the next + } + + totals.bytes += fileInfo.meta.byte_size; + + const appendFileInfo = () => { + listBody += stringFormat(entryTemplate, formatObj); + + state.current = current; + state.status = `Processing ${fileInfo.fileName}`; + state.fileInfo = formatObj; + + updateProgress(err => { + return nextFileId(err); + }); + }; + + const area = FileArea.getFileAreaByTag(fileInfo.areaTag); + + formatObj.fileId = fileId; + formatObj.areaName = _.get(area, 'name') || 'N/A'; + formatObj.areaDesc = _.get(area, 'desc') || 'N/A'; + formatObj.userRating = fileInfo.userRating || 0; + formatObj.fileName = fileInfo.fileName; + formatObj.fileSize = fileInfo.meta.byte_size; + formatObj.fileDesc = fileInfo.desc || ''; + formatObj.fileDescShort = formatObj.fileDesc.slice(0, options.descWidth); + formatObj.fileSha256 = fileInfo.fileSha256; + formatObj.fileCrc32 = fileInfo.meta.file_crc32; + formatObj.fileMd5 = fileInfo.meta.file_md5; + formatObj.fileSha1 = fileInfo.meta.file_sha1; + formatObj.uploadBy = fileInfo.meta.upload_by_username || 'N/A'; + formatObj.fileUploadTs = moment(fileInfo.uploadTimestamp).format(options.tsFormat); + formatObj.fileHashTags = fileInfo.hashTags.size > 0 ? Array.from(fileInfo.hashTags).join(', ') : 'N/A'; + formatObj.currentFile = current; + formatObj.progress = Math.floor( (current / fileIds.length) * 100 ); + + if(isAnsi(fileInfo.desc)) { + AnsiPrep( + fileInfo.desc, + { + cols : Math.min(options.descWidth, 79 - descIndent), + forceLineTerm : true, // ensure each line is term'd + asciiMode : true, // export to ASCII + fillLines : false, // don't fill up to |cols| + indent : descIndent, + }, + (err, desc) => { + if(desc) { + formatObj.fileDesc = desc; + } + return appendFileInfo(); + } + ); + } else { + const indentSpc = descIndent > 0 ? ' '.repeat(descIndent) : ''; + formatObj.fileDesc = splitTextAtTerms(formatObj.fileDesc).join(`\r\n${indentSpc}`) + '\r\n'; + return appendFileInfo(); + } + }); + }, err => { + return callback(err, listBody, headerTemplate, totals); + }); + }, + function buildHeader(listBody, headerTemplate, totals, callback) { + // header is built last such that we can have totals/etc. + + let filterAreaName; + let filterAreaDesc; + if(filterCriteria.areaTag) { + const area = FileArea.getFileAreaByTag(filterCriteria.areaTag); + filterAreaName = _.get(area, 'name') || 'N/A'; + filterAreaDesc = _.get(area, 'desc') || 'N/A'; + } else { + filterAreaName = '-ALL-'; + filterAreaDesc = 'All areas'; + } + + const headerFormatObj = { + nowTs : moment().format(options.tsFormat), + boardName : Config.general.boardName, + totalFileCount : totals.fileCount, + totalFileSize : totals.bytes, + filterAreaTag : filterCriteria.areaTag || '-ALL-', + filterAreaName : filterAreaName, + filterAreaDesc : filterAreaDesc, + filterTerms : filterCriteria.terms || '(none)', + filterHashTags : filterCriteria.tags || '(none)', + }; + + listBody = stringFormat(headerTemplate, headerFormatObj) + listBody; + return callback(null, listBody); + }, + function done(listBody, callback) { + delete state.fileInfo; + state.step = 'finished'; + state.status = 'Finished processing'; + updateProgress( () => { + return callback(null, listBody); + }); + } + ], (err, listBody) => { + return cb(err, listBody); + } + ); +};