diff --git a/core/bbs.js b/core/bbs.js index c2f54802..33301f70 100644 --- a/core/bbs.js +++ b/core/bbs.js @@ -259,6 +259,9 @@ function initialize(cb) { function listenConnections(callback) { return require('./listening_server.js').startup(callback); }, + function readyFileBaseArea(callback) { + return require('./file_base_area.js').startup(callback); + }, function readyFileAreaWeb(callback) { return require('./file_area_web.js').startup(callback); }, diff --git a/core/file_base_area_select.js b/core/file_base_area_select.js index f762ab1d..df859bea 100644 --- a/core/file_base_area_select.js +++ b/core/file_base_area_select.js @@ -65,7 +65,7 @@ exports.getModule = class FileAreaSelectModule extends MenuModule { return callback(null, availAreas); }, function prepView(availAreas, callback) { - self.prepViewController('allViews', 0, { mciMap : mciData.menu }, (err, vc) => { + self.prepViewController('allViews', 0, mciData.menu, (err, vc) => { if(err) { return callback(err); } diff --git a/core/file_base_list_export.js b/core/file_base_list_export.js new file mode 100644 index 00000000..f3f185cd --- /dev/null +++ b/core/file_base_list_export.js @@ -0,0 +1,343 @@ +/* jslint node: true */ +'use strict'; + +// ENiGMA½ +const { MenuModule } = require('./menu_module.js'); +const stringFormat = require('./string_format.js'); +const FileEntry = require('./file_entry.js'); +const FileArea = require('./file_base_area.js'); +const Config = require('./config.js').config; +const { Errors } = require('./enig_error.js'); +const { + splitTextAtTerms, + isAnsi +} = require('./string_util.js'); +const AnsiPrep = require('./ansi_prep.js'); +const Events = require('./events.js'); +const Log = require('./logger.js').log; +const DownloadQueue = require('./download_queue.js'); + +// deps +const _ = require('lodash'); +const async = require('async'); +const fs = require('graceful-fs'); +const fse = require('fs-extra'); +const paths = require('path'); +const iconv = require('iconv-lite'); +const moment = require('moment'); +const uuidv4 = require('uuid/v4'); + +/* + :TODO: document: + - config options + - header template + - entry template + - header format + - entry format +*/ + +// :TODO: compress lists > N + +exports.moduleInfo = { + name : 'File Base List Export', + desc : 'Exports file base listings for download', + author : 'NuSkooler', +}; + +const FormIds = { + main : 0, +}; + +const MciViewIds = { + main : { + status : 1, + progressBar : 2, + + customRangeStart : 10, + } +}; + +const TEMPLATE_KEYS = [ // config.templates.* + 'header', 'entry', +]; + +exports.getModule = class FileBaseListExport extends MenuModule { + + constructor(options) { + super(options); + this.config = Object.assign({}, _.get(options, 'menuConfig.config'), options.extraArgs); + + this.config.templateEncoding = this.config.templateEncoding || 'utf8'; + this.config.tsFormat = this.config.tsFormat || this.client.currentTheme.helpers.getDateTimeFormat('short'); + this.config.descWidth = this.config.descWidth || 45; // ie FILE_ID.DIZ + this.config.progBarChar = (this.config.progBarChar || '▒').charAt(0); + } + + mciReady(mciData, cb) { + super.mciReady(mciData, err => { + if(err) { + return cb(err); + } + + async.series( + [ + (callback) => this.prepViewController('main', FormIds.main, mciData.menu, callback), + (callback) => this.prepareList(callback), + ], + err => { + return cb(err); + } + ); + }); + } + + finishedLoading() { + this.prevMenu(); + } + + prepareList(cb) { + const self = this; + + const statusView = self.viewControllers.main.getView(MciViewIds.main.status); + const updateStatus = (status) => { + if(statusView) { + statusView.setText(status); + } + }; + + const progBarView = self.viewControllers.main.getView(MciViewIds.main.progressBar); + const updateProgressBar = (curr, total) => { + if(progBarView) { + const prog = Math.floor( (curr / total) * progBarView.dimens.width ); + progBarView.setText(self.config.progBarChar.repeat(prog)); + } + }; + + async.waterfall( + [ + function readTemplateFiles(callback) { + updateStatus('Preparing'); + + async.map(TEMPLATE_KEYS, (templateKey, nextKey) => { + let templatePath = _.get(self.config, [ 'templates', templateKey ]); + templatePath = templatePath || `file_list_${templateKey}.asc`; + templatePath = paths.isAbsolute(templatePath) ? templatePath : paths.join(Config.paths.misc, templatePath); + + fs.readFile(templatePath, (err, data) => { + return nextKey(err, data); + }); + }, (err, templates) => { + if(err) { + return Errors.General(err.message); + } + + // decode + ensure DOS style CRLF + templates = templates.map(tmp => iconv.decode(tmp, self.config.templateEncoding).replace(/\r?\n/g, '\r\n') ); + + // Look for the first {fileDesc} (if any) in 'entry' template & find indentation requirements + let descIndent = 0; + splitTextAtTerms(templates[1]).some(line => { + const pos = line.indexOf('{fileDesc}'); + if(pos > -1) { + descIndent = pos; + return true; // found it! + } + return false; // keep looking + }); + + return callback(null, templates[0], templates[1], descIndent); + }); + }, + function findFiles(headerTemplate, entryTemplate, descIndent, callback) { + const filterCriteria = Object.assign({}, self.config.filterCriteria); + if(!filterCriteria.areaTag) { + filterCriteria.areaTag = FileArea.getAvailableFileAreaTags(self.client); + } + + updateStatus('Gathering files for supplied criteria'); + + FileEntry.findFiles(filterCriteria, (err, fileIds) => { + // :TODO: handle empty file IDs -- bail early. + return callback(err, headerTemplate, entryTemplate, descIndent, fileIds); + }); + }, + function buildListEntries(headerTemplate, entryTemplate, descIndent, fileIds, callback) { + const formatObj = { + totalFileCount : fileIds.length, + }; + + const fileInfo = new FileEntry(); + let current = 0; + let listBody = ''; + const totals = { fileCount : fileIds.length, bytes : 0 }; + + async.eachSeries(fileIds, (fileId, nextFileId) => { + current += 1; + + fileInfo.load(fileId, err => { + if(err) { + return nextFileId(null); // failed, but try the next + } + + updateStatus(`Processing ${fileInfo.fileName}`); + + totals.bytes += fileInfo.meta.byte_size; + + updateProgressBar(current, fileIds.length); + + const appendFileInfo = () => { + listBody += stringFormat(entryTemplate, formatObj); + + self.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, formatObj); + + return nextFileId(null); + }; + + const area = FileArea.getFileAreaByTag(fileInfo.areaTag); + + formatObj.fileId = fileId; + formatObj.areaName = _.get(area, 'name') || 'N/A'; + formatObj.areaDesc = _.get(area, 'desc') || 'N/A'; + formatObj.userRating = fileInfo.userRating || 0; + formatObj.fileName = fileInfo.fileName; + formatObj.fileSize = fileInfo.meta.byte_size; + formatObj.fileDesc = fileInfo.desc || ''; + formatObj.fileDescShort = formatObj.fileDesc.slice(0, self.config.descWidth); + formatObj.fileSha256 = fileInfo.fileSha256; + formatObj.fileCrc32 = fileInfo.meta.file_crc32; + formatObj.fileMd5 = fileInfo.meta.file_md5; + formatObj.fileSha1 = fileInfo.meta.file_sha1; + formatObj.uploadBy = fileInfo.meta.upload_by_username || 'N/A'; + formatObj.fileUploadTs = moment(fileInfo.uploadTimestamp).format(self.config.tsFormat); + formatObj.fileHashTags = fileInfo.hashTags.size > 0 ? Array.from(fileInfo.hashTags).join(', ') : 'N/A'; + formatObj.currentFile = current; + formatObj.progress = Math.floor( (current / fileIds.length) * 100 ); + + if(isAnsi(fileInfo.desc)) { + AnsiPrep( + fileInfo.desc, + { + cols : Math.min(self.config.descWidth, 79 - descIndent), + forceLineTerm : true, // ensure each line is term'd + asciiMode : true, // export to ASCII + fillLines : false, // don't fill up to |cols| + indent : descIndent, + }, + (err, desc) => { + if(desc) { + formatObj.fileDesc = desc; + } + return appendFileInfo(); + } + ); + } else { + const indentSpc = descIndent > 0 ? ' '.repeat(descIndent) : ''; + formatObj.fileDesc = splitTextAtTerms(formatObj.fileDesc).join(`\r\n${indentSpc}`) + '\r\n'; + return appendFileInfo(); + } + }); + }, err => { + return callback(err, listBody, headerTemplate, totals); + }); + }, + function buildHeader(listBody, headerTemplate, totals, callback) { + // header is built last such that we can have totals/etc. + + let filterAreaName; + let filterAreaDesc; + if(self.config.filterCriteria.areaTag) { + const area = FileArea.getFileAreaByTag(self.config.filterCriteria.areaTag); + filterAreaName = _.get(area, 'name') || 'N/A'; + filterAreaDesc = _.get(area, 'desc') || 'N/A'; + } else { + filterAreaName = '-ALL-'; + filterAreaDesc = 'All areas'; + } + + const headerFormatObj = { + nowTs : moment().format(self.config.tsFormat), + boardName : Config.general.boardName, + totalFileCount : totals.fileCount, + totalFileSize : totals.bytes, + filterAreaTag : self.config.filterCriteria.areaTag || '-ALL-', + filterAreaName : filterAreaName, + filterAreaDesc : filterAreaDesc, + filterTerms : self.config.filterCriteria.terms || '(none)', + filterHashTags : self.config.filterCriteria.tags || '(none)', + }; + + listBody = stringFormat(headerTemplate, headerFormatObj) + listBody; + return callback(null, listBody); + }, + function persistList(listBody, callback) { + + updateStatus('Persisting list'); + + const sysTempDownloadArea = FileArea.getFileAreaByTag(FileArea.WellKnownAreaTags.TempDownloads); + const sysTempDownloadDir = FileArea.getAreaDefaultStorageDirectory(sysTempDownloadArea); + + fse.mkdirs(sysTempDownloadDir, err => { + if(err) { + return callback(err); + } + + const outputFileName = paths.join( + sysTempDownloadDir, + `file_list_${uuidv4()}.txt` + ); + + fs.writeFile(outputFileName, listBody, 'utf8', err => { + return callback(err, outputFileName, sysTempDownloadArea); + }); + }); + }, + function persistFileEntry(outputFileName, sysTempDownloadArea, callback) { + fse.stat(outputFileName, (err, stats) => { + const newEntry = new FileEntry({ + areaTag : sysTempDownloadArea.areaTag, + fileName : paths.basename(outputFileName), + storageTag : sysTempDownloadArea.storageTags[0], + meta : { + upload_by_username : self.client.user.username, + upload_by_user_id : self.client.user.userId, + byte_size : stats.size, + session_temp_dl : 1, // download is valid until session is over + } + }); + + newEntry.desc = 'File List Export'; + + newEntry.persist(err => { + if(!err) { + // queue it! + const dlQueue = new DownloadQueue(self.client); + dlQueue.add(newEntry); + + // clean up after ourselves when the session ends + const thisClientId = self.client.session.id; + Events.once('codes.l33t.enigma.system.disconnected', evt => { // :TODO: Make a enum for system events/etc. + if(thisClientId === _.get(evt, 'client.session.id')) { + FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => { + if(err) { + Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' ); + } else { + Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' ); + } + }); + } + }); + } + return callback(err); + }); + }); + }, + function done(callback) { + updateStatus('Exported list has been added to your download queue'); + return callback(null); + } + ], err => { + return cb(err); + } + ); + } +}; \ No newline at end of file diff --git a/core/menu_module.js b/core/menu_module.js index a6fb3b4d..97536bc7 100644 --- a/core/menu_module.js +++ b/core/menu_module.js @@ -311,7 +311,7 @@ exports.MenuModule = class MenuModule extends PluginModule { ); } - prepViewController(name, formId, artData, cb) { + prepViewController(name, formId, mciMap, cb) { if(_.isUndefined(this.viewControllers[name])) { const vcOpts = { client : this.client, @@ -322,7 +322,7 @@ exports.MenuModule = class MenuModule extends PluginModule { const loadOpts = { callingMenu : this, - mciMap : artData.mciMap, + mciMap : mciMap, formId : formId, }; @@ -345,7 +345,7 @@ exports.MenuModule = class MenuModule extends PluginModule { return cb(err); } - return this.prepViewController(name, formId, artData, cb); + return this.prepViewController(name, formId, artData.mciMap, cb); } ); } diff --git a/core/message_base_search.js b/core/message_base_search.js index b0259490..fdb17859 100644 --- a/core/message_base_search.js +++ b/core/message_base_search.js @@ -49,7 +49,7 @@ exports.getModule = class MessageBaseSearch extends MenuModule { return cb(err); } - this.prepViewController('search', 0, { mciMap : mciData.menu }, (err, vc) => { + this.prepViewController('search', 0, mciData.menu, (err, vc) => { if(err) { return cb(err); }