diff --git a/art/themes/luciano_blocktronics/offline_mail.ans b/art/themes/luciano_blocktronics/offline_mail.ans new file mode 100644 index 00000000..ffd21e9e Binary files /dev/null and b/art/themes/luciano_blocktronics/offline_mail.ans differ diff --git a/art/themes/luciano_blocktronics/qwk_export_progress.ans b/art/themes/luciano_blocktronics/qwk_export_progress.ans new file mode 100644 index 00000000..dcde7d82 Binary files /dev/null and b/art/themes/luciano_blocktronics/qwk_export_progress.ans differ diff --git a/art/themes/luciano_blocktronics/theme.hjson b/art/themes/luciano_blocktronics/theme.hjson index 6d8490dc..d812a9c3 100644 --- a/art/themes/luciano_blocktronics/theme.hjson +++ b/art/themes/luciano_blocktronics/theme.hjson @@ -288,6 +288,17 @@ } } + qwkExportPacketCurrentConfig: { + mci: { + TL1: { + width: 70 + } + TL2: { + width: 70 + } + } + } + mailMenuCreateMessage: { 0: { mci: { diff --git a/core/download_queue.js b/core/download_queue.js index 28ca3aac..d80ddf34 100644 --- a/core/download_queue.js +++ b/core/download_queue.js @@ -1,11 +1,12 @@ /* jslint node: true */ 'use strict'; -const FileEntry = require('./file_entry.js'); -const UserProps = require('./user_property.js'); +const FileEntry = require('./file_entry'); +const UserProps = require('./user_property'); +const Events = require('./events'); // deps -const { partition } = require('lodash'); +const _ = require('lodash'); module.exports = class DownloadQueue { constructor(client) { @@ -20,6 +21,10 @@ module.exports = class DownloadQueue { } } + static get(client) { + return new DownloadQueue(client); + } + get items() { return this.client.user.downloadQueue; } @@ -52,7 +57,7 @@ module.exports = class DownloadQueue { fileIds = [ fileIds ]; } - const [ remain, removed ] = partition(this.client.user.downloadQueue, e => ( -1 === fileIds.indexOf(e.fileId) )); + const [ remain, removed ] = _.partition(this.client.user.downloadQueue, e => ( -1 === fileIds.indexOf(e.fileId) )); this.client.user.downloadQueue = remain; return removed; } @@ -76,4 +81,23 @@ module.exports = class DownloadQueue { this.client.log.error( { error : e.message, property : prop }, 'Failed parsing download queue property'); } } + + addTemporaryDownload(entry) { + this.add(entry, true); // true=systemFile + + // clean up after ourselves when the session ends + const thisClientId = this.client.session.id; + Events.once(Events.getSystemEvents().ClientDisconnected, evt => { + if(thisClientId === _.get(evt, 'client.session.id')) { + FileEntry.removeEntry(entry, { removePhysFile : true }, err => { + const Log = require('./logger').log; + if(err) { + Log.warn( { fileId : entry.fileId, path : entry.filePath }, 'Failed removing temporary session download' ); + } else { + Log.debug( { fileId : entry.fileId, path : entry.filePath }, 'Removed temporary session download item' ); + } + }); + } + }); + } }; diff --git a/core/enig_error.js b/core/enig_error.js index 08a3312e..be025214 100644 --- a/core/enig_error.js +++ b/core/enig_error.js @@ -37,6 +37,7 @@ exports.Errors = { MissingParam : (reason, reasonCode) => new EnigError('Missing paramter(s)', -32008, reason, reasonCode), MissingMci : (reason, reasonCode) => new EnigError('Missing required MCI code(s)', -32009, reason, reasonCode), BadLogin : (reason, reasonCode) => new EnigError('Bad login attempt', -32010, reason, reasonCode), + UserInterrupt : (reason, reasonCode) => new EnigError('User interrupted', -32011, reason, reasonCode), }; exports.ErrorReasons = { diff --git a/core/file_base_user_list_export.js b/core/file_base_user_list_export.js index 3c00d167..594a9ffe 100644 --- a/core/file_base_user_list_export.js +++ b/core/file_base_user_list_export.js @@ -7,8 +7,6 @@ const FileEntry = require('./file_entry.js'); const FileArea = require('./file_base_area.js'); const { renderSubstr } = require('./string_util.js'); const { Errors } = require('./enig_error.js'); -const Events = require('./events.js'); -const Log = require('./logger.js').log; const DownloadQueue = require('./download_queue.js'); const { exportFileList } = require('./file_base_list_export.js'); @@ -28,7 +26,7 @@ const yazl = require('yazl'); tsFormat - timestamp format (theme 'short') descWidth - max desc width (45) progBarChar - progress bar character (▒) - compressThreshold - threshold to kick in comrpession for lists (1.44 MiB) + compressThreshold - threshold to kick in compression for lists (1.44 MiB) templates - object containing: header - filename of header template (misc/file_list_header.asc) entry - filename of entry template (misc/file_list_entry.asc) @@ -222,28 +220,14 @@ exports.getModule = class FileBaseListExport extends MenuModule { newEntry.persist(err => { if(!err) { // queue it! - const dlQueue = new DownloadQueue(self.client); - dlQueue.add(newEntry, true); // true=systemFile - - // clean up after ourselves when the session ends - const thisClientId = self.client.session.id; - Events.once(Events.getSystemEvents().ClientDisconnected, evt => { - if(thisClientId === _.get(evt, 'client.session.id')) { - FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => { - if(err) { - Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' ); - } else { - Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' ); - } - }); - } - }); + DownloadQueue.get(self.client).addTemporaryDownload(newEntry); } return callback(err); }); }, function done(callback) { // re-enable idle monitor + // :TODO: this should probably be moved down below at the end of the full waterfall self.client.startIdleMonitor(); updateStatus('Exported list has been added to your download queue'); diff --git a/core/message_base_qwk_export.js b/core/message_base_qwk_export.js new file mode 100644 index 00000000..d8efcc1d --- /dev/null +++ b/core/message_base_qwk_export.js @@ -0,0 +1,386 @@ +// ENiGMA½ +const { MenuModule } = require('./menu_module'); +const Message = require('./message'); +const { Errors } = require('./enig_error'); +const { + getMessageAreaByTag, + hasMessageConfAndAreaRead, + getAllAvailableMessageAreaTags, +} = require('./message_area'); +const FileArea = require('./file_base_area'); +const { QWKPacketWriter } = require('./qwk_mail_packet'); +const { renderSubstr } = require('./string_util'); +const Config = require('./config').get; +const FileEntry = require('./file_entry'); +const DownloadQueue = require('./download_queue'); +const { Log } = require('./logger').log; + +// deps +const async = require('async'); +const _ = require('lodash'); +const fse = require('fs-extra'); +const temptmp = require('temptmp'); +const paths = require('path'); +const UUIDv4 = require('uuid/v4'); + +const FormIds = { + main : 0, +}; + +const MciViewIds = { + main : { + status : 1, + progressBar : 2, + + customRangeStart : 10, + } +}; + +exports.moduleInfo = { + name : 'QWK Export', + desc : 'Exports a QWK Packet for download', + author : 'NuSkooler', +}; + +exports.getModule = class MessageBaseQWKExport extends MenuModule { + constructor(options) { + super(options); + + this.config = Object.assign({}, _.get(options, 'menuConfig.config'), options.extraArgs); + + this.config.progBarChar = renderSubstr( (this.config.progBarChar || '▒'), 0, 1); + this.config.bbsID = this.config.bbsID || _.get(Config(), 'messageNetworks.qwk.bbsID', 'ENIGMA'); + + this.tempName = `${UUIDv4().substr(-8).toUpperCase()}.QWK`; + this.sysTempDownloadArea = FileArea.getFileAreaByTag(FileArea.WellKnownAreaTags.TempDownloads); + } + + mciReady(mciData, cb) { + super.mciReady(mciData, err => { + if (err) { + return cb(err); + } + + async.waterfall( + [ + (callback) => { + this.prepViewController('main', FormIds.main, mciData.menu, err => { + return callback(err); + }); + }, + (callback) => { + this.temptmp = temptmp.createTrackedSession('qwkuserexp'); + this.temptmp.mkdir({ prefix : 'enigqwkwriter-'}, (err, tempDir) => { + if (err) { + return callback(err); + } + + this.tempPacketDir = tempDir; + + const sysTempDownloadDir = FileArea.getAreaDefaultStorageDirectory(this.sysTempDownloadArea); + + // ensure dir exists + fse.mkdirs(sysTempDownloadDir, err => { + return callback(err, sysTempDownloadDir); + }); + }); + }, + (sysTempDownloadDir, callback) => { + this._performExport(sysTempDownloadDir, err => { + return callback(err); + }); + }, + ], + err => { + this.temptmp.cleanup(); + + if (err) { + // :TODO: doesn't do anything currently: + if ('NORESULTS' === err.reasonCode) { + return this.gotoMenu(this.menuConfig.config.noResultsMenu || 'qwkExportNoResults'); + } + + return this.prevMenu(); + } + return cb(err); + } + ); + }); + } + + finishedLoading() { + this.prevMenu(); + } + + _getUserQWKExportOptions() { + let qwkOptions = this.client.user.getProperty('qwk_export_options'); + try { + qwkOptions = JSON.parse(qwkOptions); + } catch(e) { + qwkOptions = { + enableQWKE : true, + enableHeadersExtension : true, + enableAtKludges : true, + archiveFormat : 'application/zip', + }; + } + return qwkOptions; + } + + _getUserQWKExportAreas() { + let qwkExportAreas = this.client.user.getProperty('qwk_export_msg_areas'); + try { + qwkExportAreas = JSON.parse(qwkExportAreas); + } catch(e) { + // default to all public and private without 'since' + qwkExportAreas = getAllAvailableMessageAreaTags(this.client).map(areaTag => { + return { areaTag }; + }); + + // Include user's private area + qwkExportAreas.push({ + areaTag : Message.WellKnownAreaTags.Private, + }); + } + + return qwkExportAreas; + } + + _performExport(sysTempDownloadDir, cb) { + const statusView = this.viewControllers.main.getView(MciViewIds.main.status); + const updateStatus = (status) => { + if (statusView) { + statusView.setText(status); + } + }; + + const progBarView = this.viewControllers.main.getView(MciViewIds.main.progressBar); + const updateProgressBar = (curr, total) => { + if (progBarView) { + const prog = Math.floor( (curr / total) * progBarView.dimens.width ); + progBarView.setText(this.config.progBarChar.repeat(prog)); + } + }; + + let cancel = false; + + let lastProgUpdate = 0; + const progressHandler = (state, next) => { + // we can produce a TON of updates; only update progress at most every 3/4s + if (Date.now() - lastProgUpdate > 750) { + switch (state.step) { + case 'next_area' : + updateStatus(state.status); + updateProgressBar(0, 0); + this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.area); + break; + + case 'message' : + updateStatus(state.status); + updateProgressBar(state.current, state.total); + this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.message); + break; + + default : + break; + } + lastProgUpdate = Date.now(); + } + + return next(cancel ? Errors.UserInterrupt('User canceled') : null); + }; + + const keyPressHandler = (ch, key) => { + if('escape' === key.name) { + cancel = true; + this.client.removeListener('key press', keyPressHandler); + } + }; + + const processMessagesWithFilter = (filter, cb) => { + Message.findMessages(filter, (err, messageIds) => { + if (err) { + return cb(err); + } + + let current = 1; + async.eachSeries(messageIds, (messageId, nextMessageId) => { + const message = new Message(); + message.load({ messageId }, err => { + if (err) { + return nextMessageId(err); + } + + const progress = { + current, + message, + step : 'message', + total : messageIds.length, + status : `Writing message ${current} / ${messageIds.length}`, + }; + + progressHandler(progress, err => { + if (err) { + return nextMessageId(err); + } + + packetWriter.appendMessage(message); + current += 1; + + return nextMessageId(null); + }); + }); + }, + err => { + return cb(err); + }); + }); + }; + + const packetWriter = new QWKPacketWriter( + Object.assign(this._getUserQWKExportOptions(), { + user : this.client.user, + bbsID : this.config.bbsID, + }) + ); + + packetWriter.on('warning', warning => { + this.client.log.warn( { warning }, 'QWK packet writer warning'); + }); + + async.waterfall( + [ + (callback) => { + // don't count idle monitor while processing + this.client.stopIdleMonitor(); + + // let user cancel + this.client.on('key press', keyPressHandler); + + packetWriter.once('ready', () => { + return callback(null); + }); + + packetWriter.once('error', err => { + this.client.log.error( { error : err.message }, 'QWK packet writer error'); + cancel = true; + }); + + packetWriter.init(); + }, + (callback) => { + // For each public area -> for each message + const userExportAreas = this._getUserQWKExportAreas(); + async.eachSeries(userExportAreas, (exportArea, nextExportArea) => { + const area = getMessageAreaByTag(exportArea.areaTag); + if (!area) { + // :TODO: remove from user properties - this area does not exist + this.client.log.warn({ areaTag : exportArea.areaTag }, 'Cannot QWK export area as it does not exist'); + return nextExportArea(null); + } + + if (!hasMessageConfAndAreaRead(this.client, area)) { + this.client.log.warn({ areaTag : area.areaTag }, 'Cannot QWK export area due to ACS'); + return nextExportArea(null); + } + + const progress = { + area, + step : 'next_area', + status : `Gathering messages in ${area.name}...`, + }; + + progressHandler(progress, err => { + if (err) { + return nextExportArea(err); + } + + const filter = { + resultType : 'id', + areaTag : exportArea.areaTag, + newerThanTimestamp : exportArea.newerThanTimestamp + }; + + processMessagesWithFilter(filter, err => { + return nextExportArea(err); + }); + }); + }, + err => { + return callback(err, userExportAreas); + }); + }, + (userExportAreas, callback) => { + // Private messages to current user if the user has + // elected to export private messages + if (!(userExportAreas.find(exportArea => exportArea.areaTag === Message.WellKnownAreaTags.Private))) { + return callback(null); + } + + const filter = { + resultType : 'id', + privateTagUserId : this.client.user.userId, + // :TODO: newerThanTimestamp for private messages + //newerThanTimestamp : exportArea.newerThanTimestamp + }; + return processMessagesWithFilter(filter, callback); + }, + (callback) => { + let packetInfo; + packetWriter.once('packet', info => { + packetInfo = info; + }); + + packetWriter.once('finished', () => { + return callback(null, packetInfo); + }); + + packetWriter.finish(this.tempPacketDir); + }, + (packetInfo, callback) => { + const sysDownloadPath = paths.join(sysTempDownloadDir, this.tempName); + fse.move(packetInfo.path, sysDownloadPath, err => { + return callback(null, sysDownloadPath, packetInfo); + }); + }, + (sysDownloadPath, packetInfo, callback) => { + const newEntry = new FileEntry({ + areaTag : this.sysTempDownloadArea.areaTag, + fileName : paths.basename(sysDownloadPath), + storageTag : this.sysTempDownloadArea.storageTags[0], + meta : { + upload_by_username : this.client.user.username, + upload_by_user_id : this.client.user.userId, + byte_size : packetInfo.stats.size, + session_temp_dl : 1, // download is valid until session is over + + // :TODO: something like this: allow to override the displayed/downloaded as filename + // separate from the actual on disk filename. E.g. we could always download as "ENIGMA.QWK" + //visible_filename : paths.basename(packetInfo.path), + } + }); + + newEntry.desc = 'QWK Export'; + + newEntry.persist(err => { + if(!err) { + // queue it! + DownloadQueue.get(this.client).addTemporaryDownload(newEntry); + } + return callback(err); + }); + } + ], + err => { + this.client.startIdleMonitor(); // re-enable + this.client.removeListener('key press', keyPressHandler); + + if (!err) { + updateStatus('A QWK packet has been placed in your download queue'); + } + + return cb(err); + } + ); + } +}; \ No newline at end of file diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index eec737cb..3d613ca4 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -1242,7 +1242,12 @@ class QWKPacketWriter extends EventEmitter { files, this.workDir, err => { - return cb(err); + fs.stat(packetPath, (err, stats) => { + if (stats) { + this.emit('packet', { stats, path : packetPath } ); + } + return cb(err); + }); } ); }); diff --git a/core/user_property.js b/core/user_property.js index cc68ef09..d55a0ebe 100644 --- a/core/user_property.js +++ b/core/user_property.js @@ -29,7 +29,7 @@ module.exports = { UserComment : 'user_comment', // NYI AutoSignature : 'auto_signature', - DownloadQueue : 'dl_queue', // download_queue.js + DownloadQueue : 'dl_queue', // see download_queue.js FailedLoginAttempts : 'failed_login_attempts', AccountLockedTs : 'account_locked_timestamp', @@ -64,5 +64,6 @@ module.exports = { AuthFactor2OTP : 'auth_factor2_otp', // If present, OTP type for 2FA. See OTPTypes AuthFactor2OTPSecret : 'auth_factor2_otp_secret', // Secret used in conjunction with OTP 2FA AuthFactor2OTPBackupCodes : 'auth_factor2_otp_backup', // JSON array of backup codes + };