Merge pull request #278 from NuSkooler/qwk-menu

Qwk menu

art/themes/luciano_blocktronics/theme.hjson

@@ -288,6 +288,17 @@
 				}
 			}
 
+			qwkExportPacketCurrentConfig: {
+				mci: {
+					TL1: {
+						width: 70
+					}
+					TL2: {
+						width: 70
+					}
+				}
+			}
+
 			mailMenuCreateMessage: {
 				0: {
 					mci: {

core/download_queue.js

@@ -1,11 +1,12 @@
 /* jslint node: true */
 'use strict';
 
-const FileEntry     = require('./file_entry.js');
+const FileEntry     = require('./file_entry');
-const UserProps     = require('./user_property.js');
+const UserProps     = require('./user_property');
+const Events        = require('./events');
 
 //  deps
-const { partition } = require('lodash');
+const _ = require('lodash');
 
 module.exports = class DownloadQueue {
     constructor(client) {
@@ -20,6 +21,10 @@ module.exports = class DownloadQueue {
         }
     }
 
+    static get(client) {
+        return new DownloadQueue(client);
+    }
+
     get items() {
         return this.client.user.downloadQueue;
     }
@@ -52,7 +57,7 @@ module.exports = class DownloadQueue {
             fileIds = [ fileIds ];
         }
 
-        const [ remain, removed ] = partition(this.client.user.downloadQueue, e => ( -1 === fileIds.indexOf(e.fileId) ));
+        const [ remain, removed ] = _.partition(this.client.user.downloadQueue, e => ( -1 === fileIds.indexOf(e.fileId) ));
         this.client.user.downloadQueue = remain;
         return removed;
     }
@@ -76,4 +81,23 @@ module.exports = class DownloadQueue {
             this.client.log.error( { error : e.message, property : prop }, 'Failed parsing download queue property');
         }
     }
+
+    addTemporaryDownload(entry) {
+        this.add(entry, true);    //  true=systemFile
+
+        //  clean up after ourselves when the session ends
+        const thisClientId = this.client.session.id;
+        Events.once(Events.getSystemEvents().ClientDisconnected, evt => {
+            if(thisClientId === _.get(evt, 'client.session.id')) {
+                FileEntry.removeEntry(entry, { removePhysFile : true }, err => {
+                    const Log = require('./logger').log;
+                    if(err) {
+                        Log.warn( { fileId : entry.fileId, path : entry.filePath }, 'Failed removing temporary session download' );
+                    } else {
+                        Log.debug( { fileId : entry.fileId, path : entry.filePath }, 'Removed temporary session download item' );
+                    }
+                });
+            }
+        });
+    }
 };

core/enig_error.js

@@ -37,6 +37,7 @@ exports.Errors = {
     MissingParam        : (reason, reasonCode)  => new EnigError('Missing paramter(s)', -32008, reason, reasonCode),
     MissingMci          : (reason, reasonCode)  => new EnigError('Missing required MCI code(s)', -32009, reason, reasonCode),
     BadLogin            : (reason, reasonCode)  => new EnigError('Bad login attempt', -32010, reason, reasonCode),
+    UserInterrupt       : (reason, reasonCode)  => new EnigError('User interrupted', -32011, reason, reasonCode),
 };
 
 exports.ErrorReasons = {

core/file_base_user_list_export.js

@@ -7,8 +7,6 @@ const FileEntry             = require('./file_entry.js');
 const FileArea              = require('./file_base_area.js');
 const { renderSubstr }      = require('./string_util.js');
 const { Errors }            = require('./enig_error.js');
-const Events                = require('./events.js');
-const Log                   = require('./logger.js').log;
 const DownloadQueue         = require('./download_queue.js');
 const { exportFileList }    = require('./file_base_list_export.js');
 
@@ -28,7 +26,7 @@ const yazl              = require('yazl');
     tsFormat            - timestamp format (theme 'short')
     descWidth           - max desc width (45)
     progBarChar         - progress bar character (▒)
-    compressThreshold   - threshold to kick in comrpession for lists (1.44 MiB)
+    compressThreshold   - threshold to kick in compression for lists (1.44 MiB)
     templates           - object containing:
         header          - filename of header template (misc/file_list_header.asc)
         entry           - filename of entry template (misc/file_list_entry.asc)
@@ -222,28 +220,14 @@ exports.getModule = class FileBaseListExport extends MenuModule {
                     newEntry.persist(err => {
                         if(!err) {
                             //  queue it!
-                            const dlQueue = new DownloadQueue(self.client);
+                            DownloadQueue.get(self.client).addTemporaryDownload(newEntry);
-                            dlQueue.add(newEntry, true);    //  true=systemFile
-
-                            //  clean up after ourselves when the session ends
-                            const thisClientId = self.client.session.id;
-                            Events.once(Events.getSystemEvents().ClientDisconnected, evt => {
-                                if(thisClientId === _.get(evt, 'client.session.id')) {
-                                    FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => {
-                                        if(err) {
-                                            Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' );
-                                        } else {
-                                            Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' );
-                                        }
-                                    });
-                                }
-                            });
                         }
                         return callback(err);
                     });
                 },
                 function done(callback) {
                     //  re-enable idle monitor
+                    //  :TODO: this should probably be moved down below at the end of the full waterfall
                     self.client.startIdleMonitor();
 
                     updateStatus('Exported list has been added to your download queue');

core/message_base_qwk_export.js

@@ -0,0 +1,386 @@
+//  ENiGMA½
+const { MenuModule } = require('./menu_module');
+const Message = require('./message');
+const { Errors } = require('./enig_error');
+const {
+    getMessageAreaByTag,
+    hasMessageConfAndAreaRead,
+    getAllAvailableMessageAreaTags,
+} = require('./message_area');
+const FileArea = require('./file_base_area');
+const { QWKPacketWriter } = require('./qwk_mail_packet');
+const { renderSubstr } = require('./string_util');
+const Config = require('./config').get;
+const FileEntry = require('./file_entry');
+const DownloadQueue = require('./download_queue');
+const { Log } = require('./logger').log;
+
+//  deps
+const async = require('async');
+const _ = require('lodash');
+const fse = require('fs-extra');
+const temptmp = require('temptmp');
+const paths = require('path');
+const UUIDv4 = require('uuid/v4');
+
+const FormIds = {
+    main    : 0,
+};
+
+const MciViewIds = {
+    main : {
+        status              : 1,
+        progressBar         : 2,
+
+        customRangeStart    : 10,
+    }
+};
+
+exports.moduleInfo = {
+    name    : 'QWK Export',
+    desc    : 'Exports a QWK Packet for download',
+    author  : 'NuSkooler',
+};
+
+exports.getModule = class MessageBaseQWKExport extends MenuModule {
+    constructor(options) {
+        super(options);
+
+        this.config = Object.assign({}, _.get(options, 'menuConfig.config'), options.extraArgs);
+
+        this.config.progBarChar = renderSubstr( (this.config.progBarChar || '▒'), 0, 1);
+        this.config.bbsID       = this.config.bbsID || _.get(Config(), 'messageNetworks.qwk.bbsID', 'ENIGMA');
+
+        this.tempName = `${UUIDv4().substr(-8).toUpperCase()}.QWK`;
+        this.sysTempDownloadArea = FileArea.getFileAreaByTag(FileArea.WellKnownAreaTags.TempDownloads);
+    }
+
+    mciReady(mciData, cb) {
+        super.mciReady(mciData, err => {
+            if (err) {
+                return cb(err);
+            }
+
+            async.waterfall(
+                [
+                    (callback) => {
+                        this.prepViewController('main', FormIds.main, mciData.menu, err => {
+                            return callback(err);
+                        });
+                    },
+                    (callback) => {
+                        this.temptmp = temptmp.createTrackedSession('qwkuserexp');
+                        this.temptmp.mkdir({ prefix : 'enigqwkwriter-'}, (err, tempDir) => {
+                            if (err) {
+                                return callback(err);
+                            }
+
+                            this.tempPacketDir = tempDir;
+
+                            const sysTempDownloadDir = FileArea.getAreaDefaultStorageDirectory(this.sysTempDownloadArea);
+
+                            //  ensure dir exists
+                            fse.mkdirs(sysTempDownloadDir, err => {
+                                return callback(err, sysTempDownloadDir);
+                            });
+                        });
+                    },
+                    (sysTempDownloadDir, callback) => {
+                        this._performExport(sysTempDownloadDir, err => {
+                            return callback(err);
+                        });
+                    },
+                ],
+                err => {
+                    this.temptmp.cleanup();
+
+                    if (err) {
+                        //  :TODO: doesn't do anything currently:
+                        if ('NORESULTS' === err.reasonCode) {
+                            return this.gotoMenu(this.menuConfig.config.noResultsMenu || 'qwkExportNoResults');
+                        }
+
+                        return this.prevMenu();
+                    }
+                    return cb(err);
+                }
+            );
+        });
+    }
+
+    finishedLoading() {
+        this.prevMenu();
+    }
+
+    _getUserQWKExportOptions() {
+        let qwkOptions = this.client.user.getProperty('qwk_export_options');
+        try {
+            qwkOptions = JSON.parse(qwkOptions);
+        } catch(e) {
+            qwkOptions = {
+                enableQWKE              : true,
+                enableHeadersExtension  : true,
+                enableAtKludges         : true,
+                archiveFormat           : 'application/zip',
+            };
+        }
+        return qwkOptions;
+    }
+
+    _getUserQWKExportAreas() {
+        let qwkExportAreas = this.client.user.getProperty('qwk_export_msg_areas');
+        try {
+            qwkExportAreas = JSON.parse(qwkExportAreas);
+        } catch(e) {
+            //  default to all public and private without 'since'
+            qwkExportAreas = getAllAvailableMessageAreaTags(this.client).map(areaTag => {
+                return { areaTag };
+            });
+
+            //  Include user's private area
+            qwkExportAreas.push({
+                areaTag : Message.WellKnownAreaTags.Private,
+            });
+        }
+
+        return qwkExportAreas;
+    }
+
+    _performExport(sysTempDownloadDir, cb) {
+        const statusView = this.viewControllers.main.getView(MciViewIds.main.status);
+        const updateStatus = (status) => {
+            if (statusView) {
+                statusView.setText(status);
+            }
+        };
+
+        const progBarView = this.viewControllers.main.getView(MciViewIds.main.progressBar);
+        const updateProgressBar = (curr, total) => {
+            if (progBarView) {
+                const prog = Math.floor( (curr / total) * progBarView.dimens.width );
+                progBarView.setText(this.config.progBarChar.repeat(prog));
+            }
+        };
+
+        let cancel = false;
+
+        let lastProgUpdate = 0;
+        const progressHandler = (state, next) => {
+            //  we can produce a TON of updates; only update progress at most every 3/4s
+            if (Date.now() - lastProgUpdate > 750) {
+                switch (state.step) {
+                    case 'next_area' :
+                        updateStatus(state.status);
+                        updateProgressBar(0, 0);
+                        this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.area);
+                        break;
+
+                    case 'message' :
+                        updateStatus(state.status);
+                        updateProgressBar(state.current, state.total);
+                        this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.message);
+                        break;
+
+                    default :
+                        break;
+                }
+                lastProgUpdate = Date.now();
+            }
+
+            return next(cancel ? Errors.UserInterrupt('User canceled') : null);
+        };
+
+        const keyPressHandler = (ch, key) => {
+            if('escape' === key.name) {
+                cancel = true;
+                this.client.removeListener('key press', keyPressHandler);
+            }
+        };
+
+        const processMessagesWithFilter = (filter, cb) => {
+            Message.findMessages(filter, (err, messageIds) => {
+                if (err) {
+                    return cb(err);
+                }
+
+                let current = 1;
+                async.eachSeries(messageIds, (messageId, nextMessageId) => {
+                    const message = new Message();
+                    message.load({ messageId }, err => {
+                        if (err) {
+                            return nextMessageId(err);
+                        }
+
+                        const progress = {
+                            current,
+                            message,
+                            step    : 'message',
+                            total   : messageIds.length,
+                            status  : `Writing message ${current} / ${messageIds.length}`,
+                        };
+
+                        progressHandler(progress, err => {
+                            if (err) {
+                                return nextMessageId(err);
+                            }
+
+                            packetWriter.appendMessage(message);
+                            current += 1;
+
+                            return nextMessageId(null);
+                        });
+                    });
+                },
+                err => {
+                    return cb(err);
+                });
+            });
+        };
+
+        const packetWriter = new QWKPacketWriter(
+            Object.assign(this._getUserQWKExportOptions(), {
+                user    : this.client.user,
+                bbsID   : this.config.bbsID,
+            })
+        );
+
+        packetWriter.on('warning', warning => {
+            this.client.log.warn( { warning }, 'QWK packet writer warning');
+        });
+
+        async.waterfall(
+            [
+                (callback) => {
+                    //  don't count idle monitor while processing
+                    this.client.stopIdleMonitor();
+
+                    //  let user cancel
+                    this.client.on('key press', keyPressHandler);
+
+                    packetWriter.once('ready', () => {
+                        return callback(null);
+                    });
+
+                    packetWriter.once('error', err => {
+                        this.client.log.error( { error : err.message }, 'QWK packet writer error');
+                        cancel = true;
+                    });
+
+                    packetWriter.init();
+                },
+                (callback) => {
+                    //  For each public area -> for each message
+                    const userExportAreas = this._getUserQWKExportAreas();
+                    async.eachSeries(userExportAreas, (exportArea, nextExportArea) => {
+                        const area = getMessageAreaByTag(exportArea.areaTag);
+                        if (!area) {
+                            //  :TODO: remove from user properties - this area does not exist
+                            this.client.log.warn({ areaTag : exportArea.areaTag }, 'Cannot QWK export area as it does not exist');
+                            return nextExportArea(null);
+                        }
+
+                        if (!hasMessageConfAndAreaRead(this.client, area)) {
+                            this.client.log.warn({ areaTag : area.areaTag }, 'Cannot QWK export area due to ACS');
+                            return nextExportArea(null);
+                        }
+
+                        const progress = {
+                            area,
+                            step    : 'next_area',
+                            status  : `Gathering messages in ${area.name}...`,
+                        };
+
+                        progressHandler(progress, err => {
+                            if (err) {
+                                return nextExportArea(err);
+                            }
+
+                            const filter = {
+                                resultType          : 'id',
+                                areaTag             : exportArea.areaTag,
+                                newerThanTimestamp  : exportArea.newerThanTimestamp
+                            };
+
+                            processMessagesWithFilter(filter, err => {
+                                return nextExportArea(err);
+                            });
+                        });
+                    },
+                    err => {
+                        return callback(err, userExportAreas);
+                    });
+                },
+                (userExportAreas, callback) => {
+                    //  Private messages to current user if the user has
+                    //  elected to export private messages
+                    if (!(userExportAreas.find(exportArea => exportArea.areaTag === Message.WellKnownAreaTags.Private))) {
+                        return callback(null);
+                    }
+
+                    const filter = {
+                        resultType          : 'id',
+                        privateTagUserId    : this.client.user.userId,
+                        //  :TODO: newerThanTimestamp for private messages
+                        //newerThanTimestamp  : exportArea.newerThanTimestamp
+                    };
+                    return processMessagesWithFilter(filter, callback);
+                },
+                (callback) => {
+                    let packetInfo;
+                    packetWriter.once('packet', info => {
+                        packetInfo = info;
+                    });
+
+                    packetWriter.once('finished', () => {
+                        return callback(null, packetInfo);
+                    });
+
+                    packetWriter.finish(this.tempPacketDir);
+                },
+                (packetInfo, callback) => {
+                    const sysDownloadPath = paths.join(sysTempDownloadDir, this.tempName);
+                    fse.move(packetInfo.path, sysDownloadPath, err => {
+                        return callback(null, sysDownloadPath, packetInfo);
+                    });
+                },
+                (sysDownloadPath, packetInfo, callback) => {
+                    const newEntry = new FileEntry({
+                        areaTag     : this.sysTempDownloadArea.areaTag,
+                        fileName    : paths.basename(sysDownloadPath),
+                        storageTag  : this.sysTempDownloadArea.storageTags[0],
+                        meta        : {
+                            upload_by_username  : this.client.user.username,
+                            upload_by_user_id   : this.client.user.userId,
+                            byte_size           : packetInfo.stats.size,
+                            session_temp_dl     : 1,    //  download is valid until session is over
+
+                            //  :TODO: something like this: allow to override the displayed/downloaded as filename
+                            //  separate from the actual on disk filename. E.g. we could always download as "ENIGMA.QWK"
+                            //visible_filename    : paths.basename(packetInfo.path),
+                        }
+                    });
+
+                    newEntry.desc = 'QWK Export';
+
+                    newEntry.persist(err => {
+                        if(!err) {
+                            //  queue it!
+                            DownloadQueue.get(this.client).addTemporaryDownload(newEntry);
+                        }
+                        return callback(err);
+                    });
+                }
+            ],
+            err => {
+                this.client.startIdleMonitor(); //  re-enable
+                this.client.removeListener('key press', keyPressHandler);
+
+                if (!err) {
+                    updateStatus('A QWK packet has been placed in your download queue');
+                }
+
+                return cb(err);
+            }
+        );
+    }
+};

core/qwk_mail_packet.js

@@ -1242,7 +1242,12 @@ class QWKPacketWriter extends EventEmitter {
                     files,
                     this.workDir,
                     err => {
+                        fs.stat(packetPath, (err, stats) => {
+                            if (stats) {
+                                this.emit('packet', { stats, path : packetPath } );
+                            }
                             return cb(err);
+                        });
                     }
                 );
             });

core/user_property.js

@@ -29,7 +29,7 @@ module.exports = {
     UserComment                     : 'user_comment',               //  NYI
     AutoSignature                   : 'auto_signature',
 
-    DownloadQueue                   : 'dl_queue',                   // download_queue.js
+    DownloadQueue                   : 'dl_queue',                   // see download_queue.js
 
     FailedLoginAttempts             : 'failed_login_attempts',
     AccountLockedTs                 : 'account_locked_timestamp',
@@ -64,5 +64,6 @@ module.exports = {
     AuthFactor2OTP                  : 'auth_factor2_otp',           //  If present, OTP type for 2FA. See OTPTypes
     AuthFactor2OTPSecret            : 'auth_factor2_otp_secret',    //  Secret used in conjunction with OTP 2FA
     AuthFactor2OTPBackupCodes       : 'auth_factor2_otp_backup',    //  JSON array of backup codes
+
 };