WIP on QWK download support

core/enig_error.js

@@ -37,6 +37,7 @@ exports.Errors = {
     MissingParam        : (reason, reasonCode)  => new EnigError('Missing paramter(s)', -32008, reason, reasonCode),
     MissingMci          : (reason, reasonCode)  => new EnigError('Missing required MCI code(s)', -32009, reason, reasonCode),
     BadLogin            : (reason, reasonCode)  => new EnigError('Bad login attempt', -32010, reason, reasonCode),
+    UserInterrupt       : (reason, reasonCode)  => new EnigError('User interrupted', -32011, reason, reasonCode),
 };
 
 exports.ErrorReasons = {

core/file_base_user_list_export.js

@@ -28,7 +28,7 @@ const yazl              = require('yazl');
     tsFormat            - timestamp format (theme 'short')
     descWidth           - max desc width (45)
     progBarChar         - progress bar character (▒)
-    compressThreshold   - threshold to kick in comrpession for lists (1.44 MiB)
+    compressThreshold   - threshold to kick in compression for lists (1.44 MiB)
     templates           - object containing:
         header          - filename of header template (misc/file_list_header.asc)
         entry           - filename of entry template (misc/file_list_entry.asc)
@@ -244,6 +244,7 @@ exports.getModule = class FileBaseListExport extends MenuModule {
                 },
                 function done(callback) {
                     //  re-enable idle monitor
+                    //  :TODO: this should probably be moved down below at the end of the full waterfall
                     self.client.startIdleMonitor();
 
                     updateStatus('Exported list has been added to your download queue');

core/message_base_qwk_export.js

@@ -0,0 +1,378 @@
+//  ENiGMA½
+const { MenuModule } = require('./menu_module');
+const Message = require('./message');
+const { Errors } = require('./enig_error');
+const {
+    getMessageAreaByTag,
+    hasMessageConfAndAreaRead,
+} = require('./message_area');
+const FileArea = require('./file_base_area');
+const { QWKPacketWriter } = require('./qwk_mail_packet');
+const { renderSubstr } = require('./string_util');
+const Config = require('./config').get;
+const FileEntry = require('./file_entry');
+const Events = require('./events');
+const DownloadQueue = require('./download_queue');
+
+//  deps
+const async = require('async');
+const _ = require('lodash');
+const fse = require('fs-extra');
+const temptmp = require('temptmp');
+const paths = require('path');
+const UUIDv4 = require('uuid/v4');
+
+const FormIds = {
+    main    : 0,
+};
+
+const MciViewIds = {
+    main : {
+        status              : 1,
+        progressBar         : 2,
+
+        customRangeStart    : 10,
+    }
+};
+
+exports.moduleInfo = {
+    name    : 'QWK Export',
+    desc    : 'Exports a QWK Packet for download',
+    author  : 'NuSkooler',
+};
+
+exports.getModule = class MessageBaseQWKExport extends MenuModule {
+    constructor(options) {
+        super(options);
+
+        this.config = Object.assign({}, _.get(options, 'menuConfig.config'), options.extraArgs);
+
+        this.config.progBarChar = renderSubstr( (this.config.progBarChar || '▒'), 0, 1);
+        this.config.bbsID       = this.config.bbsID || _.get(Config(), 'messageNetworks.qwk.bbsID', 'ENIGMA');
+
+        this.tempName = `${UUIDv4().substr(-8).toUpperCase()}.QWK`;
+        this.sysTempDownloadArea = FileArea.getFileAreaByTag(FileArea.WellKnownAreaTags.TempDownloads);
+    }
+
+    mciReady(mciData, cb) {
+        super.mciReady(mciData, err => {
+            if (err) {
+                return cb(err);
+            }
+
+            async.waterfall(
+                [
+                    (callback) => {
+                        this.prepViewController('main', FormIds.main, mciData.menu, err => {
+                            return callback(err);
+                        });
+                    },
+                    (callback) => {
+                        this.temptmp = temptmp.createTrackedSession('qwkuserexp');
+                        this.temptmp.mkdir({ prefix : 'enigqwkwriter-'}, (err, tempDir) => {
+                            if (err) {
+                                return callback(err);
+                            }
+
+                            this.tempPacketDir = tempDir;
+
+                            const sysTempDownloadDir = FileArea.getAreaDefaultStorageDirectory(this.sysTempDownloadArea);
+
+                            //  ensure dir exists
+                            fse.mkdirs(sysTempDownloadDir, err => {
+                                return callback(err, sysTempDownloadDir);
+                            });
+                        });
+                    },
+                    (sysTempDownloadDir, callback) => {
+                        this.performExport(sysTempDownloadDir, err => {
+                            return callback(err);
+                        });
+                    },
+                ],
+                err => {
+                    this.temptmp.cleanup();
+
+                    if (err) {
+                        //  :TODO: doesn't do anything currently:
+                        if ('NORESULTS' === err.reasonCode) {
+                            return this.gotoMenu(this.menuConfig.config.noResultsMenu || 'qwkExportNoResults');
+                        }
+
+                        return this.prevMenu();
+                    }
+                    return cb(err);
+                }
+            );
+        });
+    }
+
+    finishedLoading() {
+        this.prevMenu();
+    }
+
+    performExport(sysTempDownloadDir, cb) {
+        const statusView = this.viewControllers.main.getView(MciViewIds.main.status);
+        const updateStatus = (status) => {
+            if (statusView) {
+                statusView.setText(status);
+            }
+        };
+
+        const progBarView = this.viewControllers.main.getView(MciViewIds.main.progressBar);
+        const updateProgressBar = (curr, total) => {
+            if (progBarView) {
+                const prog = Math.floor( (curr / total) * progBarView.dimens.width );
+                progBarView.setText(this.config.progBarChar.repeat(prog));
+            }
+        };
+
+        let cancel = false;
+
+        let lastProgUpdate = 0;
+        const progressHandler = (state, next) => {
+            //  we can produce a TON of updates; only update progress at most every 3/4s
+            if (Date.now() - lastProgUpdate > 750) {
+                switch (state.step) {
+                    case 'next_area' :
+                        updateStatus(state.status);
+                        updateProgressBar(0, 0);
+                        this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.area);
+                        break;
+
+                    case 'message' :
+                        updateStatus(state.status);
+                        updateProgressBar(state.current, state.total);
+                        this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.message);
+                        break;
+
+                    default :
+                        break;
+                }
+                lastProgUpdate = Date.now();
+            }
+
+            return next(cancel ? Errors.UserInterrupt('User canceled') : null);
+        };
+
+        const keyPressHandler = (ch, key) => {
+            if('escape' === key.name) {
+                cancel = true;
+                this.client.removeListener('key press', keyPressHandler);
+            }
+        };
+
+        const processMessagesWithFilter = (filter, cb) => {
+            Message.findMessages(filter, (err, messageIds) => {
+                if (err) {
+                    return cb(err);
+                }
+
+                let current = 1;
+                async.eachSeries(messageIds, (messageId, nextMessageId) => {
+                    const message = new Message();
+                    message.load({ messageId }, err => {
+                        if (err) {
+                            return nextMessageId(err);
+                        }
+
+                        const progress = {
+                            current,
+                            message,
+                            step    : 'message',
+                            total   : messageIds.length,
+                            status  : `Writing message ${current} / ${messageIds.length}`,
+                        };
+
+                        progressHandler(progress, err => {
+                            if (err) {
+                                return nextMessageId(err);
+                            }
+
+                            packetWriter.appendMessage(message);
+                            current += 1;
+
+                            return nextMessageId(null);
+                        });
+                    });
+                },
+                err => {
+                    return cb(err);
+                });
+            });
+        };
+
+        const packetWriter = new QWKPacketWriter({
+            user    : this.client.user,
+            bbsID   : this.config.bbsID,
+        });   //  :TODO: User configuration here
+
+        packetWriter.on('warning', warning => {
+            this.client.log.warn( { warning }, 'QWK packet writer warning');
+        });
+
+        async.waterfall(
+            [
+                (callback) => {
+                    //  don't count idle monitor while processing
+                    this.client.stopIdleMonitor();
+
+                    //  let user cancel
+                    this.client.on('key press', keyPressHandler);
+
+                    packetWriter.once('ready', () => {
+                        return callback(null);
+                    });
+
+                    packetWriter.once('error', err => {
+                        this.client.log.error( { error : err.message }, 'QWK packet writer error');
+                        cancel = true;
+                    });
+
+                    packetWriter.init();
+                },
+                (callback) => {
+                    //
+                    //  Fetch messages for user-configured area tags.
+                    //  - If private tag is present, we fetch this separately.
+                    //  - User property determines newscan timestamps dates if present for tag, else "all"
+                    //  - We have to fetch one area at a time in order to process message pointers/timestamps.
+                    //    ...this also allows for better progress.
+                    //
+                    //  TL;DR: for each area -> for each message
+                    //
+                    const exportAreas = [ //  :TODO: Load in something like this
+                        {
+                            areaTag : 'general',
+                            newerThanTimestamp : '2018-01-01',
+                        },
+                        {
+                            areaTag : 'fsx_gen',
+                        }
+                    ];
+
+                    async.eachSeries(exportAreas, (exportArea, nextExportArea) => {
+                        const area = getMessageAreaByTag(exportArea.areaTag);
+                        if (!area) {
+                            //  :TODO: remove from user properties - this area does not exist
+                            this.client.log.warn({ areaTag : exportArea.areaTag }, 'Cannot QWK export area as it does not exist');
+                            return nextExportArea(null);
+                        }
+
+                        if (!hasMessageConfAndAreaRead(this.client, area)) {
+                            this.client.log.warn({ areaTag : area.areaTag }, 'Cannot QWK export area due to ACS');
+                            return nextExportArea(null);
+                        }
+
+                        const progress = {
+                            area,
+                            step    : 'next_area',
+                            status  : `Gathering messages in ${area.name}...`,
+                        };
+
+                        progressHandler(progress, err => {
+                            if (err) {
+                                return nextExportArea(err);
+                            }
+
+                            const filter = {
+                                resultType          : 'id',
+                                areaTag             : exportArea.areaTag,
+                                newerThanTimestamp  : exportArea.newerThanTimestamp
+                            };
+
+                            processMessagesWithFilter(filter, err => {
+                                return nextExportArea(err);
+                            });
+                        });
+                    },
+                    err => {
+                        return callback(err);
+                    });
+                },
+                (callback) => {
+                    const filter = {
+                        resultType          : 'id',
+                        privateTagUserId    : this.client.user.userId,
+                        //  :TODO: newerThanTimestamp for private messages
+                        //newerThanTimestamp  : exportArea.newerThanTimestamp
+                    };
+                    return processMessagesWithFilter(filter, callback);
+                },
+                (callback) => {
+                    let packetInfo;
+                    packetWriter.once('packet', info => {
+                        packetInfo = info;
+                    });
+
+                    packetWriter.once('finished', () => {
+                        return callback(null, packetInfo);
+                    });
+
+                    packetWriter.finish(this.tempPacketDir);
+                },
+                (packetInfo, callback) => {
+                    const sysDownloadPath = paths.join(sysTempDownloadDir, this.tempName);
+                    fse.move(packetInfo.path, sysDownloadPath, err => {
+                        return callback(null, sysDownloadPath, packetInfo);
+                    });
+                },
+                (sysDownloadPath, packetInfo, callback) => {
+                    const newEntry = new FileEntry({
+                        areaTag     : this.sysTempDownloadArea.areaTag,
+                        fileName    : paths.basename(sysDownloadPath),
+                        storageTag  : this.sysTempDownloadArea.storageTags[0],
+                        meta        : {
+                            upload_by_username  : this.client.user.username,
+                            upload_by_user_id   : this.client.user.userId,
+                            byte_size           : packetInfo.stats.size,
+                            session_temp_dl     : 1,    //  download is valid until session is over
+
+                            //  :TODO: something like this: allow to override the displayed/downloaded as filename
+                            //  separate from the actual on disk filename. E.g. we could always download as "ENIGMA.QWK"
+                            visible_filename    : paths.basename(packetInfo.path),
+                        }
+                    });
+
+                    newEntry.desc = 'QWK Export';
+
+                    newEntry.persist(err => {
+                        if(!err) {
+                            //  queue it!
+                            const dlQueue = new DownloadQueue(this.client);
+                            dlQueue.add(newEntry, true);    //  true=systemFile
+
+                            //  clean up after ourselves when the session ends
+                            //  :TODO: DRY this with that in file_base_user_export
+                            const thisClientId = this.client.session.id;
+                            Events.once(Events.getSystemEvents().ClientDisconnected, evt => {
+                                if(thisClientId === _.get(evt, 'client.session.id')) {
+                                    FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => {
+                                        if(err) {
+                                            Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' );
+                                        } else {
+                                            Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' );
+                                        }
+                                    });
+                                }
+                            });
+                        }
+                        return callback(err);
+                    });
+                }
+            ],
+            err => {
+                this.client.startIdleMonitor(); //  re-enable
+                this.client.removeListener('key press', keyPressHandler);
+
+                if (!err) {
+                    updateStatus('A QWK packet has been placed in your download queue');
+                }
+
+                //  :TODO: send user to download manager with pop flags/etc.
+
+                return cb(err);
+            }
+        );
+    }
+};

core/qwk_mail_packet.js

@@ -1242,7 +1242,12 @@ class QWKPacketWriter extends EventEmitter {
                     files,
                     this.workDir,
                     err => {
-                        return cb(err);
+                        fs.stat(packetPath, (err, stats) => {
+                            if (stats) {
+                                this.emit('packet', { stats, path : packetPath } );
+                            }
+                            return cb(err);
+                        });
                     }
                 );
             });