* oputil mb qwk-export

* oputil mb qwk-dump
* Fix QWK reader encoding

core/oputil/oputil_help.js

@@ -170,6 +170,11 @@ Actions:
 
   import-areas PATH           Import areas using FidoNet *.NA or AREAS.BBS file
 
+  qwk-dump PATH               Dumps a QWK packet to stdout.
+  qwk-export [AREA_TAGS] PATH Exports one or more configured message area to a QWK
+                              packet in the directory specified by PATH. The QWK
+                              BBS ID will be obtained by the final component of PATH.
+
 import-areas arguments:
   --conf CONF_TAG             Conference tag in which to import areas
   --network NETWORK           Network name/key to associate FTN areas
@@ -177,6 +182,11 @@ import-areas arguments:
   --type TYPE                 Area import type
 
   Valid types are "bbs" and "na".
+
+qwk-export arguments:
+  --user USER                 User in which to export for. Defaults to the SysOp.
+  --after TIMESTAMP           Export only messages with a timestamp later than
+                              TIMESTAMP.
 `
 };
 

core/oputil/oputil_message_base.js

@@ -11,6 +11,7 @@ const {
     getAnswers,
     writeConfig,
 } = require('./oputil_common.js');
+
 const getHelpFor	= require('./oputil_help.js').getHelpFor;
 const Address		= require('../ftn_address.js');
 const Errors	    = require('../enig_error.js').Errors;
@@ -21,6 +22,7 @@ const paths                     = require('path');
 const fs        = require('fs');
 const hjson     = require('hjson');
 const _         = require('lodash');
+const moment    = require('moment');
 
 exports.handleMessageBaseCommand	= handleMessageBaseCommand;
 
@@ -434,42 +436,19 @@ function getImportEntries(importType, importData) {
     return importEntries;
 }
 
-function handleQWK() {
+function dumpQWKPacket() {
     const packetPath = argv._[argv._.length - 1];
-    if(argv._.length < 4 || !packetPath || 0 === packetPath.length) {
+    if(argv._.length < 3 || !packetPath || 0 === packetPath.length) {
-        return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
+        return printUsageAndSetExitCode(getHelpFor('MessageBase'), ExitCodes.ERROR);
     }
 
-    const subAction = argv._[argv._.length - 2];
-    switch (subAction) {
-        case 'dump' :
-            return dumpQWKPacket(packetPath);
-
-        case 'export' :
-            return exportQWKPacket(packetPath);
-
-        default :
-            return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
-    }
-}
-
-function dumpQWKPacket(packetPath) {
     async.waterfall(
         [
             (callback) => {
                 return initConfigAndDatabases(callback);
             },
             (callback) => {
-                ////
-                const { QWKPacketWriter } = require('../qwk_mail_packet');
-                const writer = new QWKPacketWriter({
-                    bbsID : 'XIBALBA',
-                });
-
                 const { QWKPacketReader } = require('../qwk_mail_packet');
-
-
-                writer.on('ready', () => {
                 const reader = new QWKPacketReader(packetPath);
 
                 reader.on('error', err => {
@@ -478,7 +457,7 @@ function dumpQWKPacket(packetPath) {
                 });
 
                 reader.on('done', () => {
-                        writer.finish();
+                    return callback(null);
                 });
 
                 reader.on('archive type', archiveType => {
@@ -490,49 +469,14 @@ function dumpQWKPacket(packetPath) {
                 });
 
                 reader.on('message', message => {
-                        writer.appendMessage(message);
+                    console.info('--- message ---');
+                    console.info(`To:      ${message.toUserName}`);
+                    console.info(`From:    ${message.fromUserName}`);
+                    console.info(`Subject: ${message.subject}`);
+                    console.info(`Message:\r\n${message.message}`);
                 });
 
                 reader.read();
-                });
-
-                writer.on('finished', () => {
-                    console.log('done');
-                });
-
-                writer.init();
-
-                ////
-
-                // const { QWKPacketReader } = require('../qwk_mail_packet');
-                // const reader = new QWKPacketReader(packetPath);
-
-                // reader.on('error', err => {
-                //     console.error(`ERROR: ${err.message}`);
-                //     return callback(err);
-                // });
-
-                // reader.on('done', () => {
-                //     return callback(null);
-                // });
-
-                // reader.on('archive type', archiveType => {
-                //     console.info(`-> Archive type: ${archiveType}`);
-                // });
-
-                // reader.on('creator', creator => {
-                //     console.info(`-> Creator: ${creator}`);
-                // });
-
-                // reader.on('message', message => {
-                //     console.info('--- message ---');
-                //     console.info(`To        : ${message.toUserName}`);
-                //     console.info(`From      : ${message.fromUserName}`);
-                //     console.info(`Subject   : ${message.subject}`);
-                //     console.info(`Message   :\r\n${message.message}`);
-                // });
-
-                // reader.read();
             }
         ],
         err => {
@@ -541,36 +485,43 @@ function dumpQWKPacket(packetPath) {
     )
 }
 
-function exportQWKPacket(packetPath) {
+function exportQWKPacket() {
-    //  oputil mb qwk export SPEC PATH [--user USER]
+    let packetPath = argv._[argv._.length - 1];
-    //  [areaTag1[@dateTime]],[...] PATH --user USER
+    if(argv._.length < 3 || !packetPath || 0 === packetPath.length) {
+        return printUsageAndSetExitCode(getHelpFor('MessageBase'), ExitCodes.ERROR);
+    }
 
-    //  :TODO: bbsID from PATH filename else 'ENIGMA'
+    //  oputil mb qwk-export TAGS PATH [--user USER] [--after TIMESTAMP]
+    //  [areaTag1,areaTag2,...] PATH --user USER --after TIMESTAMP
+    let bbsID = 'ENIGMA';
+    const filename = paths.basename(packetPath);
+    if (filename) {
+        const ext = paths.extname(filename);
+        bbsID = paths.basename(filename, ext);
+    }
+
+    packetPath = paths.dirname(packetPath);
 
     const posArgLen = argv._.length;
 
-    if (posArgLen < 4) {
+    let areaTags;
-        return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
+    if (4 === posArgLen) {
+        areaTags = argv._[posArgLen - 2].split(',');
+    } else {
+        areaTags = [];
     }
 
-    let areaTagSpecs = '*';
+    let newerThanTimestamp = null;
-    if (5 === posArgLen) {
+    if (argv.after) {
-        areaTagSpecs = argv._[areaTagSpecs - 2];
+        const ts = moment(argv.after);
+        if (ts.isValid()) {
+            newerThanTimestamp = ts.format();
+        }
     }
-
-
-    //const areaTagSpecs = argv._[areaTagSpecs - 2];
-
-    /*const packetPath = argv._[argv._.length - 1];
-    if(argv._.length < 4 || !packetPath || 0 === packetPath.length) {
-        return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
-    }*/
-
-    //  :TODO: parse area tags(s) and timestamps
-    const areaTags = [ 'general', 'fsx_gen' ];
 
     const userName = argv.user || '-';
 
+    let totalExported = 0;
     async.waterfall(
         [
             (callback) => {
@@ -590,14 +541,29 @@ function exportQWKPacket(packetPath) {
                     return User.getUser(userId, callback);
                 });
             },
+            (user, callback) => {
+                //  populate area tags with all available to user
+                //  if they were not explicitly supplied
+                if (!areaTags.length) {
+                    const {
+                        getAvailableMessageConferences,
+                        getAvailableMessageAreasByConfTag
+                    } = require('../../core/message_area');
+
+                    const confTags = Object.keys(getAvailableMessageConferences(null, { noClient : true }));
+                    confTags.forEach( confTag => {
+                        areaTags = areaTags.concat(Object.keys(getAvailableMessageAreasByConfTag(confTag)));
+                    });
+                }
+                return callback(null, user);
+            },
             (user, callback) => {
                 const Message = require('../message');
 
                 const filter = {
                     resultType  : 'id',
                     areaTag     : areaTags,
-
+                    newerThanTimestamp,
-                    //  :TODO: newerThanTimestamp
                 };
 
                 //  public
@@ -617,9 +583,8 @@ function exportQWKPacket(packetPath) {
             (user, Message, messageIds, callback) => {
                 const { QWKPacketWriter } = require('../qwk_mail_packet');
                 const writer = new QWKPacketWriter({
-                    //  :TODO: export needs these options
+                    bbsID,
-                    bbsID : 'XIBALBA',
+                    user,
-                    user        : user,
                 });
 
                 writer.on('ready', () => {
@@ -628,18 +593,23 @@ function exportQWKPacket(packetPath) {
                         message.load( { messageId }, err => {
                             if (!err) {
                                 writer.appendMessage(message);
+                                ++totalExported;
                             }
                             return nextMessageId(err);
                         });
                     },
                     (err) => {
-                        writer.finish('/home/nuskooler/Downloads/qwk2/');
+                        writer.finish(packetPath);
                         if (err) {
                             console.error(`Failed to write one or more messages: ${err.message}`);
                         }
                     });
                 });
 
+                writer.on('warning', err => {
+                    console.warn(`!!! ${err.reason ? err.reason : err.message}`);
+                });
+
                 writer.on('finished', () => {
                     return callback(null);
                 });
@@ -649,8 +619,10 @@ function exportQWKPacket(packetPath) {
         ],
         err => {
             if(err) {
-                console.error(err.reason ? err.reason : err.message);
+                return console.error(err.reason ? err.reason : err.message);
             }
+
+            console.info(`-> Exported ${totalExported} messages`);
         }
     );
 }
@@ -673,6 +645,7 @@ function handleMessageBaseCommand() {
     return({
         areafix	        : areaFix,
         'import-areas'  : importAreas,
-        qwk             : handleQWK,
+        'qwk-dump'      : dumpQWKPacket,
+        'qwk-export'    : exportQWKPacket,
     }[action] || errUsage)();
 }

core/qwk_mail_packet.js

@@ -493,7 +493,7 @@ class QWKPacketReader extends EventEmitter {
         }
 
         const encodingToSpec = 'cp437';
-        let encoding = encodingToSpec;
+        let encoding;
 
         const path = paths.join(this.packetInfo.tempDir, this.packetInfo.messages.filename);
         fs.open(path, 'r', (err, fd) => {
@@ -575,6 +575,7 @@ class QWKPacketReader extends EventEmitter {
                         switch (state) {
                             case 'header' :
                                 const header = MessageHeaderParser.parse(buffer);
+                                encoding = encodingToSpec;  //  reset per message
 
                                 //  massage into something a little more sane (things we can't quite do in the parser directly)
                                 ['toName', 'fromName', 'subject'].forEach(field => {
@@ -601,7 +602,7 @@ class QWKPacketReader extends EventEmitter {
 
                                 //  if we have HEADERS.DAT with a 'Utf8' override for this message,
                                 //  the overridden to/from/subject/message fields are UTF-8
-                                if (currMessage.headersExtension && currMessage.headersExtension.Utf8) {
+                                if (currMessage.headersExtension && 'true' === currMessage.headersExtension.Utf8.toLowerCase()) {
                                     encoding = 'utf8';
                                 }
 
@@ -825,7 +826,7 @@ class QWKPacketWriter extends EventEmitter {
             enableHeadersExtension = true,
             enableAtKludges = true,
             systemDomain = 'enigma-bbs',
-            bbsID = '',
+            bbsID = 'ENIGMA',
             user = null,
             archiveFormat = 'application/zip',
             forceEncoding = null,
@@ -853,7 +854,7 @@ class QWKPacketWriter extends EventEmitter {
             enableHeadersExtension  : true,
             enableAtKludges         : true,
             systemDomain            : 'enigma-bbs',
-            bbsID                   : '',
+            bbsID                   : 'ENIGMA',
             user                    : null,
             archiveFormat           :'application/zip',
             forceEncoding           : null,