Produce NDX files, various improvements to spec, etc.

core/mbf.js

@@ -0,0 +1,59 @@
+const { Errors } = require('./enig_error');
+
+//
+//  Utils for dealing with Microsoft Binary Format (MBF) used
+//  in various BASIC languages, etc.
+//
+//  - https://en.wikipedia.org/wiki/Microsoft_Binary_Format
+//  - https://stackoverflow.com/questions/2268191/how-to-convert-from-ieee-python-float-to-microsoft-basic-float
+//
+
+//  Number to 32bit MBF
+numToMbf32 = (v) => {
+    const mbf = Buffer.alloc(4);
+
+	if (0 === v) {
+		return mbf;
+	}
+
+	const ieee = Buffer.alloc(4);
+	ieee.writeFloatLE(v, 0);
+
+	const sign  = ieee[3] & 0x80;
+	let exp     = (ieee[3] << 1) | (ieee[2] >> 7);
+
+    if (exp === 0xfe) {
+		throw Errors.Invalid(`${v} cannot be converted to mbf`);
+	}
+
+	exp += 2;
+
+    mbf[3] = exp;
+	mbf[2] = sign | (ieee[2] & 0x7f);
+	mbf[1] = ieee[1];
+	mbf[0] = ieee[0];
+
+	return mbf;
+}
+
+mbf32ToNum = (mbf) => {
+	if (0 === mbf[3]) {
+		return 0.0;
+	}
+
+	const ieee	= Buffer.alloc(4);
+	const sign	= mbf[2] & 0x80;
+	const exp	= mbf[3] - 2;
+
+	ieee[3] = sign | (exp >> 1);
+	ieee[2] = (exp << 7) | (mbf[2] & 0x7f);
+	ieee[1] = mbf[1];
+	ieee[0] = mbf[0];
+
+	return ieee.readFloatLE(0);
+}
+
+module.exports = {
+	numToMbf32,
+	mbf32ToNum,
+}

core/oputil/oputil_message_base.js

@@ -445,6 +445,9 @@ function handleQWK() {
         case 'dump' :
             return dumpQWKPacket(packetPath);
 
+        case 'export' :
+            return exportQWKPacket(packetPath);
+
         default :
             return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
     }
@@ -457,7 +460,18 @@ function dumpQWKPacket(packetPath) {
                 return initConfigAndDatabases(callback);
             },
             (callback) => {
+                ////
+                const { QWKPacketWriter } = require('../qwk_mail_packet');
+                const writer = new QWKPacketWriter({
+                    bbsID : 'XIBALBA',
+                    toUser : 'NuSkooler',
+                    encoding : 'cp437',
+                });
+
                 const { QWKPacketReader } = require('../qwk_mail_packet');
+
+
+                writer.on('ready', () => {
                     const reader = new QWKPacketReader(packetPath);
 
                     reader.on('error', err => {
@@ -466,7 +480,7 @@ function dumpQWKPacket(packetPath) {
                     });
 
                     reader.on('done', () => {
-                    return callback(null);
+                        writer.finish();
                     });
 
                     reader.on('archive type', archiveType => {
@@ -478,14 +492,49 @@ function dumpQWKPacket(packetPath) {
                     });
 
                     reader.on('message', message => {
-                    console.info('--- message ---');
+                        writer.appendMessage(message);
-                    console.info(`To        : ${message.toUserName}`);
-                    console.info(`From      : ${message.fromUserName}`);
-                    console.info(`Subject   : ${message.subject}`);
-                    console.info(`Message   :\r\n${message.message}`);
                     });
 
                     reader.read();
+                });
+
+                writer.on('finished', () => {
+                    console.log('done');
+                });
+
+                writer.init();
+
+                ////
+
+                // const { QWKPacketReader } = require('../qwk_mail_packet');
+                // const reader = new QWKPacketReader(packetPath);
+
+                // reader.on('error', err => {
+                //     console.error(`ERROR: ${err.message}`);
+                //     return callback(err);
+                // });
+
+                // reader.on('done', () => {
+                //     return callback(null);
+                // });
+
+                // reader.on('archive type', archiveType => {
+                //     console.info(`-> Archive type: ${archiveType}`);
+                // });
+
+                // reader.on('creator', creator => {
+                //     console.info(`-> Creator: ${creator}`);
+                // });
+
+                // reader.on('message', message => {
+                //     console.info('--- message ---');
+                //     console.info(`To        : ${message.toUserName}`);
+                //     console.info(`From      : ${message.fromUserName}`);
+                //     console.info(`Subject   : ${message.subject}`);
+                //     console.info(`Message   :\r\n${message.message}`);
+                // });
+
+                // reader.read();
             }
         ],
         err => {
@@ -494,6 +543,120 @@ function dumpQWKPacket(packetPath) {
     )
 }
 
+function exportQWKPacket(packetPath) {
+    //  oputil mb qwk export SPEC PATH [--user USER]
+    //  [areaTag1[@dateTime]],[...] PATH --user USER
+
+    const posArgLen = argv._.length;
+
+    if (posArgLen < 4) {
+        return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
+    }
+
+    let areaTagSpecs = '*';
+    if (5 === posArgLen) {
+        areaTagSpecs = argv._[areaTagSpecs - 2];
+    }
+
+
+    //const areaTagSpecs = argv._[areaTagSpecs - 2];
+
+    /*const packetPath = argv._[argv._.length - 1];
+    if(argv._.length < 4 || !packetPath || 0 === packetPath.length) {
+        return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
+    }*/
+
+    //  :TODO: parse area tags(s) and timestamps
+    const areaTags = [ 'general', 'fsx_gen' ];
+
+    const userName = argv.user || '-';
+
+    async.waterfall(
+        [
+            (callback) => {
+                return initConfigAndDatabases(callback);
+            },
+            (callback) => {
+                const User = require('../../core/user.js');
+
+                User.getUserIdAndName(userName, (err, userId) => {
+                    if (err) {
+                        if ('-' === userName) {
+                            userId = 1;
+                        } else {
+                            return callback(err);
+                        }
+                    }
+                    return User.getUser(userId, callback);
+                });
+            },
+            (user, callback) => {
+                const Message = require('../message');
+
+                const filter = {
+                    resultType  : 'id',
+                    areaTag     : areaTags,
+
+                    //  :TODO: newerThanTimestamp
+                };
+
+                //  public
+                Message.findMessages(filter, (err, publicMessageIds) => {
+                    if (err) {
+                        return callback(err);
+                    }
+
+                    delete filter.areaTag;
+                    filter.privateTagUserId = user.userId;
+
+                    Message.findMessages(filter, (err, privateMessageIds) => {
+                        return callback(err, user, Message, privateMessageIds.concat(publicMessageIds));
+                    });
+                });
+            },
+            (user, Message, messageIds, callback) => {
+                const { QWKPacketWriter } = require('../qwk_mail_packet');
+                const writer = new QWKPacketWriter({
+                    //  :TODO: export needs these options
+                    bbsID : 'XIBALBA',
+                    toUser : 'NuSkooler',
+                    encoding : 'cp437',
+                    user        : user,
+                });
+
+                writer.on('ready', () => {
+                    async.eachSeries(messageIds, (messageId, nextMessageId) => {
+                        const message = new Message();
+                        message.load( { messageId }, err => {
+                            if (!err) {
+                                writer.appendMessage(message);
+                            }
+                            return nextMessageId(err);
+                        });
+                    },
+                    (err) => {
+                        writer.finish('/home/nuskooler/Downloads/qwk2/TEST1.QWK');
+                        if (err) {
+                            console.error(`Failed to write one or more messages: ${err.message}`);
+                        }
+                    });
+                });
+
+                writer.on('finished', () => {
+                    return callback(null);
+                });
+
+                writer.init();
+            }
+        ],
+        err => {
+            if(err) {
+                console.error(err.reason ? err.reason : err.message);
+            }
+        }
+    );
+}
+
 function handleMessageBaseCommand() {
 
     function errUsage() {

core/qwk_mail_packet.js

@@ -9,6 +9,8 @@ const {
 const StatLog = require('./stat_log');
 const Config = require('./config').get;
 const SysProps = require('./system_property');
+const UserProps = require('./user_property');
+const { numToMbf32 } = require('./mbf');
 
 const { EventEmitter } = require('events');
 const temptmp = require('temptmp');
@@ -83,8 +85,8 @@ const QWKLF                     = 0xe3;
 const QWKMessageStatusCodes = {
     UnreadPublic                : ' ',
     ReadPublic                  : '-',
-    ReadBySomeonePrivate        : '*',
     UnreadPrivate               : '+',
+    ReadPrivate                 : '*',
     UnreadCommentToSysOp        : '~',
     ReadCommentToSysOp          : '`',
     UnreadSenderPWProtected     : '%',
@@ -100,6 +102,11 @@ const QWKMessageActiveStatus = {
     Deleted : 226,
 };
 
+const QWKNetworkTagIndicator = {
+    Present     : '*',
+    NotPresent  : ' ',
+};
+
 //  See the following:
 //  -   http://fileformats.archiveteam.org/wiki/QWK
 //  -   http://wiki.synchro.net/ref:qwk
@@ -878,6 +885,9 @@ class QWKPacketWriter extends EventEmitter {
 
                     this.totalMessages = 0;
                     this.areaTagsSeen = new Set();
+                    this.personalIndex = [];    //  messages addressed to 'user'
+                    this.inboxIndex = [];       //  private messages for 'user'
+                    this.publicIndex = new Map();
 
                     return callback(null);
                 },
@@ -938,9 +948,6 @@ class QWKPacketWriter extends EventEmitter {
             }
         }
 
-        //  The actual message contents
-        //fullMessageBody += message.message;
-
         //  Sanitize line feeds (e.g. CRLF -> LF, and possibly -> QWK style below)
         splitTextAtTerms(message.message).forEach(line => {
             fullMessageBody += `${line}\n`;
@@ -961,11 +968,12 @@ class QWKPacketWriter extends EventEmitter {
         //  block padded by spaces or nulls (we use nulls)
         const fullBlocks    = Math.trunc(encodedMessage.length / QWKMessageBlockSize);
         const remainBytes   = QWKMessageBlockSize - (encodedMessage.length % QWKMessageBlockSize);
+        const totalBlocks   = fullBlocks + 1 + (remainBytes ? 1 : 0);
 
         //  The first block is always a header
         if (!this._writeMessageHeader(
             message,
-            fullBlocks + 1 + (remainBytes ? 1 : 0),
+            totalBlocks
         ))
         {
             //  we can't write this message
@@ -974,26 +982,61 @@ class QWKPacketWriter extends EventEmitter {
 
         this.messagesStream.write(encodedMessage);
 
-
         if (remainBytes) {
             this.messagesStream.write(Buffer.alloc(remainBytes, ' '));
         }
 
+        this._updateIndexTracking(message);
+
         if (this.options.enableHeadersExtension) {
             this._appendHeadersExtensionData(message);
         }
 
-        this.currentMessageOffset += fullBlocks * QWKMessageBlockSize;
+        //  next message starts at this block
-
+        this.currentMessageOffset += totalBlocks * QWKMessageBlockSize;
-        if (remainBytes)
-        {
-            this.currentMessageOffset += QWKMessageBlockSize;
-        }
 
         this.totalMessages += 1;
         this.areaTagsSeen.add(message.areaTag);
     }
 
+    _messageAddressedToUser(message) {
+        if (_.isUndefined(this.cachedCompareNames)) {
+            if (this.options.user) {
+                this.cachedCompareNames = [
+                    this.options.user.username.toLowerCase()
+                ];
+                const realName = this.options.user.getProperty(UserProps.RealName);
+                if (realName) {
+                    this.cachedCompareNames.push(realName.toLowerCase());
+                }
+            } else {
+                this.cachedCompareNames = [];
+            }
+        };
+
+        return this.cachedCompareNames.includes(message.toUserName.toLowerCase());
+    }
+
+    _updateIndexTracking(message) {
+        //  index points at start of *message* not the header for... reasons?
+        const index = (this.currentMessageOffset / QWKMessageBlockSize) + 1;
+        if (message.isPrivate()) {
+            this.inboxIndex.push(index);
+        } else {
+            if (this._messageAddressedToUser(message)) {
+                //  :TODO: add to both indexes???
+                this.personalIndex.push(index);
+            }
+
+            const areaTag = message.areaTag;
+            if (!this.publicIndex.has(areaTag)) {
+                this.publicIndex.set(areaTag, [index]);
+            } else {
+                this.publicIndex.get(areaTag).push(index);
+            }
+        }
+    }
+
     appendNewFile() {
 
     }
@@ -1019,6 +1062,9 @@ class QWKPacketWriter extends EventEmitter {
                 (callback) => {
                     return this._createControlData(callback);
                 },
+                (callback) => {
+                    return this._createIndexes(callback);
+                },
                 (callback) => {
                     return this._producePacketArchive(packetPath, callback);
                 }
@@ -1038,22 +1084,21 @@ class QWKPacketWriter extends EventEmitter {
     _producePacketArchive(packetPath, cb) {
         const archiveUtil = ArchiveUtil.getInstance();
 
-        const packetFiles = [
+        fs.readdir(this.workDir, (err, files) => {
-            'messages.dat', 'headers.dat', 'control.dat',
+            if (err) {
-        ].map(filename => {
+                return cb(err);
-            return filename;
+            }
-            //return paths.join(this.workDir, filename);
-        });
 
             archiveUtil.compressTo(
                 this.options.archiveFormat,
                 packetPath,
-            packetFiles,
+                files,
                 this.workDir,
                 err => {
                     return cb(err);
                 }
             );
+        });
     }
 
     _qwkMessageStatus(message) {
@@ -1090,15 +1135,9 @@ class QWKPacketWriter extends EventEmitter {
             return false;
         }
 
-        const netTag = ' '; //  :TODO:
-
-        this.lolMessageId = this.lolMessageId || 1;
-        //message.messageId = this.lolMessageId;
-        this.lolMessageId++;
-
         const header = Buffer.alloc(QWKMessageBlockSize, ' ');
         header.write(this._qwkMessageStatus(message), 0, 1, 'ascii');
-        header.write(asciiNum(message.messageId), 1, 'ascii');  //  :TODO: It seems Sync puts the relative, as in # of messages we've called appendMessage()?!
+        header.write(asciiNum(message.messageId), 1, 'ascii');
         header.write(message.modTimestamp.format(QWKHeaderTimestampFormat), 8, 13, 'ascii');
         header.write(message.toUserName.substr(0, 25), 21, 'ascii');
         header.write(message.fromUserName.substr(0, 25), 46, 'ascii');
@@ -1108,8 +1147,8 @@ class QWKPacketWriter extends EventEmitter {
         header.write(asciiTotalBlocks, 116, 'ascii');
         header.writeUInt8(QWKMessageActiveStatus.Active, 122);
         header.writeUInt16LE(conferenceNumber, 123);
-        header.writeUInt16LE(0, 125);   //  :TODO: Check if others actually populate this
+        header.writeUInt16LE(this.totalMessages + 1, 125);
-        header.write(netTag[0], 127, 1, 'ascii');
+        header.write(QWKNetworkTagIndicator.NotPresent, 127, 1, 'ascii');   //  :TODO: Present if for network output?
 
         this.messagesStream.write(header);
 
@@ -1117,6 +1156,9 @@ class QWKPacketWriter extends EventEmitter {
     }
 
     _getMessageConferenceNumberByAreaTag(areaTag) {
+        if (Message.isPrivateAreaTag(areaTag)) {
+            return 0;
+        }
         const areaConfig = _.get(Config(), [ 'messageNetworks', 'qwk', 'areas', areaTag ]);
         return areaConfig && areaConfig.conference;
     }
@@ -1131,6 +1173,13 @@ class QWKPacketWriter extends EventEmitter {
 
     _createControlData(cb) {
         const areas = Array.from(this.areaTagsSeen).map(areaTag => {
+            if (Message.isPrivateAreaTag(areaTag)) {
+                return {
+                    areaTag : Message.WellKnownAreaTags.Private,
+                    name    : 'Private',
+                    desc    : 'Private Messages',
+                };
+            }
             return getMessageAreaByTag(areaTag);
         });
 
@@ -1168,8 +1217,12 @@ class QWKPacketWriter extends EventEmitter {
         //  map areas as conf #\r\nDescription\r\n pairs
         areas.forEach(area => {
             const conferenceNumber = this._getMessageConferenceNumberByAreaTag(area.areaTag);
+            let desc = area.name;
+            if (area.desc) {
+                desc += ` - ${area.desc}`
+            }
             controlStream.write(`${conferenceNumber}\r\n`);
-            controlStream.write(`${area.name}\r\n`);
+            controlStream.write(`${desc}\r\n`);
         });
 
         //  :TODO: do we ever care here?!
@@ -1180,6 +1233,73 @@ class QWKPacketWriter extends EventEmitter {
         controlStream.end();
     }
 
+    _createIndexes(cb) {
+        const appendIndexData = (stream, offset) => {
+            const msb = numToMbf32(offset);
+            stream.write(msb);
+
+            //  technically, the conference #, but only as a byte, so pretty much useless
+            //  AND the filename itself is the conference number... dafuq.
+            stream.write(Buffer.from([0x00]));
+        };
+
+        async.series(
+            [
+                (callback) => {
+                    //  Create PERSONAL.NDX
+                    if (!this.personalIndex.length) {
+                        return callback(null);
+                    }
+
+                    const indexStream = fs.createWriteStream(paths.join(this.workDir, 'personal.ndx'));
+                    this.personalIndex.forEach(offset => appendIndexData(indexStream, offset));
+
+                    indexStream.on('close', err => {
+                        return callback(err);
+                    });
+
+                    indexStream.end();
+                },
+                (callback) => {
+                    //  000.NDX of private mails
+                    if (!this.inboxIndex.length) {
+                        return callback(null);
+                    }
+
+                    const indexStream = fs.createWriteStream(paths.join(this.workDir, '000.ndx'));
+                    this.inboxIndex.forEach(offset => appendIndexData(indexStream, offset));
+
+                    indexStream.on('close', err => {
+                        return callback(err);
+                    });
+
+                    indexStream.end();
+                },
+                (callback) => {
+                    //  ####.NDX
+                    async.eachSeries(this.publicIndex.keys(), (areaTag, nextArea) => {
+                        const offsets = this.publicIndex.get(areaTag);
+                        const conferenceNumber = this._getMessageConferenceNumberByAreaTag(areaTag);
+                        const indexStream = fs.createWriteStream(paths.join(this.workDir, `${conferenceNumber.toString()}.ndx`));
+                        offsets.forEach(offset => appendIndexData(indexStream, offset));
+
+                        indexStream.on('close', err => {
+                            return nextArea(err);
+                        });
+
+                        indexStream.end();
+                    },
+                    err => {
+                        return callback(err);
+                    });
+                }
+            ],
+            err => {
+                return cb(err);
+            }
+        );
+    }
+
     _makeSynchronetTimestamp(ts) {
         const syncTimestamp = ts.format('YYYYMMDDHHmmssZZ');
         const syncTZ        = UTCOffsetToSMBTZ[ts.format('Z')] || '0000';   //  :TODO: what if we don't have a map?
@@ -1215,7 +1335,7 @@ class QWKPacketWriter extends EventEmitter {
             Tags            : message.hashTags.join(' '),
 
             //  :TODO: Needs tested with Sync/etc.; Sync wants Conference *numbers*
-            Conference      : getMessageConfTagByAreaTag(message.areaTag),
+            Conference      : message.isPrivate() ? '0' : getMessageConfTagByAreaTag(message.areaTag),
 
             //  ENiGMA Headers
             MessageUUID     : message.uuid,