diff --git a/core/mbf.js b/core/mbf.js new file mode 100644 index 00000000..9c3b2f6d --- /dev/null +++ b/core/mbf.js @@ -0,0 +1,59 @@ +const { Errors } = require('./enig_error'); + +// +// Utils for dealing with Microsoft Binary Format (MBF) used +// in various BASIC languages, etc. +// +// - https://en.wikipedia.org/wiki/Microsoft_Binary_Format +// - https://stackoverflow.com/questions/2268191/how-to-convert-from-ieee-python-float-to-microsoft-basic-float +// + +// Number to 32bit MBF +numToMbf32 = (v) => { + const mbf = Buffer.alloc(4); + + if (0 === v) { + return mbf; + } + + const ieee = Buffer.alloc(4); + ieee.writeFloatLE(v, 0); + + const sign = ieee[3] & 0x80; + let exp = (ieee[3] << 1) | (ieee[2] >> 7); + + if (exp === 0xfe) { + throw Errors.Invalid(`${v} cannot be converted to mbf`); + } + + exp += 2; + + mbf[3] = exp; + mbf[2] = sign | (ieee[2] & 0x7f); + mbf[1] = ieee[1]; + mbf[0] = ieee[0]; + + return mbf; +} + +mbf32ToNum = (mbf) => { + if (0 === mbf[3]) { + return 0.0; + } + + const ieee = Buffer.alloc(4); + const sign = mbf[2] & 0x80; + const exp = mbf[3] - 2; + + ieee[3] = sign | (exp >> 1); + ieee[2] = (exp << 7) | (mbf[2] & 0x7f); + ieee[1] = mbf[1]; + ieee[0] = mbf[0]; + + return ieee.readFloatLE(0); +} + +module.exports = { + numToMbf32, + mbf32ToNum, +} \ No newline at end of file diff --git a/core/oputil/oputil_message_base.js b/core/oputil/oputil_message_base.js index e5b30c12..7d914ab2 100644 --- a/core/oputil/oputil_message_base.js +++ b/core/oputil/oputil_message_base.js @@ -445,6 +445,9 @@ function handleQWK() { case 'dump' : return dumpQWKPacket(packetPath); + case 'export' : + return exportQWKPacket(packetPath); + default : return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); } @@ -457,35 +460,81 @@ function dumpQWKPacket(packetPath) { return initConfigAndDatabases(callback); }, (callback) => { + //// + const { QWKPacketWriter } = require('../qwk_mail_packet'); + const writer = new QWKPacketWriter({ + bbsID : 'XIBALBA', + toUser : 'NuSkooler', + encoding : 'cp437', + }); + const { QWKPacketReader } = require('../qwk_mail_packet'); - const reader = new QWKPacketReader(packetPath); - reader.on('error', err => { - console.error(`ERROR: ${err.message}`); - return callback(err); + + writer.on('ready', () => { + const reader = new QWKPacketReader(packetPath); + + reader.on('error', err => { + console.error(`ERROR: ${err.message}`); + return callback(err); + }); + + reader.on('done', () => { + writer.finish(); + }); + + reader.on('archive type', archiveType => { + console.info(`-> Archive type: ${archiveType}`); + }); + + reader.on('creator', creator => { + console.info(`-> Creator: ${creator}`); + }); + + reader.on('message', message => { + writer.appendMessage(message); + }); + + reader.read(); }); - reader.on('done', () => { - return callback(null); + writer.on('finished', () => { + console.log('done'); }); - reader.on('archive type', archiveType => { - console.info(`-> Archive type: ${archiveType}`); - }); + writer.init(); - reader.on('creator', creator => { - console.info(`-> Creator: ${creator}`); - }); + //// - reader.on('message', message => { - console.info('--- message ---'); - console.info(`To : ${message.toUserName}`); - console.info(`From : ${message.fromUserName}`); - console.info(`Subject : ${message.subject}`); - console.info(`Message :\r\n${message.message}`); - }); + // const { QWKPacketReader } = require('../qwk_mail_packet'); + // const reader = new QWKPacketReader(packetPath); - reader.read(); + // reader.on('error', err => { + // console.error(`ERROR: ${err.message}`); + // return callback(err); + // }); + + // reader.on('done', () => { + // return callback(null); + // }); + + // reader.on('archive type', archiveType => { + // console.info(`-> Archive type: ${archiveType}`); + // }); + + // reader.on('creator', creator => { + // console.info(`-> Creator: ${creator}`); + // }); + + // reader.on('message', message => { + // console.info('--- message ---'); + // console.info(`To : ${message.toUserName}`); + // console.info(`From : ${message.fromUserName}`); + // console.info(`Subject : ${message.subject}`); + // console.info(`Message :\r\n${message.message}`); + // }); + + // reader.read(); } ], err => { @@ -494,6 +543,120 @@ function dumpQWKPacket(packetPath) { ) } +function exportQWKPacket(packetPath) { + // oputil mb qwk export SPEC PATH [--user USER] + // [areaTag1[@dateTime]],[...] PATH --user USER + + const posArgLen = argv._.length; + + if (posArgLen < 4) { + return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + } + + let areaTagSpecs = '*'; + if (5 === posArgLen) { + areaTagSpecs = argv._[areaTagSpecs - 2]; + } + + + //const areaTagSpecs = argv._[areaTagSpecs - 2]; + + /*const packetPath = argv._[argv._.length - 1]; + if(argv._.length < 4 || !packetPath || 0 === packetPath.length) { + return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + }*/ + + // :TODO: parse area tags(s) and timestamps + const areaTags = [ 'general', 'fsx_gen' ]; + + const userName = argv.user || '-'; + + async.waterfall( + [ + (callback) => { + return initConfigAndDatabases(callback); + }, + (callback) => { + const User = require('../../core/user.js'); + + User.getUserIdAndName(userName, (err, userId) => { + if (err) { + if ('-' === userName) { + userId = 1; + } else { + return callback(err); + } + } + return User.getUser(userId, callback); + }); + }, + (user, callback) => { + const Message = require('../message'); + + const filter = { + resultType : 'id', + areaTag : areaTags, + + // :TODO: newerThanTimestamp + }; + + // public + Message.findMessages(filter, (err, publicMessageIds) => { + if (err) { + return callback(err); + } + + delete filter.areaTag; + filter.privateTagUserId = user.userId; + + Message.findMessages(filter, (err, privateMessageIds) => { + return callback(err, user, Message, privateMessageIds.concat(publicMessageIds)); + }); + }); + }, + (user, Message, messageIds, callback) => { + const { QWKPacketWriter } = require('../qwk_mail_packet'); + const writer = new QWKPacketWriter({ + // :TODO: export needs these options + bbsID : 'XIBALBA', + toUser : 'NuSkooler', + encoding : 'cp437', + user : user, + }); + + writer.on('ready', () => { + async.eachSeries(messageIds, (messageId, nextMessageId) => { + const message = new Message(); + message.load( { messageId }, err => { + if (!err) { + writer.appendMessage(message); + } + return nextMessageId(err); + }); + }, + (err) => { + writer.finish('/home/nuskooler/Downloads/qwk2/TEST1.QWK'); + if (err) { + console.error(`Failed to write one or more messages: ${err.message}`); + } + }); + }); + + writer.on('finished', () => { + return callback(null); + }); + + writer.init(); + } + ], + err => { + if(err) { + console.error(err.reason ? err.reason : err.message); + } + } + ); +} + function handleMessageBaseCommand() { function errUsage() { diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index e5c316b1..761b71f7 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -9,6 +9,8 @@ const { const StatLog = require('./stat_log'); const Config = require('./config').get; const SysProps = require('./system_property'); +const UserProps = require('./user_property'); +const { numToMbf32 } = require('./mbf'); const { EventEmitter } = require('events'); const temptmp = require('temptmp'); @@ -83,8 +85,8 @@ const QWKLF = 0xe3; const QWKMessageStatusCodes = { UnreadPublic : ' ', ReadPublic : '-', - ReadBySomeonePrivate : '*', UnreadPrivate : '+', + ReadPrivate : '*', UnreadCommentToSysOp : '~', ReadCommentToSysOp : '`', UnreadSenderPWProtected : '%', @@ -100,6 +102,11 @@ const QWKMessageActiveStatus = { Deleted : 226, }; +const QWKNetworkTagIndicator = { + Present : '*', + NotPresent : ' ', +}; + // See the following: // - http://fileformats.archiveteam.org/wiki/QWK // - http://wiki.synchro.net/ref:qwk @@ -878,6 +885,9 @@ class QWKPacketWriter extends EventEmitter { this.totalMessages = 0; this.areaTagsSeen = new Set(); + this.personalIndex = []; // messages addressed to 'user' + this.inboxIndex = []; // private messages for 'user' + this.publicIndex = new Map(); return callback(null); }, @@ -938,9 +948,6 @@ class QWKPacketWriter extends EventEmitter { } } - // The actual message contents - //fullMessageBody += message.message; - // Sanitize line feeds (e.g. CRLF -> LF, and possibly -> QWK style below) splitTextAtTerms(message.message).forEach(line => { fullMessageBody += `${line}\n`; @@ -961,11 +968,12 @@ class QWKPacketWriter extends EventEmitter { // block padded by spaces or nulls (we use nulls) const fullBlocks = Math.trunc(encodedMessage.length / QWKMessageBlockSize); const remainBytes = QWKMessageBlockSize - (encodedMessage.length % QWKMessageBlockSize); + const totalBlocks = fullBlocks + 1 + (remainBytes ? 1 : 0); // The first block is always a header if (!this._writeMessageHeader( message, - fullBlocks + 1 + (remainBytes ? 1 : 0), + totalBlocks )) { // we can't write this message @@ -974,26 +982,61 @@ class QWKPacketWriter extends EventEmitter { this.messagesStream.write(encodedMessage); - if (remainBytes) { this.messagesStream.write(Buffer.alloc(remainBytes, ' ')); } + this._updateIndexTracking(message); + if (this.options.enableHeadersExtension) { this._appendHeadersExtensionData(message); } - this.currentMessageOffset += fullBlocks * QWKMessageBlockSize; - - if (remainBytes) - { - this.currentMessageOffset += QWKMessageBlockSize; - } + // next message starts at this block + this.currentMessageOffset += totalBlocks * QWKMessageBlockSize; this.totalMessages += 1; this.areaTagsSeen.add(message.areaTag); } + _messageAddressedToUser(message) { + if (_.isUndefined(this.cachedCompareNames)) { + if (this.options.user) { + this.cachedCompareNames = [ + this.options.user.username.toLowerCase() + ]; + const realName = this.options.user.getProperty(UserProps.RealName); + if (realName) { + this.cachedCompareNames.push(realName.toLowerCase()); + } + } else { + this.cachedCompareNames = []; + } + }; + + return this.cachedCompareNames.includes(message.toUserName.toLowerCase()); + } + + _updateIndexTracking(message) { + // index points at start of *message* not the header for... reasons? + const index = (this.currentMessageOffset / QWKMessageBlockSize) + 1; + if (message.isPrivate()) { + this.inboxIndex.push(index); + } else { + if (this._messageAddressedToUser(message)) { + // :TODO: add to both indexes??? + this.personalIndex.push(index); + } + + const areaTag = message.areaTag; + if (!this.publicIndex.has(areaTag)) { + this.publicIndex.set(areaTag, [index]); + } else { + this.publicIndex.get(areaTag).push(index); + } + } + } + appendNewFile() { } @@ -1019,6 +1062,9 @@ class QWKPacketWriter extends EventEmitter { (callback) => { return this._createControlData(callback); }, + (callback) => { + return this._createIndexes(callback); + }, (callback) => { return this._producePacketArchive(packetPath, callback); } @@ -1038,22 +1084,21 @@ class QWKPacketWriter extends EventEmitter { _producePacketArchive(packetPath, cb) { const archiveUtil = ArchiveUtil.getInstance(); - const packetFiles = [ - 'messages.dat', 'headers.dat', 'control.dat', - ].map(filename => { - return filename; - //return paths.join(this.workDir, filename); - }); - - archiveUtil.compressTo( - this.options.archiveFormat, - packetPath, - packetFiles, - this.workDir, - err => { + fs.readdir(this.workDir, (err, files) => { + if (err) { return cb(err); } - ); + + archiveUtil.compressTo( + this.options.archiveFormat, + packetPath, + files, + this.workDir, + err => { + return cb(err); + } + ); + }); } _qwkMessageStatus(message) { @@ -1090,15 +1135,9 @@ class QWKPacketWriter extends EventEmitter { return false; } - const netTag = ' '; // :TODO: - - this.lolMessageId = this.lolMessageId || 1; - //message.messageId = this.lolMessageId; - this.lolMessageId++; - const header = Buffer.alloc(QWKMessageBlockSize, ' '); header.write(this._qwkMessageStatus(message), 0, 1, 'ascii'); - header.write(asciiNum(message.messageId), 1, 'ascii'); // :TODO: It seems Sync puts the relative, as in # of messages we've called appendMessage()?! + header.write(asciiNum(message.messageId), 1, 'ascii'); header.write(message.modTimestamp.format(QWKHeaderTimestampFormat), 8, 13, 'ascii'); header.write(message.toUserName.substr(0, 25), 21, 'ascii'); header.write(message.fromUserName.substr(0, 25), 46, 'ascii'); @@ -1108,8 +1147,8 @@ class QWKPacketWriter extends EventEmitter { header.write(asciiTotalBlocks, 116, 'ascii'); header.writeUInt8(QWKMessageActiveStatus.Active, 122); header.writeUInt16LE(conferenceNumber, 123); - header.writeUInt16LE(0, 125); // :TODO: Check if others actually populate this - header.write(netTag[0], 127, 1, 'ascii'); + header.writeUInt16LE(this.totalMessages + 1, 125); + header.write(QWKNetworkTagIndicator.NotPresent, 127, 1, 'ascii'); // :TODO: Present if for network output? this.messagesStream.write(header); @@ -1117,6 +1156,9 @@ class QWKPacketWriter extends EventEmitter { } _getMessageConferenceNumberByAreaTag(areaTag) { + if (Message.isPrivateAreaTag(areaTag)) { + return 0; + } const areaConfig = _.get(Config(), [ 'messageNetworks', 'qwk', 'areas', areaTag ]); return areaConfig && areaConfig.conference; } @@ -1131,6 +1173,13 @@ class QWKPacketWriter extends EventEmitter { _createControlData(cb) { const areas = Array.from(this.areaTagsSeen).map(areaTag => { + if (Message.isPrivateAreaTag(areaTag)) { + return { + areaTag : Message.WellKnownAreaTags.Private, + name : 'Private', + desc : 'Private Messages', + }; + } return getMessageAreaByTag(areaTag); }); @@ -1168,8 +1217,12 @@ class QWKPacketWriter extends EventEmitter { // map areas as conf #\r\nDescription\r\n pairs areas.forEach(area => { const conferenceNumber = this._getMessageConferenceNumberByAreaTag(area.areaTag); + let desc = area.name; + if (area.desc) { + desc += ` - ${area.desc}` + } controlStream.write(`${conferenceNumber}\r\n`); - controlStream.write(`${area.name}\r\n`); + controlStream.write(`${desc}\r\n`); }); // :TODO: do we ever care here?! @@ -1180,6 +1233,73 @@ class QWKPacketWriter extends EventEmitter { controlStream.end(); } + _createIndexes(cb) { + const appendIndexData = (stream, offset) => { + const msb = numToMbf32(offset); + stream.write(msb); + + // technically, the conference #, but only as a byte, so pretty much useless + // AND the filename itself is the conference number... dafuq. + stream.write(Buffer.from([0x00])); + }; + + async.series( + [ + (callback) => { + // Create PERSONAL.NDX + if (!this.personalIndex.length) { + return callback(null); + } + + const indexStream = fs.createWriteStream(paths.join(this.workDir, 'personal.ndx')); + this.personalIndex.forEach(offset => appendIndexData(indexStream, offset)); + + indexStream.on('close', err => { + return callback(err); + }); + + indexStream.end(); + }, + (callback) => { + // 000.NDX of private mails + if (!this.inboxIndex.length) { + return callback(null); + } + + const indexStream = fs.createWriteStream(paths.join(this.workDir, '000.ndx')); + this.inboxIndex.forEach(offset => appendIndexData(indexStream, offset)); + + indexStream.on('close', err => { + return callback(err); + }); + + indexStream.end(); + }, + (callback) => { + // ####.NDX + async.eachSeries(this.publicIndex.keys(), (areaTag, nextArea) => { + const offsets = this.publicIndex.get(areaTag); + const conferenceNumber = this._getMessageConferenceNumberByAreaTag(areaTag); + const indexStream = fs.createWriteStream(paths.join(this.workDir, `${conferenceNumber.toString()}.ndx`)); + offsets.forEach(offset => appendIndexData(indexStream, offset)); + + indexStream.on('close', err => { + return nextArea(err); + }); + + indexStream.end(); + }, + err => { + return callback(err); + }); + } + ], + err => { + return cb(err); + } + ); + } + _makeSynchronetTimestamp(ts) { const syncTimestamp = ts.format('YYYYMMDDHHmmssZZ'); const syncTZ = UTCOffsetToSMBTZ[ts.format('Z')] || '0000'; // :TODO: what if we don't have a map? @@ -1215,7 +1335,7 @@ class QWKPacketWriter extends EventEmitter { Tags : message.hashTags.join(' '), // :TODO: Needs tested with Sync/etc.; Sync wants Conference *numbers* - Conference : getMessageConfTagByAreaTag(message.areaTag), + Conference : message.isPrivate() ? '0' : getMessageConfTagByAreaTag(message.areaTag), // ENiGMA Headers MessageUUID : message.uuid,