From 29ee9c4d58140f72c90bb8e4d122fcee3fdb0a26 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 21 Apr 2020 19:50:04 -0600 Subject: [PATCH 1/8] WIP on QWK support --- core/message.js | 19 ++ core/oputil/oputil_message_base.js | 61 ++++ core/qwk_mail_packet.js | 520 +++++++++++++++++++++++++++++ 3 files changed, 600 insertions(+) create mode 100644 core/qwk_mail_packet.js diff --git a/core/message.js b/core/message.js index 5291b82a..68f51ee4 100644 --- a/core/message.js +++ b/core/message.js @@ -87,6 +87,21 @@ const FTN_PROPERTY_NAMES = { FtnSeenBy : 'ftn_seen_by', // http://ftsc.org/docs/fts-0004.001 }; +const QWKPropertyNames = { + MessageNumber : 'qwk_msg_num', + MessageStatus : 'qwk_msg_status', // See http://wiki.synchro.net/ref:qwk for a decent list + ConferenceNumber : 'qwk_conf_num', + InReplyToNum : 'qwk_in_reply_to_num', // note that we prefer the 'InReplyToMsgId' kludge if available +}; + +const QWKKludgeNames = { + Via : 'via', + MessageId : 'msg_id', + InReplyToMsgId : 'in_reply_to_msg_id', + SyncTZ : 'synchronet_timezone', + ReplyTo : 'reply_to', +}; + // :TODO: this is a ugly hack due to bad variable names - clean it up & just _.camelCase(k)! const MESSAGE_ROW_MAP = { reply_to_message_id : 'replyToMsgId', @@ -183,6 +198,10 @@ module.exports = class Message { return FTN_PROPERTY_NAMES; } + static get QWKPropertyNames() { + return QWKPropertyNames; + } + setLocalToUserId(userId) { this.meta.System = this.meta.System || {}; this.meta.System[Message.SystemMetaNames.LocalToUserID] = userId; diff --git a/core/oputil/oputil_message_base.js b/core/oputil/oputil_message_base.js index 0f1b5cfb..e5b30c12 100644 --- a/core/oputil/oputil_message_base.js +++ b/core/oputil/oputil_message_base.js @@ -434,6 +434,66 @@ function getImportEntries(importType, importData) { return importEntries; } +function handleQWK() { + const packetPath = argv._[argv._.length - 1]; + if(argv._.length < 4 || !packetPath || 0 === packetPath.length) { + return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + } + + const subAction = argv._[argv._.length - 2]; + switch (subAction) { + case 'dump' : + return dumpQWKPacket(packetPath); + + default : + return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + } +} + +function dumpQWKPacket(packetPath) { + async.waterfall( + [ + (callback) => { + return initConfigAndDatabases(callback); + }, + (callback) => { + const { QWKPacketReader } = require('../qwk_mail_packet'); + const reader = new QWKPacketReader(packetPath); + + reader.on('error', err => { + console.error(`ERROR: ${err.message}`); + return callback(err); + }); + + reader.on('done', () => { + return callback(null); + }); + + reader.on('archive type', archiveType => { + console.info(`-> Archive type: ${archiveType}`); + }); + + reader.on('creator', creator => { + console.info(`-> Creator: ${creator}`); + }); + + reader.on('message', message => { + console.info('--- message ---'); + console.info(`To : ${message.toUserName}`); + console.info(`From : ${message.fromUserName}`); + console.info(`Subject : ${message.subject}`); + console.info(`Message :\r\n${message.message}`); + }); + + reader.read(); + } + ], + err => { + + } + ) +} + function handleMessageBaseCommand() { function errUsage() { @@ -452,5 +512,6 @@ function handleMessageBaseCommand() { return({ areafix : areaFix, 'import-areas' : importAreas, + qwk : handleQWK, }[action] || errUsage)(); } \ No newline at end of file diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js new file mode 100644 index 00000000..415822a3 --- /dev/null +++ b/core/qwk_mail_packet.js @@ -0,0 +1,520 @@ + + +const ArchiveUtil = require('./archive_util'); +const { Errors } = require('./enig_error'); +const Message = require('./message'); +const { splitTextAtTerms } = require('./string_util'); + +const { EventEmitter } = require('events'); +const temptmp = require('temptmp').createTrackedSession('qwk_packet'); +const async = require('async'); +const fs = require('graceful-fs'); +const paths = require('path'); +const { Parser } = require('binary-parser'); +const iconv = require('iconv-lite'); +const moment = require('moment'); +const _ = require('lodash'); + +const SMBTZToUTCOffset = (smbTZ) => { + // convert a Synchronet smblib TZ to a UTC offset + // see https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h + return { + // US Standard + '40F0' : '-04:00', // Atlantic + '412C' : '-05:00', // Eastern + '4168' : '-06:00', // Central + '41A4' : '-07:00', // Mountain + '41E0' : '-08:00', // Pacific + '421C' : '-09:00', // Yukon + '4258' : '-10:00', // Hawaii/Alaska + '4294' : '-11:00', // Bering + + // US Daylight + + }[smbTZ]; +}; + +const QWKMessageBlockSize = 128; + +const MessageHeaderParser = new Parser() + .endianess('little') + .string('status', { + encoding : 'ascii', + length : 1, + }) + .string('num', { // message num or conf num for REP's + encoding : 'ascii', + length : 7, + formatter : n => { + return parseInt(n); + } + }) + .string('timestamp', { + encoding : 'ascii', + length : 13, + }) + // these fields may be encoded in something other than ascii/CP437 + .array('toName', { + type : 'uint8', + length : 25, + }) + .array('fromName', { + type : 'uint8', + length : 25, + }) + .array('subject', { + type : 'uint8', + length : 25, + }) + .string('password', { + encoding : 'ascii', + length : 12, + }) + .string('replyToNum', { + encoding : 'ascii', + length : 8, + formatter : n => { + return parseInt(n); + } + }) + .string('numBlocks', { + encoding : 'ascii', + length : 6, + formatter : n => { + return parseInt(n); + } + }) + .uint8('status2') + .uint16('confNum') + .uint16('relNum') + .uint8('netTag'); + + +class QWKPacketReader extends EventEmitter { + constructor(packetPath, mode=QWKPacketReader.Modes.Guess, options = { keepTearAndOrigin : true } ) { + super(); + + this.packetPath = packetPath; + this.mode = mode; + this.options = options; + } + + static get Modes() { + return { + Guess : 'guess', // try to guess + QWK : 'qwk', // standard incoming packet + REP : 'rep', // a reply packet + }; + } + + read() { + // + // A general overview: + // + // - Find out what kind of archive we're dealing with + // - Extract to temporary location + // - Process various files + // - Emit messages we find, information about the packet, so on + // + async.waterfall( + [ + // determine packet archive type + (callback) => { + const archiveUtil = ArchiveUtil.getInstance(); + archiveUtil.detectType(this.packetPath, (err, archiveType) => { + if (err) { + return callback(err); + } + this.emit('archive type', archiveType); + return callback(null, archiveType); + }); + }, + // create a temporary location to do processing + (archiveType, callback) => { + temptmp.mkdir( { prefix : 'enigqwkpacket-'}, (err, tempDir) => { + if (err) { + return callback(err); + } + + return callback(null, archiveType, tempDir); + }); + }, + // extract it + (archiveType, tempDir, callback) => { + const archiveUtil = ArchiveUtil.getInstance(); + archiveUtil.extractTo(this.packetPath, tempDir, archiveType, err => { + if (err) { + return callback(err); + } + + return callback(null, tempDir); + }); + }, + // gather extracted file list + (tempDir, callback) => { + fs.readdir(tempDir, (err, files) => { + if (err) { + return callback(err); + } + + // Discover basic information about well known files + async.reduce( + files, + {}, + (out, filename, next) => { + const key = filename.toUpperCase(); + + switch (key) { + case 'MESSAGES.DAT' : // QWK + if (this.mode === QWKPacketReader.Modes.Guess) { + this.mode = QWKPacketReader.Modes.QWK; + } + if (this.mode === QWKPacketReader.Modes.QWK) { + out.messages = { filename }; + } + break; + + case 'ID.MSG' : + if (this.mode === QWKPacketReader.Modes.Guess) { + this.mode = Modes.REP; + } + + if (this.mode === QWKPacketReader.Modes.REP) { + out.messages = { filename }; + } + break; + + case 'HEADERS.DAT' : // Synchronet + out.headers = { filename }; + break; + + case 'VOTING.DAT' : // Synchronet + out.voting = { filename }; + break; + + case 'CONTROL.DAT' : // QWK + out.control = { filename }; + break; + + case 'DOOR.ID' : // QWK + out.door = { filename }; + break; + + case 'NETFLAGS.DAT' : // QWK + out.netflags = { filename }; + break; + + case 'NEWFILES.DAT' : // QWK + out.newfiles = { filename }; + break; + + case 'PERSONAL.NDX' : // QWK + out.personal = { filename }; + break; + + case '000.NDX' : // QWK + out.inbox = { filename }; + break; + + case 'TOREADER.EXT' : // QWKE + out.toreader = { filename }; + break; + + case 'QLR.DAT' : + out.qlr = { filename }; + break; + + default : + if (/[0-9]+\.NDX/.test(key)) { // QWK + out.pointers = out.pointers || { filenames: [] }; + out.pointers.filenames.push(filename); + } else { + out[key] = { filename }; + } + break; + } + + return next(null, out); + }, + (err, packetFileInfo) => { + this.packetInfo = Object.assign( + {}, + packetFileInfo, + { + tempDir, + defaultEncoding : 'CP437' + } + ); + return callback(null); + } + ); + }); + }, + (callback) => { + return this.processPacketFiles(callback); + }, + (tempDir, callback) => { + return callback(null); + } + ], + err => { + temptmp.cleanup(); + + if (err) { + return this.emit('error', err); + } + + this.emit('done'); + } + ); + } + + processPacketFiles(cb) { + return this.readMessages(cb); + } + + readMessages(cb) { + // :TODO: update to use proper encoding: if headers.dat specifies UTF-8, use that, else CP437 + if (!this.packetInfo.messages) { + return cb(Errors.DoesNotExist('No messages file found within QWK packet')); + } + + const encoding = this.packetInfo.defaultEncoding; + const path = paths.join(this.packetInfo.tempDir, this.packetInfo.messages.filename); + fs.open(path, 'r', (err, fd) => { + if (err) { + return cb(err); + } + + let blockCount = 0; + let currMessage = { }; + let state; + let messageBlocksRemain; + const buffer = Buffer.alloc(QWKMessageBlockSize); + + const readNextBlock = () => { + fs.read(fd, buffer, 0, QWKMessageBlockSize, null, (err, read) => { + if (err) { + return cb(err); + } + + if (0 == read) { + // we're done consuming all blocks + return fs.close(fd, err => { + return cb(err); + }); + } + + if (QWKMessageBlockSize !== read) { + return cb(Errors.Invalid(`Invalid QWK message block size. Expected ${QWKMessageBlockSize} got ${read}`)); + } + + if (0 === blockCount) { + // first 128 bytes is a space padded ID + const id = buffer.toString('ascii').trim(); + this.emit('generator', id); + state = 'header'; + } else { + switch (state) { + case 'header' : + const header = MessageHeaderParser.parse(buffer); + + // massage into something a little more sane (things we can't quite do in the parser directly) + ['toName', 'fromName', 'subject'].forEach(field => { + header[field] = iconv.decode(header[field], encoding).trim(); + }); + + header.timestamp = moment(header.timestamp, 'MM-DD-YYHH:mm'); + + currMessage = { + header, + // these may be overridden + toName : header.toName, + fromName : header.fromName, + subject : header.subject, + }; + + // remainder of blocks until the end of this message + messageBlocksRemain = header.numBlocks - 1; + state = 'message'; + break; + + case 'message' : + if (!currMessage.body) { + currMessage.body = buffer; + } else { + currMessage.body = Buffer.concat([currMessage.body, buffer]); + } + messageBlocksRemain -= 1; + + if (0 === messageBlocksRemain) { + // 1:n buffers to make up body. Decode: + // First, replace QWK style line feeds (0xe3) unless the message is UTF-8. + // If the message is UTF-8, we assume it's using standard line feeds. + if (encoding !== 'utf8') { + let i = 0; + const QWKLF = Buffer.from([0xe3]); + while (i < currMessage.body.length) { + i = currMessage.body.indexOf(QWKLF, i); + if (-1 === i) { + break; + } + currMessage.body[i] = 0x0a; + ++i; + } + } + + // + // Decode the message based on our final message encoding. Split the message + // into lines so we can extract various bits such as QWKE headers, origin, tear + // lines, etc. + // + const messageLines = splitTextAtTerms(iconv.decode(currMessage.body, encoding).trimEnd()); + const bodyLines = []; + + // + // Various kludge tags defined by QWKE, etc. + // See the following: + // - ftp://vert.synchro.net/main/BBS/qwke.txt + // - http://wiki.synchro.net/ref:qwk + // + const Kludges = { + // QWKE + To : 'To:', + From : 'From:', + Subject : 'Subject:', + + // Synchronet + Via : '@VIA:', + MsgID : '@MSGID:', + Reply : '@REPLY:', + TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h + ReplyTo : '@REPLYTO:', + }; + + let bodyState = 'kludge'; + + const MessageTrailers = { + // While technically FTN oriented, these can come from any network + // (though we'll be processing a lot of messages that routed through FTN + // at some point) + Origin : /^[ ]{1,2}\* Origin: /, + Tear : /^--- /, + }; + + const qwkKludge = {}; + const ftnProperty = {}; + + messageLines.forEach(line => { + if (0 === line.length) { + return bodyLines.push(''); + } + + switch (bodyState) { + case 'kludge' : + // :TODO: Update these to use the well known consts: + if (line.startsWith(Kludges.To)) { + currMessage.toName = line.substring(Kludges.To.length).trim(); + } else if (line.startsWith(Kludges.From)) { + currMessage.fromName = line.substring(Kludges.From.length).trim(); + } else if (line.startsWith(Kludges.Subject)) { + currMessage.subject = line.substring(Kludges.Subject.length).trim(); + } else if (line.startsWith(Kludges.Via)) { + qwkKludge.via = line; + } else if (line.startsWith(Kludges.MsgID)) { + qwkKludge.msg_id = line.substring(Kludges.MsgID.length).trim(); + } else if (line.startsWith(Kludges.Reply)) { + qwkKludge.in_reply_to_msg_id = line.substring(Kludges.Reply.length).trim(); + } else if (line.startsWith(Kludges.TZ)) { + qwkKludge.synchronet_timezone = line.substring(Kludges.TZ.length).trim(); + } else if (line.startsWith(Kludges.ReplyTo)) { + qwkKludge.reply_to = line.substring(Kludges.ReplyTo.length).trim(); + } else { + bodyState = 'body'; // past this point and up to any tear/origin/etc., is the real message body + bodyLines.push(line); + } + break; + + case 'body' : + case 'trailers' : + if (MessageTrailers.Origin.test(line)) { + ftnProperty.ftn_origin = line; + bodyState = 'trailers'; + } else if (MessageTrailers.Tear.test(line)) { + ftnProperty.ftn_tear_line = line; + bodyState = 'trailers'; + } else if ('body' === bodyState) { + bodyLines.push(line); + } + } + }); + + const message = new Message({ + toUserName : currMessage.toName, + fromUserName : currMessage.fromName, + subject : currMessage.subject, + modTimestamp : currMessage.header.timestamp, + message : bodyLines.join('\n'), + }); + + if (!_.isEmpty(qwkKludge)) { + message.meta.QwkKludge = qwkKludge; + } + + if (!_.isEmpty(ftnProperty)) { + message.meta.FtnProperty = ftnProperty; + } + + // Add in tear line and origin if requested + if (this.options.keepTearAndOrigin) { + if (ftnProperty.ftn_tear_line) { + message.message += `\r\n${ftnProperty.ftn_tear_line}\r\n`; + } + + if (ftnProperty.ftn_origin) { + message.message += `${ftnProperty.ftn_origin}\r\n`; + } + } + + // Update the timestamp if we have a valid TZ + if (_.has(message, 'meta.QwkKludge.synchronet_timezone')) { + const tzOffset = SMBTZToUTCOffset(message.meta.QwkKludge.synchronet_timezone); + if (tzOffset) { + message.modTimestamp.utcOffset(tzOffset); + } + } + + message.meta.QwkProperty = { + qwk_msg_status : currMessage.header.status, + qwk_in_reply_to_num : currMessage.header.replyToNum, + }; + + if (this.mode === QWKPacketReader.Modes.QWK) { + message.meta.QwkProperty.qwk_msg_num = currMessage.header.num; + } else { + // For REP's, prefer the larger field. + message.meta.QwkProperty.qwk_conf_num = currMessage.header.num || currMessage.header.confNum; + } + + this.emit('message', message); + state = 'header'; + } + break; + } + } + + ++blockCount; + readNextBlock(); + }); + }; + + // start reading blocks + readNextBlock(); + }); + } +}; + +module.exports = { + QWKPacketReader, +// QWKPacketWriter, +} From 6edfe95dfe9914f70282eba1ef5f052212a689e7 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 21 Apr 2020 22:14:29 -0600 Subject: [PATCH 2/8] A good amount of HEADERS.DAT support --- core/qwk_mail_packet.js | 180 ++++++++++++++++++++++++++++++++++------ package.json | 1 + yarn.lock | 19 +++++ 3 files changed, 174 insertions(+), 26 deletions(-) diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index 415822a3..9ed97bb6 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -14,6 +14,7 @@ const { Parser } = require('binary-parser'); const iconv = require('iconv-lite'); const moment = require('moment'); const _ = require('lodash'); +const IniConfigParser = require('ini-config-parser'); const SMBTZToUTCOffset = (smbTZ) => { // convert a Synchronet smblib TZ to a UTC offset @@ -242,7 +243,6 @@ class QWKPacketReader extends EventEmitter { packetFileInfo, { tempDir, - defaultEncoding : 'CP437' } ); return callback(null); @@ -270,7 +270,41 @@ class QWKPacketReader extends EventEmitter { } processPacketFiles(cb) { - return this.readMessages(cb); + async.series( + [ + (callback) => { + return this.readHeadersExtension(callback); + }, + (callback) => { + return this.readMessages(callback); + } + ], + err => { + return cb(err); + } + ) + } + + readHeadersExtension(cb) { + if (!this.packetInfo.headers) { + return cb(null); // nothing to do + } + + const path = paths.join(this.packetInfo.tempDir, this.packetInfo.headers.filename); + fs.readFile(path, { encoding : 'utf8' }, (err, iniData) => { + if (err) { + this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid (${err.message})`)); + return cb(null); // non-fatal + } + + try { + this.packetInfo.headers.ini = IniConfigParser.parse(iniData); + } catch (e) { + this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid (${e.message})`)); + } + + return cb(null); + }); } readMessages(cb) { @@ -279,13 +313,54 @@ class QWKPacketReader extends EventEmitter { return cb(Errors.DoesNotExist('No messages file found within QWK packet')); } - const encoding = this.packetInfo.defaultEncoding; + const encodingToSpec = 'cp437'; + let encoding = encodingToSpec; + const path = paths.join(this.packetInfo.tempDir, this.packetInfo.messages.filename); fs.open(path, 'r', (err, fd) => { if (err) { return cb(err); } + // Some mappings/etc. used in loops below.... + // Sync sets these in HEADERS.DAT: http://wiki.synchro.net/ref:qwk + const FTNPropertyMapping = { + 'X-FTN-AREA' : Message.FtnPropertyNames.FtnArea, + 'X-FTN-SEEN-BY' : Message.FtnPropertyNames.FtnSeenBy, + 'X-FTN-FLAGS' : Message.FtnPropertyNames + }; + + const FTNKludgeMapping = { + 'X-FTN-PATH' : 'PATH', + 'X-FTN-MSGID' : 'MSGID', + 'X-FTN-REPLY' : 'REPLY', + 'X-FTN-PID' : 'PID', + 'X-FTN-FLAGS' : 'FLAGS', + 'X-FTN-TID' : 'TID', + 'X-FTN-CHRS' : 'CHRS', + // :TODO: X-FTN-KLUDGE - not sure what this is? + }; + + // + // Various kludge tags defined by QWKE, etc. + // See the following: + // - ftp://vert.synchro.net/main/BBS/qwke.txt + // - http://wiki.synchro.net/ref:qwk + // + const Kludges = { + // QWKE + To : 'To:', + From : 'From:', + Subject : 'Subject:', + + // Synchronet + Via : '@VIA:', + MsgID : '@MSGID:', + Reply : '@REPLY:', + TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h + ReplyTo : '@REPLYTO:', + }; + let blockCount = 0; let currMessage = { }; let state; @@ -321,7 +396,8 @@ class QWKPacketReader extends EventEmitter { // massage into something a little more sane (things we can't quite do in the parser directly) ['toName', 'fromName', 'subject'].forEach(field => { - header[field] = iconv.decode(header[field], encoding).trim(); + // note: always use to-spec encoding here + header[field] = iconv.decode(header[field], encodingToSpec).trim(); }); header.timestamp = moment(header.timestamp, 'MM-DD-YYHH:mm'); @@ -334,6 +410,19 @@ class QWKPacketReader extends EventEmitter { subject : header.subject, }; + if (_.has(this.packetInfo, 'headers.ini')) { + // Sections for a message in HEADERS.DAT are by current byte offset. + // 128 = first message header = 0x80 = section [80] + const headersSectionId = (blockCount * QWKMessageBlockSize).toString(16); + currMessage.headersExtension = this.packetInfo.headers.ini[headersSectionId]; + } + + // if we have HEADERS.DAT with a 'Utf8' override for this message, + // the overridden to/from/subject/message fields are UTF-8 + if (currMessage.headersExtension && currMessage.headersExtension.Utf8) { + encoding = 'utf8'; + } + // remainder of blocks until the end of this message messageBlocksRemain = header.numBlocks - 1; state = 'message'; @@ -372,26 +461,6 @@ class QWKPacketReader extends EventEmitter { const messageLines = splitTextAtTerms(iconv.decode(currMessage.body, encoding).trimEnd()); const bodyLines = []; - // - // Various kludge tags defined by QWKE, etc. - // See the following: - // - ftp://vert.synchro.net/main/BBS/qwke.txt - // - http://wiki.synchro.net/ref:qwk - // - const Kludges = { - // QWKE - To : 'To:', - From : 'From:', - Subject : 'Subject:', - - // Synchronet - Via : '@VIA:', - MsgID : '@MSGID:', - Reply : '@REPLY:', - TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h - ReplyTo : '@REPLYTO:', - }; - let bodyState = 'kludge'; const MessageTrailers = { @@ -404,6 +473,7 @@ class QWKPacketReader extends EventEmitter { const qwkKludge = {}; const ftnProperty = {}; + const ftnKludge = {}; messageLines.forEach(line => { if (0 === line.length) { @@ -449,12 +519,60 @@ class QWKPacketReader extends EventEmitter { } }); + let messageTimestamp = currMessage.header.timestamp; + + // HEADERS.DAT support. + let useTZKludge = true; + if (currMessage.headersExtension) { + const ext = currMessage.headersExtension; + + // to and subject can be overridden yet again if entries are present + currMessage.toName = ext.To || currMessage.toName + currMessage.subject = ext.Subject || currMessage.subject; + currMessage.from = ext.Sender || currMessage.from; // why not From? Who the fuck knows. + + // possibly override message ID kludge + qwkKludge.msg_id = ext['Message-ID'] || qwkKludge.msg_id; + + // WhenWritten contains a ISO-8601-ish timestamp and a Synchronet/SMB style TZ offset: + // 20180101174837-0600 4168 + // We can use this to get a very slightly better precision on the timestamp (addition of seconds) + // over the headers value. Why not milliseconds? Who the fuck knows. + if (ext.WhenWritten) { + const whenWritten = moment(ext.WhenWritten, 'YYYYMMDDHHmmssZ'); + if (whenWritten.isValid()) { + messageTimestamp = whenWritten; + useTZKludge = false; + } + } + + if (ext.Tags) { + currMessage.hashTags = ext.Tags.split(' '); + } + + // FTN style properties/kludges represented as X-FTN-XXXX + for (let [extName, propName] of Object.entries(FTNPropertyMapping)) { + const v = ext[extName]; + if (v) { + ftnProperty[propName] = v; + } + } + + for (let [extName, kludgeName] of Object.entries(FTNKludgeMapping)) { + const v = ext[extName]; + if (v) { + ftnKludge[kludgeName] = v; + } + } + } + const message = new Message({ toUserName : currMessage.toName, fromUserName : currMessage.fromName, subject : currMessage.subject, - modTimestamp : currMessage.header.timestamp, + modTimestamp : messageTimestamp, message : bodyLines.join('\n'), + hashTags : currMessage.hashTags, }); if (!_.isEmpty(qwkKludge)) { @@ -465,6 +583,10 @@ class QWKPacketReader extends EventEmitter { message.meta.FtnProperty = ftnProperty; } + if (!_.isEmpty(ftnKludge)) { + message.meta.FtnKludge = ftnKludge; + } + // Add in tear line and origin if requested if (this.options.keepTearAndOrigin) { if (ftnProperty.ftn_tear_line) { @@ -477,7 +599,7 @@ class QWKPacketReader extends EventEmitter { } // Update the timestamp if we have a valid TZ - if (_.has(message, 'meta.QwkKludge.synchronet_timezone')) { + if (useTZKludge && _.has(message, 'meta.QwkKludge.synchronet_timezone')) { const tzOffset = SMBTZToUTCOffset(message.meta.QwkKludge.synchronet_timezone); if (tzOffset) { message.modTimestamp.utcOffset(tzOffset); @@ -491,11 +613,17 @@ class QWKPacketReader extends EventEmitter { if (this.mode === QWKPacketReader.Modes.QWK) { message.meta.QwkProperty.qwk_msg_num = currMessage.header.num; + message.meta.QwkProperty.qwk_conf_num = currMessage.header.confNum; } else { // For REP's, prefer the larger field. message.meta.QwkProperty.qwk_conf_num = currMessage.header.num || currMessage.header.confNum; } + // Another quick HEADERS.DAT fix-up + if (currMessage.headersExtension) { + message.meta.QwkProperty.qwk_conf_num = currMessage.headersExtension.Conference || message.meta.QwkProperty.qwk_conf_num; + } + this.emit('message', message); state = 'header'; } diff --git a/package.json b/package.json index 048a207f..12754271 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "hashids": "2.1.0", "hjson": "^3.2.1", "iconv-lite": "0.5.0", + "ini-config-parser": "^1.0.4", "inquirer": "^7.0.0", "later": "1.2.0", "lodash": "^4.17.15", diff --git a/yarn.lock b/yarn.lock index 453791de..cb08db2e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -312,6 +312,11 @@ code-point-at@^1.0.0: resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= +coffee-script@^1.12.4: + version "1.12.7" + resolved "https://registry.yarnpkg.com/coffee-script/-/coffee-script-1.12.7.tgz#c05dae0cb79591d05b3070a8433a98c9a89ccc53" + integrity sha512-fLeEhqwymYat/MpTPUjSKHVYYl0ec2mOyALEMLmzr5i1isuG+6jfI2j2d5oBO3VIzgUXgBVIcOT9uH1TFxBckw== + collection-visit@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0" @@ -408,6 +413,11 @@ decode-uri-component@^0.2.0: resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU= +deep-extend@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.5.1.tgz#b894a9dd90d3023fbf1c55a394fb858eb2066f1f" + integrity sha512-N8vBdOa+DF7zkRrDCsaOXoCs/E2fJfx9B9MrKnnSiHNh4ws7eSys6YQE4KvT1cecKmOASYQBhbKjeuDD9lT81w== + deep-extend@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" @@ -883,6 +893,15 @@ inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= +ini-config-parser@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/ini-config-parser/-/ini-config-parser-1.0.4.tgz#0abc75cb68c506204712d2b4861400b6adbfda78" + integrity sha512-5hLh5Cqai67pTrLQ9q/K/3EtSP2Tzu41AZzwPLSegkkMkc42dGweLgkbiocCBiBBEg2fPhs6pKmdFhwj5Ul3Bg== + dependencies: + coffee-script "^1.12.4" + deep-extend "^0.5.1" + rimraf "^2.6.1" + ini@~1.3.0: version "1.3.5" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" From d8f0601914717f65c461b89ba57e86d71467e916 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 21 Apr 2020 22:57:06 -0600 Subject: [PATCH 3/8] Add CONTROL.DAT parsing --- core/qwk_mail_packet.js | 93 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index 9ed97bb6..f670e494 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -31,12 +31,17 @@ const SMBTZToUTCOffset = (smbTZ) => { '4294' : '-11:00', // Bering // US Daylight + // :TODO: FINISH ME! }[smbTZ]; }; const QWKMessageBlockSize = 128; +// See the following: +// - http://fileformats.archiveteam.org/wiki/QWK +// - http://wiki.synchro.net/ref:qwk +// const MessageHeaderParser = new Parser() .endianess('little') .string('status', { @@ -272,6 +277,9 @@ class QWKPacketReader extends EventEmitter { processPacketFiles(cb) { async.series( [ + (callback) => { + return this.readControl(callback); + }, (callback) => { return this.readHeadersExtension(callback); }, @@ -285,6 +293,91 @@ class QWKPacketReader extends EventEmitter { ) } + readControl(cb) { + // + // CONTROL.DAT is a CRLF text file containing information about + // the originating BBS, conf number <> name mapping, etc. + // + // References: + // - http://fileformats.archiveteam.org/wiki/QWK + // + if (!this.packetInfo.control) { + return cb(Errors.DoesNotExist('No control file found within QWK packet')); + } + + const path = paths.join(this.packetInfo.tempDir, this.packetInfo.control.filename); + + // note that we read as UTF-8. Legacy says it should be CP437/ASCII + // but this seems safer for now so conference names and the like + // can be non-English for example. + fs.readFile(path, { encoding : 'utf8' }, (err, controlLines) => { + if (err) { + return cb(err); + } + + controlLines = splitTextAtTerms(controlLines); + + let state = 'header'; + const control = { confMap : {} }; + let currConfNumber; + for (let lineNumber = 0; lineNumber < controlLines.length; ++lineNumber) { + const line = controlLines[lineNumber].trim(); + switch (lineNumber) { + // first set of lines is header info + case 0 : control.bbsName = line; break; + case 1 : control.bbsLocation = line; break; + case 2 : control.bbsPhone = line; break; + case 3 : control.bbsSysOp = line; break; + case 4 : control.doorRegAndBoardID = line; break; + case 5 : control.packetCreationTime = line; break; + case 6 : control.toUser = line; break; + case 7 : break; // Qmail menu + case 8 : break; // unknown, always 0? + case 9 : break; // total messages in packet (often set to 0) + case 10 : + control.totalMessages = (parseInt(line) + 1); + state = 'confNumber'; + break; + + default : + switch (state) { + case 'confNumber' : + currConfNumber = parseInt(line); + if (isNaN(currConfNumber)) { + state = 'news'; + + control.welcomeFile = line; + } else { + state = 'confName'; + } + break; + + case 'confName' : + control.confMap[currConfNumber] = line; + state = 'confNumber'; + break; + + case 'news' : + control.newsFile = line; + state = 'logoff'; + break; + + case 'logoff' : + control.logoffFile = line; + state = 'footer'; + break; + + case 'footer' : + // some systems append additional info; we don't care. + break; + } + } + } + + return cb(null); + }); + } + readHeadersExtension(cb) { if (!this.packetInfo.headers) { return cb(null); // nothing to do From 8a81b34ed044639b754ecb0d106b9fc5fd0505c0 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 25 Apr 2020 11:25:47 -0600 Subject: [PATCH 4/8] WIP: Lots of progress with QWK reader/writer --- core/message.js | 11 +- core/message_area.js | 1 + core/qwk_mail_packet.js | 554 +++++++++++++++++++++++++++++++++++----- 3 files changed, 492 insertions(+), 74 deletions(-) diff --git a/core/message.js b/core/message.js index 68f51ee4..3649a6f6 100644 --- a/core/message.js +++ b/core/message.js @@ -49,7 +49,8 @@ const SYSTEM_META_NAMES = { const ADDRESS_FLAVOR = { Local : 'local', // local / non-remote addressing FTN : 'ftn', // FTN style - Email : 'email', + Email : 'email', // From email + QWK : 'qwk', // QWK packet }; const STATE_FLAGS0 = { @@ -94,14 +95,6 @@ const QWKPropertyNames = { InReplyToNum : 'qwk_in_reply_to_num', // note that we prefer the 'InReplyToMsgId' kludge if available }; -const QWKKludgeNames = { - Via : 'via', - MessageId : 'msg_id', - InReplyToMsgId : 'in_reply_to_msg_id', - SyncTZ : 'synchronet_timezone', - ReplyTo : 'reply_to', -}; - // :TODO: this is a ugly hack due to bad variable names - clean it up & just _.camelCase(k)! const MESSAGE_ROW_MAP = { reply_to_message_id : 'replyToMsgId', diff --git a/core/message_area.js b/core/message_area.js index 579f1c9b..ff01f1e1 100644 --- a/core/message_area.js +++ b/core/message_area.js @@ -29,6 +29,7 @@ exports.getDefaultMessageAreaTagByConfTag = getDefaultMessageAreaTagByConfTag; exports.getSuitableMessageConfAndAreaTags = getSuitableMessageConfAndAreaTags; exports.getMessageConferenceByTag = getMessageConferenceByTag; exports.getMessageAreaByTag = getMessageAreaByTag; +exports.getMessageConfTagByAreaTag = getMessageConfTagByAreaTag; exports.changeMessageConference = changeMessageConference; exports.changeMessageArea = changeMessageArea; exports.hasMessageConfAndAreaRead = hasMessageConfAndAreaRead; diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index f670e494..5dc3275f 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -1,12 +1,14 @@ - - const ArchiveUtil = require('./archive_util'); const { Errors } = require('./enig_error'); const Message = require('./message'); const { splitTextAtTerms } = require('./string_util'); +const { getMessageConfTagByAreaTag } = require('./message_area'); +const StatLog = require('./stat_log'); +const Config = require('./config').get; +const SysProps = require('./system_property'); const { EventEmitter } = require('events'); -const temptmp = require('temptmp').createTrackedSession('qwk_packet'); +const temptmp = require('temptmp'); const async = require('async'); const fs = require('graceful-fs'); const paths = require('path'); @@ -16,27 +18,64 @@ const moment = require('moment'); const _ = require('lodash'); const IniConfigParser = require('ini-config-parser'); -const SMBTZToUTCOffset = (smbTZ) => { - // convert a Synchronet smblib TZ to a UTC offset - // see https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h - return { - // US Standard - '40F0' : '-04:00', // Atlantic - '412C' : '-05:00', // Eastern - '4168' : '-06:00', // Central - '41A4' : '-07:00', // Mountain - '41E0' : '-08:00', // Pacific - '421C' : '-09:00', // Yukon - '4258' : '-10:00', // Hawaii/Alaska - '4294' : '-11:00', // Bering +const enigmaVersion = require('../package.json').version; - // US Daylight - // :TODO: FINISH ME! +// Synchronet smblib TZ to a UTC offset +// see https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h +const SMBTZToUTCOffset = { + // US Standard + '40F0' : '-04:00', // Atlantic + '412C' : '-05:00', // Eastern + '4168' : '-06:00', // Central + '41A4' : '-07:00', // Mountain + '41E0' : '-08:00', // Pacific + '421C' : '-09:00', // Yukon + '4258' : '-10:00', // Hawaii/Alaska + '4294' : '-11:00', // Bering - }[smbTZ]; + // US Daylight + 'C0F0' : '-03:00', // Atlantic + 'C12C' : '-04:00', // Eastern + 'C168' : '-05:00', // Central + 'C1A4' : '-06:00', // Mountain + 'C1E0' : '-07:00', // Pacific + 'C21C' : '-08:00', // Yukon + 'C258' : '-09:00', // Hawaii/Alaska + 'C294' : '-10:00', // Bering + + // "Non-Standard" + '2294' : '-11:00', // Midway + '21E0' : '-08:00', // Vancouver + '21A4' : '-07:00', // Edmonton + '2168' : '-06:00', // Winnipeg + '212C' : '-05:00', // Bogota + '20F0' : '-04:00', // Caracas + '20B4' : '-03:00', // Rio de Janeiro + '2078' : '-02:00', // Fernando de Noronha + '203C' : '-01:00', // Azores + '1000' : '+00:00', // London + '103C' : '+01:00', // Berlin + '1078' : '+02:00', // Athens + '10B4' : '+03:00', // Moscow + '10F0' : '+04:00', // Dubai + '110E' : '+04:30', // Kabul + '112C' : '+05:00', // Karachi + '114A' : '+05:30', // Bombay + '1159' : '+05:45', // Kathmandu + '1168' : '+06:00', // Dhaka + '11A4' : '+07:00', // Bangkok + '11E0' : '+08:00', // Hong Kong + '121C' : '+09:00', // Tokyo + '1258' : '+10:00', // Sydney + '1294' : '+11:00', // Noumea + '12D0' : '+12:00', // Wellington }; -const QWKMessageBlockSize = 128; +const UTCOffsetToSMBTZ = _.invert(SMBTZToUTCOffset); + +const QWKMessageBlockSize = 128; +const QWKHeaderTimestampFormat = 'MM-DD-YYHH:mm'; +const QWKLF = 0xe3; // See the following: // - http://fileformats.archiveteam.org/wiki/QWK @@ -95,14 +134,29 @@ const MessageHeaderParser = new Parser() .uint16('relNum') .uint8('netTag'); +const replaceCharInBuffer = (buffer, search, replace) => { + let i = 0; + search = Buffer.from([search]); + while (i < buffer.length) { + i = buffer.indexOf(search, i); + if (-1 === i) { + break; + } + buffer[i] = replace; + ++i; + } +} class QWKPacketReader extends EventEmitter { - constructor(packetPath, mode=QWKPacketReader.Modes.Guess, options = { keepTearAndOrigin : true } ) { + constructor( + packetPath, + { mode = QWKPacketReader.Modes.Guess, keepTearAndOrigin = true } = { mode : QWKPacketReader.Modes.Guess, keepTearAndOrigin : true }) + { super(); this.packetPath = packetPath; - this.mode = mode; - this.options = options; + this.options = { mode, keepTearAndOrigin }; + this.temptmp = temptmp.createTrackedSession('qwkpacketreader'); } static get Modes() { @@ -137,7 +191,7 @@ class QWKPacketReader extends EventEmitter { }, // create a temporary location to do processing (archiveType, callback) => { - temptmp.mkdir( { prefix : 'enigqwkpacket-'}, (err, tempDir) => { + this.temptmp.mkdir( { prefix : 'enigqwkreader-'}, (err, tempDir) => { if (err) { return callback(err); } @@ -172,20 +226,20 @@ class QWKPacketReader extends EventEmitter { switch (key) { case 'MESSAGES.DAT' : // QWK - if (this.mode === QWKPacketReader.Modes.Guess) { - this.mode = QWKPacketReader.Modes.QWK; + if (this.options.mode === QWKPacketReader.Modes.Guess) { + this.options.mode = QWKPacketReader.Modes.QWK; } - if (this.mode === QWKPacketReader.Modes.QWK) { + if (this.options.mode === QWKPacketReader.Modes.QWK) { out.messages = { filename }; } break; case 'ID.MSG' : - if (this.mode === QWKPacketReader.Modes.Guess) { - this.mode = Modes.REP; + if (this.options.mode === QWKPacketReader.Modes.Guess) { + this.options.mode = Modes.REP; } - if (this.mode === QWKPacketReader.Modes.REP) { + if (this.options.mode === QWKPacketReader.Modes.REP) { out.messages = { filename }; } break; @@ -258,12 +312,9 @@ class QWKPacketReader extends EventEmitter { (callback) => { return this.processPacketFiles(callback); }, - (tempDir, callback) => { - return callback(null); - } ], err => { - temptmp.cleanup(); + this.temptmp.cleanup(); if (err) { return this.emit('error', err); @@ -386,14 +437,19 @@ class QWKPacketReader extends EventEmitter { const path = paths.join(this.packetInfo.tempDir, this.packetInfo.headers.filename); fs.readFile(path, { encoding : 'utf8' }, (err, iniData) => { if (err) { - this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid (${err.message})`)); + this.emit('warning', Errors.Invalid(`Problem reading HEADERS.DAT: ${err.message}`)); return cb(null); // non-fatal } try { - this.packetInfo.headers.ini = IniConfigParser.parse(iniData); + const parserOptions = { + lineComment : false, // no line comments; consume full lines + nativeType : false, // just keep everything as strings + dotKey : false, // 'a.b.c = value' stays 'a.b.c = value' + }; + this.packetInfo.headers.ini = IniConfigParser.parse(iniData, parserOptions); } catch (e) { - this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid (${e.message})`)); + this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid: ${e.message}`)); } return cb(null); @@ -401,7 +457,6 @@ class QWKPacketReader extends EventEmitter { } readMessages(cb) { - // :TODO: update to use proper encoding: if headers.dat specifies UTF-8, use that, else CP437 if (!this.packetInfo.messages) { return cb(Errors.DoesNotExist('No messages file found within QWK packet')); } @@ -420,7 +475,6 @@ class QWKPacketReader extends EventEmitter { const FTNPropertyMapping = { 'X-FTN-AREA' : Message.FtnPropertyNames.FtnArea, 'X-FTN-SEEN-BY' : Message.FtnPropertyNames.FtnSeenBy, - 'X-FTN-FLAGS' : Message.FtnPropertyNames }; const FTNKludgeMapping = { @@ -452,6 +506,10 @@ class QWKPacketReader extends EventEmitter { Reply : '@REPLY:', TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h ReplyTo : '@REPLYTO:', + + // :TODO: Look into other non-standards + // https://github.com/wmcbrine/MultiMail/blob/master/mmail/qwk.cc + // title, @subject, etc. }; let blockCount = 0; @@ -480,7 +538,7 @@ class QWKPacketReader extends EventEmitter { if (0 === blockCount) { // first 128 bytes is a space padded ID const id = buffer.toString('ascii').trim(); - this.emit('generator', id); + this.emit('creator', id); state = 'header'; } else { switch (state) { @@ -493,7 +551,7 @@ class QWKPacketReader extends EventEmitter { header[field] = iconv.decode(header[field], encodingToSpec).trim(); }); - header.timestamp = moment(header.timestamp, 'MM-DD-YYHH:mm'); + header.timestamp = moment(header.timestamp, QWKHeaderTimestampFormat); currMessage = { header, @@ -523,7 +581,7 @@ class QWKPacketReader extends EventEmitter { case 'message' : if (!currMessage.body) { - currMessage.body = buffer; + currMessage.body = Buffer.from(buffer); } else { currMessage.body = Buffer.concat([currMessage.body, buffer]); } @@ -534,16 +592,7 @@ class QWKPacketReader extends EventEmitter { // First, replace QWK style line feeds (0xe3) unless the message is UTF-8. // If the message is UTF-8, we assume it's using standard line feeds. if (encoding !== 'utf8') { - let i = 0; - const QWKLF = Buffer.from([0xe3]); - while (i < currMessage.body.length) { - i = currMessage.body.indexOf(QWKLF, i); - if (-1 === i) { - break; - } - currMessage.body[i] = 0x0a; - ++i; - } + replaceCharInBuffer(currMessage.body, QWKLF, 0x0a); } // @@ -583,15 +632,15 @@ class QWKPacketReader extends EventEmitter { } else if (line.startsWith(Kludges.Subject)) { currMessage.subject = line.substring(Kludges.Subject.length).trim(); } else if (line.startsWith(Kludges.Via)) { - qwkKludge.via = line; + qwkKludge['@VIA'] = line; } else if (line.startsWith(Kludges.MsgID)) { - qwkKludge.msg_id = line.substring(Kludges.MsgID.length).trim(); + qwkKludge['@MSGID'] = line.substring(Kludges.MsgID.length).trim(); } else if (line.startsWith(Kludges.Reply)) { - qwkKludge.in_reply_to_msg_id = line.substring(Kludges.Reply.length).trim(); + qwkKludge['@REPLY'] = line.substring(Kludges.Reply.length).trim(); } else if (line.startsWith(Kludges.TZ)) { - qwkKludge.synchronet_timezone = line.substring(Kludges.TZ.length).trim(); + qwkKludge['@TZ'] = line.substring(Kludges.TZ.length).trim(); } else if (line.startsWith(Kludges.ReplyTo)) { - qwkKludge.reply_to = line.substring(Kludges.ReplyTo.length).trim(); + qwkKludge['@REPLYTO'] = line.substring(Kludges.ReplyTo.length).trim(); } else { bodyState = 'body'; // past this point and up to any tear/origin/etc., is the real message body bodyLines.push(line); @@ -620,12 +669,12 @@ class QWKPacketReader extends EventEmitter { const ext = currMessage.headersExtension; // to and subject can be overridden yet again if entries are present - currMessage.toName = ext.To || currMessage.toName + currMessage.toName = ext.To || currMessage.toName; currMessage.subject = ext.Subject || currMessage.subject; - currMessage.from = ext.Sender || currMessage.from; // why not From? Who the fuck knows. + currMessage.from = ext.Sender || currMessage.fromName; // why not From? Who the fuck knows. // possibly override message ID kludge - qwkKludge.msg_id = ext['Message-ID'] || qwkKludge.msg_id; + qwkKludge['@MSGID'] = ext['Message-ID'] || qwkKludge['@MSGID']; // WhenWritten contains a ISO-8601-ish timestamp and a Synchronet/SMB style TZ offset: // 20180101174837-0600 4168 @@ -640,7 +689,7 @@ class QWKPacketReader extends EventEmitter { } if (ext.Tags) { - currMessage.hashTags = ext.Tags.split(' '); + currMessage.hashTags = (ext.Tags).toString().split(' '); } // FTN style properties/kludges represented as X-FTN-XXXX @@ -668,6 +717,9 @@ class QWKPacketReader extends EventEmitter { hashTags : currMessage.hashTags, }); + // Indicate this message was imported from a QWK packet + message.meta.System[Message.SystemMetaNames.ExternalFlavor] = Message.AddressFlavor.QWK; + if (!_.isEmpty(qwkKludge)) { message.meta.QwkKludge = qwkKludge; } @@ -692,8 +744,8 @@ class QWKPacketReader extends EventEmitter { } // Update the timestamp if we have a valid TZ - if (useTZKludge && _.has(message, 'meta.QwkKludge.synchronet_timezone')) { - const tzOffset = SMBTZToUTCOffset(message.meta.QwkKludge.synchronet_timezone); + if (useTZKludge && qwkKludge['@TZ']) { + const tzOffset = SMBTZToUTCOffset[qwkKludge['@TZ']]; if (tzOffset) { message.modTimestamp.utcOffset(tzOffset); } @@ -704,7 +756,7 @@ class QWKPacketReader extends EventEmitter { qwk_in_reply_to_num : currMessage.header.replyToNum, }; - if (this.mode === QWKPacketReader.Modes.QWK) { + if (this.options.mode === QWKPacketReader.Modes.QWK) { message.meta.QwkProperty.qwk_msg_num = currMessage.header.num; message.meta.QwkProperty.qwk_conf_num = currMessage.header.confNum; } else { @@ -735,7 +787,379 @@ class QWKPacketReader extends EventEmitter { } }; +class QWKPacketWriter extends EventEmitter { + constructor( + { + enableQWKE = true, + enableHeadersExtension = true, + enableAtKludges = true, + encoding = 'cp437', + systemDomain = 'enigma-bbs', + bbsID = '', + toUser = '', + } = QWKPacketWriter.DefaultOptions) + { + super(); + + this.options = { + enableQWKE, + enableHeadersExtension, + enableAtKludges, + systemDomain, + bbsID, + toUser, + encoding : encoding.toLowerCase(), + }; + + this.temptmp = temptmp.createTrackedSession('qwkpacketwriter'); + } + + static get DefaultOptions() { + return { + enableQWKE : true, + enableHeadersExtension : true, + enableAtKludges : true, + encoding : 'cp437', + systemDomain : 'enigma-bbs', + bbsID : '', + toUser : '', + }; + } + + init() { + async.series( + [ + (callback) => { + return StatLog.init(callback); + }, + (callback) => { + this.temptmp.mkdir( { prefix : 'enigqwkwriter-'}, (err, workDir) => { + this.workDir = workDir; + return callback(err); + }); + }, + (callback) => { + this.messagesStream = fs.createWriteStream(paths.join(this.workDir, 'messages.dat')); + + if (this.options.enableHeadersExtension) { + this.headersDatStream = fs.createWriteStream(paths.join(this.workDir, 'headers.dat')); + } + + // First block is a space padded ID + const id = `Created with ENiGMA 1/2 BBS v${enigmaVersion} Copyright (c) 2015-2020 Bryan Ashby`; + this.messagesStream.write(id.padEnd(QWKMessageBlockSize, ' '), 'ascii'); + this.currentMessageOffset = QWKMessageBlockSize; + + this.totalMessages = 0; + this.areaTagsSeen = new Set(); + + return callback(null); + }, + ], + err => { + if (err) { + return this.emit('error', err); + } + + this.emit('ready'); + } + ) + } + + makeMessageIdentifier(message) { + return `<${message.messageId}.${message.messageUuid}@${this.options.systemDomain}>`; + } + + appendMessage(message) { + // + // Each message has to: + // - Append to MESSAGES.DAT + // - Append to HEADERS.DAT if enabled + // + // If this is a personal (ie: non-network) packet: + // - Produce PERSONAL.NDX + // - Produce 000.NDX with pointers to the users personal "inbox" mail + // - Produce ####.NDX with pointers to the public/conference mail + // - Produce TOREADER.EXT if QWKE support is enabled + // + + let fullMessageBody = ''; + + // Start of body is kludges if enabled + if (this.options.enableQWKE) { + if (message.toUserName.length > 25) { + fullMessageBody += `To: ${message.toUserName}\n`; + } + if (message.fromUserName.length > 25) { + fullMessageBody += `From: ${message.fromUserName}\n`; + } + if (message.subject.length > 25) { + fullMessageBody += `Subject: ${message.subject}\n`; + } + } + + if (this.options.enableAtKludges) { + // Add in original kludges (perhaps in a different order) if + // they were originally imported + if (Message.AddressFlavor.QWK == message.meta.System[Message.SystemMetaNames.ExternalFlavor]) { + if (message.meta.QwkKludge) { + for (let [kludge, value] of Object.entries(message.meta.QwkKludge)) { + fullMessageBody += `${kludge}: ${value}\n`; + }; + } + } else { + fullMessageBody += `@MSGID: ${this.makeMessageIdentifier(message)}\n`; + } + } + + // The actual message contents + fullMessageBody += message.message; + + // :TODO: sanitize line feeds -> \n ???? + + // splitTextAtTerms(message.message).forEach(line => { + // appendBodyLine(line); + // }); + + const encodedMessage = iconv.encode(fullMessageBody, this.options.encoding); + + // + // QWK spec wants line feeds as 0xe3 for some reason, so we'll have + // to replace the \n's. If we're going against the spec and using UTF-8 + // we can just leave them be. + // + if ('utf8' !== this.options.encoding) { + replaceCharInBuffer(encodedMessage, 0x0a, QWKLF); + } + + // Messages must comprise of multiples of 128 bit blocks with the last + // block padded by spaces or nulls (we use nulls) + const fullBlocks = Math.trunc(encodedMessage.length / QWKMessageBlockSize); + const remainBytes = QWKMessageBlockSize - (encodedMessage.length % QWKMessageBlockSize); + + // The first block is always a header + this._writeMessageHeader( + message, + fullBlocks + 1 + (remainBytes ? 1 : 0), + ); + + this.messagesStream.write(encodedMessage); + + + if (remainBytes) { + this.messagesStream.write(Buffer.alloc(remainBytes, 0x00)); + } + + if (this.options.enableHeadersExtension) { + this._appendHeadersExtensionData(message); + } + + this.currentMessageOffset += fullBlocks * QWKMessageBlockSize; + + if (remainBytes) + { + this.currentMessageOffset += QWKMessageBlockSize; + } + + this.totalMessages += 1; + this.areaTagsSeen.add(message.areaTag); + } + + appendNewFile() { + + } + + finish(packetPath) { + async.series( + [ + (callback) => { + this.messagesStream.on('close', () => { + return callback(null); + }); + this.messagesStream.end(); + }, + (callback) => { + if (!this.headersDatStream) { + return callback(null); + } + this.headersDatStream.on('close', () => { + return callback(null); + }); + this.headersDatStream.end(); + }, + (callback) => { + return this._createControlData(callback); + } + ], + err => { + this.temptmp.cleanup(); + + if (err) { + return this.emit('error', err); + } + + this.emit('finished'); + } + ) + } + + _writeMessageHeader(message, totalBlocks) { + const asciiNum = (n, l) => { + if (isNaN(n)) { + return ''; + } + return n.toString().substr(0, l); + }; + + const status = 'FIXME'; + const totalBlocksStr = asciiNum(totalBlocks, 6);//totalBlocks.toString().padEnd(6, ' '); + const messageStatus = 255; // :TODO: ever anything different? + const confNumber = 1004; // :TODO: areaTag -> conf mapping + const netTag = ' '; // :TODO: + + if (totalBlocksStr.length > 6) { + return this.emit('warning', Errors.General('Message too large for packet'), message); + } + + const header = Buffer.alloc(QWKMessageBlockSize, ' '); + header.write(status[0], 0, 1, 'ascii'); + header.write(asciiNum(message.messageId), 1, 'ascii'); // :TODO: It seems Sync puts the relative, as in # of messages we've called appendMessage()?! + header.write(message.modTimestamp.format(QWKHeaderTimestampFormat), 8, 13, 'ascii'); + header.write(message.toUserName.substr(0, 25), 21, 'ascii'); + header.write(message.fromUserName.substr(0, 25), 46, 'ascii'); + header.write(message.subject.substr(0, 25), 71, 'ascii'); + header.write(' '.repeat(12), 96, 'ascii'); // we don't use the password field + header.write(asciiNum(message.replyToMsgId), 108, 'ascii'); + header.write(asciiNum(totalBlocks, 6), 116, 'ascii'); + header.writeUInt8(messageStatus, 122); + header.writeUInt16LE(confNumber, 123); + header.writeUInt16LE(0, 125); // :TODO: Check if others actually populate this + header.write(netTag[0], 127, 1, 'ascii'); + + this.messagesStream.write(header); + } + + _createControlData(cb) { + const controlStream = fs.createWriteStream(paths.join(this.workDir, 'control.dat')); + controlStream.setDefaultEncoding('ascii'); + + controlStream.on('close', () => { + return cb(null); + }); + + controlStream.on('error', err => { + return cb(err); + }); + + const controlData = [ + Config().general.boardName, + 'Earth', + 'XXX-XXX-XXX', + `${StatLog.getSystemStat(SysProps.SysOpUsername)}, Sysop`, + `0000,${this.options.bbsID}`, + moment().format('MM-DD-YYYY,HH:mm:ss'), + this.options.toUser, + '', // name of Qmail menu + '0', // uh, OK + this.totalMessages.toString(), + // this next line is total conferences - 1: + // We have areaTag <> conference mapping, so the number should work out + (this.areaTagsSeen.size - 1).toString(), + + // :TODO: append all areaTag->conf number/IDs and names (13 chars max) + '0', 'First Conf', + 'HELLO', + 'BBSNEWS', + 'GOODBYE', + ]; + + controlData.forEach(line => { + controlStream.write(`${line}\r\n`); + }); + + controlStream.end(); + } + + _makeSynchronetTimestamp(ts) { + const syncTimestamp = ts.format('YYYYMMDDHHmmssZZ'); + const syncTZ = UTCOffsetToSMBTZ[ts.format('Z')] || '0000'; // :TODO: what if we don't have a map? + return `${syncTimestamp} ${syncTZ}`; + } + + _appendHeadersExtensionData(message) { + const messageData = { + // Synchronet style + Utf8 : ('utf8' === this.options.encoding ? 'true' : 'false'), + 'Message-ID' : this.makeMessageIdentifier(message), + + WhenWritten : this._makeSynchronetTimestamp(message.modTimestamp), + // WhenImported : '', // :TODO: only if we have a imported time from another external system? + ExportedFrom : `${this.options.systemID} ${message.areaTag} ${message.messageId}`, + Sender : message.fromUserName, + + // :TODO: if exporting for QWK-Net style/etc. + //SenderNetAddr + + SenderIpAddr : '127.0.0.1', // no sir, that's private. + SenderHostName : this.options.systemDomain, + // :TODO: if exported: + //SenderProtocol + Organization : 'BBS', + + //'Reply-To' : :TODO: "address to direct replies".... ?! + Subject : message.subject, + To : message.toUserName, + //ToNetAddr : :TODO: net addr to?! + + // :TODO: Only set if not imported: + Tags : message.hashTags.join(' '), + + // :TODO: Needs tested with Sync/etc.; Sync wants Conference *numbers* + Conference : getMessageConfTagByAreaTag(message.areaTag), + + // ENiGMA Headers + MessageUUID : message.uuid, + ModTimestamp : message.modTimestamp.format('YYYY-MM-DDTHH:mm:ss.SSSZ'), + AreaTag : message.areaTag, + }; + + const externalFlavor = message.meta.System[Message.SystemMetaNames.ExternalFlavor]; + if (externalFlavor === Message.AddressFlavor.FTN) { + // Add FTN properties if it came from such an origin + if (message.meta.FtnProperty) { + const ftnProp = message.meta.FtnProperty; + messageData['X-FTN-AREA'] = ftnProp[Message.FtnPropertyNames.FtnArea]; + messageData['X-FTN-SEEN-BY'] = fntProp[Message.FtnPropertyNames.FtnSeenBy]; + } + + if (message.meta.FtnKludge) { + const ftnKludge = message.meta.FtnKludge; + messageData['X-FTN-PATH'] = ftnKludge.PATH; + messageData['X-FTN-MSGID'] = ftnKludge.MSGID; + messageData['X-FTN-REPLY'] = fntKludge.REPLY; + messageData['X-FTN-PID'] = fntKludge.PID; + messageData['X-FTN-FLAGS'] = ftnKludge.FLAGS; + messageData['X-FTN-TID'] = fntKludge.TID; + messageData['X-FTN-CHRS'] = fntKludge.CHRS; + } + } else { + messageData.WhenExported = this._makeSynchronetTimestamp(moment()); + messageData.Editor = `ENiGMA 1/2 BBS FSE v${enigmaVersion}`; + } + + this.headersDatStream.write(iconv.encode(`[${this.currentMessageOffset.toString(16)}]\r\n`, this.options.encoding)); + + for (let [name, value] of Object.entries(messageData)) { + if (value) { + this.headersDatStream.write(iconv.encode(`${name}: ${value}\r\n`, this.options.encoding)); + } + } + + this.headersDatStream.write('\r\n'); + } +} + module.exports = { QWKPacketReader, -// QWKPacketWriter, + QWKPacketWriter, } From 2b7d810c777ddb96961c179305de456c50d5d894 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 27 Apr 2020 20:55:41 -0600 Subject: [PATCH 5/8] Lots of progress on packet writing, reading, etc. * Bug fixes * Create packet archive --- core/archive_util.js | 26 ++++-- core/qwk_mail_packet.js | 180 +++++++++++++++++++++++++++++++--------- 2 files changed, 160 insertions(+), 46 deletions(-) diff --git a/core/archive_util.js b/core/archive_util.js index 8549cd12..47291860 100644 --- a/core/archive_util.js +++ b/core/archive_util.js @@ -204,23 +204,37 @@ module.exports = class ArchiveUtil { }); } - compressTo(archType, archivePath, files, cb) { + compressTo(archType, archivePath, files, workDir, cb) { const archiver = this.getArchiver(archType, paths.extname(archivePath)); if(!archiver) { return cb(Errors.Invalid(`Unknown archive type: ${archType}`)); } + if (!cb && _.isFunction(workDir)) { + cb = workDir; + workDir = null; + } + const fmtObj = { archivePath : archivePath, fileList : files.join(' '), // :TODO: probably need same hack as extractTo here! }; - const args = archiver.compress.args.map( arg => stringFormat(arg, fmtObj) ); + // :TODO: DRY with extractTo() + const args = archiver.compress.args.map( arg => { + return '{fileList}' === arg ? arg : stringFormat(arg, fmtObj); + }); + + const fileListPos = args.indexOf('{fileList}'); + if(fileListPos > -1) { + // replace {fileList} with 0:n sep file list arguments + args.splice.apply(args, [fileListPos, 1].concat(files)); + } let proc; try { - proc = pty.spawn(archiver.compress.cmd, args, this.getPtyOpts()); + proc = pty.spawn(archiver.compress.cmd, args, this.getPtyOpts(workDir)); } catch(e) { return cb(Errors.ExternalProcess( `Error spawning archiver process "${archiver.compress.cmd}" with args "${args.join(' ')}": ${e.message}`) @@ -332,15 +346,15 @@ module.exports = class ArchiveUtil { }); } - getPtyOpts(extractPath) { + getPtyOpts(cwd) { const opts = { name : 'enigma-archiver', cols : 80, rows : 24, env : process.env, }; - if(extractPath) { - opts.cwd = extractPath; + if(cwd) { + opts.cwd = cwd; } // :TODO: set cwd to supplied temp path if not sepcific extract return opts; diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index 5dc3275f..e5c316b1 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -2,7 +2,10 @@ const ArchiveUtil = require('./archive_util'); const { Errors } = require('./enig_error'); const Message = require('./message'); const { splitTextAtTerms } = require('./string_util'); -const { getMessageConfTagByAreaTag } = require('./message_area'); +const { + getMessageConfTagByAreaTag, + getMessageAreaByTag, +} = require('./message_area'); const StatLog = require('./stat_log'); const Config = require('./config').get; const SysProps = require('./system_property'); @@ -77,6 +80,26 @@ const QWKMessageBlockSize = 128; const QWKHeaderTimestampFormat = 'MM-DD-YYHH:mm'; const QWKLF = 0xe3; +const QWKMessageStatusCodes = { + UnreadPublic : ' ', + ReadPublic : '-', + ReadBySomeonePrivate : '*', + UnreadPrivate : '+', + UnreadCommentToSysOp : '~', + ReadCommentToSysOp : '`', + UnreadSenderPWProtected : '%', + ReadSenderPWProtected : '^', + UnreadGroupPWProtected : '!', + ReadGroupPWProtected : '#', + PWProtectedToAll : '$', + Vote : 'V', +}; + +const QWKMessageActiveStatus = { + Active : 255, + Deleted : 226, +}; + // See the following: // - http://fileformats.archiveteam.org/wiki/QWK // - http://wiki.synchro.net/ref:qwk @@ -796,7 +819,8 @@ class QWKPacketWriter extends EventEmitter { encoding = 'cp437', systemDomain = 'enigma-bbs', bbsID = '', - toUser = '', + user = null, + archiveFormat = 'application/zip', } = QWKPacketWriter.DefaultOptions) { super(); @@ -807,7 +831,8 @@ class QWKPacketWriter extends EventEmitter { enableAtKludges, systemDomain, bbsID, - toUser, + user, + archiveFormat, encoding : encoding.toLowerCase(), }; @@ -822,7 +847,8 @@ class QWKPacketWriter extends EventEmitter { encoding : 'cp437', systemDomain : 'enigma-bbs', bbsID : '', - toUser : '', + user : null, + archiveFormat :'application/zip', }; } @@ -913,13 +939,12 @@ class QWKPacketWriter extends EventEmitter { } // The actual message contents - fullMessageBody += message.message; + //fullMessageBody += message.message; - // :TODO: sanitize line feeds -> \n ???? - - // splitTextAtTerms(message.message).forEach(line => { - // appendBodyLine(line); - // }); + // Sanitize line feeds (e.g. CRLF -> LF, and possibly -> QWK style below) + splitTextAtTerms(message.message).forEach(line => { + fullMessageBody += `${line}\n`; + }); const encodedMessage = iconv.encode(fullMessageBody, this.options.encoding); @@ -938,16 +963,20 @@ class QWKPacketWriter extends EventEmitter { const remainBytes = QWKMessageBlockSize - (encodedMessage.length % QWKMessageBlockSize); // The first block is always a header - this._writeMessageHeader( + if (!this._writeMessageHeader( message, fullBlocks + 1 + (remainBytes ? 1 : 0), - ); + )) + { + // we can't write this message + return; + } this.messagesStream.write(encodedMessage); if (remainBytes) { - this.messagesStream.write(Buffer.alloc(remainBytes, 0x00)); + this.messagesStream.write(Buffer.alloc(remainBytes, ' ')); } if (this.options.enableHeadersExtension) { @@ -989,6 +1018,9 @@ class QWKPacketWriter extends EventEmitter { }, (callback) => { return this._createControlData(callback); + }, + (callback) => { + return this._producePacketArchive(packetPath, callback); } ], err => { @@ -1003,6 +1035,41 @@ class QWKPacketWriter extends EventEmitter { ) } + _producePacketArchive(packetPath, cb) { + const archiveUtil = ArchiveUtil.getInstance(); + + const packetFiles = [ + 'messages.dat', 'headers.dat', 'control.dat', + ].map(filename => { + return filename; + //return paths.join(this.workDir, filename); + }); + + archiveUtil.compressTo( + this.options.archiveFormat, + packetPath, + packetFiles, + this.workDir, + err => { + return cb(err); + } + ); + } + + _qwkMessageStatus(message) { + // - Public vs Private + // - Look at message pointers for read status + // - If +op is exporting and this message is to +op + // - + // :TODO: this needs addressed - handle unread vs read, +op, etc. + // ....see getNewMessagesInAreaForUser(); Variant with just IDs, or just a way to get first new message ID per area? + + if (message.isPrivate()) { + return QWKMessageStatusCodes.UnreadPrivate; + } + return QWKMessageStatusCodes.UnreadPublic; + } + _writeMessageHeader(message, totalBlocks) { const asciiNum = (n, l) => { if (isNaN(n)) { @@ -1011,18 +1078,26 @@ class QWKPacketWriter extends EventEmitter { return n.toString().substr(0, l); }; - const status = 'FIXME'; - const totalBlocksStr = asciiNum(totalBlocks, 6);//totalBlocks.toString().padEnd(6, ' '); - const messageStatus = 255; // :TODO: ever anything different? - const confNumber = 1004; // :TODO: areaTag -> conf mapping - const netTag = ' '; // :TODO: - - if (totalBlocksStr.length > 6) { - return this.emit('warning', Errors.General('Message too large for packet'), message); + const asciiTotalBlocks = asciiNum(totalBlocks, 6); + if (asciiTotalBlocks.length > 6) { + this.emit('warning', Errors.General('Message too large for packet'), message); + return false; } + const conferenceNumber = this._getMessageConferenceNumberByAreaTag(message.areaTag); + if (isNaN(conferenceNumber)) { + this.emit('warning', Errors.MissingConfig(`No QWK conference mapping for areaTag ${message.areaTag}`)); + return false; + } + + const netTag = ' '; // :TODO: + + this.lolMessageId = this.lolMessageId || 1; + //message.messageId = this.lolMessageId; + this.lolMessageId++; + const header = Buffer.alloc(QWKMessageBlockSize, ' '); - header.write(status[0], 0, 1, 'ascii'); + header.write(this._qwkMessageStatus(message), 0, 1, 'ascii'); header.write(asciiNum(message.messageId), 1, 'ascii'); // :TODO: It seems Sync puts the relative, as in # of messages we've called appendMessage()?! header.write(message.modTimestamp.format(QWKHeaderTimestampFormat), 8, 13, 'ascii'); header.write(message.toUserName.substr(0, 25), 21, 'ascii'); @@ -1030,16 +1105,35 @@ class QWKPacketWriter extends EventEmitter { header.write(message.subject.substr(0, 25), 71, 'ascii'); header.write(' '.repeat(12), 96, 'ascii'); // we don't use the password field header.write(asciiNum(message.replyToMsgId), 108, 'ascii'); - header.write(asciiNum(totalBlocks, 6), 116, 'ascii'); - header.writeUInt8(messageStatus, 122); - header.writeUInt16LE(confNumber, 123); + header.write(asciiTotalBlocks, 116, 'ascii'); + header.writeUInt8(QWKMessageActiveStatus.Active, 122); + header.writeUInt16LE(conferenceNumber, 123); header.writeUInt16LE(0, 125); // :TODO: Check if others actually populate this header.write(netTag[0], 127, 1, 'ascii'); this.messagesStream.write(header); + + return true; + } + + _getMessageConferenceNumberByAreaTag(areaTag) { + const areaConfig = _.get(Config(), [ 'messageNetworks', 'qwk', 'areas', areaTag ]); + return areaConfig && areaConfig.conference; + } + + _getExportForUsername() { + return _.get(this.options, 'user.username', 'Any'); + } + + _getExportSysOpUsername() { + return StatLog.getSystemStat(SysProps.SysOpUsername) || 'SysOp'; } _createControlData(cb) { + const areas = Array.from(this.areaTagsSeen).map(areaTag => { + return getMessageAreaByTag(areaTag); + }); + const controlStream = fs.createWriteStream(paths.join(this.workDir, 'control.dat')); controlStream.setDefaultEncoding('ascii'); @@ -1051,32 +1145,38 @@ class QWKPacketWriter extends EventEmitter { return cb(err); }); - const controlData = [ + const initialControlData = [ Config().general.boardName, 'Earth', 'XXX-XXX-XXX', - `${StatLog.getSystemStat(SysProps.SysOpUsername)}, Sysop`, + `${this._getExportSysOpUsername()}, Sysop`, `0000,${this.options.bbsID}`, moment().format('MM-DD-YYYY,HH:mm:ss'), - this.options.toUser, + this._getExportForUsername(), '', // name of Qmail menu '0', // uh, OK this.totalMessages.toString(), // this next line is total conferences - 1: // We have areaTag <> conference mapping, so the number should work out (this.areaTagsSeen.size - 1).toString(), - - // :TODO: append all areaTag->conf number/IDs and names (13 chars max) - '0', 'First Conf', - 'HELLO', - 'BBSNEWS', - 'GOODBYE', ]; - controlData.forEach(line => { + initialControlData.forEach(line => { controlStream.write(`${line}\r\n`); }); + // map areas as conf #\r\nDescription\r\n pairs + areas.forEach(area => { + const conferenceNumber = this._getMessageConferenceNumberByAreaTag(area.areaTag); + controlStream.write(`${conferenceNumber}\r\n`); + controlStream.write(`${area.name}\r\n`); + }); + + // :TODO: do we ever care here?! + ['HELLO', 'BBSNEWS', 'GOODBYE'].forEach(trailer => { + controlStream.write(`${trailer}\r\n`); + }); + controlStream.end(); } @@ -1129,18 +1229,18 @@ class QWKPacketWriter extends EventEmitter { if (message.meta.FtnProperty) { const ftnProp = message.meta.FtnProperty; messageData['X-FTN-AREA'] = ftnProp[Message.FtnPropertyNames.FtnArea]; - messageData['X-FTN-SEEN-BY'] = fntProp[Message.FtnPropertyNames.FtnSeenBy]; + messageData['X-FTN-SEEN-BY'] = ftnProp[Message.FtnPropertyNames.FtnSeenBy]; } if (message.meta.FtnKludge) { const ftnKludge = message.meta.FtnKludge; messageData['X-FTN-PATH'] = ftnKludge.PATH; messageData['X-FTN-MSGID'] = ftnKludge.MSGID; - messageData['X-FTN-REPLY'] = fntKludge.REPLY; - messageData['X-FTN-PID'] = fntKludge.PID; + messageData['X-FTN-REPLY'] = ftnKludge.REPLY; + messageData['X-FTN-PID'] = ftnKludge.PID; messageData['X-FTN-FLAGS'] = ftnKludge.FLAGS; - messageData['X-FTN-TID'] = fntKludge.TID; - messageData['X-FTN-CHRS'] = fntKludge.CHRS; + messageData['X-FTN-TID'] = ftnKludge.TID; + messageData['X-FTN-CHRS'] = ftnKludge.CHRS; } } else { messageData.WhenExported = this._makeSynchronetTimestamp(moment()); From 1f1813c14a619fda8bac46afc12b1bea852bd60e Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 30 Apr 2020 22:07:29 -0600 Subject: [PATCH 6/8] Produce NDX files, various improvements to spec, etc. --- core/mbf.js | 59 +++++++++ core/oputil/oputil_message_base.js | 203 ++++++++++++++++++++++++++--- core/qwk_mail_packet.js | 194 +++++++++++++++++++++------ 3 files changed, 399 insertions(+), 57 deletions(-) create mode 100644 core/mbf.js diff --git a/core/mbf.js b/core/mbf.js new file mode 100644 index 00000000..9c3b2f6d --- /dev/null +++ b/core/mbf.js @@ -0,0 +1,59 @@ +const { Errors } = require('./enig_error'); + +// +// Utils for dealing with Microsoft Binary Format (MBF) used +// in various BASIC languages, etc. +// +// - https://en.wikipedia.org/wiki/Microsoft_Binary_Format +// - https://stackoverflow.com/questions/2268191/how-to-convert-from-ieee-python-float-to-microsoft-basic-float +// + +// Number to 32bit MBF +numToMbf32 = (v) => { + const mbf = Buffer.alloc(4); + + if (0 === v) { + return mbf; + } + + const ieee = Buffer.alloc(4); + ieee.writeFloatLE(v, 0); + + const sign = ieee[3] & 0x80; + let exp = (ieee[3] << 1) | (ieee[2] >> 7); + + if (exp === 0xfe) { + throw Errors.Invalid(`${v} cannot be converted to mbf`); + } + + exp += 2; + + mbf[3] = exp; + mbf[2] = sign | (ieee[2] & 0x7f); + mbf[1] = ieee[1]; + mbf[0] = ieee[0]; + + return mbf; +} + +mbf32ToNum = (mbf) => { + if (0 === mbf[3]) { + return 0.0; + } + + const ieee = Buffer.alloc(4); + const sign = mbf[2] & 0x80; + const exp = mbf[3] - 2; + + ieee[3] = sign | (exp >> 1); + ieee[2] = (exp << 7) | (mbf[2] & 0x7f); + ieee[1] = mbf[1]; + ieee[0] = mbf[0]; + + return ieee.readFloatLE(0); +} + +module.exports = { + numToMbf32, + mbf32ToNum, +} \ No newline at end of file diff --git a/core/oputil/oputil_message_base.js b/core/oputil/oputil_message_base.js index e5b30c12..7d914ab2 100644 --- a/core/oputil/oputil_message_base.js +++ b/core/oputil/oputil_message_base.js @@ -445,6 +445,9 @@ function handleQWK() { case 'dump' : return dumpQWKPacket(packetPath); + case 'export' : + return exportQWKPacket(packetPath); + default : return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); } @@ -457,35 +460,81 @@ function dumpQWKPacket(packetPath) { return initConfigAndDatabases(callback); }, (callback) => { + //// + const { QWKPacketWriter } = require('../qwk_mail_packet'); + const writer = new QWKPacketWriter({ + bbsID : 'XIBALBA', + toUser : 'NuSkooler', + encoding : 'cp437', + }); + const { QWKPacketReader } = require('../qwk_mail_packet'); - const reader = new QWKPacketReader(packetPath); - reader.on('error', err => { - console.error(`ERROR: ${err.message}`); - return callback(err); + + writer.on('ready', () => { + const reader = new QWKPacketReader(packetPath); + + reader.on('error', err => { + console.error(`ERROR: ${err.message}`); + return callback(err); + }); + + reader.on('done', () => { + writer.finish(); + }); + + reader.on('archive type', archiveType => { + console.info(`-> Archive type: ${archiveType}`); + }); + + reader.on('creator', creator => { + console.info(`-> Creator: ${creator}`); + }); + + reader.on('message', message => { + writer.appendMessage(message); + }); + + reader.read(); }); - reader.on('done', () => { - return callback(null); + writer.on('finished', () => { + console.log('done'); }); - reader.on('archive type', archiveType => { - console.info(`-> Archive type: ${archiveType}`); - }); + writer.init(); - reader.on('creator', creator => { - console.info(`-> Creator: ${creator}`); - }); + //// - reader.on('message', message => { - console.info('--- message ---'); - console.info(`To : ${message.toUserName}`); - console.info(`From : ${message.fromUserName}`); - console.info(`Subject : ${message.subject}`); - console.info(`Message :\r\n${message.message}`); - }); + // const { QWKPacketReader } = require('../qwk_mail_packet'); + // const reader = new QWKPacketReader(packetPath); - reader.read(); + // reader.on('error', err => { + // console.error(`ERROR: ${err.message}`); + // return callback(err); + // }); + + // reader.on('done', () => { + // return callback(null); + // }); + + // reader.on('archive type', archiveType => { + // console.info(`-> Archive type: ${archiveType}`); + // }); + + // reader.on('creator', creator => { + // console.info(`-> Creator: ${creator}`); + // }); + + // reader.on('message', message => { + // console.info('--- message ---'); + // console.info(`To : ${message.toUserName}`); + // console.info(`From : ${message.fromUserName}`); + // console.info(`Subject : ${message.subject}`); + // console.info(`Message :\r\n${message.message}`); + // }); + + // reader.read(); } ], err => { @@ -494,6 +543,120 @@ function dumpQWKPacket(packetPath) { ) } +function exportQWKPacket(packetPath) { + // oputil mb qwk export SPEC PATH [--user USER] + // [areaTag1[@dateTime]],[...] PATH --user USER + + const posArgLen = argv._.length; + + if (posArgLen < 4) { + return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + } + + let areaTagSpecs = '*'; + if (5 === posArgLen) { + areaTagSpecs = argv._[areaTagSpecs - 2]; + } + + + //const areaTagSpecs = argv._[areaTagSpecs - 2]; + + /*const packetPath = argv._[argv._.length - 1]; + if(argv._.length < 4 || !packetPath || 0 === packetPath.length) { + return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + }*/ + + // :TODO: parse area tags(s) and timestamps + const areaTags = [ 'general', 'fsx_gen' ]; + + const userName = argv.user || '-'; + + async.waterfall( + [ + (callback) => { + return initConfigAndDatabases(callback); + }, + (callback) => { + const User = require('../../core/user.js'); + + User.getUserIdAndName(userName, (err, userId) => { + if (err) { + if ('-' === userName) { + userId = 1; + } else { + return callback(err); + } + } + return User.getUser(userId, callback); + }); + }, + (user, callback) => { + const Message = require('../message'); + + const filter = { + resultType : 'id', + areaTag : areaTags, + + // :TODO: newerThanTimestamp + }; + + // public + Message.findMessages(filter, (err, publicMessageIds) => { + if (err) { + return callback(err); + } + + delete filter.areaTag; + filter.privateTagUserId = user.userId; + + Message.findMessages(filter, (err, privateMessageIds) => { + return callback(err, user, Message, privateMessageIds.concat(publicMessageIds)); + }); + }); + }, + (user, Message, messageIds, callback) => { + const { QWKPacketWriter } = require('../qwk_mail_packet'); + const writer = new QWKPacketWriter({ + // :TODO: export needs these options + bbsID : 'XIBALBA', + toUser : 'NuSkooler', + encoding : 'cp437', + user : user, + }); + + writer.on('ready', () => { + async.eachSeries(messageIds, (messageId, nextMessageId) => { + const message = new Message(); + message.load( { messageId }, err => { + if (!err) { + writer.appendMessage(message); + } + return nextMessageId(err); + }); + }, + (err) => { + writer.finish('/home/nuskooler/Downloads/qwk2/TEST1.QWK'); + if (err) { + console.error(`Failed to write one or more messages: ${err.message}`); + } + }); + }); + + writer.on('finished', () => { + return callback(null); + }); + + writer.init(); + } + ], + err => { + if(err) { + console.error(err.reason ? err.reason : err.message); + } + } + ); +} + function handleMessageBaseCommand() { function errUsage() { diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index e5c316b1..761b71f7 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -9,6 +9,8 @@ const { const StatLog = require('./stat_log'); const Config = require('./config').get; const SysProps = require('./system_property'); +const UserProps = require('./user_property'); +const { numToMbf32 } = require('./mbf'); const { EventEmitter } = require('events'); const temptmp = require('temptmp'); @@ -83,8 +85,8 @@ const QWKLF = 0xe3; const QWKMessageStatusCodes = { UnreadPublic : ' ', ReadPublic : '-', - ReadBySomeonePrivate : '*', UnreadPrivate : '+', + ReadPrivate : '*', UnreadCommentToSysOp : '~', ReadCommentToSysOp : '`', UnreadSenderPWProtected : '%', @@ -100,6 +102,11 @@ const QWKMessageActiveStatus = { Deleted : 226, }; +const QWKNetworkTagIndicator = { + Present : '*', + NotPresent : ' ', +}; + // See the following: // - http://fileformats.archiveteam.org/wiki/QWK // - http://wiki.synchro.net/ref:qwk @@ -878,6 +885,9 @@ class QWKPacketWriter extends EventEmitter { this.totalMessages = 0; this.areaTagsSeen = new Set(); + this.personalIndex = []; // messages addressed to 'user' + this.inboxIndex = []; // private messages for 'user' + this.publicIndex = new Map(); return callback(null); }, @@ -938,9 +948,6 @@ class QWKPacketWriter extends EventEmitter { } } - // The actual message contents - //fullMessageBody += message.message; - // Sanitize line feeds (e.g. CRLF -> LF, and possibly -> QWK style below) splitTextAtTerms(message.message).forEach(line => { fullMessageBody += `${line}\n`; @@ -961,11 +968,12 @@ class QWKPacketWriter extends EventEmitter { // block padded by spaces or nulls (we use nulls) const fullBlocks = Math.trunc(encodedMessage.length / QWKMessageBlockSize); const remainBytes = QWKMessageBlockSize - (encodedMessage.length % QWKMessageBlockSize); + const totalBlocks = fullBlocks + 1 + (remainBytes ? 1 : 0); // The first block is always a header if (!this._writeMessageHeader( message, - fullBlocks + 1 + (remainBytes ? 1 : 0), + totalBlocks )) { // we can't write this message @@ -974,26 +982,61 @@ class QWKPacketWriter extends EventEmitter { this.messagesStream.write(encodedMessage); - if (remainBytes) { this.messagesStream.write(Buffer.alloc(remainBytes, ' ')); } + this._updateIndexTracking(message); + if (this.options.enableHeadersExtension) { this._appendHeadersExtensionData(message); } - this.currentMessageOffset += fullBlocks * QWKMessageBlockSize; - - if (remainBytes) - { - this.currentMessageOffset += QWKMessageBlockSize; - } + // next message starts at this block + this.currentMessageOffset += totalBlocks * QWKMessageBlockSize; this.totalMessages += 1; this.areaTagsSeen.add(message.areaTag); } + _messageAddressedToUser(message) { + if (_.isUndefined(this.cachedCompareNames)) { + if (this.options.user) { + this.cachedCompareNames = [ + this.options.user.username.toLowerCase() + ]; + const realName = this.options.user.getProperty(UserProps.RealName); + if (realName) { + this.cachedCompareNames.push(realName.toLowerCase()); + } + } else { + this.cachedCompareNames = []; + } + }; + + return this.cachedCompareNames.includes(message.toUserName.toLowerCase()); + } + + _updateIndexTracking(message) { + // index points at start of *message* not the header for... reasons? + const index = (this.currentMessageOffset / QWKMessageBlockSize) + 1; + if (message.isPrivate()) { + this.inboxIndex.push(index); + } else { + if (this._messageAddressedToUser(message)) { + // :TODO: add to both indexes??? + this.personalIndex.push(index); + } + + const areaTag = message.areaTag; + if (!this.publicIndex.has(areaTag)) { + this.publicIndex.set(areaTag, [index]); + } else { + this.publicIndex.get(areaTag).push(index); + } + } + } + appendNewFile() { } @@ -1019,6 +1062,9 @@ class QWKPacketWriter extends EventEmitter { (callback) => { return this._createControlData(callback); }, + (callback) => { + return this._createIndexes(callback); + }, (callback) => { return this._producePacketArchive(packetPath, callback); } @@ -1038,22 +1084,21 @@ class QWKPacketWriter extends EventEmitter { _producePacketArchive(packetPath, cb) { const archiveUtil = ArchiveUtil.getInstance(); - const packetFiles = [ - 'messages.dat', 'headers.dat', 'control.dat', - ].map(filename => { - return filename; - //return paths.join(this.workDir, filename); - }); - - archiveUtil.compressTo( - this.options.archiveFormat, - packetPath, - packetFiles, - this.workDir, - err => { + fs.readdir(this.workDir, (err, files) => { + if (err) { return cb(err); } - ); + + archiveUtil.compressTo( + this.options.archiveFormat, + packetPath, + files, + this.workDir, + err => { + return cb(err); + } + ); + }); } _qwkMessageStatus(message) { @@ -1090,15 +1135,9 @@ class QWKPacketWriter extends EventEmitter { return false; } - const netTag = ' '; // :TODO: - - this.lolMessageId = this.lolMessageId || 1; - //message.messageId = this.lolMessageId; - this.lolMessageId++; - const header = Buffer.alloc(QWKMessageBlockSize, ' '); header.write(this._qwkMessageStatus(message), 0, 1, 'ascii'); - header.write(asciiNum(message.messageId), 1, 'ascii'); // :TODO: It seems Sync puts the relative, as in # of messages we've called appendMessage()?! + header.write(asciiNum(message.messageId), 1, 'ascii'); header.write(message.modTimestamp.format(QWKHeaderTimestampFormat), 8, 13, 'ascii'); header.write(message.toUserName.substr(0, 25), 21, 'ascii'); header.write(message.fromUserName.substr(0, 25), 46, 'ascii'); @@ -1108,8 +1147,8 @@ class QWKPacketWriter extends EventEmitter { header.write(asciiTotalBlocks, 116, 'ascii'); header.writeUInt8(QWKMessageActiveStatus.Active, 122); header.writeUInt16LE(conferenceNumber, 123); - header.writeUInt16LE(0, 125); // :TODO: Check if others actually populate this - header.write(netTag[0], 127, 1, 'ascii'); + header.writeUInt16LE(this.totalMessages + 1, 125); + header.write(QWKNetworkTagIndicator.NotPresent, 127, 1, 'ascii'); // :TODO: Present if for network output? this.messagesStream.write(header); @@ -1117,6 +1156,9 @@ class QWKPacketWriter extends EventEmitter { } _getMessageConferenceNumberByAreaTag(areaTag) { + if (Message.isPrivateAreaTag(areaTag)) { + return 0; + } const areaConfig = _.get(Config(), [ 'messageNetworks', 'qwk', 'areas', areaTag ]); return areaConfig && areaConfig.conference; } @@ -1131,6 +1173,13 @@ class QWKPacketWriter extends EventEmitter { _createControlData(cb) { const areas = Array.from(this.areaTagsSeen).map(areaTag => { + if (Message.isPrivateAreaTag(areaTag)) { + return { + areaTag : Message.WellKnownAreaTags.Private, + name : 'Private', + desc : 'Private Messages', + }; + } return getMessageAreaByTag(areaTag); }); @@ -1168,8 +1217,12 @@ class QWKPacketWriter extends EventEmitter { // map areas as conf #\r\nDescription\r\n pairs areas.forEach(area => { const conferenceNumber = this._getMessageConferenceNumberByAreaTag(area.areaTag); + let desc = area.name; + if (area.desc) { + desc += ` - ${area.desc}` + } controlStream.write(`${conferenceNumber}\r\n`); - controlStream.write(`${area.name}\r\n`); + controlStream.write(`${desc}\r\n`); }); // :TODO: do we ever care here?! @@ -1180,6 +1233,73 @@ class QWKPacketWriter extends EventEmitter { controlStream.end(); } + _createIndexes(cb) { + const appendIndexData = (stream, offset) => { + const msb = numToMbf32(offset); + stream.write(msb); + + // technically, the conference #, but only as a byte, so pretty much useless + // AND the filename itself is the conference number... dafuq. + stream.write(Buffer.from([0x00])); + }; + + async.series( + [ + (callback) => { + // Create PERSONAL.NDX + if (!this.personalIndex.length) { + return callback(null); + } + + const indexStream = fs.createWriteStream(paths.join(this.workDir, 'personal.ndx')); + this.personalIndex.forEach(offset => appendIndexData(indexStream, offset)); + + indexStream.on('close', err => { + return callback(err); + }); + + indexStream.end(); + }, + (callback) => { + // 000.NDX of private mails + if (!this.inboxIndex.length) { + return callback(null); + } + + const indexStream = fs.createWriteStream(paths.join(this.workDir, '000.ndx')); + this.inboxIndex.forEach(offset => appendIndexData(indexStream, offset)); + + indexStream.on('close', err => { + return callback(err); + }); + + indexStream.end(); + }, + (callback) => { + // ####.NDX + async.eachSeries(this.publicIndex.keys(), (areaTag, nextArea) => { + const offsets = this.publicIndex.get(areaTag); + const conferenceNumber = this._getMessageConferenceNumberByAreaTag(areaTag); + const indexStream = fs.createWriteStream(paths.join(this.workDir, `${conferenceNumber.toString()}.ndx`)); + offsets.forEach(offset => appendIndexData(indexStream, offset)); + + indexStream.on('close', err => { + return nextArea(err); + }); + + indexStream.end(); + }, + err => { + return callback(err); + }); + } + ], + err => { + return cb(err); + } + ); + } + _makeSynchronetTimestamp(ts) { const syncTimestamp = ts.format('YYYYMMDDHHmmssZZ'); const syncTZ = UTCOffsetToSMBTZ[ts.format('Z')] || '0000'; // :TODO: what if we don't have a map? @@ -1215,7 +1335,7 @@ class QWKPacketWriter extends EventEmitter { Tags : message.hashTags.join(' '), // :TODO: Needs tested with Sync/etc.; Sync wants Conference *numbers* - Conference : getMessageConfTagByAreaTag(message.areaTag), + Conference : message.isPrivate() ? '0' : getMessageConfTagByAreaTag(message.areaTag), // ENiGMA Headers MessageUUID : message.uuid, From 8817113364322f5a96f6f42c756b7fa4d14a36aa Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 2 May 2020 13:34:28 -0600 Subject: [PATCH 7/8] * Create bundle filenames to spec * Better cp437 vs utf8 vs other encoding support * Add some CP437 and related utils --- core/cp437util.js | 55 +++++++++++++ core/message.js | 17 ++++ core/oputil/oputil_message_base.js | 8 +- core/qwk_mail_packet.js | 121 ++++++++++++++++++++++++----- core/string_util.js | 15 ++++ 5 files changed, 191 insertions(+), 25 deletions(-) create mode 100644 core/cp437util.js diff --git a/core/cp437util.js b/core/cp437util.js new file mode 100644 index 00000000..32425d3a --- /dev/null +++ b/core/cp437util.js @@ -0,0 +1,55 @@ + + +const CP437UnicodeTable = [ + '\u0000', '\u0001', '\u0002', '\u0003', '\u0004', '\u0005', '\u0006', + '\u0007', '\u0008', '\u0009', '\u000A', '\u000B', '\u000C', '\u000D', + '\u000E', '\u000F', '\u0010', '\u0011', '\u0012', '\u0013', '\u0014', + '\u0015', '\u0016', '\u0017', '\u0018', '\u0019', '\u001A', '\u001B', + '\u001C', '\u001D', '\u001E', '\u001F', '\u0020', '\u0021', '\u0022', + '\u0023', '\u0024', '\u0025', '\u0026', '\u0027', '\u0028', '\u0029', + '\u002A', '\u002B', '\u002C', '\u002D', '\u002E', '\u002F', '\u0030', + '\u0031', '\u0032', '\u0033', '\u0034', '\u0035', '\u0036', '\u0037', + '\u0038', '\u0039', '\u003A', '\u003B', '\u003C', '\u003D', '\u003E', + '\u003F', '\u0040', '\u0041', '\u0042', '\u0043', '\u0044', '\u0045', + '\u0046', '\u0047', '\u0048', '\u0049', '\u004A', '\u004B', '\u004C', + '\u004D', '\u004E', '\u004F', '\u0050', '\u0051', '\u0052', '\u0053', + '\u0054', '\u0055', '\u0056', '\u0057', '\u0058', '\u0059', '\u005A', + '\u005B', '\u005C', '\u005D', '\u005E', '\u005F', '\u0060', '\u0061', + '\u0062', '\u0063', '\u0064', '\u0065', '\u0066', '\u0067', '\u0068', + '\u0069', '\u006A', '\u006B', '\u006C', '\u006D', '\u006E', '\u006F', + '\u0070', '\u0071', '\u0072', '\u0073', '\u0074', '\u0075', '\u0076', + '\u0077', '\u0078', '\u0079', '\u007A', '\u007B', '\u007C', '\u007D', + '\u007E', '\u007F', '\u00C7', '\u00FC', '\u00E9', '\u00E2', '\u00E4', + '\u00E0', '\u00E5', '\u00E7', '\u00EA', '\u00EB', '\u00E8', '\u00EF', + '\u00EE', '\u00EC', '\u00C4', '\u00C5', '\u00C9', '\u00E6', '\u00C6', + '\u00F4', '\u00F6', '\u00F2', '\u00FB', '\u00F9', '\u00FF', '\u00D6', + '\u00DC', '\u00A2', '\u00A3', '\u00A5', '\u20A7', '\u0192', '\u00E1', + '\u00ED', '\u00F3', '\u00FA', '\u00F1', '\u00D1', '\u00AA', '\u00BA', + '\u00BF', '\u2310', '\u00AC', '\u00BD', '\u00BC', '\u00A1', '\u00AB', + '\u00BB', '\u2591', '\u2592', '\u2593', '\u2502', '\u2524', '\u2561', + '\u2562', '\u2556', '\u2555', '\u2563', '\u2551', '\u2557', '\u255D', + '\u255C', '\u255B', '\u2510', '\u2514', '\u2534', '\u252C', '\u251C', + '\u2500', '\u253C', '\u255E', '\u255F', '\u255A', '\u2554', '\u2569', + '\u2566', '\u2560', '\u2550', '\u256C', '\u2567', '\u2568', '\u2564', + '\u2565', '\u2559', '\u2558', '\u2552', '\u2553', '\u256B', '\u256A', + '\u2518', '\u250C', '\u2588', '\u2584', '\u258C', '\u2590', '\u2580', + '\u03B1', '\u00DF', '\u0393', '\u03C0', '\u03A3', '\u03C3', '\u00B5', + '\u03C4', '\u03A6', '\u0398', '\u03A9', '\u03B4', '\u221E', '\u03C6', + '\u03B5', '\u2229', '\u2261', '\u00B1', '\u2265', '\u2264', '\u2320', + '\u2321', '\u00F7', '\u2248', '\u00B0', '\u2219', '\u00B7', '\u221A', + '\u207F', '\u00B2', '\u25A0', '\u00A0' +]; + +const NonCP437EncodableRegExp = /[^\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\u0008\u0009\u000A\u000B\u000C\u000D\u000E\u000F\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001A\u001B\u001C\u001D\u001E\u001F\u0020\u0021\u0022\u0023\u0024\u0025\u0026\u0027\u0028\u0029\u002A\u002B\u002C\u002D\u002E\u002F\u0030\u0031\u0032\u0033\u0034\u0035\u0036\u0037\u0038\u0039\u003A\u003B\u003C\u003D\u003E\u003F\u0040\u0041\u0042\u0043\u0044\u0045\u0046\u0047\u0048\u0049\u004A\u004B\u004C\u004D\u004E\u004F\u0050\u0051\u0052\u0053\u0054\u0055\u0056\u0057\u0058\u0059\u005A\u005B\u005C\u005D\u005E\u005F\u0060\u0061\u0062\u0063\u0064\u0065\u0066\u0067\u0068\u0069\u006A\u006B\u006C\u006D\u006E\u006F\u0070\u0071\u0072\u0073\u0074\u0075\u0076\u0077\u0078\u0079\u007A\u007B\u007C\u007D\u007E\u007F\u00C7\u00FC\u00E9\u00E2\u00E4\u00E0\u00E5\u00E7\u00EA\u00EB\u00E8\u00EF\u00EE\u00EC\u00C4\u00C5\u00C9\u00E6\u00C6\u00F4\u00F6\u00F2\u00FB\u00F9\u00FF\u00D6\u00DC\u00A2\u00A3\u00A5\u20A7\u0192\u00E1\u00ED\u00F3\u00FA\u00F1\u00D1\u00AA\u00BA\u00BF\u2310\u00AC\u00BD\u00BC\u00A1\u00AB\u00BB\u2591\u2592\u2593\u2502\u2524\u2561\u2562\u2556\u2555\u2563\u2551\u2557\u255D\u255C\u255B\u2510\u2514\u2534\u252C\u251C\u2500\u253C\u255E\u255F\u255A\u2554\u2569\u2566\u2560\u2550\u256C\u2567\u2568\u2564\u2565\u2559\u2558\u2552\u2553\u256B\u256A\u2518\u250C\u2588\u2584\u258C\u2590\u2580\u03B1\u00DF\u0393\u03C0\u03A3\u03C3\u00B5\u03C4\u03A6\u0398\u03A9\u03B4\u221E\u03C6\u03B5\u2229\u2261\u00B1\u2265\u2264\u2320\u2321\u00F7\u2248\u00B0\u2219\u00B7\u221A\u207F\u00B2\u25A0\u00A0]/; +const isCP437Encodable = (s) => { + if (!s.length) { + return true; + } + + return !NonCP437EncodableRegExp.test(s); +} + +module.exports = { + CP437UnicodeTable, + isCP437Encodable, +} \ No newline at end of file diff --git a/core/message.js b/core/message.js index 3649a6f6..c763bbd2 100644 --- a/core/message.js +++ b/core/message.js @@ -11,6 +11,9 @@ const { sanitizeString, getISOTimestampString } = require('./database.js'); +const { isCP437Encodable } = require('./cp437util'); +const { containsNonLatinCodepoints } = require('./string_util'); + const { isAnsi, isFormattedLine, splitTextAtTerms, @@ -145,6 +148,20 @@ module.exports = class Message { return null !== _.get(this, 'meta.System.remote_from_user', null); } + isCP437Encodable() { + return isCP437Encodable(this.toUserName) && + isCP437Encodable(this.fromUserName) && + isCP437Encodable(this.subject) && + isCP437Encodable(this.message); + } + + containsNonLatinCodepoints() { + return containsNonLatinCodepoints(this.toUserName) || + containsNonLatinCodepoints(this.fromUserName) || + containsNonLatinCodepoints(this.subject) || + containsNonLatinCodepoints(this.message); + } + /* :TODO: finish me static checkUserHasDeleteRights(user, messageIdOrUuid, cb) { diff --git a/core/oputil/oputil_message_base.js b/core/oputil/oputil_message_base.js index 7d914ab2..6f677c52 100644 --- a/core/oputil/oputil_message_base.js +++ b/core/oputil/oputil_message_base.js @@ -464,8 +464,6 @@ function dumpQWKPacket(packetPath) { const { QWKPacketWriter } = require('../qwk_mail_packet'); const writer = new QWKPacketWriter({ bbsID : 'XIBALBA', - toUser : 'NuSkooler', - encoding : 'cp437', }); const { QWKPacketReader } = require('../qwk_mail_packet'); @@ -547,6 +545,8 @@ function exportQWKPacket(packetPath) { // oputil mb qwk export SPEC PATH [--user USER] // [areaTag1[@dateTime]],[...] PATH --user USER + // :TODO: bbsID from PATH filename else 'ENIGMA' + const posArgLen = argv._.length; if (posArgLen < 4) { @@ -619,8 +619,6 @@ function exportQWKPacket(packetPath) { const writer = new QWKPacketWriter({ // :TODO: export needs these options bbsID : 'XIBALBA', - toUser : 'NuSkooler', - encoding : 'cp437', user : user, }); @@ -635,7 +633,7 @@ function exportQWKPacket(packetPath) { }); }, (err) => { - writer.finish('/home/nuskooler/Downloads/qwk2/TEST1.QWK'); + writer.finish('/home/nuskooler/Downloads/qwk2/'); if (err) { console.error(`Failed to write one or more messages: ${err.message}`); } diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index 761b71f7..b7f53ebc 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -11,6 +11,7 @@ const Config = require('./config').get; const SysProps = require('./system_property'); const UserProps = require('./user_property'); const { numToMbf32 } = require('./mbf'); +const { getEncodingFromCharacterSetIdentifier } = require('./ftn_util'); const { EventEmitter } = require('events'); const temptmp = require('temptmp'); @@ -823,11 +824,11 @@ class QWKPacketWriter extends EventEmitter { enableQWKE = true, enableHeadersExtension = true, enableAtKludges = true, - encoding = 'cp437', systemDomain = 'enigma-bbs', bbsID = '', user = null, archiveFormat = 'application/zip', + forceEncoding = null, } = QWKPacketWriter.DefaultOptions) { super(); @@ -840,7 +841,7 @@ class QWKPacketWriter extends EventEmitter { bbsID, user, archiveFormat, - encoding : encoding.toLowerCase(), + forceEncoding : forceEncoding ? forceEncoding.toLowerCase() : null, }; this.temptmp = temptmp.createTrackedSession('qwkpacketwriter'); @@ -851,11 +852,11 @@ class QWKPacketWriter extends EventEmitter { enableQWKE : true, enableHeadersExtension : true, enableAtKludges : true, - encoding : 'cp437', systemDomain : 'enigma-bbs', bbsID : '', user : null, archiveFormat :'application/zip', + forceEncoding : null, }; } @@ -945,6 +946,8 @@ class QWKPacketWriter extends EventEmitter { } } else { fullMessageBody += `@MSGID: ${this.makeMessageIdentifier(message)}\n`; + fullMessageBody += `@TZ: ${UTCOffsetToSMBTZ[moment().format('Z')]}\n`; + // :TODO: REPLY and REPLYTO } } @@ -953,14 +956,16 @@ class QWKPacketWriter extends EventEmitter { fullMessageBody += `${line}\n`; }); - const encodedMessage = iconv.encode(fullMessageBody, this.options.encoding); + const encoding = this._getEncoding(message); + + const encodedMessage = iconv.encode(fullMessageBody, encoding); // // QWK spec wants line feeds as 0xe3 for some reason, so we'll have // to replace the \n's. If we're going against the spec and using UTF-8 // we can just leave them be. // - if ('utf8' !== this.options.encoding) { + if ('utf8' !== encoding) { replaceCharInBuffer(encodedMessage, 0x0a, QWKLF); } @@ -989,7 +994,7 @@ class QWKPacketWriter extends EventEmitter { this._updateIndexTracking(message); if (this.options.enableHeadersExtension) { - this._appendHeadersExtensionData(message); + this._appendHeadersExtensionData(message, encoding); } // next message starts at this block @@ -999,6 +1004,38 @@ class QWKPacketWriter extends EventEmitter { this.areaTagsSeen.add(message.areaTag); } + _getEncoding(message) { + if (this.options.forceEncoding) { + return this.options.forceEncoding; + } + + // If the system has stored an explicit encoding, use that. + let encoding = _.get(message.meta, 'System.explicit_encoding'); + if (encoding) { + return encoding; + } + + // If the message is already tagged with a supported encoding + // indicator such as FTN-style CHRS, try to use that. + encoding = _.get(message.meta, 'FtnKludge.CHRS'); + if (encoding) { + // convert from CHRS to something standard + encoding = getEncodingFromCharacterSetIdentifier(encoding); + if (encoding) { + return encoding; + } + } + + // The to-spec default is CP437/ASCII. If it can be encoded as + // such then do so. + if (message.isCP437Encodable()) { + return 'cp437'; + } + + // Something more modern... + return 'utf8'; + } + _messageAddressedToUser(message) { if (_.isUndefined(this.cachedCompareNames)) { if (this.options.user) { @@ -1041,7 +1078,7 @@ class QWKPacketWriter extends EventEmitter { } - finish(packetPath) { + finish(packetDirectory) { async.series( [ (callback) => { @@ -1066,7 +1103,7 @@ class QWKPacketWriter extends EventEmitter { return this._createIndexes(callback); }, (callback) => { - return this._producePacketArchive(packetPath, callback); + return this._producePacketArchive(packetDirectory, callback); } ], err => { @@ -1081,7 +1118,44 @@ class QWKPacketWriter extends EventEmitter { ) } - _producePacketArchive(packetPath, cb) { + _getNextAvailPacketFileName(packetDirectory, cb) { + // + // According to http://wiki.synchro.net/ref:qwk filenames should + // start with .QWK -> .QW1 ... .QW9 -> .Q10 ... .Q99 + // + let digits = 0; + async.doWhilst( callback => { + let ext; + if (0 === digits) { + ext = 'QWK'; + } else if (digits < 10) { + ext = `QW${digits}`; + } else if (digits < 100) { + ext = `Q${digits}`; + } else { + return callback(Errors.UnexpectedState(`Unable to choose a valid QWK output filename`)); + } + + ++digits; + + const filename = `${this.options.bbsID}.${ext}`; + fs.stat(paths.join(packetDirectory, filename), (err, stats) => { + if (err && 'ENOENT' === err.code) { + return callback(null, filename); + } else { + return callback(null, null); + } + }); + }, + (filename, callback) => { + return callback(null, filename ? false : true); + }, + (err, filename) => { + return cb(err, filename); + }); + } + + _producePacketArchive(packetDirectory, cb) { const archiveUtil = ArchiveUtil.getInstance(); fs.readdir(this.workDir, (err, files) => { @@ -1089,15 +1163,22 @@ class QWKPacketWriter extends EventEmitter { return cb(err); } - archiveUtil.compressTo( - this.options.archiveFormat, - packetPath, - files, - this.workDir, - err => { + this._getNextAvailPacketFileName(packetDirectory, (err, filename) => { + if (err) { return cb(err); } - ); + + const packetPath = paths.join(packetDirectory, filename); + archiveUtil.compressTo( + this.options.archiveFormat, + packetPath, + files, + this.workDir, + err => { + return cb(err); + } + ); + }); }); } @@ -1306,10 +1387,10 @@ class QWKPacketWriter extends EventEmitter { return `${syncTimestamp} ${syncTZ}`; } - _appendHeadersExtensionData(message) { + _appendHeadersExtensionData(message, encoding) { const messageData = { // Synchronet style - Utf8 : ('utf8' === this.options.encoding ? 'true' : 'false'), + Utf8 : ('utf8' === encoding ? 'true' : 'false'), 'Message-ID' : this.makeMessageIdentifier(message), WhenWritten : this._makeSynchronetTimestamp(message.modTimestamp), @@ -1367,11 +1448,11 @@ class QWKPacketWriter extends EventEmitter { messageData.Editor = `ENiGMA 1/2 BBS FSE v${enigmaVersion}`; } - this.headersDatStream.write(iconv.encode(`[${this.currentMessageOffset.toString(16)}]\r\n`, this.options.encoding)); + this.headersDatStream.write(iconv.encode(`[${this.currentMessageOffset.toString(16)}]\r\n`, encoding)); for (let [name, value] of Object.entries(messageData)) { if (value) { - this.headersDatStream.write(iconv.encode(`${name}: ${value}\r\n`, this.options.encoding)); + this.headersDatStream.write(iconv.encode(`${name}: ${value}\r\n`, encoding)); } } diff --git a/core/string_util.js b/core/string_util.js index fa9a9097..cde7ac3e 100644 --- a/core/string_util.js +++ b/core/string_util.js @@ -13,6 +13,7 @@ exports.pad = pad; exports.insert = insert; exports.replaceAt = replaceAt; exports.isPrintable = isPrintable; +exports.containsNonLatinCodepoints = containsNonLatinCodepoints; exports.stripAllLineFeeds = stripAllLineFeeds; exports.debugEscapedString = debugEscapedString; exports.stringFromNullTermBuffer = stringFromNullTermBuffer; @@ -196,6 +197,20 @@ function isPrintable(s) { return !RE_NON_PRINTABLE.test(s); } +const NonLatinCodePointsRegExp = /[^\u0000-\u00ff]/; + +function containsNonLatinCodepoints(s) { + if (!s.length) { + return false; + } + + if (s.charCodeAt(0) > 255) { + return true; + } + + return NonLatinCodepointsRegEx.test(s); +} + function stripAllLineFeeds(s) { return s.replace(/\r?\n|[\r\u2028\u2029]/g, ''); } From c7a543e87eeedadfbb82934d08085b91c7a3f096 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 2 May 2020 16:48:24 -0600 Subject: [PATCH 8/8] * oputil mb qwk-export * oputil mb qwk-dump * Fix QWK reader encoding --- core/oputil/oputil_help.js | 10 ++ core/oputil/oputil_message_base.js | 209 +++++++++++++---------------- core/qwk_mail_packet.js | 9 +- 3 files changed, 106 insertions(+), 122 deletions(-) diff --git a/core/oputil/oputil_help.js b/core/oputil/oputil_help.js index abcf61fe..3fd55575 100644 --- a/core/oputil/oputil_help.js +++ b/core/oputil/oputil_help.js @@ -170,6 +170,11 @@ Actions: import-areas PATH Import areas using FidoNet *.NA or AREAS.BBS file + qwk-dump PATH Dumps a QWK packet to stdout. + qwk-export [AREA_TAGS] PATH Exports one or more configured message area to a QWK + packet in the directory specified by PATH. The QWK + BBS ID will be obtained by the final component of PATH. + import-areas arguments: --conf CONF_TAG Conference tag in which to import areas --network NETWORK Network name/key to associate FTN areas @@ -177,6 +182,11 @@ import-areas arguments: --type TYPE Area import type Valid types are "bbs" and "na". + +qwk-export arguments: + --user USER User in which to export for. Defaults to the SysOp. + --after TIMESTAMP Export only messages with a timestamp later than + TIMESTAMP. ` }; diff --git a/core/oputil/oputil_message_base.js b/core/oputil/oputil_message_base.js index 6f677c52..e29a30dc 100644 --- a/core/oputil/oputil_message_base.js +++ b/core/oputil/oputil_message_base.js @@ -10,17 +10,19 @@ const { initConfigAndDatabases, getAnswers, writeConfig, -} = require('./oputil_common.js'); -const getHelpFor = require('./oputil_help.js').getHelpFor; -const Address = require('../ftn_address.js'); -const Errors = require('../enig_error.js').Errors; +} = require('./oputil_common.js'); + +const getHelpFor = require('./oputil_help.js').getHelpFor; +const Address = require('../ftn_address.js'); +const Errors = require('../enig_error.js').Errors; // deps -const async = require('async'); -const paths = require('path'); -const fs = require('fs'); -const hjson = require('hjson'); -const _ = require('lodash'); +const async = require('async'); +const paths = require('path'); +const fs = require('fs'); +const hjson = require('hjson'); +const _ = require('lodash'); +const moment = require('moment'); exports.handleMessageBaseCommand = handleMessageBaseCommand; @@ -434,105 +436,47 @@ function getImportEntries(importType, importData) { return importEntries; } -function handleQWK() { +function dumpQWKPacket() { const packetPath = argv._[argv._.length - 1]; - if(argv._.length < 4 || !packetPath || 0 === packetPath.length) { - return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + if(argv._.length < 3 || !packetPath || 0 === packetPath.length) { + return printUsageAndSetExitCode(getHelpFor('MessageBase'), ExitCodes.ERROR); } - const subAction = argv._[argv._.length - 2]; - switch (subAction) { - case 'dump' : - return dumpQWKPacket(packetPath); - - case 'export' : - return exportQWKPacket(packetPath); - - default : - return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); - } -} - -function dumpQWKPacket(packetPath) { async.waterfall( [ (callback) => { return initConfigAndDatabases(callback); }, (callback) => { - //// - const { QWKPacketWriter } = require('../qwk_mail_packet'); - const writer = new QWKPacketWriter({ - bbsID : 'XIBALBA', - }); - const { QWKPacketReader } = require('../qwk_mail_packet'); + const reader = new QWKPacketReader(packetPath); - - writer.on('ready', () => { - const reader = new QWKPacketReader(packetPath); - - reader.on('error', err => { - console.error(`ERROR: ${err.message}`); - return callback(err); - }); - - reader.on('done', () => { - writer.finish(); - }); - - reader.on('archive type', archiveType => { - console.info(`-> Archive type: ${archiveType}`); - }); - - reader.on('creator', creator => { - console.info(`-> Creator: ${creator}`); - }); - - reader.on('message', message => { - writer.appendMessage(message); - }); - - reader.read(); + reader.on('error', err => { + console.error(`ERROR: ${err.message}`); + return callback(err); }); - writer.on('finished', () => { - console.log('done'); + reader.on('done', () => { + return callback(null); }); - writer.init(); + reader.on('archive type', archiveType => { + console.info(`-> Archive type: ${archiveType}`); + }); - //// + reader.on('creator', creator => { + console.info(`-> Creator: ${creator}`); + }); - // const { QWKPacketReader } = require('../qwk_mail_packet'); - // const reader = new QWKPacketReader(packetPath); + reader.on('message', message => { + console.info('--- message ---'); + console.info(`To: ${message.toUserName}`); + console.info(`From: ${message.fromUserName}`); + console.info(`Subject: ${message.subject}`); + console.info(`Message:\r\n${message.message}`); + }); - // reader.on('error', err => { - // console.error(`ERROR: ${err.message}`); - // return callback(err); - // }); - - // reader.on('done', () => { - // return callback(null); - // }); - - // reader.on('archive type', archiveType => { - // console.info(`-> Archive type: ${archiveType}`); - // }); - - // reader.on('creator', creator => { - // console.info(`-> Creator: ${creator}`); - // }); - - // reader.on('message', message => { - // console.info('--- message ---'); - // console.info(`To : ${message.toUserName}`); - // console.info(`From : ${message.fromUserName}`); - // console.info(`Subject : ${message.subject}`); - // console.info(`Message :\r\n${message.message}`); - // }); - - // reader.read(); + reader.read(); } ], err => { @@ -541,36 +485,43 @@ function dumpQWKPacket(packetPath) { ) } -function exportQWKPacket(packetPath) { - // oputil mb qwk export SPEC PATH [--user USER] - // [areaTag1[@dateTime]],[...] PATH --user USER +function exportQWKPacket() { + let packetPath = argv._[argv._.length - 1]; + if(argv._.length < 3 || !packetPath || 0 === packetPath.length) { + return printUsageAndSetExitCode(getHelpFor('MessageBase'), ExitCodes.ERROR); + } - // :TODO: bbsID from PATH filename else 'ENIGMA' + // oputil mb qwk-export TAGS PATH [--user USER] [--after TIMESTAMP] + // [areaTag1,areaTag2,...] PATH --user USER --after TIMESTAMP + let bbsID = 'ENIGMA'; + const filename = paths.basename(packetPath); + if (filename) { + const ext = paths.extname(filename); + bbsID = paths.basename(filename, ext); + } + + packetPath = paths.dirname(packetPath); const posArgLen = argv._.length; - if (posArgLen < 4) { - return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + let areaTags; + if (4 === posArgLen) { + areaTags = argv._[posArgLen - 2].split(','); + } else { + areaTags = []; } - let areaTagSpecs = '*'; - if (5 === posArgLen) { - areaTagSpecs = argv._[areaTagSpecs - 2]; + let newerThanTimestamp = null; + if (argv.after) { + const ts = moment(argv.after); + if (ts.isValid()) { + newerThanTimestamp = ts.format(); + } } - - //const areaTagSpecs = argv._[areaTagSpecs - 2]; - - /*const packetPath = argv._[argv._.length - 1]; - if(argv._.length < 4 || !packetPath || 0 === packetPath.length) { - return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); - }*/ - - // :TODO: parse area tags(s) and timestamps - const areaTags = [ 'general', 'fsx_gen' ]; - const userName = argv.user || '-'; + let totalExported = 0; async.waterfall( [ (callback) => { @@ -590,14 +541,29 @@ function exportQWKPacket(packetPath) { return User.getUser(userId, callback); }); }, + (user, callback) => { + // populate area tags with all available to user + // if they were not explicitly supplied + if (!areaTags.length) { + const { + getAvailableMessageConferences, + getAvailableMessageAreasByConfTag + } = require('../../core/message_area'); + + const confTags = Object.keys(getAvailableMessageConferences(null, { noClient : true })); + confTags.forEach( confTag => { + areaTags = areaTags.concat(Object.keys(getAvailableMessageAreasByConfTag(confTag))); + }); + } + return callback(null, user); + }, (user, callback) => { const Message = require('../message'); const filter = { resultType : 'id', areaTag : areaTags, - - // :TODO: newerThanTimestamp + newerThanTimestamp, }; // public @@ -617,9 +583,8 @@ function exportQWKPacket(packetPath) { (user, Message, messageIds, callback) => { const { QWKPacketWriter } = require('../qwk_mail_packet'); const writer = new QWKPacketWriter({ - // :TODO: export needs these options - bbsID : 'XIBALBA', - user : user, + bbsID, + user, }); writer.on('ready', () => { @@ -628,18 +593,23 @@ function exportQWKPacket(packetPath) { message.load( { messageId }, err => { if (!err) { writer.appendMessage(message); + ++totalExported; } return nextMessageId(err); }); }, (err) => { - writer.finish('/home/nuskooler/Downloads/qwk2/'); + writer.finish(packetPath); if (err) { console.error(`Failed to write one or more messages: ${err.message}`); } }); }); + writer.on('warning', err => { + console.warn(`!!! ${err.reason ? err.reason : err.message}`); + }); + writer.on('finished', () => { return callback(null); }); @@ -649,8 +619,10 @@ function exportQWKPacket(packetPath) { ], err => { if(err) { - console.error(err.reason ? err.reason : err.message); + return console.error(err.reason ? err.reason : err.message); } + + console.info(`-> Exported ${totalExported} messages`); } ); } @@ -673,6 +645,7 @@ function handleMessageBaseCommand() { return({ areafix : areaFix, 'import-areas' : importAreas, - qwk : handleQWK, + 'qwk-dump' : dumpQWKPacket, + 'qwk-export' : exportQWKPacket, }[action] || errUsage)(); } \ No newline at end of file diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index b7f53ebc..5aa5deac 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -493,7 +493,7 @@ class QWKPacketReader extends EventEmitter { } const encodingToSpec = 'cp437'; - let encoding = encodingToSpec; + let encoding; const path = paths.join(this.packetInfo.tempDir, this.packetInfo.messages.filename); fs.open(path, 'r', (err, fd) => { @@ -575,6 +575,7 @@ class QWKPacketReader extends EventEmitter { switch (state) { case 'header' : const header = MessageHeaderParser.parse(buffer); + encoding = encodingToSpec; // reset per message // massage into something a little more sane (things we can't quite do in the parser directly) ['toName', 'fromName', 'subject'].forEach(field => { @@ -601,7 +602,7 @@ class QWKPacketReader extends EventEmitter { // if we have HEADERS.DAT with a 'Utf8' override for this message, // the overridden to/from/subject/message fields are UTF-8 - if (currMessage.headersExtension && currMessage.headersExtension.Utf8) { + if (currMessage.headersExtension && 'true' === currMessage.headersExtension.Utf8.toLowerCase()) { encoding = 'utf8'; } @@ -825,7 +826,7 @@ class QWKPacketWriter extends EventEmitter { enableHeadersExtension = true, enableAtKludges = true, systemDomain = 'enigma-bbs', - bbsID = '', + bbsID = 'ENIGMA', user = null, archiveFormat = 'application/zip', forceEncoding = null, @@ -853,7 +854,7 @@ class QWKPacketWriter extends EventEmitter { enableHeadersExtension : true, enableAtKludges : true, systemDomain : 'enigma-bbs', - bbsID : '', + bbsID : 'ENIGMA', user : null, archiveFormat :'application/zip', forceEncoding : null,