From 6edfe95dfe9914f70282eba1ef5f052212a689e7 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 21 Apr 2020 22:14:29 -0600 Subject: [PATCH] A good amount of HEADERS.DAT support --- core/qwk_mail_packet.js | 180 ++++++++++++++++++++++++++++++++++------ package.json | 1 + yarn.lock | 19 +++++ 3 files changed, 174 insertions(+), 26 deletions(-) diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index 415822a3..9ed97bb6 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -14,6 +14,7 @@ const { Parser } = require('binary-parser'); const iconv = require('iconv-lite'); const moment = require('moment'); const _ = require('lodash'); +const IniConfigParser = require('ini-config-parser'); const SMBTZToUTCOffset = (smbTZ) => { // convert a Synchronet smblib TZ to a UTC offset @@ -242,7 +243,6 @@ class QWKPacketReader extends EventEmitter { packetFileInfo, { tempDir, - defaultEncoding : 'CP437' } ); return callback(null); @@ -270,7 +270,41 @@ class QWKPacketReader extends EventEmitter { } processPacketFiles(cb) { - return this.readMessages(cb); + async.series( + [ + (callback) => { + return this.readHeadersExtension(callback); + }, + (callback) => { + return this.readMessages(callback); + } + ], + err => { + return cb(err); + } + ) + } + + readHeadersExtension(cb) { + if (!this.packetInfo.headers) { + return cb(null); // nothing to do + } + + const path = paths.join(this.packetInfo.tempDir, this.packetInfo.headers.filename); + fs.readFile(path, { encoding : 'utf8' }, (err, iniData) => { + if (err) { + this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid (${err.message})`)); + return cb(null); // non-fatal + } + + try { + this.packetInfo.headers.ini = IniConfigParser.parse(iniData); + } catch (e) { + this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid (${e.message})`)); + } + + return cb(null); + }); } readMessages(cb) { @@ -279,13 +313,54 @@ class QWKPacketReader extends EventEmitter { return cb(Errors.DoesNotExist('No messages file found within QWK packet')); } - const encoding = this.packetInfo.defaultEncoding; + const encodingToSpec = 'cp437'; + let encoding = encodingToSpec; + const path = paths.join(this.packetInfo.tempDir, this.packetInfo.messages.filename); fs.open(path, 'r', (err, fd) => { if (err) { return cb(err); } + // Some mappings/etc. used in loops below.... + // Sync sets these in HEADERS.DAT: http://wiki.synchro.net/ref:qwk + const FTNPropertyMapping = { + 'X-FTN-AREA' : Message.FtnPropertyNames.FtnArea, + 'X-FTN-SEEN-BY' : Message.FtnPropertyNames.FtnSeenBy, + 'X-FTN-FLAGS' : Message.FtnPropertyNames + }; + + const FTNKludgeMapping = { + 'X-FTN-PATH' : 'PATH', + 'X-FTN-MSGID' : 'MSGID', + 'X-FTN-REPLY' : 'REPLY', + 'X-FTN-PID' : 'PID', + 'X-FTN-FLAGS' : 'FLAGS', + 'X-FTN-TID' : 'TID', + 'X-FTN-CHRS' : 'CHRS', + // :TODO: X-FTN-KLUDGE - not sure what this is? + }; + + // + // Various kludge tags defined by QWKE, etc. + // See the following: + // - ftp://vert.synchro.net/main/BBS/qwke.txt + // - http://wiki.synchro.net/ref:qwk + // + const Kludges = { + // QWKE + To : 'To:', + From : 'From:', + Subject : 'Subject:', + + // Synchronet + Via : '@VIA:', + MsgID : '@MSGID:', + Reply : '@REPLY:', + TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h + ReplyTo : '@REPLYTO:', + }; + let blockCount = 0; let currMessage = { }; let state; @@ -321,7 +396,8 @@ class QWKPacketReader extends EventEmitter { // massage into something a little more sane (things we can't quite do in the parser directly) ['toName', 'fromName', 'subject'].forEach(field => { - header[field] = iconv.decode(header[field], encoding).trim(); + // note: always use to-spec encoding here + header[field] = iconv.decode(header[field], encodingToSpec).trim(); }); header.timestamp = moment(header.timestamp, 'MM-DD-YYHH:mm'); @@ -334,6 +410,19 @@ class QWKPacketReader extends EventEmitter { subject : header.subject, }; + if (_.has(this.packetInfo, 'headers.ini')) { + // Sections for a message in HEADERS.DAT are by current byte offset. + // 128 = first message header = 0x80 = section [80] + const headersSectionId = (blockCount * QWKMessageBlockSize).toString(16); + currMessage.headersExtension = this.packetInfo.headers.ini[headersSectionId]; + } + + // if we have HEADERS.DAT with a 'Utf8' override for this message, + // the overridden to/from/subject/message fields are UTF-8 + if (currMessage.headersExtension && currMessage.headersExtension.Utf8) { + encoding = 'utf8'; + } + // remainder of blocks until the end of this message messageBlocksRemain = header.numBlocks - 1; state = 'message'; @@ -372,26 +461,6 @@ class QWKPacketReader extends EventEmitter { const messageLines = splitTextAtTerms(iconv.decode(currMessage.body, encoding).trimEnd()); const bodyLines = []; - // - // Various kludge tags defined by QWKE, etc. - // See the following: - // - ftp://vert.synchro.net/main/BBS/qwke.txt - // - http://wiki.synchro.net/ref:qwk - // - const Kludges = { - // QWKE - To : 'To:', - From : 'From:', - Subject : 'Subject:', - - // Synchronet - Via : '@VIA:', - MsgID : '@MSGID:', - Reply : '@REPLY:', - TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h - ReplyTo : '@REPLYTO:', - }; - let bodyState = 'kludge'; const MessageTrailers = { @@ -404,6 +473,7 @@ class QWKPacketReader extends EventEmitter { const qwkKludge = {}; const ftnProperty = {}; + const ftnKludge = {}; messageLines.forEach(line => { if (0 === line.length) { @@ -449,12 +519,60 @@ class QWKPacketReader extends EventEmitter { } }); + let messageTimestamp = currMessage.header.timestamp; + + // HEADERS.DAT support. + let useTZKludge = true; + if (currMessage.headersExtension) { + const ext = currMessage.headersExtension; + + // to and subject can be overridden yet again if entries are present + currMessage.toName = ext.To || currMessage.toName + currMessage.subject = ext.Subject || currMessage.subject; + currMessage.from = ext.Sender || currMessage.from; // why not From? Who the fuck knows. + + // possibly override message ID kludge + qwkKludge.msg_id = ext['Message-ID'] || qwkKludge.msg_id; + + // WhenWritten contains a ISO-8601-ish timestamp and a Synchronet/SMB style TZ offset: + // 20180101174837-0600 4168 + // We can use this to get a very slightly better precision on the timestamp (addition of seconds) + // over the headers value. Why not milliseconds? Who the fuck knows. + if (ext.WhenWritten) { + const whenWritten = moment(ext.WhenWritten, 'YYYYMMDDHHmmssZ'); + if (whenWritten.isValid()) { + messageTimestamp = whenWritten; + useTZKludge = false; + } + } + + if (ext.Tags) { + currMessage.hashTags = ext.Tags.split(' '); + } + + // FTN style properties/kludges represented as X-FTN-XXXX + for (let [extName, propName] of Object.entries(FTNPropertyMapping)) { + const v = ext[extName]; + if (v) { + ftnProperty[propName] = v; + } + } + + for (let [extName, kludgeName] of Object.entries(FTNKludgeMapping)) { + const v = ext[extName]; + if (v) { + ftnKludge[kludgeName] = v; + } + } + } + const message = new Message({ toUserName : currMessage.toName, fromUserName : currMessage.fromName, subject : currMessage.subject, - modTimestamp : currMessage.header.timestamp, + modTimestamp : messageTimestamp, message : bodyLines.join('\n'), + hashTags : currMessage.hashTags, }); if (!_.isEmpty(qwkKludge)) { @@ -465,6 +583,10 @@ class QWKPacketReader extends EventEmitter { message.meta.FtnProperty = ftnProperty; } + if (!_.isEmpty(ftnKludge)) { + message.meta.FtnKludge = ftnKludge; + } + // Add in tear line and origin if requested if (this.options.keepTearAndOrigin) { if (ftnProperty.ftn_tear_line) { @@ -477,7 +599,7 @@ class QWKPacketReader extends EventEmitter { } // Update the timestamp if we have a valid TZ - if (_.has(message, 'meta.QwkKludge.synchronet_timezone')) { + if (useTZKludge && _.has(message, 'meta.QwkKludge.synchronet_timezone')) { const tzOffset = SMBTZToUTCOffset(message.meta.QwkKludge.synchronet_timezone); if (tzOffset) { message.modTimestamp.utcOffset(tzOffset); @@ -491,11 +613,17 @@ class QWKPacketReader extends EventEmitter { if (this.mode === QWKPacketReader.Modes.QWK) { message.meta.QwkProperty.qwk_msg_num = currMessage.header.num; + message.meta.QwkProperty.qwk_conf_num = currMessage.header.confNum; } else { // For REP's, prefer the larger field. message.meta.QwkProperty.qwk_conf_num = currMessage.header.num || currMessage.header.confNum; } + // Another quick HEADERS.DAT fix-up + if (currMessage.headersExtension) { + message.meta.QwkProperty.qwk_conf_num = currMessage.headersExtension.Conference || message.meta.QwkProperty.qwk_conf_num; + } + this.emit('message', message); state = 'header'; } diff --git a/package.json b/package.json index 048a207f..12754271 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "hashids": "2.1.0", "hjson": "^3.2.1", "iconv-lite": "0.5.0", + "ini-config-parser": "^1.0.4", "inquirer": "^7.0.0", "later": "1.2.0", "lodash": "^4.17.15", diff --git a/yarn.lock b/yarn.lock index 453791de..cb08db2e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -312,6 +312,11 @@ code-point-at@^1.0.0: resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= +coffee-script@^1.12.4: + version "1.12.7" + resolved "https://registry.yarnpkg.com/coffee-script/-/coffee-script-1.12.7.tgz#c05dae0cb79591d05b3070a8433a98c9a89ccc53" + integrity sha512-fLeEhqwymYat/MpTPUjSKHVYYl0ec2mOyALEMLmzr5i1isuG+6jfI2j2d5oBO3VIzgUXgBVIcOT9uH1TFxBckw== + collection-visit@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0" @@ -408,6 +413,11 @@ decode-uri-component@^0.2.0: resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU= +deep-extend@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.5.1.tgz#b894a9dd90d3023fbf1c55a394fb858eb2066f1f" + integrity sha512-N8vBdOa+DF7zkRrDCsaOXoCs/E2fJfx9B9MrKnnSiHNh4ws7eSys6YQE4KvT1cecKmOASYQBhbKjeuDD9lT81w== + deep-extend@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" @@ -883,6 +893,15 @@ inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= +ini-config-parser@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/ini-config-parser/-/ini-config-parser-1.0.4.tgz#0abc75cb68c506204712d2b4861400b6adbfda78" + integrity sha512-5hLh5Cqai67pTrLQ9q/K/3EtSP2Tzu41AZzwPLSegkkMkc42dGweLgkbiocCBiBBEg2fPhs6pKmdFhwj5Ul3Bg== + dependencies: + coffee-script "^1.12.4" + deep-extend "^0.5.1" + rimraf "^2.6.1" + ini@~1.3.0: version "1.3.5" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927"