moon
/
enigma-bbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

A good amount of HEADERS.DAT support

This commit is contained in:
Bryan Ashby 2020-04-21 22:14:29 -06:00
parent 29ee9c4d58
commit 6edfe95dfe
No known key found for this signature in database
GPG Key ID: B49EB437951D2542
3 changed files with 174 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

180
core/qwk_mail_packet.js
View File

@ -14,6 +14,7 @@ const { Parser } = require('binary-parser');
const iconv = require('iconv-lite');
const moment = require('moment');
const _ = require('lodash');
const IniConfigParser = require('ini-config-parser');
const SMBTZToUTCOffset = (smbTZ) => {
// convert a Synchronet smblib TZ to a UTC offset
@ -242,7 +243,6 @@ class QWKPacketReader extends EventEmitter {
packetFileInfo,
{
tempDir,
defaultEncoding : 'CP437'
}
);
return callback(null);
@ -270,7 +270,41 @@ class QWKPacketReader extends EventEmitter {
}
processPacketFiles(cb) {
return this.readMessages(cb);
async.series(
[
(callback) => {
return this.readHeadersExtension(callback);
},
(callback) => {
return this.readMessages(callback);
}
],
err => {
return cb(err);
}
)
}
readHeadersExtension(cb) {
if (!this.packetInfo.headers) {
return cb(null); // nothing to do
}
const path = paths.join(this.packetInfo.tempDir, this.packetInfo.headers.filename);
fs.readFile(path, { encoding : 'utf8' }, (err, iniData) => {
if (err) {
this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid (${err.message})`));
return cb(null); // non-fatal
}
try {
this.packetInfo.headers.ini = IniConfigParser.parse(iniData);
} catch (e) {
this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid (${e.message})`));
}
return cb(null);
});
}
readMessages(cb) {
@ -279,13 +313,54 @@ class QWKPacketReader extends EventEmitter {
return cb(Errors.DoesNotExist('No messages file found within QWK packet'));
}
const encoding = this.packetInfo.defaultEncoding;
const encodingToSpec = 'cp437';
let encoding = encodingToSpec;
const path = paths.join(this.packetInfo.tempDir, this.packetInfo.messages.filename);
fs.open(path, 'r', (err, fd) => {
if (err) {
return cb(err);
}
// Some mappings/etc. used in loops below....
// Sync sets these in HEADERS.DAT: http://wiki.synchro.net/ref:qwk
const FTNPropertyMapping = {
'X-FTN-AREA' : Message.FtnPropertyNames.FtnArea,
'X-FTN-SEEN-BY' : Message.FtnPropertyNames.FtnSeenBy,
'X-FTN-FLAGS' : Message.FtnPropertyNames
};
const FTNKludgeMapping = {
'X-FTN-PATH' : 'PATH',
'X-FTN-MSGID' : 'MSGID',
'X-FTN-REPLY' : 'REPLY',
'X-FTN-PID' : 'PID',
'X-FTN-FLAGS' : 'FLAGS',
'X-FTN-TID' : 'TID',
'X-FTN-CHRS' : 'CHRS',
// :TODO: X-FTN-KLUDGE - not sure what this is?
};
//
// Various kludge tags defined by QWKE, etc.
// See the following:
// - ftp://vert.synchro.net/main/BBS/qwke.txt
// - http://wiki.synchro.net/ref:qwk
//
const Kludges = {
// QWKE
To : 'To:',
From : 'From:',
Subject : 'Subject:',
// Synchronet
Via : '@VIA:',
MsgID : '@MSGID:',
Reply : '@REPLY:',
TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h
ReplyTo : '@REPLYTO:',
};
let blockCount = 0;
let currMessage = { };
let state;
@ -321,7 +396,8 @@ class QWKPacketReader extends EventEmitter {
// massage into something a little more sane (things we can't quite do in the parser directly)
['toName', 'fromName', 'subject'].forEach(field => {
header[field] = iconv.decode(header[field], encoding).trim();
// note: always use to-spec encoding here
header[field] = iconv.decode(header[field], encodingToSpec).trim();
});
header.timestamp = moment(header.timestamp, 'MM-DD-YYHH:mm');
@ -334,6 +410,19 @@ class QWKPacketReader extends EventEmitter {
subject : header.subject,
};
if (_.has(this.packetInfo, 'headers.ini')) {
// Sections for a message in HEADERS.DAT are by current byte offset.
// 128 = first message header = 0x80 = section [80]
const headersSectionId = (blockCount * QWKMessageBlockSize).toString(16);
currMessage.headersExtension = this.packetInfo.headers.ini[headersSectionId];
}
// if we have HEADERS.DAT with a 'Utf8' override for this message,
// the overridden to/from/subject/message fields are UTF-8
if (currMessage.headersExtension && currMessage.headersExtension.Utf8) {
encoding = 'utf8';
}
// remainder of blocks until the end of this message
messageBlocksRemain = header.numBlocks - 1;
state = 'message';
@ -372,26 +461,6 @@ class QWKPacketReader extends EventEmitter {
const messageLines = splitTextAtTerms(iconv.decode(currMessage.body, encoding).trimEnd());
const bodyLines = [];
//
// Various kludge tags defined by QWKE, etc.
// See the following:
// - ftp://vert.synchro.net/main/BBS/qwke.txt
// - http://wiki.synchro.net/ref:qwk
//
const Kludges = {
// QWKE
To : 'To:',
From : 'From:',
Subject : 'Subject:',
// Synchronet
Via : '@VIA:',
MsgID : '@MSGID:',
Reply : '@REPLY:',
TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h
ReplyTo : '@REPLYTO:',
};
let bodyState = 'kludge';
const MessageTrailers = {
@ -404,6 +473,7 @@ class QWKPacketReader extends EventEmitter {
const qwkKludge = {};
const ftnProperty = {};
const ftnKludge = {};
messageLines.forEach(line => {
if (0 === line.length) {
@ -449,12 +519,60 @@ class QWKPacketReader extends EventEmitter {
}
});
let messageTimestamp = currMessage.header.timestamp;
// HEADERS.DAT support.
let useTZKludge = true;
if (currMessage.headersExtension) {
const ext = currMessage.headersExtension;
// to and subject can be overridden yet again if entries are present
currMessage.toName = ext.To || currMessage.toName
currMessage.subject = ext.Subject || currMessage.subject;
currMessage.from = ext.Sender || currMessage.from; // why not From? Who the fuck knows.
// possibly override message ID kludge
qwkKludge.msg_id = ext['Message-ID'] || qwkKludge.msg_id;
// WhenWritten contains a ISO-8601-ish timestamp and a Synchronet/SMB style TZ offset:
// 20180101174837-0600 4168
// We can use this to get a very slightly better precision on the timestamp (addition of seconds)
// over the headers value. Why not milliseconds? Who the fuck knows.
if (ext.WhenWritten) {
const whenWritten = moment(ext.WhenWritten, 'YYYYMMDDHHmmssZ');
if (whenWritten.isValid()) {
messageTimestamp = whenWritten;
useTZKludge = false;
}
}
if (ext.Tags) {
currMessage.hashTags = ext.Tags.split(' ');
}
// FTN style properties/kludges represented as X-FTN-XXXX
for (let [extName, propName] of Object.entries(FTNPropertyMapping)) {
const v = ext[extName];
if (v) {
ftnProperty[propName] = v;
}
}
for (let [extName, kludgeName] of Object.entries(FTNKludgeMapping)) {
const v = ext[extName];
if (v) {
ftnKludge[kludgeName] = v;
}
}
}
const message = new Message({
toUserName : currMessage.toName,
fromUserName : currMessage.fromName,
subject : currMessage.subject,
modTimestamp : currMessage.header.timestamp,
modTimestamp : messageTimestamp,
message : bodyLines.join('\n'),
hashTags : currMessage.hashTags,
});
if (!_.isEmpty(qwkKludge)) {
@ -465,6 +583,10 @@ class QWKPacketReader extends EventEmitter {
message.meta.FtnProperty = ftnProperty;
}
if (!_.isEmpty(ftnKludge)) {
message.meta.FtnKludge = ftnKludge;
}
// Add in tear line and origin if requested
if (this.options.keepTearAndOrigin) {
if (ftnProperty.ftn_tear_line) {
@ -477,7 +599,7 @@ class QWKPacketReader extends EventEmitter {
}
// Update the timestamp if we have a valid TZ
if (_.has(message, 'meta.QwkKludge.synchronet_timezone')) {
if (useTZKludge && _.has(message, 'meta.QwkKludge.synchronet_timezone')) {
const tzOffset = SMBTZToUTCOffset(message.meta.QwkKludge.synchronet_timezone);
if (tzOffset) {
message.modTimestamp.utcOffset(tzOffset);
@ -491,11 +613,17 @@ class QWKPacketReader extends EventEmitter {
if (this.mode === QWKPacketReader.Modes.QWK) {
message.meta.QwkProperty.qwk_msg_num = currMessage.header.num;
message.meta.QwkProperty.qwk_conf_num = currMessage.header.confNum;
} else {
// For REP's, prefer the larger field.
message.meta.QwkProperty.qwk_conf_num = currMessage.header.num || currMessage.header.confNum;
}
// Another quick HEADERS.DAT fix-up
if (currMessage.headersExtension) {
message.meta.QwkProperty.qwk_conf_num = currMessage.headersExtension.Conference || message.meta.QwkProperty.qwk_conf_num;
}
this.emit('message', message);
state = 'header';
}

1
package.json
View File

@ -33,6 +33,7 @@
"hashids": "2.1.0",
"hjson": "^3.2.1",
"iconv-lite": "0.5.0",
"ini-config-parser": "^1.0.4",
"inquirer": "^7.0.0",
"later": "1.2.0",
"lodash": "^4.17.15",

19
yarn.lock
View File

@ -312,6 +312,11 @@ code-point-at@^1.0.0:
resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77"
integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=
coffee-script@^1.12.4:
version "1.12.7"
resolved "https://registry.yarnpkg.com/coffee-script/-/coffee-script-1.12.7.tgz#c05dae0cb79591d05b3070a8433a98c9a89ccc53"
integrity sha512-fLeEhqwymYat/MpTPUjSKHVYYl0ec2mOyALEMLmzr5i1isuG+6jfI2j2d5oBO3VIzgUXgBVIcOT9uH1TFxBckw==
collection-visit@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0"
@ -408,6 +413,11 @@ decode-uri-component@^0.2.0:
resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545"
integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=
deep-extend@^0.5.1:
version "0.5.1"
resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.5.1.tgz#b894a9dd90d3023fbf1c55a394fb858eb2066f1f"
integrity sha512-N8vBdOa+DF7zkRrDCsaOXoCs/E2fJfx9B9MrKnnSiHNh4ws7eSys6YQE4KvT1cecKmOASYQBhbKjeuDD9lT81w==
deep-extend@^0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac"
@ -883,6 +893,15 @@ inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3:
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=
ini-config-parser@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/ini-config-parser/-/ini-config-parser-1.0.4.tgz#0abc75cb68c506204712d2b4861400b6adbfda78"
integrity sha512-5hLh5Cqai67pTrLQ9q/K/3EtSP2Tzu41AZzwPLSegkkMkc42dGweLgkbiocCBiBBEg2fPhs6pKmdFhwj5Ul3Bg==
dependencies:
coffee-script "^1.12.4"
deep-extend "^0.5.1"
rimraf "^2.6.1"
ini@~1.3.0:
version "1.3.5"
resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927"