WIP on QWK support
This commit is contained in:
Bryan Ashby
parent
91939f46cf
commit
29ee9c4d58
|
|
@ -87,6 +87,21 @@ const FTN_PROPERTY_NAMES = {
|
|||
FtnSeenBy : 'ftn_seen_by', // http://ftsc.org/docs/fts-0004.001
|
||||
};
|
||||
|
||||
const QWKPropertyNames = {
|
||||
MessageNumber : 'qwk_msg_num',
|
||||
MessageStatus : 'qwk_msg_status', // See http://wiki.synchro.net/ref:qwk for a decent list
|
||||
ConferenceNumber : 'qwk_conf_num',
|
||||
InReplyToNum : 'qwk_in_reply_to_num', // note that we prefer the 'InReplyToMsgId' kludge if available
|
||||
};
|
||||
|
||||
const QWKKludgeNames = {
|
||||
Via : 'via',
|
||||
MessageId : 'msg_id',
|
||||
InReplyToMsgId : 'in_reply_to_msg_id',
|
||||
SyncTZ : 'synchronet_timezone',
|
||||
ReplyTo : 'reply_to',
|
||||
};
|
||||
|
||||
// :TODO: this is a ugly hack due to bad variable names - clean it up & just _.camelCase(k)!
|
||||
const MESSAGE_ROW_MAP = {
|
||||
reply_to_message_id : 'replyToMsgId',
|
||||
|
|
@ -183,6 +198,10 @@ module.exports = class Message {
|
|||
return FTN_PROPERTY_NAMES;
|
||||
}
|
||||
|
||||
static get QWKPropertyNames() {
|
||||
return QWKPropertyNames;
|
||||
}
|
||||
|
||||
setLocalToUserId(userId) {
|
||||
this.meta.System = this.meta.System || {};
|
||||
this.meta.System[Message.SystemMetaNames.LocalToUserID] = userId;
|
||||
|
|
|
|||
|
|
@ -434,6 +434,66 @@ function getImportEntries(importType, importData) {
|
|||
return importEntries;
|
||||
}
|
||||
|
||||
function handleQWK() {
|
||||
const packetPath = argv._[argv._.length - 1];
|
||||
if(argv._.length < 4 || !packetPath || 0 === packetPath.length) {
|
||||
return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
|
||||
}
|
||||
|
||||
const subAction = argv._[argv._.length - 2];
|
||||
switch (subAction) {
|
||||
case 'dump' :
|
||||
return dumpQWKPacket(packetPath);
|
||||
|
||||
default :
|
||||
return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
function dumpQWKPacket(packetPath) {
|
||||
async.waterfall(
|
||||
[
|
||||
(callback) => {
|
||||
return initConfigAndDatabases(callback);
|
||||
},
|
||||
(callback) => {
|
||||
const { QWKPacketReader } = require('../qwk_mail_packet');
|
||||
const reader = new QWKPacketReader(packetPath);
|
||||
|
||||
reader.on('error', err => {
|
||||
console.error(`ERROR: ${err.message}`);
|
||||
return callback(err);
|
||||
});
|
||||
|
||||
reader.on('done', () => {
|
||||
return callback(null);
|
||||
});
|
||||
|
||||
reader.on('archive type', archiveType => {
|
||||
console.info(`-> Archive type: ${archiveType}`);
|
||||
});
|
||||
|
||||
reader.on('creator', creator => {
|
||||
console.info(`-> Creator: ${creator}`);
|
||||
});
|
||||
|
||||
reader.on('message', message => {
|
||||
console.info('--- message ---');
|
||||
console.info(`To : ${message.toUserName}`);
|
||||
console.info(`From : ${message.fromUserName}`);
|
||||
console.info(`Subject : ${message.subject}`);
|
||||
console.info(`Message :\r\n${message.message}`);
|
||||
});
|
||||
|
||||
reader.read();
|
||||
}
|
||||
],
|
||||
err => {
|
||||
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function handleMessageBaseCommand() {
|
||||
|
||||
function errUsage() {
|
||||
|
|
@ -452,5 +512,6 @@ function handleMessageBaseCommand() {
|
|||
return({
|
||||
areafix : areaFix,
|
||||
'import-areas' : importAreas,
|
||||
qwk : handleQWK,
|
||||
}[action] || errUsage)();
|
||||
}
|
||||
|
|
@ -0,0 +1,520 @@
|
|||
|
||||
|
||||
const ArchiveUtil = require('./archive_util');
|
||||
const { Errors } = require('./enig_error');
|
||||
const Message = require('./message');
|
||||
const { splitTextAtTerms } = require('./string_util');
|
||||
|
||||
const { EventEmitter } = require('events');
|
||||
const temptmp = require('temptmp').createTrackedSession('qwk_packet');
|
||||
const async = require('async');
|
||||
const fs = require('graceful-fs');
|
||||
const paths = require('path');
|
||||
const { Parser } = require('binary-parser');
|
||||
const iconv = require('iconv-lite');
|
||||
const moment = require('moment');
|
||||
const _ = require('lodash');
|
||||
|
||||
const SMBTZToUTCOffset = (smbTZ) => {
|
||||
// convert a Synchronet smblib TZ to a UTC offset
|
||||
// see https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h
|
||||
return {
|
||||
// US Standard
|
||||
'40F0' : '-04:00', // Atlantic
|
||||
'412C' : '-05:00', // Eastern
|
||||
'4168' : '-06:00', // Central
|
||||
'41A4' : '-07:00', // Mountain
|
||||
'41E0' : '-08:00', // Pacific
|
||||
'421C' : '-09:00', // Yukon
|
||||
'4258' : '-10:00', // Hawaii/Alaska
|
||||
'4294' : '-11:00', // Bering
|
||||
|
||||
// US Daylight
|
||||
|
||||
}[smbTZ];
|
||||
};
|
||||
|
||||
const QWKMessageBlockSize = 128;
|
||||
|
||||
const MessageHeaderParser = new Parser()
|
||||
.endianess('little')
|
||||
.string('status', {
|
||||
encoding : 'ascii',
|
||||
length : 1,
|
||||
})
|
||||
.string('num', { // message num or conf num for REP's
|
||||
encoding : 'ascii',
|
||||
length : 7,
|
||||
formatter : n => {
|
||||
return parseInt(n);
|
||||
}
|
||||
})
|
||||
.string('timestamp', {
|
||||
encoding : 'ascii',
|
||||
length : 13,
|
||||
})
|
||||
// these fields may be encoded in something other than ascii/CP437
|
||||
.array('toName', {
|
||||
type : 'uint8',
|
||||
length : 25,
|
||||
})
|
||||
.array('fromName', {
|
||||
type : 'uint8',
|
||||
length : 25,
|
||||
})
|
||||
.array('subject', {
|
||||
type : 'uint8',
|
||||
length : 25,
|
||||
})
|
||||
.string('password', {
|
||||
encoding : 'ascii',
|
||||
length : 12,
|
||||
})
|
||||
.string('replyToNum', {
|
||||
encoding : 'ascii',
|
||||
length : 8,
|
||||
formatter : n => {
|
||||
return parseInt(n);
|
||||
}
|
||||
})
|
||||
.string('numBlocks', {
|
||||
encoding : 'ascii',
|
||||
length : 6,
|
||||
formatter : n => {
|
||||
return parseInt(n);
|
||||
}
|
||||
})
|
||||
.uint8('status2')
|
||||
.uint16('confNum')
|
||||
.uint16('relNum')
|
||||
.uint8('netTag');
|
||||
|
||||
|
||||
class QWKPacketReader extends EventEmitter {
|
||||
constructor(packetPath, mode=QWKPacketReader.Modes.Guess, options = { keepTearAndOrigin : true } ) {
|
||||
super();
|
||||
|
||||
this.packetPath = packetPath;
|
||||
this.mode = mode;
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
static get Modes() {
|
||||
return {
|
||||
Guess : 'guess', // try to guess
|
||||
QWK : 'qwk', // standard incoming packet
|
||||
REP : 'rep', // a reply packet
|
||||
};
|
||||
}
|
||||
|
||||
read() {
|
||||
//
|
||||
// A general overview:
|
||||
//
|
||||
// - Find out what kind of archive we're dealing with
|
||||
// - Extract to temporary location
|
||||
// - Process various files
|
||||
// - Emit messages we find, information about the packet, so on
|
||||
//
|
||||
async.waterfall(
|
||||
[
|
||||
// determine packet archive type
|
||||
(callback) => {
|
||||
const archiveUtil = ArchiveUtil.getInstance();
|
||||
archiveUtil.detectType(this.packetPath, (err, archiveType) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
this.emit('archive type', archiveType);
|
||||
return callback(null, archiveType);
|
||||
});
|
||||
},
|
||||
// create a temporary location to do processing
|
||||
(archiveType, callback) => {
|
||||
temptmp.mkdir( { prefix : 'enigqwkpacket-'}, (err, tempDir) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
return callback(null, archiveType, tempDir);
|
||||
});
|
||||
},
|
||||
// extract it
|
||||
(archiveType, tempDir, callback) => {
|
||||
const archiveUtil = ArchiveUtil.getInstance();
|
||||
archiveUtil.extractTo(this.packetPath, tempDir, archiveType, err => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
return callback(null, tempDir);
|
||||
});
|
||||
},
|
||||
// gather extracted file list
|
||||
(tempDir, callback) => {
|
||||
fs.readdir(tempDir, (err, files) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
// Discover basic information about well known files
|
||||
async.reduce(
|
||||
files,
|
||||
{},
|
||||
(out, filename, next) => {
|
||||
const key = filename.toUpperCase();
|
||||
|
||||
switch (key) {
|
||||
case 'MESSAGES.DAT' : // QWK
|
||||
if (this.mode === QWKPacketReader.Modes.Guess) {
|
||||
this.mode = QWKPacketReader.Modes.QWK;
|
||||
}
|
||||
if (this.mode === QWKPacketReader.Modes.QWK) {
|
||||
out.messages = { filename };
|
||||
}
|
||||
break;
|
||||
|
||||
case 'ID.MSG' :
|
||||
if (this.mode === QWKPacketReader.Modes.Guess) {
|
||||
this.mode = Modes.REP;
|
||||
}
|
||||
|
||||
if (this.mode === QWKPacketReader.Modes.REP) {
|
||||
out.messages = { filename };
|
||||
}
|
||||
break;
|
||||
|
||||
case 'HEADERS.DAT' : // Synchronet
|
||||
out.headers = { filename };
|
||||
break;
|
||||
|
||||
case 'VOTING.DAT' : // Synchronet
|
||||
out.voting = { filename };
|
||||
break;
|
||||
|
||||
case 'CONTROL.DAT' : // QWK
|
||||
out.control = { filename };
|
||||
break;
|
||||
|
||||
case 'DOOR.ID' : // QWK
|
||||
out.door = { filename };
|
||||
break;
|
||||
|
||||
case 'NETFLAGS.DAT' : // QWK
|
||||
out.netflags = { filename };
|
||||
break;
|
||||
|
||||
case 'NEWFILES.DAT' : // QWK
|
||||
out.newfiles = { filename };
|
||||
break;
|
||||
|
||||
case 'PERSONAL.NDX' : // QWK
|
||||
out.personal = { filename };
|
||||
break;
|
||||
|
||||
case '000.NDX' : // QWK
|
||||
out.inbox = { filename };
|
||||
break;
|
||||
|
||||
case 'TOREADER.EXT' : // QWKE
|
||||
out.toreader = { filename };
|
||||
break;
|
||||
|
||||
case 'QLR.DAT' :
|
||||
out.qlr = { filename };
|
||||
break;
|
||||
|
||||
default :
|
||||
if (/[0-9]+\.NDX/.test(key)) { // QWK
|
||||
out.pointers = out.pointers || { filenames: [] };
|
||||
out.pointers.filenames.push(filename);
|
||||
} else {
|
||||
out[key] = { filename };
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
return next(null, out);
|
||||
},
|
||||
(err, packetFileInfo) => {
|
||||
this.packetInfo = Object.assign(
|
||||
{},
|
||||
packetFileInfo,
|
||||
{
|
||||
tempDir,
|
||||
defaultEncoding : 'CP437'
|
||||
}
|
||||
);
|
||||
return callback(null);
|
||||
}
|
||||
);
|
||||
});
|
||||
},
|
||||
(callback) => {
|
||||
return this.processPacketFiles(callback);
|
||||
},
|
||||
(tempDir, callback) => {
|
||||
return callback(null);
|
||||
}
|
||||
],
|
||||
err => {
|
||||
temptmp.cleanup();
|
||||
|
||||
if (err) {
|
||||
return this.emit('error', err);
|
||||
}
|
||||
|
||||
this.emit('done');
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
processPacketFiles(cb) {
|
||||
return this.readMessages(cb);
|
||||
}
|
||||
|
||||
readMessages(cb) {
|
||||
// :TODO: update to use proper encoding: if headers.dat specifies UTF-8, use that, else CP437
|
||||
if (!this.packetInfo.messages) {
|
||||
return cb(Errors.DoesNotExist('No messages file found within QWK packet'));
|
||||
}
|
||||
|
||||
const encoding = this.packetInfo.defaultEncoding;
|
||||
const path = paths.join(this.packetInfo.tempDir, this.packetInfo.messages.filename);
|
||||
fs.open(path, 'r', (err, fd) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
|
||||
let blockCount = 0;
|
||||
let currMessage = { };
|
||||
let state;
|
||||
let messageBlocksRemain;
|
||||
const buffer = Buffer.alloc(QWKMessageBlockSize);
|
||||
|
||||
const readNextBlock = () => {
|
||||
fs.read(fd, buffer, 0, QWKMessageBlockSize, null, (err, read) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
|
||||
if (0 == read) {
|
||||
// we're done consuming all blocks
|
||||
return fs.close(fd, err => {
|
||||
return cb(err);
|
||||
});
|
||||
}
|
||||
|
||||
if (QWKMessageBlockSize !== read) {
|
||||
return cb(Errors.Invalid(`Invalid QWK message block size. Expected ${QWKMessageBlockSize} got ${read}`));
|
||||
}
|
||||
|
||||
if (0 === blockCount) {
|
||||
// first 128 bytes is a space padded ID
|
||||
const id = buffer.toString('ascii').trim();
|
||||
this.emit('generator', id);
|
||||
state = 'header';
|
||||
} else {
|
||||
switch (state) {
|
||||
case 'header' :
|
||||
const header = MessageHeaderParser.parse(buffer);
|
||||
|
||||
// massage into something a little more sane (things we can't quite do in the parser directly)
|
||||
['toName', 'fromName', 'subject'].forEach(field => {
|
||||
header[field] = iconv.decode(header[field], encoding).trim();
|
||||
});
|
||||
|
||||
header.timestamp = moment(header.timestamp, 'MM-DD-YYHH:mm');
|
||||
|
||||
currMessage = {
|
||||
header,
|
||||
// these may be overridden
|
||||
toName : header.toName,
|
||||
fromName : header.fromName,
|
||||
subject : header.subject,
|
||||
};
|
||||
|
||||
// remainder of blocks until the end of this message
|
||||
messageBlocksRemain = header.numBlocks - 1;
|
||||
state = 'message';
|
||||
break;
|
||||
|
||||
case 'message' :
|
||||
if (!currMessage.body) {
|
||||
currMessage.body = buffer;
|
||||
} else {
|
||||
currMessage.body = Buffer.concat([currMessage.body, buffer]);
|
||||
}
|
||||
messageBlocksRemain -= 1;
|
||||
|
||||
if (0 === messageBlocksRemain) {
|
||||
// 1:n buffers to make up body. Decode:
|
||||
// First, replace QWK style line feeds (0xe3) unless the message is UTF-8.
|
||||
// If the message is UTF-8, we assume it's using standard line feeds.
|
||||
if (encoding !== 'utf8') {
|
||||
let i = 0;
|
||||
const QWKLF = Buffer.from([0xe3]);
|
||||
while (i < currMessage.body.length) {
|
||||
i = currMessage.body.indexOf(QWKLF, i);
|
||||
if (-1 === i) {
|
||||
break;
|
||||
}
|
||||
currMessage.body[i] = 0x0a;
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Decode the message based on our final message encoding. Split the message
|
||||
// into lines so we can extract various bits such as QWKE headers, origin, tear
|
||||
// lines, etc.
|
||||
//
|
||||
const messageLines = splitTextAtTerms(iconv.decode(currMessage.body, encoding).trimEnd());
|
||||
const bodyLines = [];
|
||||
|
||||
//
|
||||
// Various kludge tags defined by QWKE, etc.
|
||||
// See the following:
|
||||
// - ftp://vert.synchro.net/main/BBS/qwke.txt
|
||||
// - http://wiki.synchro.net/ref:qwk
|
||||
//
|
||||
const Kludges = {
|
||||
// QWKE
|
||||
To : 'To:',
|
||||
From : 'From:',
|
||||
Subject : 'Subject:',
|
||||
|
||||
// Synchronet
|
||||
Via : '@VIA:',
|
||||
MsgID : '@MSGID:',
|
||||
Reply : '@REPLY:',
|
||||
TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h
|
||||
ReplyTo : '@REPLYTO:',
|
||||
};
|
||||
|
||||
let bodyState = 'kludge';
|
||||
|
||||
const MessageTrailers = {
|
||||
// While technically FTN oriented, these can come from any network
|
||||
// (though we'll be processing a lot of messages that routed through FTN
|
||||
// at some point)
|
||||
Origin : /^[ ]{1,2}\* Origin: /,
|
||||
Tear : /^--- /,
|
||||
};
|
||||
|
||||
const qwkKludge = {};
|
||||
const ftnProperty = {};
|
||||
|
||||
messageLines.forEach(line => {
|
||||
if (0 === line.length) {
|
||||
return bodyLines.push('');
|
||||
}
|
||||
|
||||
switch (bodyState) {
|
||||
case 'kludge' :
|
||||
// :TODO: Update these to use the well known consts:
|
||||
if (line.startsWith(Kludges.To)) {
|
||||
currMessage.toName = line.substring(Kludges.To.length).trim();
|
||||
} else if (line.startsWith(Kludges.From)) {
|
||||
currMessage.fromName = line.substring(Kludges.From.length).trim();
|
||||
} else if (line.startsWith(Kludges.Subject)) {
|
||||
currMessage.subject = line.substring(Kludges.Subject.length).trim();
|
||||
} else if (line.startsWith(Kludges.Via)) {
|
||||
qwkKludge.via = line;
|
||||
} else if (line.startsWith(Kludges.MsgID)) {
|
||||
qwkKludge.msg_id = line.substring(Kludges.MsgID.length).trim();
|
||||
} else if (line.startsWith(Kludges.Reply)) {
|
||||
qwkKludge.in_reply_to_msg_id = line.substring(Kludges.Reply.length).trim();
|
||||
} else if (line.startsWith(Kludges.TZ)) {
|
||||
qwkKludge.synchronet_timezone = line.substring(Kludges.TZ.length).trim();
|
||||
} else if (line.startsWith(Kludges.ReplyTo)) {
|
||||
qwkKludge.reply_to = line.substring(Kludges.ReplyTo.length).trim();
|
||||
} else {
|
||||
bodyState = 'body'; // past this point and up to any tear/origin/etc., is the real message body
|
||||
bodyLines.push(line);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'body' :
|
||||
case 'trailers' :
|
||||
if (MessageTrailers.Origin.test(line)) {
|
||||
ftnProperty.ftn_origin = line;
|
||||
bodyState = 'trailers';
|
||||
} else if (MessageTrailers.Tear.test(line)) {
|
||||
ftnProperty.ftn_tear_line = line;
|
||||
bodyState = 'trailers';
|
||||
} else if ('body' === bodyState) {
|
||||
bodyLines.push(line);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const message = new Message({
|
||||
toUserName : currMessage.toName,
|
||||
fromUserName : currMessage.fromName,
|
||||
subject : currMessage.subject,
|
||||
modTimestamp : currMessage.header.timestamp,
|
||||
message : bodyLines.join('\n'),
|
||||
});
|
||||
|
||||
if (!_.isEmpty(qwkKludge)) {
|
||||
message.meta.QwkKludge = qwkKludge;
|
||||
}
|
||||
|
||||
if (!_.isEmpty(ftnProperty)) {
|
||||
message.meta.FtnProperty = ftnProperty;
|
||||
}
|
||||
|
||||
// Add in tear line and origin if requested
|
||||
if (this.options.keepTearAndOrigin) {
|
||||
if (ftnProperty.ftn_tear_line) {
|
||||
message.message += `\r\n${ftnProperty.ftn_tear_line}\r\n`;
|
||||
}
|
||||
|
||||
if (ftnProperty.ftn_origin) {
|
||||
message.message += `${ftnProperty.ftn_origin}\r\n`;
|
||||
}
|
||||
}
|
||||
|
||||
// Update the timestamp if we have a valid TZ
|
||||
if (_.has(message, 'meta.QwkKludge.synchronet_timezone')) {
|
||||
const tzOffset = SMBTZToUTCOffset(message.meta.QwkKludge.synchronet_timezone);
|
||||
if (tzOffset) {
|
||||
message.modTimestamp.utcOffset(tzOffset);
|
||||
}
|
||||
}
|
||||
|
||||
message.meta.QwkProperty = {
|
||||
qwk_msg_status : currMessage.header.status,
|
||||
qwk_in_reply_to_num : currMessage.header.replyToNum,
|
||||
};
|
||||
|
||||
if (this.mode === QWKPacketReader.Modes.QWK) {
|
||||
message.meta.QwkProperty.qwk_msg_num = currMessage.header.num;
|
||||
} else {
|
||||
// For REP's, prefer the larger field.
|
||||
message.meta.QwkProperty.qwk_conf_num = currMessage.header.num || currMessage.header.confNum;
|
||||
}
|
||||
|
||||
this.emit('message', message);
|
||||
state = 'header';
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
++blockCount;
|
||||
readNextBlock();
|
||||
});
|
||||
};
|
||||
|
||||
// start reading blocks
|
||||
readNextBlock();
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
QWKPacketReader,
|
||||
// QWKPacketWriter,
|
||||
}
|
||||
Loading…
Reference in New Issue