Produce NDX files, various improvements to spec, etc.

This commit is contained in:
Bryan Ashby 2020-04-30 22:07:29 -06:00
parent 2b7d810c77
commit 1f1813c14a
No known key found for this signature in database
GPG Key ID: B49EB437951D2542
3 changed files with 399 additions and 57 deletions

59
core/mbf.js Normal file
View File

@ -0,0 +1,59 @@
const { Errors } = require('./enig_error');
//
// Utils for dealing with Microsoft Binary Format (MBF) used
// in various BASIC languages, etc.
//
// - https://en.wikipedia.org/wiki/Microsoft_Binary_Format
// - https://stackoverflow.com/questions/2268191/how-to-convert-from-ieee-python-float-to-microsoft-basic-float
//
// Number to 32bit MBF
numToMbf32 = (v) => {
const mbf = Buffer.alloc(4);
if (0 === v) {
return mbf;
}
const ieee = Buffer.alloc(4);
ieee.writeFloatLE(v, 0);
const sign = ieee[3] & 0x80;
let exp = (ieee[3] << 1) | (ieee[2] >> 7);
if (exp === 0xfe) {
throw Errors.Invalid(`${v} cannot be converted to mbf`);
}
exp += 2;
mbf[3] = exp;
mbf[2] = sign | (ieee[2] & 0x7f);
mbf[1] = ieee[1];
mbf[0] = ieee[0];
return mbf;
}
mbf32ToNum = (mbf) => {
if (0 === mbf[3]) {
return 0.0;
}
const ieee = Buffer.alloc(4);
const sign = mbf[2] & 0x80;
const exp = mbf[3] - 2;
ieee[3] = sign | (exp >> 1);
ieee[2] = (exp << 7) | (mbf[2] & 0x7f);
ieee[1] = mbf[1];
ieee[0] = mbf[0];
return ieee.readFloatLE(0);
}
module.exports = {
numToMbf32,
mbf32ToNum,
}

View File

@ -445,6 +445,9 @@ function handleQWK() {
case 'dump' :
return dumpQWKPacket(packetPath);
case 'export' :
return exportQWKPacket(packetPath);
default :
return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
}
@ -457,35 +460,81 @@ function dumpQWKPacket(packetPath) {
return initConfigAndDatabases(callback);
},
(callback) => {
////
const { QWKPacketWriter } = require('../qwk_mail_packet');
const writer = new QWKPacketWriter({
bbsID : 'XIBALBA',
toUser : 'NuSkooler',
encoding : 'cp437',
});
const { QWKPacketReader } = require('../qwk_mail_packet');
const reader = new QWKPacketReader(packetPath);
reader.on('error', err => {
console.error(`ERROR: ${err.message}`);
return callback(err);
writer.on('ready', () => {
const reader = new QWKPacketReader(packetPath);
reader.on('error', err => {
console.error(`ERROR: ${err.message}`);
return callback(err);
});
reader.on('done', () => {
writer.finish();
});
reader.on('archive type', archiveType => {
console.info(`-> Archive type: ${archiveType}`);
});
reader.on('creator', creator => {
console.info(`-> Creator: ${creator}`);
});
reader.on('message', message => {
writer.appendMessage(message);
});
reader.read();
});
reader.on('done', () => {
return callback(null);
writer.on('finished', () => {
console.log('done');
});
reader.on('archive type', archiveType => {
console.info(`-> Archive type: ${archiveType}`);
});
writer.init();
reader.on('creator', creator => {
console.info(`-> Creator: ${creator}`);
});
////
reader.on('message', message => {
console.info('--- message ---');
console.info(`To : ${message.toUserName}`);
console.info(`From : ${message.fromUserName}`);
console.info(`Subject : ${message.subject}`);
console.info(`Message :\r\n${message.message}`);
});
// const { QWKPacketReader } = require('../qwk_mail_packet');
// const reader = new QWKPacketReader(packetPath);
reader.read();
// reader.on('error', err => {
// console.error(`ERROR: ${err.message}`);
// return callback(err);
// });
// reader.on('done', () => {
// return callback(null);
// });
// reader.on('archive type', archiveType => {
// console.info(`-> Archive type: ${archiveType}`);
// });
// reader.on('creator', creator => {
// console.info(`-> Creator: ${creator}`);
// });
// reader.on('message', message => {
// console.info('--- message ---');
// console.info(`To : ${message.toUserName}`);
// console.info(`From : ${message.fromUserName}`);
// console.info(`Subject : ${message.subject}`);
// console.info(`Message :\r\n${message.message}`);
// });
// reader.read();
}
],
err => {
@ -494,6 +543,120 @@ function dumpQWKPacket(packetPath) {
)
}
function exportQWKPacket(packetPath) {
// oputil mb qwk export SPEC PATH [--user USER]
// [areaTag1[@dateTime]],[...] PATH --user USER
const posArgLen = argv._.length;
if (posArgLen < 4) {
return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
}
let areaTagSpecs = '*';
if (5 === posArgLen) {
areaTagSpecs = argv._[areaTagSpecs - 2];
}
//const areaTagSpecs = argv._[areaTagSpecs - 2];
/*const packetPath = argv._[argv._.length - 1];
if(argv._.length < 4 || !packetPath || 0 === packetPath.length) {
return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
}*/
// :TODO: parse area tags(s) and timestamps
const areaTags = [ 'general', 'fsx_gen' ];
const userName = argv.user || '-';
async.waterfall(
[
(callback) => {
return initConfigAndDatabases(callback);
},
(callback) => {
const User = require('../../core/user.js');
User.getUserIdAndName(userName, (err, userId) => {
if (err) {
if ('-' === userName) {
userId = 1;
} else {
return callback(err);
}
}
return User.getUser(userId, callback);
});
},
(user, callback) => {
const Message = require('../message');
const filter = {
resultType : 'id',
areaTag : areaTags,
// :TODO: newerThanTimestamp
};
// public
Message.findMessages(filter, (err, publicMessageIds) => {
if (err) {
return callback(err);
}
delete filter.areaTag;
filter.privateTagUserId = user.userId;
Message.findMessages(filter, (err, privateMessageIds) => {
return callback(err, user, Message, privateMessageIds.concat(publicMessageIds));
});
});
},
(user, Message, messageIds, callback) => {
const { QWKPacketWriter } = require('../qwk_mail_packet');
const writer = new QWKPacketWriter({
// :TODO: export needs these options
bbsID : 'XIBALBA',
toUser : 'NuSkooler',
encoding : 'cp437',
user : user,
});
writer.on('ready', () => {
async.eachSeries(messageIds, (messageId, nextMessageId) => {
const message = new Message();
message.load( { messageId }, err => {
if (!err) {
writer.appendMessage(message);
}
return nextMessageId(err);
});
},
(err) => {
writer.finish('/home/nuskooler/Downloads/qwk2/TEST1.QWK');
if (err) {
console.error(`Failed to write one or more messages: ${err.message}`);
}
});
});
writer.on('finished', () => {
return callback(null);
});
writer.init();
}
],
err => {
if(err) {
console.error(err.reason ? err.reason : err.message);
}
}
);
}
function handleMessageBaseCommand() {
function errUsage() {

View File

@ -9,6 +9,8 @@ const {
const StatLog = require('./stat_log');
const Config = require('./config').get;
const SysProps = require('./system_property');
const UserProps = require('./user_property');
const { numToMbf32 } = require('./mbf');
const { EventEmitter } = require('events');
const temptmp = require('temptmp');
@ -83,8 +85,8 @@ const QWKLF = 0xe3;
const QWKMessageStatusCodes = {
UnreadPublic : ' ',
ReadPublic : '-',
ReadBySomeonePrivate : '*',
UnreadPrivate : '+',
ReadPrivate : '*',
UnreadCommentToSysOp : '~',
ReadCommentToSysOp : '`',
UnreadSenderPWProtected : '%',
@ -100,6 +102,11 @@ const QWKMessageActiveStatus = {
Deleted : 226,
};
const QWKNetworkTagIndicator = {
Present : '*',
NotPresent : ' ',
};
// See the following:
// - http://fileformats.archiveteam.org/wiki/QWK
// - http://wiki.synchro.net/ref:qwk
@ -878,6 +885,9 @@ class QWKPacketWriter extends EventEmitter {
this.totalMessages = 0;
this.areaTagsSeen = new Set();
this.personalIndex = []; // messages addressed to 'user'
this.inboxIndex = []; // private messages for 'user'
this.publicIndex = new Map();
return callback(null);
},
@ -938,9 +948,6 @@ class QWKPacketWriter extends EventEmitter {
}
}
// The actual message contents
//fullMessageBody += message.message;
// Sanitize line feeds (e.g. CRLF -> LF, and possibly -> QWK style below)
splitTextAtTerms(message.message).forEach(line => {
fullMessageBody += `${line}\n`;
@ -961,11 +968,12 @@ class QWKPacketWriter extends EventEmitter {
// block padded by spaces or nulls (we use nulls)
const fullBlocks = Math.trunc(encodedMessage.length / QWKMessageBlockSize);
const remainBytes = QWKMessageBlockSize - (encodedMessage.length % QWKMessageBlockSize);
const totalBlocks = fullBlocks + 1 + (remainBytes ? 1 : 0);
// The first block is always a header
if (!this._writeMessageHeader(
message,
fullBlocks + 1 + (remainBytes ? 1 : 0),
totalBlocks
))
{
// we can't write this message
@ -974,26 +982,61 @@ class QWKPacketWriter extends EventEmitter {
this.messagesStream.write(encodedMessage);
if (remainBytes) {
this.messagesStream.write(Buffer.alloc(remainBytes, ' '));
}
this._updateIndexTracking(message);
if (this.options.enableHeadersExtension) {
this._appendHeadersExtensionData(message);
}
this.currentMessageOffset += fullBlocks * QWKMessageBlockSize;
if (remainBytes)
{
this.currentMessageOffset += QWKMessageBlockSize;
}
// next message starts at this block
this.currentMessageOffset += totalBlocks * QWKMessageBlockSize;
this.totalMessages += 1;
this.areaTagsSeen.add(message.areaTag);
}
_messageAddressedToUser(message) {
if (_.isUndefined(this.cachedCompareNames)) {
if (this.options.user) {
this.cachedCompareNames = [
this.options.user.username.toLowerCase()
];
const realName = this.options.user.getProperty(UserProps.RealName);
if (realName) {
this.cachedCompareNames.push(realName.toLowerCase());
}
} else {
this.cachedCompareNames = [];
}
};
return this.cachedCompareNames.includes(message.toUserName.toLowerCase());
}
_updateIndexTracking(message) {
// index points at start of *message* not the header for... reasons?
const index = (this.currentMessageOffset / QWKMessageBlockSize) + 1;
if (message.isPrivate()) {
this.inboxIndex.push(index);
} else {
if (this._messageAddressedToUser(message)) {
// :TODO: add to both indexes???
this.personalIndex.push(index);
}
const areaTag = message.areaTag;
if (!this.publicIndex.has(areaTag)) {
this.publicIndex.set(areaTag, [index]);
} else {
this.publicIndex.get(areaTag).push(index);
}
}
}
appendNewFile() {
}
@ -1019,6 +1062,9 @@ class QWKPacketWriter extends EventEmitter {
(callback) => {
return this._createControlData(callback);
},
(callback) => {
return this._createIndexes(callback);
},
(callback) => {
return this._producePacketArchive(packetPath, callback);
}
@ -1038,22 +1084,21 @@ class QWKPacketWriter extends EventEmitter {
_producePacketArchive(packetPath, cb) {
const archiveUtil = ArchiveUtil.getInstance();
const packetFiles = [
'messages.dat', 'headers.dat', 'control.dat',
].map(filename => {
return filename;
//return paths.join(this.workDir, filename);
});
archiveUtil.compressTo(
this.options.archiveFormat,
packetPath,
packetFiles,
this.workDir,
err => {
fs.readdir(this.workDir, (err, files) => {
if (err) {
return cb(err);
}
);
archiveUtil.compressTo(
this.options.archiveFormat,
packetPath,
files,
this.workDir,
err => {
return cb(err);
}
);
});
}
_qwkMessageStatus(message) {
@ -1090,15 +1135,9 @@ class QWKPacketWriter extends EventEmitter {
return false;
}
const netTag = ' '; // :TODO:
this.lolMessageId = this.lolMessageId || 1;
//message.messageId = this.lolMessageId;
this.lolMessageId++;
const header = Buffer.alloc(QWKMessageBlockSize, ' ');
header.write(this._qwkMessageStatus(message), 0, 1, 'ascii');
header.write(asciiNum(message.messageId), 1, 'ascii'); // :TODO: It seems Sync puts the relative, as in # of messages we've called appendMessage()?!
header.write(asciiNum(message.messageId), 1, 'ascii');
header.write(message.modTimestamp.format(QWKHeaderTimestampFormat), 8, 13, 'ascii');
header.write(message.toUserName.substr(0, 25), 21, 'ascii');
header.write(message.fromUserName.substr(0, 25), 46, 'ascii');
@ -1108,8 +1147,8 @@ class QWKPacketWriter extends EventEmitter {
header.write(asciiTotalBlocks, 116, 'ascii');
header.writeUInt8(QWKMessageActiveStatus.Active, 122);
header.writeUInt16LE(conferenceNumber, 123);
header.writeUInt16LE(0, 125); // :TODO: Check if others actually populate this
header.write(netTag[0], 127, 1, 'ascii');
header.writeUInt16LE(this.totalMessages + 1, 125);
header.write(QWKNetworkTagIndicator.NotPresent, 127, 1, 'ascii'); // :TODO: Present if for network output?
this.messagesStream.write(header);
@ -1117,6 +1156,9 @@ class QWKPacketWriter extends EventEmitter {
}
_getMessageConferenceNumberByAreaTag(areaTag) {
if (Message.isPrivateAreaTag(areaTag)) {
return 0;
}
const areaConfig = _.get(Config(), [ 'messageNetworks', 'qwk', 'areas', areaTag ]);
return areaConfig && areaConfig.conference;
}
@ -1131,6 +1173,13 @@ class QWKPacketWriter extends EventEmitter {
_createControlData(cb) {
const areas = Array.from(this.areaTagsSeen).map(areaTag => {
if (Message.isPrivateAreaTag(areaTag)) {
return {
areaTag : Message.WellKnownAreaTags.Private,
name : 'Private',
desc : 'Private Messages',
};
}
return getMessageAreaByTag(areaTag);
});
@ -1168,8 +1217,12 @@ class QWKPacketWriter extends EventEmitter {
// map areas as conf #\r\nDescription\r\n pairs
areas.forEach(area => {
const conferenceNumber = this._getMessageConferenceNumberByAreaTag(area.areaTag);
let desc = area.name;
if (area.desc) {
desc += ` - ${area.desc}`
}
controlStream.write(`${conferenceNumber}\r\n`);
controlStream.write(`${area.name}\r\n`);
controlStream.write(`${desc}\r\n`);
});
// :TODO: do we ever care here?!
@ -1180,6 +1233,73 @@ class QWKPacketWriter extends EventEmitter {
controlStream.end();
}
_createIndexes(cb) {
const appendIndexData = (stream, offset) => {
const msb = numToMbf32(offset);
stream.write(msb);
// technically, the conference #, but only as a byte, so pretty much useless
// AND the filename itself is the conference number... dafuq.
stream.write(Buffer.from([0x00]));
};
async.series(
[
(callback) => {
// Create PERSONAL.NDX
if (!this.personalIndex.length) {
return callback(null);
}
const indexStream = fs.createWriteStream(paths.join(this.workDir, 'personal.ndx'));
this.personalIndex.forEach(offset => appendIndexData(indexStream, offset));
indexStream.on('close', err => {
return callback(err);
});
indexStream.end();
},
(callback) => {
// 000.NDX of private mails
if (!this.inboxIndex.length) {
return callback(null);
}
const indexStream = fs.createWriteStream(paths.join(this.workDir, '000.ndx'));
this.inboxIndex.forEach(offset => appendIndexData(indexStream, offset));
indexStream.on('close', err => {
return callback(err);
});
indexStream.end();
},
(callback) => {
// ####.NDX
async.eachSeries(this.publicIndex.keys(), (areaTag, nextArea) => {
const offsets = this.publicIndex.get(areaTag);
const conferenceNumber = this._getMessageConferenceNumberByAreaTag(areaTag);
const indexStream = fs.createWriteStream(paths.join(this.workDir, `${conferenceNumber.toString()}.ndx`));
offsets.forEach(offset => appendIndexData(indexStream, offset));
indexStream.on('close', err => {
return nextArea(err);
});
indexStream.end();
},
err => {
return callback(err);
});
}
],
err => {
return cb(err);
}
);
}
_makeSynchronetTimestamp(ts) {
const syncTimestamp = ts.format('YYYYMMDDHHmmssZZ');
const syncTZ = UTCOffsetToSMBTZ[ts.format('Z')] || '0000'; // :TODO: what if we don't have a map?
@ -1215,7 +1335,7 @@ class QWKPacketWriter extends EventEmitter {
Tags : message.hashTags.join(' '),
// :TODO: Needs tested with Sync/etc.; Sync wants Conference *numbers*
Conference : getMessageConfTagByAreaTag(message.areaTag),
Conference : message.isPrivate() ? '0' : getMessageConfTagByAreaTag(message.areaTag),
// ENiGMA Headers
MessageUUID : message.uuid,