Produce NDX files, various improvements to spec, etc.
This commit is contained in:
parent
2b7d810c77
commit
1f1813c14a
|
@ -0,0 +1,59 @@
|
|||
const { Errors } = require('./enig_error');
|
||||
|
||||
//
|
||||
// Utils for dealing with Microsoft Binary Format (MBF) used
|
||||
// in various BASIC languages, etc.
|
||||
//
|
||||
// - https://en.wikipedia.org/wiki/Microsoft_Binary_Format
|
||||
// - https://stackoverflow.com/questions/2268191/how-to-convert-from-ieee-python-float-to-microsoft-basic-float
|
||||
//
|
||||
|
||||
// Number to 32bit MBF
|
||||
numToMbf32 = (v) => {
|
||||
const mbf = Buffer.alloc(4);
|
||||
|
||||
if (0 === v) {
|
||||
return mbf;
|
||||
}
|
||||
|
||||
const ieee = Buffer.alloc(4);
|
||||
ieee.writeFloatLE(v, 0);
|
||||
|
||||
const sign = ieee[3] & 0x80;
|
||||
let exp = (ieee[3] << 1) | (ieee[2] >> 7);
|
||||
|
||||
if (exp === 0xfe) {
|
||||
throw Errors.Invalid(`${v} cannot be converted to mbf`);
|
||||
}
|
||||
|
||||
exp += 2;
|
||||
|
||||
mbf[3] = exp;
|
||||
mbf[2] = sign | (ieee[2] & 0x7f);
|
||||
mbf[1] = ieee[1];
|
||||
mbf[0] = ieee[0];
|
||||
|
||||
return mbf;
|
||||
}
|
||||
|
||||
mbf32ToNum = (mbf) => {
|
||||
if (0 === mbf[3]) {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
const ieee = Buffer.alloc(4);
|
||||
const sign = mbf[2] & 0x80;
|
||||
const exp = mbf[3] - 2;
|
||||
|
||||
ieee[3] = sign | (exp >> 1);
|
||||
ieee[2] = (exp << 7) | (mbf[2] & 0x7f);
|
||||
ieee[1] = mbf[1];
|
||||
ieee[0] = mbf[0];
|
||||
|
||||
return ieee.readFloatLE(0);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
numToMbf32,
|
||||
mbf32ToNum,
|
||||
}
|
|
@ -445,6 +445,9 @@ function handleQWK() {
|
|||
case 'dump' :
|
||||
return dumpQWKPacket(packetPath);
|
||||
|
||||
case 'export' :
|
||||
return exportQWKPacket(packetPath);
|
||||
|
||||
default :
|
||||
return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
|
||||
}
|
||||
|
@ -457,7 +460,18 @@ function dumpQWKPacket(packetPath) {
|
|||
return initConfigAndDatabases(callback);
|
||||
},
|
||||
(callback) => {
|
||||
////
|
||||
const { QWKPacketWriter } = require('../qwk_mail_packet');
|
||||
const writer = new QWKPacketWriter({
|
||||
bbsID : 'XIBALBA',
|
||||
toUser : 'NuSkooler',
|
||||
encoding : 'cp437',
|
||||
});
|
||||
|
||||
const { QWKPacketReader } = require('../qwk_mail_packet');
|
||||
|
||||
|
||||
writer.on('ready', () => {
|
||||
const reader = new QWKPacketReader(packetPath);
|
||||
|
||||
reader.on('error', err => {
|
||||
|
@ -466,7 +480,7 @@ function dumpQWKPacket(packetPath) {
|
|||
});
|
||||
|
||||
reader.on('done', () => {
|
||||
return callback(null);
|
||||
writer.finish();
|
||||
});
|
||||
|
||||
reader.on('archive type', archiveType => {
|
||||
|
@ -478,14 +492,49 @@ function dumpQWKPacket(packetPath) {
|
|||
});
|
||||
|
||||
reader.on('message', message => {
|
||||
console.info('--- message ---');
|
||||
console.info(`To : ${message.toUserName}`);
|
||||
console.info(`From : ${message.fromUserName}`);
|
||||
console.info(`Subject : ${message.subject}`);
|
||||
console.info(`Message :\r\n${message.message}`);
|
||||
writer.appendMessage(message);
|
||||
});
|
||||
|
||||
reader.read();
|
||||
});
|
||||
|
||||
writer.on('finished', () => {
|
||||
console.log('done');
|
||||
});
|
||||
|
||||
writer.init();
|
||||
|
||||
////
|
||||
|
||||
// const { QWKPacketReader } = require('../qwk_mail_packet');
|
||||
// const reader = new QWKPacketReader(packetPath);
|
||||
|
||||
// reader.on('error', err => {
|
||||
// console.error(`ERROR: ${err.message}`);
|
||||
// return callback(err);
|
||||
// });
|
||||
|
||||
// reader.on('done', () => {
|
||||
// return callback(null);
|
||||
// });
|
||||
|
||||
// reader.on('archive type', archiveType => {
|
||||
// console.info(`-> Archive type: ${archiveType}`);
|
||||
// });
|
||||
|
||||
// reader.on('creator', creator => {
|
||||
// console.info(`-> Creator: ${creator}`);
|
||||
// });
|
||||
|
||||
// reader.on('message', message => {
|
||||
// console.info('--- message ---');
|
||||
// console.info(`To : ${message.toUserName}`);
|
||||
// console.info(`From : ${message.fromUserName}`);
|
||||
// console.info(`Subject : ${message.subject}`);
|
||||
// console.info(`Message :\r\n${message.message}`);
|
||||
// });
|
||||
|
||||
// reader.read();
|
||||
}
|
||||
],
|
||||
err => {
|
||||
|
@ -494,6 +543,120 @@ function dumpQWKPacket(packetPath) {
|
|||
)
|
||||
}
|
||||
|
||||
function exportQWKPacket(packetPath) {
|
||||
// oputil mb qwk export SPEC PATH [--user USER]
|
||||
// [areaTag1[@dateTime]],[...] PATH --user USER
|
||||
|
||||
const posArgLen = argv._.length;
|
||||
|
||||
if (posArgLen < 4) {
|
||||
return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
|
||||
}
|
||||
|
||||
let areaTagSpecs = '*';
|
||||
if (5 === posArgLen) {
|
||||
areaTagSpecs = argv._[areaTagSpecs - 2];
|
||||
}
|
||||
|
||||
|
||||
//const areaTagSpecs = argv._[areaTagSpecs - 2];
|
||||
|
||||
/*const packetPath = argv._[argv._.length - 1];
|
||||
if(argv._.length < 4 || !packetPath || 0 === packetPath.length) {
|
||||
return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR);
|
||||
}*/
|
||||
|
||||
// :TODO: parse area tags(s) and timestamps
|
||||
const areaTags = [ 'general', 'fsx_gen' ];
|
||||
|
||||
const userName = argv.user || '-';
|
||||
|
||||
async.waterfall(
|
||||
[
|
||||
(callback) => {
|
||||
return initConfigAndDatabases(callback);
|
||||
},
|
||||
(callback) => {
|
||||
const User = require('../../core/user.js');
|
||||
|
||||
User.getUserIdAndName(userName, (err, userId) => {
|
||||
if (err) {
|
||||
if ('-' === userName) {
|
||||
userId = 1;
|
||||
} else {
|
||||
return callback(err);
|
||||
}
|
||||
}
|
||||
return User.getUser(userId, callback);
|
||||
});
|
||||
},
|
||||
(user, callback) => {
|
||||
const Message = require('../message');
|
||||
|
||||
const filter = {
|
||||
resultType : 'id',
|
||||
areaTag : areaTags,
|
||||
|
||||
// :TODO: newerThanTimestamp
|
||||
};
|
||||
|
||||
// public
|
||||
Message.findMessages(filter, (err, publicMessageIds) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
delete filter.areaTag;
|
||||
filter.privateTagUserId = user.userId;
|
||||
|
||||
Message.findMessages(filter, (err, privateMessageIds) => {
|
||||
return callback(err, user, Message, privateMessageIds.concat(publicMessageIds));
|
||||
});
|
||||
});
|
||||
},
|
||||
(user, Message, messageIds, callback) => {
|
||||
const { QWKPacketWriter } = require('../qwk_mail_packet');
|
||||
const writer = new QWKPacketWriter({
|
||||
// :TODO: export needs these options
|
||||
bbsID : 'XIBALBA',
|
||||
toUser : 'NuSkooler',
|
||||
encoding : 'cp437',
|
||||
user : user,
|
||||
});
|
||||
|
||||
writer.on('ready', () => {
|
||||
async.eachSeries(messageIds, (messageId, nextMessageId) => {
|
||||
const message = new Message();
|
||||
message.load( { messageId }, err => {
|
||||
if (!err) {
|
||||
writer.appendMessage(message);
|
||||
}
|
||||
return nextMessageId(err);
|
||||
});
|
||||
},
|
||||
(err) => {
|
||||
writer.finish('/home/nuskooler/Downloads/qwk2/TEST1.QWK');
|
||||
if (err) {
|
||||
console.error(`Failed to write one or more messages: ${err.message}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
writer.on('finished', () => {
|
||||
return callback(null);
|
||||
});
|
||||
|
||||
writer.init();
|
||||
}
|
||||
],
|
||||
err => {
|
||||
if(err) {
|
||||
console.error(err.reason ? err.reason : err.message);
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function handleMessageBaseCommand() {
|
||||
|
||||
function errUsage() {
|
||||
|
|
|
@ -9,6 +9,8 @@ const {
|
|||
const StatLog = require('./stat_log');
|
||||
const Config = require('./config').get;
|
||||
const SysProps = require('./system_property');
|
||||
const UserProps = require('./user_property');
|
||||
const { numToMbf32 } = require('./mbf');
|
||||
|
||||
const { EventEmitter } = require('events');
|
||||
const temptmp = require('temptmp');
|
||||
|
@ -83,8 +85,8 @@ const QWKLF = 0xe3;
|
|||
const QWKMessageStatusCodes = {
|
||||
UnreadPublic : ' ',
|
||||
ReadPublic : '-',
|
||||
ReadBySomeonePrivate : '*',
|
||||
UnreadPrivate : '+',
|
||||
ReadPrivate : '*',
|
||||
UnreadCommentToSysOp : '~',
|
||||
ReadCommentToSysOp : '`',
|
||||
UnreadSenderPWProtected : '%',
|
||||
|
@ -100,6 +102,11 @@ const QWKMessageActiveStatus = {
|
|||
Deleted : 226,
|
||||
};
|
||||
|
||||
const QWKNetworkTagIndicator = {
|
||||
Present : '*',
|
||||
NotPresent : ' ',
|
||||
};
|
||||
|
||||
// See the following:
|
||||
// - http://fileformats.archiveteam.org/wiki/QWK
|
||||
// - http://wiki.synchro.net/ref:qwk
|
||||
|
@ -878,6 +885,9 @@ class QWKPacketWriter extends EventEmitter {
|
|||
|
||||
this.totalMessages = 0;
|
||||
this.areaTagsSeen = new Set();
|
||||
this.personalIndex = []; // messages addressed to 'user'
|
||||
this.inboxIndex = []; // private messages for 'user'
|
||||
this.publicIndex = new Map();
|
||||
|
||||
return callback(null);
|
||||
},
|
||||
|
@ -938,9 +948,6 @@ class QWKPacketWriter extends EventEmitter {
|
|||
}
|
||||
}
|
||||
|
||||
// The actual message contents
|
||||
//fullMessageBody += message.message;
|
||||
|
||||
// Sanitize line feeds (e.g. CRLF -> LF, and possibly -> QWK style below)
|
||||
splitTextAtTerms(message.message).forEach(line => {
|
||||
fullMessageBody += `${line}\n`;
|
||||
|
@ -961,11 +968,12 @@ class QWKPacketWriter extends EventEmitter {
|
|||
// block padded by spaces or nulls (we use nulls)
|
||||
const fullBlocks = Math.trunc(encodedMessage.length / QWKMessageBlockSize);
|
||||
const remainBytes = QWKMessageBlockSize - (encodedMessage.length % QWKMessageBlockSize);
|
||||
const totalBlocks = fullBlocks + 1 + (remainBytes ? 1 : 0);
|
||||
|
||||
// The first block is always a header
|
||||
if (!this._writeMessageHeader(
|
||||
message,
|
||||
fullBlocks + 1 + (remainBytes ? 1 : 0),
|
||||
totalBlocks
|
||||
))
|
||||
{
|
||||
// we can't write this message
|
||||
|
@ -974,26 +982,61 @@ class QWKPacketWriter extends EventEmitter {
|
|||
|
||||
this.messagesStream.write(encodedMessage);
|
||||
|
||||
|
||||
if (remainBytes) {
|
||||
this.messagesStream.write(Buffer.alloc(remainBytes, ' '));
|
||||
}
|
||||
|
||||
this._updateIndexTracking(message);
|
||||
|
||||
if (this.options.enableHeadersExtension) {
|
||||
this._appendHeadersExtensionData(message);
|
||||
}
|
||||
|
||||
this.currentMessageOffset += fullBlocks * QWKMessageBlockSize;
|
||||
|
||||
if (remainBytes)
|
||||
{
|
||||
this.currentMessageOffset += QWKMessageBlockSize;
|
||||
}
|
||||
// next message starts at this block
|
||||
this.currentMessageOffset += totalBlocks * QWKMessageBlockSize;
|
||||
|
||||
this.totalMessages += 1;
|
||||
this.areaTagsSeen.add(message.areaTag);
|
||||
}
|
||||
|
||||
_messageAddressedToUser(message) {
|
||||
if (_.isUndefined(this.cachedCompareNames)) {
|
||||
if (this.options.user) {
|
||||
this.cachedCompareNames = [
|
||||
this.options.user.username.toLowerCase()
|
||||
];
|
||||
const realName = this.options.user.getProperty(UserProps.RealName);
|
||||
if (realName) {
|
||||
this.cachedCompareNames.push(realName.toLowerCase());
|
||||
}
|
||||
} else {
|
||||
this.cachedCompareNames = [];
|
||||
}
|
||||
};
|
||||
|
||||
return this.cachedCompareNames.includes(message.toUserName.toLowerCase());
|
||||
}
|
||||
|
||||
_updateIndexTracking(message) {
|
||||
// index points at start of *message* not the header for... reasons?
|
||||
const index = (this.currentMessageOffset / QWKMessageBlockSize) + 1;
|
||||
if (message.isPrivate()) {
|
||||
this.inboxIndex.push(index);
|
||||
} else {
|
||||
if (this._messageAddressedToUser(message)) {
|
||||
// :TODO: add to both indexes???
|
||||
this.personalIndex.push(index);
|
||||
}
|
||||
|
||||
const areaTag = message.areaTag;
|
||||
if (!this.publicIndex.has(areaTag)) {
|
||||
this.publicIndex.set(areaTag, [index]);
|
||||
} else {
|
||||
this.publicIndex.get(areaTag).push(index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
appendNewFile() {
|
||||
|
||||
}
|
||||
|
@ -1019,6 +1062,9 @@ class QWKPacketWriter extends EventEmitter {
|
|||
(callback) => {
|
||||
return this._createControlData(callback);
|
||||
},
|
||||
(callback) => {
|
||||
return this._createIndexes(callback);
|
||||
},
|
||||
(callback) => {
|
||||
return this._producePacketArchive(packetPath, callback);
|
||||
}
|
||||
|
@ -1038,22 +1084,21 @@ class QWKPacketWriter extends EventEmitter {
|
|||
_producePacketArchive(packetPath, cb) {
|
||||
const archiveUtil = ArchiveUtil.getInstance();
|
||||
|
||||
const packetFiles = [
|
||||
'messages.dat', 'headers.dat', 'control.dat',
|
||||
].map(filename => {
|
||||
return filename;
|
||||
//return paths.join(this.workDir, filename);
|
||||
});
|
||||
fs.readdir(this.workDir, (err, files) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
|
||||
archiveUtil.compressTo(
|
||||
this.options.archiveFormat,
|
||||
packetPath,
|
||||
packetFiles,
|
||||
files,
|
||||
this.workDir,
|
||||
err => {
|
||||
return cb(err);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
_qwkMessageStatus(message) {
|
||||
|
@ -1090,15 +1135,9 @@ class QWKPacketWriter extends EventEmitter {
|
|||
return false;
|
||||
}
|
||||
|
||||
const netTag = ' '; // :TODO:
|
||||
|
||||
this.lolMessageId = this.lolMessageId || 1;
|
||||
//message.messageId = this.lolMessageId;
|
||||
this.lolMessageId++;
|
||||
|
||||
const header = Buffer.alloc(QWKMessageBlockSize, ' ');
|
||||
header.write(this._qwkMessageStatus(message), 0, 1, 'ascii');
|
||||
header.write(asciiNum(message.messageId), 1, 'ascii'); // :TODO: It seems Sync puts the relative, as in # of messages we've called appendMessage()?!
|
||||
header.write(asciiNum(message.messageId), 1, 'ascii');
|
||||
header.write(message.modTimestamp.format(QWKHeaderTimestampFormat), 8, 13, 'ascii');
|
||||
header.write(message.toUserName.substr(0, 25), 21, 'ascii');
|
||||
header.write(message.fromUserName.substr(0, 25), 46, 'ascii');
|
||||
|
@ -1108,8 +1147,8 @@ class QWKPacketWriter extends EventEmitter {
|
|||
header.write(asciiTotalBlocks, 116, 'ascii');
|
||||
header.writeUInt8(QWKMessageActiveStatus.Active, 122);
|
||||
header.writeUInt16LE(conferenceNumber, 123);
|
||||
header.writeUInt16LE(0, 125); // :TODO: Check if others actually populate this
|
||||
header.write(netTag[0], 127, 1, 'ascii');
|
||||
header.writeUInt16LE(this.totalMessages + 1, 125);
|
||||
header.write(QWKNetworkTagIndicator.NotPresent, 127, 1, 'ascii'); // :TODO: Present if for network output?
|
||||
|
||||
this.messagesStream.write(header);
|
||||
|
||||
|
@ -1117,6 +1156,9 @@ class QWKPacketWriter extends EventEmitter {
|
|||
}
|
||||
|
||||
_getMessageConferenceNumberByAreaTag(areaTag) {
|
||||
if (Message.isPrivateAreaTag(areaTag)) {
|
||||
return 0;
|
||||
}
|
||||
const areaConfig = _.get(Config(), [ 'messageNetworks', 'qwk', 'areas', areaTag ]);
|
||||
return areaConfig && areaConfig.conference;
|
||||
}
|
||||
|
@ -1131,6 +1173,13 @@ class QWKPacketWriter extends EventEmitter {
|
|||
|
||||
_createControlData(cb) {
|
||||
const areas = Array.from(this.areaTagsSeen).map(areaTag => {
|
||||
if (Message.isPrivateAreaTag(areaTag)) {
|
||||
return {
|
||||
areaTag : Message.WellKnownAreaTags.Private,
|
||||
name : 'Private',
|
||||
desc : 'Private Messages',
|
||||
};
|
||||
}
|
||||
return getMessageAreaByTag(areaTag);
|
||||
});
|
||||
|
||||
|
@ -1168,8 +1217,12 @@ class QWKPacketWriter extends EventEmitter {
|
|||
// map areas as conf #\r\nDescription\r\n pairs
|
||||
areas.forEach(area => {
|
||||
const conferenceNumber = this._getMessageConferenceNumberByAreaTag(area.areaTag);
|
||||
let desc = area.name;
|
||||
if (area.desc) {
|
||||
desc += ` - ${area.desc}`
|
||||
}
|
||||
controlStream.write(`${conferenceNumber}\r\n`);
|
||||
controlStream.write(`${area.name}\r\n`);
|
||||
controlStream.write(`${desc}\r\n`);
|
||||
});
|
||||
|
||||
// :TODO: do we ever care here?!
|
||||
|
@ -1180,6 +1233,73 @@ class QWKPacketWriter extends EventEmitter {
|
|||
controlStream.end();
|
||||
}
|
||||
|
||||
_createIndexes(cb) {
|
||||
const appendIndexData = (stream, offset) => {
|
||||
const msb = numToMbf32(offset);
|
||||
stream.write(msb);
|
||||
|
||||
// technically, the conference #, but only as a byte, so pretty much useless
|
||||
// AND the filename itself is the conference number... dafuq.
|
||||
stream.write(Buffer.from([0x00]));
|
||||
};
|
||||
|
||||
async.series(
|
||||
[
|
||||
(callback) => {
|
||||
// Create PERSONAL.NDX
|
||||
if (!this.personalIndex.length) {
|
||||
return callback(null);
|
||||
}
|
||||
|
||||
const indexStream = fs.createWriteStream(paths.join(this.workDir, 'personal.ndx'));
|
||||
this.personalIndex.forEach(offset => appendIndexData(indexStream, offset));
|
||||
|
||||
indexStream.on('close', err => {
|
||||
return callback(err);
|
||||
});
|
||||
|
||||
indexStream.end();
|
||||
},
|
||||
(callback) => {
|
||||
// 000.NDX of private mails
|
||||
if (!this.inboxIndex.length) {
|
||||
return callback(null);
|
||||
}
|
||||
|
||||
const indexStream = fs.createWriteStream(paths.join(this.workDir, '000.ndx'));
|
||||
this.inboxIndex.forEach(offset => appendIndexData(indexStream, offset));
|
||||
|
||||
indexStream.on('close', err => {
|
||||
return callback(err);
|
||||
});
|
||||
|
||||
indexStream.end();
|
||||
},
|
||||
(callback) => {
|
||||
// ####.NDX
|
||||
async.eachSeries(this.publicIndex.keys(), (areaTag, nextArea) => {
|
||||
const offsets = this.publicIndex.get(areaTag);
|
||||
const conferenceNumber = this._getMessageConferenceNumberByAreaTag(areaTag);
|
||||
const indexStream = fs.createWriteStream(paths.join(this.workDir, `${conferenceNumber.toString()}.ndx`));
|
||||
offsets.forEach(offset => appendIndexData(indexStream, offset));
|
||||
|
||||
indexStream.on('close', err => {
|
||||
return nextArea(err);
|
||||
});
|
||||
|
||||
indexStream.end();
|
||||
},
|
||||
err => {
|
||||
return callback(err);
|
||||
});
|
||||
}
|
||||
],
|
||||
err => {
|
||||
return cb(err);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
_makeSynchronetTimestamp(ts) {
|
||||
const syncTimestamp = ts.format('YYYYMMDDHHmmssZZ');
|
||||
const syncTZ = UTCOffsetToSMBTZ[ts.format('Z')] || '0000'; // :TODO: what if we don't have a map?
|
||||
|
@ -1215,7 +1335,7 @@ class QWKPacketWriter extends EventEmitter {
|
|||
Tags : message.hashTags.join(' '),
|
||||
|
||||
// :TODO: Needs tested with Sync/etc.; Sync wants Conference *numbers*
|
||||
Conference : getMessageConfTagByAreaTag(message.areaTag),
|
||||
Conference : message.isPrivate() ? '0' : getMessageConfTagByAreaTag(message.areaTag),
|
||||
|
||||
// ENiGMA Headers
|
||||
MessageUUID : message.uuid,
|
||||
|
|
Loading…
Reference in New Issue