moon
/
enigma-bbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Change FTN packet read() to use async iterator 

* createMessageUuidAlternate(): Mmethod for FTN message v5 UUID generation when no MSGID to work with
* parseAbbreviatedNetNodeList() now works properly
* Add core/uuid_util.js for various UUID utilities such as v5 named UUID generation
* Fix message meta load/retrieval
* Add lookup for REPLY kludge -> MSGID -> local reply IDs
* Fix SEEN-BY additions @ export
* Don't override MSGIDs if they already exist
* Store MSGID @ export so it can be inspected later
* Add import functionality (working, but WIP!)
* Clean up bundles and packets after import
This commit is contained in:
Bryan Ashby 2016-03-08 22:30:04 -07:00
parent 6094bed07f
commit ad0296addf
7 changed files with 628 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/database.js
View File

@ -175,7 +175,7 @@ function createMessageBaseTables() {
' meta_category INTEGER NOT NULL,' + ' meta_category INTEGER NOT NULL,' +
' meta_name VARCHAR NOT NULL,' + ' meta_name VARCHAR NOT NULL,' +
' meta_value VARCHAR NOT NULL,' + ' meta_value VARCHAR NOT NULL,' +
' UNIQUE(message_id, meta_category, meta_name, meta_value),' + ' UNIQUE(message_id, meta_category, meta_name, meta_value),' + // why unique here?
' FOREIGN KEY(message_id) REFERENCES message(message_id)' + ' FOREIGN KEY(message_id) REFERENCES message(message_id)' +
');' ');'
); );

2
core/ftn_address.js
View File

@ -107,7 +107,7 @@ module.exports = class Address {
} }
*/ */
isMatch(pattern) { isPatternMatch(pattern) {
const addr = this.getMatchAddr(pattern); const addr = this.getMatchAddr(pattern);
if(addr) { if(addr) {
return ( return (

213
core/ftn_mail_packet.js
View File

@ -32,6 +32,7 @@ const FTN_PACKET_HEADER_SIZE = 58; // fixed header size
const FTN_PACKET_HEADER_TYPE = 2; const FTN_PACKET_HEADER_TYPE = 2;
const FTN_PACKET_MESSAGE_TYPE = 2; const FTN_PACKET_MESSAGE_TYPE = 2;
const FTN_PACKET_BAUD_TYPE_2_2 = 2; const FTN_PACKET_BAUD_TYPE_2_2 = 2;
const NULL_TERM_BUFFER = new Buffer( [ 0x00 ] );
// SAUCE magic header + version ("00") // SAUCE magic header + version ("00")
const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00'); const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00');
@ -171,7 +172,6 @@ exports.PacketHeader = PacketHeader;
// http://walon.org/pub/fidonet/FTSC-nodelists-etc./pkt-types.txt // http://walon.org/pub/fidonet/FTSC-nodelists-etc./pkt-types.txt
// //
function Packet() { function Packet() {
var self = this; var self = this;
this.parsePacketHeader = function(packetBuffer, cb) { this.parsePacketHeader = function(packetBuffer, cb) {
@ -509,115 +509,105 @@ function Packet() {
} }
); );
}; };
this.parsePacketMessages = function(packetBuffer, iterator, cb) {
binary.parse(packetBuffer)
.word16lu('messageType')
.word16lu('ftn_orig_node')
.word16lu('ftn_dest_node')
.word16lu('ftn_orig_network')
.word16lu('ftn_dest_network')
.word16lu('ftn_attr_flags')
.word16lu('ftn_cost')
.scan('modDateTime', NULL_TERM_BUFFER) // :TODO: 20 bytes max
.scan('toUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max
.scan('fromUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max
.scan('subject', NULL_TERM_BUFFER) // :TODO: 72 bytes max6
.scan('message', NULL_TERM_BUFFER)
.tap(function tapped(msgData) { // no arrow function; want classic this
if(!msgData.messageType) {
// end marker -- no more messages
return cb(null);
}
if(FTN_PACKET_MESSAGE_TYPE != msgData.messageType) {
return cb(new Error('Unsupported message type: ' + msgData.messageType));
}
const read =
14 + // fixed header size
msgData.modDateTime.length + 1 +
msgData.toUserName.length + 1 +
msgData.fromUserName.length + 1 +
msgData.subject.length + 1 +
msgData.message.length + 1;
//
// Convert null terminated arrays to strings
//
let convMsgData = {};
[ 'modDateTime', 'toUserName', 'fromUserName', 'subject' ].forEach(k => {
convMsgData[k] = iconv.decode(msgData[k], 'CP437');
});
this.parsePacketMessages = function(messagesBuffer, iterator, cb) { //
const NULL_TERM_BUFFER = new Buffer( [ 0 ] ); // The message body itself is a special beast as it may
// contain an origin line, kludges, SAUCE in the case
var count = 0; // of ANSI files, etc.
//
let msg = new Message( {
toUserName : convMsgData.toUserName,
fromUserName : convMsgData.fromUserName,
subject : convMsgData.subject,
modTimestamp : ftn.getDateFromFtnDateTime(convMsgData.modDateTime),
});
msg.meta.FtnProperty = {};
msg.meta.FtnProperty.ftn_orig_node = msgData.ftn_orig_node;
msg.meta.FtnProperty.ftn_dest_node = msgData.ftn_dest_node;
msg.meta.FtnProperty.ftn_orig_network = msgData.ftn_orig_network;
msg.meta.FtnProperty.ftn_dest_network = msgData.ftn_dest_network;
msg.meta.FtnProperty.ftn_attr_flags = msgData.ftn_attr_flags;
msg.meta.FtnProperty.ftn_cost = msgData.ftn_cost;
binary.stream(messagesBuffer).loop(function looper(end, vars) { self.processMessageBody(msgData.message, messageBodyData => {
// msg.message = messageBodyData.message;
// Some variable names used here match up directly with well known msg.meta.FtnKludge = messageBodyData.kludgeLines;
// meta data names used with FTN messages.
// if(messageBodyData.tearLine) {
this msg.meta.FtnProperty.ftn_tear_line = messageBodyData.tearLine;
.word16lu('messageType')
.word16lu('ftn_orig_node')
.word16lu('ftn_dest_node')
.word16lu('ftn_orig_network')
.word16lu('ftn_dest_network')
.word16lu('ftn_attr_flags')
.word16lu('ftn_cost')
.scan('modDateTime', NULL_TERM_BUFFER) // :TODO: 20 bytes max
.scan('toUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max
.scan('fromUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max
.scan('subject', NULL_TERM_BUFFER) // :TODO: 72 bytes max
.scan('message', NULL_TERM_BUFFER)
.tap(function tapped(msgData) {
if(!msgData.messageType) {
// end marker -- no more messages
end();
return;
}
if(FTN_PACKET_MESSAGE_TYPE != msgData.messageType) {
end();
// :TODO: This is probably a bug if we hit a bad message after at leats one iterate
cb(new Error('Unsupported message type: ' + msgData.messageType));
return;
} }
++count; if(messageBodyData.seenBy.length > 0) {
msg.meta.FtnProperty.ftn_seen_by = messageBodyData.seenBy;
// }
// Convert null terminated arrays to strings
// if(messageBodyData.area) {
let convMsgData = {}; msg.meta.FtnProperty.ftn_area = messageBodyData.area;
[ 'modDateTime', 'toUserName', 'fromUserName', 'subject' ].forEach(k => { }
convMsgData[k] = iconv.decode(msgData[k], 'CP437');
}); if(messageBodyData.originLine) {
msg.meta.FtnProperty.ftn_origin = messageBodyData.originLine;
// }
// The message body itself is a special beast as it may
// contain special origin lines, kludges, SAUCE in the case const nextBuf = packetBuffer.slice(read);
// of ANSI files, etc. if(nextBuf.length > 0) {
// let next = function(e) {
let msg = new Message( { if(e) {
toUserName : convMsgData.toUserName, cb(e);
fromUserName : convMsgData.fromUserName, } else {
subject : convMsgData.subject, self.parsePacketMessages(nextBuf, iterator, cb);
modTimestamp : ftn.getDateFromFtnDateTime(convMsgData.modDateTime), }
}); };
msg.meta.FtnProperty = {};
msg.meta.FtnProperty.ftn_orig_node = msgData.ftn_orig_node;
msg.meta.FtnProperty.ftn_dest_node = msgData.ftn_dest_node;
msg.meta.FtnProperty.ftn_orig_network = msgData.ftn_orig_network;
msg.meta.FtnProperty.ftn_dest_network = msgData.ftn_dest_network;
msg.meta.FtnProperty.ftn_attr_flags = msgData.ftn_attr_flags;
msg.meta.FtnProperty.ftn_cost = msgData.ftn_cost;
self.processMessageBody(msgData.message, function processed(messageBodyData) {
msg.message = messageBodyData.message;
msg.meta.FtnKludge = messageBodyData.kludgeLines;
if(messageBodyData.tearLine) {
msg.meta.FtnProperty.ftn_tear_line = messageBodyData.tearLine;
}
if(messageBodyData.seenBy.length > 0) {
msg.meta.FtnProperty.ftn_seen_by = messageBodyData.seenBy;
}
if(messageBodyData.area) {
msg.meta.FtnProperty.ftn_area = messageBodyData.area;
}
if(messageBodyData.originLine) {
msg.meta.FtnProperty.ftn_origin = messageBodyData.originLine;
}
//
// Update message UUID, if possible, based on MSGID and AREA
//
if(_.isString(msg.meta.FtnKludge.MSGID) &&
_.isString(msg.meta.FtnProperty.ftn_area) &&
msg.meta.FtnKludge.MSGID.length > 0 &&
msg.meta.FtnProperty.ftn_area.length > 0)
{
msg.uuid = ftn.createMessageUuid(
msg.meta.FtnKludge.MSGID,
msg.meta.FtnProperty.area);
}
iterator('message', msg);
--count; iterator('message', msg, next);
if(0 === count) { } else {
cb(null); cb(null);
} }
}) });
}); });
});
}; };
this.getMessageEntryBuffer = function(message, options) { this.getMessageEntryBuffer = function(message, options) {
let basicHeader = new Buffer(34); let basicHeader = new Buffer(34);
@ -664,7 +654,7 @@ function Packet() {
}); });
} }
} }
// //
// FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001
// AREA:CONFERENCE // AREA:CONFERENCE
@ -818,10 +808,15 @@ function Packet() {
[ [
function processHeader(callback) { function processHeader(callback) {
self.parsePacketHeader(packetBuffer, (err, header) => { self.parsePacketHeader(packetBuffer, (err, header) => {
if(!err) { if(err) {
iterator('header', header); return callback(err);
} }
callback(err);
let next = function(e) {
callback(e);
};
iterator('header', header, next);
}); });
}, },
function processMessages(callback) { function processMessages(callback) {
@ -881,7 +876,7 @@ Packet.prototype.read = function(pathOrBuffer, iterator, cb) {
}); });
} }
], ],
function complete(err) { err => {
cb(err); cb(err);
} }
); );

113
core/ftn_util.js
View File

@ -4,6 +4,7 @@
let Config = require('./config.js').config; let Config = require('./config.js').config;
let Address = require('./ftn_address.js'); let Address = require('./ftn_address.js');
let FNV1a = require('./fnv1a.js'); let FNV1a = require('./fnv1a.js');
let createNamedUUID = require('./uuid_util.js').createNamedUUID;
let _ = require('lodash'); let _ = require('lodash');
let assert = require('assert'); let assert = require('assert');
@ -22,6 +23,7 @@ let packageJson = require('../package.json');
exports.stringToNullPaddedBuffer = stringToNullPaddedBuffer; exports.stringToNullPaddedBuffer = stringToNullPaddedBuffer;
exports.getMessageSerialNumber = getMessageSerialNumber; exports.getMessageSerialNumber = getMessageSerialNumber;
exports.createMessageUuid = createMessageUuid; exports.createMessageUuid = createMessageUuid;
exports.createMessageUuidAlternate = createMessageUuidAlternate;
exports.getDateFromFtnDateTime = getDateFromFtnDateTime; exports.getDateFromFtnDateTime = getDateFromFtnDateTime;
exports.getDateTimeString = getDateTimeString; exports.getDateTimeString = getDateTimeString;
@ -100,42 +102,43 @@ function getDateTimeString(m) {
return m.format('DD MMM YY HH:mm:ss'); return m.format('DD MMM YY HH:mm:ss');
} }
//
// Create a v5 named UUID given a message ID ("MSGID") and
// FTN area tag ("AREA").
//
// This is similar to CrashMail
// See https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c
//
function createMessageUuid(ftnMsgId, ftnArea) { function createMessageUuid(ftnMsgId, ftnArea) {
// assert(_.isString(ftnMsgId));
// v5 UUID generation code based on the work here: assert(_.isString(ftnArea));
// https://github.com/download13/uuidv5/blob/master/uuid.js
//
// Note: CrashMail uses MSGID + AREA, so we go with that as well:
// https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c
//
if(!Buffer.isBuffer(ftnMsgId)) {
ftnMsgId = iconv.encode(ftnMsgId, 'CP437');
}
ftnArea = ftnArea || ''; // AREA is optional ftnMsgId = iconv.encode(ftnMsgId, 'CP437');
if(!Buffer.isBuffer(ftnArea)) { ftnArea = iconv.encode(ftnArea.toUpperCase(), 'CP437');
ftnArea = iconv.encode(ftnArea, 'CP437');
}
const ns = new Buffer(ENIGMA_FTN_MSGID_NAMESPACE); return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnMsgId, ftnArea ] )));
};
let digest = createHash('sha1').update( //
Buffer.concat([ ns, ftnMsgId, ftnArea ])).digest(); // Create a v5 named UUID given a FTN area tag ("AREA"),
// create/modified date, subject, and message body
let u = new Buffer(16); //
// This method should be used as a backup for when a MSGID is
// bbbb - bb - bb - bb - bbbbbb // not available in which createMessageUuid() above should be
digest.copy(u, 0, 0, 4); // time_low // used instead.
digest.copy(u, 4, 4, 6); // time_mid //
digest.copy(u, 6, 6, 8); // time_hi_and_version function createMessageUuidAlternate(ftnArea, modTimestamp, subject, msgBody) {
assert(_.isString(ftnArea));
u[6] = (u[6] & 0x0f) | 0x50; // version, 4 most significant bits are set to version 5 (0101) assert(_.isDate(modTimestamp) || moment.isMoment(modTimestamp));
u[8] = (digest[8] & 0x3f) | 0x80; // clock_seq_hi_and_reserved, 2msb are set to 10 assert(_.isString(subject));
u[9] = digest[9]; assert(_.isString(msgBody));
ftnArea = iconv.encode(ftnArea.toUpperCase(), 'CP437');
modTimestamp = iconv.encode(getDateTimeString(modTimestamp), 'CP437');
subject = iconv.encode(subject.toUpperCase().trim(), 'CP437');
msgBody = iconv.encode(msgBody.replace(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g, '').trim(), 'CP437');
digest.copy(u, 10, 10, 16); return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnArea, modTimestamp, subject, msgBody ] )));
return uuid.unparse(u); // to string
} }
function getMessageSerialNumber(messageId) { function getMessageSerialNumber(messageId) {
@ -274,6 +277,9 @@ function getAbbreviatedNetNodeList(netNodes) {
let abbrList = ''; let abbrList = '';
let currNet; let currNet;
netNodes.forEach(netNode => { netNodes.forEach(netNode => {
if(_.isString(netNode)) {
netNode = Address.fromString(netNode);
}
if(currNet !== netNode.net) { if(currNet !== netNode.net) {
abbrList += `${netNode.net}/`; abbrList += `${netNode.net}/`;
currNet = netNode.net; currNet = netNode.net;
@ -284,29 +290,24 @@ function getAbbreviatedNetNodeList(netNodes) {
return abbrList.trim(); // remove trailing space return abbrList.trim(); // remove trailing space
} }
//
// Parse an abbreviated net/node list commonly used for SEEN-BY and PATH
//
function parseAbbreviatedNetNodeList(netNodes) { function parseAbbreviatedNetNodeList(netNodes) {
// const re = /([0-9]+)\/([0-9]+)\s?|([0-9]+)\s?/g;
// Make sure we have an array of objects. let net;
// Allow for a single object or string(s) let m;
// let results = [];
if(!_.isArray(netNodes)) { while(null !== (m = re.exec(netNodes))) {
if(_.isString(netNodes)) { if(m[1] && m[2]) {
netNodes = netNodes.split(' '); net = parseInt(m[1]);
} else { results.push(new Address( { net : net, node : parseInt(m[2]) } ));
netNodes = [ netNodes ]; } else if(net) {
} results.push(new Address( { net : net, node : parseInt(m[3]) } ));
} }
}
//
// Convert any strings to parsed address objects return results;
//
return netNodes.map(a => {
if(_.isObject(a)) {
return a;
} else {
return Address.fromString(a);
}
});
} }
// //
@ -348,8 +349,12 @@ function getUpdatedSeenByEntries(existingEntries, additions) {
if(!_.isArray(existingEntries)) { if(!_.isArray(existingEntries)) {
existingEntries = [ existingEntries ]; existingEntries = [ existingEntries ];
} }
if(!_.isString(additions)) {
additions = parseAbbreviatedNetNodeList(getAbbreviatedNetNodeList(additions));
}
additions = parseAbbreviatedNetNodeList(additions).sort(Address.getComparator()); additions = additions.sort(Address.getComparator());
// //
// For now, we'll just append a new SEEN-BY entry // For now, we'll just append a new SEEN-BY entry

231
core/message.js
View File

@ -62,22 +62,6 @@ function Message(options) {
ts = ts || new Date(); ts = ts || new Date();
return ts.toISOString(); return ts.toISOString();
}; };
/*
Object.defineProperty(this, 'messageId', {
get : function() {
return messageId;
}
});
Object.defineProperty(this, 'areaId', {
get : function() { return areaId; },
set : function(i) {
areaId = i;
}
});
*/
} }
Message.WellKnownAreaTags = { Message.WellKnownAreaTags = {
@ -115,9 +99,7 @@ Message.FtnPropertyNames = {
FtnDestZone : 'ftn_dest_zone', FtnDestZone : 'ftn_dest_zone',
FtnOrigPoint : 'ftn_orig_point', FtnOrigPoint : 'ftn_orig_point',
FtnDestPoint : 'ftn_dest_point', FtnDestPoint : 'ftn_dest_point',
FtnAttribute : 'ftn_attribute', FtnAttribute : 'ftn_attribute',
FtnTearLine : 'ftn_tear_line', // http://ftsc.org/docs/fts-0004.001 FtnTearLine : 'ftn_tear_line', // http://ftsc.org/docs/fts-0004.001
@ -136,6 +118,118 @@ Message.prototype.setLocalFromUserId = function(userId) {
this.meta.System.local_from_user_id = userId; this.meta.System.local_from_user_id = userId;
}; };
Message.getMessageIdByUuid = function(uuid, cb) {
msgDb.get(
`SELECT message_id
FROM message
WHERE message_uuid = ?
LIMIT 1;`,
[ uuid ],
(err, row) => {
if(err) {
cb(err);
} else {
const success = (row && row.message_id);
cb(success ? null : new Error('No match'), success ? row.message_id : null);
}
}
);
};
Message.getMessageIdsByMetaValue = function(category, name, value, cb) {
msgDb.all(
`SELECT message_id
FROM message_meta
WHERE meta_category = ? AND meta_name = ? AND meta_value = ?;`,
[ category, name, value ],
(err, rows) => {
if(err) {
cb(err);
} else {
cb(null, rows.map(r => parseInt(r.message_id))); // return array of ID(s)
}
}
);
};
Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name, cb) {
async.waterfall(
[
function getMessageId(callback) {
Message.getMessageIdByUuid(uuid, (err, messageId) => {
callback(err, messageId);
});
},
function getMetaValue(messageId, callback) {
const sql =
`SELECT meta_value
FROM message_meta
WHERE message_id = ? AND message_category = ? AND meta_name = ?;`;
msgDb.all(sql, [ messageId, category, name ], (err, rows) => {
if(err) {
return callback(err);
}
if(0 === rows.length) {
return callback(new Error('No value for category/name'));
}
// single values are returned without an array
if(1 === rows.length) {
return callback(null, rows[0].meta_value);
}
callback(null, rows.map(r => r.meta_value));
});
}
],
(err, value) => {
cb(err, value);
}
);
};
Message.prototype.loadMeta = function(cb) {
/*
Example of loaded this.meta:
meta: {
System: {
local_to_user_id: 1234,
},
FtnProperty: {
ftn_seen_by: [ "1/102 103", "2/42 52 65" ]
}
}
*/
const sql =
`SELECT meta_category, meta_name, meta_value
FROM message_meta
WHERE message_id = ?;`;
let self = this;
msgDb.each(sql, [ this.messageId ], (err, row) => {
if(!(row.meta_category in self.meta)) {
self.meta[row.meta_category] = { };
self.meta[row.meta_category][row.meta_name] = row.meta_value;
} else {
if(!(row.meta_name in self.meta[row.meta_category])) {
self.meta[row.meta_category][row.meta_name] = row.meta_value;
} else {
if(_.isString(self.meta[row.meta_category][row.meta_name])) {
self.meta[row.meta_category][row.meta_name] = [ self.meta[row.meta_category][row.meta_name] ];
}
self.meta[row.meta_category][row.meta_name].push(row.meta_value);
}
}
}, err => {
cb(err);
});
};
Message.prototype.load = function(options, cb) { Message.prototype.load = function(options, cb) {
assert(_.isString(options.uuid)); assert(_.isString(options.uuid));
@ -168,8 +262,9 @@ Message.prototype.load = function(options, cb) {
); );
}, },
function loadMessageMeta(callback) { function loadMessageMeta(callback) {
// :TODO: self.loadMeta(err => {
callback(null); callback(err);
});
}, },
function loadHashTags(callback) { function loadHashTags(callback) {
// :TODO: // :TODO:
@ -188,27 +283,59 @@ Message.prototype.load = function(options, cb) {
); );
}; };
Message.prototype.persistMetaValue = function(category, name, value, cb) {
const metaStmt = msgDb.prepare(
`INSERT INTO message_meta (message_id, meta_category, meta_name, meta_value)
VALUES (?, ?, ?, ?);`);
if(!_.isArray(value)) {
value = [ value ];
}
let self = this;
async.each(value, (v, next) => {
metaStmt.run(self.messageId, category, name, v, err => {
next(err);
});
}, err => {
cb(err);
});
};
Message.startTransaction = function(cb) {
msgDb.run('BEGIN;', err => {
cb(err);
});
};
Message.endTransaction = function(hadError, cb) {
msgDb.run(hadError ? 'ROLLBACK;' : 'COMMIT;', err => {
cb(err);
});
};
Message.prototype.persist = function(cb) { Message.prototype.persist = function(cb) {
if(!this.isValid()) { if(!this.isValid()) {
cb(new Error('Cannot persist invalid message!')); return cb(new Error('Cannot persist invalid message!'));
return;
} }
var self = this; let self = this;
async.series( async.series(
[ [
function beginTransaction(callback) { function beginTransaction(callback) {
msgDb.run('BEGIN;', function transBegin(err) { Message.startTransaction(err => {
callback(err); callback(err);
}); });
}, },
function storeMessage(callback) { function storeMessage(callback) {
msgDb.run( msgDb.run(
'INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) ' + `INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp)
'VALUES (?, ?, ?, ?, ?, ?, ?, ?);', [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ], VALUES (?, ?, ?, ?, ?, ?, ?, ?);`,
function msgInsert(err) { [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ],
function inserted(err) { // use for this scope
if(!err) { if(!err) {
self.messageId = this.lastID; self.messageId = this.lastID;
} }
@ -221,26 +348,30 @@ Message.prototype.persist = function(cb) {
if(!self.meta) { if(!self.meta) {
callback(null); callback(null);
} else { } else {
// :TODO: this should be it's own method such that meta can be updated /*
var metaStmt = msgDb.prepare( Example of self.meta:
'INSERT INTO message_meta (message_id, meta_category, meta_name, meta_value) ' +
'VALUES (?, ?, ?, ?);'); meta: {
System: {
for(var metaCategroy in self.meta) { local_to_user_id: 1234,
async.each(Object.keys(self.meta[metaCategroy]), function meta(metaName, next) { },
metaStmt.run(self.messageId, Message.MetaCategories[metaCategroy], metaName, self.meta[metaCategroy][metaName], function inserted(err) { FtnProperty: {
next(err); ftn_seen_by: [ "1/102 103", "2/42 52 65" ]
});
}, function complete(err) {
if(!err) {
metaStmt.finalize(function finalized() {
callback(null);
});
} else {
callback(err);
} }
}
*/
async.each(Object.keys(self.meta), (category, nextCat) => {
async.each(Object.keys(self.meta[category]), (name, nextName) => {
self.persistMetaValue(category, name, self.meta[category][name], err => {
nextName(err);
});
}, err => {
nextCat(err);
}); });
}
}, err => {
callback(err);
});
} }
}, },
function storeHashTags(callback) { function storeHashTags(callback) {
@ -248,9 +379,9 @@ Message.prototype.persist = function(cb) {
callback(null); callback(null);
} }
], ],
function complete(err) { err => {
msgDb.run(err ? 'ROLLBACK;' : 'COMMIT;', function transEnd(err) { Message.endTransaction(err, transErr => {
cb(err, self.messageId); cb(err ? err : transErr, self.messageId);
}); });
} }
); );

315
core/scanner_tossers/ftn_bso.js
View File

@ -20,6 +20,7 @@ let async = require('async');
let fs = require('fs'); let fs = require('fs');
let later = require('later'); let later = require('later');
let temp = require('temp').track(); // track() cleans up temp dir/files for us let temp = require('temp').track(); // track() cleans up temp dir/files for us
let assert = require('assert');
exports.moduleInfo = { exports.moduleInfo = {
name : 'FTN BSO', name : 'FTN BSO',
@ -72,13 +73,44 @@ function FTNMessageScanTossModule() {
return(networkName === defaultNetworkName && address.zone === this.moduleConfig.defaultZone); return(networkName === defaultNetworkName && address.zone === this.moduleConfig.defaultZone);
}; };
this.getNetworkNameByAddress = function(address) { this.getNetworkNameByAddress = function(remoteAddress) {
return _.findKey(Config.messageNetworks.ftn.networks, network => { return _.findKey(Config.messageNetworks.ftn.networks, network => {
const networkAddress = Address.fromString(network.localAddress); const localAddress = Address.fromString(network.localAddress);
return !_.isUndefined(networkAddress) && address.isEqual(networkAddress); return !_.isUndefined(localAddress) && localAddress.isEqual(remoteAddress);
}); });
}; };
this.getNetworkNameByAddressPattern = function(remoteAddressPattern) {
return _.findKey(Config.messageNetworks.ftn.networks, network => {
const localAddress = Address.fromString(network.localAddress);
return !_.isUndefined(localAddress) && localAddress.isPatternMatch(remoteAddressPattern);
});
};
this.getLocalAreaTagByFtnAreaTag = function(ftnAreaTag) {
return _.findKey(Config.messageNetworks.ftn.areas, areaConf => {
return areaConf.tag === ftnAreaTag;
});
};
/*
this.getSeenByAddresses = function(messageSeenBy) {
if(!_.isArray(messageSeenBy)) {
messageSeenBy = [ messageSeenBy ];
}
let seenByAddrs = [];
messageSeenBy.forEach(sb => {
seenByAddrs = seenByAddrs.concat(ftnUtil.parseAbbreviatedNetNodeList(sb));
});
return seenByAddrs;
};
*/
this.messageHasValidMSGID = function(msg) {
return _.isString(msg.meta.FtnKludge.MSGID) && msg.meta.FtnKludge.MSGID.length > 0;
};
this.getOutgoingPacketDir = function(networkName, destAddress) { this.getOutgoingPacketDir = function(networkName, destAddress) {
let dir = this.moduleConfig.paths.outbound; let dir = this.moduleConfig.paths.outbound;
if(!this.isDefaultDomainZone(networkName, destAddress)) { if(!this.isDefaultDomainZone(networkName, destAddress)) {
@ -164,41 +196,6 @@ function FTNMessageScanTossModule() {
}); });
}; };
this.exportMessage = function(message, options, cb) {
this.prepareMessage(message, options);
let packet = new ftnMailPacket.Packet();
let packetHeader = new ftnMailPacket.PacketHeader(
options.network.localAddress,
options.destAddress,
options.nodeConfig.packetType);
packetHeader.password = options.nodeConfig.packetPassword || '';
if(message.isPrivate()) {
// :TODO: this should actually be checking for isNetMail()!!
} else {
const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.destAddress);
mkdirp(outgoingDir, err => {
if(err) {
return cb(err);
}
this.getOutgoingBundleFileName(outgoingDir, options.network.localAddress, options.destAddress, (err, path) => {
console.log(path);
});
packet.write(
this.getOutgoingPacketFileName(outgoingDir, message),
packetHeader,
[ message ],
{ encoding : options.encoding }
);
});
}
};
this.prepareMessage = function(message, options) { this.prepareMessage = function(message, options) {
// //
// Set various FTN kludges/etc. // Set various FTN kludges/etc.
@ -241,7 +238,8 @@ function FTNMessageScanTossModule() {
// When exporting messages, we should create/update SEEN-BY // When exporting messages, we should create/update SEEN-BY
// with remote address(s) we are exporting to. // with remote address(s) we are exporting to.
// //
const seenByAdditions = [ options.network.localAddress ].concat(Config.messageNetworks.ftn.areas[message.areaTag].uplinks); const seenByAdditions =
[ `${options.network.localAddress.net}/${options.network.localAddress.node}` ].concat(Config.messageNetworks.ftn.areas[message.areaTag].uplinks);
message.meta.FtnProperty.ftn_seen_by = message.meta.FtnProperty.ftn_seen_by =
ftnUtil.getUpdatedSeenByEntries(message.meta.FtnProperty.ftn_seen_by, seenByAdditions); ftnUtil.getUpdatedSeenByEntries(message.meta.FtnProperty.ftn_seen_by, seenByAdditions);
@ -256,8 +254,14 @@ function FTNMessageScanTossModule() {
// //
// Additional kludges // Additional kludges
//
// Check for existence of MSGID as we may already have stored it from a previous
// export that failed to finish
// //
message.meta.FtnKludge.MSGID = ftnUtil.getMessageIdentifier(message, options.network.localAddress); if(!message.meta.FtnKludge.MSGID) {
message.meta.FtnKludge.MSGID = ftnUtil.getMessageIdentifier(message, options.network.localAddress);
}
message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset(); message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset();
if(!message.meta.FtnKludge.PID) { if(!message.meta.FtnKludge.PID) {
@ -369,7 +373,7 @@ function FTNMessageScanTossModule() {
this.getNodeConfigKeyForUplink = function(uplink) { this.getNodeConfigKeyForUplink = function(uplink) {
// :TODO: sort by least # of '*' & take top? // :TODO: sort by least # of '*' & take top?
const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => { const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => {
return Address.fromString(addr).isMatch(uplink); return Address.fromString(addr).isPatternMatch(uplink);
})[0]; })[0];
return nodeKey; return nodeKey;
@ -451,6 +455,19 @@ function FTNMessageScanTossModule() {
ws.write(msgBuf); ws.write(msgBuf);
} }
callback(null); callback(null);
},
function updateStoredMeta(callback) {
//
// We want to store some meta as if we had imported
// this message for later reference
//
if(message.meta.FtnKludge.MSGID) {
message.persistMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.MSGID, err => {
callback(err);
});
} else {
callback(null);
}
} }
], ],
err => { err => {
@ -607,37 +624,144 @@ function FTNMessageScanTossModule() {
}, cb); // complete }, cb); // complete
}; };
this.importMessagesFromPacketFile = function(packetPath, cb) { this.setReplyToMsgIdFtnReplyKludge = function(message, cb) {
const packet = new ftnMailPacket.Packet(); //
// Given a FTN REPLY kludge, set |message.replyToMsgId|, if possible,
// by looking up an associated MSGID kludge meta.
//
// See also: http://ftsc.org/docs/fts-0009.001
//
if(!_.isString(message.meta.FtnKludge.REPLY)) {
// nothing to do
return cb();
}
// :TODO: packet.read() should have a way to cancel iteration... Message.getMessageIdsByMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.REPLY, (err, msgIds) => {
let localNetworkName; if(!err) {
packet.read(packetPath, (entryType, entryData) => { assert(1 === msgIds.length);
message.replyToMsgId = msgIds[0];
}
cb();
});
};
this.importNetMailToArea = function(localAreaTag, header, message, cb) {
async.series(
[
function validateDestinationAddress(callback) {
/*
const messageDestAddress = new Address({
node : message.meta.FtnProperty.ftn_dest_node,
net : message.meta.FtnProperty.ftn_dest_network,
});
*/
const localNetworkPattern = `${message.meta.FtnProperty.ftn_dest_network}/${message.meta.FtnProperty.ftn_dest_node}`;
const localNetworkName = self.getNetworkNameByAddressPattern(localNetworkPattern);
callback(_.isString(localNetworkName) ? null : new Error('Packet destination is not us'));
},
function basicSetup(callback) {
message.areaTag = localAreaTag;
//
// If duplicates are NOT allowed in the area (the default), we need to update
// the message UUID using data available to us. Duplicate UUIDs are internally
// not allowed in our local database.
//
if(!Config.messageNetworks.ftn.areas[localAreaTag].allowDupes) {
if(self.messageHasValidMSGID(message)) {
// Update UUID with our preferred generation method
message.uuid = ftnUtil.createMessageUuid(
message.meta.FtnKludge.MSGID,
message.meta.FtnProperty.ftn_area);
} else {
// Update UUID with alternate/backup generation method
message.uuid = ftnUtil.createMessageUuidAlternate(
message.meta.FtnProperty.ftn_area,
message.modTimestamp,
message.subject,
message.message);
}
}
callback(null);
},
function setReplyToMessageId(callback) {
self.setReplyToMsgIdFtnReplyKludge(message, () => {
callback(null);
});
},
function persistImport(callback) {
message.persist(err => {
callback(err);
});
}
], err => {
cb(err);
}
);
};
//
// Ref. implementations on import:
// * https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/pkt.c
// https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/handle.c
//
this.importMessagesFromPacketFile = function(packetPath, cb) {
let packetHeader;
new ftnMailPacket.Packet().read(packetPath, (entryType, entryData, next) => {
if('header' === entryType) { if('header' === entryType) {
// packetHeader = entryData;
// Discover if this packet is for one of our network(s)
// const localNetworkName = self.getNetworkNameByAddress(packetHeader.destAddress);
localNetworkName = self.getNetworkNameByAddress(entryData.destAddress); if(!_.isString(localNetworkName)) {
next(new Error('No configuration for this packet'));
} else if(localNetworkName && 'message' === entryType) { } else {
const message = entryData; // so we ref something reasonable :) next(null);
}
} else if('message' === entryType) {
const message = entryData;
const areaTag = message.meta.FtnProperty.ftn_area; const areaTag = message.meta.FtnProperty.ftn_area;
// :TODO: we need to know if this message is a dupe - UUID will be the same if MSGID, but if not present... what to do?
// :TODO: lookup and set message.areaTag if match
// :TODO: check SEEN-BY for echo
// :TODO: Handle area vs Netmail - Via, etc.
// :TODO: Handle PATH
// :TODO: handle REPLY kludges... set local ID when possible
if(areaTag) { if(areaTag) {
// //
// Find local area tag // EchoMail
//
const localAreaTag = self.getLocalAreaTagByFtnAreaTag(areaTag);
if(localAreaTag) {
self.importNetMailToArea(localAreaTag, packetHeader, message, err => {
if(err) {
if('SQLITE_CONSTRAINT' === err.code) {
Log.info(
{ subject : message.subject, uuid : message.uuid },
'Not importing non-unique message');
return next(null);
}
}
next(err);
});
} else {
//
// No local area configured for this import
//
// :TODO: Handle the "catch all" case, if configured
}
} else {
//
// NetMail
// //
} }
} }
}, err => { }, err => {
cb(err); cb(err);
}); }; });
};
this.importPacketFilesFromDirectory = function(importDir, cb) { this.importPacketFilesFromDirectory = function(importDir, cb) {
async.waterfall( async.waterfall(
@ -672,9 +796,9 @@ function FTNMessageScanTossModule() {
// :TODO: rename to .bad, perhaps move to a rejects dir + log // :TODO: rename to .bad, perhaps move to a rejects dir + log
nextFile(); nextFile();
} else { } else {
//fs.unlink(fullPath, err => { fs.unlink(fullPath, err => {
nextFile(); nextFile();
//}); });
} }
}, err => { }, err => {
callback(err); callback(err);
@ -687,7 +811,9 @@ function FTNMessageScanTossModule() {
); );
}; };
this.importMessagesFromDirectory = function(importDir, cb) { this.importMessagesFromDirectory = function(inboundType, importDir, cb) {
let tempDirectory;
async.waterfall( async.waterfall(
[ [
// start with .pkt files // start with .pkt files
@ -712,21 +838,37 @@ function FTNMessageScanTossModule() {
}, },
function createTempDir(bundleFiles, callback) { function createTempDir(bundleFiles, callback) {
temp.mkdir('enigftnimport-', (err, tempDir) => { temp.mkdir('enigftnimport-', (err, tempDir) => {
callback(err, bundleFiles, tempDir); tempDirectory = tempDir;
callback(err, bundleFiles);
}); });
}, },
function importBundles(bundleFiles, tempDir, callback) { function importBundles(bundleFiles, callback) {
let rejects = [];
async.each(bundleFiles, (bundleFile, nextFile) => { async.each(bundleFiles, (bundleFile, nextFile) => {
if(_.isUndefined(bundleFile.archName)) { if(_.isUndefined(bundleFile.archName)) {
// :TODO: log? Log.info(
{ fileName : bundleFile.path },
'Unknown bundle archive type');
rejects.push(bundleFile.path);
return nextFile(); // unknown archive type return nextFile(); // unknown archive type
} }
self.archUtil.extractTo( self.archUtil.extractTo(
bundleFile.path, bundleFile.path,
tempDir, tempDirectory,
bundleFile.archName, bundleFile.archName,
err => { err => {
if(err) {
Log.info(
{ fileName : bundleFile.path, error : err.toString() },
'Failed to extract bundle');
rejects.push(bundleFile.path);
}
nextFile(); nextFile();
} }
); );
@ -738,14 +880,39 @@ function FTNMessageScanTossModule() {
// //
// All extracted - import .pkt's // All extracted - import .pkt's
// //
self.importPacketFilesFromDirectory(tempDir, err => { self.importPacketFilesFromDirectory(tempDirectory, err => {
callback(err); callback(null, bundleFiles, rejects);
}); });
}); });
},
function handleProcessedBundleFiles(bundleFiles, rejects, callback) {
async.each(bundleFiles, (bundleFile, nextFile) => {
if(rejects.indexOf(bundleFile.path) > -1) {
// :TODO: rename to .bad, perhaps move to a rejects dir + log
nextFile();
} else {
fs.unlink(bundleFile.path, err => {
nextFile();
});
}
}, err => {
callback(err);
});
} }
], ],
err => { err => {
cb(err); if(tempDirectory) {
temp.cleanup( (errIgnored, stats) => {
Log.trace(
Object.assign(stats, { tempDir : tempDirectory } ),
'Temporary directory cleaned up'
);
cb(err); // orig err
});
} else {
cb(err);
}
} }
); );
}; };
@ -818,8 +985,8 @@ FTNMessageScanTossModule.prototype.performImport = function(cb) {
var self = this; var self = this;
async.each( [ 'inbound', 'secInbound' ], (importDir, nextDir) => { async.each( [ 'inbound', 'secInbound' ], (inboundType, nextDir) => {
self.importMessagesFromDirectory(self.moduleConfig.paths[importDir], err => { self.importMessagesFromDirectory(inboundType, self.moduleConfig.paths[inboundType], err => {
nextDir(); nextDir();
}); });
@ -879,7 +1046,7 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) {
const newLastScanId = msgRows[msgRows.length - 1].message_id; const newLastScanId = msgRows[msgRows.length - 1].message_id;
Log.info( Log.info(
{ messagesExported : msgRows.length, newLastScanId : newLastScanId }, { areaTag : areaTag, messagesExported : msgRows.length, newLastScanId : newLastScanId },
'Export complete'); 'Export complete');
callback(err, newLastScanId); callback(err, newLastScanId);

41
core/uuid_util.js Normal file
View File

@ -0,0 +1,41 @@
/* jslint node: true */
'use strict';
let uuid = require('node-uuid');
let assert = require('assert');
let _ = require('lodash');
let createHash = require('crypto').createHash;
exports.createNamedUUID = createNamedUUID;
function createNamedUUID(namespaceUuid, key) {
//
// v5 UUID generation code based on the work here:
// https://github.com/download13/uuidv5/blob/master/uuid.js
//
if(!Buffer.isBuffer(namespaceUuid)) {
namespaceUuid = new Buffer(namespaceUuid);
}
if(!Buffer.isBuffer(key)) {
key = new Buffer(key);
}
let digest = createHash('sha1').update(
Buffer.concat( [ namespaceUuid, key ] )).digest();
let u = new Buffer(16);
// bbbb - bb - bb - bb - bbbbbb
digest.copy(u, 0, 0, 4); // time_low
digest.copy(u, 4, 4, 6); // time_mid
digest.copy(u, 6, 6, 8); // time_hi_and_version
u[6] = (u[6] & 0x0f) | 0x50; // version, 4 most significant bits are set to version 5 (0101)
u[8] = (digest[8] & 0x3f) | 0x80; // clock_seq_hi_and_reserved, 2msb are set to 10
u[9] = digest[9];
digest.copy(u, 10, 10, 16);
return u;
}