From ad0296addff3cf6b6e488b2a5c3517a220a8624e Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 8 Mar 2016 22:30:04 -0700 Subject: [PATCH] * Change FTN packet read() to use async iterator * createMessageUuidAlternate(): Mmethod for FTN message v5 UUID generation when no MSGID to work with * parseAbbreviatedNetNodeList() now works properly * Add core/uuid_util.js for various UUID utilities such as v5 named UUID generation * Fix message meta load/retrieval * Add lookup for REPLY kludge -> MSGID -> local reply IDs * Fix SEEN-BY additions @ export * Don't override MSGIDs if they already exist * Store MSGID @ export so it can be inspected later * Add import functionality (working, but WIP!) * Clean up bundles and packets after import --- core/database.js | 2 +- core/ftn_address.js | 2 +- core/ftn_mail_packet.js | 213 +++++++++++---------- core/ftn_util.js | 113 ++++++------ core/message.js | 231 ++++++++++++++++++----- core/scanner_tossers/ftn_bso.js | 315 ++++++++++++++++++++++++-------- core/uuid_util.js | 41 +++++ 7 files changed, 628 insertions(+), 289 deletions(-) create mode 100644 core/uuid_util.js diff --git a/core/database.js b/core/database.js index 3e3b6faf..35e9cc87 100644 --- a/core/database.js +++ b/core/database.js @@ -175,7 +175,7 @@ function createMessageBaseTables() { ' meta_category INTEGER NOT NULL,' + ' meta_name VARCHAR NOT NULL,' + ' meta_value VARCHAR NOT NULL,' + - ' UNIQUE(message_id, meta_category, meta_name, meta_value),' + + ' UNIQUE(message_id, meta_category, meta_name, meta_value),' + // why unique here? ' FOREIGN KEY(message_id) REFERENCES message(message_id)' + ');' ); diff --git a/core/ftn_address.js b/core/ftn_address.js index 880828d9..3e849b55 100644 --- a/core/ftn_address.js +++ b/core/ftn_address.js @@ -107,7 +107,7 @@ module.exports = class Address { } */ - isMatch(pattern) { + isPatternMatch(pattern) { const addr = this.getMatchAddr(pattern); if(addr) { return ( diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index ecc0f1c8..5f4aeac1 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -32,6 +32,7 @@ const FTN_PACKET_HEADER_SIZE = 58; // fixed header size const FTN_PACKET_HEADER_TYPE = 2; const FTN_PACKET_MESSAGE_TYPE = 2; const FTN_PACKET_BAUD_TYPE_2_2 = 2; +const NULL_TERM_BUFFER = new Buffer( [ 0x00 ] ); // SAUCE magic header + version ("00") const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00'); @@ -171,7 +172,6 @@ exports.PacketHeader = PacketHeader; // http://walon.org/pub/fidonet/FTSC-nodelists-etc./pkt-types.txt // function Packet() { - var self = this; this.parsePacketHeader = function(packetBuffer, cb) { @@ -509,115 +509,105 @@ function Packet() { } ); }; + + this.parsePacketMessages = function(packetBuffer, iterator, cb) { + binary.parse(packetBuffer) + .word16lu('messageType') + .word16lu('ftn_orig_node') + .word16lu('ftn_dest_node') + .word16lu('ftn_orig_network') + .word16lu('ftn_dest_network') + .word16lu('ftn_attr_flags') + .word16lu('ftn_cost') + .scan('modDateTime', NULL_TERM_BUFFER) // :TODO: 20 bytes max + .scan('toUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max + .scan('fromUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max + .scan('subject', NULL_TERM_BUFFER) // :TODO: 72 bytes max6 + .scan('message', NULL_TERM_BUFFER) + .tap(function tapped(msgData) { // no arrow function; want classic this + if(!msgData.messageType) { + // end marker -- no more messages + return cb(null); + } + + if(FTN_PACKET_MESSAGE_TYPE != msgData.messageType) { + return cb(new Error('Unsupported message type: ' + msgData.messageType)); + } + + const read = + 14 + // fixed header size + msgData.modDateTime.length + 1 + + msgData.toUserName.length + 1 + + msgData.fromUserName.length + 1 + + msgData.subject.length + 1 + + msgData.message.length + 1; + + // + // Convert null terminated arrays to strings + // + let convMsgData = {}; + [ 'modDateTime', 'toUserName', 'fromUserName', 'subject' ].forEach(k => { + convMsgData[k] = iconv.decode(msgData[k], 'CP437'); + }); - this.parsePacketMessages = function(messagesBuffer, iterator, cb) { - const NULL_TERM_BUFFER = new Buffer( [ 0 ] ); - - var count = 0; + // + // The message body itself is a special beast as it may + // contain an origin line, kludges, SAUCE in the case + // of ANSI files, etc. + // + let msg = new Message( { + toUserName : convMsgData.toUserName, + fromUserName : convMsgData.fromUserName, + subject : convMsgData.subject, + modTimestamp : ftn.getDateFromFtnDateTime(convMsgData.modDateTime), + }); + + msg.meta.FtnProperty = {}; + msg.meta.FtnProperty.ftn_orig_node = msgData.ftn_orig_node; + msg.meta.FtnProperty.ftn_dest_node = msgData.ftn_dest_node; + msg.meta.FtnProperty.ftn_orig_network = msgData.ftn_orig_network; + msg.meta.FtnProperty.ftn_dest_network = msgData.ftn_dest_network; + msg.meta.FtnProperty.ftn_attr_flags = msgData.ftn_attr_flags; + msg.meta.FtnProperty.ftn_cost = msgData.ftn_cost; - binary.stream(messagesBuffer).loop(function looper(end, vars) { - // - // Some variable names used here match up directly with well known - // meta data names used with FTN messages. - // - this - .word16lu('messageType') - .word16lu('ftn_orig_node') - .word16lu('ftn_dest_node') - .word16lu('ftn_orig_network') - .word16lu('ftn_dest_network') - .word16lu('ftn_attr_flags') - .word16lu('ftn_cost') - .scan('modDateTime', NULL_TERM_BUFFER) // :TODO: 20 bytes max - .scan('toUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max - .scan('fromUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max - .scan('subject', NULL_TERM_BUFFER) // :TODO: 72 bytes max - .scan('message', NULL_TERM_BUFFER) - .tap(function tapped(msgData) { - if(!msgData.messageType) { - // end marker -- no more messages - end(); - return; - } - - if(FTN_PACKET_MESSAGE_TYPE != msgData.messageType) { - end(); - // :TODO: This is probably a bug if we hit a bad message after at leats one iterate - cb(new Error('Unsupported message type: ' + msgData.messageType)); - return; + self.processMessageBody(msgData.message, messageBodyData => { + msg.message = messageBodyData.message; + msg.meta.FtnKludge = messageBodyData.kludgeLines; + + if(messageBodyData.tearLine) { + msg.meta.FtnProperty.ftn_tear_line = messageBodyData.tearLine; } - ++count; - - // - // Convert null terminated arrays to strings - // - let convMsgData = {}; - [ 'modDateTime', 'toUserName', 'fromUserName', 'subject' ].forEach(k => { - convMsgData[k] = iconv.decode(msgData[k], 'CP437'); - }); - - // - // The message body itself is a special beast as it may - // contain special origin lines, kludges, SAUCE in the case - // of ANSI files, etc. - // - let msg = new Message( { - toUserName : convMsgData.toUserName, - fromUserName : convMsgData.fromUserName, - subject : convMsgData.subject, - modTimestamp : ftn.getDateFromFtnDateTime(convMsgData.modDateTime), - }); - - msg.meta.FtnProperty = {}; - msg.meta.FtnProperty.ftn_orig_node = msgData.ftn_orig_node; - msg.meta.FtnProperty.ftn_dest_node = msgData.ftn_dest_node; - msg.meta.FtnProperty.ftn_orig_network = msgData.ftn_orig_network; - msg.meta.FtnProperty.ftn_dest_network = msgData.ftn_dest_network; - msg.meta.FtnProperty.ftn_attr_flags = msgData.ftn_attr_flags; - msg.meta.FtnProperty.ftn_cost = msgData.ftn_cost; - - self.processMessageBody(msgData.message, function processed(messageBodyData) { - msg.message = messageBodyData.message; - msg.meta.FtnKludge = messageBodyData.kludgeLines; - - if(messageBodyData.tearLine) { - msg.meta.FtnProperty.ftn_tear_line = messageBodyData.tearLine; - } - if(messageBodyData.seenBy.length > 0) { - msg.meta.FtnProperty.ftn_seen_by = messageBodyData.seenBy; - } - if(messageBodyData.area) { - msg.meta.FtnProperty.ftn_area = messageBodyData.area; - } - if(messageBodyData.originLine) { - msg.meta.FtnProperty.ftn_origin = messageBodyData.originLine; - } - - // - // Update message UUID, if possible, based on MSGID and AREA - // - if(_.isString(msg.meta.FtnKludge.MSGID) && - _.isString(msg.meta.FtnProperty.ftn_area) && - msg.meta.FtnKludge.MSGID.length > 0 && - msg.meta.FtnProperty.ftn_area.length > 0) - { - msg.uuid = ftn.createMessageUuid( - msg.meta.FtnKludge.MSGID, - msg.meta.FtnProperty.area); - } - - iterator('message', msg); + if(messageBodyData.seenBy.length > 0) { + msg.meta.FtnProperty.ftn_seen_by = messageBodyData.seenBy; + } + + if(messageBodyData.area) { + msg.meta.FtnProperty.ftn_area = messageBodyData.area; + } + + if(messageBodyData.originLine) { + msg.meta.FtnProperty.ftn_origin = messageBodyData.originLine; + } + + const nextBuf = packetBuffer.slice(read); + if(nextBuf.length > 0) { + let next = function(e) { + if(e) { + cb(e); + } else { + self.parsePacketMessages(nextBuf, iterator, cb); + } + }; - --count; - if(0 === count) { - cb(null); - } - }) - }); - }); + iterator('message', msg, next); + } else { + cb(null); + } + }); + }); }; - + this.getMessageEntryBuffer = function(message, options) { let basicHeader = new Buffer(34); @@ -664,7 +654,7 @@ function Packet() { }); } } - + // // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 // AREA:CONFERENCE @@ -818,10 +808,15 @@ function Packet() { [ function processHeader(callback) { self.parsePacketHeader(packetBuffer, (err, header) => { - if(!err) { - iterator('header', header); + if(err) { + return callback(err); } - callback(err); + + let next = function(e) { + callback(e); + }; + + iterator('header', header, next); }); }, function processMessages(callback) { @@ -881,7 +876,7 @@ Packet.prototype.read = function(pathOrBuffer, iterator, cb) { }); } ], - function complete(err) { + err => { cb(err); } ); diff --git a/core/ftn_util.js b/core/ftn_util.js index 3a347b2b..595ffca7 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -4,6 +4,7 @@ let Config = require('./config.js').config; let Address = require('./ftn_address.js'); let FNV1a = require('./fnv1a.js'); +let createNamedUUID = require('./uuid_util.js').createNamedUUID; let _ = require('lodash'); let assert = require('assert'); @@ -22,6 +23,7 @@ let packageJson = require('../package.json'); exports.stringToNullPaddedBuffer = stringToNullPaddedBuffer; exports.getMessageSerialNumber = getMessageSerialNumber; exports.createMessageUuid = createMessageUuid; +exports.createMessageUuidAlternate = createMessageUuidAlternate; exports.getDateFromFtnDateTime = getDateFromFtnDateTime; exports.getDateTimeString = getDateTimeString; @@ -100,42 +102,43 @@ function getDateTimeString(m) { return m.format('DD MMM YY HH:mm:ss'); } +// +// Create a v5 named UUID given a message ID ("MSGID") and +// FTN area tag ("AREA"). +// +// This is similar to CrashMail +// See https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c +// function createMessageUuid(ftnMsgId, ftnArea) { - // - // v5 UUID generation code based on the work here: - // https://github.com/download13/uuidv5/blob/master/uuid.js - // - // Note: CrashMail uses MSGID + AREA, so we go with that as well: - // https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c - // - if(!Buffer.isBuffer(ftnMsgId)) { - ftnMsgId = iconv.encode(ftnMsgId, 'CP437'); - } + assert(_.isString(ftnMsgId)); + assert(_.isString(ftnArea)); - ftnArea = ftnArea || ''; // AREA is optional - if(!Buffer.isBuffer(ftnArea)) { - ftnArea = iconv.encode(ftnArea, 'CP437'); - } + ftnMsgId = iconv.encode(ftnMsgId, 'CP437'); + ftnArea = iconv.encode(ftnArea.toUpperCase(), 'CP437'); - const ns = new Buffer(ENIGMA_FTN_MSGID_NAMESPACE); + return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnMsgId, ftnArea ] ))); +}; - let digest = createHash('sha1').update( - Buffer.concat([ ns, ftnMsgId, ftnArea ])).digest(); - - let u = new Buffer(16); - - // bbbb - bb - bb - bb - bbbbbb - digest.copy(u, 0, 0, 4); // time_low - digest.copy(u, 4, 4, 6); // time_mid - digest.copy(u, 6, 6, 8); // time_hi_and_version - - u[6] = (u[6] & 0x0f) | 0x50; // version, 4 most significant bits are set to version 5 (0101) - u[8] = (digest[8] & 0x3f) | 0x80; // clock_seq_hi_and_reserved, 2msb are set to 10 - u[9] = digest[9]; +// +// Create a v5 named UUID given a FTN area tag ("AREA"), +// create/modified date, subject, and message body +// +// This method should be used as a backup for when a MSGID is +// not available in which createMessageUuid() above should be +// used instead. +// +function createMessageUuidAlternate(ftnArea, modTimestamp, subject, msgBody) { + assert(_.isString(ftnArea)); + assert(_.isDate(modTimestamp) || moment.isMoment(modTimestamp)); + assert(_.isString(subject)); + assert(_.isString(msgBody)); + + ftnArea = iconv.encode(ftnArea.toUpperCase(), 'CP437'); + modTimestamp = iconv.encode(getDateTimeString(modTimestamp), 'CP437'); + subject = iconv.encode(subject.toUpperCase().trim(), 'CP437'); + msgBody = iconv.encode(msgBody.replace(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g, '').trim(), 'CP437'); - digest.copy(u, 10, 10, 16); - - return uuid.unparse(u); // to string + return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnArea, modTimestamp, subject, msgBody ] ))); } function getMessageSerialNumber(messageId) { @@ -274,6 +277,9 @@ function getAbbreviatedNetNodeList(netNodes) { let abbrList = ''; let currNet; netNodes.forEach(netNode => { + if(_.isString(netNode)) { + netNode = Address.fromString(netNode); + } if(currNet !== netNode.net) { abbrList += `${netNode.net}/`; currNet = netNode.net; @@ -284,29 +290,24 @@ function getAbbreviatedNetNodeList(netNodes) { return abbrList.trim(); // remove trailing space } +// +// Parse an abbreviated net/node list commonly used for SEEN-BY and PATH +// function parseAbbreviatedNetNodeList(netNodes) { - // - // Make sure we have an array of objects. - // Allow for a single object or string(s) - // - if(!_.isArray(netNodes)) { - if(_.isString(netNodes)) { - netNodes = netNodes.split(' '); - } else { - netNodes = [ netNodes ]; - } - } - - // - // Convert any strings to parsed address objects - // - return netNodes.map(a => { - if(_.isObject(a)) { - return a; - } else { - return Address.fromString(a); - } - }); + const re = /([0-9]+)\/([0-9]+)\s?|([0-9]+)\s?/g; + let net; + let m; + let results = []; + while(null !== (m = re.exec(netNodes))) { + if(m[1] && m[2]) { + net = parseInt(m[1]); + results.push(new Address( { net : net, node : parseInt(m[2]) } )); + } else if(net) { + results.push(new Address( { net : net, node : parseInt(m[3]) } )); + } + } + + return results; } // @@ -348,8 +349,12 @@ function getUpdatedSeenByEntries(existingEntries, additions) { if(!_.isArray(existingEntries)) { existingEntries = [ existingEntries ]; } + + if(!_.isString(additions)) { + additions = parseAbbreviatedNetNodeList(getAbbreviatedNetNodeList(additions)); + } - additions = parseAbbreviatedNetNodeList(additions).sort(Address.getComparator()); + additions = additions.sort(Address.getComparator()); // // For now, we'll just append a new SEEN-BY entry diff --git a/core/message.js b/core/message.js index 1a5ddd3b..30b72d7c 100644 --- a/core/message.js +++ b/core/message.js @@ -62,22 +62,6 @@ function Message(options) { ts = ts || new Date(); return ts.toISOString(); }; - - /* - Object.defineProperty(this, 'messageId', { - get : function() { - return messageId; - } - }); - - Object.defineProperty(this, 'areaId', { - get : function() { return areaId; }, - set : function(i) { - areaId = i; - } - }); - - */ } Message.WellKnownAreaTags = { @@ -115,9 +99,7 @@ Message.FtnPropertyNames = { FtnDestZone : 'ftn_dest_zone', FtnOrigPoint : 'ftn_orig_point', FtnDestPoint : 'ftn_dest_point', - - - + FtnAttribute : 'ftn_attribute', FtnTearLine : 'ftn_tear_line', // http://ftsc.org/docs/fts-0004.001 @@ -136,6 +118,118 @@ Message.prototype.setLocalFromUserId = function(userId) { this.meta.System.local_from_user_id = userId; }; +Message.getMessageIdByUuid = function(uuid, cb) { + msgDb.get( + `SELECT message_id + FROM message + WHERE message_uuid = ? + LIMIT 1;`, + [ uuid ], + (err, row) => { + if(err) { + cb(err); + } else { + const success = (row && row.message_id); + cb(success ? null : new Error('No match'), success ? row.message_id : null); + } + } + ); +}; + +Message.getMessageIdsByMetaValue = function(category, name, value, cb) { + msgDb.all( + `SELECT message_id + FROM message_meta + WHERE meta_category = ? AND meta_name = ? AND meta_value = ?;`, + [ category, name, value ], + (err, rows) => { + if(err) { + cb(err); + } else { + cb(null, rows.map(r => parseInt(r.message_id))); // return array of ID(s) + } + } + ); +}; + +Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name, cb) { + async.waterfall( + [ + function getMessageId(callback) { + Message.getMessageIdByUuid(uuid, (err, messageId) => { + callback(err, messageId); + }); + }, + function getMetaValue(messageId, callback) { + const sql = + `SELECT meta_value + FROM message_meta + WHERE message_id = ? AND message_category = ? AND meta_name = ?;`; + + msgDb.all(sql, [ messageId, category, name ], (err, rows) => { + if(err) { + return callback(err); + } + + if(0 === rows.length) { + return callback(new Error('No value for category/name')); + } + + // single values are returned without an array + if(1 === rows.length) { + return callback(null, rows[0].meta_value); + } + + callback(null, rows.map(r => r.meta_value)); + }); + } + ], + (err, value) => { + cb(err, value); + } + ); +}; + +Message.prototype.loadMeta = function(cb) { + /* + Example of loaded this.meta: + + meta: { + System: { + local_to_user_id: 1234, + }, + FtnProperty: { + ftn_seen_by: [ "1/102 103", "2/42 52 65" ] + } + } + */ + + const sql = + `SELECT meta_category, meta_name, meta_value + FROM message_meta + WHERE message_id = ?;`; + + let self = this; + msgDb.each(sql, [ this.messageId ], (err, row) => { + if(!(row.meta_category in self.meta)) { + self.meta[row.meta_category] = { }; + self.meta[row.meta_category][row.meta_name] = row.meta_value; + } else { + if(!(row.meta_name in self.meta[row.meta_category])) { + self.meta[row.meta_category][row.meta_name] = row.meta_value; + } else { + if(_.isString(self.meta[row.meta_category][row.meta_name])) { + self.meta[row.meta_category][row.meta_name] = [ self.meta[row.meta_category][row.meta_name] ]; + } + + self.meta[row.meta_category][row.meta_name].push(row.meta_value); + } + } + }, err => { + cb(err); + }); +}; + Message.prototype.load = function(options, cb) { assert(_.isString(options.uuid)); @@ -168,8 +262,9 @@ Message.prototype.load = function(options, cb) { ); }, function loadMessageMeta(callback) { - // :TODO: - callback(null); + self.loadMeta(err => { + callback(err); + }); }, function loadHashTags(callback) { // :TODO: @@ -188,27 +283,59 @@ Message.prototype.load = function(options, cb) { ); }; +Message.prototype.persistMetaValue = function(category, name, value, cb) { + const metaStmt = msgDb.prepare( + `INSERT INTO message_meta (message_id, meta_category, meta_name, meta_value) + VALUES (?, ?, ?, ?);`); + + if(!_.isArray(value)) { + value = [ value ]; + } + + let self = this; + + async.each(value, (v, next) => { + metaStmt.run(self.messageId, category, name, v, err => { + next(err); + }); + }, err => { + cb(err); + }); +}; + +Message.startTransaction = function(cb) { + msgDb.run('BEGIN;', err => { + cb(err); + }); +}; + +Message.endTransaction = function(hadError, cb) { + msgDb.run(hadError ? 'ROLLBACK;' : 'COMMIT;', err => { + cb(err); + }); +}; + Message.prototype.persist = function(cb) { if(!this.isValid()) { - cb(new Error('Cannot persist invalid message!')); - return; + return cb(new Error('Cannot persist invalid message!')); } - var self = this; - + let self = this; + async.series( [ function beginTransaction(callback) { - msgDb.run('BEGIN;', function transBegin(err) { + Message.startTransaction(err => { callback(err); }); }, function storeMessage(callback) { msgDb.run( - 'INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) ' + - 'VALUES (?, ?, ?, ?, ?, ?, ?, ?);', [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ], - function msgInsert(err) { + `INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) + VALUES (?, ?, ?, ?, ?, ?, ?, ?);`, + [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ], + function inserted(err) { // use for this scope if(!err) { self.messageId = this.lastID; } @@ -221,26 +348,30 @@ Message.prototype.persist = function(cb) { if(!self.meta) { callback(null); } else { - // :TODO: this should be it's own method such that meta can be updated - var metaStmt = msgDb.prepare( - 'INSERT INTO message_meta (message_id, meta_category, meta_name, meta_value) ' + - 'VALUES (?, ?, ?, ?);'); - - for(var metaCategroy in self.meta) { - async.each(Object.keys(self.meta[metaCategroy]), function meta(metaName, next) { - metaStmt.run(self.messageId, Message.MetaCategories[metaCategroy], metaName, self.meta[metaCategroy][metaName], function inserted(err) { - next(err); - }); - }, function complete(err) { - if(!err) { - metaStmt.finalize(function finalized() { - callback(null); - }); - } else { - callback(err); + /* + Example of self.meta: + + meta: { + System: { + local_to_user_id: 1234, + }, + FtnProperty: { + ftn_seen_by: [ "1/102 103", "2/42 52 65" ] } + } + */ + async.each(Object.keys(self.meta), (category, nextCat) => { + async.each(Object.keys(self.meta[category]), (name, nextName) => { + self.persistMetaValue(category, name, self.meta[category][name], err => { + nextName(err); + }); + }, err => { + nextCat(err); }); - } + + }, err => { + callback(err); + }); } }, function storeHashTags(callback) { @@ -248,9 +379,9 @@ Message.prototype.persist = function(cb) { callback(null); } ], - function complete(err) { - msgDb.run(err ? 'ROLLBACK;' : 'COMMIT;', function transEnd(err) { - cb(err, self.messageId); + err => { + Message.endTransaction(err, transErr => { + cb(err ? err : transErr, self.messageId); }); } ); diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 41a87f93..8a83526c 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -20,6 +20,7 @@ let async = require('async'); let fs = require('fs'); let later = require('later'); let temp = require('temp').track(); // track() cleans up temp dir/files for us +let assert = require('assert'); exports.moduleInfo = { name : 'FTN BSO', @@ -72,13 +73,44 @@ function FTNMessageScanTossModule() { return(networkName === defaultNetworkName && address.zone === this.moduleConfig.defaultZone); }; - this.getNetworkNameByAddress = function(address) { + this.getNetworkNameByAddress = function(remoteAddress) { return _.findKey(Config.messageNetworks.ftn.networks, network => { - const networkAddress = Address.fromString(network.localAddress); - return !_.isUndefined(networkAddress) && address.isEqual(networkAddress); + const localAddress = Address.fromString(network.localAddress); + return !_.isUndefined(localAddress) && localAddress.isEqual(remoteAddress); }); }; + this.getNetworkNameByAddressPattern = function(remoteAddressPattern) { + return _.findKey(Config.messageNetworks.ftn.networks, network => { + const localAddress = Address.fromString(network.localAddress); + return !_.isUndefined(localAddress) && localAddress.isPatternMatch(remoteAddressPattern); + }); + }; + + this.getLocalAreaTagByFtnAreaTag = function(ftnAreaTag) { + return _.findKey(Config.messageNetworks.ftn.areas, areaConf => { + return areaConf.tag === ftnAreaTag; + }); + }; + + /* + this.getSeenByAddresses = function(messageSeenBy) { + if(!_.isArray(messageSeenBy)) { + messageSeenBy = [ messageSeenBy ]; + } + + let seenByAddrs = []; + messageSeenBy.forEach(sb => { + seenByAddrs = seenByAddrs.concat(ftnUtil.parseAbbreviatedNetNodeList(sb)); + }); + return seenByAddrs; + }; + */ + + this.messageHasValidMSGID = function(msg) { + return _.isString(msg.meta.FtnKludge.MSGID) && msg.meta.FtnKludge.MSGID.length > 0; + }; + this.getOutgoingPacketDir = function(networkName, destAddress) { let dir = this.moduleConfig.paths.outbound; if(!this.isDefaultDomainZone(networkName, destAddress)) { @@ -164,41 +196,6 @@ function FTNMessageScanTossModule() { }); }; - this.exportMessage = function(message, options, cb) { - this.prepareMessage(message, options); - - let packet = new ftnMailPacket.Packet(); - - let packetHeader = new ftnMailPacket.PacketHeader( - options.network.localAddress, - options.destAddress, - options.nodeConfig.packetType); - - packetHeader.password = options.nodeConfig.packetPassword || ''; - - if(message.isPrivate()) { - // :TODO: this should actually be checking for isNetMail()!! - } else { - const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.destAddress); - - mkdirp(outgoingDir, err => { - if(err) { - return cb(err); - } - this.getOutgoingBundleFileName(outgoingDir, options.network.localAddress, options.destAddress, (err, path) => { - console.log(path); - }); - packet.write( - this.getOutgoingPacketFileName(outgoingDir, message), - packetHeader, - [ message ], - { encoding : options.encoding } - ); - }); - } - - }; - this.prepareMessage = function(message, options) { // // Set various FTN kludges/etc. @@ -241,7 +238,8 @@ function FTNMessageScanTossModule() { // When exporting messages, we should create/update SEEN-BY // with remote address(s) we are exporting to. // - const seenByAdditions = [ options.network.localAddress ].concat(Config.messageNetworks.ftn.areas[message.areaTag].uplinks); + const seenByAdditions = + [ `${options.network.localAddress.net}/${options.network.localAddress.node}` ].concat(Config.messageNetworks.ftn.areas[message.areaTag].uplinks); message.meta.FtnProperty.ftn_seen_by = ftnUtil.getUpdatedSeenByEntries(message.meta.FtnProperty.ftn_seen_by, seenByAdditions); @@ -256,8 +254,14 @@ function FTNMessageScanTossModule() { // // Additional kludges + // + // Check for existence of MSGID as we may already have stored it from a previous + // export that failed to finish // - message.meta.FtnKludge.MSGID = ftnUtil.getMessageIdentifier(message, options.network.localAddress); + if(!message.meta.FtnKludge.MSGID) { + message.meta.FtnKludge.MSGID = ftnUtil.getMessageIdentifier(message, options.network.localAddress); + } + message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset(); if(!message.meta.FtnKludge.PID) { @@ -369,7 +373,7 @@ function FTNMessageScanTossModule() { this.getNodeConfigKeyForUplink = function(uplink) { // :TODO: sort by least # of '*' & take top? const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => { - return Address.fromString(addr).isMatch(uplink); + return Address.fromString(addr).isPatternMatch(uplink); })[0]; return nodeKey; @@ -451,6 +455,19 @@ function FTNMessageScanTossModule() { ws.write(msgBuf); } callback(null); + }, + function updateStoredMeta(callback) { + // + // We want to store some meta as if we had imported + // this message for later reference + // + if(message.meta.FtnKludge.MSGID) { + message.persistMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.MSGID, err => { + callback(err); + }); + } else { + callback(null); + } } ], err => { @@ -607,37 +624,144 @@ function FTNMessageScanTossModule() { }, cb); // complete }; - this.importMessagesFromPacketFile = function(packetPath, cb) { - const packet = new ftnMailPacket.Packet(); + this.setReplyToMsgIdFtnReplyKludge = function(message, cb) { + // + // Given a FTN REPLY kludge, set |message.replyToMsgId|, if possible, + // by looking up an associated MSGID kludge meta. + // + // See also: http://ftsc.org/docs/fts-0009.001 + // + if(!_.isString(message.meta.FtnKludge.REPLY)) { + // nothing to do + return cb(); + } - // :TODO: packet.read() should have a way to cancel iteration... - let localNetworkName; - packet.read(packetPath, (entryType, entryData) => { + Message.getMessageIdsByMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.REPLY, (err, msgIds) => { + if(!err) { + assert(1 === msgIds.length); + message.replyToMsgId = msgIds[0]; + } + cb(); + }); + }; + + this.importNetMailToArea = function(localAreaTag, header, message, cb) { + async.series( + [ + function validateDestinationAddress(callback) { + /* + const messageDestAddress = new Address({ + node : message.meta.FtnProperty.ftn_dest_node, + net : message.meta.FtnProperty.ftn_dest_network, + }); + */ + + const localNetworkPattern = `${message.meta.FtnProperty.ftn_dest_network}/${message.meta.FtnProperty.ftn_dest_node}`; + + const localNetworkName = self.getNetworkNameByAddressPattern(localNetworkPattern); + + callback(_.isString(localNetworkName) ? null : new Error('Packet destination is not us')); + }, + function basicSetup(callback) { + message.areaTag = localAreaTag; + + // + // If duplicates are NOT allowed in the area (the default), we need to update + // the message UUID using data available to us. Duplicate UUIDs are internally + // not allowed in our local database. + // + if(!Config.messageNetworks.ftn.areas[localAreaTag].allowDupes) { + if(self.messageHasValidMSGID(message)) { + // Update UUID with our preferred generation method + message.uuid = ftnUtil.createMessageUuid( + message.meta.FtnKludge.MSGID, + message.meta.FtnProperty.ftn_area); + } else { + // Update UUID with alternate/backup generation method + message.uuid = ftnUtil.createMessageUuidAlternate( + message.meta.FtnProperty.ftn_area, + message.modTimestamp, + message.subject, + message.message); + } + } + + callback(null); + }, + function setReplyToMessageId(callback) { + self.setReplyToMsgIdFtnReplyKludge(message, () => { + callback(null); + }); + }, + function persistImport(callback) { + message.persist(err => { + callback(err); + }); + } + ], err => { + cb(err); + } + ); + }; + + // + // Ref. implementations on import: + // * https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/pkt.c + // https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/handle.c + // + this.importMessagesFromPacketFile = function(packetPath, cb) { + let packetHeader; + + new ftnMailPacket.Packet().read(packetPath, (entryType, entryData, next) => { if('header' === entryType) { - // - // Discover if this packet is for one of our network(s) - // - localNetworkName = self.getNetworkNameByAddress(entryData.destAddress); - - } else if(localNetworkName && 'message' === entryType) { - const message = entryData; // so we ref something reasonable :) + packetHeader = entryData; + + const localNetworkName = self.getNetworkNameByAddress(packetHeader.destAddress); + if(!_.isString(localNetworkName)) { + next(new Error('No configuration for this packet')); + } else { + next(null); + } + + } else if('message' === entryType) { + const message = entryData; const areaTag = message.meta.FtnProperty.ftn_area; - // :TODO: we need to know if this message is a dupe - UUID will be the same if MSGID, but if not present... what to do? - // :TODO: lookup and set message.areaTag if match - // :TODO: check SEEN-BY for echo - // :TODO: Handle area vs Netmail - Via, etc. - // :TODO: Handle PATH - // :TODO: handle REPLY kludges... set local ID when possible if(areaTag) { // - // Find local area tag + // EchoMail + // + const localAreaTag = self.getLocalAreaTagByFtnAreaTag(areaTag); + if(localAreaTag) { + self.importNetMailToArea(localAreaTag, packetHeader, message, err => { + if(err) { + if('SQLITE_CONSTRAINT' === err.code) { + Log.info( + { subject : message.subject, uuid : message.uuid }, + 'Not importing non-unique message'); + + return next(null); + } + } + + next(err); + }); + } else { + // + // No local area configured for this import + // + // :TODO: Handle the "catch all" case, if configured + } + } else { + // + // NetMail // } } }, err => { cb(err); - }); }; + }); + }; this.importPacketFilesFromDirectory = function(importDir, cb) { async.waterfall( @@ -672,9 +796,9 @@ function FTNMessageScanTossModule() { // :TODO: rename to .bad, perhaps move to a rejects dir + log nextFile(); } else { - //fs.unlink(fullPath, err => { + fs.unlink(fullPath, err => { nextFile(); - //}); + }); } }, err => { callback(err); @@ -687,7 +811,9 @@ function FTNMessageScanTossModule() { ); }; - this.importMessagesFromDirectory = function(importDir, cb) { + this.importMessagesFromDirectory = function(inboundType, importDir, cb) { + let tempDirectory; + async.waterfall( [ // start with .pkt files @@ -712,21 +838,37 @@ function FTNMessageScanTossModule() { }, function createTempDir(bundleFiles, callback) { temp.mkdir('enigftnimport-', (err, tempDir) => { - callback(err, bundleFiles, tempDir); + tempDirectory = tempDir; + callback(err, bundleFiles); }); }, - function importBundles(bundleFiles, tempDir, callback) { + function importBundles(bundleFiles, callback) { + let rejects = []; + async.each(bundleFiles, (bundleFile, nextFile) => { if(_.isUndefined(bundleFile.archName)) { - // :TODO: log? + Log.info( + { fileName : bundleFile.path }, + 'Unknown bundle archive type'); + + rejects.push(bundleFile.path); + return nextFile(); // unknown archive type } self.archUtil.extractTo( bundleFile.path, - tempDir, + tempDirectory, bundleFile.archName, err => { + if(err) { + Log.info( + { fileName : bundleFile.path, error : err.toString() }, + 'Failed to extract bundle'); + + rejects.push(bundleFile.path); + } + nextFile(); } ); @@ -738,14 +880,39 @@ function FTNMessageScanTossModule() { // // All extracted - import .pkt's // - self.importPacketFilesFromDirectory(tempDir, err => { - callback(err); + self.importPacketFilesFromDirectory(tempDirectory, err => { + callback(null, bundleFiles, rejects); }); }); + }, + function handleProcessedBundleFiles(bundleFiles, rejects, callback) { + async.each(bundleFiles, (bundleFile, nextFile) => { + if(rejects.indexOf(bundleFile.path) > -1) { + // :TODO: rename to .bad, perhaps move to a rejects dir + log + nextFile(); + } else { + fs.unlink(bundleFile.path, err => { + nextFile(); + }); + } + }, err => { + callback(err); + }); } ], err => { - cb(err); + if(tempDirectory) { + temp.cleanup( (errIgnored, stats) => { + Log.trace( + Object.assign(stats, { tempDir : tempDirectory } ), + 'Temporary directory cleaned up' + ); + + cb(err); // orig err + }); + } else { + cb(err); + } } ); }; @@ -818,8 +985,8 @@ FTNMessageScanTossModule.prototype.performImport = function(cb) { var self = this; - async.each( [ 'inbound', 'secInbound' ], (importDir, nextDir) => { - self.importMessagesFromDirectory(self.moduleConfig.paths[importDir], err => { + async.each( [ 'inbound', 'secInbound' ], (inboundType, nextDir) => { + self.importMessagesFromDirectory(inboundType, self.moduleConfig.paths[inboundType], err => { nextDir(); }); @@ -879,7 +1046,7 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { const newLastScanId = msgRows[msgRows.length - 1].message_id; Log.info( - { messagesExported : msgRows.length, newLastScanId : newLastScanId }, + { areaTag : areaTag, messagesExported : msgRows.length, newLastScanId : newLastScanId }, 'Export complete'); callback(err, newLastScanId); diff --git a/core/uuid_util.js b/core/uuid_util.js new file mode 100644 index 00000000..00e8840c --- /dev/null +++ b/core/uuid_util.js @@ -0,0 +1,41 @@ +/* jslint node: true */ +'use strict'; + +let uuid = require('node-uuid'); +let assert = require('assert'); +let _ = require('lodash'); +let createHash = require('crypto').createHash; + +exports.createNamedUUID = createNamedUUID; + +function createNamedUUID(namespaceUuid, key) { + // + // v5 UUID generation code based on the work here: + // https://github.com/download13/uuidv5/blob/master/uuid.js + // + if(!Buffer.isBuffer(namespaceUuid)) { + namespaceUuid = new Buffer(namespaceUuid); + } + + if(!Buffer.isBuffer(key)) { + key = new Buffer(key); + } + + let digest = createHash('sha1').update( + Buffer.concat( [ namespaceUuid, key ] )).digest(); + + let u = new Buffer(16); + + // bbbb - bb - bb - bb - bbbbbb + digest.copy(u, 0, 0, 4); // time_low + digest.copy(u, 4, 4, 6); // time_mid + digest.copy(u, 6, 6, 8); // time_hi_and_version + + u[6] = (u[6] & 0x0f) | 0x50; // version, 4 most significant bits are set to version 5 (0101) + u[8] = (digest[8] & 0x3f) | 0x80; // clock_seq_hi_and_reserved, 2msb are set to 10 + u[9] = digest[9]; + + digest.copy(u, 10, 10, 16); + + return u; +} \ No newline at end of file