moon
/
enigma-bbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Change FTN packet read() to use async iterator 

* createMessageUuidAlternate(): Mmethod for FTN message v5 UUID generation when no MSGID to work with
* parseAbbreviatedNetNodeList() now works properly
* Add core/uuid_util.js for various UUID utilities such as v5 named UUID generation
* Fix message meta load/retrieval
* Add lookup for REPLY kludge -> MSGID -> local reply IDs
* Fix SEEN-BY additions @ export
* Don't override MSGIDs if they already exist
* Store MSGID @ export so it can be inspected later
* Add import functionality (working, but WIP!)
* Clean up bundles and packets after import
This commit is contained in:
Bryan Ashby 2016-03-08 22:30:04 -07:00
parent 6094bed07f
commit ad0296addf
7 changed files with 628 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/database.js
View File

@ -175,7 +175,7 @@ function createMessageBaseTables() {
' meta_category INTEGER NOT NULL,' +
' meta_name VARCHAR NOT NULL,' +
' meta_value VARCHAR NOT NULL,' +
' UNIQUE(message_id, meta_category, meta_name, meta_value),' +
' UNIQUE(message_id, meta_category, meta_name, meta_value),' + // why unique here?
' FOREIGN KEY(message_id) REFERENCES message(message_id)' +
');'
);

2
core/ftn_address.js
View File

@ -107,7 +107,7 @@ module.exports = class Address {
}
*/
isMatch(pattern) {
isPatternMatch(pattern) {
const addr = this.getMatchAddr(pattern);
if(addr) {
return (

81
core/ftn_mail_packet.js
View File

@ -32,6 +32,7 @@ const FTN_PACKET_HEADER_SIZE = 58; // fixed header size
const FTN_PACKET_HEADER_TYPE = 2;
const FTN_PACKET_MESSAGE_TYPE = 2;
const FTN_PACKET_BAUD_TYPE_2_2 = 2;
const NULL_TERM_BUFFER = new Buffer( [ 0x00 ] );
// SAUCE magic header + version ("00")
const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00');
@ -171,7 +172,6 @@ exports.PacketHeader = PacketHeader;
// http://walon.org/pub/fidonet/FTSC-nodelists-etc./pkt-types.txt
//
function Packet() {
var self = this;
this.parsePacketHeader = function(packetBuffer, cb) {
@ -510,17 +510,8 @@ function Packet() {
);
};
this.parsePacketMessages = function(messagesBuffer, iterator, cb) {
const NULL_TERM_BUFFER = new Buffer( [ 0 ] );
var count = 0;
binary.stream(messagesBuffer).loop(function looper(end, vars) {
//
// Some variable names used here match up directly with well known
// meta data names used with FTN messages.
//
this
this.parsePacketMessages = function(packetBuffer, iterator, cb) {
binary.parse(packetBuffer)
.word16lu('messageType')
.word16lu('ftn_orig_node')
.word16lu('ftn_dest_node')
@ -531,23 +522,25 @@ function Packet() {
.scan('modDateTime', NULL_TERM_BUFFER) // :TODO: 20 bytes max
.scan('toUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max
.scan('fromUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max
.scan('subject', NULL_TERM_BUFFER) // :TODO: 72 bytes max
.scan('subject', NULL_TERM_BUFFER) // :TODO: 72 bytes max6
.scan('message', NULL_TERM_BUFFER)
.tap(function tapped(msgData) {
.tap(function tapped(msgData) { // no arrow function; want classic this
if(!msgData.messageType) {
// end marker -- no more messages
end();
return;
return cb(null);
}
if(FTN_PACKET_MESSAGE_TYPE != msgData.messageType) {
end();
// :TODO: This is probably a bug if we hit a bad message after at leats one iterate
cb(new Error('Unsupported message type: ' + msgData.messageType));
return;
return cb(new Error('Unsupported message type: ' + msgData.messageType));
}
++count;
const read =
14 + // fixed header size
msgData.modDateTime.length + 1 +
msgData.toUserName.length + 1 +
msgData.fromUserName.length + 1 +
msgData.subject.length + 1 +
msgData.message.length + 1;
//
// Convert null terminated arrays to strings
@ -559,7 +552,7 @@ function Packet() {
//
// The message body itself is a special beast as it may
// contain special origin lines, kludges, SAUCE in the case
// contain an origin line, kludges, SAUCE in the case
// of ANSI files, etc.
//
let msg = new Message( {
@ -577,43 +570,40 @@ function Packet() {
msg.meta.FtnProperty.ftn_attr_flags = msgData.ftn_attr_flags;
msg.meta.FtnProperty.ftn_cost = msgData.ftn_cost;
self.processMessageBody(msgData.message, function processed(messageBodyData) {
self.processMessageBody(msgData.message, messageBodyData => {
msg.message = messageBodyData.message;
msg.meta.FtnKludge = messageBodyData.kludgeLines;
if(messageBodyData.tearLine) {
msg.meta.FtnProperty.ftn_tear_line = messageBodyData.tearLine;
}
if(messageBodyData.seenBy.length > 0) {
msg.meta.FtnProperty.ftn_seen_by = messageBodyData.seenBy;
}
if(messageBodyData.area) {
msg.meta.FtnProperty.ftn_area = messageBodyData.area;
}
if(messageBodyData.originLine) {
msg.meta.FtnProperty.ftn_origin = messageBodyData.originLine;
}
//
// Update message UUID, if possible, based on MSGID and AREA
//
if(_.isString(msg.meta.FtnKludge.MSGID) &&
_.isString(msg.meta.FtnProperty.ftn_area) &&
msg.meta.FtnKludge.MSGID.length > 0 &&
msg.meta.FtnProperty.ftn_area.length > 0)
{
msg.uuid = ftn.createMessageUuid(
msg.meta.FtnKludge.MSGID,
msg.meta.FtnProperty.area);
const nextBuf = packetBuffer.slice(read);
if(nextBuf.length > 0) {
let next = function(e) {
if(e) {
cb(e);
} else {
self.parsePacketMessages(nextBuf, iterator, cb);
}
};
iterator('message', msg);
--count;
if(0 === count) {
iterator('message', msg, next);
} else {
cb(null);
}
})
});
});
};
@ -818,10 +808,15 @@ function Packet() {
[
function processHeader(callback) {
self.parsePacketHeader(packetBuffer, (err, header) => {
if(!err) {
iterator('header', header);
if(err) {
return callback(err);
}
callback(err);
let next = function(e) {
callback(e);
};
iterator('header', header, next);
});
},
function processMessages(callback) {
@ -881,7 +876,7 @@ Packet.prototype.read = function(pathOrBuffer, iterator, cb) {
});
}
],
function complete(err) {
err => {
cb(err);
}
);

105
core/ftn_util.js
View File

@ -4,6 +4,7 @@
let Config = require('./config.js').config;
let Address = require('./ftn_address.js');
let FNV1a = require('./fnv1a.js');
let createNamedUUID = require('./uuid_util.js').createNamedUUID;
let _ = require('lodash');
let assert = require('assert');
@ -22,6 +23,7 @@ let packageJson = require('../package.json');
exports.stringToNullPaddedBuffer = stringToNullPaddedBuffer;
exports.getMessageSerialNumber = getMessageSerialNumber;
exports.createMessageUuid = createMessageUuid;
exports.createMessageUuidAlternate = createMessageUuidAlternate;
exports.getDateFromFtnDateTime = getDateFromFtnDateTime;
exports.getDateTimeString = getDateTimeString;
@ -100,42 +102,43 @@ function getDateTimeString(m) {
return m.format('DD MMM YY HH:mm:ss');
}
//
// Create a v5 named UUID given a message ID ("MSGID") and
// FTN area tag ("AREA").
//
// This is similar to CrashMail
// See https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c
//
function createMessageUuid(ftnMsgId, ftnArea) {
//
// v5 UUID generation code based on the work here:
// https://github.com/download13/uuidv5/blob/master/uuid.js
//
// Note: CrashMail uses MSGID + AREA, so we go with that as well:
// https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c
//
if(!Buffer.isBuffer(ftnMsgId)) {
assert(_.isString(ftnMsgId));
assert(_.isString(ftnArea));
ftnMsgId = iconv.encode(ftnMsgId, 'CP437');
}
ftnArea = iconv.encode(ftnArea.toUpperCase(), 'CP437');
ftnArea = ftnArea || ''; // AREA is optional
if(!Buffer.isBuffer(ftnArea)) {
ftnArea = iconv.encode(ftnArea, 'CP437');
}
return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnMsgId, ftnArea ] )));
};
const ns = new Buffer(ENIGMA_FTN_MSGID_NAMESPACE);
//
// Create a v5 named UUID given a FTN area tag ("AREA"),
// create/modified date, subject, and message body
//
// This method should be used as a backup for when a MSGID is
// not available in which createMessageUuid() above should be
// used instead.
//
function createMessageUuidAlternate(ftnArea, modTimestamp, subject, msgBody) {
assert(_.isString(ftnArea));
assert(_.isDate(modTimestamp) || moment.isMoment(modTimestamp));
assert(_.isString(subject));
assert(_.isString(msgBody));
let digest = createHash('sha1').update(
Buffer.concat([ ns, ftnMsgId, ftnArea ])).digest();
ftnArea = iconv.encode(ftnArea.toUpperCase(), 'CP437');
modTimestamp = iconv.encode(getDateTimeString(modTimestamp), 'CP437');
subject = iconv.encode(subject.toUpperCase().trim(), 'CP437');
msgBody = iconv.encode(msgBody.replace(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g, '').trim(), 'CP437');
let u = new Buffer(16);
// bbbb - bb - bb - bb - bbbbbb
digest.copy(u, 0, 0, 4); // time_low
digest.copy(u, 4, 4, 6); // time_mid
digest.copy(u, 6, 6, 8); // time_hi_and_version
u[6] = (u[6] & 0x0f) | 0x50; // version, 4 most significant bits are set to version 5 (0101)
u[8] = (digest[8] & 0x3f) | 0x80; // clock_seq_hi_and_reserved, 2msb are set to 10
u[9] = digest[9];
digest.copy(u, 10, 10, 16);
return uuid.unparse(u); // to string
return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnArea, modTimestamp, subject, msgBody ] )));
}
function getMessageSerialNumber(messageId) {
@ -274,6 +277,9 @@ function getAbbreviatedNetNodeList(netNodes) {
let abbrList = '';
let currNet;
netNodes.forEach(netNode => {
if(_.isString(netNode)) {
netNode = Address.fromString(netNode);
}
if(currNet !== netNode.net) {
abbrList += `${netNode.net}/`;
currNet = netNode.net;
@ -284,29 +290,24 @@ function getAbbreviatedNetNodeList(netNodes) {
return abbrList.trim(); // remove trailing space
}
//
// Parse an abbreviated net/node list commonly used for SEEN-BY and PATH
//
function parseAbbreviatedNetNodeList(netNodes) {
//
// Make sure we have an array of objects.
// Allow for a single object or string(s)
//
if(!_.isArray(netNodes)) {
if(_.isString(netNodes)) {
netNodes = netNodes.split(' ');
} else {
netNodes = [ netNodes ];
const re = /([0-9]+)\/([0-9]+)\s?|([0-9]+)\s?/g;
let net;
let m;
let results = [];
while(null !== (m = re.exec(netNodes))) {
if(m[1] && m[2]) {
net = parseInt(m[1]);
results.push(new Address( { net : net, node : parseInt(m[2]) } ));
} else if(net) {
results.push(new Address( { net : net, node : parseInt(m[3]) } ));
}
}
//
// Convert any strings to parsed address objects
//
return netNodes.map(a => {
if(_.isObject(a)) {
return a;
} else {
return Address.fromString(a);
}
});
return results;
}
//
@ -349,7 +350,11 @@ function getUpdatedSeenByEntries(existingEntries, additions) {
existingEntries = [ existingEntries ];
}
additions = parseAbbreviatedNetNodeList(additions).sort(Address.getComparator());
if(!_.isString(additions)) {
additions = parseAbbreviatedNetNodeList(getAbbreviatedNetNodeList(additions));
}
additions = additions.sort(Address.getComparator());
//
// For now, we'll just append a new SEEN-BY entry

221
core/message.js
View File

@ -62,22 +62,6 @@ function Message(options) {
ts = ts || new Date();
return ts.toISOString();
};
/*
Object.defineProperty(this, 'messageId', {
get : function() {
return messageId;
}
});
Object.defineProperty(this, 'areaId', {
get : function() { return areaId; },
set : function(i) {
areaId = i;
}
});
*/
}
Message.WellKnownAreaTags = {
@ -116,8 +100,6 @@ Message.FtnPropertyNames = {
FtnOrigPoint : 'ftn_orig_point',
FtnDestPoint : 'ftn_dest_point',
FtnAttribute : 'ftn_attribute',
FtnTearLine : 'ftn_tear_line', // http://ftsc.org/docs/fts-0004.001
@ -136,6 +118,118 @@ Message.prototype.setLocalFromUserId = function(userId) {
this.meta.System.local_from_user_id = userId;
};
Message.getMessageIdByUuid = function(uuid, cb) {
msgDb.get(
`SELECT message_id
FROM message
WHERE message_uuid = ?
LIMIT 1;`,
[ uuid ],
(err, row) => {
if(err) {
cb(err);
} else {
const success = (row && row.message_id);
cb(success ? null : new Error('No match'), success ? row.message_id : null);
}
}
);
};
Message.getMessageIdsByMetaValue = function(category, name, value, cb) {
msgDb.all(
`SELECT message_id
FROM message_meta
WHERE meta_category = ? AND meta_name = ? AND meta_value = ?;`,
[ category, name, value ],
(err, rows) => {
if(err) {
cb(err);
} else {
cb(null, rows.map(r => parseInt(r.message_id))); // return array of ID(s)
}
}
);
};
Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name, cb) {
async.waterfall(
[
function getMessageId(callback) {
Message.getMessageIdByUuid(uuid, (err, messageId) => {
callback(err, messageId);
});
},
function getMetaValue(messageId, callback) {
const sql =
`SELECT meta_value
FROM message_meta
WHERE message_id = ? AND message_category = ? AND meta_name = ?;`;
msgDb.all(sql, [ messageId, category, name ], (err, rows) => {
if(err) {
return callback(err);
}
if(0 === rows.length) {
return callback(new Error('No value for category/name'));
}
// single values are returned without an array
if(1 === rows.length) {
return callback(null, rows[0].meta_value);
}
callback(null, rows.map(r => r.meta_value));
});
}
],
(err, value) => {
cb(err, value);
}
);
};
Message.prototype.loadMeta = function(cb) {
/*
Example of loaded this.meta:
meta: {
System: {
local_to_user_id: 1234,
},
FtnProperty: {
ftn_seen_by: [ "1/102 103", "2/42 52 65" ]
}
}
*/
const sql =
`SELECT meta_category, meta_name, meta_value
FROM message_meta
WHERE message_id = ?;`;
let self = this;
msgDb.each(sql, [ this.messageId ], (err, row) => {
if(!(row.meta_category in self.meta)) {
self.meta[row.meta_category] = { };
self.meta[row.meta_category][row.meta_name] = row.meta_value;
} else {
if(!(row.meta_name in self.meta[row.meta_category])) {
self.meta[row.meta_category][row.meta_name] = row.meta_value;
} else {
if(_.isString(self.meta[row.meta_category][row.meta_name])) {
self.meta[row.meta_category][row.meta_name] = [ self.meta[row.meta_category][row.meta_name] ];
}
self.meta[row.meta_category][row.meta_name].push(row.meta_value);
}
}
}, err => {
cb(err);
});
};
Message.prototype.load = function(options, cb) {
assert(_.isString(options.uuid));
@ -168,8 +262,9 @@ Message.prototype.load = function(options, cb) {
);
},
function loadMessageMeta(callback) {
// :TODO:
callback(null);
self.loadMeta(err => {
callback(err);
});
},
function loadHashTags(callback) {
// :TODO:
@ -188,27 +283,59 @@ Message.prototype.load = function(options, cb) {
);
};
Message.prototype.persistMetaValue = function(category, name, value, cb) {
const metaStmt = msgDb.prepare(
`INSERT INTO message_meta (message_id, meta_category, meta_name, meta_value)
VALUES (?, ?, ?, ?);`);
if(!_.isArray(value)) {
value = [ value ];
}
let self = this;
async.each(value, (v, next) => {
metaStmt.run(self.messageId, category, name, v, err => {
next(err);
});
}, err => {
cb(err);
});
};
Message.startTransaction = function(cb) {
msgDb.run('BEGIN;', err => {
cb(err);
});
};
Message.endTransaction = function(hadError, cb) {
msgDb.run(hadError ? 'ROLLBACK;' : 'COMMIT;', err => {
cb(err);
});
};
Message.prototype.persist = function(cb) {
if(!this.isValid()) {
cb(new Error('Cannot persist invalid message!'));
return;
return cb(new Error('Cannot persist invalid message!'));
}
var self = this;
let self = this;
async.series(
[
function beginTransaction(callback) {
msgDb.run('BEGIN;', function transBegin(err) {
Message.startTransaction(err => {
callback(err);
});
},
function storeMessage(callback) {
msgDb.run(
'INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) ' +
'VALUES (?, ?, ?, ?, ?, ?, ?, ?);', [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ],
function msgInsert(err) {
`INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp)
VALUES (?, ?, ?, ?, ?, ?, ?, ?);`,
[ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ],
function inserted(err) { // use for this scope
if(!err) {
self.messageId = this.lastID;
}
@ -221,36 +348,40 @@ Message.prototype.persist = function(cb) {
if(!self.meta) {
callback(null);
} else {
// :TODO: this should be it's own method such that meta can be updated
var metaStmt = msgDb.prepare(
'INSERT INTO message_meta (message_id, meta_category, meta_name, meta_value) ' +
'VALUES (?, ?, ?, ?);');
/*
Example of self.meta:
for(var metaCategroy in self.meta) {
async.each(Object.keys(self.meta[metaCategroy]), function meta(metaName, next) {
metaStmt.run(self.messageId, Message.MetaCategories[metaCategroy], metaName, self.meta[metaCategroy][metaName], function inserted(err) {
next(err);
meta: {
System: {
local_to_user_id: 1234,
},
FtnProperty: {
ftn_seen_by: [ "1/102 103", "2/42 52 65" ]
}
}
*/
async.each(Object.keys(self.meta), (category, nextCat) => {
async.each(Object.keys(self.meta[category]), (name, nextName) => {
self.persistMetaValue(category, name, self.meta[category][name], err => {
nextName(err);
});
}, function complete(err) {
if(!err) {
metaStmt.finalize(function finalized() {
callback(null);
}, err => {
nextCat(err);
});
} else {
}, err => {
callback(err);
}
});
}
}
},
function storeHashTags(callback) {
// :TODO: hash tag support
callback(null);
}
],
function complete(err) {
msgDb.run(err ? 'ROLLBACK;' : 'COMMIT;', function transEnd(err) {
cb(err, self.messageId);
err => {
Message.endTransaction(err, transErr => {
cb(err ? err : transErr, self.messageId);
});
}
);

307
core/scanner_tossers/ftn_bso.js
View File

@ -20,6 +20,7 @@ let async = require('async');
let fs = require('fs');
let later = require('later');
let temp = require('temp').track(); // track() cleans up temp dir/files for us
let assert = require('assert');
exports.moduleInfo = {
name : 'FTN BSO',
@ -72,13 +73,44 @@ function FTNMessageScanTossModule() {
return(networkName === defaultNetworkName && address.zone === this.moduleConfig.defaultZone);
};
this.getNetworkNameByAddress = function(address) {
this.getNetworkNameByAddress = function(remoteAddress) {
return _.findKey(Config.messageNetworks.ftn.networks, network => {
const networkAddress = Address.fromString(network.localAddress);
return !_.isUndefined(networkAddress) && address.isEqual(networkAddress);
const localAddress = Address.fromString(network.localAddress);
return !_.isUndefined(localAddress) && localAddress.isEqual(remoteAddress);
});
};
this.getNetworkNameByAddressPattern = function(remoteAddressPattern) {
return _.findKey(Config.messageNetworks.ftn.networks, network => {
const localAddress = Address.fromString(network.localAddress);
return !_.isUndefined(localAddress) && localAddress.isPatternMatch(remoteAddressPattern);
});
};
this.getLocalAreaTagByFtnAreaTag = function(ftnAreaTag) {
return _.findKey(Config.messageNetworks.ftn.areas, areaConf => {
return areaConf.tag === ftnAreaTag;
});
};
/*
this.getSeenByAddresses = function(messageSeenBy) {
if(!_.isArray(messageSeenBy)) {
messageSeenBy = [ messageSeenBy ];
}
let seenByAddrs = [];
messageSeenBy.forEach(sb => {
seenByAddrs = seenByAddrs.concat(ftnUtil.parseAbbreviatedNetNodeList(sb));
});
return seenByAddrs;
};
*/
this.messageHasValidMSGID = function(msg) {
return _.isString(msg.meta.FtnKludge.MSGID) && msg.meta.FtnKludge.MSGID.length > 0;
};
this.getOutgoingPacketDir = function(networkName, destAddress) {
let dir = this.moduleConfig.paths.outbound;
if(!this.isDefaultDomainZone(networkName, destAddress)) {
@ -164,41 +196,6 @@ function FTNMessageScanTossModule() {
});
};
this.exportMessage = function(message, options, cb) {
this.prepareMessage(message, options);
let packet = new ftnMailPacket.Packet();
let packetHeader = new ftnMailPacket.PacketHeader(
options.network.localAddress,
options.destAddress,
options.nodeConfig.packetType);
packetHeader.password = options.nodeConfig.packetPassword || '';
if(message.isPrivate()) {
// :TODO: this should actually be checking for isNetMail()!!
} else {
const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.destAddress);
mkdirp(outgoingDir, err => {
if(err) {
return cb(err);
}
this.getOutgoingBundleFileName(outgoingDir, options.network.localAddress, options.destAddress, (err, path) => {
console.log(path);
});
packet.write(
this.getOutgoingPacketFileName(outgoingDir, message),
packetHeader,
[ message ],
{ encoding : options.encoding }
);
});
}
};
this.prepareMessage = function(message, options) {
//
// Set various FTN kludges/etc.
@ -241,7 +238,8 @@ function FTNMessageScanTossModule() {
// When exporting messages, we should create/update SEEN-BY
// with remote address(s) we are exporting to.
//
const seenByAdditions = [ options.network.localAddress ].concat(Config.messageNetworks.ftn.areas[message.areaTag].uplinks);
const seenByAdditions =
[ `${options.network.localAddress.net}/${options.network.localAddress.node}` ].concat(Config.messageNetworks.ftn.areas[message.areaTag].uplinks);
message.meta.FtnProperty.ftn_seen_by =
ftnUtil.getUpdatedSeenByEntries(message.meta.FtnProperty.ftn_seen_by, seenByAdditions);
@ -257,7 +255,13 @@ function FTNMessageScanTossModule() {
//
// Additional kludges
//
// Check for existence of MSGID as we may already have stored it from a previous
// export that failed to finish
//
if(!message.meta.FtnKludge.MSGID) {
message.meta.FtnKludge.MSGID = ftnUtil.getMessageIdentifier(message, options.network.localAddress);
}
message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset();
if(!message.meta.FtnKludge.PID) {
@ -369,7 +373,7 @@ function FTNMessageScanTossModule() {
this.getNodeConfigKeyForUplink = function(uplink) {
// :TODO: sort by least # of '*' & take top?
const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => {
return Address.fromString(addr).isMatch(uplink);
return Address.fromString(addr).isPatternMatch(uplink);
})[0];
return nodeKey;
@ -451,6 +455,19 @@ function FTNMessageScanTossModule() {
ws.write(msgBuf);
}
callback(null);
},
function updateStoredMeta(callback) {
//
// We want to store some meta as if we had imported
// this message for later reference
//
if(message.meta.FtnKludge.MSGID) {
message.persistMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.MSGID, err => {
callback(err);
});
} else {
callback(null);
}
}
],
err => {
@ -607,37 +624,144 @@ function FTNMessageScanTossModule() {
}, cb); // complete
};
this.setReplyToMsgIdFtnReplyKludge = function(message, cb) {
//
// Given a FTN REPLY kludge, set |message.replyToMsgId|, if possible,
// by looking up an associated MSGID kludge meta.
//
// See also: http://ftsc.org/docs/fts-0009.001
//
if(!_.isString(message.meta.FtnKludge.REPLY)) {
// nothing to do
return cb();
}
Message.getMessageIdsByMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.REPLY, (err, msgIds) => {
if(!err) {
assert(1 === msgIds.length);
message.replyToMsgId = msgIds[0];
}
cb();
});
};
this.importNetMailToArea = function(localAreaTag, header, message, cb) {
async.series(
[
function validateDestinationAddress(callback) {
/*
const messageDestAddress = new Address({
node : message.meta.FtnProperty.ftn_dest_node,
net : message.meta.FtnProperty.ftn_dest_network,
});
*/
const localNetworkPattern = `${message.meta.FtnProperty.ftn_dest_network}/${message.meta.FtnProperty.ftn_dest_node}`;
const localNetworkName = self.getNetworkNameByAddressPattern(localNetworkPattern);
callback(_.isString(localNetworkName) ? null : new Error('Packet destination is not us'));
},
function basicSetup(callback) {
message.areaTag = localAreaTag;
//
// If duplicates are NOT allowed in the area (the default), we need to update
// the message UUID using data available to us. Duplicate UUIDs are internally
// not allowed in our local database.
//
if(!Config.messageNetworks.ftn.areas[localAreaTag].allowDupes) {
if(self.messageHasValidMSGID(message)) {
// Update UUID with our preferred generation method
message.uuid = ftnUtil.createMessageUuid(
message.meta.FtnKludge.MSGID,
message.meta.FtnProperty.ftn_area);
} else {
// Update UUID with alternate/backup generation method
message.uuid = ftnUtil.createMessageUuidAlternate(
message.meta.FtnProperty.ftn_area,
message.modTimestamp,
message.subject,
message.message);
}
}
callback(null);
},
function setReplyToMessageId(callback) {
self.setReplyToMsgIdFtnReplyKludge(message, () => {
callback(null);
});
},
function persistImport(callback) {
message.persist(err => {
callback(err);
});
}
], err => {
cb(err);
}
);
};
//
// Ref. implementations on import:
// * https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/pkt.c
// https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/handle.c
//
this.importMessagesFromPacketFile = function(packetPath, cb) {
const packet = new ftnMailPacket.Packet();
let packetHeader;
// :TODO: packet.read() should have a way to cancel iteration...
let localNetworkName;
packet.read(packetPath, (entryType, entryData) => {
new ftnMailPacket.Packet().read(packetPath, (entryType, entryData, next) => {
if('header' === entryType) {
//
// Discover if this packet is for one of our network(s)
//
localNetworkName = self.getNetworkNameByAddress(entryData.destAddress);
packetHeader = entryData;
} else if(localNetworkName && 'message' === entryType) {
const message = entryData; // so we ref something reasonable :)
const localNetworkName = self.getNetworkNameByAddress(packetHeader.destAddress);
if(!_.isString(localNetworkName)) {
next(new Error('No configuration for this packet'));
} else {
next(null);
}
} else if('message' === entryType) {
const message = entryData;
const areaTag = message.meta.FtnProperty.ftn_area;
// :TODO: we need to know if this message is a dupe - UUID will be the same if MSGID, but if not present... what to do?
// :TODO: lookup and set message.areaTag if match
// :TODO: check SEEN-BY for echo
// :TODO: Handle area vs Netmail - Via, etc.
// :TODO: Handle PATH
// :TODO: handle REPLY kludges... set local ID when possible
if(areaTag) {
//
// Find local area tag
// EchoMail
//
const localAreaTag = self.getLocalAreaTagByFtnAreaTag(areaTag);
if(localAreaTag) {
self.importNetMailToArea(localAreaTag, packetHeader, message, err => {
if(err) {
if('SQLITE_CONSTRAINT' === err.code) {
Log.info(
{ subject : message.subject, uuid : message.uuid },
'Not importing non-unique message');
return next(null);
}
}
next(err);
});
} else {
//
// No local area configured for this import
//
// :TODO: Handle the "catch all" case, if configured
}
} else {
//
// NetMail
//
}
}
}, err => {
cb(err);
}); };
});
};
this.importPacketFilesFromDirectory = function(importDir, cb) {
async.waterfall(
@ -672,9 +796,9 @@ function FTNMessageScanTossModule() {
// :TODO: rename to .bad, perhaps move to a rejects dir + log
nextFile();
} else {
//fs.unlink(fullPath, err => {
fs.unlink(fullPath, err => {
nextFile();
//});
});
}
}, err => {
callback(err);
@ -687,7 +811,9 @@ function FTNMessageScanTossModule() {
);
};
this.importMessagesFromDirectory = function(importDir, cb) {
this.importMessagesFromDirectory = function(inboundType, importDir, cb) {
let tempDirectory;
async.waterfall(
[
// start with .pkt files
@ -712,21 +838,37 @@ function FTNMessageScanTossModule() {
},
function createTempDir(bundleFiles, callback) {
temp.mkdir('enigftnimport-', (err, tempDir) => {
callback(err, bundleFiles, tempDir);
tempDirectory = tempDir;
callback(err, bundleFiles);
});
},
function importBundles(bundleFiles, tempDir, callback) {
function importBundles(bundleFiles, callback) {
let rejects = [];
async.each(bundleFiles, (bundleFile, nextFile) => {
if(_.isUndefined(bundleFile.archName)) {
// :TODO: log?
Log.info(
{ fileName : bundleFile.path },
'Unknown bundle archive type');
rejects.push(bundleFile.path);
return nextFile(); // unknown archive type
}
self.archUtil.extractTo(
bundleFile.path,
tempDir,
tempDirectory,
bundleFile.archName,
err => {
if(err) {
Log.info(
{ fileName : bundleFile.path, error : err.toString() },
'Failed to extract bundle');
rejects.push(bundleFile.path);
}
nextFile();
}
);
@ -738,15 +880,40 @@ function FTNMessageScanTossModule() {
//
// All extracted - import .pkt's
//
self.importPacketFilesFromDirectory(tempDir, err => {
callback(err);
self.importPacketFilesFromDirectory(tempDirectory, err => {
callback(null, bundleFiles, rejects);
});
});
},
function handleProcessedBundleFiles(bundleFiles, rejects, callback) {
async.each(bundleFiles, (bundleFile, nextFile) => {
if(rejects.indexOf(bundleFile.path) > -1) {
// :TODO: rename to .bad, perhaps move to a rejects dir + log
nextFile();
} else {
fs.unlink(bundleFile.path, err => {
nextFile();
});
}
}, err => {
callback(err);
});
}
],
err => {
if(tempDirectory) {
temp.cleanup( (errIgnored, stats) => {
Log.trace(
Object.assign(stats, { tempDir : tempDirectory } ),
'Temporary directory cleaned up'
);
cb(err); // orig err
});
} else {
cb(err);
}
}
);
};
}
@ -818,8 +985,8 @@ FTNMessageScanTossModule.prototype.performImport = function(cb) {
var self = this;
async.each( [ 'inbound', 'secInbound' ], (importDir, nextDir) => {
self.importMessagesFromDirectory(self.moduleConfig.paths[importDir], err => {
async.each( [ 'inbound', 'secInbound' ], (inboundType, nextDir) => {
self.importMessagesFromDirectory(inboundType, self.moduleConfig.paths[inboundType], err => {
nextDir();
});
@ -879,7 +1046,7 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) {
const newLastScanId = msgRows[msgRows.length - 1].message_id;
Log.info(
{ messagesExported : msgRows.length, newLastScanId : newLastScanId },
{ areaTag : areaTag, messagesExported : msgRows.length, newLastScanId : newLastScanId },
'Export complete');
callback(err, newLastScanId);

41
core/uuid_util.js Normal file
View File

@ -0,0 +1,41 @@
/* jslint node: true */
'use strict';
let uuid = require('node-uuid');
let assert = require('assert');
let _ = require('lodash');
let createHash = require('crypto').createHash;
exports.createNamedUUID = createNamedUUID;
function createNamedUUID(namespaceUuid, key) {
//
// v5 UUID generation code based on the work here:
// https://github.com/download13/uuidv5/blob/master/uuid.js
//
if(!Buffer.isBuffer(namespaceUuid)) {
namespaceUuid = new Buffer(namespaceUuid);
}
if(!Buffer.isBuffer(key)) {
key = new Buffer(key);
}
let digest = createHash('sha1').update(
Buffer.concat( [ namespaceUuid, key ] )).digest();
let u = new Buffer(16);
// bbbb - bb - bb - bb - bbbbbb
digest.copy(u, 0, 0, 4); // time_low
digest.copy(u, 4, 4, 6); // time_mid
digest.copy(u, 6, 6, 8); // time_hi_and_version
u[6] = (u[6] & 0x0f) | 0x50; // version, 4 most significant bits are set to version 5 (0101)
u[8] = (digest[8] & 0x3f) | 0x80; // clock_seq_hi_and_reserved, 2msb are set to 10
u[9] = digest[9];
digest.copy(u, 10, 10, 16);
return u;
}