* Change FTN packet read() to use async iterator

* createMessageUuidAlternate(): Mmethod for FTN message v5 UUID generation when no MSGID to work with
* parseAbbreviatedNetNodeList() now works properly
* Add core/uuid_util.js for various UUID utilities such as v5 named UUID generation
* Fix message meta load/retrieval
* Add lookup for REPLY kludge -> MSGID -> local reply IDs
* Fix SEEN-BY additions @ export
* Don't override MSGIDs if they already exist
* Store MSGID @ export so it can be inspected later
* Add import functionality (working, but WIP!)
* Clean up bundles and packets after import

core/database.js

@@ -175,7 +175,7 @@ function createMessageBaseTables() {
 		'	meta_category	INTEGER NOT NULL,'							+
 		'	meta_name		VARCHAR NOT NULL,'							+
 		'	meta_value		VARCHAR NOT NULL,'							+
-		'	UNIQUE(message_id, meta_category, meta_name, meta_value),'	+
+		'	UNIQUE(message_id, meta_category, meta_name, meta_value),'	+	//	why unique here? 
 		'	FOREIGN KEY(message_id) REFERENCES message(message_id)'		+
 		');'
 	);

core/ftn_address.js

@@ -107,7 +107,7 @@ module.exports = class Address {
 	}
 	*/
 
-	isMatch(pattern) {
+	isPatternMatch(pattern) {
 		const addr = this.getMatchAddr(pattern);
 		if(addr) {
 			return (

core/ftn_mail_packet.js

@@ -32,6 +32,7 @@ const FTN_PACKET_HEADER_SIZE	= 58;	//	fixed header size
 const FTN_PACKET_HEADER_TYPE	= 2;
 const FTN_PACKET_MESSAGE_TYPE	= 2;
 const FTN_PACKET_BAUD_TYPE_2_2	= 2;
+const NULL_TERM_BUFFER			= new Buffer( [ 0x00 ] );
 
 //	SAUCE magic header + version ("00")
 const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00');
@@ -171,7 +172,6 @@ exports.PacketHeader = PacketHeader;
 //		http://walon.org/pub/fidonet/FTSC-nodelists-etc./pkt-types.txt
 //
 function Packet() {
-
 	var self = this;
 
 	this.parsePacketHeader = function(packetBuffer, cb) {
@@ -510,17 +510,8 @@ function Packet() {
 		);
 	};
 	
-	this.parsePacketMessages = function(messagesBuffer, iterator, cb) {
+	this.parsePacketMessages = function(packetBuffer, iterator, cb) {
-		const NULL_TERM_BUFFER = new Buffer( [ 0 ] );
+		binary.parse(packetBuffer)
-		
-		var count = 0;
-
-		binary.stream(messagesBuffer).loop(function looper(end, vars) {
-			//
-			//	Some variable names used here match up directly with well known
-			//	meta data names used with FTN messages.
-			//
-			this
 			.word16lu('messageType')
 			.word16lu('ftn_orig_node')
 			.word16lu('ftn_dest_node')
@@ -531,23 +522,25 @@ function Packet() {
 			.scan('modDateTime', NULL_TERM_BUFFER)	//	:TODO: 20 bytes max
 			.scan('toUserName', NULL_TERM_BUFFER)	//	:TODO: 36 bytes max
 			.scan('fromUserName', NULL_TERM_BUFFER)	//	:TODO: 36 bytes max
-				.scan('subject', NULL_TERM_BUFFER)		//	:TODO: 72 bytes max
+			.scan('subject', NULL_TERM_BUFFER)		//	:TODO: 72 bytes max6
 			.scan('message', NULL_TERM_BUFFER)
-				.tap(function tapped(msgData) {
+			.tap(function tapped(msgData) {	//	no arrow function; want classic this
 				if(!msgData.messageType) {
 					//	end marker -- no more messages			
-						end();						
+					return cb(null);
-						return;
 				}
 				
 				if(FTN_PACKET_MESSAGE_TYPE != msgData.messageType) {
-						end();
+					return cb(new Error('Unsupported message type: ' + msgData.messageType));
-						//	:TODO: This is probably a bug if we hit a bad message after at leats one iterate
-						cb(new Error('Unsupported message type: ' + msgData.messageType));
-						return;
 				}
 				
-					++count;
+				const read = 
+					14 +								//	fixed header size
+					msgData.modDateTime.length + 1 +
+					msgData.toUserName.length + 1 +
+					msgData.fromUserName.length + 1 +
+					msgData.subject.length + 1 +
+					msgData.message.length + 1;
 				
 				//
 				//	Convert null terminated arrays to strings
@@ -559,7 +552,7 @@ function Packet() {
 
 				//
 				//	The message body itself is a special beast as it may
-					//	contain special origin lines, kludges, SAUCE in the case
+				//	contain an origin line, kludges, SAUCE in the case
 				//	of ANSI files, etc.
 				//
 				let msg = new Message( {
@@ -577,43 +570,40 @@ function Packet() {
 				msg.meta.FtnProperty.ftn_attr_flags		= msgData.ftn_attr_flags;
 				msg.meta.FtnProperty.ftn_cost			= msgData.ftn_cost;
 
-					self.processMessageBody(msgData.message, function processed(messageBodyData) {
+				self.processMessageBody(msgData.message, messageBodyData => {
 					msg.message 		= messageBodyData.message;
 					msg.meta.FtnKludge	= messageBodyData.kludgeLines;
 			
 					if(messageBodyData.tearLine) {
 						msg.meta.FtnProperty.ftn_tear_line = messageBodyData.tearLine;
 					}
+					
 					if(messageBodyData.seenBy.length > 0) {
 						msg.meta.FtnProperty.ftn_seen_by = messageBodyData.seenBy;
 					}
+					
 					if(messageBodyData.area) {
 						msg.meta.FtnProperty.ftn_area = messageBodyData.area;
 					}
+					
 					if(messageBodyData.originLine) {
 						msg.meta.FtnProperty.ftn_origin = messageBodyData.originLine;
 					}			
 					
-						//
+					const nextBuf = packetBuffer.slice(read);
-						//	Update message UUID, if possible, based on MSGID and AREA
+					if(nextBuf.length > 0) {
-						//
+						let next = function(e) {
-						if(_.isString(msg.meta.FtnKludge.MSGID) &&
+							if(e) {
-							_.isString(msg.meta.FtnProperty.ftn_area) &&
+								cb(e);
-							msg.meta.FtnKludge.MSGID.length > 0 &&
+							} else {
-							msg.meta.FtnProperty.ftn_area.length > 0)
+								self.parsePacketMessages(nextBuf, iterator, cb);
-						{
-							msg.uuid = ftn.createMessageUuid(
-								msg.meta.FtnKludge.MSGID,
-								msg.meta.FtnProperty.area);
 							}
+						};
 						
-						iterator('message', msg);
+						iterator('message', msg, next);
-						
+					} else {
-						--count;
-						if(0 === count) {
 						cb(null);
 					}
-					})					
 				});			
 			});
 	};
@@ -818,10 +808,15 @@ function Packet() {
 			[
 				function processHeader(callback) {
 					self.parsePacketHeader(packetBuffer, (err, header) => {
-						if(!err) {
+						if(err) {
-							iterator('header', header);
+							return callback(err);
 						}
-						callback(err);
+						
+						let next = function(e) {
+							callback(e);
+						};
+						
+						iterator('header', header, next);
 					});
 				},
 				function processMessages(callback) {
@@ -881,7 +876,7 @@ Packet.prototype.read = function(pathOrBuffer, iterator, cb) {
 				});
 			}
 		],
-		function complete(err) {
+		err => {
 			cb(err);
 		}
 	);

core/ftn_util.js

@@ -4,6 +4,7 @@
 let Config			= require('./config.js').config;
 let Address			= require('./ftn_address.js');
 let FNV1a			= require('./fnv1a.js');
+let createNamedUUID	= require('./uuid_util.js').createNamedUUID;
 
 let _				= require('lodash');
 let assert			= require('assert');
@@ -22,6 +23,7 @@ let packageJson 	= require('../package.json');
 exports.stringToNullPaddedBuffer	= stringToNullPaddedBuffer;
 exports.getMessageSerialNumber		= getMessageSerialNumber;
 exports.createMessageUuid			= createMessageUuid;
+exports.createMessageUuidAlternate	= createMessageUuidAlternate;
 exports.getDateFromFtnDateTime		= getDateFromFtnDateTime;
 exports.getDateTimeString			= getDateTimeString;
 
@@ -100,42 +102,43 @@ function getDateTimeString(m) {
 	return m.format('DD MMM YY  HH:mm:ss');
 }
 
+//
+//	Create a v5 named UUID given a message ID ("MSGID") and
+//	FTN area tag ("AREA").
+//
+//	This is similar to CrashMail
+//	See https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c
+//
 function createMessageUuid(ftnMsgId, ftnArea) {
-	//
+	assert(_.isString(ftnMsgId));
-	//	v5 UUID generation code based on the work here:
+	assert(_.isString(ftnArea));
-	//	https://github.com/download13/uuidv5/blob/master/uuid.js
+
-	//
-	//	Note: CrashMail uses MSGID + AREA, so we go with that as well:
-	//	https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c
-	//
-	if(!Buffer.isBuffer(ftnMsgId)) {
 	ftnMsgId	= iconv.encode(ftnMsgId, 'CP437');
-	}
+	ftnArea		= iconv.encode(ftnArea.toUpperCase(), 'CP437');
 	
-	ftnArea = ftnArea || '';	//	AREA is optional
+	return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnMsgId, ftnArea ] )));
-	if(!Buffer.isBuffer(ftnArea)) {
+};
-		ftnArea = iconv.encode(ftnArea, 'CP437');
-	}
 
-	const ns = new Buffer(ENIGMA_FTN_MSGID_NAMESPACE);
+//
+//	Create a v5 named UUID given a FTN area tag ("AREA"),
+//	create/modified date, subject, and message body
+//
+//	This method should be used as a backup for when a MSGID is
+//	not available in which createMessageUuid() above should be
+//	used instead.
+//
+function createMessageUuidAlternate(ftnArea, modTimestamp, subject, msgBody) {
+	assert(_.isString(ftnArea));
+	assert(_.isDate(modTimestamp) || moment.isMoment(modTimestamp));
+	assert(_.isString(subject));
+	assert(_.isString(msgBody));
 		
-	let digest = createHash('sha1').update(
+	ftnArea			= iconv.encode(ftnArea.toUpperCase(), 'CP437');
-		Buffer.concat([ ns, ftnMsgId, ftnArea ])).digest();
+	modTimestamp	= iconv.encode(getDateTimeString(modTimestamp), 'CP437');
+	subject			= iconv.encode(subject.toUpperCase().trim(), 'CP437');
+	msgBody			= iconv.encode(msgBody.replace(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g, '').trim(), 'CP437');
 	
-	let u = new Buffer(16);
+	return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnArea, modTimestamp, subject, msgBody ] )));
-
-	// bbbb - bb - bb - bb - bbbbbb
-	digest.copy(u, 0, 0, 4);			// time_low
-	digest.copy(u, 4, 4, 6);			// time_mid
-	digest.copy(u, 6, 6, 8);			// time_hi_and_version
-
-	u[6] = (u[6] & 0x0f) | 0x50;		// version, 4 most significant bits are set to version 5 (0101)
-	u[8] = (digest[8] & 0x3f) | 0x80;	// clock_seq_hi_and_reserved, 2msb are set to 10
-	u[9] = digest[9];
-	
-	digest.copy(u, 10, 10, 16);
-
-	return uuid.unparse(u);	//	to string
 }
 
 function getMessageSerialNumber(messageId) {
@@ -274,6 +277,9 @@ function getAbbreviatedNetNodeList(netNodes) {
 	let abbrList = '';
 	let currNet;
 	netNodes.forEach(netNode => {
+		if(_.isString(netNode)) {
+			netNode = Address.fromString(netNode);
+		}
 		if(currNet !== netNode.net) {
 			abbrList += `${netNode.net}/`;
 			currNet = netNode.net;
@@ -284,29 +290,24 @@ function getAbbreviatedNetNodeList(netNodes) {
 	return abbrList.trim();	//	remove trailing space
 }
 
+//
+//	Parse an abbreviated net/node list commonly used for SEEN-BY and PATH
+//
 function parseAbbreviatedNetNodeList(netNodes) {
-	//
+    const re = /([0-9]+)\/([0-9]+)\s?|([0-9]+)\s?/g;
-	//	Make sure we have an array of objects.
+    let net;
-	//	Allow for a single object or string(s)
+    let m;
-	//
+    let results = [];	
-	if(!_.isArray(netNodes)) {
+    while(null !== (m = re.exec(netNodes))) {
-		if(_.isString(netNodes)) {
+        if(m[1] && m[2]) {
-			netNodes = netNodes.split(' ');
+            net = parseInt(m[1]);
-		} else {
+            results.push(new Address( { net : net, node : parseInt(m[2]) } ));
-			netNodes = [ netNodes ];
+        } else if(net) {
+            results.push(new Address( { net : net, node : parseInt(m[3]) } ));
         }
     }
     
-	//
+    return results;
-	//	Convert any strings to parsed address objects
-	//
-	return netNodes.map(a => {
-		if(_.isObject(a)) {
-			return a;			
-		} else {
-			return Address.fromString(a);
-		}
-	});
 }
 
 //
@@ -349,7 +350,11 @@ function getUpdatedSeenByEntries(existingEntries, additions) {
 		existingEntries = [ existingEntries ];
 	}
 	
-	additions = parseAbbreviatedNetNodeList(additions).sort(Address.getComparator());
+	if(!_.isString(additions)) {
+		additions = parseAbbreviatedNetNodeList(getAbbreviatedNetNodeList(additions)); 
+	}
+
+	additions = additions.sort(Address.getComparator());
 
 	//
 	//	For now, we'll just append a new SEEN-BY entry

core/message.js

@@ -62,22 +62,6 @@ function Message(options) {
 		ts = ts || new Date();
 		return ts.toISOString();
 	};
-
-	/*
-	Object.defineProperty(this, 'messageId', {
-		get : function() {
-			return messageId;
-		}
-	});
-
-	Object.defineProperty(this, 'areaId', {
-		get : function() { return areaId; },
-		set : function(i) {
-			areaId = i;
-		}
-	});
-
-	*/
 }
 
 Message.WellKnownAreaTags = {
@@ -116,8 +100,6 @@ Message.FtnPropertyNames = {
 	FtnOrigPoint		: 'ftn_orig_point',
 	FtnDestPoint		: 'ftn_dest_point',
 		
-	
-	
 	FtnAttribute		: 'ftn_attribute',
 
 	FtnTearLine			: 'ftn_tear_line',		//	http://ftsc.org/docs/fts-0004.001
@@ -136,6 +118,118 @@ Message.prototype.setLocalFromUserId = function(userId) {
 	this.meta.System.local_from_user_id = userId;
 };
 
+Message.getMessageIdByUuid = function(uuid, cb) {
+	msgDb.get(
+		`SELECT message_id
+		FROM message
+		WHERE message_uuid = ?
+		LIMIT 1;`, 
+		[ uuid ], 
+		(err, row) => {
+			if(err) {
+				cb(err);
+			} else {
+				const success = (row && row.message_id);
+				cb(success ? null : new Error('No match'), success ? row.message_id : null);
+			}
+		}
+	);
+};
+
+Message.getMessageIdsByMetaValue = function(category, name, value, cb) {
+	msgDb.all(
+		`SELECT message_id
+		FROM message_meta
+		WHERE meta_category = ? AND meta_name = ? AND meta_value = ?;`,
+		[ category, name, value ],
+		(err, rows) => {
+			if(err) {
+				cb(err);
+			} else {
+				cb(null, rows.map(r => parseInt(r.message_id)));	//	return array of ID(s)
+			}
+		}
+	);
+};
+
+Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name, cb) {	
+	async.waterfall(
+		[
+			function getMessageId(callback) {
+				Message.getMessageIdByUuid(uuid, (err, messageId) => {
+					callback(err, messageId);
+				});
+			},
+			function getMetaValue(messageId, callback) {
+				const sql = 
+					`SELECT meta_value
+					FROM message_meta
+					WHERE message_id = ? AND message_category = ? AND meta_name = ?;`;
+					
+				msgDb.all(sql, [ messageId, category, name ], (err, rows) => {
+					if(err) {
+						return callback(err);
+					}
+					
+					if(0 === rows.length) {
+						return callback(new Error('No value for category/name'));
+					}
+					
+					//	single values are returned without an array
+					if(1 === rows.length) {
+						return callback(null, rows[0].meta_value);
+					}
+					
+					callback(null, rows.map(r => r.meta_value));
+				});
+			}
+		],
+		(err, value) => {
+			cb(err, value);
+		}
+	);
+};
+
+Message.prototype.loadMeta = function(cb) {
+	/*
+		Example of loaded this.meta:
+		
+		meta: {
+			System: {
+				local_to_user_id: 1234,				
+			},
+			FtnProperty: {
+				ftn_seen_by: [ "1/102 103", "2/42 52 65" ]
+			}
+		}					
+	*/	
+	
+	const sql = 
+		`SELECT meta_category, meta_name, meta_value
+		FROM message_meta
+		WHERE message_id = ?;`;
+		
+	let self = this;
+	msgDb.each(sql, [ this.messageId ], (err, row) => {
+		if(!(row.meta_category in self.meta)) {
+			self.meta[row.meta_category] = { };
+			self.meta[row.meta_category][row.meta_name] = row.meta_value;
+		} else {
+			if(!(row.meta_name in self.meta[row.meta_category])) {
+				self.meta[row.meta_category][row.meta_name] = row.meta_value; 
+			} else {
+				if(_.isString(self.meta[row.meta_category][row.meta_name])) {
+					self.meta[row.meta_category][row.meta_name] = [ self.meta[row.meta_category][row.meta_name] ];					
+				}
+				
+				self.meta[row.meta_category][row.meta_name].push(row.meta_value);
+			}
+		}
+	}, err => {
+		cb(err);
+	});
+};
+
 Message.prototype.load = function(options, cb) {
 	assert(_.isString(options.uuid));
 
@@ -168,8 +262,9 @@ Message.prototype.load = function(options, cb) {
 				);
 			},
 			function loadMessageMeta(callback) {
-				//	:TODO:
+				self.loadMeta(err => {
-				callback(null);
+					callback(err);
+				});
 			},
 			function loadHashTags(callback) {
 				//	:TODO:
@@ -188,27 +283,59 @@ Message.prototype.load = function(options, cb) {
 	);
 };
 
+Message.prototype.persistMetaValue = function(category, name, value, cb) {
+	const metaStmt = msgDb.prepare(
+		`INSERT INTO message_meta (message_id, meta_category, meta_name, meta_value) 
+		VALUES (?, ?, ?, ?);`);
+		
+	if(!_.isArray(value)) {
+		value = [ value ];
+	}
+	
+	let self = this;
+	
+	async.each(value, (v, next) => {
+		metaStmt.run(self.messageId, category, name, v, err => {
+			next(err);
+		});
+	}, err => {
+		cb(err);
+	});
+};
+
+Message.startTransaction = function(cb) {
+	msgDb.run('BEGIN;', err => {
+		cb(err);
+	});
+};
+
+Message.endTransaction = function(hadError, cb) {
+	msgDb.run(hadError ? 'ROLLBACK;' : 'COMMIT;', err => {
+		cb(err);
+	});	
+};
+
 Message.prototype.persist = function(cb) {
 
 	if(!this.isValid()) {
-		cb(new Error('Cannot persist invalid message!'));
+		return cb(new Error('Cannot persist invalid message!'));
-		return;
 	}
 
-	var self = this;
+	let self = this;
 	
 	async.series(
 		[
 			function beginTransaction(callback) {
-				msgDb.run('BEGIN;', function transBegin(err) {
+				Message.startTransaction(err => {
 					callback(err);
 				});
 			},
 			function storeMessage(callback) {
 				msgDb.run(
-					'INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) ' +
+					`INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp)
-					'VALUES (?, ?, ?, ?, ?, ?, ?, ?);', [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ],
+					VALUES (?, ?, ?, ?, ?, ?, ?, ?);`, 
-					function msgInsert(err) {
+					[ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ],
+					function inserted(err) {	//	use for this scope
 						if(!err) {
 							self.messageId = this.lastID;
 						}
@@ -221,36 +348,40 @@ Message.prototype.persist = function(cb) {
 				if(!self.meta) {
 					callback(null);
 				} else {
-					//	:TODO: this should be it's own method such that meta can be updated
+					/*
-					var metaStmt = msgDb.prepare(
+						Example of self.meta:
-						'INSERT INTO message_meta (message_id, meta_category, meta_name, meta_value) ' + 
-						'VALUES (?, ?, ?, ?);');
 						
-					for(var metaCategroy in self.meta) {
+						meta: {
-						async.each(Object.keys(self.meta[metaCategroy]), function meta(metaName, next) {
+							System: {
-							metaStmt.run(self.messageId, Message.MetaCategories[metaCategroy], metaName, self.meta[metaCategroy][metaName], function inserted(err) {
+								local_to_user_id: 1234,				
-								next(err);
+							},
+							FtnProperty: {
+								ftn_seen_by: [ "1/102 103", "2/42 52 65" ]
+							}
+						}					
+					*/
+					async.each(Object.keys(self.meta), (category, nextCat) => {
+						async.each(Object.keys(self.meta[category]), (name, nextName) => {
+							self.persistMetaValue(category, name, self.meta[category][name], err => {
+								nextName(err);
 							});
-						}, function complete(err) {
+						}, err => {
-							if(!err) {
+							nextCat(err);
-								metaStmt.finalize(function finalized() {
-									callback(null);
 						});
-							} else {
+						 
+					}, err => {
 						callback(err);
-							}
 					});					
 				}
-				}
 			},
 			function storeHashTags(callback) {
 				//	:TODO: hash tag support
 				callback(null);
 			}
 		],
-		function complete(err) {
+		err => {
-			msgDb.run(err ? 'ROLLBACK;' : 'COMMIT;', function transEnd(err) {
+			Message.endTransaction(err, transErr => {
-				cb(err, self.messageId);
+				cb(err ? err : transErr, self.messageId);
 			});
 		}
 	);

core/scanner_tossers/ftn_bso.js

@@ -20,6 +20,7 @@ let async					= require('async');
 let fs						= require('fs');
 let later					= require('later');
 let temp					= require('temp').track();	//	track() cleans up temp dir/files for us
+let assert					= require('assert');
 
 exports.moduleInfo = {
 	name	: 'FTN BSO',
@@ -72,13 +73,44 @@ function FTNMessageScanTossModule() {
 		return(networkName === defaultNetworkName && address.zone === this.moduleConfig.defaultZone);
 	};
 	
-	this.getNetworkNameByAddress = function(address) {
+	this.getNetworkNameByAddress = function(remoteAddress) {
 		return _.findKey(Config.messageNetworks.ftn.networks, network => {
-			const networkAddress = Address.fromString(network.localAddress);
+			const localAddress = Address.fromString(network.localAddress);			
-			return !_.isUndefined(networkAddress) && address.isEqual(networkAddress);
+			return !_.isUndefined(localAddress) && localAddress.isEqual(remoteAddress);
 		});
 	};
 	
+	this.getNetworkNameByAddressPattern = function(remoteAddressPattern) {
+		return _.findKey(Config.messageNetworks.ftn.networks, network => {
+			const localAddress = Address.fromString(network.localAddress);			
+			return !_.isUndefined(localAddress) && localAddress.isPatternMatch(remoteAddressPattern);
+		});	
+	};
+	
+	this.getLocalAreaTagByFtnAreaTag = function(ftnAreaTag) {
+		return _.findKey(Config.messageNetworks.ftn.areas, areaConf => {
+			return areaConf.tag === ftnAreaTag;
+		});
+	};
+	
+	/*
+	this.getSeenByAddresses = function(messageSeenBy) {
+		if(!_.isArray(messageSeenBy)) {
+			messageSeenBy = [ messageSeenBy ];
+		}
+		
+		let seenByAddrs = [];
+		messageSeenBy.forEach(sb => {
+			seenByAddrs = seenByAddrs.concat(ftnUtil.parseAbbreviatedNetNodeList(sb));
+		});
+		return seenByAddrs;
+	};
+	*/
+	
+	this.messageHasValidMSGID = function(msg) {
+		return _.isString(msg.meta.FtnKludge.MSGID) && msg.meta.FtnKludge.MSGID.length > 0;	
+	};
+	
 	this.getOutgoingPacketDir = function(networkName, destAddress) {
 		let dir = this.moduleConfig.paths.outbound;
 		if(!this.isDefaultDomainZone(networkName, destAddress)) {
@@ -164,41 +196,6 @@ function FTNMessageScanTossModule() {
 		});
 	};
 	
-	this.exportMessage = function(message, options, cb) {
-		this.prepareMessage(message, options);
-
-		let packet = new ftnMailPacket.Packet();
-
-		let packetHeader = new ftnMailPacket.PacketHeader(
-			options.network.localAddress,
-			options.destAddress,
-			options.nodeConfig.packetType);
-
-		packetHeader.password = options.nodeConfig.packetPassword || '';
-				
-		if(message.isPrivate()) {
-			//	:TODO: this should actually be checking for isNetMail()!!
-		} else {
-			const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.destAddress);
-			
-			mkdirp(outgoingDir, err => {
-				if(err) {
-					return cb(err);
-				}
-				this.getOutgoingBundleFileName(outgoingDir, options.network.localAddress, options.destAddress, (err, path) => {
-						console.log(path);
-				});
-				packet.write(
-					this.getOutgoingPacketFileName(outgoingDir, message), 
-					packetHeader, 
-					[ message ],
-					{ encoding : options.encoding }
-					);
-			});
-		}
-		
-	};
-
 	this.prepareMessage = function(message, options) {
 		//
 		//	Set various FTN kludges/etc.
@@ -241,7 +238,8 @@ function FTNMessageScanTossModule() {
 			//	When exporting messages, we should create/update SEEN-BY
 			//	with remote address(s) we are exporting to.
 			//
-			const seenByAdditions = [ options.network.localAddress ].concat(Config.messageNetworks.ftn.areas[message.areaTag].uplinks);
+			const seenByAdditions = 
+				[ `${options.network.localAddress.net}/${options.network.localAddress.node}` ].concat(Config.messageNetworks.ftn.areas[message.areaTag].uplinks);
 			message.meta.FtnProperty.ftn_seen_by = 
 				ftnUtil.getUpdatedSeenByEntries(message.meta.FtnProperty.ftn_seen_by, seenByAdditions);
 
@@ -257,7 +255,13 @@ function FTNMessageScanTossModule() {
 		//
 		//	Additional kludges
 		//
+		//	Check for existence of MSGID as we may already have stored it from a previous
+		//	export that failed to finish
+		//	
+		if(!message.meta.FtnKludge.MSGID) {
 			message.meta.FtnKludge.MSGID = ftnUtil.getMessageIdentifier(message, options.network.localAddress);
+		}
+			
 		message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset();
 		
 		if(!message.meta.FtnKludge.PID) {
@@ -369,7 +373,7 @@ function FTNMessageScanTossModule() {
 	this.getNodeConfigKeyForUplink = function(uplink) {
 		//	:TODO: sort by least # of '*' & take top?
 		const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => {
-			return Address.fromString(addr).isMatch(uplink);
+			return Address.fromString(addr).isPatternMatch(uplink);
 		})[0];
 
 		return nodeKey;
@@ -451,6 +455,19 @@ function FTNMessageScanTossModule() {
 							ws.write(msgBuf);
 						}
 						callback(null);
+					},
+					function updateStoredMeta(callback) {
+						//
+						//	We want to store some meta as if we had imported
+						//	this message for later reference
+						//
+						if(message.meta.FtnKludge.MSGID) {
+							message.persistMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.MSGID, err => {
+								callback(err);
+							});
+						} else {
+							callback(null);
+						}					
 					}
 				], 
 				err => {
@@ -607,37 +624,144 @@ function FTNMessageScanTossModule() {
 		}, cb);	//	complete
 	};
 	
+	this.setReplyToMsgIdFtnReplyKludge = function(message, cb) {
+		//
+		//	Given a FTN REPLY kludge, set |message.replyToMsgId|, if possible,
+		//	by looking up an associated MSGID kludge meta.
+		//
+		//	See also: http://ftsc.org/docs/fts-0009.001
+		//
+		if(!_.isString(message.meta.FtnKludge.REPLY)) {
+			//	nothing to do
+			return cb();
+		}
+		
+		Message.getMessageIdsByMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.REPLY, (err, msgIds) => {
+			if(!err) {
+				assert(1 === msgIds.length);
+				message.replyToMsgId = msgIds[0];
+			}
+			cb();
+		});
+	};
+	
+	this.importNetMailToArea = function(localAreaTag, header, message, cb) {
+		async.series(
+			[
+				function validateDestinationAddress(callback) {
+					/*
+					const messageDestAddress = new Address({
+						node	: message.meta.FtnProperty.ftn_dest_node,
+						net		: message.meta.FtnProperty.ftn_dest_network,
+					});
+					*/
+					
+					const localNetworkPattern = `${message.meta.FtnProperty.ftn_dest_network}/${message.meta.FtnProperty.ftn_dest_node}`;
+					
+					const localNetworkName = self.getNetworkNameByAddressPattern(localNetworkPattern);
+					
+					callback(_.isString(localNetworkName) ? null : new Error('Packet destination is not us'));
+				},
+				function basicSetup(callback) {
+					message.areaTag = localAreaTag;
+					
+					//
+					//	If duplicates are NOT allowed in the area (the default), we need to update
+					//	the message UUID using data available to us. Duplicate UUIDs are internally
+					//	not allowed in our local database.
+	 				//
+					if(!Config.messageNetworks.ftn.areas[localAreaTag].allowDupes) {
+						if(self.messageHasValidMSGID(message)) {
+							//	Update UUID with our preferred generation method
+							message.uuid = ftnUtil.createMessageUuid(
+								message.meta.FtnKludge.MSGID,
+								message.meta.FtnProperty.ftn_area);
+						} else {
+							//	Update UUID with alternate/backup generation method
+							message.uuid = ftnUtil.createMessageUuidAlternate(
+								message.meta.FtnProperty.ftn_area, 
+								message.modTimestamp,
+								message.subject,
+								message.message);
+						}
+					}
+					
+					callback(null);	
+				},
+				function setReplyToMessageId(callback) {
+					self.setReplyToMsgIdFtnReplyKludge(message, () => {
+						callback(null);
+					});
+				},
+				function persistImport(callback) {
+					message.persist(err => {
+						callback(err);						
+					});
+				}
+			], err => {
+				cb(err);	
+			}
+		);
+	};
+	
+	//
+	//	Ref. implementations on import: 
+	//	*	https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/pkt.c
+	//		https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/handle.c
+	//
 	this.importMessagesFromPacketFile = function(packetPath, cb) {
-		const packet = new ftnMailPacket.Packet();
+		let packetHeader;
 				
-		//	:TODO: packet.read() should have a way to cancel iteration...
+		new ftnMailPacket.Packet().read(packetPath, (entryType, entryData, next) => {
-		let localNetworkName;
-		packet.read(packetPath, (entryType, entryData) => {
 			if('header' === entryType) {
-				//
+				packetHeader = entryData;
-				//	Discover if this packet is for one of our network(s)
-				//
-				localNetworkName = self.getNetworkNameByAddress(entryData.destAddress);
 				
-			} else if(localNetworkName && 'message' === entryType) {
+				const localNetworkName = self.getNetworkNameByAddress(packetHeader.destAddress);
-				const message = entryData;	//	so we ref something reasonable :)
+				if(!_.isString(localNetworkName)) {
+					next(new Error('No configuration for this packet'));
+				} else {
+					next(null);
+				}
+				
+			} else if('message' === entryType) {
+				const message = entryData;
 				const areaTag = message.meta.FtnProperty.ftn_area;
 				
-				//	:TODO: we need to know if this message is a dupe - UUID will be the same if MSGID, but if not present... what to do?
-				//	:TODO: lookup and set message.areaTag if match
-				//	:TODO: check SEEN-BY for echo
-				//	:TODO: Handle area vs Netmail - Via, etc.
-				//	:TODO: Handle PATH
-				//	:TODO: handle REPLY kludges... set local ID when possible
 				if(areaTag) {
 					//
-					//	Find local area tag
+					//	EchoMail
+					//
+					const localAreaTag = self.getLocalAreaTagByFtnAreaTag(areaTag);
+					if(localAreaTag) {
+						self.importNetMailToArea(localAreaTag, packetHeader, message, err => {
+							if(err) {
+								if('SQLITE_CONSTRAINT' === err.code) {
+									Log.info(
+										{ subject : message.subject, uuid : message.uuid }, 
+										'Not importing non-unique message');
+									
+									return next(null);
+								}
+							}
+															
+							next(err);
+						});
+					} else {
+						//
+						//	No local area configured for this import
+						//
+						//	:TODO: Handle the "catch all" case, if configured 
+					}
+				} else {
+					//
+					//	NetMail
 					//
 				}
 			}
 		}, err => {
 			cb(err);
-		});	};
+		});
+	};
 	
 	this.importPacketFilesFromDirectory = function(importDir, cb) {
 		async.waterfall(
@@ -672,9 +796,9 @@ function FTNMessageScanTossModule() {
 							//	:TODO: rename to .bad, perhaps move to a rejects dir + log
 							nextFile();					
 						} else {
-							//fs.unlink(fullPath, err => {
+							fs.unlink(fullPath, err => {
 								nextFile();
-							//});
+							});
 						}
 					}, err => {
 						callback(err);
@@ -687,7 +811,9 @@ function FTNMessageScanTossModule() {
 		);
 	};
 	
-	this.importMessagesFromDirectory = function(importDir, cb) {
+	this.importMessagesFromDirectory = function(inboundType, importDir, cb) {
+		let tempDirectory;
+		
 		async.waterfall(
 			[
 				//	start with .pkt files
@@ -712,21 +838,37 @@ function FTNMessageScanTossModule() {
 				},
 				function createTempDir(bundleFiles, callback) {
 					temp.mkdir('enigftnimport-', (err, tempDir) => {
-						callback(err, bundleFiles, tempDir);
+						tempDirectory = tempDir;
+						callback(err, bundleFiles);
 					});
 				}, 
-				function importBundles(bundleFiles, tempDir, callback) {
+				function importBundles(bundleFiles, callback) {
+					let rejects = [];
+					
 					async.each(bundleFiles, (bundleFile, nextFile) => {
 						if(_.isUndefined(bundleFile.archName)) {
-							//	:TODO: log?
+							Log.info( 
+								{ fileName : bundleFile.path },
+								'Unknown bundle archive type');
+					
+							rejects.push(bundleFile.path);
+										
 							return nextFile();	//	unknown archive type
 						}
 						
 						self.archUtil.extractTo(
 							bundleFile.path,
-							tempDir,
+							tempDirectory,
 							bundleFile.archName,
 							err => {
+								if(err) {									
+									Log.info(
+										{ fileName : bundleFile.path, error : err.toString() },
+										'Failed to extract bundle');
+										
+									rejects.push(bundleFile.path);
+								}
+								
 								nextFile();		
 							}
 						);
@@ -738,15 +880,40 @@ function FTNMessageScanTossModule() {
 						//
 						//	All extracted - import .pkt's
 						//
-						self.importPacketFilesFromDirectory(tempDir, err => {
+						self.importPacketFilesFromDirectory(tempDirectory, err => {
-							callback(err);
+							callback(null, bundleFiles, rejects);
 						});					
 					});
+				},
+				function handleProcessedBundleFiles(bundleFiles, rejects, callback) {
+					async.each(bundleFiles, (bundleFile, nextFile) => {
+						if(rejects.indexOf(bundleFile.path) > -1) {
+							//	:TODO: rename to .bad, perhaps move to a rejects dir + log
+							nextFile();					
+						} else {
+							fs.unlink(bundleFile.path, err => {
+								nextFile();
+							});
+						}
+					}, err => {
+						callback(err);
+					});
 				}
 			],
 			err => {
+				if(tempDirectory) {
+					temp.cleanup( (errIgnored, stats) => {
+						Log.trace(
+							Object.assign(stats, { tempDir : tempDirectory } ),
+							'Temporary directory cleaned up'
+						);
+						
+						cb(err);	//	orig err
+					});
+				} else {				
 					cb(err);
 				}
+			}
 		);
 	};
 }
@@ -818,8 +985,8 @@ FTNMessageScanTossModule.prototype.performImport = function(cb) {
 	
 	var self = this;
 	
-	async.each( [ 'inbound', 'secInbound' ], (importDir, nextDir) => {
+	async.each( [ 'inbound', 'secInbound' ], (inboundType, nextDir) => {
-		self.importMessagesFromDirectory(self.moduleConfig.paths[importDir], err => {
+		self.importMessagesFromDirectory(inboundType, self.moduleConfig.paths[inboundType], err => {
 			
 			nextDir();
 		});
@@ -879,7 +1046,7 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) {
 						const newLastScanId = msgRows[msgRows.length - 1].message_id; 
 						
 						Log.info(
-							{ messagesExported : msgRows.length, newLastScanId : newLastScanId }, 
+							{ areaTag : areaTag, messagesExported : msgRows.length, newLastScanId : newLastScanId }, 
 							'Export complete');
 							
 						callback(err, newLastScanId);

core/uuid_util.js

@@ -0,0 +1,41 @@
+/* jslint node: true */
+'use strict';
+
+let uuid		= require('node-uuid');
+let assert      = require('assert');
+let _           = require('lodash');
+let createHash	= require('crypto').createHash;
+
+exports.createNamedUUID	= createNamedUUID;
+
+function createNamedUUID(namespaceUuid, key) {
+	//
+	//	v5 UUID generation code based on the work here:
+	//	https://github.com/download13/uuidv5/blob/master/uuid.js
+	//	
+    if(!Buffer.isBuffer(namespaceUuid)) {
+        namespaceUuid = new Buffer(namespaceUuid);
+    }
+	
+	if(!Buffer.isBuffer(key)) {
+		key = new Buffer(key);
+	}
+    
+	let digest = createHash('sha1').update(
+		Buffer.concat( [ namespaceUuid, key ] )).digest();
+
+	let u = new Buffer(16);
+
+	// bbbb - bb - bb - bb - bbbbbb
+	digest.copy(u, 0, 0, 4);			// time_low
+	digest.copy(u, 4, 4, 6);			// time_mid
+	digest.copy(u, 6, 6, 8);			// time_hi_and_version
+
+	u[6] = (u[6] & 0x0f) | 0x50;		// version, 4 most significant bits are set to version 5 (0101)
+	u[8] = (digest[8] & 0x3f) | 0x80;	// clock_seq_hi_and_reserved, 2msb are set to 10
+	u[9] = digest[9];
+	
+	digest.copy(u, 10, 10, 16);
+	
+	return u;
+}