* Changed scan check to use new System state_flags0 meta to skip already imported/exported msgs

* Use moment.js for Message modTimestamp
* Remove user_message_status stuff
* Add REPLY kludge support @ export
* Use TID vs PID kludge @ export (spec)
* Start work on @immediate - nearly complete

core/database.js

@@ -204,17 +204,6 @@ function createMessageBaseTables() {
 		');'
 	);
 	
-	//	:TODO: Not currently used 
-	dbs.message.run(
-		'CREATE TABLE IF NOT EXISTS user_message_status ('	+
-		'	user_id		INTEGER NOT NULL,'					+
-		'	message_id	INTEGER NOT NULL,'					+
-		'	status		INTEGER NOT NULL,'					+
-		'	UNIQUE(user_id, message_id, status),'			+
-		'	FOREIGN KEY(user_id) REFERENCES user(id)'		+
-		');'
-	);
-	
 	dbs.message.run(
 		`CREATE TABLE IF NOT EXISTS message_area_last_scan (
 				scan_toss		VARCHAR NOT NULL,

core/ftn_util.js

@@ -68,7 +68,8 @@ function getDateFromFtnDateTime(dateTime) {
 	//		"27 Feb 15  00:00:03"
 	//
 	//	:TODO: Use moment.js here
-	return (new Date(Date.parse(dateTime))).toISOString();
+	return moment(Date.parse(dateTime));	//	Date.parse() allows funky formats
+//	return (new Date(Date.parse(dateTime))).toISOString();
 }
 
 function getDateTimeString(m) {

core/message.js

@@ -1,14 +1,15 @@
 /* jslint node: true */
 'use strict';
 
-var msgDb			= require('./database.js').dbs.message;
+let msgDb			= require('./database.js').dbs.message;
-var wordWrapText	= require('./word_wrap.js').wordWrapText;
+let wordWrapText	= require('./word_wrap.js').wordWrapText;
-var ftnUtil			= require('./ftn_util.js');
+let ftnUtil			= require('./ftn_util.js');
 
-var uuid			= require('node-uuid');
+let uuid			= require('node-uuid');
-var async			= require('async');
+let async			= require('async');
-var _				= require('lodash');
+let _				= require('lodash');
-var assert			= require('assert');
+let assert			= require('assert');
+let moment			= require('moment');
 
 module.exports = Message;
 
@@ -24,10 +25,10 @@ function Message(options) {
 	this.subject		= options.subject || '';
 	this.message		= options.message || '';
 	
-	if(_.isDate(options.modTimestamp)) {
+	if(_.isDate(options.modTimestamp) || moment.isMoment(options.modTimestamp)) {
-		this.modTimestamp = options.modTimestamp;
+		this.modTimestamp = moment(options.modTimestamp);
 	} else if(_.isString(options.modTimestamp)) {
-		this.modTimestamp = new Date(options.modTimestamp);
+		this.modTimestamp = moment(options.modTimestamp);
 	}
 
 	this.viewCount		= options.viewCount || 0;
@@ -44,11 +45,8 @@ function Message(options) {
 		this.meta = options.meta;
 	}
 
-//	this.meta			= options.meta || {};
 	this.hashTags		= options.hashTags || [];
 
-	var self			= this;
-
 	this.isValid = function() {
 		//	:TODO: validate as much as possible
 		return true;
@@ -59,8 +57,8 @@ function Message(options) {
 	};
 
 	this.getMessageTimestampString = function(ts) {
-		ts = ts || new Date();
+		ts = ts || moment();
-		return ts.toISOString();
+		return ts.format('YYYY-MM-DDTHH:mm:ss.SSSZ');
 	};
 }
 
@@ -70,13 +68,7 @@ Message.WellKnownAreaTags = {
 	Bulletin	: 'local_bulletin',
 };
 
-//	:TODO: This doesn't seem like a good way to go -- perhaps only for local/user2user, or just use
+//	:TODO: FTN stuff really doesn't belong here - move it elsewhere and/or just use the names directly when needed
-//	a system similar to the "last read" for general areas
-Message.Status = {
-	New		: 0,
-	Read	: 1,
-};
-
 Message.MetaCategories = {
 	System				: 1,			//	ENiGMA1/2 stuff
 	FtnProperty			: 2,			//	Various FTN network properties, ftn_cost, ftn_origin, ...
@@ -86,6 +78,13 @@ Message.MetaCategories = {
 Message.SystemMetaNames = {
 	LocalToUserID			: 'local_to_user_id',
 	LocalFromUserID			: 'local_from_user_id',
+	StateFlags0				: 'state_flags0',		//	See Message.StateFlags0
+};
+
+Message.StateFlags0 = {
+	None		: 0x00000000,
+	Imported	: 0x00000001,	//	imported from foreign system
+	Exported	: 0x00000002,	//	exported to foreign system
 };
 
 Message.FtnPropertyNames = {	
@@ -152,7 +151,31 @@ Message.getMessageIdsByMetaValue = function(category, name, value, cb) {
 	);
 };
 
-Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name, cb) {	
+Message.getMetaValuesByMessageId = function(messageId, category, name, cb) {
+	const sql = 
+		`SELECT meta_value
+		FROM message_meta
+		WHERE message_id = ? AND meta_category = ? AND meta_name = ?;`;
+	
+	msgDb.all(sql, [ messageId, category, name ], (err, rows) => {
+		if(err) {
+			return cb(err);
+		}
+		
+		if(0 === rows.length) {
+			return cb(new Error('No value for category/name'));
+		}
+		
+		//	single values are returned without an array
+		if(1 === rows.length) {
+			return cb(null, rows[0].meta_value);
+		}
+		
+		cb(null, rows.map(r => r.meta_value));	//	map to array of values only
+	});
+};
+
+Message.getMetaValuesByMessageUuid = function(uuid, category, name, cb) {	
 	async.waterfall(
 		[
 			function getMessageId(callback) {
@@ -160,32 +183,14 @@ Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name,
 					callback(err, messageId);
 				});
 			},
-			function getMetaValue(messageId, callback) {
+			function getMetaValues(messageId, callback) {
-				const sql = 
+				Message.getMetaValuesByMessageId(messageId, category, name, (err, values) => {
-					`SELECT meta_value
+					callback(err, values);
-					FROM message_meta
-					WHERE message_id = ? AND message_category = ? AND meta_name = ?;`;
-					
-				msgDb.all(sql, [ messageId, category, name ], (err, rows) => {
-					if(err) {
-						return callback(err);
-					}
-					
-					if(0 === rows.length) {
-						return callback(new Error('No value for category/name'));
-					}
-					
-					//	single values are returned without an array
-					if(1 === rows.length) {
-						return callback(null, rows[0].meta_value);
-					}
-					
-					callback(null, rows.map(r => r.meta_value));
 				});
 			}
 		],
-		(err, value) => {
+		(err, values) => {
-			cb(err, value);
+			cb(err, values);
 		}
 	);
 };
@@ -254,7 +259,7 @@ Message.prototype.load = function(options, cb) {
 						self.fromUserName	= msgRow.from_user_name;
 						self.subject		= msgRow.subject;
 						self.message		= msgRow.message;
-						self.modTimestamp	= msgRow.modified_timestamp;
+						self.modTimestamp	= moment(msgRow.modified_timestamp);
 						self.viewCount		= msgRow.view_count;
 
 						callback(err);
@@ -269,12 +274,6 @@ Message.prototype.load = function(options, cb) {
 			function loadHashTags(callback) {
 				//	:TODO:
 				callback(null);
-			},
-			function loadMessageStatus(callback) {
-				if(options.user) {
-					//	:TODO: Load from user_message_status
-				}
-				callback(null);
 			}
 		],
 		function complete(err) {

core/scanner_tossers/ftn_bso.js

@@ -30,12 +30,11 @@ exports.moduleInfo = {
 
 /*
 	:TODO:
-	* Add bundle timer (arcmail)
+	* Support (approx) max bundle size 
-	* Queue until time elapses / fixed time interval
+	* Support NetMail
-	* Pakcets append until >= max byte size
+		* NetMail needs explicit isNetMail()  check
-	* [if arch type is not empty): Packets -> bundle until max byte size -> repeat process
+		* NetMail filename / location / etc. is still unknown - need to post on groups & get real answers
-	* NetMail needs explicit isNetMail()  check
+	
-	* NetMail filename / location / etc. is still unknown - need to post on groups & get real answers
 */ 
 
 exports.getModule = FTNMessageScanTossModule;
@@ -242,9 +241,7 @@ function FTNMessageScanTossModule() {
 		message.meta.FtnProperty.ftn_dest_node		= options.destAddress.node;
 		message.meta.FtnProperty.ftn_orig_network	= options.network.localAddress.net;
 		message.meta.FtnProperty.ftn_dest_network	= options.destAddress.net;
-		//	:TODO: attr1 & 2
 		message.meta.FtnProperty.ftn_cost			= 0;
-		
 		message.meta.FtnProperty.ftn_tear_line		= ftnUtil.getTearLine();		
 
 		//	:TODO: Need an explicit isNetMail() check
@@ -309,14 +306,14 @@ function FTNMessageScanTossModule() {
 			
 		message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset();
 			
-		if(!message.meta.FtnKludge.PID) {
+		//
-			message.meta.FtnKludge.PID = ftnUtil.getProductIdentifier();
+		//	According to FSC-0046:
-		}
+		//	
-		
+		//	"When a Conference Mail processor adds a TID to a message, it may not
-		if(!message.meta.FtnKludge.TID) {
+		//	add a PID. An existing TID should, however, be replaced. TIDs follow
-			//	:TODO: Create TID!!
+		//	the same format used for PIDs, as explained above."
-			//message.meta.FtnKludge.TID = 
+		//
-		}
+		message.meta.FtnKludge.TID = ftnUtil.getProductIdentifier(); 
 		
 		//
 		//	Determine CHRS and actual internal encoding name
@@ -332,12 +329,31 @@ function FTNMessageScanTossModule() {
 		options.encoding = encoding;	//	save for later
 		message.meta.FtnKludge.CHRS = ftnUtil.getCharacterSetIdentifierByEncoding(encoding);
 		//	:TODO: FLAGS kludge? 		
-		//	:TODO: Add REPLY kludge if appropriate
-		
 	};
 	
+	this.setReplyKludgeFromReplyToMsgId = function(message, cb) {
+		//
+		//	Look up MSGID kludge for |message.replyToMsgId|, if any.
+		//	If found, we can create a REPLY kludge with the previously
+		//	discovered MSGID.
+		//
+		
+		if(0 === message.replyToMsgId) {
+			return cb(null);	//	nothing to do
+		}
+		
+		Message.getMetaValuesByMessageId(message.replyToMsgId, 'FtnKludge', 'MSGID', (err, msgIdVal) => {
+			assert(_.isString(msgIdVal));
+			
+			if(!err) {			
+				//	got a MSGID - create a REPLY
+				message.meta.FtnKludge.REPLY = msgIdVal;
+			}
+			
+			cb(null);	//	this method always passes
+		});	
+	};
 	
-	//	:TODO: change to something like isAreaConfigValid
 	//	check paths, Addresses, etc.
 	this.isAreaConfigValid = function(areaConfig) {
 		if(!_.isString(areaConfig.tag) || !_.isString(areaConfig.network)) {
@@ -415,7 +431,7 @@ function FTNMessageScanTossModule() {
 		});
 	};
 	
-	this.getNodeConfigKeyForUplink = function(uplink) {
+	this.getNodeConfigKeyByAddress = function(uplink) {
 		//	:TODO: sort by least # of '*' & take top?
 		const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => {
 			return Address.fromString(addr).isPatternMatch(uplink);
@@ -457,10 +473,16 @@ function FTNMessageScanTossModule() {
 					},
 					function loadMessage(callback) {
 						message.load( { uuid : msgUuid }, err => {
-							if(!err) {
+							if(err) {
-								self.prepareMessage(message, exportOpts);
+								return callback(err);
 							}
-							callback(err);
+							
+							//	General preperation
+							self.prepareMessage(message, exportOpts);
+							
+							self.setReplyKludgeFromReplyToMsgId(message, err => {
+								callback(err);
+							});
 						});
 					},
 					function createNewPacket(callback) {			
@@ -502,7 +524,12 @@ function FTNMessageScanTossModule() {
 						}
 						callback(null);
 					},
-					function updateStoredMeta(callback) {
+					function storeStateFlags0Meta(callback) {
+						message.persistMetaValue('System', 'state_flags0', Message.StateFlags0.Exported.toString(), err => {
+							callback(err);
+						});
+					},
+					function storeMsgIdMeta(callback) {
 						//
 						//	We want to store some meta as if we had imported
 						//	this message for later reference
@@ -577,7 +604,7 @@ function FTNMessageScanTossModule() {
 		
 	this.exportMessagesToUplinks = function(messageUuids, areaConfig, cb) {
 		async.each(areaConfig.uplinks, (uplink, nextUplink) => {
-			const nodeConfigKey = self.getNodeConfigKeyForUplink(uplink);
+			const nodeConfigKey = self.getNodeConfigKeyByAddress(uplink);
 			if(!nodeConfigKey) {
 				return nextUplink();
 			}
@@ -657,7 +684,6 @@ function FTNMessageScanTossModule() {
 								const newPath = paths.join(outgoingDir, paths.basename(oldPath));
 								fs.rename(oldPath, newPath, err => {
 									if(err) {
-										//	:TODO: Log this - but move on to the next file
 										Log.warn(
 											{ oldPath : oldPath, newPath : newPath },
 											'Failed moving temporary bundle file!');
@@ -756,6 +782,10 @@ function FTNMessageScanTossModule() {
 					});
 				},
 				function persistImport(callback) {
+					//	mark as imported
+					message.meta.System.StateFlags0 = Message.StateFlags0.Imported.toString();
+					
+					//	save to disc
 					message.persist(err => {
 						callback(err);						
 					});
@@ -783,6 +813,8 @@ function FTNMessageScanTossModule() {
 				if(!_.isString(localNetworkName)) {
 					next(new Error('No configuration for this packet'));
 				} else {
+					
+					//	:TODO: password needs validated - need to determine if it will use the same node config (which can have wildcards) or something else?!					
 					next(null);
 				}
 				
@@ -1011,6 +1043,10 @@ FTNMessageScanTossModule.prototype.startup = function(cb) {
 				if(exportSchedule.watchFile) {
 					//	:TODO: monitor file for changes/existance with gaze
 				}
+				
+				if(_.isBoolean(exportSchedule.immediate)) {
+					this.exportImmediate = exportSchedule.immediate;
+				}
 			}
 			
 			const importSchedule = this.parseScheduleString(this.moduleConfig.schedule.import);
@@ -1043,6 +1079,10 @@ FTNMessageScanTossModule.prototype.shutdown = function(cb) {
 		this.exportTimer.clear();
 	}
 	
+	if(this.importTimer) {
+		this.importTimer.clear();
+	}
+	
 	//
 	//	Clean up temp dir/files we created
 	//
@@ -1088,6 +1128,7 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) {
 	//	Additionally exclude messages that have a ftn_attr_flags FtnProperty meta
 	//	as those came via import!
 	//
+	/*	
 	const getNewUuidsSql = 
 		`SELECT message_id, message_uuid
 		FROM message m
@@ -1096,6 +1137,23 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) {
 			FROM message_meta 
 			WHERE message_id = m.message_id AND meta_category = 'FtnProperty' AND meta_name = 'ftn_attr_flags') = 0
 		ORDER BY message_id;`;
+	*/
+	
+	//
+	//	Select all messages with a |message_id| > |lastScanId|.
+	//	Additionally exclude messages with the System state_flags0 which will be present for
+	//	imported or already exported messages
+	//
+	//	NOTE: If StateFlags0 starts to use additional bits, we'll likely need to check them here!
+	//
+	const getNewUuidsSql = 
+		`SELECT message_id, message_uuid
+		FROM message m
+		WHERE area_tag = ? AND message_id > ? AND
+			(SELECT COUNT(message_id) 
+			FROM message_meta 
+			WHERE message_id = m.message_id AND meta_category = 'System' AND meta_name = 'state_flags0') = 0
+		ORDER BY message_id;`;
 		
 	var self = this;		
 		
@@ -1156,5 +1214,26 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) {
 
 FTNMessageScanTossModule.prototype.record = function(message) {
 	//
-	//	:TODO: If @immediate, we should do something here!
+	//	This module works off schedules, but we do support @immediate for export
+	//
+	if(true !== this.exportImmediate || !this.hasValidConfiguration()) {
+		return;
+	}
+	
+	if(message.isPrivate()) {
+		//	:TODO: support NetMail
+	} else if(message.areaTag) {
+		const areaConfig = Config.messageNetworks.ftn.areas[message.areaTag];
+		if(!this.isAreaConfigValid(areaConfig)) {
+			return;
+		}
+		
+		//	:TODO: We must share a check to block export with schedule/timer when this is exporting...
+		//	:TODO: Messages must be marked as "exported" else we will export this particular message again later @ schedule/timer
+		//	...if getNewUuidsSql in performExport checks for MSGID existence also we can omit already-exported messages 
+		
+		this.exportMessagesToUplinks( [ message.uuid ], areaConfig, err => {
+		});
+		
+	}		
 };