* msg_network.js: Management of message network modules (start/stop/etc.)

* Minor updates to ES6 in some areas
* Better bbs.js startup seq
* Better iterator support for loadModulesForCategory()
* Start work on loading message network modules & tieing in record() (WIP)
* FTN PacketHeader is now a ES6 class
* Various FTN utils, e.g. Via line creation

core/bbs.js

@@ -5,36 +5,37 @@
 //SegfaultHandler.registerHandler('enigma-bbs-segfault.log');
 
 //	ENiGMA½
-var conf		= require('./config.js');
+let conf		= require('./config.js');
-var logger		= require('./logger.js');
+let logger		= require('./logger.js');
-var miscUtil	= require('./misc_util.js');
+let miscUtil	= require('./misc_util.js');
-var database	= require('./database.js');
+let database	= require('./database.js');
-var clientConns	= require('./client_connections.js');
+let clientConns	= require('./client_connections.js');
 
-var paths		= require('path');
+let paths		= require('path');
-var async		= require('async');
+let async		= require('async');
-var util		= require('util');
+let util		= require('util');
-var _			= require('lodash');
+let _			= require('lodash');
-var assert		= require('assert');
+let assert		= require('assert');
-var mkdirp 		= require('mkdirp');
+let mkdirp 		= require('mkdirp');
 
-exports.bbsMain			= bbsMain;
+//	our main entry point
+exports.bbsMain	= bbsMain;
 
 function bbsMain() {
 	async.waterfall(
 		[
 			function processArgs(callback) {
-				const args = parseArgs();
+				const args = process.argv.slice(2);
 
 				var configPath;
 
 				if(args.indexOf('--help') > 0) {
 					//	:TODO: display help
 				} else {
-					var argCount = args.length;
+					let argCount = args.length;
-					for(var i = 0; i < argCount; ++i) {
+					for(let i = 0; i < argCount; ++i) {
-						var arg = args[i];
+						const arg = args[i];
-						if('--config' == arg) {
+						if('--config' === arg) {
 							configPath = args[i + 1];
 						}
 					}
@@ -70,25 +71,19 @@ function bbsMain() {
 					}
 					callback(err);
 				});
+			},
+			function listenConnections(callback) {
+				startListening(callback);
 			}
 		],
 		function complete(err) {
-			if(!err) {
+			if(err) {
-				startListening();
+				logger.log.error(err);
 			}
 		}
 	);
 }
 
-function parseArgs() {
-	var args = [];
-	process.argv.slice(2).forEach(function(val, index, array) {
-		args.push(val);
-	});
-
-	return args;
-}
-
 function initialize(cb) {
 	async.series(
 		[
@@ -171,6 +166,9 @@ function initialize(cb) {
 						});
 					}
 				});
+			},
+			function readyMessageNetworkSupport(callback) {
+				require('./msg_network.js').startup(callback);	
 			}
 		],
 		function onComplete(err) {
@@ -179,29 +177,30 @@ function initialize(cb) {
 	);
 }
 
-function startListening() {
+function startListening(cb) {
 	if(!conf.config.servers) {
 		//	:TODO: Log error ... output to stderr as well. We can do it all with the logger
-		logger.log.error('No servers configured');
+		//logger.log.error('No servers configured');
-		return [];
+		cb(new Error('No servers configured'));
+		return;
 	}
 
-	var moduleUtil = require('./module_util.js');	//	late load so we get Config
+	let moduleUtil = require('./module_util.js');	//	late load so we get Config
 
-	moduleUtil.loadModulesForCategory('servers', function onServerModule(err, module) {
+	moduleUtil.loadModulesForCategory('servers', (err, module) => {
 		if(err) {
 			logger.log.info(err);
 			return;
 		}
 
-		var port = parseInt(module.runtime.config.port);
+		const port = parseInt(module.runtime.config.port);
 		if(isNaN(port)) {
 			logger.log.error( { port : module.runtime.config.port, server : module.moduleInfo.name }, 'Cannot load server (Invalid port)');
 			return;
 		}
 
-		var moduleInst = new module.getModule();
+		const moduleInst	= new module.getModule();
-		var server = moduleInst.createServer();
+		const server		= moduleInst.createServer();
 
 		//	:TODO: handle maxConnections, e.g. conf.maxConnections
 
@@ -262,7 +261,11 @@ function startListening() {
 		});
 
 		server.listen(port);
-		logger.log.info({ server : module.moduleInfo.name, port : port }, 'Listening for connections');
+
+		logger.log.info(
+			{ server : module.moduleInfo.name, port : port }, 'Listening for connections');
+	}, err => {
+		cb(err);
 	});
 }
 

core/config.js

@@ -228,12 +228,6 @@ function getDefaultConfig() {
 			}
 		},
 
-		messages : {
-			areas : [
-				{ name : 'private_mail', desc : 'Private Email', groups : [ 'users' ] }
-			]
-		},
-
 		networks : {
 			/*
 			networkName : {	//	e.g. fidoNet
@@ -248,6 +242,17 @@ function getDefaultConfig() {
 			*/
 		},
 		
+		scannerTossers : {
+			ftn_bso : {
+				paths : {
+
+				},
+
+				maxPacketByteSize : 256000,
+				maxBundleByteSize : 256000,
+			}
+		},
+
 		misc : {
 			idleLogoutSeconds	: 60 * 6,	//	6m
 		},

core/fse.js

@@ -309,7 +309,7 @@ function FullScreenEditorModule(options) {
 						//	in NetRunner:
 						self.client.term.rawWrite(ansi.reset() + ansi.deleteLine(3));
 						
-						//self.client.term.rawWrite(ansi.reset() + ansi.eraseLine(2))
+						self.client.term.rawWrite(ansi.reset() + ansi.eraseLine(2))
 					}
 					callback(null);
 				},

core/ftn_address.js

@@ -34,7 +34,7 @@ module.exports = class Address {
 			);
 	}
 
-	isMatch(pattern) {
+	getMatchAddr(pattern) {
 		const m = FTN_PATTERN_REGEXP.exec(pattern);
 		if(m) {
 			let addr = { };
@@ -81,6 +81,35 @@ module.exports = class Address {
 				addr.domain = '*';
 			}
 
+			return addr;
+		}
+	}
+
+	/*
+	getMatchScore(pattern) {
+		let score = 0;
+		const addr = this.getMatchAddr(pattern);
+		if(addr) {
+			const PARTS = [ 'net', 'node', 'zone', 'point', 'domain' ];
+			for(let i = 0; i < PARTS.length; ++i) {
+				const member = PARTS[i];
+				if(this[member] === addr[member]) {
+					score += 2;
+				} else if('*' === addr[member]) {
+					score += 1;
+				} else {
+					break;
+				}
+			}
+		}
+
+		return score;
+	}
+	*/
+
+	isMatch(pattern) {
+		const addr = this.getMatchAddr(pattern);
+		if(addr) {
 			return (
 				('*' === addr.net || this.net === addr.net) &&
 				('*' === addr.node || this.node === addr.node) &&

core/ftn_mail_packet.js

@@ -1,7 +1,6 @@
 /* jslint node: true */
 'use strict';
 
-//var MailPacket		= require('./mail_packet.js');
 let ftn				= require('./ftn_util.js');
 let Message			= require('./message.js');
 let sauce			= require('./sauce.js');
@@ -18,7 +17,6 @@ let iconv			= require('iconv-lite');
 let buffers			= require('buffers');
 let moment			= require('moment');
 
-exports.PacketHeader	= PacketHeader;
 exports.Packet			= Packet;
 
 /*
@@ -40,6 +38,117 @@ const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00');
 
 const FTN_MESSAGE_KLUDGE_PREFIX	= '\x01';
 
+class PacketHeader {
+	constructor(origAddr, destAddr, created, version) {
+		const EMPTY_ADDRESS = {
+			node	: 0,
+			net		: 0,
+			zone	: 0,
+			point	: 0,
+		};
+
+		this.packetVersion = version || '2+';
+
+		this.origAddress 	= origAddr || EMPTY_ADDRESS;
+		this.destAddress 	= destAddr || EMPTY_ADDRESS;
+		this.created		= created || moment();
+
+		//	uncommon to set the following explicitly
+		this.prodCodeLo		= 0xfe;	//	http://ftsc.org/docs/fta-1005.003
+		this.prodRevLo		= 0;
+		this.baud			= 0;
+		this.packetType		= FTN_PACKET_HEADER_TYPE;
+		this.password		= '';
+		this.prodData 		= 0x47694e45;	//	"ENiG"
+
+		this.capWord			= 0x0001;
+		this.capWordValidate	= ((this.capWord & 0xff) << 8) | ((this.capWord >> 8) & 0xff);
+	}
+
+	get origAddress() {
+		let addr = new Address({
+			node	: this.origNode,
+			zone	: this.origZone,
+		});
+
+		if(this.origPoint) {
+			addr.point	= this.origPoint;
+			addr.net	= this.auxNet;
+		} else {
+			addr.net	= this.origNet;
+		}
+
+		return addr;
+	}
+
+	set origAddress(address) {
+		if(_.isString(address)) {
+			address = Address.fromString(address);
+		}
+
+		this.origNode = address.node;
+
+		//	See FSC-48
+		if(address.point) {
+			this.origNet	= -1;
+			this.auxNet		= address.net;
+		} else {
+			this.origNet	= address.net;
+			this.auxNet		= 0;
+		}
+
+		this.origZone	= address.zone;
+		this.origZone2	= address.zone;
+		this.origPoint	= address.point || 0;
+	}
+
+	get destAddress() {
+		let addr = new Address({
+			node	: this.destNode,
+			net		: this.destNet,
+			zone	: this.destZone,
+		});
+
+		if(this.destPoint) {
+			addr.point = this.destPoint;
+		}
+
+		return addr;
+	}
+
+	set destAddress(address) {
+		if(_.isString(address)) {
+			address = Address.fromString(address);
+		}
+
+		this.destNode	= address.node;
+		this.destNet	= address.net;
+		this.destZone	= address.zone;
+		this.destZone2	= address.zone;
+		this.destPoint	= address.point || 0;
+	}
+
+	get created() {
+		return moment(this);	//	use year, month, etc. properties
+	}
+
+	set created(momentCreated) {
+		if(!moment.isMoment(momentCreated)) {
+			created = moment(momentCreated);
+		}
+
+		this.year	= momentCreated.year();
+		this.month	= momentCreated.month();
+		this.day	= momentCreated.day();
+		this.hour	= momentCreated.hour();
+		this.minute	= momentCreated.minute();
+		this.second	= momentCreated.second();
+	}
+}
+
+exports.PacketHeader = PacketHeader;
+
+/*
 function PacketHeader(options) {
 }
 
@@ -120,7 +229,7 @@ PacketHeader.prototype.setCreated = function(created) {
 PacketHeader.prototype.setPassword = function(password) {
 	this.password = password.substr(0, 8);
 }
-
+*/
 
 //
 //	Read/Write FTN packets with support for the following formats:
@@ -762,7 +871,7 @@ Packet.prototype.write = function(path, packetHeader, messages) {
 		);
 };
 
-
+/*
 const LOCAL_ADDRESS = {
 	zone	: 46,
 	net		: 1,
@@ -776,9 +885,8 @@ const REMOTE_ADDRESS = {
 	node : 218,
 };
 
-var packetHeader1 = new PacketHeader();
+
-packetHeader1.init(LOCAL_ADDRESS, REMOTE_ADDRESS);
+var packetHeader1 = new PacketHeader(LOCAL_ADDRESS, REMOTE_ADDRESS);
-console.log(packetHeader1);
 
 var packet = new Packet();
 var theHeader;
@@ -796,23 +904,13 @@ packet.read(
 
 			messagesToWrite.push(msg);
 
-			/*
-			if(!written) {
-				written = true;
-
-				let messages = [ msg ];
-				Packet.write('/home/nuskooler/Downloads/ftnout/test1.pkt', theHeader, messages, err => {
-
-				});
-
-			}*/
-
 			let address = {
 				zone	: 46,
 				net		: 1,
 				node	: 232,
 				domain	: 'l33t.codes',
 			};
+			
 			msg.areaTag = 'agn_bbs';
 			msg.messageId = 1234;
 			console.log(ftn.getMessageIdentifier(msg, address));
@@ -825,6 +923,9 @@ packet.read(
 				msg.meta.FtnProperty.ftn_seen_by, '1/107 4/22 4/25 4/10'));
 			console.log(ftn.getUpdatedPathEntries(
 				msg.meta.FtnKludge['PATH'], '1:365/50'))
+			console.log('Via: ' + ftn.getVia(address))
+			console.log(Address.fromString('46:1/232.4@l33t.codes').isMatch('*:1/232.*'))
+			//console.log(Address.fromString('46:1/232.4@l33t.codes').getMatchScore('46:1/232'))
 		}
 	},
 	function completion(err) {
@@ -837,3 +938,4 @@ packet.read(
 		});
 	}
 );
+*/

core/ftn_util.js

@@ -28,6 +28,7 @@ exports.getProductIdentifier		= getProductIdentifier;
 exports.getUTCTimeZoneOffset		= getUTCTimeZoneOffset;
 exports.getOrigin					= getOrigin;
 exports.getTearLine					= getTearLine;
+exports.getVia						= getVia;
 exports.getAbbreviatedNetNodeList	= getAbbreviatedNetNodeList;
 exports.parseAbbreviatedNetNodeList	= parseAbbreviatedNetNodeList;
 exports.getUpdatedSeenByEntries		= getUpdatedSeenByEntries;
@@ -181,6 +182,9 @@ function getMessageIdentifier(message, address) {
 //	Return a FSC-0046.005 Product Identifier or "PID"
 //	http://ftsc.org/docs/fsc-0046.005
 //
+//	Note that we use a variant on the spec for <serial>
+//	in which (<os>; <arch>; <nodeVer>) is used instead
+//
 function getProductIdentifier() {
 	const version = packageJson.version
 		.replace(/\-/g, '.')
@@ -235,6 +239,28 @@ function getTearLine() {
 	return `--- ENiGMA 1/2 v{$packageJson.version} (${os.platform()}; ${os.arch()}; ${nodeVer})`;
 }
 
+//
+//	Return a FRL-1005.001 "Via" line
+//	http://ftsc.org/docs/frl-1005.001
+//
+function getVia(address) {
+	/*
+		FRL-1005.001 states teh following format:
+
+		^AVia: <FTN Address> @YYYYMMDD.HHMMSS[.Precise][.Time Zone] 
+	    <Program Name> <Version> [Serial Number]<CR>
+	*/
+	const addrStr	= new Address(address).toString('5D');
+	const dateTime	= moment().utc().format('YYYYMMDD.HHmmSS.SSSS.UTC');
+
+	const version	= packageJson.version
+		.replace(/\-/g, '.')
+		.replace(/alpha/,'a')
+		.replace(/beta/,'b');
+
+	return `${addrStr} @${dateTime} ENiGMA1/2 ${version}`;
+}
+
 function getAbbreviatedNetNodeList(netNodes) {
 	let abbrList = '';
 	let currNet;

core/message_area.js

@@ -1,15 +1,16 @@
 /* jslint node: true */
 'use strict';
 
-var msgDb			= require('./database.js').dbs.message;
+let msgDb			= require('./database.js').dbs.message;
-var Config			= require('./config.js').config;
+let Config			= require('./config.js').config;
-var Message			= require('./message.js');
+let Message			= require('./message.js');
-var Log				= require('./logger.js').log;
+let Log				= require('./logger.js').log;
-var checkAcs        = require('./acs_util.js').checkAcs;
+let checkAcs        = require('./acs_util.js').checkAcs;
+let msgNetRecord	= require('./msg_network.js').recordMessage;
 
-var async			= require('async');
+let async			= require('async');
-var _				= require('lodash');
+let _				= require('lodash');
-var assert			= require('assert');
+let assert			= require('assert');
 
 exports.getAvailableMessageConferences      = getAvailableMessageConferences;
 exports.getSortedAvailMessageConferences	= getSortedAvailMessageConferences;
@@ -439,3 +440,17 @@ function updateMessageAreaLastReadId(userId, areaTag, messageId, cb) {
 		}
 	);
 }
+
+function persistMessage(message, cb) {
+	async.series(
+		[
+			function persistMessageToDisc(callback) {
+				message.persist(callback);
+			},
+			function recordToMessageNetworks(callback) {
+				msgNetRecord(message, callback);
+			}
+		],
+		cb
+	);
+}

core/module_util.js

@@ -1,20 +1,22 @@
 /* jslint node: true */
 'use strict';
 
-var Config		= require('./config.js').config;
+//	ENiGMA½
-var miscUtil	= require('./misc_util.js');
+let Config		= require('./config.js').config;
+let miscUtil	= require('./misc_util.js');
 
-var fs 			= require('fs');
+//	standard/deps
-var paths		= require('path');
+let fs 			= require('fs');
-var _			= require('lodash');
+let paths		= require('path');
-var assert		= require('assert');
+let _			= require('lodash');
+let assert		= require('assert');
+let async		= require('async');
 
 //	exports
 exports.loadModuleEx			= loadModuleEx;
 exports.loadModule				= loadModule;
 exports.loadModulesForCategory	= loadModulesForCategory;
 
-
 function loadModuleEx(options, cb) {
 	assert(_.isObject(options));
 	assert(_.isString(options.name));
@@ -44,7 +46,7 @@ function loadModuleEx(options, cb) {
 		return;
 	}
 
-		//	Ref configuration, if any, for convience to the module
+	//	Ref configuration, if any, for convience to the module
 	mod.runtime = { config : modConfig };
 
 	cb(null, mod);
@@ -63,18 +65,27 @@ function loadModule(name, category, cb) {
 	});
 }
 
-function loadModulesForCategory(category, iterator) {
+function loadModulesForCategory(category, iterator, complete) {
-	var path = Config.paths[category];
 	
-	fs.readdir(path, function onFiles(err, files) {
+	fs.readdir(Config.paths[category], (err, files) => {
 		if(err) {
-			cb(err);
+			iterator(err);
 			return;
 		}
 
-		var filtered = files.filter(function onFilter(file) { return '.js' === paths.extname(file); });
+		const jsModules = files.filter(file => {
-		filtered.forEach(function onFile(file) {
+			return '.js' === paths.extname(file);
-			loadModule(paths.basename(file, '.js'), category, iterator);
+		});
+
+		async.each(jsModules, (file, next) => {
+			loadModule(paths.basename(file, '.js'), category, (err, mod) => {
+				iterator(err, mod);
+				next();
+			});
+		}, err => {
+			if(complete) {
+				complete(err);
+			}
 		});
 	});
 }

core/msg_network.js

@@ -0,0 +1,60 @@
+/* jslint node: true */
+'use strict';
+
+//	ENiGMA½
+let loadModulesForCategory	= require('./module_util.js').loadModulesForCategory;
+
+//	standard/deps
+let async					= require('async');
+
+exports.startup			= startup
+exports.shutdown		= shutdown;
+exports.recordMessage	= recordMessage;
+
+let msgNetworkModules = [];
+
+function startup(cb) {
+	async.series(
+		[
+			function loadModules(callback) {
+				loadModulesForCategory('scannerTossers', (err, module) => {
+					if(!err) {
+						const modInst = new module.getModule();
+
+						modInst.startup(err => {
+							if(!err) {
+								msgNetworkModules.push(modInst);
+							}
+						});
+					}
+				}, err => {
+					callback(err);
+				});
+			}
+		],
+		cb
+	);
+}
+
+function shutdown() {
+	msgNetworkModules.forEach(mod => {
+		mod.shutdown();
+	});
+
+	msgNetworkModules = [];
+}
+
+function recordMessage(message, cb) {
+	//
+	//	Give all message network modules (scanner/tossers)
+	//	a chance to do something with |message|. Any or all can
+	//	choose to ignore it.
+	//
+	async.each(msgNetworkModules, (modInst, next) => {
+		modInst.record(message, err => {
+			next();
+		});
+	}, err => {
+		cb(err);
+	});
+}

core/scanner_tossers/ftn_bso.js

@@ -2,12 +2,15 @@
 'use strict';
 
 //	ENiGMA½
-let MessageScanTossModule		= require('../scan_toss_module.js').MessageScanTossModule;
+let MessageScanTossModule	= require('../msg_scan_toss_module.js').MessageScanTossModule;
-let Config						= require('../config.js').config;
+let Config					= require('../config.js').config;
-let ftnMailpacket				= require('../ftn_mail_packet.js');
+let ftnMailpacket			= require('../ftn_mail_packet.js');
-let ftnUtil						= require('../ftn_util.js');
+let ftnUtil					= require('../ftn_util.js');
+let Address					= require('../ftn_address.js');
+let Log						= require('../logger.js').log;
 
-let moment						= require('moment');
+let moment					= require('moment');
+let _						= require('lodash');
 
 exports.moduleInfo = {
 	name	: 'FTN',
@@ -20,7 +23,9 @@ exports.getModule = FTNMessageScanTossModule;
 function FTNMessageScanTossModule() {
 	MessageScanTossModule.call(this);
 
-	this.config = Config.scannerTossers.ftn_bso;
+	if(_.has(Config, 'scannerTossers.ftn_bso')) {
+		this.config = Config.scannerTossers.ftn_bso;
+	}
 
 	this.createMessagePacket = function(message, config) {
 		this.prepareMessage(message);
@@ -50,10 +55,18 @@ function FTNMessageScanTossModule() {
 		message.meta.FtnProperty.ftn_tear_line		= ftnUtil.getTearLine();
 		message.meta.FtnProperty.ftn_origin			= ftnUtil.getOrigin(config.network.localAddress);
 
-		if(message.areaTag) {
+		if(message.isPrivate()) {
-			message.meta.FtnProperty.ftn_area = message.areaTag;
+			//
+			//	NetMail messages need a FRL-1005.001 "Via" line
+			//	http://ftsc.org/docs/frl-1005.001
+			//
+			if(_.isString(message.meta.FtnKludge['Via'])) {
+				message.meta.FtnKludge['Via'] = [ message.meta.FtnKludge['Via'] ];
+			}
+			message.meta.FtnKludge['Via'] = message.meta.FtnKludge['Via'] || [];
+			message.meta.FtnKludge['Via'].push(ftnUtil.getVia(config.network.localAddress));
 		} else {
-			//	:TODO: add "Via" line -- FSP-1010
+			message.meta.FtnProperty.ftn_area = message.areaTag;
 		}
 
 		//
@@ -81,14 +94,42 @@ function FTNMessageScanTossModule() {
 require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule);
 
 FTNMessageScanTossModule.prototype.startup = function(cb) {
+	Log.info('FidoNet Scanner/Tosser starting up');
+
 	cb(null);
 };
 
 FTNMessageScanTossModule.prototype.shutdown = function(cb) {
+	Log.info('FidoNet Scanner/Tosser shutting down');
+
 	cb(null);
 };
 
 FTNMessageScanTossModule.prototype.record = function(message, cb) {
+	if(!_.has(Config, [ 'messageNetworks', 'ftn', 'areas', message.areaTag ])) {
+		return;
+	}
+
+	const area = Config.messageNetworks.ftn.areas[message.areaTag];
+	if(!_.isString(area.ftnArea) || !_.isArray(area.uplinks)) {
+		//	:TODO: should probably log a warning here
+		return;
+	}
+
+	//
+	//	For each uplink, find the best configuration match
+	//
+	area.uplinks.forEach(uplink => {
+		//	:TODO: sort by least # of '*' & take top?
+		let matchNodes = _.filter(Object.keys(Config.scannerTossers.ftn_bso.nodes), addr => {
+			return Address.fromString(addr).isMatch(uplink);
+		});
+
+		if(matchNodes.length > 0) {
+			const nodeKey = matchNodes[0];
+
+		}
+	});
 
 
 	cb(null);