NetMail non-HUB fixes

* Properly separate FTN *packet* header vs *message* header DST/SRC information
* Change routes{} handling: These are now *require* for out-of-HUB routing such that Enig will know where to send messages

core/ftn_mail_packet.js

@@ -524,13 +524,13 @@ function Packet(options) {
 		);
 	};
 
-	this.parsePacketMessages = function(packetBuffer, iterator, cb) {
+	this.parsePacketMessages = function(header, packetBuffer, iterator, cb) {
 		binary.parse(packetBuffer)
 			.word16lu('messageType')
-			.word16lu('ftn_orig_node')
+			.word16lu('ftn_msg_orig_node')
-			.word16lu('ftn_dest_node')
+			.word16lu('ftn_msg_dest_node')
-			.word16lu('ftn_orig_network')
+			.word16lu('ftn_msg_orig_net')
-			.word16lu('ftn_dest_network')
+			.word16lu('ftn_msg_dest_net')
 			.word16lu('ftn_attr_flags')
 			.word16lu('ftn_cost')
 			.scan('modDateTime', NULL_TERM_BUFFER)	//	:TODO: 20 bytes max
@@ -569,20 +569,28 @@ function Packet(options) {
 				//	contain an origin line, kludges, SAUCE in the case
 				//	of ANSI files, etc.
 				//
-				let msg = new Message( {
+				const msg = new Message( {
 					toUserName		: convMsgData.toUserName,
 					fromUserName	: convMsgData.fromUserName,
 					subject			: convMsgData.subject,
 					modTimestamp	: ftn.getDateFromFtnDateTime(convMsgData.modDateTime),
 				});
 
-				msg.meta.FtnProperty = {};
+				//	:TODO: When non-private (e.g. EchoMail), attempt to extract SRC from MSGID vs headers, when avail (or Orgin line? research further)
-				msg.meta.FtnProperty.ftn_orig_node		= msgData.ftn_orig_node;
+				msg.meta.FtnProperty = {
-				msg.meta.FtnProperty.ftn_dest_node		= msgData.ftn_dest_node;
+					ftn_orig_node		: header.origNode,
-				msg.meta.FtnProperty.ftn_orig_network 	= msgData.ftn_orig_network;
+					ftn_dest_node		: header.destNode,
-				msg.meta.FtnProperty.ftn_dest_network	= msgData.ftn_dest_network;
+					ftn_orig_network	: header.origNet,
-				msg.meta.FtnProperty.ftn_attr_flags		= msgData.ftn_attr_flags;
+					ftn_dest_network	: header.destNet,
-				msg.meta.FtnProperty.ftn_cost			= msgData.ftn_cost;
+
+					ftn_attr_flags		: msgData.ftn_attr_flags,
+					ftn_cost			: msgData.ftn_cost,
+
+					ftn_msg_orig_node	: msgData.ftn_msg_orig_node,
+					ftn_msg_dest_node	: msgData.ftn_msg_dest_node,
+					ftn_msg_orig_net	: msgData.ftn_msg_orig_net,
+					ftn_msg_dest_net	: msgData.ftn_msg_dest_net,
+				};
 
 				self.processMessageBody(msgData.message, messageBodyData => {
 					msg.message 		= messageBodyData.message;
@@ -622,11 +630,11 @@ function Packet(options) {
 
 					const nextBuf = packetBuffer.slice(read);
 					if(nextBuf.length > 0) {
-						let next = function(e) {
+						const next = function(e) {
 							if(e) {
 								cb(e);
 							} else {
-								self.parsePacketMessages(nextBuf, iterator, cb);
+								self.parsePacketMessages(header, nextBuf, iterator, cb);
 							}
 						};
 
@@ -651,6 +659,10 @@ function Packet(options) {
 			Message.FtnPropertyNames.FtnOrigPoint,
 			Message.FtnPropertyNames.FtnDestPoint,
 			Message.FtnPropertyNames.FtnAttribute,
+			Message.FtnPropertyNames.FtnMsgOrigNode,
+			Message.FtnPropertyNames.FtnMsgDestNode,
+			Message.FtnPropertyNames.FtnMsgOrigNet,
+			Message.FtnPropertyNames.FtnMsgDestNet,
 		].forEach( propName => {
 			if(message.meta.FtnProperty[propName]) {
 				message.meta.FtnProperty[propName] = parseInt(message.meta.FtnProperty[propName]) || 0;
@@ -658,6 +670,25 @@ function Packet(options) {
 		});
 	};
 
+	this.writeMessageHeader = function(message, buf) {
+		//	ensure address FtnProperties are numbers
+		self.sanatizeFtnProperties(message);
+
+		const destNode	= message.meta.FtnProperty.ftn_msg_dest_node || message.meta.FtnProperty.ftn_dest_node;
+		const destNet	= message.meta.FtnProperty.ftn_msg_dest_net || message.meta.FtnProperty.ftn_dest_network;
+
+		buf.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0);
+		buf.writeUInt16LE(message.meta.FtnProperty.ftn_orig_node, 2);
+		buf.writeUInt16LE(destNode, 4);
+		buf.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6);
+		buf.writeUInt16LE(destNet, 8);
+		buf.writeUInt16LE(message.meta.FtnProperty.ftn_attr_flags, 10);
+		buf.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12);
+
+		const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0');
+		dateTimeBuffer.copy(buf, 14);
+	};
+
 	this.getMessageEntryBuffer = function(message, options, cb) {
 
 		function getAppendMeta(k, m, sepChar=':') {
@@ -678,20 +709,7 @@ function Packet(options) {
 			[
 				function prepareHeaderAndKludges(callback) {
 					const basicHeader = new Buffer(34);
-
+					self.writeMessageHeader(message, basicHeader);			
-					//	ensure address FtnProperties are numbers
-					self.sanatizeFtnProperties(message);
-
-					basicHeader.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0);
-					basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_node, 2);
-					basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_node, 4);
-					basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6);
-					basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_network, 8);
-					basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_attr_flags, 10);
-					basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12);
-
-					const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0');
-					dateTimeBuffer.copy(basicHeader, 14);
 
 					//
 					//	To, from, and subject must be NULL term'd and have max lengths as per spec.
@@ -808,17 +826,7 @@ function Packet(options) {
 
 	this.writeMessage = function(message, ws, options) {
 		let basicHeader = new Buffer(34);
-
+		self.writeMessageHeader(message, basicHeader);
-		basicHeader.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0);
-		basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_node, 2);
-		basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_node, 4);
-		basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6);
-		basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_network, 8);
-		basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_attr_flags, 10);
-		basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12);
-
-		const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0');
-		dateTimeBuffer.copy(basicHeader, 14);
 
 		ws.write(basicHeader);
 
@@ -911,7 +919,7 @@ function Packet(options) {
 	};
 
 	this.parsePacketBuffer = function(packetBuffer, iterator, cb) {
-		async.series(
+		async.waterfall(
 			[
 				function processHeader(callback) {
 					self.parsePacketHeader(packetBuffer, (err, header) => {
@@ -919,15 +927,16 @@ function Packet(options) {
 							return callback(err);
 						}
 
-						let next = function(e) {
+						const next = function(e) {
-							callback(e);
+							return callback(e, header);
 						};
 
 						iterator('header', header, next);
 					});
 				},
-				function processMessages(callback) {
+				function processMessages(header, callback) {
 					self.parsePacketMessages(
+						header,
 						packetBuffer.slice(FTN_PACKET_HEADER_SIZE),
 						iterator,
 						callback);

core/message.js

@@ -116,8 +116,10 @@ Message.StateFlags0 = {
 };
 
 Message.FtnPropertyNames = {
+	//	packet header oriented
 	FtnOrigNode			: 'ftn_orig_node',
 	FtnDestNode			: 'ftn_dest_node',
+	//	:TODO: rename these to ftn_*_net vs network - ensure things won't break, may need mapping
 	FtnOrigNetwork		: 'ftn_orig_network',
 	FtnDestNetwork		: 'ftn_dest_network',
 	FtnAttrFlags		: 'ftn_attr_flags',
@@ -127,6 +129,12 @@ Message.FtnPropertyNames = {
 	FtnOrigPoint		: 'ftn_orig_point',
 	FtnDestPoint		: 'ftn_dest_point',
 
+	//	message header oriented
+	FtnMsgOrigNode		: 'ftn_msg_orig_node',
+	FtnMsgDestNode		: 'ftn_msg_dest_node',
+	FtnMsgOrigNet		: 'ftn_msg_orig_net',
+	FtnMsgDestNet		: 'ftn_msg_dest_net',
+
 	FtnAttribute		: 'ftn_attribute',
 
 	FtnTearLine			: 'ftn_tear_line',		//	http://ftsc.org/docs/fts-0004.001

core/scanner_tossers/ftn_bso.js

@@ -306,12 +306,26 @@ function FTNMessageScanTossModule() {
 		//
 		const localAddress = new Address(options.network.localAddress);	//	ensure we have an Address obj not a string version
 
+		//	:TODO: create Address.toMeta() / similar
 		message.meta.FtnProperty = message.meta.FtnProperty || {};
 		message.meta.FtnKludge = message.meta.FtnKludge || {};
 
 		message.meta.FtnProperty.ftn_orig_node			= localAddress.node;
 		message.meta.FtnProperty.ftn_orig_network		= localAddress.net;
 		message.meta.FtnProperty.ftn_cost				= 0;
+		message.meta.FtnProperty.ftn_msg_orig_node		= localAddress.node;
+		message.meta.FtnProperty.ftn_msg_orig_net	= localAddress.net;
+
+		const destAddress = options.routeAddress || options.destAddress;
+		message.meta.FtnProperty.ftn_dest_node		= destAddress.node;
+		message.meta.FtnProperty.ftn_dest_network	= destAddress.net;
+
+		if(destAddress.zone) {
+			message.meta.FtnProperty.ftn_dest_zone	= destAddress.zone;
+		}
+		if(destAddress.point) {
+			message.meta.FtnProperty.ftn_dest_point	= destAddress.point;
+		}
 
 		//	tear line and origin can both go in EchoMail & NetMail
 		message.meta.FtnProperty.ftn_tear_line		= ftnUtil.getTearLine();
@@ -320,9 +334,11 @@ function FTNMessageScanTossModule() {
 		let ftnAttribute = ftnMailPacket.Packet.Attribute.Local;	//	message from our system
 
 		if(self.isNetMailMessage(message)) {
-			//	These should be set for Private/NetMail already
+			//
-			assert(_.isNumber(parseInt(message.meta.FtnProperty.ftn_dest_node)));
+			//	Set route and message destination properties -- they may differ
-			assert(_.isNumber(parseInt(message.meta.FtnProperty.ftn_dest_network)));
+			//
+			message.meta.FtnProperty.ftn_msg_dest_node		= options.destAddress.node;
+			message.meta.FtnProperty.ftn_msg_dest_net	= options.destAddress.net;
 
 			ftnAttribute |= ftnMailPacket.Packet.Attribute.Private;
 
@@ -353,10 +369,6 @@ function FTNMessageScanTossModule() {
 				message.meta.FtnKludge.TOPT = options.destAddress.point;
 			}
 		} else {
-			//	We need to set some destination info for EchoMail
-			message.meta.FtnProperty.ftn_dest_node		= options.destAddress.node;
-			message.meta.FtnProperty.ftn_dest_network	= options.destAddress.net;
-
 			//
 			//	Set appropriate attribute flag for export type
 			//
@@ -573,7 +585,7 @@ function FTNMessageScanTossModule() {
 
 					const packetHeader = new ftnMailPacket.PacketHeader(
 						exportOpts.network.localAddress,
-						exportOpts.destAddress,
+						exportOpts.routeAddress,
 						exportOpts.nodeConfig.packetType
 					);
 
@@ -801,57 +813,44 @@ function FTNMessageScanTossModule() {
 		return _.find(routes, (route, addrWildcard) => {
 			return dstAddr.isPatternMatch(addrWildcard);
 		});
-
-		/*
-		const route = _.find(routes, (route, addrWildcard) => {
-			return dstAddr.isPatternMatch(addrWildcard);
-		});
-
-		if(route && route.address) {
-			return Address.fromString(route.address);
-		}
-		*/
 	};
 
-	this.getAcceptableNetMailNetworkInfoFromAddress = function(dstAddr, cb) {
+	this.getNetMailRouteInfoFromAddress = function(destAddress, cb) {
 		//
-		//	Attempt to find an acceptable network configuration using the following
+		//	Attempt to find route information for |destAddress|:
-		//	lookup order (most to least explicit config):
 		//
 		//	1) Routes: messageNetworks.ftn.netMail.routes{} -> scannerTossers.ftn_bso.nodes{} -> config
-		//		- Where we send may not be where dstAddress is (it's routed!)
+		//		- Where we send may not be where destAddress is (it's routed!)
 		//	2) Direct to nodes: scannerTossers.ftn_bso.nodes{} -> config
-		//		- Where we send is direct to dstAddr
+		//		- Where we send is direct to destAddress
 		//
 		//	In both cases, attempt to look up Zone:Net/* to discover local "from" network/address
 		//	falling back to Config.scannerTossers.ftn_bso.defaultNetwork
 		//
-		const route = this.getNetMailRoute(dstAddr);
+		const route = this.getNetMailRoute(destAddress);
 
 		let routeAddress;
 		let networkName;
+		let isRouted;
 		if(route) {
 			routeAddress	= Address.fromString(route.address);
 			networkName		= route.network;
+			isRouted		= true;
 		} else {
-			routeAddress = dstAddr;
+			routeAddress	= destAddress;
+			isRouted		= false;
 		}
 
-		networkName = networkName || 
+		networkName = networkName || this.getNetworkNameByAddress(routeAddress);
-			this.getNetworkNameByAddressPattern(`${routeAddress.zone}:${routeAddress.net}/*`) ||
-			Config.scannerTossers.ftn_bso.defaultNetwork
-			;
 
 		const config = _.find(this.moduleConfig.nodes, (node, nodeAddrWildcard) => {
 			return routeAddress.isPatternMatch(nodeAddrWildcard);
-		}) || {
+		}) || { packetType : '2+', encoding	: Config.scannerTossers.ftn_bso.packetMsgEncoding };
-			packetType	: '2+',
-			encoding	: Config.scannerTossers.ftn_bso.packetMsgEncoding,
-		};
 
+		//	we should never be failing here; we may just be using defaults.
 		return cb(
-			config ? null : Errors.DoesNotExist(`No configuration found for ${dstAddr.toString()}`),
+			networkName ? null : Errors.DoesNotExist(`No NetMail route for ${destAddress.toString()}`),
-			config, routeAddress, networkName
+			{ destAddress, routeAddress, networkName, config, isRouted }
 		);
 	};
 
@@ -876,21 +875,22 @@ function FTNMessageScanTossModule() {
 					function discoverUplink(callback) {
 						const dstAddr = new Address(message.meta.System[Message.SystemMetaNames.RemoteToUser]);
 
-						return self.getAcceptableNetMailNetworkInfoFromAddress(dstAddr, (err, config, routeAddress, networkName) => {
+						self.getNetMailRouteInfoFromAddress(dstAddr, (err, routeInfo) => {
 							if(err) {
 								return callback(err);
 							}
 
-							exportOpts.nodeConfig	= config;
+							exportOpts.nodeConfig	= routeInfo.config;
-							exportOpts.destAddress	= routeAddress;
+							exportOpts.destAddress	= dstAddr;
-							exportOpts.fileCase		= config.fileCase || 'lower';
+							exportOpts.routeAddress	= routeInfo.routeAddress;
-							exportOpts.network		= Config.messageNetworks.ftn.networks[networkName];
+							exportOpts.fileCase		= routeInfo.config.fileCase || 'lower';
-							exportOpts.networkName	= networkName;
+							exportOpts.network		= Config.messageNetworks.ftn.networks[routeInfo.networkName];
+							exportOpts.networkName	= routeInfo.networkName;
 							exportOpts.outgoingDir	= self.getOutgoingEchoMailPacketDir(exportOpts.networkName, exportOpts.destAddress);
-							exportOpts.exportType	= self.getExportType(config);
+							exportOpts.exportType	= self.getExportType(routeInfo.config);
 
 							if(!exportOpts.network) {
-								return callback(Errors.DoesNotExist(`No configuration found for network ${networkName}`));
+								return callback(Errors.DoesNotExist(`No configuration found for network ${routeInfo.networkName}`));
 							}
 
 							return callback(null);
@@ -937,12 +937,15 @@ function FTNMessageScanTossModule() {
 				],
 				err => {
 					if(err) {
-						Log.warn( { error :err.message }, 'Error exporting message' );
+						Log.warn( { error : err.message }, 'Error exporting message' );
 					}
 					return nextMessageOrUuid(null);
 				}
 			);
 		}, err => {
+			if(err) {
+				Log.warn( { error : err.message }, 'Error(s) during NetMail export');
+			}
 			return cb(err);
 		});
 	};
@@ -962,6 +965,7 @@ function FTNMessageScanTossModule() {
 				fileCase		: self.moduleConfig.nodes[nodeConfigKey].fileCase || 'lower',
 			};
 
+			
 			if(_.isString(exportOpts.network.localAddress)) {
 				exportOpts.network.localAddress = Address.fromString(exportOpts.network.localAddress);
 			}
@@ -2031,8 +2035,7 @@ function FTNMessageScanTossModule() {
 	this.isNetMailMessage = function(message) {
 		return message.isPrivate() &&
 			null === _.get(message, 'meta.System.LocalToUserID', null) &&
-			Message.AddressFlavor.FTN === _.get(message, 'meta.System.external_flavor', null)
+			Message.AddressFlavor.FTN === _.get(message, 'meta.System.external_flavor', null);
-			;
 	};
 }