* Fix messages with no origin line

* Fix end of message/termination detection for FTN packets
* Start of archive support -- one use will be FTN archives
* Work on FTN ArcMail/bundles

core/archive_util.js

@@ -0,0 +1,114 @@
+/* jslint node: true */
+'use strict';
+
+//	ENiGMA½
+let Config		= require('./config.js').config;
+
+//	base/modules
+let fs			= require('fs');
+let _			= require('lodash');
+let pty			= require('ptyw.js');
+
+module.exports = class ArchiveUtil {
+	
+	constructor() {
+		this.archivers = {};
+		this.longestSignature = 0;
+	}
+
+	init() {
+		//
+		//	Load configuration
+		//
+		if(_.has(Config, 'archivers')) {
+			Object.keys(Config.archivers).forEach(archKey => {
+				const arch = Config.archivers[archKey];
+				if(!_.isString(arch.sig) || 
+					!_.isString(arch.compressCmd) ||
+					!_.isString(arch.decompressCmd) ||
+					!_.isArray(arch.compressArgs) ||
+					!_.isArray(arch.decompressArgs))
+				{
+					//	:TODO: log warning
+					return;
+				}
+
+				const archiver = {
+					compressCmd		: arch.compressCmd,
+					compressArgs	: arch.compressArgs,
+					decompressCmd	: arch.decompressCmd,
+					decompressArgs	: arch.decompressArgs,
+					sig				: new Buffer(arch.sig, 'hex'),
+					offset			: arch.offset || 0,
+				};
+				
+				this.archivers[archKey] = archiver;
+				
+				if(archiver.offset + archiver.sig.length > this.longestSignature) {
+					this.longestSignature = archiver.offset + archiver.sig.length;
+				}
+			});
+		}
+	}
+
+	detectType(path, cb) {
+		fs.open(path, 'r', (err, fd) => {
+			if(err) {
+				cb(err);
+				return;
+			}
+			
+			let buf = new Buffer(this.longestSignature);
+			fs.read(fd, buf, 0, buf.length, 0, (err, bytesRead) => {
+				if(err) {
+					cb(err);
+					return;
+				}
+
+				//	return first match
+				const detected = _.find(this.archivers, arch => {
+					const lenNeeded = arch.offset + arch.sig.length;
+					
+					if(buf.length < lenNeeded) {
+						return false;
+					}
+
+					const comp = buf.slice(arch.offset, arch.offset + arch.sig.length);
+					return (arch.sig.equals(comp));
+				});
+
+				cb(detected ? null : new Error('Unknown type'), detected);
+			});			
+		});
+	}
+
+	compressTo(archType, archivePath, files, cb) {
+		const archiver = this.archivers[archType];
+		if(!archiver) {
+			cb(new Error('Unknown archive type: ' + archType));
+			return;
+		}
+
+		let args = _.clone(archiver.compressArgs);	//	don't much with orig
+		for(let i = 0; i < args.length; ++i) {
+			args[i] = args[i].format({
+				archivePath	: archivePath,
+				fileList	: files.join(' '),
+			});
+		}
+
+		let comp = pty.spawn(archiver.compressCmd, args, {
+			cols : 80,
+			rows : 24,
+			//	:TODO: cwd
+		});
+
+		comp.on('exit', exitCode => {
+			cb(0 === exitCode ? null : new Error('Compression failed with exit code: ' + exitCode));
+		});
+	}
+
+	extractTo(archivePath, extractPath, archType, cb) {
+
+	}
+}

core/config.js

@@ -209,6 +209,18 @@ function getDefaultConfig() {
 			}
 		},
 
+		archivers : {
+			zip : {
+				name			: "PKZip",
+				sig				: "504b0304",
+				offset			: 0,
+				compressCmd		: "7z",
+				compressArgs	: [ "a", "-tzip", "{archivePath}", "{fileList}" ],
+				decompressCmd	: "7z",
+				decompressArgs	: [ "e", "-o{extractDir}", "{archivePath}" ]
+			}
+		},
+
 		messageConferences : {
 			system_internal : {
 				name 	: 'System Internal',
@@ -231,13 +243,13 @@ function getDefaultConfig() {
 		scannerTossers : {
 			ftn_bso : {
 				paths : {
-					outbound	: paths.join(__dirname, './../mail/out/'),
+					outbound	: paths.join(__dirname, './../mail/ftn_out/'),
-					inbound		: paths.join(__dirname, './../mail/in/'),
+					inbound		: paths.join(__dirname, './../mail/ftn_in/'),
-					secInbound	: paths.join(__dirname, './../mail/secin/'),
+					secInbound	: paths.join(__dirname, './../mail/ftn_secin/'),
 				},
 
-				maxPacketByteSize : 256000,
+				maxPacketByteSize : 512000,		//	512k, before placing messages in a new pkt
-				maxBundleByteSize : 256000,
+				maxBundleByteSize : 2048000,	//	2M, before creating another archive
 			}
 		},
 

core/ftn_mail_packet.js

@@ -419,36 +419,34 @@ function Packet() {
 					//	Decode |messageBodyBuffer| using |encoding| defaulted or detected above
 					//
 					//	:TODO: Look into \xec thing more - document
-					const messageLines = iconv.decode(messageBodyBuffer, encoding).replace(/[\xec\n]/g, '').split(/\r/g);
+					const messageLines = iconv.decode(messageBodyBuffer, encoding).replace(/\xec/g, '').split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);
-					let preOrigin = true;
+					let endOfMessage = true;
 
 					messageLines.forEach(line => {
 						if(0 === line.length) {
 							messageBodyData.message.push('');
 							return;
 						}
-
+						
-						if(preOrigin) {
+						if(line.startsWith('AREA:')) {
-							if(line.startsWith('AREA:')) {
+							messageBodyData.area = line.substring(line.indexOf(':') + 1).trim();
-								messageBodyData.area = line.substring(line.indexOf(':') + 1).trim();
+						} else if(line.startsWith('--- ')) {
-							} else if(line.startsWith('--- ')) {
+							//	Tear Lines are tracked allowing for specialized display/etc.
-								//	Tear Lines are tracked allowing for specialized display/etc.
+							messageBodyData.tearLine = line;
-								messageBodyData.tearLine = line;
+						} else if(/[ ]{1,2}(\* )?Origin\: /.test(line)) {	//	To spec is "  * Origin: ..."
-							} else if(/[ ]{1,2}(\* )?Origin\: /.test(line)) {	//	To spec is "  * Origin: ..."
+							messageBodyData.originLine = line;
-								messageBodyData.originLine = line;
+							endOfMessage = false;	//	Anything past origin is not part of the message body
-								preOrigin = false;
+						} else if(line.startsWith('SEEN-BY:')) {
-							} else if(FTN_MESSAGE_KLUDGE_PREFIX === line.charAt(0)) {
+							endOfMessage = true;	//	Anything past the first SEEN-BY is not part of the message body
-								addKludgeLine(line.slice(1));
+							messageBodyData.seenBy.push(line.substring(line.indexOf(':') + 1).trim());
-							} else {
+						} else if(FTN_MESSAGE_KLUDGE_PREFIX === line.charAt(0)) {
-								//	regular ol' message line
+							if('PATH:' === line.slice(1, 6)) {
-								messageBodyData.message.push(line);
+								endOfMessage = true;	//	Anything pats the first PATH is not part of the message body
 							}
-						} else {
+							addKludgeLine(line.slice(1));
-							if(line.startsWith('SEEN-BY:')) {
+						} else if(endOfMessage) {
-								messageBodyData.seenBy.push(line.substring(line.indexOf(':') + 1).trim());
+							//	regular ol' message line
-							} else if(FTN_MESSAGE_KLUDGE_PREFIX === line.charAt(0)) {
+							messageBodyData.message.push(line);
-								addKludgeLine(line.slice(1));
-							} 
 						}
 					});
 
@@ -486,7 +484,7 @@ function Packet() {
 				.scan('subject', NULL_TERM_BUFFER)		//	:TODO: 72 bytes max
 				.scan('message', NULL_TERM_BUFFER)
 				.tap(function tapped(msgData) {
-					if(!msgData.ftn_orig_node) {
+					if(!msgData.messageType) {
 						//	end marker -- no more messages
 						end();						
 						return;

core/message.js

@@ -110,8 +110,6 @@ Message.FtnPropertyNames = {
 	FtnOrigNetwork		: 'ftn_orig_network',
 	FtnDestNetwork		: 'ftn_dest_network',
 	FtnAttrFlags		: 'ftn_attr_flags',
-	//FtnAttrFlags1		: 'ftn_attr_flags1',
-	//FtnAttrFlags2		: 'ftn_attr_flags2',
 	FtnCost				: 'ftn_cost',
 	FtnOrigZone			: 'ftn_orig_zone',
 	FtnDestZone			: 'ftn_dest_zone',

core/scanner_tossers/ftn_bso.js

@@ -8,11 +8,14 @@ let ftnMailPacket			= require('../ftn_mail_packet.js');
 let ftnUtil					= require('../ftn_util.js');
 let Address					= require('../ftn_address.js');
 let Log						= require('../logger.js').log;
+let ArchiveUtil				= require('../archive_util.js');
 
 let moment					= require('moment');
 let _						= require('lodash');
 let paths					= require('path');
 let mkdirp 					= require('mkdirp');
+let async					= require('async');
+let fs						= require('fs');
 
 exports.moduleInfo = {
 	name	: 'FTN',
@@ -35,6 +38,9 @@ exports.getModule = FTNMessageScanTossModule;
 function FTNMessageScanTossModule() {
 	MessageScanTossModule.call(this);
 
+	this.archUtil = new ArchiveUtil();
+	this.archUtil.init();
+
 	if(_.has(Config, 'scannerTossers.ftn_bso')) {
 		this.moduleConfig = Config.scannerTossers.ftn_bso;
 	}
@@ -43,10 +49,10 @@ function FTNMessageScanTossModule() {
 		return(networkName === this.moduleConfig.defaultNetwork && address.zone === this.moduleConfig.defaultZone);
 	}
 	
-	this.getOutgoingPacketDir = function(networkName, remoteAddress) {
+	this.getOutgoingPacketDir = function(networkName, destAddress) {
 		let dir = this.moduleConfig.paths.outbound;
-		if(!this.isDefaultDomainZone(networkName, remoteAddress)) {
+		if(!this.isDefaultDomainZone(networkName, destAddress)) {
-			const hexZone = `000${remoteAddress.zone.toString(16)}`.substr(-3);
+			const hexZone = `000${destAddress.zone.toString(16)}`.substr(-3);
 			dir = paths.join(dir, `${networkName.toLowerCase()}.${hexZone}`);
 		}
 		return dir;
@@ -75,6 +81,59 @@ function FTNMessageScanTossModule() {
 		return paths.join(basePath, `${name}.${ext}`);
 	};
 
+	this.getOutgoingFlowFileName = function(basePath, destAddress, exportType, extSuffix) {
+		if(destAddress.point) {
+
+		} else {
+			//
+			//	Use |destAddress| nnnnNNNN.??? where nnnn is dest net and NNNN is dest
+			//	node. This seems to match what Mystic does
+			//
+			return `${Math.abs(destAddress.net)}${Math.abs(destAddress.node)}.${exportType[1]}${extSuffix}`;
+		}
+	};
+	
+	this.getOutgoingBundleFileName = function(basePath, sourceAddress, destAddress, cb) {
+		//
+		//	Base filename is constructed as such:
+		//	*	If this |destAddress| is *not* a point address, we use NNNNnnnn where 
+		//		NNNN is 0 padded hex of dest net - source net and and nnnn is 0 padded 
+		//		hex of dest node - source node.
+		//	*	If |destAddress| is a point, NNNN becomes 0000 and nnnn becomes 'p' +
+		//		3 digit 0 padded hex point
+		//
+		//	Extension is dd? where dd is Su...Mo and ? is 0...Z as collisions arise
+		//
+		var basename;
+		if(destAddress.point) {
+			const pointHex = `000${destAddress.point}`.substr(-3);
+			basename = `0000p${pointHex}`;
+		} else {
+			basename = 
+				`0000${Math.abs(sourceAddress.net - destAddress.net).toString(16)}`.substr(-4) + 
+				`0000${Math.abs(sourceAddress.node - destAddress.node).toString(16)}`.substr(-4);			
+		}
+		
+		//
+		//	We need to now find the first entry that does not exist starting
+		//	with dd0 to ddz
+		//
+		const EXT_SUFFIXES = '0123456789abcdefghijklmnopqrstuvwxyz'.split('');		
+		let fileName = `${basename}.${moment().format('dd').toLowerCase()}`;
+		async.detectSeries(EXT_SUFFIXES, (suffix, callback) => {
+			const checkFileName = fileName + suffix; 			
+			fs.stat(paths.join(basePath, checkFileName), (err, stats) => {
+				callback((err && 'ENOENT' === err.code) ? true : false);
+			});
+		}, finalSuffix => {
+			if(finalSuffix) {
+				cb(null, paths.join(basePath, fileName + finalSuffix));
+			} else {
+				cb(new Error('Could not acquire a bundle filename!'));
+			}
+		});
+	};
+
 	this.createMessagePacket = function(message, options) {
 		this.prepareMessage(message, options);
 
@@ -82,7 +141,7 @@ function FTNMessageScanTossModule() {
 
 		let packetHeader = new ftnMailPacket.PacketHeader(
 			options.network.localAddress,
-			options.remoteAddress,
+			options.destAddress,
 			options.nodeConfig.packetType);
 
 		packetHeader.password = options.nodeConfig.packetPassword || '';
@@ -90,12 +149,15 @@ function FTNMessageScanTossModule() {
 		if(message.isPrivate()) {
 			//	:TODO: this should actually be checking for isNetMail()!!
 		} else {
-			const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.remoteAddress);
+			const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.destAddress);
 			
 			mkdirp(outgoingDir, err => {
 				if(err) {
 					//	:TODO: Handle me!!
 				} else {
+					this.getOutgoingBundleFileName(outgoingDir, options.network.localAddress, options.destAddress, (err, path) => {
+						console.log(path);
+					});
 					packet.write(
 						this.getOutgoingPacketFileName(outgoingDir, message), 
 						packetHeader, 
@@ -116,16 +178,20 @@ function FTNMessageScanTossModule() {
 		message.meta.FtnKludge = message.meta.FtnKludge || {};
 		
 		message.meta.FtnProperty.ftn_orig_node		= options.network.localAddress.node;
-		message.meta.FtnProperty.ftn_dest_node		= options.remoteAddress.node;
+		message.meta.FtnProperty.ftn_dest_node		= options.destAddress.node;
 		message.meta.FtnProperty.ftn_orig_network	= options.network.localAddress.net;
-		message.meta.FtnProperty.ftn_dest_network	= options.remoteAddress.net;
+		message.meta.FtnProperty.ftn_dest_network	= options.destAddress.net;
 		//	:TODO: attr1 & 2
 		message.meta.FtnProperty.ftn_cost			= 0;
 		
 		message.meta.FtnProperty.ftn_tear_line		= ftnUtil.getTearLine();		
 
 		//	:TODO: Need an explicit isNetMail() check
+		let ftnAttribute = 0;
+		
 		if(message.isPrivate()) {
+			ftnAttribute |= ftnMailPacket.Packet.Attribute.Private;
+			
 			//
 			//	NetMail messages need a FRL-1005.001 "Via" line
 			//	http://ftsc.org/docs/frl-1005.001
@@ -159,6 +225,8 @@ function FTNMessageScanTossModule() {
 				ftnUtil.getUpdatedPathEntries(message.meta.FtnKludge.PATH, options.network.localAddress);
 		}
 		
+		message.meta.FtnProperty.ftn_attr_flags = ftnAttribute;
+		
 		//
 		//	Additional kludges
 		//	
@@ -249,7 +317,7 @@ FTNMessageScanTossModule.prototype.record = function(message) {
 			const processOptions = {
 				nodeConfig		: this.moduleConfig.nodes[nodeKey],
 				network			: Config.messageNetworks.ftn.networks[areaConfig.network],
-				remoteAddress	: Address.fromString(uplink),
+				destAddress		: Address.fromString(uplink),
 				networkName		: areaConfig.network,
 			};