diff --git a/core/archive_util.js b/core/archive_util.js new file mode 100644 index 00000000..ba7d3c4e --- /dev/null +++ b/core/archive_util.js @@ -0,0 +1,114 @@ +/* jslint node: true */ +'use strict'; + +// ENiGMA½ +let Config = require('./config.js').config; + +// base/modules +let fs = require('fs'); +let _ = require('lodash'); +let pty = require('ptyw.js'); + +module.exports = class ArchiveUtil { + + constructor() { + this.archivers = {}; + this.longestSignature = 0; + } + + init() { + // + // Load configuration + // + if(_.has(Config, 'archivers')) { + Object.keys(Config.archivers).forEach(archKey => { + const arch = Config.archivers[archKey]; + if(!_.isString(arch.sig) || + !_.isString(arch.compressCmd) || + !_.isString(arch.decompressCmd) || + !_.isArray(arch.compressArgs) || + !_.isArray(arch.decompressArgs)) + { + // :TODO: log warning + return; + } + + const archiver = { + compressCmd : arch.compressCmd, + compressArgs : arch.compressArgs, + decompressCmd : arch.decompressCmd, + decompressArgs : arch.decompressArgs, + sig : new Buffer(arch.sig, 'hex'), + offset : arch.offset || 0, + }; + + this.archivers[archKey] = archiver; + + if(archiver.offset + archiver.sig.length > this.longestSignature) { + this.longestSignature = archiver.offset + archiver.sig.length; + } + }); + } + } + + detectType(path, cb) { + fs.open(path, 'r', (err, fd) => { + if(err) { + cb(err); + return; + } + + let buf = new Buffer(this.longestSignature); + fs.read(fd, buf, 0, buf.length, 0, (err, bytesRead) => { + if(err) { + cb(err); + return; + } + + // return first match + const detected = _.find(this.archivers, arch => { + const lenNeeded = arch.offset + arch.sig.length; + + if(buf.length < lenNeeded) { + return false; + } + + const comp = buf.slice(arch.offset, arch.offset + arch.sig.length); + return (arch.sig.equals(comp)); + }); + + cb(detected ? null : new Error('Unknown type'), detected); + }); + }); + } + + compressTo(archType, archivePath, files, cb) { + const archiver = this.archivers[archType]; + if(!archiver) { + cb(new Error('Unknown archive type: ' + archType)); + return; + } + + let args = _.clone(archiver.compressArgs); // don't much with orig + for(let i = 0; i < args.length; ++i) { + args[i] = args[i].format({ + archivePath : archivePath, + fileList : files.join(' '), + }); + } + + let comp = pty.spawn(archiver.compressCmd, args, { + cols : 80, + rows : 24, + // :TODO: cwd + }); + + comp.on('exit', exitCode => { + cb(0 === exitCode ? null : new Error('Compression failed with exit code: ' + exitCode)); + }); + } + + extractTo(archivePath, extractPath, archType, cb) { + + } +} diff --git a/core/config.js b/core/config.js index 85865ee3..76521f97 100644 --- a/core/config.js +++ b/core/config.js @@ -209,6 +209,18 @@ function getDefaultConfig() { } }, + archivers : { + zip : { + name : "PKZip", + sig : "504b0304", + offset : 0, + compressCmd : "7z", + compressArgs : [ "a", "-tzip", "{archivePath}", "{fileList}" ], + decompressCmd : "7z", + decompressArgs : [ "e", "-o{extractDir}", "{archivePath}" ] + } + }, + messageConferences : { system_internal : { name : 'System Internal', @@ -231,13 +243,13 @@ function getDefaultConfig() { scannerTossers : { ftn_bso : { paths : { - outbound : paths.join(__dirname, './../mail/out/'), - inbound : paths.join(__dirname, './../mail/in/'), - secInbound : paths.join(__dirname, './../mail/secin/'), + outbound : paths.join(__dirname, './../mail/ftn_out/'), + inbound : paths.join(__dirname, './../mail/ftn_in/'), + secInbound : paths.join(__dirname, './../mail/ftn_secin/'), }, - maxPacketByteSize : 256000, - maxBundleByteSize : 256000, + maxPacketByteSize : 512000, // 512k, before placing messages in a new pkt + maxBundleByteSize : 2048000, // 2M, before creating another archive } }, diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index 34801e6c..a0d1b8a3 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -419,36 +419,34 @@ function Packet() { // Decode |messageBodyBuffer| using |encoding| defaulted or detected above // // :TODO: Look into \xec thing more - document - const messageLines = iconv.decode(messageBodyBuffer, encoding).replace(/[\xec\n]/g, '').split(/\r/g); - let preOrigin = true; + const messageLines = iconv.decode(messageBodyBuffer, encoding).replace(/\xec/g, '').split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g); + let endOfMessage = true; messageLines.forEach(line => { if(0 === line.length) { messageBodyData.message.push(''); return; } - - if(preOrigin) { - if(line.startsWith('AREA:')) { - messageBodyData.area = line.substring(line.indexOf(':') + 1).trim(); - } else if(line.startsWith('--- ')) { - // Tear Lines are tracked allowing for specialized display/etc. - messageBodyData.tearLine = line; - } else if(/[ ]{1,2}(\* )?Origin\: /.test(line)) { // To spec is " * Origin: ..." - messageBodyData.originLine = line; - preOrigin = false; - } else if(FTN_MESSAGE_KLUDGE_PREFIX === line.charAt(0)) { - addKludgeLine(line.slice(1)); - } else { - // regular ol' message line - messageBodyData.message.push(line); + + if(line.startsWith('AREA:')) { + messageBodyData.area = line.substring(line.indexOf(':') + 1).trim(); + } else if(line.startsWith('--- ')) { + // Tear Lines are tracked allowing for specialized display/etc. + messageBodyData.tearLine = line; + } else if(/[ ]{1,2}(\* )?Origin\: /.test(line)) { // To spec is " * Origin: ..." + messageBodyData.originLine = line; + endOfMessage = false; // Anything past origin is not part of the message body + } else if(line.startsWith('SEEN-BY:')) { + endOfMessage = true; // Anything past the first SEEN-BY is not part of the message body + messageBodyData.seenBy.push(line.substring(line.indexOf(':') + 1).trim()); + } else if(FTN_MESSAGE_KLUDGE_PREFIX === line.charAt(0)) { + if('PATH:' === line.slice(1, 6)) { + endOfMessage = true; // Anything pats the first PATH is not part of the message body } - } else { - if(line.startsWith('SEEN-BY:')) { - messageBodyData.seenBy.push(line.substring(line.indexOf(':') + 1).trim()); - } else if(FTN_MESSAGE_KLUDGE_PREFIX === line.charAt(0)) { - addKludgeLine(line.slice(1)); - } + addKludgeLine(line.slice(1)); + } else if(endOfMessage) { + // regular ol' message line + messageBodyData.message.push(line); } }); @@ -486,7 +484,7 @@ function Packet() { .scan('subject', NULL_TERM_BUFFER) // :TODO: 72 bytes max .scan('message', NULL_TERM_BUFFER) .tap(function tapped(msgData) { - if(!msgData.ftn_orig_node) { + if(!msgData.messageType) { // end marker -- no more messages end(); return; diff --git a/core/message.js b/core/message.js index f506f91b..1a5ddd3b 100644 --- a/core/message.js +++ b/core/message.js @@ -110,8 +110,6 @@ Message.FtnPropertyNames = { FtnOrigNetwork : 'ftn_orig_network', FtnDestNetwork : 'ftn_dest_network', FtnAttrFlags : 'ftn_attr_flags', - //FtnAttrFlags1 : 'ftn_attr_flags1', - //FtnAttrFlags2 : 'ftn_attr_flags2', FtnCost : 'ftn_cost', FtnOrigZone : 'ftn_orig_zone', FtnDestZone : 'ftn_dest_zone', diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 71e95aae..fa211146 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -8,11 +8,14 @@ let ftnMailPacket = require('../ftn_mail_packet.js'); let ftnUtil = require('../ftn_util.js'); let Address = require('../ftn_address.js'); let Log = require('../logger.js').log; +let ArchiveUtil = require('../archive_util.js'); let moment = require('moment'); let _ = require('lodash'); let paths = require('path'); let mkdirp = require('mkdirp'); +let async = require('async'); +let fs = require('fs'); exports.moduleInfo = { name : 'FTN', @@ -35,6 +38,9 @@ exports.getModule = FTNMessageScanTossModule; function FTNMessageScanTossModule() { MessageScanTossModule.call(this); + this.archUtil = new ArchiveUtil(); + this.archUtil.init(); + if(_.has(Config, 'scannerTossers.ftn_bso')) { this.moduleConfig = Config.scannerTossers.ftn_bso; } @@ -43,10 +49,10 @@ function FTNMessageScanTossModule() { return(networkName === this.moduleConfig.defaultNetwork && address.zone === this.moduleConfig.defaultZone); } - this.getOutgoingPacketDir = function(networkName, remoteAddress) { + this.getOutgoingPacketDir = function(networkName, destAddress) { let dir = this.moduleConfig.paths.outbound; - if(!this.isDefaultDomainZone(networkName, remoteAddress)) { - const hexZone = `000${remoteAddress.zone.toString(16)}`.substr(-3); + if(!this.isDefaultDomainZone(networkName, destAddress)) { + const hexZone = `000${destAddress.zone.toString(16)}`.substr(-3); dir = paths.join(dir, `${networkName.toLowerCase()}.${hexZone}`); } return dir; @@ -75,6 +81,59 @@ function FTNMessageScanTossModule() { return paths.join(basePath, `${name}.${ext}`); }; + this.getOutgoingFlowFileName = function(basePath, destAddress, exportType, extSuffix) { + if(destAddress.point) { + + } else { + // + // Use |destAddress| nnnnNNNN.??? where nnnn is dest net and NNNN is dest + // node. This seems to match what Mystic does + // + return `${Math.abs(destAddress.net)}${Math.abs(destAddress.node)}.${exportType[1]}${extSuffix}`; + } + }; + + this.getOutgoingBundleFileName = function(basePath, sourceAddress, destAddress, cb) { + // + // Base filename is constructed as such: + // * If this |destAddress| is *not* a point address, we use NNNNnnnn where + // NNNN is 0 padded hex of dest net - source net and and nnnn is 0 padded + // hex of dest node - source node. + // * If |destAddress| is a point, NNNN becomes 0000 and nnnn becomes 'p' + + // 3 digit 0 padded hex point + // + // Extension is dd? where dd is Su...Mo and ? is 0...Z as collisions arise + // + var basename; + if(destAddress.point) { + const pointHex = `000${destAddress.point}`.substr(-3); + basename = `0000p${pointHex}`; + } else { + basename = + `0000${Math.abs(sourceAddress.net - destAddress.net).toString(16)}`.substr(-4) + + `0000${Math.abs(sourceAddress.node - destAddress.node).toString(16)}`.substr(-4); + } + + // + // We need to now find the first entry that does not exist starting + // with dd0 to ddz + // + const EXT_SUFFIXES = '0123456789abcdefghijklmnopqrstuvwxyz'.split(''); + let fileName = `${basename}.${moment().format('dd').toLowerCase()}`; + async.detectSeries(EXT_SUFFIXES, (suffix, callback) => { + const checkFileName = fileName + suffix; + fs.stat(paths.join(basePath, checkFileName), (err, stats) => { + callback((err && 'ENOENT' === err.code) ? true : false); + }); + }, finalSuffix => { + if(finalSuffix) { + cb(null, paths.join(basePath, fileName + finalSuffix)); + } else { + cb(new Error('Could not acquire a bundle filename!')); + } + }); + }; + this.createMessagePacket = function(message, options) { this.prepareMessage(message, options); @@ -82,7 +141,7 @@ function FTNMessageScanTossModule() { let packetHeader = new ftnMailPacket.PacketHeader( options.network.localAddress, - options.remoteAddress, + options.destAddress, options.nodeConfig.packetType); packetHeader.password = options.nodeConfig.packetPassword || ''; @@ -90,12 +149,15 @@ function FTNMessageScanTossModule() { if(message.isPrivate()) { // :TODO: this should actually be checking for isNetMail()!! } else { - const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.remoteAddress); + const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.destAddress); mkdirp(outgoingDir, err => { if(err) { // :TODO: Handle me!! } else { + this.getOutgoingBundleFileName(outgoingDir, options.network.localAddress, options.destAddress, (err, path) => { + console.log(path); + }); packet.write( this.getOutgoingPacketFileName(outgoingDir, message), packetHeader, @@ -116,16 +178,20 @@ function FTNMessageScanTossModule() { message.meta.FtnKludge = message.meta.FtnKludge || {}; message.meta.FtnProperty.ftn_orig_node = options.network.localAddress.node; - message.meta.FtnProperty.ftn_dest_node = options.remoteAddress.node; + message.meta.FtnProperty.ftn_dest_node = options.destAddress.node; message.meta.FtnProperty.ftn_orig_network = options.network.localAddress.net; - message.meta.FtnProperty.ftn_dest_network = options.remoteAddress.net; + message.meta.FtnProperty.ftn_dest_network = options.destAddress.net; // :TODO: attr1 & 2 message.meta.FtnProperty.ftn_cost = 0; message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); // :TODO: Need an explicit isNetMail() check + let ftnAttribute = 0; + if(message.isPrivate()) { + ftnAttribute |= ftnMailPacket.Packet.Attribute.Private; + // // NetMail messages need a FRL-1005.001 "Via" line // http://ftsc.org/docs/frl-1005.001 @@ -159,6 +225,8 @@ function FTNMessageScanTossModule() { ftnUtil.getUpdatedPathEntries(message.meta.FtnKludge.PATH, options.network.localAddress); } + message.meta.FtnProperty.ftn_attr_flags = ftnAttribute; + // // Additional kludges // @@ -249,7 +317,7 @@ FTNMessageScanTossModule.prototype.record = function(message) { const processOptions = { nodeConfig : this.moduleConfig.nodes[nodeKey], network : Config.messageNetworks.ftn.networks[areaConfig.network], - remoteAddress : Address.fromString(uplink), + destAddress : Address.fromString(uplink), networkName : areaConfig.network, };