diff --git a/core/config.js b/core/config.js index 468732c9..85865ee3 100644 --- a/core/config.js +++ b/core/config.js @@ -227,25 +227,13 @@ function getDefaultConfig() { } } }, - - networks : { - /* - networkName : { // e.g. fidoNet - address : { - zone : 0, - net : 0, - node : 0, - point : 0, - domain : 'l33t.codes' - } - } - */ - }, scannerTossers : { ftn_bso : { paths : { - + outbound : paths.join(__dirname, './../mail/out/'), + inbound : paths.join(__dirname, './../mail/in/'), + secInbound : paths.join(__dirname, './../mail/secin/'), }, maxPacketByteSize : 256000, diff --git a/core/fnv1a.js b/core/fnv1a.js new file mode 100644 index 00000000..f7714936 --- /dev/null +++ b/core/fnv1a.js @@ -0,0 +1,50 @@ +/* jslint node: true */ +'use strict'; + +let _ = require('lodash'); + +// FNV-1a based on work here: https://github.com/wiedi/node-fnv +module.exports = class FNV1a { + constructor(data) { + this.hash = 0x811c9dc5; + + if(!_.isUndefined(data)) { + this.update(data); + } + } + + update(data) { + if(_.isNumber(data)) { + data = data.toString(); + } + + if(_.isString(data)) { + data = new Buffer(data); + } + + if(!Buffer.isBuffer(data)) { + throw new Error('data must be String or Buffer!'); + } + + for(let b of data) { + this.hash = this.hash ^ b; + this.hash += + (this.hash << 24) + (this.hash << 8) + (this.hash << 7) + + (this.hash << 4) + (this.hash << 1); + } + + return this; + } + + digest(encoding) { + encoding = encoding || 'binary'; + let buf = new Buffer(4); + buf.writeInt32BE(this.hash & 0xffffffff, 0); + return buf.toString(encoding); + } + + get value() { + return this.hash & 0xffffffff; + } +} + diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index 74950582..34801e6c 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -39,7 +39,7 @@ const FTN_MESSAGE_SAUCE_HEADER = new Buffer('SAUCE00'); const FTN_MESSAGE_KLUDGE_PREFIX = '\x01'; class PacketHeader { - constructor(origAddr, destAddr, created, version) { + constructor(origAddr, destAddr, version, created) { const EMPTY_ADDRESS = { node : 0, net : 0, @@ -139,7 +139,7 @@ class PacketHeader { this.year = momentCreated.year(); this.month = momentCreated.month(); - this.day = momentCreated.day(); + this.day = momentCreated.date(); // day of month this.hour = momentCreated.hour(); this.minute = momentCreated.minute(); this.second = momentCreated.second(); @@ -148,89 +148,6 @@ class PacketHeader { exports.PacketHeader = PacketHeader; -/* -function PacketHeader(options) { -} - -PacketHeader.prototype.init = function(origAddr, destAddr, created, version) { - version = version || '2+'; - - const EMPTY_ADDRESS = { - node : 0, - net : 0, - zone : 0, - point : 0, - }; - - this.packetVersion = version; - - this.setOrigAddress(origAddr || EMPTY_ADDRESS); - this.setDestAddress(destAddr || EMPTY_ADDRESS); - this.setCreated(created || moment()); - - // uncommon to set the following explicitly - this.prodCodeLo = 0xfe; // http://ftsc.org/docs/fta-1005.003 - this.prodRevLo = 0; - this.baud = 0; - this.packetType = FTN_PACKET_HEADER_TYPE; - this.password = ''; - this.prodData = 0x47694e45; // "ENiG" - - this.capWord = 0x0001; - this.capWordValidate = ((this.capWord & 0xff) << 8) | ((this.capWord >> 8) & 0xff); - - return this; -}; - -PacketHeader.prototype.setOrigAddress = function(address) { - this.origNode = address.node; - - // See FSC-48 - if(address.point) { - this.origNet = -1; - this.auxNet = address.net; - } else { - this.origNet = address.net; - this.auxNet = 0; - } - - this.origZone = address.zone; - this.origZone2 = address.zone; - this.origPoint = address.point || 0; - - return this; -}; - -PacketHeader.prototype.setDestAddress = function(address) { - this.destNode = address.node; - this.destNet = address.net; - this.destZone = address.zone; - this.destZone2 = address.zone; - this.destPoint = address.point || 0; - - return this; -}; - -PacketHeader.prototype.setCreated = function(created) { - if(!moment.isMoment(created)) { - created = moment(created); - } - - this.year = created.year(); - this.month = created.month(); - this.day = created.day(); - this.hour = created.hour(); - this.minute = created.minute(); - this.second = created.second(); - - return this; -}; - -PacketHeader.prototype.setPassword = function(password) { - this.password = password.substr(0, 8); -} -*/ - // // Read/Write FTN packets with support for the following formats: // @@ -338,15 +255,19 @@ function Packet() { } } - // - // Date/time components into something more reasonable - // Note: The names above match up with object members moment() allows - // - packetHeader.created = moment(packetHeader); + packetHeader.created = moment({ + year : packetHeader.year, + month : packetHeader.month, + date : packetHeader.day, + hour : packetHeader.hour, + minute : packetHeader.minute, + second : packetHeader.second + }); let ph = new PacketHeader(); _.assign(ph, packetHeader); + cb(null, ph); }); }; @@ -358,7 +279,7 @@ function Packet() { buffer.writeUInt16LE(packetHeader.destNode, 2); buffer.writeUInt16LE(packetHeader.created.year(), 4); buffer.writeUInt16LE(packetHeader.created.month(), 6); - buffer.writeUInt16LE(packetHeader.created.date(), 8); + buffer.writeUInt16LE(packetHeader.created.date(), 8); // day of month buffer.writeUInt16LE(packetHeader.created.hour(), 10); buffer.writeUInt16LE(packetHeader.created.minute(), 12); buffer.writeUInt16LE(packetHeader.created.second(), 14); @@ -374,45 +295,6 @@ function Packet() { buffer.writeUInt16LE(packetHeader.origZone, 34); buffer.writeUInt16LE(packetHeader.destZone, 36); - - // :TODO: update header information appropriately for 2 and 2.2 - switch(packetHeader.packetVersion) { - case '2' : - // filler... - packetHeader.auxNet = 0; - packetHeader.capWordValidate = 0; - packetHeader.prodCodeHi = 0; - packetHeader.prodRevLo = 0; - packetHeader.capWord = 0; - packetHeader.origZone2 = 0; - packetHeader.destZone2 = 0; - packetHeader.origPoint = 0; - packetHeader.destPoint = 0; - break; - - case '2.2' : - packetHeader.day = 0; - packetHeader.hour = 0; - packetHeader.minute = 0; - packetHeader.second = 0; - - // :TODO: copy over fields from 2+ -> overriden fields here! - - packetHeader.baud = FTN_PACKET_BAUD_TYPE_2_2; - break; - - case '2+' : - const capWordValidateSwapped = - ((packetHeader.capWordValidate & 0xff) << 8) | - ((packetHeader.capWordValidate >> 8) & 0xff); - - packetHeader.capWordValidate = capWordValidateSwapped; - - // :TODO: set header appropriate if point - break; - - } - buffer.writeUInt16LE(packetHeader.auxNet, 38); buffer.writeUInt16LE(packetHeader.capWordValidate, 40); buffer.writeUInt8(packetHeader.prodCodeHi, 42); @@ -422,12 +304,6 @@ function Packet() { buffer.writeUInt16LE(packetHeader.destZone2, 48); buffer.writeUInt16LE(packetHeader.origPoint, 50); buffer.writeUInt16LE(packetHeader.destPoint, 52); - - // Store in "ENiG" in prodData unless we already have something useful - if(0 === packetHeader.prodData) { - packetHeader.prodData = 0x47694e45; - } - buffer.writeUInt32LE(packetHeader.prodData, 54); ws.write(buffer); @@ -479,6 +355,8 @@ function Packet() { messageBodyData.kludgeLines[key] = value; } } + + let encoding = 'cp437'; async.series( [ @@ -503,10 +381,45 @@ function Packet() { callback(null); } }, + function extractChrsAndDetermineEncoding(callback) { + // + // From FTS-5003.001: + // "The CHRS control line is formatted as follows: + // + // ^ACHRS: + // + // Where is a character string of no more than eight (8) + // ASCII characters identifying the character set or character encoding + // scheme used, and level is a positive integer value describing what + // level of CHRS the message is written in." + // + // Also according to the spec, the deprecated "CHARSET" value may be used + // :TODO: Look into CHARSET more - should we bother supporting it? + // :TODO: See encodingFromHeader() for CHRS/CHARSET support @ https://github.com/Mithgol/node-fidonet-jam + const FTN_CHRS_PREFIX = new Buffer( [ 0x01, 0x43, 0x48, 0x52, 0x53, 0x3a, 0x20 ] ); // "\x01CHRS:" + const FTN_CHRS_SUFFIX = new Buffer( [ 0x0d ] ); + binary.parse(messageBodyBuffer) + .scan('prefix', FTN_CHRS_PREFIX) + .scan('content', FTN_CHRS_SUFFIX) + .tap(chrsData => { + if(chrsData.prefix && chrsData.content && chrsData.content.length > 0) { + const chrs = iconv.decode(chrsData.content, 'CP437'); + const chrsEncoding = ftn.getEncodingFromCharacterSetIdentifier(chrs); + if(chrsEncoding) { + encoding = chrsEncoding; + } + callback(null); + } else { + callback(null); + } + }); + }, function extractMessageData(callback) { - const messageLines = - iconv.decode(messageBodyBuffer, 'CP437').replace(/[\xec\n]/g, '').split(/\r/g); - + // + // Decode |messageBodyBuffer| using |encoding| defaulted or detected above + // + // :TODO: Look into \xec thing more - document + const messageLines = iconv.decode(messageBodyBuffer, encoding).replace(/[\xec\n]/g, '').split(/\r/g); let preOrigin = true; messageLines.forEach(line => { @@ -566,8 +479,6 @@ function Packet() { .word16lu('ftn_orig_network') .word16lu('ftn_dest_network') .word16lu('ftn_attr_flags') - //.word8('ftn_attr_flags1') - //.word8('ftn_attr_flags2') .word16lu('ftn_cost') .scan('modDateTime', NULL_TERM_BUFFER) // :TODO: 20 bytes max .scan('toUserName', NULL_TERM_BUFFER) // :TODO: 36 bytes max @@ -616,8 +527,6 @@ function Packet() { msg.meta.FtnProperty.ftn_orig_network = msgData.ftn_orig_network; msg.meta.FtnProperty.ftn_dest_network = msgData.ftn_dest_network; msg.meta.FtnProperty.ftn_attr_flags = msgData.ftn_attr_flags; - //msg.meta.FtnProperty.ftn_attr_flags1 = msgData.ftn_attr_flags1; - //msg.meta.FtnProperty.ftn_attr_flags2 = msgData.ftn_attr_flags2; msg.meta.FtnProperty.ftn_cost = msgData.ftn_cost; self.processMessageBody(msgData.message, function processed(messageBodyData) { @@ -661,7 +570,7 @@ function Packet() { }); }; - this.writeMessage = function(message, ws) { + this.writeMessage = function(message, ws, options) { let basicHeader = new Buffer(34); basicHeader.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0); @@ -670,8 +579,6 @@ function Packet() { basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6); basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_network, 8); basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_attr_flags, 10); - //basicHeader.writeUInt8(message.meta.FtnProperty.ftn_attr_flags1, 10); - //basicHeader.writeUInt8(message.meta.FtnProperty.ftn_attr_flags2, 11); basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12); const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0'); @@ -731,7 +638,7 @@ function Packet() { } }); - msgBody += message.message; + msgBody += message.message + '\r'; // // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 @@ -756,7 +663,9 @@ function Packet() { appendMeta('\x01PATH', message.meta.FtnKludge['PATH']); - ws.write(iconv.encode(msgBody + '\0', 'CP437')); + // + // :TODO: We should encode based on config and add the proper kludge here! + ws.write(iconv.encode(msgBody + '\0', options.encoding)); }; this.parsePacketBuffer = function(packetBuffer, iterator, cb) { @@ -839,19 +748,13 @@ Packet.prototype.writeStream = function(ws, messages, options) { } if(_.isObject(options.packetHeader)) { - /* - let packetHeader = options.packetHeader; - - // default header - packetHeader.packetVersion = packetHeader.packetVersion || '2+'; - packetHeader.created = packetHeader.created || moment(); - packetHeader.baud = packetHeader.baud || 0; - */ this.writePacketHeader(options.packetHeader, ws); } + + options.encoding = options.encoding || 'utf8'; messages.forEach(msg => { - this.writeMessage(msg, ws); + this.writeMessage(msg, ws, options); }); if(true === options.terminatePacket) { @@ -859,10 +762,12 @@ Packet.prototype.writeStream = function(ws, messages, options) { } } -Packet.prototype.write = function(path, packetHeader, messages) { +Packet.prototype.write = function(path, packetHeader, messages, options) { if(!_.isArray(messages)) { messages = [ messages ]; } + + options = options || { encoding : 'utf8' }; // utf-8 = 'CHRS UTF-8 4' this.writeStream( fs.createWriteStream(path), // :TODO: specify mode/etc. @@ -870,72 +775,3 @@ Packet.prototype.write = function(path, packetHeader, messages) { { packetHeader : packetHeader, terminatePacket : true } ); }; - -/* -const LOCAL_ADDRESS = { - zone : 46, - net : 1, - node : 232, - domain : 'l33t.codes', -}; - -const REMOTE_ADDRESS = { - zone : 1, - net : 2, - node : 218, -}; - - -var packetHeader1 = new PacketHeader(LOCAL_ADDRESS, REMOTE_ADDRESS); - -var packet = new Packet(); -var theHeader; -var written = false; -let messagesToWrite = []; -packet.read( - process.argv[2], - function iterator(dataType, data) { - if('header' === dataType) { - theHeader = data; - console.log(theHeader); - } else if('message' === dataType) { - const msg = data; - console.log(msg); - - messagesToWrite.push(msg); - - let address = { - zone : 46, - net : 1, - node : 232, - domain : 'l33t.codes', - }; - - msg.areaTag = 'agn_bbs'; - msg.messageId = 1234; - console.log(ftn.getMessageIdentifier(msg, address)); - console.log(ftn.getProductIdentifier()) - //console.log(ftn.getOrigin(address)) - //console.log(ftn.parseAddress('46:1/232.4@l33t.codes')) - console.log(Address.fromString('46:1/232.4@l33t.codes')); - console.log(ftn.getUTCTimeZoneOffset()) - console.log(ftn.getUpdatedSeenByEntries( - msg.meta.FtnProperty.ftn_seen_by, '1/107 4/22 4/25 4/10')); - console.log(ftn.getUpdatedPathEntries( - msg.meta.FtnKludge['PATH'], '1:365/50')) - console.log('Via: ' + ftn.getVia(address)) - console.log(Address.fromString('46:1/232.4@l33t.codes').isMatch('*:1/232.*')) - //console.log(Address.fromString('46:1/232.4@l33t.codes').getMatchScore('46:1/232')) - } - }, - function completion(err) { - console.log('complete!') - console.log(err); - - - console.log(messagesToWrite.length) - packet.write('/home/nuskooler/Downloads/ftnout/test1.pkt', theHeader, messagesToWrite, err => { - }); - } -); -*/ \ No newline at end of file diff --git a/core/ftn_util.js b/core/ftn_util.js index dbc1d2ea..5172b218 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -1,24 +1,26 @@ /* jslint node: true */ 'use strict'; -var Config = require('./config.js').config; -var Address = require('./ftn_address.js'); +let Config = require('./config.js').config; +let Address = require('./ftn_address.js'); +let FNV1a = require('./fnv1a.js'); -var _ = require('lodash'); -var assert = require('assert'); -var binary = require('binary'); -var fs = require('fs'); -var util = require('util'); -var iconv = require('iconv-lite'); -var moment = require('moment'); -var createHash = require('crypto').createHash; -var uuid = require('node-uuid'); -var os = require('os'); +let _ = require('lodash'); +let assert = require('assert'); +let binary = require('binary'); +let fs = require('fs'); +let util = require('util'); +let iconv = require('iconv-lite'); +let moment = require('moment'); +let createHash = require('crypto').createHash; +let uuid = require('node-uuid'); +let os = require('os'); -var packageJson = require('../package.json'); +let packageJson = require('../package.json'); // :TODO: Remove "Ftn" from most of these -- it's implied in the module exports.stringToNullPaddedBuffer = stringToNullPaddedBuffer; +exports.getMessageSerialNumber = getMessageSerialNumber; exports.createMessageUuid = createMessageUuid; exports.getDateFromFtnDateTime = getDateFromFtnDateTime; exports.getDateTimeString = getDateTimeString; @@ -34,6 +36,9 @@ exports.parseAbbreviatedNetNodeList = parseAbbreviatedNetNodeList; exports.getUpdatedSeenByEntries = getUpdatedSeenByEntries; exports.getUpdatedPathEntries = getUpdatedPathEntries; +exports.getCharacterSetIdentifierByEncoding = getCharacterSetIdentifierByEncoding; +exports.getEncodingFromCharacterSetIdentifier = getEncodingFromCharacterSetIdentifier; + exports.getQuotePrefix = getQuotePrefix; // @@ -134,8 +139,11 @@ function createMessageUuid(ftnMsgId, ftnArea) { } function getMessageSerialNumber(message) { - return ('00000000' + ((Math.floor((Date.now() - Date.UTC(2016, 1, 1)) / 1000) + - message.messageId)).toString(16)).substr(-8); + const msSinceEnigmaEpoc = (Date.now() - Date.UTC(2016, 1, 1)); + const hash = Math.abs(new FNV1a(msSinceEnigmaEpoc + message.messageId).value).toString(16); + return `00000000${hash}`.substr(-8); + // return ('00000000' + ((Math.floor((Date.now() - Date.UTC(2016, 1, 1)) / 1000) + + // message.messageId)).toString(16)).substr(-8); } // @@ -236,7 +244,8 @@ function getOrigin(address) { } function getTearLine() { - return `--- ENiGMA 1/2 v{$packageJson.version} (${os.platform()}; ${os.arch()}; ${nodeVer})`; + const nodeVer = process.version.substr(1); // remove 'v' prefix + return `--- ENiGMA 1/2 v${packageJson.version} (${os.platform()}; ${os.arch()}; ${nodeVer})`; } // @@ -363,3 +372,77 @@ function getUpdatedPathEntries(existingEntries, localAddress) { return existingEntries; } + +// +// Return FTS-5000.001 "CHRS" value +// http://ftsc.org/docs/fts-5003.001 +// +const ENCODING_TO_FTS_5003_001_CHARS = { + // level 1 - generally should not be used + ascii : [ 'ASCII', 1 ], + 'us-ascii' : [ 'ASCII', 1 ], + + // level 2 - 8 bit, ASCII based + cp437 : [ 'CP437', 2 ], + cp850 : [ 'CP850', 2 ], + + // level 3 - reserved + + // level 4 + utf8 : [ 'UTF-8', 4 ], + 'utf-8' : [ 'UTF-8', 4 ], +}; + + +function getCharacterSetIdentifierByEncoding(encodingName) { + const value = ENCODING_TO_FTS_5003_001_CHARS[encodingName.toLowerCase()]; + return value ? `${value[0]} ${value[1]}` : encodingName.toUpperCase(); +} + +function getEncodingFromCharacterSetIdentifier(chrs) { + const ident = chrs.split(' ')[0].toUpperCase(); + + // :TODO: fill in the rest!!! + return { + // level 1 + 'ASCII' : 'iso-646-1', + 'DUTCH' : 'iso-646', + 'FINNISH' : 'iso-646-10', + 'FRENCH' : 'iso-646', + 'CANADIAN' : 'iso-646', + 'GERMAN' : 'iso-646', + 'ITALIAN' : 'iso-646', + 'NORWEIG' : 'iso-646', + 'PORTU' : 'iso-646', + 'SPANISH' : 'iso-656', + 'SWEDISH' : 'iso-646-10', + 'SWISS' : 'iso-646', + 'UK' : 'iso-646', + 'ISO-10' : 'iso-646-10', + + // level 2 + 'CP437' : 'cp437', + 'CP850' : 'cp850', + 'CP852' : 'cp852', + 'CP866' : 'cp866', + 'CP848' : 'cp848', + 'CP1250' : 'cp1250', + 'CP1251' : 'cp1251', + 'CP1252' : 'cp1252', + 'CP10000' : 'macroman', + 'LATIN-1' : 'iso-8859-1', + 'LATIN-2' : 'iso-8859-2', + 'LATIN-5' : 'iso-8859-9', + 'LATIN-9' : 'iso-8859-15', + + // level 4 + 'UTF-8' : 'utf8', + + // deprecated stuff + 'IBMPC' : 'cp1250', // :TODO: validate + '+7_FIDO' : 'cp866', + '+7' : 'cp866', + 'MAC' : 'macroman', // :TODO: validate + + }[ident]; +} \ No newline at end of file diff --git a/core/message_area.js b/core/message_area.js index 06bb5234..4d47b31e 100644 --- a/core/message_area.js +++ b/core/message_area.js @@ -26,6 +26,7 @@ exports.getMessageListForArea = getMessageListForArea; exports.getNewMessagesInAreaForUser = getNewMessagesInAreaForUser; exports.getMessageAreaLastReadId = getMessageAreaLastReadId; exports.updateMessageAreaLastReadId = updateMessageAreaLastReadId; +exports.persistMessage = persistMessage; const CONF_AREA_RW_ACS_DEFAULT = 'GM[users]'; const AREA_MANAGE_ACS_DEFAULT = 'GM[sysops]'; diff --git a/core/msg_network.js b/core/msg_network.js index 5354ffdc..030c544e 100644 --- a/core/msg_network.js +++ b/core/msg_network.js @@ -51,9 +51,8 @@ function recordMessage(message, cb) { // choose to ignore it. // async.each(msgNetworkModules, (modInst, next) => { - modInst.record(message, err => { - next(); - }); + modInst.record(message); + next(); }, err => { cb(err); }); diff --git a/core/msg_scan_toss_module.js b/core/msg_scan_toss_module.js index e396f44d..8172d77f 100644 --- a/core/msg_scan_toss_module.js +++ b/core/msg_scan_toss_module.js @@ -20,6 +20,5 @@ MessageScanTossModule.prototype.shutdown = function(cb) { cb(null); }; -MessageScanTossModule.prototype.record = function(message, cb) { - cb(null); +MessageScanTossModule.prototype.record = function(message) { }; \ No newline at end of file diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 3cc6c0cf..71e95aae 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -4,13 +4,15 @@ // ENiGMA½ let MessageScanTossModule = require('../msg_scan_toss_module.js').MessageScanTossModule; let Config = require('../config.js').config; -let ftnMailpacket = require('../ftn_mail_packet.js'); +let ftnMailPacket = require('../ftn_mail_packet.js'); let ftnUtil = require('../ftn_util.js'); let Address = require('../ftn_address.js'); let Log = require('../logger.js').log; let moment = require('moment'); let _ = require('lodash'); +let paths = require('path'); +let mkdirp = require('mkdirp'); exports.moduleInfo = { name : 'FTN', @@ -18,75 +20,191 @@ exports.moduleInfo = { author : 'NuSkooler', }; +/* + :TODO: + * Add bundle timer (arcmail) + * Queue until time elapses / fixed time interval + * Pakcets append until >= max byte size + * [if arch type is not empty): Packets -> bundle until max byte size -> repeat process + * NetMail needs explicit isNetMail() check + * NetMail filename / location / etc. is still unknown - need to post on groups & get real answers +*/ + exports.getModule = FTNMessageScanTossModule; function FTNMessageScanTossModule() { MessageScanTossModule.call(this); if(_.has(Config, 'scannerTossers.ftn_bso')) { - this.config = Config.scannerTossers.ftn_bso; + this.moduleConfig = Config.scannerTossers.ftn_bso; } + + this.isDefaultDomainZone = function(networkName, address) { + return(networkName === this.moduleConfig.defaultNetwork && address.zone === this.moduleConfig.defaultZone); + } + + this.getOutgoingPacketDir = function(networkName, remoteAddress) { + let dir = this.moduleConfig.paths.outbound; + if(!this.isDefaultDomainZone(networkName, remoteAddress)) { + const hexZone = `000${remoteAddress.zone.toString(16)}`.substr(-3); + dir = paths.join(dir, `${networkName.toLowerCase()}.${hexZone}`); + } + return dir; + }; + + this.getOutgoingPacketFileName = function(basePath, message, isTemp) { + // + // Generating an outgoing packet file name comes with a few issues: + // * We must use DOS 8.3 filenames due to legacy systems that receive + // the packet not understanding LFNs + // * We need uniqueness; This is especially important with packets that + // end up in bundles and on the receiving/remote system where conflicts + // with other systems could also occur + // + // There are a lot of systems in use here for the name: + // * HEX CRC16/32 of data + // * HEX UNIX timestamp + // * Mystic at least at one point, used Hex8(day of month + seconds past midnight + hundredths of second) + // See https://groups.google.com/forum/#!searchin/alt.bbs.mystic/netmail$20filename/alt.bbs.mystic/m1xLnY8i1pU/YnG2excdl6MJ + // * SBBSEcho uses DDHHMMSS - see https://github.com/ftnapps/pkg-sbbs/blob/master/docs/fidonet.txt + // * We already have a system for 8-character serial number gernation that is + // used for e.g. in FTS-0009.001 MSGIDs... let's use that! + // + const name = ftnUtil.getMessageSerialNumber(message); + const ext = (true === isTemp) ? 'pk_' : 'pkt'; + return paths.join(basePath, `${name}.${ext}`); + }; - this.createMessagePacket = function(message, config) { - this.prepareMessage(message); + this.createMessagePacket = function(message, options) { + this.prepareMessage(message, options); let packet = new ftnMailPacket.Packet(); - let packetHeader = new ftnMailpacket.PacketHeader(); - packetHeader.init( - config.network.localAddress, - config.remoteAddress); + let packetHeader = new ftnMailPacket.PacketHeader( + options.network.localAddress, + options.remoteAddress, + options.nodeConfig.packetType); - packetHeader.setPassword(config.remoteNode.packetPassword || ''); + packetHeader.password = options.nodeConfig.packetPassword || ''; + + if(message.isPrivate()) { + // :TODO: this should actually be checking for isNetMail()!! + } else { + const outgoingDir = this.getOutgoingPacketDir(options.networkName, options.remoteAddress); + + mkdirp(outgoingDir, err => { + if(err) { + // :TODO: Handle me!! + } else { + packet.write( + this.getOutgoingPacketFileName(outgoingDir, message), + packetHeader, + [ message ], + { encoding : options.encoding } + ); + } + }); + } + }; - this.prepareMessage = function(message, config) { + this.prepareMessage = function(message, options) { // // Set various FTN kludges/etc. // message.meta.FtnProperty = message.meta.FtnProperty || {}; - message.meta.FtnProperty.ftn_orig_node = config.network.localAddress.node; - message.meta.FtnProperty.ftn_dest_node = config.remoteAddress.node; - message.meta.FtnProperty.ftn_orig_network = config.network.localAddress.net; - message.meta.FtnProperty.ftn_dest_network = config.remoteAddress.net; + message.meta.FtnKludge = message.meta.FtnKludge || {}; + + message.meta.FtnProperty.ftn_orig_node = options.network.localAddress.node; + message.meta.FtnProperty.ftn_dest_node = options.remoteAddress.node; + message.meta.FtnProperty.ftn_orig_network = options.network.localAddress.net; + message.meta.FtnProperty.ftn_dest_network = options.remoteAddress.net; // :TODO: attr1 & 2 message.meta.FtnProperty.ftn_cost = 0; + + message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); - message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); - message.meta.FtnProperty.ftn_origin = ftnUtil.getOrigin(config.network.localAddress); - + // :TODO: Need an explicit isNetMail() check if(message.isPrivate()) { // // NetMail messages need a FRL-1005.001 "Via" line // http://ftsc.org/docs/frl-1005.001 // - if(_.isString(message.meta.FtnKludge['Via'])) { - message.meta.FtnKludge['Via'] = [ message.meta.FtnKludge['Via'] ]; + if(_.isString(message.meta.FtnKludge.Via)) { + message.meta.FtnKludge.Via = [ message.meta.FtnKludge.Via ]; } - message.meta.FtnKludge['Via'] = message.meta.FtnKludge['Via'] || []; - message.meta.FtnKludge['Via'].push(ftnUtil.getVia(config.network.localAddress)); + message.meta.FtnKludge.Via = message.meta.FtnKludge.Via || []; + message.meta.FtnKludge.Via.push(ftnUtil.getVia(options.network.localAddress)); } else { - message.meta.FtnProperty.ftn_area = message.areaTag; + // + // EchoMail requires some additional properties & kludges + // + message.meta.FtnProperty.ftn_origin = ftnUtil.getOrigin(options.network.localAddress); + message.meta.FtnProperty.ftn_area = Config.messageNetworks.ftn.areas[message.areaTag].tag; + + // + // When exporting messages, we should create/update SEEN-BY + // with remote address(s) we are exporting to. + // + message.meta.FtnProperty.ftn_seen_by = + ftnUtil.getUpdatedSeenByEntries( + message.meta.FtnProperty.ftn_seen_by, + Config.messageNetworks.ftn.areas[message.areaTag].uplinks + ); + + // + // And create/update PATH for ourself + // + message.meta.FtnKludge.PATH = + ftnUtil.getUpdatedPathEntries(message.meta.FtnKludge.PATH, options.network.localAddress); } - + // - // When exporting messages, we should create/update SEEN-BY - // with remote address(s) we are exporting to. + // Additional kludges + // + message.meta.FtnKludge.MSGID = ftnUtil.getMessageIdentifier(message, options.network.localAddress); + message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset(); + + if(!message.meta.FtnKludge.PID) { + message.meta.FtnKludge.PID = ftnUtil.getProductIdentifier(); + } + + if(!message.meta.FtnKludge.TID) { + // :TODO: Create TID!! + //message.meta.FtnKludge.TID = + } + // - message.meta.FtnProperty.ftn_seen_by = - ftnUtil.getUpdatedSeenByEntries( - message.meta.FtnProperty.ftn_seen_by, - Config.messageNetworks.ftn.areas[message.areaTag].uplinks - ); - - // - // And create/update PATH for ourself - // - message.meta.FtnKludge['PATH'] = - ftnUtil.getUpdatedPathEntries( - message.meta.FtnKludge['PATH'], - config.network.localAddress.node - ); + // Determine CHRS and actual internal encoding name + // Try to preserve anything already here + let encoding = options.nodeConfig.encoding || 'utf8'; + if(message.meta.FtnKludge.CHRS) { + const encFromChars = ftnUtil.getEncodingFromCharacterSetIdentifier(message.meta.FtnKludge.CHRS); + if(encFromChars) { + encoding = encFromChars; + } + } + + options.encoding = encoding; // save for later + message.meta.FtnKludge.CHRS = ftnUtil.getCharacterSetIdentifierByEncoding(encoding); + // :TODO: FLAGS kludge? + // :TODO: Add REPLY kludge if appropriate + + }; + + + // :TODO: change to something like isAreaConfigValid + // check paths, Addresses, etc. + this.isAreaConfigComplete = function(areaConfig) { + if(!_.isString(areaConfig.tag) || !_.isString(areaConfig.network)) { + return false; + } + + if(_.isString(areaConfig.uplinks)) { + areaConfig.uplinks = areaConfig.uplinks.split(' '); + } + + return (_.isArray(areaConfig.uplinks)); }; } @@ -95,23 +213,25 @@ require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); FTNMessageScanTossModule.prototype.startup = function(cb) { Log.info('FidoNet Scanner/Tosser starting up'); - - cb(null); + + FTNMessageScanTossModule.super_.prototype.startup.call(this, cb); }; FTNMessageScanTossModule.prototype.shutdown = function(cb) { Log.info('FidoNet Scanner/Tosser shutting down'); - cb(null); + FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb); }; -FTNMessageScanTossModule.prototype.record = function(message, cb) { - if(!_.has(Config, [ 'messageNetworks', 'ftn', 'areas', message.areaTag ])) { +FTNMessageScanTossModule.prototype.record = function(message) { + if(!_.has(this, 'moduleConfig.nodes') || + !_.has(Config, [ 'messageNetworks', 'ftn', 'areas', message.areaTag ])) + { return; } - const area = Config.messageNetworks.ftn.areas[message.areaTag]; - if(!_.isString(area.ftnArea) || !_.isArray(area.uplinks)) { + const areaConfig = Config.messageNetworks.ftn.areas[message.areaTag]; + if(!this.isAreaConfigComplete(areaConfig)) { // :TODO: should probably log a warning here return; } @@ -119,20 +239,31 @@ FTNMessageScanTossModule.prototype.record = function(message, cb) { // // For each uplink, find the best configuration match // - area.uplinks.forEach(uplink => { + areaConfig.uplinks.forEach(uplink => { // :TODO: sort by least # of '*' & take top? - let matchNodes = _.filter(Object.keys(Config.scannerTossers.ftn_bso.nodes), addr => { + const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => { return Address.fromString(addr).isMatch(uplink); - }); - - if(matchNodes.length > 0) { - const nodeKey = matchNodes[0]; + })[0]; + if(nodeKey) { + const processOptions = { + nodeConfig : this.moduleConfig.nodes[nodeKey], + network : Config.messageNetworks.ftn.networks[areaConfig.network], + remoteAddress : Address.fromString(uplink), + networkName : areaConfig.network, + }; + + if(_.isString(processOptions.network.localAddress)) { + // :TODO: move/cache this - e.g. @ startup(). Think about due to Config cache + processOptions.network.localAddress = Address.fromString(processOptions.network.localAddress); + } + + // :TODO: Validate the rest of the matching config -- or do that elsewhere, e.g. startup() + + this.createMessagePacket(message, processOptions); } }); - - cb(null); // :TODO: should perhaps record in batches - e.g. start an event, record // to temp location until time is hit or N achieved such that if multiple diff --git a/mods/msg_area_post_fse.js b/mods/msg_area_post_fse.js index 2b0c488d..16292cea 100644 --- a/mods/msg_area_post_fse.js +++ b/mods/msg_area_post_fse.js @@ -1,12 +1,13 @@ /* jslint node: true */ 'use strict'; -var FullScreenEditorModule = require('../core/fse.js').FullScreenEditorModule; -var Message = require('../core/message.js').Message; -var user = require('../core/user.js'); +let FullScreenEditorModule = require('../core/fse.js').FullScreenEditorModule; +//var Message = require('../core/message.js').Message; +let persistMessage = require('../core/message_area.js').persistMessage; +let user = require('../core/user.js'); -var _ = require('lodash'); -var async = require('async'); +let _ = require('lodash'); +let async = require('async'); exports.getModule = AreaPostFSEModule; @@ -36,9 +37,12 @@ function AreaPostFSEModule(options) { }); }, function saveMessage(callback) { + persistMessage(msg, callback); + /* msg.persist(function persisted(err) { callback(err); }); + */ } ], function complete(err) {