/* jslint node: true */ 'use strict'; // ENiGMA½ const MessageScanTossModule = require('../msg_scan_toss_module.js').MessageScanTossModule; const Config = require('../config.js').get; const ftnMailPacket = require('../ftn_mail_packet.js'); const ftnUtil = require('../ftn_util.js'); const Address = require('../ftn_address.js'); const Log = require('../logger.js').log; const ArchiveUtil = require('../archive_util.js'); const msgDb = require('../database.js').dbs.message; const Message = require('../message.js'); const TicFileInfo = require('../tic_file_info.js'); const Errors = require('../enig_error.js').Errors; const FileEntry = require('../file_entry.js'); const scanFile = require('../file_base_area.js').scanFile; const getFileAreaByTag = require('../file_base_area.js').getFileAreaByTag; const getDescFromFileName = require('../file_base_area.js').getDescFromFileName; const copyFileWithCollisionHandling = require('../file_util.js').copyFileWithCollisionHandling; const getAreaStorageDirectoryByTag = require('../file_base_area.js').getAreaStorageDirectoryByTag; const isValidStorageTag = require('../file_base_area.js').isValidStorageTag; const User = require('../user.js'); const StatLog = require('../stat_log.js'); const SysProps = require('../system_property.js'); // deps const moment = require('moment'); const _ = require('lodash'); const paths = require('path'); const async = require('async'); const fs = require('graceful-fs'); const later = require('@breejs/later'); const temptmp = require('temptmp').createTrackedSession('ftn_bso'); const assert = require('assert'); const sane = require('sane'); const fse = require('fs-extra'); const iconv = require('iconv-lite'); const { v4: UUIDv4 } = require('uuid'); exports.moduleInfo = { name: 'FTN BSO', desc: 'BSO style message scanner/tosser for FTN networks', author: 'NuSkooler', }; /* :TODO: * Support (approx) max bundle size * Validate packet passwords!!!! => secure vs insecure landing areas */ exports.getModule = FTNMessageScanTossModule; const SCHEDULE_REGEXP = /(?:^|or )?(@watch:|@immediate)([^\0]+)?$/; function FTNMessageScanTossModule() { MessageScanTossModule.call(this); const self = this; this.archUtil = ArchiveUtil.getInstance(); const config = Config(); if (_.has(config, 'scannerTossers.ftn_bso')) { this.moduleConfig = config.scannerTossers.ftn_bso; } this.getDefaultNetworkName = function () { if (this.moduleConfig.defaultNetwork) { return this.moduleConfig.defaultNetwork.toLowerCase(); } const networkNames = Object.keys(config.messageNetworks.ftn.networks); if (1 === networkNames.length) { return networkNames[0].toLowerCase(); } }; this.getDefaultZone = function (networkName) { const config = Config(); if (_.isNumber(config.messageNetworks.ftn.networks[networkName].defaultZone)) { return config.messageNetworks.ftn.networks[networkName].defaultZone; } // non-explicit: default to local address zone const networkLocalAddress = config.messageNetworks.ftn.networks[networkName].localAddress; if (networkLocalAddress) { const addr = Address.fromString(networkLocalAddress); return addr.zone; } }; /* this.isDefaultDomainZone = function(networkName, address) { const defaultNetworkName = this.getDefaultNetworkName(); return(networkName === defaultNetworkName && address.zone === this.moduleConfig.defaultZone); }; */ this.getNetworkNameByAddress = function (remoteAddress) { return _.findKey(Config().messageNetworks.ftn.networks, network => { const localAddress = Address.fromString(network.localAddress); return !_.isUndefined(localAddress) && localAddress.isEqual(remoteAddress); }); }; this.getNetworkNameByAddressPattern = function (remoteAddressPattern) { return _.findKey(Config().messageNetworks.ftn.networks, network => { const localAddress = Address.fromString(network.localAddress); return ( !_.isUndefined(localAddress) && localAddress.isPatternMatch(remoteAddressPattern) ); }); }; this.getLocalAreaTagByFtnAreaTag = function (ftnAreaTag) { ftnAreaTag = ftnAreaTag.toUpperCase(); // always compare upper return _.findKey(Config().messageNetworks.ftn.areas, areaConf => { return _.isString(areaConf.tag) && areaConf.tag.toUpperCase() === ftnAreaTag; }); }; this.getExportType = function (nodeConfig) { return _.isString(nodeConfig.exportType) ? nodeConfig.exportType.toLowerCase() : 'crash'; }; /* this.getSeenByAddresses = function(messageSeenBy) { if(!_.isArray(messageSeenBy)) { messageSeenBy = [ messageSeenBy ]; } let seenByAddrs = []; messageSeenBy.forEach(sb => { seenByAddrs = seenByAddrs.concat(ftnUtil.parseAbbreviatedNetNodeList(sb)); }); return seenByAddrs; }; */ this.messageHasValidMSGID = function (msg) { return ( _.isString(msg.meta.FtnKludge.MSGID) && msg.meta.FtnKludge.MSGID.length > 0 ); }; /* this.getOutgoingEchoMailPacketDir = function(networkName, destAddress) { let dir = this.moduleConfig.paths.outbound; if(!this.isDefaultDomainZone(networkName, destAddress)) { const hexZone = `000${destAddress.zone.toString(16)}`.substr(-3); dir = paths.join(dir, `${networkName.toLowerCase()}.${hexZone}`); } return dir; }; */ this.getOutgoingEchoMailPacketDir = function (networkName, destAddress) { networkName = networkName.toLowerCase(); let dir = this.moduleConfig.paths.outbound; const defaultNetworkName = this.getDefaultNetworkName(); const defaultZone = this.getDefaultZone(networkName); let zoneExt; if (defaultZone !== destAddress.zone) { zoneExt = '.' + `000${destAddress.zone.toString(16)}`.substr(-3); } else { zoneExt = ''; } if (defaultNetworkName === networkName) { dir = paths.join(dir, `outbound${zoneExt}`); } else { dir = paths.join(dir, `${networkName}${zoneExt}`); } return dir; }; this.getOutgoingPacketFileName = function (basePath, messageId, isTemp, fileCase) { // // Generating an outgoing packet file name comes with a few issues: // * We must use DOS 8.3 filenames due to legacy systems that receive // the packet not understanding LFNs // * We need uniqueness; This is especially important with packets that // end up in bundles and on the receiving/remote system where conflicts // with other systems could also occur // // There are a lot of systems in use here for the name: // * HEX CRC16/32 of data // * HEX UNIX timestamp // * Mystic at least at one point, used Hex8(day of month + seconds past midnight + hundredths of second) // See https://groups.google.com/forum/#!searchin/alt.bbs.mystic/netmail$20filename/alt.bbs.mystic/m1xLnY8i1pU/YnG2excdl6MJ // * SBBSEcho uses DDHHMMSS - see https://github.com/ftnapps/pkg-sbbs/blob/master/docs/fidonet.txt // * We already have a system for 8-character serial number gernation that is // used for e.g. in FTS-0009.001 MSGIDs... let's use that! // const name = ftnUtil.getMessageSerialNumber(messageId); const ext = true === isTemp ? 'pk_' : 'pkt'; let fileName = `${name}.${ext}`; if ('upper' === fileCase) { fileName = fileName.toUpperCase(); } return paths.join(basePath, fileName); }; this.getOutgoingFlowFileExtension = function ( destAddress, flowType, exportType, fileCase ) { let ext; switch (flowType) { case 'mail': ext = `${exportType.toLowerCase()[0]}ut`; break; case 'ref': ext = `${exportType.toLowerCase()[0]}lo`; break; case 'busy': ext = 'bsy'; break; case 'request': ext = 'req'; break; case 'requests': ext = 'hrq'; break; } if ('upper' === fileCase) { ext = ext.toUpperCase(); } return ext; }; this.getOutgoingFlowFileName = function ( basePath, destAddress, flowType, exportType, fileCase ) { // // Refs // * http://ftsc.org/docs/fts-5005.003 // * http://wiki.synchro.net/ref:fidonet_files#flow_files // let controlFileBaseName; let pointDir; const ext = self.getOutgoingFlowFileExtension( destAddress, flowType, exportType, fileCase ); const netComponent = `0000${destAddress.net.toString(16)}`.substr(-4); const nodeComponent = `0000${destAddress.node.toString(16)}`.substr(-4); if (destAddress.point) { // point's go in an extra subdir, e.g. outbound/NNNNnnnn.pnt/00000001.pnt (for a point of 1) pointDir = `${netComponent}${nodeComponent}.pnt`; controlFileBaseName = `00000000${destAddress.point.toString(16)}`.substr(-8); } else { pointDir = ''; // // Use |destAddress| nnnnNNNN.??? where nnnn is dest net and NNNN is dest // node. This seems to match what Mystic does // controlFileBaseName = `${netComponent}${nodeComponent}`; } // // From FTS-5005.003: "Lower case filenames are prefered if supported by the file system." // ...but we let the user override. // if ('upper' === fileCase) { controlFileBaseName = controlFileBaseName.toUpperCase(); pointDir = pointDir.toUpperCase(); } return paths.join(basePath, pointDir, `${controlFileBaseName}.${ext}`); }; this.flowFileAppendRefs = function (filePath, fileRefs, directive, cb) { // // We have to ensure the *directory* of |filePath| exists here esp. // for cases such as point destinations where a subdir may be // present in the path that doesn't yet exist. // const flowFileDir = paths.dirname(filePath); fse.mkdirs(flowFileDir, () => { // note not checking err; let's try appendFile const appendLines = fileRefs.reduce((content, ref) => { return content + `${directive}${ref}\n`; }, ''); fs.appendFile(filePath, appendLines, err => { return cb(err); }); }); }; this.getOutgoingBundleFileName = function (basePath, sourceAddress, destAddress, cb) { // // Base filename is constructed as such: // * If this |destAddress| is *not* a point address, we use NNNNnnnn where // NNNN is 0 padded hex of dest net - source net and and nnnn is 0 padded // hex of dest node - source node. // * If |destAddress| is a point, NNNN becomes 0000 and nnnn becomes 'p' + // 3 digit 0 padded hex point // // Extension is dd? where dd is Su...Mo and ? is 0...Z as collisions arise // let basename; if (destAddress.point) { const pointHex = `000${destAddress.point}`.substr(-3); basename = `0000p${pointHex}`; } else { basename = `0000${Math.abs(sourceAddress.net - destAddress.net).toString( 16 )}`.substr(-4) + `0000${Math.abs(sourceAddress.node - destAddress.node).toString( 16 )}`.substr(-4); } // // We need to now find the first entry that does not exist starting // with dd0 to ddz // const EXT_SUFFIXES = '0123456789abcdefghijklmnopqrstuvwxyz'.split(''); let fileName = `${basename}.${moment().format('dd').toLowerCase()}`; async.detectSeries( EXT_SUFFIXES, (suffix, callback) => { const checkFileName = fileName + suffix; fs.stat(paths.join(basePath, checkFileName), err => { callback(null, err && 'ENOENT' === err.code ? true : false); }); }, (err, finalSuffix) => { if (finalSuffix) { return cb(null, paths.join(basePath, fileName + finalSuffix)); } return cb(new Error('Could not acquire a bundle filename!')); } ); }; this.prepareMessage = function (message, options) { // // Set various FTN kludges/etc. // const localAddress = new Address(options.network.localAddress); // ensure we have an Address obj not a string version // :TODO: create Address.toMeta() / similar message.meta.FtnProperty = message.meta.FtnProperty || {}; message.meta.FtnKludge = message.meta.FtnKludge || {}; message.meta.FtnProperty.ftn_orig_node = localAddress.node; message.meta.FtnProperty.ftn_orig_network = localAddress.net; message.meta.FtnProperty.ftn_cost = 0; message.meta.FtnProperty.ftn_msg_orig_node = localAddress.node; message.meta.FtnProperty.ftn_msg_orig_net = localAddress.net; const destAddress = options.routeAddress || options.destAddress; message.meta.FtnProperty.ftn_dest_node = destAddress.node; message.meta.FtnProperty.ftn_dest_network = destAddress.net; if (destAddress.zone) { message.meta.FtnProperty.ftn_dest_zone = destAddress.zone; } if (destAddress.point) { message.meta.FtnProperty.ftn_dest_point = destAddress.point; } // tear line and origin can both go in EchoMail & NetMail message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine(); message.meta.FtnProperty.ftn_origin = ftnUtil.getOrigin(localAddress); let ftnAttribute = ftnMailPacket.Packet.Attribute.Local; // message from our system const config = Config(); if (self.isNetMailMessage(message)) { // // Set route and message destination properties -- they may differ // message.meta.FtnProperty.ftn_msg_dest_node = options.destAddress.node; message.meta.FtnProperty.ftn_msg_dest_net = options.destAddress.net; ftnAttribute |= ftnMailPacket.Packet.Attribute.Private; // // NetMail messages need a FRL-1005.001 "Via" line // http://ftsc.org/docs/frl-1005.001 // // :TODO: We need to do this when FORWARDING NetMail /* if(_.isString(message.meta.FtnKludge.Via)) { message.meta.FtnKludge.Via = [ message.meta.FtnKludge.Via ]; } message.meta.FtnKludge.Via = message.meta.FtnKludge.Via || []; message.meta.FtnKludge.Via.push(ftnUtil.getVia(options.network.localAddress)); */ // // We need to set INTL, and possibly FMPT and/or TOPT // See http://retro.fidoweb.ru/docs/index=ftsc&doc=FTS-4001&enc=mac // message.meta.FtnKludge.INTL = ftnUtil.getIntl( options.destAddress, localAddress ); if (_.isNumber(localAddress.point) && localAddress.point > 0) { message.meta.FtnKludge.FMPT = localAddress.point; } if (_.isNumber(options.destAddress.point) && options.destAddress.point > 0) { message.meta.FtnKludge.TOPT = options.destAddress.point; } } else { // // Set appropriate attribute flag for export type // switch (this.getExportType(options.nodeConfig)) { case 'crash': ftnAttribute |= ftnMailPacket.Packet.Attribute.Crash; break; case 'hold': ftnAttribute |= ftnMailPacket.Packet.Attribute.Hold; break; // :TODO: Others? } // // EchoMail requires some additional properties & kludges // message.meta.FtnProperty.ftn_area = config.messageNetworks.ftn.areas[message.areaTag].tag; // // When exporting messages, we should create/update SEEN-BY // with remote address(s) we are exporting to. // const seenByAdditions = [`${localAddress.net}/${localAddress.node}`].concat( config.messageNetworks.ftn.areas[message.areaTag].uplinks ); message.meta.FtnProperty.ftn_seen_by = ftnUtil.getUpdatedSeenByEntries( message.meta.FtnProperty.ftn_seen_by, seenByAdditions ); // // And create/update PATH for ourself // message.meta.FtnKludge.PATH = ftnUtil.getUpdatedPathEntries( message.meta.FtnKludge.PATH, localAddress ); } message.meta.FtnProperty.ftn_attr_flags = ftnAttribute; // // Additional kludges // // Check for existence of MSGID as we may already have stored it from a previous // export that failed to finish // if (!message.meta.FtnKludge.MSGID) { message.meta.FtnKludge.MSGID = ftnUtil.getMessageIdentifier( message, localAddress, message.isPrivate() // true = isNetMail ); } message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset(); // // According to FSC-0046: // // "When a Conference Mail processor adds a TID to a message, it may not // add a PID. An existing TID should, however, be replaced. TIDs follow // the same format used for PIDs, as explained above." // message.meta.FtnKludge.TID = ftnUtil.getProductIdentifier(); // // Determine CHRS and actual internal encoding name. If the message has an // explicit encoding set, use it. Otherwise, try to preserve any CHRS/encoding already set. // let encoding = options.nodeConfig.encoding || config.scannerTossers.ftn_bso.packetMsgEncoding || 'utf8'; const explicitEncoding = _.get(message.meta, 'System.explicit_encoding'); if (explicitEncoding) { encoding = explicitEncoding; } else if (message.meta.FtnKludge.CHRS) { const encFromChars = ftnUtil.getEncodingFromCharacterSetIdentifier( message.meta.FtnKludge.CHRS ); if (encFromChars) { encoding = encFromChars; } } // // Ensure we ended up with something useable. If not, back to utf8! // if (!iconv.encodingExists(encoding)) { Log.debug({ encoding: encoding }, 'Unknown encoding. Falling back to utf8'); encoding = 'utf8'; } options.encoding = encoding; // save for later message.meta.FtnKludge.CHRS = ftnUtil.getCharacterSetIdentifierByEncoding(encoding); }; this.setReplyKludgeFromReplyToMsgId = function (message, cb) { // // Look up MSGID kludge for |message.replyToMsgId|, if any. // If found, we can create a REPLY kludge with the previously // discovered MSGID. // if (0 === message.replyToMsgId) { return cb(null); // nothing to do } Message.getMetaValuesByMessageId( message.replyToMsgId, 'FtnKludge', 'MSGID', (err, msgIdVal) => { if (!err) { assert( _.isString(msgIdVal), 'Expected string but got ' + typeof msgIdVal + ' (' + msgIdVal + ')' ); // got a MSGID - create a REPLY message.meta.FtnKludge.REPLY = msgIdVal; } cb(null); // this method always passes } ); }; // check paths, Addresses, etc. this.isAreaConfigValid = function (areaConfig) { if ( !areaConfig || !_.isString(areaConfig.tag) || !_.isString(areaConfig.network) ) { return false; } if (_.isString(areaConfig.uplinks)) { areaConfig.uplinks = areaConfig.uplinks.split(' '); } return _.isArray(areaConfig.uplinks); }; this.hasValidConfiguration = function ({ shouldLog = false } = {}) { const hasNodes = _.has(this, 'moduleConfig.nodes'); const hasAreas = _.has(Config(), 'messageNetworks.ftn.areas'); if (!hasNodes && !hasAreas) { if (shouldLog) { Log.warn( { 'scannerTossers.ftn_bso.nodes': hasNodes, 'messageNetworks.ftn.areas': hasAreas, }, 'Missing one or more required configuration blocks' ); } return false; } // :TODO: need to check more! return true; }; this.parseScheduleString = function (schedStr) { if (!schedStr) { return; // nothing to parse! } let schedule = {}; const m = SCHEDULE_REGEXP.exec(schedStr); if (m) { schedStr = schedStr.substr(0, m.index).trim(); if ('@watch:' === m[1]) { schedule.watchFile = m[2]; } else if ('@immediate' === m[1]) { schedule.immediate = true; } } if (schedStr.length > 0) { const sched = later.parse.text(schedStr); if (-1 === sched.error) { schedule.sched = sched; } } // return undefined if we couldn't parse out anything useful if (!_.isEmpty(schedule)) { return schedule; } }; this.getAreaLastScanId = function (areaTag, cb) { const sql = `SELECT area_tag, message_id FROM message_area_last_scan WHERE scan_toss = "ftn_bso" AND area_tag = ? LIMIT 1;`; msgDb.get(sql, [areaTag], (err, row) => { return cb(err, row ? row.message_id : 0); }); }; this.setAreaLastScanId = function (areaTag, lastScanId, cb) { const sql = `REPLACE INTO message_area_last_scan (scan_toss, area_tag, message_id) VALUES ("ftn_bso", ?, ?);`; msgDb.run(sql, [areaTag, lastScanId], err => { return cb(err); }); }; this.getNodeConfigByAddress = function (addr) { addr = _.isString(addr) ? Address.fromString(addr) : addr; // :TODO: sort wildcard nodes{} entries by most->least explicit according to FTN hierarchy return _.find(this.moduleConfig.nodes, (node, nodeAddrWildcard) => { return addr.isPatternMatch(nodeAddrWildcard); }); }; this.exportNetMailMessagePacket = function (message, exportOpts, cb) { // // For NetMail, we always create a *single* packet per message. // async.series( [ function generalPrep(callback) { self.prepareMessage(message, exportOpts); return self.setReplyKludgeFromReplyToMsgId(message, callback); }, function createPacket(callback) { const packet = new ftnMailPacket.Packet(); const packetHeader = new ftnMailPacket.PacketHeader( exportOpts.network.localAddress, exportOpts.routeAddress, exportOpts.nodeConfig.packetType ); packetHeader.password = exportOpts.nodeConfig.packetPassword || ''; // use current message ID for filename seed exportOpts.pktFileName = self.getOutgoingPacketFileName( self.exportTempDir, message.messageId, false, // createTempPacket=false exportOpts.fileCase ); const ws = fs.createWriteStream(exportOpts.pktFileName); packet.writeHeader(ws, packetHeader); packet.getMessageEntryBuffer(message, exportOpts, (err, msgBuf) => { if (err) { return callback(err); } ws.write(msgBuf); packet.writeTerminator(ws); ws.end(); ws.once('finish', () => { return callback(null); }); }); }, ], err => { return cb(err); } ); }; this.exportMessagesByUuid = function (messageUuids, exportOpts, cb) { // // This method has a lot of madness going on: // - Try to stuff messages into packets until we've hit the target size // - We need to wait for write streams to finish before proceeding in many cases // or data will be cut off when closing and creating a new stream // let exportedFiles = []; let currPacketSize = self.moduleConfig.packetTargetByteSize; let packet; let ws; let remainMessageBuf; let remainMessageId; const createTempPacket = !_.isString(exportOpts.nodeConfig.archiveType) || 0 === exportOpts.nodeConfig.archiveType.length; function finalizePacket(cb) { packet.writeTerminator(ws); ws.end(); ws.once('finish', () => { return cb(null); }); } async.each( messageUuids, (msgUuid, nextUuid) => { let message = new Message(); async.series( [ function finalizePrevious(callback) { if ( packet && currPacketSize >= self.moduleConfig.packetTargetByteSize ) { return finalizePacket(callback); } else { callback(null); } }, function loadMessage(callback) { message.load({ uuid: msgUuid }, err => { if (err) { return callback(err); } // General preperation self.prepareMessage(message, exportOpts); self.setReplyKludgeFromReplyToMsgId(message, err => { callback(err); }); }); }, function createNewPacket(callback) { if ( currPacketSize >= self.moduleConfig.packetTargetByteSize ) { packet = new ftnMailPacket.Packet(); const packetHeader = new ftnMailPacket.PacketHeader( exportOpts.network.localAddress, exportOpts.destAddress, exportOpts.nodeConfig.packetType ); packetHeader.password = exportOpts.nodeConfig.packetPassword || ''; // use current message ID for filename seed const pktFileName = self.getOutgoingPacketFileName( self.exportTempDir, message.messageId, createTempPacket, exportOpts.fileCase ); exportedFiles.push(pktFileName); ws = fs.createWriteStream(pktFileName); currPacketSize = packet.writeHeader(ws, packetHeader); if (remainMessageBuf) { currPacketSize += packet.writeMessageEntry( ws, remainMessageBuf ); remainMessageBuf = null; } } callback(null); }, function appendMessage(callback) { packet.getMessageEntryBuffer( message, exportOpts, (err, msgBuf) => { if (err) { return callback(err); } currPacketSize += msgBuf.length; if ( currPacketSize >= self.moduleConfig.packetTargetByteSize ) { remainMessageBuf = msgBuf; // save for next packet remainMessageId = message.messageId; } else { ws.write(msgBuf); } return callback(null); } ); }, function storeStateFlags0Meta(callback) { message.persistMetaValue( 'System', 'state_flags0', Message.StateFlags0.Exported.toString(), err => { callback(err); } ); }, function storeMsgIdMeta(callback) { // // We want to store some meta as if we had imported // this message for later reference // if (message.meta.FtnKludge.MSGID) { message.persistMetaValue( 'FtnKludge', 'MSGID', message.meta.FtnKludge.MSGID, err => { callback(err); } ); } else { callback(null); } }, ], err => { nextUuid(err); } ); }, err => { if (err) { cb(err); } else { async.series( [ function terminateLast(callback) { if (packet) { return finalizePacket(callback); } else { callback(null); } }, function writeRemainPacket(callback) { if (remainMessageBuf) { // :TODO: DRY this with the code above -- they are basically identical packet = new ftnMailPacket.Packet(); const packetHeader = new ftnMailPacket.PacketHeader( exportOpts.network.localAddress, exportOpts.destAddress, exportOpts.nodeConfig.packetType ); packetHeader.password = exportOpts.nodeConfig.packetPassword || ''; // use current message ID for filename seed const pktFileName = self.getOutgoingPacketFileName( self.exportTempDir, remainMessageId, createTempPacket, exportOpts.filleCase ); exportedFiles.push(pktFileName); ws = fs.createWriteStream(pktFileName); packet.writeHeader(ws, packetHeader); ws.write(remainMessageBuf); return finalizePacket(callback); } else { callback(null); } }, ], err => { cb(err, exportedFiles); } ); } } ); }; this.getNetMailRoute = function (dstAddr) { // // Route full|wildcard -> full adddress/network lookup // const routes = _.get(Config(), 'scannerTossers.ftn_bso.netMail.routes'); if (!routes) { return; } return _.find(routes, (route, addrWildcard) => { return dstAddr.isPatternMatch(addrWildcard); }); }; this.getNetMailRouteInfoFromAddress = function (destAddress, cb) { // // Attempt to find route information for |destAddress|: // // 1) Routes: scannerTossers.ftn_bso.netMail.routes{} -> scannerTossers.ftn_bso.nodes{} -> config // - Where we send may not be where destAddress is (it's routed!) // 2) Direct to nodes: scannerTossers.ftn_bso.nodes{} -> config // - Where we send is direct to destAddress // // In both cases, attempt to look up Zone:Net/* to discover local "from" network/address // falling back to Config.scannerTossers.ftn_bso.defaultNetwork // const route = this.getNetMailRoute(destAddress); let routeAddress; let networkName; let isRouted; if (route) { routeAddress = Address.fromString(route.address); networkName = route.network; isRouted = true; } else { routeAddress = destAddress; isRouted = false; } networkName = networkName || this.getNetworkNameByAddress(routeAddress); const config = _.find(this.moduleConfig.nodes, (node, nodeAddrWildcard) => { return routeAddress.isPatternMatch(nodeAddrWildcard); }) || { packetType: '2+', encoding: Config().scannerTossers.ftn_bso.packetMsgEncoding, }; // we should never be failing here; we may just be using defaults. return cb( networkName ? null : Errors.DoesNotExist(`No NetMail route for ${destAddress.toString()}`), { destAddress, routeAddress, networkName, config, isRouted } ); }; this.exportNetMailMessagesToUplinks = function (messagesOrMessageUuids, cb) { // for each message/UUID, find where to send the thing async.each( messagesOrMessageUuids, (msgOrUuid, nextMessageOrUuid) => { const exportOpts = {}; const message = new Message(); async.series( [ function loadMessage(callback) { if (_.isString(msgOrUuid)) { message.load({ uuid: msgOrUuid }, err => { return callback(err, message); }); } else { return callback(null, msgOrUuid); } }, function discoverUplink(callback) { const dstAddr = new Address( message.meta.System[Message.SystemMetaNames.RemoteToUser] ); self.getNetMailRouteInfoFromAddress( dstAddr, (err, routeInfo) => { if (err) { return callback(err); } exportOpts.nodeConfig = routeInfo.config; exportOpts.destAddress = dstAddr; exportOpts.routeAddress = routeInfo.routeAddress; exportOpts.fileCase = routeInfo.config.fileCase || 'lower'; exportOpts.network = Config().messageNetworks.ftn.networks[ routeInfo.networkName ]; exportOpts.networkName = routeInfo.networkName; exportOpts.outgoingDir = self.getOutgoingEchoMailPacketDir( exportOpts.networkName, exportOpts.destAddress ); exportOpts.exportType = self.getExportType( routeInfo.config ); if (!exportOpts.network) { return callback( Errors.DoesNotExist( `No configuration found for network ${routeInfo.networkName}` ) ); } return callback(null); } ); }, function createOutgoingDir(callback) { // ensure outgoing NetMail directory exists return fse.mkdirs(exportOpts.outgoingDir, callback); }, function exportPacket(callback) { return self.exportNetMailMessagePacket( message, exportOpts, callback ); }, function moveToOutgoing(callback) { const newExt = exportOpts.fileCase === 'lower' ? '.pkt' : '.PKT'; exportOpts.exportedToPath = paths.join( exportOpts.outgoingDir, `${paths.basename( exportOpts.pktFileName, paths.extname(exportOpts.pktFileName) )}${newExt}` ); return fse.move( exportOpts.pktFileName, exportOpts.exportedToPath, callback ); }, function prepareFloFile(callback) { const flowFilePath = self.getOutgoingFlowFileName( exportOpts.outgoingDir, exportOpts.routeAddress, 'ref', exportOpts.exportType, exportOpts.fileCase ); return self.flowFileAppendRefs( flowFilePath, [exportOpts.exportedToPath], '^', callback ); }, function storeStateFlags0Meta(callback) { return message.persistMetaValue( 'System', 'state_flags0', Message.StateFlags0.Exported.toString(), callback ); }, function storeMsgIdMeta(callback) { // Store meta as if we had imported this message -- for later reference if (message.meta.FtnKludge.MSGID) { return message.persistMetaValue( 'FtnKludge', 'MSGID', message.meta.FtnKludge.MSGID, callback ); } return callback(null); }, ], err => { if (err) { Log.warn( { error: err.message }, `Error exporting message: ${err.message}` ); } return nextMessageOrUuid(null); } ); }, err => { if (err) { Log.warn({ error: err.message }, 'Error(s) during NetMail export'); } return cb(err); } ); }; this.exportEchoMailMessagesToUplinks = function (messageUuids, areaConfig, cb) { const config = Config(); async.each( areaConfig.uplinks, (uplink, nextUplink) => { const nodeConfig = self.getNodeConfigByAddress(uplink); if (!nodeConfig) { return nextUplink(); } const exportOpts = { nodeConfig, network: config.messageNetworks.ftn.networks[areaConfig.network], destAddress: Address.fromString(uplink), networkName: areaConfig.network, fileCase: nodeConfig.fileCase || 'lower', }; if (_.isString(exportOpts.network.localAddress)) { exportOpts.network.localAddress = Address.fromString( exportOpts.network.localAddress ); } const outgoingDir = self.getOutgoingEchoMailPacketDir( exportOpts.networkName, exportOpts.destAddress ); const exportType = self.getExportType(exportOpts.nodeConfig); async.waterfall( [ function createOutgoingDir(callback) { fse.mkdirs(outgoingDir, err => { callback(err); }); }, function exportToTempArea(callback) { self.exportMessagesByUuid(messageUuids, exportOpts, callback); }, function createArcMailBundle(exportedFileNames, callback) { if ( self.archUtil.haveArchiver( exportOpts.nodeConfig.archiveType ) ) { // :TODO: support bundleTargetByteSize: // // Compress to a temp location then we'll move it in the next step // // Note that we must use the *final* output dir for getOutgoingBundleFileName() // as it checks for collisions in bundle names! // self.getOutgoingBundleFileName( outgoingDir, exportOpts.network.localAddress, exportOpts.destAddress, (err, bundlePath) => { if (err) { return callback(err); } // adjust back to temp path const tempBundlePath = paths.join( self.exportTempDir, paths.basename(bundlePath) ); self.archUtil.compressTo( exportOpts.nodeConfig.archiveType, tempBundlePath, exportedFileNames, err => { callback(err, [tempBundlePath]); } ); } ); } else { callback(null, exportedFileNames); } }, function moveFilesToOutgoing(exportedFileNames, callback) { async.each( exportedFileNames, (oldPath, nextFile) => { const ext = paths.extname(oldPath).toLowerCase(); if ('.pk_' === ext.toLowerCase()) { // // For a given temporary .pk_ file, we need to move it to the outoing // directory with the appropriate BSO style filename. // const newExt = self.getOutgoingFlowFileExtension( exportOpts.destAddress, 'mail', exportType, exportOpts.fileCase ); const newPath = paths.join( outgoingDir, `${paths.basename(oldPath, ext)}${newExt}` ); fse.move(oldPath, newPath, nextFile); } else { const newPath = paths.join( outgoingDir, paths.basename(oldPath) ); fse.move(oldPath, newPath, err => { if (err) { Log.warn( { oldPath: oldPath, newPath: newPath, error: err.toString(), }, 'Failed moving temporary bundle file!' ); return nextFile(); } // // For bundles, we need to append to the appropriate flow file // const flowFilePath = self.getOutgoingFlowFileName( outgoingDir, exportOpts.destAddress, 'ref', exportType, exportOpts.fileCase ); // directive of '^' = delete file after transfer self.flowFileAppendRefs( flowFilePath, [newPath], '^', err => { if (err) { Log.warn( { path: flowFilePath }, 'Failed appending flow reference record!' ); } nextFile(); } ); }); } }, callback ); }, ], err => { // :TODO: do something with |err| ? if (err) { Log.warn(err.message); } nextUplink(); } ); }, cb ); // complete }; this.setReplyToMsgIdFtnReplyKludge = function (message, cb) { // // Given a FTN REPLY kludge, set |message.replyToMsgId|, if possible, // by looking up an associated MSGID kludge meta. // // See also: http://ftsc.org/docs/fts-0009.001 // if (!_.isString(message.meta.FtnKludge.REPLY)) { // nothing to do return cb(); } Message.getMessageIdsByMetaValue( 'FtnKludge', 'MSGID', message.meta.FtnKludge.REPLY, (err, msgIds) => { if (msgIds && msgIds.length > 0) { // expect a single match, but dupe checking is not perfect - warn otherwise if (1 === msgIds.length) { message.replyToMsgId = msgIds[0]; } else { Log.warn( { msgIds: msgIds, replyKludge: message.meta.FtnKludge.REPLY }, 'Found 2:n MSGIDs matching REPLY kludge!' ); } } cb(); } ); }; this.getLocalUserNameFromAlias = function (lookup) { lookup = lookup.toLowerCase(); const aliases = _.get(Config(), 'messageNetworks.ftn.netMail.aliases'); if (!aliases) { return lookup; // keep orig } const alias = _.find(aliases, (localName, alias) => { return alias.toLowerCase() === lookup; }); return alias || lookup; }; this.getAddressesFromNetMailMessage = function (message) { const intlKludge = _.get(message, 'meta.FtnKludge.INTL'); if (!intlKludge) { return {}; } let [to, from] = intlKludge.split(' '); if (!to || !from) { return {}; } const fromPoint = _.get(message, 'meta.FtnKludge.FMPT'); const toPoint = _.get(message, 'meta.FtnKludge.TOPT'); if (fromPoint) { from += `.${fromPoint}`; } if (toPoint) { to += `.${toPoint}`; } return { to: Address.fromString(to), from: Address.fromString(from) }; }; this.importMailToArea = function (config, header, message, cb) { async.series( [ function validateDestinationAddress(callback) { const localNetworkPattern = `${message.meta.FtnProperty.ftn_dest_network}/${message.meta.FtnProperty.ftn_dest_node}`; const localNetworkName = self.getNetworkNameByAddressPattern(localNetworkPattern); return callback( _.isString(localNetworkName) ? null : new Error('Packet destination is not us') ); }, function checkForDupeMSGID(callback) { // // If we have a MSGID, don't allow a dupe // if (!_.has(message.meta, 'FtnKludge.MSGID')) { return callback(null); } Message.getMessageIdsByMetaValue( 'FtnKludge', 'MSGID', message.meta.FtnKludge.MSGID, (err, msgIds) => { if (msgIds && msgIds.length > 0) { const err = new Error('Duplicate MSGID'); err.code = 'DUPE_MSGID'; return callback(err); } return callback(null); } ); }, function basicSetup(callback) { message.areaTag = config.localAreaTag; // indicate this was imported from FTN message.meta.System[Message.SystemMetaNames.ExternalFlavor] = Message.AddressFlavor.FTN; // // If we *allow* dupes (disabled by default), then just generate // a random UUID. Otherwise, don't assign the UUID just yet. It will be // generated at persist() time and should be consistent across import/exports // if ( true === _.get( Config(), [ 'messageNetworks', 'ftn', 'areas', config.localAreaTag, 'allowDupes', ], false ) ) { // just generate a UUID & therefor always allow for dupes message.messageUuid = UUIDv4(); } return callback(null); }, function setReplyToMessageId(callback) { self.setReplyToMsgIdFtnReplyKludge(message, () => { return callback(null); }); }, function setupPrivateMessage(callback) { // // If this is a private message (e.g. NetMail) we set the local user ID // if (Message.WellKnownAreaTags.Private !== config.localAreaTag) { return callback(null); } // // Create a meta value for the *remote* from user. In the case here with FTN, // their fully qualified FTN from address // const { from } = self.getAddressesFromNetMailMessage(message); if (!from) { return callback( Errors.Invalid( 'Cannot import FTN NetMail without valid INTL line' ) ); } message.meta.System[Message.SystemMetaNames.RemoteFromUser] = from.toString(); const lookupName = self.getLocalUserNameFromAlias(message.toUserName); User.getUserIdAndNameByLookup( lookupName, (err, localToUserId, localUserName) => { if (err) { // // Couldn't find a local username. If the toUserName itself is a FTN address // we can only assume the message is to the +op, else we'll have to fail. // const toUserNameAsAddress = Address.fromString( message.toUserName ); if ( toUserNameAsAddress && toUserNameAsAddress.isValid() ) { Log.info( { toUserName: message.toUserName, fromUserName: message.fromUserName, }, 'No local "to" username for FTN message. Appears to be a FTN address only; assuming addressed to SysOp' ); User.getUserName( User.RootUserID, (err, sysOpUserName) => { if (err) { return callback( Errors.UnexpectedState( 'Failed to get SysOp user information' ) ); } message.meta.System[ Message.SystemMetaNames.LocalToUserID ] = User.RootUserID; message.toUserName = sysOpUserName; return callback(null); } ); } else { return callback( Errors.DoesNotExist( `Could not get local user ID for "${message.toUserName}": ${err.message}` ) ); } } // we do this after such that error cases can be preserved above if (lookupName !== message.toUserName) { message.toUserName = localUserName; } // set the meta information - used elsewhere for retrieval message.meta.System[Message.SystemMetaNames.LocalToUserID] = localToUserId; return callback(null); } ); }, function persistImport(callback) { // mark as imported message.meta.System.state_flags0 = Message.StateFlags0.Imported.toString(); // save to disc message.persist(err => { if (!message.isPrivate()) { StatLog.incrementNonPersistentSystemStat( SysProps.MessageTotalCount, 1 ); StatLog.incrementNonPersistentSystemStat( SysProps.MessagesToday, 1 ); } return callback(err); }); }, ], err => { cb(err); } ); }; this.appendTearAndOrigin = function (message) { if (message.meta.FtnProperty.ftn_tear_line) { message.message += `\r\n${message.meta.FtnProperty.ftn_tear_line}\r\n`; } if (message.meta.FtnProperty.ftn_origin) { message.message += `${message.meta.FtnProperty.ftn_origin}\r\n`; } }; // // Ref. implementations on import: // * https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/pkt.c // https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/handle.c // this.importMessagesFromPacketFile = function (packetPath, password, cb) { let packetHeader; const packetOpts = { keepTearAndOrigin: false }; // needed so we can calc message UUID without these; we'll add later let importStats = { packetPath, areaSuccess: {}, // areaTag->count areaFail: {}, // areaTag->count otherFail: 0, }; new ftnMailPacket.Packet(packetOpts).read( packetPath, (entryType, entryData, next) => { if ('header' === entryType) { packetHeader = entryData; const localNetworkName = self.getNetworkNameByAddress( packetHeader.destAddress ); if (!_.isString(localNetworkName)) { const addrString = new Address( packetHeader.destAddress ).toString(); return next( new Error( `No local configuration for packet addressed to ${addrString}` ) ); } else { // :TODO: password needs validated - need to determine if it will use the same node config (which can have wildcards) or something else?! return next(null); } } else if ('message' === entryType) { const message = entryData; const areaTag = message.meta.FtnProperty.ftn_area; let localAreaTag; if (areaTag) { localAreaTag = self.getLocalAreaTagByFtnAreaTag(areaTag); if (!localAreaTag) { // // No local area configured for this import // // :TODO: Handle the "catch all" area bucket case if configured -> email with area info/etc.? catchAll: enabled, areaTag, prefixMsg Log.warn( { areaTag: areaTag }, `No local message area for "${areaTag}"` ); // bump generic failure importStats.otherFail += 1; return next(null); } } else { // // No area tag: If marked private in attributes, this is a NetMail // if ( message.meta.FtnProperty.ftn_attr_flags & ftnMailPacket.Packet.Attribute.Private ) { localAreaTag = Message.WellKnownAreaTags.Private; } else { Log.warn('Non-private message without area tag'); importStats.otherFail += 1; return next(null); } } message.messageUuid = Message.createMessageUUID( localAreaTag, message.modTimestamp, message.subject, message.message ); self.appendTearAndOrigin(message); const importConfig = { localAreaTag }; self.importMailToArea(importConfig, packetHeader, message, err => { if (err) { // bump area fail stats importStats.areaFail[localAreaTag] = (importStats.areaFail[localAreaTag] || 0) + 1; if ( 'SQLITE_CONSTRAINT' === err.code || 'DUPE_MSGID' === err.code ) { const msgId = _.has(message.meta, 'FtnKludge.MSGID') ? message.meta.FtnKludge.MSGID : 'N/A'; Log.info( { area: localAreaTag, subject: message.subject, uuid: message.messageUuid, MSGID: msgId, }, `Not importing non-unique message "${message.subject}" to ${localAreaTag}` ); return next(null); } } else { // bump area success importStats.areaSuccess[localAreaTag] = (importStats.areaSuccess[localAreaTag] || 0) + 1; } return next(err); }); } }, err => { // // try to produce something helpful in the log // const makeCount = obj => { return obj ? _.reduce( obj, (sum, c) => { return sum + c; }, 0 ) : 0; }; const totalFail = makeCount(importStats.areaFail) + importStats.otherFail; const packetFileName = paths.basename(packetPath); if (err || totalFail > 0) { if (err) { Object.assign(importStats, { error: err.message }); } Log.warn( importStats, `Packet ${packetFileName} import reported ${totalFail} error(s)` ); } else { const totalSuccess = makeCount(importStats.areaSuccess); Log.info( importStats, `Packet ${packetFileName} imported with ${totalSuccess} new message(s)` ); } cb(err); } ); }; this.maybeArchiveImportFile = function (origPath, type, status, cb) { // // type : pkt|tic|bundle // status : good|reject // // Status of "good" is only applied to pkt files & placed // in |retain| if set. This is generally used for debugging only. // let archivePath; const ts = moment().format('YYYY-MM-DDTHH.mm.ss.SSS'); const fn = paths.basename(origPath); if ('good' === status && type === 'pkt') { if (!_.isString(self.moduleConfig.paths.retain)) { return cb(null); } archivePath = paths.join( self.moduleConfig.paths.retain, `good-pkt-${ts}--${fn}` ); } else if ('good' !== status) { archivePath = paths.join( self.moduleConfig.paths.reject, `${status}-${type}--${ts}-${fn}` ); } else { return cb(null); // don't archive non-good/pkt files } Log.debug( { origPath: origPath, archivePath: archivePath, type: type, status: status }, 'Archiving import file' ); fse.copy(origPath, archivePath, err => { if (err) { Log.warn( { error: err.message, origPath: origPath, archivePath: archivePath, type: type, status: status, }, 'Failed to archive packet file' ); } return cb(null); // never fatal }); }; this.importPacketFilesFromDirectory = function (importDir, password, cb) { async.waterfall( [ function getPacketFiles(callback) { fs.readdir(importDir, (err, files) => { if (err) { return callback(err); } callback( null, files.filter(f => '.pkt' === paths.extname(f).toLowerCase()) ); }); }, function importPacketFiles(packetFiles, callback) { let rejects = []; async.eachSeries( packetFiles, (packetFile, nextFile) => { self.importMessagesFromPacketFile( paths.join(importDir, packetFile), '', err => { if (err) { Log.debug( { path: paths.join(importDir, packetFile), error: err.toString(), }, `Failed to import packet file "${paths.basename( packetFile )}"` ); rejects.push(packetFile); } nextFile(); } ); }, err => { // :TODO: Handle err! we should try to keep going though... callback(err, packetFiles, rejects); } ); }, function handleProcessedFiles(packetFiles, rejects, callback) { async.each( packetFiles, (packetFile, nextFile) => { // possibly archive, then remove original const fullPath = paths.join(importDir, packetFile); self.maybeArchiveImportFile( fullPath, 'pkt', rejects.includes(packetFile) ? 'reject' : 'good', () => { fs.unlink(fullPath, () => { return nextFile(null); }); } ); }, err => { callback(err); } ); }, ], err => { cb(err); } ); }; this.importFromDirectory = function (inboundType, importDir, cb) { async.waterfall( [ // start with .pkt files function importPacketFiles(callback) { self.importPacketFilesFromDirectory(importDir, '', err => { callback(err); }); }, function discoverBundles(callback) { fs.readdir(importDir, (err, files) => { // :TODO: if we do much more of this, probably just use the glob module const bundleRegExp = /\.(su|mo|tu|we|th|fr|sa)[0-9a-z]/i; files = files.filter(f => { const fext = paths.extname(f); return bundleRegExp.test(fext); }); async.map( files, (file, transform) => { const fullPath = paths.join(importDir, file); self.archUtil.detectType(fullPath, (err, archName) => { transform(null, { path: fullPath, archName: archName, }); }); }, (err, bundleFiles) => { callback(err, bundleFiles); } ); }); }, function importBundles(bundleFiles, callback) { let rejects = []; async.each( bundleFiles, (bundleFile, nextFile) => { if (_.isUndefined(bundleFile.archName)) { Log.warn( { fileName: bundleFile.path }, 'Unknown bundle archive type' ); rejects.push(bundleFile.path); return nextFile(); // unknown archive type } Log.debug({ bundleFile: bundleFile }, 'Processing bundle'); self.archUtil.extractTo( bundleFile.path, self.importTempDir, bundleFile.archName, err => { if (err) { Log.warn( { path: bundleFile.path, error: err.message }, 'Failed to extract bundle' ); rejects.push(bundleFile.path); } nextFile(); } ); }, err => { if (err) { return callback(err); } // // All extracted - import .pkt's // self.importPacketFilesFromDirectory( self.importTempDir, '', () => { // :TODO: handle |err| callback(null, bundleFiles, rejects); } ); } ); }, function handleProcessedBundleFiles(bundleFiles, rejects, callback) { async.each( bundleFiles, (bundleFile, nextFile) => { self.maybeArchiveImportFile( bundleFile.path, 'bundle', rejects.includes(bundleFile.path) ? 'reject' : 'good', () => { fs.unlink(bundleFile.path, err => { if (err) { Log.error( { path: bundleFile.path, error: err.message, }, 'Failed unlinking bundle' ); } return nextFile(null); }); } ); }, err => { callback(err); } ); }, function importTicFiles(callback) { self.processTicFilesInDirectory(importDir, err => { return callback(err); }); }, ], err => { cb(err); } ); }; this.createTempDirectories = function (cb) { temptmp.mkdir({ prefix: 'enigftnexport-' }, (err, tempDir) => { if (err) { return cb(err); } self.exportTempDir = tempDir; temptmp.mkdir({ prefix: 'enigftnimport-' }, (err, tempDir) => { self.importTempDir = tempDir; cb(err); }); }); }; // Starts an export block - returns true if we can proceed this.exportingStart = function () { if (!this.exportRunning) { this.exportRunning = true; return true; } return false; }; // ends an export block this.exportingEnd = function (cb) { this.exportRunning = false; if (cb) { return cb(null); } }; this.copyTicAttachment = function (src, dst, isUpdate, cb) { if (isUpdate) { fse.copy(src, dst, { overwrite: true }, err => { return cb(err, dst); }); } else { copyFileWithCollisionHandling(src, dst, (err, finalPath) => { return cb(err, finalPath); }); } }; this.getLocalAreaTagsForTic = function () { const config = Config(); return _.union( Object.keys(config.scannerTossers.ftn_bso.ticAreas || {}), Object.keys(config.fileBase.areas) ); }; this.processSingleTicFile = function (ticFileInfo, cb) { Log.debug( { tic: ticFileInfo.path, file: ticFileInfo.getAsString('File') }, 'Processing TIC file' ); async.waterfall( [ function generalValidation(callback) { const sysConfig = Config(); const config = { nodes: sysConfig.scannerTossers.ftn_bso.nodes, defaultPassword: sysConfig.scannerTossers.ftn_bso.tic.password, localAreaTags: self.getLocalAreaTagsForTic(), }; ticFileInfo.validate(config, (err, localInfo) => { if (err) { Log.trace({ reason: err.message }, 'Validation failure'); return callback(err); } // We may need to map |localAreaTag| back to real areaTag if it's a mapping/alias const mappedLocalAreaTag = _.get( Config().scannerTossers.ftn_bso, ['ticAreas', localInfo.areaTag] ); if (mappedLocalAreaTag) { if (_.isString(mappedLocalAreaTag.areaTag)) { localInfo.areaTag = mappedLocalAreaTag.areaTag; localInfo.hashTags = mappedLocalAreaTag.hashTags; // override default for node localInfo.storageTag = mappedLocalAreaTag.storageTag; // override default } else if (_.isString(mappedLocalAreaTag)) { localInfo.areaTag = mappedLocalAreaTag; } } return callback(null, localInfo); }); }, function findExistingItem(localInfo, callback) { // // We will need to look for an existing item to replace/update if: // a) The TIC file has a "Replaces" field // b) The general or node specific |allowReplace| is true // // Replace specifies a DOS 8.3 *pattern* which is allowed to have // ? and * characters. For example, RETRONET.* // // Lastly, we will only replace if the item is in the same/specified area // and that come from the same origin as a previous entry. // const allowReplace = _.get( Config().scannerTossers.ftn_bso.nodes, [localInfo.node, 'tic', 'allowReplace'], Config().scannerTossers.ftn_bso.tic.allowReplace ); const replaces = ticFileInfo.getAsString('Replaces'); if (!allowReplace || !replaces) { return callback(null, localInfo); } const metaPairs = [ { name: 'short_file_name', value: replaces.toUpperCase(), // we store upper as well wildcards: true, // value may contain wildcards }, { name: 'tic_origin', value: ticFileInfo.getAsString('Origin'), }, ]; FileEntry.findFiles( { metaPairs: metaPairs, areaTag: localInfo.areaTag }, (err, fileIds) => { if (err) { return callback(err); } // 0:1 allowed if (1 === fileIds.length) { localInfo.existingFileId = fileIds[0]; // fetch old filename - we may need to remove it if replacing with a new name FileEntry.loadBasicEntry( localInfo.existingFileId, {}, (err, info) => { if (info) { Log.trace( { fileId: localInfo.existingFileId, oldFileName: info.fileName, oldStorageTag: info.storageTag, }, 'Existing TIC file target to be replaced' ); localInfo.oldFileName = info.fileName; localInfo.oldStorageTag = info.storageTag; } return callback(null, localInfo); // continue even if we couldn't find an old match } ); } else if (fileIds.length > 1) { return callback( Errors.General( `More than one existing entry for TIC in ${ localInfo.areaTag } ([${fileIds.join(', ')}])` ) ); } else { return callback(null, localInfo); } } ); }, function scan(localInfo, callback) { const scanOpts = { sha256: localInfo.sha256, // *may* have already been calculated meta: { // some TIC-related metadata we always want short_file_name: ticFileInfo .getAsString('File') .toUpperCase(), // upper to ensure no case issues later; this should be a DOS 8.3 name tic_origin: ticFileInfo.getAsString('Origin'), tic_desc: ticFileInfo.getAsString('Desc'), upload_by_username: _.get( Config().scannerTossers.ftn_bso.nodes, [localInfo.node, 'tic', 'uploadBy'], Config().scannerTossers.ftn_bso.tic.uploadBy ), }, }; const ldesc = ticFileInfo.getAsString('Ldesc', '\n'); if (ldesc) { scanOpts.meta.tic_ldesc = ldesc; } // // We may have TIC auto-tagging for this node and/or specific (remote) area // const hashTags = localInfo.hashTags || _.get(Config().scannerTossers.ftn_bso.nodes, [ localInfo.node, 'tic', 'hashTags', ]); // catch-all*/ if (hashTags) { scanOpts.hashTags = new Set(hashTags.split(/[\s,]+/)); } if (localInfo.crc32) { scanOpts.meta.file_crc32 = localInfo.crc32.toString(16); // again, *may* have already been calculated } scanFile(ticFileInfo.filePath, scanOpts, (err, fileEntry) => { if (err) { Log.trace({ reason: err.message }, 'Scanning failed'); } localInfo.fileEntry = fileEntry; return callback(err, localInfo); }); }, function store(localInfo, callback) { // // Move file to final area storage and persist to DB // const areaInfo = getFileAreaByTag(localInfo.areaTag); if (!areaInfo) { return callback( Errors.UnexpectedState( `Could not get area for tag ${localInfo.areaTag}` ) ); } const storageTag = localInfo.storageTag || areaInfo.storageTags[0]; if (!isValidStorageTag(storageTag)) { return callback( Errors.Invalid(`Invalid storage tag: ${storageTag}`) ); } localInfo.fileEntry.storageTag = storageTag; localInfo.fileEntry.areaTag = localInfo.areaTag; localInfo.fileEntry.fileName = ticFileInfo.longFileName; // // We may now have two descriptions: from .DIZ/etc. or the TIC itself. // Determine which one to use using |descPriority| and availability. // // We will still fallback as needed from -> -> // const descPriority = _.get( Config().scannerTossers.ftn_bso.nodes, [localInfo.node, 'tic', 'descPriority'], Config().scannerTossers.ftn_bso.tic.descPriority ); if ('tic' === descPriority) { const origDesc = localInfo.fileEntry.desc; localInfo.fileEntry.desc = ticFileInfo.getAsString('Ldesc') || origDesc || getDescFromFileName(ticFileInfo.filePath); } else { // see if we got desc from .DIZ/etc. const fromDescFile = 'descFile' === localInfo.fileEntry.descSrc; localInfo.fileEntry.desc = fromDescFile ? localInfo.fileEntry.desc : ticFileInfo.getAsString('Ldesc'); localInfo.fileEntry.desc = localInfo.fileEntry.desc || getDescFromFileName(ticFileInfo.filePath); } const areaStorageDir = getAreaStorageDirectoryByTag(storageTag); if (!areaStorageDir) { return callback( Errors.UnexpectedState( `Could not get storage directory for tag ${localInfo.areaTag}` ) ); } const isUpdate = localInfo.existingFileId ? true : false; if (isUpdate) { // we need to *update* an existing record/file localInfo.fileEntry.fileId = localInfo.existingFileId; } const dst = paths.join(areaStorageDir, localInfo.fileEntry.fileName); self.copyTicAttachment( ticFileInfo.filePath, dst, isUpdate, (err, finalPath) => { if (err) { Log.info( { reason: err.message }, 'Failed to copy TIC attachment' ); return callback(err); } if (dst !== finalPath) { localInfo.fileEntry.fileName = paths.basename(finalPath); } localInfo.newPath = dst; localInfo.fileEntry.persist(isUpdate, err => { return callback(err, localInfo); }); } ); }, // :TODO: from here, we need to re-toss files if needed, before they are removed function cleanupOldFile(localInfo, callback) { if (!localInfo.existingFileId) { return callback(null, localInfo); } const oldStorageDir = getAreaStorageDirectoryByTag( localInfo.oldStorageTag ); const oldPath = paths.join(oldStorageDir, localInfo.oldFileName); // if we updated a file in place, don't delete it! if (localInfo.newPath === oldPath) { Log.trace( { path: oldPath }, 'TIC file replaced in place. Nothing to remove.' ); return callback(null, localInfo); } fs.unlink(oldPath, err => { if (err) { Log.warn( { error: err.message, oldPath: oldPath }, 'Failed removing old physical file during TIC replacement' ); } else { Log.trace( { oldPath: oldPath }, 'Removed old physical file during TIC replacement' ); } return callback(null, localInfo); // continue even if err }); }, ], (err, localInfo) => { if (err) { Log.error( { error: err.message, reason: err.reason, tic: ticFileInfo.filePath, }, `Failed to import/update TIC for "${paths.basename( ticFileInfo.filePath )}"` ); } else { Log.info( { tic: ticFileInfo.path, file: ticFileInfo.filePath, area: localInfo.areaTag, }, `TIC imported "${paths.basename(ticFileInfo.filePath)}" -> ${ localInfo.areaTag }` ); } return cb(err); } ); }; this.removeAssocTicFiles = function (ticFileInfo, cb) { async.each( [ticFileInfo.path, ticFileInfo.filePath], (path, nextPath) => { fs.unlink(path, err => { if (err && 'ENOENT' !== err.code) { // don't log when the file doesn't exist Log.warn( { error: err.message, path: path }, 'Failed unlinking TIC file' ); } return nextPath(null); }); }, err => { return cb(err); } ); }; this.performEchoMailExport = function (cb) { // // Select all messages with a |message_id| > |lastScanId|. // Additionally exclude messages with the System state_flags0 which will be present for // imported or already exported messages // // NOTE: If StateFlags0 starts to use additional bits, we'll likely need to check them here! // const getNewUuidsSql = `SELECT message_id, message_uuid FROM message m WHERE area_tag = ? AND message_id > ? AND (SELECT COUNT(message_id) FROM message_meta WHERE message_id = m.message_id AND meta_category = 'System' AND meta_name = 'state_flags0') = 0 ORDER BY message_id;`; // we shouldn't, but be sure we don't try to pick up private mail here const config = Config(); const areaTags = Object.keys(config.messageNetworks.ftn.areas).filter( areaTag => Message.WellKnownAreaTags.Private !== areaTag ); async.each( areaTags, (areaTag, nextArea) => { const areaConfig = config.messageNetworks.ftn.areas[areaTag]; if (!this.isAreaConfigValid(areaConfig)) { return nextArea(); } // // For each message that is newer than that of the last scan // we need to export to each configured associated uplink(s) // async.waterfall( [ function getLastScanId(callback) { self.getAreaLastScanId(areaTag, callback); }, function getNewUuids(lastScanId, callback) { msgDb.all( getNewUuidsSql, [areaTag, lastScanId], (err, rows) => { if (err) { callback(err); } else { if (0 === rows.length) { let nothingToDoErr = new Error( 'Nothing to do!' ); nothingToDoErr.noRows = true; callback(nothingToDoErr); } else { callback(null, rows); } } } ); }, function exportToConfiguredUplinks(msgRows, callback) { const uuidsOnly = msgRows.map(r => r.message_uuid); // convert to array of UUIDs only self.exportEchoMailMessagesToUplinks( uuidsOnly, areaConfig, err => { const newLastScanId = msgRows[msgRows.length - 1].message_id; Log.info( { areaTag: areaTag, messagesExported: msgRows.length, newLastScanId: newLastScanId, }, 'Export complete' ); callback(err, newLastScanId); } ); }, function updateLastScanId(newLastScanId, callback) { self.setAreaLastScanId(areaTag, newLastScanId, callback); }, ], () => { return nextArea(); } ); }, err => { return cb(err); } ); }; this.performNetMailExport = function (cb) { // // Select all messages with a |message_id| > |lastScanId| in the private area // that are schedule for export to FTN-style networks. // // Just like EchoMail, we additionally exclude messages with the System state_flags0 // which will be present for imported or already exported messages // // // :TODO: fill out the rest of the consts here // :TODO: this statement is crazy ugly -- use JOIN / NOT EXISTS for state_flags & 0x02 const getNewUuidsSql = `SELECT message_id, message_uuid FROM message m WHERE area_tag = '${Message.WellKnownAreaTags.Private}' AND message_id > ? AND (SELECT COUNT(message_id) FROM message_meta WHERE message_id = m.message_id AND meta_category = 'System' AND (meta_name = 'state_flags0' OR meta_name = 'local_to_user_id') ) = 0 AND (SELECT COUNT(message_id) FROM message_meta WHERE message_id = m.message_id AND meta_category = 'System' AND meta_name = '${Message.SystemMetaNames.ExternalFlavor}' AND meta_value = '${Message.AddressFlavor.FTN}' ) = 1 ORDER BY message_id; `; async.waterfall( [ function getLastScanId(callback) { return self.getAreaLastScanId( Message.WellKnownAreaTags.Private, callback ); }, function getNewUuids(lastScanId, callback) { msgDb.all(getNewUuidsSql, [lastScanId], (err, rows) => { if (err) { return callback(err); } if (0 === rows.length) { return cb(null); // note |cb| -- early bail out! } return callback(null, rows); }); }, function exportMessages(rows, callback) { const messageUuids = rows.map(r => r.message_uuid); return self.exportNetMailMessagesToUplinks(messageUuids, callback); }, ], err => { return cb(err); } ); }; this.isNetMailMessage = function (message) { return ( message.isPrivate() && null === _.get(message, 'meta.System.LocalToUserID', null) && Message.AddressFlavor.FTN === _.get(message, 'meta.System.external_flavor', null) ); }; } require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule); // :TODO: *scheduled* portion of this stuff should probably use event_scheduler - @immediate would still use record(). FTNMessageScanTossModule.prototype.processTicFilesInDirectory = function (importDir, cb) { // :TODO: pass in 'inbound' vs 'secInbound' -- pass along to processSingleTicFile() where password will be checked const self = this; async.waterfall( [ function findTicFiles(callback) { fs.readdir(importDir, (err, files) => { if (err) { return callback(err); } return callback( null, files.filter(f => '.tic' === paths.extname(f).toLowerCase()) ); }); }, function gatherInfo(ticFiles, callback) { const ticFilesInfo = []; async.each( ticFiles, (fileName, nextFile) => { const fullPath = paths.join(importDir, fileName); TicFileInfo.createFromFile(fullPath, (err, ticInfo) => { if (err) { Log.warn( { error: err.message, path: fullPath }, 'Failed reading TIC file' ); } else { ticFilesInfo.push(ticInfo); } return nextFile(null); }); }, err => { return callback(err, ticFilesInfo); } ); }, function process(ticFilesInfo, callback) { async.eachSeries( ticFilesInfo, (ticFileInfo, nextTicInfo) => { self.processSingleTicFile(ticFileInfo, err => { if (err) { // :TODO: If ENOENT -OR- failed due to CRC mismatch: create a pending state & try again later; the "attached" file may not yet be ready. // archive rejected TIC stuff (.TIC + attach) async.each( [ticFileInfo.path, ticFileInfo.filePath], (path, nextPath) => { if (!path) { // possibly rejected due to "File" not existing/etc. return nextPath(null); } self.maybeArchiveImportFile( path, 'tic', 'reject', () => { return nextPath(null); } ); }, () => { self.removeAssocTicFiles(ticFileInfo, () => { return nextTicInfo(null); }); } ); } else { self.removeAssocTicFiles(ticFileInfo, () => { return nextTicInfo(null); }); } }); }, err => { return callback(err); } ); }, ], err => { return cb(err); } ); }; FTNMessageScanTossModule.prototype.startup = function (cb) { Log.info(`${exports.moduleInfo.name} Scanner/Tosser starting up`); this.hasValidConfiguration({ shouldLog: true }); // just check and log let importing = false; let self = this; function tryImportNow(reasonDesc, extraInfo) { if (!importing) { importing = true; Log.info( Object.assign({ module: exports.moduleInfo.name }, extraInfo), reasonDesc ); self.performImport(() => { importing = false; }); } } this.createTempDirectories(err => { if (err) { Log.warn({ error: err.toStrong() }, 'Failed creating temporary directories!'); return cb(err); } if (_.isObject(this.moduleConfig.schedule)) { const exportSchedule = this.parseScheduleString( this.moduleConfig.schedule.export ); if (exportSchedule) { Log.debug( { schedule: this.moduleConfig.schedule.export, schedOK: -1 === _.get(exportSchedule, 'sched.error'), next: exportSchedule.sched ? moment(later.schedule(exportSchedule.sched).next(1)).format( 'ddd, MMM Do, YYYY @ h:m:ss a' ) : 'N/A', immediate: exportSchedule.immediate ? true : false, }, 'Export schedule loaded' ); if (exportSchedule.sched) { this.exportTimer = later.setInterval(() => { if (this.exportingStart()) { Log.info( { module: exports.moduleInfo.name }, 'Performing scheduled message scan/export...' ); this.performExport(() => { this.exportingEnd(); }); } }, exportSchedule.sched); } if (_.isBoolean(exportSchedule.immediate)) { this.exportImmediate = exportSchedule.immediate; } } const importSchedule = this.parseScheduleString( this.moduleConfig.schedule.import ); if (importSchedule) { Log.debug( { schedule: this.moduleConfig.schedule.import, schedOK: -1 === _.get(importSchedule, 'sched.error'), next: importSchedule.sched ? moment(later.schedule(importSchedule.sched).next(1)).format( 'ddd, MMM Do, YYYY @ h:m:ss a' ) : 'N/A', watchFile: _.isString(importSchedule.watchFile) ? importSchedule.watchFile : 'None', }, 'Import schedule loaded' ); if (importSchedule.sched) { this.importTimer = later.setInterval(() => { tryImportNow('Performing scheduled message import/toss...'); }, importSchedule.sched); } if (_.isString(importSchedule.watchFile)) { const watcher = sane(paths.dirname(importSchedule.watchFile), { glob: `**/${paths.basename(importSchedule.watchFile)}`, }); const makeImportMsg = (e, path) => { return `Import/toss due to @watch[${e}] "${paths.basename( path )}"`; }; ['change', 'add', 'delete'].forEach(event => { watcher.on(event, (fileName, fileRoot) => { const eventPath = paths.join(fileRoot, fileName); if (eventPath === importSchedule.watchFile) { tryImportNow(makeImportMsg(event, eventPath), { eventPath, event, }); } }); }); // // If the watch file already exists, kick off now // https://github.com/NuSkooler/enigma-bbs/issues/122 // fse.access(importSchedule.watchFile, fse.constants.R_OK, err => { if (!err) { // exists and we can read tryImportNow( makeImportMsg('exists', importSchedule.watchFile), { eventPath: importSchedule.watchFile, event: 'exists', } ); } }); } } } FTNMessageScanTossModule.super_.prototype.startup.call(this, cb); }); }; FTNMessageScanTossModule.prototype.shutdown = function (cb) { Log.info('FidoNet Scanner/Tosser shutting down'); if (this.exportTimer) { this.exportTimer.clear(); } if (this.importTimer) { this.importTimer.clear(); } // // Clean up temp dir/files we created // temptmp.cleanup(paths => { const fullStats = { exportDir: this.exportTempDir, importTemp: this.importTempDir, paths: paths, sessionId: temptmp.sessionId, }; Log.trace(fullStats, 'Temporary directories cleaned up'); FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb); }); FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb); }; FTNMessageScanTossModule.prototype.performImport = function (cb) { if (!this.hasValidConfiguration()) { return cb(Errors.MissingConfig('Invalid or missing configuration')); } const self = this; async.each( ['inbound', 'secInbound'], (inboundType, nextDir) => { const importDir = self.moduleConfig.paths[inboundType]; self.importFromDirectory(inboundType, importDir, err => { if (err) { Log.trace( { importDir, error: err.message }, 'Cannot perform FTN import for directory' ); } return nextDir(null); }); }, cb ); }; FTNMessageScanTossModule.prototype.performExport = function (cb) { // // We're only concerned with areas related to FTN. For each area, loop though // and let's find out what messages need exported. // if (!this.hasValidConfiguration()) { return cb(Errors.MissingConfig('Invalid or missing configuration')); } const self = this; async.eachSeries( ['EchoMail', 'NetMail'], (type, nextType) => { self[`perform${type}Export`](err => { if (err) { Log.warn({ type, error: err.message }, 'Error(s) during export'); } return nextType(null); // try next, always }); }, () => { return cb(null); } ); }; FTNMessageScanTossModule.prototype.record = function (message) { // // This module works off schedules, but we do support @immediate for export // if (true !== this.exportImmediate || !this.hasValidConfiguration()) { return; } const info = { uuid: message.messageUuid, subject: message.subject }; function exportLog(err) { if (err) { Log.warn(info, 'Failed exporting message'); } else { Log.info(info, 'Message exported'); } } if (this.isNetMailMessage(message)) { Object.assign(info, { type: 'NetMail' }); if (this.exportingStart()) { this.exportNetMailMessagesToUplinks([message.messageUuid], err => { this.exportingEnd(() => exportLog(err)); }); } } else if (message.areaTag) { Object.assign(info, { type: 'EchoMail' }); const areaConfig = Config().messageNetworks.ftn.areas[message.areaTag]; if (!this.isAreaConfigValid(areaConfig)) { return; } if (this.exportingStart()) { this.exportEchoMailMessagesToUplinks( [message.messageUuid], areaConfig, err => { this.exportingEnd(() => exportLog(err)); } ); } } };