enigma-bbs/core/scanner_tossers/ftn_bso.js

/* jslint node: true */
'use strict';

//  ENiGMA½
const MessageScanTossModule             = require('../msg_scan_toss_module.js').MessageScanTossModule;
const Config                            = require('../config.js').get;
const ftnMailPacket                     = require('../ftn_mail_packet.js');
const ftnUtil                           = require('../ftn_util.js');
const Address                           = require('../ftn_address.js');
const Log                               = require('../logger.js').log;
const ArchiveUtil                       = require('../archive_util.js');
const msgDb                             = require('../database.js').dbs.message;
const Message                           = require('../message.js');
const TicFileInfo                       = require('../tic_file_info.js');
const Errors                            = require('../enig_error.js').Errors;
const FileEntry                         = require('../file_entry.js');
const scanFile                          = require('../file_base_area.js').scanFile;
const getFileAreaByTag                  = require('../file_base_area.js').getFileAreaByTag;
const getDescFromFileName               = require('../file_base_area.js').getDescFromFileName;
const copyFileWithCollisionHandling     = require('../file_util.js').copyFileWithCollisionHandling;
const getAreaStorageDirectoryByTag      = require('../file_base_area.js').getAreaStorageDirectoryByTag;
const isValidStorageTag                 = require('../file_base_area.js').isValidStorageTag;
const User                              = require('../user.js');

//  deps
const moment                = require('moment');
const _                     = require('lodash');
const paths                 = require('path');
const async                 = require('async');
const fs                    = require('graceful-fs');
const later                 = require('later');
const temptmp               = require('temptmp').createTrackedSession('ftn_bso');
const assert                = require('assert');
const sane                  = require('sane');
const fse                   = require('fs-extra');
const iconv                 = require('iconv-lite');
const uuidV4                = require('uuid/v4');

exports.moduleInfo = {
    name    : 'FTN BSO',
    desc    : 'BSO style message scanner/tosser for FTN networks',
    author  : 'NuSkooler',
};

/*
    :TODO:
    * Support (approx) max bundle size
    * Validate packet passwords!!!!
        => secure vs insecure landing areas
*/

exports.getModule = FTNMessageScanTossModule;

const SCHEDULE_REGEXP   = /(?:^|or )?(@watch:|@immediate)([^\0]+)?$/;

function FTNMessageScanTossModule() {
    MessageScanTossModule.call(this);

    const self = this;

    this.archUtil = ArchiveUtil.getInstance();

    const config = Config();
    if(_.has(config, 'scannerTossers.ftn_bso')) {
        this.moduleConfig = config.scannerTossers.ftn_bso;
    }

    this.getDefaultNetworkName = function() {
        if(this.moduleConfig.defaultNetwork) {
            return this.moduleConfig.defaultNetwork.toLowerCase();
        }

        const networkNames = Object.keys(config.messageNetworks.ftn.networks);
        if(1 === networkNames.length) {
            return networkNames[0].toLowerCase();
        }
    };

    this.getDefaultZone = function(networkName) {
        const config = Config();
        if(_.isNumber(config.messageNetworks.ftn.networks[networkName].defaultZone)) {
            return config.messageNetworks.ftn.networks[networkName].defaultZone;
        }

        //  non-explicit: default to local address zone
        const networkLocalAddress = config.messageNetworks.ftn.networks[networkName].localAddress;
        if(networkLocalAddress) {
            const addr = Address.fromString(networkLocalAddress);
            return addr.zone;
        }
    };

    /*
    this.isDefaultDomainZone = function(networkName, address) {
        const defaultNetworkName    = this.getDefaultNetworkName();
        return(networkName === defaultNetworkName && address.zone === this.moduleConfig.defaultZone);
    };
    */

    this.getNetworkNameByAddress = function(remoteAddress) {
        return _.findKey(Config().messageNetworks.ftn.networks, network => {
            const localAddress = Address.fromString(network.localAddress);
            return !_.isUndefined(localAddress) && localAddress.isEqual(remoteAddress);
        });
    };

    this.getNetworkNameByAddressPattern = function(remoteAddressPattern) {
        return _.findKey(Config().messageNetworks.ftn.networks, network => {
            const localAddress = Address.fromString(network.localAddress);
            return !_.isUndefined(localAddress) && localAddress.isPatternMatch(remoteAddressPattern);
        });
    };

    this.getLocalAreaTagByFtnAreaTag = function(ftnAreaTag) {
        ftnAreaTag = ftnAreaTag.toUpperCase();  //  always compare upper
        return _.findKey(Config().messageNetworks.ftn.areas, areaConf => {
            return _.isString(areaConf.tag) && areaConf.tag.toUpperCase() === ftnAreaTag;
        });
    };

    this.getExportType = function(nodeConfig) {
        return _.isString(nodeConfig.exportType) ? nodeConfig.exportType.toLowerCase() : 'crash';
    };

    /*
    this.getSeenByAddresses = function(messageSeenBy) {
        if(!_.isArray(messageSeenBy)) {
            messageSeenBy = [ messageSeenBy ];
        }

        let seenByAddrs = [];
        messageSeenBy.forEach(sb => {
            seenByAddrs = seenByAddrs.concat(ftnUtil.parseAbbreviatedNetNodeList(sb));
        });
        return seenByAddrs;
    };
    */

    this.messageHasValidMSGID = function(msg) {
        return _.isString(msg.meta.FtnKludge.MSGID) && msg.meta.FtnKludge.MSGID.length > 0;
    };

    /*
    this.getOutgoingEchoMailPacketDir = function(networkName, destAddress) {
        let dir = this.moduleConfig.paths.outbound;
        if(!this.isDefaultDomainZone(networkName, destAddress)) {
            const hexZone = `000${destAddress.zone.toString(16)}`.substr(-3);
            dir = paths.join(dir, `${networkName.toLowerCase()}.${hexZone}`);
        }
        return dir;
    };
    */

    this.getOutgoingEchoMailPacketDir = function(networkName, destAddress) {
        networkName = networkName.toLowerCase();

        let dir = this.moduleConfig.paths.outbound;

        const defaultNetworkName    = this.getDefaultNetworkName();
        const defaultZone           = this.getDefaultZone(networkName);

        let zoneExt;
        if(defaultZone !== destAddress.zone) {
            zoneExt = '.' + `000${destAddress.zone.toString(16)}`.substr(-3);
        } else {
            zoneExt = '';
        }

        if(defaultNetworkName === networkName) {
            dir = paths.join(dir, `outbound${zoneExt}`);
        } else {
            dir = paths.join(dir, `${networkName}${zoneExt}`);
        }

        return dir;
    };

    this.getOutgoingPacketFileName = function(basePath, messageId, isTemp, fileCase) {
        //
        //  Generating an outgoing packet file name comes with a few issues:
        //  *   We must use DOS 8.3 filenames due to legacy systems that receive
        //      the packet not understanding LFNs
        //  *   We need uniqueness; This is especially important with packets that
        //      end up in bundles and on the receiving/remote system where conflicts
        //      with other systems could also occur
        //
        //  There are a lot of systems in use here for the name:
        //  *   HEX CRC16/32 of data
        //  *   HEX UNIX timestamp
        //  *   Mystic at least at one point, used Hex8(day of month + seconds past midnight + hundredths of second)
        //      See https://groups.google.com/forum/#!searchin/alt.bbs.mystic/netmail$20filename/alt.bbs.mystic/m1xLnY8i1pU/YnG2excdl6MJ
        //  *   SBBSEcho uses DDHHMMSS - see https://github.com/ftnapps/pkg-sbbs/blob/master/docs/fidonet.txt
        //  *   We already have a system for 8-character serial number gernation that is
        //      used for e.g. in FTS-0009.001 MSGIDs... let's use that!
        //
        const name      = ftnUtil.getMessageSerialNumber(messageId);
        const ext       = (true === isTemp) ? 'pk_' : 'pkt';

        let fileName = `${name}.${ext}`;
        if('upper' === fileCase) {
            fileName = fileName.toUpperCase();
        }

        return paths.join(basePath, fileName);
    };

    this.getOutgoingFlowFileExtension = function(destAddress, flowType, exportType, fileCase) {
        let ext;

        switch(flowType) {
            case 'mail'     : ext = `${exportType.toLowerCase()[0]}ut`; break;
            case 'ref'      : ext = `${exportType.toLowerCase()[0]}lo`; break;
            case 'busy'     : ext = 'bsy'; break;
            case 'request'  : ext = 'req'; break;
            case 'requests' : ext = 'hrq'; break;
        }

        if('upper' === fileCase) {
            ext = ext.toUpperCase();
        }

        return ext;
    };

    this.getOutgoingFlowFileName = function(basePath, destAddress, flowType, exportType, fileCase) {
        //
        //  Refs
        //  * http://ftsc.org/docs/fts-5005.003
        //  * http://wiki.synchro.net/ref:fidonet_files#flow_files
        //
        let controlFileBaseName;
        let pointDir;

        const ext = self.getOutgoingFlowFileExtension(
            destAddress,
            flowType,
            exportType,
            fileCase
        );

        const netComponent  = `0000${destAddress.net.toString(16)}`.substr(-4);
        const nodeComponent = `0000${destAddress.node.toString(16)}`.substr(-4);

        if(destAddress.point) {
            //  point's go in an extra subdir, e.g. outbound/NNNNnnnn.pnt/00000001.pnt (for a point of 1)
            pointDir            = `${netComponent}${nodeComponent}.pnt`;
            controlFileBaseName = `00000000${destAddress.point.toString(16)}`.substr(-8);
        } else {
            pointDir = '';

            //
            //  Use |destAddress| nnnnNNNN.??? where nnnn is dest net and NNNN is dest
            //  node. This seems to match what Mystic does
            //
            controlFileBaseName = `${netComponent}${nodeComponent}`;
        }

        //
        //  From FTS-5005.003: "Lower case filenames are prefered if supported by the file system."
        //  ...but we let the user override.
        //
        if('upper' === fileCase) {
            controlFileBaseName = controlFileBaseName.toUpperCase();
            pointDir            = pointDir.toUpperCase();
        }

        return paths.join(basePath, pointDir, `${controlFileBaseName}.${ext}`);
    };

    this.flowFileAppendRefs = function(filePath, fileRefs, directive, cb) {
        //
        //  We have to ensure the *directory* of |filePath| exists here esp.
        //  for cases such as point destinations where a subdir may be
        //  present in the path that doesn't yet exist.
        //
        const flowFileDir = paths.dirname(filePath);
        fse.mkdirs(flowFileDir, () => { //  note not checking err; let's try appendFile
            const appendLines = fileRefs.reduce( (content, ref) => {
                return content + `${directive}${ref}\n`;
            }, '');

            fs.appendFile(filePath, appendLines, err => {
                return cb(err);
            });
        });
    };

    this.getOutgoingBundleFileName = function(basePath, sourceAddress, destAddress, cb) {
        //
        //  Base filename is constructed as such:
        //  *   If this |destAddress| is *not* a point address, we use NNNNnnnn where
        //      NNNN is 0 padded hex of dest net - source net and and nnnn is 0 padded
        //      hex of dest node - source node.
        //  *   If |destAddress| is a point, NNNN becomes 0000 and nnnn becomes 'p' +
        //      3 digit 0 padded hex point
        //
        //  Extension is dd? where dd is Su...Mo and ? is 0...Z as collisions arise
        //
        let basename;
        if(destAddress.point) {
            const pointHex = `000${destAddress.point}`.substr(-3);
            basename = `0000p${pointHex}`;
        } else {
            basename =
                `0000${Math.abs(sourceAddress.net - destAddress.net).toString(16)}`.substr(-4) +
                `0000${Math.abs(sourceAddress.node - destAddress.node).toString(16)}`.substr(-4);
        }

        //
        //  We need to now find the first entry that does not exist starting
        //  with dd0 to ddz
        //
        const EXT_SUFFIXES = '0123456789abcdefghijklmnopqrstuvwxyz'.split('');
        let fileName = `${basename}.${moment().format('dd').toLowerCase()}`;
        async.detectSeries(EXT_SUFFIXES, (suffix, callback) => {
            const checkFileName = fileName + suffix;
            fs.stat(paths.join(basePath, checkFileName), err => {
                callback(null, (err && 'ENOENT' === err.code) ? true : false);
            });
        }, (err, finalSuffix) => {
            if(finalSuffix) {
                return cb(null, paths.join(basePath, fileName + finalSuffix));
            }

            return cb(new Error('Could not acquire a bundle filename!'));
        });
    };

    this.prepareMessage = function(message, options) {
        //
        //  Set various FTN kludges/etc.
        //
        const localAddress = new Address(options.network.localAddress); //  ensure we have an Address obj not a string version

        //  :TODO: create Address.toMeta() / similar
        message.meta.FtnProperty = message.meta.FtnProperty || {};
        message.meta.FtnKludge = message.meta.FtnKludge || {};

        message.meta.FtnProperty.ftn_orig_node          = localAddress.node;
        message.meta.FtnProperty.ftn_orig_network       = localAddress.net;
        message.meta.FtnProperty.ftn_cost               = 0;
        message.meta.FtnProperty.ftn_msg_orig_node      = localAddress.node;
        message.meta.FtnProperty.ftn_msg_orig_net   = localAddress.net;

        const destAddress = options.routeAddress || options.destAddress;
        message.meta.FtnProperty.ftn_dest_node      = destAddress.node;
        message.meta.FtnProperty.ftn_dest_network   = destAddress.net;

        if(destAddress.zone) {
            message.meta.FtnProperty.ftn_dest_zone  = destAddress.zone;
        }
        if(destAddress.point) {
            message.meta.FtnProperty.ftn_dest_point = destAddress.point;
        }

        //  tear line and origin can both go in EchoMail & NetMail
        message.meta.FtnProperty.ftn_tear_line      = ftnUtil.getTearLine();
        message.meta.FtnProperty.ftn_origin         = ftnUtil.getOrigin(localAddress);

        let ftnAttribute = ftnMailPacket.Packet.Attribute.Local;    //  message from our system

        const config = Config();
        if(self.isNetMailMessage(message)) {
            //
            //  Set route and message destination properties -- they may differ
            //
            message.meta.FtnProperty.ftn_msg_dest_node      = options.destAddress.node;
            message.meta.FtnProperty.ftn_msg_dest_net   = options.destAddress.net;

            ftnAttribute |= ftnMailPacket.Packet.Attribute.Private;

            //
            //  NetMail messages need a FRL-1005.001 "Via" line
            //  http://ftsc.org/docs/frl-1005.001
            //
            //  :TODO:  We need to do this when FORWARDING NetMail
            /*
            if(_.isString(message.meta.FtnKludge.Via)) {
                message.meta.FtnKludge.Via = [ message.meta.FtnKludge.Via ];
            }
            message.meta.FtnKludge.Via = message.meta.FtnKludge.Via || [];
            message.meta.FtnKludge.Via.push(ftnUtil.getVia(options.network.localAddress));
            */

            //
            //  We need to set INTL, and possibly FMPT and/or TOPT
            //  See http://retro.fidoweb.ru/docs/index=ftsc&doc=FTS-4001&enc=mac
            //
            message.meta.FtnKludge.INTL = ftnUtil.getIntl(options.destAddress, localAddress);

            if(_.isNumber(localAddress.point) && localAddress.point > 0) {
                message.meta.FtnKludge.FMPT = localAddress.point;
            }

            if(_.isNumber(options.destAddress.point) && options.destAddress.point > 0) {
                message.meta.FtnKludge.TOPT = options.destAddress.point;
            }
        } else {
            //
            //  Set appropriate attribute flag for export type
            //
            switch(this.getExportType(options.nodeConfig)) {
                case 'crash'    : ftnAttribute |= ftnMailPacket.Packet.Attribute.Crash; break;
                case 'hold'     : ftnAttribute |= ftnMailPacket.Packet.Attribute.Hold; break;
                //  :TODO: Others?
            }

            //
            //  EchoMail requires some additional properties & kludges
            //
            message.meta.FtnProperty.ftn_area = config.messageNetworks.ftn.areas[message.areaTag].tag;

            //
            //  When exporting messages, we should create/update SEEN-BY
            //  with remote address(s) we are exporting to.
            //
            const seenByAdditions =
                [ `${localAddress.net}/${localAddress.node}` ].concat(config.messageNetworks.ftn.areas[message.areaTag].uplinks);
            message.meta.FtnProperty.ftn_seen_by =
                ftnUtil.getUpdatedSeenByEntries(message.meta.FtnProperty.ftn_seen_by, seenByAdditions);

            //
            //  And create/update PATH for ourself
            //
            message.meta.FtnKludge.PATH = ftnUtil.getUpdatedPathEntries(message.meta.FtnKludge.PATH, localAddress);
        }

        message.meta.FtnProperty.ftn_attr_flags = ftnAttribute;

        //
        //  Additional kludges
        //
        //  Check for existence of MSGID as we may already have stored it from a previous
        //  export that failed to finish
        //
        if(!message.meta.FtnKludge.MSGID) {
            message.meta.FtnKludge.MSGID = ftnUtil.getMessageIdentifier(
                message,
                localAddress,
                message.isPrivate() // true = isNetMail
            );
        }

        message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset();

        //
        //  According to FSC-0046:
        //
        //  "When a Conference Mail processor adds a TID to a message, it may not
        //  add a PID. An existing TID should, however, be replaced. TIDs follow
        //  the same format used for PIDs, as explained above."
        //
        message.meta.FtnKludge.TID = ftnUtil.getProductIdentifier();

        //
        //  Determine CHRS and actual internal encoding name. If the message has an
        //  explicit encoding set, use it. Otherwise, try to preserve any CHRS/encoding already set.
        //
        let encoding = options.nodeConfig.encoding || config.scannerTossers.ftn_bso.packetMsgEncoding || 'utf8';
        const explicitEncoding = _.get(message.meta, 'System.explicit_encoding');
        if(explicitEncoding) {
            encoding = explicitEncoding;
        } else if(message.meta.FtnKludge.CHRS) {
            const encFromChars = ftnUtil.getEncodingFromCharacterSetIdentifier(message.meta.FtnKludge.CHRS);
            if(encFromChars) {
                encoding = encFromChars;
            }
        }

        //
        //  Ensure we ended up with something useable. If not, back to utf8!
        //
        if(!iconv.encodingExists(encoding)) {
            Log.debug( { encoding : encoding }, 'Unknown encoding. Falling back to utf8');
            encoding = 'utf8';
        }

        options.encoding = encoding;    //  save for later
        message.meta.FtnKludge.CHRS = ftnUtil.getCharacterSetIdentifierByEncoding(encoding);
    };

    this.setReplyKludgeFromReplyToMsgId = function(message, cb) {
        //
        //  Look up MSGID kludge for |message.replyToMsgId|, if any.
        //  If found, we can create a REPLY kludge with the previously
        //  discovered MSGID.
        //

        if(0 === message.replyToMsgId) {
            return cb(null);    //  nothing to do
        }

        Message.getMetaValuesByMessageId(message.replyToMsgId, 'FtnKludge', 'MSGID', (err, msgIdVal) => {
            if(!err) {
                assert(_.isString(msgIdVal), 'Expected string but got ' + (typeof msgIdVal) + ' (' + msgIdVal + ')');
                //  got a MSGID - create a REPLY
                message.meta.FtnKludge.REPLY = msgIdVal;
            }

            cb(null);   //  this method always passes
        });
    };

    //  check paths, Addresses, etc.
    this.isAreaConfigValid = function(areaConfig) {
        if(!areaConfig || !_.isString(areaConfig.tag) || !_.isString(areaConfig.network)) {
            return false;
        }

        if(_.isString(areaConfig.uplinks)) {
            areaConfig.uplinks = areaConfig.uplinks.split(' ');
        }

        return (_.isArray(areaConfig.uplinks));
    };


    this.hasValidConfiguration = function() {
        if(!_.has(this, 'moduleConfig.nodes') || !_.has(Config(), 'messageNetworks.ftn.areas')) {
            return false;
        }

        //  :TODO: need to check more!

        return true;
    };

    this.parseScheduleString = function(schedStr) {
        if(!schedStr) {
            return; //  nothing to parse!
        }

        let schedule = {};

        const m = SCHEDULE_REGEXP.exec(schedStr);
        if(m) {
            schedStr = schedStr.substr(0, m.index).trim();

            if('@watch:' === m[1]) {
                schedule.watchFile = m[2];
            } else if('@immediate' === m[1]) {
                schedule.immediate = true;
            }
        }

        if(schedStr.length > 0) {
            const sched = later.parse.text(schedStr);
            if(-1 === sched.error) {
                schedule.sched = sched;
            }
        }

        //  return undefined if we couldn't parse out anything useful
        if(!_.isEmpty(schedule)) {
            return schedule;
        }
    };

    this.getAreaLastScanId = function(areaTag, cb) {
        const sql =
            `SELECT area_tag, message_id
            FROM message_area_last_scan
            WHERE scan_toss = "ftn_bso" AND area_tag = ?
            LIMIT 1;`;

        msgDb.get(sql, [ areaTag ], (err, row) => {
            return cb(err, row ? row.message_id : 0);
        });
    };

    this.setAreaLastScanId = function(areaTag, lastScanId, cb) {
        const sql =
            `REPLACE INTO message_area_last_scan (scan_toss, area_tag, message_id)
            VALUES ("ftn_bso", ?, ?);`;

        msgDb.run(sql, [ areaTag, lastScanId ], err => {
            return cb(err);
        });
    };

    this.getNodeConfigByAddress = function(addr) {
        addr = _.isString(addr) ? Address.fromString(addr) : addr;

        //  :TODO: sort wildcard nodes{} entries by most->least explicit according to FTN hierarchy
        return _.find(this.moduleConfig.nodes, (node, nodeAddrWildcard) => {
            return addr.isPatternMatch(nodeAddrWildcard);
        });
    };

    this.exportNetMailMessagePacket = function(message, exportOpts, cb) {
        //
        //  For NetMail, we always create a *single* packet per message.
        //
        async.series(
            [
                function generalPrep(callback) {
                    self.prepareMessage(message, exportOpts);

                    return self.setReplyKludgeFromReplyToMsgId(message, callback);
                },
                function createPacket(callback) {
                    const packet = new ftnMailPacket.Packet();

                    const packetHeader = new ftnMailPacket.PacketHeader(
                        exportOpts.network.localAddress,
                        exportOpts.routeAddress,
                        exportOpts.nodeConfig.packetType
                    );

                    packetHeader.password = exportOpts.nodeConfig.packetPassword || '';

                    //  use current message ID for filename seed
                    exportOpts.pktFileName = self.getOutgoingPacketFileName(
                        self.exportTempDir,
                        message.messageId,
                        false,  //  createTempPacket=false
                        exportOpts.fileCase
                    );

                    const ws = fs.createWriteStream(exportOpts.pktFileName);

                    packet.writeHeader(ws, packetHeader);

                    packet.getMessageEntryBuffer(message, exportOpts, (err, msgBuf) => {
                        if(err) {
                            return callback(err);
                        }

                        ws.write(msgBuf);

                        packet.writeTerminator(ws);

                        ws.end();
                        ws.once('finish', () => {
                            return callback(null);
                        });
                    });
                }
            ],
            err => {
                return cb(err);
            }
        );
    };

    this.exportMessagesByUuid = function(messageUuids, exportOpts, cb) {
        //
        //  This method has a lot of madness going on:
        //  - Try to stuff messages into packets until we've hit the target size
        //  - We need to wait for write streams to finish before proceeding in many cases
        //    or data will be cut off when closing and creating a new stream
        //
        let exportedFiles   = [];
        let currPacketSize  = self.moduleConfig.packetTargetByteSize;
        let packet;
        let ws;
        let remainMessageBuf;
        let remainMessageId;
        const createTempPacket = !_.isString(exportOpts.nodeConfig.archiveType) || 0 === exportOpts.nodeConfig.archiveType.length;

        function finalizePacket(cb) {
            packet.writeTerminator(ws);
            ws.end();
            ws.once('finish', () => {
                return cb(null);
            });
        }

        async.each(messageUuids, (msgUuid, nextUuid) => {
            let message = new Message();

            async.series(
                [
                    function finalizePrevious(callback) {
                        if(packet && currPacketSize >= self.moduleConfig.packetTargetByteSize) {
                            return finalizePacket(callback);
                        } else {
                            callback(null);
                        }
                    },
                    function loadMessage(callback) {
                        message.load( { uuid : msgUuid }, err => {
                            if(err) {
                                return callback(err);
                            }

                            //  General preperation
                            self.prepareMessage(message, exportOpts);

                            self.setReplyKludgeFromReplyToMsgId(message, err => {
                                callback(err);
                            });
                        });
                    },
                    function createNewPacket(callback) {
                        if(currPacketSize >= self.moduleConfig.packetTargetByteSize) {
                            packet = new ftnMailPacket.Packet();

                            const packetHeader = new ftnMailPacket.PacketHeader(
                                exportOpts.network.localAddress,
                                exportOpts.destAddress,
                                exportOpts.nodeConfig.packetType);

                            packetHeader.password = exportOpts.nodeConfig.packetPassword || '';

                            //  use current message ID for filename seed
                            const pktFileName = self.getOutgoingPacketFileName(
                                self.exportTempDir,
                                message.messageId,
                                createTempPacket,
                                exportOpts.fileCase
                            );

                            exportedFiles.push(pktFileName);

                            ws = fs.createWriteStream(pktFileName);

                            currPacketSize = packet.writeHeader(ws, packetHeader);

                            if(remainMessageBuf) {
                                currPacketSize += packet.writeMessageEntry(ws, remainMessageBuf);
                                remainMessageBuf = null;
                            }
                        }

                        callback(null);
                    },
                    function appendMessage(callback) {
                        packet.getMessageEntryBuffer(message, exportOpts, (err, msgBuf) => {
                            if(err) {
                                return callback(err);
                            }

                            currPacketSize  += msgBuf.length;

                            if(currPacketSize >= self.moduleConfig.packetTargetByteSize) {
                                remainMessageBuf    = msgBuf;   //  save for next packet
                                remainMessageId     = message.messageId;
                            } else {
                                ws.write(msgBuf);
                            }

                            return callback(null);
                        });
                    },
                    function storeStateFlags0Meta(callback) {
                        message.persistMetaValue('System', 'state_flags0', Message.StateFlags0.Exported.toString(), err => {
                            callback(err);
                        });
                    },
                    function storeMsgIdMeta(callback) {
                        //
                        //  We want to store some meta as if we had imported
                        //  this message for later reference
                        //
                        if(message.meta.FtnKludge.MSGID) {
                            message.persistMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.MSGID, err => {
                                callback(err);
                            });
                        } else {
                            callback(null);
                        }
                    }
                ],
                err => {
                    nextUuid(err);
                }
            );
        }, err => {
            if(err) {
                cb(err);
            } else {
                async.series(
                    [
                        function terminateLast(callback) {
                            if(packet) {
                                return finalizePacket(callback);
                            } else {
                                callback(null);
                            }
                        },
                        function writeRemainPacket(callback) {
                            if(remainMessageBuf) {
                                //  :TODO: DRY this with the code above -- they are basically identical
                                packet = new ftnMailPacket.Packet();

                                const packetHeader = new ftnMailPacket.PacketHeader(
                                    exportOpts.network.localAddress,
                                    exportOpts.destAddress,
                                    exportOpts.nodeConfig.packetType);

                                packetHeader.password = exportOpts.nodeConfig.packetPassword || '';

                                //  use current message ID for filename seed
                                const pktFileName = self.getOutgoingPacketFileName(
                                    self.exportTempDir,
                                    remainMessageId,
                                    createTempPacket,
                                    exportOpts.filleCase
                                );

                                exportedFiles.push(pktFileName);

                                ws = fs.createWriteStream(pktFileName);

                                packet.writeHeader(ws, packetHeader);
                                ws.write(remainMessageBuf);
                                return finalizePacket(callback);
                            } else {
                                callback(null);
                            }
                        }
                    ],
                    err => {
                        cb(err, exportedFiles);
                    }
                );
            }
        });
    };

    this.getNetMailRoute = function(dstAddr) {
        //
        //  Route full|wildcard -> full adddress/network lookup
        //
        const routes = _.get(Config(), 'scannerTossers.ftn_bso.netMail.routes');
        if(!routes) {
            return;
        }

        return _.find(routes, (route, addrWildcard) => {
            return dstAddr.isPatternMatch(addrWildcard);
        });
    };

    this.getNetMailRouteInfoFromAddress = function(destAddress, cb) {
        //
        //  Attempt to find route information for |destAddress|:
        //
        //  1) Routes: scannerTossers.ftn_bso.netMail.routes{} -> scannerTossers.ftn_bso.nodes{} -> config
        //      - Where we send may not be where destAddress is (it's routed!)
        //  2) Direct to nodes: scannerTossers.ftn_bso.nodes{} -> config
        //      - Where we send is direct to destAddress
        //
        //  In both cases, attempt to look up Zone:Net/* to discover local "from" network/address
        //  falling back to Config.scannerTossers.ftn_bso.defaultNetwork
        //
        const route = this.getNetMailRoute(destAddress);

        let routeAddress;
        let networkName;
        let isRouted;
        if(route) {
            routeAddress    = Address.fromString(route.address);
            networkName     = route.network;
            isRouted        = true;
        } else {
            routeAddress    = destAddress;
            isRouted        = false;
        }

        networkName = networkName || this.getNetworkNameByAddress(routeAddress);

        const config = _.find(this.moduleConfig.nodes, (node, nodeAddrWildcard) => {
            return routeAddress.isPatternMatch(nodeAddrWildcard);
        }) || { packetType : '2+', encoding : Config().scannerTossers.ftn_bso.packetMsgEncoding };

        //  we should never be failing here; we may just be using defaults.
        return cb(
            networkName ? null : Errors.DoesNotExist(`No NetMail route for ${destAddress.toString()}`),
            { destAddress, routeAddress, networkName, config, isRouted }
        );
    };

    this.exportNetMailMessagesToUplinks = function(messagesOrMessageUuids, cb) {
        //  for each message/UUID, find where to send the thing
        async.each(messagesOrMessageUuids, (msgOrUuid, nextMessageOrUuid) => {

            const exportOpts = {};
            const message = new Message();

            async.series(
                [
                    function loadMessage(callback) {
                        if(_.isString(msgOrUuid)) {
                            message.load( { uuid : msgOrUuid }, err => {
                                return callback(err, message);
                            });
                        } else {
                            return callback(null, msgOrUuid);
                        }
                    },
                    function discoverUplink(callback) {
                        const dstAddr = new Address(message.meta.System[Message.SystemMetaNames.RemoteToUser]);

                        self.getNetMailRouteInfoFromAddress(dstAddr, (err, routeInfo) => {
                            if(err) {
                                return callback(err);
                            }

                            exportOpts.nodeConfig   = routeInfo.config;
                            exportOpts.destAddress  = dstAddr;
                            exportOpts.routeAddress = routeInfo.routeAddress;
                            exportOpts.fileCase     = routeInfo.config.fileCase || 'lower';
                            exportOpts.network      = Config().messageNetworks.ftn.networks[routeInfo.networkName];
                            exportOpts.networkName  = routeInfo.networkName;
                            exportOpts.outgoingDir  = self.getOutgoingEchoMailPacketDir(exportOpts.networkName, exportOpts.destAddress);
                            exportOpts.exportType   = self.getExportType(routeInfo.config);

                            if(!exportOpts.network) {
                                return callback(Errors.DoesNotExist(`No configuration found for network ${routeInfo.networkName}`));
                            }

                            return callback(null);
                        });
                    },
                    function createOutgoingDir(callback) {
                        //  ensure outgoing NetMail directory exists
                        return fse.mkdirs(exportOpts.outgoingDir, callback);
                    },
                    function exportPacket(callback) {
                        return self.exportNetMailMessagePacket(message, exportOpts, callback);
                    },
                    function moveToOutgoing(callback) {
                        const newExt = exportOpts.fileCase === 'lower' ? '.pkt' : '.PKT';
                        exportOpts.exportedToPath = paths.join(
                            exportOpts.outgoingDir,
                            `${paths.basename(exportOpts.pktFileName, paths.extname(exportOpts.pktFileName))}${newExt}`
                        );

                        return fse.move(exportOpts.pktFileName, exportOpts.exportedToPath, callback);
                    },
                    function prepareFloFile(callback) {
                        const flowFilePath = self.getOutgoingFlowFileName(
                            exportOpts.outgoingDir,
                            exportOpts.routeAddress,
                            'ref',
                            exportOpts.exportType,
                            exportOpts.fileCase
                        );

                        return self.flowFileAppendRefs(flowFilePath, [ exportOpts.exportedToPath ], '^', callback);
                    },
                    function storeStateFlags0Meta(callback) {
                        return message.persistMetaValue('System', 'state_flags0', Message.StateFlags0.Exported.toString(), callback);
                    },
                    function storeMsgIdMeta(callback) {
                        //  Store meta as if we had imported this message -- for later reference
                        if(message.meta.FtnKludge.MSGID) {
                            return message.persistMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.MSGID, callback);
                        }

                        return callback(null);
                    }
                ],
                err => {
                    if(err) {
                        Log.warn( { error : err.message }, 'Error exporting message' );
                    }
                    return nextMessageOrUuid(null);
                }
            );
        }, err => {
            if(err) {
                Log.warn( { error : err.message }, 'Error(s) during NetMail export');
            }
            return cb(err);
        });
    };

    this.exportEchoMailMessagesToUplinks = function(messageUuids, areaConfig, cb) {
        const config = Config();
        async.each(areaConfig.uplinks, (uplink, nextUplink) => {
            const nodeConfig = self.getNodeConfigByAddress(uplink);
            if(!nodeConfig) {
                return nextUplink();
            }

            const exportOpts = {
                nodeConfig,
                network         : config.messageNetworks.ftn.networks[areaConfig.network],
                destAddress     : Address.fromString(uplink),
                networkName     : areaConfig.network,
                fileCase        : nodeConfig.fileCase || 'lower',
            };

            if(_.isString(exportOpts.network.localAddress)) {
                exportOpts.network.localAddress = Address.fromString(exportOpts.network.localAddress);
            }

            const outgoingDir   = self.getOutgoingEchoMailPacketDir(exportOpts.networkName, exportOpts.destAddress);
            const exportType    = self.getExportType(exportOpts.nodeConfig);

            async.waterfall(
                [
                    function createOutgoingDir(callback) {
                        fse.mkdirs(outgoingDir, err => {
                            callback(err);
                        });
                    },
                    function exportToTempArea(callback) {
                        self.exportMessagesByUuid(messageUuids, exportOpts, callback);
                    },
                    function createArcMailBundle(exportedFileNames, callback) {
                        if(self.archUtil.haveArchiver(exportOpts.nodeConfig.archiveType)) {
                            //  :TODO: support bundleTargetByteSize:
                            //
                            //  Compress to a temp location then we'll move it in the next step
                            //
                            //  Note that we must use the *final* output dir for getOutgoingBundleFileName()
                            //  as it checks for collisions in bundle names!
                            //
                            self.getOutgoingBundleFileName(outgoingDir, exportOpts.network.localAddress, exportOpts.destAddress, (err, bundlePath) => {
                                if(err) {
                                    return callback(err);
                                }

                                //  adjust back to temp path
                                const tempBundlePath = paths.join(self.exportTempDir, paths.basename(bundlePath));

                                self.archUtil.compressTo(
                                    exportOpts.nodeConfig.archiveType,
                                    tempBundlePath,
                                    exportedFileNames, err => {
                                        callback(err, [ tempBundlePath ] );
                                    }
                                );
                            });
                        } else {
                            callback(null, exportedFileNames);
                        }
                    },
                    function moveFilesToOutgoing(exportedFileNames, callback) {
                        async.each(exportedFileNames, (oldPath, nextFile) => {
                            const ext = paths.extname(oldPath).toLowerCase();
                            if('.pk_' === ext.toLowerCase()) {
                                //
                                //  For a given temporary .pk_ file, we need to move it to the outoing
                                //  directory with the appropriate BSO style filename.
                                //
                                const newExt = self.getOutgoingFlowFileExtension(
                                    exportOpts.destAddress,
                                    'mail',
                                    exportType,
                                    exportOpts.fileCase
                                );

                                const newPath = paths.join(
                                    outgoingDir,
                                    `${paths.basename(oldPath, ext)}${newExt}`);

                                fse.move(oldPath, newPath, nextFile);
                            } else {
                                const newPath = paths.join(outgoingDir, paths.basename(oldPath));
                                fse.move(oldPath, newPath, err => {
                                    if(err) {
                                        Log.warn(
                                            { oldPath : oldPath, newPath : newPath, error : err.toString() },
                                            'Failed moving temporary bundle file!');

                                        return nextFile();
                                    }

                                    //
                                    //  For bundles, we need to append to the appropriate flow file
                                    //
                                    const flowFilePath = self.getOutgoingFlowFileName(
                                        outgoingDir,
                                        exportOpts.destAddress,
                                        'ref',
                                        exportType,
                                        exportOpts.fileCase
                                    );

                                    //  directive of '^' = delete file after transfer
                                    self.flowFileAppendRefs(flowFilePath, [ newPath ], '^', err => {
                                        if(err) {
                                            Log.warn( { path : flowFilePath }, 'Failed appending flow reference record!');
                                        }
                                        nextFile();
                                    });
                                });
                            }
                        }, callback);
                    }
                ],
                err => {
                    //  :TODO: do something with |err| ?
                    if(err) {
                        Log.warn(err.message);
                    }
                    nextUplink();
                }
            );
        }, cb); //  complete
    };

    this.setReplyToMsgIdFtnReplyKludge = function(message, cb) {
        //
        //  Given a FTN REPLY kludge, set |message.replyToMsgId|, if possible,
        //  by looking up an associated MSGID kludge meta.
        //
        //  See also: http://ftsc.org/docs/fts-0009.001
        //
        if(!_.isString(message.meta.FtnKludge.REPLY)) {
            //  nothing to do
            return cb();
        }

        Message.getMessageIdsByMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.REPLY, (err, msgIds) => {
            if(msgIds && msgIds.length > 0) {
                //  expect a single match, but dupe checking is not perfect - warn otherwise
                if(1 === msgIds.length) {
                    message.replyToMsgId = msgIds[0];
                } else {
                    Log.warn( { msgIds : msgIds, replyKludge :  message.meta.FtnKludge.REPLY }, 'Found 2:n MSGIDs matching REPLY kludge!');
                }
            }
            cb();
        });
    };

    this.getLocalUserNameFromAlias = function(lookup) {
        lookup = lookup.toLowerCase();

        const aliases = _.get(Config(), 'messageNetworks.ftn.netMail.aliases');
        if(!aliases) {
            return lookup;  //  keep orig
        }

        const alias = _.find(aliases, (localName, alias) => {
            return alias.toLowerCase() === lookup;
        });

        return alias || lookup;
    };

    this.getAddressesFromNetMailMessage = function(message) {
        const intlKludge = _.get(message, 'meta.FtnKludge.INTL');

        if(!intlKludge) {
            return {};
        }

        let [ to, from ] = intlKludge.split(' ');
        if(!to || !from) {
            return {};
        }

        const fromPoint = _.get(message, 'meta.FtnKludge.FMPT');
        const toPoint   = _.get(message, 'meta.FtnKludge.TOPT');

        if(fromPoint) {
            from += `.${fromPoint}`;
        }

        if(toPoint) {
            to += `.${toPoint}`;
        }

        return { to : Address.fromString(to), from : Address.fromString(from) };
    };

    this.importMailToArea = function(config, header, message, cb) {
        async.series(
            [
                function validateDestinationAddress(callback) {
                    const localNetworkPattern = `${message.meta.FtnProperty.ftn_dest_network}/${message.meta.FtnProperty.ftn_dest_node}`;
                    const localNetworkName = self.getNetworkNameByAddressPattern(localNetworkPattern);

                    return callback(_.isString(localNetworkName) ? null : new Error('Packet destination is not us'));
                },
                function checkForDupeMSGID(callback) {
                    //
                    //  If we have a MSGID, don't allow a dupe
                    //
                    if(!_.has(message.meta, 'FtnKludge.MSGID')) {
                        return callback(null);
                    }

                    Message.getMessageIdsByMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.MSGID, (err, msgIds) => {
                        if(msgIds && msgIds.length > 0) {
                            const err = new Error('Duplicate MSGID');
                            err.code = 'DUPE_MSGID';
                            return callback(err);
                        }

                        return callback(null);
                    });
                },
                function basicSetup(callback) {
                    message.areaTag = config.localAreaTag;

                    //  indicate this was imported from FTN
                    message.meta.System[Message.SystemMetaNames.ExternalFlavor] = Message.AddressFlavor.FTN;

                    //
                    //  If we *allow* dupes (disabled by default), then just generate
                    //  a random UUID. Otherwise, don't assign the UUID just yet. It will be
                    //  generated at persist() time and should be consistent across import/exports
                    //
                    if(true === _.get(Config(), [ 'messageNetworks', 'ftn', 'areas', config.localAreaTag, 'allowDupes' ], false)) {
                        //  just generate a UUID & therefor always allow for dupes
                        message.uuid = uuidV4();
                    }

                    return callback(null);
                },
                function setReplyToMessageId(callback) {
                    self.setReplyToMsgIdFtnReplyKludge(message, () => {
                        return callback(null);
                    });
                },
                function setupPrivateMessage(callback) {
                    //
                    //  If this is a private message (e.g. NetMail) we set the local user ID
                    //
                    if(Message.WellKnownAreaTags.Private !== config.localAreaTag) {
                        return callback(null);
                    }

                    //
                    //  Create a meta value for the *remote* from user. In the case here with FTN,
                    //  their fully qualified FTN from address
                    //
                    const { from } = self.getAddressesFromNetMailMessage(message);

                    if(!from) {
                        return callback(Errors.Invalid('Cannot import FTN NetMail without valid INTL line'));
                    }

                    message.meta.System[Message.SystemMetaNames.RemoteFromUser] = from.toString();

                    const lookupName = self.getLocalUserNameFromAlias(message.toUserName);

                    User.getUserIdAndNameByLookup(lookupName, (err, localToUserId, localUserName) => {
                        if(err) {
                            //
                            //  Couldn't find a local username. If the toUserName itself is a FTN address
                            //  we can only assume the message is to the +op, else we'll have to fail.
                            //
                            const toUserNameAsAddress = Address.fromString(message.toUserName);
                            if(toUserNameAsAddress.isValid()) {

                                Log.info(
                                    { toUserName : message.toUserName, fromUserName : message.fromUserName },
                                    'No local "to" username for FTN message. Appears to be a FTN address only; assuming addressed to SysOp'
                                );

                                User.getUserName(User.RootUserID, (err, sysOpUserName) => {
                                    if(err) {
                                        return callback(Errors.UnexpectedState('Failed to get SysOp user information'));
                                    }

                                    message.meta.System[Message.SystemMetaNames.LocalToUserID] = User.RootUserID;
                                    message.toUserName = sysOpUserName;
                                    return callback(null);
                                });
                            } else {
                                return callback(Errors.DoesNotExist(`Could not get local user ID for "${message.toUserName}": ${err.message}`));
                            }
                        }

                        //  we do this after such that error cases can be preseved above
                        if(lookupName !== message.toUserName) {
                            message.toUserName = localUserName;
                        }

                        //  set the meta information - used elsehwere for retrieval
                        message.meta.System[Message.SystemMetaNames.LocalToUserID] = localToUserId;
                        return callback(null);
                    });
                },
                function persistImport(callback) {
                    //  mark as imported
                    message.meta.System.state_flags0 = Message.StateFlags0.Imported.toString();

                    //  save to disc
                    message.persist(err => {
                        return callback(err);
                    });
                }
            ],
            err => {
                cb(err);
            }
        );
    };

    this.appendTearAndOrigin = function(message) {
        if(message.meta.FtnProperty.ftn_tear_line) {
            message.message += `\r\n${message.meta.FtnProperty.ftn_tear_line}\r\n`;
        }

        if(message.meta.FtnProperty.ftn_origin) {
            message.message += `${message.meta.FtnProperty.ftn_origin}\r\n`;
        }
    };

    //
    //  Ref. implementations on import:
    //  *   https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/pkt.c
    //      https://github.com/larsks/crashmail/blob/26e5374710c7868dab3d834be14bf4041041aae5/crashmail/handle.c
    //
    this.importMessagesFromPacketFile = function(packetPath, password, cb) {
        let packetHeader;

        const packetOpts = { keepTearAndOrigin : false };   //  needed so we can calc message UUID without these; we'll add later

        let importStats = {
            areaSuccess : {},   //  areaTag->count
            areaFail    : {},   //  areaTag->count
            otherFail   : 0,
        };

        new ftnMailPacket.Packet(packetOpts).read(packetPath, (entryType, entryData, next) => {
            if('header' === entryType) {
                packetHeader = entryData;

                const localNetworkName = self.getNetworkNameByAddress(packetHeader.destAddress);
                if(!_.isString(localNetworkName)) {
                    const addrString = new Address(packetHeader.destAddress).toString();
                    return next(new Error(`No local configuration for packet addressed to ${addrString}`));
                } else {

                    //  :TODO: password needs validated - need to determine if it will use the same node config (which can have wildcards) or something else?!
                    return next(null);
                }

            } else if('message' === entryType) {
                const message = entryData;
                const areaTag = message.meta.FtnProperty.ftn_area;

                let localAreaTag;
                if(areaTag) {
                    localAreaTag = self.getLocalAreaTagByFtnAreaTag(areaTag);

                    if(!localAreaTag) {
                        //
                        //  No local area configured for this import
                        //
                        //  :TODO: Handle the "catch all" area bucket case if configured
                        Log.warn( { areaTag : areaTag }, 'No local area configured for this packet file!');

                        //  bump generic failure
                        importStats.otherFail += 1;

                        return next(null);
                    }
                } else {
                    //
                    //  No area tag: If marked private in attributes, this is a NetMail
                    //
                    if(message.meta.FtnProperty.ftn_attr_flags & ftnMailPacket.Packet.Attribute.Private) {
                        localAreaTag = Message.WellKnownAreaTags.Private;
                    } else {
                        Log.warn('Non-private message without area tag');
                        importStats.otherFail += 1;
                        return next(null);
                    }
                }

                message.uuid = Message.createMessageUUID(
                    localAreaTag,
                    message.modTimestamp,
                    message.subject,
                    message.message);

                self.appendTearAndOrigin(message);

                const importConfig = {
                    localAreaTag    : localAreaTag,
                };

                self.importMailToArea(importConfig, packetHeader, message, err => {
                    if(err) {
                        //  bump area fail stats
                        importStats.areaFail[localAreaTag] = (importStats.areaFail[localAreaTag] || 0) + 1;

                        if('SQLITE_CONSTRAINT' === err.code || 'DUPE_MSGID' === err.code) {
                            const msgId = _.has(message.meta, 'FtnKludge.MSGID') ? message.meta.FtnKludge.MSGID : 'N/A';
                            Log.info(
                                { area : localAreaTag, subject : message.subject, uuid : message.uuid, MSGID : msgId },
                                'Not importing non-unique message');

                            return next(null);
                        }
                    } else {
                        //  bump area success
                        importStats.areaSuccess[localAreaTag] = (importStats.areaSuccess[localAreaTag] || 0) + 1;
                    }

                    return next(err);
                });
            }
        }, err => {
            //
            //  try to produce something helpful in the log
            //
            const finalStats = Object.assign(importStats, { packetPath : packetPath } );
            if(err || Object.keys(finalStats.areaFail).length > 0) {
                if(err) {
                    Object.assign(finalStats, { error : err.message } );
                }

                Log.warn(finalStats, 'Import completed with error(s)');
            } else {
                Log.info(finalStats, 'Import complete');
            }

            cb(err);
        });
    };

    this.maybeArchiveImportFile = function(origPath, type, status, cb) {
        //
        //  type    : pkt|tic|bundle
        //  status  : good|reject
        //
        //  Status of "good" is only applied to pkt files & placed
        //  in |retain| if set. This is generally used for debugging only.
        //
        let archivePath;
        const ts = moment().format('YYYY-MM-DDTHH.mm.ss.SSS');
        const fn = paths.basename(origPath);

        if('good' === status && type === 'pkt') {
            if(!_.isString(self.moduleConfig.paths.retain)) {
                return cb(null);
            }

            archivePath = paths.join(self.moduleConfig.paths.retain, `good-pkt-${ts}--${fn}`);
        } else if('good' !== status) {
            archivePath = paths.join(self.moduleConfig.paths.reject, `${status}-${type}--${ts}-${fn}`);
        } else {
            return cb(null);    //  don't archive non-good/pkt files
        }

        Log.debug( { origPath : origPath, archivePath : archivePath, type : type, status : status }, 'Archiving import file');

        fse.copy(origPath, archivePath, err => {
            if(err) {
                Log.warn( { error : err.message, origPath : origPath, archivePath : archivePath, type : type, status : status }, 'Failed to archive packet file');
            }

            return cb(null);    //  never fatal
        });
    };

    this.importPacketFilesFromDirectory = function(importDir, password, cb) {
        async.waterfall(
            [
                function getPacketFiles(callback) {
                    fs.readdir(importDir, (err, files) => {
                        if(err) {
                            return callback(err);
                        }
                        callback(null, files.filter(f => '.pkt' === paths.extname(f).toLowerCase()));
                    });
                },
                function importPacketFiles(packetFiles, callback) {
                    let rejects = [];
                    async.eachSeries(packetFiles, (packetFile, nextFile) => {
                        self.importMessagesFromPacketFile(paths.join(importDir, packetFile), '', err => {
                            if(err) {
                                Log.debug(
                                    { path : paths.join(importDir, packetFile), error : err.toString() },
                                    'Failed to import packet file');

                                rejects.push(packetFile);
                            }
                            nextFile();
                        });
                    }, err => {
                        //  :TODO: Handle err! we should try to keep going though...
                        callback(err, packetFiles, rejects);
                    });
                },
                function handleProcessedFiles(packetFiles, rejects, callback) {
                    async.each(packetFiles, (packetFile, nextFile) => {
                        //  possibly archive, then remove original
                        const fullPath = paths.join(importDir, packetFile);
                        self.maybeArchiveImportFile(
                            fullPath,
                            'pkt',
                            rejects.includes(packetFile) ? 'reject' : 'good',
                            () => {
                                fs.unlink(fullPath, () => {
                                    return nextFile(null);
                                });
                            }
                        );
                    }, err => {
                        callback(err);
                    });
                }
            ],
            err => {
                cb(err);
            }
        );
    };

    this.importFromDirectory = function(inboundType, importDir, cb) {
        async.waterfall(
            [
                //  start with .pkt files
                function importPacketFiles(callback) {
                    self.importPacketFilesFromDirectory(importDir, '', err => {
                        callback(err);
                    });
                },
                function discoverBundles(callback) {
                    fs.readdir(importDir, (err, files) => {
                        //  :TODO: if we do much more of this, probably just use the glob module
                        const bundleRegExp = /\.(su|mo|tu|we|th|fr|sa)[0-9a-z]/i;
                        files = files.filter(f => {
                            const fext = paths.extname(f);
                            return bundleRegExp.test(fext);
                        });

                        async.map(files, (file, transform) => {
                            const fullPath = paths.join(importDir, file);
                            self.archUtil.detectType(fullPath, (err, archName) => {
                                transform(null, { path : fullPath, archName : archName } );
                            });
                        }, (err, bundleFiles) => {
                            callback(err, bundleFiles);
                        });
                    });
                },
                function importBundles(bundleFiles, callback) {
                    let rejects = [];

                    async.each(bundleFiles, (bundleFile, nextFile) => {
                        if(_.isUndefined(bundleFile.archName)) {
                            Log.warn(
                                { fileName : bundleFile.path },
                                'Unknown bundle archive type');

                            rejects.push(bundleFile.path);

                            return nextFile();  //  unknown archive type
                        }

                        Log.debug( { bundleFile : bundleFile }, 'Processing bundle' );

                        self.archUtil.extractTo(
                            bundleFile.path,
                            self.importTempDir,
                            bundleFile.archName,
                            err => {
                                if(err) {
                                    Log.warn(
                                        { path : bundleFile.path, error : err.message },
                                        'Failed to extract bundle');

                                    rejects.push(bundleFile.path);
                                }

                                nextFile();
                            }
                        );
                    }, err => {
                        if(err) {
                            return callback(err);
                        }

                        //
                        //  All extracted - import .pkt's
                        //
                        self.importPacketFilesFromDirectory(self.importTempDir, '', () => {
                            //  :TODO: handle |err|
                            callback(null, bundleFiles, rejects);
                        });
                    });
                },
                function handleProcessedBundleFiles(bundleFiles, rejects, callback) {
                    async.each(bundleFiles, (bundleFile, nextFile) => {
                        self.maybeArchiveImportFile(
                            bundleFile.path,
                            'bundle',
                            rejects.includes(bundleFile.path) ? 'reject' : 'good',
                            () => {
                                fs.unlink(bundleFile.path, err => {
                                    if(err) {
                                        Log.error( { path : bundleFile.path, error : err.message }, 'Failed unlinking bundle');
                                    }
                                    return nextFile(null);
                                });
                            }
                        );
                    }, err => {
                        callback(err);
                    });
                },
                function importTicFiles(callback) {
                    self.processTicFilesInDirectory(importDir, err => {
                        return callback(err);
                    });
                }
            ],
            err => {
                cb(err);
            }
        );
    };

    this.createTempDirectories = function(cb) {
        temptmp.mkdir( { prefix : 'enigftnexport-' }, (err, tempDir) => {
            if(err) {
                return cb(err);
            }

            self.exportTempDir = tempDir;

            temptmp.mkdir( { prefix : 'enigftnimport-' }, (err, tempDir) => {
                self.importTempDir = tempDir;

                cb(err);
            });
        });
    };

    //  Starts an export block - returns true if we can proceed
    this.exportingStart = function() {
        if(!this.exportRunning) {
            this.exportRunning = true;
            return true;
        }

        return false;
    };

    //  ends an export block
    this.exportingEnd = function(cb) {
        this.exportRunning = false;

        if(cb) {
            return cb(null);
        }
    };

    this.copyTicAttachment = function(src, dst, isUpdate, cb) {
        if(isUpdate) {
            fse.copy(src, dst, { overwrite : true }, err => {
                return cb(err, dst);
            });
        } else {
            copyFileWithCollisionHandling(src, dst, (err, finalPath) => {
                return cb(err, finalPath);
            });
        }
    };

    this.getLocalAreaTagsForTic = function() {
        const config = Config();
        return _.union(Object.keys(config.scannerTossers.ftn_bso.ticAreas || {} ), Object.keys(config.fileBase.areas));
    };

    this.processSingleTicFile = function(ticFileInfo, cb) {
        Log.debug( { tic : ticFileInfo.path, file : ticFileInfo.getAsString('File') }, 'Processing TIC file');

        async.waterfall(
            [
                function generalValidation(callback) {
                    const sysConfig = Config();
                    const config = {
                        nodes           : sysConfig.scannerTossers.ftn_bso.nodes,
                        defaultPassword : sysConfig.scannerTossers.ftn_bso.tic.password,
                        localAreaTags   : self.getLocalAreaTagsForTic(),
                    };

                    return ticFileInfo.validate(config, (err, localInfo) => {
                        if(err) {
                            Log.trace( { reason : err.message }, 'Validation failure');
                            return callback(err);
                        }

                        //  We may need to map |localAreaTag| back to real areaTag if it's a mapping/alias
                        const mappedLocalAreaTag = _.get(Config().scannerTossers.ftn_bso, [ 'ticAreas', localInfo.areaTag ]);

                        if(mappedLocalAreaTag) {
                            if(_.isString(mappedLocalAreaTag.areaTag)) {
                                localInfo.areaTag       = mappedLocalAreaTag.areaTag;
                                localInfo.hashTags      = mappedLocalAreaTag.hashTags;      //  override default for node
                                localInfo.storageTag    = mappedLocalAreaTag.storageTag;    //  override default
                            } else if(_.isString(mappedLocalAreaTag)) {
                                localInfo.areaTag = mappedLocalAreaTag;
                            }
                        }

                        return callback(null, localInfo);
                    });
                },
                function findExistingItem(localInfo, callback) {
                    //
                    //  We will need to look for an existing item to replace/update if:
                    //  a) The TIC file has a "Replaces" field
                    //  b) The general or node specific |allowReplace| is true
                    //
                    //  Replace specifies a DOS 8.3 *pattern* which is allowed to have
                    //  ? and * characters. For example, RETRONET.*
                    //
                    //  Lastly, we will only replace if the item is in the same/specified area
                    //  and that come from the same origin as a previous entry.
                    //
                    const allowReplace  = _.get(Config().scannerTossers.ftn_bso.nodes, [ localInfo.node, 'tic', 'allowReplace' ], Config().scannerTossers.ftn_bso.tic.allowReplace);
                    const replaces      = ticFileInfo.getAsString('Replaces');

                    if(!allowReplace || !replaces) {
                        return callback(null, localInfo);
                    }

                    const metaPairs = [
                        {
                            name        : 'short_file_name',
                            value       : replaces.toUpperCase(),   //  we store upper as well
                            wildcards   : true, //  value may contain wildcards
                        },
                        {
                            name    : 'tic_origin',
                            value   : ticFileInfo.getAsString('Origin'),
                        }
                    ];

                    FileEntry.findFiles( { metaPairs : metaPairs, areaTag : localInfo.areaTag }, (err, fileIds) => {
                        if(err) {
                            return callback(err);
                        }

                        //  0:1 allowed
                        if(1 === fileIds.length) {
                            localInfo.existingFileId = fileIds[0];

                            //  fetch old filename - we may need to remove it if replacing with a new name
                            FileEntry.loadBasicEntry(localInfo.existingFileId, {}, (err, info) => {
                                if(info) {
                                    Log.trace(
                                        { fileId : localInfo.existingFileId, oldFileName : info.fileName, oldStorageTag : info.storageTag },
                                        'Existing TIC file target to be replaced'
                                    );

                                    localInfo.oldFileName   = info.fileName;
                                    localInfo.oldStorageTag = info.storageTag;
                                }
                                return callback(null, localInfo);   //  continue even if we couldn't find an old match
                            });
                        } else if(fileIds.legnth > 1) {
                            return callback(Errors.General(`More than one existing entry for TIC in ${localInfo.areaTag} ([${fileIds.join(', ')}])`));
                        } else {
                            return callback(null, localInfo);
                        }
                    });
                },
                function scan(localInfo, callback) {
                    const scanOpts = {
                        sha256      : localInfo.sha256, //  *may* have already been calculated
                        meta        : {
                            //  some TIC-related metadata we always want
                            short_file_name     : ticFileInfo.getAsString('File').toUpperCase(),    //  upper to ensure no case issues later; this should be a DOS 8.3 name
                            tic_origin          : ticFileInfo.getAsString('Origin'),
                            tic_desc            : ticFileInfo.getAsString('Desc'),
                            upload_by_username  : _.get(Config().scannerTossers.ftn_bso.nodes, [ localInfo.node, 'tic', 'uploadBy' ], Config().scannerTossers.ftn_bso.tic.uploadBy),
                        }
                    };

                    const ldesc = ticFileInfo.getAsString('Ldesc', '\n');
                    if(ldesc) {
                        scanOpts.meta.tic_ldesc = ldesc;
                    }

                    //
                    //  We may have TIC auto-tagging for this node and/or specific (remote) area
                    //
                    const hashTags =
                        localInfo.hashTags ||
                        _.get(Config().scannerTossers.ftn_bso.nodes, [ localInfo.node, 'tic', 'hashTags' ] );       //  catch-all*/

                    if(hashTags) {
                        scanOpts.hashTags = new Set(hashTags.split(/[\s,]+/));
                    }

                    if(localInfo.crc32) {
                        scanOpts.meta.file_crc32 = localInfo.crc32.toString(16);    //  again, *may* have already been calculated
                    }

                    scanFile(
                        ticFileInfo.filePath,
                        scanOpts,
                        (err, fileEntry) => {
                            if(err) {
                                Log.trace( { reason : err.message }, 'Scanning failed');
                            }

                            localInfo.fileEntry = fileEntry;
                            return callback(err, localInfo);
                        }
                    );
                },
                function store(localInfo, callback) {
                    //
                    //  Move file to final area storage and persist to DB
                    //
                    const areaInfo = getFileAreaByTag(localInfo.areaTag);
                    if(!areaInfo) {
                        return callback(Errors.UnexpectedState(`Could not get area for tag ${localInfo.areaTag}`));
                    }

                    const storageTag = localInfo.storageTag || areaInfo.storageTags[0];
                    if(!isValidStorageTag(storageTag)) {
                        return callback(Errors.Invalid(`Invalid storage tag: ${storageTag}`));
                    }

                    localInfo.fileEntry.storageTag  = storageTag;
                    localInfo.fileEntry.areaTag     = localInfo.areaTag;
                    localInfo.fileEntry.fileName    = ticFileInfo.longFileName;

                    //
                    //  We may now have two descriptions: from .DIZ/etc. or the TIC itself.
                    //  Determine which one to use using |descPriority| and availability.
                    //
                    //  We will still fallback as needed from <priority1> -> <priority2> -> <fromFileName>
                    //
                    const descPriority = _.get(
                        Config().scannerTossers.ftn_bso.nodes, [ localInfo.node, 'tic', 'descPriority' ],
                        Config().scannerTossers.ftn_bso.tic.descPriority
                    );

                    if('tic' === descPriority) {
                        const origDesc = localInfo.fileEntry.desc;
                        localInfo.fileEntry.desc = ticFileInfo.getAsString('Ldesc') || origDesc || getDescFromFileName(ticFileInfo.filePath);
                    } else {
                        //  see if we got desc from .DIZ/etc.
                        const fromDescFile = 'descFile' === localInfo.fileEntry.descSrc;
                        localInfo.fileEntry.desc = fromDescFile ? localInfo.fileEntry.desc : ticFileInfo.getAsString('Ldesc');
                        localInfo.fileEntry.desc = localInfo.fileEntry.desc || getDescFromFileName(ticFileInfo.filePath);
                    }

                    const areaStorageDir = getAreaStorageDirectoryByTag(storageTag);
                    if(!areaStorageDir) {
                        return callback(Errors.UnexpectedState(`Could not get storage directory for tag ${localInfo.areaTag}`));
                    }

                    const isUpdate = localInfo.existingFileId ? true : false;

                    if(isUpdate) {
                        //  we need to *update* an existing record/file
                        localInfo.fileEntry.fileId = localInfo.existingFileId;
                    }

                    const dst = paths.join(areaStorageDir, localInfo.fileEntry.fileName);

                    self.copyTicAttachment(ticFileInfo.filePath, dst, isUpdate, (err, finalPath) => {
                        if(err) {
                            Log.info( { reason : err.message }, 'Failed to copy TIC attachment');
                            return callback(err);
                        }

                        if(dst !== finalPath) {
                            localInfo.fileEntry.fileName = paths.basename(finalPath);
                        }

                        localInfo.fileEntry.persist(isUpdate, err => {
                            return callback(err, localInfo);
                        });
                    });
                },
                //  :TODO: from here, we need to re-toss files if needed, before they are removed
                function cleanupOldFile(localInfo, callback) {
                    if(!localInfo.existingFileId) {
                        return callback(null, localInfo);
                    }

                    const oldStorageDir = getAreaStorageDirectoryByTag(localInfo.oldStorageTag);
                    const oldPath       = paths.join(oldStorageDir, localInfo.oldFileName);

                    fs.unlink(oldPath, err => {
                        if(err) {
                            Log.warn( { error : err.message, oldPath : oldPath }, 'Failed removing old physical file during TIC replacement');
                        } else {
                            Log.debug( { oldPath : oldPath }, 'Removed old physical file during TIC replacement');
                        }
                        return callback(null, localInfo);   //  continue even if err
                    });
                },
            ],
            (err, localInfo) => {
                if(err) {
                    Log.error( { error : err.message, reason : err.reason, tic : ticFileInfo.filePath }, 'Failed import/update TIC record' );
                } else {
                    Log.debug(
                        { tic : ticFileInfo.path, file : ticFileInfo.filePath, area : localInfo.areaTag },
                        'TIC imported successfully'
                    );
                }
                return cb(err);
            }
        );
    };

    this.removeAssocTicFiles = function(ticFileInfo, cb) {
        async.each( [ ticFileInfo.path, ticFileInfo.filePath ], (path, nextPath) => {
            fs.unlink(path, err => {
                if(err && 'ENOENT' !== err.code) {  //  don't log when the file doesn't exist
                    Log.warn( { error : err.message, path : path }, 'Failed unlinking TIC file');
                }
                return nextPath(null);
            });
        }, err => {
            return cb(err);
        });
    };


    this.performEchoMailExport = function(cb) {
        //
        //  Select all messages with a |message_id| > |lastScanId|.
        //  Additionally exclude messages with the System state_flags0 which will be present for
        //  imported or already exported messages
        //
        //  NOTE: If StateFlags0 starts to use additional bits, we'll likely need to check them here!
        //
        const getNewUuidsSql =
            `SELECT message_id, message_uuid
            FROM message m
            WHERE area_tag = ? AND message_id > ? AND
                (SELECT COUNT(message_id) 
                FROM message_meta 
                WHERE message_id = m.message_id AND meta_category = 'System' AND meta_name = 'state_flags0') = 0
            ORDER BY message_id;`
            ;

        //  we shouldn't, but be sure we don't try to pick up private mail here
        const config = Config();
        const areaTags = Object.keys(config.messageNetworks.ftn.areas)
            .filter(areaTag => Message.WellKnownAreaTags.Private !== areaTag);

        async.each(areaTags, (areaTag, nextArea) => {
            const areaConfig = config.messageNetworks.ftn.areas[areaTag];
            if(!this.isAreaConfigValid(areaConfig)) {
                return nextArea();
            }

            //
            //  For each message that is newer than that of the last scan
            //  we need to export to each configured associated uplink(s)
            //
            async.waterfall(
                [
                    function getLastScanId(callback) {
                        self.getAreaLastScanId(areaTag, callback);
                    },
                    function getNewUuids(lastScanId, callback) {
                        msgDb.all(getNewUuidsSql, [ areaTag, lastScanId ], (err, rows) => {
                            if(err) {
                                callback(err);
                            } else {
                                if(0 === rows.length) {
                                    let nothingToDoErr = new Error('Nothing to do!');
                                    nothingToDoErr.noRows = true;
                                    callback(nothingToDoErr);
                                } else {
                                    callback(null, rows);
                                }
                            }
                        });
                    },
                    function exportToConfiguredUplinks(msgRows, callback) {
                        const uuidsOnly = msgRows.map(r => r.message_uuid); //  convert to array of UUIDs only
                        self.exportEchoMailMessagesToUplinks(uuidsOnly, areaConfig, err => {
                            const newLastScanId = msgRows[msgRows.length - 1].message_id;

                            Log.info(
                                { areaTag : areaTag, messagesExported : msgRows.length, newLastScanId : newLastScanId },
                                'Export complete');

                            callback(err, newLastScanId);
                        });
                    },
                    function updateLastScanId(newLastScanId, callback) {
                        self.setAreaLastScanId(areaTag, newLastScanId, callback);
                    }
                ],
                () => {
                    return nextArea();
                }
            );
        },
        err => {
            return cb(err);
        });
    };

    this.performNetMailExport = function(cb) {
        //
        //  Select all messages with a |message_id| > |lastScanId| in the private area
        //  that are schedule for export to FTN-style networks.
        //
        //  Just like EchoMail, we additionally exclude messages with the System state_flags0
        //  which will be present for imported or already exported messages
        //
        //
        //  :TODO: fill out the rest of the consts here
        //  :TODO: this statement is crazy ugly -- use JOIN / NOT EXISTS for state_flags & 0x02
        const getNewUuidsSql =
            `SELECT message_id, message_uuid
            FROM message m
            WHERE area_tag = '${Message.WellKnownAreaTags.Private}' AND message_id > ? AND 
                (SELECT COUNT(message_id)
                FROM message_meta
                WHERE message_id = m.message_id
                    AND meta_category = 'System'
                    AND (meta_name = 'state_flags0' OR meta_name = 'local_to_user_id')
                ) = 0
            AND
                (SELECT COUNT(message_id)
                FROM message_meta
                WHERE message_id = m.message_id
                    AND meta_category = 'System' 
                    AND meta_name = '${Message.SystemMetaNames.ExternalFlavor}'
                    AND meta_value = '${Message.AddressFlavor.FTN}'
                ) = 1
            ORDER BY message_id;
            `;

        async.waterfall(
            [
                function getLastScanId(callback) {
                    return self.getAreaLastScanId(Message.WellKnownAreaTags.Private, callback);
                },
                function getNewUuids(lastScanId, callback) {
                    msgDb.all(getNewUuidsSql, [ lastScanId ], (err, rows) => {
                        if(err) {
                            return callback(err);
                        }

                        if(0 === rows.length) {
                            return cb(null);    //  note |cb| -- early bail out!
                        }

                        return callback(null, rows);
                    });
                },
                function exportMessages(rows, callback) {
                    const messageUuids = rows.map(r => r.message_uuid);
                    return self.exportNetMailMessagesToUplinks(messageUuids, callback);
                }
            ],
            err => {
                return cb(err);
            }
        );
    };

    this.isNetMailMessage = function(message) {
        return message.isPrivate() &&
            null === _.get(message, 'meta.System.LocalToUserID', null) &&
            Message.AddressFlavor.FTN === _.get(message, 'meta.System.external_flavor', null);
    };
}

require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule);

//  :TODO: *scheduled* portion of this stuff should probably use event_scheduler - @immediate would still use record().

FTNMessageScanTossModule.prototype.processTicFilesInDirectory = function(importDir, cb) {
    //  :TODO: pass in 'inbound' vs 'secInbound' -- pass along to processSingleTicFile() where password will be checked

    const self = this;
    async.waterfall(
        [
            function findTicFiles(callback) {
                fs.readdir(importDir, (err, files) => {
                    if(err) {
                        return callback(err);
                    }

                    return callback(null, files.filter(f => '.tic' === paths.extname(f).toLowerCase()));
                });
            },
            function gatherInfo(ticFiles, callback) {
                const ticFilesInfo = [];

                async.each(ticFiles, (fileName, nextFile) => {
                    const fullPath = paths.join(importDir, fileName);

                    TicFileInfo.createFromFile(fullPath, (err, ticInfo) => {
                        if(err) {
                            Log.warn( { error : err.message, path : fullPath }, 'Failed reading TIC file');
                        } else {
                            ticFilesInfo.push(ticInfo);
                        }

                        return nextFile(null);
                    });
                },
                err => {
                    return callback(err, ticFilesInfo);
                });
            },
            function process(ticFilesInfo, callback) {
                async.eachSeries(ticFilesInfo, (ticFileInfo, nextTicInfo) => {
                    self.processSingleTicFile(ticFileInfo, err => {
                        if(err) {
                            //  archive rejected TIC stuff (.TIC + attach)
                            async.each( [ ticFileInfo.path, ticFileInfo.filePath ], (path, nextPath) => {
                                if(!path) { //  possibly rejected due to "File" not existing/etc.
                                    return nextPath(null);
                                }

                                self.maybeArchiveImportFile(
                                    path,
                                    'tic',
                                    'reject',
                                    () => {
                                        return nextPath(null);
                                    }
                                );
                            },
                            () => {
                                self.removeAssocTicFiles(ticFileInfo, () => {
                                    return nextTicInfo(null);
                                });
                            });
                        } else {
                            self.removeAssocTicFiles(ticFileInfo, () => {
                                return nextTicInfo(null);
                            });
                        }
                    });
                }, err => {
                    return callback(err);
                });
            }
        ],
        err => {
            return cb(err);
        }
    );
};

FTNMessageScanTossModule.prototype.startup = function(cb) {
    Log.info(`${exports.moduleInfo.name} Scanner/Tosser starting up`);

    let importing = false;

    let self = this;

    function tryImportNow(reasonDesc, extraInfo) {
        if(!importing) {
            importing = true;

            Log.info( Object.assign({ module : exports.moduleInfo.name }, extraInfo), reasonDesc);

            self.performImport( () => {
                importing = false;
            });
        }
    }

    this.createTempDirectories(err => {
        if(err) {
            Log.warn( { error : err.toStrong() }, 'Failed creating temporary directories!');
            return cb(err);
        }

        if(_.isObject(this.moduleConfig.schedule)) {
            const exportSchedule = this.parseScheduleString(this.moduleConfig.schedule.export);
            if(exportSchedule) {
                Log.debug(
                    {
                        schedule    : this.moduleConfig.schedule.export,
                        schedOK     : -1 === exportSchedule.sched.error,
                        next        : moment(later.schedule(exportSchedule.sched).next(1)).format('ddd, MMM Do, YYYY @ h:m:ss a'),
                        immediate   : exportSchedule.immediate ? true : false,
                    },
                    'Export schedule loaded'
                );

                if(exportSchedule.sched) {
                    this.exportTimer = later.setInterval( () => {
                        if(this.exportingStart()) {
                            Log.info( { module : exports.moduleInfo.name }, 'Performing scheduled message scan/export...');

                            this.performExport( () => {
                                this.exportingEnd();
                            });
                        }
                    }, exportSchedule.sched);
                }

                if(_.isBoolean(exportSchedule.immediate)) {
                    this.exportImmediate = exportSchedule.immediate;
                }
            }

            const importSchedule = this.parseScheduleString(this.moduleConfig.schedule.import);
            if(importSchedule) {
                Log.debug(
                    {
                        schedule    : this.moduleConfig.schedule.import,
                        schedOK     : -1 === importSchedule.sched.error,
                        next        : moment(later.schedule(importSchedule.sched).next(1)).format('ddd, MMM Do, YYYY @ h:m:ss a'),
                        watchFile   : _.isString(importSchedule.watchFile) ? importSchedule.watchFile : 'None',
                    },
                    'Import schedule loaded'
                );

                if(importSchedule.sched) {
                    this.importTimer = later.setInterval( () => {
                        tryImportNow('Performing scheduled message import/toss...');
                    }, importSchedule.sched);
                }

                if(_.isString(importSchedule.watchFile)) {
                    const watcher = sane(
                        paths.dirname(importSchedule.watchFile),
                        {
                            glob : `**/${paths.basename(importSchedule.watchFile)}`
                        }
                    );

                    [ 'change', 'add', 'delete' ].forEach(event => {
                        watcher.on(event, (fileName, fileRoot) => {
                            const eventPath = paths.join(fileRoot, fileName);
                            if(paths.join(fileRoot, fileName) === importSchedule.watchFile) {
                                tryImportNow('Performing import/toss due to @watch', { eventPath, event } );
                            }
                        });
                    });

                    //
                    //  If the watch file already exists, kick off now
                    //  https://github.com/NuSkooler/enigma-bbs/issues/122
                    //
                    fse.exists(importSchedule.watchFile, exists => {
                        if(exists) {
                            tryImportNow('Performing import/toss due to @watch', { eventPath : importSchedule.watchFile, event : 'initial exists' } );
                        }
                    });
                }
            }
        }

        FTNMessageScanTossModule.super_.prototype.startup.call(this, cb);
    });
};

FTNMessageScanTossModule.prototype.shutdown = function(cb) {
    Log.info('FidoNet Scanner/Tosser shutting down');

    if(this.exportTimer) {
        this.exportTimer.clear();
    }

    if(this.importTimer) {
        this.importTimer.clear();
    }

    //
    //  Clean up temp dir/files we created
    //
    temptmp.cleanup( paths => {
        const fullStats = {
            exportDir   : this.exportTempDir,
            importTemp  : this.importTempDir,
            paths       : paths,
            sessionId   : temptmp.sessionId,
        };

        Log.trace(fullStats, 'Temporary directories cleaned up');

        FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb);
    });

    FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb);
};

FTNMessageScanTossModule.prototype.performImport = function(cb) {
    if(!this.hasValidConfiguration()) {
        return cb(new Error('Missing or invalid configuration'));
    }

    const self = this;

    async.each( [ 'inbound', 'secInbound' ], (inboundType, nextDir) => {
        self.importFromDirectory(inboundType, self.moduleConfig.paths[inboundType], () => {
            return nextDir(null);
        });
    }, cb);
};

FTNMessageScanTossModule.prototype.performExport = function(cb) {
    //
    //  We're only concerned with areas related to FTN. For each area, loop though
    //  and let's find out what messages need exported.
    //
    if(!this.hasValidConfiguration()) {
        return cb(new Error('Missing or invalid configuration'));
    }

    const self = this;

    async.eachSeries( [ 'EchoMail', 'NetMail' ], (type, nextType) => {
        self[`perform${type}Export`]( err => {
            if(err) {
                Log.warn( { error : err.message, type : type }, 'Error(s) during export' );
            }
            return nextType(null);  //  try next, always
        });
    }, () => {
        return cb(null);
    });
};

FTNMessageScanTossModule.prototype.record = function(message) {
    //
    //  This module works off schedules, but we do support @immediate for export
    //
    if(true !== this.exportImmediate || !this.hasValidConfiguration()) {
        return;
    }

    const info = { uuid : message.uuid, subject : message.subject };

    function exportLog(err) {
        if(err) {
            Log.warn(info, 'Failed exporting message');
        } else {
            Log.info(info, 'Message exported');
        }
    }

    if(this.isNetMailMessage(message)) {
        Object.assign(info, { type : 'NetMail' } );

        if(this.exportingStart()) {
            this.exportNetMailMessagesToUplinks( [ message.uuid ], err => {
                this.exportingEnd( () => exportLog(err) );
            });
        }
    } else if(message.areaTag) {
        Object.assign(info, { type : 'EchoMail' } );

        const areaConfig = Config().messageNetworks.ftn.areas[message.areaTag];
        if(!this.isAreaConfigValid(areaConfig)) {
            return;
        }

        if(this.exportingStart()) {
            this.exportEchoMailMessagesToUplinks( [ message.uuid ], areaConfig, err => {
                this.exportingEnd( () => exportLog(err) );
            });
        }
    }
};