enigma-bbs/core/qwk_mail_packet.js

const ArchiveUtil = require('./archive_util');
const { Errors } = require('./enig_error');
const Message = require('./message');
const { splitTextAtTerms } = require('./string_util');
const {
    getMessageConfTagByAreaTag,
    getMessageAreaByTag,
} = require('./message_area');
const StatLog = require('./stat_log');
const Config = require('./config').get;
const SysProps = require('./system_property');
const UserProps = require('./user_property');
const { numToMbf32 } = require('./mbf');
const { getEncodingFromCharacterSetIdentifier } = require('./ftn_util');

const { EventEmitter } = require('events');
const temptmp = require('temptmp');
const async = require('async');
const fs = require('graceful-fs');
const paths = require('path');
const { Parser } = require('binary-parser');
const iconv = require('iconv-lite');
const moment = require('moment');
const _ = require('lodash');
const IniConfigParser = require('ini-config-parser');

const enigmaVersion = require('../package.json').version;

//  Synchronet smblib TZ to a UTC offset
//  see https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h
const SMBTZToUTCOffset = {
    //  US Standard
    '40F0'  : '-04:00', //  Atlantic
    '412C'  : '-05:00', //  Eastern
    '4168'  : '-06:00', //  Central
    '41A4'  : '-07:00', //  Mountain
    '41E0'  : '-08:00', //  Pacific
    '421C'  : '-09:00', //  Yukon
    '4258'  : '-10:00', //  Hawaii/Alaska
    '4294'  : '-11:00', //  Bering

    //  US Daylight
    'C0F0'  : '-03:00', //  Atlantic
    'C12C'	: '-04:00', //  Eastern
    'C168'	: '-05:00', //  Central
    'C1A4'	: '-06:00', //  Mountain
    'C1E0'	: '-07:00', //  Pacific
    'C21C'	: '-08:00', //  Yukon
    'C258'  : '-09:00', //  Hawaii/Alaska
    'C294'  : '-10:00', //  Bering

    //  "Non-Standard"
    '2294'  : '-11:00', //  Midway
    '21E0'  : '-08:00', //  Vancouver
    '21A4'  : '-07:00', //  Edmonton
    '2168'  : '-06:00', //  Winnipeg
    '212C'  : '-05:00', //  Bogota
    '20F0'  : '-04:00', //  Caracas
    '20B4'  : '-03:00', //  Rio de Janeiro
    '2078'  : '-02:00', //  Fernando de Noronha
    '203C'  : '-01:00', //  Azores
    '1000'  : '+00:00', //  London
    '103C'  : '+01:00', //  Berlin
    '1078'  : '+02:00', //  Athens
    '10B4'  : '+03:00', //  Moscow
    '10F0'  : '+04:00', //  Dubai
    '110E'  : '+04:30', //  Kabul
    '112C'  : '+05:00', //  Karachi
    '114A'  : '+05:30', //  Bombay
    '1159'  : '+05:45', //  Kathmandu
    '1168'  : '+06:00', //  Dhaka
    '11A4'  : '+07:00', //  Bangkok
    '11E0'  : '+08:00', //  Hong Kong
    '121C'  : '+09:00', //  Tokyo
    '1258'  : '+10:00', //  Sydney
    '1294'  : '+11:00', //  Noumea
    '12D0'  : '+12:00', //  Wellington
};

const UTCOffsetToSMBTZ   = _.invert(SMBTZToUTCOffset);

const QWKMessageBlockSize       = 128;
const QWKHeaderTimestampFormat  = 'MM-DD-YYHH:mm';
const QWKLF                     = 0xe3;

const QWKMessageStatusCodes = {
    UnreadPublic                : ' ',
    ReadPublic                  : '-',
    UnreadPrivate               : '+',
    ReadPrivate                 : '*',
    UnreadCommentToSysOp        : '~',
    ReadCommentToSysOp          : '`',
    UnreadSenderPWProtected     : '%',
    ReadSenderPWProtected       : '^',
    UnreadGroupPWProtected      : '!',
    ReadGroupPWProtected        : '#',
    PWProtectedToAll            : '$',
    Vote                        : 'V',
};

const QWKMessageActiveStatus = {
    Active  : 255,
    Deleted : 226,
};

const QWKNetworkTagIndicator = {
    Present     : '*',
    NotPresent  : ' ',
};

//  See the following:
//  -   http://fileformats.archiveteam.org/wiki/QWK
//  -   http://wiki.synchro.net/ref:qwk
//
const MessageHeaderParser = new Parser()
    .endianess('little')
    .string('status', {
        encoding    : 'ascii',
        length      : 1,
    })
    .string('num', {    //  message num or conf num for REP's
        encoding    : 'ascii',
        length      : 7,
        formatter   : n => {
            return parseInt(n);
        }
    })
    .string('timestamp', {
        encoding    : 'ascii',
        length      : 13,
    })
    //  these fields may be encoded in something other than ascii/CP437
    .array('toName', {
        type    : 'uint8',
        length  : 25,
    })
    .array('fromName', {
        type    : 'uint8',
        length  : 25,
    })
    .array('subject', {
        type    : 'uint8',
        length  : 25,
    })
    .string('password', {
        encoding    : 'ascii',
        length      : 12,
    })
    .string('replyToNum', {
        encoding    : 'ascii',
        length      : 8,
        formatter   : n => {
            return parseInt(n);
        }
    })
    .string('numBlocks', {
        encoding    : 'ascii',
        length      : 6,
        formatter   : n => {
            return parseInt(n);
        }
    })
    .uint8('status2')
    .uint16('confNum')
    .uint16('relNum')
    .uint8('netTag');

const replaceCharInBuffer = (buffer, search, replace) => {
    let i = 0;
    search = Buffer.from([search]);
    while (i < buffer.length) {
        i = buffer.indexOf(search, i);
        if (-1 === i) {
            break;
        }
        buffer[i] = replace;
        ++i;
    }
}

class QWKPacketReader extends EventEmitter {
    constructor(
        packetPath,
        { mode = QWKPacketReader.Modes.Guess, keepTearAndOrigin = true } = { mode : QWKPacketReader.Modes.Guess, keepTearAndOrigin : true })
    {
        super();

        this.packetPath = packetPath;
        this.options    = { mode, keepTearAndOrigin };
        this.temptmp    = temptmp.createTrackedSession('qwkpacketreader');
    }

    static get Modes() {
        return {
            Guess   : 'guess',  //  try to guess
            QWK     : 'qwk',    //  standard incoming packet
            REP     : 'rep',    //  a reply packet
        };
    }

    read() {
        //
        //  A general overview:
        //
        //  - Find out what kind of archive we're dealing with
        //  - Extract to temporary location
        //  - Process various files
        //  - Emit messages we find, information about the packet, so on
        //
        async.waterfall(
            [
                //  determine packet archive type
                (callback) => {
                    const archiveUtil = ArchiveUtil.getInstance();
                    archiveUtil.detectType(this.packetPath, (err, archiveType) => {
                        if (err) {
                            return callback(err);
                        }
                        this.emit('archive type', archiveType);
                        return callback(null, archiveType);
                    });
                },
                //  create a temporary location to do processing
                (archiveType, callback) => {
                    this.temptmp.mkdir( { prefix : 'enigqwkreader-'}, (err, tempDir) => {
                        if (err) {
                            return callback(err);
                        }

                        return callback(null, archiveType, tempDir);
                    });
                },
                //  extract it
                (archiveType, tempDir, callback) => {
                    const archiveUtil = ArchiveUtil.getInstance();
                    archiveUtil.extractTo(this.packetPath, tempDir, archiveType, err => {
                        if (err) {
                            return callback(err);
                        }

                        return callback(null, tempDir);
                    });
                },
                //  gather extracted file list
                (tempDir, callback) => {
                    fs.readdir(tempDir, (err, files) => {
                        if (err) {
                            return callback(err);
                        }

                        //  Discover basic information about well known files
                        async.reduce(
                            files,
                            {},
                            (out, filename, next) => {
                                const key = filename.toUpperCase();

                                switch (key) {
                                    case 'MESSAGES.DAT' :   //  QWK
                                        if (this.options.mode === QWKPacketReader.Modes.Guess) {
                                            this.options.mode = QWKPacketReader.Modes.QWK;
                                        }
                                        if (this.options.mode === QWKPacketReader.Modes.QWK) {
                                            out.messages = { filename };
                                        }
                                        break;

                                    case 'ID.MSG' :
                                        if (this.options.mode === QWKPacketReader.Modes.Guess) {
                                            this.options.mode = Modes.REP;
                                        }

                                        if (this.options.mode === QWKPacketReader.Modes.REP) {
                                            out.messages = { filename };
                                        }
                                        break;

                                    case 'HEADERS.DAT' :    //  Synchronet
                                        out.headers = { filename };
                                        break;

                                    case 'VOTING.DAT' : //  Synchronet
                                        out.voting = { filename };
                                        break;

                                    case 'CONTROL.DAT' :    //  QWK
                                        out.control = { filename };
                                        break;

                                    case 'DOOR.ID' :    //  QWK
                                        out.door = { filename };
                                        break;

                                    case 'NETFLAGS.DAT' :   //  QWK
                                        out.netflags = { filename };
                                        break;

                                    case 'NEWFILES.DAT' :   //  QWK
                                        out.newfiles = { filename };
                                        break;

                                    case 'PERSONAL.NDX' : //    QWK
                                        out.personal = { filename };
                                        break;

                                    case '000.NDX' : // QWK
                                        out.inbox = { filename };
                                        break;

                                    case 'TOREADER.EXT' :   //  QWKE
                                        out.toreader = { filename };
                                        break;

                                    case 'QLR.DAT' :
                                        out.qlr = { filename };
                                        break;

                                    default :
                                        if (/[0-9]+\.NDX/.test(key)) {  //  QWK
                                            out.pointers = out.pointers || { filenames: [] };
                                            out.pointers.filenames.push(filename);
                                        } else {
                                            out[key] = { filename };
                                        }
                                        break;
                                }

                                return next(null, out);
                            },
                            (err, packetFileInfo) => {
                                this.packetInfo = Object.assign(
                                    {},
                                    packetFileInfo,
                                    {
                                        tempDir,
                                    }
                                );
                                return callback(null);
                            }
                        );
                    });
                },
                (callback) => {
                    return this.processPacketFiles(callback);
                },
            ],
            err => {
                this.temptmp.cleanup();

                if (err) {
                    return this.emit('error', err);
                }

                this.emit('done');
            }
        );
    }

    processPacketFiles(cb) {
        async.series(
            [
                (callback) => {
                    return this.readControl(callback);
                },
                (callback) => {
                    return this.readHeadersExtension(callback);
                },
                (callback) => {
                    return this.readMessages(callback);
                }
            ],
            err => {
                return cb(err);
            }
        )
    }

    readControl(cb) {
        //
        //  CONTROL.DAT is a CRLF text file containing information about
        //  the originating BBS, conf number <> name mapping, etc.
        //
        //  References:
        //  -   http://fileformats.archiveteam.org/wiki/QWK
        //
        if (!this.packetInfo.control) {
            return cb(Errors.DoesNotExist('No control file found within QWK packet'));
        }

        const path = paths.join(this.packetInfo.tempDir, this.packetInfo.control.filename);

        //  note that we read as UTF-8. Legacy says it should be CP437/ASCII
        //  but this seems safer for now so conference names and the like
        //  can be non-English for example.
        fs.readFile(path, { encoding : 'utf8' }, (err, controlLines) => {
            if (err) {
                return cb(err);
            }

            controlLines = splitTextAtTerms(controlLines);

            let state = 'header';
            const control = { confMap : {} };
            let currConfNumber;
            for (let lineNumber = 0; lineNumber < controlLines.length; ++lineNumber) {
                const line = controlLines[lineNumber].trim();
                switch (lineNumber) {
                    //  first set of lines is header info
                    case 0  : control.bbsName = line; break;
                    case 1  : control.bbsLocation = line; break;
                    case 2  : control.bbsPhone = line; break;
                    case 3  : control.bbsSysOp = line; break;
                    case 4  : control.doorRegAndBoardID = line; break;
                    case 5  : control.packetCreationTime = line; break;
                    case 6  : control.toUser = line; break;
                    case 7  : break; //  Qmail menu
                    case 8  : break; //  unknown, always 0?
                    case 9  : break; //  total messages in packet (often set to 0)
                    case 10 :
                        control.totalMessages = (parseInt(line) + 1);
                        state = 'confNumber';
                        break;

                    default :
                        switch (state) {
                            case 'confNumber' :
                                currConfNumber = parseInt(line);
                                if (isNaN(currConfNumber)) {
                                    state = 'news';

                                    control.welcomeFile = line;
                                } else {
                                    state = 'confName';
                                }
                                break;

                            case 'confName' :
                                control.confMap[currConfNumber] = line;
                                state = 'confNumber';
                                break;

                            case 'news' :
                                control.newsFile = line;
                                state = 'logoff';
                                break;

                            case 'logoff' :
                                control.logoffFile = line;
                                state = 'footer';
                                break;

                            case 'footer' :
                                //  some systems append additional info; we don't care.
                                break;
                        }
                }
            }

             return cb(null);
        });
    }

    readHeadersExtension(cb) {
        if (!this.packetInfo.headers) {
            return cb(null);    //  nothing to do
        }

        const path = paths.join(this.packetInfo.tempDir, this.packetInfo.headers.filename);
        fs.readFile(path, { encoding : 'utf8' }, (err, iniData) => {
            if (err) {
                this.emit('warning', Errors.Invalid(`Problem reading HEADERS.DAT: ${err.message}`));
                return cb(null);    //  non-fatal
            }

            try {
                const parserOptions = {
                    lineComment : false,    //  no line comments; consume full lines
                    nativeType  : false,    //  just keep everything as strings
                    dotKey      : false,    //  'a.b.c = value' stays 'a.b.c = value'
                };
                this.packetInfo.headers.ini = IniConfigParser.parse(iniData, parserOptions);
            } catch (e) {
                this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid: ${e.message}`));
            }

            return cb(null);
        });
    }

    readMessages(cb) {
        if (!this.packetInfo.messages) {
            return cb(Errors.DoesNotExist('No messages file found within QWK packet'));
        }

        const encodingToSpec = 'cp437';
        let encoding = encodingToSpec;

        const path = paths.join(this.packetInfo.tempDir, this.packetInfo.messages.filename);
        fs.open(path, 'r', (err, fd) => {
            if (err) {
                return cb(err);
            }

            //  Some mappings/etc. used in loops below....
            //  Sync sets these in HEADERS.DAT: http://wiki.synchro.net/ref:qwk
            const FTNPropertyMapping = {
                'X-FTN-AREA'    : Message.FtnPropertyNames.FtnArea,
                'X-FTN-SEEN-BY' : Message.FtnPropertyNames.FtnSeenBy,
            };

            const FTNKludgeMapping = {
                'X-FTN-PATH'    : 'PATH',
                'X-FTN-MSGID'   : 'MSGID',
                'X-FTN-REPLY'   : 'REPLY',
                'X-FTN-PID'     : 'PID',
                'X-FTN-FLAGS'   : 'FLAGS',
                'X-FTN-TID'     : 'TID',
                'X-FTN-CHRS'    : 'CHRS',
                //  :TODO: X-FTN-KLUDGE - not sure what this is?
            };

            //
            //  Various kludge tags defined by QWKE, etc.
            //  See the following:
            //  - ftp://vert.synchro.net/main/BBS/qwke.txt
            //  - http://wiki.synchro.net/ref:qwk
            //
            const Kludges = {
                //  QWKE
                To      : 'To:',
                From    : 'From:',
                Subject : 'Subject:',

                //  Synchronet
                Via     : '@VIA:',
                MsgID   : '@MSGID:',
                Reply   : '@REPLY:',
                TZ      : '@TZ:',       //  https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h
                ReplyTo : '@REPLYTO:',

                //  :TODO: Look into other non-standards
                //  https://github.com/wmcbrine/MultiMail/blob/master/mmail/qwk.cc
                //  title, @subject, etc.
            };

            let blockCount = 0;
            let currMessage = { };
            let state;
            let messageBlocksRemain;
            const buffer = Buffer.alloc(QWKMessageBlockSize);

            const readNextBlock = () => {
                fs.read(fd, buffer, 0, QWKMessageBlockSize, null, (err, read) => {
                    if (err) {
                        return cb(err);
                    }

                    if (0 == read) {
                        //  we're done consuming all blocks
                        return fs.close(fd, err => {
                            return cb(err);
                        });
                    }

                    if (QWKMessageBlockSize !== read) {
                        return cb(Errors.Invalid(`Invalid QWK message block size. Expected ${QWKMessageBlockSize} got ${read}`));
                    }

                    if (0 === blockCount) {
                        //  first 128 bytes is a space padded ID
                        const id = buffer.toString('ascii').trim();
                        this.emit('creator', id);
                        state = 'header';
                    } else {
                        switch (state) {
                            case 'header' :
                                const header = MessageHeaderParser.parse(buffer);

                                //  massage into something a little more sane (things we can't quite do in the parser directly)
                                ['toName', 'fromName', 'subject'].forEach(field => {
                                    //  note: always use to-spec encoding here
                                    header[field] = iconv.decode(header[field], encodingToSpec).trim();
                                });

                                header.timestamp = moment(header.timestamp, QWKHeaderTimestampFormat);

                                currMessage = {
                                    header,
                                    //  these may be overridden
                                    toName      : header.toName,
                                    fromName    : header.fromName,
                                    subject     : header.subject,
                                };

                                if (_.has(this.packetInfo, 'headers.ini')) {
                                    //  Sections for a message in HEADERS.DAT are by current byte offset.
                                    //  128 = first message header = 0x80 = section [80]
                                    const headersSectionId = (blockCount * QWKMessageBlockSize).toString(16);
                                    currMessage.headersExtension = this.packetInfo.headers.ini[headersSectionId];
                                }

                                //  if we have HEADERS.DAT with a 'Utf8' override for this message,
                                //  the overridden to/from/subject/message fields are UTF-8
                                if (currMessage.headersExtension && currMessage.headersExtension.Utf8) {
                                    encoding = 'utf8';
                                }

                                //  remainder of blocks until the end of this message
                                messageBlocksRemain = header.numBlocks - 1;
                                state = 'message';
                                break;

                            case 'message' :
                                if (!currMessage.body) {
                                    currMessage.body = Buffer.from(buffer);
                                } else {
                                    currMessage.body = Buffer.concat([currMessage.body, buffer]);
                                }
                                messageBlocksRemain -= 1;

                                if (0 === messageBlocksRemain) {
                                    //  1:n buffers to make up body. Decode:
                                    //  First, replace QWK style line feeds (0xe3) unless the message is UTF-8.
                                    //  If the message is UTF-8, we assume it's using standard line feeds.
                                    if (encoding !== 'utf8') {
                                        replaceCharInBuffer(currMessage.body, QWKLF, 0x0a);
                                    }

                                    //
                                    //  Decode the message based on our final message encoding. Split the message
                                    //  into lines so we can extract various bits such as QWKE headers, origin, tear
                                    //  lines, etc.
                                    //
                                    const messageLines = splitTextAtTerms(iconv.decode(currMessage.body, encoding).trimEnd());
                                    const bodyLines = [];

                                    let bodyState = 'kludge';

                                    const MessageTrailers = {
                                        //  While technically FTN oriented, these can come from any network
                                        //  (though we'll be processing a lot of messages that routed through FTN
                                        //  at some point)
                                        Origin  : /^[ ]{1,2}\* Origin: /,
                                        Tear    : /^--- /,
                                    };

                                    const qwkKludge = {};
                                    const ftnProperty = {};
                                    const ftnKludge = {};

                                    messageLines.forEach(line => {
                                        if (0 === line.length) {
                                            return bodyLines.push('');
                                        }

                                        switch (bodyState) {
                                            case 'kludge' :
                                                //  :TODO: Update these to use the well known consts:
                                                if (line.startsWith(Kludges.To)) {
                                                    currMessage.toName = line.substring(Kludges.To.length).trim();
                                                } else if (line.startsWith(Kludges.From)) {
                                                    currMessage.fromName = line.substring(Kludges.From.length).trim();
                                                } else if (line.startsWith(Kludges.Subject)) {
                                                    currMessage.subject = line.substring(Kludges.Subject.length).trim();
                                                } else if (line.startsWith(Kludges.Via)) {
                                                    qwkKludge['@VIA'] = line;
                                                } else if (line.startsWith(Kludges.MsgID)) {
                                                    qwkKludge['@MSGID'] = line.substring(Kludges.MsgID.length).trim();
                                                } else if (line.startsWith(Kludges.Reply)) {
                                                    qwkKludge['@REPLY'] = line.substring(Kludges.Reply.length).trim();
                                                } else if (line.startsWith(Kludges.TZ)) {
                                                    qwkKludge['@TZ'] = line.substring(Kludges.TZ.length).trim();
                                                } else if (line.startsWith(Kludges.ReplyTo)) {
                                                    qwkKludge['@REPLYTO'] = line.substring(Kludges.ReplyTo.length).trim();
                                                } else {
                                                    bodyState = 'body'; // past this point and up to any tear/origin/etc., is the real message body
                                                    bodyLines.push(line);
                                                }
                                                break;

                                            case 'body' :
                                            case 'trailers' :
                                                if (MessageTrailers.Origin.test(line)) {
                                                    ftnProperty.ftn_origin = line;
                                                    bodyState = 'trailers';
                                                } else if (MessageTrailers.Tear.test(line)) {
                                                    ftnProperty.ftn_tear_line = line;
                                                    bodyState = 'trailers';
                                                } else if ('body' === bodyState) {
                                                    bodyLines.push(line);
                                                }
                                        }
                                    });

                                    let messageTimestamp = currMessage.header.timestamp;

                                    //  HEADERS.DAT support.
                                    let useTZKludge = true;
                                    if (currMessage.headersExtension) {
                                        const ext = currMessage.headersExtension;

                                        //  to and subject can be overridden yet again if entries are present
                                        currMessage.toName  = ext.To || currMessage.toName;
                                        currMessage.subject = ext.Subject || currMessage.subject;
                                        currMessage.from    = ext.Sender || currMessage.fromName;   //  why not From? Who the fuck knows.

                                        //  possibly override message ID kludge
                                        qwkKludge['@MSGID'] = ext['Message-ID'] || qwkKludge['@MSGID'];

                                        //  WhenWritten contains a ISO-8601-ish timestamp and a Synchronet/SMB style TZ offset:
                                        //  20180101174837-0600  4168
                                        //  We can use this to get a very slightly better precision on the timestamp (addition of seconds)
                                        //  over the headers value. Why not milliseconds? Who the fuck knows.
                                        if (ext.WhenWritten) {
                                            const whenWritten = moment(ext.WhenWritten, 'YYYYMMDDHHmmssZ');
                                            if (whenWritten.isValid()) {
                                                messageTimestamp = whenWritten;
                                                useTZKludge = false;
                                            }
                                        }

                                        if (ext.Tags) {
                                            currMessage.hashTags = (ext.Tags).toString().split(' ');
                                        }

                                        //  FTN style properties/kludges represented as X-FTN-XXXX
                                        for (let [extName, propName] of Object.entries(FTNPropertyMapping)) {
                                            const v = ext[extName];
                                            if (v) {
                                                ftnProperty[propName] = v;
                                            }
                                        }

                                        for (let [extName, kludgeName] of Object.entries(FTNKludgeMapping)) {
                                            const v = ext[extName];
                                            if (v) {
                                                ftnKludge[kludgeName] = v;
                                            }
                                        }
                                    }

                                    const message = new Message({
                                        toUserName      : currMessage.toName,
                                        fromUserName    : currMessage.fromName,
                                        subject         : currMessage.subject,
                                        modTimestamp    : messageTimestamp,
                                        message         : bodyLines.join('\n'),
                                        hashTags        : currMessage.hashTags,
                                    });

                                    //  Indicate this message was imported from a QWK packet
                                    message.meta.System[Message.SystemMetaNames.ExternalFlavor] = Message.AddressFlavor.QWK;

                                    if (!_.isEmpty(qwkKludge)) {
                                        message.meta.QwkKludge = qwkKludge;
                                    }

                                    if (!_.isEmpty(ftnProperty)) {
                                        message.meta.FtnProperty = ftnProperty;
                                    }

                                    if (!_.isEmpty(ftnKludge)) {
                                        message.meta.FtnKludge = ftnKludge;
                                    }

                                    //  Add in tear line and origin if requested
                                    if (this.options.keepTearAndOrigin) {
                                        if (ftnProperty.ftn_tear_line) {
                                            message.message += `\r\n${ftnProperty.ftn_tear_line}\r\n`;
                                        }

                                        if (ftnProperty.ftn_origin) {
                                            message.message += `${ftnProperty.ftn_origin}\r\n`;
                                        }
                                    }

                                    //  Update the timestamp if we have a valid TZ
                                    if (useTZKludge && qwkKludge['@TZ']) {
                                        const tzOffset = SMBTZToUTCOffset[qwkKludge['@TZ']];
                                        if (tzOffset) {
                                            message.modTimestamp.utcOffset(tzOffset);
                                        }
                                    }

                                    message.meta.QwkProperty = {
                                        qwk_msg_status          : currMessage.header.status,
                                        qwk_in_reply_to_num     : currMessage.header.replyToNum,
                                    };

                                    if (this.options.mode === QWKPacketReader.Modes.QWK) {
                                        message.meta.QwkProperty.qwk_msg_num = currMessage.header.num;
                                        message.meta.QwkProperty.qwk_conf_num = currMessage.header.confNum;
                                    } else {
                                        //  For REP's, prefer the larger field.
                                        message.meta.QwkProperty.qwk_conf_num = currMessage.header.num || currMessage.header.confNum;
                                    }

                                    //  Another quick HEADERS.DAT fix-up
                                    if (currMessage.headersExtension) {
                                        message.meta.QwkProperty.qwk_conf_num = currMessage.headersExtension.Conference || message.meta.QwkProperty.qwk_conf_num;
                                    }

                                    this.emit('message', message);
                                    state = 'header';
                                }
                                break;
                        }
                    }

                    ++blockCount;
                    readNextBlock();
                });
            };

            //  start reading blocks
            readNextBlock();
        });
    }
};

class QWKPacketWriter extends EventEmitter {
    constructor(
        {
            enableQWKE = true,
            enableHeadersExtension = true,
            enableAtKludges = true,
            systemDomain = 'enigma-bbs',
            bbsID = '',
            user = null,
            archiveFormat = 'application/zip',
            forceEncoding = null,
        } = QWKPacketWriter.DefaultOptions)
    {
        super();

        this.options = {
            enableQWKE,
            enableHeadersExtension,
            enableAtKludges,
            systemDomain,
            bbsID,
            user,
            archiveFormat,
            forceEncoding : forceEncoding ? forceEncoding.toLowerCase() : null,
        };

        this.temptmp = temptmp.createTrackedSession('qwkpacketwriter');
    }

    static get DefaultOptions() {
        return {
            enableQWKE              : true,
            enableHeadersExtension  : true,
            enableAtKludges         : true,
            systemDomain            : 'enigma-bbs',
            bbsID                   : '',
            user                    : null,
            archiveFormat           :'application/zip',
            forceEncoding           : null,
        };
    }

    init() {
        async.series(
            [
                (callback) => {
                    return StatLog.init(callback);
                },
                (callback) => {
                    this.temptmp.mkdir( { prefix : 'enigqwkwriter-'}, (err, workDir) => {
                        this.workDir = workDir;
                        return callback(err);
                    });
                },
                (callback) => {
                    this.messagesStream = fs.createWriteStream(paths.join(this.workDir, 'messages.dat'));

                    if (this.options.enableHeadersExtension) {
                        this.headersDatStream = fs.createWriteStream(paths.join(this.workDir, 'headers.dat'));
                    }

                    //  First block is a space padded ID
                    const id = `Created with ENiGMA 1/2 BBS v${enigmaVersion} Copyright (c) 2015-2020 Bryan Ashby`;
                    this.messagesStream.write(id.padEnd(QWKMessageBlockSize, ' '), 'ascii');
                    this.currentMessageOffset = QWKMessageBlockSize;

                    this.totalMessages = 0;
                    this.areaTagsSeen = new Set();
                    this.personalIndex = [];    //  messages addressed to 'user'
                    this.inboxIndex = [];       //  private messages for 'user'
                    this.publicIndex = new Map();

                    return callback(null);
                },
            ],
            err => {
                if (err) {
                    return this.emit('error', err);
                }

                this.emit('ready');
            }
        )
    }

    makeMessageIdentifier(message) {
        return `<${message.messageId}.${message.messageUuid}@${this.options.systemDomain}>`;
    }

    appendMessage(message) {
        //
        //  Each message has to:
        //  - Append to MESSAGES.DAT
        //  - Append to HEADERS.DAT if enabled
        //
        //  If this is a personal (ie: non-network) packet:
        //  - Produce PERSONAL.NDX
        //  - Produce 000.NDX with pointers to the users personal "inbox" mail
        //  - Produce ####.NDX with pointers to the public/conference mail
        //  - Produce TOREADER.EXT if QWKE support is enabled
        //

        let fullMessageBody = '';

        //  Start of body is kludges if enabled
        if (this.options.enableQWKE) {
            if (message.toUserName.length > 25) {
                fullMessageBody += `To: ${message.toUserName}\n`;
            }
            if (message.fromUserName.length > 25) {
                fullMessageBody += `From: ${message.fromUserName}\n`;
            }
            if (message.subject.length > 25) {
                fullMessageBody += `Subject: ${message.subject}\n`;
            }
        }

        if (this.options.enableAtKludges) {
            //  Add in original kludges (perhaps in a different order) if
            //  they were originally imported
            if (Message.AddressFlavor.QWK == message.meta.System[Message.SystemMetaNames.ExternalFlavor]) {
                if (message.meta.QwkKludge) {
                    for (let [kludge, value] of Object.entries(message.meta.QwkKludge)) {
                        fullMessageBody += `${kludge}: ${value}\n`;
                    };
                }
            } else {
                fullMessageBody += `@MSGID: ${this.makeMessageIdentifier(message)}\n`;
                fullMessageBody += `@TZ: ${UTCOffsetToSMBTZ[moment().format('Z')]}\n`;
                //  :TODO: REPLY and REPLYTO
            }
        }

        //  Sanitize line feeds (e.g. CRLF -> LF, and possibly -> QWK style below)
        splitTextAtTerms(message.message).forEach(line => {
            fullMessageBody += `${line}\n`;
        });

        const encoding = this._getEncoding(message);

        const encodedMessage = iconv.encode(fullMessageBody, encoding);

        //
        //  QWK spec wants line feeds as 0xe3 for some reason, so we'll have
        //  to replace the \n's. If we're going against the spec and using UTF-8
        //  we can just leave them be.
        //
        if ('utf8' !== encoding) {
            replaceCharInBuffer(encodedMessage, 0x0a, QWKLF);
        }

        //  Messages must comprise of multiples of 128 bit blocks with the last
        //  block padded by spaces or nulls (we use nulls)
        const fullBlocks    = Math.trunc(encodedMessage.length / QWKMessageBlockSize);
        const remainBytes   = QWKMessageBlockSize - (encodedMessage.length % QWKMessageBlockSize);
        const totalBlocks   = fullBlocks + 1 + (remainBytes ? 1 : 0);

        //  The first block is always a header
        if (!this._writeMessageHeader(
            message,
            totalBlocks
        ))
        {
            //  we can't write this message
            return;
        }

        this.messagesStream.write(encodedMessage);

        if (remainBytes) {
            this.messagesStream.write(Buffer.alloc(remainBytes, ' '));
        }

        this._updateIndexTracking(message);

        if (this.options.enableHeadersExtension) {
            this._appendHeadersExtensionData(message, encoding);
        }

        //  next message starts at this block
        this.currentMessageOffset += totalBlocks * QWKMessageBlockSize;

        this.totalMessages += 1;
        this.areaTagsSeen.add(message.areaTag);
    }

    _getEncoding(message) {
        if (this.options.forceEncoding) {
            return this.options.forceEncoding;
        }

        //  If the system has stored an explicit encoding, use that.
        let encoding = _.get(message.meta, 'System.explicit_encoding');
        if (encoding) {
            return encoding;
        }

        //  If the message is already tagged with a supported encoding
        //  indicator such as FTN-style CHRS, try to use that.
        encoding = _.get(message.meta, 'FtnKludge.CHRS');
        if (encoding) {
            //  convert from CHRS to something standard
            encoding = getEncodingFromCharacterSetIdentifier(encoding);
            if (encoding) {
                return encoding;
            }
        }

        //  The to-spec default is CP437/ASCII. If it can be encoded as
        //  such then do so.
        if (message.isCP437Encodable()) {
            return 'cp437';
        }

        //  Something more modern...
        return 'utf8';
    }

    _messageAddressedToUser(message) {
        if (_.isUndefined(this.cachedCompareNames)) {
            if (this.options.user) {
                this.cachedCompareNames = [
                    this.options.user.username.toLowerCase()
                ];
                const realName = this.options.user.getProperty(UserProps.RealName);
                if (realName) {
                    this.cachedCompareNames.push(realName.toLowerCase());
                }
            } else {
                this.cachedCompareNames = [];
            }
        };

        return this.cachedCompareNames.includes(message.toUserName.toLowerCase());
    }

    _updateIndexTracking(message) {
        //  index points at start of *message* not the header for... reasons?
        const index = (this.currentMessageOffset / QWKMessageBlockSize) + 1;
        if (message.isPrivate()) {
            this.inboxIndex.push(index);
        } else {
            if (this._messageAddressedToUser(message)) {
                //  :TODO: add to both indexes???
                this.personalIndex.push(index);
            }

            const areaTag = message.areaTag;
            if (!this.publicIndex.has(areaTag)) {
                this.publicIndex.set(areaTag, [index]);
            } else {
                this.publicIndex.get(areaTag).push(index);
            }
        }
    }

    appendNewFile() {

    }

    finish(packetDirectory) {
        async.series(
            [
                (callback) => {
                    this.messagesStream.on('close', () => {
                        return callback(null);
                    });
                    this.messagesStream.end();
                },
                (callback) => {
                    if (!this.headersDatStream) {
                        return callback(null);
                    }
                    this.headersDatStream.on('close', () => {
                        return callback(null);
                    });
                    this.headersDatStream.end();
                },
                (callback) => {
                    return this._createControlData(callback);
                },
                (callback) => {
                    return this._createIndexes(callback);
                },
                (callback) => {
                    return this._producePacketArchive(packetDirectory, callback);
                }
            ],
            err => {
                this.temptmp.cleanup();

                if (err) {
                    return this.emit('error', err);
                }

                this.emit('finished');
            }
        )
    }

    _getNextAvailPacketFileName(packetDirectory, cb) {
        //
        //  According to http://wiki.synchro.net/ref:qwk filenames should
        //  start with .QWK -> .QW1 ... .QW9 -> .Q10 ... .Q99
        //
        let digits = 0;
        async.doWhilst( callback => {
            let ext;
            if (0 === digits) {
                ext = 'QWK';
            } else if (digits < 10) {
                ext = `QW${digits}`;
            } else if (digits < 100) {
                ext = `Q${digits}`;
            } else {
                return callback(Errors.UnexpectedState(`Unable to choose a valid QWK output filename`));
            }

            ++digits;

            const filename = `${this.options.bbsID}.${ext}`;
            fs.stat(paths.join(packetDirectory, filename), (err, stats) => {
                if (err && 'ENOENT' === err.code) {
                    return callback(null, filename);
                } else {
                    return callback(null, null);
                }
            });
        },
        (filename, callback) => {
            return callback(null, filename ? false : true);
        },
        (err, filename) => {
            return cb(err, filename);
        });
    }

    _producePacketArchive(packetDirectory, cb) {
        const archiveUtil = ArchiveUtil.getInstance();

        fs.readdir(this.workDir, (err, files) => {
            if (err) {
                return cb(err);
            }

            this._getNextAvailPacketFileName(packetDirectory, (err, filename) => {
                if (err) {
                    return cb(err);
                }

                const packetPath = paths.join(packetDirectory, filename);
                archiveUtil.compressTo(
                    this.options.archiveFormat,
                    packetPath,
                    files,
                    this.workDir,
                    err => {
                        return cb(err);
                    }
                );
            });
        });
    }

    _qwkMessageStatus(message) {
        //  - Public vs Private
        //  - Look at message pointers for read status
        //  - If +op is exporting and this message is to +op
        //  -
        //  :TODO: this needs addressed - handle unread vs read, +op, etc.
        //  ....see getNewMessagesInAreaForUser(); Variant with just IDs, or just a way to get first new message ID per area?

        if (message.isPrivate()) {
            return QWKMessageStatusCodes.UnreadPrivate;
        }
        return QWKMessageStatusCodes.UnreadPublic;
    }

    _writeMessageHeader(message, totalBlocks) {
        const asciiNum = (n, l) => {
            if (isNaN(n)) {
                return '';
            }
            return n.toString().substr(0, l);
        };

        const asciiTotalBlocks = asciiNum(totalBlocks, 6);
        if (asciiTotalBlocks.length > 6) {
            this.emit('warning', Errors.General('Message too large for packet'), message);
            return false;
        }

        const conferenceNumber = this._getMessageConferenceNumberByAreaTag(message.areaTag);
        if (isNaN(conferenceNumber)) {
            this.emit('warning', Errors.MissingConfig(`No QWK conference mapping for areaTag ${message.areaTag}`));
            return false;
        }

        const header = Buffer.alloc(QWKMessageBlockSize, ' ');
        header.write(this._qwkMessageStatus(message), 0, 1, 'ascii');
        header.write(asciiNum(message.messageId), 1, 'ascii');
        header.write(message.modTimestamp.format(QWKHeaderTimestampFormat), 8, 13, 'ascii');
        header.write(message.toUserName.substr(0, 25), 21, 'ascii');
        header.write(message.fromUserName.substr(0, 25), 46, 'ascii');
        header.write(message.subject.substr(0, 25), 71, 'ascii');
        header.write(' '.repeat(12), 96, 'ascii');  //  we don't use the password field
        header.write(asciiNum(message.replyToMsgId), 108, 'ascii');
        header.write(asciiTotalBlocks, 116, 'ascii');
        header.writeUInt8(QWKMessageActiveStatus.Active, 122);
        header.writeUInt16LE(conferenceNumber, 123);
        header.writeUInt16LE(this.totalMessages + 1, 125);
        header.write(QWKNetworkTagIndicator.NotPresent, 127, 1, 'ascii');   //  :TODO: Present if for network output?

        this.messagesStream.write(header);

        return true;
    }

    _getMessageConferenceNumberByAreaTag(areaTag) {
        if (Message.isPrivateAreaTag(areaTag)) {
            return 0;
        }
        const areaConfig = _.get(Config(), [ 'messageNetworks', 'qwk', 'areas', areaTag ]);
        return areaConfig && areaConfig.conference;
    }

    _getExportForUsername() {
        return _.get(this.options, 'user.username', 'Any');
    }

    _getExportSysOpUsername() {
        return StatLog.getSystemStat(SysProps.SysOpUsername) || 'SysOp';
    }

    _createControlData(cb) {
        const areas = Array.from(this.areaTagsSeen).map(areaTag => {
            if (Message.isPrivateAreaTag(areaTag)) {
                return {
                    areaTag : Message.WellKnownAreaTags.Private,
                    name    : 'Private',
                    desc    : 'Private Messages',
                };
            }
            return getMessageAreaByTag(areaTag);
        });

        const controlStream = fs.createWriteStream(paths.join(this.workDir, 'control.dat'));
        controlStream.setDefaultEncoding('ascii');

        controlStream.on('close', () => {
            return cb(null);
        });

        controlStream.on('error', err => {
            return cb(err);
        });

        const initialControlData = [
            Config().general.boardName,
            'Earth',
            'XXX-XXX-XXX',
            `${this._getExportSysOpUsername()}, Sysop`,
            `0000,${this.options.bbsID}`,
            moment().format('MM-DD-YYYY,HH:mm:ss'),
            this._getExportForUsername(),
            '',     // name of Qmail menu
            '0',    // uh, OK
            this.totalMessages.toString(),
            //  this next line is total conferences - 1:
            //  We have areaTag <> conference mapping, so the number should work out
            (this.areaTagsSeen.size - 1).toString(),
        ];

        initialControlData.forEach(line => {
            controlStream.write(`${line}\r\n`);
        });

        //  map areas as conf #\r\nDescription\r\n pairs
        areas.forEach(area => {
            const conferenceNumber = this._getMessageConferenceNumberByAreaTag(area.areaTag);
            let desc = area.name;
            if (area.desc) {
                desc += ` - ${area.desc}`
            }
            controlStream.write(`${conferenceNumber}\r\n`);
            controlStream.write(`${desc}\r\n`);
        });

        //  :TODO: do we ever care here?!
        ['HELLO', 'BBSNEWS', 'GOODBYE'].forEach(trailer => {
            controlStream.write(`${trailer}\r\n`);
        });

        controlStream.end();
    }

    _createIndexes(cb) {
        const appendIndexData = (stream, offset) => {
            const msb = numToMbf32(offset);
            stream.write(msb);

            //  technically, the conference #, but only as a byte, so pretty much useless
            //  AND the filename itself is the conference number... dafuq.
            stream.write(Buffer.from([0x00]));
        };

        async.series(
            [
                (callback) => {
                    //  Create PERSONAL.NDX
                    if (!this.personalIndex.length) {
                        return callback(null);
                    }

                    const indexStream = fs.createWriteStream(paths.join(this.workDir, 'personal.ndx'));
                    this.personalIndex.forEach(offset => appendIndexData(indexStream, offset));

                    indexStream.on('close', err => {
                        return callback(err);
                    });

                    indexStream.end();
                },
                (callback) => {
                    //  000.NDX of private mails
                    if (!this.inboxIndex.length) {
                        return callback(null);
                    }

                    const indexStream = fs.createWriteStream(paths.join(this.workDir, '000.ndx'));
                    this.inboxIndex.forEach(offset => appendIndexData(indexStream, offset));

                    indexStream.on('close', err => {
                        return callback(err);
                    });

                    indexStream.end();
                },
                (callback) => {
                    //  ####.NDX
                    async.eachSeries(this.publicIndex.keys(), (areaTag, nextArea) => {
                        const offsets = this.publicIndex.get(areaTag);
                        const conferenceNumber = this._getMessageConferenceNumberByAreaTag(areaTag);
                        const indexStream = fs.createWriteStream(paths.join(this.workDir, `${conferenceNumber.toString()}.ndx`));
                        offsets.forEach(offset => appendIndexData(indexStream, offset));

                        indexStream.on('close', err => {
                            return nextArea(err);
                        });

                        indexStream.end();
                    },
                    err => {
                        return callback(err);
                    });
                }
            ],
            err => {
                return cb(err);
            }
        );
    }

    _makeSynchronetTimestamp(ts) {
        const syncTimestamp = ts.format('YYYYMMDDHHmmssZZ');
        const syncTZ        = UTCOffsetToSMBTZ[ts.format('Z')] || '0000';   //  :TODO: what if we don't have a map?
        return `${syncTimestamp} ${syncTZ}`;
    }

    _appendHeadersExtensionData(message, encoding) {
        const messageData = {
            //  Synchronet style
            Utf8            : ('utf8' === encoding ? 'true' : 'false'),
            'Message-ID'    : this.makeMessageIdentifier(message),

            WhenWritten     : this._makeSynchronetTimestamp(message.modTimestamp),
            // WhenImported    : '',   //  :TODO: only if we have a imported time from another external system?
            ExportedFrom    : `${this.options.systemID} ${message.areaTag} ${message.messageId}`,
            Sender          : message.fromUserName,

            //  :TODO: if exporting for QWK-Net style/etc.
            //SenderNetAddr

            SenderIpAddr    : '127.0.0.1',  //  no sir, that's private.
            SenderHostName  : this.options.systemDomain,
            //  :TODO: if exported:
            //SenderProtocol
            Organization    : 'BBS',

            //'Reply-To'      : :TODO: "address to direct replies".... ?!
            Subject         : message.subject,
            To              : message.toUserName,
            //ToNetAddr     : :TODO: net addr to?!

            //  :TODO: Only set if not imported:
            Tags            : message.hashTags.join(' '),

            //  :TODO: Needs tested with Sync/etc.; Sync wants Conference *numbers*
            Conference      : message.isPrivate() ? '0' : getMessageConfTagByAreaTag(message.areaTag),

            //  ENiGMA Headers
            MessageUUID     : message.uuid,
            ModTimestamp    : message.modTimestamp.format('YYYY-MM-DDTHH:mm:ss.SSSZ'),
            AreaTag         : message.areaTag,
        };

        const externalFlavor = message.meta.System[Message.SystemMetaNames.ExternalFlavor];
        if (externalFlavor === Message.AddressFlavor.FTN) {
            //  Add FTN properties if it came from such an origin
            if (message.meta.FtnProperty) {
                const ftnProp = message.meta.FtnProperty;
                messageData['X-FTN-AREA']       = ftnProp[Message.FtnPropertyNames.FtnArea];
                messageData['X-FTN-SEEN-BY']    = ftnProp[Message.FtnPropertyNames.FtnSeenBy];
            }

            if (message.meta.FtnKludge) {
                const ftnKludge = message.meta.FtnKludge;
                messageData['X-FTN-PATH']   = ftnKludge.PATH;
                messageData['X-FTN-MSGID']  = ftnKludge.MSGID;
                messageData['X-FTN-REPLY']  = ftnKludge.REPLY;
                messageData['X-FTN-PID']    = ftnKludge.PID;
                messageData['X-FTN-FLAGS']  = ftnKludge.FLAGS;
                messageData['X-FTN-TID']    = ftnKludge.TID;
                messageData['X-FTN-CHRS']   = ftnKludge.CHRS;
            }
        } else {
            messageData.WhenExported    = this._makeSynchronetTimestamp(moment());
            messageData.Editor          = `ENiGMA 1/2 BBS FSE v${enigmaVersion}`;
        }

        this.headersDatStream.write(iconv.encode(`[${this.currentMessageOffset.toString(16)}]\r\n`, encoding));

        for (let [name, value] of Object.entries(messageData)) {
            if (value) {
                this.headersDatStream.write(iconv.encode(`${name}: ${value}\r\n`, encoding));
            }
        }

        this.headersDatStream.write('\r\n');
    }
}

module.exports = {
    QWKPacketReader,
    QWKPacketWriter,
}