From 3cd6d59e12b131c3d20d49b156e6eae7522c5dfb Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 19 Nov 2019 20:37:46 -0700 Subject: [PATCH 01/95] First 0.0.11-beta commit - create the branch and prep work --- UPGRADE.md | 2 ++ WHATSNEW.md | 3 +++ docs/art/mci.md | 32 ++++++++++++++++---------------- package.json | 2 +- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/UPGRADE.md b/UPGRADE.md index bd0a47b0..7f1fa96d 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -40,6 +40,8 @@ npm install Report your issue on Xibalba BBS, hop in #enigma-bbs on FreeNode and chat, or [file a issue on GitHub](https://github.com/NuSkooler/enigma-bbs/issues). +# 0.0.10-alpha to 0.0.11-beta + # 0.0.9-alpha to 0.0.10-alpha * Security related files such as private keys and certs are now looked for in `config/security` by default. * Default archive handler for zip files has switched to InfoZip due to a bug in the latest p7Zip packages causing "volume not found" errors. Ensure you have the InfoZip `zip` and `unzip` commands in ENiGMA's path. You can switch back to 7Zip by overriding `archiveHandler` for `application/zip` in your `config.hjson` under `fileTypes` to `7Zip`. diff --git a/WHATSNEW.md b/WHATSNEW.md index bc8cc9f8..9668431d 100644 --- a/WHATSNEW.md +++ b/WHATSNEW.md @@ -1,6 +1,9 @@ # Whats New This document attempts to track **major** changes and additions in ENiGMA½. For details, see GitHub. +## 0.0.11-beta +* Upgraded from `alpha` to `beta` -- The software is far along and mature enough at this point! + ## 0.0.10-alpha + `oputil.js user rename USERNAME NEWNAME` + `my_messages.js` module (defaulted to "m" at the message menu) to list public messages addressed to the currently logged in user. Takes into account their username and `real_name` property. diff --git a/docs/art/mci.md b/docs/art/mci.md index 60bb8ae6..42091d98 100644 --- a/docs/art/mci.md +++ b/docs/art/mci.md @@ -2,22 +2,22 @@ layout: page title: MCI Codes --- -ENiGMA½ supports a variety of MCI codes. Some **predefined** codes produce information about the current user, system, -or other statistics while others are used to instantiate a **View**. MCI codes are two characters in length and are -prefixed with a percent (%) symbol. Some MCI codes have additional options that may be set directly from the code itself -while others -- and more advanced options -- are controlled via the current theme. Standard (non-focus) and focus colors +ENiGMA½ supports a variety of MCI codes. Some **predefined** codes produce information about the current user, system, +or other statistics while others are used to instantiate a **View**. MCI codes are two characters in length and are +prefixed with a percent (%) symbol. Some MCI codes have additional options that may be set directly from the code itself +while others -- and more advanced options -- are controlled via the current theme. Standard (non-focus) and focus colors are set by placing duplicate codes back to back in art files. ## Predefined MCI Codes -There are many predefined MCI codes that can be used anywhere on the system (placed in any art file). More are added all -the time so also check out [core/predefined_mci.js](https://github.com/NuSkooler/enigma-bbs/blob/master/core/mci_view_factory.js) +There are many predefined MCI codes that can be used anywhere on the system (placed in any art file). More are added all +the time so also check out [core/predefined_mci.js](https://github.com/NuSkooler/enigma-bbs/blob/master/core/mci_view_factory.js) for a full listing. Many codes attempt to pay homage to Oblivion/2, iNiQUiTY, etc. | Code | Description | |------|--------------| | `BN` | Board Name | -| `VL` | Version *label*, e.g. "ENiGMA½ v0.0.10-alpha" | -| `VN` | Version *number*, eg.. "0.0.10-alpha" | +| `VL` | Version *label*, e.g. "ENiGMA½ v0.0.11-beta" | +| `VN` | Version *number*, eg.. "0.0.11-beta" | | `SN` | SysOp username | | `SR` | SysOp real name | | `SL` | SysOp location | @@ -75,7 +75,7 @@ for a full listing. Many codes attempt to pay homage to Oblivion/2, iNiQUiTY, et | `SD` | Total downloads, system wide | | `SO` | Total downloaded amount, system wide (formatted to appropriate bytes/megs/etc.) | | `SU` | Total uploads, system wide | -| `SP` | Total uploaded amount, system wide (formatted to appropriate bytes/megs/etc.) | +| `SP` | Total uploaded amount, system wide (formatted to appropriate bytes/megs/etc.) | | `TF` | Total number of files on the system | | `TB` | Total amount of files on the system (formatted to appropriate bytes/megs/gigs/etc.) | | `TP` | Total messages posted/imported to the system *currently* | @@ -93,7 +93,7 @@ Some additional special case codes also exist: ## Views -A **View** is a control placed on a **form** that can display variable data or collect input. One example of a View is +A **View** is a control placed on a **form** that can display variable data or collect input. One example of a View is a Vertical Menu (`%VM`): Old-school BBSers may recognize this as a lightbar menu. | Code | Name | Description | @@ -103,14 +103,14 @@ a Vertical Menu (`%VM`): Old-school BBSers may recognize this as a lightbar menu | `ME` | Masked Edit Text | Collect user input using a *mask* | | `MT` | Multi Line Text Edit | Multi line edit control | | `BT` | Button | A button | -| `VM` | Vertical Menu | A vertical menu aka a vertical lightbar | -| `HM` | Horizontal Menu | A horizontal menu aka a horizontal lightbar | +| `VM` | Vertical Menu | A vertical menu aka a vertical lightbar | +| `HM` | Horizontal Menu | A horizontal menu aka a horizontal lightbar | | `SM` | Spinner Menu | A spinner input control | -| `TM` | Toggle Menu | A toggle menu commonly used for Yes/No style input | +| `TM` | Toggle Menu | A toggle menu commonly used for Yes/No style input | | `KE` | Key Entry | A *single* key input control | -Peek at [/core/mci_view_factory.js](https://github.com/NuSkooler/enigma-bbs/blob/master/core/mci_view_factory.js) to +Peek at [/core/mci_view_factory.js](https://github.com/NuSkooler/enigma-bbs/blob/master/core/mci_view_factory.js) to see additional information. @@ -132,7 +132,7 @@ Predefined MCI codes and other Views can have properties set via `menu.hjson` an | `itemFormat` | Sets the format for a list entry. See **Entry Formatting** below | | `focusItemFormat` | Sets the format for a focused list entry. See **Entry Formatting** below | -These are just a few of the properties set on various views. *Use the source Luke*, as well as taking a look at the default +These are just a few of the properties set on various views. *Use the source Luke*, as well as taking a look at the default `menu.hjson` and `theme.hjson` files! ### Custom Properties @@ -144,7 +144,7 @@ Standard style types available for `textStyle` and `focusTextStyle`: | Style | Description | |----------|--------------| -| `normal` | Leaves text as-is. This is the default. | +| `normal` | Leaves text as-is. This is the default. | | `upper` | ENIGMA BULLETIN BOARD SOFTWARE | | `lower` | enigma bulletin board software | | `title` | Enigma Bulletin Board Software | diff --git a/package.json b/package.json index 0d9b23f0..212dcc6a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "enigma-bbs", - "version": "0.0.10-alpha", + "version": "0.0.11-beta", "description": "ENiGMA½ Bulletin Board System", "author": "Bryan Ashby ", "license": "BSD-2-Clause", From 57358b0f2c89b1cebab703d55aaf75c8a5067d79 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 19 Nov 2019 20:41:09 -0700 Subject: [PATCH 02/95] Bump versions in install.sh for branch --- README.md | 2 +- docs/installation/install-script.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e7bb1204..477935a3 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ ENiGMA has been tested with many terminals. However, the following are suggested ## Installation On *nix type systems: ``` -curl -o- https://raw.githubusercontent.com/NuSkooler/enigma-bbs/master/misc/install.sh | bash +curl -o- https://raw.githubusercontent.com/NuSkooler/enigma-bbs/0.0.11-beta/misc/install.sh | bash ``` Please see [Installation Methods](https://nuskooler.github.io/enigma-bbs/installation/installation-methods.html) for Windows, Docker, and so on... diff --git a/docs/installation/install-script.md b/docs/installation/install-script.md index 2f92db79..564a6973 100644 --- a/docs/installation/install-script.md +++ b/docs/installation/install-script.md @@ -6,10 +6,10 @@ title: Install Script Under most Linux/UNIX like environments (Linux, BSD, OS X, ...) new users can simply execute the `install.sh` script to get everything up and running. Cut + paste the following into your terminal: ``` -curl -o- https://raw.githubusercontent.com/NuSkooler/enigma-bbs/master/misc/install.sh | bash +curl -o- https://raw.githubusercontent.com/NuSkooler/enigma-bbs/0.0.11-beta/misc/install.sh | bash ``` -You may review the [installation script](https://raw.githubusercontent.com/NuSkooler/enigma-bbs/master/misc/install.sh) +You may review the [installation script](https://raw.githubusercontent.com/NuSkooler/enigma-bbs/0.0.11-beta/misc/install.sh) on GitHub before running it. The script will install nvm, Node.js 6 and grab the latest ENiGMA BBS from GitHub. It will also guide you through creating a basic configuration file, and recommend some packages to install. From 998df3386a590d863d1fd6f1d45b30c787c0f540 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 19 Nov 2019 21:32:21 -0700 Subject: [PATCH 03/95] Bump Node.js req to 12+ --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 212dcc6a..8b620932 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,6 @@ }, "devDependencies": {}, "engines": { - "node": ">=8" + "node": ">=12" } } From 0328ad5b8083260c6a90b2198d80cb97e8b46217 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 20 Nov 2019 19:33:33 -0700 Subject: [PATCH 04/95] Node.js 12 LTS to docs & install script --- UPGRADE.md | 1 + WHATSNEW.md | 1 + misc/install.sh | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/UPGRADE.md b/UPGRADE.md index 7f1fa96d..3fcd35a8 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -41,6 +41,7 @@ Report your issue on Xibalba BBS, hop in #enigma-bbs on FreeNode and chat, or [file a issue on GitHub](https://github.com/NuSkooler/enigma-bbs/issues). # 0.0.10-alpha to 0.0.11-beta +* Node.js 12.x LTS is now in use. Follow standard Node.js upgrade procedures (e.g.: `nvm install 12 && nvm use 12`). # 0.0.9-alpha to 0.0.10-alpha * Security related files such as private keys and certs are now looked for in `config/security` by default. diff --git a/WHATSNEW.md b/WHATSNEW.md index 9668431d..9736d82f 100644 --- a/WHATSNEW.md +++ b/WHATSNEW.md @@ -3,6 +3,7 @@ This document attempts to track **major** changes and additions in ENiGMA½. For ## 0.0.11-beta * Upgraded from `alpha` to `beta` -- The software is far along and mature enough at this point! +* Development is now against Node.js 12.x LTS. Other versions may work but are not currently supported! ## 0.0.10-alpha + `oputil.js user rename USERNAME NEWNAME` diff --git a/misc/install.sh b/misc/install.sh index 5da8e0e4..ed81d205 100755 --- a/misc/install.sh +++ b/misc/install.sh @@ -2,7 +2,7 @@ { # this ensures the entire script is downloaded before execution -ENIGMA_NODE_VERSION=${ENIGMA_NODE_VERSION:=10} +ENIGMA_NODE_VERSION=${ENIGMA_NODE_VERSION:=12} ENIGMA_BRANCH=${ENIGMA_BRANCH:=master} ENIGMA_INSTALL_DIR=${ENIGMA_INSTALL_DIR:=$HOME/enigma-bbs} ENIGMA_SOURCE=${ENIGMA_SOURCE:=https://github.com/NuSkooler/enigma-bbs.git} From 37638957b43dd670945d82f01e483e7ad9bbf153 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Fri, 22 Nov 2019 21:44:24 -0700 Subject: [PATCH 05/95] Fix issues with corrupt FTN packets --- core/ftn_mail_packet.js | 40 ++++++++++++++++++++++++++++++++-------- util/dump_ftn_packet.js | 5 ++++- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index cc0dde3e..92e28270 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -552,13 +552,18 @@ function Packet(options) { .uint16le('ftn_msg_dest_net') .uint16le('ftn_attr_flags') .uint16le('ftn_cost') - // :TODO: use string() for these if https://github.com/keichi/binary-parser/issues/33 is resolved + // + // It would be nice to just string() these, but we want CP437 which requires + // iconv. Another option would be to use a formatter, but until issue 33 + // (https://github.com/keichi/binary-parser/issues/33) is fixed, this is cumbersome. + // .array('modDateTime', { type : 'uint8', - readUntil : b => 0x00 === b, + length : 20, // FTS-0001.016: 20 bytes }) .array('toUserName', { type : 'uint8', + // :TODO: array needs some soft of 'limit' field readUntil : b => 0x00 === b, }) .array('fromUserName', { @@ -585,16 +590,35 @@ function Packet(options) { // // Convert null terminated arrays to strings // + // From FTS-0001.016: + // * modDateTime: 20 bytes exactly (see above) + // * toUserName and fromUserName: *max* 36 bytes, aka "up to"; null terminated + // * subject: *max* 72 bytes, aka "up to"; null terminated + // * message: Unbounded & null terminated + // + // For everything above but message, we can get away with assuming CP437 + // and probably even just "ascii" for most cases. The message field is + // much more complex so we'll look for encoding kludges, detection, etc. + // later on. + // + if(msgData.modDateTime.length != 20) { + return cb(Errors.Invalid(`FTN packet DateTime field must be 20 bytes (got ${msgData.modDateTime.length})`)); + } + if(msgData.toUserName.length > 36) { + return cb(Errors.Invalid(`FTN packet toUserName field must be 36 bytes max (got ${msgData.toUserName.length})`)); + } + if(msgData.fromUserName.length > 36) { + return cb(Errors.Invalid(`FTN packet fromUserName field must be 36 bytes max (got ${msgData.fromUserName.length})`)); + } + if(msgData.subject.length > 72) { + return cb(Errors.Invalid(`FTN packet subject field must be 72 bytes max (got ${msgData.subject.length})`)); + } + + // Arrays of CP437 bytes -> String [ 'modDateTime', 'toUserName', 'fromUserName', 'subject' ].forEach(k => { msgData[k] = strUtil.stringFromNullTermBuffer(msgData[k], 'CP437'); }); - // Technically the following fields have length limits as per fts-0001.016: - // * modDateTime : 20 bytes - // * toUserName : 36 bytes - // * fromUserName : 36 bytes - // * subject : 72 bytes - // // The message body itself is a special beast as it may // contain an origin line, kludges, SAUCE in the case diff --git a/util/dump_ftn_packet.js b/util/dump_ftn_packet.js index 144d18f3..88eeece8 100755 --- a/util/dump_ftn_packet.js +++ b/util/dump_ftn_packet.js @@ -42,7 +42,10 @@ function main() { return next(null); }, - () => { + (err) => { + if(err) { + return console.error(`Error processing packet: ${err.message}`); + } console.info(''); console.info('--- EOF --- '); console.info(''); From 2c9a68d0b1940f632e7337af29d3c679b10d5812 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 23 Nov 2019 10:08:42 -0700 Subject: [PATCH 06/95] Update sqlite3-trans package - get lodash updates along with it --- package.json | 2 +- yarn.lock | 15 +++++---------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 8b620932..048a207f 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "sane": "4.1.0", "sanitize-filename": "^1.6.3", "sqlite3": "^4.1.0", - "sqlite3-trans": "^1.2.1", + "sqlite3-trans": "^1.2.2", "ssh2": "0.8.6", "temptmp": "^1.1.0", "uuid": "^3.3.3", diff --git a/yarn.lock b/yarn.lock index b13f2369..453791de 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1138,11 +1138,6 @@ lodash@^4.17.15: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== -lodash@^4.17.4: - version "4.17.11" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d" - integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg== - lru-cache@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-5.1.1.tgz#1da27e6710271947695daf6848e847f01d84b920" @@ -1918,12 +1913,12 @@ split2@^3.0.0: dependencies: readable-stream "^3.0.0" -sqlite3-trans@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/sqlite3-trans/-/sqlite3-trans-1.2.1.tgz#642dff9f6da53d533ccd264b49e68c8818542255" - integrity sha512-KLtR+PBZN/moxDTKWTwWypkunDCJ0oi5vknjht8omjUXswwUEf+MX2DKtgQB1V5Tsjgc4mL4mHjv9zp7+FHs5g== +sqlite3-trans@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/sqlite3-trans/-/sqlite3-trans-1.2.2.tgz#faf268cc8d04dfd1a4854d64a70a229bdb50609f" + integrity sha512-+c2je0JMgPeNYHM7vMwEv/nHqOMYa5NNgQDcUyFkVMJ5QHATOQ+GywJptlVbkRCjgSTctmighfWLwUHPlkXbSQ== dependencies: - lodash "^4.17.4" + lodash "^4.17.15" sqlite3@^4.1.0: version "4.1.0" From cd3b8d5e76d2a05a7e3d5d25f8efdb39737394f3 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 5 Dec 2019 20:48:13 -0700 Subject: [PATCH 07/95] Low hanging fruit: Don't re-create binary parsers constantly --- core/ftn_mail_packet.js | 138 ++++++++++++++++++----------------- core/sauce.js | 40 +++++----- core/servers/login/telnet.js | 69 +++++++++--------- 3 files changed, 126 insertions(+), 121 deletions(-) diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index 92e28270..781ed929 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -165,6 +165,74 @@ exports.PacketHeader = PacketHeader; // * Writeup on differences between type 2, 2.2, and 2+: // http://walon.org/pub/fidonet/FTSC-nodelists-etc./pkt-types.txt // +const PacketHeaderParser = new Parser() + .uint16le('origNode') + .uint16le('destNode') + .uint16le('year') + .uint16le('month') + .uint16le('day') + .uint16le('hour') + .uint16le('minute') + .uint16le('second') + .uint16le('baud') + .uint16le('packetType') + .uint16le('origNet') + .uint16le('destNet') + .int8('prodCodeLo') + .int8('prodRevLo') // aka serialNo + .buffer('password', { length : 8 }) // can't use string; need CP437 - see https://github.com/keichi/binary-parser/issues/33 + .uint16le('origZone') + .uint16le('destZone') + // + // The following is "filler" in FTS-0001, specifics in + // FSC-0045 and FSC-0048 + // + .uint16le('auxNet') + .uint16le('capWordValidate') + .int8('prodCodeHi') + .int8('prodRevHi') + .uint16le('capWord') + .uint16le('origZone2') + .uint16le('destZone2') + .uint16le('origPoint') + .uint16le('destPoint') + .uint32le('prodData'); + +const MessageHeaderParser = new Parser() + .uint16le('messageType') + .uint16le('ftn_msg_orig_node') + .uint16le('ftn_msg_dest_node') + .uint16le('ftn_msg_orig_net') + .uint16le('ftn_msg_dest_net') + .uint16le('ftn_attr_flags') + .uint16le('ftn_cost') + // + // It would be nice to just string() these, but we want CP437 which requires + // iconv. Another option would be to use a formatter, but until issue 33 + // (https://github.com/keichi/binary-parser/issues/33) is fixed, this is cumbersome. + // + .array('modDateTime', { + type : 'uint8', + length : 20, // FTS-0001.016: 20 bytes + }) + .array('toUserName', { + type : 'uint8', + // :TODO: array needs some soft of 'limit' field + readUntil : b => 0x00 === b, + }) + .array('fromUserName', { + type : 'uint8', + readUntil : b => 0x00 === b, + }) + .array('subject', { + type : 'uint8', + readUntil : b => 0x00 === b, + }) + .array('message', { + type : 'uint8', + readUntil : b => 0x00 === b, + }); + function Packet(options) { var self = this; @@ -175,39 +243,7 @@ function Packet(options) { let packetHeader; try { - packetHeader = new Parser() - .uint16le('origNode') - .uint16le('destNode') - .uint16le('year') - .uint16le('month') - .uint16le('day') - .uint16le('hour') - .uint16le('minute') - .uint16le('second') - .uint16le('baud') - .uint16le('packetType') - .uint16le('origNet') - .uint16le('destNet') - .int8('prodCodeLo') - .int8('prodRevLo') // aka serialNo - .buffer('password', { length : 8 }) // can't use string; need CP437 - see https://github.com/keichi/binary-parser/issues/33 - .uint16le('origZone') - .uint16le('destZone') - // - // The following is "filler" in FTS-0001, specifics in - // FSC-0045 and FSC-0048 - // - .uint16le('auxNet') - .uint16le('capWordValidate') - .int8('prodCodeHi') - .int8('prodRevHi') - .uint16le('capWord') - .uint16le('origZone2') - .uint16le('destZone2') - .uint16le('origPoint') - .uint16le('destPoint') - .uint32le('prodData') - .parse(packetBuffer); + packetHeader = PacketHeaderParser.parse(packetBuffer); } catch(e) { return Errors.Invalid(`Unable to parse FTN packet header: ${e.message}`); } @@ -544,41 +580,7 @@ function Packet(options) { let msgData; try { - msgData = new Parser() - .uint16le('messageType') - .uint16le('ftn_msg_orig_node') - .uint16le('ftn_msg_dest_node') - .uint16le('ftn_msg_orig_net') - .uint16le('ftn_msg_dest_net') - .uint16le('ftn_attr_flags') - .uint16le('ftn_cost') - // - // It would be nice to just string() these, but we want CP437 which requires - // iconv. Another option would be to use a formatter, but until issue 33 - // (https://github.com/keichi/binary-parser/issues/33) is fixed, this is cumbersome. - // - .array('modDateTime', { - type : 'uint8', - length : 20, // FTS-0001.016: 20 bytes - }) - .array('toUserName', { - type : 'uint8', - // :TODO: array needs some soft of 'limit' field - readUntil : b => 0x00 === b, - }) - .array('fromUserName', { - type : 'uint8', - readUntil : b => 0x00 === b, - }) - .array('subject', { - type : 'uint8', - readUntil : b => 0x00 === b, - }) - .array('message', { - type : 'uint8', - readUntil : b => 0x00 === b, - }) - .parse(packetBuffer); + msgData = MessageHeaderParser.parse(packetBuffer); } catch(e) { return cb(Errors.Invalid(`Failed to parse FTN message header: ${e.message}`)); } diff --git a/core/sauce.js b/core/sauce.js index 7d5f52fd..40434861 100644 --- a/core/sauce.js +++ b/core/sauce.js @@ -26,6 +26,25 @@ exports.SAUCE_SIZE = SAUCE_SIZE; // const SAUCE_VALID_DATA_TYPES = [0, 1, 2, 3, 4, 5, 6, 7, 8 ]; +const SAUCEParser = new Parser() + .buffer('id', { length : 5 } ) + .buffer('version', { length : 2 } ) + .buffer('title', { length: 35 } ) + .buffer('author', { length : 20 } ) + .buffer('group', { length: 20 } ) + .buffer('date', { length: 8 } ) + .uint32le('fileSize') + .int8('dataType') + .int8('fileType') + .uint16le('tinfo1') + .uint16le('tinfo2') + .uint16le('tinfo3') + .uint16le('tinfo4') + .int8('numComments') + .int8('flags') + // :TODO: does this need to be optional? + .buffer('tinfos', { length: 22 } ); // SAUCE 00.5 + function readSAUCE(data, cb) { if(data.length < SAUCE_SIZE) { return cb(Errors.DoesNotExist('No SAUCE record present')); @@ -33,30 +52,11 @@ function readSAUCE(data, cb) { let sauceRec; try { - sauceRec = new Parser() - .buffer('id', { length : 5 } ) - .buffer('version', { length : 2 } ) - .buffer('title', { length: 35 } ) - .buffer('author', { length : 20 } ) - .buffer('group', { length: 20 } ) - .buffer('date', { length: 8 } ) - .uint32le('fileSize') - .int8('dataType') - .int8('fileType') - .uint16le('tinfo1') - .uint16le('tinfo2') - .uint16le('tinfo3') - .uint16le('tinfo4') - .int8('numComments') - .int8('flags') - // :TODO: does this need to be optional? - .buffer('tinfos', { length: 22 } ) // SAUCE 00.5 - .parse(data.slice(data.length - SAUCE_SIZE)); + sauceRec = SAUCEParser.parse(data.slice(data.length - SAUCE_SIZE)); } catch(e) { return cb(Errors.Invalid('Invalid SAUCE record')); } - if(!SAUCE_ID.equals(sauceRec.id)) { return cb(Errors.DoesNotExist('No SAUCE record present')); } diff --git a/core/servers/login/telnet.js b/core/servers/login/telnet.js index 1b892e71..20ec6cd9 100644 --- a/core/servers/login/telnet.js +++ b/core/servers/login/telnet.js @@ -192,6 +192,18 @@ OPTION_IMPLS[OPTIONS.SUPPRESS_GO_AHEAD] = function(bufs, i, event) { return event; }; +const TermTypeCmdParser = new Parser() + .uint8('iac1') + .uint8('sb') + .uint8('opt') + .uint8('is') + .array('ttype', { + type : 'uint8', + readUntil : b => 255 === b, // 255=COMMANDS.IAC + }) + // note we read iac2 above + .uint8('se'); + OPTION_IMPLS[OPTIONS.TERMINAL_TYPE] = function(bufs, i, event) { if(event.commandCode !== COMMANDS.SB) { OPTION_IMPLS.NO_ARGS(bufs, i, event); @@ -208,18 +220,7 @@ OPTION_IMPLS[OPTIONS.TERMINAL_TYPE] = function(bufs, i, event) { let ttypeCmd; try { - ttypeCmd = new Parser() - .uint8('iac1') - .uint8('sb') - .uint8('opt') - .uint8('is') - .array('ttype', { - type : 'uint8', - readUntil : b => 255 === b, // 255=COMMANDS.IAC - }) - // note we read iac2 above - .uint8('se') - .parse(bufs.toBuffer()); + ttypeCmd = TermTypeCmdParser.parse(bufs.toBuffer()); } catch(e) { Log.debug( { error : e }, 'Failed parsing TTYP telnet command'); return event; @@ -242,6 +243,15 @@ OPTION_IMPLS[OPTIONS.TERMINAL_TYPE] = function(bufs, i, event) { return event; }; +const NawsCmdParser = new Parser() + .uint8('iac1') + .uint8('sb') + .uint8('opt') + .uint16be('width') + .uint16be('height') + .uint8('iac2') + .uint8('se'); + OPTION_IMPLS[OPTIONS.WINDOW_SIZE] = function(bufs, i, event) { if(event.commandCode !== COMMANDS.SB) { OPTION_IMPLS.NO_ARGS(bufs, i, event); @@ -253,15 +263,7 @@ OPTION_IMPLS[OPTIONS.WINDOW_SIZE] = function(bufs, i, event) { let nawsCmd; try { - nawsCmd = new Parser() - .uint8('iac1') - .uint8('sb') - .uint8('opt') - .uint16be('width') - .uint16be('height') - .uint8('iac2') - .uint8('se') - .parse(bufs.splice(0, 9).toBuffer()); + nawsCmd = NawsCmdParser.parse(bufs.splice(0, 9).toBuffer()); } catch(e) { Log.debug( { error : e }, 'Failed parsing NAWS telnet command'); return event; @@ -282,6 +284,18 @@ OPTION_IMPLS[OPTIONS.WINDOW_SIZE] = function(bufs, i, event) { // Build an array of delimiters for parsing NEW_ENVIRONMENT[_DEP] //const NEW_ENVIRONMENT_DELIMITERS = _.values(NEW_ENVIRONMENT_COMMANDS); +const EnvCmdParser = new Parser() + .uint8('iac1') + .uint8('sb') + .uint8('opt') + .uint8('isOrInfo') // IS=initial, INFO=updates + .array('envBlock', { + type : 'uint8', + readUntil : b => 255 === b, // 255=COMMANDS.IAC + }) + // note we consume IAC above + .uint8('se'); + // Handle the deprecated RFC 1408 & the updated RFC 1572: OPTION_IMPLS[OPTIONS.NEW_ENVIRONMENT_DEP] = OPTION_IMPLS[OPTIONS.NEW_ENVIRONMENT] = function(bufs, i, event) { @@ -306,18 +320,7 @@ OPTION_IMPLS[OPTIONS.NEW_ENVIRONMENT] = function(bufs, i, event) { let envCmd; try { - envCmd = new Parser() - .uint8('iac1') - .uint8('sb') - .uint8('opt') - .uint8('isOrInfo') // IS=initial, INFO=updates - .array('envBlock', { - type : 'uint8', - readUntil : b => 255 === b, // 255=COMMANDS.IAC - }) - // note we consume IAC above - .uint8('se') - .parse(bufs.splice(0, bufs.length).toBuffer()); + envCmd = EnvCmdParser.parse(bufs.splice(0, bufs.length).toBuffer()); } catch(e) { Log.debug( { error : e }, 'Failed parsing NEW-ENVIRON telnet command'); return event; From 3f454c44c2f91be5408293d1b0bb2106c20fe049 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 23 Dec 2019 20:06:15 -0700 Subject: [PATCH 08/95] Don't hang on unzip prompting --- art/general/CONNECT1.ANS | Bin 2052 -> 0 bytes art/general/NEWSCAN.ANS | Bin 152 -> 211 bytes core/config.js | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 art/general/CONNECT1.ANS diff --git a/art/general/CONNECT1.ANS b/art/general/CONNECT1.ANS deleted file mode 100644 index d1a870bcd2a2b09a4f4b19fcd530769831631c2c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2052 zcmb_dOHRWu5N%gT*|H;>9Km%HDwI{J5MqIZgkY01_$b^3dYwedRd_QVf8uHA1~00{ z^LcOHjO}_beQWyt!BzEeu%~+1^?U24sfL07U>{Z(_iSt}ZJ)(F)>m`j-87MUWJ*aay7Me`W+NL0LRdF|S}XqI-6bOA5xxhCv~FZ7cqqH{ha8o4TS zY$+m|oUWhaycxnK>a6(E$fTNVM8}ARLGG7RtWt2y)sBOk^hwLCc_&Kc0F5&ppF&F# zfsS`N<}xrez&bmXP!QHmLSbDxs+&r}@A4K|l}>}i6f6oIu4}|sMZDM-;m0Cs^mUa> zeN~NgwaBbuJdB&@VzucO(_m7n)^mvkq_XAWP$tk2*!iO&EYNb6b&3)M68U*uk_asS zN}%VQ8ThMN(j~?#EfPKfPiF{(p(_CCt-~qcM}xwdTQ^uba(gyM5bK8(z0BkFU+= J^U;6B-(LakpR@n~ diff --git a/art/general/NEWSCAN.ANS b/art/general/NEWSCAN.ANS index 96371880565ba4b6a107a8de58776d00101ef3d3..2762c9f121da8f79978854a1dc49911230c4e6ee 100644 GIT binary patch literal 211 zcmb1+Hn27^ur@Z&Xnp@E?^0|NsiV*mpSkOl%_PbXi6 LFn31?4^9FA%#jJk diff --git a/core/config.js b/core/config.js index 44b5f4cc..66ffab14 100644 --- a/core/config.js +++ b/core/config.js @@ -678,7 +678,7 @@ function getDefaultConfig() { }, decompress : { cmd : 'unzip', - args : [ '{archivePath}', '-d', '{extractPath}' ], + args : [ '-n', '{archivePath}', '-d', '{extractPath}' ], }, list : { cmd : 'unzip', @@ -688,7 +688,7 @@ function getDefaultConfig() { }, extract : { cmd : 'unzip', - args : [ '{archivePath}', '{fileList}', '-d', '{extractPath}' ], + args : [ '-n', '{archivePath}', '{fileList}', '-d', '{extractPath}' ], } }, From ea828ba8123a71e3f834185ae5b525a4cd4e6357 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 1 Jan 2020 14:10:47 -0700 Subject: [PATCH 09/95] Update copyrights --- LICENSE.TXT | 2 +- README.md | 2 +- core/connect.js | 2 +- docs/installation/testing.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/LICENSE.TXT b/LICENSE.TXT index 74697ba9..af51c707 100644 --- a/LICENSE.TXT +++ b/LICENSE.TXT @@ -1,4 +1,4 @@ -Copyright (c) 2015-2019, Bryan D. Ashby +Copyright (c) 2015-2020, Bryan D. Ashby All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.md b/README.md index 477935a3..e099dcdb 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Please see [Installation Methods](https://nuskooler.github.io/enigma-bbs/install ## License Released under the [BSD 2-clause](https://opensource.org/licenses/BSD-2-Clause) license: -Copyright (c) 2015-2019, Bryan D. Ashby +Copyright (c) 2015-2020, Bryan D. Ashby All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/core/connect.js b/core/connect.js index d5b03de4..64c4ea3e 100644 --- a/core/connect.js +++ b/core/connect.js @@ -199,7 +199,7 @@ function displayBanner(term) { // note: intentional formatting: term.pipeWrite(` |06Connected to |02EN|10i|02GMA|10½ |06BBS version |12|VN -|06Copyright (c) 2014-2019 Bryan Ashby |14- |12http://l33t.codes/ +|06Copyright (c) 2014-2020 Bryan Ashby |14- |12http://l33t.codes/ |06Updates & source |14- |12https://github.com/NuSkooler/enigma-bbs/ |00` ); diff --git a/docs/installation/testing.md b/docs/installation/testing.md index b23616f2..bfd2de26 100644 --- a/docs/installation/testing.md +++ b/docs/installation/testing.md @@ -13,7 +13,7 @@ _Note that if you've used the [Docker](docker) installation method, you've alrea If everything went OK: ```bash -ENiGMA½ Copyright (c) 2014-2019 Bryan Ashby +ENiGMA½ Copyright (c) 2014-2020, Bryan Ashby _____________________ _____ ____________________ __________\_ / \__ ____/\_ ____ \ /____/ / _____ __ \ / ______/ // /___jp! // __|___// | \// |// | \// | | \// \ /___ /_____ From 29ee9c4d58140f72c90bb8e4d122fcee3fdb0a26 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 21 Apr 2020 19:50:04 -0600 Subject: [PATCH 10/95] WIP on QWK support --- core/message.js | 19 ++ core/oputil/oputil_message_base.js | 61 ++++ core/qwk_mail_packet.js | 520 +++++++++++++++++++++++++++++ 3 files changed, 600 insertions(+) create mode 100644 core/qwk_mail_packet.js diff --git a/core/message.js b/core/message.js index 5291b82a..68f51ee4 100644 --- a/core/message.js +++ b/core/message.js @@ -87,6 +87,21 @@ const FTN_PROPERTY_NAMES = { FtnSeenBy : 'ftn_seen_by', // http://ftsc.org/docs/fts-0004.001 }; +const QWKPropertyNames = { + MessageNumber : 'qwk_msg_num', + MessageStatus : 'qwk_msg_status', // See http://wiki.synchro.net/ref:qwk for a decent list + ConferenceNumber : 'qwk_conf_num', + InReplyToNum : 'qwk_in_reply_to_num', // note that we prefer the 'InReplyToMsgId' kludge if available +}; + +const QWKKludgeNames = { + Via : 'via', + MessageId : 'msg_id', + InReplyToMsgId : 'in_reply_to_msg_id', + SyncTZ : 'synchronet_timezone', + ReplyTo : 'reply_to', +}; + // :TODO: this is a ugly hack due to bad variable names - clean it up & just _.camelCase(k)! const MESSAGE_ROW_MAP = { reply_to_message_id : 'replyToMsgId', @@ -183,6 +198,10 @@ module.exports = class Message { return FTN_PROPERTY_NAMES; } + static get QWKPropertyNames() { + return QWKPropertyNames; + } + setLocalToUserId(userId) { this.meta.System = this.meta.System || {}; this.meta.System[Message.SystemMetaNames.LocalToUserID] = userId; diff --git a/core/oputil/oputil_message_base.js b/core/oputil/oputil_message_base.js index 0f1b5cfb..e5b30c12 100644 --- a/core/oputil/oputil_message_base.js +++ b/core/oputil/oputil_message_base.js @@ -434,6 +434,66 @@ function getImportEntries(importType, importData) { return importEntries; } +function handleQWK() { + const packetPath = argv._[argv._.length - 1]; + if(argv._.length < 4 || !packetPath || 0 === packetPath.length) { + return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + } + + const subAction = argv._[argv._.length - 2]; + switch (subAction) { + case 'dump' : + return dumpQWKPacket(packetPath); + + default : + return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + } +} + +function dumpQWKPacket(packetPath) { + async.waterfall( + [ + (callback) => { + return initConfigAndDatabases(callback); + }, + (callback) => { + const { QWKPacketReader } = require('../qwk_mail_packet'); + const reader = new QWKPacketReader(packetPath); + + reader.on('error', err => { + console.error(`ERROR: ${err.message}`); + return callback(err); + }); + + reader.on('done', () => { + return callback(null); + }); + + reader.on('archive type', archiveType => { + console.info(`-> Archive type: ${archiveType}`); + }); + + reader.on('creator', creator => { + console.info(`-> Creator: ${creator}`); + }); + + reader.on('message', message => { + console.info('--- message ---'); + console.info(`To : ${message.toUserName}`); + console.info(`From : ${message.fromUserName}`); + console.info(`Subject : ${message.subject}`); + console.info(`Message :\r\n${message.message}`); + }); + + reader.read(); + } + ], + err => { + + } + ) +} + function handleMessageBaseCommand() { function errUsage() { @@ -452,5 +512,6 @@ function handleMessageBaseCommand() { return({ areafix : areaFix, 'import-areas' : importAreas, + qwk : handleQWK, }[action] || errUsage)(); } \ No newline at end of file diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js new file mode 100644 index 00000000..415822a3 --- /dev/null +++ b/core/qwk_mail_packet.js @@ -0,0 +1,520 @@ + + +const ArchiveUtil = require('./archive_util'); +const { Errors } = require('./enig_error'); +const Message = require('./message'); +const { splitTextAtTerms } = require('./string_util'); + +const { EventEmitter } = require('events'); +const temptmp = require('temptmp').createTrackedSession('qwk_packet'); +const async = require('async'); +const fs = require('graceful-fs'); +const paths = require('path'); +const { Parser } = require('binary-parser'); +const iconv = require('iconv-lite'); +const moment = require('moment'); +const _ = require('lodash'); + +const SMBTZToUTCOffset = (smbTZ) => { + // convert a Synchronet smblib TZ to a UTC offset + // see https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h + return { + // US Standard + '40F0' : '-04:00', // Atlantic + '412C' : '-05:00', // Eastern + '4168' : '-06:00', // Central + '41A4' : '-07:00', // Mountain + '41E0' : '-08:00', // Pacific + '421C' : '-09:00', // Yukon + '4258' : '-10:00', // Hawaii/Alaska + '4294' : '-11:00', // Bering + + // US Daylight + + }[smbTZ]; +}; + +const QWKMessageBlockSize = 128; + +const MessageHeaderParser = new Parser() + .endianess('little') + .string('status', { + encoding : 'ascii', + length : 1, + }) + .string('num', { // message num or conf num for REP's + encoding : 'ascii', + length : 7, + formatter : n => { + return parseInt(n); + } + }) + .string('timestamp', { + encoding : 'ascii', + length : 13, + }) + // these fields may be encoded in something other than ascii/CP437 + .array('toName', { + type : 'uint8', + length : 25, + }) + .array('fromName', { + type : 'uint8', + length : 25, + }) + .array('subject', { + type : 'uint8', + length : 25, + }) + .string('password', { + encoding : 'ascii', + length : 12, + }) + .string('replyToNum', { + encoding : 'ascii', + length : 8, + formatter : n => { + return parseInt(n); + } + }) + .string('numBlocks', { + encoding : 'ascii', + length : 6, + formatter : n => { + return parseInt(n); + } + }) + .uint8('status2') + .uint16('confNum') + .uint16('relNum') + .uint8('netTag'); + + +class QWKPacketReader extends EventEmitter { + constructor(packetPath, mode=QWKPacketReader.Modes.Guess, options = { keepTearAndOrigin : true } ) { + super(); + + this.packetPath = packetPath; + this.mode = mode; + this.options = options; + } + + static get Modes() { + return { + Guess : 'guess', // try to guess + QWK : 'qwk', // standard incoming packet + REP : 'rep', // a reply packet + }; + } + + read() { + // + // A general overview: + // + // - Find out what kind of archive we're dealing with + // - Extract to temporary location + // - Process various files + // - Emit messages we find, information about the packet, so on + // + async.waterfall( + [ + // determine packet archive type + (callback) => { + const archiveUtil = ArchiveUtil.getInstance(); + archiveUtil.detectType(this.packetPath, (err, archiveType) => { + if (err) { + return callback(err); + } + this.emit('archive type', archiveType); + return callback(null, archiveType); + }); + }, + // create a temporary location to do processing + (archiveType, callback) => { + temptmp.mkdir( { prefix : 'enigqwkpacket-'}, (err, tempDir) => { + if (err) { + return callback(err); + } + + return callback(null, archiveType, tempDir); + }); + }, + // extract it + (archiveType, tempDir, callback) => { + const archiveUtil = ArchiveUtil.getInstance(); + archiveUtil.extractTo(this.packetPath, tempDir, archiveType, err => { + if (err) { + return callback(err); + } + + return callback(null, tempDir); + }); + }, + // gather extracted file list + (tempDir, callback) => { + fs.readdir(tempDir, (err, files) => { + if (err) { + return callback(err); + } + + // Discover basic information about well known files + async.reduce( + files, + {}, + (out, filename, next) => { + const key = filename.toUpperCase(); + + switch (key) { + case 'MESSAGES.DAT' : // QWK + if (this.mode === QWKPacketReader.Modes.Guess) { + this.mode = QWKPacketReader.Modes.QWK; + } + if (this.mode === QWKPacketReader.Modes.QWK) { + out.messages = { filename }; + } + break; + + case 'ID.MSG' : + if (this.mode === QWKPacketReader.Modes.Guess) { + this.mode = Modes.REP; + } + + if (this.mode === QWKPacketReader.Modes.REP) { + out.messages = { filename }; + } + break; + + case 'HEADERS.DAT' : // Synchronet + out.headers = { filename }; + break; + + case 'VOTING.DAT' : // Synchronet + out.voting = { filename }; + break; + + case 'CONTROL.DAT' : // QWK + out.control = { filename }; + break; + + case 'DOOR.ID' : // QWK + out.door = { filename }; + break; + + case 'NETFLAGS.DAT' : // QWK + out.netflags = { filename }; + break; + + case 'NEWFILES.DAT' : // QWK + out.newfiles = { filename }; + break; + + case 'PERSONAL.NDX' : // QWK + out.personal = { filename }; + break; + + case '000.NDX' : // QWK + out.inbox = { filename }; + break; + + case 'TOREADER.EXT' : // QWKE + out.toreader = { filename }; + break; + + case 'QLR.DAT' : + out.qlr = { filename }; + break; + + default : + if (/[0-9]+\.NDX/.test(key)) { // QWK + out.pointers = out.pointers || { filenames: [] }; + out.pointers.filenames.push(filename); + } else { + out[key] = { filename }; + } + break; + } + + return next(null, out); + }, + (err, packetFileInfo) => { + this.packetInfo = Object.assign( + {}, + packetFileInfo, + { + tempDir, + defaultEncoding : 'CP437' + } + ); + return callback(null); + } + ); + }); + }, + (callback) => { + return this.processPacketFiles(callback); + }, + (tempDir, callback) => { + return callback(null); + } + ], + err => { + temptmp.cleanup(); + + if (err) { + return this.emit('error', err); + } + + this.emit('done'); + } + ); + } + + processPacketFiles(cb) { + return this.readMessages(cb); + } + + readMessages(cb) { + // :TODO: update to use proper encoding: if headers.dat specifies UTF-8, use that, else CP437 + if (!this.packetInfo.messages) { + return cb(Errors.DoesNotExist('No messages file found within QWK packet')); + } + + const encoding = this.packetInfo.defaultEncoding; + const path = paths.join(this.packetInfo.tempDir, this.packetInfo.messages.filename); + fs.open(path, 'r', (err, fd) => { + if (err) { + return cb(err); + } + + let blockCount = 0; + let currMessage = { }; + let state; + let messageBlocksRemain; + const buffer = Buffer.alloc(QWKMessageBlockSize); + + const readNextBlock = () => { + fs.read(fd, buffer, 0, QWKMessageBlockSize, null, (err, read) => { + if (err) { + return cb(err); + } + + if (0 == read) { + // we're done consuming all blocks + return fs.close(fd, err => { + return cb(err); + }); + } + + if (QWKMessageBlockSize !== read) { + return cb(Errors.Invalid(`Invalid QWK message block size. Expected ${QWKMessageBlockSize} got ${read}`)); + } + + if (0 === blockCount) { + // first 128 bytes is a space padded ID + const id = buffer.toString('ascii').trim(); + this.emit('generator', id); + state = 'header'; + } else { + switch (state) { + case 'header' : + const header = MessageHeaderParser.parse(buffer); + + // massage into something a little more sane (things we can't quite do in the parser directly) + ['toName', 'fromName', 'subject'].forEach(field => { + header[field] = iconv.decode(header[field], encoding).trim(); + }); + + header.timestamp = moment(header.timestamp, 'MM-DD-YYHH:mm'); + + currMessage = { + header, + // these may be overridden + toName : header.toName, + fromName : header.fromName, + subject : header.subject, + }; + + // remainder of blocks until the end of this message + messageBlocksRemain = header.numBlocks - 1; + state = 'message'; + break; + + case 'message' : + if (!currMessage.body) { + currMessage.body = buffer; + } else { + currMessage.body = Buffer.concat([currMessage.body, buffer]); + } + messageBlocksRemain -= 1; + + if (0 === messageBlocksRemain) { + // 1:n buffers to make up body. Decode: + // First, replace QWK style line feeds (0xe3) unless the message is UTF-8. + // If the message is UTF-8, we assume it's using standard line feeds. + if (encoding !== 'utf8') { + let i = 0; + const QWKLF = Buffer.from([0xe3]); + while (i < currMessage.body.length) { + i = currMessage.body.indexOf(QWKLF, i); + if (-1 === i) { + break; + } + currMessage.body[i] = 0x0a; + ++i; + } + } + + // + // Decode the message based on our final message encoding. Split the message + // into lines so we can extract various bits such as QWKE headers, origin, tear + // lines, etc. + // + const messageLines = splitTextAtTerms(iconv.decode(currMessage.body, encoding).trimEnd()); + const bodyLines = []; + + // + // Various kludge tags defined by QWKE, etc. + // See the following: + // - ftp://vert.synchro.net/main/BBS/qwke.txt + // - http://wiki.synchro.net/ref:qwk + // + const Kludges = { + // QWKE + To : 'To:', + From : 'From:', + Subject : 'Subject:', + + // Synchronet + Via : '@VIA:', + MsgID : '@MSGID:', + Reply : '@REPLY:', + TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h + ReplyTo : '@REPLYTO:', + }; + + let bodyState = 'kludge'; + + const MessageTrailers = { + // While technically FTN oriented, these can come from any network + // (though we'll be processing a lot of messages that routed through FTN + // at some point) + Origin : /^[ ]{1,2}\* Origin: /, + Tear : /^--- /, + }; + + const qwkKludge = {}; + const ftnProperty = {}; + + messageLines.forEach(line => { + if (0 === line.length) { + return bodyLines.push(''); + } + + switch (bodyState) { + case 'kludge' : + // :TODO: Update these to use the well known consts: + if (line.startsWith(Kludges.To)) { + currMessage.toName = line.substring(Kludges.To.length).trim(); + } else if (line.startsWith(Kludges.From)) { + currMessage.fromName = line.substring(Kludges.From.length).trim(); + } else if (line.startsWith(Kludges.Subject)) { + currMessage.subject = line.substring(Kludges.Subject.length).trim(); + } else if (line.startsWith(Kludges.Via)) { + qwkKludge.via = line; + } else if (line.startsWith(Kludges.MsgID)) { + qwkKludge.msg_id = line.substring(Kludges.MsgID.length).trim(); + } else if (line.startsWith(Kludges.Reply)) { + qwkKludge.in_reply_to_msg_id = line.substring(Kludges.Reply.length).trim(); + } else if (line.startsWith(Kludges.TZ)) { + qwkKludge.synchronet_timezone = line.substring(Kludges.TZ.length).trim(); + } else if (line.startsWith(Kludges.ReplyTo)) { + qwkKludge.reply_to = line.substring(Kludges.ReplyTo.length).trim(); + } else { + bodyState = 'body'; // past this point and up to any tear/origin/etc., is the real message body + bodyLines.push(line); + } + break; + + case 'body' : + case 'trailers' : + if (MessageTrailers.Origin.test(line)) { + ftnProperty.ftn_origin = line; + bodyState = 'trailers'; + } else if (MessageTrailers.Tear.test(line)) { + ftnProperty.ftn_tear_line = line; + bodyState = 'trailers'; + } else if ('body' === bodyState) { + bodyLines.push(line); + } + } + }); + + const message = new Message({ + toUserName : currMessage.toName, + fromUserName : currMessage.fromName, + subject : currMessage.subject, + modTimestamp : currMessage.header.timestamp, + message : bodyLines.join('\n'), + }); + + if (!_.isEmpty(qwkKludge)) { + message.meta.QwkKludge = qwkKludge; + } + + if (!_.isEmpty(ftnProperty)) { + message.meta.FtnProperty = ftnProperty; + } + + // Add in tear line and origin if requested + if (this.options.keepTearAndOrigin) { + if (ftnProperty.ftn_tear_line) { + message.message += `\r\n${ftnProperty.ftn_tear_line}\r\n`; + } + + if (ftnProperty.ftn_origin) { + message.message += `${ftnProperty.ftn_origin}\r\n`; + } + } + + // Update the timestamp if we have a valid TZ + if (_.has(message, 'meta.QwkKludge.synchronet_timezone')) { + const tzOffset = SMBTZToUTCOffset(message.meta.QwkKludge.synchronet_timezone); + if (tzOffset) { + message.modTimestamp.utcOffset(tzOffset); + } + } + + message.meta.QwkProperty = { + qwk_msg_status : currMessage.header.status, + qwk_in_reply_to_num : currMessage.header.replyToNum, + }; + + if (this.mode === QWKPacketReader.Modes.QWK) { + message.meta.QwkProperty.qwk_msg_num = currMessage.header.num; + } else { + // For REP's, prefer the larger field. + message.meta.QwkProperty.qwk_conf_num = currMessage.header.num || currMessage.header.confNum; + } + + this.emit('message', message); + state = 'header'; + } + break; + } + } + + ++blockCount; + readNextBlock(); + }); + }; + + // start reading blocks + readNextBlock(); + }); + } +}; + +module.exports = { + QWKPacketReader, +// QWKPacketWriter, +} From 6edfe95dfe9914f70282eba1ef5f052212a689e7 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 21 Apr 2020 22:14:29 -0600 Subject: [PATCH 11/95] A good amount of HEADERS.DAT support --- core/qwk_mail_packet.js | 180 ++++++++++++++++++++++++++++++++++------ package.json | 1 + yarn.lock | 19 +++++ 3 files changed, 174 insertions(+), 26 deletions(-) diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index 415822a3..9ed97bb6 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -14,6 +14,7 @@ const { Parser } = require('binary-parser'); const iconv = require('iconv-lite'); const moment = require('moment'); const _ = require('lodash'); +const IniConfigParser = require('ini-config-parser'); const SMBTZToUTCOffset = (smbTZ) => { // convert a Synchronet smblib TZ to a UTC offset @@ -242,7 +243,6 @@ class QWKPacketReader extends EventEmitter { packetFileInfo, { tempDir, - defaultEncoding : 'CP437' } ); return callback(null); @@ -270,7 +270,41 @@ class QWKPacketReader extends EventEmitter { } processPacketFiles(cb) { - return this.readMessages(cb); + async.series( + [ + (callback) => { + return this.readHeadersExtension(callback); + }, + (callback) => { + return this.readMessages(callback); + } + ], + err => { + return cb(err); + } + ) + } + + readHeadersExtension(cb) { + if (!this.packetInfo.headers) { + return cb(null); // nothing to do + } + + const path = paths.join(this.packetInfo.tempDir, this.packetInfo.headers.filename); + fs.readFile(path, { encoding : 'utf8' }, (err, iniData) => { + if (err) { + this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid (${err.message})`)); + return cb(null); // non-fatal + } + + try { + this.packetInfo.headers.ini = IniConfigParser.parse(iniData); + } catch (e) { + this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid (${e.message})`)); + } + + return cb(null); + }); } readMessages(cb) { @@ -279,13 +313,54 @@ class QWKPacketReader extends EventEmitter { return cb(Errors.DoesNotExist('No messages file found within QWK packet')); } - const encoding = this.packetInfo.defaultEncoding; + const encodingToSpec = 'cp437'; + let encoding = encodingToSpec; + const path = paths.join(this.packetInfo.tempDir, this.packetInfo.messages.filename); fs.open(path, 'r', (err, fd) => { if (err) { return cb(err); } + // Some mappings/etc. used in loops below.... + // Sync sets these in HEADERS.DAT: http://wiki.synchro.net/ref:qwk + const FTNPropertyMapping = { + 'X-FTN-AREA' : Message.FtnPropertyNames.FtnArea, + 'X-FTN-SEEN-BY' : Message.FtnPropertyNames.FtnSeenBy, + 'X-FTN-FLAGS' : Message.FtnPropertyNames + }; + + const FTNKludgeMapping = { + 'X-FTN-PATH' : 'PATH', + 'X-FTN-MSGID' : 'MSGID', + 'X-FTN-REPLY' : 'REPLY', + 'X-FTN-PID' : 'PID', + 'X-FTN-FLAGS' : 'FLAGS', + 'X-FTN-TID' : 'TID', + 'X-FTN-CHRS' : 'CHRS', + // :TODO: X-FTN-KLUDGE - not sure what this is? + }; + + // + // Various kludge tags defined by QWKE, etc. + // See the following: + // - ftp://vert.synchro.net/main/BBS/qwke.txt + // - http://wiki.synchro.net/ref:qwk + // + const Kludges = { + // QWKE + To : 'To:', + From : 'From:', + Subject : 'Subject:', + + // Synchronet + Via : '@VIA:', + MsgID : '@MSGID:', + Reply : '@REPLY:', + TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h + ReplyTo : '@REPLYTO:', + }; + let blockCount = 0; let currMessage = { }; let state; @@ -321,7 +396,8 @@ class QWKPacketReader extends EventEmitter { // massage into something a little more sane (things we can't quite do in the parser directly) ['toName', 'fromName', 'subject'].forEach(field => { - header[field] = iconv.decode(header[field], encoding).trim(); + // note: always use to-spec encoding here + header[field] = iconv.decode(header[field], encodingToSpec).trim(); }); header.timestamp = moment(header.timestamp, 'MM-DD-YYHH:mm'); @@ -334,6 +410,19 @@ class QWKPacketReader extends EventEmitter { subject : header.subject, }; + if (_.has(this.packetInfo, 'headers.ini')) { + // Sections for a message in HEADERS.DAT are by current byte offset. + // 128 = first message header = 0x80 = section [80] + const headersSectionId = (blockCount * QWKMessageBlockSize).toString(16); + currMessage.headersExtension = this.packetInfo.headers.ini[headersSectionId]; + } + + // if we have HEADERS.DAT with a 'Utf8' override for this message, + // the overridden to/from/subject/message fields are UTF-8 + if (currMessage.headersExtension && currMessage.headersExtension.Utf8) { + encoding = 'utf8'; + } + // remainder of blocks until the end of this message messageBlocksRemain = header.numBlocks - 1; state = 'message'; @@ -372,26 +461,6 @@ class QWKPacketReader extends EventEmitter { const messageLines = splitTextAtTerms(iconv.decode(currMessage.body, encoding).trimEnd()); const bodyLines = []; - // - // Various kludge tags defined by QWKE, etc. - // See the following: - // - ftp://vert.synchro.net/main/BBS/qwke.txt - // - http://wiki.synchro.net/ref:qwk - // - const Kludges = { - // QWKE - To : 'To:', - From : 'From:', - Subject : 'Subject:', - - // Synchronet - Via : '@VIA:', - MsgID : '@MSGID:', - Reply : '@REPLY:', - TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h - ReplyTo : '@REPLYTO:', - }; - let bodyState = 'kludge'; const MessageTrailers = { @@ -404,6 +473,7 @@ class QWKPacketReader extends EventEmitter { const qwkKludge = {}; const ftnProperty = {}; + const ftnKludge = {}; messageLines.forEach(line => { if (0 === line.length) { @@ -449,12 +519,60 @@ class QWKPacketReader extends EventEmitter { } }); + let messageTimestamp = currMessage.header.timestamp; + + // HEADERS.DAT support. + let useTZKludge = true; + if (currMessage.headersExtension) { + const ext = currMessage.headersExtension; + + // to and subject can be overridden yet again if entries are present + currMessage.toName = ext.To || currMessage.toName + currMessage.subject = ext.Subject || currMessage.subject; + currMessage.from = ext.Sender || currMessage.from; // why not From? Who the fuck knows. + + // possibly override message ID kludge + qwkKludge.msg_id = ext['Message-ID'] || qwkKludge.msg_id; + + // WhenWritten contains a ISO-8601-ish timestamp and a Synchronet/SMB style TZ offset: + // 20180101174837-0600 4168 + // We can use this to get a very slightly better precision on the timestamp (addition of seconds) + // over the headers value. Why not milliseconds? Who the fuck knows. + if (ext.WhenWritten) { + const whenWritten = moment(ext.WhenWritten, 'YYYYMMDDHHmmssZ'); + if (whenWritten.isValid()) { + messageTimestamp = whenWritten; + useTZKludge = false; + } + } + + if (ext.Tags) { + currMessage.hashTags = ext.Tags.split(' '); + } + + // FTN style properties/kludges represented as X-FTN-XXXX + for (let [extName, propName] of Object.entries(FTNPropertyMapping)) { + const v = ext[extName]; + if (v) { + ftnProperty[propName] = v; + } + } + + for (let [extName, kludgeName] of Object.entries(FTNKludgeMapping)) { + const v = ext[extName]; + if (v) { + ftnKludge[kludgeName] = v; + } + } + } + const message = new Message({ toUserName : currMessage.toName, fromUserName : currMessage.fromName, subject : currMessage.subject, - modTimestamp : currMessage.header.timestamp, + modTimestamp : messageTimestamp, message : bodyLines.join('\n'), + hashTags : currMessage.hashTags, }); if (!_.isEmpty(qwkKludge)) { @@ -465,6 +583,10 @@ class QWKPacketReader extends EventEmitter { message.meta.FtnProperty = ftnProperty; } + if (!_.isEmpty(ftnKludge)) { + message.meta.FtnKludge = ftnKludge; + } + // Add in tear line and origin if requested if (this.options.keepTearAndOrigin) { if (ftnProperty.ftn_tear_line) { @@ -477,7 +599,7 @@ class QWKPacketReader extends EventEmitter { } // Update the timestamp if we have a valid TZ - if (_.has(message, 'meta.QwkKludge.synchronet_timezone')) { + if (useTZKludge && _.has(message, 'meta.QwkKludge.synchronet_timezone')) { const tzOffset = SMBTZToUTCOffset(message.meta.QwkKludge.synchronet_timezone); if (tzOffset) { message.modTimestamp.utcOffset(tzOffset); @@ -491,11 +613,17 @@ class QWKPacketReader extends EventEmitter { if (this.mode === QWKPacketReader.Modes.QWK) { message.meta.QwkProperty.qwk_msg_num = currMessage.header.num; + message.meta.QwkProperty.qwk_conf_num = currMessage.header.confNum; } else { // For REP's, prefer the larger field. message.meta.QwkProperty.qwk_conf_num = currMessage.header.num || currMessage.header.confNum; } + // Another quick HEADERS.DAT fix-up + if (currMessage.headersExtension) { + message.meta.QwkProperty.qwk_conf_num = currMessage.headersExtension.Conference || message.meta.QwkProperty.qwk_conf_num; + } + this.emit('message', message); state = 'header'; } diff --git a/package.json b/package.json index 048a207f..12754271 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "hashids": "2.1.0", "hjson": "^3.2.1", "iconv-lite": "0.5.0", + "ini-config-parser": "^1.0.4", "inquirer": "^7.0.0", "later": "1.2.0", "lodash": "^4.17.15", diff --git a/yarn.lock b/yarn.lock index 453791de..cb08db2e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -312,6 +312,11 @@ code-point-at@^1.0.0: resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= +coffee-script@^1.12.4: + version "1.12.7" + resolved "https://registry.yarnpkg.com/coffee-script/-/coffee-script-1.12.7.tgz#c05dae0cb79591d05b3070a8433a98c9a89ccc53" + integrity sha512-fLeEhqwymYat/MpTPUjSKHVYYl0ec2mOyALEMLmzr5i1isuG+6jfI2j2d5oBO3VIzgUXgBVIcOT9uH1TFxBckw== + collection-visit@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/collection-visit/-/collection-visit-1.0.0.tgz#4bc0373c164bc3291b4d368c829cf1a80a59dca0" @@ -408,6 +413,11 @@ decode-uri-component@^0.2.0: resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" integrity sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU= +deep-extend@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.5.1.tgz#b894a9dd90d3023fbf1c55a394fb858eb2066f1f" + integrity sha512-N8vBdOa+DF7zkRrDCsaOXoCs/E2fJfx9B9MrKnnSiHNh4ws7eSys6YQE4KvT1cecKmOASYQBhbKjeuDD9lT81w== + deep-extend@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" @@ -883,6 +893,15 @@ inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= +ini-config-parser@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/ini-config-parser/-/ini-config-parser-1.0.4.tgz#0abc75cb68c506204712d2b4861400b6adbfda78" + integrity sha512-5hLh5Cqai67pTrLQ9q/K/3EtSP2Tzu41AZzwPLSegkkMkc42dGweLgkbiocCBiBBEg2fPhs6pKmdFhwj5Ul3Bg== + dependencies: + coffee-script "^1.12.4" + deep-extend "^0.5.1" + rimraf "^2.6.1" + ini@~1.3.0: version "1.3.5" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" From d8f0601914717f65c461b89ba57e86d71467e916 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 21 Apr 2020 22:57:06 -0600 Subject: [PATCH 12/95] Add CONTROL.DAT parsing --- core/qwk_mail_packet.js | 93 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index 9ed97bb6..f670e494 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -31,12 +31,17 @@ const SMBTZToUTCOffset = (smbTZ) => { '4294' : '-11:00', // Bering // US Daylight + // :TODO: FINISH ME! }[smbTZ]; }; const QWKMessageBlockSize = 128; +// See the following: +// - http://fileformats.archiveteam.org/wiki/QWK +// - http://wiki.synchro.net/ref:qwk +// const MessageHeaderParser = new Parser() .endianess('little') .string('status', { @@ -272,6 +277,9 @@ class QWKPacketReader extends EventEmitter { processPacketFiles(cb) { async.series( [ + (callback) => { + return this.readControl(callback); + }, (callback) => { return this.readHeadersExtension(callback); }, @@ -285,6 +293,91 @@ class QWKPacketReader extends EventEmitter { ) } + readControl(cb) { + // + // CONTROL.DAT is a CRLF text file containing information about + // the originating BBS, conf number <> name mapping, etc. + // + // References: + // - http://fileformats.archiveteam.org/wiki/QWK + // + if (!this.packetInfo.control) { + return cb(Errors.DoesNotExist('No control file found within QWK packet')); + } + + const path = paths.join(this.packetInfo.tempDir, this.packetInfo.control.filename); + + // note that we read as UTF-8. Legacy says it should be CP437/ASCII + // but this seems safer for now so conference names and the like + // can be non-English for example. + fs.readFile(path, { encoding : 'utf8' }, (err, controlLines) => { + if (err) { + return cb(err); + } + + controlLines = splitTextAtTerms(controlLines); + + let state = 'header'; + const control = { confMap : {} }; + let currConfNumber; + for (let lineNumber = 0; lineNumber < controlLines.length; ++lineNumber) { + const line = controlLines[lineNumber].trim(); + switch (lineNumber) { + // first set of lines is header info + case 0 : control.bbsName = line; break; + case 1 : control.bbsLocation = line; break; + case 2 : control.bbsPhone = line; break; + case 3 : control.bbsSysOp = line; break; + case 4 : control.doorRegAndBoardID = line; break; + case 5 : control.packetCreationTime = line; break; + case 6 : control.toUser = line; break; + case 7 : break; // Qmail menu + case 8 : break; // unknown, always 0? + case 9 : break; // total messages in packet (often set to 0) + case 10 : + control.totalMessages = (parseInt(line) + 1); + state = 'confNumber'; + break; + + default : + switch (state) { + case 'confNumber' : + currConfNumber = parseInt(line); + if (isNaN(currConfNumber)) { + state = 'news'; + + control.welcomeFile = line; + } else { + state = 'confName'; + } + break; + + case 'confName' : + control.confMap[currConfNumber] = line; + state = 'confNumber'; + break; + + case 'news' : + control.newsFile = line; + state = 'logoff'; + break; + + case 'logoff' : + control.logoffFile = line; + state = 'footer'; + break; + + case 'footer' : + // some systems append additional info; we don't care. + break; + } + } + } + + return cb(null); + }); + } + readHeadersExtension(cb) { if (!this.packetInfo.headers) { return cb(null); // nothing to do From 8a81b34ed044639b754ecb0d106b9fc5fd0505c0 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 25 Apr 2020 11:25:47 -0600 Subject: [PATCH 13/95] WIP: Lots of progress with QWK reader/writer --- core/message.js | 11 +- core/message_area.js | 1 + core/qwk_mail_packet.js | 554 +++++++++++++++++++++++++++++++++++----- 3 files changed, 492 insertions(+), 74 deletions(-) diff --git a/core/message.js b/core/message.js index 68f51ee4..3649a6f6 100644 --- a/core/message.js +++ b/core/message.js @@ -49,7 +49,8 @@ const SYSTEM_META_NAMES = { const ADDRESS_FLAVOR = { Local : 'local', // local / non-remote addressing FTN : 'ftn', // FTN style - Email : 'email', + Email : 'email', // From email + QWK : 'qwk', // QWK packet }; const STATE_FLAGS0 = { @@ -94,14 +95,6 @@ const QWKPropertyNames = { InReplyToNum : 'qwk_in_reply_to_num', // note that we prefer the 'InReplyToMsgId' kludge if available }; -const QWKKludgeNames = { - Via : 'via', - MessageId : 'msg_id', - InReplyToMsgId : 'in_reply_to_msg_id', - SyncTZ : 'synchronet_timezone', - ReplyTo : 'reply_to', -}; - // :TODO: this is a ugly hack due to bad variable names - clean it up & just _.camelCase(k)! const MESSAGE_ROW_MAP = { reply_to_message_id : 'replyToMsgId', diff --git a/core/message_area.js b/core/message_area.js index 579f1c9b..ff01f1e1 100644 --- a/core/message_area.js +++ b/core/message_area.js @@ -29,6 +29,7 @@ exports.getDefaultMessageAreaTagByConfTag = getDefaultMessageAreaTagByConfTag; exports.getSuitableMessageConfAndAreaTags = getSuitableMessageConfAndAreaTags; exports.getMessageConferenceByTag = getMessageConferenceByTag; exports.getMessageAreaByTag = getMessageAreaByTag; +exports.getMessageConfTagByAreaTag = getMessageConfTagByAreaTag; exports.changeMessageConference = changeMessageConference; exports.changeMessageArea = changeMessageArea; exports.hasMessageConfAndAreaRead = hasMessageConfAndAreaRead; diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index f670e494..5dc3275f 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -1,12 +1,14 @@ - - const ArchiveUtil = require('./archive_util'); const { Errors } = require('./enig_error'); const Message = require('./message'); const { splitTextAtTerms } = require('./string_util'); +const { getMessageConfTagByAreaTag } = require('./message_area'); +const StatLog = require('./stat_log'); +const Config = require('./config').get; +const SysProps = require('./system_property'); const { EventEmitter } = require('events'); -const temptmp = require('temptmp').createTrackedSession('qwk_packet'); +const temptmp = require('temptmp'); const async = require('async'); const fs = require('graceful-fs'); const paths = require('path'); @@ -16,27 +18,64 @@ const moment = require('moment'); const _ = require('lodash'); const IniConfigParser = require('ini-config-parser'); -const SMBTZToUTCOffset = (smbTZ) => { - // convert a Synchronet smblib TZ to a UTC offset - // see https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h - return { - // US Standard - '40F0' : '-04:00', // Atlantic - '412C' : '-05:00', // Eastern - '4168' : '-06:00', // Central - '41A4' : '-07:00', // Mountain - '41E0' : '-08:00', // Pacific - '421C' : '-09:00', // Yukon - '4258' : '-10:00', // Hawaii/Alaska - '4294' : '-11:00', // Bering +const enigmaVersion = require('../package.json').version; - // US Daylight - // :TODO: FINISH ME! +// Synchronet smblib TZ to a UTC offset +// see https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h +const SMBTZToUTCOffset = { + // US Standard + '40F0' : '-04:00', // Atlantic + '412C' : '-05:00', // Eastern + '4168' : '-06:00', // Central + '41A4' : '-07:00', // Mountain + '41E0' : '-08:00', // Pacific + '421C' : '-09:00', // Yukon + '4258' : '-10:00', // Hawaii/Alaska + '4294' : '-11:00', // Bering - }[smbTZ]; + // US Daylight + 'C0F0' : '-03:00', // Atlantic + 'C12C' : '-04:00', // Eastern + 'C168' : '-05:00', // Central + 'C1A4' : '-06:00', // Mountain + 'C1E0' : '-07:00', // Pacific + 'C21C' : '-08:00', // Yukon + 'C258' : '-09:00', // Hawaii/Alaska + 'C294' : '-10:00', // Bering + + // "Non-Standard" + '2294' : '-11:00', // Midway + '21E0' : '-08:00', // Vancouver + '21A4' : '-07:00', // Edmonton + '2168' : '-06:00', // Winnipeg + '212C' : '-05:00', // Bogota + '20F0' : '-04:00', // Caracas + '20B4' : '-03:00', // Rio de Janeiro + '2078' : '-02:00', // Fernando de Noronha + '203C' : '-01:00', // Azores + '1000' : '+00:00', // London + '103C' : '+01:00', // Berlin + '1078' : '+02:00', // Athens + '10B4' : '+03:00', // Moscow + '10F0' : '+04:00', // Dubai + '110E' : '+04:30', // Kabul + '112C' : '+05:00', // Karachi + '114A' : '+05:30', // Bombay + '1159' : '+05:45', // Kathmandu + '1168' : '+06:00', // Dhaka + '11A4' : '+07:00', // Bangkok + '11E0' : '+08:00', // Hong Kong + '121C' : '+09:00', // Tokyo + '1258' : '+10:00', // Sydney + '1294' : '+11:00', // Noumea + '12D0' : '+12:00', // Wellington }; -const QWKMessageBlockSize = 128; +const UTCOffsetToSMBTZ = _.invert(SMBTZToUTCOffset); + +const QWKMessageBlockSize = 128; +const QWKHeaderTimestampFormat = 'MM-DD-YYHH:mm'; +const QWKLF = 0xe3; // See the following: // - http://fileformats.archiveteam.org/wiki/QWK @@ -95,14 +134,29 @@ const MessageHeaderParser = new Parser() .uint16('relNum') .uint8('netTag'); +const replaceCharInBuffer = (buffer, search, replace) => { + let i = 0; + search = Buffer.from([search]); + while (i < buffer.length) { + i = buffer.indexOf(search, i); + if (-1 === i) { + break; + } + buffer[i] = replace; + ++i; + } +} class QWKPacketReader extends EventEmitter { - constructor(packetPath, mode=QWKPacketReader.Modes.Guess, options = { keepTearAndOrigin : true } ) { + constructor( + packetPath, + { mode = QWKPacketReader.Modes.Guess, keepTearAndOrigin = true } = { mode : QWKPacketReader.Modes.Guess, keepTearAndOrigin : true }) + { super(); this.packetPath = packetPath; - this.mode = mode; - this.options = options; + this.options = { mode, keepTearAndOrigin }; + this.temptmp = temptmp.createTrackedSession('qwkpacketreader'); } static get Modes() { @@ -137,7 +191,7 @@ class QWKPacketReader extends EventEmitter { }, // create a temporary location to do processing (archiveType, callback) => { - temptmp.mkdir( { prefix : 'enigqwkpacket-'}, (err, tempDir) => { + this.temptmp.mkdir( { prefix : 'enigqwkreader-'}, (err, tempDir) => { if (err) { return callback(err); } @@ -172,20 +226,20 @@ class QWKPacketReader extends EventEmitter { switch (key) { case 'MESSAGES.DAT' : // QWK - if (this.mode === QWKPacketReader.Modes.Guess) { - this.mode = QWKPacketReader.Modes.QWK; + if (this.options.mode === QWKPacketReader.Modes.Guess) { + this.options.mode = QWKPacketReader.Modes.QWK; } - if (this.mode === QWKPacketReader.Modes.QWK) { + if (this.options.mode === QWKPacketReader.Modes.QWK) { out.messages = { filename }; } break; case 'ID.MSG' : - if (this.mode === QWKPacketReader.Modes.Guess) { - this.mode = Modes.REP; + if (this.options.mode === QWKPacketReader.Modes.Guess) { + this.options.mode = Modes.REP; } - if (this.mode === QWKPacketReader.Modes.REP) { + if (this.options.mode === QWKPacketReader.Modes.REP) { out.messages = { filename }; } break; @@ -258,12 +312,9 @@ class QWKPacketReader extends EventEmitter { (callback) => { return this.processPacketFiles(callback); }, - (tempDir, callback) => { - return callback(null); - } ], err => { - temptmp.cleanup(); + this.temptmp.cleanup(); if (err) { return this.emit('error', err); @@ -386,14 +437,19 @@ class QWKPacketReader extends EventEmitter { const path = paths.join(this.packetInfo.tempDir, this.packetInfo.headers.filename); fs.readFile(path, { encoding : 'utf8' }, (err, iniData) => { if (err) { - this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid (${err.message})`)); + this.emit('warning', Errors.Invalid(`Problem reading HEADERS.DAT: ${err.message}`)); return cb(null); // non-fatal } try { - this.packetInfo.headers.ini = IniConfigParser.parse(iniData); + const parserOptions = { + lineComment : false, // no line comments; consume full lines + nativeType : false, // just keep everything as strings + dotKey : false, // 'a.b.c = value' stays 'a.b.c = value' + }; + this.packetInfo.headers.ini = IniConfigParser.parse(iniData, parserOptions); } catch (e) { - this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid (${e.message})`)); + this.emit('warning', Errors.Invalid(`HEADERS.DAT file appears to be invalid: ${e.message}`)); } return cb(null); @@ -401,7 +457,6 @@ class QWKPacketReader extends EventEmitter { } readMessages(cb) { - // :TODO: update to use proper encoding: if headers.dat specifies UTF-8, use that, else CP437 if (!this.packetInfo.messages) { return cb(Errors.DoesNotExist('No messages file found within QWK packet')); } @@ -420,7 +475,6 @@ class QWKPacketReader extends EventEmitter { const FTNPropertyMapping = { 'X-FTN-AREA' : Message.FtnPropertyNames.FtnArea, 'X-FTN-SEEN-BY' : Message.FtnPropertyNames.FtnSeenBy, - 'X-FTN-FLAGS' : Message.FtnPropertyNames }; const FTNKludgeMapping = { @@ -452,6 +506,10 @@ class QWKPacketReader extends EventEmitter { Reply : '@REPLY:', TZ : '@TZ:', // https://github.com/kvadevack/synchronet/blob/master/src/smblib/smbdefs.h ReplyTo : '@REPLYTO:', + + // :TODO: Look into other non-standards + // https://github.com/wmcbrine/MultiMail/blob/master/mmail/qwk.cc + // title, @subject, etc. }; let blockCount = 0; @@ -480,7 +538,7 @@ class QWKPacketReader extends EventEmitter { if (0 === blockCount) { // first 128 bytes is a space padded ID const id = buffer.toString('ascii').trim(); - this.emit('generator', id); + this.emit('creator', id); state = 'header'; } else { switch (state) { @@ -493,7 +551,7 @@ class QWKPacketReader extends EventEmitter { header[field] = iconv.decode(header[field], encodingToSpec).trim(); }); - header.timestamp = moment(header.timestamp, 'MM-DD-YYHH:mm'); + header.timestamp = moment(header.timestamp, QWKHeaderTimestampFormat); currMessage = { header, @@ -523,7 +581,7 @@ class QWKPacketReader extends EventEmitter { case 'message' : if (!currMessage.body) { - currMessage.body = buffer; + currMessage.body = Buffer.from(buffer); } else { currMessage.body = Buffer.concat([currMessage.body, buffer]); } @@ -534,16 +592,7 @@ class QWKPacketReader extends EventEmitter { // First, replace QWK style line feeds (0xe3) unless the message is UTF-8. // If the message is UTF-8, we assume it's using standard line feeds. if (encoding !== 'utf8') { - let i = 0; - const QWKLF = Buffer.from([0xe3]); - while (i < currMessage.body.length) { - i = currMessage.body.indexOf(QWKLF, i); - if (-1 === i) { - break; - } - currMessage.body[i] = 0x0a; - ++i; - } + replaceCharInBuffer(currMessage.body, QWKLF, 0x0a); } // @@ -583,15 +632,15 @@ class QWKPacketReader extends EventEmitter { } else if (line.startsWith(Kludges.Subject)) { currMessage.subject = line.substring(Kludges.Subject.length).trim(); } else if (line.startsWith(Kludges.Via)) { - qwkKludge.via = line; + qwkKludge['@VIA'] = line; } else if (line.startsWith(Kludges.MsgID)) { - qwkKludge.msg_id = line.substring(Kludges.MsgID.length).trim(); + qwkKludge['@MSGID'] = line.substring(Kludges.MsgID.length).trim(); } else if (line.startsWith(Kludges.Reply)) { - qwkKludge.in_reply_to_msg_id = line.substring(Kludges.Reply.length).trim(); + qwkKludge['@REPLY'] = line.substring(Kludges.Reply.length).trim(); } else if (line.startsWith(Kludges.TZ)) { - qwkKludge.synchronet_timezone = line.substring(Kludges.TZ.length).trim(); + qwkKludge['@TZ'] = line.substring(Kludges.TZ.length).trim(); } else if (line.startsWith(Kludges.ReplyTo)) { - qwkKludge.reply_to = line.substring(Kludges.ReplyTo.length).trim(); + qwkKludge['@REPLYTO'] = line.substring(Kludges.ReplyTo.length).trim(); } else { bodyState = 'body'; // past this point and up to any tear/origin/etc., is the real message body bodyLines.push(line); @@ -620,12 +669,12 @@ class QWKPacketReader extends EventEmitter { const ext = currMessage.headersExtension; // to and subject can be overridden yet again if entries are present - currMessage.toName = ext.To || currMessage.toName + currMessage.toName = ext.To || currMessage.toName; currMessage.subject = ext.Subject || currMessage.subject; - currMessage.from = ext.Sender || currMessage.from; // why not From? Who the fuck knows. + currMessage.from = ext.Sender || currMessage.fromName; // why not From? Who the fuck knows. // possibly override message ID kludge - qwkKludge.msg_id = ext['Message-ID'] || qwkKludge.msg_id; + qwkKludge['@MSGID'] = ext['Message-ID'] || qwkKludge['@MSGID']; // WhenWritten contains a ISO-8601-ish timestamp and a Synchronet/SMB style TZ offset: // 20180101174837-0600 4168 @@ -640,7 +689,7 @@ class QWKPacketReader extends EventEmitter { } if (ext.Tags) { - currMessage.hashTags = ext.Tags.split(' '); + currMessage.hashTags = (ext.Tags).toString().split(' '); } // FTN style properties/kludges represented as X-FTN-XXXX @@ -668,6 +717,9 @@ class QWKPacketReader extends EventEmitter { hashTags : currMessage.hashTags, }); + // Indicate this message was imported from a QWK packet + message.meta.System[Message.SystemMetaNames.ExternalFlavor] = Message.AddressFlavor.QWK; + if (!_.isEmpty(qwkKludge)) { message.meta.QwkKludge = qwkKludge; } @@ -692,8 +744,8 @@ class QWKPacketReader extends EventEmitter { } // Update the timestamp if we have a valid TZ - if (useTZKludge && _.has(message, 'meta.QwkKludge.synchronet_timezone')) { - const tzOffset = SMBTZToUTCOffset(message.meta.QwkKludge.synchronet_timezone); + if (useTZKludge && qwkKludge['@TZ']) { + const tzOffset = SMBTZToUTCOffset[qwkKludge['@TZ']]; if (tzOffset) { message.modTimestamp.utcOffset(tzOffset); } @@ -704,7 +756,7 @@ class QWKPacketReader extends EventEmitter { qwk_in_reply_to_num : currMessage.header.replyToNum, }; - if (this.mode === QWKPacketReader.Modes.QWK) { + if (this.options.mode === QWKPacketReader.Modes.QWK) { message.meta.QwkProperty.qwk_msg_num = currMessage.header.num; message.meta.QwkProperty.qwk_conf_num = currMessage.header.confNum; } else { @@ -735,7 +787,379 @@ class QWKPacketReader extends EventEmitter { } }; +class QWKPacketWriter extends EventEmitter { + constructor( + { + enableQWKE = true, + enableHeadersExtension = true, + enableAtKludges = true, + encoding = 'cp437', + systemDomain = 'enigma-bbs', + bbsID = '', + toUser = '', + } = QWKPacketWriter.DefaultOptions) + { + super(); + + this.options = { + enableQWKE, + enableHeadersExtension, + enableAtKludges, + systemDomain, + bbsID, + toUser, + encoding : encoding.toLowerCase(), + }; + + this.temptmp = temptmp.createTrackedSession('qwkpacketwriter'); + } + + static get DefaultOptions() { + return { + enableQWKE : true, + enableHeadersExtension : true, + enableAtKludges : true, + encoding : 'cp437', + systemDomain : 'enigma-bbs', + bbsID : '', + toUser : '', + }; + } + + init() { + async.series( + [ + (callback) => { + return StatLog.init(callback); + }, + (callback) => { + this.temptmp.mkdir( { prefix : 'enigqwkwriter-'}, (err, workDir) => { + this.workDir = workDir; + return callback(err); + }); + }, + (callback) => { + this.messagesStream = fs.createWriteStream(paths.join(this.workDir, 'messages.dat')); + + if (this.options.enableHeadersExtension) { + this.headersDatStream = fs.createWriteStream(paths.join(this.workDir, 'headers.dat')); + } + + // First block is a space padded ID + const id = `Created with ENiGMA 1/2 BBS v${enigmaVersion} Copyright (c) 2015-2020 Bryan Ashby`; + this.messagesStream.write(id.padEnd(QWKMessageBlockSize, ' '), 'ascii'); + this.currentMessageOffset = QWKMessageBlockSize; + + this.totalMessages = 0; + this.areaTagsSeen = new Set(); + + return callback(null); + }, + ], + err => { + if (err) { + return this.emit('error', err); + } + + this.emit('ready'); + } + ) + } + + makeMessageIdentifier(message) { + return `<${message.messageId}.${message.messageUuid}@${this.options.systemDomain}>`; + } + + appendMessage(message) { + // + // Each message has to: + // - Append to MESSAGES.DAT + // - Append to HEADERS.DAT if enabled + // + // If this is a personal (ie: non-network) packet: + // - Produce PERSONAL.NDX + // - Produce 000.NDX with pointers to the users personal "inbox" mail + // - Produce ####.NDX with pointers to the public/conference mail + // - Produce TOREADER.EXT if QWKE support is enabled + // + + let fullMessageBody = ''; + + // Start of body is kludges if enabled + if (this.options.enableQWKE) { + if (message.toUserName.length > 25) { + fullMessageBody += `To: ${message.toUserName}\n`; + } + if (message.fromUserName.length > 25) { + fullMessageBody += `From: ${message.fromUserName}\n`; + } + if (message.subject.length > 25) { + fullMessageBody += `Subject: ${message.subject}\n`; + } + } + + if (this.options.enableAtKludges) { + // Add in original kludges (perhaps in a different order) if + // they were originally imported + if (Message.AddressFlavor.QWK == message.meta.System[Message.SystemMetaNames.ExternalFlavor]) { + if (message.meta.QwkKludge) { + for (let [kludge, value] of Object.entries(message.meta.QwkKludge)) { + fullMessageBody += `${kludge}: ${value}\n`; + }; + } + } else { + fullMessageBody += `@MSGID: ${this.makeMessageIdentifier(message)}\n`; + } + } + + // The actual message contents + fullMessageBody += message.message; + + // :TODO: sanitize line feeds -> \n ???? + + // splitTextAtTerms(message.message).forEach(line => { + // appendBodyLine(line); + // }); + + const encodedMessage = iconv.encode(fullMessageBody, this.options.encoding); + + // + // QWK spec wants line feeds as 0xe3 for some reason, so we'll have + // to replace the \n's. If we're going against the spec and using UTF-8 + // we can just leave them be. + // + if ('utf8' !== this.options.encoding) { + replaceCharInBuffer(encodedMessage, 0x0a, QWKLF); + } + + // Messages must comprise of multiples of 128 bit blocks with the last + // block padded by spaces or nulls (we use nulls) + const fullBlocks = Math.trunc(encodedMessage.length / QWKMessageBlockSize); + const remainBytes = QWKMessageBlockSize - (encodedMessage.length % QWKMessageBlockSize); + + // The first block is always a header + this._writeMessageHeader( + message, + fullBlocks + 1 + (remainBytes ? 1 : 0), + ); + + this.messagesStream.write(encodedMessage); + + + if (remainBytes) { + this.messagesStream.write(Buffer.alloc(remainBytes, 0x00)); + } + + if (this.options.enableHeadersExtension) { + this._appendHeadersExtensionData(message); + } + + this.currentMessageOffset += fullBlocks * QWKMessageBlockSize; + + if (remainBytes) + { + this.currentMessageOffset += QWKMessageBlockSize; + } + + this.totalMessages += 1; + this.areaTagsSeen.add(message.areaTag); + } + + appendNewFile() { + + } + + finish(packetPath) { + async.series( + [ + (callback) => { + this.messagesStream.on('close', () => { + return callback(null); + }); + this.messagesStream.end(); + }, + (callback) => { + if (!this.headersDatStream) { + return callback(null); + } + this.headersDatStream.on('close', () => { + return callback(null); + }); + this.headersDatStream.end(); + }, + (callback) => { + return this._createControlData(callback); + } + ], + err => { + this.temptmp.cleanup(); + + if (err) { + return this.emit('error', err); + } + + this.emit('finished'); + } + ) + } + + _writeMessageHeader(message, totalBlocks) { + const asciiNum = (n, l) => { + if (isNaN(n)) { + return ''; + } + return n.toString().substr(0, l); + }; + + const status = 'FIXME'; + const totalBlocksStr = asciiNum(totalBlocks, 6);//totalBlocks.toString().padEnd(6, ' '); + const messageStatus = 255; // :TODO: ever anything different? + const confNumber = 1004; // :TODO: areaTag -> conf mapping + const netTag = ' '; // :TODO: + + if (totalBlocksStr.length > 6) { + return this.emit('warning', Errors.General('Message too large for packet'), message); + } + + const header = Buffer.alloc(QWKMessageBlockSize, ' '); + header.write(status[0], 0, 1, 'ascii'); + header.write(asciiNum(message.messageId), 1, 'ascii'); // :TODO: It seems Sync puts the relative, as in # of messages we've called appendMessage()?! + header.write(message.modTimestamp.format(QWKHeaderTimestampFormat), 8, 13, 'ascii'); + header.write(message.toUserName.substr(0, 25), 21, 'ascii'); + header.write(message.fromUserName.substr(0, 25), 46, 'ascii'); + header.write(message.subject.substr(0, 25), 71, 'ascii'); + header.write(' '.repeat(12), 96, 'ascii'); // we don't use the password field + header.write(asciiNum(message.replyToMsgId), 108, 'ascii'); + header.write(asciiNum(totalBlocks, 6), 116, 'ascii'); + header.writeUInt8(messageStatus, 122); + header.writeUInt16LE(confNumber, 123); + header.writeUInt16LE(0, 125); // :TODO: Check if others actually populate this + header.write(netTag[0], 127, 1, 'ascii'); + + this.messagesStream.write(header); + } + + _createControlData(cb) { + const controlStream = fs.createWriteStream(paths.join(this.workDir, 'control.dat')); + controlStream.setDefaultEncoding('ascii'); + + controlStream.on('close', () => { + return cb(null); + }); + + controlStream.on('error', err => { + return cb(err); + }); + + const controlData = [ + Config().general.boardName, + 'Earth', + 'XXX-XXX-XXX', + `${StatLog.getSystemStat(SysProps.SysOpUsername)}, Sysop`, + `0000,${this.options.bbsID}`, + moment().format('MM-DD-YYYY,HH:mm:ss'), + this.options.toUser, + '', // name of Qmail menu + '0', // uh, OK + this.totalMessages.toString(), + // this next line is total conferences - 1: + // We have areaTag <> conference mapping, so the number should work out + (this.areaTagsSeen.size - 1).toString(), + + // :TODO: append all areaTag->conf number/IDs and names (13 chars max) + '0', 'First Conf', + 'HELLO', + 'BBSNEWS', + 'GOODBYE', + ]; + + controlData.forEach(line => { + controlStream.write(`${line}\r\n`); + }); + + controlStream.end(); + } + + _makeSynchronetTimestamp(ts) { + const syncTimestamp = ts.format('YYYYMMDDHHmmssZZ'); + const syncTZ = UTCOffsetToSMBTZ[ts.format('Z')] || '0000'; // :TODO: what if we don't have a map? + return `${syncTimestamp} ${syncTZ}`; + } + + _appendHeadersExtensionData(message) { + const messageData = { + // Synchronet style + Utf8 : ('utf8' === this.options.encoding ? 'true' : 'false'), + 'Message-ID' : this.makeMessageIdentifier(message), + + WhenWritten : this._makeSynchronetTimestamp(message.modTimestamp), + // WhenImported : '', // :TODO: only if we have a imported time from another external system? + ExportedFrom : `${this.options.systemID} ${message.areaTag} ${message.messageId}`, + Sender : message.fromUserName, + + // :TODO: if exporting for QWK-Net style/etc. + //SenderNetAddr + + SenderIpAddr : '127.0.0.1', // no sir, that's private. + SenderHostName : this.options.systemDomain, + // :TODO: if exported: + //SenderProtocol + Organization : 'BBS', + + //'Reply-To' : :TODO: "address to direct replies".... ?! + Subject : message.subject, + To : message.toUserName, + //ToNetAddr : :TODO: net addr to?! + + // :TODO: Only set if not imported: + Tags : message.hashTags.join(' '), + + // :TODO: Needs tested with Sync/etc.; Sync wants Conference *numbers* + Conference : getMessageConfTagByAreaTag(message.areaTag), + + // ENiGMA Headers + MessageUUID : message.uuid, + ModTimestamp : message.modTimestamp.format('YYYY-MM-DDTHH:mm:ss.SSSZ'), + AreaTag : message.areaTag, + }; + + const externalFlavor = message.meta.System[Message.SystemMetaNames.ExternalFlavor]; + if (externalFlavor === Message.AddressFlavor.FTN) { + // Add FTN properties if it came from such an origin + if (message.meta.FtnProperty) { + const ftnProp = message.meta.FtnProperty; + messageData['X-FTN-AREA'] = ftnProp[Message.FtnPropertyNames.FtnArea]; + messageData['X-FTN-SEEN-BY'] = fntProp[Message.FtnPropertyNames.FtnSeenBy]; + } + + if (message.meta.FtnKludge) { + const ftnKludge = message.meta.FtnKludge; + messageData['X-FTN-PATH'] = ftnKludge.PATH; + messageData['X-FTN-MSGID'] = ftnKludge.MSGID; + messageData['X-FTN-REPLY'] = fntKludge.REPLY; + messageData['X-FTN-PID'] = fntKludge.PID; + messageData['X-FTN-FLAGS'] = ftnKludge.FLAGS; + messageData['X-FTN-TID'] = fntKludge.TID; + messageData['X-FTN-CHRS'] = fntKludge.CHRS; + } + } else { + messageData.WhenExported = this._makeSynchronetTimestamp(moment()); + messageData.Editor = `ENiGMA 1/2 BBS FSE v${enigmaVersion}`; + } + + this.headersDatStream.write(iconv.encode(`[${this.currentMessageOffset.toString(16)}]\r\n`, this.options.encoding)); + + for (let [name, value] of Object.entries(messageData)) { + if (value) { + this.headersDatStream.write(iconv.encode(`${name}: ${value}\r\n`, this.options.encoding)); + } + } + + this.headersDatStream.write('\r\n'); + } +} + module.exports = { QWKPacketReader, -// QWKPacketWriter, + QWKPacketWriter, } From e1b0a3553b72a98869dbcec97a0e1f27cbdde4a0 Mon Sep 17 00:00:00 2001 From: squintgit Date: Sun, 26 Apr 2020 18:20:42 -0500 Subject: [PATCH 14/95] Update testing.md Adding instructions for shutting down server, which may not be immediately clear to new sysops --- docs/installation/testing.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/installation/testing.md b/docs/installation/testing.md index bfd2de26..576b28fd 100644 --- a/docs/installation/testing.md +++ b/docs/installation/testing.md @@ -26,7 +26,9 @@ _____________________ _____ ____________________ __________\_ / System started! ``` -Grab your favourite telnet client, connect to localhost:8888 and test out your installation. +Grab your favourite telnet client, connect to localhost:8888 and test out your installation. + +To shut down the server, press Ctrl-C. ## Points of Interest @@ -44,4 +46,4 @@ If you don't have any telnet software, these are compatible with ENiGMA½: * [NetRunner](http://mysticbbs.com/downloads.html) * [MagiTerm](https://magickabbs.com/index.php/magiterm/) * [VTX](https://github.com/codewar65/VTX_ClientServer) (Browser based) -* [fTelnet](https://www.ftelnet.ca/) (Browser based) \ No newline at end of file +* [fTelnet](https://www.ftelnet.ca/) (Browser based) From 2b7d810c777ddb96961c179305de456c50d5d894 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 27 Apr 2020 20:55:41 -0600 Subject: [PATCH 15/95] Lots of progress on packet writing, reading, etc. * Bug fixes * Create packet archive --- core/archive_util.js | 26 ++++-- core/qwk_mail_packet.js | 180 +++++++++++++++++++++++++++++++--------- 2 files changed, 160 insertions(+), 46 deletions(-) diff --git a/core/archive_util.js b/core/archive_util.js index 8549cd12..47291860 100644 --- a/core/archive_util.js +++ b/core/archive_util.js @@ -204,23 +204,37 @@ module.exports = class ArchiveUtil { }); } - compressTo(archType, archivePath, files, cb) { + compressTo(archType, archivePath, files, workDir, cb) { const archiver = this.getArchiver(archType, paths.extname(archivePath)); if(!archiver) { return cb(Errors.Invalid(`Unknown archive type: ${archType}`)); } + if (!cb && _.isFunction(workDir)) { + cb = workDir; + workDir = null; + } + const fmtObj = { archivePath : archivePath, fileList : files.join(' '), // :TODO: probably need same hack as extractTo here! }; - const args = archiver.compress.args.map( arg => stringFormat(arg, fmtObj) ); + // :TODO: DRY with extractTo() + const args = archiver.compress.args.map( arg => { + return '{fileList}' === arg ? arg : stringFormat(arg, fmtObj); + }); + + const fileListPos = args.indexOf('{fileList}'); + if(fileListPos > -1) { + // replace {fileList} with 0:n sep file list arguments + args.splice.apply(args, [fileListPos, 1].concat(files)); + } let proc; try { - proc = pty.spawn(archiver.compress.cmd, args, this.getPtyOpts()); + proc = pty.spawn(archiver.compress.cmd, args, this.getPtyOpts(workDir)); } catch(e) { return cb(Errors.ExternalProcess( `Error spawning archiver process "${archiver.compress.cmd}" with args "${args.join(' ')}": ${e.message}`) @@ -332,15 +346,15 @@ module.exports = class ArchiveUtil { }); } - getPtyOpts(extractPath) { + getPtyOpts(cwd) { const opts = { name : 'enigma-archiver', cols : 80, rows : 24, env : process.env, }; - if(extractPath) { - opts.cwd = extractPath; + if(cwd) { + opts.cwd = cwd; } // :TODO: set cwd to supplied temp path if not sepcific extract return opts; diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index 5dc3275f..e5c316b1 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -2,7 +2,10 @@ const ArchiveUtil = require('./archive_util'); const { Errors } = require('./enig_error'); const Message = require('./message'); const { splitTextAtTerms } = require('./string_util'); -const { getMessageConfTagByAreaTag } = require('./message_area'); +const { + getMessageConfTagByAreaTag, + getMessageAreaByTag, +} = require('./message_area'); const StatLog = require('./stat_log'); const Config = require('./config').get; const SysProps = require('./system_property'); @@ -77,6 +80,26 @@ const QWKMessageBlockSize = 128; const QWKHeaderTimestampFormat = 'MM-DD-YYHH:mm'; const QWKLF = 0xe3; +const QWKMessageStatusCodes = { + UnreadPublic : ' ', + ReadPublic : '-', + ReadBySomeonePrivate : '*', + UnreadPrivate : '+', + UnreadCommentToSysOp : '~', + ReadCommentToSysOp : '`', + UnreadSenderPWProtected : '%', + ReadSenderPWProtected : '^', + UnreadGroupPWProtected : '!', + ReadGroupPWProtected : '#', + PWProtectedToAll : '$', + Vote : 'V', +}; + +const QWKMessageActiveStatus = { + Active : 255, + Deleted : 226, +}; + // See the following: // - http://fileformats.archiveteam.org/wiki/QWK // - http://wiki.synchro.net/ref:qwk @@ -796,7 +819,8 @@ class QWKPacketWriter extends EventEmitter { encoding = 'cp437', systemDomain = 'enigma-bbs', bbsID = '', - toUser = '', + user = null, + archiveFormat = 'application/zip', } = QWKPacketWriter.DefaultOptions) { super(); @@ -807,7 +831,8 @@ class QWKPacketWriter extends EventEmitter { enableAtKludges, systemDomain, bbsID, - toUser, + user, + archiveFormat, encoding : encoding.toLowerCase(), }; @@ -822,7 +847,8 @@ class QWKPacketWriter extends EventEmitter { encoding : 'cp437', systemDomain : 'enigma-bbs', bbsID : '', - toUser : '', + user : null, + archiveFormat :'application/zip', }; } @@ -913,13 +939,12 @@ class QWKPacketWriter extends EventEmitter { } // The actual message contents - fullMessageBody += message.message; + //fullMessageBody += message.message; - // :TODO: sanitize line feeds -> \n ???? - - // splitTextAtTerms(message.message).forEach(line => { - // appendBodyLine(line); - // }); + // Sanitize line feeds (e.g. CRLF -> LF, and possibly -> QWK style below) + splitTextAtTerms(message.message).forEach(line => { + fullMessageBody += `${line}\n`; + }); const encodedMessage = iconv.encode(fullMessageBody, this.options.encoding); @@ -938,16 +963,20 @@ class QWKPacketWriter extends EventEmitter { const remainBytes = QWKMessageBlockSize - (encodedMessage.length % QWKMessageBlockSize); // The first block is always a header - this._writeMessageHeader( + if (!this._writeMessageHeader( message, fullBlocks + 1 + (remainBytes ? 1 : 0), - ); + )) + { + // we can't write this message + return; + } this.messagesStream.write(encodedMessage); if (remainBytes) { - this.messagesStream.write(Buffer.alloc(remainBytes, 0x00)); + this.messagesStream.write(Buffer.alloc(remainBytes, ' ')); } if (this.options.enableHeadersExtension) { @@ -989,6 +1018,9 @@ class QWKPacketWriter extends EventEmitter { }, (callback) => { return this._createControlData(callback); + }, + (callback) => { + return this._producePacketArchive(packetPath, callback); } ], err => { @@ -1003,6 +1035,41 @@ class QWKPacketWriter extends EventEmitter { ) } + _producePacketArchive(packetPath, cb) { + const archiveUtil = ArchiveUtil.getInstance(); + + const packetFiles = [ + 'messages.dat', 'headers.dat', 'control.dat', + ].map(filename => { + return filename; + //return paths.join(this.workDir, filename); + }); + + archiveUtil.compressTo( + this.options.archiveFormat, + packetPath, + packetFiles, + this.workDir, + err => { + return cb(err); + } + ); + } + + _qwkMessageStatus(message) { + // - Public vs Private + // - Look at message pointers for read status + // - If +op is exporting and this message is to +op + // - + // :TODO: this needs addressed - handle unread vs read, +op, etc. + // ....see getNewMessagesInAreaForUser(); Variant with just IDs, or just a way to get first new message ID per area? + + if (message.isPrivate()) { + return QWKMessageStatusCodes.UnreadPrivate; + } + return QWKMessageStatusCodes.UnreadPublic; + } + _writeMessageHeader(message, totalBlocks) { const asciiNum = (n, l) => { if (isNaN(n)) { @@ -1011,18 +1078,26 @@ class QWKPacketWriter extends EventEmitter { return n.toString().substr(0, l); }; - const status = 'FIXME'; - const totalBlocksStr = asciiNum(totalBlocks, 6);//totalBlocks.toString().padEnd(6, ' '); - const messageStatus = 255; // :TODO: ever anything different? - const confNumber = 1004; // :TODO: areaTag -> conf mapping - const netTag = ' '; // :TODO: - - if (totalBlocksStr.length > 6) { - return this.emit('warning', Errors.General('Message too large for packet'), message); + const asciiTotalBlocks = asciiNum(totalBlocks, 6); + if (asciiTotalBlocks.length > 6) { + this.emit('warning', Errors.General('Message too large for packet'), message); + return false; } + const conferenceNumber = this._getMessageConferenceNumberByAreaTag(message.areaTag); + if (isNaN(conferenceNumber)) { + this.emit('warning', Errors.MissingConfig(`No QWK conference mapping for areaTag ${message.areaTag}`)); + return false; + } + + const netTag = ' '; // :TODO: + + this.lolMessageId = this.lolMessageId || 1; + //message.messageId = this.lolMessageId; + this.lolMessageId++; + const header = Buffer.alloc(QWKMessageBlockSize, ' '); - header.write(status[0], 0, 1, 'ascii'); + header.write(this._qwkMessageStatus(message), 0, 1, 'ascii'); header.write(asciiNum(message.messageId), 1, 'ascii'); // :TODO: It seems Sync puts the relative, as in # of messages we've called appendMessage()?! header.write(message.modTimestamp.format(QWKHeaderTimestampFormat), 8, 13, 'ascii'); header.write(message.toUserName.substr(0, 25), 21, 'ascii'); @@ -1030,16 +1105,35 @@ class QWKPacketWriter extends EventEmitter { header.write(message.subject.substr(0, 25), 71, 'ascii'); header.write(' '.repeat(12), 96, 'ascii'); // we don't use the password field header.write(asciiNum(message.replyToMsgId), 108, 'ascii'); - header.write(asciiNum(totalBlocks, 6), 116, 'ascii'); - header.writeUInt8(messageStatus, 122); - header.writeUInt16LE(confNumber, 123); + header.write(asciiTotalBlocks, 116, 'ascii'); + header.writeUInt8(QWKMessageActiveStatus.Active, 122); + header.writeUInt16LE(conferenceNumber, 123); header.writeUInt16LE(0, 125); // :TODO: Check if others actually populate this header.write(netTag[0], 127, 1, 'ascii'); this.messagesStream.write(header); + + return true; + } + + _getMessageConferenceNumberByAreaTag(areaTag) { + const areaConfig = _.get(Config(), [ 'messageNetworks', 'qwk', 'areas', areaTag ]); + return areaConfig && areaConfig.conference; + } + + _getExportForUsername() { + return _.get(this.options, 'user.username', 'Any'); + } + + _getExportSysOpUsername() { + return StatLog.getSystemStat(SysProps.SysOpUsername) || 'SysOp'; } _createControlData(cb) { + const areas = Array.from(this.areaTagsSeen).map(areaTag => { + return getMessageAreaByTag(areaTag); + }); + const controlStream = fs.createWriteStream(paths.join(this.workDir, 'control.dat')); controlStream.setDefaultEncoding('ascii'); @@ -1051,32 +1145,38 @@ class QWKPacketWriter extends EventEmitter { return cb(err); }); - const controlData = [ + const initialControlData = [ Config().general.boardName, 'Earth', 'XXX-XXX-XXX', - `${StatLog.getSystemStat(SysProps.SysOpUsername)}, Sysop`, + `${this._getExportSysOpUsername()}, Sysop`, `0000,${this.options.bbsID}`, moment().format('MM-DD-YYYY,HH:mm:ss'), - this.options.toUser, + this._getExportForUsername(), '', // name of Qmail menu '0', // uh, OK this.totalMessages.toString(), // this next line is total conferences - 1: // We have areaTag <> conference mapping, so the number should work out (this.areaTagsSeen.size - 1).toString(), - - // :TODO: append all areaTag->conf number/IDs and names (13 chars max) - '0', 'First Conf', - 'HELLO', - 'BBSNEWS', - 'GOODBYE', ]; - controlData.forEach(line => { + initialControlData.forEach(line => { controlStream.write(`${line}\r\n`); }); + // map areas as conf #\r\nDescription\r\n pairs + areas.forEach(area => { + const conferenceNumber = this._getMessageConferenceNumberByAreaTag(area.areaTag); + controlStream.write(`${conferenceNumber}\r\n`); + controlStream.write(`${area.name}\r\n`); + }); + + // :TODO: do we ever care here?! + ['HELLO', 'BBSNEWS', 'GOODBYE'].forEach(trailer => { + controlStream.write(`${trailer}\r\n`); + }); + controlStream.end(); } @@ -1129,18 +1229,18 @@ class QWKPacketWriter extends EventEmitter { if (message.meta.FtnProperty) { const ftnProp = message.meta.FtnProperty; messageData['X-FTN-AREA'] = ftnProp[Message.FtnPropertyNames.FtnArea]; - messageData['X-FTN-SEEN-BY'] = fntProp[Message.FtnPropertyNames.FtnSeenBy]; + messageData['X-FTN-SEEN-BY'] = ftnProp[Message.FtnPropertyNames.FtnSeenBy]; } if (message.meta.FtnKludge) { const ftnKludge = message.meta.FtnKludge; messageData['X-FTN-PATH'] = ftnKludge.PATH; messageData['X-FTN-MSGID'] = ftnKludge.MSGID; - messageData['X-FTN-REPLY'] = fntKludge.REPLY; - messageData['X-FTN-PID'] = fntKludge.PID; + messageData['X-FTN-REPLY'] = ftnKludge.REPLY; + messageData['X-FTN-PID'] = ftnKludge.PID; messageData['X-FTN-FLAGS'] = ftnKludge.FLAGS; - messageData['X-FTN-TID'] = fntKludge.TID; - messageData['X-FTN-CHRS'] = fntKludge.CHRS; + messageData['X-FTN-TID'] = ftnKludge.TID; + messageData['X-FTN-CHRS'] = ftnKludge.CHRS; } } else { messageData.WhenExported = this._makeSynchronetTimestamp(moment()); From 1f1813c14a619fda8bac46afc12b1bea852bd60e Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 30 Apr 2020 22:07:29 -0600 Subject: [PATCH 16/95] Produce NDX files, various improvements to spec, etc. --- core/mbf.js | 59 +++++++++ core/oputil/oputil_message_base.js | 203 ++++++++++++++++++++++++++--- core/qwk_mail_packet.js | 194 +++++++++++++++++++++------ 3 files changed, 399 insertions(+), 57 deletions(-) create mode 100644 core/mbf.js diff --git a/core/mbf.js b/core/mbf.js new file mode 100644 index 00000000..9c3b2f6d --- /dev/null +++ b/core/mbf.js @@ -0,0 +1,59 @@ +const { Errors } = require('./enig_error'); + +// +// Utils for dealing with Microsoft Binary Format (MBF) used +// in various BASIC languages, etc. +// +// - https://en.wikipedia.org/wiki/Microsoft_Binary_Format +// - https://stackoverflow.com/questions/2268191/how-to-convert-from-ieee-python-float-to-microsoft-basic-float +// + +// Number to 32bit MBF +numToMbf32 = (v) => { + const mbf = Buffer.alloc(4); + + if (0 === v) { + return mbf; + } + + const ieee = Buffer.alloc(4); + ieee.writeFloatLE(v, 0); + + const sign = ieee[3] & 0x80; + let exp = (ieee[3] << 1) | (ieee[2] >> 7); + + if (exp === 0xfe) { + throw Errors.Invalid(`${v} cannot be converted to mbf`); + } + + exp += 2; + + mbf[3] = exp; + mbf[2] = sign | (ieee[2] & 0x7f); + mbf[1] = ieee[1]; + mbf[0] = ieee[0]; + + return mbf; +} + +mbf32ToNum = (mbf) => { + if (0 === mbf[3]) { + return 0.0; + } + + const ieee = Buffer.alloc(4); + const sign = mbf[2] & 0x80; + const exp = mbf[3] - 2; + + ieee[3] = sign | (exp >> 1); + ieee[2] = (exp << 7) | (mbf[2] & 0x7f); + ieee[1] = mbf[1]; + ieee[0] = mbf[0]; + + return ieee.readFloatLE(0); +} + +module.exports = { + numToMbf32, + mbf32ToNum, +} \ No newline at end of file diff --git a/core/oputil/oputil_message_base.js b/core/oputil/oputil_message_base.js index e5b30c12..7d914ab2 100644 --- a/core/oputil/oputil_message_base.js +++ b/core/oputil/oputil_message_base.js @@ -445,6 +445,9 @@ function handleQWK() { case 'dump' : return dumpQWKPacket(packetPath); + case 'export' : + return exportQWKPacket(packetPath); + default : return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); } @@ -457,35 +460,81 @@ function dumpQWKPacket(packetPath) { return initConfigAndDatabases(callback); }, (callback) => { + //// + const { QWKPacketWriter } = require('../qwk_mail_packet'); + const writer = new QWKPacketWriter({ + bbsID : 'XIBALBA', + toUser : 'NuSkooler', + encoding : 'cp437', + }); + const { QWKPacketReader } = require('../qwk_mail_packet'); - const reader = new QWKPacketReader(packetPath); - reader.on('error', err => { - console.error(`ERROR: ${err.message}`); - return callback(err); + + writer.on('ready', () => { + const reader = new QWKPacketReader(packetPath); + + reader.on('error', err => { + console.error(`ERROR: ${err.message}`); + return callback(err); + }); + + reader.on('done', () => { + writer.finish(); + }); + + reader.on('archive type', archiveType => { + console.info(`-> Archive type: ${archiveType}`); + }); + + reader.on('creator', creator => { + console.info(`-> Creator: ${creator}`); + }); + + reader.on('message', message => { + writer.appendMessage(message); + }); + + reader.read(); }); - reader.on('done', () => { - return callback(null); + writer.on('finished', () => { + console.log('done'); }); - reader.on('archive type', archiveType => { - console.info(`-> Archive type: ${archiveType}`); - }); + writer.init(); - reader.on('creator', creator => { - console.info(`-> Creator: ${creator}`); - }); + //// - reader.on('message', message => { - console.info('--- message ---'); - console.info(`To : ${message.toUserName}`); - console.info(`From : ${message.fromUserName}`); - console.info(`Subject : ${message.subject}`); - console.info(`Message :\r\n${message.message}`); - }); + // const { QWKPacketReader } = require('../qwk_mail_packet'); + // const reader = new QWKPacketReader(packetPath); - reader.read(); + // reader.on('error', err => { + // console.error(`ERROR: ${err.message}`); + // return callback(err); + // }); + + // reader.on('done', () => { + // return callback(null); + // }); + + // reader.on('archive type', archiveType => { + // console.info(`-> Archive type: ${archiveType}`); + // }); + + // reader.on('creator', creator => { + // console.info(`-> Creator: ${creator}`); + // }); + + // reader.on('message', message => { + // console.info('--- message ---'); + // console.info(`To : ${message.toUserName}`); + // console.info(`From : ${message.fromUserName}`); + // console.info(`Subject : ${message.subject}`); + // console.info(`Message :\r\n${message.message}`); + // }); + + // reader.read(); } ], err => { @@ -494,6 +543,120 @@ function dumpQWKPacket(packetPath) { ) } +function exportQWKPacket(packetPath) { + // oputil mb qwk export SPEC PATH [--user USER] + // [areaTag1[@dateTime]],[...] PATH --user USER + + const posArgLen = argv._.length; + + if (posArgLen < 4) { + return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + } + + let areaTagSpecs = '*'; + if (5 === posArgLen) { + areaTagSpecs = argv._[areaTagSpecs - 2]; + } + + + //const areaTagSpecs = argv._[areaTagSpecs - 2]; + + /*const packetPath = argv._[argv._.length - 1]; + if(argv._.length < 4 || !packetPath || 0 === packetPath.length) { + return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + }*/ + + // :TODO: parse area tags(s) and timestamps + const areaTags = [ 'general', 'fsx_gen' ]; + + const userName = argv.user || '-'; + + async.waterfall( + [ + (callback) => { + return initConfigAndDatabases(callback); + }, + (callback) => { + const User = require('../../core/user.js'); + + User.getUserIdAndName(userName, (err, userId) => { + if (err) { + if ('-' === userName) { + userId = 1; + } else { + return callback(err); + } + } + return User.getUser(userId, callback); + }); + }, + (user, callback) => { + const Message = require('../message'); + + const filter = { + resultType : 'id', + areaTag : areaTags, + + // :TODO: newerThanTimestamp + }; + + // public + Message.findMessages(filter, (err, publicMessageIds) => { + if (err) { + return callback(err); + } + + delete filter.areaTag; + filter.privateTagUserId = user.userId; + + Message.findMessages(filter, (err, privateMessageIds) => { + return callback(err, user, Message, privateMessageIds.concat(publicMessageIds)); + }); + }); + }, + (user, Message, messageIds, callback) => { + const { QWKPacketWriter } = require('../qwk_mail_packet'); + const writer = new QWKPacketWriter({ + // :TODO: export needs these options + bbsID : 'XIBALBA', + toUser : 'NuSkooler', + encoding : 'cp437', + user : user, + }); + + writer.on('ready', () => { + async.eachSeries(messageIds, (messageId, nextMessageId) => { + const message = new Message(); + message.load( { messageId }, err => { + if (!err) { + writer.appendMessage(message); + } + return nextMessageId(err); + }); + }, + (err) => { + writer.finish('/home/nuskooler/Downloads/qwk2/TEST1.QWK'); + if (err) { + console.error(`Failed to write one or more messages: ${err.message}`); + } + }); + }); + + writer.on('finished', () => { + return callback(null); + }); + + writer.init(); + } + ], + err => { + if(err) { + console.error(err.reason ? err.reason : err.message); + } + } + ); +} + function handleMessageBaseCommand() { function errUsage() { diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index e5c316b1..761b71f7 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -9,6 +9,8 @@ const { const StatLog = require('./stat_log'); const Config = require('./config').get; const SysProps = require('./system_property'); +const UserProps = require('./user_property'); +const { numToMbf32 } = require('./mbf'); const { EventEmitter } = require('events'); const temptmp = require('temptmp'); @@ -83,8 +85,8 @@ const QWKLF = 0xe3; const QWKMessageStatusCodes = { UnreadPublic : ' ', ReadPublic : '-', - ReadBySomeonePrivate : '*', UnreadPrivate : '+', + ReadPrivate : '*', UnreadCommentToSysOp : '~', ReadCommentToSysOp : '`', UnreadSenderPWProtected : '%', @@ -100,6 +102,11 @@ const QWKMessageActiveStatus = { Deleted : 226, }; +const QWKNetworkTagIndicator = { + Present : '*', + NotPresent : ' ', +}; + // See the following: // - http://fileformats.archiveteam.org/wiki/QWK // - http://wiki.synchro.net/ref:qwk @@ -878,6 +885,9 @@ class QWKPacketWriter extends EventEmitter { this.totalMessages = 0; this.areaTagsSeen = new Set(); + this.personalIndex = []; // messages addressed to 'user' + this.inboxIndex = []; // private messages for 'user' + this.publicIndex = new Map(); return callback(null); }, @@ -938,9 +948,6 @@ class QWKPacketWriter extends EventEmitter { } } - // The actual message contents - //fullMessageBody += message.message; - // Sanitize line feeds (e.g. CRLF -> LF, and possibly -> QWK style below) splitTextAtTerms(message.message).forEach(line => { fullMessageBody += `${line}\n`; @@ -961,11 +968,12 @@ class QWKPacketWriter extends EventEmitter { // block padded by spaces or nulls (we use nulls) const fullBlocks = Math.trunc(encodedMessage.length / QWKMessageBlockSize); const remainBytes = QWKMessageBlockSize - (encodedMessage.length % QWKMessageBlockSize); + const totalBlocks = fullBlocks + 1 + (remainBytes ? 1 : 0); // The first block is always a header if (!this._writeMessageHeader( message, - fullBlocks + 1 + (remainBytes ? 1 : 0), + totalBlocks )) { // we can't write this message @@ -974,26 +982,61 @@ class QWKPacketWriter extends EventEmitter { this.messagesStream.write(encodedMessage); - if (remainBytes) { this.messagesStream.write(Buffer.alloc(remainBytes, ' ')); } + this._updateIndexTracking(message); + if (this.options.enableHeadersExtension) { this._appendHeadersExtensionData(message); } - this.currentMessageOffset += fullBlocks * QWKMessageBlockSize; - - if (remainBytes) - { - this.currentMessageOffset += QWKMessageBlockSize; - } + // next message starts at this block + this.currentMessageOffset += totalBlocks * QWKMessageBlockSize; this.totalMessages += 1; this.areaTagsSeen.add(message.areaTag); } + _messageAddressedToUser(message) { + if (_.isUndefined(this.cachedCompareNames)) { + if (this.options.user) { + this.cachedCompareNames = [ + this.options.user.username.toLowerCase() + ]; + const realName = this.options.user.getProperty(UserProps.RealName); + if (realName) { + this.cachedCompareNames.push(realName.toLowerCase()); + } + } else { + this.cachedCompareNames = []; + } + }; + + return this.cachedCompareNames.includes(message.toUserName.toLowerCase()); + } + + _updateIndexTracking(message) { + // index points at start of *message* not the header for... reasons? + const index = (this.currentMessageOffset / QWKMessageBlockSize) + 1; + if (message.isPrivate()) { + this.inboxIndex.push(index); + } else { + if (this._messageAddressedToUser(message)) { + // :TODO: add to both indexes??? + this.personalIndex.push(index); + } + + const areaTag = message.areaTag; + if (!this.publicIndex.has(areaTag)) { + this.publicIndex.set(areaTag, [index]); + } else { + this.publicIndex.get(areaTag).push(index); + } + } + } + appendNewFile() { } @@ -1019,6 +1062,9 @@ class QWKPacketWriter extends EventEmitter { (callback) => { return this._createControlData(callback); }, + (callback) => { + return this._createIndexes(callback); + }, (callback) => { return this._producePacketArchive(packetPath, callback); } @@ -1038,22 +1084,21 @@ class QWKPacketWriter extends EventEmitter { _producePacketArchive(packetPath, cb) { const archiveUtil = ArchiveUtil.getInstance(); - const packetFiles = [ - 'messages.dat', 'headers.dat', 'control.dat', - ].map(filename => { - return filename; - //return paths.join(this.workDir, filename); - }); - - archiveUtil.compressTo( - this.options.archiveFormat, - packetPath, - packetFiles, - this.workDir, - err => { + fs.readdir(this.workDir, (err, files) => { + if (err) { return cb(err); } - ); + + archiveUtil.compressTo( + this.options.archiveFormat, + packetPath, + files, + this.workDir, + err => { + return cb(err); + } + ); + }); } _qwkMessageStatus(message) { @@ -1090,15 +1135,9 @@ class QWKPacketWriter extends EventEmitter { return false; } - const netTag = ' '; // :TODO: - - this.lolMessageId = this.lolMessageId || 1; - //message.messageId = this.lolMessageId; - this.lolMessageId++; - const header = Buffer.alloc(QWKMessageBlockSize, ' '); header.write(this._qwkMessageStatus(message), 0, 1, 'ascii'); - header.write(asciiNum(message.messageId), 1, 'ascii'); // :TODO: It seems Sync puts the relative, as in # of messages we've called appendMessage()?! + header.write(asciiNum(message.messageId), 1, 'ascii'); header.write(message.modTimestamp.format(QWKHeaderTimestampFormat), 8, 13, 'ascii'); header.write(message.toUserName.substr(0, 25), 21, 'ascii'); header.write(message.fromUserName.substr(0, 25), 46, 'ascii'); @@ -1108,8 +1147,8 @@ class QWKPacketWriter extends EventEmitter { header.write(asciiTotalBlocks, 116, 'ascii'); header.writeUInt8(QWKMessageActiveStatus.Active, 122); header.writeUInt16LE(conferenceNumber, 123); - header.writeUInt16LE(0, 125); // :TODO: Check if others actually populate this - header.write(netTag[0], 127, 1, 'ascii'); + header.writeUInt16LE(this.totalMessages + 1, 125); + header.write(QWKNetworkTagIndicator.NotPresent, 127, 1, 'ascii'); // :TODO: Present if for network output? this.messagesStream.write(header); @@ -1117,6 +1156,9 @@ class QWKPacketWriter extends EventEmitter { } _getMessageConferenceNumberByAreaTag(areaTag) { + if (Message.isPrivateAreaTag(areaTag)) { + return 0; + } const areaConfig = _.get(Config(), [ 'messageNetworks', 'qwk', 'areas', areaTag ]); return areaConfig && areaConfig.conference; } @@ -1131,6 +1173,13 @@ class QWKPacketWriter extends EventEmitter { _createControlData(cb) { const areas = Array.from(this.areaTagsSeen).map(areaTag => { + if (Message.isPrivateAreaTag(areaTag)) { + return { + areaTag : Message.WellKnownAreaTags.Private, + name : 'Private', + desc : 'Private Messages', + }; + } return getMessageAreaByTag(areaTag); }); @@ -1168,8 +1217,12 @@ class QWKPacketWriter extends EventEmitter { // map areas as conf #\r\nDescription\r\n pairs areas.forEach(area => { const conferenceNumber = this._getMessageConferenceNumberByAreaTag(area.areaTag); + let desc = area.name; + if (area.desc) { + desc += ` - ${area.desc}` + } controlStream.write(`${conferenceNumber}\r\n`); - controlStream.write(`${area.name}\r\n`); + controlStream.write(`${desc}\r\n`); }); // :TODO: do we ever care here?! @@ -1180,6 +1233,73 @@ class QWKPacketWriter extends EventEmitter { controlStream.end(); } + _createIndexes(cb) { + const appendIndexData = (stream, offset) => { + const msb = numToMbf32(offset); + stream.write(msb); + + // technically, the conference #, but only as a byte, so pretty much useless + // AND the filename itself is the conference number... dafuq. + stream.write(Buffer.from([0x00])); + }; + + async.series( + [ + (callback) => { + // Create PERSONAL.NDX + if (!this.personalIndex.length) { + return callback(null); + } + + const indexStream = fs.createWriteStream(paths.join(this.workDir, 'personal.ndx')); + this.personalIndex.forEach(offset => appendIndexData(indexStream, offset)); + + indexStream.on('close', err => { + return callback(err); + }); + + indexStream.end(); + }, + (callback) => { + // 000.NDX of private mails + if (!this.inboxIndex.length) { + return callback(null); + } + + const indexStream = fs.createWriteStream(paths.join(this.workDir, '000.ndx')); + this.inboxIndex.forEach(offset => appendIndexData(indexStream, offset)); + + indexStream.on('close', err => { + return callback(err); + }); + + indexStream.end(); + }, + (callback) => { + // ####.NDX + async.eachSeries(this.publicIndex.keys(), (areaTag, nextArea) => { + const offsets = this.publicIndex.get(areaTag); + const conferenceNumber = this._getMessageConferenceNumberByAreaTag(areaTag); + const indexStream = fs.createWriteStream(paths.join(this.workDir, `${conferenceNumber.toString()}.ndx`)); + offsets.forEach(offset => appendIndexData(indexStream, offset)); + + indexStream.on('close', err => { + return nextArea(err); + }); + + indexStream.end(); + }, + err => { + return callback(err); + }); + } + ], + err => { + return cb(err); + } + ); + } + _makeSynchronetTimestamp(ts) { const syncTimestamp = ts.format('YYYYMMDDHHmmssZZ'); const syncTZ = UTCOffsetToSMBTZ[ts.format('Z')] || '0000'; // :TODO: what if we don't have a map? @@ -1215,7 +1335,7 @@ class QWKPacketWriter extends EventEmitter { Tags : message.hashTags.join(' '), // :TODO: Needs tested with Sync/etc.; Sync wants Conference *numbers* - Conference : getMessageConfTagByAreaTag(message.areaTag), + Conference : message.isPrivate() ? '0' : getMessageConfTagByAreaTag(message.areaTag), // ENiGMA Headers MessageUUID : message.uuid, From 8817113364322f5a96f6f42c756b7fa4d14a36aa Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 2 May 2020 13:34:28 -0600 Subject: [PATCH 17/95] * Create bundle filenames to spec * Better cp437 vs utf8 vs other encoding support * Add some CP437 and related utils --- core/cp437util.js | 55 +++++++++++++ core/message.js | 17 ++++ core/oputil/oputil_message_base.js | 8 +- core/qwk_mail_packet.js | 121 ++++++++++++++++++++++++----- core/string_util.js | 15 ++++ 5 files changed, 191 insertions(+), 25 deletions(-) create mode 100644 core/cp437util.js diff --git a/core/cp437util.js b/core/cp437util.js new file mode 100644 index 00000000..32425d3a --- /dev/null +++ b/core/cp437util.js @@ -0,0 +1,55 @@ + + +const CP437UnicodeTable = [ + '\u0000', '\u0001', '\u0002', '\u0003', '\u0004', '\u0005', '\u0006', + '\u0007', '\u0008', '\u0009', '\u000A', '\u000B', '\u000C', '\u000D', + '\u000E', '\u000F', '\u0010', '\u0011', '\u0012', '\u0013', '\u0014', + '\u0015', '\u0016', '\u0017', '\u0018', '\u0019', '\u001A', '\u001B', + '\u001C', '\u001D', '\u001E', '\u001F', '\u0020', '\u0021', '\u0022', + '\u0023', '\u0024', '\u0025', '\u0026', '\u0027', '\u0028', '\u0029', + '\u002A', '\u002B', '\u002C', '\u002D', '\u002E', '\u002F', '\u0030', + '\u0031', '\u0032', '\u0033', '\u0034', '\u0035', '\u0036', '\u0037', + '\u0038', '\u0039', '\u003A', '\u003B', '\u003C', '\u003D', '\u003E', + '\u003F', '\u0040', '\u0041', '\u0042', '\u0043', '\u0044', '\u0045', + '\u0046', '\u0047', '\u0048', '\u0049', '\u004A', '\u004B', '\u004C', + '\u004D', '\u004E', '\u004F', '\u0050', '\u0051', '\u0052', '\u0053', + '\u0054', '\u0055', '\u0056', '\u0057', '\u0058', '\u0059', '\u005A', + '\u005B', '\u005C', '\u005D', '\u005E', '\u005F', '\u0060', '\u0061', + '\u0062', '\u0063', '\u0064', '\u0065', '\u0066', '\u0067', '\u0068', + '\u0069', '\u006A', '\u006B', '\u006C', '\u006D', '\u006E', '\u006F', + '\u0070', '\u0071', '\u0072', '\u0073', '\u0074', '\u0075', '\u0076', + '\u0077', '\u0078', '\u0079', '\u007A', '\u007B', '\u007C', '\u007D', + '\u007E', '\u007F', '\u00C7', '\u00FC', '\u00E9', '\u00E2', '\u00E4', + '\u00E0', '\u00E5', '\u00E7', '\u00EA', '\u00EB', '\u00E8', '\u00EF', + '\u00EE', '\u00EC', '\u00C4', '\u00C5', '\u00C9', '\u00E6', '\u00C6', + '\u00F4', '\u00F6', '\u00F2', '\u00FB', '\u00F9', '\u00FF', '\u00D6', + '\u00DC', '\u00A2', '\u00A3', '\u00A5', '\u20A7', '\u0192', '\u00E1', + '\u00ED', '\u00F3', '\u00FA', '\u00F1', '\u00D1', '\u00AA', '\u00BA', + '\u00BF', '\u2310', '\u00AC', '\u00BD', '\u00BC', '\u00A1', '\u00AB', + '\u00BB', '\u2591', '\u2592', '\u2593', '\u2502', '\u2524', '\u2561', + '\u2562', '\u2556', '\u2555', '\u2563', '\u2551', '\u2557', '\u255D', + '\u255C', '\u255B', '\u2510', '\u2514', '\u2534', '\u252C', '\u251C', + '\u2500', '\u253C', '\u255E', '\u255F', '\u255A', '\u2554', '\u2569', + '\u2566', '\u2560', '\u2550', '\u256C', '\u2567', '\u2568', '\u2564', + '\u2565', '\u2559', '\u2558', '\u2552', '\u2553', '\u256B', '\u256A', + '\u2518', '\u250C', '\u2588', '\u2584', '\u258C', '\u2590', '\u2580', + '\u03B1', '\u00DF', '\u0393', '\u03C0', '\u03A3', '\u03C3', '\u00B5', + '\u03C4', '\u03A6', '\u0398', '\u03A9', '\u03B4', '\u221E', '\u03C6', + '\u03B5', '\u2229', '\u2261', '\u00B1', '\u2265', '\u2264', '\u2320', + '\u2321', '\u00F7', '\u2248', '\u00B0', '\u2219', '\u00B7', '\u221A', + '\u207F', '\u00B2', '\u25A0', '\u00A0' +]; + +const NonCP437EncodableRegExp = /[^\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\u0008\u0009\u000A\u000B\u000C\u000D\u000E\u000F\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001A\u001B\u001C\u001D\u001E\u001F\u0020\u0021\u0022\u0023\u0024\u0025\u0026\u0027\u0028\u0029\u002A\u002B\u002C\u002D\u002E\u002F\u0030\u0031\u0032\u0033\u0034\u0035\u0036\u0037\u0038\u0039\u003A\u003B\u003C\u003D\u003E\u003F\u0040\u0041\u0042\u0043\u0044\u0045\u0046\u0047\u0048\u0049\u004A\u004B\u004C\u004D\u004E\u004F\u0050\u0051\u0052\u0053\u0054\u0055\u0056\u0057\u0058\u0059\u005A\u005B\u005C\u005D\u005E\u005F\u0060\u0061\u0062\u0063\u0064\u0065\u0066\u0067\u0068\u0069\u006A\u006B\u006C\u006D\u006E\u006F\u0070\u0071\u0072\u0073\u0074\u0075\u0076\u0077\u0078\u0079\u007A\u007B\u007C\u007D\u007E\u007F\u00C7\u00FC\u00E9\u00E2\u00E4\u00E0\u00E5\u00E7\u00EA\u00EB\u00E8\u00EF\u00EE\u00EC\u00C4\u00C5\u00C9\u00E6\u00C6\u00F4\u00F6\u00F2\u00FB\u00F9\u00FF\u00D6\u00DC\u00A2\u00A3\u00A5\u20A7\u0192\u00E1\u00ED\u00F3\u00FA\u00F1\u00D1\u00AA\u00BA\u00BF\u2310\u00AC\u00BD\u00BC\u00A1\u00AB\u00BB\u2591\u2592\u2593\u2502\u2524\u2561\u2562\u2556\u2555\u2563\u2551\u2557\u255D\u255C\u255B\u2510\u2514\u2534\u252C\u251C\u2500\u253C\u255E\u255F\u255A\u2554\u2569\u2566\u2560\u2550\u256C\u2567\u2568\u2564\u2565\u2559\u2558\u2552\u2553\u256B\u256A\u2518\u250C\u2588\u2584\u258C\u2590\u2580\u03B1\u00DF\u0393\u03C0\u03A3\u03C3\u00B5\u03C4\u03A6\u0398\u03A9\u03B4\u221E\u03C6\u03B5\u2229\u2261\u00B1\u2265\u2264\u2320\u2321\u00F7\u2248\u00B0\u2219\u00B7\u221A\u207F\u00B2\u25A0\u00A0]/; +const isCP437Encodable = (s) => { + if (!s.length) { + return true; + } + + return !NonCP437EncodableRegExp.test(s); +} + +module.exports = { + CP437UnicodeTable, + isCP437Encodable, +} \ No newline at end of file diff --git a/core/message.js b/core/message.js index 3649a6f6..c763bbd2 100644 --- a/core/message.js +++ b/core/message.js @@ -11,6 +11,9 @@ const { sanitizeString, getISOTimestampString } = require('./database.js'); +const { isCP437Encodable } = require('./cp437util'); +const { containsNonLatinCodepoints } = require('./string_util'); + const { isAnsi, isFormattedLine, splitTextAtTerms, @@ -145,6 +148,20 @@ module.exports = class Message { return null !== _.get(this, 'meta.System.remote_from_user', null); } + isCP437Encodable() { + return isCP437Encodable(this.toUserName) && + isCP437Encodable(this.fromUserName) && + isCP437Encodable(this.subject) && + isCP437Encodable(this.message); + } + + containsNonLatinCodepoints() { + return containsNonLatinCodepoints(this.toUserName) || + containsNonLatinCodepoints(this.fromUserName) || + containsNonLatinCodepoints(this.subject) || + containsNonLatinCodepoints(this.message); + } + /* :TODO: finish me static checkUserHasDeleteRights(user, messageIdOrUuid, cb) { diff --git a/core/oputil/oputil_message_base.js b/core/oputil/oputil_message_base.js index 7d914ab2..6f677c52 100644 --- a/core/oputil/oputil_message_base.js +++ b/core/oputil/oputil_message_base.js @@ -464,8 +464,6 @@ function dumpQWKPacket(packetPath) { const { QWKPacketWriter } = require('../qwk_mail_packet'); const writer = new QWKPacketWriter({ bbsID : 'XIBALBA', - toUser : 'NuSkooler', - encoding : 'cp437', }); const { QWKPacketReader } = require('../qwk_mail_packet'); @@ -547,6 +545,8 @@ function exportQWKPacket(packetPath) { // oputil mb qwk export SPEC PATH [--user USER] // [areaTag1[@dateTime]],[...] PATH --user USER + // :TODO: bbsID from PATH filename else 'ENIGMA' + const posArgLen = argv._.length; if (posArgLen < 4) { @@ -619,8 +619,6 @@ function exportQWKPacket(packetPath) { const writer = new QWKPacketWriter({ // :TODO: export needs these options bbsID : 'XIBALBA', - toUser : 'NuSkooler', - encoding : 'cp437', user : user, }); @@ -635,7 +633,7 @@ function exportQWKPacket(packetPath) { }); }, (err) => { - writer.finish('/home/nuskooler/Downloads/qwk2/TEST1.QWK'); + writer.finish('/home/nuskooler/Downloads/qwk2/'); if (err) { console.error(`Failed to write one or more messages: ${err.message}`); } diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index 761b71f7..b7f53ebc 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -11,6 +11,7 @@ const Config = require('./config').get; const SysProps = require('./system_property'); const UserProps = require('./user_property'); const { numToMbf32 } = require('./mbf'); +const { getEncodingFromCharacterSetIdentifier } = require('./ftn_util'); const { EventEmitter } = require('events'); const temptmp = require('temptmp'); @@ -823,11 +824,11 @@ class QWKPacketWriter extends EventEmitter { enableQWKE = true, enableHeadersExtension = true, enableAtKludges = true, - encoding = 'cp437', systemDomain = 'enigma-bbs', bbsID = '', user = null, archiveFormat = 'application/zip', + forceEncoding = null, } = QWKPacketWriter.DefaultOptions) { super(); @@ -840,7 +841,7 @@ class QWKPacketWriter extends EventEmitter { bbsID, user, archiveFormat, - encoding : encoding.toLowerCase(), + forceEncoding : forceEncoding ? forceEncoding.toLowerCase() : null, }; this.temptmp = temptmp.createTrackedSession('qwkpacketwriter'); @@ -851,11 +852,11 @@ class QWKPacketWriter extends EventEmitter { enableQWKE : true, enableHeadersExtension : true, enableAtKludges : true, - encoding : 'cp437', systemDomain : 'enigma-bbs', bbsID : '', user : null, archiveFormat :'application/zip', + forceEncoding : null, }; } @@ -945,6 +946,8 @@ class QWKPacketWriter extends EventEmitter { } } else { fullMessageBody += `@MSGID: ${this.makeMessageIdentifier(message)}\n`; + fullMessageBody += `@TZ: ${UTCOffsetToSMBTZ[moment().format('Z')]}\n`; + // :TODO: REPLY and REPLYTO } } @@ -953,14 +956,16 @@ class QWKPacketWriter extends EventEmitter { fullMessageBody += `${line}\n`; }); - const encodedMessage = iconv.encode(fullMessageBody, this.options.encoding); + const encoding = this._getEncoding(message); + + const encodedMessage = iconv.encode(fullMessageBody, encoding); // // QWK spec wants line feeds as 0xe3 for some reason, so we'll have // to replace the \n's. If we're going against the spec and using UTF-8 // we can just leave them be. // - if ('utf8' !== this.options.encoding) { + if ('utf8' !== encoding) { replaceCharInBuffer(encodedMessage, 0x0a, QWKLF); } @@ -989,7 +994,7 @@ class QWKPacketWriter extends EventEmitter { this._updateIndexTracking(message); if (this.options.enableHeadersExtension) { - this._appendHeadersExtensionData(message); + this._appendHeadersExtensionData(message, encoding); } // next message starts at this block @@ -999,6 +1004,38 @@ class QWKPacketWriter extends EventEmitter { this.areaTagsSeen.add(message.areaTag); } + _getEncoding(message) { + if (this.options.forceEncoding) { + return this.options.forceEncoding; + } + + // If the system has stored an explicit encoding, use that. + let encoding = _.get(message.meta, 'System.explicit_encoding'); + if (encoding) { + return encoding; + } + + // If the message is already tagged with a supported encoding + // indicator such as FTN-style CHRS, try to use that. + encoding = _.get(message.meta, 'FtnKludge.CHRS'); + if (encoding) { + // convert from CHRS to something standard + encoding = getEncodingFromCharacterSetIdentifier(encoding); + if (encoding) { + return encoding; + } + } + + // The to-spec default is CP437/ASCII. If it can be encoded as + // such then do so. + if (message.isCP437Encodable()) { + return 'cp437'; + } + + // Something more modern... + return 'utf8'; + } + _messageAddressedToUser(message) { if (_.isUndefined(this.cachedCompareNames)) { if (this.options.user) { @@ -1041,7 +1078,7 @@ class QWKPacketWriter extends EventEmitter { } - finish(packetPath) { + finish(packetDirectory) { async.series( [ (callback) => { @@ -1066,7 +1103,7 @@ class QWKPacketWriter extends EventEmitter { return this._createIndexes(callback); }, (callback) => { - return this._producePacketArchive(packetPath, callback); + return this._producePacketArchive(packetDirectory, callback); } ], err => { @@ -1081,7 +1118,44 @@ class QWKPacketWriter extends EventEmitter { ) } - _producePacketArchive(packetPath, cb) { + _getNextAvailPacketFileName(packetDirectory, cb) { + // + // According to http://wiki.synchro.net/ref:qwk filenames should + // start with .QWK -> .QW1 ... .QW9 -> .Q10 ... .Q99 + // + let digits = 0; + async.doWhilst( callback => { + let ext; + if (0 === digits) { + ext = 'QWK'; + } else if (digits < 10) { + ext = `QW${digits}`; + } else if (digits < 100) { + ext = `Q${digits}`; + } else { + return callback(Errors.UnexpectedState(`Unable to choose a valid QWK output filename`)); + } + + ++digits; + + const filename = `${this.options.bbsID}.${ext}`; + fs.stat(paths.join(packetDirectory, filename), (err, stats) => { + if (err && 'ENOENT' === err.code) { + return callback(null, filename); + } else { + return callback(null, null); + } + }); + }, + (filename, callback) => { + return callback(null, filename ? false : true); + }, + (err, filename) => { + return cb(err, filename); + }); + } + + _producePacketArchive(packetDirectory, cb) { const archiveUtil = ArchiveUtil.getInstance(); fs.readdir(this.workDir, (err, files) => { @@ -1089,15 +1163,22 @@ class QWKPacketWriter extends EventEmitter { return cb(err); } - archiveUtil.compressTo( - this.options.archiveFormat, - packetPath, - files, - this.workDir, - err => { + this._getNextAvailPacketFileName(packetDirectory, (err, filename) => { + if (err) { return cb(err); } - ); + + const packetPath = paths.join(packetDirectory, filename); + archiveUtil.compressTo( + this.options.archiveFormat, + packetPath, + files, + this.workDir, + err => { + return cb(err); + } + ); + }); }); } @@ -1306,10 +1387,10 @@ class QWKPacketWriter extends EventEmitter { return `${syncTimestamp} ${syncTZ}`; } - _appendHeadersExtensionData(message) { + _appendHeadersExtensionData(message, encoding) { const messageData = { // Synchronet style - Utf8 : ('utf8' === this.options.encoding ? 'true' : 'false'), + Utf8 : ('utf8' === encoding ? 'true' : 'false'), 'Message-ID' : this.makeMessageIdentifier(message), WhenWritten : this._makeSynchronetTimestamp(message.modTimestamp), @@ -1367,11 +1448,11 @@ class QWKPacketWriter extends EventEmitter { messageData.Editor = `ENiGMA 1/2 BBS FSE v${enigmaVersion}`; } - this.headersDatStream.write(iconv.encode(`[${this.currentMessageOffset.toString(16)}]\r\n`, this.options.encoding)); + this.headersDatStream.write(iconv.encode(`[${this.currentMessageOffset.toString(16)}]\r\n`, encoding)); for (let [name, value] of Object.entries(messageData)) { if (value) { - this.headersDatStream.write(iconv.encode(`${name}: ${value}\r\n`, this.options.encoding)); + this.headersDatStream.write(iconv.encode(`${name}: ${value}\r\n`, encoding)); } } diff --git a/core/string_util.js b/core/string_util.js index fa9a9097..cde7ac3e 100644 --- a/core/string_util.js +++ b/core/string_util.js @@ -13,6 +13,7 @@ exports.pad = pad; exports.insert = insert; exports.replaceAt = replaceAt; exports.isPrintable = isPrintable; +exports.containsNonLatinCodepoints = containsNonLatinCodepoints; exports.stripAllLineFeeds = stripAllLineFeeds; exports.debugEscapedString = debugEscapedString; exports.stringFromNullTermBuffer = stringFromNullTermBuffer; @@ -196,6 +197,20 @@ function isPrintable(s) { return !RE_NON_PRINTABLE.test(s); } +const NonLatinCodePointsRegExp = /[^\u0000-\u00ff]/; + +function containsNonLatinCodepoints(s) { + if (!s.length) { + return false; + } + + if (s.charCodeAt(0) > 255) { + return true; + } + + return NonLatinCodepointsRegEx.test(s); +} + function stripAllLineFeeds(s) { return s.replace(/\r?\n|[\r\u2028\u2029]/g, ''); } From c7a543e87eeedadfbb82934d08085b91c7a3f096 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 2 May 2020 16:48:24 -0600 Subject: [PATCH 18/95] * oputil mb qwk-export * oputil mb qwk-dump * Fix QWK reader encoding --- core/oputil/oputil_help.js | 10 ++ core/oputil/oputil_message_base.js | 209 +++++++++++++---------------- core/qwk_mail_packet.js | 9 +- 3 files changed, 106 insertions(+), 122 deletions(-) diff --git a/core/oputil/oputil_help.js b/core/oputil/oputil_help.js index abcf61fe..3fd55575 100644 --- a/core/oputil/oputil_help.js +++ b/core/oputil/oputil_help.js @@ -170,6 +170,11 @@ Actions: import-areas PATH Import areas using FidoNet *.NA or AREAS.BBS file + qwk-dump PATH Dumps a QWK packet to stdout. + qwk-export [AREA_TAGS] PATH Exports one or more configured message area to a QWK + packet in the directory specified by PATH. The QWK + BBS ID will be obtained by the final component of PATH. + import-areas arguments: --conf CONF_TAG Conference tag in which to import areas --network NETWORK Network name/key to associate FTN areas @@ -177,6 +182,11 @@ import-areas arguments: --type TYPE Area import type Valid types are "bbs" and "na". + +qwk-export arguments: + --user USER User in which to export for. Defaults to the SysOp. + --after TIMESTAMP Export only messages with a timestamp later than + TIMESTAMP. ` }; diff --git a/core/oputil/oputil_message_base.js b/core/oputil/oputil_message_base.js index 6f677c52..e29a30dc 100644 --- a/core/oputil/oputil_message_base.js +++ b/core/oputil/oputil_message_base.js @@ -10,17 +10,19 @@ const { initConfigAndDatabases, getAnswers, writeConfig, -} = require('./oputil_common.js'); -const getHelpFor = require('./oputil_help.js').getHelpFor; -const Address = require('../ftn_address.js'); -const Errors = require('../enig_error.js').Errors; +} = require('./oputil_common.js'); + +const getHelpFor = require('./oputil_help.js').getHelpFor; +const Address = require('../ftn_address.js'); +const Errors = require('../enig_error.js').Errors; // deps -const async = require('async'); -const paths = require('path'); -const fs = require('fs'); -const hjson = require('hjson'); -const _ = require('lodash'); +const async = require('async'); +const paths = require('path'); +const fs = require('fs'); +const hjson = require('hjson'); +const _ = require('lodash'); +const moment = require('moment'); exports.handleMessageBaseCommand = handleMessageBaseCommand; @@ -434,105 +436,47 @@ function getImportEntries(importType, importData) { return importEntries; } -function handleQWK() { +function dumpQWKPacket() { const packetPath = argv._[argv._.length - 1]; - if(argv._.length < 4 || !packetPath || 0 === packetPath.length) { - return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + if(argv._.length < 3 || !packetPath || 0 === packetPath.length) { + return printUsageAndSetExitCode(getHelpFor('MessageBase'), ExitCodes.ERROR); } - const subAction = argv._[argv._.length - 2]; - switch (subAction) { - case 'dump' : - return dumpQWKPacket(packetPath); - - case 'export' : - return exportQWKPacket(packetPath); - - default : - return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); - } -} - -function dumpQWKPacket(packetPath) { async.waterfall( [ (callback) => { return initConfigAndDatabases(callback); }, (callback) => { - //// - const { QWKPacketWriter } = require('../qwk_mail_packet'); - const writer = new QWKPacketWriter({ - bbsID : 'XIBALBA', - }); - const { QWKPacketReader } = require('../qwk_mail_packet'); + const reader = new QWKPacketReader(packetPath); - - writer.on('ready', () => { - const reader = new QWKPacketReader(packetPath); - - reader.on('error', err => { - console.error(`ERROR: ${err.message}`); - return callback(err); - }); - - reader.on('done', () => { - writer.finish(); - }); - - reader.on('archive type', archiveType => { - console.info(`-> Archive type: ${archiveType}`); - }); - - reader.on('creator', creator => { - console.info(`-> Creator: ${creator}`); - }); - - reader.on('message', message => { - writer.appendMessage(message); - }); - - reader.read(); + reader.on('error', err => { + console.error(`ERROR: ${err.message}`); + return callback(err); }); - writer.on('finished', () => { - console.log('done'); + reader.on('done', () => { + return callback(null); }); - writer.init(); + reader.on('archive type', archiveType => { + console.info(`-> Archive type: ${archiveType}`); + }); - //// + reader.on('creator', creator => { + console.info(`-> Creator: ${creator}`); + }); - // const { QWKPacketReader } = require('../qwk_mail_packet'); - // const reader = new QWKPacketReader(packetPath); + reader.on('message', message => { + console.info('--- message ---'); + console.info(`To: ${message.toUserName}`); + console.info(`From: ${message.fromUserName}`); + console.info(`Subject: ${message.subject}`); + console.info(`Message:\r\n${message.message}`); + }); - // reader.on('error', err => { - // console.error(`ERROR: ${err.message}`); - // return callback(err); - // }); - - // reader.on('done', () => { - // return callback(null); - // }); - - // reader.on('archive type', archiveType => { - // console.info(`-> Archive type: ${archiveType}`); - // }); - - // reader.on('creator', creator => { - // console.info(`-> Creator: ${creator}`); - // }); - - // reader.on('message', message => { - // console.info('--- message ---'); - // console.info(`To : ${message.toUserName}`); - // console.info(`From : ${message.fromUserName}`); - // console.info(`Subject : ${message.subject}`); - // console.info(`Message :\r\n${message.message}`); - // }); - - // reader.read(); + reader.read(); } ], err => { @@ -541,36 +485,43 @@ function dumpQWKPacket(packetPath) { ) } -function exportQWKPacket(packetPath) { - // oputil mb qwk export SPEC PATH [--user USER] - // [areaTag1[@dateTime]],[...] PATH --user USER +function exportQWKPacket() { + let packetPath = argv._[argv._.length - 1]; + if(argv._.length < 3 || !packetPath || 0 === packetPath.length) { + return printUsageAndSetExitCode(getHelpFor('MessageBase'), ExitCodes.ERROR); + } - // :TODO: bbsID from PATH filename else 'ENIGMA' + // oputil mb qwk-export TAGS PATH [--user USER] [--after TIMESTAMP] + // [areaTag1,areaTag2,...] PATH --user USER --after TIMESTAMP + let bbsID = 'ENIGMA'; + const filename = paths.basename(packetPath); + if (filename) { + const ext = paths.extname(filename); + bbsID = paths.basename(filename, ext); + } + + packetPath = paths.dirname(packetPath); const posArgLen = argv._.length; - if (posArgLen < 4) { - return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); + let areaTags; + if (4 === posArgLen) { + areaTags = argv._[posArgLen - 2].split(','); + } else { + areaTags = []; } - let areaTagSpecs = '*'; - if (5 === posArgLen) { - areaTagSpecs = argv._[areaTagSpecs - 2]; + let newerThanTimestamp = null; + if (argv.after) { + const ts = moment(argv.after); + if (ts.isValid()) { + newerThanTimestamp = ts.format(); + } } - - //const areaTagSpecs = argv._[areaTagSpecs - 2]; - - /*const packetPath = argv._[argv._.length - 1]; - if(argv._.length < 4 || !packetPath || 0 === packetPath.length) { - return printUsageAndSetExitCode(getHelpFor('QWK'), ExitCodes.ERROR); - }*/ - - // :TODO: parse area tags(s) and timestamps - const areaTags = [ 'general', 'fsx_gen' ]; - const userName = argv.user || '-'; + let totalExported = 0; async.waterfall( [ (callback) => { @@ -590,14 +541,29 @@ function exportQWKPacket(packetPath) { return User.getUser(userId, callback); }); }, + (user, callback) => { + // populate area tags with all available to user + // if they were not explicitly supplied + if (!areaTags.length) { + const { + getAvailableMessageConferences, + getAvailableMessageAreasByConfTag + } = require('../../core/message_area'); + + const confTags = Object.keys(getAvailableMessageConferences(null, { noClient : true })); + confTags.forEach( confTag => { + areaTags = areaTags.concat(Object.keys(getAvailableMessageAreasByConfTag(confTag))); + }); + } + return callback(null, user); + }, (user, callback) => { const Message = require('../message'); const filter = { resultType : 'id', areaTag : areaTags, - - // :TODO: newerThanTimestamp + newerThanTimestamp, }; // public @@ -617,9 +583,8 @@ function exportQWKPacket(packetPath) { (user, Message, messageIds, callback) => { const { QWKPacketWriter } = require('../qwk_mail_packet'); const writer = new QWKPacketWriter({ - // :TODO: export needs these options - bbsID : 'XIBALBA', - user : user, + bbsID, + user, }); writer.on('ready', () => { @@ -628,18 +593,23 @@ function exportQWKPacket(packetPath) { message.load( { messageId }, err => { if (!err) { writer.appendMessage(message); + ++totalExported; } return nextMessageId(err); }); }, (err) => { - writer.finish('/home/nuskooler/Downloads/qwk2/'); + writer.finish(packetPath); if (err) { console.error(`Failed to write one or more messages: ${err.message}`); } }); }); + writer.on('warning', err => { + console.warn(`!!! ${err.reason ? err.reason : err.message}`); + }); + writer.on('finished', () => { return callback(null); }); @@ -649,8 +619,10 @@ function exportQWKPacket(packetPath) { ], err => { if(err) { - console.error(err.reason ? err.reason : err.message); + return console.error(err.reason ? err.reason : err.message); } + + console.info(`-> Exported ${totalExported} messages`); } ); } @@ -673,6 +645,7 @@ function handleMessageBaseCommand() { return({ areafix : areaFix, 'import-areas' : importAreas, - qwk : handleQWK, + 'qwk-dump' : dumpQWKPacket, + 'qwk-export' : exportQWKPacket, }[action] || errUsage)(); } \ No newline at end of file diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index b7f53ebc..5aa5deac 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -493,7 +493,7 @@ class QWKPacketReader extends EventEmitter { } const encodingToSpec = 'cp437'; - let encoding = encodingToSpec; + let encoding; const path = paths.join(this.packetInfo.tempDir, this.packetInfo.messages.filename); fs.open(path, 'r', (err, fd) => { @@ -575,6 +575,7 @@ class QWKPacketReader extends EventEmitter { switch (state) { case 'header' : const header = MessageHeaderParser.parse(buffer); + encoding = encodingToSpec; // reset per message // massage into something a little more sane (things we can't quite do in the parser directly) ['toName', 'fromName', 'subject'].forEach(field => { @@ -601,7 +602,7 @@ class QWKPacketReader extends EventEmitter { // if we have HEADERS.DAT with a 'Utf8' override for this message, // the overridden to/from/subject/message fields are UTF-8 - if (currMessage.headersExtension && currMessage.headersExtension.Utf8) { + if (currMessage.headersExtension && 'true' === currMessage.headersExtension.Utf8.toLowerCase()) { encoding = 'utf8'; } @@ -825,7 +826,7 @@ class QWKPacketWriter extends EventEmitter { enableHeadersExtension = true, enableAtKludges = true, systemDomain = 'enigma-bbs', - bbsID = '', + bbsID = 'ENIGMA', user = null, archiveFormat = 'application/zip', forceEncoding = null, @@ -853,7 +854,7 @@ class QWKPacketWriter extends EventEmitter { enableHeadersExtension : true, enableAtKludges : true, systemDomain : 'enigma-bbs', - bbsID : '', + bbsID : 'ENIGMA', user : null, archiveFormat :'application/zip', forceEncoding : null, From b32dae9b485da22b85905868b1bcbf59a514d37c Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 3 May 2020 10:42:57 -0600 Subject: [PATCH 19/95] Auto-generate conf numbers for user export mode --- core/message_area.js | 15 ++++++++ core/oputil/oputil_message_base.js | 8 +--- core/qwk_mail_packet.js | 59 +++++++++++++++++++++++++++++- 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/core/message_area.js b/core/message_area.js index ff01f1e1..30ad14ed 100644 --- a/core/message_area.js +++ b/core/message_area.js @@ -24,6 +24,7 @@ exports.getAvailableMessageConferences = getAvailableMessageConferences; exports.getSortedAvailMessageConferences = getSortedAvailMessageConferences; exports.getAvailableMessageAreasByConfTag = getAvailableMessageAreasByConfTag; exports.getSortedAvailMessageAreasByConfTag = getSortedAvailMessageAreasByConfTag; +exports.getAllAvailableMessageAreaTags = getAllAvailableMessageAreaTags; exports.getDefaultMessageConferenceTag = getDefaultMessageConferenceTag; exports.getDefaultMessageAreaTagByConfTag = getDefaultMessageAreaTagByConfTag; exports.getSuitableMessageConfAndAreaTags = getSuitableMessageConfAndAreaTags; @@ -140,6 +141,20 @@ function getSortedAvailMessageAreasByConfTag(confTag, options) { return areas; } +function getAllAvailableMessageAreaTags(client, options) { + const areaTags = []; + + // mask over older messy APIs for now + const confOpts = Object.assign({}, options, { noClient : client ? false : true }); + const areaOpts = Object.assign({}, options, { client }); + + Object.keys(getAvailableMessageConferences(client, confOpts)).forEach(confTag => { + areaTags.push(...Object.keys(getAvailableMessageAreasByConfTag(confTag, areaOpts))); + }); + + return areaTags; +} + function getDefaultMessageConferenceTag(client, disableAcsCheck) { // // Find the first conference marked 'default'. If found, diff --git a/core/oputil/oputil_message_base.js b/core/oputil/oputil_message_base.js index e29a30dc..ea996c6c 100644 --- a/core/oputil/oputil_message_base.js +++ b/core/oputil/oputil_message_base.js @@ -546,14 +546,10 @@ function exportQWKPacket() { // if they were not explicitly supplied if (!areaTags.length) { const { - getAvailableMessageConferences, - getAvailableMessageAreasByConfTag + getAllAvailableMessageAreaTags } = require('../../core/message_area'); - const confTags = Object.keys(getAvailableMessageConferences(null, { noClient : true })); - confTags.forEach( confTag => { - areaTags = areaTags.concat(Object.keys(getAvailableMessageAreasByConfTag(confTag))); - }); + areaTags = getAllAvailableMessageAreaTags(); } return callback(null, user); }, diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index 5aa5deac..be524422 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -5,6 +5,7 @@ const { splitTextAtTerms } = require('./string_util'); const { getMessageConfTagByAreaTag, getMessageAreaByTag, + getAllAvailableMessageAreaTags, } = require('./message_area'); const StatLog = require('./stat_log'); const Config = require('./config').get; @@ -822,6 +823,7 @@ class QWKPacketReader extends EventEmitter { class QWKPacketWriter extends EventEmitter { constructor( { + mode = QWKPacketWriter.Modes.User, enableQWKE = true, enableHeadersExtension = true, enableAtKludges = true, @@ -835,6 +837,7 @@ class QWKPacketWriter extends EventEmitter { super(); this.options = { + mode, enableQWKE, enableHeadersExtension, enableAtKludges, @@ -846,10 +849,13 @@ class QWKPacketWriter extends EventEmitter { }; this.temptmp = temptmp.createTrackedSession('qwkpacketwriter'); + + this.areaTagConfMap = {}; } static get DefaultOptions() { return { + mode : QWKPacketWriter.Modes.User, enableQWKE : true, enableHeadersExtension : true, enableAtKludges : true, @@ -861,6 +867,13 @@ class QWKPacketWriter extends EventEmitter { }; } + static get Modes() { + return { + User : 'user', // creation of a packet for a user (non-network); non-mapped confs allowed + Network : 'network', // creation of a packet for QWK network + }; + } + init() { async.series( [ @@ -873,6 +886,48 @@ class QWKPacketWriter extends EventEmitter { return callback(err); }); }, + (callback) => { + // + // Prepare areaTag -> conference number mapping: + // - In User mode, areaTags's that are not explicitly configured + // will have their conference number auto-generated. + // - In Network mode areaTags's missing a configuration will not + // be mapped, and thus skipped. + // + const configuredAreas = _.get(Config(), 'messageNetworks.qwk.areas'); + if (configuredAreas) { + Object.keys(configuredAreas).forEach(areaTag => { + const confNumber = configuredAreas[areaTag].conference; + if (confNumber) { + this.areaTagConfMap[areaTag] = confNumber; + } + }); + } + + if (this.options.mode === QWKPacketWriter.Modes.User) { + // All the rest + let confNumber = 1; + const usedConfNumbers = new Set(Object.values(this.areaTagConfMap)); + getAllAvailableMessageAreaTags().forEach(areaTag => { + if (this.areaTagConfMap[areaTag]) { + return; + } + + while (confNumber < 65537 && usedConfNumbers.has(confNumber)) { + ++confNumber; + } + + if (confNumber === 65536) { // sanity... + this.emit('warning', Errors.General(`To many conferences`)); + } else { + this.areaTagConfMap[areaTag] = confNumber; + ++confNumber; + } + }); + } + + return callback(null); + }, (callback) => { this.messagesStream = fs.createWriteStream(paths.join(this.workDir, 'messages.dat')); @@ -1241,8 +1296,8 @@ class QWKPacketWriter extends EventEmitter { if (Message.isPrivateAreaTag(areaTag)) { return 0; } - const areaConfig = _.get(Config(), [ 'messageNetworks', 'qwk', 'areas', areaTag ]); - return areaConfig && areaConfig.conference; + + return this.areaTagConfMap[areaTag]; } _getExportForUsername() { From c6bde65be04f3240e2b87c8497500429b8a39e54 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 3 May 2020 11:10:40 -0600 Subject: [PATCH 20/95] Minor fixes, tidy up FTS CHRS table a bit --- core/ftn_util.js | 28 +++++++++++++++------------- core/qwk_mail_packet.js | 15 ++++++++++++--- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/core/ftn_util.js b/core/ftn_util.js index e4637554..9d53c9a3 100644 --- a/core/ftn_util.js +++ b/core/ftn_util.js @@ -375,26 +375,28 @@ function getCharacterSetIdentifierByEncoding(encodingName) { return value ? `${value[0]} ${value[1]}` : encodingName.toUpperCase(); } +// http://ftsc.org/docs/fts-5003.001 +// http://www.unicode.org/L2/L1999/99325-N.htm function getEncodingFromCharacterSetIdentifier(chrs) { const ident = chrs.split(' ')[0].toUpperCase(); // :TODO: fill in the rest!!! return { // level 1 - 'ASCII' : 'iso-646-1', - 'DUTCH' : 'iso-646', - 'FINNISH' : 'iso-646-10', - 'FRENCH' : 'iso-646', - 'CANADIAN' : 'iso-646', - 'GERMAN' : 'iso-646', - 'ITALIAN' : 'iso-646', - 'NORWEIG' : 'iso-646', - 'PORTU' : 'iso-646', + 'ASCII' : 'ascii', // ISO-646-1 + 'DUTCH' : 'ascii', // ISO-646 + 'FINNISH' : 'ascii', // ISO-646-10 + 'FRENCH' : 'ascii', // ISO-646 + 'CANADIAN' : 'ascii', // ISO-646 + 'GERMAN' : 'ascii', // ISO-646 + 'ITALIAN' : 'ascii', // ISO-646 + 'NORWEIG' : 'ascii', // ISO-646 + 'PORTU' : 'ascii', // ISO-646 'SPANISH' : 'iso-656', - 'SWEDISH' : 'iso-646-10', - 'SWISS' : 'iso-646', - 'UK' : 'iso-646', - 'ISO-10' : 'iso-646-10', + 'SWEDISH' : 'ascii', // ISO-646-10 + 'SWISS' : 'ascii', // ISO-646 + 'UK' : 'ascii', // ISO-646 + 'ISO-10' : 'ascii', // ISO-646-10 // level 2 'CP437' : 'cp437', diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index be524422..c1e58bc0 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -963,6 +963,15 @@ class QWKPacketWriter extends EventEmitter { return `<${message.messageId}.${message.messageUuid}@${this.options.systemDomain}>`; } + _encodeWithFallback(s, encoding) { + try { + return iconv.encode(s, encoding); + } catch (e) { + this.emit('warning', Errors.General(`Failed to encode buffer using ${encoding}; Falling back to 'ascii'`)); + return iconv.encode(s, 'ascii'); + } + } + appendMessage(message) { // // Each message has to: @@ -1014,7 +1023,7 @@ class QWKPacketWriter extends EventEmitter { const encoding = this._getEncoding(message); - const encodedMessage = iconv.encode(fullMessageBody, encoding); + const encodedMessage = this._encodeWithFallback(fullMessageBody, encoding); // // QWK spec wants line feeds as 0xe3 for some reason, so we'll have @@ -1504,11 +1513,11 @@ class QWKPacketWriter extends EventEmitter { messageData.Editor = `ENiGMA 1/2 BBS FSE v${enigmaVersion}`; } - this.headersDatStream.write(iconv.encode(`[${this.currentMessageOffset.toString(16)}]\r\n`, encoding)); + this.headersDatStream.write(this._encodeWithFallback(`[${this.currentMessageOffset.toString(16)}]\r\n`, encoding)); for (let [name, value] of Object.entries(messageData)) { if (value) { - this.headersDatStream.write(iconv.encode(`${name}: ${value}\r\n`, encoding)); + this.headersDatStream.write(this._encodeWithFallback(`${name}: ${value}\r\n`, encoding)); } } From 2b1747a8f451deb639b2de2a0c89f75014a3613c Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 3 May 2020 11:17:33 -0600 Subject: [PATCH 21/95] Fix NDX filenames --- core/qwk_mail_packet.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index c1e58bc0..69617866 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -1426,7 +1426,7 @@ class QWKPacketWriter extends EventEmitter { async.eachSeries(this.publicIndex.keys(), (areaTag, nextArea) => { const offsets = this.publicIndex.get(areaTag); const conferenceNumber = this._getMessageConferenceNumberByAreaTag(areaTag); - const indexStream = fs.createWriteStream(paths.join(this.workDir, `${conferenceNumber.toString()}.ndx`)); + const indexStream = fs.createWriteStream(paths.join(this.workDir, `${conferenceNumber.toString().padStart(4, '0')}.ndx`)); offsets.forEach(offset => appendIndexData(indexStream, offset)); indexStream.on('close', err => { From 65a5eabe9c2ce4f75308d3e28ddfdc5f31914a74 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 3 May 2020 11:40:54 -0600 Subject: [PATCH 22/95] Try this --- core/qwk_mail_packet.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index 69617866..f42271ff 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -1412,7 +1412,7 @@ class QWKPacketWriter extends EventEmitter { return callback(null); } - const indexStream = fs.createWriteStream(paths.join(this.workDir, '000.ndx')); + const indexStream = fs.createWriteStream(paths.join(this.workDir, '0000.ndx')); this.inboxIndex.forEach(offset => appendIndexData(indexStream, offset)); indexStream.on('close', err => { From 21374d6e6df3ccc8f8fd60dc5306fb34f09cd03b Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 3 May 2020 11:48:03 -0600 Subject: [PATCH 23/95] Minor --- core/qwk_mail_packet.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index f42271ff..ef2cbef9 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -913,11 +913,12 @@ class QWKPacketWriter extends EventEmitter { return; } - while (confNumber < 65537 && usedConfNumbers.has(confNumber)) { + while (confNumber < 10001 && usedConfNumbers.has(confNumber)) { ++confNumber; } - if (confNumber === 65536) { // sanity... + // we can go up to 65535 for some things, but NDX files are limited to 9999 + if (confNumber === 10000) { // sanity... this.emit('warning', Errors.General(`To many conferences`)); } else { this.areaTagConfMap[areaTag] = confNumber; @@ -1412,7 +1413,7 @@ class QWKPacketWriter extends EventEmitter { return callback(null); } - const indexStream = fs.createWriteStream(paths.join(this.workDir, '0000.ndx')); + const indexStream = fs.createWriteStream(paths.join(this.workDir, '000.ndx')); this.inboxIndex.forEach(offset => appendIndexData(indexStream, offset)); indexStream.on('close', err => { From 7fb10418f0381bad0257dde6f0b91c483bcbe1dc Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 3 May 2020 12:01:31 -0600 Subject: [PATCH 24/95] Start auto-gen confs at 1000 --- core/qwk_mail_packet.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index ef2cbef9..eec737cb 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -906,7 +906,8 @@ class QWKPacketWriter extends EventEmitter { if (this.options.mode === QWKPacketWriter.Modes.User) { // All the rest - let confNumber = 1; + // Start at 1000 to work around what seems to be a bug with some readers + let confNumber = 1000; const usedConfNumbers = new Set(Object.values(this.areaTagConfMap)); getAllAvailableMessageAreaTags().forEach(areaTag => { if (this.areaTagConfMap[areaTag]) { From 67ac86ac053a7d6e04bb1f9c0fe77f16771bb580 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 5 May 2020 19:01:47 -0600 Subject: [PATCH 25/95] Add initial QWK docs, update FTN, etc. --- core/oputil/oputil_help.js | 2 + core/oputil/oputil_message_base.js | 11 ++- docs/_includes/nav.md | 2 + docs/messageareas/ftn.md | 105 ++++++++++++++++++++++++ docs/messageareas/message-networks.md | 110 ++++---------------------- docs/messageareas/qwk.md | 52 ++++++++++++ 6 files changed, 185 insertions(+), 97 deletions(-) create mode 100644 docs/messageareas/ftn.md create mode 100644 docs/messageareas/qwk.md diff --git a/core/oputil/oputil_help.js b/core/oputil/oputil_help.js index 3fd55575..0d132445 100644 --- a/core/oputil/oputil_help.js +++ b/core/oputil/oputil_help.js @@ -187,6 +187,8 @@ qwk-export arguments: --user USER User in which to export for. Defaults to the SysOp. --after TIMESTAMP Export only messages with a timestamp later than TIMESTAMP. + --no-qwke Disable QWKE extensions. + --no-synchronet Disable Synchronet style extensions. ` }; diff --git a/core/oputil/oputil_message_base.js b/core/oputil/oputil_message_base.js index ea996c6c..1790b1e5 100644 --- a/core/oputil/oputil_message_base.js +++ b/core/oputil/oputil_message_base.js @@ -521,6 +521,13 @@ function exportQWKPacket() { const userName = argv.user || '-'; + const writerOptions = { + enableQWKE : !(false === argv.qwke), + enableHeadersExtension : !(false === argv.synchronet), + enableAtKludges : !(false === argv.synchronet), + archiveFormat : argv.format || 'application/zip' + }; + let totalExported = 0; async.waterfall( [ @@ -578,10 +585,10 @@ function exportQWKPacket() { }, (user, Message, messageIds, callback) => { const { QWKPacketWriter } = require('../qwk_mail_packet'); - const writer = new QWKPacketWriter({ + const writer = new QWKPacketWriter(Object.assign(writerOptions, { bbsID, user, - }); + })); writer.on('ready', () => { async.eachSeries(messageIds, (messageId, nextMessageId) => { diff --git a/docs/_includes/nav.md b/docs/_includes/nav.md index e51a8734..4fb9a399 100644 --- a/docs/_includes/nav.md +++ b/docs/_includes/nav.md @@ -41,6 +41,8 @@ - [Message networks]({{ site.baseurl }}{% link messageareas/message-networks.md %}) - [BSO Import & Export]({{ site.baseurl }}{% link messageareas/bso-import-export.md %}) - [Netmail]({{ site.baseurl }}{% link messageareas/netmail.md %}) + - [QWK]({{ site.baseurl }}{% link messageareas/qwk.md %}) + - [FTN]({{ site.baseurl }}{% link messageareas/ftn.md %}) - Art - [General]({{ site.baseurl }}{% link art/general.md %}) diff --git a/docs/messageareas/ftn.md b/docs/messageareas/ftn.md new file mode 100644 index 00000000..f1bb5366 --- /dev/null +++ b/docs/messageareas/ftn.md @@ -0,0 +1,105 @@ +--- +layout: page +title: FidoNet-Style Networks (FTN) +--- + +## FidoNet-Style Networks (FTN) + +TODO: preamble + +### Configuration + +1. `messageNetworks.ftn.networks`: declares available networks. +2. `messageNetworks.ftn.areas`: establishes local area mappings and per-area specifics. +3. `scannerTossers.ftn_bso`: general configuration for the scanner/tosser (import/export). This is also where we configure per-node settings. + +:information_source: ENiGMA½'s `ftn_bso` module is **not a mailer** and makes **no attempts** to perform packet transport! An external utility such as Binkd is required for this task. + +#### Networks +The `networks` block is a per-network configuration where each entry's ID (or "key") may be referenced elsewhere in `config.hjson`. For example, consider two networks: ArakNet (`araknet`) and fsxNet (`fsxnet`): + +```hjson +{ + messageNetworks: { + ftn: { + networks: { + // it is recommended to use lowercase network tags + fsxnet: { + defaultZone: 21 + localAddress: "21:1/121" + } + + araknet: { + defaultZone: 10 + localAddress: "10:101/9" + } + } + } + } +} +``` + +#### Areas +The `areas` section describes a mapping of local **area tags** configured in your `messageConferences` (see [Configuring a Message Area](configuring-a-message-area.md)) to a message network (described above), a FTN specific area tag, and remote uplink address(s). This section can be thought of similar to the *AREAS.BBS* file used by other BBS packages. + +When ENiGMA½ imports messages, they will be placed in the local area that matches key under `areas` while exported messages will be sent to the relevant `network`. + +| Config Item | Required | Description | +|-------------|----------|----------------------------------------------------------| +| `network` | :+1: | Associated network from the `networks` section above | +| `tag` | :+1: | FTN area tag (ie: `FSX_GEN`) | +| `uplinks` | :+1: | An array of FTN address uplink(s) for this network | + +Example: +```hjson +{ + messageNetworks: { + ftn: { + areas: { + // it is recommended to use lowercase area tags + fsx_general: // *local* tag found within messageConferences + network: fsxnet // that we are mapping to this network + tag: FSX_GEN // ...and this remote FTN-specific tag + uplinks: [ "21:1/100" ] // a single string also allowed here + } + } + } + } +} +``` + +:information_source: You can import `AREAS.BBS` or FTN style `.NA` files using [oputil](/docs/admin/oputil.md)! + +#### A More Complete Example +Below is a more complete *example* illustrating some of the concepts above: + +```hjson +{ + messageNetworks: { + ftn: { + networks: { + fsxnet: { + defaultZone: 21 + localAddress: "21:1/121" + } + } + + areas: { + fsx_general: { + network: fsxnet + + // ie as found in your info packs .NA file + tag: FSX_GEN + + uplinks: [ "21:1/100" ] + } + } + } + } +} +``` + +:information_source: Remember for a complete FTN experience, you'll probably also want to configure [FTN/BSO scanner/tosser](bso-import-export.md) settings. + +#### FTN/BSO Scanner Tosser +Please see the [FTN/BSO Scanner/Tosser](bso-import-export.md) documentation for information on this area. \ No newline at end of file diff --git a/docs/messageareas/message-networks.md b/docs/messageareas/message-networks.md index 5838da53..7fbd80c2 100644 --- a/docs/messageareas/message-networks.md +++ b/docs/messageareas/message-networks.md @@ -3,103 +3,23 @@ layout: page title: Message Networks --- ## Message Networks -ENiGMA½ considers all non-ENiGMA½, non-local messages (and their networks, such as FTN "external". That is, messages are only imported and exported from/to such a networks. Configuring such external message networks in ENiGMA½ requires three sections in your `config.hjson`. +ENiGMA½ considers all non-ENiGMA½, non-local messages (and their networks, such as FidoNet-Style (FTN) "external". That is, messages are only imported and exported from/to such a networks. Configuring such external message networks in ENiGMA½ requires three sections in your `config.hjson`. -1. `messageNetworks..networks`: declares available networks. -2. `messageNetworks..areas`: establishes local area mappings and per-area specifics. -3. `scannerTossers.`: general configuration for the scanner/tosser (import/export). This is also where we configure per-node settings. +All message network configuration occurs under the `messageNetworks.` block in `config.hjson` (where name is something such as `ftn` or `qwk`). Similarly, if a scanner/tosser module exists for the network it can be configured under `scannerTossers.`. An example of this is the [FTN/BSO scanner/tosser](bso-import-export.md) module where name is `ftn_bso`. -### FTN Networks +The most basic of external message network configurations generally comprises of two sections within `config.hjson`: + +1. `messageNetworks..networks`: Global/general configuration for a particular network where `` is for example `ftn` or `qwk`. +2. `messageNetworks..areas`: Provides mapping of ENiGMA½ **area tags** to their external counterparts. + +Finally, a related section under `scannerTossers.` may provide configuration for scanning (importing) and tossing (exporting) messages for a particular network type. As an example, FidoNet-Style networks often work with BinkleyTerm Style Outbound (BSO) and thus the [FTN/BSO scanner/tosser](bso-import-export.md) module. + +### Supported Networks + +#### FidoNet-Style (FTN) FidoNet and FidoNet style (FTN) networks as well as a [FTN/BSO scanner/tosser](bso-import-export.md) (`ftn_bso` module) are configured via the `messageNetworks.ftn` and `scannerTossers.ftn_bso` blocks in `config.hjson`. -:information_source: ENiGMA½'s `ftn_bso` module is **not a mailer** and makes **no attempts** to perform packet transport! An external utility such as Binkd is required for this! +See [FidoNet-Style Networks](ftn.md) for more information. -#### Networks -The `networks` block a per-network configuration where each entry's key may be referenced elsewhere in `config.hjson`. - -Example: the following example declares two networks: `araknet` and `fsxnet`: -```hjson -{ - messageNetworks: { - ftn: { - networks: { - // it is recommended to use lowercase network tags - fsxnet: { - defaultZone: 21 - localAddress: "21:1/121" - } - - araknet: { - defaultZone: 10 - localAddress: "10:101/9" - } - } - } - } -} -``` - -#### Areas -The `areas` section describes a mapping of local **area tags** configured in your `messageConferences` (see [Configuring a Message Area](configuring-a-message-area.md)) to a message network (described above), a FTN specific area tag, and remote uplink address(s). This section can be thought of similar to the *AREAS.BBS* file used by other BBS packages. - -When ENiGMA½ imports messages, they will be placed in the local area that matches key under `areas` while exported messages will be sent to the relevant `network`. - -| Config Item | Required | Description | -|-------------|----------|----------------------------------------------------------| -| `network` | :+1: | Associated network from the `networks` section above | -| `tag` | :+1: | FTN area tag (ie: `FSX_GEN`) | -| `uplinks` | :+1: | An array of FTN address uplink(s) for this network | - -Example: -```hjson -{ - messageNetworks: { - ftn: { - areas: { - // it is recommended to use lowercase area tags - fsx_general: // *local* tag found within messageConferences - network: fsxnet // that we are mapping to this network - tag: FSX_GEN // ...and this remote FTN-specific tag - uplinks: [ "21:1/100" ] // a single string also allowed here - } - } - } - } -} -``` - -:information_source: You can import `AREAS.BBS` or FTN style `.NA` files using [oputil](/docs/admin/oputil.md)! - -### A More Complete Example -Below is a more complete *example* illustrating some of the concepts above: - -```hjson -{ - messageNetworks: { - ftn: { - networks: { - fsxnet: { - defaultZone: 21 - localAddress: "21:1/121" - } - } - - areas: { - fsx_general: { - network: fsxnet - - // ie as found in your info packs .NA file - tag: FSX_GEN - - uplinks: [ "21:1/100" ] - } - } - } - } -} -``` - -:information_source: Remember for a complete FTN experience, you'll probably also want to configure [FTN/BSO scanner/tosser](bso-import-export.md) settings. - -### FTN/BSO Scanner Tosser -Please see the [FTN/BSO Scanner/Tosser](bso-import-export.md) documentation for information on this area. +#### QWK +See [QWK and QWK-Net Style Networks](qwk.md) for more information. diff --git a/docs/messageareas/qwk.md b/docs/messageareas/qwk.md new file mode 100644 index 00000000..6c72afb0 --- /dev/null +++ b/docs/messageareas/qwk.md @@ -0,0 +1,52 @@ +--- +layout: page +title: QWK Support +--- + +## QWK and QWK-Net Style Networks +As like all other such as FidoNet-Style (FTN) networks, ENiGMA½ considers QWK external to the system but can import and export the format. + +### Supported Standards +QWK must be considered a semi-standard as there are many implementations. What follows is a short & incomplete list of such standards ENiGMA½ supports: +* The basic [QWK packet format](http://fileformats.archiveteam.org/wiki/QWK). +* [QWKE extensions](https://github.com/wwivbbs/wwiv/blob/master/specs/qwk/qwke.txt). +* [Synchronet BBS style extensions](http://wiki.synchro.net/ref:qwk) such as `HEADERS.DAT`, `@` kludges, and UTF-8 handling. + + +### Configuration +QWK configuration occurs in the `messageNetworks.qwk` config block of `config.hjson`. As QWK wants to deal with conference numbers and ENiGMA½ uses area tags (conferences and conference tags are only used for logical grouping), a mapping can be made. + +:information_source: During a regular, non QWK-Net exports, conference numbers can be auto-generated. Note that for QWK-Net style networks, you will need to create mappings however. + +:TODO: information on QWK-Net type setup here + +Example: +```hjson +{ + messageNetworks: { + qwk: { + areas: { + general: { // local ENiGMA½ area tag + conference: 1 // conference number to map to + } + } + } + } +} +``` + +### oputil +The `oputil.js` utility can export packet files, dump the messages of a packet to stdout, etc. + +TODO: Examples, etc. +TODO: Link to oputil, update --help there. + +### Offline Readers +A few of the offline readers that have been tested with QWK packet files produced by ENiGMA: + +| Software | Status | Notes | +|----------|--------|-------| +| MultiMail/Win v0.52 | Supported | Private mail seems to break even with bundles from other systems | +| SkyReader/W32 v1.00 | Supported | Works well. No QWKE or HEADERS.DAT support. Gets confused with low conference numbers. | + +There are also [many other readers](https://www.softwolves.pp.se/old/2000/faq/bwprod) for various systems. \ No newline at end of file From f56e30442a379c4d06380b77a5b62d34943368e8 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 5 May 2020 21:18:58 -0600 Subject: [PATCH 26/95] Suggestion: option to trigger List of pending users. #272 --- core/oputil/oputil_help.js | 9 +++++ core/oputil/oputil_user.js | 82 +++++++++++++++++++++++++++++++++----- 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/core/oputil/oputil_help.js b/core/oputil/oputil_help.js index 0d132445..c3767c92 100644 --- a/core/oputil/oputil_help.js +++ b/core/oputil/oputil_help.js @@ -59,6 +59,15 @@ Actions: group USERNAME [+|-]GROUP Adds (+) or removes (-) user from a group + list [FILTER] List users with optional FILTER. + + Valid filters: + all : All users (default). + disabled : Disabled users. + inactive : Inactive users. + active : Active (regular) users. + locked : Locked users. + info arguments: --security Include security information in output diff --git a/core/oputil/oputil_user.js b/core/oputil/oputil_user.js index 78ea08ca..376ee166 100644 --- a/core/oputil/oputil_user.js +++ b/core/oputil/oputil_user.js @@ -460,6 +460,64 @@ function twoFactorAuthOTP(user) { ); } +function listUsers() { + // oputil user list [disabled|inactive|active|locked|all] + // :TODO: --after TIMESTAMP (new users) + // :TODO: --sort name|id + let listWhat; + if (argv._.length > 2) { + listWhat = argv._[argv._.length - 1]; + } else { + listWhat = 'all'; + } + + const User = require('../../core/user'); + if (![ 'all' ].concat(Object.keys(User.AccountStatus)).includes(listWhat)) { + return printUsageAndSetExitCode(getHelpFor('User'), ExitCodes.ERROR); + } + + async.waterfall( + [ + (callback) => { + const UserProps = require('../../core/user_property'); + + const userListOpts = { + properties : [ + UserProps.AccountStatus, + ], + }; + + User.getUserList(userListOpts, (err, userList) => { + if (err) { + return callback(err); + } + + if ('all' === listWhat) { + return callback(null, userList); + } + + const accountStatusFilter = User.AccountStatus[listWhat].toString(); + + return callback(null, userList.filter(user => { + return user[UserProps.AccountStatus] === accountStatusFilter; + })); + }); + }, + (userList, callback) => { + userList.forEach(user => { + + console.info(`${user.userId}: ${user.userName}`); + }); + }, + ], + err => { + if(err) { + return console.error(err.reason ? err.reason : err.message); + } + } + ); +} + function handleUserCommand() { function errUsage() { return printUsageAndSetExitCode(getHelpFor('User'), ExitCodes.ERROR); @@ -470,20 +528,25 @@ function handleUserCommand() { } const action = argv._[1]; - const usernameIdx = [ - 'pw', 'pass', 'passwd', 'password', - 'group', - 'mv', 'rename', - '2fa-otp', 'otp' - ].includes(action) ? argv._.length - 2 : argv._.length - 1; - const userName = argv._[usernameIdx]; + const userRequired = ![ 'list' ].includes(action); - if(!userName) { + let userName; + if (userRequired) { + const usernameIdx = [ + 'pw', 'pass', 'passwd', 'password', + 'group', + 'mv', 'rename', + '2fa-otp', 'otp' + ].includes(action) ? argv._.length - 2 : argv._.length - 1; + userName = argv._[usernameIdx]; + } + + if(!userName && userRequired) { return errUsage(); } initAndGetUser(userName, (err, user) => { - if(err) { + if(userName && err) { process.exitCode = ExitCodes.ERROR; return console.error(err.message); } @@ -512,6 +575,7 @@ function handleUserCommand() { '2fa-otp' : twoFactorAuthOTP, otp : twoFactorAuthOTP, + list : listUsers, }[action] || errUsage)(user, action); }); } \ No newline at end of file From ec994211e06004f75c89adf072c346c920dea8a2 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 6 May 2020 11:12:19 -0600 Subject: [PATCH 27/95] More doc updates --- docs/admin/oputil.md | 17 ++++++++++++++++- docs/messageareas/ftn.md | 10 +++++----- docs/messageareas/message-networks.md | 11 +++++------ docs/messageareas/qwk.md | 11 +++-------- 4 files changed, 29 insertions(+), 20 deletions(-) diff --git a/docs/admin/oputil.md b/docs/admin/oputil.md index 0b05f7aa..e591de9f 100644 --- a/docs/admin/oputil.md +++ b/docs/admin/oputil.md @@ -291,17 +291,32 @@ Actions: import-areas PATH Import areas using FidoNet *.NA or AREAS.BBS file + qwk-dump PATH Dumps a QWK packet to stdout. + qwk-export [AREA_TAGS] PATH Exports one or more configured message area to a QWK + packet in the directory specified by PATH. The QWK + BBS ID will be obtained by the final component of PATH. + import-areas arguments: --conf CONF_TAG Conference tag in which to import areas --network NETWORK Network name/key to associate FTN areas --uplinks UL1,UL2,... One or more uplinks (comma separated) --type TYPE Area import type - Valid types are "bbs" and "na" + Valid types are "bbs" and "na". + +qwk-export arguments: + --user USER User in which to export for. Defaults to the SysOp. + --after TIMESTAMP Export only messages with a timestamp later than + TIMESTAMP. + --no-qwke Disable QWKE extensions. + --no-synchronet Disable Synchronet style extensions. ``` | Action | Description | Examples | |-----------|-------------------|---------------------------------------| | `import-areas` | Imports areas using a FidoNet style *.NA or AREAS.BBS formatted file. Optionally maps areas to FTN networks. | `./oputil.js config import-areas /some/path/l33tnet.na` | +| `areafix` | Utility for sending AreaFix mails without logging into the system | | +| `qwk-dump` | Dump a QWK packet to stdout | `./oputil.js mb qwk-dump /path/to/XIBALBA.QWK` | +| `qwk-export` | Export messages to a QWK packet | `./oputil.js mb qwk-export /path/to/XIBALBA.QWK` | When using the `import-areas` action, you will be prompted for any missing additional arguments described in "import-areas args". diff --git a/docs/messageareas/ftn.md b/docs/messageareas/ftn.md index f1bb5366..34f90862 100644 --- a/docs/messageareas/ftn.md +++ b/docs/messageareas/ftn.md @@ -4,14 +4,14 @@ title: FidoNet-Style Networks (FTN) --- ## FidoNet-Style Networks (FTN) - -TODO: preamble +[FidoNet](https://en.wikipedia.org/wiki/FidoNet) proper and other FidoNet-Style networks are supported by ENiGMA½. A bit of configuration and you'll be up and running in no time! ### Configuration +Getting a fully running FTN enabled system requires a few configuration points: -1. `messageNetworks.ftn.networks`: declares available networks. -2. `messageNetworks.ftn.areas`: establishes local area mappings and per-area specifics. -3. `scannerTossers.ftn_bso`: general configuration for the scanner/tosser (import/export). This is also where we configure per-node settings. +1. `messageNetworks.ftn.networks`: Declares available networks. That is, networks you wish to sync up with. +2. `messageNetworks.ftn.areas`: Establishes local area mappings (ENiGMA½ to/from FTN area tags) and per-area specific configurations. +3. `scannerTossers.ftn_bso`: General configuration for the scanner/tosser (import/export) process. This is also where we configure per-node (uplink) settings. :information_source: ENiGMA½'s `ftn_bso` module is **not a mailer** and makes **no attempts** to perform packet transport! An external utility such as Binkd is required for this task. diff --git a/docs/messageareas/message-networks.md b/docs/messageareas/message-networks.md index 7fbd80c2..52a4227e 100644 --- a/docs/messageareas/message-networks.md +++ b/docs/messageareas/message-networks.md @@ -3,18 +3,17 @@ layout: page title: Message Networks --- ## Message Networks -ENiGMA½ considers all non-ENiGMA½, non-local messages (and their networks, such as FidoNet-Style (FTN) "external". That is, messages are only imported and exported from/to such a networks. Configuring such external message networks in ENiGMA½ requires three sections in your `config.hjson`. +ENiGMA½ supports external networks such as FidoNet-Style (FTN) and QWK by the way of importing and exporting to/from it's own internal format. This allows for a very flexible system that can easily be extended by creating new network modules. -All message network configuration occurs under the `messageNetworks.` block in `config.hjson` (where name is something such as `ftn` or `qwk`). Similarly, if a scanner/tosser module exists for the network it can be configured under `scannerTossers.`. An example of this is the [FTN/BSO scanner/tosser](bso-import-export.md) module where name is `ftn_bso`. - -The most basic of external message network configurations generally comprises of two sections within `config.hjson`: +All message network configuration occurs under the `messageNetworks.` block in `config.hjson` (where name is something such as `ftn` or `qwk`). The most basic of external message network configurations generally comprises of two sections: 1. `messageNetworks..networks`: Global/general configuration for a particular network where `` is for example `ftn` or `qwk`. 2. `messageNetworks..areas`: Provides mapping of ENiGMA½ **area tags** to their external counterparts. -Finally, a related section under `scannerTossers.` may provide configuration for scanning (importing) and tossing (exporting) messages for a particular network type. As an example, FidoNet-Style networks often work with BinkleyTerm Style Outbound (BSO) and thus the [FTN/BSO scanner/tosser](bso-import-export.md) module. +:information_source: A related section under `scannerTossers.` may provide configuration for scanning (importing) and tossing (exporting) messages for a particular network type. As an example, FidoNet-Style networks often work with BinkleyTerm Style Outbound (BSO) and thus the [FTN/BSO scanner/tosser](bso-import-export.md) (`ftn_bso`) module. -### Supported Networks +### Currently Supported Networks +The following networks are supported out of the box. Remember that you can create modules to add others if desired! #### FidoNet-Style (FTN) FidoNet and FidoNet style (FTN) networks as well as a [FTN/BSO scanner/tosser](bso-import-export.md) (`ftn_bso` module) are configured via the `messageNetworks.ftn` and `scannerTossers.ftn_bso` blocks in `config.hjson`. diff --git a/docs/messageareas/qwk.md b/docs/messageareas/qwk.md index 6c72afb0..964551a3 100644 --- a/docs/messageareas/qwk.md +++ b/docs/messageareas/qwk.md @@ -4,7 +4,7 @@ title: QWK Support --- ## QWK and QWK-Net Style Networks -As like all other such as FidoNet-Style (FTN) networks, ENiGMA½ considers QWK external to the system but can import and export the format. +As like all other networks such as FidoNet-Style (FTN) networks, ENiGMA½ considers QWK external to the system but can import and export the format. ### Supported Standards QWK must be considered a semi-standard as there are many implementations. What follows is a short & incomplete list of such standards ENiGMA½ supports: @@ -18,8 +18,6 @@ QWK configuration occurs in the `messageNetworks.qwk` config block of `config.hj :information_source: During a regular, non QWK-Net exports, conference numbers can be auto-generated. Note that for QWK-Net style networks, you will need to create mappings however. -:TODO: information on QWK-Net type setup here - Example: ```hjson { @@ -36,13 +34,10 @@ Example: ``` ### oputil -The `oputil.js` utility can export packet files, dump the messages of a packet to stdout, etc. - -TODO: Examples, etc. -TODO: Link to oputil, update --help there. +The `oputil.js` utility can export packet files, dump the messages of a packet to stdout, etc. See [the oputil documentation](/docs/admin/oputil.md) for more information. ### Offline Readers -A few of the offline readers that have been tested with QWK packet files produced by ENiGMA: +A few of the offline readers that have been tested with QWK packet files produced by ENiGMA½: | Software | Status | Notes | |----------|--------|-------| From 59ee52ea6fb9b82ed1e303b87334b2c7c27e6193 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 6 May 2020 17:34:37 -0600 Subject: [PATCH 28/95] Tidy + some logging --- core/scanner_tossers/ftn_bso.js | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 3fec4fc3..046df546 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -1931,8 +1931,8 @@ function FTNMessageScanTossModule() { `SELECT message_id, message_uuid FROM message m WHERE area_tag = ? AND message_id > ? AND - (SELECT COUNT(message_id) - FROM message_meta + (SELECT COUNT(message_id) + FROM message_meta WHERE message_id = m.message_id AND meta_category = 'System' AND meta_name = 'state_flags0') = 0 ORDER BY message_id;` ; @@ -2012,7 +2012,7 @@ function FTNMessageScanTossModule() { const getNewUuidsSql = `SELECT message_id, message_uuid FROM message m - WHERE area_tag = '${Message.WellKnownAreaTags.Private}' AND message_id > ? AND + WHERE area_tag = '${Message.WellKnownAreaTags.Private}' AND message_id > ? AND (SELECT COUNT(message_id) FROM message_meta WHERE message_id = m.message_id @@ -2023,7 +2023,7 @@ function FTNMessageScanTossModule() { (SELECT COUNT(message_id) FROM message_meta WHERE message_id = m.message_id - AND meta_category = 'System' + AND meta_category = 'System' AND meta_name = '${Message.SystemMetaNames.ExternalFlavor}' AND meta_value = '${Message.AddressFlavor.FTN}' ) = 1 @@ -2287,13 +2287,18 @@ FTNMessageScanTossModule.prototype.shutdown = function(cb) { FTNMessageScanTossModule.prototype.performImport = function(cb) { if(!this.hasValidConfiguration()) { - return cb(new Error('Missing or invalid configuration')); + return cb(Errors.MissingConfig('Invalid or missing configuration')); } const self = this; async.each( [ 'inbound', 'secInbound' ], (inboundType, nextDir) => { - self.importFromDirectory(inboundType, self.moduleConfig.paths[inboundType], () => { + const importDir = self.moduleConfig.paths[inboundType]; + self.importFromDirectory(inboundType, importDir, err => { + if (err) { + Log.warn({ importDir, error : err.message }, 'Error(s) during import'); + } + return nextDir(null); }); }, cb); @@ -2305,7 +2310,7 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { // and let's find out what messages need exported. // if(!this.hasValidConfiguration()) { - return cb(new Error('Missing or invalid configuration')); + return cb(Errors.MissingConfig('Invalid or missing configuration')); } const self = this; @@ -2313,7 +2318,7 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) { async.eachSeries( [ 'EchoMail', 'NetMail' ], (type, nextType) => { self[`perform${type}Export`]( err => { if(err) { - Log.warn( { error : err.message, type : type }, 'Error(s) during export' ); + Log.warn( { type, error : err.message }, 'Error(s) during export' ); } return nextType(null); // try next, always }); From 387dfb3b5dfbb7ce3a4a976bc1f6f56d6f86cc07 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 6 May 2020 17:47:13 -0600 Subject: [PATCH 29/95] Don't idle logout if a transfer is making progress --- core/client.js | 12 ++++++++---- core/file_transfer.js | 11 +++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/core/client.js b/core/client.js index 566aec5d..a6cfae52 100644 --- a/core/client.js +++ b/core/client.js @@ -84,7 +84,7 @@ function Client(/*input, output*/) { this.user = new User(); this.currentTheme = { info : { name : 'N/A', description : 'None' } }; - this.lastKeyPressMs = Date.now(); + this.lastActivityTime = Date.now(); this.menuStack = new MenuStack(this); this.acs = new ACS( { client : this, user : this.user } ); this.mciCache = {}; @@ -406,7 +406,7 @@ function Client(/*input, output*/) { self.log.trace( { key : key, ch : escape(ch) }, 'User keyboard input'); // jshint ignore:line } - self.lastKeyPressMs = Date.now(); + self.lastActivityTime = Date.now(); if(!self.ignoreInput) { self.emit('key press', ch, key); @@ -438,7 +438,7 @@ Client.prototype.startIdleMonitor = function() { this.stopIdleMonitor(); } - this.lastKeyPressMs = Date.now(); + this.lastActivityTime = Date.now(); // // Every 1m, check for idle. @@ -476,7 +476,7 @@ Client.prototype.startIdleMonitor = function() { // use override value if set idleLogoutSeconds = this.idleLogoutSecondsOverride || idleLogoutSeconds; - if(idleLogoutSeconds > 0 && (nowMs - this.lastKeyPressMs >= (idleLogoutSeconds * 1000))) { + if(idleLogoutSeconds > 0 && (nowMs - this.lastActivityTime >= (idleLogoutSeconds * 1000))) { this.emit('idle timeout'); } }, 1000 * 60); @@ -489,6 +489,10 @@ Client.prototype.stopIdleMonitor = function() { } }; +Client.prototype.explicitActivityTimeUpdate = function() { + this.lastActivityTime = Date.now(); +} + Client.prototype.overrideIdleLogoutSeconds = function(seconds) { this.idleLogoutSecondsOverride = seconds; }; diff --git a/core/file_transfer.js b/core/file_transfer.js index 5f0138e2..ff9216d6 100644 --- a/core/file_transfer.js +++ b/core/file_transfer.js @@ -378,7 +378,16 @@ exports.getModule = class TransferFileModule extends MenuModule { const externalProc = pty.spawn(cmd, args, spawnOpts); + let dataHits = 0; + const updateActivity = () => { + if (0 === (dataHits++ % 4)) { + this.client.explicitActivityTimeUpdate(); + } + }; + this.client.setTemporaryDirectDataHandler(data => { + updateActivity(); + // needed for things like sz/rz if(external.escapeTelnet) { const tmp = data.toString('binary').replace(/\xff{2}/g, '\xff'); // de-escape @@ -389,6 +398,8 @@ exports.getModule = class TransferFileModule extends MenuModule { }); externalProc.on('data', data => { + updateActivity(); + // needed for things like sz/rz if(external.escapeTelnet) { const tmp = data.toString('binary').replace(/\xff/g, '\xff\xff'); // escape From 3732914c4a71871793e3fa75c8982b8f387850a3 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 6 May 2020 17:50:44 -0600 Subject: [PATCH 30/95] Add extra log --- core/scanner_tossers/ftn_bso.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 046df546..9bc272fe 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -2151,6 +2151,10 @@ FTNMessageScanTossModule.prototype.processTicFilesInDirectory = function(importD FTNMessageScanTossModule.prototype.startup = function(cb) { Log.info(`${exports.moduleInfo.name} Scanner/Tosser starting up`); + if (!this.hasValidConfiguration()) { + Log.debug('No configuration present'); + } + let importing = false; let self = this; From d6dce82a92dfc6c350c1f52621fccc48577cada6 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 7 May 2020 19:00:32 -0600 Subject: [PATCH 31/95] Clean up log a bit --- core/scanner_tossers/ftn_bso.js | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 9bc272fe..488dcc8e 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -517,8 +517,20 @@ function FTNMessageScanTossModule() { }; - this.hasValidConfiguration = function() { - if(!_.has(this, 'moduleConfig.nodes') || !_.has(Config(), 'messageNetworks.ftn.areas')) { + this.hasValidConfiguration = function({shouldLog = false} = {}) { + const hasNodes = _.has(this, 'moduleConfig.nodes'); + const hasAreas = _.has(Config(), 'messageNetworks.ftn.areas'); + + if(!hasNodes && !hasAreas) { + if (shouldLog) { + Log.warn( + { + 'scannerTossers.ftn_bso.nodes' : hasNodes, + 'messageNetworks.ftn.areas' : hasAreas, + }, + 'Missing one or more required configuration blocks' + ); + } return false; } @@ -2151,9 +2163,7 @@ FTNMessageScanTossModule.prototype.processTicFilesInDirectory = function(importD FTNMessageScanTossModule.prototype.startup = function(cb) { Log.info(`${exports.moduleInfo.name} Scanner/Tosser starting up`); - if (!this.hasValidConfiguration()) { - Log.debug('No configuration present'); - } + this.hasValidConfiguration({ shouldLog : true }); // just check and log let importing = false; @@ -2300,7 +2310,7 @@ FTNMessageScanTossModule.prototype.performImport = function(cb) { const importDir = self.moduleConfig.paths[inboundType]; self.importFromDirectory(inboundType, importDir, err => { if (err) { - Log.warn({ importDir, error : err.message }, 'Error(s) during import'); + Log.trace({ importDir, error : err.message }, 'Cannot perform FTN import for directory'); } return nextDir(null); From 14f7ca9dcc4403086d2ab8c25f680741cd8f3b72 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 7 May 2020 20:02:12 -0600 Subject: [PATCH 32/95] Add Ability to Import All File Areas at Once #271 * Simple wildcard support for area tags param --- core/file_base_area.js | 11 +++++++++++ core/oputil/oputil_file_base.js | 19 ++++++++++++++++++- core/oputil/oputil_help.js | 8 ++++++-- core/string_util.js | 6 ++++++ 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/core/file_base_area.js b/core/file_base_area.js index e3887b84..e9926111 100644 --- a/core/file_base_area.js +++ b/core/file_base_area.js @@ -17,6 +17,7 @@ const StatLog = require('./stat_log.js'); const UserProps = require('./user_property.js'); const SysProps = require('./system_property.js'); const SAUCE = require('./sauce.js'); +const { wildcardMatch } = require('./string_util'); // deps const _ = require('lodash'); @@ -40,6 +41,7 @@ exports.getAreaDefaultStorageDirectory = getAreaDefaultStorageDirectory; exports.getAreaStorageLocations = getAreaStorageLocations; exports.getDefaultFileAreaTag = getDefaultFileAreaTag; exports.getFileAreaByTag = getFileAreaByTag; +exports.getFileAreasByTagWildcardRule = getFileAreasByTagWildcardRule; exports.getFileEntryPath = getFileEntryPath; exports.changeFileAreaWithOptions = changeFileAreaWithOptions; exports.scanFile = scanFile; @@ -143,6 +145,15 @@ function getFileAreaByTag(areaTag) { } } +function getFileAreasByTagWildcardRule(rule) { + const areaTags = Object.keys(Config().fileBase.areas) + .filter(areaTag => { + return !isInternalArea(areaTag) && wildcardMatch(areaTag, rule); + }); + + return areaTags.map(areaTag => getFileAreaByTag(areaTag)); +} + function changeFileAreaWithOptions(client, areaTag, options, cb) { async.waterfall( [ diff --git a/core/oputil/oputil_file_base.js b/core/oputil/oputil_file_base.js index 67962fc0..308d3581 100644 --- a/core/oputil/oputil_file_base.js +++ b/core/oputil/oputil_file_base.js @@ -521,7 +521,24 @@ function scanFileAreas() { }); }, function scanAreas(callback) { - fileArea = require('../../core/file_base_area.js'); + fileArea = require('../../core/file_base_area'); + + // Further expand any wildcards + let areaAndStorageInfoExpanded = []; + options.areaAndStorageInfo.forEach(info => { + if (info.areaTag.indexOf('*') > -1) { + const areas = fileArea.getFileAreasByTagWildcardRule(info.areaTag); + areas.forEach(area => { + areaAndStorageInfoExpanded.push(Object.assign({}, info, { + areaTag : area.areaTag, + })); + }); + } else { + areaAndStorageInfoExpanded.push(info); + } + }); + + options.areaAndStorageInfo = areaAndStorageInfoExpanded; async.eachSeries(options.areaAndStorageInfo, (areaAndStorage, nextAreaTag) => { const areaInfo = fileArea.getFileAreaByTag(areaAndStorage.areaTag); diff --git a/core/oputil/oputil_help.js b/core/oputil/oputil_help.js index c3767c92..50bc3b5e 100644 --- a/core/oputil/oputil_help.js +++ b/core/oputil/oputil_help.js @@ -101,8 +101,12 @@ cat arguments: Actions: scan AREA_TAG[@STORAGE_TAG] Scan specified area - May contain optional GLOB as last parameter. - Example: ./oputil.js fb scan d0pew4r3z *.zip + Tips: + - May contain optional GLOB as last parameter. + Example: ./oputil.js fb scan d0pew4r3z *.zip + + - AREA_TAG may contain simple wildcards. + Example: ./oputil.js fb scan *warez* info CRITERIA Display information about areas and/or files diff --git a/core/string_util.js b/core/string_util.js index cde7ac3e..2f6596c8 100644 --- a/core/string_util.js +++ b/core/string_util.js @@ -29,6 +29,7 @@ exports.isAnsi = isAnsi; exports.isAnsiLine = isAnsiLine; exports.isFormattedLine = isFormattedLine; exports.splitTextAtTerms = splitTextAtTerms; +exports.wildcardMatch = wildcardMatch; // :TODO: create Unicode version of this const VOWELS = [ @@ -474,3 +475,8 @@ function isAnsi(input) { function splitTextAtTerms(s) { return s.split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g); } + +function wildcardMatch(input, rule) { + const escapeRegex = (s) => s.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1"); + return new RegExp("^" + rule.split("*").map(escapeRegex).join(".*") + "$").test(input); +} \ No newline at end of file From ee992278e818135fc5499ef84a54d08e2807c7b6 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 10 May 2020 21:56:05 -0600 Subject: [PATCH 33/95] WIP on QWK download support --- core/enig_error.js | 1 + core/file_base_user_list_export.js | 3 +- core/message_base_qwk_export.js | 378 +++++++++++++++++++++++++++++ core/qwk_mail_packet.js | 7 +- 4 files changed, 387 insertions(+), 2 deletions(-) create mode 100644 core/message_base_qwk_export.js diff --git a/core/enig_error.js b/core/enig_error.js index 08a3312e..be025214 100644 --- a/core/enig_error.js +++ b/core/enig_error.js @@ -37,6 +37,7 @@ exports.Errors = { MissingParam : (reason, reasonCode) => new EnigError('Missing paramter(s)', -32008, reason, reasonCode), MissingMci : (reason, reasonCode) => new EnigError('Missing required MCI code(s)', -32009, reason, reasonCode), BadLogin : (reason, reasonCode) => new EnigError('Bad login attempt', -32010, reason, reasonCode), + UserInterrupt : (reason, reasonCode) => new EnigError('User interrupted', -32011, reason, reasonCode), }; exports.ErrorReasons = { diff --git a/core/file_base_user_list_export.js b/core/file_base_user_list_export.js index 3c00d167..1307144c 100644 --- a/core/file_base_user_list_export.js +++ b/core/file_base_user_list_export.js @@ -28,7 +28,7 @@ const yazl = require('yazl'); tsFormat - timestamp format (theme 'short') descWidth - max desc width (45) progBarChar - progress bar character (▒) - compressThreshold - threshold to kick in comrpession for lists (1.44 MiB) + compressThreshold - threshold to kick in compression for lists (1.44 MiB) templates - object containing: header - filename of header template (misc/file_list_header.asc) entry - filename of entry template (misc/file_list_entry.asc) @@ -244,6 +244,7 @@ exports.getModule = class FileBaseListExport extends MenuModule { }, function done(callback) { // re-enable idle monitor + // :TODO: this should probably be moved down below at the end of the full waterfall self.client.startIdleMonitor(); updateStatus('Exported list has been added to your download queue'); diff --git a/core/message_base_qwk_export.js b/core/message_base_qwk_export.js new file mode 100644 index 00000000..b79a6411 --- /dev/null +++ b/core/message_base_qwk_export.js @@ -0,0 +1,378 @@ +// ENiGMA½ +const { MenuModule } = require('./menu_module'); +const Message = require('./message'); +const { Errors } = require('./enig_error'); +const { + getMessageAreaByTag, + hasMessageConfAndAreaRead, +} = require('./message_area'); +const FileArea = require('./file_base_area'); +const { QWKPacketWriter } = require('./qwk_mail_packet'); +const { renderSubstr } = require('./string_util'); +const Config = require('./config').get; +const FileEntry = require('./file_entry'); +const Events = require('./events'); +const DownloadQueue = require('./download_queue'); + +// deps +const async = require('async'); +const _ = require('lodash'); +const fse = require('fs-extra'); +const temptmp = require('temptmp'); +const paths = require('path'); +const UUIDv4 = require('uuid/v4'); + +const FormIds = { + main : 0, +}; + +const MciViewIds = { + main : { + status : 1, + progressBar : 2, + + customRangeStart : 10, + } +}; + +exports.moduleInfo = { + name : 'QWK Export', + desc : 'Exports a QWK Packet for download', + author : 'NuSkooler', +}; + +exports.getModule = class MessageBaseQWKExport extends MenuModule { + constructor(options) { + super(options); + + this.config = Object.assign({}, _.get(options, 'menuConfig.config'), options.extraArgs); + + this.config.progBarChar = renderSubstr( (this.config.progBarChar || '▒'), 0, 1); + this.config.bbsID = this.config.bbsID || _.get(Config(), 'messageNetworks.qwk.bbsID', 'ENIGMA'); + + this.tempName = `${UUIDv4().substr(-8).toUpperCase()}.QWK`; + this.sysTempDownloadArea = FileArea.getFileAreaByTag(FileArea.WellKnownAreaTags.TempDownloads); + } + + mciReady(mciData, cb) { + super.mciReady(mciData, err => { + if (err) { + return cb(err); + } + + async.waterfall( + [ + (callback) => { + this.prepViewController('main', FormIds.main, mciData.menu, err => { + return callback(err); + }); + }, + (callback) => { + this.temptmp = temptmp.createTrackedSession('qwkuserexp'); + this.temptmp.mkdir({ prefix : 'enigqwkwriter-'}, (err, tempDir) => { + if (err) { + return callback(err); + } + + this.tempPacketDir = tempDir; + + const sysTempDownloadDir = FileArea.getAreaDefaultStorageDirectory(this.sysTempDownloadArea); + + // ensure dir exists + fse.mkdirs(sysTempDownloadDir, err => { + return callback(err, sysTempDownloadDir); + }); + }); + }, + (sysTempDownloadDir, callback) => { + this.performExport(sysTempDownloadDir, err => { + return callback(err); + }); + }, + ], + err => { + this.temptmp.cleanup(); + + if (err) { + // :TODO: doesn't do anything currently: + if ('NORESULTS' === err.reasonCode) { + return this.gotoMenu(this.menuConfig.config.noResultsMenu || 'qwkExportNoResults'); + } + + return this.prevMenu(); + } + return cb(err); + } + ); + }); + } + + finishedLoading() { + this.prevMenu(); + } + + performExport(sysTempDownloadDir, cb) { + const statusView = this.viewControllers.main.getView(MciViewIds.main.status); + const updateStatus = (status) => { + if (statusView) { + statusView.setText(status); + } + }; + + const progBarView = this.viewControllers.main.getView(MciViewIds.main.progressBar); + const updateProgressBar = (curr, total) => { + if (progBarView) { + const prog = Math.floor( (curr / total) * progBarView.dimens.width ); + progBarView.setText(this.config.progBarChar.repeat(prog)); + } + }; + + let cancel = false; + + let lastProgUpdate = 0; + const progressHandler = (state, next) => { + // we can produce a TON of updates; only update progress at most every 3/4s + if (Date.now() - lastProgUpdate > 750) { + switch (state.step) { + case 'next_area' : + updateStatus(state.status); + updateProgressBar(0, 0); + this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.area); + break; + + case 'message' : + updateStatus(state.status); + updateProgressBar(state.current, state.total); + this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.message); + break; + + default : + break; + } + lastProgUpdate = Date.now(); + } + + return next(cancel ? Errors.UserInterrupt('User canceled') : null); + }; + + const keyPressHandler = (ch, key) => { + if('escape' === key.name) { + cancel = true; + this.client.removeListener('key press', keyPressHandler); + } + }; + + const processMessagesWithFilter = (filter, cb) => { + Message.findMessages(filter, (err, messageIds) => { + if (err) { + return cb(err); + } + + let current = 1; + async.eachSeries(messageIds, (messageId, nextMessageId) => { + const message = new Message(); + message.load({ messageId }, err => { + if (err) { + return nextMessageId(err); + } + + const progress = { + current, + message, + step : 'message', + total : messageIds.length, + status : `Writing message ${current} / ${messageIds.length}`, + }; + + progressHandler(progress, err => { + if (err) { + return nextMessageId(err); + } + + packetWriter.appendMessage(message); + current += 1; + + return nextMessageId(null); + }); + }); + }, + err => { + return cb(err); + }); + }); + }; + + const packetWriter = new QWKPacketWriter({ + user : this.client.user, + bbsID : this.config.bbsID, + }); // :TODO: User configuration here + + packetWriter.on('warning', warning => { + this.client.log.warn( { warning }, 'QWK packet writer warning'); + }); + + async.waterfall( + [ + (callback) => { + // don't count idle monitor while processing + this.client.stopIdleMonitor(); + + // let user cancel + this.client.on('key press', keyPressHandler); + + packetWriter.once('ready', () => { + return callback(null); + }); + + packetWriter.once('error', err => { + this.client.log.error( { error : err.message }, 'QWK packet writer error'); + cancel = true; + }); + + packetWriter.init(); + }, + (callback) => { + // + // Fetch messages for user-configured area tags. + // - If private tag is present, we fetch this separately. + // - User property determines newscan timestamps dates if present for tag, else "all" + // - We have to fetch one area at a time in order to process message pointers/timestamps. + // ...this also allows for better progress. + // + // TL;DR: for each area -> for each message + // + const exportAreas = [ // :TODO: Load in something like this + { + areaTag : 'general', + newerThanTimestamp : '2018-01-01', + }, + { + areaTag : 'fsx_gen', + } + ]; + + async.eachSeries(exportAreas, (exportArea, nextExportArea) => { + const area = getMessageAreaByTag(exportArea.areaTag); + if (!area) { + // :TODO: remove from user properties - this area does not exist + this.client.log.warn({ areaTag : exportArea.areaTag }, 'Cannot QWK export area as it does not exist'); + return nextExportArea(null); + } + + if (!hasMessageConfAndAreaRead(this.client, area)) { + this.client.log.warn({ areaTag : area.areaTag }, 'Cannot QWK export area due to ACS'); + return nextExportArea(null); + } + + const progress = { + area, + step : 'next_area', + status : `Gathering messages in ${area.name}...`, + }; + + progressHandler(progress, err => { + if (err) { + return nextExportArea(err); + } + + const filter = { + resultType : 'id', + areaTag : exportArea.areaTag, + newerThanTimestamp : exportArea.newerThanTimestamp + }; + + processMessagesWithFilter(filter, err => { + return nextExportArea(err); + }); + }); + }, + err => { + return callback(err); + }); + }, + (callback) => { + const filter = { + resultType : 'id', + privateTagUserId : this.client.user.userId, + // :TODO: newerThanTimestamp for private messages + //newerThanTimestamp : exportArea.newerThanTimestamp + }; + return processMessagesWithFilter(filter, callback); + }, + (callback) => { + let packetInfo; + packetWriter.once('packet', info => { + packetInfo = info; + }); + + packetWriter.once('finished', () => { + return callback(null, packetInfo); + }); + + packetWriter.finish(this.tempPacketDir); + }, + (packetInfo, callback) => { + const sysDownloadPath = paths.join(sysTempDownloadDir, this.tempName); + fse.move(packetInfo.path, sysDownloadPath, err => { + return callback(null, sysDownloadPath, packetInfo); + }); + }, + (sysDownloadPath, packetInfo, callback) => { + const newEntry = new FileEntry({ + areaTag : this.sysTempDownloadArea.areaTag, + fileName : paths.basename(sysDownloadPath), + storageTag : this.sysTempDownloadArea.storageTags[0], + meta : { + upload_by_username : this.client.user.username, + upload_by_user_id : this.client.user.userId, + byte_size : packetInfo.stats.size, + session_temp_dl : 1, // download is valid until session is over + + // :TODO: something like this: allow to override the displayed/downloaded as filename + // separate from the actual on disk filename. E.g. we could always download as "ENIGMA.QWK" + visible_filename : paths.basename(packetInfo.path), + } + }); + + newEntry.desc = 'QWK Export'; + + newEntry.persist(err => { + if(!err) { + // queue it! + const dlQueue = new DownloadQueue(this.client); + dlQueue.add(newEntry, true); // true=systemFile + + // clean up after ourselves when the session ends + // :TODO: DRY this with that in file_base_user_export + const thisClientId = this.client.session.id; + Events.once(Events.getSystemEvents().ClientDisconnected, evt => { + if(thisClientId === _.get(evt, 'client.session.id')) { + FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => { + if(err) { + Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' ); + } else { + Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' ); + } + }); + } + }); + } + return callback(err); + }); + } + ], + err => { + this.client.startIdleMonitor(); // re-enable + this.client.removeListener('key press', keyPressHandler); + + if (!err) { + updateStatus('A QWK packet has been placed in your download queue'); + } + + // :TODO: send user to download manager with pop flags/etc. + + return cb(err); + } + ); + } +}; \ No newline at end of file diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index eec737cb..3d613ca4 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -1242,7 +1242,12 @@ class QWKPacketWriter extends EventEmitter { files, this.workDir, err => { - return cb(err); + fs.stat(packetPath, (err, stats) => { + if (stats) { + this.emit('packet', { stats, path : packetPath } ); + } + return cb(err); + }); } ); }); From 9174b7b7100cca5555e80f27ef9b52ac58670e64 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 11 May 2020 19:52:01 -0600 Subject: [PATCH 34/95] Code cleanup, use some defaults --- core/download_queue.js | 32 ++++++++-- core/file_base_user_list_export.js | 19 +----- core/message_base_qwk_export.js | 96 ++++++++++++++++-------------- core/user_property.js | 3 +- 4 files changed, 81 insertions(+), 69 deletions(-) diff --git a/core/download_queue.js b/core/download_queue.js index 28ca3aac..1eb4023e 100644 --- a/core/download_queue.js +++ b/core/download_queue.js @@ -1,11 +1,12 @@ /* jslint node: true */ 'use strict'; -const FileEntry = require('./file_entry.js'); -const UserProps = require('./user_property.js'); +const FileEntry = require('./file_entry'); +const UserProps = require('./user_property'); +const Events = require('./events'); // deps -const { partition } = require('lodash'); +const _ = require('lodash'); module.exports = class DownloadQueue { constructor(client) { @@ -20,6 +21,10 @@ module.exports = class DownloadQueue { } } + static get(client) { + return new DownloadQueue(client); + } + get items() { return this.client.user.downloadQueue; } @@ -52,7 +57,7 @@ module.exports = class DownloadQueue { fileIds = [ fileIds ]; } - const [ remain, removed ] = partition(this.client.user.downloadQueue, e => ( -1 === fileIds.indexOf(e.fileId) )); + const [ remain, removed ] = _.partition(this.client.user.downloadQueue, e => ( -1 === fileIds.indexOf(e.fileId) )); this.client.user.downloadQueue = remain; return removed; } @@ -76,4 +81,23 @@ module.exports = class DownloadQueue { this.client.log.error( { error : e.message, property : prop }, 'Failed parsing download queue property'); } } + + addTemporaryDownload(entry) { + this.add(entry, true); // true=systemFile + + // clean up after ourselves when the session ends + const thisClientId = this.client.session.id; + Events.once(Events.getSystemEvents().ClientDisconnected, evt => { + if(thisClientId === _.get(evt, 'client.session.id')) { + FileEntry.removeEntry(entry, { removePhysFile : true }, err => { + const Log = require('./logger').log; + if(err) { + Log.warn( { fileId : entry.fileId, path : outputFileName }, 'Failed removing temporary session download' ); + } else { + Log.debug( { fileId : entry.fileId, path : outputFileName }, 'Removed temporary session download item' ); + } + }); + } + }); + } }; diff --git a/core/file_base_user_list_export.js b/core/file_base_user_list_export.js index 1307144c..594a9ffe 100644 --- a/core/file_base_user_list_export.js +++ b/core/file_base_user_list_export.js @@ -7,8 +7,6 @@ const FileEntry = require('./file_entry.js'); const FileArea = require('./file_base_area.js'); const { renderSubstr } = require('./string_util.js'); const { Errors } = require('./enig_error.js'); -const Events = require('./events.js'); -const Log = require('./logger.js').log; const DownloadQueue = require('./download_queue.js'); const { exportFileList } = require('./file_base_list_export.js'); @@ -222,22 +220,7 @@ exports.getModule = class FileBaseListExport extends MenuModule { newEntry.persist(err => { if(!err) { // queue it! - const dlQueue = new DownloadQueue(self.client); - dlQueue.add(newEntry, true); // true=systemFile - - // clean up after ourselves when the session ends - const thisClientId = self.client.session.id; - Events.once(Events.getSystemEvents().ClientDisconnected, evt => { - if(thisClientId === _.get(evt, 'client.session.id')) { - FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => { - if(err) { - Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' ); - } else { - Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' ); - } - }); - } - }); + DownloadQueue.get(self.client).addTemporaryDownload(newEntry); } return callback(err); }); diff --git a/core/message_base_qwk_export.js b/core/message_base_qwk_export.js index b79a6411..a6d86952 100644 --- a/core/message_base_qwk_export.js +++ b/core/message_base_qwk_export.js @@ -5,14 +5,15 @@ const { Errors } = require('./enig_error'); const { getMessageAreaByTag, hasMessageConfAndAreaRead, + getAllAvailableMessageAreaTags, } = require('./message_area'); const FileArea = require('./file_base_area'); const { QWKPacketWriter } = require('./qwk_mail_packet'); const { renderSubstr } = require('./string_util'); const Config = require('./config').get; const FileEntry = require('./file_entry'); -const Events = require('./events'); const DownloadQueue = require('./download_queue'); +const { Log } = require('./logger').log; // deps const async = require('async'); @@ -85,7 +86,7 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { }); }, (sysTempDownloadDir, callback) => { - this.performExport(sysTempDownloadDir, err => { + this._performExport(sysTempDownloadDir, err => { return callback(err); }); }, @@ -111,7 +112,41 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { this.prevMenu(); } - performExport(sysTempDownloadDir, cb) { + _getUserQWKExportOptions() { + let qwkOptions = this.client.user.getProperty('qwk_export_options'); + try { + qwkOptions = JSON.parse(qwkOptions); + } catch(e) { + qwkOptions = { + enableQWKE : true, + enableHeadersExtension : true, + enableAtKludges : true, + archiveFormat : 'application/zip', + }; + } + return qwkOptions; + } + + _getUserQWKExportAreas() { + let qwkExportAreas = this.client.user.getProperty('qwk_export_msg_areas'); + try { + qwkExportAreas = JSON.parse(qwkExportAreas); + } catch(e) { + // default to all public and private without 'since' + qwkExportAreas = getAllAvailableMessageAreaTags(this.client).map(areaTag => { + return { areaTag }; + }); + + // Include user's private area + qwkExportAreas.push({ + areaTag : Message.WellKnownAreaTags.Private, + }); + } + + return qwkExportAreas; + } + + _performExport(sysTempDownloadDir, cb) { const statusView = this.viewControllers.main.getView(MciViewIds.main.status); const updateStatus = (status) => { if (statusView) { @@ -202,10 +237,12 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { }); }; - const packetWriter = new QWKPacketWriter({ - user : this.client.user, - bbsID : this.config.bbsID, - }); // :TODO: User configuration here + const packetWriter = new QWKPacketWriter( + Object.assign(this._getUserQWKExportOptions(), { + user : this.client.user, + bbsID : this.config.bbsID, + }) + ); packetWriter.on('warning', warning => { this.client.log.warn( { warning }, 'QWK packet writer warning'); @@ -232,26 +269,9 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { packetWriter.init(); }, (callback) => { - // - // Fetch messages for user-configured area tags. - // - If private tag is present, we fetch this separately. - // - User property determines newscan timestamps dates if present for tag, else "all" - // - We have to fetch one area at a time in order to process message pointers/timestamps. - // ...this also allows for better progress. - // - // TL;DR: for each area -> for each message - // - const exportAreas = [ // :TODO: Load in something like this - { - areaTag : 'general', - newerThanTimestamp : '2018-01-01', - }, - { - areaTag : 'fsx_gen', - } - ]; - - async.eachSeries(exportAreas, (exportArea, nextExportArea) => { + // For each public area -> for each message + const userExportAreas = this._getUserQWKExportAreas(); + async.eachSeries(userExportAreas, (exportArea, nextExportArea) => { const area = getMessageAreaByTag(exportArea.areaTag); if (!area) { // :TODO: remove from user properties - this area does not exist @@ -291,6 +311,8 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { }); }, (callback) => { + // private messages to current user + // :TODO: Only if user property has private area tag const filter = { resultType : 'id', privateTagUserId : this.client.user.userId, @@ -339,23 +361,7 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { newEntry.persist(err => { if(!err) { // queue it! - const dlQueue = new DownloadQueue(this.client); - dlQueue.add(newEntry, true); // true=systemFile - - // clean up after ourselves when the session ends - // :TODO: DRY this with that in file_base_user_export - const thisClientId = this.client.session.id; - Events.once(Events.getSystemEvents().ClientDisconnected, evt => { - if(thisClientId === _.get(evt, 'client.session.id')) { - FileEntry.removeEntry(newEntry, { removePhysFile : true }, err => { - if(err) { - Log.warn( { fileId : newEntry.fileId, path : outputFileName }, 'Failed removing temporary session download' ); - } else { - Log.debug( { fileId : newEntry.fileId, path : outputFileName }, 'Removed temporary session download item' ); - } - }); - } - }); + DownloadQueue.get(this.client).addTemporaryDownload(newEntry); } return callback(err); }); @@ -369,8 +375,6 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { updateStatus('A QWK packet has been placed in your download queue'); } - // :TODO: send user to download manager with pop flags/etc. - return cb(err); } ); diff --git a/core/user_property.js b/core/user_property.js index cc68ef09..d55a0ebe 100644 --- a/core/user_property.js +++ b/core/user_property.js @@ -29,7 +29,7 @@ module.exports = { UserComment : 'user_comment', // NYI AutoSignature : 'auto_signature', - DownloadQueue : 'dl_queue', // download_queue.js + DownloadQueue : 'dl_queue', // see download_queue.js FailedLoginAttempts : 'failed_login_attempts', AccountLockedTs : 'account_locked_timestamp', @@ -64,5 +64,6 @@ module.exports = { AuthFactor2OTP : 'auth_factor2_otp', // If present, OTP type for 2FA. See OTPTypes AuthFactor2OTPSecret : 'auth_factor2_otp_secret', // Secret used in conjunction with OTP 2FA AuthFactor2OTPBackupCodes : 'auth_factor2_otp_backup', // JSON array of backup codes + }; From a56546cf3f7c6eff63e257ce33b0fd391b159795 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 11 May 2020 19:57:25 -0600 Subject: [PATCH 35/95] Only export private messages if the user has elected to do so --- core/message_base_qwk_export.js | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/core/message_base_qwk_export.js b/core/message_base_qwk_export.js index a6d86952..d8efcc1d 100644 --- a/core/message_base_qwk_export.js +++ b/core/message_base_qwk_export.js @@ -307,12 +307,16 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { }); }, err => { - return callback(err); + return callback(err, userExportAreas); }); }, - (callback) => { - // private messages to current user - // :TODO: Only if user property has private area tag + (userExportAreas, callback) => { + // Private messages to current user if the user has + // elected to export private messages + if (!(userExportAreas.find(exportArea => exportArea.areaTag === Message.WellKnownAreaTags.Private))) { + return callback(null); + } + const filter = { resultType : 'id', privateTagUserId : this.client.user.userId, @@ -352,7 +356,7 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { // :TODO: something like this: allow to override the displayed/downloaded as filename // separate from the actual on disk filename. E.g. we could always download as "ENIGMA.QWK" - visible_filename : paths.basename(packetInfo.path), + //visible_filename : paths.basename(packetInfo.path), } }); From 681a6e84498cab7258f5152f655d3b78022181ff Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 11 May 2020 20:09:48 -0600 Subject: [PATCH 36/95] Add theme for QWK export --- art/themes/luciano_blocktronics/theme.hjson | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/art/themes/luciano_blocktronics/theme.hjson b/art/themes/luciano_blocktronics/theme.hjson index 6d8490dc..d812a9c3 100644 --- a/art/themes/luciano_blocktronics/theme.hjson +++ b/art/themes/luciano_blocktronics/theme.hjson @@ -288,6 +288,17 @@ } } + qwkExportPacketCurrentConfig: { + mci: { + TL1: { + width: 70 + } + TL2: { + width: 70 + } + } + } + mailMenuCreateMessage: { 0: { mci: { From c8472fe2f07666dd3121ef0f4bb8596f18060d53 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 11 May 2020 20:13:50 -0600 Subject: [PATCH 37/95] Add some asserts for QWK export --- art/themes/luciano_blocktronics/offline_mail.ans | Bin 0 -> 3636 bytes .../luciano_blocktronics/qwk_export_progress.ans | Bin 0 -> 240 bytes 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 art/themes/luciano_blocktronics/offline_mail.ans create mode 100644 art/themes/luciano_blocktronics/qwk_export_progress.ans diff --git a/art/themes/luciano_blocktronics/offline_mail.ans b/art/themes/luciano_blocktronics/offline_mail.ans new file mode 100644 index 0000000000000000000000000000000000000000..ffd21e9e933ad5dee09db050336670438c4bd08a GIT binary patch literal 3636 zcmb`JL2lee5Jly!fUL3!G8-O&!;v^Pv{D9kAiz!n1j$A#A)1BY(6Rz4%CRI}BS#<+ z-X!q*zq*?odjw>WF-)=9)m8uhs;=(Ed3|zJpRDTcuv(liPgYGgEY6RbZq%Q;>-l-C zXWb3M5W{i7LX2g$)Dn-FTb{6Zz?VM^+0R<*<0Jx9aA`x z#cJJERSXyB#~VXvG%p~IWal}WL*t%+37y~hoPZ~&@j=fZ2Mv)GNt^N>TtsE=Z|ujB z|G1gv`#vmi7avE)&8y8g4q&g~%Q!^fWm-3~u#c#L2iGz!nEJyi@W!-sD8wqQ#MiQW z;6>i0eUJ`2d9svRGaqQW?$EZ~WH`exT4qdR*6xkDP)%Vbg(fT!Fs-b5;qLzf+9DWO zc6q2o0=(q{!)nz%=-Xya*Iy`*L<2KJo-f8P%L=VYstCRtHp!P5j6=`A0#bq}m{Qth z9H?HRBZW%RkM*|V?Ui1zLRJ22sgXTBpR?*I$=2_t)?RiZJ5#gBCdWQqciT{+?Y4h? zy#C|1y19Q}eZ2VX{q|Oy$B0Bi0F5~VoV!OnLq)b(c?B0`)@tVb0ClGhWrkprOb~Wp zsiSfuwh*wZ3DaHjBuLx+d9}T-F0b!D{C<6LS>4=i@3vS$tQcz$3xj^^p)AJ7q4GX* zR2cz+XHKn#q`>M5?gThYD;Up~rk&t?u8U@t_a@buVzk}O-PLV%dtG%GS0C&tW!>Be zs^KGGA7w5uCL=JSULa|!OppTp$bQ7AQFzzYQ2_C`VW_kRl1n+$&R1h9>2ZT*?|;TN zmB1q@q&ulOsVnalYIkX%DnwXgiI1C2w2k5-x=`AGtYYJ-NpqAWm~IST5|#kTNw{w6 z;I6ZUa$IxF;Egt&HksSPOa|1op}Sz1?hRicsk=}+sdq%JD6|rWC+%Ow)iJ30000|4 z`UB5K2?-35^?XVwC?``sOcD~lroB`}?kwL@`5<{dL&>VJh^dSfnWIVh0S1fL?z{2T zc(TY?WDnp{#ik_4&!oGFD*)&w(xoj(W=WpmAWaY79#nF};2&!h^Bi15&zQ5Hm@ZQ@ zQo8pbS$1W@9l;clN++fJmS%LiE!}Q>OOQ`f8F8w+ z2Y-Z~Z?JG$o$%*8%e6_iy6rem-gCTrD1Pu)WGoy%k@XN*ZOJCsoM0M=cA&at__A9N zutY_O=R8wdsJ;kC;7rE_!JQNnRI=h8_XPCHK4!%z|QG zGUkvHDU~g+_Mo})EnOS4Ku!bkT+6OVF}DaRXWbvdvxMb75mRQWjkqtWBJfMWjYoSZ z=$fB>LO^>b2({qs_mCe1o=Clz{g#l|~p zgS{$6&Nazlm+sPF_l;4t{AK9O^@$^RAN0tO*SR{ACJ9v@&DUnS@??;f zB?Z^3U*b|8@9u^+Xg5rziwPdoh!4O0`Sh2~>$=9r|MhF?rml~d_1|9|9K3k()4?|f S{(XJ+-Mi}7?@wb?o}U2vk;_N` literal 0 HcmV?d00001 diff --git a/art/themes/luciano_blocktronics/qwk_export_progress.ans b/art/themes/luciano_blocktronics/qwk_export_progress.ans new file mode 100644 index 0000000000000000000000000000000000000000..dcde7d82c0d3f83d257ce80062a0dc20737b7420 GIT binary patch literal 240 zcmb1+Hn27^ur@Z&<&uszHpo?wjyAM5Hp=}6WEh*}I)!@rxOn=xD+GpnD+D+?d%K2! zBn)zOA@Tt}u8zU33gM2PU~!;6E-s)9&_J_X)es*;AP-0x0SyIljku(O9YdX64Ga{h l2aF7i3=B*S4NDmq7#JA?7}$U`5D0rZ`6`6DJ3@GH5&)`jD?k7M literal 0 HcmV?d00001 From b562ba34b0ff26848b66ab73f111bd948df26af0 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 11 May 2020 20:31:35 -0600 Subject: [PATCH 38/95] Fix bad log --- core/download_queue.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/download_queue.js b/core/download_queue.js index 1eb4023e..d80ddf34 100644 --- a/core/download_queue.js +++ b/core/download_queue.js @@ -92,9 +92,9 @@ module.exports = class DownloadQueue { FileEntry.removeEntry(entry, { removePhysFile : true }, err => { const Log = require('./logger').log; if(err) { - Log.warn( { fileId : entry.fileId, path : outputFileName }, 'Failed removing temporary session download' ); + Log.warn( { fileId : entry.fileId, path : entry.filePath }, 'Failed removing temporary session download' ); } else { - Log.debug( { fileId : entry.fileId, path : outputFileName }, 'Removed temporary session download item' ); + Log.debug( { fileId : entry.fileId, path : entry.filePath }, 'Removed temporary session download item' ); } }); } From fbec46d1b91d3d93c88ed29e71b15f53388c983c Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 11 May 2020 21:54:20 -0600 Subject: [PATCH 39/95] Fix Message.findMessages() if private tag passed in, fix QWK export --- core/message.js | 29 ++++++++++++++++------------- core/message_base_qwk_export.js | 5 ++++- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/core/message.js b/core/message.js index c763bbd2..b410f105 100644 --- a/core/message.js +++ b/core/message.js @@ -288,7 +288,7 @@ module.exports = class Message { filter.extraFields = [] filter.privateTagUserId = - if set, only private messages belonging to are processed - - any other areaTag or confTag filters will be ignored + - areaTags filter ignored - if NOT present, private areas are skipped *=NYI @@ -364,20 +364,23 @@ module.exports = class Message { )`); } else { if(filter.areaTag && filter.areaTag.length > 0) { - if(Array.isArray(filter.areaTag)) { - const areaList = filter.areaTag - .filter(t => t != Message.WellKnownAreaTags.Private) - .map(t => `"${t}"`).join(', '); - if(areaList.length > 0) { - appendWhereClause(`m.area_tag IN(${areaList})`); - } - } else if(_.isString(filter.areaTag) && Message.WellKnownAreaTags.Private !== filter.areaTag) { - appendWhereClause(`m.area_tag = "${filter.areaTag}"`); + if (!Array.isArray(filter.areaTag)) { + filter.areaTag = [ filter.areaTag ]; } - } - // explicit exclude of Private - appendWhereClause(`m.area_tag != "${Message.WellKnownAreaTags.Private}"`, 'AND'); + const areaList = filter.areaTag + .filter(t => t !== Message.WellKnownAreaTags.Private) + .map(t => `"${t}"`).join(', '); + if(areaList.length > 0) { + appendWhereClause(`m.area_tag IN(${areaList})`); + } else { + // nothing to do; no areas remain + return cb(null, []); + } + } else { + // explicit exclude of Private + appendWhereClause(`m.area_tag != "${Message.WellKnownAreaTags.Private}"`, 'AND'); + } } if(_.isNumber(filter.replyToMessageId)) { diff --git a/core/message_base_qwk_export.js b/core/message_base_qwk_export.js index d8efcc1d..dcdd605e 100644 --- a/core/message_base_qwk_export.js +++ b/core/message_base_qwk_export.js @@ -270,7 +270,10 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { }, (callback) => { // For each public area -> for each message - const userExportAreas = this._getUserQWKExportAreas(); + const userExportAreas = this._getUserQWKExportAreas() + .filter(exportArea => { + return exportArea.areaTag !== Message.WellKnownAreaTags.Private; + }); async.eachSeries(userExportAreas, (exportArea, nextExportArea) => { const area = getMessageAreaByTag(exportArea.areaTag); if (!area) { From f63464c501c35274ac12c8068940e2ed0c53a7ed Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 12 May 2020 09:12:55 -0600 Subject: [PATCH 40/95] Very slight display improvements, include conf in info --- core/message_base_qwk_export.js | 9 ++++++--- core/qwk_mail_packet.js | 8 ++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/core/message_base_qwk_export.js b/core/message_base_qwk_export.js index dcdd605e..6d748074 100644 --- a/core/message_base_qwk_export.js +++ b/core/message_base_qwk_export.js @@ -4,6 +4,7 @@ const Message = require('./message'); const { Errors } = require('./enig_error'); const { getMessageAreaByTag, + getMessageConferenceByTag, hasMessageConfAndAreaRead, getAllAvailableMessageAreaTags, } = require('./message_area'); @@ -216,7 +217,7 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { message, step : 'message', total : messageIds.length, - status : `Writing message ${current} / ${messageIds.length}`, + status : `${_.truncate(message.subject, { length : 25 })} (${current} / ${messageIds.length})`, }; progressHandler(progress, err => { @@ -276,7 +277,8 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { }); async.eachSeries(userExportAreas, (exportArea, nextExportArea) => { const area = getMessageAreaByTag(exportArea.areaTag); - if (!area) { + const conf = getMessageConferenceByTag(area.confTag); + if (!area || !conf) { // :TODO: remove from user properties - this area does not exist this.client.log.warn({ areaTag : exportArea.areaTag }, 'Cannot QWK export area as it does not exist'); return nextExportArea(null); @@ -288,9 +290,10 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { } const progress = { + conf, area, step : 'next_area', - status : `Gathering messages in ${area.name}...`, + status : `Gathering in ${conf.name} - ${area.name}...`, }; progressHandler(progress, err => { diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index 3d613ca4..c89ae714 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -5,6 +5,7 @@ const { splitTextAtTerms } = require('./string_util'); const { getMessageConfTagByAreaTag, getMessageAreaByTag, + getMessageConferenceByTag, getAllAvailableMessageAreaTags, } = require('./message_area'); const StatLog = require('./stat_log'); @@ -1370,10 +1371,9 @@ class QWKPacketWriter extends EventEmitter { // map areas as conf #\r\nDescription\r\n pairs areas.forEach(area => { const conferenceNumber = this._getMessageConferenceNumberByAreaTag(area.areaTag); - let desc = area.name; - if (area.desc) { - desc += ` - ${area.desc}` - } + const conf = getMessageConferenceByTag(area.confTag); + const desc = `${conf.name} - ${area.name}`; + controlStream.write(`${conferenceNumber}\r\n`); controlStream.write(`${desc}\r\n`); }); From db91d6d6b7ec7be29e76eddcb8f5dfca89a88dfb Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 12 May 2020 18:53:47 -0600 Subject: [PATCH 41/95] * Update QWK area pointers upon export * Specialize 0 messages exported --- core/enig_error.js | 1 + core/message_base_qwk_export.js | 68 +++++++++++++++++++++++++-------- 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/core/enig_error.js b/core/enig_error.js index be025214..4d88cfa1 100644 --- a/core/enig_error.js +++ b/core/enig_error.js @@ -38,6 +38,7 @@ exports.Errors = { MissingMci : (reason, reasonCode) => new EnigError('Missing required MCI code(s)', -32009, reason, reasonCode), BadLogin : (reason, reasonCode) => new EnigError('Bad login attempt', -32010, reason, reasonCode), UserInterrupt : (reason, reasonCode) => new EnigError('User interrupted', -32011, reason, reasonCode), + NothingToDo : (reason, reasonCode) => new EnigError('Nothing to do', -32012, reason, reasonCode), }; exports.ErrorReasons = { diff --git a/core/message_base_qwk_export.js b/core/message_base_qwk_export.js index 6d748074..53682471 100644 --- a/core/message_base_qwk_export.js +++ b/core/message_base_qwk_export.js @@ -14,7 +14,7 @@ const { renderSubstr } = require('./string_util'); const Config = require('./config').get; const FileEntry = require('./file_entry'); const DownloadQueue = require('./download_queue'); -const { Log } = require('./logger').log; +const { getISOTimestampString } = require('./database'); // deps const async = require('async'); @@ -23,6 +23,7 @@ const fse = require('fs-extra'); const temptmp = require('temptmp'); const paths = require('path'); const UUIDv4 = require('uuid/v4'); +const moment = require('moment'); const FormIds = { main : 0, @@ -37,6 +38,11 @@ const MciViewIds = { } }; +const UserProperties = { + ExportOptions : 'qwk_export_options', + ExportAreas : 'qwk_export_msg_areas', +}; + exports.moduleInfo = { name : 'QWK Export', desc : 'Exports a QWK Packet for download', @@ -114,7 +120,7 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { } _getUserQWKExportOptions() { - let qwkOptions = this.client.user.getProperty('qwk_export_options'); + let qwkOptions = this.client.user.getProperty(UserProperties.ExportOptions); try { qwkOptions = JSON.parse(qwkOptions); } catch(e) { @@ -129,9 +135,14 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { } _getUserQWKExportAreas() { - let qwkExportAreas = this.client.user.getProperty('qwk_export_msg_areas'); + let qwkExportAreas = this.client.user.getProperty(UserProperties.ExportAreas); try { - qwkExportAreas = JSON.parse(qwkExportAreas); + qwkExportAreas = JSON.parse(qwkExportAreas).map(exportArea => { + if (exportArea.newerThanTimestamp) { + exportArea.newerThanTimestamp = moment(exportArea.newerThanTimestamp); + } + return exportArea; + }); } catch(e) { // default to all public and private without 'since' qwkExportAreas = getAllAvailableMessageAreaTags(this.client).map(areaTag => { @@ -173,13 +184,13 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { case 'next_area' : updateStatus(state.status); updateProgressBar(0, 0); - this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.area); + this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state); break; case 'message' : updateStatus(state.status); updateProgressBar(state.current, state.total); - this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state.message); + this.updateCustomViewTextsWithFilter('main', MciViewIds.main.customRangeStart, state); break; default : @@ -198,6 +209,7 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { } }; + let totalExported = 0; const processMessagesWithFilter = (filter, cb) => { Message.findMessages(filter, (err, messageIds) => { if (err) { @@ -213,11 +225,12 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { } const progress = { - current, message, - step : 'message', - total : messageIds.length, - status : `${_.truncate(message.subject, { length : 25 })} (${current} / ${messageIds.length})`, + step : 'message', + total : ++totalExported, + areaCurrent : current, + areaCount : messageIds.length, + status : `${_.truncate(message.subject, { length : 25 })} (${current} / ${messageIds.length})`, }; progressHandler(progress, err => { @@ -271,11 +284,13 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { }, (callback) => { // For each public area -> for each message - const userExportAreas = this._getUserQWKExportAreas() + const userExportAreas = this._getUserQWKExportAreas(); + + const publicExportAreas = userExportAreas .filter(exportArea => { return exportArea.areaTag !== Message.WellKnownAreaTags.Private; }); - async.eachSeries(userExportAreas, (exportArea, nextExportArea) => { + async.eachSeries(publicExportAreas, (exportArea, nextExportArea) => { const area = getMessageAreaByTag(exportArea.areaTag); const conf = getMessageConferenceByTag(area.confTag); if (!area || !conf) { @@ -319,15 +334,15 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { (userExportAreas, callback) => { // Private messages to current user if the user has // elected to export private messages - if (!(userExportAreas.find(exportArea => exportArea.areaTag === Message.WellKnownAreaTags.Private))) { + const privateExportArea = userExportAreas.find(exportArea => exportArea.areaTag === Message.WellKnownAreaTags.Private); + if (!privateExportArea) { return callback(null); } const filter = { resultType : 'id', privateTagUserId : this.client.user.userId, - // :TODO: newerThanTimestamp for private messages - //newerThanTimestamp : exportArea.newerThanTimestamp + newerThanTimestamp : privateExportArea.newerThanTimestamp, }; return processMessagesWithFilter(filter, callback); }, @@ -344,6 +359,10 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { packetWriter.finish(this.tempPacketDir); }, (packetInfo, callback) => { + if (0 === totalExported) { + return callback(Errors.NothingToDo('No messages exported')); + } + const sysDownloadPath = paths.join(sysTempDownloadDir, this.tempName); fse.move(packetInfo.path, sysDownloadPath, err => { return callback(null, sysDownloadPath, packetInfo); @@ -375,7 +394,21 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { } return callback(err); }); - } + }, + (callback) => { + // update user's export area dates; they can always change/reset them again + const updatedUserExportAreas = this._getUserQWKExportAreas().map(exportArea => { + return Object.assign(exportArea, { + newerThanTimestamp : getISOTimestampString(), + }); + }); + + return this.client.user.persistProperty( + UserProperties.ExportAreas, + JSON.stringify(updatedUserExportAreas), + callback + ); + }, ], err => { this.client.startIdleMonitor(); // re-enable @@ -383,6 +416,9 @@ exports.getModule = class MessageBaseQWKExport extends MenuModule { if (!err) { updateStatus('A QWK packet has been placed in your download queue'); + } else if (err.code === Errors.NothingToDo().code) { + updateStatus('No messages to export with current criteria'); + err = null; } return cb(err); From 714f32f6956cf57361dafedaec98ac3df5741b71 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 13 May 2020 19:04:11 -0600 Subject: [PATCH 42/95] Add Node.js version to startup log --- core/bbs.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/bbs.js b/core/bbs.js index 98ba1b7a..e29b8d27 100644 --- a/core/bbs.js +++ b/core/bbs.js @@ -189,8 +189,12 @@ function initialize(cb) { function basicInit(callback) { logger.init(); logger.log.info( - { version : require('../package.json').version }, - '**** ENiGMA½ Bulletin Board System Starting Up! ****'); + { + version : require('../package.json').version, + nodeVersion : process.version, + }, + '**** ENiGMA½ Bulletin Board System Starting Up! ****' + ); process.on('SIGINT', shutdownSystem); From fd6bb47427ef603736a5a3bb0e477c232b623934 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 13 May 2020 19:30:57 -0600 Subject: [PATCH 43/95] Fix yet another bug with node IDs * Pick appropriate slot * Log nodeId vs clientId/etc. for less confusion --- core/client.js | 2 +- core/client_connections.js | 24 ++++++++++++------------ core/download_queue.js | 4 ++-- core/login_server_module.js | 4 ++-- core/user_login.js | 9 +++++---- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/core/client.js b/core/client.js index a6cfae52..40872fa2 100644 --- a/core/client.js +++ b/core/client.js @@ -96,7 +96,7 @@ function Client(/*input, output*/) { Object.defineProperty(this, 'node', { get : function() { - return self.session.id + 1; + return self.session.id; } }); diff --git a/core/client_connections.js b/core/client_connections.js index e2c8d577..d1a6be6e 100644 --- a/core/client_connections.js +++ b/core/client_connections.js @@ -61,27 +61,27 @@ function getActiveConnectionList(authUsersOnly) { function addNewClient(client, clientSock) { // - // Assign ID/client ID to next lowest & available # + // Find a node ID "slot" // - let id = 0; - for(let i = 0; i < clientConnections.length; ++i) { - if(clientConnections[i].id > id) { - break; + let nodeId; + for (nodeId = 1; nodeId < Number.MAX_SAFE_INTEGER; ++nodeId) { + const existing = clientConnections.find(client => nodeId === client.node); + if (!existing) { + break; // available slot } - id++; } - client.session.id = id; - const remoteAddress = client.remoteAddress = clientSock.remoteAddress; + client.session.id = nodeId; + const remoteAddress = client.remoteAddress = clientSock.remoteAddress; // create a unique identifier one-time ID for this session - client.session.uniqueId = new hashids('ENiGMA½ClientSession').encode([ id, moment().valueOf() ]); + client.session.uniqueId = new hashids('ENiGMA½ClientSession').encode([ nodeId, moment().valueOf() ]); clientConnections.push(client); clientConnections.sort( (c1, c2) => c1.session.id - c2.session.id); // Create a client specific logger // Note that this will be updated @ login with additional information - client.log = logger.log.child( { clientId : id, sessionId : client.session.uniqueId } ); + client.log = logger.log.child( { nodeId, sessionId : client.session.uniqueId } ); const connInfo = { remoteAddress : remoteAddress, @@ -101,7 +101,7 @@ function addNewClient(client, clientSock) { { client : client, connectionCount : clientConnections.length } ); - return id; + return nodeId; } function removeClient(client) { @@ -114,7 +114,7 @@ function removeClient(client) { logger.log.info( { connectionCount : clientConnections.length, - clientId : client.session.id + nodeId : client.node, }, 'Client disconnected' ); diff --git a/core/download_queue.js b/core/download_queue.js index d80ddf34..4380ded1 100644 --- a/core/download_queue.js +++ b/core/download_queue.js @@ -86,9 +86,9 @@ module.exports = class DownloadQueue { this.add(entry, true); // true=systemFile // clean up after ourselves when the session ends - const thisClientId = this.client.session.id; + const thisUniqueId = this.client.session.uniqueId; Events.once(Events.getSystemEvents().ClientDisconnected, evt => { - if(thisClientId === _.get(evt, 'client.session.id')) { + if(thisUniqueId === _.get(evt, 'client.session.uniqueId')) { FileEntry.removeEntry(entry, { removePhysFile : true }, err => { const Log = require('./logger').log; if(err) { diff --git a/core/login_server_module.js b/core/login_server_module.js index a08abfe9..da3e06de 100644 --- a/core/login_server_module.js +++ b/core/login_server_module.js @@ -72,12 +72,12 @@ module.exports = class LoginServerModule extends ServerModule { }); client.on('error', err => { - logger.log.info({ clientId : client.session.id, error : err.message }, 'Connection error'); + logger.log.info({ nodeId : client.node, error : err.message }, 'Connection error'); }); client.on('close', err => { const logFunc = err ? logger.log.info : logger.log.debug; - logFunc( { clientId : client.session.id }, 'Connection closed'); + logFunc( { nodeId : client.node }, 'Connection closed'); clientConns.removeClient(client); }); diff --git a/core/user_login.js b/core/user_login.js index 99faa1a2..3db6b5cc 100644 --- a/core/user_login.js +++ b/core/user_login.js @@ -81,9 +81,9 @@ function userLogin(client, username, password, options, cb) { if(existingClientConnection) { client.log.info( { - existingClientId : existingClientConnection.session.id, - username : user.username, - userId : user.userId + existingNodeId : existingClientConnection.node, + username : user.username, + userId : user.userId }, 'Already logged in' ); @@ -97,11 +97,12 @@ function userLogin(client, username, password, options, cb) { // update client logger with addition of username client.log = logger.log.child( { - clientId : client.log.fields.clientId, + nodeId : client.log.fields.nodeId, sessionId : client.log.fields.sessionId, username : user.username, } ); + client.log.info('Successful login'); // User's unique session identifier is the same as the connection itself From b7cac2d38f517738528e4c0ff5cd62efc2b52d73 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 13 May 2020 20:19:45 -0600 Subject: [PATCH 44/95] Fix sliceAtEOF() bug causing short ANSI's to get borked when e.g. display achievements --- core/art.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/art.js b/core/art.js index 38594985..0ff4835f 100644 --- a/core/art.js +++ b/core/art.js @@ -49,7 +49,7 @@ function getFontNameFromSAUCE(sauce) { function sliceAtEOF(data, eofMarker) { let eof = data.length; - const stopPos = Math.max(data.length - (256), 0); // 256 = 2 * sizeof(SAUCE) + const stopPos = Math.max(data.length - 256, 0); // 256 = 2 * sizeof(SAUCE) for(let i = eof - 1; i > stopPos; i--) { if(eofMarker === data[i]) { @@ -57,9 +57,16 @@ function sliceAtEOF(data, eofMarker) { break; } } - if(eof === data.length || eof < 128) { + + if (eof === data.length) { + return data; // nothing to do + } + + // try to prevent goofs + if (eof < 128 && 'SAUCE00' !== data.slice(eof + 1, eof + 8).toString()) { return data; } + return data.slice(0, eof); } From 60955cac1155395db615491179adbcf00ba5e398 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 19:46:00 -0600 Subject: [PATCH 45/95] Fix Message.uuid vs Message.messageUuid * Just use Message.messageUuid which better aligns with message_uuid in DB, messageId, etc. * Keep uuid as property for now just in case --- core/message.js | 29 ++++++++++++++++++++++------- core/msg_area_post_fse.js | 2 +- core/qwk_mail_packet.js | 2 +- core/scanner_tossers/ftn_bso.js | 12 ++++++------ 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/core/message.js b/core/message.js index b410f105..b45e868c 100644 --- a/core/message.js +++ b/core/message.js @@ -107,15 +107,23 @@ const MESSAGE_ROW_MAP = { module.exports = class Message { constructor( { - messageId = 0, areaTag = Message.WellKnownAreaTags.Invalid, uuid, replyToMsgId = 0, - toUserName = '', fromUserName = '', subject = '', message = '', modTimestamp = moment(), - meta, hashTags = [], + messageId = 0, + areaTag = Message.WellKnownAreaTags.Invalid, + uuid, + replyToMsgId = 0, + toUserName = '', + fromUserName = '', + subject = '', + message = '', + modTimestamp = moment(), + meta, + hashTags = [], } = { } ) { this.messageId = messageId; this.areaTag = areaTag; - this.uuid = uuid; + this.messageUuid = uuid; this.replyToMsgId = replyToMsgId; this.toUserName = toUserName; this.fromUserName = fromUserName; @@ -134,6 +142,10 @@ module.exports = class Message { this.hashTags = hashTags; } + get uuid() { // deprecated, will be removed in the near future + return this.messageUuid; + } + isValid() { return true; } // :TODO: obviously useless; look into this or remove it static isPrivateAreaTag(areaTag) { @@ -684,8 +696,8 @@ module.exports = class Message { function storeMessage(trans, callback) { // generate a UUID for this message if required (general case) const msgTimestamp = moment(); - if(!self.uuid) { - self.uuid = Message.createMessageUUID( + if(!self.messageUuid) { + self.messageUuid = Message.createMessageUUID( self.areaTag, msgTimestamp, self.subject, @@ -696,7 +708,10 @@ module.exports = class Message { trans.run( `INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) VALUES (?, ?, ?, ?, ?, ?, ?, ?);`, - [ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, getISOTimestampString(msgTimestamp) ], + [ + self.areaTag, self.messageUuid, self.replyToMsgId, self.toUserName, + self.fromUserName, self.subject, self.message, getISOTimestampString(msgTimestamp) + ], function inserted(err) { // use non-arrow function for 'this' scope if(!err) { self.messageId = this.lastID; diff --git a/core/msg_area_post_fse.js b/core/msg_area_post_fse.js index f47717b2..0c92bbfb 100644 --- a/core/msg_area_post_fse.js +++ b/core/msg_area_post_fse.js @@ -50,7 +50,7 @@ exports.getModule = class AreaPostFSEModule extends FullScreenEditorModule { } else { // note: not logging 'from' here as it's part of client.log.xxxx() self.client.log.info( - { to : msg.toUserName, subject : msg.subject, uuid : msg.uuid }, + { to : msg.toUserName, subject : msg.subject, uuid : msg.messageUuid }, 'Message persisted' ); } diff --git a/core/qwk_mail_packet.js b/core/qwk_mail_packet.js index c89ae714..de42abf8 100644 --- a/core/qwk_mail_packet.js +++ b/core/qwk_mail_packet.js @@ -1491,7 +1491,7 @@ class QWKPacketWriter extends EventEmitter { Conference : message.isPrivate() ? '0' : getMessageConfTagByAreaTag(message.areaTag), // ENiGMA Headers - MessageUUID : message.uuid, + MessageUUID : message.messageUuid, ModTimestamp : message.modTimestamp.format('YYYY-MM-DDTHH:mm:ss.SSSZ'), AreaTag : message.areaTag, }; diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 488dcc8e..17cd61b8 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -1215,7 +1215,7 @@ function FTNMessageScanTossModule() { // if(true === _.get(Config(), [ 'messageNetworks', 'ftn', 'areas', config.localAreaTag, 'allowDupes' ], false)) { // just generate a UUID & therefor always allow for dupes - message.uuid = uuidV4(); + message.messageUuid = uuidV4(); } return callback(null); @@ -1378,7 +1378,7 @@ function FTNMessageScanTossModule() { } } - message.uuid = Message.createMessageUUID( + message.messageUuid = Message.createMessageUUID( localAreaTag, message.modTimestamp, message.subject, @@ -1398,7 +1398,7 @@ function FTNMessageScanTossModule() { if('SQLITE_CONSTRAINT' === err.code || 'DUPE_MSGID' === err.code) { const msgId = _.has(message.meta, 'FtnKludge.MSGID') ? message.meta.FtnKludge.MSGID : 'N/A'; Log.info( - { area : localAreaTag, subject : message.subject, uuid : message.uuid, MSGID : msgId }, + { area : localAreaTag, subject : message.subject, uuid : message.messageUuid, MSGID : msgId }, 'Not importing non-unique message'); return next(null); @@ -2349,7 +2349,7 @@ FTNMessageScanTossModule.prototype.record = function(message) { return; } - const info = { uuid : message.uuid, subject : message.subject }; + const info = { uuid : message.messageUuid, subject : message.subject }; function exportLog(err) { if(err) { @@ -2363,7 +2363,7 @@ FTNMessageScanTossModule.prototype.record = function(message) { Object.assign(info, { type : 'NetMail' } ); if(this.exportingStart()) { - this.exportNetMailMessagesToUplinks( [ message.uuid ], err => { + this.exportNetMailMessagesToUplinks( [ message.messageUuid ], err => { this.exportingEnd( () => exportLog(err) ); }); } @@ -2376,7 +2376,7 @@ FTNMessageScanTossModule.prototype.record = function(message) { } if(this.exportingStart()) { - this.exportEchoMailMessagesToUplinks( [ message.uuid ], areaConfig, err => { + this.exportEchoMailMessagesToUplinks( [ message.messageUuid ], areaConfig, err => { this.exportingEnd( () => exportLog(err) ); }); } From 8ee3317bd2e47234fc2cb6c7d345fec30dc62bb3 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 20:08:02 -0600 Subject: [PATCH 46/95] Updates on art docs --- docs/art/general.md | 67 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/docs/art/general.md b/docs/art/general.md index a0620de3..18c438ec 100644 --- a/docs/art/general.md +++ b/docs/art/general.md @@ -7,13 +7,36 @@ One of the most basic elements of BBS customization is through it's artwork. ENi As a general rule, art files live in one of two places: -1. The `art/general` directory. This is where you place command non-themed art files. -2. Within a theme such as `art/themes/super_fancy_theme`. +1. The `art/general` directory. This is where you place common/non-themed art files. +2. Within a _theme_ such as `art/themes/super_fancy_theme`. -### Menu Entries -While art can be displayed programmatically such as from a custom module, the most basic and common form is via `menu.hjson` entries. This usually falls into one of two forms: a "standard" entry where a single `art` spec is utilized or a entry for a custom module where multiple pieces are declared and used. The second style usually takes the form of a `config.art` block with two or more entries. +### Art in Menus +While art can be displayed programmatically such as from a custom module, the most basic and common form is via `menu.hjson` entries. This usually falls into one of two forms. -A menu entry has a few elements that control how art is choosen and displayed. First, the `art` *spec* tells teh system how to look for the art asset. Second, the `config` block can further control aspecs of lookup and display: +**Form 1**: A "standard" entry where a single `art` spec is utilized: +```hjson +{ + mainMenu: { + art: main_menu.ans + } +} +``` + +**Form 2**: An entry for a custom module where multiple pieces are declared and used. The second style usually takes the form of a `config.art` block with two or more entries: +```hjson +{ + nodeMessage: { + config: { + art: { + header: node_msg_header + footer: node_msg_footer + } + } + } +} +``` + +A menu entry has a few elements that control how art is selected and displayed. First, the `art` *spec* tells the system how to look for the art asset. Second, the `config` block can further control aspects of lookup and display. The following table describes such entries: | Item | Description| |------|------------| @@ -23,13 +46,13 @@ A menu entry has a few elements that control how art is choosen and displayed. F | `cls` | Clear the screen before display if set to `true`. | | `random` | Set to `false` to explicitly disable random lookup. | | `types` | An optional array of types (aka file extensions) to consider for lookup. For example : `[ '.ans', '.asc' ]` | -| `readSauce` | May be set to `false` if you need to explictly disable SAUCE support. | +| `readSauce` | May be set to `false` if you need to explicitly disable SAUCE support. | #### Art Spec -It was mentioned that the `art` member is a *spec*. The value of a `art` member controls how the system looks for an asset. The following forms are supported: +In the section above it is mentioned that the `art` member is a *spec*. The value of a `art` spec controls how the system looks for an asset. The following forms are supported: * `FOO`: The system will look for `FOO.ANS`, `FOO.ASC`, `FOO.TXT`, etc. using the default search path. Unless otherwise specified if `FOO1.ANS`, `FOO2.ANS`, and so on exist, a random selection will be made. -* `FOO.ANS`: By specifying an extension, only that type will be searched for. +* `FOO.ANS`: By specifying an extension, only the exact match will be searched for. * `rel/path/to/BAR.ANS`: Only match a path (relative to the system's `art` directory). * `/path/to/BAZ.ANS`: Exact path only. @@ -40,8 +63,27 @@ ENiGMA½ uses a fallback system for art selection. When a menu entry calls for a 3. In the system default theme directory. 4. In the `art/general` directory. +#### ACS-Driven Conditionals +The [ACS](/docs/configuration/acs.md) system can be used to make conditional art selection choices. To do this, provide an array of possible values in your art spec. As an example: +```hjson +{ + fancyMenu: { + art: [ + { + acs: GM[l33t] + art: leet_art.ans + } + { + // default + art: newb.asc + } + ] + } +} +``` + #### SyncTERM Style Fonts -ENiGMA½ can set a [SyncTERM](http://syncterm.bbsdev.net/) style font for art display. This is supported by many popular BBS terminals besides just SyncTERM and is common for displaying Amiga style fonts for example. The system will use the `font` specifier or look for a font declared in an artworks SAUCE record (unless `readSauce` is `false`). +ENiGMA½ can set a [SyncTERM](http://syncterm.bbsdev.net/) style font for art display. This is supported by many other popular BBS terminals as well. A common usage is for displaying Amiga style fonts for example. The system will use the `font` specifier or look for a font declared in an artworks SAUCE record (unless `readSauce` is `false`). The most common fonts are probably as follows: @@ -96,7 +138,7 @@ See [this specification](https://github.com/protomouse/synchronet/blob/master/sr #### SyncTERM Style Baud Rates The `baudRate` member can set a [SyncTERM](http://syncterm.bbsdev.net/) style emulated baud rate. May be `300`, `600`, `1200`, `2400`, `4800`, `9600`, `19200`, `38400`, `57600`, `76800`, or `115200`. A value of `ulimited`, `off`, or `0` resets (disables) the rate. See [this specification](https://github.com/protomouse/synchronet/blob/master/src/conio/cterm.txt) for more information. -## Common Example +### Common Example ```hjson fullLogoffSequenceRandomBoardAd: { art: OTHRBBS @@ -108,4 +150,7 @@ fullLogoffSequenceRandomBoardAd: { cls: true } } -``` \ No newline at end of file +``` + +### See Also +See also the [Show Art Module](/docs/modding/show-art.md) for more advanced art display! \ No newline at end of file From fc22db546252cfd8cd6f242b58a971ce08125f9d Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 20:44:11 -0600 Subject: [PATCH 47/95] FTN/BSO doc tidy up --- core/mime_util.js | 2 +- docs/messageareas/bso-import-export.md | 94 +++++++++++++------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/core/mime_util.js b/core/mime_util.js index 857b967c..ddf44432 100644 --- a/core/mime_util.js +++ b/core/mime_util.js @@ -35,7 +35,7 @@ function startup(cb) { function resolveMimeType(query) { if(mimeTypes.extensions[query]) { - return query; // alreaed a mime-type + return query; // already a mime-type } return mimeTypes.lookup(query) || undefined; // lookup() returns false; we want undefined diff --git a/docs/messageareas/bso-import-export.md b/docs/messageareas/bso-import-export.md index fac7bc6d..6fc6298e 100644 --- a/docs/messageareas/bso-import-export.md +++ b/docs/messageareas/bso-import-export.md @@ -5,58 +5,23 @@ title: BSO Import / Export ## BSO Import / Export The scanner/tosser module `ftn_bso` provides **B**inkley **S**tyle **O**utbound (BSO) import/toss and scan/export of messages EchoMail and NetMail messages. Configuration is supplied in `config.hjson` under `scannerTossers.ftn_bso`. -:information_source: ENiGMA½'s `ftn_bso` module is not a mailer and **makes no attempts** to perfrom packet transport! An external [mailer](http://www.filegate.net/bbsmailers.htm) such as [Binkd](https://github.com/pgul/binkd) is required for this! +:information_source: ENiGMA½'s `ftn_bso` module is not a mailer and **makes no attempts to perform packet transport**! An external [mailer](http://www.filegate.net/bbsmailers.htm) such as [Binkd](https://github.com/pgul/binkd) is required for this task. +### Configuration Let's look at some of the basic configuration: | Config Item | Required | Description | |-------------|----------|----------------------------------------------------------| -| `schedule` | :+1: | Sets `import` and `export` schedules. [Later style text parsing](https://bunkat.github.io/later/parsers.html#text) supported. `import` also can utilize a `@watch:` syntax while `export` additionally supports `@immediate`. | +| `schedule` | :+1: | Sets `import` and `export` schedules. [Later style text parsing](https://bunkat.github.io/later/parsers.html#text) supported. `import` also can utilize a `@watch:` syntax while `export` additionally supports `@immediate`. | | `packetMsgEncoding` | :-1: | Override default `utf8` encoding. -| `defaultNetwork` | :-1: | Explicitly set default network (by tag in `messageNetworks.ftn.networks`). If not set, the first found is used. | -| `nodes` | :+1: | Per-node settings. Entries (keys) here support wildcards for a portion of the FTN-style address (e.g.: `21:1/*`). `archiveType` may be set to a FTN supported archive extention that the system supports (TODO); if unset, only .PKT files are produced. `encoding` may be set to override `packetMsgEncoding` on a per-node basis. If the node requires a packet password, set `packetPassword` | -| `paths` | :-1: | An optional configuration block that can set a additional paths or override defaults. See "Paths" below. | +| `defaultNetwork` | :-1: | Explicitly set default network (by tag found within `messageNetworks.ftn.networks`). If not set, the first found is used. | +| `nodes` | :+1: | Per-node settings. Entries (keys) here support wildcards for a portion of the FTN-style address (e.g.: `21:1/*`). See **Nodes** below. +| `paths` | :-1: | An optional configuration block that can set a additional paths or override defaults. See **Paths** below. | | `packetTargetByteSize` | :-1: | Overrides the system *target* packet (.pkt) size of 512000 bytes (512k) | | `bundleTargetByteSize` | :-1: | Overrides the system *target* ArcMail bundle size of 2048000 bytes (2M) | -### Paths -Paths for packet files work out of the box and are relative to your install directory. If you want to configure `reject` or `retain` to keep rejected/imported packet files respectively, set those values. You may override defaults as well. - -| Key | Description | Default | -|-----|-------------|---------| -| `outbound` | *Base* path to write outbound (exported) packet files and bundles. | `enigma-bbs/mail/ftn_out/` | -| `inbound` | *Base* path to write inbound (ie: those written by an external mailer) packet files an bundles. | `enigma-bbs/mail/ftn_in/` | -| `secInbound` | *Base* path to write **secure** inbound packet files and bundles. | `enigma-bbs/mail/ftn_secin/` | -| `reject` | Path in which to write rejected packet files. | No default | -| `retain` | Path in which to write imported packet files. Useful for debugging or if you wish to archive the raw .pkt files. | No default | - - -## Scheduling -Schedules can be defined for importing and exporting via `import` and `export` under `schedule`. Each entry is allowed a "free form" text and/or special indicators for immediate export or watch file triggers. - - * `@immediate`: A message will be immediately exported if this trigger is defined in a schedule. Only used for `export`. - * `@watch:/path/to/file`: This trigger watches the path specified for changes and will trigger an import or export when such events occur. Only used for `import`. - * Free form [Later style](https://bunkat.github.io/later/parsers.html#text) text — can be things like `at 5:00 pm` or `every 2 hours`. - -See [Later text parsing documentation](http://bunkat.github.io/later/parsers.html#text) for more information. - -### Example Schedule Configuration - -```hjson -{ - scannerTossers: { - ftn_bso: { - schedule: { - import: every 1 hours or @watch:/path/to/watchfile.ext - export: every 1 hours or @immediate - } - } - } -} -``` - -## Nodes -The `nodes` section defines how to export messages for one or more uplinks. +#### Nodes +The `nodes` section defines how to export messages for one or more uplinks. A node entry starts with a [FTN address](http://ftsc.org/docs/old/fsp-1028.001) (up to 5D) **as a key** in `config.hjson`. This key may contain wildcard(s) for net/zone/node/point/domain. @@ -65,7 +30,7 @@ A node entry starts with a [FTN address](http://ftsc.org/docs/old/fsp-1028.001) | `packetType` | :-1: | `2`, `2.2`, or `2+`. Defaults to `2+` for modern mailer compatiability. | | `packetPassword` | :-1: | Optional password for the packet | | `encoding` | :-1: | Encoding to use for message bodies; Defaults to `utf-8`. | -| `archiveType` | :-1: | Specifies the archive type (by extension) for ArcMail bundles. This should be `zip` for most setups. Other valid examples include `arc`, `arj`, `lhz`, `pak`, `sqz`, or `zoo`. See [Archivers](docs/configuration/archivers.md) for more information. | +| `archiveType` | :-1: | Specifies the archive type (by extension or MIME type) for ArcMail bundles. This should be `zip` (or `application/zip`) for most setups. Other valid examples include `arc`, `arj`, `lhz`, `pak`, `sqz`, or `zoo`. See [Archivers](docs/configuration/archivers.md) for more information. | **Example**: ```hjson @@ -85,7 +50,42 @@ A node entry starts with a [FTN address](http://ftsc.org/docs/old/fsp-1028.001) } ``` -## A More Complete Example +#### Paths +Paths for packet files work out of the box and are relative to your install directory. If you want to configure `reject` or `retain` to keep rejected/imported packet files respectively, set those values. You may override defaults as well. + +| Key | Description | Default | +|-----|-------------|---------| +| `outbound` | *Base* path to write outbound (exported) packet files and bundles. | `enigma-bbs/mail/ftn_out/` | +| `inbound` | *Base* path to write inbound (ie: those written by an external mailer) packet files an bundles. | `enigma-bbs/mail/ftn_in/` | +| `secInbound` | *Base* path to write **secure** inbound packet files and bundles. | `enigma-bbs/mail/ftn_secin/` | +| `reject` | Path in which to write rejected packet files. | No default | +| `retain` | Path in which to write imported packet files. Useful for debugging or if you wish to archive the raw .pkt files. | No default | + +### Scheduling +Schedules can be defined for importing and exporting via `import` and `export` under `schedule`. Each entry is allowed a "free form" text and/or special indicators for immediate export or watch file triggers. + +* `@immediate`: A message will be immediately exported if this trigger is defined in a schedule. Only used for `export`. +* `@watch:/path/to/file`: This trigger watches the path specified for changes and will trigger an import or export when such events occur. Only used for `import`. +* Free form [Later style](https://bunkat.github.io/later/parsers.html#text) text — can be things like `at 5:00 pm` or `every 2 hours`. + +See [Later text parsing documentation](http://bunkat.github.io/later/parsers.html#text) for more information. + +#### Example Schedule Configuration + +```hjson +{ + scannerTossers: { + ftn_bso: { + schedule: { + import: every 1 hours or @watch:/path/to/watchfile.ext + export: every 1 hours or @immediate + } + } + } +} +``` + +### A More Complete Example Below is a more complete example showing the sections described above. ```hjson @@ -149,7 +149,7 @@ do done ``` -Now, create an Event Scheuler entry in your `config.hjson`. As an example: +Now, create an Event Scheduler entry in your `config.hjson`. As an example: ```hjson eventScheduler: { events: { @@ -163,4 +163,4 @@ eventScheduler: { ``` ## Additional Resources -* [Blog entry on setting up ENiGMA + Binkd on CentOS7](https://l33t.codes/enigma-12-binkd-on-centos-7/). Note that this references an **older version**, so be wary of the `config.hjson` refernces! +[Blog entry on setting up ENiGMA + Binkd on CentOS7](https://l33t.codes/enigma-12-binkd-on-centos-7/). Note that this references an **older version**, so be wary of the `config.hjson` references! From 82a1e22e422dd3b9cda04ba37ede8013a23571cc Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 20:58:13 -0600 Subject: [PATCH 48/95] File area doc tidy --- docs/filebase/first-file-area.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/filebase/first-file-area.md b/docs/filebase/first-file-area.md index 46d8d649..e61e5a25 100644 --- a/docs/filebase/first-file-area.md +++ b/docs/filebase/first-file-area.md @@ -3,15 +3,15 @@ layout: page title: Configuring a File Base --- ## Configuring a File Base -ENiGMA½ offers a powerful and flexible file base. Configuration of file the file base and areas is handled via the `fileBase` section of `config.hjson`. +ENiGMA½ offers a powerful and flexible file base. Configuration of file the file base and areas is handled via the `fileBase` section of `config.hjson`. ## ENiGMA½ File Base Key Concepts First, there are some core concepts you should understand: * Storage Tags -* Areas (and Area Tags) +* Area Tags ### Storage Tags -*Storage Tags* define paths to physical (file) storage locations that are referenced in a file *Area* entry. Each entry may be either a fully qualified path or a relative path. Relative paths are relative to the value set by the `areaStoragePrefix` key (defaults to `/path/to/enigma-bbs/file_base`). +*Storage Tags* define paths to physical (filesystem) storage locations that are referenced in a file *Area* entry. Each entry may be either a fully qualified path or a relative path. Relative paths are relative to the value set by the `fileBase.areaStoragePrefix` key (defaults to `/path/to/enigma-bbs/file_base`). Below is an example defining some storage tags using the relative and fully qualified forms: @@ -28,7 +28,7 @@ storageTags: { :information_source: Remember that paths are case sensitive on most non-Windows systems! ### Areas -File base *Areas* are configured using the `fileBase::areas` configuration block in `config.hjson`. Valid members for an area are as follows: +File base *Areas* are configured using the `fileBase.areas` configuration block in `config.hjson`. Each entry's block starts with an *area tag*. Valid members for an area are as follows: | Item | Required | Description | |--------|---------------|------------------| @@ -41,7 +41,7 @@ Example areas section: ```hjson areas: { - retro_pc: { + retro_pc: { // an area tag! name: Retro PC desc: Oldschool PC/DOS storageTags: [ "retro_pc_dos", "retro_pc_bbs" ] @@ -55,6 +55,7 @@ This combines the two concepts described above. When viewing the file areas from ```hjson fileBase: { + // override the default relative location areaStoragePrefix: /enigma-bbs/file_base storageTags: { From b3abe3ad407541f4f814144b2530836a7fbd387c Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 21:20:27 -0600 Subject: [PATCH 49/95] More doc tidy --- docs/filebase/index.md | 6 ++++++ docs/filebase/tic-support.md | 31 ++++++++++++++++--------------- docs/filebase/uploads.md | 4 ++-- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/docs/filebase/index.md b/docs/filebase/index.md index d5dac134..0e14e732 100644 --- a/docs/filebase/index.md +++ b/docs/filebase/index.md @@ -20,3 +20,9 @@ ENiGMA½ has strayed away from the old familiar setup here and instead takes a m * Duplicates are checked for by cryptographically secure [SHA-256](https://en.wikipedia.org/wiki/SHA-2) hashes. * Support for many archive and file formats. External utilities can easily be added to the configuration to extend for additional formats. * Much, much more! + +### Modding +The default ENiGMA½ approach for file areas may not be for everyone. Remember that you can mod everything your setup! Some inspirational examples: +* A more traditional set of areas and scrolling file listings. +* An S/X style integration of message areas and file areas. +* Something completely different! Some tweaks are possible without any code while others may require creating new JavaScript modules to use instead of the defaults. diff --git a/docs/filebase/tic-support.md b/docs/filebase/tic-support.md index 9d092fb1..4c128bab 100644 --- a/docs/filebase/tic-support.md +++ b/docs/filebase/tic-support.md @@ -2,9 +2,10 @@ layout: page title: TIC Support --- -ENiGMA½ supports TIC files. This is handled by mapping TIC areas to local file areas. +## TIC Support +ENiGMA½ supports FidoNet-Style TIC file attachments by mapping TIC areas to local file areas. -Under a given node defined in the `ftn_bso` config section in `config.hjson` (see +Under a given node defined in the `ftn_bso` config section in `config.hjson` (see [BSO Import/Export](../messageareas/bso-import-export)), TIC configuration may be supplied: ```hjson @@ -28,11 +29,9 @@ Under a given node defined in the `ftn_bso` config section in `config.hjson` (se } ``` -You then need to configure the mapping between TIC areas you want to carry, and the file -base area and storage tag for them to be tossed to. Optionally you can also add hashtags to the tossed -files to assist users in searching for files: +You then need to configure the mapping between TIC areas you want to carry, and the file base area and storage tag for them to be tossed to. Optionally you can also add hashtags to the tossed files to assist users in searching for files: -````hjson +```hjson ticAreas: { agn_node: { areaTag: msgNetworks @@ -41,21 +40,20 @@ ticAreas: { } } -```` -Multiple TIC areas can be mapped to a single file base area. +``` +Multiple TIC areas can be mapped to a single file base area. -## Example Configuration +### Example Configuration +An example configuration linking file base areas, FTN BSO node configuration and TIC area configuration. -An example configuration linking filebase areas, FTN BSO node configuration and TIC area configuration. - -````hjson +```hjson fileBase: { areaStoragePrefix: /home/bbs/file_areas/ - + storageTags: { msg_network: "msg_network" } - + areas: { msgNetworks: { name: Message Networks @@ -97,4 +95,7 @@ ticAreas: { hashTags: agoranet,infopack } } -```` \ No newline at end of file +``` + +## See Also +[Message Networks](/docs/messageareas/message-networks.md) diff --git a/docs/filebase/uploads.md b/docs/filebase/uploads.md index 8e1e2530..72b901c9 100644 --- a/docs/filebase/uploads.md +++ b/docs/filebase/uploads.md @@ -3,9 +3,9 @@ layout: page title: Uploads --- ## Uploads -The default ACS for file areas areas in ENiGMA½ is to allow read (viewing of the area), and downloads for users while only permitting SysOps to write (upload). See [File Base ACS](acs.md) for more information. +The default ACS for file areas in ENiGMA½ is to allow regular users 'read' and sysops 'read/write'. Read ACS includes listing and downloading while write allows for uploading. See [File Base ACS](acs.md) for more information. -To allow uploads to a particular area, change the ACS level for `write`. For example: +To change ACS for a particular area, create an `acs` block specifying ACS for `read` and/or `write`. For example, let's allow regular users (in the "users" group) to upload to an area: ```hjson uploads: { name: Uploads From 1f691c6246c2e98d991421906c3626ddd1a43fb2 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 21:22:15 -0600 Subject: [PATCH 50/95] Better example --- docs/filebase/uploads.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/filebase/uploads.md b/docs/filebase/uploads.md index 72b901c9..795c0781 100644 --- a/docs/filebase/uploads.md +++ b/docs/filebase/uploads.md @@ -5,7 +5,7 @@ title: Uploads ## Uploads The default ACS for file areas in ENiGMA½ is to allow regular users 'read' and sysops 'read/write'. Read ACS includes listing and downloading while write allows for uploading. See [File Base ACS](acs.md) for more information. -To change ACS for a particular area, create an `acs` block specifying ACS for `read` and/or `write`. For example, let's allow regular users (in the "users" group) to upload to an area: +Let's allow regular users (in the "users" group) to upload to an area: ```hjson uploads: { name: Uploads From 4ddf79087ccbc47d7a0e69a295c8b6e1a4166f6b Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 21:28:28 -0600 Subject: [PATCH 51/95] Update to ugprade docs --- docs/admin/updating.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/admin/updating.md b/docs/admin/updating.md index 235737b7..ee396b4b 100644 --- a/docs/admin/updating.md +++ b/docs/admin/updating.md @@ -3,18 +3,18 @@ layout: page title: Updating --- ## Updating your Installation -Updating ENiGMA½ can be a bit of a learning curve compared to other systems. Especially when running off of a development branch (such as `0.0.9-alpha` being the recommended branch as of this writing), you'll want frequent updates. +Updating ENiGMA½ can be a bit of a learning curve compared to other systems.Especially when running from Git cloned source, you'll want frequent updates. ## Steps In general the steps are as follows: 1. `cd /path/to/enigma-bbs` 2. `git pull` 3. `npm update` or `yarn` to refresh any new or updated modules. -4. Merge updates to `config/menu_template.hjson` to your `config/yourbbsname-menu.hjson` file. +4. Merge updates to `config/menu_template.hjson` to your `config/yourbbsname-menu.hjson` file (or simply use the template as a reference to spot any newly added default menus that you may wish to have on your system as well!). 5. If there are updates to the `art/themes/luciano_blocktronics/theme.hjson` file and you have a custom theme, you may want to look at them as well. Visual diff tools such as [DiffMerge](https://www.sourcegear.com/diffmerge/downloads.php) (free, works on all major platforms) can be very helpful here. -Remember to also keep an eye on [WHATSNEW](/WHATSNEW.md) and [UPGRADE](/UPGRADE.md)! +:information_source: Remember to also keep an eye on [WHATSNEW](/WHATSNEW.md) and [UPGRADE](/UPGRADE.md)! From 8409b80d698c9c723231f2f1024b33e3cdee6e93 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 21:31:21 -0600 Subject: [PATCH 52/95] Slightly better --- docs/admin/updating.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/admin/updating.md b/docs/admin/updating.md index ee396b4b..8fda4692 100644 --- a/docs/admin/updating.md +++ b/docs/admin/updating.md @@ -3,18 +3,19 @@ layout: page title: Updating --- ## Updating your Installation -Updating ENiGMA½ can be a bit of a learning curve compared to other systems.Especially when running from Git cloned source, you'll want frequent updates. +Updating ENiGMA½ can be a bit of a learning curve compared to other systems. Especially when running from Git cloned source, you'll want frequent updates. ## Steps -In general the steps are as follows: +In _general_ the steps are as follows: 1. `cd /path/to/enigma-bbs` 2. `git pull` 3. `npm update` or `yarn` to refresh any new or updated modules. 4. Merge updates to `config/menu_template.hjson` to your `config/yourbbsname-menu.hjson` file (or simply use the template as a reference to spot any newly added default menus that you may wish to have on your system as well!). 5. If there are updates to the `art/themes/luciano_blocktronics/theme.hjson` file and you have a custom theme, you may want to look at them as well. -Visual diff tools such as [DiffMerge](https://www.sourcegear.com/diffmerge/downloads.php) (free, works on all major platforms) can be very helpful here. +:information_source: Always keep an eye on [WHATSNEW](/WHATSNEW.md) and [UPGRADE](/UPGRADE.md)! + +:information_source: Visual diff tools such as [DiffMerge](https://www.sourcegear.com/diffmerge/downloads.php) (free, works on all major platforms) can be very helpful for the tasks outlined above! -:information_source: Remember to also keep an eye on [WHATSNEW](/WHATSNEW.md) and [UPGRADE](/UPGRADE.md)! From b76bb8a9bd4f3887cde6f2e5d0901b93cfcdfd4b Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 21:49:18 -0600 Subject: [PATCH 53/95] Minor dep updates --- package.json | 4 ++-- yarn.lock | 23 ++++++++--------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index 12754271..908c48c3 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "retro" ], "dependencies": { - "async": "3.1.0", + "async": "3.2.0", "binary-parser": "^1.5.0", "buffers": "github:NuSkooler/node-buffers", "bunyan": "^1.8.12", @@ -55,7 +55,7 @@ "temptmp": "^1.1.0", "uuid": "^3.3.3", "uuid-parse": "1.1.0", - "ws": "^7.2.0", + "ws": "^7.3.0", "xxhash": "^0.3.0", "yazl": "^2.5.1" }, diff --git a/yarn.lock b/yarn.lock index cb08db2e..1c3624aa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -124,15 +124,10 @@ assign-symbols@^1.0.0: resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367" integrity sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c= -async-limiter@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.0.tgz#78faed8c3d074ab81f22b4e985d79e8738f720f8" - integrity sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg== - -async@3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/async/-/async-3.1.0.tgz#42b3b12ae1b74927b5217d8c0016baaf62463772" - integrity sha512-4vx/aaY6j/j3Lw3fbCHNWP0pPaTCew3F6F3hYyl/tHs/ndmV1q7NW9T5yuJ2XAGwdQrP+6Wu20x06U4APo/iQQ== +async@3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720" + integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw== asynckit@^0.4.0: version "0.4.0" @@ -2276,12 +2271,10 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= -ws@^7.2.0: - version "7.2.0" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.2.0.tgz#422eda8c02a4b5dba7744ba66eebbd84bcef0ec7" - integrity sha512-+SqNqFbwTm/0DC18KYzIsMTnEWpLwJsiasW/O17la4iDRRIO9uaHbvKiAS3AHgTiuuWerK/brj4O6MYZkei9xg== - dependencies: - async-limiter "^1.0.0" +ws@^7.3.0: + version "7.3.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-7.3.0.tgz#4b2f7f219b3d3737bc1a2fbf145d825b94d38ffd" + integrity sha512-iFtXzngZVXPGgpTlP1rBqsUK82p9tKqsWRPg5L56egiljujJT3vGAYnHANvFxBieXrTFavhzhxW52jnaWV+w2w== xtend@~4.0.1: version "4.0.1" From 9ed7c049ad191d03dfafa16bf6cd94b1733862fc Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 22:05:17 -0600 Subject: [PATCH 54/95] Update uuid dep --- core/file_base_filter.js | 4 ++-- core/file_base_user_list_export.js | 4 ++-- core/message_base_qwk_export.js | 2 +- core/scanner_tossers/ftn_bso.js | 4 ++-- package.json | 2 +- yarn.lock | 8 ++++---- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/core/file_base_filter.js b/core/file_base_filter.js index d72b3eea..ecf857fa 100644 --- a/core/file_base_filter.js +++ b/core/file_base_filter.js @@ -5,7 +5,7 @@ const UserProps = require('./user_property.js'); // deps const _ = require('lodash'); -const uuidV4 = require('uuid/v4'); +const { v4 : UUIDv4 } = require('uuid'); module.exports = class FileBaseFilters { constructor(client) { @@ -41,7 +41,7 @@ module.exports = class FileBaseFilters { } add(filterInfo) { - const filterUuid = uuidV4(); + const filterUuid = UUIDv4(); filterInfo.tags = this.cleanTags(filterInfo.tags); diff --git a/core/file_base_user_list_export.js b/core/file_base_user_list_export.js index 594a9ffe..c3f3b6f8 100644 --- a/core/file_base_user_list_export.js +++ b/core/file_base_user_list_export.js @@ -17,7 +17,7 @@ const fs = require('graceful-fs'); const fse = require('fs-extra'); const paths = require('path'); const moment = require('moment'); -const uuidv4 = require('uuid/v4'); +const { v4 : UUIDv4 } = require('uuid'); const yazl = require('yazl'); /* @@ -188,7 +188,7 @@ exports.getModule = class FileBaseListExport extends MenuModule { const outputFileName = paths.join( sysTempDownloadDir, - `file_list_${uuidv4().substr(-8)}_${moment().format('YYYY-MM-DD')}.txt` + `file_list_${UUIDv4().substr(-8)}_${moment().format('YYYY-MM-DD')}.txt` ); fs.writeFile(outputFileName, listBody, 'utf8', err => { diff --git a/core/message_base_qwk_export.js b/core/message_base_qwk_export.js index 53682471..2f89c1bd 100644 --- a/core/message_base_qwk_export.js +++ b/core/message_base_qwk_export.js @@ -22,7 +22,7 @@ const _ = require('lodash'); const fse = require('fs-extra'); const temptmp = require('temptmp'); const paths = require('path'); -const UUIDv4 = require('uuid/v4'); +const { v4 : UUIDv4 } = require('uuid'); const moment = require('moment'); const FormIds = { diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 17cd61b8..99a537ec 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -36,7 +36,7 @@ const assert = require('assert'); const sane = require('sane'); const fse = require('fs-extra'); const iconv = require('iconv-lite'); -const uuidV4 = require('uuid/v4'); +const { v4 : UUIDv4 } = require('uuid'); exports.moduleInfo = { name : 'FTN BSO', @@ -1215,7 +1215,7 @@ function FTNMessageScanTossModule() { // if(true === _.get(Config(), [ 'messageNetworks', 'ftn', 'areas', config.localAreaTag, 'allowDupes' ], false)) { // just generate a UUID & therefor always allow for dupes - message.messageUuid = uuidV4(); + message.messageUuid = UUIDv4(); } return callback(null); diff --git a/package.json b/package.json index 908c48c3..efdd1d17 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "sqlite3-trans": "^1.2.2", "ssh2": "0.8.6", "temptmp": "^1.1.0", - "uuid": "^3.3.3", + "uuid": "^8.0.0", "uuid-parse": "1.1.0", "ws": "^7.3.0", "xxhash": "^0.3.0", diff --git a/yarn.lock b/yarn.lock index 1c3624aa..f5fb93f3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2231,10 +2231,10 @@ uuid@^3.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== -uuid@^3.3.3: - version "3.3.3" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.3.tgz#4568f0216e78760ee1dbf3a4d2cf53e224112866" - integrity sha512-pW0No1RGHgzlpHJO1nsVrHKpOEIxkGg1xB+v0ZmdNH5OAeAwzAVrCnI2/6Mtx+Uys6iaylxa+D3g4j63IKKjSQ== +uuid@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c" + integrity sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw== verror@1.10.0: version "1.10.0" From 0e5d9ddbd1e4a7c212c708e886f07771a9e5d384 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 22:12:03 -0600 Subject: [PATCH 55/95] Update sqlite3 (minor) --- package.json | 2 +- yarn.lock | 311 ++------------------------------------------------- 2 files changed, 11 insertions(+), 302 deletions(-) diff --git a/package.json b/package.json index efdd1d17..d22f18f3 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "rlogin": "^1.0.0", "sane": "4.1.0", "sanitize-filename": "^1.6.3", - "sqlite3": "^4.1.0", + "sqlite3": "^4.2.0", "sqlite3-trans": "^1.2.2", "ssh2": "0.8.6", "temptmp": "^1.1.0", diff --git a/yarn.lock b/yarn.lock index f5fb93f3..e2b5b944 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15,16 +15,6 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== -ajv@^5.3.0: - version "5.5.2" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.5.2.tgz#73b5eeca3fab653e3d3f9422b341ad42205dc965" - integrity sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU= - dependencies: - co "^4.6.0" - fast-deep-equal "^1.0.0" - fast-json-stable-stringify "^2.0.0" - json-schema-traverse "^0.3.0" - ansi-escapes@^4.2.1: version "4.2.1" resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.2.1.tgz#4dccdb846c3eee10f6d64dea66273eab90c37228" @@ -107,18 +97,13 @@ array-unique@^0.3.2: resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428" integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg= -asn1@~0.2.0, asn1@~0.2.3: +asn1@~0.2.0: version "0.2.4" resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" integrity sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg== dependencies: safer-buffer "~2.1.0" -assert-plus@1.0.0, assert-plus@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" - integrity sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU= - assign-symbols@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367" @@ -129,26 +114,11 @@ async@3.2.0: resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720" integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw== -asynckit@^0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" - integrity sha1-x57Zf380y48robyXkLzDZkdLS3k= - atob@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== -aws-sign2@~0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" - integrity sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg= - -aws4@^1.8.0: - version "1.8.0" - resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f" - integrity sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ== - balanced-match@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" @@ -167,7 +137,7 @@ base@^0.11.1: mixin-deep "^1.2.0" pascalcase "^0.1.1" -bcrypt-pbkdf@^1.0.0, bcrypt-pbkdf@^1.0.2: +bcrypt-pbkdf@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" integrity sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4= @@ -251,11 +221,6 @@ capture-exit@^2.0.0: dependencies: rsvp "^4.8.4" -caseless@~0.12.0: - version "0.12.0" - resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" - integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= - chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" @@ -297,11 +262,6 @@ cli-width@^2.0.0: resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.0.tgz#ff19ede8a9a5e579324147b0c11f0fbcbabed639" integrity sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk= -co@^4.6.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" - integrity sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ= - code-point-at@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" @@ -332,20 +292,6 @@ color-name@1.1.3: resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= -combined-stream@1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.6.tgz#723e7df6e801ac5613113a7e445a9b69cb632818" - integrity sha1-cj599ugBrFYTETp+RFqbactjKBg= - dependencies: - delayed-stream "~1.0.0" - -combined-stream@~1.0.6: - version "1.0.7" - resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.7.tgz#2d1d24317afb8abe95d6d2c0b07b57813539d828" - integrity sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w== - dependencies: - delayed-stream "~1.0.0" - component-emitter@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.2.1.tgz#137918d6d78283f7df7a6b7c5a63e140e69425e6" @@ -366,7 +312,7 @@ copy-descriptor@^0.1.0: resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d" integrity sha1-Z29us8OZl8LuGsOpJP1hJHSPV40= -core-util-is@1.0.2, core-util-is@~1.0.0: +core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= @@ -382,13 +328,6 @@ cross-spawn@^6.0.0: shebang-command "^1.2.0" which "^1.2.9" -dashdash@^1.12.0: - version "1.14.1" - resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" - integrity sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA= - dependencies: - assert-plus "^1.0.0" - debug@^2.1.2, debug@^2.2.0, debug@^2.3.3: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -452,11 +391,6 @@ del@^3.0.0: pify "^3.0.0" rimraf "^2.2.8" -delayed-stream@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" - integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= - delegates@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" @@ -484,14 +418,6 @@ dtrace-provider@~0.8: dependencies: nan "^2.10.0" -ecc-jsbn@~0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" - integrity sha1-OoOpBOVDUyh4dMVkt1SThoSamMk= - dependencies: - jsbn "~0.1.0" - safer-buffer "^2.1.0" - emoji-regex@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" @@ -560,11 +486,6 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2: assign-symbols "^1.0.0" is-extendable "^1.0.1" -extend@~3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" - integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== - external-editor@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/external-editor/-/external-editor-3.0.3.tgz#5866db29a97826dbe4bf3afd24070ead9ea43a27" @@ -588,26 +509,6 @@ extglob@^2.0.4: snapdragon "^0.8.1" to-regex "^3.0.1" -extsprintf@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" - integrity sha1-lpGEQOMEGnpBT4xS48V06zw+HgU= - -extsprintf@^1.2.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" - integrity sha1-4mifjzVvrWLMplo6kcXfX5VRaS8= - -fast-deep-equal@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz#c053477817c86b51daa853c81e059b733d023614" - integrity sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ= - -fast-json-stable-stringify@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2" - integrity sha1-1RQsDK7msRifh9OnYREGT4bIu/I= - fb-watchman@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/fb-watchman/-/fb-watchman-2.0.0.tgz#54e9abf7dfa2f26cd9b1636c588c1afc05de5d58" @@ -637,20 +538,6 @@ for-in@^1.0.2: resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" integrity sha1-gQaNKVqBQuwKxybG4iAMMPttXoA= -forever-agent@~0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" - integrity sha1-+8cfDEGt6zf5bFd60e1C2P2sypE= - -form-data@~2.3.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.2.tgz#4970498be604c20c005d4f5c23aecd21d6b49099" - integrity sha1-SXBJi+YEwgwAXU9cI67NIda0kJk= - dependencies: - asynckit "^0.4.0" - combined-stream "1.0.6" - mime-types "^2.1.12" - fragment-cache@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/fragment-cache/-/fragment-cache-0.2.1.tgz#4290fad27f13e89be7f33799c6bc5a0abfff0d19" @@ -713,13 +600,6 @@ get-value@^2.0.3, get-value@^2.0.6: resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" integrity sha1-3BXKHGcjh8p2vTesCjlbogQqLCg= -getpass@^0.1.1: - version "0.1.7" - resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" - integrity sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo= - dependencies: - assert-plus "^1.0.0" - glob@7.1.6: version "7.1.6" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" @@ -781,19 +661,6 @@ graceful-fs@^4.2.3: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.3.tgz#4a12ff1b60376ef09862c2093edd908328be8423" integrity sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ== -har-schema@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" - integrity sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI= - -har-validator@~5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.1.0.tgz#44657f5688a22cfd4b72486e81b3a3fb11742c29" - integrity sha512-+qnmNjI4OfH2ipQ9VQOw23bBd/ibtfbVdK2fYbY4acTDqKTW/YDp9McimZdDbG8iV9fZizUqQMD5xvriB146TA== - dependencies: - ajv "^5.3.0" - har-schema "^2.0.0" - has-flag@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" @@ -845,15 +712,6 @@ hjson@^3.2.1: resolved "https://registry.yarnpkg.com/hjson/-/hjson-3.2.1.tgz#20de41dc87fc9a10d1557d0230b0e02afb1b09ac" integrity sha512-OhhrFMeC7dVuA1xvxuXGTv/yTdhTvbe8hz+3LgVNsfi9+vgz0sF/RrkuX8eegpKaMc9cwYwydImBH6iePoJtdQ== -http-signature@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" - integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE= - dependencies: - assert-plus "^1.0.0" - jsprim "^1.2.2" - sshpk "^1.7.0" - iconv-lite@0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550" @@ -1044,11 +902,6 @@ is-stream@^1.1.0: resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ= -is-typedarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" - integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= - is-windows@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d" @@ -1076,31 +929,6 @@ isobject@^3.0.0, isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8= -isstream@~0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" - integrity sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo= - -jsbn@~0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" - integrity sha1-peZUwuWi3rXyAdls77yoDA7y9RM= - -json-schema-traverse@^0.3.0: - version "0.3.1" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" - integrity sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A= - -json-schema@0.2.3: - version "0.2.3" - resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" - integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM= - -json-stringify-safe@~5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" - integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= - jsonfile@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" @@ -1108,16 +936,6 @@ jsonfile@^4.0.0: optionalDependencies: graceful-fs "^4.1.6" -jsprim@^1.2.2: - version "1.4.1" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" - integrity sha1-MT5mvB5cwG5Di8G3SZwuXFastqI= - dependencies: - assert-plus "1.0.0" - extsprintf "1.3.0" - json-schema "0.2.3" - verror "1.10.0" - kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0: version "3.2.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" @@ -1202,11 +1020,6 @@ mime-db@1.42.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.42.0.tgz#3e252907b4c7adb906597b4b65636272cf9e7bac" integrity sha512-UbfJCR4UAVRNgMpfImz05smAXK7+c+ZntjaA26ANtkXLlOe947Aag5zdIcKQULAiF9Cq4WxBi9jUs5zkA84bYQ== -mime-db@~1.36.0: - version "1.36.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.36.0.tgz#5020478db3c7fe93aad7bbcc4dcf869c43363397" - integrity sha512-L+xvyD9MkoYMXb1jAmzI/lWYAxAMCPvIBSWur0PZ5nOf5euahRLVqH//FKW9mWp2lkqUgYiXPgkzfMUFi4zVDw== - mime-types@2.1.25: version "2.1.25" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.25.tgz#39772d46621f93e2a80a856c53b86a62156a6437" @@ -1214,13 +1027,6 @@ mime-types@2.1.25: dependencies: mime-db "1.42.0" -mime-types@^2.1.12, mime-types@~2.1.19: - version "2.1.20" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.20.tgz#930cb719d571e903738520f8470911548ca2cc19" - integrity sha512-HrkrPaP9vGuWbLK1B1FfgAkbqNjIuy4eHlIYnFi7kamZyLLrGlo2mpcx0bBmNpKqBtYtAfGbodDddIgddSJC2A== - dependencies: - mime-db "~1.36.0" - mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" @@ -1452,11 +1258,6 @@ number-is-nan@^1.0.0: resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= -oauth-sign@~0.9.0: - version "0.9.0" - resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" - integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== - object-assign@^4.0.1, object-assign@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" @@ -1554,11 +1355,6 @@ path-key@^2.0.0, path-key@^2.0.1: resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A= -performance-now@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" - integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= - pify@^2.0.0: version "2.3.0" resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" @@ -1591,11 +1387,6 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" integrity sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw== -psl@^1.1.24: - version "1.1.29" - resolved "https://registry.yarnpkg.com/psl/-/psl-1.1.29.tgz#60f580d360170bb722a797cc704411e6da850c67" - integrity sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ== - pump@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" @@ -1604,21 +1395,11 @@ pump@^3.0.0: end-of-stream "^1.1.0" once "^1.3.1" -punycode@^1.4.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" - integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= - qrcode-generator@^1.4.4: version "1.4.4" resolved "https://registry.yarnpkg.com/qrcode-generator/-/qrcode-generator-1.4.4.tgz#63f771224854759329a99048806a53ed278740e7" integrity sha512-HM7yY8O2ilqhmULxGMpcHSF1EhJJ9yBj8gvDEuZ6M+KGJ0YY2hKpnXvRD+hZPLrDVck3ExIGhmPtSdcjC+guuw== -qs@~6.5.2: - version "6.5.2" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" - integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== - rc@^1.2.7: version "1.2.8" resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" @@ -1674,32 +1455,6 @@ repeat-string@^1.6.1: resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= -request@^2.87.0: - version "2.88.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" - integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg== - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.8.0" - caseless "~0.12.0" - combined-stream "~1.0.6" - extend "~3.0.2" - forever-agent "~0.6.1" - form-data "~2.3.2" - har-validator "~5.1.0" - http-signature "~1.2.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.19" - oauth-sign "~0.9.0" - performance-now "^2.1.0" - qs "~6.5.2" - safe-buffer "^5.1.2" - tough-cookie "~2.4.3" - tunnel-agent "^0.6.0" - uuid "^3.3.2" - resolve-url@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a" @@ -1756,7 +1511,7 @@ rxjs@^6.4.0: dependencies: tslib "^1.9.0" -safe-buffer@^5.0.1, safe-buffer@^5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: +safe-buffer@^5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== @@ -1773,7 +1528,7 @@ safe-regex@^1.1.0: dependencies: ret "~0.1.10" -"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: +"safer-buffer@>= 2.1.2 < 3", safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== @@ -1934,14 +1689,13 @@ sqlite3-trans@^1.2.2: dependencies: lodash "^4.17.15" -sqlite3@^4.1.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-4.1.0.tgz#e051fb9c133be15726322a69e2e37ec560368380" - integrity sha512-RvqoKxq+8pDHsJo7aXxsFR18i+dU2Wp5o12qAJOV5LNcDt+fgJsc2QKKg3sIRfXrN9ZjzY1T7SNe/DFVqAXjaw== +sqlite3@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-4.2.0.tgz#49026d665e9fc4f922e56fb9711ba5b4c85c4901" + integrity sha512-roEOz41hxui2Q7uYnWsjMOTry6TcNUNmp8audCx18gF10P2NknwdpF+E+HKvz/F2NvPKGGBF4NGc+ZPQ+AABwg== dependencies: nan "^2.12.1" node-pre-gyp "^0.11.0" - request "^2.87.0" ssh2-streams@~0.4.7: version "0.4.7" @@ -1959,22 +1713,6 @@ ssh2@0.8.6: dependencies: ssh2-streams "~0.4.7" -sshpk@^1.7.0: - version "1.14.2" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.14.2.tgz#c6fc61648a3d9c4e764fd3fcdf4ea105e492ba98" - integrity sha1-xvxhZIo9nE52T9P8306hBeSSupg= - dependencies: - asn1 "~0.2.3" - assert-plus "^1.0.0" - dashdash "^1.12.0" - getpass "^0.1.1" - safer-buffer "^2.0.2" - optionalDependencies: - bcrypt-pbkdf "^1.0.0" - ecc-jsbn "~0.1.1" - jsbn "~0.1.0" - tweetnacl "~0.14.0" - static-extend@^0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/static-extend/-/static-extend-0.1.2.tgz#60809c39cbff55337226fd5e0b520f341f1fb5c6" @@ -2141,14 +1879,6 @@ to-regex@^3.0.1, to-regex@^3.0.2: regex-not "^1.0.2" safe-regex "^1.1.0" -tough-cookie@~2.4.3: - version "2.4.3" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781" - integrity sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ== - dependencies: - psl "^1.1.24" - punycode "^1.4.1" - truncate-utf8-bytes@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz#405923909592d56f78a5818434b0b78489ca5f2b" @@ -2161,14 +1891,7 @@ tslib@^1.9.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.9.3.tgz#d7e4dd79245d85428c4d7e4822a79917954ca286" integrity sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ== -tunnel-agent@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" - integrity sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0= - dependencies: - safe-buffer "^5.0.1" - -tweetnacl@^0.14.3, tweetnacl@~0.14.0: +tweetnacl@^0.14.3: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= @@ -2226,25 +1949,11 @@ uuid-parse@1.1.0: resolved "https://registry.yarnpkg.com/uuid-parse/-/uuid-parse-1.1.0.tgz#7061c5a1384ae0e1f943c538094597e1b5f3a65b" integrity sha512-OdmXxA8rDsQ7YpNVbKSJkNzTw2I+S5WsbMDnCtIWSQaosNAcWtFuI/YK1TjzUI6nbkgiqEyh8gWngfcv8Asd9A== -uuid@^3.3.2: - version "3.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" - integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== - uuid@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c" integrity sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw== -verror@1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" - integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA= - dependencies: - assert-plus "^1.0.0" - core-util-is "1.0.2" - extsprintf "^1.2.0" - walker@~1.0.5: version "1.0.7" resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.7.tgz#2f7f9b8fd10d677262b18a884e28d19618e028fb" From 009688c3b6c7135a9343a54452bbf7212718dddf Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 22:30:57 -0600 Subject: [PATCH 56/95] SSH dep update --- package.json | 2 +- yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index d22f18f3..6b030312 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "sanitize-filename": "^1.6.3", "sqlite3": "^4.2.0", "sqlite3-trans": "^1.2.2", - "ssh2": "0.8.6", + "ssh2": "0.8.9", "temptmp": "^1.1.0", "uuid": "^8.0.0", "uuid-parse": "1.1.0", diff --git a/yarn.lock b/yarn.lock index e2b5b944..5c8f7793 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1697,21 +1697,21 @@ sqlite3@^4.2.0: nan "^2.12.1" node-pre-gyp "^0.11.0" -ssh2-streams@~0.4.7: - version "0.4.7" - resolved "https://registry.yarnpkg.com/ssh2-streams/-/ssh2-streams-0.4.7.tgz#093b89069de9cf5f06feff0601a5301471b01611" - integrity sha512-JhF8BNfeguOqVHOLhXjzLlRKlUP8roAEhiT/y+NcBQCqpRUupLNrRf2M+549OPNVGx21KgKktug4P3MY/IvTig== +ssh2-streams@~0.4.10: + version "0.4.10" + resolved "https://registry.yarnpkg.com/ssh2-streams/-/ssh2-streams-0.4.10.tgz#48ef7e8a0e39d8f2921c30521d56dacb31d23a34" + integrity sha512-8pnlMjvnIZJvmTzUIIA5nT4jr2ZWNNVHwyXfMGdRJbug9TpI3kd99ffglgfSWqujVv/0gxwMsDn9j9RVst8yhQ== dependencies: asn1 "~0.2.0" bcrypt-pbkdf "^1.0.2" streamsearch "~0.1.2" -ssh2@0.8.6: - version "0.8.6" - resolved "https://registry.yarnpkg.com/ssh2/-/ssh2-0.8.6.tgz#dcc62e1d3b9e58a21f711f5186f043e4e792e6da" - integrity sha512-T0cPmEtmtC8WxSupicFDjx3vVUdNXO8xu2a/D5bjt8ixOUCe387AgvxU3mJgEHpu7+Sq1ZYx4d3P2pl/yxMH+w== +ssh2@0.8.9: + version "0.8.9" + resolved "https://registry.yarnpkg.com/ssh2/-/ssh2-0.8.9.tgz#54da3a6c4ba3daf0d8477a538a481326091815f3" + integrity sha512-GmoNPxWDMkVpMFa9LVVzQZHF6EW3WKmBwL+4/GeILf2hFmix5Isxm7Amamo8o7bHiU0tC+wXsGcUXOxp8ChPaw== dependencies: - ssh2-streams "~0.4.7" + ssh2-streams "~0.4.10" static-extend@^0.1.1: version "0.1.2" From f27ba9f6e0208e0fc0788c2da451e89ce1a40ddc Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 22:39:49 -0600 Subject: [PATCH 57/95] fs-extra dep update --- package.json | 2 +- yarn.lock | 36 ++++++++++++++++++++++-------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 6b030312..1ca96316 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "buffers": "github:NuSkooler/node-buffers", "bunyan": "^1.8.12", "exiftool": "^0.0.3", - "fs-extra": "8.1.0", + "fs-extra": "9.0.0", "glob": "7.1.6", "graceful-fs": "^4.2.3", "hashids": "2.1.0", diff --git a/yarn.lock b/yarn.lock index 5c8f7793..e220adc7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -114,6 +114,11 @@ async@3.2.0: resolved "https://registry.yarnpkg.com/async/-/async-3.2.0.tgz#b3a2685c5ebb641d3de02d161002c60fc9f85720" integrity sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw== +at-least-node@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/at-least-node/-/at-least-node-1.0.0.tgz#602cd4b46e844ad4effc92a8011a3c46e0238dc2" + integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg== + atob@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" @@ -553,14 +558,15 @@ from2@^2.3.0: inherits "^2.0.1" readable-stream "^2.0.0" -fs-extra@8.1.0: - version "8.1.0" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0" - integrity sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g== +fs-extra@9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.0.0.tgz#b6afc31036e247b2466dc99c29ae797d5d4580a3" + integrity sha512-pmEYSk3vYsG/bF651KPUXZ+hvjpgWYw/Gc7W9NFUe3ZVLczKKWIij3IKpOrQcdw4TILtibFslZ0UmR8Vvzig4g== dependencies: + at-least-node "^1.0.0" graceful-fs "^4.2.0" - jsonfile "^4.0.0" - universalify "^0.1.0" + jsonfile "^6.0.1" + universalify "^1.0.0" fs-minipass@^1.2.5: version "1.2.5" @@ -929,10 +935,12 @@ isobject@^3.0.0, isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8= -jsonfile@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" - integrity sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss= +jsonfile@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.0.1.tgz#98966cba214378c8c84b82e085907b40bf614179" + integrity sha512-jR2b5v7d2vIOust+w3wtFKZIfpC2pnRmFAhAC/BuweZFQR8qZzxH1OyrQ10HmdVYiXWkYUqPVsz91cG7EL2FBg== + dependencies: + universalify "^1.0.0" optionalDependencies: graceful-fs "^4.1.6" @@ -1911,10 +1919,10 @@ union-value@^1.0.0: is-extendable "^0.1.1" set-value "^0.4.3" -universalify@^0.1.0: - version "0.1.2" - resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" - integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== +universalify@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-1.0.0.tgz#b61a1da173e8435b2fe3c67d29b9adf8594bd16d" + integrity sha512-rb6X1W158d7pRQBg5gkR8uPaSfiids68LTJQYOtEUhoJUWBdaQHsuT/EUduxXYxcrt4r5PJ4fuHW1MHT6p0qug== unset-value@^1.0.0: version "1.0.0" From dff3e9d0c8deb58b09404fe34fa171f65f80435b Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 22:42:56 -0600 Subject: [PATCH 58/95] Update mime-types --- package.json | 2 +- yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 1ca96316..7cba5d4d 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "later": "1.2.0", "lodash": "^4.17.15", "lru-cache": "^5.1.1", - "mime-types": "2.1.25", + "mime-types": "2.1.27", "minimist": "1.2.0", "moment": "^2.24.0", "nntp-server": "^1.0.3", diff --git a/yarn.lock b/yarn.lock index e220adc7..4ca8df12 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1023,17 +1023,17 @@ micromatch@^3.1.4: snapdragon "^0.8.1" to-regex "^3.0.2" -mime-db@1.42.0: - version "1.42.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.42.0.tgz#3e252907b4c7adb906597b4b65636272cf9e7bac" - integrity sha512-UbfJCR4UAVRNgMpfImz05smAXK7+c+ZntjaA26ANtkXLlOe947Aag5zdIcKQULAiF9Cq4WxBi9jUs5zkA84bYQ== +mime-db@1.44.0: + version "1.44.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92" + integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg== -mime-types@2.1.25: - version "2.1.25" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.25.tgz#39772d46621f93e2a80a856c53b86a62156a6437" - integrity sha512-5KhStqB5xpTAeGqKBAMgwaYMnQik7teQN4IAzC7npDv6kzeU6prfkR67bc87J1kWMPGkoaZSq1npmexMgkmEVg== +mime-types@2.1.27: + version "2.1.27" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f" + integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w== dependencies: - mime-db "1.42.0" + mime-db "1.44.0" mimic-fn@^2.1.0: version "2.1.0" From 8736f5c3b9208391bff9b0a13ad29a3fad481637 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 22:45:04 -0600 Subject: [PATCH 59/95] graceful-fs update --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 7cba5d4d..4c772eee 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "exiftool": "^0.0.3", "fs-extra": "9.0.0", "glob": "7.1.6", - "graceful-fs": "^4.2.3", + "graceful-fs": "^4.2.4", "hashids": "2.1.0", "hjson": "^3.2.1", "iconv-lite": "0.5.0", diff --git a/yarn.lock b/yarn.lock index 4ca8df12..f0171f7f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -662,10 +662,10 @@ graceful-fs@^4.2.0: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.0.tgz#8d8fdc73977cb04104721cb53666c1ca64cd328b" integrity sha512-jpSvDPV4Cq/bgtpndIWbI5hmYxhQGHPC4d4cqBPb4DLniCfhJokdXhwhaDuLBGLQdvvRum/UiX6ECVIPvDXqdg== -graceful-fs@^4.2.3: - version "4.2.3" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.3.tgz#4a12ff1b60376ef09862c2093edd908328be8423" - integrity sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ== +graceful-fs@^4.2.4: + version "4.2.4" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb" + integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== has-flag@^3.0.0: version "3.0.0" From 76a662dcd5ee75e7cfad776326f3011d3a88a99c Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 22:48:17 -0600 Subject: [PATCH 60/95] Minor dep updates --- package.json | 4 ++-- yarn.lock | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 4c772eee..46ab9793 100644 --- a/package.json +++ b/package.json @@ -30,9 +30,9 @@ "fs-extra": "9.0.0", "glob": "7.1.6", "graceful-fs": "^4.2.4", - "hashids": "2.1.0", + "hashids": "2.2.1", "hjson": "^3.2.1", - "iconv-lite": "0.5.0", + "iconv-lite": "0.5.1", "ini-config-parser": "^1.0.4", "inquirer": "^7.0.0", "later": "1.2.0", diff --git a/yarn.lock b/yarn.lock index f0171f7f..ddad5668 100644 --- a/yarn.lock +++ b/yarn.lock @@ -708,20 +708,20 @@ has-values@^1.0.0: is-number "^3.0.0" kind-of "^4.0.0" -hashids@2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/hashids/-/hashids-2.1.0.tgz#96eec6de3081a76dcd839650a6a26e6081e729d3" - integrity sha512-N53K2p7TrwKLNHKHcEDH+qpiAgO9JfyPEg8Tfy4fB9AcVhwxlTanJ55HVV9BQJQ6ajM1Wfmtl2wgKuEbcucolw== +hashids@2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/hashids/-/hashids-2.2.1.tgz#ad0c600f0083aa0df7451dfd184e53db34f71289" + integrity sha512-+hQeKWwpSDiWFeu/3jKUvwboE4Z035gR6FnpscbHPOEEjCbgv2px9/Mlb3O0nOTRyZOw4MMFRYfVL3zctOV6OQ== hjson@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/hjson/-/hjson-3.2.1.tgz#20de41dc87fc9a10d1557d0230b0e02afb1b09ac" integrity sha512-OhhrFMeC7dVuA1xvxuXGTv/yTdhTvbe8hz+3LgVNsfi9+vgz0sF/RrkuX8eegpKaMc9cwYwydImBH6iePoJtdQ== -iconv-lite@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.0.tgz#59cdde0a2a297cc2aeb0c6445a195ee89f127550" - integrity sha512-NnEhI9hIEKHOzJ4f697DMz9IQEXr/MMJ5w64vN2/4Ai+wRnvV7SBrL0KLoRlwaKVghOc7LQ5YkPLuX146b6Ydw== +iconv-lite@0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.5.1.tgz#b2425d3c7b18f7219f2ca663d103bddb91718d64" + integrity sha512-ONHr16SQvKZNSqjQT9gy5z24Jw+uqfO02/ngBSBoqChZ+W8qXX7GPRa1RoUnzGADw8K63R1BXUMzarCVQBpY8Q== dependencies: safer-buffer ">= 2.1.2 < 3" From ec1ee9d79665fe370c2c55b7bbe42e2ae6a207ef Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 22:51:30 -0600 Subject: [PATCH 61/95] Update moment --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 46ab9793..f1d5302e 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "lru-cache": "^5.1.1", "mime-types": "2.1.27", "minimist": "1.2.0", - "moment": "^2.24.0", + "moment": "^2.25.3", "nntp-server": "^1.0.3", "node-pty": "^0.9.0", "nodemailer": "^6.3.1", diff --git a/yarn.lock b/yarn.lock index ddad5668..c957c129 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1092,10 +1092,10 @@ moment@^2.10.6: resolved "https://registry.yarnpkg.com/moment/-/moment-2.22.2.tgz#3c257f9839fc0e93ff53149632239eb90783ff66" integrity sha1-PCV/mDn8DpP/UxSWMiOeuQeD/2Y= -moment@^2.24.0: - version "2.24.0" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b" - integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg== +moment@^2.25.3: + version "2.25.3" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.25.3.tgz#252ff41319cf41e47761a1a88cab30edfe9808c0" + integrity sha512-PuYv0PHxZvzc15Sp8ybUCoQ+xpyPWvjOuK72a5ovzp2LI32rJXOiIfyoFoYvG3s6EwwrdkMyWuRiEHSZRLJNdg== ms@2.0.0: version "2.0.0" From 19c0f75610f7d944ca9e511eab35fe3a47c8ce05 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 23:04:55 -0600 Subject: [PATCH 62/95] binary parser update --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index f1d5302e..d2c5e77f 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ ], "dependencies": { "async": "3.2.0", - "binary-parser": "^1.5.0", + "binary-parser": "^1.6.2", "buffers": "github:NuSkooler/node-buffers", "bunyan": "^1.8.12", "exiftool": "^0.0.3", diff --git a/yarn.lock b/yarn.lock index c957c129..207f207c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -149,10 +149,10 @@ bcrypt-pbkdf@^1.0.2: dependencies: tweetnacl "^0.14.3" -binary-parser@^1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/binary-parser/-/binary-parser-1.5.0.tgz#3e50de3a5076badbacd760e833e7d94892b9e9fa" - integrity sha512-z+hqNSnO7trFDPLihjUGTwlSTbcIzLYSCwnbiasFkRvCIY9F3ZTex7Mlm9UAP3w5mfHD3KxejnWFPJjtsVVMuw== +binary-parser@^1.6.2: + version "1.6.2" + resolved "https://registry.yarnpkg.com/binary-parser/-/binary-parser-1.6.2.tgz#8410a82ffd9403271ec182bd91e63a09cee88cbe" + integrity sha512-cYAhKB51A9T/uylDvMK7uAYaPLWLwlferNOpnQ0E0fuO73yPi7kWaWiOm22BvuKxCbggmkiFN0VkuLg6gc+KQQ== brace-expansion@^1.1.7: version "1.1.11" From ef022d6a009258d12676d93f83533341794e2b68 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 14 May 2020 23:07:30 -0600 Subject: [PATCH 63/95] More minor dep updates --- package.json | 6 +-- yarn.lock | 133 ++++++++++++++++++++++++++++----------------------- 2 files changed, 77 insertions(+), 62 deletions(-) diff --git a/package.json b/package.json index d2c5e77f..0a68e584 100644 --- a/package.json +++ b/package.json @@ -34,16 +34,16 @@ "hjson": "^3.2.1", "iconv-lite": "0.5.1", "ini-config-parser": "^1.0.4", - "inquirer": "^7.0.0", + "inquirer": "^7.1.0", "later": "1.2.0", "lodash": "^4.17.15", "lru-cache": "^5.1.1", "mime-types": "2.1.27", - "minimist": "1.2.0", + "minimist": "1.2.5", "moment": "^2.25.3", "nntp-server": "^1.0.3", "node-pty": "^0.9.0", - "nodemailer": "^6.3.1", + "nodemailer": "^6.4.6", "otplib": "11.0.1", "qrcode-generator": "^1.4.4", "rlogin": "^1.0.0", diff --git a/yarn.lock b/yarn.lock index 207f207c..30d2470a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10,6 +10,11 @@ exec-sh "^0.3.2" minimist "^1.2.0" +"@types/color-name@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@types/color-name/-/color-name-1.1.1.tgz#1c1261bbeaa10a8055bbc5d8ab84b7b2afc846a0" + integrity sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ== + abbrev@1: version "1.1.1" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" @@ -37,12 +42,18 @@ ansi-regex@^4.1.0: resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997" integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== -ansi-styles@^3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" - integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== +ansi-regex@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.0.tgz#388539f55179bf39339c81af30a654d69f87cb75" + integrity sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg== + +ansi-styles@^4.1.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.2.1.tgz#90ae75c424d008d2624c5bf29ead3177ebfcf359" + integrity sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA== dependencies: - color-convert "^1.9.0" + "@types/color-name" "^1.1.1" + color-convert "^2.0.1" anymatch@^2.0.0: version "2.0.0" @@ -226,14 +237,13 @@ capture-exit@^2.0.0: dependencies: rsvp "^4.8.4" -chalk@^2.4.2: - version "2.4.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" - integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== +chalk@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-3.0.0.tgz#3f73c2bf526591f574cc492c51e2456349f844e4" + integrity sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg== dependencies: - ansi-styles "^3.2.1" - escape-string-regexp "^1.0.5" - supports-color "^5.3.0" + ansi-styles "^4.1.0" + supports-color "^7.1.0" chardet@^0.7.0: version "0.7.0" @@ -285,17 +295,17 @@ collection-visit@^1.0.0: map-visit "^1.0.0" object-visit "^1.0.0" -color-convert@^1.9.0: - version "1.9.3" - resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" - integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== dependencies: - color-name "1.1.3" + color-name "~1.1.4" -color-name@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" - integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== component-emitter@^1.2.1: version "1.2.1" @@ -667,10 +677,10 @@ graceful-fs@^4.2.4: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.4.tgz#2256bde14d3632958c465ebc96dc467ca07a29fb" integrity sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw== -has-flag@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" - integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= +has-flag@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== has-unicode@^2.0.0: version "2.0.1" @@ -766,23 +776,23 @@ ini@~1.3.0: resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" integrity sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw== -inquirer@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-7.0.0.tgz#9e2b032dde77da1db5db804758b8fea3a970519a" - integrity sha512-rSdC7zelHdRQFkWnhsMu2+2SO41mpv2oF2zy4tMhmiLWkcKbOAs87fWAJhVXttKVwhdZvymvnuM95EyEXg2/tQ== +inquirer@^7.1.0: + version "7.1.0" + resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-7.1.0.tgz#1298a01859883e17c7264b82870ae1034f92dd29" + integrity sha512-5fJMWEmikSYu0nv/flMc475MhGbB7TSPd/2IpFV4I4rMklboCH2rQjYY5kKiYGHqUF9gvaambupcJFFG9dvReg== dependencies: ansi-escapes "^4.2.1" - chalk "^2.4.2" + chalk "^3.0.0" cli-cursor "^3.1.0" cli-width "^2.0.0" external-editor "^3.0.3" figures "^3.0.0" lodash "^4.17.15" mute-stream "0.0.8" - run-async "^2.2.0" - rxjs "^6.4.0" + run-async "^2.4.0" + rxjs "^6.5.3" string-width "^4.1.0" - strip-ansi "^5.1.0" + strip-ansi "^6.0.0" through "^2.3.6" is-accessor-descriptor@^0.1.6: @@ -898,11 +908,6 @@ is-plain-object@^2.0.1, is-plain-object@^2.0.3, is-plain-object@^2.0.4: dependencies: isobject "^3.0.1" -is-promise@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.1.0.tgz#79a2a9ece7f096e80f36d2b2f3bc16c1ff4bf3fa" - integrity sha1-eaKp7OfwlugPNtKy87wWwf9L8/o= - is-stream@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" @@ -1052,7 +1057,12 @@ minimist@0.0.8: resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" integrity sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0= -minimist@1.2.0, minimist@^1.1.1, minimist@^1.2.0: +minimist@1.2.5: + version "1.2.5" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602" + integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw== + +minimist@^1.1.1, minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" integrity sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ= @@ -1211,10 +1221,10 @@ node-pty@^0.9.0: dependencies: nan "^2.14.0" -nodemailer@^6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.3.1.tgz#2784beebac6b9f014c424c54dbdcc5c4d1221346" - integrity sha512-j0BsSyaMlyadEDEypK/F+xlne2K5m6wzPYMXS/yxKI0s7jmT1kBx6GEKRVbZmyYfKOsjkeC/TiMVDJBI/w5gMQ== +nodemailer@^6.4.6: + version "6.4.6" + resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.4.6.tgz#d37f504f6560b36616f646a606894fe18819107f" + integrity sha512-/kJ+FYVEm2HuUlw87hjSqTss+GU35D4giOpdSfGp7DO+5h6RlJj7R94YaYHOkoxu1CSaM0d3WRBtCzwXrY6MKA== nopt@^4.0.1: version "4.0.1" @@ -1505,17 +1515,15 @@ rsvp@^4.8.4: resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-4.8.5.tgz#c8f155311d167f68f21e168df71ec5b083113734" integrity sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA== -run-async@^2.2.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.3.0.tgz#0371ab4ae0bdd720d4166d7dfda64ff7a445a6c0" - integrity sha1-A3GrSuC91yDUFm19/aZP96RFpsA= - dependencies: - is-promise "^2.1.0" +run-async@^2.4.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.4.1.tgz#8440eccf99ea3e70bd409d49aab88e10c189a455" + integrity sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ== -rxjs@^6.4.0: - version "6.4.0" - resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.4.0.tgz#f3bb0fe7bda7fb69deac0c16f17b50b0b8790504" - integrity sha512-Z9Yfa11F6B9Sg/BK9MnqnQ+aQYicPLtilXBp2yUtDt2JRCE0h26d33EnfO3ZxoNxG0T92OUucP3Ct7cpfkdFfw== +rxjs@^6.5.3: + version "6.5.5" + resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.5.5.tgz#c5c884e3094c8cfee31bf27eb87e54ccfc87f9ec" + integrity sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ== dependencies: tslib "^1.9.0" @@ -1788,13 +1796,20 @@ strip-ansi@^4.0.0: dependencies: ansi-regex "^3.0.0" -strip-ansi@^5.1.0, strip-ansi@^5.2.0: +strip-ansi@^5.2.0: version "5.2.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-5.2.0.tgz#8c9a536feb6afc962bdfa5b104a5091c1ad9c0ae" integrity sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA== dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.0.tgz#0b1571dd7669ccd4f3e06e14ef1eed26225ae532" + integrity sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w== + dependencies: + ansi-regex "^5.0.0" + strip-eof@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf" @@ -1805,12 +1820,12 @@ strip-json-comments@~2.0.1: resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo= -supports-color@^5.3.0: - version "5.5.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" - integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== +supports-color@^7.1.0: + version "7.1.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.1.0.tgz#68e32591df73e25ad1c4b49108a2ec507962bfd1" + integrity sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g== dependencies: - has-flag "^3.0.0" + has-flag "^4.0.0" tar@^4: version "4.4.6" From f23027ba1de78845d61cd7359bf76cf8c0fb7185 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Fri, 15 May 2020 18:45:26 -0600 Subject: [PATCH 64/95] More doc updates, add telnet-bridge.md --- docs/_includes/nav.md | 1 + docs/modding/local-doors.md | 19 ++++--- docs/modding/show-art.md | 4 +- docs/modding/telnet-bridge.md | 96 +++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 9 deletions(-) create mode 100644 docs/modding/telnet-bridge.md diff --git a/docs/_includes/nav.md b/docs/_includes/nav.md index 4fb9a399..c4303fac 100644 --- a/docs/_includes/nav.md +++ b/docs/_includes/nav.md @@ -67,6 +67,7 @@ - BBSLink - Combatnet - Exodus + - [Telnet Bridge]({{ site.baseurl }}{% link modding/telnet-bridge.md %}) - [Existing Mods]({{ site.baseurl }}{% link modding/existing-mods.md %}) - [File Area List]({{ site.baseurl }}{% link modding/file-area-list.md %}) - [Last Callers]({{ site.baseurl }}{% link modding/last-callers.md %}) diff --git a/docs/modding/local-doors.md b/docs/modding/local-doors.md index 4ab8037b..2a4a1338 100644 --- a/docs/modding/local-doors.md +++ b/docs/modding/local-doors.md @@ -3,7 +3,7 @@ layout: page title: Local Doors --- ## Local Doors -ENiGMA½ has many ways to add doors to your system. In addition to the many built in door server modules, local doors are of course also supported using the ! The `abracadabra` module! +ENiGMA½ has many ways to add doors to your system. In addition to the [many built in door server modules](door-servers.md), local doors are of course also supported using the ! The `abracadabra` module! ## The abracadabra Module The `abracadabra` module provides a generic and flexible solution for many door types. Through this module you can execute native processes & scripts directly, and perform I/O through standard I/O (stdio) or a temporary TCP server. @@ -17,7 +17,7 @@ The `abracadabra` `config` block can contain the following members: | `dropFileType` | :+1: | Specifies the type of dropfile to generate (See **Dropfile Types** below). | | `cmd` | :+1: | Path to executable to launch. | | `args` | :-1: | Array of argument(s) to pass to `cmd`. See **Argument Variables** below for information on variables that can be used here. -| `cwd` | :-1: | Sets the Current Working Directory (CWD) for `cmd`. Defaults to the directory of `cmd`. | +| `cwd` | :-1: | Sets the Current Working Directory (CWD) for `cmd`. Defaults to the directory of `cmd`. | | `nodeMax` | :-1: | Max number of nodes that can access this door at once. Uses `name` as a tracking key. | | `tooManyArt` | :-1: | Art spec to display if too many instances are already in use. | | `io` | :-1: | How to process input/output (I/O). Can be `stdio` or `socket`. When using `stdio`, I/O is handled via standard stdin/stdout. When using `socket` a temporary socket server is spawned that can be connected back to. The server listens on localhost on `{srvPort}` (See **Argument Variables** below for more information). Default value is `stdio`. | @@ -99,8 +99,8 @@ doorPimpWars: { cmd: /usr/bin/dosemu args: [ "-quiet", - "-f", - "/path/to/dosemu.conf", + "-f", + "/path/to/dosemu.conf", "X:\\PW\\START.BAT {dropFile} {node}" ], nodeMax: 1 @@ -149,7 +149,7 @@ Please see the [bivrost!](https://github.com/NuSkooler/bivrost) documentation fo Pre-built binaries of bivrost! have been released under [Phenom Productions](https://www.phenomprod.com/) and can be found on various boards. #### Alternative Workarounds -Alternative workarounds include Telnet Bridge (`telnet_bridge` module) to hook up Telnet-accessible (including local) door servers -- It may also be possible bridge via [NET2BBS](http://pcmicro.com/netfoss/guide/net2bbs.html). +Alternative workarounds include [Telnet Bridge module](telnet-bridge.md) to hook up Telnet-accessible (including local) door servers -- It may also be possible bridge via [NET2BBS](http://pcmicro.com/netfoss/guide/net2bbs.html). ### QEMU with abracadabra [QEMU](http://wiki.qemu.org/Main_Page) provides a robust, cross platform solution for launching doors under many platforms (likely anywhere Node.js is supported and ENiGMA½ can run). Note however that there is an important and major caveat: **Multiple instances of a particular door/OS image should not be run at once!** Being more flexible means being a bit more complex. Let's look at an example for running L.O.R.D. under a UNIX like system such as Linux or FreeBSD. @@ -223,8 +223,13 @@ doorLORD: { } ``` +## See Also +* [Telnet Bridge](telnet-bridge.md) +* [Door Servers](door-servers.md) + ## Additional Resources -### DOSBox +### DOS Emulation +* [DOSEMU](http://www.dosemu.org/) * [DOSBox-X](https://github.com/joncampbell123/dosbox-x) ### Door Downloads & Support Sites @@ -233,4 +238,4 @@ doorLORD: { * http://bbstorrents.bbses.info/ #### L.O.R.D. -* http://lord.lordlegacy.com/ \ No newline at end of file +* http://lord.lordlegacy.com/ diff --git a/docs/modding/show-art.md b/docs/modding/show-art.md index c00d7009..5ffce10c 100644 --- a/docs/modding/show-art.md +++ b/docs/modding/show-art.md @@ -29,8 +29,8 @@ showWithExtraArgs: { If the `showWithExtraArgs` menu was entered and passed `extraArgs` as the following: ```json { - fizzBang : true, - fooBaz : "LOLART" + "fizzBang" : true, + "fooBaz" : "LOLART" } ``` diff --git a/docs/modding/telnet-bridge.md b/docs/modding/telnet-bridge.md new file mode 100644 index 00000000..36e1b3e6 --- /dev/null +++ b/docs/modding/telnet-bridge.md @@ -0,0 +1,96 @@ +--- +layout: page +title: Telnet Bridge +--- +## Telnet Bridge +The `telnet_bridge` module allows "bridged" Telnet connections from your board to other Telnet services (such as other BBSes!). + +## Configuration +### Config Block +Available `config` entries: +* `host`: Hostname or IP address to connect to. +* `port`: Port to connect to. Defaults to the standard Telnet port of `23`. +* `font`: A SyncTERM style font. Useful for example if you would like to connect form a "DOS" style BBS to an Amiga. See [the general art documentation on SyncTERM Style Fonts](/docs/art/general.md). + +### Example +Below is an example `menu.hjson` entry that would connect to [Xibalba](https://xibalba.l33t.codes): + +```hjson +{ + telnetBridgeXibalba: { + desc: Xibalba BBS + module: telnet_bridge + config: { + host: xibalba.l33t.codes + port: 45510 + } + } +} +``` + +### Using Extra Args +The `telnet_bridge` module can also accept standard `extraArgs` of the same configuration arguments described above. This can be illustrated with an example: + +```hjson +telnetBridgeMenu: { + desc: Telnet Bridge + art: telnet_bridge + config: { + font: cp437 + } + form: { + 0: { + mci: { + VM1: { + argName: selection + + items: [ + { + board: BLACK Flag + soft: Mystic + data: bf + } + { + board: Xibalba + soft: ENiGMA½ + data: xib + } + ] + + // sort by 'board' fields above + sort: board + submit: true + } + } + + submit: { + *: [ + { + value: { "selection" : "bf" } + action: @menu:telnetBridgeFromExtraFlags + extraArgs: { + host: blackflag.acid.org + } + } + { + value: { "selection" : "xib" } + action: @menu:telnetBridgeFromExtraFlags + extraArgs: { + host: xibalba.l33t.codes + port: 44510 + } + } + ] + } + } + } +} + +telnetBridgeFromExtraFlags: { + desc: Telnet Bridge + module: telnet_bridge +} +``` + +Here we've created a lightbar menu with custom items in which we'd use `itemFormat`'s with in a theme. When the user selects an item, the `telnetBridgeFromExtraFlags` menu is instantiated using the supplied `extraArgs`. + From 02fa3b014b28fbc4cfa7b04c13d9c36d28f2a9f2 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 16 May 2020 17:37:51 -0600 Subject: [PATCH 65/95] ANSI FILE_ID.DIZ sometimes do not display correctly #244 * When parsing ANSI files, handle UNIX style LF's only vs "standard" DOS CRLF pairs --- core/ansi_escape_parser.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/ansi_escape_parser.js b/core/ansi_escape_parser.js index 9786ea4c..88063bef 100644 --- a/core/ansi_escape_parser.js +++ b/core/ansi_escape_parser.js @@ -87,6 +87,7 @@ function ANSIEscapeParser(options) { let pos = 0; let start = 0; let charCode; + let lastCharCode; while(pos < len) { charCode = text.charCodeAt(pos) & 0xff; // 8bit clean @@ -102,6 +103,12 @@ function ANSIEscapeParser(options) { break; case LF : + // Handle ANSI saved with UNIX-style LF's only + // vs the CRLF pairs + if (lastCharCode !== CR) { + self.column = 1; + } + self.emit('literal', text.slice(start, pos)); start = pos; @@ -126,6 +133,7 @@ function ANSIEscapeParser(options) { } ++pos; + lastCharCode = charCode; } // From ad2382c1f28a354b92a0a4d55683d660ca198b56 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 17 May 2020 19:42:05 -0600 Subject: [PATCH 66/95] Add a couple items to whatsnew --- WHATSNEW.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/WHATSNEW.md b/WHATSNEW.md index 9736d82f..bf6ac243 100644 --- a/WHATSNEW.md +++ b/WHATSNEW.md @@ -4,6 +4,9 @@ This document attempts to track **major** changes and additions in ENiGMA½. For ## 0.0.11-beta * Upgraded from `alpha` to `beta` -- The software is far along and mature enough at this point! * Development is now against Node.js 12.x LTS. Other versions may work but are not currently supported! +* [QWK support](/docs/messageareas/qwk.md) +* `oputil fb scan *areaTagWildcard*` scans all areas in which wildcard is matched. + ## 0.0.10-alpha + `oputil.js user rename USERNAME NEWNAME` From 75787b61079c4cfe91498d415f4d4d8d89565843 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 17 May 2020 20:22:16 -0600 Subject: [PATCH 67/95] Start work on new telnet server --- core/config.js | 5 ++++ core/servers/login/telnet2.js | 43 +++++++++++++++++++++++++++++++++++ package.json | 3 ++- yarn.lock | 9 +++++++- 4 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 core/servers/login/telnet2.js diff --git a/core/config.js b/core/config.js index 66ffab14..76ee98f5 100644 --- a/core/config.js +++ b/core/config.js @@ -285,6 +285,11 @@ function getDefaultConfig() { }, loginServers : { + telnet2: { + port: 8810, + enabled : true, + firstMenu: 'telnetConnected', + }, telnet : { port : 8888, enabled : true, diff --git a/core/servers/login/telnet2.js b/core/servers/login/telnet2.js new file mode 100644 index 00000000..a19fe675 --- /dev/null +++ b/core/servers/login/telnet2.js @@ -0,0 +1,43 @@ +// ENiGMA½ +const LoginServerModule = require('../../login_server_module'); +const Client = require('../../client'); + +// deps +const net = require('net'); +const { TelnetSocket, TelnetSpec } = require('telnet-socket'); + +const ModuleInfo = exports.moduleInfo = { + name : 'Telnet', + desc : 'Telnet Server', + author : 'NuSkooler', + isSecure : false, + packageName : 'codes.l33t.enigma.telnet.server.v2', +}; + + + +class TelnetClient extends Client { + constructor(socket) { + super(); + + this.setInputOutput(socket, socket); + this.telnetSocket = new TelnetSocket(socket); + + // :TODO: banner + } +}; + +exports.getModule = class TelnetServerModule extends LoginServerModule { + constructor() { + super(); + } + + createServer(cb) { + this.server = net.createServer( socket => { + const client = new TelnetClient(socket); + this.handleNewClient(client, socket, ModuleInfo); + }); + + return cb(null); + } +}; diff --git a/package.json b/package.json index 0a68e584..6bb5a613 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,8 @@ "uuid-parse": "1.1.0", "ws": "^7.3.0", "xxhash": "^0.3.0", - "yazl": "^2.5.1" + "yazl": "^2.5.1", + "telnet-socket" : "github:NuSkooler/telnet-socket" }, "devDependencies": {}, "engines": { diff --git a/yarn.lock b/yarn.lock index 30d2470a..5dd80c0e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -160,7 +160,7 @@ bcrypt-pbkdf@^1.0.2: dependencies: tweetnacl "^0.14.3" -binary-parser@^1.6.2: +binary-parser@^1.5.0, binary-parser@^1.6.2: version "1.6.2" resolved "https://registry.yarnpkg.com/binary-parser/-/binary-parser-1.6.2.tgz#8410a82ffd9403271ec182bd91e63a09cee88cbe" integrity sha512-cYAhKB51A9T/uylDvMK7uAYaPLWLwlferNOpnQ0E0fuO73yPi7kWaWiOm22BvuKxCbggmkiFN0VkuLg6gc+KQQ== @@ -1840,6 +1840,13 @@ tar@^4: safe-buffer "^5.1.2" yallist "^3.0.2" +"telnet-socket@github:NuSkooler/telnet-socket": + version "0.1.0" + resolved "https://codeload.github.com/NuSkooler/telnet-socket/tar.gz/a29ea2edaec98bb844f2b0d27fa0977f33eaa7bf" + dependencies: + binary-parser "^1.5.0" + buffers "github:NuSkooler/node-buffers" + temptmp@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/temptmp/-/temptmp-1.1.0.tgz#bfbbff858d7f7d59c563fbf069758a7775ecd431" From 1018485e8e55e5be284502080f27698c02cb0207 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 17 May 2020 21:36:48 -0600 Subject: [PATCH 68/95] WIP --- core/config.js | 5 -- core/servers/login/telnet2.js | 126 ++++++++++++++++++++++++++++++++-- 2 files changed, 120 insertions(+), 11 deletions(-) diff --git a/core/config.js b/core/config.js index 76ee98f5..66ffab14 100644 --- a/core/config.js +++ b/core/config.js @@ -285,11 +285,6 @@ function getDefaultConfig() { }, loginServers : { - telnet2: { - port: 8810, - enabled : true, - firstMenu: 'telnetConnected', - }, telnet : { port : 8888, enabled : true, diff --git a/core/servers/login/telnet2.js b/core/servers/login/telnet2.js index a19fe675..3127dfb8 100644 --- a/core/servers/login/telnet2.js +++ b/core/servers/login/telnet2.js @@ -1,10 +1,16 @@ // ENiGMA½ const LoginServerModule = require('../../login_server_module'); -const Client = require('../../client'); +const { Client } = require('../../client'); +const Config = require('../../config').get; +const { log: Log } = require('../../logger'); // deps const net = require('net'); -const { TelnetSocket, TelnetSpec } = require('telnet-socket'); +const { + TelnetSocket, + TelnetSpec: { Options, Commands } +} = require('telnet-socket'); +const { inherits } = require('util'); const ModuleInfo = exports.moduleInfo = { name : 'Telnet', @@ -16,17 +22,104 @@ const ModuleInfo = exports.moduleInfo = { -class TelnetClient extends Client { +class TelnetClient { constructor(socket) { - super(); + Client.apply(this, socket, socket); this.setInputOutput(socket, socket); - this.telnetSocket = new TelnetSocket(socket); + this.socket = new TelnetSocket(socket); - // :TODO: banner + // + // Wait up to 3s to hear about from our terminal type request + // then go ahead and move on... + // + setTimeout(() => { + this.clientReady(); + }, 3000); + + this.socket.on('WILL', command => { + switch (command.option) { + case Options.LINEMODE : + return this.socket.dont.linemode(); + + case Options.TTYPE : + return this.socket.sb.send.ttype(); + + case Options.NEW_ENVIRON : + return this.socket.sb.send.new_environ(); + + default : + break; + } + }); + + this.socket.on('SB', command => { + switch (command.option) { + case Options.TTYPE : + this.setTermType(command.optionData.ttype); + this.clientReady(); + break; + + case Options.NEW_ENVIRON : + { + if ('unknown' === this.term.termType) { + const term = + command.optionData.vars.find(nv => nv.TERM) || + command.optionData.userVars.find(nv => nv.TERM); + if (term) { + this.setTermType(term); + } + } + + command.optionData.vars.forEach(nv => { + console.log(nv); + }); + } + break; + + case Options.NAWS : + this.term.termWidth = command.optionData.width; + this.term.termHeight = command.optionData.height; + + // :TODO: update env, see old telnet.js + + this.clearMciCache(); + + // :TODO: Log negotiation + break; + } + }); + + this.banner(); + } + + clientReady() { + if (this.clientReadyHandled) { + return; // already processed + } + + this.clientReadyHandled = true; + this.emit('ready', { firstMenu : Config().loginServers.telnet.firstMenu } ); + } + + banner() { + this.socket.do.echo(); + this.socket.will.echo(); // we'll echo back + + this.socket.will.sga(); + this.socket.do.sga(); + + this.socket.do.transmit_binary(); + this.socket.will.transmit_binary(); + + this.socket.do.ttype(); + this.socket.do.naws(); + this.socket.do.new_environ(); } }; +inherits(TelnetClient, Client); + exports.getModule = class TelnetServerModule extends LoginServerModule { constructor() { super(); @@ -38,6 +131,27 @@ exports.getModule = class TelnetServerModule extends LoginServerModule { this.handleNewClient(client, socket, ModuleInfo); }); + this.server.on('error', err => { + Log.info( { error : err.message }, 'Telnet server error'); + }); + return cb(null); } + + listen(cb) { + const config = Config(); + //const port = parseInt(config.loginServers.telnet.port); + const port = 8810; // :TODO: Put me back ;) + if(isNaN(port)) { + Log.error( { server : ModuleInfo.name, port : config.loginServers.telnet.port }, 'Cannot load server (invalid port)' ); + return cb(Errors.Invalid(`Invalid port: ${config.loginServers.telnet.port}`)); + } + + this.server.listen(port, config.loginServers.telnet.address, err => { + if(!err) { + Log.info( { server : ModuleInfo.name, port : port }, 'Listening for connections' ); + } + return cb(err); + }); + } }; From e517e31b9405aa099b89617e6a1666e99734d18c Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 17 May 2020 22:35:03 -0600 Subject: [PATCH 69/95] Working fairly well --- core/servers/login/telnet2.js | 72 +++++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/core/servers/login/telnet2.js b/core/servers/login/telnet2.js index 3127dfb8..3e38d988 100644 --- a/core/servers/login/telnet2.js +++ b/core/servers/login/telnet2.js @@ -20,8 +20,6 @@ const ModuleInfo = exports.moduleInfo = { packageName : 'codes.l33t.enigma.telnet.server.v2', }; - - class TelnetClient { constructor(socket) { Client.apply(this, socket, socket); @@ -37,6 +35,33 @@ class TelnetClient { this.clientReady(); }, 3000); + this.socket.on('data', data => { + this.emit('data', data); + }); + + this.socket.on('error', err => { + // :TODO: Log me + return this.emit('end'); + }); + + this.socket.on('end', () => { + this.emit('end'); + }); + + this.socket.on('DO', command => { + switch (command.option) { + case Options.ECHO : + return this.socket.will.echo(); + + default : + return this.socket.command(Commands.WONT, command.option); + } + }); + + this.socket.on('DONT', command => { + // :TODO: Log me + }); + this.socket.on('WILL', command => { switch (command.option) { case Options.LINEMODE : @@ -53,6 +78,10 @@ class TelnetClient { } }); + this.socket.on('WONT', command => { + // :TODO: see telnet.js handling + }); + this.socket.on('SB', command => { switch (command.option) { case Options.TTYPE : @@ -78,18 +107,39 @@ class TelnetClient { break; case Options.NAWS : - this.term.termWidth = command.optionData.width; - this.term.termHeight = command.optionData.height; + { + const { width, height } = command.optionData; - // :TODO: update env, see old telnet.js + this.term.termWidth = width; + this.term.termHeight = height; - this.clearMciCache(); + if (width) { + this.term.env.COLUMNS = width; + } - // :TODO: Log negotiation + if (height) { + this.term.env.ROWS = height; + } + + this.clearMciCache(); + + // :TODO: Log negotiation + } break; } }); + this.socket.on('IP', command => { + // :TODO: Log me + return this.disconnect(); + }); + + this.socket.on('AYT', () => { + // :TODO: Log me + return this.socket.write('\b'); + }); + + // kick off negotiations this.banner(); } @@ -102,6 +152,14 @@ class TelnetClient { this.emit('ready', { firstMenu : Config().loginServers.telnet.firstMenu } ); } + disconnect() { + try { + return this.socket.rawSocket.end(); + } catch (e) { + // ignored + } + } + banner() { this.socket.do.echo(); this.socket.will.echo(); // we'll echo back From a1ac6dfc67e12fdbca7376652032c3be9fa38e4b Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 18 May 2020 00:33:14 -0600 Subject: [PATCH 70/95] Shim in dataHandler for toggling from elsewhere --- core/servers/login/telnet2.js | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/core/servers/login/telnet2.js b/core/servers/login/telnet2.js index 3e38d988..7f63659e 100644 --- a/core/servers/login/telnet2.js +++ b/core/servers/login/telnet2.js @@ -35,9 +35,11 @@ class TelnetClient { this.clientReady(); }, 3000); - this.socket.on('data', data => { + this.dataHandler = function(data) { this.emit('data', data); - }); + }.bind(this); + + this.socket.on('data', this.dataHandler); this.socket.on('error', err => { // :TODO: Log me @@ -79,7 +81,14 @@ class TelnetClient { }); this.socket.on('WONT', command => { - // :TODO: see telnet.js handling + switch (command.option) { + case Options.NEW_ENVIRON : + return this.socket.dont.new_environ(); + + default : + // :TODO: Log me + break; + } }); this.socket.on('SB', command => { @@ -143,6 +152,10 @@ class TelnetClient { this.banner(); } + // dataHandler(data) { + // this.emit('data', data); + // } + clientReady() { if (this.clientReadyHandled) { return; // already processed From 6d307ec06ba7c589fb175764677c1e911a87ddc3 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 18 May 2020 18:41:23 -0600 Subject: [PATCH 71/95] Clean up a bit. Still more to do... --- core/servers/login/telnet.js | 1028 ++++++------------------------- core/servers/login/telnet2.js | 228 ------- core/servers/login/websocket.js | 2 +- 3 files changed, 204 insertions(+), 1054 deletions(-) delete mode 100644 core/servers/login/telnet2.js diff --git a/core/servers/login/telnet.js b/core/servers/login/telnet.js index 20ec6cd9..111167fb 100644 --- a/core/servers/login/telnet.js +++ b/core/servers/login/telnet.js @@ -1,841 +1,232 @@ -/* jslint node: true */ -'use strict'; - // ENiGMA½ -const baseClient = require('../../client.js'); -const Log = require('../../logger.js').log; -const LoginServerModule = require('../../login_server_module.js'); -const Config = require('../../config.js').get; -const EnigAssert = require('../../enigma_assert.js'); -const { stringFromNullTermBuffer } = require('../../string_util.js'); -const { Errors } = require('../../enig_error.js'); +const LoginServerModule = require('../../login_server_module'); +const { Client } = require('../../client'); +const Config = require('../../config').get; +const { log: Log } = require('../../logger'); // deps -const net = require('net'); -const buffers = require('buffers'); -const { Parser } = require('binary-parser'); -const util = require('util'); +const net = require('net'); +const { + TelnetSocket, + TelnetSpec: { Options, Commands } +} = require('telnet-socket'); +const { inherits } = require('util'); const ModuleInfo = exports.moduleInfo = { name : 'Telnet', - desc : 'Telnet Server', + desc : 'Telnet Server v2', author : 'NuSkooler', isSecure : false, - packageName : 'codes.l33t.enigma.telnet.server', + packageName : 'codes.l33t.enigma.telnet.server.v2', }; -exports.TelnetClient = TelnetClient; +class TelnetClient { + constructor(socket) { + Client.apply(this, socket, socket); -// -// Telnet Protocol Resources -// * http://pcmicro.com/netfoss/telnet.html -// * http://mud-dev.wikidot.com/telnet:negotiation -// + this.setInputOutput(socket, socket); + this.socket = new TelnetSocket(socket); -/* - TODO: - * Various (much lesser used) Telnet command coverage -*/ - -const COMMANDS = { - SE : 240, // End of Sub-Negotation Parameters - NOP : 241, // No Operation - DM : 242, // Data Mark - BRK : 243, // Break - IP : 244, // Interrupt Process - AO : 245, // Abort Output - AYT : 246, // Are You There? - EC : 247, // Erase Character - EL : 248, // Erase Line - GA : 249, // Go Ahead - SB : 250, // Start Sub-Negotiation Parameters - WILL : 251, // - WONT : 252, - DO : 253, - DONT : 254, - IAC : 255, // (Data Byte) -}; - -// -// Resources: -// * http://www.faqs.org/rfcs/rfc1572.html -// -const SB_COMMANDS = { - IS : 0, - SEND : 1, - INFO : 2, -}; - -// -// Telnet Options -// -// Resources -// * http://mars.netanya.ac.il/~unesco/cdrom/booklet/HTML/NETWORKING/node300.html -// * http://www.networksorcery.com/enp/protocol/telnet.htm -// -const OPTIONS = { - TRANSMIT_BINARY : 0, // http://tools.ietf.org/html/rfc856 - ECHO : 1, // http://tools.ietf.org/html/rfc857 - // RECONNECTION : 2 - SUPPRESS_GO_AHEAD : 3, // aka 'SGA': RFC 858 @ http://tools.ietf.org/html/rfc858 - //APPROX_MESSAGE_SIZE : 4 - STATUS : 5, // http://tools.ietf.org/html/rfc859 - TIMING_MARK : 6, // http://tools.ietf.org/html/rfc860 - //RC_TRANS_AND_ECHO : 7, // aka 'RCTE' @ http://www.rfc-base.org/txt/rfc-726.txt - //OUPUT_LINE_WIDTH : 8, - //OUTPUT_PAGE_SIZE : 9, // - //OUTPUT_CARRIAGE_RETURN_DISP : 10, // RFC 652 - //OUTPUT_HORIZ_TABSTOPS : 11, // RFC 653 - //OUTPUT_HORIZ_TAB_DISP : 12, // RFC 654 - //OUTPUT_FORMFEED_DISP : 13, // RFC 655 - //OUTPUT_VERT_TABSTOPS : 14, // RFC 656 - //OUTPUT_VERT_TAB_DISP : 15, // RFC 657 - //OUTPUT_LF_DISP : 16, // RFC 658 - //EXTENDED_ASCII : 17, // RFC 659 - //LOGOUT : 18, // RFC 727 - //BYTE_MACRO : 19, // RFC 753 - //DATA_ENTRY_TERMINAL : 20, // RFC 1043 - //SUPDUP : 21, // RFC 736 - //SUPDUP_OUTPUT : 22, // RFC 749 - SEND_LOCATION : 23, // RFC 779 - TERMINAL_TYPE : 24, // aka 'TTYPE': RFC 1091 @ http://tools.ietf.org/html/rfc1091 - //END_OF_RECORD : 25, // RFC 885 - //TACACS_USER_ID : 26, // RFC 927 - //OUTPUT_MARKING : 27, // RFC 933 - //TERMINCAL_LOCATION_NUMBER : 28, // RFC 946 - //TELNET_3270_REGIME : 29, // RFC 1041 - WINDOW_SIZE : 31, // aka 'NAWS': RFC 1073 @ http://tools.ietf.org/html/rfc1073 - TERMINAL_SPEED : 32, // RFC 1079 @ http://tools.ietf.org/html/rfc1079 - REMOTE_FLOW_CONTROL : 33, // RFC 1072 @ http://tools.ietf.org/html/rfc1372 - LINEMODE : 34, // RFC 1184 @ http://tools.ietf.org/html/rfc1184 - X_DISPLAY_LOCATION : 35, // aka 'XDISPLOC': RFC 1096 @ http://tools.ietf.org/html/rfc1096 - NEW_ENVIRONMENT_DEP : 36, // aka 'NEW-ENVIRON': RFC 1408 @ http://tools.ietf.org/html/rfc1408 (note: RFC 1572 is an update to this) - AUTHENTICATION : 37, // RFC 2941 @ http://tools.ietf.org/html/rfc2941 - ENCRYPT : 38, // RFC 2946 @ http://tools.ietf.org/html/rfc2946 - NEW_ENVIRONMENT : 39, // aka 'NEW-ENVIRON': RFC 1572 @ http://tools.ietf.org/html/rfc1572 (note: update to RFC 1408) - //TN3270E : 40, // RFC 2355 - //XAUTH : 41, - //CHARSET : 42, // RFC 2066 - //REMOTE_SERIAL_PORT : 43, - //COM_PORT_CONTROL : 44, // RFC 2217 - //SUPRESS_LOCAL_ECHO : 45, - //START_TLS : 46, - //KERMIT : 47, // RFC 2840 - //SEND_URL : 48, - //FORWARD_X : 49, - - //PRAGMA_LOGON : 138, - //SSPI_LOGON : 139, - //PRAGMA_HEARTBEAT : 140 - - ARE_YOU_THERE : 246, // aka 'AYT' RFC 854 @ https://tools.ietf.org/html/rfc854 - - EXTENDED_OPTIONS_LIST : 255, // RFC 861 (STD 32) -}; - -// Commands used within NEW_ENVIRONMENT[_DEP] -const NEW_ENVIRONMENT_COMMANDS = { - VAR : 0, - VALUE : 1, - ESC : 2, - USERVAR : 3, -}; - -const IAC_BUF = Buffer.from([ COMMANDS.IAC ]); -const IAC_SE_BUF = Buffer.from([ COMMANDS.IAC, COMMANDS.SE ]); - -const COMMAND_NAMES = Object.keys(COMMANDS).reduce(function(names, name) { - names[COMMANDS[name]] = name.toLowerCase(); - return names; -}, {}); - -const COMMAND_IMPLS = {}; -[ 'do', 'dont', 'will', 'wont', 'sb' ].forEach(function(command) { - const code = COMMANDS[command.toUpperCase()]; - COMMAND_IMPLS[code] = function(bufs, i, event) { - if(bufs.length < (i + 1)) { - return MORE_DATA_REQUIRED; - } - return parseOption(bufs, i, event); - }; -}); - -// :TODO: See TooTallNate's telnet.js: Handle COMMAND_IMPL for IAC in binary mode - -// Create option names such as 'transmit binary' -> OPTIONS.TRANSMIT_BINARY -const OPTION_NAMES = Object.keys(OPTIONS).reduce(function(names, name) { - names[OPTIONS[name]] = name.toLowerCase().replace(/_/g, ' '); - return names; -}, {}); - -function unknownOption(bufs, i, event) { - Log.warn( { bufs : bufs, i : i, event : event }, 'Unknown Telnet option'); - event.buf = bufs.splice(0, i).toBuffer(); - return event; -} - -const OPTION_IMPLS = {}; -// :TODO: fill in the rest... -OPTION_IMPLS.NO_ARGS = -OPTION_IMPLS[OPTIONS.ECHO] = -OPTION_IMPLS[OPTIONS.STATUS] = -OPTION_IMPLS[OPTIONS.LINEMODE] = -OPTION_IMPLS[OPTIONS.TRANSMIT_BINARY] = -OPTION_IMPLS[OPTIONS.AUTHENTICATION] = -OPTION_IMPLS[OPTIONS.TERMINAL_SPEED] = -OPTION_IMPLS[OPTIONS.REMOTE_FLOW_CONTROL] = -OPTION_IMPLS[OPTIONS.X_DISPLAY_LOCATION] = -OPTION_IMPLS[OPTIONS.SEND_LOCATION] = -OPTION_IMPLS[OPTIONS.ARE_YOU_THERE] = -OPTION_IMPLS[OPTIONS.SUPPRESS_GO_AHEAD] = function(bufs, i, event) { - event.buf = bufs.splice(0, i).toBuffer(); - return event; -}; - -const TermTypeCmdParser = new Parser() - .uint8('iac1') - .uint8('sb') - .uint8('opt') - .uint8('is') - .array('ttype', { - type : 'uint8', - readUntil : b => 255 === b, // 255=COMMANDS.IAC - }) - // note we read iac2 above - .uint8('se'); - -OPTION_IMPLS[OPTIONS.TERMINAL_TYPE] = function(bufs, i, event) { - if(event.commandCode !== COMMANDS.SB) { - OPTION_IMPLS.NO_ARGS(bufs, i, event); - } else { - // We need 4 bytes header + data + IAC SE - if(bufs.length < 7) { - return MORE_DATA_REQUIRED; - } - - const end = bufs.indexOf(IAC_SE_BUF, 5); // look past header bytes - if(-1 === end) { - return MORE_DATA_REQUIRED; - } - - let ttypeCmd; - try { - ttypeCmd = TermTypeCmdParser.parse(bufs.toBuffer()); - } catch(e) { - Log.debug( { error : e }, 'Failed parsing TTYP telnet command'); - return event; - } - - EnigAssert(COMMANDS.IAC === ttypeCmd.iac1); - EnigAssert(COMMANDS.SB === ttypeCmd.sb); - EnigAssert(OPTIONS.TERMINAL_TYPE === ttypeCmd.opt); - EnigAssert(SB_COMMANDS.IS === ttypeCmd.is); - EnigAssert(ttypeCmd.ttype.length > 0); - // note we found IAC_SE above - - // some terminals such as NetRunner provide a NULL-terminated buffer - // slice to remove IAC - event.ttype = stringFromNullTermBuffer(ttypeCmd.ttype.slice(0, -1), 'ascii'); - - bufs.splice(0, end); - } - - return event; -}; - -const NawsCmdParser = new Parser() - .uint8('iac1') - .uint8('sb') - .uint8('opt') - .uint16be('width') - .uint16be('height') - .uint8('iac2') - .uint8('se'); - -OPTION_IMPLS[OPTIONS.WINDOW_SIZE] = function(bufs, i, event) { - if(event.commandCode !== COMMANDS.SB) { - OPTION_IMPLS.NO_ARGS(bufs, i, event); - } else { - // we need 9 bytes - if(bufs.length < 9) { - return MORE_DATA_REQUIRED; - } - - let nawsCmd; - try { - nawsCmd = NawsCmdParser.parse(bufs.splice(0, 9).toBuffer()); - } catch(e) { - Log.debug( { error : e }, 'Failed parsing NAWS telnet command'); - return event; - } - - EnigAssert(COMMANDS.IAC === nawsCmd.iac1); - EnigAssert(COMMANDS.SB === nawsCmd.sb); - EnigAssert(OPTIONS.WINDOW_SIZE === nawsCmd.opt); - EnigAssert(COMMANDS.IAC === nawsCmd.iac2); - EnigAssert(COMMANDS.SE === nawsCmd.se); - - event.cols = event.columns = event.width = nawsCmd.width; - event.rows = event.height = nawsCmd.height; - } - return event; -}; - -// Build an array of delimiters for parsing NEW_ENVIRONMENT[_DEP] -//const NEW_ENVIRONMENT_DELIMITERS = _.values(NEW_ENVIRONMENT_COMMANDS); - -const EnvCmdParser = new Parser() - .uint8('iac1') - .uint8('sb') - .uint8('opt') - .uint8('isOrInfo') // IS=initial, INFO=updates - .array('envBlock', { - type : 'uint8', - readUntil : b => 255 === b, // 255=COMMANDS.IAC - }) - // note we consume IAC above - .uint8('se'); - -// Handle the deprecated RFC 1408 & the updated RFC 1572: -OPTION_IMPLS[OPTIONS.NEW_ENVIRONMENT_DEP] = -OPTION_IMPLS[OPTIONS.NEW_ENVIRONMENT] = function(bufs, i, event) { - if(event.commandCode !== COMMANDS.SB) { - OPTION_IMPLS.NO_ARGS(bufs, i, event); - } else { // - // We need 4 bytes header + + IAC SE - // Many terminals send a empty list: - // IAC SB NEW-ENVIRON IS IAC SE + // Wait up to 3s to hear about from our terminal type request + // then go ahead and move on... // - if(bufs.length < 6) { - return MORE_DATA_REQUIRED; - } + setTimeout(() => { + this._clientReady(); + }, 3000); - let end = bufs.indexOf(IAC_SE_BUF, 4); // look past header bytes - if(-1 === end) { - return MORE_DATA_REQUIRED; - } + this.dataHandler = function(data) { + this.emit('data', data); + }.bind(this); - // :TODO: It's likely that we could do all the env name/value parsing directly in Parser. + this.socket.on('data', this.dataHandler); - let envCmd; - try { - envCmd = EnvCmdParser.parse(bufs.splice(0, bufs.length).toBuffer()); - } catch(e) { - Log.debug( { error : e }, 'Failed parsing NEW-ENVIRON telnet command'); - return event; - } + this.socket.on('error', err => { + this._logDebug( { error : err.message }, 'Socket error'); + return this.emit('end'); + }); - EnigAssert(COMMANDS.IAC === envCmd.iac1); - EnigAssert(COMMANDS.SB === envCmd.sb); - EnigAssert(OPTIONS.NEW_ENVIRONMENT === envCmd.opt || OPTIONS.NEW_ENVIRONMENT_DEP === envCmd.opt); - EnigAssert(SB_COMMANDS.IS === envCmd.isOrInfo || SB_COMMANDS.INFO === envCmd.isOrInfo); + this.socket.on('end', () => { + this.emit('end'); + }); - if(OPTIONS.NEW_ENVIRONMENT_DEP === envCmd.opt) { - // :TODO: we should probably support this for legacy clients? - Log.warn('Handling deprecated RFC 1408 NEW-ENVIRON'); - } + this.socket.on('DO', command => { + switch (command.option) { + case Options.ECHO : + return this.socket.will.echo(); - const envBuf = envCmd.envBlock.slice(0, -1); // remove IAC - - if(envBuf.length < 4) { // TYPE + single char name + sep + single char value - // empty env block - return event; - } - - const States = { - Name : 1, - Value : 2, - }; - - let state = States.Name; - const setVars = {}; - const delVars = []; - let varName; - // :TODO: handle ESC type!!! - while(envBuf.length) { - switch(state) { - case States.Name : - { - const type = parseInt(envBuf.splice(0, 1)); - if(![ NEW_ENVIRONMENT_COMMANDS.VAR, NEW_ENVIRONMENT_COMMANDS.USERVAR, NEW_ENVIRONMENT_COMMANDS.ESC ].includes(type)) { - return event; // fail :( - } - - let nameEnd = envBuf.indexOf(NEW_ENVIRONMENT_COMMANDS.VALUE); - if(-1 === nameEnd) { - nameEnd = envBuf.length; - } - - varName = envBuf.splice(0, nameEnd); - if(!varName) { - return event; // something is wrong. - } - - varName = Buffer.from(varName).toString('ascii'); - - const next = parseInt(envBuf.splice(0, 1)); - if(NEW_ENVIRONMENT_COMMANDS.VALUE === next) { - state = States.Value; - } else { - state = States.Name; - delVars.push(varName); // no value; del this var - } - } - break; - - case States.Value : - { - let valueEnd = envBuf.indexOf(NEW_ENVIRONMENT_COMMANDS.VAR); - if(-1 === valueEnd) { - valueEnd = envBuf.indexOf(NEW_ENVIRONMENT_COMMANDS.USERVAR); - } - if(-1 === valueEnd) { - valueEnd = envBuf.length; - } - - let value = envBuf.splice(0, valueEnd); - if(value) { - value = Buffer.from(value).toString('ascii'); - setVars[varName] = value; - } - state = States.Name; - } - break; - } - } - - // :TODO: Handle deleting previously set vars via delVars - event.type = envCmd.isOrInfo; - event.envVars = setVars; - } - - return event; -}; - -const MORE_DATA_REQUIRED = 0xfeedface; - -function parseBufs(bufs) { - EnigAssert(bufs.length >= 2); - EnigAssert(bufs.get(0) === COMMANDS.IAC); - return parseCommand(bufs, 1, {}); -} - -function parseCommand(bufs, i, event) { - const command = bufs.get(i); // :TODO: fix deprecation... [i] is not the same - event.commandCode = command; - event.command = COMMAND_NAMES[command]; - - const handler = COMMAND_IMPLS[command]; - if(handler) { - return handler(bufs, i + 1, event); - } else { - if(2 !== bufs.length) { - Log.warn( { bufsLength : bufs.length }, 'Expected bufs length of 2'); // expected: IAC + COMMAND - } - - event.buf = bufs.splice(0, 2).toBuffer(); - return event; - } -} - -function parseOption(bufs, i, event) { - const option = bufs.get(i); // :TODO: fix deprecation... [i] is not the same - event.optionCode = option; - event.option = OPTION_NAMES[option]; - - const handler = OPTION_IMPLS[option]; - return handler ? handler(bufs, i + 1, event) : unknownOption(bufs, i + 1, event); -} - - -function TelnetClient(input, output) { - baseClient.Client.apply(this, arguments); - - const self = this; - - let bufs = buffers(); - this.bufs = bufs; - - this.sentDont = {}; // DON'T's we've already sent - - this.setInputOutput(input, output); - - this.negotiationsComplete = false; // are we in the 'negotiation' phase? - this.didReady = false; // have we emit the 'ready' event? - - this.subNegotiationState = { - newEnvironRequested : false, - }; - - this.dataHandler = function(b) { - if(!Buffer.isBuffer(b)) { - EnigAssert(false, `Cannot push non-buffer ${typeof b}`); - return; - } - - bufs.push(b); - - let i; - while((i = bufs.indexOf(IAC_BUF)) >= 0) { - - // - // Some clients will send even IAC separate from data - // - if(bufs.length <= (i + 1)) { - i = MORE_DATA_REQUIRED; - break; - } - - EnigAssert(bufs.length > (i + 1)); - - if(i > 0) { - self.emit('data', bufs.splice(0, i).toBuffer()); - } - - i = parseBufs(bufs); - - if(MORE_DATA_REQUIRED === i) { - break; - } else if(i) { - if(i.option) { - self.emit(i.option, i); // "transmit binary", "echo", ... - } - - self.handleTelnetEvent(i); - - if(i.data) { - self.emit('data', i.data); - } - } - } - - if(MORE_DATA_REQUIRED !== i && bufs.length > 0) { - // - // Standard data payload. This can still be "non-user" data - // such as ANSI control, but we don't handle that here. - // - self.emit('data', bufs.splice(0).toBuffer()); - } - }; - - this.input.on('data', this.dataHandler); - - this.input.on('end', () => { - self.emit('end'); - }); - - this.input.on('error', err => { - this.connectionDebug( { err : err }, 'Socket error' ); - return self.emit('end'); - }); - - this.connectionTrace = (info, msg) => { - if(Config().loginServers.telnet.traceConnections) { - const logger = self.log || Log; - return logger.trace(info, `Telnet: ${msg}`); - } - }; - - this.connectionDebug = (info, msg) => { - const logger = self.log || Log; - return logger.debug(info, `Telnet: ${msg}`); - }; - - this.connectionWarn = (info, msg) => { - const logger = self.log || Log; - return logger.warn(info, `Telnet: ${msg}`); - }; - - this.readyNow = () => { - if(!this.didReady) { - this.didReady = true; - this.emit('ready', { firstMenu : Config().loginServers.telnet.firstMenu } ); - } - }; - - this.disconnect = function() { - try { - return this.output.end.apply(this.output, arguments); - } - catch(e) { - // nothing - } - }; -} - -util.inherits(TelnetClient, baseClient.Client); - -/////////////////////////////////////////////////////////////////////////////// -// Telnet Command/Option handling -/////////////////////////////////////////////////////////////////////////////// -TelnetClient.prototype.handleTelnetEvent = function(evt) { - - if(!evt.command) { - return this.connectionWarn( { evt : evt }, 'No command for event'); - } - - // handler name e.g. 'handleWontCommand' - const handlerName = `handle${evt.command.charAt(0).toUpperCase()}${evt.command.substr(1)}Command`; - - if(this[handlerName]) { - // specialized - this[handlerName](evt); - } else { - // generic-ish - this.handleMiscCommand(evt); - } -}; - -TelnetClient.prototype.handleWillCommand = function(evt) { - if('terminal type' === evt.option) { - // - // See RFC 1091 @ http://www.faqs.org/rfcs/rfc1091.html - // - this.requestTerminalType(); - } else if('new environment' === evt.option) { - // - // See RFC 1572 @ http://www.faqs.org/rfcs/rfc1572.html - // - this.requestNewEnvironment(); - } else { - // :TODO: temporary: - this.connectionTrace(evt, 'WILL'); - } -}; - -TelnetClient.prototype.handleWontCommand = function(evt) { - if(this.sentDont[evt.option]) { - return this.connectionTrace(evt, 'WONT - DON\'T already sent'); - } - - this.sentDont[evt.option] = true; - - if('new environment' === evt.option) { - this.dont.new_environment(); - } else { - this.connectionTrace(evt, 'WONT'); - } -}; - -TelnetClient.prototype.handleDoCommand = function(evt) { - // :TODO: handle the rest, e.g. echo nd the like - - if('linemode' === evt.option) { - // - // Client wants to enable linemode editing. Denied. - // - this.wont.linemode(); - } else if('encrypt' === evt.option) { - // - // Client wants to enable encryption. Denied. - // - this.wont.encrypt(); - } else { - // :TODO: temporary: - this.connectionTrace(evt, 'DO'); - } -}; - -TelnetClient.prototype.handleDontCommand = function(evt) { - this.connectionTrace(evt, 'DONT'); -}; - -TelnetClient.prototype.handleSbCommand = function(evt) { - const self = this; - - if('terminal type' === evt.option) { - // - // See RFC 1091 @ http://www.faqs.org/rfcs/rfc1091.html - // - // :TODO: According to RFC 1091 @ http://www.faqs.org/rfcs/rfc1091.html - // We should keep asking until we see a repeat. From there, determine the best type/etc. - self.setTermType(evt.ttype); - - self.negotiationsComplete = true; // :TODO: throw in a array of what we've taken care. Complete = array satisified or timeout - - self.readyNow(); - } else if('new environment' === evt.option) { - // - // Handling is as follows: - // * Map 'TERM' -> 'termType' and only update if ours is 'unknown' - // * Map COLUMNS -> 'termWidth' and only update if ours is 0 - // * Map ROWS -> 'termHeight' and only update if ours is 0 - // * Add any new variables, ignore any existing - // - Object.keys(evt.envVars || {} ).forEach(function onEnv(name) { - if('TERM' === name && 'unknown' === self.term.termType) { - self.setTermType(evt.envVars[name]); - } else if('COLUMNS' === name && 0 === self.term.termWidth) { - self.term.termWidth = parseInt(evt.envVars[name]); - self.clearMciCache(); // term size changes = invalidate cache - self.connectionDebug({ termWidth : self.term.termWidth, source : 'NEW-ENVIRON'}, 'Window width updated'); - } else if('ROWS' === name && 0 === self.term.termHeight) { - self.term.termHeight = parseInt(evt.envVars[name]); - self.clearMciCache(); // term size changes = invalidate cache - self.connectionDebug({ termHeight : self.term.termHeight, source : 'NEW-ENVIRON'}, 'Window height updated'); - } else { - if(name in self.term.env) { - - EnigAssert( - SB_COMMANDS.INFO === evt.type || SB_COMMANDS.IS === evt.type, - 'Unexpected type: ' + evt.type - ); - - self.connectionWarn( - { varName : name, value : evt.envVars[name], existingValue : self.term.env[name] }, - 'Environment variable already exists' - ); - } else { - self.term.env[name] = evt.envVars[name]; - self.connectionDebug( { varName : name, value : evt.envVars[name] }, 'New environment variable' ); - } + default : + return this.socket.command(Commands.WONT, command.option); } }); - } else if('window size' === evt.option) { - // - // Update termWidth & termHeight. - // Set LINES and COLUMNS environment variables as well. - // - self.term.termWidth = evt.width; - self.term.termHeight = evt.height; + this.socket.on('DONT', command => { + this._logTrace(command, 'DONT'); + }); - if(evt.width > 0) { - self.term.env.COLUMNS = evt.height; + this.socket.on('WILL', command => { + switch (command.option) { + case Options.LINEMODE : + return this.socket.dont.linemode(); + + case Options.TTYPE : + return this.socket.sb.send.ttype(); + + case Options.NEW_ENVIRON : + return this.socket.sb.send.new_environ( + [ 'ROWS', 'COLUMNS', 'TERM', 'TERM_PROGRAM' ] + ); + + default : + break; + } + }); + + this.socket.on('WONT', command => { + switch (command.option) { + case Options.NEW_ENVIRON : + return this.socket.dont.new_environ(); + + default : + return this._logTrace(command, 'WONT'); + } + }); + + this.socket.on('SB', command => { + switch (command.option) { + case Options.TTYPE : + this.setTermType(command.optionData.ttype); + return this._clientReady(); + + case Options.NEW_ENVIRON : + { + this._logDebug( + { vars : command.optionData.vars, userVars : command.optionData.userVars }, + 'New environment received' + ); + + // get a value from vars with fallback of user vars + const getValue = (name) => { + return command.optionData.vars.find(nv => nv.name === name) || + command.optionData.userVars.find(nv => nv.name === name); + }; + + if ('unknown' === this.term.termType) { + // allow from vars or user vars + const term = getValue('TERM') || getValue('TERM_PROGRAM'); + if (term) { + this.setTermType(term.value); + } + } + + if (0 === this.term.termHeight || 0 === this.term.termWidth) { + const updateTermSize = (what) => { + const value = parseInt(getValue(what)); + if (value) { + this.term[what === 'ROWS' ? 'termHeight' : 'termWidth'] = value; + this.clearMciCache(); + this._logDebug( + { [ what ] : value, source : 'NEW-ENVIRON' }, + 'Window size updated' + ); + } + }; + + updateTermSize('ROWS'); + updateTermSize('COLUMNS'); + } + } + break; + + case Options.NAWS : + { + const { width, height } = command.optionData; + + this.term.termWidth = width; + this.term.termHeight = height; + + if (width) { + this.term.env.COLUMNS = width; + } + + if (height) { + this.term.env.ROWS = height; + } + + this.clearMciCache(); + + this._logDebug( + { width, height, source : 'NAWS' }, + 'Windows size updated' + ); + } + break; + + default : + return this._logTrace(command, 'SB'); + } + }); + + this.socket.on('IP', command => { + this._logDebug(command, 'Interrupt Process (IP) - Ending session'); + return this.disconnect(); + }); + + this.socket.on('AYT', () => { + this.socket.write('\b'); + return this._logTrace(command, 'Are You There (AYT) - Replied'); + }); + + // kick off negotiations + this._banner(); + } + + disconnect() { + try { + return this.socket.rawSocket.end(); + } catch (e) { + // ignored + } + } + + _logTrace(info, msg) { + if (Config().loginServers.telnet.traceConnections) { + const log = this.log || Log; + return log.trace(info, `Telnet: ${msg}`); + } + } + + _logDebug(info, msg) { + const log = this.log || Log; + return log.debug(info, `Telnet: ${msg}`); + } + + _clientReady() { + if (this.clientReadyHandled) { + return; // already processed } - if(evt.height > 0) { - self.term.env.ROWS = evt.height; - } + this.clientReadyHandled = true; + this.emit('ready', { firstMenu : Config().loginServers.telnet.firstMenu } ); + } - self.clearMciCache(); // term size changes = invalidate cache + _banner() { + this.socket.do.echo(); + this.socket.will.echo(); // we'll echo back - self.connectionDebug({ termWidth : evt.width , termHeight : evt.height, source : 'NAWS' }, 'Window size updated'); - } else { - self.connectionDebug(evt, 'SB'); + this.socket.will.sga(); + this.socket.do.sga(); + + this.socket.do.transmit_binary(); + this.socket.will.transmit_binary(); + + this.socket.do.ttype(); + this.socket.do.naws(); + this.socket.do.new_environ(); } }; -const IGNORED_COMMANDS = [ - COMMANDS.EL, COMMANDS.GA, COMMANDS.NOP, COMMANDS.DM, COMMANDS.BRK -]; - - -TelnetClient.prototype.handleMiscCommand = function(evt) { - EnigAssert(evt.command !== 'undefined' && evt.command.length > 0); - - // - // See: - // * RFC 854 @ http://tools.ietf.org/html/rfc854 - // - if('ip' === evt.command) { - // Interrupt Process (IP) - this.log.debug('Interrupt Process (IP) - Ending'); - - this.input.end(); - } else if('ayt' === evt.command) { - this.output.write('\b'); - - this.log.debug('Are You There (AYT) - Replied "\\b"'); - } else if(IGNORED_COMMANDS.indexOf(evt.commandCode)) { - this.log.trace({ command : evt.command, commandCode : evt.commandCode }, 'Ignoring command'); - } else { - this.log.warn({ evt : evt }, 'Unknown command'); - } -}; - -TelnetClient.prototype.requestTerminalType = function() { - const buf = Buffer.from( [ - COMMANDS.IAC, - COMMANDS.SB, - OPTIONS.TERMINAL_TYPE, - SB_COMMANDS.SEND, - COMMANDS.IAC, - COMMANDS.SE ]); - this.output.write(buf); -}; - -const WANTED_ENVIRONMENT_VAR_BUFS = [ - Buffer.from( 'LINES' ), - Buffer.from( 'COLUMNS' ), - Buffer.from( 'TERM' ), - Buffer.from( 'TERM_PROGRAM' ) -]; - -TelnetClient.prototype.requestNewEnvironment = function() { - - if(this.subNegotiationState.newEnvironRequested) { - this.log.debug('New environment already requested'); - return; - } - - const self = this; - - const bufs = buffers(); - bufs.push(Buffer.from( [ - COMMANDS.IAC, - COMMANDS.SB, - OPTIONS.NEW_ENVIRONMENT, - SB_COMMANDS.SEND ] - )); - - for(let i = 0; i < WANTED_ENVIRONMENT_VAR_BUFS.length; ++i) { - bufs.push(Buffer.from( [ NEW_ENVIRONMENT_COMMANDS.VAR ] ), WANTED_ENVIRONMENT_VAR_BUFS[i] ); - } - - bufs.push(Buffer.from([ NEW_ENVIRONMENT_COMMANDS.USERVAR, COMMANDS.IAC, COMMANDS.SE ])); - - self.output.write(bufs.toBuffer()); - - this.subNegotiationState.newEnvironRequested = true; -}; - -TelnetClient.prototype.banner = function() { - this.will.echo(); - - this.will.suppress_go_ahead(); - this.do.suppress_go_ahead(); - - this.do.transmit_binary(); - this.will.transmit_binary(); - - this.do.terminal_type(); - - this.do.window_size(); - this.do.new_environment(); -}; - -function Command(command, client) { - this.command = COMMANDS[command.toUpperCase()]; - this.client = client; -} - -// Create Command objects with echo, transmit_binary, ... -Object.keys(OPTIONS).forEach(function(name) { - const code = OPTIONS[name]; - - Command.prototype[name.toLowerCase()] = function() { - const buf = Buffer.alloc(3); - buf[0] = COMMANDS.IAC; - buf[1] = this.command; - buf[2] = code; - return this.client.output.write(buf); - }; -}); - -// Create do, dont, etc. methods on Client -['do', 'dont', 'will', 'wont'].forEach(function(command) { - const get = function() { - return new Command(command, this); - }; - - Object.defineProperty(TelnetClient.prototype, command, { - get : get, - enumerable : true, - configurable : true - }); -}); +inherits(TelnetClient, Client); exports.getModule = class TelnetServerModule extends LoginServerModule { constructor() { @@ -843,24 +234,9 @@ exports.getModule = class TelnetServerModule extends LoginServerModule { } createServer(cb) { - this.server = net.createServer( sock => { - const client = new TelnetClient(sock, sock); - - client.banner(); - - this.handleNewClient(client, sock, ModuleInfo); - - // - // Set a timeout and attempt to proceed even if we don't know - // the term type yet, which is the preferred trigger - // for moving along - // - setTimeout( () => { - if(!client.didReady) { - Log.info('Proceeding after 3s without knowing term type'); - client.readyNow(); - } - }, 3000); + this.server = net.createServer( socket => { + const client = new TelnetClient(socket); + this.handleNewClient(client, socket, ModuleInfo); }); this.server.on('error', err => { @@ -886,3 +262,5 @@ exports.getModule = class TelnetServerModule extends LoginServerModule { }); } }; + +exports.TelnetClient = TelnetClient; // WebSockets is a wrapper on top of this diff --git a/core/servers/login/telnet2.js b/core/servers/login/telnet2.js deleted file mode 100644 index 7f63659e..00000000 --- a/core/servers/login/telnet2.js +++ /dev/null @@ -1,228 +0,0 @@ -// ENiGMA½ -const LoginServerModule = require('../../login_server_module'); -const { Client } = require('../../client'); -const Config = require('../../config').get; -const { log: Log } = require('../../logger'); - -// deps -const net = require('net'); -const { - TelnetSocket, - TelnetSpec: { Options, Commands } -} = require('telnet-socket'); -const { inherits } = require('util'); - -const ModuleInfo = exports.moduleInfo = { - name : 'Telnet', - desc : 'Telnet Server', - author : 'NuSkooler', - isSecure : false, - packageName : 'codes.l33t.enigma.telnet.server.v2', -}; - -class TelnetClient { - constructor(socket) { - Client.apply(this, socket, socket); - - this.setInputOutput(socket, socket); - this.socket = new TelnetSocket(socket); - - // - // Wait up to 3s to hear about from our terminal type request - // then go ahead and move on... - // - setTimeout(() => { - this.clientReady(); - }, 3000); - - this.dataHandler = function(data) { - this.emit('data', data); - }.bind(this); - - this.socket.on('data', this.dataHandler); - - this.socket.on('error', err => { - // :TODO: Log me - return this.emit('end'); - }); - - this.socket.on('end', () => { - this.emit('end'); - }); - - this.socket.on('DO', command => { - switch (command.option) { - case Options.ECHO : - return this.socket.will.echo(); - - default : - return this.socket.command(Commands.WONT, command.option); - } - }); - - this.socket.on('DONT', command => { - // :TODO: Log me - }); - - this.socket.on('WILL', command => { - switch (command.option) { - case Options.LINEMODE : - return this.socket.dont.linemode(); - - case Options.TTYPE : - return this.socket.sb.send.ttype(); - - case Options.NEW_ENVIRON : - return this.socket.sb.send.new_environ(); - - default : - break; - } - }); - - this.socket.on('WONT', command => { - switch (command.option) { - case Options.NEW_ENVIRON : - return this.socket.dont.new_environ(); - - default : - // :TODO: Log me - break; - } - }); - - this.socket.on('SB', command => { - switch (command.option) { - case Options.TTYPE : - this.setTermType(command.optionData.ttype); - this.clientReady(); - break; - - case Options.NEW_ENVIRON : - { - if ('unknown' === this.term.termType) { - const term = - command.optionData.vars.find(nv => nv.TERM) || - command.optionData.userVars.find(nv => nv.TERM); - if (term) { - this.setTermType(term); - } - } - - command.optionData.vars.forEach(nv => { - console.log(nv); - }); - } - break; - - case Options.NAWS : - { - const { width, height } = command.optionData; - - this.term.termWidth = width; - this.term.termHeight = height; - - if (width) { - this.term.env.COLUMNS = width; - } - - if (height) { - this.term.env.ROWS = height; - } - - this.clearMciCache(); - - // :TODO: Log negotiation - } - break; - } - }); - - this.socket.on('IP', command => { - // :TODO: Log me - return this.disconnect(); - }); - - this.socket.on('AYT', () => { - // :TODO: Log me - return this.socket.write('\b'); - }); - - // kick off negotiations - this.banner(); - } - - // dataHandler(data) { - // this.emit('data', data); - // } - - clientReady() { - if (this.clientReadyHandled) { - return; // already processed - } - - this.clientReadyHandled = true; - this.emit('ready', { firstMenu : Config().loginServers.telnet.firstMenu } ); - } - - disconnect() { - try { - return this.socket.rawSocket.end(); - } catch (e) { - // ignored - } - } - - banner() { - this.socket.do.echo(); - this.socket.will.echo(); // we'll echo back - - this.socket.will.sga(); - this.socket.do.sga(); - - this.socket.do.transmit_binary(); - this.socket.will.transmit_binary(); - - this.socket.do.ttype(); - this.socket.do.naws(); - this.socket.do.new_environ(); - } -}; - -inherits(TelnetClient, Client); - -exports.getModule = class TelnetServerModule extends LoginServerModule { - constructor() { - super(); - } - - createServer(cb) { - this.server = net.createServer( socket => { - const client = new TelnetClient(socket); - this.handleNewClient(client, socket, ModuleInfo); - }); - - this.server.on('error', err => { - Log.info( { error : err.message }, 'Telnet server error'); - }); - - return cb(null); - } - - listen(cb) { - const config = Config(); - //const port = parseInt(config.loginServers.telnet.port); - const port = 8810; // :TODO: Put me back ;) - if(isNaN(port)) { - Log.error( { server : ModuleInfo.name, port : config.loginServers.telnet.port }, 'Cannot load server (invalid port)' ); - return cb(Errors.Invalid(`Invalid port: ${config.loginServers.telnet.port}`)); - } - - this.server.listen(port, config.loginServers.telnet.address, err => { - if(!err) { - Log.info( { server : ModuleInfo.name, port : port }, 'Listening for connections' ); - } - return cb(err); - }); - } -}; diff --git a/core/servers/login/websocket.js b/core/servers/login/websocket.js index dadcdbe6..1cff80e4 100644 --- a/core/servers/login/websocket.js +++ b/core/servers/login/websocket.js @@ -95,7 +95,7 @@ function WebSocketClient(ws, req, serverType) { ws.isConnectionAlive = true; }); - TelnetClient.call(this, this.socketBridge, this.socketBridge); + TelnetClient.call(this, this.socketBridge); Log.trace( { headers : req.headers }, 'WebSocket connection headers' ); From e85ba322cefd2b40b4c76110d7fa52af55957ec0 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 18 May 2020 19:19:30 -0600 Subject: [PATCH 72/95] WebSockets work with new telnet server --- core/servers/login/telnet.js | 34 +++---- core/servers/login/websocket.js | 174 +++++++++++++++++--------------- 2 files changed, 106 insertions(+), 102 deletions(-) diff --git a/core/servers/login/telnet.js b/core/servers/login/telnet.js index 111167fb..7bb938a4 100644 --- a/core/servers/login/telnet.js +++ b/core/servers/login/telnet.js @@ -176,9 +176,6 @@ class TelnetClient { this.socket.write('\b'); return this._logTrace(command, 'Are You There (AYT) - Replied'); }); - - // kick off negotiations - this._banner(); } disconnect() { @@ -189,6 +186,21 @@ class TelnetClient { } } + banner() { + this.socket.do.echo(); + this.socket.will.echo(); // we'll echo back + + this.socket.will.sga(); + this.socket.do.sga(); + + this.socket.do.transmit_binary(); + this.socket.will.transmit_binary(); + + this.socket.do.ttype(); + this.socket.do.naws(); + this.socket.do.new_environ(); + } + _logTrace(info, msg) { if (Config().loginServers.telnet.traceConnections) { const log = this.log || Log; @@ -209,21 +221,6 @@ class TelnetClient { this.clientReadyHandled = true; this.emit('ready', { firstMenu : Config().loginServers.telnet.firstMenu } ); } - - _banner() { - this.socket.do.echo(); - this.socket.will.echo(); // we'll echo back - - this.socket.will.sga(); - this.socket.do.sga(); - - this.socket.do.transmit_binary(); - this.socket.will.transmit_binary(); - - this.socket.do.ttype(); - this.socket.do.naws(); - this.socket.do.new_environ(); - } }; inherits(TelnetClient, Client); @@ -236,6 +233,7 @@ exports.getModule = class TelnetServerModule extends LoginServerModule { createServer(cb) { this.server = net.createServer( socket => { const client = new TelnetClient(socket); + client.banner(); // start negotiations this.handleNewClient(client, socket, ModuleInfo); }); diff --git a/core/servers/login/websocket.js b/core/servers/login/websocket.js index 1cff80e4..f0cdc0b1 100644 --- a/core/servers/login/websocket.js +++ b/core/servers/login/websocket.js @@ -24,100 +24,106 @@ const ModuleInfo = exports.moduleInfo = { packageName : 'codes.l33t.enigma.websocket.server', }; -function WebSocketClient(ws, req, serverType) { +class WebSocketClient extends TelnetClient { + constructor(ws, req, serverType) { + // + // This bridge makes accessible various calls that client sub classes + // want to access on I/O socket + // + const socketBridge = new class SocketBridge extends Writable { + constructor(ws) { + super(); + this.ws = ws; + } - Object.defineProperty(this, 'isSecure', { - get : () => ('secure' === serverType || true === this.proxied) ? true : false, - }); + setClient(client) { + this.client = client; + } - const self = this; + end() { + return ws.close(); + } - this.dataHandler = function(data) { - if(self.pipedDest) { - self.pipedDest.write(data); + write(data, cb) { + cb = cb || ( () => { /* eat it up */} ); // handle data writes after close + + return this.ws.send(data, { binary : true }, cb); + } + + pipe(dest) { + Log.trace('WebSocket SocketBridge pipe()'); + this.client.pipedDest = dest; + } + + unpipe() { + Log.trace('WebSocket SocketBridge unpipe()'); + this.client.pipedDest = null; + } + + resume() { + Log.trace('WebSocket SocketBridge resume()'); + } + + get remoteAddress() { + // Support X-Forwarded-For and X-Real-IP headers for proxied connections + return (this.client.proxied && (req.headers['x-forwarded-for'] || req.headers['x-real-ip'])) || req.connection.remoteAddress; + } + }(ws); + + // :TODO: this is quite the clusterfuck... + super(socketBridge); + this.socketBridge = socketBridge; + this.serverType = serverType; + + this.socketBridge.setClient(this); + + this.dataHandler = function(data) { + if(this.pipedDest) { + this.pipedDest.write(data); + } else { + this.socketBridge.emit('data', data); + } + }.bind(this); + + ws.on('message', this.dataHandler); + + ws.on('close', () => { + // we'll remove client connection which will in turn end() via our SocketBridge above + return this.emit('end'); + }); + + // + // Monitor connection status with ping/pong + // + ws.on('pong', () => { + Log.trace(`Pong from ${this.socketBridge.remoteAddress}`); + ws.isConnectionAlive = true; + }); + + Log.trace( { headers : req.headers }, 'WebSocket connection headers' ); + + // + // If the config allows it, look for 'x-forwarded-proto' as "https" + // to override |isSecure| + // + if(true === _.get(Config(), 'loginServers.webSocket.proxied') && + 'https' === req.headers['x-forwarded-proto']) + { + Log.debug(`Assuming secure connection due to X-Forwarded-Proto of "${req.headers['x-forwarded-proto']}"`); + this.proxied = true; } else { - self.socketBridge.emit('data', data); - } - }; - - // - // This bridge makes accessible various calls that client sub classes - // want to access on I/O socket - // - this.socketBridge = new class SocketBridge extends Writable { - constructor(ws) { - super(); - this.ws = ws; + this.proxied = false; } - end() { - return ws.close(); - } - - write(data, cb) { - cb = cb || ( () => { /* eat it up */} ); // handle data writes after close - - return this.ws.send(data, { binary : true }, cb); - } - - pipe(dest) { - Log.trace('WebSocket SocketBridge pipe()'); - self.pipedDest = dest; - } - - unpipe() { - Log.trace('WebSocket SocketBridge unpipe()'); - self.pipedDest = null; - } - - resume() { - Log.trace('WebSocket SocketBridge resume()'); - } - - get remoteAddress() { - // Support X-Forwarded-For and X-Real-IP headers for proxied connections - return (self.proxied && (req.headers['x-forwarded-for'] || req.headers['x-real-ip'])) || req.connection.remoteAddress; - } - }(ws); - - ws.on('message', this.dataHandler); - - ws.on('close', () => { - // we'll remove client connection which will in turn end() via our SocketBridge above - return this.emit('end'); - }); - - // - // Monitor connection status with ping/pong - // - ws.on('pong', () => { - Log.trace(`Pong from ${this.socketBridge.remoteAddress}`); - ws.isConnectionAlive = true; - }); - - TelnetClient.call(this, this.socketBridge); - - Log.trace( { headers : req.headers }, 'WebSocket connection headers' ); - - // - // If the config allows it, look for 'x-forwarded-proto' as "https" - // to override |isSecure| - // - if(true === _.get(Config(), 'loginServers.webSocket.proxied') && - 'https' === req.headers['x-forwarded-proto']) - { - Log.debug(`Assuming secure connection due to X-Forwarded-Proto of "${req.headers['x-forwarded-proto']}"`); - this.proxied = true; - } else { - this.proxied = false; + // start handshake process + this.banner(); } - // start handshake process - this.banner(); + get isSecure() { + return ('secure' === this.serverType || true === this.proxied) ? true : false; + } } -require('util').inherits(WebSocketClient, TelnetClient); - const WSS_SERVER_TYPES = [ 'insecure', 'secure' ]; exports.getModule = class WebSocketLoginServer extends LoginServerModule { From 65d68f33dc33c3d9014989c119d73f10b20ef576 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 20 May 2020 20:23:09 -0600 Subject: [PATCH 73/95] Allow passthrough --- core/servers/login/telnet.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/core/servers/login/telnet.js b/core/servers/login/telnet.js index 7bb938a4..318dd9fc 100644 --- a/core/servers/login/telnet.js +++ b/core/servers/login/telnet.js @@ -24,8 +24,8 @@ class TelnetClient { constructor(socket) { Client.apply(this, socket, socket); - this.setInputOutput(socket, socket); this.socket = new TelnetSocket(socket); + this.setInputOutput(this.socket, this.socket); // // Wait up to 3s to hear about from our terminal type request @@ -50,6 +50,8 @@ class TelnetClient { this.emit('end'); }); + // :TODO: handle 'command error' event + this.socket.on('DO', command => { switch (command.option) { case Options.ECHO : @@ -178,6 +180,14 @@ class TelnetClient { }); } + get dataPassthrough() { + return this.socket.passthrough; + } + + set dataPassthrough(passthrough) { + this.socket.passthrough = passthrough; + } + disconnect() { try { return this.socket.rawSocket.end(); From 72301104ffb817b2175bf50ffbc3284f8968ed48 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 20 May 2020 20:24:12 -0600 Subject: [PATCH 74/95] Update passthrough state --- core/client.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/client.js b/core/client.js index 40872fa2..88c9ea0f 100644 --- a/core/client.js +++ b/core/client.js @@ -107,11 +107,13 @@ function Client(/*input, output*/) { }); this.setTemporaryDirectDataHandler = function(handler) { + this.dataPassthrough = true; // let implementations do with what they will here this.input.removeAllListeners('data'); this.input.on('data', handler); }; this.restoreDataHandler = function() { + this.dataPassthrough = false; this.input.removeAllListeners('data'); this.input.on('data', this.dataHandler); }; From 03b79fa9f294bc840f3d7a6586d19a7ef3a22707 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 20 May 2020 20:24:26 -0600 Subject: [PATCH 75/95] TODO for IAC processing --- core/file_transfer.js | 1 + 1 file changed, 1 insertion(+) diff --git a/core/file_transfer.js b/core/file_transfer.js index ff9216d6..c5d5030c 100644 --- a/core/file_transfer.js +++ b/core/file_transfer.js @@ -390,6 +390,7 @@ exports.getModule = class TransferFileModule extends MenuModule { // needed for things like sz/rz if(external.escapeTelnet) { + // :TODO: do this faster for already-buffers... const tmp = data.toString('binary').replace(/\xff{2}/g, '\xff'); // de-escape externalProc.write(Buffer.from(tmp, 'binary')); } else { From bbae082f7dd2c39253f7ec274670ff76dd93a998 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 20 May 2020 20:28:55 -0600 Subject: [PATCH 76/95] Temp: Use local mod for development --- package.json | 2 +- yarn.lock | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 6bb5a613..e0e5f0d1 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "ws": "^7.3.0", "xxhash": "^0.3.0", "yazl": "^2.5.1", - "telnet-socket" : "github:NuSkooler/telnet-socket" + "telnet-socket" : "../telnet-socket" }, "devDependencies": {}, "engines": { diff --git a/yarn.lock b/yarn.lock index 5dd80c0e..4f91f7e2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1840,9 +1840,8 @@ tar@^4: safe-buffer "^5.1.2" yallist "^3.0.2" -"telnet-socket@github:NuSkooler/telnet-socket": +telnet-socket@../telnet-socket: version "0.1.0" - resolved "https://codeload.github.com/NuSkooler/telnet-socket/tar.gz/a29ea2edaec98bb844f2b0d27fa0977f33eaa7bf" dependencies: binary-parser "^1.5.0" buffers "github:NuSkooler/node-buffers" From 1e6c577cd3001cc2cad7b4bf284efa6002896ad8 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 21 May 2020 19:16:40 -0600 Subject: [PATCH 77/95] More ANSI-BBS compliant... still some to go --- core/servers/login/telnet.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/core/servers/login/telnet.js b/core/servers/login/telnet.js index 318dd9fc..fe0393c2 100644 --- a/core/servers/login/telnet.js +++ b/core/servers/login/telnet.js @@ -42,7 +42,7 @@ class TelnetClient { this.socket.on('data', this.dataHandler); this.socket.on('error', err => { - this._logDebug( { error : err.message }, 'Socket error'); + this._logDebug({ error : err.message }, 'Socket error'); return this.emit('end'); }); @@ -50,12 +50,17 @@ class TelnetClient { this.emit('end'); }); - // :TODO: handle 'command error' event + this.socket.on('command error', (command, err) => { + this._logDebug({ command, error : err.message }, 'Command error'); + }); this.socket.on('DO', command => { switch (command.option) { - case Options.ECHO : - return this.socket.will.echo(); + case Options.SGA : + return this.socket.will.sga(); + + case Options.TRANSMIT_BINARY : + return this.socket.will.transmit_binary(); default : return this.socket.command(Commands.WONT, command.option); @@ -68,9 +73,6 @@ class TelnetClient { this.socket.on('WILL', command => { switch (command.option) { - case Options.LINEMODE : - return this.socket.dont.linemode(); - case Options.TTYPE : return this.socket.sb.send.ttype(); @@ -197,8 +199,7 @@ class TelnetClient { } banner() { - this.socket.do.echo(); - this.socket.will.echo(); // we'll echo back + this.socket.will.echo(); // we'll echo back this.socket.will.sga(); this.socket.do.sga(); From 2a93de94876c8881be802081e11a0e12227e83a5 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 21 May 2020 21:22:15 -0600 Subject: [PATCH 78/95] Yet more ANSI-BBS related updates --- core/servers/login/telnet.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/servers/login/telnet.js b/core/servers/login/telnet.js index fe0393c2..39f8abbb 100644 --- a/core/servers/login/telnet.js +++ b/core/servers/login/telnet.js @@ -56,11 +56,14 @@ class TelnetClient { this.socket.on('DO', command => { switch (command.option) { + // We've already stated we WILL do the following via + // the banner - some terminals will ask over and over + // if we respond to a DO with a WILL, so just don't + // do anything... case Options.SGA : - return this.socket.will.sga(); - + case Options.ECHO : case Options.TRANSMIT_BINARY : - return this.socket.will.transmit_binary(); + break; default : return this.socket.command(Commands.WONT, command.option); @@ -199,6 +202,7 @@ class TelnetClient { } banner() { + this.socket.do.echo(); this.socket.will.echo(); // we'll echo back this.socket.will.sga(); From 5aaf568ed55b79c5e1af9af56ff0d06ccbd8178b Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 21 May 2020 21:22:49 -0600 Subject: [PATCH 79/95] Fix bug reported by user before I forget --- core/door_util.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/door_util.js b/core/door_util.js index 6517f1be..24c2cd80 100644 --- a/core/door_util.js +++ b/core/door_util.js @@ -16,6 +16,10 @@ function trackDoorRunBegin(client, doorTag) { } function trackDoorRunEnd(trackInfo) { + if (!trackInfo) { + return; + } + const { startTime, client, doorTag } = trackInfo; const diff = moment.duration(moment().diff(startTime)); From eeba6095607a6ecd872abbc80bee2c2c7b230bff Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Fri, 22 May 2020 11:41:59 -0600 Subject: [PATCH 80/95] This is broken WIP. Goign to refactor, but on the road... --- core/servers/login/websocket.js | 89 ++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 25 deletions(-) diff --git a/core/servers/login/websocket.js b/core/servers/login/websocket.js index f0cdc0b1..a8e9ad1b 100644 --- a/core/servers/login/websocket.js +++ b/core/servers/login/websocket.js @@ -15,6 +15,7 @@ const http = require('http'); const https = require('https'); const fs = require('graceful-fs'); const Writable = require('stream'); +const { Duplex } = require('stream'); const forEachSeries = require('async/forEachSeries'); const ModuleInfo = exports.moduleInfo = { @@ -30,45 +31,83 @@ class WebSocketClient extends TelnetClient { // This bridge makes accessible various calls that client sub classes // want to access on I/O socket // - const socketBridge = new class SocketBridge extends Writable { + const socketBridge = new class SocketBridge extends Duplex { constructor(ws) { super(); this.ws = ws; + + this.ws.on('close', err => this.emit('close', err)); + //this.ws.on('connect', () => this.emit('connect')); + //this.ws.on('drain', () => this.emit('drain')); + //this.ws.on('end', () => this.emit('end')); + this.ws.on('error', err => this.emit('error', err)); + + //this.ws.on('ready', () => this.emit('ready')); + //this.ws.on('timeout', () => this.emit('timeout')); + this.ws.on('data', data => this._data(data)); } setClient(client) { this.client = client; } - end() { - return ws.close(); - } - - write(data, cb) { - cb = cb || ( () => { /* eat it up */} ); // handle data writes after close - - return this.ws.send(data, { binary : true }, cb); - } - - pipe(dest) { - Log.trace('WebSocket SocketBridge pipe()'); - this.client.pipedDest = dest; - } - - unpipe() { - Log.trace('WebSocket SocketBridge unpipe()'); - this.client.pipedDest = null; - } - - resume() { - Log.trace('WebSocket SocketBridge resume()'); - } - get remoteAddress() { // Support X-Forwarded-For and X-Real-IP headers for proxied connections return (this.client.proxied && (req.headers['x-forwarded-for'] || req.headers['x-real-ip'])) || req.connection.remoteAddress; } + + _write(data, encoding, cb) { + cb = cb || ( () => { /* eat it up */} ); // handle data writes after close + return this.ws.send(data, { binary : true }, cb); + } + + _read() { + // dummy + } + + _data(data) { + this.push(data); + } }(ws); + // const socketBridge = new class SocketBridge extends Writable { + // constructor(ws) { + // super(); + // this.ws = ws; + // } + + // setClient(client) { + // this.client = client; + // } + + // end() { + // return ws.close(); + // } + + // write(data, cb) { + // cb = cb || ( () => { /* eat it up */} ); // handle data writes after close + + // return this.ws.send(data, { binary : true }, cb); + // } + + // pipe(dest) { + // Log.trace('WebSocket SocketBridge pipe()'); + // this.client.pipedDest = dest; + // } + + // unpipe() { + // Log.trace('WebSocket SocketBridge unpipe()'); + // this.client.pipedDest = null; + // } + + // resume() { + // Log.trace('WebSocket SocketBridge resume()'); + // } + + // get remoteAddress() { + // // Support X-Forwarded-For and X-Real-IP headers for proxied connections + // return (this.client.proxied && (req.headers['x-forwarded-for'] || req.headers['x-real-ip'])) || req.connection.remoteAddress; + // } + // }(ws); // :TODO: this is quite the clusterfuck... super(socketBridge); From 2234a717059f6e281ab62857c46c510a290d2328 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Fri, 22 May 2020 12:55:52 -0600 Subject: [PATCH 81/95] Working much nicer --- core/servers/login/websocket.js | 85 ++++----------------------------- 1 file changed, 10 insertions(+), 75 deletions(-) diff --git a/core/servers/login/websocket.js b/core/servers/login/websocket.js index a8e9ad1b..fcf0ea07 100644 --- a/core/servers/login/websocket.js +++ b/core/servers/login/websocket.js @@ -27,24 +27,15 @@ const ModuleInfo = exports.moduleInfo = { class WebSocketClient extends TelnetClient { constructor(ws, req, serverType) { - // - // This bridge makes accessible various calls that client sub classes - // want to access on I/O socket - // - const socketBridge = new class SocketBridge extends Duplex { + // allow WebSocket to act like a Duplex (socket) + const wsDuplex = new class WebSocketDuplex extends Duplex { constructor(ws) { super(); this.ws = ws; this.ws.on('close', err => this.emit('close', err)); - //this.ws.on('connect', () => this.emit('connect')); - //this.ws.on('drain', () => this.emit('drain')); - //this.ws.on('end', () => this.emit('end')); this.ws.on('error', err => this.emit('error', err)); - - //this.ws.on('ready', () => this.emit('ready')); - //this.ws.on('timeout', () => this.emit('timeout')); - this.ws.on('data', data => this._data(data)); + this.ws.on('message', data => this._data(data)); } setClient(client) { @@ -69,73 +60,22 @@ class WebSocketClient extends TelnetClient { this.push(data); } }(ws); - // const socketBridge = new class SocketBridge extends Writable { - // constructor(ws) { - // super(); - // this.ws = ws; - // } - // setClient(client) { - // this.client = client; - // } + super(wsDuplex); + wsDuplex.setClient(this); - // end() { - // return ws.close(); - // } - - // write(data, cb) { - // cb = cb || ( () => { /* eat it up */} ); // handle data writes after close - - // return this.ws.send(data, { binary : true }, cb); - // } - - // pipe(dest) { - // Log.trace('WebSocket SocketBridge pipe()'); - // this.client.pipedDest = dest; - // } - - // unpipe() { - // Log.trace('WebSocket SocketBridge unpipe()'); - // this.client.pipedDest = null; - // } - - // resume() { - // Log.trace('WebSocket SocketBridge resume()'); - // } - - // get remoteAddress() { - // // Support X-Forwarded-For and X-Real-IP headers for proxied connections - // return (this.client.proxied && (req.headers['x-forwarded-for'] || req.headers['x-real-ip'])) || req.connection.remoteAddress; - // } - // }(ws); - - // :TODO: this is quite the clusterfuck... - super(socketBridge); - this.socketBridge = socketBridge; - this.serverType = serverType; - - this.socketBridge.setClient(this); - - this.dataHandler = function(data) { - if(this.pipedDest) { - this.pipedDest.write(data); - } else { - this.socketBridge.emit('data', data); - } - }.bind(this); - - ws.on('message', this.dataHandler); - - ws.on('close', () => { + wsDuplex.on('close', () => { // we'll remove client connection which will in turn end() via our SocketBridge above return this.emit('end'); }); + this.serverType = serverType; + // // Monitor connection status with ping/pong // ws.on('pong', () => { - Log.trace(`Pong from ${this.socketBridge.remoteAddress}`); + Log.trace(`Pong from ${wsDuplex.remoteAddress}`); ws.isConnectionAlive = true; }); @@ -261,7 +201,7 @@ exports.getModule = class WebSocketLoginServer extends LoginServerModule { server.wsServer.on('connection', (ws, req) => { const webSocketClient = new WebSocketClient(ws, req, serverType); - this.handleNewClient(webSocketClient, webSocketClient.socketBridge, ModuleInfo); + this.handleNewClient(webSocketClient, webSocketClient.socket, ModuleInfo); }); Log.info( { server : serverName, port : port }, 'Listening for connections' ); @@ -272,9 +212,4 @@ exports.getModule = class WebSocketLoginServer extends LoginServerModule { cb(err); }); } - - webSocketConnection(conn) { - const webSocketClient = new WebSocketClient(conn); - this.handleNewClient(webSocketClient, webSocketClient.socketShim, ModuleInfo); - } }; From 0bf3031d9f2dfa86eae073fe912c873af2c21571 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 26 May 2020 22:55:51 -0600 Subject: [PATCH 82/95] WIP on better data handling for file transfer --- WHATSNEW.md | 2 +- core/config.js | 3 +- core/file_transfer.js | 121 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 115 insertions(+), 11 deletions(-) diff --git a/WHATSNEW.md b/WHATSNEW.md index bf6ac243..21626d92 100644 --- a/WHATSNEW.md +++ b/WHATSNEW.md @@ -6,7 +6,7 @@ This document attempts to track **major** changes and additions in ENiGMA½. For * Development is now against Node.js 12.x LTS. Other versions may work but are not currently supported! * [QWK support](/docs/messageareas/qwk.md) * `oputil fb scan *areaTagWildcard*` scans all areas in which wildcard is matched. - +* The archiver configuration `escapeTelnet` has been renamed `escapeIACs`. Support for the old value will be removed in the future. ## 0.0.10-alpha + `oputil.js user rename USERNAME NEWNAME` diff --git a/core/config.js b/core/config.js index 66ffab14..7e467709 100644 --- a/core/config.js +++ b/core/config.js @@ -865,8 +865,7 @@ function getDefaultConfig() { recvArgs : [ '--zmodem', '--binary', '--restricted', '--keep-uppercase', // dumps to CWD which is set to {uploadDir} ], - // :TODO: can we not just use --escape ? - escapeTelnet : true, // set to true to escape Telnet codes such as IAC + processIACs : true, // escape/de-escape IACs (0xff) } } }, diff --git a/core/file_transfer.js b/core/file_transfer.js index c5d5030c..bb4427b7 100644 --- a/core/file_transfer.js +++ b/core/file_transfer.js @@ -364,6 +364,16 @@ exports.getModule = class TransferFileModule extends MenuModule { const external = this.protocolConfig.external; const cmd = external[`${this.direction}Cmd`]; + // support for handlers that need IACs taken care of over Telnet/etc. + const processIACs = + external.processIACs || + external.escapeTelnet; // deprecated name + + // :TODO: we should only do this when over Telnet (or derived, such as WebSockets)? + + const IAC = Buffer.from([255]); + const EscapedIAC = Buffer.from([255, 255]); + this.client.log.debug( { cmd : cmd, args : args, tempDir : this.recvDirectory, direction : this.direction }, 'Executing external protocol' @@ -385,14 +395,92 @@ exports.getModule = class TransferFileModule extends MenuModule { } }; + // const procBuffer = []; + + // let buffering = false; + // const procWrite = (data) => { + // if (!externalProc._socket) { + // return externalProc.write(data); + // } + + // if (buffering) { + // return procBuffer.push(data); + // } + + // // while (procBuffer.length && externalProc._socket.writable) { + // // externalProc.write(procBuffer.unshift()); + // // } + + // if (externalProc._socket.writable) { + // externalProc.write(data); + // } else { + // // last write put us into buffer mode + // externalProc._socket.once('drain', () => { + // while (procBuffer.length && externalProc._socket.writable) { + // externalProc.write(procBuffer.unshift()); + // } + // buffering = !externalProc._socket.writable; + // }); + // buffering = true; + // procBuffer.push(data); + // } + // }; + + // const writeData = (data) => { + // updateActivity(); + + // if(processIACs) { + // let iacPos = data.indexOf(EscapedIAC); + // if (-1 === iacPos) { + // return procWrite(data); + // } + + // // at least one double (escaped) IAC + // let lastPos = 0; + // while (iacPos > -1) { + // let rem = iacPos - lastPos; + // if (rem >= 0) { + // procWrite(data.slice(lastPos, iacPos + 1)); + // } + // lastPos = iacPos + 2; + // iacPos = data.indexOf(EscapedIAC, lastPos); + // } + + // if (lastPos < data.length) { + // procWrite(data.slice(lastPos)); + // } + // } else { + // procWrite(data); + // } + // }; + this.client.setTemporaryDirectDataHandler(data => { updateActivity(); - // needed for things like sz/rz - if(external.escapeTelnet) { - // :TODO: do this faster for already-buffers... - const tmp = data.toString('binary').replace(/\xff{2}/g, '\xff'); // de-escape - externalProc.write(Buffer.from(tmp, 'binary')); + if(processIACs) { + let iacPos = data.indexOf(EscapedIAC); + if (-1 === iacPos) { + return externalProc.write(data); + } + + // at least one double (escaped) IAC + externalProc._socket.cork(); + let lastPos = 0; + while (iacPos > -1) { + let rem = iacPos - lastPos; + if (rem >= 0) { + externalProc.write(data.slice(lastPos, iacPos + 1)); + } + lastPos = iacPos + 2; + iacPos = data.indexOf(EscapedIAC, lastPos); + } + + if (lastPos < data.length) { + externalProc.write(data.slice(lastPos)); + } + externalProc._socket.uncork(); + // const tmp = data.toString('binary').replace(/\xff{2}/g, '\xff'); // de-escape + // externalProc.write(Buffer.from(tmp, 'binary')); } else { externalProc.write(data); } @@ -402,9 +490,26 @@ exports.getModule = class TransferFileModule extends MenuModule { updateActivity(); // needed for things like sz/rz - if(external.escapeTelnet) { - const tmp = data.toString('binary').replace(/\xff/g, '\xff\xff'); // escape - this.client.term.rawWrite(Buffer.from(tmp, 'binary')); + if(processIACs) { + let iacPos = data.indexOf(IAC); + if (-1 === iacPos) { + return this.client.term.rawWrite(data); + } + + // Has at least a single IAC + let lastPos = 0; + while (iacPos !== -1) { + if (iacPos - lastPos > 0) { + this.client.term.rawWrite(data.slice(lastPos, iacPos)); + } + this.client.term.rawWrite(EscapedIAC); + lastPos = iacPos + 1; + iacPos = data.indexOf(IAC, lastPos); + } + + if (lastPos < data.length) { + this.client.term.rawWrite(data.slice(lastPos)); + } } else { this.client.term.rawWrite(data); } From a1c9d8538f4c0fbee4fbb872835296d0d45a24e4 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 27 May 2020 19:24:45 -0600 Subject: [PATCH 83/95] Don't cork --- core/file_transfer.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/file_transfer.js b/core/file_transfer.js index bb4427b7..20e60e07 100644 --- a/core/file_transfer.js +++ b/core/file_transfer.js @@ -464,7 +464,6 @@ exports.getModule = class TransferFileModule extends MenuModule { } // at least one double (escaped) IAC - externalProc._socket.cork(); let lastPos = 0; while (iacPos > -1) { let rem = iacPos - lastPos; @@ -478,7 +477,6 @@ exports.getModule = class TransferFileModule extends MenuModule { if (lastPos < data.length) { externalProc.write(data.slice(lastPos)); } - externalProc._socket.uncork(); // const tmp = data.toString('binary').replace(/\xff{2}/g, '\xff'); // de-escape // externalProc.write(Buffer.from(tmp, 'binary')); } else { From 4dea9bdeb615316578a58645c70dd35dd6d9d23f Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 27 May 2020 19:25:15 -0600 Subject: [PATCH 84/95] Remove dead code --- core/file_transfer.js | 59 ------------------------------------------- 1 file changed, 59 deletions(-) diff --git a/core/file_transfer.js b/core/file_transfer.js index 20e60e07..90f38667 100644 --- a/core/file_transfer.js +++ b/core/file_transfer.js @@ -395,65 +395,6 @@ exports.getModule = class TransferFileModule extends MenuModule { } }; - // const procBuffer = []; - - // let buffering = false; - // const procWrite = (data) => { - // if (!externalProc._socket) { - // return externalProc.write(data); - // } - - // if (buffering) { - // return procBuffer.push(data); - // } - - // // while (procBuffer.length && externalProc._socket.writable) { - // // externalProc.write(procBuffer.unshift()); - // // } - - // if (externalProc._socket.writable) { - // externalProc.write(data); - // } else { - // // last write put us into buffer mode - // externalProc._socket.once('drain', () => { - // while (procBuffer.length && externalProc._socket.writable) { - // externalProc.write(procBuffer.unshift()); - // } - // buffering = !externalProc._socket.writable; - // }); - // buffering = true; - // procBuffer.push(data); - // } - // }; - - // const writeData = (data) => { - // updateActivity(); - - // if(processIACs) { - // let iacPos = data.indexOf(EscapedIAC); - // if (-1 === iacPos) { - // return procWrite(data); - // } - - // // at least one double (escaped) IAC - // let lastPos = 0; - // while (iacPos > -1) { - // let rem = iacPos - lastPos; - // if (rem >= 0) { - // procWrite(data.slice(lastPos, iacPos + 1)); - // } - // lastPos = iacPos + 2; - // iacPos = data.indexOf(EscapedIAC, lastPos); - // } - - // if (lastPos < data.length) { - // procWrite(data.slice(lastPos)); - // } - // } else { - // procWrite(data); - // } - // }; - this.client.setTemporaryDirectDataHandler(data => { updateActivity(); // needed for things like sz/rz From 87a23bd3c889b2b7ab758fe59ad9147744d0e73f Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 27 May 2020 21:05:19 -0600 Subject: [PATCH 85/95] Telnet Bridge updates * Use telnet-socket TelnetSpec/etc. consts & helpers * Enable passthrough --- core/file_transfer.js | 1 + core/telnet_bridge.js | 43 +++++++++++++++++++++++++------------------ 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/core/file_transfer.js b/core/file_transfer.js index 90f38667..68daf2db 100644 --- a/core/file_transfer.js +++ b/core/file_transfer.js @@ -397,6 +397,7 @@ exports.getModule = class TransferFileModule extends MenuModule { this.client.setTemporaryDirectDataHandler(data => { updateActivity(); + // needed for things like sz/rz if(processIACs) { let iacPos = data.indexOf(EscapedIAC); diff --git a/core/telnet_bridge.js b/core/telnet_bridge.js index a3a4d672..5c43ced6 100644 --- a/core/telnet_bridge.js +++ b/core/telnet_bridge.js @@ -11,7 +11,16 @@ const async = require('async'); const _ = require('lodash'); const net = require('net'); const EventEmitter = require('events'); -const buffers = require('buffers'); + +const { + TelnetSocket, + TelnetSpec : + { + Commands, + Options, + SubNegotiationCommands, + }, +} = require('telnet-socket'); /* Expected configuration block: @@ -33,7 +42,10 @@ exports.moduleInfo = { author : 'Andrew Pamment', }; -const IAC_DO_TERM_TYPE = Buffer.from( [ 255, 253, 24 ] ); +const IAC_DO_TERM_TYPE = TelnetSocket.commandBuffer( + Commands.DO, + Options.TTYPE, +); class TelnetClientConnection extends EventEmitter { constructor(client) { @@ -46,6 +58,7 @@ class TelnetClientConnection extends EventEmitter { restorePipe() { if(!this.pipeRestored) { this.pipeRestored = true; + this.client.dataPassthrough = false; // client may have bailed if(null !== _.get(this, 'client.term.output', null)) { @@ -62,6 +75,7 @@ class TelnetClientConnection extends EventEmitter { this.emit('connected'); this.pipeRestored = false; + this.client.dataPassthrough = true; this.client.term.output.pipe(this.bridgeConnection); }); @@ -69,7 +83,7 @@ class TelnetClientConnection extends EventEmitter { this.client.term.rawWrite(data); // - // Wait for a terminal type request, and send it eactly once. + // Wait for a terminal type request, and send it exactly once. // This is enough (in additional to other negotiations handled in telnet.js) // to get us in on most systems // @@ -110,25 +124,18 @@ class TelnetClientConnection extends EventEmitter { // Create a TERMINAL-TYPE sub negotiation buffer using the // actual/current terminal type. // - let bufs = buffers(); - - bufs.push(Buffer.from( + const sendTermType = TelnetSocket.commandBuffer( + Commands.SB, + Options.TTYPE, [ - 255, // IAC - 250, // SB - 24, // TERMINAL-TYPE - 0, // IS + SubNegotiationCommands.IS, + ...Buffer.from(this.client.term.termType), // e.g. "ansi" + Commands.IAC, + Commands.SE, ] - )); - - bufs.push( - Buffer.from(this.client.term.termType), // e.g. "ansi" - Buffer.from( [ 255, 240 ] ) // IAC, SE ); - - return bufs.toBuffer(); + return sendTermType; } - } exports.getModule = class TelnetBridgeModule extends MenuModule { From 438a3161d0a42f94bf9a2a88ff04296f75c987ad Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 27 May 2020 21:31:51 -0600 Subject: [PATCH 86/95] Fix up remoteAddress for WebSocket connections --- core/servers/login/websocket.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/core/servers/login/websocket.js b/core/servers/login/websocket.js index fcf0ea07..42a22723 100644 --- a/core/servers/login/websocket.js +++ b/core/servers/login/websocket.js @@ -38,13 +38,17 @@ class WebSocketClient extends TelnetClient { this.ws.on('message', data => this._data(data)); } - setClient(client) { + setClient(client, httpRequest) { this.client = client; + + // Support X-Forwarded-For and X-Real-IP headers for proxied connections + this.resolvedRemoteAddress = + (this.client.proxied && (httpRequest.headers['x-forwarded-for'] || httpRequest.headers['x-real-ip'])) || + httpRequest.connection.remoteAddress; } get remoteAddress() { - // Support X-Forwarded-For and X-Real-IP headers for proxied connections - return (this.client.proxied && (req.headers['x-forwarded-for'] || req.headers['x-real-ip'])) || req.connection.remoteAddress; + return this.resolvedRemoteAddress; } _write(data, encoding, cb) { @@ -62,7 +66,10 @@ class WebSocketClient extends TelnetClient { }(ws); super(wsDuplex); - wsDuplex.setClient(this); + wsDuplex.setClient(this, req); + + // fudge remoteAddress on socket, which is now TelnetSocket + this.socket.remoteAddress = wsDuplex.remoteAddress; wsDuplex.on('close', () => { // we'll remove client connection which will in turn end() via our SocketBridge above From fb22d89328ba1d14eed9c00caaa8faefe260d7ad Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 27 May 2020 21:52:28 -0600 Subject: [PATCH 87/95] Use telnet-socket published package --- package.json | 2 +- yarn.lock | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e0e5f0d1..4effb16b 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "ws": "^7.3.0", "xxhash": "^0.3.0", "yazl": "^2.5.1", - "telnet-socket" : "../telnet-socket" + "telnet-socket" : "^0.2.1" }, "devDependencies": {}, "engines": { diff --git a/yarn.lock b/yarn.lock index 4f91f7e2..12199dfe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -160,7 +160,7 @@ bcrypt-pbkdf@^1.0.2: dependencies: tweetnacl "^0.14.3" -binary-parser@^1.5.0, binary-parser@^1.6.2: +binary-parser@1.6.2, binary-parser@^1.6.2: version "1.6.2" resolved "https://registry.yarnpkg.com/binary-parser/-/binary-parser-1.6.2.tgz#8410a82ffd9403271ec182bd91e63a09cee88cbe" integrity sha512-cYAhKB51A9T/uylDvMK7uAYaPLWLwlferNOpnQ0E0fuO73yPi7kWaWiOm22BvuKxCbggmkiFN0VkuLg6gc+KQQ== @@ -1840,10 +1840,12 @@ tar@^4: safe-buffer "^5.1.2" yallist "^3.0.2" -telnet-socket@../telnet-socket: - version "0.1.0" +telnet-socket@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/telnet-socket/-/telnet-socket-0.2.1.tgz#1b7d74152a90740a630c166070ab6e751f50fda1" + integrity sha512-8o5kIq5CGvaEuVbVTl40Lw9NCz2v68Il2Skb7iMLuid23kWV3Sq5AAg5lIl5o0H7agoJkEGNlmMRp/6+saDaNA== dependencies: - binary-parser "^1.5.0" + binary-parser "1.6.2" buffers "github:NuSkooler/node-buffers" temptmp@^1.1.0: From a15abc60627a4886c1914bf10e2bb623c7d6f640 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Fri, 29 May 2020 23:25:34 -0600 Subject: [PATCH 88/95] Use telnet-socket 0.2.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4effb16b..19cf7662 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "ws": "^7.3.0", "xxhash": "^0.3.0", "yazl": "^2.5.1", - "telnet-socket" : "^0.2.1" + "telnet-socket" : "^0.2.2" }, "devDependencies": {}, "engines": { From 78b78e8c79e6d5a174c95c90d89b0cac88ca7a4f Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 30 May 2020 22:39:27 -0600 Subject: [PATCH 89/95] Doc updates --- core/oputil/oputil_user.js | 4 ++- docs/_includes/nav.md | 1 + docs/admin/administration.md | 43 +++++++++++++++++++++++++ docs/admin/updating.md | 30 +++++++++++------ docs/troubleshooting/monitoring-logs.md | 6 ++-- 5 files changed, 70 insertions(+), 14 deletions(-) create mode 100644 docs/admin/administration.md diff --git a/core/oputil/oputil_user.js b/core/oputil/oputil_user.js index 376ee166..07f9227a 100644 --- a/core/oputil/oputil_user.js +++ b/core/oputil/oputil_user.js @@ -462,7 +462,9 @@ function twoFactorAuthOTP(user) { function listUsers() { // oputil user list [disabled|inactive|active|locked|all] - // :TODO: --after TIMESTAMP (new users) + // :TODO: --created-since SPEC and --last-called SPEC + // --created-since SPEC + // SPEC can be TIMESTAMP or e.g. "-1hour" or "-90days" // :TODO: --sort name|id let listWhat; if (argv._.length > 2) { diff --git a/docs/_includes/nav.md b/docs/_includes/nav.md index c4303fac..6b335617 100644 --- a/docs/_includes/nav.md +++ b/docs/_includes/nav.md @@ -89,6 +89,7 @@ - [Auto Signature Editor]({{ site.baseurl }}{% link modding/autosig-edit.md %}) - Administration + - [Administration]({{ site.baseurl }}{% link admin/administration.md %}) - [oputil]({{ site.baseurl }}{% link admin/oputil.md %}) - [Updating]({{ site.baseurl }}{% link admin/updating.md %}) diff --git a/docs/admin/administration.md b/docs/admin/administration.md new file mode 100644 index 00000000..8d15fba5 --- /dev/null +++ b/docs/admin/administration.md @@ -0,0 +1,43 @@ +--- +layout: page +title: Administration +--- + +# Administration + +## Keeping Up to Date +See [Updating](updating.md). + +## Viewing Logs +See [Monitoring Logs](/docs/troubleshooting/monitoring-logs.md). + +## Managing Users +User management is currently handled via the [oputil CLI](oputil.md). + +## Backing Up Your System +It is *highly* recommended to perform **regular backups** of your system. Nothing is worse than spending a lot of time setting up a system only to have to go away unexpectedly! + +In general, simply creating a copy/archive of your system is enough for the default configuration. If you have changed default paths to point outside of your main ENiGMA½ installation take special care to ensure these are preserved as well. Database files may be in a state of flux when simply copying files. See **Database Backups** below for details on consistent backups. + +### Database Backups +[SQLite's CLI backup command](https://sqlite.org/cli.html#special_commands_to_sqlite3_dot_commands_) can be used for creating database backup files. This can be performed as an additional step to a full backup to ensure the database is backed up in a consistent state (whereas simply copying the files does not make any guarantees). + +As an example, consider the following Bash script that creates foo.sqlite3.backup files: + +```bash +for dbfile in /path/to/enigma-bbs/db/*.sqlite3; do + sqlite3 $dbfile ".backup '/path/to/db_backup/$(basename $dbfile).backup'" +done +``` + +### Backup Tools +There are many backup solutions available across all platforms. Configuration of such tools is outside the scope of this documentation. With that said, the author has had great success with [Borg](https://www.borgbackup.org/). + +## General Maintenance Tasks +### Vacuuming Database Files +SQLite database files become less performant over time and waste space. It is recommended to periodically vacuum your databases. Before proceeding, you should make a backup! + +Example: +```bash +sqlite3 ./db/message.sqlite3 "vacuum;" +``` \ No newline at end of file diff --git a/docs/admin/updating.md b/docs/admin/updating.md index 8fda4692..3cf93768 100644 --- a/docs/admin/updating.md +++ b/docs/admin/updating.md @@ -2,20 +2,30 @@ layout: page title: Updating --- -## Updating your Installation -Updating ENiGMA½ can be a bit of a learning curve compared to other systems. Especially when running from Git cloned source, you'll want frequent updates. +# Updating +Keeping your system up to date ensures you have the latest fixes, features, and general improvements. Updating ENiGMA½ can be a bit of a learning curve compared to traditional binary-release systems you may be used to, especially when running from Git cloned source. -## Steps -In _general_ the steps are as follows: -1. `cd /path/to/enigma-bbs` -2. `git pull` -3. `npm update` or `yarn` to refresh any new or updated modules. -4. Merge updates to `config/menu_template.hjson` to your `config/yourbbsname-menu.hjson` file (or simply use the template as a reference to spot any newly added default menus that you may wish to have on your system as well!). +## Updating From Source +If you have installed using Git source (if you used the `install.sh` script) follow these general steps to update your system: + +1. **Back up your system**! +2. Pull down the latest source: +```bash +cd /path/to/enigma-bbs +git pull +``` +3. :bulb: Review `WHATSNEW.md` and `UPDATE.md` for any specific instructions or changes to be aware of. +4. Update your dependencies: +```bash +npm install # or 'yarn' +``` +4. Merge updates from `config/menu_template.hjson` to your `config/yourbbsname-menu.hjson` file (or simply use the template as a reference to spot any newly added default menus that you may wish to have on your system as well!). 5. If there are updates to the `art/themes/luciano_blocktronics/theme.hjson` file and you have a custom theme, you may want to look at them as well. - -:information_source: Always keep an eye on [WHATSNEW](/WHATSNEW.md) and [UPGRADE](/UPGRADE.md)! +6. Finally, restart your running ENiGMA½ instance. :information_source: Visual diff tools such as [DiffMerge](https://www.sourcegear.com/diffmerge/downloads.php) (free, works on all major platforms) can be very helpful for the tasks outlined above! +:information_source: It is recommended to tail the logs and poke around a bit after an update. + diff --git a/docs/troubleshooting/monitoring-logs.md b/docs/troubleshooting/monitoring-logs.md index 28a3773a..4fa27a5c 100644 --- a/docs/troubleshooting/monitoring-logs.md +++ b/docs/troubleshooting/monitoring-logs.md @@ -3,7 +3,7 @@ layout: page title: Monitoring Logs --- ## Monitoring Logs -ENiGMA½ does not produce much to stdout. Logs are produced by Bunyan which outputs each entry as a JSON object. +ENiGMA½ does not produce much to stdout. Logs are produced by [Bunyan](https://github.com/trentm/node-bunyan) which outputs each entry as a JSON object. Start by installing bunyan and making it available on your path: @@ -11,11 +11,11 @@ Start by installing bunyan and making it available on your path: npm install bunyan -g ``` -or with Yarn: +or via Yarn: ```bash yarn global add bunyan ``` - + To tail logs in a colorized and pretty format, issue the following command: ```bash tail -F /path/to/enigma-bbs/logs/enigma-bbs.log | bunyan From 29ef0935c3cef51081c3bd33b919ea386afc2995 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 31 May 2020 11:49:32 -0600 Subject: [PATCH 90/95] Don't send back don't in respond to wont new_environ --- core/servers/login/telnet.js | 8 +------- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/core/servers/login/telnet.js b/core/servers/login/telnet.js index 39f8abbb..cf90b92f 100644 --- a/core/servers/login/telnet.js +++ b/core/servers/login/telnet.js @@ -90,13 +90,7 @@ class TelnetClient { }); this.socket.on('WONT', command => { - switch (command.option) { - case Options.NEW_ENVIRON : - return this.socket.dont.new_environ(); - - default : - return this._logTrace(command, 'WONT'); - } + return this._logTrace(command, 'WONT'); }); this.socket.on('SB', command => { diff --git a/yarn.lock b/yarn.lock index 12199dfe..fb655a43 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1840,10 +1840,10 @@ tar@^4: safe-buffer "^5.1.2" yallist "^3.0.2" -telnet-socket@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/telnet-socket/-/telnet-socket-0.2.1.tgz#1b7d74152a90740a630c166070ab6e751f50fda1" - integrity sha512-8o5kIq5CGvaEuVbVTl40Lw9NCz2v68Il2Skb7iMLuid23kWV3Sq5AAg5lIl5o0H7agoJkEGNlmMRp/6+saDaNA== +telnet-socket@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/telnet-socket/-/telnet-socket-0.2.2.tgz#9abf29bbd022f2bc1176fd35ab62ca94e62916c1" + integrity sha512-fKcpbCNn8NQ1wBshjliTLh4UmbXq0iYhoPsc48RTJfgdisxQOyLJe8dMwH26ezx03pEM4GWpg9ge+g4DGz7c/Q== dependencies: binary-parser "1.6.2" buffers "github:NuSkooler/node-buffers" From 74078939cc85e07e06aa167630404e67084c4a56 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 31 May 2020 14:54:33 -0600 Subject: [PATCH 91/95] DO should be a DONT for echo --- core/servers/login/telnet.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/servers/login/telnet.js b/core/servers/login/telnet.js index cf90b92f..240ce06b 100644 --- a/core/servers/login/telnet.js +++ b/core/servers/login/telnet.js @@ -196,8 +196,8 @@ class TelnetClient { } banner() { - this.socket.do.echo(); - this.socket.will.echo(); // we'll echo back + this.socket.dont.echo(); // don't echo characters + this.socket.will.echo(); // ...we'll echo them back this.socket.will.sga(); this.socket.do.sga(); From dd0a6730615e1837b9f064be6b4ecb1f58384937 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 31 May 2020 22:17:56 -0600 Subject: [PATCH 92/95] Doc updates on servers --- docs/servers/ssh.md | 5 +-- docs/servers/telnet.md | 5 +-- docs/servers/web-server.md | 66 ++++++++++++++++++++------------------ 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/docs/servers/ssh.md b/docs/servers/ssh.md index c576f38e..2f8b7769 100644 --- a/docs/servers/ssh.md +++ b/docs/servers/ssh.md @@ -15,7 +15,8 @@ Entries available under `config.loginServers.ssh`: | `firstMenu` | :-1: | First menu an SSH connected user is presented with. Defaults to `sshConnected`. | | `firstMenuNewUser` | :-1: | Menu presented to user when logging in with one of the usernames found within `users.newUserNames` in your `config.hjson`. Examples include `new` and `apply`. | | `enabled` | :+1: | Set to `true` to enable the SSH server. | -| `port` | :-1: | Override the default port of `8443`. | +| `port` | :-1: | Override the default port of `8443`. | +| `address` | :-1: | Sets an explicit bind address. | | `algorithms` | :-1: | Configuration block for SSH algorithms. Includes keys of `kex`, `cipher`, `hmac`, and `compress`. See the algorithms section in the [ssh2-streams](https://github.com/mscdex/ssh2-streams#ssh2stream-methods) documentation for details. For defaults set by ENiGMA½, see `core/config.js`. | `traceConnections` | :-1: | Set to `true` to enable full trace-level information on SSH connections. @@ -29,7 +30,7 @@ Entries available under `config.loginServers.ssh`: port: 8889 privateKeyPem: /path/to/ssh_private_key.pem privateKeyPass: sup3rs3kr3tpa55 - } + } } } ``` diff --git a/docs/servers/telnet.md b/docs/servers/telnet.md index ccefc966..fb4baddb 100644 --- a/docs/servers/telnet.md +++ b/docs/servers/telnet.md @@ -8,10 +8,11 @@ The Telnet *login server* provides a standard **non-secure** Telnet login experi ## Configuration The following configuration can be made in `config.hjson` under the `loginServers.telnet` block: -| Item | Required | Description | +| Key | Required | Description | |------|----------|-------------| | `enabled` | :-1: Defaults to `true`. Set to `false` to disable Telnet | | `port` | :-1: | Override the default port of `8888`. | +| `address` | :-1: | Sets an explicit bind address. | | `firstMenu` | :-1: | First menu a telnet connected user is presented with. Defaults to `telnetConnected`. | ### Example Configuration @@ -21,7 +22,7 @@ The following configuration can be made in `config.hjson` under the `loginServer telnet: { enabled: true port: 8888 - } + } } } ``` diff --git a/docs/servers/web-server.md b/docs/servers/web-server.md index 816c28a2..4bd015b3 100644 --- a/docs/servers/web-server.md +++ b/docs/servers/web-server.md @@ -2,13 +2,10 @@ layout: page title: Web Server --- -ENiGMA½ comes with a built in *content server* for supporting both HTTP and HTTPS. Currently the -[File Bases](file_base.md) registers routes for file downloads, and static files can also be served -for your BBS. Other features will likely come in the future or you can easily write your own! +ENiGMA½ comes with a built in *content server* for supporting both HTTP and HTTPS. Currently the [File Bases](file_base.md) registers routes for file downloads, and static files can also be served for your BBS. Other features will likely come in the future or you can easily write your own! -## Configuration -By default the web server is not enabled. To enable it, you will need to at a minimum configure two keys in -the `contentServers::web` section of `config.hjson`: +# Configuration +By default the web server is not enabled. To enable it, you will need to at a minimum configure two keys in the `contentServers.web` section of `config.hjson`: ```hjson contentServers: { @@ -17,39 +14,44 @@ contentServers: { http: { enabled: true + port: 8080 } } } ``` -This will configure HTTP for port 8080 (override with `port`). To additionally enable HTTPS, you will need a -PEM encoded SSL certificate and private key. [LetsEncrypt](https://letsencrypt.org/) supply free trusted -certificates that work perfectly with ENiGMA½. +The following is a table of all configuration keys available under `contentServers.web`: +| Key | Required | Description | +|------|----------|-------------| +| `domain` | :+1: | Sets the domain, e.g. `bbs.yourdomain.com`. | +| `http` | :-1: | Sub configuration for HTTP (non-secure) connections. See **HTTP Configuration** below. | +| `overrideUrlPrefix` | :-1: | Instructs the system to be explicit when handing out URLs. Useful if your server is behind a transparent proxy. | -Once obtained, simply enable the HTTPS server: +### HTTP Configuration +Entries available under `contentServers.web.http`: -```hjson -contentServers: { - web: { - domain: bbs.yourdomain.com - // set 'overrideUrlPrefix' if for example, you use a transparent proxy in front of ENiGMA and need to be explicit about URLs the system hands out - overrideUrlPrefix: https://bbs.yourdomain.com - https: { - enabled: true - port: 8443 - certPem: /path/to/your/cert.pem - keyPem: /path/to/your/cert_private_key.pem - } - } -} -``` +| Key | Required | Description | +|------|----------|-------------| +| `enable` | :+1: | Set to `true` to enable this server. +| `port` | :-1: | Override the default port of `8080`. | +| `address` | :-1: | Sets an explicit bind address. | -If no certificate paths are supplied, ENiGMA½ will assume the defaults of `/config/https_cert.pem` and -`/config/https_cert_key.pem` accordingly. +### HTTPS Configuration +Entries available under `contentServers.web.htt2`: -### Static Routes -Static files live relative to the `contentServers::web::staticRoot` path which defaults to `enigma-bbs/www`. +| Key | Required | Description | +|------|----------|-------------| +| `enable` | :+1: | Set to `true` to enable this server. +| `port` | :-1: | Override the default port of `8080`. | +| `address` | :-1: | Sets an explicit bind address. | +| `certPem` | :+1: | Overrides the default certificate path of `/config/https_cert.pem`. Certificate must be in PEM format. See **Certificates** below. | +| `keyPem` | :+1: | Overrides the default certificate key path of `/config/https_cert_key.pem`. Key must be in PEM format. See **Certificates** below. | -### Custom Error Pages -Customized error pages can be created for [HTTP error codes](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#4xx_Client_Error) -by providing a `.html` file in the *static routes* area. For example: `404.html`. +#### Certificates +If you don't have a TLS certificate for your domain, a good source for a certificate can be [LetsEncrypt](https://letsencrypt.org/) who supplies free and trusted TLS certificates. + +## Static Routes +Static files live relative to the `contentServers.web.staticRoot` path which defaults to `enigma-bbs/www`. + +## Custom Error Pages +Customized error pages can be created for [HTTP error codes](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#4xx_Client_Error) by providing a `.html` file in the *static routes* area. For example: `404.html`. From 10ec5d4271f537dbbcdc759e533cd3d625e07653 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 1 Jun 2020 19:15:07 -0600 Subject: [PATCH 93/95] Minor doc fixes --- docs/servers/web-server.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/servers/web-server.md b/docs/servers/web-server.md index 4bd015b3..5675c4c6 100644 --- a/docs/servers/web-server.md +++ b/docs/servers/web-server.md @@ -2,7 +2,7 @@ layout: page title: Web Server --- -ENiGMA½ comes with a built in *content server* for supporting both HTTP and HTTPS. Currently the [File Bases](file_base.md) registers routes for file downloads, and static files can also be served for your BBS. Other features will likely come in the future or you can easily write your own! +ENiGMA½ comes with a built in *content server* for supporting both HTTP and HTTPS. Currently the [File Bases](file_base.md) registers routes for file downloads, password reset email links are handled via the server, and static files can also be served for your BBS. Other features will likely come in the future or you can easily write your own! # Configuration By default the web server is not enabled. To enable it, you will need to at a minimum configure two keys in the `contentServers.web` section of `config.hjson`: @@ -37,7 +37,7 @@ Entries available under `contentServers.web.http`: | `address` | :-1: | Sets an explicit bind address. | ### HTTPS Configuration -Entries available under `contentServers.web.htt2`: +Entries available under `contentServers.web.https`: | Key | Required | Description | |------|----------|-------------| From 5b20c3ec38ecfbf7bd673c26f96833e61f7783bb Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 1 Jun 2020 19:22:15 -0600 Subject: [PATCH 94/95] Crash using VTX Client #284 * Fix try/catch around JSON.parse() * Ensure we don't try to replace on null/undefined no matter what --- core/servers/chat/mrc_multiplexer.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/servers/chat/mrc_multiplexer.js b/core/servers/chat/mrc_multiplexer.js index 67ca9dc6..34991de7 100644 --- a/core/servers/chat/mrc_multiplexer.js +++ b/core/servers/chat/mrc_multiplexer.js @@ -249,11 +249,10 @@ exports.getModule = class MrcModule extends ServerModule { receiveFromClient(username, message) { try { message = JSON.parse(message); + this.sendToMrcServer(message.from_user, message.from_room, message.to_user, message.to_site, message.to_room, message.body); } catch (e) { Log.debug({ server : 'MRC', user : username, message : message }, 'Dodgy message received from client'); } - - this.sendToMrcServer(message.from_user, message.from_room, message.to_user, message.to_site, message.to_room, message.body); } /** @@ -264,11 +263,11 @@ exports.getModule = class MrcModule extends ServerModule { const line = [ fromUser, this.boardName, - sanitiseRoomName(fromRoom), + sanitiseRoomName(fromRoom || ''), sanitiseName(toUser || ''), sanitiseName(toSite || ''), sanitiseRoomName(toRoom || ''), - sanitiseMessage(messageBody) + sanitiseMessage(messageBody || '') ].join('~') + '~'; // Log.debug({ server : 'MRC', data : line }, 'Sending data'); From 40739d028f8c960f61c6896b5bb117f902cdc0ca Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 1 Jun 2020 21:18:12 -0600 Subject: [PATCH 95/95] Update to telnet-socket 0.2.3 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 19cf7662..aee75136 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "ws": "^7.3.0", "xxhash": "^0.3.0", "yazl": "^2.5.1", - "telnet-socket" : "^0.2.2" + "telnet-socket" : "^0.2.3" }, "devDependencies": {}, "engines": { diff --git a/yarn.lock b/yarn.lock index fb655a43..0954c769 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1840,10 +1840,10 @@ tar@^4: safe-buffer "^5.1.2" yallist "^3.0.2" -telnet-socket@^0.2.2: - version "0.2.2" - resolved "https://registry.yarnpkg.com/telnet-socket/-/telnet-socket-0.2.2.tgz#9abf29bbd022f2bc1176fd35ab62ca94e62916c1" - integrity sha512-fKcpbCNn8NQ1wBshjliTLh4UmbXq0iYhoPsc48RTJfgdisxQOyLJe8dMwH26ezx03pEM4GWpg9ge+g4DGz7c/Q== +telnet-socket@^0.2.3: + version "0.2.3" + resolved "https://registry.yarnpkg.com/telnet-socket/-/telnet-socket-0.2.3.tgz#0ffdc64ea957cb64f8ac5287d45a857f1c05a16e" + integrity sha512-PbZycTkGq6VcVUa35FYFySx4pCzmJo4xoMX6cimls1/kv/lrgMfddKfgjBKt6HQuokkkDfieDhGLq/L/P2Unaw== dependencies: binary-parser "1.6.2" buffers "github:NuSkooler/node-buffers"