moon
/
enigma-bbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

NetMail avail to oputil & export - WIP

This commit is contained in:
Bryan Ashby 2017-12-31 17:54:11 -07:00
parent 1c5a00313b
commit fc40641eeb
12 changed files with 961 additions and 420 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.eslintrc.json
View File

@ -24,6 +24,7 @@
"error",
"always"
],
"comma-dangle": 0
"comma-dangle": 0,
"no-trailing-spaces" :"warn"
}
}

14
config/prompt.hjson
View File

@ -72,6 +72,20 @@
}
}
loginSequenceFlavorSelect: {
art: LOGINSEL
mci: {
TM1: {
argName: promptValue
items: [ "yes", "no" ]
focus: true
focusItemIndex: 1
hotKeys: { Y: 0, N: 1 }
hotKeySubmit: true
}
}
}
loginGlobalNewScan: {
art: GNSPMPT
mci: {

1
core/config.js
View File

@ -620,6 +620,7 @@ function getDefaultConfig() {
inbound : paths.join(__dirname, './../mail/ftn_in/'),
secInbound : paths.join(__dirname, './../mail/ftn_secin/'),
reject : paths.join(__dirname, './../mail/reject/'), // bad pkt, bundles, TIC attachments that fail any check, etc.
outboundNetMail : paths.join(__dirname, './../mail/ftn_netmail_out/'),
// set 'retain' to a valid path to keep good pkt files
},

48
core/ftn_mail_packet.js
View File

@ -393,9 +393,19 @@ function Packet(options) {
};
function addKludgeLine(line) {
//
// We have to special case INTL/TOPT/FMPT as they don't contain
// a ':' name/value separator like the rest of the kludge lines... because stupdity.
//
let key = line.substr(0, 4);
let value;
if( ['INTL', 'TOPT', 'FMPT' ].includes(key)) {
value = line.substr(4).trim();
} else {
const sepIndex = line.indexOf(':');
const key = line.substr(0, sepIndex).toUpperCase();
const value = line.substr(sepIndex + 1).trim();
key = line.substr(0, sepIndex).toUpperCase();
value = line.substr(sepIndex + 1).trim();
}
//
// Allow mapped value to be either a key:value if there is only
@ -639,7 +649,7 @@ function Packet(options) {
this.getMessageEntryBuffer = function(message, options, cb) {
function getAppendMeta(k, m) {
function getAppendMeta(k, m, sepChar=':') {
let append = '';
if(m) {
let a = m;
@ -647,7 +657,7 @@ function Packet(options) {
a = [ a ];
}
a.forEach(v => {
append += `${k}: ${v}\r`;
append += `${k}${sepChar} ${v}\r`;
});
}
return append;
@ -693,10 +703,21 @@ function Packet(options) {
msgBody += `AREA:${message.meta.FtnProperty.ftn_area}\r`; // note: no ^A (0x01)
}
// :TODO: DRY with similar function in this file!
Object.keys(message.meta.FtnKludge).forEach(k => {
// we want PATH to be last
if('PATH' !== k) {
switch(k) {
case 'PATH' :
break; // skip & save for last
case 'FMPT' :
case 'TOPT' :
case 'INTL' :
msgBody += getAppendMeta(`\x01${k}`, message.meta.FtnKludge[k], ''); // no sepChar
break;
default :
msgBody += getAppendMeta(`\x01${k}`, message.meta.FtnKludge[k]);
break;
}
});
@ -810,14 +831,14 @@ function Packet(options) {
// :TODO: Put this in it's own method
let msgBody = '';
function appendMeta(k, m) {
function appendMeta(k, m, sepChar=':') {
if(m) {
let a = m;
if(!_.isArray(a)) {
a = [ a ];
}
a.forEach(v => {
msgBody += `${k}: ${v}\r`;
msgBody += `${k}${sepChar} ${v}\r`;
});
}
}
@ -832,9 +853,14 @@ function Packet(options) {
}
Object.keys(message.meta.FtnKludge).forEach(k => {
// we want PATH to be last
if('PATH' !== k) {
appendMeta(`\x01${k}`, message.meta.FtnKludge[k]);
switch(k) {
case 'PATH' : break; // skip & save for last
case 'FMPT' :
case 'TOPT' :
case 'INTL' : appendMeta(`\x01${k}`, message.meta.FtnKludge[k], ''); break; // no sepChar
default : appendMeta(`\x01${k}`, message.meta.FtnKludge[k]); break;
}
});

15
core/ftn_util.js
View File

@ -26,6 +26,7 @@ exports.getUTCTimeZoneOffset = getUTCTimeZoneOffset;
exports.getOrigin = getOrigin;
exports.getTearLine = getTearLine;
exports.getVia = getVia;
exports.getIntl = getIntl;
exports.getAbbreviatedNetNodeList = getAbbreviatedNetNodeList;
exports.parseAbbreviatedNetNodeList = parseAbbreviatedNetNodeList;
exports.getUpdatedSeenByEntries = getUpdatedSeenByEntries;
@ -222,6 +223,20 @@ function getVia(address) {
return `${addrStr} @${dateTime} ENiGMA1/2 ${version}`;
}
//
// Creates a INTL kludge value as per FTS-4001
// http://retro.fidoweb.ru/docs/index=ftsc&doc=FTS-4001&enc=mac
//
function getIntl(toAddress, fromAddress) {
//
// INTL differs from 'standard' kludges in that there is no ':' after "INTL"
//
// "<SOH>"INTL "<destination address>" "<origin address><CR>"
// "...These addresses shall be given on the form <zone>:<net>/<node>"
//
return `${toAddress.toString('3D')} ${fromAddress.toString('3D')}`;
}
function getAbbreviatedNetNodeList(netNodes) {
let abbrList = '';
let currNet;

6
core/message.js
View File

@ -124,11 +124,13 @@ Message.FtnPropertyNames = {
// Note: kludges are stored with their names as-is
Message.prototype.setLocalToUserId = function(userId) {
this.meta.System.local_to_user_id = userId;
this.meta.System = this.meta.System || {};
this.meta.System[Message.SystemMetaNames.LocalToUserID] = userId;
};
Message.prototype.setLocalFromUserId = function(userId) {
this.meta.System.local_from_user_id = userId;
this.meta.System = this.meta.System || {};
this.meta.System[Message.SystemMetaNames.LocalFromUserID] = userId;
};
Message.createMessageUUID = function(areaTag, modTimestamp, subject, body) {

2
core/oputil/oputil_file_base.js
View File

@ -8,7 +8,7 @@ const argv = require('./oputil_common.js').argv;
const initConfigAndDatabases = require('./oputil_common.js').initConfigAndDatabases;
const getHelpFor = require('./oputil_help.js').getHelpFor;
const getAreaAndStorage = require('./oputil_common.js').getAreaAndStorage;
const Errors = require('../../core/enig_error.js').Errors;
const Errors = require('../enig_error.js').Errors;
const async = require('async');
const fs = require('graceful-fs');

8
core/oputil/oputil_help.js
View File

@ -84,6 +84,14 @@ general information:
FILENAME_WC filename with * and ? wildcard support. may match 0:n entries
SHA full or partial SHA-256
FILE_ID a file identifier. see file.sqlite3
`,
MessageBase :
`usage: oputil.js mb <action> [<args>]
actions:
areafix CMD1 CMD2 ... ADDR sends an AreaFix NetMail to ADDR with the supplied command(s)
one or more commands may be supplied. commands that are multi
part such as "%COMPRESS ZIP" should be quoted.
`
};

20
core/oputil/oputil_main.js
View File

@ -7,6 +7,7 @@ const argv = require('./oputil_common.js').argv;
const printUsageAndSetExitCode = require('./oputil_common.js').printUsageAndSetExitCode;
const handleUserCommand = require('./oputil_user.js').handleUserCommand;
const handleFileBaseCommand = require('./oputil_file_base.js').handleFileBaseCommand;
const handleMessageBaseCommand = require('./oputil_message_base.js').handleMessageBaseCommand;
const handleConfigCommand = require('./oputil_config.js').handleConfigCommand;
const getHelpFor = require('./oputil_help.js').getHelpFor;
@ -26,19 +27,10 @@ module.exports = function() {
}
switch(argv._[0]) {
case 'user' :
handleUserCommand();
break;
case 'config' :
handleConfigCommand();
break;
case 'fb' :
handleFileBaseCommand();
break;
default:
return printUsageAndSetExitCode(getHelpFor('General'), ExitCodes.BAD_COMMAND);
case 'user' : return handleUserCommand();
case 'config' : return handleConfigCommand();
case 'fb' : return handleFileBaseCommand();
case 'mb' : return handleMessageBaseCommand();
default : return printUsageAndSetExitCode(getHelpFor('General'), ExitCodes.BAD_COMMAND);
}
};

150
core/oputil/oputil_message_base.js Normal file
View File

@ -0,0 +1,150 @@
/* jslint node: true */
/* eslint-disable no-console */
'use strict';
const printUsageAndSetExitCode = require('./oputil_common.js').printUsageAndSetExitCode;
const ExitCodes = require('./oputil_common.js').ExitCodes;
const argv = require('./oputil_common.js').argv;
const initConfigAndDatabases = require('./oputil_common.js').initConfigAndDatabases;
const getHelpFor = require('./oputil_help.js').getHelpFor;
const Address = require('../ftn_address.js');
const Errors = require('../enig_error.js').Errors;
// deps
const async = require('async');
exports.handleMessageBaseCommand = handleMessageBaseCommand;
function areaFix() {
//
// oputil mb areafix CMD1 CMD2 ... ADDR [--password PASS]
//
if(argv._.length < 3) {
return printUsageAndSetExitCode(
getHelpFor('MessageBase'),
ExitCodes.ERROR
);
}
async.waterfall(
[
function init(callback) {
return initConfigAndDatabases(callback);
},
function validateAddress(callback) {
const addrArg = argv._.slice(-1)[0];
const ftnAddr = Address.fromString(addrArg);
if(!ftnAddr) {
return callback(Errors.Invalid(`"${addrArg}" is not a valid FTN address`));
}
//
// We need to validate the address targets a system we know unless
// the --force option is used
//
// :TODO:
return callback(null, ftnAddr);
},
function fetchFromUser(ftnAddr, callback) {
//
// --from USER || +op from system
//
// If possible, we want the user ID of the supplied user as well
//
const User = require('../user.js');
if(argv.from) {
User.getUserIdAndName(argv.from, (err, userId, fromName) => {
if(err) {
return callback(null, ftnAddr, argv.from, 0);
}
// fromName is the same as argv.from, but case may be differnet (yet correct)
return callback(null, ftnAddr, fromName, userId);
});
} else {
User.getUserName(User.RootUserID, (err, fromName) => {
return callback(null, ftnAddr, fromName || 'SysOp', err ? 0 : User.RootUserID);
});
}
},
function createMessage(ftnAddr, fromName, fromUserId, callback) {
//
// Build message as commands separated by line feed
//
// We need to remove quotes from arguments. These are required
// in the case of e.g. removing an area: "-SOME_AREA" would end
// up confusing minimist, therefor they must be quoted: "'-SOME_AREA'"
//
const messageBody = argv._.slice(2, -1).map(arg => {
return arg.replace(/["']/g, '');
}).join('\n') + '\n';
const Message = require('../message.js');
const message = new Message({
toUserName : argv.to || 'AreaFix',
fromUserName : fromName,
subject : argv.password || '',
message : messageBody,
areaTag : Message.WellKnownAreaTags.Private, // mark private
meta : {
FtnProperty : {
[ Message.FtnPropertyNames.FtnDestZone ] : ftnAddr.zone,
[ Message.FtnPropertyNames.FtnDestNetwork ] : ftnAddr.net,
[ Message.FtnPropertyNames.FtnDestNode ] : ftnAddr.node,
}
}
});
if(ftnAddr.point) {
message.meta.FtnProperty[Message.FtnPropertyNames.FtnDestPoint] = ftnAddr.point;
}
if(0 !== fromUserId) {
message.setLocalFromUserId(fromUserId);
}
return callback(null, message);
},
function persistMessage(message, callback) {
//
// :TODO:
// - Persist message in private outgoing (sysop out box)
// - Make necessary changes such that the message is exported properly
//
console.log(message);
message.persist(err => {
return callback(err);
});
}
],
err => {
if(err) {
process.exitCode = ExitCodes.ERROR;
console.error(`${err.message}${err.reason ? ': ' + err.reason : ''}`);
}
}
);
}
function handleMessageBaseCommand() {
function errUsage() {
return printUsageAndSetExitCode(
getHelpFor('MessageBase'),
ExitCodes.ERROR
);
}
if(true === argv.help) {
return errUsage();
}
const action = argv._[1];
return({
areafix : areaFix,
}[action] || errUsage)();
}

524
core/scanner_tossers/ftn_bso.js
View File

@ -59,7 +59,7 @@ const SCHEDULE_REGEXP = /(?:^|or )?(@watch\:|@immediate)([^\0]+)?$/;
function FTNMessageScanTossModule() {
MessageScanTossModule.call(this);
let self = this;
const self = this;
this.archUtil = ArchiveUtil.getInstance();
@ -78,7 +78,6 @@ function FTNMessageScanTossModule() {
}
};
this.getDefaultZone = function(networkName) {
if(_.isNumber(Config.messageNetworks.ftn.networks[networkName].defaultZone)) {
return Config.messageNetworks.ftn.networks[networkName].defaultZone;
@ -143,7 +142,7 @@ function FTNMessageScanTossModule() {
};
/*
this.getOutgoingPacketDir = function(networkName, destAddress) {
this.getOutgoingEchoMailPacketDir = function(networkName, destAddress) {
let dir = this.moduleConfig.paths.outbound;
if(!this.isDefaultDomainZone(networkName, destAddress)) {
const hexZone = `000${destAddress.zone.toString(16)}`.substr(-3);
@ -153,7 +152,7 @@ function FTNMessageScanTossModule() {
};
*/
this.getOutgoingPacketDir = function(networkName, destAddress) {
this.getOutgoingEchoMailPacketDir = function(networkName, destAddress) {
networkName = networkName.toLowerCase();
let dir = this.moduleConfig.paths.outbound;
@ -311,18 +310,26 @@ function FTNMessageScanTossModule() {
message.meta.FtnProperty = message.meta.FtnProperty || {};
message.meta.FtnKludge = message.meta.FtnKludge || {};
// :TODO: Only set DEST information for EchoMail - NetMail should already have it in message
message.meta.FtnProperty.ftn_orig_node = options.network.localAddress.node;
message.meta.FtnProperty.ftn_dest_node = options.destAddress.node;
//message.meta.FtnProperty.ftn_dest_node = options.destAddress.node;
message.meta.FtnProperty.ftn_orig_network = options.network.localAddress.net;
message.meta.FtnProperty.ftn_dest_network = options.destAddress.net;
//message.meta.FtnProperty.ftn_dest_network = options.destAddress.net;
message.meta.FtnProperty.ftn_cost = 0;
message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine();
// :TODO: Need an explicit isNetMail() check
// tear line and origin can both go in EchoMail & NetMail
message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine();
message.meta.FtnProperty.ftn_origin = ftnUtil.getOrigin(options.network.localAddress);
// :TODO: Need an explicit isNetMail() check or more generally
let ftnAttribute =
ftnMailPacket.Packet.Attribute.Local; // message from our system
if(message.isPrivate()) {
// These should be set for Private/NetMail already
assert(_.isNumber(parseInt(message.meta.FtnProperty.ftn_dest_node)));
assert(_.isNumber(parseInt(message.meta.FtnProperty.ftn_dest_network)));
ftnAttribute |= ftnMailPacket.Packet.Attribute.Private;
//
@ -334,7 +341,25 @@ function FTNMessageScanTossModule() {
}
message.meta.FtnKludge.Via = message.meta.FtnKludge.Via || [];
message.meta.FtnKludge.Via.push(ftnUtil.getVia(options.network.localAddress));
//
// We need to set INTL, and possibly FMPT and/or TOPT
// See http://retro.fidoweb.ru/docs/index=ftsc&doc=FTS-4001&enc=mac
//
message.meta.FtnKludge.INTL = ftnUtil.getIntl(options.network.localAddress, options.destAddress);
if(_.isNumber(options.network.localAddress.point) && options.network.localAddress.point > 0) {
message.meta.FtnKludge.FMPT = options.network.localAddress.point;
}
if(_.get(message, 'meta.FtnProperty.ftn_dest_point', 0) > 0) {
message.meta.FtnKludge.TOPT = message.meta.FtnProperty.ftn_dest_point;
}
} else {
// We need to set some destination info for EchoMail
message.meta.FtnProperty.ftn_dest_node = options.destAddress.node;
message.meta.FtnProperty.ftn_dest_network = options.destAddress.net;
//
// Set appropriate attribute flag for export type
//
@ -347,7 +372,6 @@ function FTNMessageScanTossModule() {
//
// EchoMail requires some additional properties & kludges
//
message.meta.FtnProperty.ftn_origin = ftnUtil.getOrigin(options.network.localAddress);
message.meta.FtnProperty.ftn_area = Config.messageNetworks.ftn.areas[message.areaTag].tag;
//
@ -502,7 +526,7 @@ function FTNMessageScanTossModule() {
LIMIT 1;`;
msgDb.get(sql, [ areaTag ], (err, row) => {
cb(err, row ? row.message_id : 0);
return cb(err, row ? row.message_id : 0);
});
};
@ -512,12 +536,21 @@ function FTNMessageScanTossModule() {
VALUES ("ftn_bso", ?, ?);`;
msgDb.run(sql, [ areaTag, lastScanId ], err => {
cb(err);
return cb(err);
});
};
this.getNodeConfigByAddress = function(addr) {
addr = _.isString(addr) ? Address.fromString(addr) : addr;
// :TODO: sort wildcard nodes{} entries by most->least explicit according to FTN hierarchy
return _.find(this.moduleConfig.nodes, (node, nodeAddrWildcard) => {
return addr.isPatternMatch(nodeAddrWildcard);
});
};
// :TODO: deprecate this in favor of getNodeConfigByAddress()
this.getNodeConfigKeyByAddress = function(uplink) {
// :TODO: sort by least # of '*' & take top?
const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => {
return Address.fromString(addr).isPatternMatch(uplink);
})[0];
@ -525,6 +558,63 @@ function FTNMessageScanTossModule() {
return nodeKey;
};
this.exportNetMailMessagePacket = function(message, exportOpts, cb) {
//
// For NetMail, we always create a *single* packet per message.
//
async.series(
[
function generalPrep(callback) {
// :TODO: Double check Via spec -- seems like it may be wrong
self.prepareMessage(message, exportOpts);
return self.setReplyKludgeFromReplyToMsgId(message, callback);
},
function createPacket(callback) {
const packet = new ftnMailPacket.Packet();
const packetHeader = new ftnMailPacket.PacketHeader(
exportOpts.network.localAddress,
exportOpts.destAddress,
exportOpts.nodeConfig.packetType
);
packetHeader.password = exportOpts.nodeConfig.packetPassword || '';
// use current message ID for filename seed
exportOpts.pktFileName = self.getOutgoingPacketFileName(
self.exportTempDir,
message.messageId,
false, // createTempPacket=false
exportOpts.fileCase
);
const ws = fs.createWriteStream(exportOpts.pktFileName);
packet.writeHeader(ws, packetHeader);
packet.getMessageEntryBuffer(message, exportOpts, (err, msgBuf) => {
if(err) {
return callback(err);
}
ws.write(msgBuf);
packet.writeTerminator(ws);
ws.end();
ws.once('finish', () => {
return callback(null);
});
});
}
],
err => {
return cb(err);
}
);
};
this.exportMessagesByUuid = function(messageUuids, exportOpts, cb) {
//
// This method has a lot of madness going on:
@ -701,7 +791,157 @@ function FTNMessageScanTossModule() {
});
};
this.exportMessagesToUplinks = function(messageUuids, areaConfig, cb) {
this.getNetMailRoute = function(dstAddr) {
//
// messageNetworks.ftn.netMail.routes{} full|wildcard -> full adddress lookup
//
const routes = _.get(Config, 'messageNetworks.ftn.netMail.routes');
if(!routes) {
return;
}
return _.find(routes, (route, addrWildcard) => {
return dstAddr.isPatternMatch(addrWildcard);
});
/*
const route = _.find(routes, (route, addrWildcard) => {
return dstAddr.isPatternMatch(addrWildcard);
});
if(route && route.address) {
return Address.fromString(route.address);
}
*/
};
this.getAcceptableNetMailNetworkInfoFromAddress = function(dstAddr, cb) {
//
// Attempt to find an acceptable network configuration using the following
// lookup order (most to least explicit config):
//
// 1) Routes: messageNetworks.ftn.netMail.routes{} -> scannerTossers.ftn_bso.nodes{} -> config
// - Where we send may not be where dstAddress is (it's routed!); use network found in route
// for local address
// 2) Direct to nodes: scannerTossers.ftn_bso.nodes{} -> config
// - Where we send is direct to dstAddr; use scannerTossers.ftn_bso.defaultNetwork to
// for local address
// 3) Nodelist DB lookup (use default config)
// - Where we send is direct to dstAddr
//
//const routeAddress = this.getNetMailRouteAddress(dstAddr) || dstAddr;
const route = this.getNetMailRoute(dstAddr);
let routeAddress;
let networkName;
if(route) {
routeAddress = Address.fromString(route.address);
networkName = route.network || Config.scannerTossers.ftn_bso.defaultNetwork;
} else {
routeAddress = dstAddr;
networkName = Config.scannerTossers.ftn_bso.defaultNetwork;
}
const config = _.find(this.moduleConfig.nodes, (node, nodeAddrWildcard) => {
return routeAddress.isPatternMatch(nodeAddrWildcard);
}) || {
packetType : '2+',
encoding : Config.scannerTossers.ftn_bso.packetMsgEncoding,
};
return cb(
config ? null : Errors.DoesNotExist(`No configuration found for ${dstAddr.toString()}`),
config, routeAddress, networkName
);
};
this.exportNetMailMessagesToUplinks = function(messagesOrMessageUuids, cb) {
// for each message/UUID, find where to send the thing
async.each(messagesOrMessageUuids, (msgOrUuid, nextMessageOrUuid) => {
const exportOpts = {};
const message = new Message();
async.series(
[
function loadMessage(callback) {
if(_.isString(msgOrUuid)) {
message.load( { uuid : msgOrUuid }, err => {
return callback(err, message);
});
} else {
return callback(null, msgOrUuid);
}
},
function discoverUplink(callback) {
const dstAddr = new Address({
zone : parseInt(message.meta.FtnProperty.ftn_dest_zone),
net : parseInt(message.meta.FtnProperty.ftn_dest_network),
node : parseInt(message.meta.FtnProperty.ftn_dest_node),
point : parseInt(message.meta.FtnProperty.ftn_dest_point) || null, // point is optional
});
return self.getAcceptableNetMailNetworkInfoFromAddress(dstAddr, (err, config, routeAddress, networkName) => {
if(err) {
return callback(err);
}
exportOpts.nodeConfig = config;
exportOpts.destAddress = routeAddress;
exportOpts.fileCase = config.fileCase || 'lower';
exportOpts.network = Config.messageNetworks.ftn.networks[networkName];
exportOpts.networkName = networkName;
if(!exportOpts.network) {
return callback(Errors.DoesNotExist(`No configuration found for network ${networkName}`));
}
return callback(null);
});
},
function createOutgoingDir(callback) {
// ensure outgoing NetMail directory exists
return fse.mkdirs(Config.scannerTossers.ftn_bso.paths.outboundNetMail, callback);
},
function exportPacket(callback) {
return self.exportNetMailMessagePacket(message, exportOpts, callback);
},
function moveToOutgoing(callback) {
const newExt = exportOpts.fileCase === 'lower' ? '.pkt' : '.PKT';
const newPath = paths.join(
Config.scannerTossers.ftn_bso.paths.outboundNetMail,
`${paths.basename(exportOpts.pktFileName, paths.extname(exportOpts.pktFileName))}${newExt}`
);
return fse.move(exportOpts.pktFileName, newPath, callback);
},
function storeStateFlags0Meta(callback) {
return message.persistMetaValue('System', 'state_flags0', Message.StateFlags0.Exported.toString(), callback);
},
function storeMsgIdMeta(callback) {
// Store meta as if we had imported this message -- for later reference
if(message.meta.FtnKludge.MSGID) {
return message.persistMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.MSGID, callback);
}
return callback(null);
}
],
err => {
if(err) {
Log.warn( { error :err.message }, 'Error exporting message' );
}
return nextMessageOrUuid(null);
}
);
}, err => {
return cb(err);
});
};
this.exportEchoMailMessagesToUplinks = function(messageUuids, areaConfig, cb) {
async.each(areaConfig.uplinks, (uplink, nextUplink) => {
const nodeConfigKey = self.getNodeConfigKeyByAddress(uplink);
if(!nodeConfigKey) {
@ -720,7 +960,7 @@ function FTNMessageScanTossModule() {
exportOpts.network.localAddress = Address.fromString(exportOpts.network.localAddress);
}
const outgoingDir = self.getOutgoingPacketDir(exportOpts.networkName, exportOpts.destAddress);
const outgoingDir = self.getOutgoingEchoMailPacketDir(exportOpts.networkName, exportOpts.destAddress);
const exportType = self.getExportType(exportOpts.nodeConfig);
async.waterfall(
@ -1253,13 +1493,17 @@ function FTNMessageScanTossModule() {
};
// ends an export block
this.exportingEnd = function() {
this.exportingEnd = function(cb) {
this.exportRunning = false;
if(cb) {
return cb(null);
}
};
this.copyTicAttachment = function(src, dst, isUpdate, cb) {
if(isUpdate) {
fse.copy(src, dst, err => {
fse.copy(src, dst, { overwrite : true }, err => {
return cb(err, dst);
});
} else {
@ -1274,8 +1518,6 @@ function FTNMessageScanTossModule() {
};
this.processSingleTicFile = function(ticFileInfo, cb) {
const self = this;
Log.debug( { tic : ticFileInfo.path, file : ticFileInfo.getAsString('File') }, 'Processing TIC file');
async.waterfall(
@ -1527,6 +1769,145 @@ function FTNMessageScanTossModule() {
return cb(err);
});
};
this.performEchoMailExport = function(cb) {
//
// Select all messages with a |message_id| > |lastScanId|.
// Additionally exclude messages with the System state_flags0 which will be present for
// imported or already exported messages
//
// NOTE: If StateFlags0 starts to use additional bits, we'll likely need to check them here!
//
const getNewUuidsSql =
`SELECT message_id, message_uuid
FROM message m
WHERE area_tag = ? AND message_id > ? AND
(SELECT COUNT(message_id)
FROM message_meta
WHERE message_id = m.message_id AND meta_category = 'System' AND meta_name = 'state_flags0') = 0
ORDER BY message_id;`
;
async.each(Object.keys(Config.messageNetworks.ftn.areas), (areaTag, nextArea) => {
const areaConfig = Config.messageNetworks.ftn.areas[areaTag];
if(!this.isAreaConfigValid(areaConfig)) {
return nextArea();
}
//
// For each message that is newer than that of the last scan
// we need to export to each configured associated uplink(s)
//
async.waterfall(
[
function getLastScanId(callback) {
self.getAreaLastScanId(areaTag, callback);
},
function getNewUuids(lastScanId, callback) {
msgDb.all(getNewUuidsSql, [ areaTag, lastScanId ], (err, rows) => {
if(err) {
callback(err);
} else {
if(0 === rows.length) {
let nothingToDoErr = new Error('Nothing to do!');
nothingToDoErr.noRows = true;
callback(nothingToDoErr);
} else {
callback(null, rows);
}
}
});
},
function exportToConfiguredUplinks(msgRows, callback) {
const uuidsOnly = msgRows.map(r => r.message_uuid); // convert to array of UUIDs only
self.exportEchoMailMessagesToUplinks(uuidsOnly, areaConfig, err => {
const newLastScanId = msgRows[msgRows.length - 1].message_id;
Log.info(
{ areaTag : areaTag, messagesExported : msgRows.length, newLastScanId : newLastScanId },
'Export complete');
callback(err, newLastScanId);
});
},
function updateLastScanId(newLastScanId, callback) {
self.setAreaLastScanId(areaTag, newLastScanId, callback);
}
],
() => {
return nextArea();
}
);
},
err => {
return cb(err);
});
};
this.performNetMailExport = function(cb) {
//
// Select all messages with a |message_id| > |lastScanId| in the private area
// that also *do not* have a local user ID meta value but *do* have a FTN dest
// network meta value.
//
// Just like EchoMail, we additionally exclude messages with the System state_flags0
// which will be present for imported or already exported messages
//
// NOTE: If StateFlags0 starts to use additional bits, we'll likely need to check them here!
//
// :TODO: fill out the rest of the consts here
// :TODO: this statement is crazy ugly
const getNewUuidsSql =
`SELECT message_id, message_uuid
FROM message m
WHERE area_tag = '${Message.WellKnownAreaTags.Private}' AND message_id > ? AND
(SELECT COUNT(message_id)
FROM message_meta
WHERE message_id = m.message_id AND meta_category = 'System' AND
(meta_name = 'state_flags0' OR meta_name='local_to_user_id')) = 0
AND
(SELECT COUNT(message_id)
FROM message_meta
WHERE message_id = m.message_id AND meta_category='FtnProperty' AND meta_name='ftn_dest_network') = 1
ORDER BY message_id;
`;
async.waterfall(
[
function getLastScanId(callback) {
return self.getAreaLastScanId(Message.WellKnownAreaTags.Private, callback);
},
function getNewUuids(lastScanId, callback) {
msgDb.all(getNewUuidsSql, [ lastScanId ], (err, rows) => {
if(err) {
return callback(err);
}
if(0 === rows.length) {
return cb(null); // note |cb| -- early bail out!
}
return callback(null, rows);
});
},
function exportMessages(rows, callback) {
const messageUuids = rows.map(r => r.message_uuid);
return self.exportNetMailMessagesToUplinks(messageUuids, callback);
}
],
err => {
return cb(err);
}
);
};
this.isNetMailMessage = function(message) {
return message.isPrivate() &&
null === _.get(message.meta, 'System.LocalToUserID', null) &&
null !== _.get(message.meta, 'FtnProperty.ftn_dest_network')
;
};
}
require('util').inherits(FTNMessageScanTossModule, MessageScanTossModule);
@ -1769,76 +2150,17 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) {
return cb(new Error('Missing or invalid configuration'));
}
//
// Select all messages with a |message_id| > |lastScanId|.
// Additionally exclude messages with the System state_flags0 which will be present for
// imported or already exported messages
//
// NOTE: If StateFlags0 starts to use additional bits, we'll likely need to check them here!
//
const getNewUuidsSql =
`SELECT message_id, message_uuid
FROM message m
WHERE area_tag = ? AND message_id > ? AND
(SELECT COUNT(message_id)
FROM message_meta
WHERE message_id = m.message_id AND meta_category = 'System' AND meta_name = 'state_flags0') = 0
ORDER BY message_id;`;
const self = this;
let self = this;
async.each(Object.keys(Config.messageNetworks.ftn.areas), (areaTag, nextArea) => {
const areaConfig = Config.messageNetworks.ftn.areas[areaTag];
if(!this.isAreaConfigValid(areaConfig)) {
return nextArea();
}
//
// For each message that is newer than that of the last scan
// we need to export to each configured associated uplink(s)
//
async.waterfall(
[
function getLastScanId(callback) {
self.getAreaLastScanId(areaTag, callback);
},
function getNewUuids(lastScanId, callback) {
msgDb.all(getNewUuidsSql, [ areaTag, lastScanId ], (err, rows) => {
async.eachSeries( [ 'EchoMail', 'NetMail' ], (type, nextType) => {
self[`perform${type}Export`]( err => {
if(err) {
callback(err);
} else {
if(0 === rows.length) {
let nothingToDoErr = new Error('Nothing to do!');
nothingToDoErr.noRows = true;
callback(nothingToDoErr);
} else {
callback(null, rows);
}
Log.warn( { error : err.message, type : type }, 'Error(s) during export' );
}
return nextType(null); // try next, always
});
},
function exportToConfiguredUplinks(msgRows, callback) {
const uuidsOnly = msgRows.map(r => r.message_uuid); // convert to array of UUIDs only
self.exportMessagesToUplinks(uuidsOnly, areaConfig, err => {
const newLastScanId = msgRows[msgRows.length - 1].message_id;
Log.info(
{ areaTag : areaTag, messagesExported : msgRows.length, newLastScanId : newLastScanId },
'Export complete');
callback(err, newLastScanId);
});
},
function updateLastScanId(newLastScanId, callback) {
self.setAreaLastScanId(areaTag, newLastScanId, callback);
}
],
() => {
return nextArea();
}
);
}, err => {
return cb(err);
}, () => {
return cb(null);
});
};
@ -1850,25 +2172,35 @@ FTNMessageScanTossModule.prototype.record = function(message) {
return;
}
if(message.isPrivate()) {
// :TODO: support NetMail
const info = { uuid : message.uuid, subject : message.subject };
function exportLog(err) {
if(err) {
Log.warn(info, 'Failed exporting message');
} else {
Log.info(info, 'Message exported');
}
}
if(this.isNetMailMessage(message)) {
Object.assign(info, { type : 'NetMail' } );
if(this.exportingStart()) {
this.exportNetMailMessagesToUplinks( [ message.uuid ], err => {
this.exportingEnd( () => exportLog(err) );
});
}
} else if(message.areaTag) {
Object.assign(info, { type : 'EchoMail' } );
const areaConfig = Config.messageNetworks.ftn.areas[message.areaTag];
if(!this.isAreaConfigValid(areaConfig)) {
return;
}
if(this.exportingStart()) {
this.exportMessagesToUplinks( [ message.uuid ], areaConfig, err => {
const info = { uuid : message.uuid, subject : message.subject };
if(err) {
Log.warn(info, 'Failed exporting message');
} else {
Log.info(info, 'Message exported');
}
this.exportingEnd();
this.exportEchoMailMessagesToUplinks( [ message.uuid ], areaConfig, err => {
this.exportingEnd( () => exportLog(err) );
});
}
}

12
package.json
View File

@ -27,19 +27,19 @@
"buffers": "NuSkooler/node-buffers",
"bunyan": "^1.8.12",
"exiftool": "^0.0.3",
"fs-extra": "^4.0.1",
"fs-extra": "^5.0.0",
"glob": "^7.1.2",
"graceful-fs": "^4.1.11",
"hashids": "^1.1.1",
"hjson": "^3.1.0",
"iconv-lite": "^0.4.18",
"inquirer": "^3.2.3",
"inquirer": "^4.0.1",
"later": "1.2.0",
"lodash": "^4.17.4",
"mime-types": "^2.1.17",
"minimist": "1.2.x",
"moment": "^2.18.1",
"node-glob": "^1.2.0",
"nodemailer": "^4.1.0",
"moment": "^2.20.0",
"nodemailer": "^4.4.1",
"ptyw.js": "NuSkooler/ptyw.js",
"rlogin": "^1.0.0",
"sane": "^2.2.0",
@ -50,7 +50,7 @@
"temptmp": "^1.0.0",
"uuid": "^3.1.0",
"uuid-parse": "^1.0.0",
"ws": "^3.1.0",
"ws": "^3.3.3",
"xxhash": "^0.2.4",
"yazl": "^2.4.2"
},