* Better logging of import/export stats & schedules

* Some code cleanup
* Remove all mkdirp module stuff -- use fs-extra

core/bbs.js

@@ -16,7 +16,7 @@ let async		= require('async');
 let util		= require('util');
 let _			= require('lodash');
 let assert		= require('assert');
-let mkdirp 		= require('mkdirp');
+let mkdirs		= require('fs-extra').mkdirs;
 
 //	our main entry point
 exports.bbsMain	= bbsMain;
@@ -89,7 +89,7 @@ function initialize(cb) {
 		[
 			function createMissingDirectories(callback) {
 				async.each(Object.keys(conf.config.paths), function entry(pathKey, next) {
-					mkdirp(conf.config.paths[pathKey], function dirCreated(err) {
+					mkdirs(conf.config.paths[pathKey], function dirCreated(err) {
 						if(err) {
 							console.error('Could not create path: ' + conf.config.paths[pathKey] + ': ' + err.toString());
 						}

core/scanner_tossers/ftn_bso.js

@@ -742,7 +742,7 @@ function FTNMessageScanTossModule() {
 		});
 	};
 	
-	this.importNetMailToArea = function(localAreaTag, header, message, cb) {
+	this.importEchoMailToArea = function(localAreaTag, header, message, cb) {
 		async.series(
 			[
 				function validateDestinationAddress(callback) {			
@@ -807,6 +807,12 @@ function FTNMessageScanTossModule() {
 		let packetHeader;
 				
         const packetOpts = { keepTearAndOrigin : true };
+		
+		let importStats = {
+			areaSuccess	: {},	//	areaTag->count
+			areaFail	: {},	//	areaTag->count
+			otherFail	: 0,
+		};
         
 		new ftnMailPacket.Packet(packetOpts).read(packetPath, (entryType, entryData, next) => {
 			if('header' === entryType) {
@@ -831,15 +837,21 @@ function FTNMessageScanTossModule() {
 					//
 					const localAreaTag = self.getLocalAreaTagByFtnAreaTag(areaTag);
 					if(localAreaTag) {
-						self.importNetMailToArea(localAreaTag, packetHeader, message, err => {
+						self.importEchoMailToArea(localAreaTag, packetHeader, message, err => {
 							if(err) {
+								//	bump area fail stats
+								importStats.areaFail[localAreaTag] = (importStats.areaFail[localAreaTag] || 0) + 1;
+								
 								if('SQLITE_CONSTRAINT' === err.code) {
 									Log.info(
-										{ subject : message.subject, uuid : message.uuid }, 
+										{ area : localAreaTag, subject : message.subject, uuid : message.uuid }, 
 										'Not importing non-unique message');
 									
 									return next(null);
 								}
+							} else {
+								//	bump area success
+								importStats.areaSuccess[localAreaTag] = (importStats.areaSuccess[localAreaTag] || 0) + 1;
 							}
 															
 							next(err);
@@ -849,14 +861,25 @@ function FTNMessageScanTossModule() {
 						//	No local area configured for this import
 						//
 						//	:TODO: Handle the "catch all" case, if configured 
+						Log.warn( { areaTag : areaTag }, 'No local area configured for this packet file!');
+						
+						//	bump generic failure
+						importStats.otherFail += 1;
+						
+						return next(null);
 					}
 				} else {
 					//
 					//	NetMail
 					//
+					Log.warn('NetMail import not yet implemented!');
+					return next(null);
 				}
 			}
 		}, err => {
+			const finalStats = Object.assign(importStats, { packetPath : packetPath } );
+			Log.info(finalStats, 'Import complete');
+			
 			cb(err);
 		});
 	};
@@ -1070,6 +1093,15 @@ FTNMessageScanTossModule.prototype.startup = function(cb) {
 		if(_.isObject(this.moduleConfig.schedule)) {
 			const exportSchedule = this.parseScheduleString(this.moduleConfig.schedule.export);
 			if(exportSchedule) {
+				Log.debug(
+					{ 
+						schedule	: this.moduleConfig.schedule.export,
+						schedOK		: -1 === exportSchedule.sched.error,
+						immediate	: exportSchedule.immediate ? true : false,
+					},
+					'Export schedule loaded'
+				);
+				
 				if(exportSchedule.sched && this.exportingStart()) {
 					this.exportTimer = later.setInterval( () => {
 						
@@ -1087,7 +1119,16 @@ FTNMessageScanTossModule.prototype.startup = function(cb) {
 			}
 			
 			const importSchedule = this.parseScheduleString(this.moduleConfig.schedule.import);
-			if(importSchedule) {			
+			if(importSchedule) {
+				Log.debug(
+					{
+						schedule	: this.moduleConfig.schedule.import,
+						schedOK		: -1 === importSchedule.sched.error,
+						watchFile	: _.isString(importSchedule.watchFile) ? importSchedule.watchFile : 'None',
+					},
+					'Import schedule loaded'
+				);
+				
 				if(importSchedule.sched) {					
 					this.importTimer = later.setInterval( () => {
 						tryImportNow('Performing scheduled message import/toss...');						
@@ -1161,22 +1202,6 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) {
 		return cb(new Error('Missing or invalid configuration'));
 	}
 	
-	//
-	//	Select all messages that have a message_id > our last scan ID.
-	//	Additionally exclude messages that have a ftn_attr_flags FtnProperty meta
-	//	as those came via import!
-	//
-	/*	
-	const getNewUuidsSql = 
-		`SELECT message_id, message_uuid
-		FROM message m
-		WHERE area_tag = ? AND message_id > ? AND
-			(SELECT COUNT(message_id) 
-			FROM message_meta 
-			WHERE message_id = m.message_id AND meta_category = 'FtnProperty' AND meta_name = 'ftn_attr_flags') = 0
-		ORDER BY message_id;`;
-	*/
-	
 	//
 	//	Select all messages with a |message_id| > |lastScanId|.
 	//	Additionally exclude messages with the System state_flags0 which will be present for
@@ -1193,7 +1218,7 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) {
 			WHERE message_id = m.message_id AND meta_category = 'System' AND meta_name = 'state_flags0') = 0
 		ORDER BY message_id;`;
 		
-	var self = this;		
+	let self = this;		
 		
 	async.each(Object.keys(Config.messageNetworks.ftn.areas), (areaTag, nextArea) => {
 		const areaConfig = Config.messageNetworks.ftn.areas[areaTag];

mods/abracadabra.js

@@ -1,26 +1,26 @@
 /* jslint node: true */
 'use strict';
 
-var MenuModule			= require('../core/menu_module.js').MenuModule;
+let MenuModule			= require('../core/menu_module.js').MenuModule;
-var DropFile			= require('../core/dropfile.js').DropFile;
+let DropFile			= require('../core/dropfile.js').DropFile;
-var door				= require('../core/door.js');
+let door				= require('../core/door.js');
-var theme				= require('../core/theme.js');
+let theme				= require('../core/theme.js');
-var ansi				= require('../core/ansi_term.js');
+let ansi				= require('../core/ansi_term.js');
 
-var async				= require('async');
+let async				= require('async');
-var assert				= require('assert');
+let assert				= require('assert');
-var mkdirp 				= require('mkdirp');
+let paths				= require('path');
-var paths				= require('path');
+let _					= require('lodash');
-var _					= require('lodash');
+let net					= require('net');
-var net					= require('net');
+let mkdirs				= require('fs-extra').mkdirs;
 
 //	:TODO: This should really be a system module... needs a little work to allow for such
 
 exports.getModule		= AbracadabraModule;
 
-var activeDoorNodeInstances = {};
+let activeDoorNodeInstances = {};
 
-var doorInstances = {};	//	name -> { count : <instCount>, { <nodeNum> : <inst> } }
+let doorInstances = {};	//	name -> { count : <instCount>, { <nodeNum> : <inst> } }
 
 exports.moduleInfo = {
 	name	: 'Abracadabra',
@@ -66,7 +66,7 @@ exports.moduleInfo = {
 function AbracadabraModule(options) {
 	MenuModule.call(this, options);
 
-	var self = this;
+	let self = this;
 
 	this.config = options.menuConfig.config;
 
@@ -128,7 +128,7 @@ function AbracadabraModule(options) {
 					self.dropFile	= new DropFile(self.client, self.config.dropFileType);
 					var fullPath	= self.dropFile.fullPath;
 
-					mkdirp(paths.dirname(fullPath), function dirCreated(err) {
+					mkdirs(paths.dirname(fullPath), function dirCreated(err) {
 						if(err) {
 							callback(err);
 						} else {
@@ -153,7 +153,7 @@ function AbracadabraModule(options) {
 
 	this.runDoor = function() {
 
-		var exeInfo = {
+		const exeInfo = {
 			cmd			: self.config.cmd,
 			args		: self.config.args,
 			io			: self.config.io || 'stdio',
@@ -163,9 +163,9 @@ function AbracadabraModule(options) {
 			//inhSocket	: self.client.output._handle.fd,
 		};
 
-		var doorInstance = new door.Door(self.client, exeInfo);
+		const doorInstance = new door.Door(self.client, exeInfo);
 
-		doorInstance.on('finished', function doorFinished() {
+		doorInstance.on('finished', () => {
 			self.prevMenu();
 		});
 

package.json

@@ -23,7 +23,6 @@
     "later": "1.2.0",
     "lodash": "^3.10.1",
     "minimist": "1.2.x",
-    "mkdirp": "0.5.x",
     "moment": "^2.11.0",
     "node-uuid": "^1.4.7",
     "ptyw.js": "^0.3.7",