Merge branch 'master' of ssh://numinibsd/git/base/enigma-bbs

README.md

@@ -1,6 +1,6 @@
 # ENiGMA½ BBS Software
 
-![alt text](http://i325.photobucket.com/albums/k361/request4spam/enigma.ans_zps05w2ey4s.png "ENiGMA½ BBS")
+![alt text](docs/images/enigma-bbs.png "ENiGMA½ BBS")
 
 ENiGMA½ is a modern BBS software with a nostalgic flair!
 
@@ -9,26 +9,25 @@ ENiGMA½ is a modern BBS software with a nostalgic flair!
  * Multi platform: Anywhere [Node.js](https://nodejs.org/) runs likely works (known to work under Linux, FreeBSD, OpenBSD, OS X and Windows)
  * Unlimited multi node support (for all those BBS "callers"!)
  * **Highly** customizable via [HJSON](http://hjson.org/) based configuration, menus, and themes in addition to JavaScript based mods
- * MCI support for lightbars, toggles, input areas, and so on plus many other other bells and whistles
+ * [MCI support](docs/mci.md) for lightbars, toggles, input areas, and so on plus many other other bells and whistles
- * Telnet & **SSH** access built in. Additional servers are easy to implement
+ * Telnet, **SSH**, and both secure and non-secure [WebSocket](https://en.wikipedia.org/wiki/WebSocket) access built in! Additional servers are easy to implement
  * [CP437](http://www.ascii-codes.com/) and UTF-8 output
  * [SyncTerm](http://syncterm.bbsdev.net/) style font and baud emulation support. Display PC/DOS and Amiga style artwork as it's intended! In general, ANSI-BBS / [cterm.txt](http://cvs.synchro.net/cgi-bin/viewcvs.cgi/*checkout*/src/conio/cterm.txt?content-type=text%2Fplain&revision=HEAD) / [bansi.txt](http://www.bbsdocumentary.com/library/PROGRAMS/GRAPHICS/ANSI/bansi.txt) are followed for expected BBS behavior
- * [SAUCE](http://www.acid.org/info/sauce/sauce.htm) support
+ * Full [SAUCE](http://www.acid.org/info/sauce/sauce.htm) support
  * Renegade style pipe color codes
  * [SQLite](http://sqlite.org/) storage of users, message areas, and so on
  * Strong [PBKDF2](https://en.wikipedia.org/wiki/PBKDF2) backed password encryption
- * [Door support](docs/doors.md) including common dropfile formats for legacy DOS doors. Built in [BBSLink](http://bbslink.net/), and [DoorParty](http://forums.throwbackbbs.com/) support!
+ * [Door support](docs/doors.md) including common dropfile formats for legacy DOS doors. Built in [BBSLink](http://bbslink.net/), [DoorParty](http://forums.throwbackbbs.com/), and [Exodus](https://oddnetwork.org/exodus/) support!
  * [Bunyan](https://github.com/trentm/node-bunyan) logging
  * [Message networks](docs/msg_networks.md) with FidoNet Type Network (FTN) + BinkleyTerm Style Outbound (BSO) message import/export
  * [Gazelle](https://github.com/WhatCD/Gazelle) inspirted File Bases including fast fully indexed full text search (FTS), #tags, and HTTP(S) temporary download URLs using a built in [web server](docs/web_server.md). Legacy X/Y/Z modem also supported!
  * Upload processor supporting [FILE_ID.DIZ](https://en.wikipedia.org/wiki/FILE_ID.DIZ) and [NFO](https://en.wikipedia.org/wiki/.nfo) extraction, year estimation, and more!
+ * ANSI support in the Full Screen Editor (FSE), file descriptions, and so on
  
 ## In the Works
 * More ES6+ usage, and **documentation**!
 * More ACS support coverage
 * SysOp dashboard (ye ol' WFC)
-* Missing functionality such as message FTS, user coloring of messages in the FST, etc.
-* String localization
 * A lot more! Feel free to request features via [the issue tracker](https://github.com/NuSkooler/enigma-bbs/issues)
 
 ## Known Issues
@@ -40,19 +39,20 @@ See [the issue tracker](https://github.com/NuSkooler/enigma-bbs/issues) for more
 * Use [the issue tracker](https://github.com/NuSkooler/enigma-bbs/issues)
 * **Discussion on a ENiGMA BBS!** (see Boards below)
 * IRC: **#enigma-bbs** on **chat.freenode.net**
+* Discussion on [fsxNet](http://bbs.geek.nz/#fsxNet) available on many boards
 * Email: bryan -at- l33t.codes
 * [Facebook ENiGMA½ group](https://www.facebook.com/groups/enigmabbs/)
-* ENiGMA discussion on [fsxNet](http://bbs.geek.nz/#fsxNet)
 
 ## Terminal Clients
 ENiGMA has been tested with many terminals. However, the following are suggested for BBSing:
+* [VTX](https://github.com/codewar65/VTX_ClientServer) (Try [Xibalba using VTX](https://l33t.codes/vtx/xibalba.html)!)
 * [SyncTERM](http://syncterm.bbsdev.net/)
 * [EtherTerm](https://github.com/M-griffin/EtherTerm)
 * [NetRunner](http://mysticbbs.com/downloads.html)
 
 ## Boards
-* WQH: :skull: Xibalba :skull: (**telnet://xibalba.l33t.codes:44510**)
+* WQH: :skull: [Xibalba](https://l33t.codes/xibalba-bbs) :skull: (**telnet://xibalba.l33t.codes:44510** or via SSH secure on port 44511)
-* Exotica: (**telnet://andrew.homeunix.org:2023**)
+* [Exotica](https://exoticabbs.com/): (**telnet://exoticabbs.com:8888**)
 * [force9](http://bbs.force9.org/): (**telnet://bbs.force9.org**)
 
 
@@ -60,10 +60,11 @@ ENiGMA has been tested with many terminals. However, the following are suggested
 ```
 curl -o- https://raw.githubusercontent.com/NuSkooler/enigma-bbs/master/misc/install.sh | bash
 ```
-<br>
+
-(See the [Quickstart](docs/index.md#quickstart) for more information)
+Please see the [Quickstart](docs/index.md) for more information.
 
 ## Special Thanks
+* [Daniel Mecklenburg Jr.](https://github.com/codewar65) for the awesome VTX terminal and general coding talk
 * [M. Brutman](http://www.brutman.com/), author of [mTCP](http://www.brutman.com/mTCP/mTCP.html) (Interwebs for DOS!)
 * [M. Griffin](https://github.com/M-griffin), author of [Enthral BBS](https://github.com/M-griffin/Enthral), [Oblivion/2 XRM](https://github.com/M-griffin/Oblivion2-XRM) and [EtherTerm](https://github.com/M-griffin/EtherTerm)!
 * [Caphood](http://www.reddit.com/user/Caphood), supreme SysOp of [BLACK ƒlag](http://www.bbsnexus.com/directory/listing/blackflag.html) BBS

core/ansi_escape_parser.js

@@ -44,14 +44,10 @@ function ANSIEscapeParser(options) {
 		self.row	+= rows;
 
 		self.column	= Math.max(self.column, 1);
-		self.column	= Math.min(self.column, self.termWidth);
+		self.column	= Math.min(self.column, self.termWidth);	//	can't move past term width
 		self.row	= Math.max(self.row, 1);
-		self.row	= Math.min(self.row, self.termHeight);
-
-//		self.emit('move cursor', self.column, self.row);
 		
 		self.positionUpdated();
-		//self.rowUpdated();
 	};
 
 	self.saveCursorPosition = function() {
@@ -85,22 +81,18 @@ function ANSIEscapeParser(options) {
 	};
 
 	function literal(text) {
-		let charCode;		
-		let pos;
-		let start	= 0;
 		const len	= text.length;
+		let pos		= 0;
+		let start	= 0;
+		let charCode;
 
-		function emitLiteral() {
+		while(pos < len) {
+			charCode = text.charCodeAt(pos) & 0xff;	//	8bit clean
+
+			switch(charCode) {
+				case CR :
 					self.emit('literal', text.slice(start, pos));
 					start = pos;
-		}
-
-		for(pos = 0; pos < len; ++pos) {
-			charCode = text.charCodeAt(pos) & 0xff;	//	ensure 8bit clean
-
-			switch(charCode) {
-				case CR : 
-					emitLiteral();
 
 					self.column = 1;
 					
@@ -108,7 +100,8 @@ function ANSIEscapeParser(options) {
 					break;
 
 				case LF :
-					emitLiteral();
+					self.emit('literal', text.slice(start, pos));
+					start = pos;
 
 					self.row += 1;
 
@@ -116,73 +109,37 @@ function ANSIEscapeParser(options) {
 					break;
 
 				default :
-					if(self.column > self.termWidth) {
+					if(self.column === self.termWidth) {
-						//
+						self.emit('literal', text.slice(start, pos + 1));
-						//	Emit data up to this point so it can be drawn before the postion update
+						start = pos + 1;
-						//
-						emitLiteral();
 
 						self.column = 1;
 						self.row	+= 1;
 
 						self.positionUpdated();
-
-						
-					} else {
-						self.column += 1;
-					}
-					break;
-			}
-		}
-
-		self.emit('literal', text.slice(start));
-
-		if(self.column > self.termWidth) {
-			self.column = 1;
-			self.row	+= 1;
-			self.positionUpdated();
-		}
-	}
-
-	function literal2(text) {
-		var charCode;
-
-		var len = text.length;
-		for(var i = 0; i < len; i++) {
-			charCode = text.charCodeAt(i) & 0xff;	//	ensure 8 bit
-			switch(charCode) {
-				case CR : 
-					self.column = 1;
-					break;
-
-				case LF : 
-					self.row++;
-					self.positionUpdated();
-					//self.rowUpdated();		
-					break;
-
-				default :
-					//	wrap
-					if(self.column > self.termWidth) {
-						self.column = 1;
-						self.row++;
-						//self.rowUpdated();
-						self.positionUpdated();
 					} else {
 						self.column += 1;
 					}
 					break;
 			}
 
-			if(self.row === self.termHeight) {
+			++pos;
-				self.scrollBack		+= 1;
+		}
-				self.row			-= 1;
+
+		//
+		//	Finalize this chunk
+		//
+		if(self.column > self.termWidth) {
+			self.column = 1;
+			self.row	+= 1;
 			
 			self.positionUpdated();
 		}
-		}
 
-		self.emit('literal', text);
+		const rem = text.slice(start);
+		if(rem) {
+			self.emit('literal', rem);
+		}
 	}
 
 	function getProcessedMCI(mci) {
@@ -238,10 +195,10 @@ function ANSIEscapeParser(options) {
 				});
 
 				if(self.mciReplaceChar.length > 0) {
-					//self.emit('chunk', ansi.getSGRFromGraphicRendition(self.graphicRenditionForErase));
 					const sgrCtrl = ansi.getSGRFromGraphicRendition(self.graphicRenditionForErase);
+					
 					self.emit('control', sgrCtrl, 'm', sgrCtrl.slice(2).split(/[\;m]/).slice(0, 3));
-					//self.emit('control', ansi.getSGRFromGraphicRendition(self.graphicRenditionForErase)); 
+
 					literal(new Array(match[0].length + 1).join(self.mciReplaceChar));
 				} else {
 					literal(match[0]);
@@ -436,6 +393,8 @@ function ANSIEscapeParser(options) {
 
 			//	set graphic rendition
 			case 'm' :
+				self.graphicRendition.reset = false;
+
 				for(let i = 0, len = args.length; i < len; ++i) {
 					arg = args[i];
 
@@ -453,8 +412,12 @@ function ANSIEscapeParser(options) {
 								delete self.graphicRendition.negative;
 								delete self.graphicRendition.invisible;
 
-								self.graphicRendition.fg = 39;
+								delete self.graphicRendition.fg;
-								self.graphicRendition.bg = 49;
+								delete self.graphicRendition.bg;
+
+								self.graphicRendition.reset = true;
+								//self.graphicRendition.fg = 39;
+								//self.graphicRendition.bg = 49;
 								break;
 
 							case 1 :
@@ -490,6 +453,8 @@ function ANSIEscapeParser(options) {
 						}
 					}
 				}
+
+				self.emit('sgr update', self.graphicRendition);
 				break;	//	m
 
 			//	:TODO: s, u, K

core/ansi_prep.js

@@ -0,0 +1,212 @@
+/* jslint node: true */
+'use strict';
+
+//	ENiGMA½
+const ANSIEscapeParser	= require('./ansi_escape_parser.js').ANSIEscapeParser;
+const ANSI				= require('./ansi_term.js');
+const { 
+	splitTextAtTerms,
+	renderStringLength
+}						= require('./string_util.js');
+
+//	deps
+const _					= require('lodash');
+
+module.exports = function ansiPrep(input, options, cb) {
+	if(!input) {
+		return cb(null, '');
+	}
+
+	options.termWidth			= options.termWidth 	|| 80;
+	options.termHeight			= options.termHeight	|| 25;
+	options.cols				= options.cols || options.termWidth		|| 80;
+	options.rows				= options.rows || options.termHeight	|| 'auto';
+	options.startCol			= options.startCol || 1;
+	options.exportMode			= options.exportMode || false;
+
+	//	in auto we start out at 25 rows, but can always expand for more
+	const canvas = Array.from( { length : 'auto' === options.rows ? 25 : options.rows }, () => Array.from( { length : options.cols}, () => new Object() ) );
+	const parser = new ANSIEscapeParser( { termHeight : options.termHeight, termWidth : options.termWidth } );
+
+	const state = {
+		row	: 0,
+		col	: 0,
+	};
+
+	let lastRow = 0;
+
+	function ensureRow(row) {
+		if(canvas[row]) {
+			return;
+		}
+		
+		canvas[row] = Array.from( { length : options.cols}, () => new Object() );
+	}
+
+	parser.on('position update', (row, col) => {
+		state.row	= row - 1;
+		state.col	= col - 1;
+
+		if(0 === state.col) {
+			state.initialSgr = state.lastSgr;
+		}
+
+		lastRow		= Math.max(state.row, lastRow);
+	});
+
+	parser.on('literal', literal => {
+		//
+		//	CR/LF are handled for 'position update'; we don't need the chars themselves
+		//
+		literal = literal.replace(/\r?\n|[\r\u2028\u2029]/g, '');
+
+		for(let c of literal) {
+			if(state.col < options.cols && ('auto' === options.rows || state.row < options.rows)) {
+				ensureRow(state.row);
+
+				if(0 === state.col) {
+					canvas[state.row][state.col].initialSgr = state.initialSgr;
+				}
+
+				canvas[state.row][state.col].char = c;
+
+				if(state.sgr) {
+					canvas[state.row][state.col].sgr	= _.clone(state.sgr);
+					state.lastSgr						= canvas[state.row][state.col].sgr;
+					state.sgr							= null;
+				}
+			}
+
+			state.col += 1;
+		}
+	});
+
+	parser.on('sgr update', sgr => {
+		ensureRow(state.row);
+
+		if(state.col < options.cols) {
+			canvas[state.row][state.col].sgr	= _.clone(sgr);
+			state.lastSgr						= canvas[state.row][state.col].sgr;
+		} else {
+			state.sgr = sgr;
+		}
+	});
+
+	function getLastPopulatedColumn(row) {
+		let col = row.length;
+		while(--col > 0) {
+			if(row[col].char || row[col].sgr) {
+				break;
+			}
+		}
+		return col;
+	}
+
+	parser.on('complete', () => {
+		let output = '';
+		let line;
+		let sgr;
+
+		canvas.slice(0, lastRow + 1).forEach(row => {
+			const lastCol = getLastPopulatedColumn(row) + 1;
+
+			let i;
+			line = '';
+			for(i = 0; i < lastCol; ++i) {
+				const col = row[i];
+
+				sgr = 0 === i ? 
+					col.initialSgr ? ANSI.getSGRFromGraphicRendition(col.initialSgr) : '' :
+					'';
+				
+				if(col.sgr) {
+					sgr += ANSI.getSGRFromGraphicRendition(col.sgr);
+				}
+
+				line += `${sgr}${col.char || ' '}`;
+			}
+
+			output += line;
+
+			if(i < row.length) {
+				output += `${ANSI.blackBG()}${row.slice(i).map( () => ' ').join('')}`;//${lastSgr}`;
+			}
+
+			if(options.startCol + i < options.termWidth || options.forceLineTerm) {
+				output += '\r\n';
+			}
+		});
+
+		if(options.exportMode) {
+			//
+			//	If we're in export mode, we do some additional hackery:
+			//	
+			//	* Hard wrap ALL lines at <= 79 *characters* (not visible columns)
+			//	  if a line must wrap early, we'll place a ESC[A ESC[<N>C where <N>
+			//	  represents chars to get back to the position we were previously at
+			//
+			//	* Replace contig spaces with ESC[<N>C as well to save... space.
+			//
+			//	:TODO: this would be better to do as part of the processing above, but this will do for now
+			const MAX_CHARS	= 79 - 8;	//	79 max, - 8 for max ESC seq's we may prefix a line with
+			let exportOutput = '';			
+			
+			let m;
+			let afterSeq;
+			let wantMore;
+			let renderStart;
+
+			splitTextAtTerms(output).forEach(fullLine => {
+				renderStart = 0;
+
+				while(fullLine.length > 0) {
+					let splitAt;
+					const ANSI_REGEXP = ANSI.getFullMatchRegExp();
+					wantMore = true;
+
+					while((m = ANSI_REGEXP.exec(fullLine))) {
+						afterSeq = m.index + m[0].length;
+
+						if(afterSeq < MAX_CHARS) {
+							//	after current seq
+							splitAt = afterSeq;
+						} else {
+							if(m.index < MAX_CHARS) {
+								//	before last found seq
+								splitAt = m.index;
+								wantMore = false;	//	can't eat up any more
+							}
+							
+							break;	//	seq's beyond this point are >= MAX_CHARS
+						}
+					}
+
+					if(splitAt) {
+						if(wantMore) {
+							splitAt = Math.min(fullLine.length, MAX_CHARS - 1);
+						}
+					} else {
+						splitAt = Math.min(fullLine.length, MAX_CHARS - 1);
+					}
+
+					const part = fullLine.slice(0, splitAt);
+					fullLine = fullLine.slice(splitAt);
+					renderStart += renderStringLength(part);
+					exportOutput += `${part}\r\n`;
+
+					if(fullLine.length > 0) {	//	more to go for this line?
+						exportOutput += `${ANSI.up()}${ANSI.right(renderStart)}`;	
+					} else {
+						exportOutput += ANSI.up();
+					}
+				}
+			});
+
+			return cb(null, exportOutput);
+		}
+
+		return cb(null, output);
+	});
+
+	parser.parse(input);
+};

core/ansi_term.js

@@ -17,12 +17,22 @@
 //		* http://www.bbsdocumentary.com/library/PROGRAMS/GRAPHICS/ANSI/ansisys.txt
 //		* http://academic.evergreen.edu/projects/biophysics/technotes/program/ansi_esc.htm
 //
+//	VTX
+//		* https://github.com/codewar65/VTX_ClientServer/blob/master/vtx.txt
+//
 //	General
 //		* http://en.wikipedia.org/wiki/ANSI_escape_code
 //		* http://www.inwap.com/pdp10/ansicode.txt
 //
 //	Other Implementations
 //		* https://github.com/chjj/term.js/blob/master/src/term.js
+//
+//
+//	For a board, we need to support the semi-standard ANSI-BBS "spec" which
+//	is bastardized mix of DOS ANSI.SYS, cterm.txt, bansi.txt and a little other.
+//	This gives us NetRunner, SyncTERM, EtherTerm, most *nix terminals, compatibilitiy
+//	with legit oldschool DOS terminals, and so on. 
+//
 
 //	ENiGMA½
 const miscUtil	= require('./misc_util.js');
@@ -31,6 +41,7 @@ const miscUtil	= require('./misc_util.js');
 const assert	= require('assert');
 const _			= require('lodash');
 
+exports.getFullMatchRegExp			= getFullMatchRegExp;
 exports.getFGColorValue				= getFGColorValue;
 exports.getBGColorValue				= getBGColorValue;
 exports.sgr							= sgr;
@@ -172,6 +183,12 @@ const SGRValues = {
 	whiteBG			: 47,
 };
 
+function getFullMatchRegExp(flags = 'g') {
+	//	:TODO: expand this a bit - see strip-ansi/etc.
+	//	:TODO: \u009b ?
+	return new RegExp(/[\u001b][[()#;?]*([0-9]{1,4}(?:;[0-9]{0,4})*)?([0-9A-ORZcf-npqrsuy=><])/, flags);	//	eslint-disable-line no-control-regex
+}
+
 function getFGColorValue(name) {
 	return SGRValues[name];
 }
@@ -289,7 +306,6 @@ const FONT_ALIAS_TO_SYNCTERM_MAP = {
 	'amiga_microknight'		: 'microknight',
 	'amiga_microknight+'	: 'microknight_plus',
 
-
 	'atari'					: 'atari',
 	'atarist'				: 'atari',
 
@@ -399,10 +415,6 @@ function getSGRFromGraphicRendition(graphicRendition, initialReset) {
 		}
 	});
 
-	if(!styleCount) {
-		sgrSeq.push(0);
-	}
-
 	if(graphicRendition.fg) {
 		sgrSeq.push(graphicRendition.fg);
 	}
@@ -411,7 +423,7 @@ function getSGRFromGraphicRendition(graphicRendition, initialReset) {
 		sgrSeq.push(graphicRendition.bg);
 	}
 
-	if(initialReset) {
+	if(0 === styleCount || initialReset) {
 		sgrSeq.unshift(0);
 	}
 
@@ -473,4 +485,3 @@ function setEmulatedBaudRate(rate) {
 	}[rate] || 0;
 	return 0 === speed ? exports.emulationSpeed() : exports.emulationSpeed(1, speed);
 }
-

core/art.js

@@ -157,18 +157,19 @@ function getArt(name, options, cb) {
 			//  Ignore anything not allowed in |options.types|
 			//
 			const fext = paths.extname(file);
-			if(options.types.indexOf(fext.toLowerCase()) < 0) {
+			if(!options.types.includes(fext.toLowerCase())) {
 				return false;
 			}
 
 			const bn = paths.basename(file, fext).toLowerCase();
 			if(options.random) {
 				const suppliedBn = paths.basename(name, fext).toLowerCase();
+			
 				//
 				//  Random selection enabled. We'll allow for
 				//  basename1.ext, basename2.ext, ...
 				//
-				if(bn.indexOf(suppliedBn) !== 0) {
+				if(!bn.startsWith(suppliedBn)) {
 					return false;
 				}
 
@@ -241,6 +242,10 @@ function display(client, art, options, cb) {
 	options.mciReplaceChar 	= options.mciReplaceChar || ' ';
 	options.disableMciCache	= options.disableMciCache || false;
 
+	//	:TODO: this is going to be broken into two approaches controlled via options:
+	//	1) Standard - use internal tracking of locations for MCI -- no CPR's/etc.
+	//	2) CPR driven
+
 	if(!_.isBoolean(options.iceColors)) {
 		//	try to detect from SAUCE
 		if(_.has(options, 'sauce.ansiFlags') && (options.sauce.ansiFlags & (1 << 0))) {
@@ -282,7 +287,6 @@ function display(client, art, options, cb) {
 		return cb(null, mciMap, extraInfo);
 	}
 
-
 	if(!options.disableMciCache) {		
 		artHash	= farmhash.hash32(art);
 
@@ -335,14 +339,14 @@ function display(client, art, options, cb) {
 				}
 
 				mciCprQueue.push(mapKey);
-				client.term.write(ansi.queryPos(), false);
+				client.term.rawWrite(ansi.queryPos());
 			}
 
 		});
 	}
 
 	ansiParser.on('literal', literal => client.term.write(literal, false) );	
-	ansiParser.on('control', control => client.term.write(control, false) );
+	ansiParser.on('control', control => client.term.rawWrite(control) );
 
 	ansiParser.on('complete', () => {
 		parseComplete = true;
@@ -352,29 +356,35 @@ function display(client, art, options, cb) {
 		}		
 	});
 
-	let ansiFontSeq;
+	let initSeq = '';
 	if(options.font) {
-		ansiFontSeq = ansi.setSyncTermFontWithAlias(options.font);
+		initSeq = ansi.setSyncTermFontWithAlias(options.font);
 	} else if(options.sauce) {
 		let fontName = getFontNameFromSAUCE(options.sauce);
 		if(fontName) {
 			fontName = ansi.getSyncTERMFontFromAlias(fontName);
 		}
 
-		//	don't set default (CP437) from SAUCE
+		//
-		if(fontName && 'cp437' !== fontName) {
+		//	Set SyncTERM font if we're switching only. Most terminals
-			ansiFontSeq = ansi.setSyncTERMFont(fontName);
+		//	that support this ESC sequence can only show *one* font
+		//	at a time. This applies to detection only (e.g. SAUCE).
+		//	If explicit, we'll set it no matter what (above)
+		//
+		if(fontName && client.term.currentSyncFont != fontName) {
+			client.term.currentSyncFont = fontName;
+			initSeq = ansi.setSyncTERMFont(fontName);
 		}
 	}
 
-	if(ansiFontSeq) {
-		client.term.write(ansiFontSeq, false);
-	}
-
 	if(options.iceColors) {
-		client.term.write(ansi.blinkToBrightIntensity(), false);
+		initSeq += ansi.blinkToBrightIntensity();
+	}
+
+	if(initSeq) {
+		client.term.rawWrite(initSeq);
 	}
 
 	ansiParser.reset(art);
-	ansiParser.parse();	
+	return ansiParser.parse();
 }

core/bbs.js

@@ -9,7 +9,6 @@
 const conf			= require('./config.js');
 const logger		= require('./logger.js');
 const database		= require('./database.js');
-const clientConns	= require('./client_connections.js');
 const resolvePath	= require('./misc_util.js').resolvePath;
 
 //	deps
@@ -84,7 +83,7 @@ function main() {
 					}
 					return callback(err);
 				});
-			},
+			}
 		],
 		function complete(err) {
 			//	note this is escaped:
@@ -111,11 +110,12 @@ function shutdownSystem() {
 	async.series(
 		[
 			function closeConnections(callback) {
-				const activeConnections = clientConns.getActiveConnections();
+				const ClientConns = require('./client_connections.js');
+				const activeConnections = ClientConns.getActiveConnections();
 				let i = activeConnections.length;
 				while(i--) {
 					activeConnections[i].term.write('\n\nServer is shutting down NOW! Disconnecting...\n\n');
-					clientConns.removeClient(activeConnections[i]);
+					ClientConns.removeClient(activeConnections[i]);
 				}
 				callback(null);
 			},
@@ -179,6 +179,9 @@ function initialize(cb) {
 			function initDatabases(callback) {
 				return database.initializeDatabases(callback);
 			},
+			function initMimeTypes(callback) {
+				return require('./mime_util.js').startup(callback);	
+			},
 			function initStatLog(callback) {
 				return require('./stat_log.js').init(callback);
 			},
@@ -237,6 +240,9 @@ function initialize(cb) {
 			function readyMessageNetworkSupport(callback) {
 				return require('./msg_network.js').startup(callback);
 			},
+			function readyEvents(callback) {
+				return require('./events.js').startup(callback);
+			},
 			function listenConnections(callback) {
 				return require('./listening_server.js').startup(callback);
 			},

core/client.js

@@ -120,6 +120,7 @@ function Client(input, output) {
 			//	* Irssi ConnectBot (Android)
 			//
 			'63;1;2'		: 'arctel',
+			'50;86;84;88'	: 'vtx',
 		}[deviceAttr];
 
 		if(!termClient) {
@@ -491,3 +492,13 @@ Client.prototype.defaultHandlerMissingMod = function(err) {
 	return handler;
 };
 
+Client.prototype.terminalSupports = function(query) {
+	switch(query) {
+		case 'vtx_audio' :
+			//	https://github.com/codewar65/VTX_ClientServer/blob/master/vtx.txt
+			return this.termClient === 'vtx';
+		
+		default : 
+			return false;
+	}
+};

core/client_connections.js

@@ -3,6 +3,7 @@
 
 //	ENiGMA½
 const logger			= require('./logger.js');
+const Events            = require('./events.js');
 
 //	deps
 const _					= require('lodash');
@@ -76,6 +77,8 @@ function addNewClient(client, clientSock) {
 
 	client.log.info(connInfo, 'Client connected');
 
+	Events.emit('codes.l33t.enigma.system.connected', { client : client, connectionCount : clientConnections.length } );
+
 	return id;
 }
 
@@ -93,6 +96,8 @@ function removeClient(client) {
 			},
 			'Client disconnected'
 			);
+
+		Events.emit('codes.l33t.enigma.system.disconnected', { client : client, connectionCount : clientConnections.length } );
 	}
 }
 

core/client_term.js

@@ -32,6 +32,8 @@ function ClientTerminal(output) {
 	var termWidth		= 0;
 	var termClient		= 'unknown';
 
+	this.currentSyncFont	= 'not_set';
+
 	//	Raw values set by e.g. telnet NAWS, ENVIRONMENT, etc.
 	this.env			= {};
 
@@ -169,7 +171,7 @@ ClientTerminal.prototype.pipeWrite = function(s, spec, cb) {
 	var conv = {
 		enigma		: enigmaToAnsi,
 		renegade	: renegadeToAnsi,
-	}[spec] || enigmaToAnsi;
+	}[spec] || renegadeToAnsi;
 	
 	this.write(conv(s, this), null, cb);	//	null = use default for |convertLineFeeds|
 };
@@ -183,3 +185,4 @@ ClientTerminal.prototype.encode = function(s, convertLineFeeds) {
 	return iconv.encode(s, this.outputEncoding);
 };
 
+

core/color_codes.js

@@ -23,6 +23,8 @@ exports.pipeToAnsi			= exports.renegadeToAnsi		= renegadeToAnsi;
 //	* fromWWIV(): <ctrl-c><0-7>
 //	* fromSyncronet(): <ctrl-a><colorCode>
 //	See http://wiki.synchro.net/custom:colors
+
+//	:TODO: rid of enigmaToAnsi() -- never really use. Instead, create bbsToAnsi() that supports renegade, PCB, WWIV, etc...
 function enigmaToAnsi(s, client) {
 	if(-1 == s.indexOf('|')) {
 		return s;	//	no pipe codes present

core/config.js

@@ -100,6 +100,11 @@ function init(configPath, options, cb) {
 		],
 		function complete(err, mergedConfig) {
 			exports.config = mergedConfig;
+
+			exports.config.get = function(path) {
+				return _.get(exports.config, path);
+			};
+
 			return cb(err);
 		}
 	);
@@ -316,6 +321,24 @@ function getDefaultConfig() {
 				longDescUtil	: 'Exiftool',
 			},
 			//
+			//	Video
+			//
+			'video/mp4' : {
+				desc			: 'MPEG Video',
+				shortDescUtil	: 'Exiftool2Desc',
+				longDescUtil	: 'Exiftool',
+			},
+			'video/x-matroska ' : {
+				desc			: 'Matroska Video',
+				shortDescUtil	: 'Exiftool2Desc',
+				longDescUtil	: 'Exiftool',
+			},
+			'video/x-msvideo' : {
+				desc			: 'Audio Video Interleave',
+				shortDescUtil	: 'Exiftool2Desc',
+				longDescUtil	: 'Exiftool',
+			},
+			//
 			//	Images
 			//
 			'image/jpeg'	: {
@@ -347,6 +370,12 @@ function getDefaultConfig() {
 				offset			: 0,
 				archiveHandler	: '7Zip',
 			},
+			/*
+			'application/x-cbr' : {
+				desc			: 'Comic Book Archive',
+				sig				: '504b0304',
+			},
+			*/
 			'application/x-arj' : {
 				desc			: 'ARJ Archive',
 				sig				: '60ea',
@@ -363,7 +392,7 @@ function getDefaultConfig() {
 				desc			: 'Gzip Archive',
 				sig				: '1f8b',
 				offset			: 0,
-				archiveHandler	: '7Zip',
+				archiveHandler	: 'TarGz',
 			},
 			//	:TODO: application/x-bzip
 			'application/x-bzip2' : {
@@ -474,6 +503,22 @@ function getDefaultConfig() {
 						cmd			: 'unrar',
 						args		: [ 'e', '{archivePath}', '{extractPath}', '{fileList}' ],
 					}
+				},
+
+				TarGz : {
+					decompress		: {
+						cmd			: 'tar',
+						args		: [ '-xf', '{archivePath}', '-C', '{extractPath}', '--strip-components=1' ],
+					},
+					list			: {
+						cmd			: 'tar',
+						args		: [ '-tvf', '{archivePath}' ],
+						entryMatch	: '^[drwx\\-]{10}\\s[A-Za-z0-9\\/]+\\s+([0-9]+)\\s[0-9]{4}\\-[0-9]{2}\\-[0-9]{2}\\s[0-9]{2}\\:[0-9]{2}\\s([^\\r\\n]+)$',						
+					},
+					extract			: {
+						cmd			: 'tar',
+						args		: [ '-xvf', '{archivePath}', '-C', '{extractPath}', '{fileList}' ],
+					}
 				}
 			},
 		},
@@ -584,6 +629,8 @@ function getDefaultConfig() {
 				//
 				packetTargetByteSize	: 512000,		//	512k, before placing messages in a new pkt
 				bundleTargetByteSize	: 2048000,		//	2M, before creating another archive
+				packetMsgEncoding		: 'utf8',		//	default packet encoding. Override per node if desired.
+				packetAnsiMsgEncoding	: 'cp437',		//	packet encoding for *ANSI ART* messages
 
 				tic : {
 					secureInOnly	: true,				//	only bring in from secure inbound (|secInbound| path, password protected)

core/connect.js

@@ -3,6 +3,7 @@
 
 //	ENiGMA½
 const ansi		= require('./ansi_term.js');
+const Events    = require('./events.js');
 
 //	deps
 const async		= require('async');
@@ -175,6 +176,9 @@ function connectEntry(client, nextMenu) {
 			//
 			displayBanner(term);
 
+			// fire event
+			Events.emit('codes.l33t.enigma.system.term_detected', { client : client } );
+
 			setTimeout( () => {
 				return client.menuStack.goto(nextMenu);
 			}, 500);

core/database.js

@@ -71,9 +71,13 @@ function initializeDatabases(cb) {
 	});
 }
 
+function enableForeignKeys(db) {
+	db.run('PRAGMA foreign_keys = ON;');
+}
+
 const DB_INIT_TABLE = {
 	system : (cb) => {
-		dbs.system.run('PRAGMA foreign_keys = ON;');
+		enableForeignKeys(dbs.system);
 
 		//	Various stat/event logging - see stat_log.js
 		dbs.system.run(
@@ -110,7 +114,7 @@ const DB_INIT_TABLE = {
 	},
 
 	user : (cb) => {
-		dbs.user.run('PRAGMA foreign_keys = ON;');
+		enableForeignKeys(dbs.user);
 
 		dbs.user.run(
 			`CREATE TABLE IF NOT EXISTS user ( 
@@ -152,7 +156,7 @@ const DB_INIT_TABLE = {
 	},
 
 	message : (cb) => {
-		dbs.message.run('PRAGMA foreign_keys = ON;');
+		enableForeignKeys(dbs.message);
 
 		dbs.message.run(
 			`CREATE TABLE IF NOT EXISTS message (
@@ -260,7 +264,7 @@ const DB_INIT_TABLE = {
 	},
 
 	file : (cb) => {
-		dbs.file.run('PRAGMA foreign_keys = ON;');
+		enableForeignKeys(dbs.file);
 
 		dbs.file.run(
 			//	:TODO: should any of this be unique -- file_sha256 unless dupes are allowed on the system

core/descript_ion_file.js

@@ -0,0 +1,72 @@
+/* jslint node: true */
+'use strict';
+
+//	deps
+const fs			= require('graceful-fs');
+const iconv			= require('iconv-lite');
+const async			= require('async');
+
+module.exports = class DescriptIonFile {
+	constructor() {
+		this.entries = new Map();
+	}
+
+	get(fileName) {
+		return this.entries.get(fileName);
+	}
+
+	getDescription(fileName) {
+		const entry = this.get(fileName);
+		if(entry) {
+			return entry.desc;
+		}
+	}
+
+	static createFromFile(path, cb) {
+		fs.readFile(path, (err, descData) => {
+			if(err) {
+				return cb(err);
+			}
+
+			const descIonFile = new DescriptIonFile();
+
+			//	DESCRIPT.ION entries are terminated with a CR and/or LF
+			const lines = iconv.decode(descData, 'cp437').split(/\r?\n/g);
+
+			async.each(lines, (entryData, nextLine) => {
+				//
+				//	We allow quoted (long) filenames or non-quoted filenames.
+				//	FILENAME<SPC>DESC<0x04><program data><CR/LF>
+				//
+				const parts = entryData.match(/^(?:(?:"([^"]+)" )|(?:([^ ]+) ))([^\x04]+)\x04(.)[^\r\n]*$/);	//	eslint-disable-line no-control-regex
+				if(!parts) {
+					return nextLine(null);
+				}
+
+				const fileName	= parts[1] || parts[2];
+
+				//
+				//	Un-escape CR/LF's
+				//	- escapped \r and/or \n
+				//	- BBBS style @n - See https://www.bbbs.net/sysop.html
+				//
+				const desc		= parts[3].replace(/\\r\\n|\\n|[^@]@n/g, '\r\n');
+
+				descIonFile.entries.set(
+					fileName,
+					{
+						desc		: desc,
+						programId	: parts[4],
+						programData	: parts[5],
+					}
+				);
+
+				return nextLine(null);
+			},
+			() => {
+				return cb(null, descIonFile);
+			});
+		});
+	}
+};
+

core/door_party.js

@@ -97,6 +97,10 @@ exports.getModule = class DoorPartyModule extends MenuModule {
 						});
 					});
 
+					sshClient.on('error', err => {
+						self.client.log.info(`DoorParty SSH client error: ${err.message}`);
+					});
+					
 					sshClient.on('close', () => {
 						restorePipe();
 						callback(null);

core/events.js

@@ -0,0 +1,73 @@
+/* jslint node: true */
+'use strict';
+
+const paths				= require('path');
+const events			= require('events');
+const Log				= require('./logger.js').log;
+
+//	deps
+const _					= require('lodash');
+const async				= require('async');
+const glob				= require('glob');
+
+module.exports = new class Events extends events.EventEmitter {
+	constructor() {
+		super();
+	}
+
+	addListener(event, listener) {
+		Log.trace( { event : event }, 'Registering event listener');
+		return super.addListener(event, listener);
+	}
+
+	emit(event, ...args) {
+		Log.trace( { event : event }, 'Emitting event');
+		return super.emit(event, args);
+	}
+
+	on(event, listener) {
+		Log.trace( { event : event }, 'Registering event listener');
+		return super.on(event, listener);
+	}
+
+	once(event, listener) {
+		Log.trace( { event : event }, 'Registering single use event listener');
+		return super.once(event, listener);
+	}
+
+	removeListener(event, listener) {
+		Log.trace( { event : event }, 'Removing listener');
+		return super.removeListener(event, listener);
+	}
+
+	startup(cb) {
+		async.each(require('./module_util.js').getModulePaths(), (modulePath, nextPath) => {
+			glob('*{.js,/*.js}', { cwd : modulePath }, (err, files) => {
+				if(err) {
+					return nextPath(err);
+				}
+
+				async.each(files, (moduleName, nextModule) => {
+					modulePath = paths.join(modulePath, moduleName);
+
+					try {
+						const mod = require(modulePath);
+						
+						if(_.isFunction(mod.registerEvents)) {
+							//	:TODO: ... or just systemInit() / systemShutdown() & mods could call Events.on() / Events.removeListener() ?
+							mod.registerEvents(this);
+						}
+					} catch(e) {
+
+					}
+
+					return nextModule(null);
+				}, err => {
+					return nextPath(err);
+				});
+			});
+		}, err => {
+			return cb(err);
+		});
+	}
+};

core/exodus.js

@@ -0,0 +1,231 @@
+/* jslint node: true */
+'use strict';
+
+//	ENiGMA½
+const MenuModule			= require('../core/menu_module.js').MenuModule;
+const resetScreen			= require('../core/ansi_term.js').resetScreen;
+const Config				= require('./config.js').config;
+const Errors				= require('./enig_error.js').Errors;
+const Log					= require('./logger.js').log;
+const getEnigmaUserAgent	= require('./misc_util.js').getEnigmaUserAgent;
+
+//	deps
+const async			= require('async');
+const _				= require('lodash');
+const joinPath		= require('path').join;
+const crypto		= require('crypto');
+const moment		= require('moment');
+const https			= require('https');
+const querystring	= require('querystring');
+const fs			= require('fs');
+const SSHClient		= require('ssh2').Client;
+
+/*
+	Configuration block:
+
+	
+	someDoor: {
+		module: exodus
+		config: {
+			//	defaults
+			ticketHost: oddnetwork.org
+			ticketPort: 1984
+			ticketPath: /exodus
+			rejectUnauthorized: false // set to true to allow untrusted CA's (dangerous!)
+			sshHost: oddnetwork.org
+			sshPort: 22
+			sshUser: exodus
+			sshKeyPem: /path/to/enigma-bbs/misc/exodus.id_rsa
+
+			//	optional
+			caPem: /path/to/cacerts.pem	// see https://curl.haxx.se/docs/caextract.html
+
+			//	required
+			board: XXXX
+			key: XXXX
+			door: some_door
+		}
+	}
+*/
+
+exports.moduleInfo = {
+	name	: 'Exodus',
+	desc	: 'Exodus Door Server Access Module - https://oddnetwork.org/exodus/',
+	author	: 'NuSkooler',
+};
+
+exports.getModule = class ExodusModule extends MenuModule {
+	constructor(options) {
+		super(options);
+
+		this.config						= options.menuConfig.config || {};
+		this.config.ticketHost			= this.config.ticketHost || 'oddnetwork.org';
+		this.config.ticketPort			= this.config.ticketPort || 1984,
+		this.config.ticketPath			= this.config.ticketPath || '/exodus';		
+		this.config.rejectUnauthorized	= _.get(this.config, 'rejectUnauthorized', true);
+		this.config.sshHost				= this.config.sshHost || this.config.ticketHost;
+		this.config.sshPort				= this.config.sshPort || 22;
+		this.config.sshUser				= this.config.sshUser || 'exodus_server';
+		this.config.sshKeyPem			= this.config.sshKeyPem || joinPath(Config.paths.misc, 'exodus.id_rsa');
+	}
+
+	initSequence() {
+
+		const self				= this;
+		let clientTerminated	= false;
+
+		async.waterfall(
+			[
+				function validateConfig(callback) {
+					//	very basic validation on optionals
+					async.each( [ 'board', 'key', 'door' ], (key, next) => {
+						return _.isString(self.config[key]) ? next(null) : next(Errors.MissingConfig(`Config requires "${key}"!`));
+					}, callback);
+				},
+				function loadCertAuthorities(callback) {
+					if(!_.isString(self.config.caPem)) {
+						return callback(null, null);
+					}
+
+					fs.readFile(self.config.caPem, (err, certAuthorities) => {
+						return callback(err, certAuthorities);
+					});
+				},
+				function getTicket(certAuthorities, callback) {
+					const now		= moment.utc().unix();
+					const sha256	= crypto.createHash('sha256').update(`${self.config.key}${now}`).digest('hex');
+					const token		= `${sha256}|${now}`;
+
+					const postData	= querystring.stringify({
+						token		: token,
+						board		: self.config.board,
+						user		: self.client.user.username,
+						door		: self.config.door,
+					});
+
+					const reqOptions = {
+						hostname			: self.config.ticketHost,
+						port				: self.config.ticketPort,
+						path				: self.config.ticketPath,
+						rejectUnauthorized	: self.config.rejectUnauthorized,
+						method				: 'POST',
+						headers				: {
+							'Content-Type'		: 'application/x-www-form-urlencoded',
+							'Content-Length'	: postData.length,
+							'User-Agent'		: getEnigmaUserAgent(),
+						}
+					};
+
+					if(certAuthorities) {
+						reqOptions.ca = certAuthorities;
+					}
+
+					let ticket = '';
+					const req = https.request(reqOptions, res => {
+						res.on('data', data => {
+							ticket += data;
+						});
+
+						res.on('end', () => {
+							if(ticket.length !== 36) {
+								return callback(Errors.Invalid(`Invalid Exodus ticket: ${ticket}`));
+							}
+
+							return callback(null, ticket);
+						});
+					});
+
+					req.on('error', err => {
+						return callback(Errors.General(`Exodus error: ${err.message}`));
+					});
+
+					req.write(postData);
+					req.end();
+				},
+				function loadPrivateKey(ticket, callback) {
+					fs.readFile(self.config.sshKeyPem, (err, privateKey) => {
+						return callback(err, ticket, privateKey);
+					});
+				},
+				function establishSecureConnection(ticket, privateKey, callback) {
+
+					let pipeRestored = false;
+					let pipedStream;
+
+					function restorePipe() {
+						if(pipedStream && !pipeRestored && !clientTerminated) {
+							self.client.term.output.unpipe(pipedStream);
+							self.client.term.output.resume();
+						}
+					}
+
+					self.client.term.write(resetScreen());
+					self.client.term.write('Connecting to Exodus server, please wait...\n');
+
+					const sshClient = new SSHClient();
+
+					const window = {
+						rows		: self.client.term.termHeight,
+						cols		: self.client.term.termWidth,
+						width		: 0,
+						height		: 0,
+						term		: 'vt100',	//	Want to pass |self.client.term.termClient| here, but we end up getting hung up on :(
+					};
+
+					const options = {
+						env : {
+							exodus : ticket,
+						},
+					};
+
+					sshClient.on('ready', () => {
+						self.client.once('end', () => {
+							self.client.log.info('Connection ended. Terminating Exodus connection');
+							clientTerminated = true;
+							return sshClient.end();
+						});
+
+						sshClient.shell(window, options, (err, stream) => {
+							pipedStream = stream;	//	:TODO: ewwwwwwwww hack
+							self.client.term.output.pipe(stream);
+
+							stream.on('data', d => {
+								return self.client.term.rawWrite(d);
+							});
+
+							stream.on('close', () => {
+								restorePipe();
+								return sshClient.end();
+							});
+
+							stream.on('error', err => {
+								Log.warn( { error : err.message }, 'Exodus SSH client stream error');
+							});
+						});
+					});
+
+					sshClient.on('close', () => {
+						restorePipe();
+						return callback(null);
+					});
+
+					sshClient.connect({
+						host		: self.config.sshHost,
+						port		: self.config.sshPort,
+						username	: self.config.sshUser,
+						privateKey	: privateKey,
+					});
+				}
+			],
+			err => {
+				if(err) {
+					self.client.log.warn( { error : err.message }, 'Exodus error');
+				}
+
+				if(!clientTerminated) {
+					self.prevMenu();
+				}
+			}
+		);
+	}
+};

core/file_area_web.js

@@ -284,7 +284,7 @@ class FileAreaWebAccess {
 
 					resp.on('finish', () => {
 						//	transfer completed fully
-						this.updateDownloadStatsForUserId(servedItem.userId, stats.size);
+						this.updateDownloadStatsForUserIdAndSystemAndSystem(servedItem.userId, stats.size);
 					});
 
 					const headers = {
@@ -301,7 +301,7 @@ class FileAreaWebAccess {
 		});
 	}
 
-	updateDownloadStatsForUserId(userId, dlBytes, cb) {
+	updateDownloadStatsForUserIdAndSystem(userId, dlBytes, cb) {
 		async.waterfall(
 			[
 				function fetchActiveUser(callback) {

core/file_base_filter.js

@@ -1,6 +1,7 @@
 /* jslint node: true */
 'use strict';
 
+//	deps
 const _						= require('lodash');
 const uuidV4				= require('uuid/v4');
 
@@ -65,14 +66,14 @@ module.exports = class FileBaseFilters {
 		let filtersProperty = this.client.user.properties.file_base_filters;
 		let defaulted;
 		if(!filtersProperty) {
-			filtersProperty = JSON.stringify(FileBaseFilters.getDefaultFilters());
+			filtersProperty = JSON.stringify(FileBaseFilters.getBuiltInSystemFilters());
 			defaulted = true;
 		}
 
 		try {
 			this.filters = JSON.parse(filtersProperty);
 		} catch(e) {
-			this.filters = FileBaseFilters.getDefaultFilters();	//	something bad happened; reset everything back to defaults :(
+			this.filters = FileBaseFilters.getBuiltInSystemFilters();	//	something bad happened; reset everything back to defaults :(
 			defaulted = true;
 			this.client.log.error( { error : e.message, property : filtersProperty }, 'Failed parsing file base filters property' );
 		}
@@ -107,18 +108,20 @@ module.exports = class FileBaseFilters {
 		return false;
 	}
 
-	static getDefaultFilters() {
+	static getBuiltInSystemFilters() {
-		const filters = {};
+		const U_LATEST	= '7458b09d-40ab-4f9b-a0d7-0cf866646329';
 
-		const uuid = uuidV4();
+		const filters = {
-		filters[uuid] = {
+			[ U_LATEST ] : {
-			name	: 'Default',
+				name	: 'By Date Added',
 				areaTag	: '',	//	all
 				terms	: '',	//	*
 				tags	: '',	//	*
 				order	: 'descending',
 				sort	: 'upload_timestamp',
-			uuid	: uuid,
+				uuid	: U_LATEST,
+				system	: true,
+			}
 		};
 
 		return filters;
@@ -127,4 +130,20 @@ module.exports = class FileBaseFilters {
 	static getActiveFilter(client) {
 		return new FileBaseFilters(client).get(client.user.properties.file_base_filter_active_uuid);
 	}
+
+	static getFileBaseLastViewedFileIdByUser(user) {
+		return parseInt((user.properties.user_file_base_last_viewed || 0));
+	}
+
+	static setFileBaseLastViewedFileIdForUser(user, fileId, cb) {
+		const current = FileBaseFilters.getFileBaseLastViewedFileIdByUser(user);
+		if(fileId < current) {
+			if(cb) {
+				cb(null);
+			}
+			return;
+		}
+
+		return user.persistProperty('user_file_base_last_viewed', fileId, cb);
+	}
 };

core/file_entry.js

@@ -510,6 +510,14 @@ module.exports = class FileEntry {
 			);
 		}
 
+		if(_.isString(filter.newerThanTimestamp) && filter.newerThanTimestamp.length > 0) {
+			appendWhereClause(`DATETIME(f.upload_timestamp) > DATETIME("${filter.newerThanTimestamp}", "+1 seconds")`);
+		}
+
+		if(_.isNumber(filter.newerThanFileId)) {
+			appendWhereClause(`f.file_id > ${filter.newerThanFileId}`);
+		}
+
 		sql += `${sqlWhere} ${sqlOrderBy};`;
 
 		const matchingFileIds = [];

core/fse.js

@@ -10,10 +10,11 @@ const Message						= require('./message.js');
 const updateMessageAreaLastReadId	= require('./message_area.js').updateMessageAreaLastReadId;
 const getMessageAreaByTag			= require('./message_area.js').getMessageAreaByTag;
 const User							= require('./user.js');
-const cleanControlCodes				= require('./string_util.js').cleanControlCodes;
 const StatLog						= require('./stat_log.js');
 const stringFormat					= require('./string_format.js');
 const MessageAreaConfTempSwitcher	= require('./mod_mixins.js').MessageAreaConfTempSwitcher;
+const { isAnsi, cleanControlCodes, insert }	= require('./string_util.js');
+const Config						= require('./config.js').config;
 
 //	deps
 const async							= require('async');
@@ -168,7 +169,6 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul
 				}
 				cb(newFocusViewId);
 			},
-			
 			headerSubmit : function(formData, extraArgs, cb) {
 				self.switchToBody();
 				return cb(null);
@@ -210,21 +210,24 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul
 			},
 			appendQuoteEntry: function(formData, extraArgs, cb) {
 				//	:TODO: Dont' use magic # ID's here			
-				var quoteMsgView = self.viewControllers.quoteBuilder.getView(1);
+				const quoteMsgView = self.viewControllers.quoteBuilder.getView(1);
 				
 				if(self.newQuoteBlock) {
 					self.newQuoteBlock = false;
+
+					//	:TODO: If replying to ANSI, add a blank sepration line here
+
 					quoteMsgView.addText(self.getQuoteByHeader());
 				}
 				
-				var quoteText = self.viewControllers.quoteBuilder.getView(3).getItem(formData.value.quote);
+				const quoteText = self.viewControllers.quoteBuilder.getView(3).getItem(formData.value.quote);
 				quoteMsgView.addText(quoteText);
 
 				//
 				//	If this is *not* the last item, advance. Otherwise, do nothing as we
 				//	don't want to jump back to the top and repeat already quoted lines
 				//
-				var quoteListView = self.viewControllers.quoteBuilder.getView(3);
+				const quoteListView = self.viewControllers.quoteBuilder.getView(3);
 				if(quoteListView.getData() !== quoteListView.getCount() - 1) {
 					quoteListView.focusNext();
 				} else {
@@ -316,22 +319,38 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul
 		}
 	}
 
-	buildMessage() {
+	buildMessage(cb) {
 		const headerValues = this.viewControllers.header.getFormData().value;
 
-		var msgOpts = {
+		const msgOpts = {
 			areaTag			: this.messageAreaTag,
 			toUserName		: headerValues.to,
 			fromUserName	: this.client.user.username,
 			subject			: headerValues.subject,
-			message			: this.viewControllers.body.getFormData().value.message,
+			//	:TODO: don't hard code 1 here:
+			message			: this.viewControllers.body.getView(1).getData( { forceLineTerms : this.replyIsAnsi } ),
 		};
 
 		if(this.isReply()) {
 			msgOpts.replyToMsgId	= this.replyToMessage.messageId;
+
+			if(this.replyIsAnsi) {
+				//
+				//	Ensure first characters indicate ANSI for detection down
+				//	the line (other boards/etc.). We also set explicit_encoding
+				//	to packetAnsiMsgEncoding (generally cp437) as various boards 
+				//	really don't like ANSI messages in UTF-8 encoding (they should!)
+				//
+				msgOpts.meta		= { System : { 'explicit_encoding' : Config.scannerTossers.ftn_bso.packetAnsiMsgEncoding || 'cp437' } };
+				//	:TODO: change to <ansi>\r\nESC[A<message>
+				//msgOpts.message		= `${ansi.reset()}${ansi.eraseData(2)}${ansi.goto(1,1)}${msgOpts.message}`;
+				msgOpts.message		= `${ansi.reset()}${ansi.eraseData(2)}${ansi.goto(1,1)}\r\n${ansi.up()}${msgOpts.message}`;
+			}
 		}
 
 		this.message = new Message(msgOpts);
+
+		return cb(null);
 	}
 	
 	setMessage(message) {
@@ -344,9 +363,35 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul
 					this.initHeaderViewMode();
 					this.initFooterViewMode();
 
-					var bodyMessageView = this.viewControllers.body.getView(1);
+					const bodyMessageView	= this.viewControllers.body.getView(1);
+					let msg					= this.message.message;
+
 					if(bodyMessageView && _.has(this, 'message.message')) {
-						bodyMessageView.setText(cleanControlCodes(this.message.message));
+						//
+						//	We handle ANSI messages differently than standard messages -- this is required as
+						//	we don't want to do things like word wrap ANSI, but instead, trust that it's formatted
+						//	how the author wanted it
+						//
+						if(isAnsi(msg)) {
+							//
+							//	Find tearline - we want to color it differently.
+							//
+							const tearLinePos = this.message.getTearLinePosition(msg);
+
+							if(tearLinePos > -1) {
+								msg = insert(msg, tearLinePos, bodyMessageView.getSGRFor('text'));
+							}
+
+							bodyMessageView.setAnsi(
+								msg.replace(/\r?\n/g, '\r\n'),	//	messages are stored with CRLF -> LF
+								{
+									prepped				: false,
+									forceLineTerm		: true,
+								}
+							);
+						} else {
+							bodyMessageView.setText(cleanControlCodes(msg));
+						}
 					}
 				}
 			}
@@ -360,9 +405,10 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul
 			[
 				function buildIfNecessary(callback) {
 					if(self.isEditMode()) {
-						self.buildMessage();	//	creates initial self.message
+						return self.buildMessage(callback);	//	creates initial self.message
 					}
-					callback(null);
+
+					return callback(null);
 				},
 				function populateLocalUserInfo(callback) {
 					if(self.isLocalEmail()) {
@@ -659,6 +705,7 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul
 							break;
 							
 						case 'edit' :
+							{
 								const fromView = self.viewControllers.header.getView(1);
 								const area = getMessageAreaByTag(self.messageAreaTag);
 								if(area && area.realNames) {
@@ -670,6 +717,7 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul
 								if(self.replyToMessage) {
 									self.initHeaderReplyEditMode();
 								}
+							}
 							break;
 					}
 
@@ -848,9 +896,31 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul
 					}
 				},
 				function loadQuoteLines(callback) {
-					var quoteView = self.viewControllers.quoteBuilder.getView(3);
+					const quoteView = self.viewControllers.quoteBuilder.getView(3);
-					quoteView.setItems(self.replyToMessage.getQuoteLines(quoteView.dimens.width));
+					const bodyView	= self.viewControllers.body.getView(1);
-					callback(null);
+
+					self.replyToMessage.getQuoteLines(
+						{
+							termWidth			: self.client.term.termWidth,
+							termHeight			: self.client.term.termHeight,
+							cols				: quoteView.dimens.width,
+							startCol			: quoteView.position.col,
+							ansiResetSgr		: bodyView.styleSGR1,
+							ansiFocusPrefixSgr	: quoteView.styleSGR2,
+						},
+						(err, quoteLines, focusQuoteLines, replyIsAnsi) => {
+							if(err) {
+								return callback(err);
+							}
+
+							self.replyIsAnsi = replyIsAnsi;
+
+							quoteView.setItems(quoteLines);
+							quoteView.setFocusItems(focusQuoteLines);
+
+							return callback(null);
+						}
+					);
 				},
 				function setViewFocus(callback) {
 					self.viewControllers.quoteBuilder.getView(1).setFocus(false);
@@ -922,14 +992,17 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul
 	
 	quoteBuilderFinalize() {
 		//	:TODO: fix magic #'s
-		var quoteMsgView	= this.viewControllers.quoteBuilder.getView(1);
+		const quoteMsgView	= this.viewControllers.quoteBuilder.getView(1);
-		var msgView			= this.viewControllers.body.getView(1);
+		const msgView		= this.viewControllers.body.getView(1);
 				
-		var quoteLines 		= quoteMsgView.getData();
+		let quoteLines 		= quoteMsgView.getData();
 		
 		if(quoteLines.trim().length > 0) {
-			msgView.addText(quoteMsgView.getData() + '\n');
+			if(this.replyIsAnsi) {
-		
+				const bodyMessageView = this.viewControllers.body.getView(1);
+				quoteLines += `${ansi.normal()}${bodyMessageView.getSGRFor('text')}`;
+			}
+			msgView.addText(`${quoteLines}\n`);
 		}
 		
 		quoteMsgView.setText('');

core/ftn_mail_packet.js

@@ -7,6 +7,7 @@ const sauce			= require('./sauce.js');
 const Address		= require('./ftn_address.js');
 const strUtil		= require('./string_util.js');
 const Log			= require('./logger.js').log;
+const ansiPrep		= require('./ansi_prep.js');
 
 const _				= require('lodash');
 const assert		= require('assert');
@@ -480,8 +481,8 @@ function Packet(options) {
 						Log.debug( { encoding : encoding, error : e.toString() }, 'Error decoding. Falling back to ASCII');
 						decoded = iconv.decode(messageBodyBuffer, 'ascii');
 					}
-					//const messageLines = iconv.decode(messageBodyBuffer, encoding).replace(/\xec/g, '').split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);
+					
-					const messageLines	= decoded.replace(/\xec/g, '').split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);
+					const messageLines	= strUtil.splitTextAtTerms(decoded.replace(/\xec/g, ''));
 					let endOfMessage	= false;
 
 					messageLines.forEach(line => {
@@ -636,8 +637,26 @@ function Packet(options) {
 			});
 	};
 		
-	this.getMessageEntryBuffer = function(message, options) {
+	this.getMessageEntryBuffer = function(message, options, cb) {
-		let basicHeader = new Buffer(34);
+		
+		function getAppendMeta(k, m) {
+			let append = '';
+			if(m) {
+				let a = m;
+				if(!_.isArray(a)) {
+					a = [ a ];
+				}
+				a.forEach(v => {
+					append += `${k}: ${v}\r`;
+				});
+			}
+			return append;
+		}
+
+		async.waterfall(
+			[
+				function prepareHeaderAndKludges(callback) {
+					const basicHeader = new Buffer(34);
 		
 					basicHeader.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0);
 					basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_node, 2);
@@ -650,17 +669,12 @@ function Packet(options) {
 					const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0');
 					dateTimeBuffer.copy(basicHeader, 14);
 
-		//	toUserName & fromUserName: up to 36 bytes in length, NULL term'd
+					//
-		//	:TODO: DRY...
+					//	To, from, and subject must be NULL term'd and have max lengths as per spec.
-		let toUserNameBuf = iconv.encode(message.toUserName + '\0', 'CP437').slice(0, 36);
+					//
-		toUserNameBuf[toUserNameBuf.length - 1] = '\0';	//	ensure it's null term'd
+					const toUserNameBuf 	= strUtil.stringToNullTermBuffer(message.toUserName, { encoding : 'cp437', maxBufLen : 36 } );
-		
+					const fromUserNameBuf	= strUtil.stringToNullTermBuffer(message.fromUserName, { encoding : 'cp437', maxBufLen : 36 } );
-		let fromUserNameBuf = iconv.encode(message.fromUserName + '\0', 'CP437').slice(0, 36);
+					const subjectBuf		= strUtil.stringToNullTermBuffer(message.subject, { encoding : 'cp437', maxBufLen : 72 } );
-		fromUserNameBuf[fromUserNameBuf.length - 1] = '\0';	//	ensure it's null term'd
-		
-		//	subject: up to 72 bytes in length, NULL term'd
-		let subjectBuf = iconv.encode(message.subject + '\0', 'CP437').slice(0, 72);
-		subjectBuf[subjectBuf.length - 1] = '\0';	//	ensure it's null term'd
 
 					//
 					//	message: unbound length, NULL term'd
@@ -668,21 +682,8 @@ function Packet(options) {
 					//	We need to build in various special lines - kludges, area,
 					//	seen-by, etc.
 					//
-		//	:TODO: Put this in it's own method
 					let msgBody = '';
 
-		function appendMeta(k, m) {
-			if(m) {
-				let a = m;
-				if(!_.isArray(a)) {
-					a = [ a ];
-				}
-				a.forEach(v => {
-					msgBody += `${k}: ${v}\r`;
-				});
-			}
-		}
-		
 					//
 					//	FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001
 					//	AREA:CONFERENCE
@@ -695,11 +696,32 @@ function Packet(options) {
 					Object.keys(message.meta.FtnKludge).forEach(k => {
 						//	we want PATH to be last
 						if('PATH' !== k) {
-				appendMeta(`\x01${k}`, message.meta.FtnKludge[k]);
+							msgBody += getAppendMeta(`\x01${k}`, message.meta.FtnKludge[k]);
 						}
 					});
 
-		msgBody += message.message + '\r';
+					return callback(null, basicHeader, toUserNameBuf, fromUserNameBuf, subjectBuf, msgBody);
+				},
+				function prepareAnsiMessageBody(basicHeader, toUserNameBuf, fromUserNameBuf, subjectBuf, msgBody, callback) {
+					if(!strUtil.isAnsi(message.message)) {
+						return callback(null, basicHeader, toUserNameBuf, fromUserNameBuf, subjectBuf, msgBody, message.message);
+					}
+
+					ansiPrep(
+						message.message,
+						{
+							cols				: 80,
+							rows				: 'auto',
+							forceLineTerm		: true,
+							exportMode			: true,
+						},
+						(err, preppedMsg) => {
+							return callback(null, basicHeader, toUserNameBuf, fromUserNameBuf, subjectBuf, msgBody, preppedMsg || message.message);
+						}
+					);
+				},
+				function addMessageBody(basicHeader, toUserNameBuf, fromUserNameBuf, subjectBuf, msgBody, preppedMsg, callback) {
+					msgBody += preppedMsg + '\r';
 
 					//
 					//	FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001
@@ -720,9 +742,8 @@ function Packet(options) {
 					//	FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001
 					//	SEEN-BY and PATH should be the last lines of a message
 					//
-		appendMeta('SEEN-BY', message.meta.FtnProperty.ftn_seen_by);	//	note: no ^A (0x01)
+					msgBody += getAppendMeta('SEEN-BY', message.meta.FtnProperty.ftn_seen_by);	//	note: no ^A (0x01)
-
+					msgBody += getAppendMeta('\x01PATH', message.meta.FtnKludge['PATH']);
-		appendMeta('\x01PATH', message.meta.FtnKludge['PATH']);
 					
 					let msgBodyEncoded;
 					try {
@@ -731,13 +752,22 @@ function Packet(options) {
 						msgBodyEncoded = iconv.encode(msgBody + '\0', 'ascii');
 					}
 					
-		return Buffer.concat( [ 
+					return callback(
+						null, 
+						Buffer.concat( [ 
 							basicHeader, 
 							toUserNameBuf, 
 							fromUserNameBuf, 
 							subjectBuf,
 							msgBodyEncoded 
-		]);
+						])
+					);
+				}
+			],
+			(err, msgEntryBuffer) => {
+				return cb(err, msgEntryBuffer);
+			}
+		);
 	};
 
 	this.writeMessage = function(message, ws, options) {

core/ftn_util.js

@@ -4,6 +4,7 @@
 let Config					= require('./config.js').config;
 let Address					= require('./ftn_address.js');
 let FNV1a					= require('./fnv1a.js');
+const getCleanEnigmaVersion	= require('./misc_util.js').getCleanEnigmaVersion;
 
 let _						= require('lodash');
 let iconv					= require('iconv-lite');
@@ -146,11 +147,7 @@ function getMessageIdentifier(message, address) {
 //	in which (<os>; <arch>; <nodeVer>) is used instead
 //
 function getProductIdentifier() {
-	const version = packageJson.version
+	const version = getCleanEnigmaVersion();
-		.replace(/\-/g, '.')
-		.replace(/alpha/,'a')
-		.replace(/beta/,'b');
-
 	const nodeVer = process.version.substr(1);	//	remove 'v' prefix
 
 	return `ENiGMA1/2 ${version} (${os.platform()}; ${os.arch()}; ${nodeVer})`;
@@ -166,10 +163,23 @@ function getUTCTimeZoneOffset() {
 	return moment().format('ZZ').replace(/\+/, '');
 }
 
-//	Get a FSC-0032 style quote prefixes
+//
+//	Get a FSC-0032 style quote prefix
+//	http://ftsc.org/docs/fsc-0032.001
+//	
 function getQuotePrefix(name) {
-	//	:TODO: Add support for real names (e.g. with spaces) -> initials
+	let initials;
-	return ' ' + name[0].toUpperCase() + name[1].toLowerCase() + '> ';
+	
+	const parts = name.split(' ');
+	if(parts.length > 1) {
+		//	First & Last initials - (Bryan Ashby -> BA)
+		initials = `${parts[0].slice(0, 1)}${parts[parts.length - 1].slice(0, 1)}`.toUpperCase();
+	} else {
+		//	Just use the first two - (NuSkooler -> Nu)
+		initials = _.capitalize(name.slice(0, 2));
+	}
+
+	return ` ${initials}> `;
 }
 
 //

core/logger.js

@@ -62,7 +62,7 @@ module.exports = class Log {
 			//	Use a regexp -- we don't know how nested fields we want to seek and destroy may be
 			//
 			return JSON.parse(
-				JSON.stringify(obj).replace(/"(password)"\s?:\s?"([^"]+)"/, (match, valueName) => {
+				JSON.stringify(obj).replace(/"(password|key|authCode)"\s?:\s?"([^"]+)"/, (match, valueName) => {
 					return `"${valueName}":"********"`;
 				})
 			);

core/menu_module.js

@@ -325,11 +325,14 @@ exports.MenuModule = class MenuModule extends PluginModule {
 				formId			: formId,
 			};
 
-			return vc.loadFromMenuConfig(loadOpts, cb);
+			return vc.loadFromMenuConfig(loadOpts, err => {
+				return cb(err, vc);
+			});
 		}
 
 		this.viewControllers[name].setFocus(true);
-		return cb(null);
+
+		return cb(null, this.viewControllers[name]);
 	}
 
 	prepViewControllerWithArt(name, formId, options, cb) {

core/menu_view.js

@@ -113,6 +113,14 @@ MenuView.prototype.focusPrevious = function() {
 	this.emit('index update', this.focusedItemIndex);
 };
 
+MenuView.prototype.focusNextPageItem = function() {
+	this.emit('index update', this.focusedItemIndex);
+};
+
+MenuView.prototype.focusPreviousPageItem = function() {
+	this.emit('index update', this.focusedItemIndex);
+};
+
 MenuView.prototype.setFocusItemIndex = function(index) {
 	this.focusedItemIndex = index;
 };

core/message.js

@@ -6,6 +6,16 @@ const wordWrapText			= require('./word_wrap.js').wordWrapText;
 const ftnUtil				= require('./ftn_util.js');
 const createNamedUUID		= require('./uuid_util.js').createNamedUUID;
 const getISOTimestampString	= require('./database.js').getISOTimestampString;
+const Errors				= require('./enig_error.js').Errors;
+const ANSI					= require('./ansi_term.js');
+
+const { 
+	isAnsi, isFormattedLine,
+	splitTextAtTerms, 
+	renderSubstr
+}							= require('./string_util.js');
+
+const ansiPrep				= require('./ansi_prep.js');
 
 //	deps
 const uuidParse				= require('uuid-parse');
@@ -82,6 +92,7 @@ Message.SystemMetaNames = {
 	LocalToUserID			: 'local_to_user_id',
 	LocalFromUserID			: 'local_from_user_id',
 	StateFlags0				: 'state_flags0',		//	See Message.StateFlags0
+	ExplicitEncoding		: 'explicit_encoding',	//	Explicitly set encoding when exporting/etc.
 };
 
 Message.StateFlags0 = {
@@ -429,47 +440,192 @@ Message.prototype.getFTNQuotePrefix = function(source) {
 	return ftnUtil.getQuotePrefix(this[source]);
 };
 
-Message.prototype.getQuoteLines = function(width, options) {
+Message.prototype.getTearLinePosition = function(input) {
-	//	:TODO: options.maxBlankLines = 1
+	const m = input.match(/^--- .+$(?![\s\S]*^--- .+$)/m);
+	return m ? m.index : -1;
+};
 
-	options = options || {};
+Message.prototype.getQuoteLines = function(options, cb) {
-	
+	if(!options.termWidth || !options.termHeight || !options.cols) {
-	//
+		return cb(Errors.MissingParam());
-	//	Include FSC-0032 style quote prefixes?
-	//
-	//	See http://ftsc.org/docs/fsc-0032.001
-	//
-	if(!_.isBoolean(options.includePrefix)) {
-		options.includePrefix = true;
 	}
 	
-	var quoteLines = [];
+	options.startCol			= options.startCol || 1;
+	options.includePrefix 		= _.get(options, 'includePrefix', true);
+	options.ansiResetSgr		= options.ansiResetSgr || ANSI.getSGRFromGraphicRendition( { fg : 39, bg : 49 }, true);
+	options.ansiFocusPrefixSgr	= options.ansiFocusPrefixSgr || ANSI.getSGRFromGraphicRendition( { intensity : 'bold', fg : 39, bg : 49 } );
+	options.isAnsi				= options.isAnsi || isAnsi(this.message);	//	:TODO: If this.isAnsi, use that setting
 	
-	var origLines = this.message
+	/*
-		.trim()
+		Some long text that needs to be wrapped and quoted should look right after
-		.replace(/\b/g, '')
+		doing so, don't ya think? yeah I think so		
-		.split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);	
 
-	var quotePrefix = '';	//	we need this init even if blank
+		Nu> Some long text that needs to be wrapped and quoted should look right 
-	if(options.includePrefix) {
+		Nu> after doing so, don't ya think? yeah I think so
-		quotePrefix = this.getFTNQuotePrefix(options.prefixSource || 'fromUserName');
-	}
 
-	var wrapOpts = {
+		Ot> Nu> Some long text that needs to be wrapped and quoted should look 
-		width		: width - quotePrefix.length,
+		Ot> Nu> right after doing so, don't ya think? yeah I think so
+
+	*/
+	const quotePrefix = options.includePrefix ? this.getFTNQuotePrefix(options.prefixSource || 'fromUserName') : '';
+
+	function getWrapped(text, extraPrefix) {
+		extraPrefix = extraPrefix ? ` ${extraPrefix}` : '';
+
+		const wrapOpts = {
+			width		: options.cols - (quotePrefix.length + extraPrefix.length),
 			tabHandling	: 'expand',
 			tabWidth	: 4,
 		};
 		
-	function addPrefix(l) {
+		return wordWrapText(text, wrapOpts).wrapped.map( (w, i) => {
-		return quotePrefix + l;
+			return i === 0 ? `${quotePrefix}${w}` : `${quotePrefix}${extraPrefix}${w}`;
+		});
 	}
 
-	var wrapped;
+	function getFormattedLine(line) {
-	for(var i = 0; i < origLines.length; ++i) {
+		//	for pre-formatted text, we just append a line truncated to fit
-		wrapped = wordWrapText(origLines[i], wrapOpts).wrapped;
+		let newLen;
-		Array.prototype.push.apply(quoteLines, _.map(wrapped, addPrefix));
+		const total = line.length + quotePrefix.length;
+
+		if(total > options.cols) {
+			newLen = options.cols - total;
+		} else {
+			newLen = total;
 		}
 
-	return quoteLines;
+		return `${quotePrefix}${line.slice(0, newLen)}`;
+	}
+
+	if(options.isAnsi) {
+		ansiPrep(
+			this.message.replace(/\r?\n/g, '\r\n'),	//	normalized LF -> CRLF
+			{
+				termWidth		: options.termWidth,
+				termHeight		: options.termHeight,
+				cols			: options.cols,
+				rows			: 'auto',
+				startCol		: options.startCol,
+				forceLineTerm	: true,				
+			},
+			(err, prepped) => {
+				prepped = prepped || this.message;
+				
+				let lastSgr = '';
+				const split = splitTextAtTerms(prepped);
+				
+				const quoteLines		= [];
+				const focusQuoteLines	= [];
+
+				//
+				//	Do not include quote prefixes (e.g. XX> ) on ANSI replies (and therefor quote builder)
+				//	as while this works in ENiGMA, other boards such as Mystic, WWIV, etc. will try to 
+				//	strip colors, colorize the lines, etc. If we exclude the prefixes, this seems to do
+				//	the trick and allow them to leave them alone!
+				//
+				split.forEach(l => {
+					quoteLines.push(`${lastSgr}${l}`);
+					
+					focusQuoteLines.push(`${options.ansiFocusPrefixSgr}>${lastSgr}${renderSubstr(l, 1, l.length - 1)}`);
+					lastSgr = (l.match(/(?:\x1b\x5b)[\?=;0-9]*m(?!.*(?:\x1b\x5b)[\?=;0-9]*m)/) || [])[0] || '';	//	eslint-disable-line no-control-regex
+				});
+
+				quoteLines[quoteLines.length - 1] += options.ansiResetSgr;
+				
+				return cb(null, quoteLines, focusQuoteLines, true);
+			}
+		);
+	} else {
+		const QUOTE_RE	= /^ ((?:[A-Za-z0-9]{2}\> )+(?:[A-Za-z0-9]{2}\>)*) */;
+		const quoted	= [];
+		const input		= _.trimEnd(this.message).replace(/\b/g, '');
+	
+		//	find *last* tearline
+		let tearLinePos = this.getTearLinePosition(input);
+		tearLinePos = -1 === tearLinePos ? input.length : tearLinePos;	//	we just want the index or the entire string
+		
+		input.slice(0, tearLinePos).split(/\r\n\r\n|\n\n/).forEach(paragraph => {
+			//
+			//	For each paragraph, a state machine:
+			//	- New line - line
+			//	- New (pre)quoted line - quote_line
+			//	- Continuation of new/quoted line
+			//
+			//	Also:
+			//	- Detect pre-formatted lines & try to keep them as-is
+			//
+			let state;
+			let buf = '';
+			let quoteMatch;
+
+			if(quoted.length > 0) {
+				//
+				//	Preserve paragraph seperation.
+				//
+				//	FSC-0032 states something about leaving blank lines fully blank
+				//	(without a prefix) but it seems nicer (and more consistent with other systems)
+				//	to put 'em in.
+				//
+				quoted.push(quotePrefix);
+			}
+
+			paragraph.split(/\r?\n/).forEach(line => {
+				if(0 === line.trim().length) {
+					//	see blank line notes above
+					return quoted.push(quotePrefix);
+				}
+
+				quoteMatch = line.match(QUOTE_RE);
+
+				switch(state) {
+					case 'line' :
+						if(quoteMatch) {
+							if(isFormattedLine(line)) {
+								quoted.push(getFormattedLine(line.replace(/\s/, '')));
+							} else {
+								quoted.push(...getWrapped(buf, quoteMatch[1]));
+								state = 'quote_line';
+								buf = line;
+							}
+						} else {
+							buf += ` ${line}`;
+						}
+						break;
+						
+					case 'quote_line' :
+						if(quoteMatch) {
+							const rem = line.slice(quoteMatch[0].length);
+							if(!buf.startsWith(quoteMatch[0])) {
+								quoted.push(...getWrapped(buf, quoteMatch[1]));
+								buf = rem;
+							} else {
+								buf += ` ${rem}`;
+							}
+						} else {
+							quoted.push(...getWrapped(buf));
+							buf = line;
+							state = 'line';
+						}
+						break;
+							
+					default :
+						if(isFormattedLine(line)) {
+							quoted.push(getFormattedLine(line));
+						} else {
+							state	= quoteMatch ? 'quote_line' : 'line';
+							buf		= 'line' === state ? line : line.replace(/\s/, '');	//	trim *first* leading space, if any
+						}
+						break;
+				}		
+			});
+			
+			quoted.push(...getWrapped(buf, quoteMatch ? quoteMatch[1] : null));
+		});
+		
+		input.slice(tearLinePos).split(/\r?\n/).forEach(l => {
+			quoted.push(...getWrapped(l));
+		});
+
+		return cb(null, quoted, null, false);
+	}
 };

core/mime_util.js

@@ -1,10 +1,38 @@
 /* jslint node: true */
 'use strict';
 
+//	deps
+const _			= require('lodash');
+
 const mimeTypes	= require('mime-types');
 
+exports.startup				= startup;
 exports.resolveMimeType		= resolveMimeType;
 
+function startup(cb) {
+	//
+	//	Add in types (not yet) supported by mime-db -- and therefor, mime-types
+	//
+	const ADDITIONAL_EXT_MIMETYPES = {
+		arj				: 'application/x-arj',
+		ans				: 'text/x-ansi',
+		gz				: 'application/gzip',	//	not in mime-types 2.1.15 :(
+	};
+
+	_.forEach(ADDITIONAL_EXT_MIMETYPES, (mimeType, ext) => {
+		//	don't override any entries
+		if(!_.isString(mimeTypes.types[ext])) {
+			mimeTypes[ext] = mimeType;
+		}
+
+		if(!mimeTypes.extensions[mimeType]) {
+			mimeTypes.extensions[mimeType] = [ ext ];
+		}
+	});
+
+	return cb(null);
+}
+
 function resolveMimeType(query) {
 	if(mimeTypes.extensions[query]) {
 		return query;	//	alreaed a mime-type

core/misc_util.js

@@ -1,12 +1,17 @@
 /* jslint node: true */
 'use strict';
 
-var paths					= require('path');
+const paths			= require('path');
+
+const os			= require('os');
+const packageJson 	= require('../package.json');
 
 exports.isProduction			= isProduction;
 exports.isDevelopment			= isDevelopment;
 exports.valueWithDefault		= valueWithDefault;
 exports.resolvePath				= resolvePath;
+exports.getCleanEnigmaVersion	= getCleanEnigmaVersion;
+exports.getEnigmaUserAgent		= getEnigmaUserAgent;
 
 function isProduction() {
 	var env = process.env.NODE_ENV || 'dev';
@@ -28,3 +33,20 @@ function resolvePath(path) {
 	}
 	return paths.resolve(path);
 }
+
+function getCleanEnigmaVersion() {
+	return packageJson.version
+		.replace(/\-/g, '.')
+		.replace(/alpha/,'a')
+		.replace(/beta/,'b')
+		;
+}
+
+//	See also ftn_util.js getTearLine() & getProductIdentifier()
+function getEnigmaUserAgent() {
+	//	can't have 1/2 or ½ in User-Agent according to RFC 1945  :(
+	const version = getCleanEnigmaVersion();
+	const nodeVer = process.version.substr(1);	//	remove 'v' prefix
+
+	return `ENiGMA-BBS/${version} (${os.platform()}; ${os.arch()}; ${nodeVer})`;
+}

core/module_util.js

@@ -15,6 +15,7 @@ const async		= require('async');
 exports.loadModuleEx			= loadModuleEx;
 exports.loadModule				= loadModule;
 exports.loadModulesForCategory	= loadModulesForCategory;
+exports.getModulePaths			= getModulePaths;
 
 function loadModuleEx(options, cb) {
 	assert(_.isObject(options));
@@ -97,3 +98,12 @@ function loadModulesForCategory(category, iterator, complete) {
 		});
 	});
 }
+
+function getModulePaths() {
+	return [
+		Config.paths.mods,
+		Config.paths.loginServers,
+		Config.paths.contentServers,
+		Config.paths.scannerTossers,
+	];
+}

core/multi_line_edit_text_view.js

@@ -6,6 +6,7 @@ const strUtil		= require('./string_util.js');
 const ansi			= require('./ansi_term.js');
 const colorCodes	= require('./color_codes.js');
 const wordWrapText	= require('./word_wrap.js').wordWrapText;
+const ansiPrep		= require('./ansi_prep.js');
 
 const assert		= require('assert');
 const _				= require('lodash');
@@ -62,7 +63,7 @@ const _				= require('lodash');
 //	*
 
 
-var SPECIAL_KEY_MAP_DEFAULT = {
+const SPECIAL_KEY_MAP_DEFAULT = {
 	'line feed'		: [ 'return' ],
 	exit			: [ 'esc' ],
 	backspace		: [ 'backspace' ],
@@ -165,43 +166,50 @@ function MultiLineEditTextView(options) {
 		return self.textLines.length;
 	};
 
+	this.toggleTextCursor = function(action) {
+		self.client.term.rawWrite(`${self.getSGRFor('text')}${'hide' === action ? ansi.hideCursor() : ansi.showCursor()}`);
+	};
+
 	this.redrawRows = function(startRow, endRow) {
-		self.client.term.rawWrite(self.getSGRFor('text') + ansi.hideCursor());
+		self.toggleTextCursor('hide');
 
-		var startIndex	= self.getTextLinesIndex(startRow);
+		const startIndex	= self.getTextLinesIndex(startRow);
-		var endIndex	= Math.min(self.getTextLinesIndex(endRow), self.textLines.length);
+		const endIndex		= Math.min(self.getTextLinesIndex(endRow), self.textLines.length);
-		var absPos		= self.getAbsolutePosition(startRow, 0);
+		const absPos		= self.getAbsolutePosition(startRow, 0);
 
-		for(var i = startIndex; i < endIndex; ++i) {
+		for(let i = startIndex; i < endIndex; ++i) {
+			//${self.getSGRFor('text')}
 			self.client.term.write(				
-				ansi.goto(absPos.row++, absPos.col) +
+				`${ansi.goto(absPos.row++, absPos.col)}${self.getRenderText(i)}`, 
-				self.getRenderText(i), false);
+				false	//	convertLineFeeds
+			);
 		}
 
-		self.client.term.rawWrite(ansi.showCursor());
+		self.toggleTextCursor('show');
 
 		return absPos.row - self.position.row;	//	row we ended on
 	};
 
 	this.eraseRows = function(startRow, endRow) {
-		self.client.term.rawWrite(self.getSGRFor('text') + ansi.hideCursor());
+		self.toggleTextCursor('hide');
 
-		var absPos		= self.getAbsolutePosition(startRow, 0);
+		const absPos		= self.getAbsolutePosition(startRow, 0);
-		var absPosEnd	= self.getAbsolutePosition(endRow, 0);
+		const absPosEnd		= self.getAbsolutePosition(endRow, 0);
-		var eraseFiller	= new Array(self.dimens.width).join(' ');
+		const eraseFiller	= ' '.repeat(self.dimens.width);//new Array(self.dimens.width).join(' ');
 
 		while(absPos.row < absPosEnd.row) {
 			self.client.term.write(
-				ansi.goto(absPos.row++, absPos.col) +
+				`${ansi.goto(absPos.row++, absPos.col)}${eraseFiller}`,
-				eraseFiller, false);
+				false	//	convertLineFeeds
+			);
 		}
 
-		self.client.term.rawWrite(ansi.showCursor());
+		self.toggleTextCursor('show');
 	};
 
 	this.redrawVisibleArea = function() {
 		assert(self.topVisibleIndex <= self.textLines.length);
-		var lastRow = self.redrawRows(0, self.dimens.height);
+		const lastRow = self.redrawRows(0, self.dimens.height);
 
 		self.eraseRows(lastRow, self.dimens.height);
 		/*
@@ -255,11 +263,14 @@ function MultiLineEditTextView(options) {
 	};
 
 	this.getRenderText = function(index) {
-		var text = self.getVisibleText(index);
+		let text 		= self.getVisibleText(index);
-		var remain	= self.dimens.width - text.length;
+		const remain	= self.dimens.width - text.length;
+
 		if(remain > 0) {
-			text += new Array(remain + 1).join(' ');
+			text += ' '.repeat(remain + 1);
+//			text += new Array(remain + 1).join(' ');
 		}
+
 		return text;
 	};
 
@@ -273,14 +284,15 @@ function MultiLineEditTextView(options) {
 		return lines;
 	};
 
-	this.getOutputText = function(startIndex, endIndex, eolMarker) {
+	this.getOutputText = function(startIndex, endIndex, eolMarker, options) {
-		let lines   = self.getTextLines(startIndex, endIndex);
+		const lines	= self.getTextLines(startIndex, endIndex);
 		let text	= '';
-		var re      = new RegExp('\\t{1,' + (self.tabWidth) + '}', 'g');
+		const re	= new RegExp('\\t{1,' + (self.tabWidth) + '}', 'g');
 
 		lines.forEach(line => {
 			text += line.text.replace(re, '\t');
-			if(eolMarker && line.eol) {
+			
+			if(options.forceLineTerms || (eolMarker && line.eol)) {
 				text += eolMarker;
 			}
 		});
@@ -491,25 +503,90 @@ function MultiLineEditTextView(options) {
 		return new Array(self.getRemainingTabWidth(col)).join(expandChar);
 	};
 
-	this.getStringLength = function(s) {
-		return self.isPreviewMode() ? colorCodes.enigmaStrLen(s) : s.length;
-	};
-
 	this.wordWrapSingleLine = function(s, tabHandling, width) {
 		if(!_.isNumber(width)) {
 			width = self.dimens.width;
 		}
 
 		return wordWrapText(
-			s, {
+			s, 
+			{
 				width		: width,
 				tabHandling	: tabHandling || 'expand',
 				tabWidth	: self.tabWidth,
 				tabChar		: '\t',
-			});
+			}
+		);
+	};
+
+	this.setTextLines = function(lines, index, termWithEol) {
+		if(0 === index && (0 === self.textLines.length || (self.textLines.length === 1 && '' === self.textLines[0].text) )) {
+			//	quick path: just set the things
+			self.textLines = lines.slice(0, -1).map(l => {
+				return { text : l };
+			}).concat( { text : lines[lines.length - 1], eol : termWithEol } );
+		} else {
+			//	insert somewhere in textLines...
+			if(index > self.textLines.length) {
+				//	fill with empty
+				self.textLines.splice(
+					self.textLines.length,
+					0,
+					...Array.from( { length : index - self.textLines.length } ).map( () => { return { text : '' }; } )
+				);
+			}
+
+			const newLines = lines.slice(0, -1).map(l => {
+				return { text : l };
+			}).concat( { text : lines[lines.length - 1], eol : termWithEol } );
+
+			self.textLines.splice(
+				index,
+				0,
+				...newLines
+			);
+		}
+	};
+
+	this.setAnsiWithOptions = function(ansi, options, cb) {
+
+		function setLines(text) {
+			text = strUtil.splitTextAtTerms(text);
+
+			let index = 0;
+
+			text.forEach(line => {
+				self.setTextLines( [ line ], index, true);	//	true=termWithEol
+				index += 1;
+			});
+
+			self.cursorStartOfDocument();
+
+			if(cb) {
+				return cb(null);
+			}
+		}
+
+		if(options.prepped) {
+			return setLines(ansi);
+		}
+
+		ansiPrep(
+			ansi,
+			{
+				termWidth			: this.client.term.termWidth,
+				termHeight			: this.client.term.termHeight,
+				cols				: this.dimens.width,
+				rows				: 'auto',
+				startCol			: this.position.col,
+				forceLineTerm		: options.forceLineTerm,
+			},
+			(err, preppedAnsi) => {
+				return setLines(err ? ansi : preppedAnsi);
+			}
+		);
 	};
 
-	//	:TODO: rename to insertRawText()
 	this.insertRawText = function(text, index, col) {
 		//
 		//	Perform the following on |text|:
@@ -542,23 +619,19 @@ function MultiLineEditTextView(options) {
 			index = self.textLines.length;
 		}
 
-		text = text
+		text = strUtil.splitTextAtTerms(text);
-			.replace(/\b/g, '')
-			.split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);
 
 		let wrapped;
-
+		text.forEach(line => {
-		for(let i = 0; i < text.length; ++i) {
 			wrapped = self.wordWrapSingleLine(
-				text[i], 	//	input
+				line,	 	//	line to wrap
 				'expand', 	//	tabHandling
-				self.dimens.width).wrapped;
+				self.dimens.width
+			).wrapped;
 
-			for(let j = 0; j < wrapped.length - 1; ++j) {
+			self.setTextLines(wrapped, index, true);	//	true=termWithEol
-				self.textLines.splice(index++, 0, { text : wrapped[j] } );
+			index += wrapped.length;
-			}
+		});
-			self.textLines.splice(index++, 0, { text : wrapped[wrapped.length - 1], eol : true } );
-		}
 	};
 
 	this.getAbsolutePosition = function(row, col) {
@@ -996,8 +1069,6 @@ MultiLineEditTextView.prototype.setFocus = function(focused) {
 };
 
 MultiLineEditTextView.prototype.setText = function(text) {
-	//text = require('graceful-fs').readFileSync('/home/nuskooler/Downloads/test_text.txt', { encoding : 'utf-8'});
-
 	this.textLines = [ ];
 	this.addText(text);
 	/*this.insertRawText(text);
@@ -1009,6 +1080,11 @@ MultiLineEditTextView.prototype.setText = function(text) {
 	}*/
 };
 
+MultiLineEditTextView.prototype.setAnsi = function(ansi, options = { prepped : false }, cb) {
+	this.textLines = [ ];
+	return this.setAnsiWithOptions(ansi, options, cb);
+};
+
 MultiLineEditTextView.prototype.addText = function(text) {
 	this.insertRawText(text);
 
@@ -1019,8 +1095,8 @@ MultiLineEditTextView.prototype.addText = function(text) {
 	}
 };
 
-MultiLineEditTextView.prototype.getData = function() {
+MultiLineEditTextView.prototype.getData = function(options = { forceLineTerms : false } ) {
-	return this.getOutputText(0, this.textLines.length, '\r\n');
+	return this.getOutputText(0, this.textLines.length, '\r\n', options);
 };
 
 MultiLineEditTextView.prototype.setPropertyValue = function(propName, value) {
@@ -1032,7 +1108,9 @@ MultiLineEditTextView.prototype.setPropertyValue = function(propName, value) {
 			} 
 			break;
 
-		case 'autoScroll'	: this.autoScroll = value; break;
+		case 'autoScroll'	: 
+			this.autoScroll = value;
+			break;
 
 		case 'tabSwitchesView' :
 			this.tabSwitchesView = value; 

core/new_scan.js

@@ -6,6 +6,9 @@ const msgArea			= require('./message_area.js');
 const MenuModule		= require('./menu_module.js').MenuModule;
 const ViewController	= require('./view_controller.js').ViewController;
 const stringFormat		= require('./string_format.js');
+const FileEntry			= require('./file_entry.js');
+const FileBaseFilters	= require('./file_base_filter.js');
+const Errors			= require('./enig_error.js').Errors;
 
 //	deps
 const _					= require('lodash');
@@ -30,13 +33,21 @@ const MciCodeIds = {
 	ScanStatusList	: 2,	//	VM2 (appends)
 };
 
+const Steps = {
+	MessageConfs	: 'messageConferences',
+	FileBase		: 'fileBase',
+	
+	Finished		: 'finished',
+};
+
 exports.getModule = class NewScanModule extends MenuModule {
 	constructor(options) {
 		super(options);
 
-		this.newScanFullExit	= _.has(options, 'lastMenuResult.fullExit') ? options.lastMenuResult.fullExit : false;
 
-		this.currentStep		= 'messageConferences';
+		this.newScanFullExit	= _.get(options, 'lastMenuResult.fullExit', false);
+
+		this.currentStep		= Steps.MessageConfs;
 		this.currentScanAux		= {};
 
 		//  :TODO: Make this conf/area specific:
@@ -49,13 +60,6 @@ exports.getModule = class NewScanModule extends MenuModule {
 
 	updateScanStatus(statusText) {
 		this.setViewText('allViews', MciCodeIds.ScanStatusLabel, statusText);
-
-		/*
-		view = vc.getView(MciCodeIds.ScanStatusList);
-		//	:TODO: MenuView needs appendItem()
-		if(view) {			
-		}
-		*/
 	}
     
 	newScanMessageConference(cb) {
@@ -88,32 +92,19 @@ exports.getModule = class NewScanModule extends MenuModule {
 		}
         
 		const currentConf = this.sortedMessageConfs[this.currentScanAux.conf];
-		const self			= this;
 
-		async.series(
+		this.newScanMessageArea(currentConf, () => {
-			[
+			if(this.sortedMessageConfs.length > this.currentScanAux.conf + 1) {
-				function scanArea(callback) {
+				this.currentScanAux.conf += 1;                            
-					//self.currentScanAux.area = self.currentScanAux.area || 0;
+				this.currentScanAux.area = 0;
 				
-					self.newScanMessageArea(currentConf, () => {
+				return this.newScanMessageConference(cb);  //  recursive to next conf
-						if(self.sortedMessageConfs.length > self.currentScanAux.conf + 1) {
-							self.currentScanAux.conf += 1;                            
-							self.currentScanAux.area = 0;
-							
-							self.newScanMessageConference(cb);  //  recursive to next conf
-							//callback(null);
-						} else {
-							self.updateScanStatus(self.scanCompleteMsg);
-							callback(new Error('No more conferences'));
 			}
+
+			this.updateScanStatus(this.scanCompleteMsg);
+			return cb(Errors.DoesNotExist('No more conferences'));
 		});        
 	}
-			],
-			err => {
-				return cb(err);
-			}
-        );
-	}
 	
 	newScanMessageArea(conf, cb) {
         //  :TODO: it would be nice to cache this - must be done by conf!
@@ -134,7 +125,7 @@ exports.getModule = class NewScanModule extends MenuModule {
 						return callback(null);
 					} else {
 						self.updateScanStatus(self.scanCompleteMsg);
-						return callback(new Error('No more areas'));	//	this will stop our scan
+						return callback(Errors.DoesNotExist('No more areas'));	//	this will stop our scan
 					}
 				},
 				function updateStatusScanStarted(callback) {
@@ -173,6 +164,28 @@ exports.getModule = class NewScanModule extends MenuModule {
 		);
 	}
 
+	newScanFileBase(cb) {
+		//	:TODO: add in steps
+		FileEntry.findFiles(
+			{ newerThanFileId : FileBaseFilters.getFileBaseLastViewedFileIdByUser(this.client.user) },
+			(err, fileIds) => {
+				if(err || 0 === fileIds.length) {
+					return cb(err ? err : Errors.DoesNotExist('No more new files'));
+				}
+
+				FileBaseFilters.setFileBaseLastViewedFileIdForUser( this.client.user, fileIds[0] );
+
+				const menuOpts = {
+					extraArgs : {
+						fileList : fileIds,				
+					},
+				};
+
+				return this.gotoMenu(this.menuConfig.config.newScanFileBaseList || 'newScanFileBaseList', menuOpts);
+			}
+		);
+	}
+
 	getSaveState() {
 		return {
 			currentStep 	: this.currentStep,
@@ -185,6 +198,26 @@ exports.getModule = class NewScanModule extends MenuModule {
 		this.currentScanAux	= savedState.currentScanAux;
 	}
 
+	performScanCurrentStep(cb) {
+		switch(this.currentStep) {
+			case Steps.MessageConfs : 
+				this.newScanMessageConference( () => {									
+					this.currentStep = Steps.FileBase;
+					return this.performScanCurrentStep(cb);
+				});
+				break;
+				
+			case Steps.FileBase :
+				this.newScanFileBase( () => {
+					this.currentStep = Steps.Finished;
+					return this.performScanCurrentStep(cb);	
+				});
+				break;
+				
+			default : return cb(null);
+		}
+	}
+
 	mciReady(mciData, cb) {
 		if(this.newScanFullExit) {
 			//	user has canceled the entire scan @ message list view
@@ -213,15 +246,7 @@ exports.getModule = class NewScanModule extends MenuModule {
 						vc.loadFromMenuConfig(loadOpts, callback);
 					},
 					function performCurrentStepScan(callback) {
-						switch(self.currentStep) {
+						return self.performScanCurrentStep(callback);
-							case 'messageConferences' :
-								self.newScanMessageConference( () => {
-									callback(null); //  finished
-								});
-								break;	
-								
-							default : return callback(null);
-						}
 					}
 				],
 				err => {

core/oputil/oputil_file_base.js

@@ -34,10 +34,25 @@ exports.handleFileBaseCommand			= handleFileBaseCommand;
 
 let fileArea;	//	required during init
 
-function finalizeEntryAndPersist(fileEntry, cb) {
+function finalizeEntryAndPersist(fileEntry, descHandler, cb) {
 	async.series(
 		[
-			function getDescIfNeeded(callback) {
+			function getDescFromHandlerIfNeeded(callback) {
+				if((fileEntry.desc && fileEntry.desc.length > 0 ) && !argv['desc-file']) {
+					return callback(null);	//	we have a desc already and are NOT overriding with desc file
+				}
+
+				if(!descHandler) {
+					return callback(null);	//	not much we can do!
+				}
+
+				const desc = descHandler.getDescription(fileEntry.fileName);
+				if(desc) {
+					fileEntry.desc = desc;
+				}
+				return callback(null);
+			},
+			function getDescFromUserIfNeeded(callback) {
 				if(false === argv.prompt || ( fileEntry.desc && fileEntry.desc.length > 0 ) ) {
 					return callback(null);
 				}
@@ -70,6 +85,18 @@ function finalizeEntryAndPersist(fileEntry, cb) {
 	);
 }
 
+const SCAN_EXCLUDE_FILENAMES = [ 'DESCRIPT.ION', 'FILES.BBS' ];
+
+function loadDescHandler(path, cb) {
+	const DescIon = require('../../core/descript_ion_file.js');
+
+	//	:TODO: support FILES.BBS also
+
+	DescIon.createFromFile(path, (err, descHandler) => {
+		return cb(err, descHandler);
+	});
+}
+
 function scanFileAreaForChanges(areaInfo, options, cb) {
 
 	const storageLocations = fileArea.getAreaStorageLocations(areaInfo).filter(sl => {
@@ -79,9 +106,18 @@ function scanFileAreaForChanges(areaInfo, options, cb) {
 	});
 	
 	async.eachSeries(storageLocations, (storageLoc, nextLocation) => {
-		async.series(
+		async.waterfall(
 			[
-				function scanPhysFiles(callback) {
+				function initDescFile(callback) {
+					if(options.descFileHandler) {
+						return callback(null, options.descFileHandler);	//	we're going to use the global handler
+					}
+
+					loadDescHandler(paths.join(storageLoc.dir, 'DESCRIPT.ION'), (err, descHandler) => {
+						return callback(null, descHandler);
+					});
+				},
+				function scanPhysFiles(descHandler, callback) {
 					const physDir = storageLoc.dir;
 
 					fs.readdir(physDir, (err, files) => {
@@ -92,6 +128,11 @@ function scanFileAreaForChanges(areaInfo, options, cb) {
 						async.eachSeries(files, (fileName, nextFile) => {
 							const fullPath = paths.join(physDir, fileName);
 
+							if(SCAN_EXCLUDE_FILENAMES.includes(fileName.toUpperCase())) {
+								console.info(`Excluding ${fullPath}`);
+								return nextFile(null);
+							}
+
 							fs.stat(fullPath, (err, stats) => {
 								if(err) {
 									//	:TODO: Log me!
@@ -117,8 +158,6 @@ function scanFileAreaForChanges(areaInfo, options, cb) {
 											return nextFile(null);	//	try next anyway
 										}								
 
-										
-
 										if(dupeEntries.length > 0) {
 											//	:TODO: Handle duplidates -- what to do here???
 											console.info('Dupe');
@@ -131,7 +170,7 @@ function scanFileAreaForChanges(areaInfo, options, cb) {
 												});
 											}
 
-											finalizeEntryAndPersist(fileEntry, err => {
+											finalizeEntryAndPersist(fileEntry, descHandler, err => {
 												return nextFile(err);
 											});
 										}
@@ -304,6 +343,8 @@ function scanFileAreas() {
 		options.tags = tags.split(',');
 	}
 
+	options.descFile = argv['desc-file'];	//	--desc-file or --desc-file PATH
+	
 	options.areaAndStorageInfo = getAreaAndStorage(argv._.slice(2));
 
 	async.series(
@@ -311,6 +352,21 @@ function scanFileAreas() {
 			function init(callback) {
 				return initConfigAndDatabases(callback);
 			},
+			function initGlobalDescHandler(callback) {		
+				//
+				//	If options.descFile is a String, it represents a FILE|PATH. We'll init
+				//	the description handler now. Else, we'll attempt to look for a description 
+				//	file in each storage location.
+				//
+				if(!_.isString(options.descFile)) {
+					return callback(null);
+				}
+
+				loadDescHandler(options.descFile, (err, descHandler) => {
+					options.descFileHandler = descHandler;
+					return callback(null);
+				});
+			},
 			function scanAreas(callback) {
 				fileArea = require('../../core/file_base_area.js');
 

core/oputil/oputil_help.js

@@ -61,6 +61,10 @@ actions:
 
 scan args:
   --tags TAG1,TAG2,...         specify tag(s) to assign to discovered entries
+  --desc-file [PATH]           prefer file descriptions from DESCRIPT.ION file over
+                               other sources such as FILE_ID.DIZ.
+                               if PATH is specified, use DESCRIPT.ION at PATH instead
+                               of looking in specific storage locations
 
 info args:
   --show-desc                  display short description, if any

core/predefined_mci.js

@@ -37,7 +37,7 @@ function setNextRandomRumor(cb) {
 	});
 }
 
-function getRatio(client, propA, propB) {
+function getUserRatio(client, propA, propB) {
 	const a = StatLog.getUserStatNum(client.user, propA);
 	const b	= StatLog.getUserStatNum(client.user, propB);
 	const ratio	= ~~((a / b) * 100);
@@ -48,6 +48,10 @@ function userStatAsString(client, statName, defaultValue) {
 	return (StatLog.getUserStat(client.user, statName) || defaultValue).toString();
 }
 
+function sysStatAsString(statName, defaultValue) {
+	return (StatLog.getSystemStat(statName) || defaultValue).toString();
+}
+
 const PREDEFINED_MCI_GENERATORS = {
 	//
 	//	Board
@@ -84,7 +88,7 @@ const PREDEFINED_MCI_GENERATORS = {
 	UT	: function themeId(client) { return userStatAsString(client, 'theme_id', ''); },
 	UC	: function loginCount(client) { return userStatAsString(client, 'login_count', 0); },
 	ND	: function connectedNode(client) { return client.node.toString(); },
-	IP	: function clientIpAddress(client) { return client.remoteAddress; },
+	IP	: function clientIpAddress(client) { return client.remoteAddress.replace(/^::ffff:/, ''); },	//	convert any :ffff: IPv4's to 32bit version
 	ST	: function serverName(client) { return client.session.serverName; },
 	FN	: function activeFileBaseFilterName(client) {
 		const activeFilter = FileBaseFilters.getActiveFilter(client);
@@ -101,15 +105,15 @@ const PREDEFINED_MCI_GENERATORS = {
 		return formatByteSize(byteSize, true);	//	true=withAbbr
 	},
 	NR	: function userUpDownRatio(client) {	//	Obv/2
-		return getRatio(client, 'ul_total_count', 'dl_total_count');
+		return getUserRatio(client, 'ul_total_count', 'dl_total_count');
 	},
 	KR	: function userUpDownByteRatio(client) {	//	Obv/2 uses KR=upload/download Kbyte ratio
-		return getRatio(client, 'ul_total_bytes', 'dl_total_bytes');
+		return getUserRatio(client, 'ul_total_bytes', 'dl_total_bytes');
 	},
 
 	MS	: function accountCreatedclient(client) { return moment(client.user.properties.account_created).format(client.currentTheme.helpers.getDateFormat()); },
 	PS	: function userPostCount(client) { return userStatAsString(client, 'post_count', 0); },
-	PC	: function userPostCallRatio(client) { return getRatio(client, 'post_count', 'login_count'); },
+	PC	: function userPostCallRatio(client) { return getUserRatio(client, 'post_count', 'login_count'); },
 
 	MD	: function currentMenuDescription(client) {
 		return _.has(client, 'currentMenuModule.menuConfig.desc') ? client.currentMenuModule.menuConfig.desc : '';
@@ -187,7 +191,16 @@ const PREDEFINED_MCI_GENERATORS = {
 	//
 	//	:TODO: DD - Today's # of downloads (iNiQUiTY)
 	//	
-	//	:TODO: System stat log for total ul/dl, total ul/dl bytes
+	SD	: function systemNumDownloads() { return sysStatAsString('dl_total_count', 0); },
+	SO	: function systemByteDownload() {
+		const byteSize = StatLog.getSystemStatNum('dl_total_bytes');
+		return formatByteSize(byteSize, true);	//	true=withAbbr
+	},
+	SU	: function systemNumUploads() { return sysStatAsString('ul_total_count', 0); },
+	SP	: function systemByteUpload() {
+		const byteSize = StatLog.getSystemStatNum('ul_total_bytes');
+		return formatByteSize(byteSize, true);	//	true=withAbbr
+	},
 
 	//	:TODO: PT - Messages posted *today* (Obv/2)
 	//		-> Include FTN/etc.

core/scanner_tossers/ftn_bso.js

@@ -293,14 +293,14 @@ function FTNMessageScanTossModule() {
 		async.detectSeries(EXT_SUFFIXES, (suffix, callback) => {
 			const checkFileName = fileName + suffix; 			
 			fs.stat(paths.join(basePath, checkFileName), err => {
-				callback((err && 'ENOENT' === err.code) ? true : false);
+				callback(null, (err && 'ENOENT' === err.code) ? true : false);
 			});
-		}, finalSuffix => {
+		}, (err, finalSuffix) => {
 			if(finalSuffix) {
-				cb(null, paths.join(basePath, fileName + finalSuffix));
+				return cb(null, paths.join(basePath, fileName + finalSuffix));
-			} else {
-				cb(new Error('Could not acquire a bundle filename!'));
 			}
+			
+			return cb(new Error('Could not acquire a bundle filename!'));			
 		});
 	};
 	
@@ -390,11 +390,14 @@ function FTNMessageScanTossModule() {
 		message.meta.FtnKludge.TID = ftnUtil.getProductIdentifier(); 
 		
 		//
-		//	Determine CHRS and actual internal encoding name
+		//	Determine CHRS and actual internal encoding name. If the message has an
-		//	Try to preserve anything already here
+		//	explicit encoding set, use it. Otherwise, try to preserve any CHRS/encoding already set.
 		//
-		let encoding = options.nodeConfig.encoding || 'utf8';
+		let encoding = options.nodeConfig.encoding || Config.scannerTossers.ftn_bso.packetMsgEncoding || 'utf8';
-		if(message.meta.FtnKludge.CHRS) {
+		const explicitEncoding = _.get(message.meta, 'System.explicit_encoding');
+		if(explicitEncoding) {
+			encoding = explicitEncoding;
+		} else if(message.meta.FtnKludge.CHRS) {
 			const encFromChars = ftnUtil.getEncodingFromCharacterSetIdentifier(message.meta.FtnKludge.CHRS);
 			if(encFromChars) {
 				encoding = encFromChars;
@@ -605,7 +608,11 @@ function FTNMessageScanTossModule() {
 						callback(null);					
 					},
 					function appendMessage(callback) {
-						const msgBuf	= packet.getMessageEntryBuffer(message, exportOpts);
+						packet.getMessageEntryBuffer(message, exportOpts, (err, msgBuf) => {
+							if(err) {
+								return callback(err);
+							}
+
 							currPacketSize	+= msgBuf.length;
 						
 							if(currPacketSize >= self.moduleConfig.packetTargetByteSize) {
@@ -614,7 +621,9 @@ function FTNMessageScanTossModule() {
 							} else {
 								ws.write(msgBuf);
 							}
-						callback(null);
+							
+							return callback(null);
+						});												
 					},
 					function storeStateFlags0Meta(callback) {
 						message.persistMetaValue('System', 'state_flags0', Message.StateFlags0.Exported.toString(), err => {

core/servers/content/web.js

@@ -116,7 +116,7 @@ exports.getModule = class WebServerModule extends ServerModule {
 			//	additional options
 			Object.assign(options, Config.contentServers.web.https.options || {} );
 
-			this.httpsServer = https.createServer(options, this.routeRequest);			
+			this.httpsServer = https.createServer(options, (req, resp) => this.routeRequest(req, resp) );			
 		}
 	}
 

core/servers/login/telnet.js

@@ -6,12 +6,12 @@ const baseClient		= require('../../client.js');
 const Log				= require('../../logger.js').log;
 const LoginServerModule	= require('../../login_server_module.js');
 const Config			= require('../../config.js').config;
+const EnigAssert		= require('../../enigma_assert.js');
 
 //	deps
 const net 			= require('net');
 const buffers		= require('buffers');
 const binary		= require('binary');
-const assert		= require('assert');
 const util			= require('util');
 
 //var debug	= require('debug')('telnet');
@@ -184,6 +184,10 @@ const OPTION_NAMES = Object.keys(OPTIONS).reduce(function(names, name) {
 	return names;
 }, {});
 
+function unknownOption(bufs, i, event) {
+	Log.warn( { bufs : bufs, i : i, event : event }, 'Unknown Telnet option');
+}
+
 const OPTION_IMPLS = {};
 //	:TODO: fill in the rest...
 OPTION_IMPLS.NO_ARGS						=
@@ -224,10 +228,10 @@ OPTION_IMPLS[OPTIONS.TERMINAL_TYPE] = function(bufs, i, event) {
 			.word8('ttype')
 			.word8('is')
 			.tap(function(vars) {
-				assert(vars.iac1 === COMMANDS.IAC);
+				EnigAssert(vars.iac1 === COMMANDS.IAC);
-				assert(vars.sb === COMMANDS.SB);
+				EnigAssert(vars.sb === COMMANDS.SB);
-				assert(vars.ttype === OPTIONS.TERMINAL_TYPE);
+				EnigAssert(vars.ttype === OPTIONS.TERMINAL_TYPE);
-				assert(vars.is === SB_COMMANDS.IS);
+				EnigAssert(vars.is === SB_COMMANDS.IS);
 			});
 
 		//	eat up the rest
@@ -275,11 +279,11 @@ OPTION_IMPLS[OPTIONS.WINDOW_SIZE] = function(bufs, i, event) {
 			.word8('iac2')
 			.word8('se')
 			.tap(function(vars) {
-				assert(vars.iac1 == COMMANDS.IAC);
+				EnigAssert(vars.iac1 == COMMANDS.IAC);
-				assert(vars.sb == COMMANDS.SB);
+				EnigAssert(vars.sb == COMMANDS.SB);
-				assert(vars.naws == OPTIONS.WINDOW_SIZE);
+				EnigAssert(vars.naws == OPTIONS.WINDOW_SIZE);
-				assert(vars.iac2 == COMMANDS.IAC);
+				EnigAssert(vars.iac2 == COMMANDS.IAC);
-				assert(vars.se == COMMANDS.SE);
+				EnigAssert(vars.se == COMMANDS.SE);
 
 				event.cols	= event.columns	= event.width = vars.width;
 				event.rows	= event.height = vars.height;
@@ -322,10 +326,10 @@ OPTION_IMPLS[OPTIONS.NEW_ENVIRONMENT]		= function(bufs, i, event) {
 			.word8('newEnv')
 			.word8('isOrInfo')	//	initial=IS, updates=INFO
 			.tap(function(vars) {
-				assert(vars.iac1 === COMMANDS.IAC);
+				EnigAssert(vars.iac1 === COMMANDS.IAC);
-				assert(vars.sb === COMMANDS.SB);
+				EnigAssert(vars.sb === COMMANDS.SB);
-				assert(vars.newEnv === OPTIONS.NEW_ENVIRONMENT || vars.newEnv === OPTIONS.NEW_ENVIRONMENT_DEP);
+				EnigAssert(vars.newEnv === OPTIONS.NEW_ENVIRONMENT || vars.newEnv === OPTIONS.NEW_ENVIRONMENT_DEP);
-				assert(vars.isOrInfo === SB_COMMANDS.IS || vars.isOrInfo === SB_COMMANDS.INFO);
+				EnigAssert(vars.isOrInfo === SB_COMMANDS.IS || vars.isOrInfo === SB_COMMANDS.INFO);
 
 				event.type = vars.isOrInfo;
 
@@ -394,8 +398,8 @@ OPTION_IMPLS[OPTIONS.NEW_ENVIRONMENT]		= function(bufs, i, event) {
 const MORE_DATA_REQUIRED	= 0xfeedface;
 
 function parseBufs(bufs) {
-	assert(bufs.length >= 2);
+	EnigAssert(bufs.length >= 2);
-	assert(bufs.get(0) === COMMANDS.IAC);
+	EnigAssert(bufs.get(0) === COMMANDS.IAC);
 	return parseCommand(bufs, 1, {});
 }
 
@@ -421,7 +425,9 @@ function parseOption(bufs, i, event) {
 	const option		= bufs.get(i);	//	:TODO: fix deprecation... [i] is not the same
 	event.optionCode	= option;
 	event.option		= OPTION_NAMES[option];
-	return OPTION_IMPLS[option](bufs, i + 1, event);
+
+	const handler = OPTION_IMPLS[option];
+	return handler ? handler(bufs, i + 1, event) : unknownOption(bufs, i + 1, event);
 }
 
 
@@ -433,6 +439,8 @@ function TelnetClient(input, output) {
 	let bufs	= buffers();
 	this.bufs	= bufs;
 
+	this.sentDont = {};	//	DON'T's we've already sent
+
 	this.setInputOutput(input, output);
 
 	this.negotiationsComplete	= false;	//	are we in the 'negotiation' phase?
@@ -453,6 +461,11 @@ function TelnetClient(input, output) {
 	};
 
 	this.dataHandler = function(b) {
+		if(!Buffer.isBuffer(b)) {
+			EnigAssert(false, `Cannot push non-buffer ${typeof b}`);
+			return;
+		}
+
 		bufs.push(b);
 
 		let i;
@@ -466,7 +479,7 @@ function TelnetClient(input, output) {
 				break;
 			}
 
-			assert(bufs.length > (i + 1));
+			EnigAssert(bufs.length > (i + 1));
 			
 			if(i > 0) {
 				self.emit('data', bufs.splice(0, i).toBuffer());
@@ -476,7 +489,7 @@ function TelnetClient(input, output) {
 			
 			if(MORE_DATA_REQUIRED === i) {
 				break;				
-			} else {
+			} else if(i) {
 				if(i.option) {
 					self.emit(i.option, i);	//	"transmit binary", "echo", ...
 				}
@@ -505,15 +518,26 @@ function TelnetClient(input, output) {
 	});
 
 	this.input.on('error', err => {
-		self.log.debug( { err : err }, 'Socket error');
+		this.connectionDebug( { err : err }, 'Socket error' );
-		self.emit('end');
+		return self.emit('end');
 	});
 
-	this.connectionDebug = (info, msg) => {
+	this.connectionTrace = (info, msg) => {
 		if(Config.loginServers.telnet.traceConnections) {
-			self.log.trace(info, 'Telnet: ' + msg);
+			const logger = self.log || Log;
+			return logger.trace(info, `Telnet: ${msg}`);
 		}
 	};
+
+	this.connectionDebug = (info, msg) => {
+		const logger = self.log || Log;
+		return logger.debug(info, `Telnet: ${msg}`);
+	};
+
+	this.connectionWarn = (info, msg) => {
+		const logger = self.log || Log;
+		return logger.warn(info, `Telnet: ${msg}`);
+	};
 }
 
 util.inherits(TelnetClient, baseClient.Client);
@@ -522,6 +546,11 @@ util.inherits(TelnetClient, baseClient.Client);
 //	Telnet Command/Option handling
 ///////////////////////////////////////////////////////////////////////////////
 TelnetClient.prototype.handleTelnetEvent = function(evt) {
+	
+	if(!evt.command) {
+		return this.connectionWarn( { evt : evt }, 'No command for event');
+	}
+
 	//	handler name e.g. 'handleWontCommand'
 	const handlerName = `handle${evt.command.charAt(0).toUpperCase()}${evt.command.substr(1)}Command`;
 
@@ -547,15 +576,21 @@ TelnetClient.prototype.handleWillCommand = function(evt) {
 		this.requestNewEnvironment();
 	} else {
 		//	:TODO: temporary:
-		this.connectionDebug(evt, 'WILL');
+		this.connectionTrace(evt, 'WILL');
 	}
 };
 
 TelnetClient.prototype.handleWontCommand = function(evt) {
+	if(this.sentDont[evt.option]) {
+		return this.connectionTrace(evt, 'WONT - DON\'T already sent');
+	}
+
+	this.sentDont[evt.option] = true;
+
 	if('new environment' === evt.option) {		
 		this.dont.new_environment();
 	} else {
-		this.connectionDebug(evt, 'WONT');
+		this.connectionTrace(evt, 'WONT');
 	}
 };
 
@@ -574,12 +609,12 @@ TelnetClient.prototype.handleDoCommand = function(evt) {
 		this.wont.encrypt();
 	} else {
 		//	:TODO: temporary:
-		this.connectionDebug(evt, 'DO');
+		this.connectionTrace(evt, 'DO');
 	}
 };
 
 TelnetClient.prototype.handleDontCommand = function(evt) {
-	this.connectionDebug(evt, 'DONT');
+	this.connectionTrace(evt, 'DONT');
 };
 
 TelnetClient.prototype.handleSbCommand = function(evt) {
@@ -613,24 +648,26 @@ TelnetClient.prototype.handleSbCommand = function(evt) {
 			} else if('COLUMNS' === name && 0 === self.term.termWidth) {
 				self.term.termWidth = parseInt(evt.envVars[name]);
 				self.clearMciCache();	//	term size changes = invalidate cache
-				self.log.debug({ termWidth : self.term.termWidth, source : 'NEW-ENVIRON'}, 'Window width updated');
+				self.connectionDebug({ termWidth : self.term.termWidth, source : 'NEW-ENVIRON'}, 'Window width updated');
 			} else if('ROWS' === name && 0 === self.term.termHeight) {
 				self.term.termHeight = parseInt(evt.envVars[name]);
 				self.clearMciCache();	//	term size changes = invalidate cache
-				self.log.debug({ termHeight : self.term.termHeight, source : 'NEW-ENVIRON'}, 'Window height updated');
+				self.connectionDebug({ termHeight : self.term.termHeight, source : 'NEW-ENVIRON'}, 'Window height updated');
 			} else {			
 				if(name in self.term.env) {
-					assert(
-						SB_COMMANDS.INFO === evt.type || SB_COMMANDS.IS === evt.type, 
-						'Unexpected type: ' + evt.type);
 
-					self.log.warn(
+					EnigAssert(
+						SB_COMMANDS.INFO === evt.type || SB_COMMANDS.IS === evt.type, 
+						'Unexpected type: ' + evt.type
+					);
+
+					self.connectionWarn(
 						{ varName : name, value : evt.envVars[name], existingValue : self.term.env[name] }, 
-						'Environment variable already exists');
+						'Environment variable already exists'
+					);
 				} else {
 					self.term.env[name] = evt.envVars[name];
-					self.log.debug(
+					self.connectionDebug( { varName : name, value : evt.envVars[name] }, 'New environment variable' );
-						{ varName : name, value : evt.envVars[name] }, 'New environment variable');
 				}
 			}
 		});
@@ -653,9 +690,9 @@ TelnetClient.prototype.handleSbCommand = function(evt) {
 
 		self.clearMciCache();	//	term size changes = invalidate cache
 
-		self.log.debug({ termWidth : evt.width , termHeight : evt.height, source : 'NAWS' }, 'Window size updated');
+		self.connectionDebug({ termWidth : evt.width , termHeight : evt.height, source : 'NAWS' }, 'Window size updated');
 	} else {
-		self.log(evt, 'SB');
+		self.connectionDebug(evt, 'SB');
 	}
 };
 
@@ -666,7 +703,7 @@ const IGNORED_COMMANDS = [];
 
 
 TelnetClient.prototype.handleMiscCommand = function(evt) {
-	assert(evt.command !== 'undefined' && evt.command.length > 0);
+	EnigAssert(evt.command !== 'undefined' && evt.command.length > 0);
 
 	//
 	//	See:

core/servers/login/websocket.js

@@ -13,7 +13,7 @@ const WebSocketServer	= require('ws').Server;
 const http				= require('http');
 const https				= require('https');
 const fs				= require('graceful-fs');
-const EventEmitter		= require('events');
+const Writable			= require('stream');
 
 const ModuleInfo = exports.moduleInfo = {
 	name		: 'WebSocket',
@@ -34,20 +34,31 @@ function WebSocketClient(ws, req, serverType) {
 	//	This bridge makes accessible various calls that client sub classes
 	//	want to access on I/O socket
 	//
-	this.socketBridge = new class SocketBridge extends EventEmitter {
+	this.socketBridge = new class SocketBridge extends Writable {
 		constructor(ws) {
 			super();
 			this.ws = ws;
 		}
 
 		end() {
-			return ws.terminate();			
+			return ws.close();
 		}
 
 		write(data, cb) {
+			cb = cb || ( () => { /* eat it up */} );	//	handle data writes after close
+
 			return this.ws.send(data, { binary : true }, cb);
 		}
 
+		//	we need to fake some streaming work
+		unpipe() {
+			Log.trace('WebSocket SocketBridge unpipe()');
+		}
+
+		resume() {
+			Log.trace('WebSocket SocketBridge resume()');
+		}
+
 		get remoteAddress() {
 			//	Support X-Forwarded-For and X-Real-IP headers for proxied connections
 			return (self.proxied && (req.headers['x-forwarded-for'] || req.headers['x-real-ip'])) || req.connection.remoteAddress;

core/stat_log.js

@@ -92,6 +92,10 @@ class StatLog {
 
 	getSystemStat(statName) { return this.systemStats[statName]; }
 
+	getSystemStatNum(statName) {
+		return parseInt(this.getSystemStat(statName)) || 0;
+	}
+
 	incrementSystemStat(statName, incrementBy, cb) {
 		incrementBy = incrementBy || 1;
 

core/string_util.js

@@ -8,20 +8,26 @@ const ANSI				= require('./ansi_term.js');
 
 //	deps
 const iconv				= require('iconv-lite');
+const _					= require('lodash');
 
 exports.stylizeString				= stylizeString;
 exports.pad							= pad;
+exports.insert						= insert;
 exports.replaceAt					= replaceAt;
 exports.isPrintable					= isPrintable;
 exports.stripAllLineFeeds			= stripAllLineFeeds;
 exports.debugEscapedString			= debugEscapedString;
 exports.stringFromNullTermBuffer	= stringFromNullTermBuffer;
+exports.stringToNullTermBuffer		= stringToNullTermBuffer;
 exports.renderSubstr				= renderSubstr;
 exports.renderStringLength			= renderStringLength;
 exports.formatByteSizeAbbr			= formatByteSizeAbbr;
 exports.formatByteSize				= formatByteSize;
 exports.cleanControlCodes			= cleanControlCodes;
-exports.createCleanAnsi				= createCleanAnsi;
+exports.isAnsi						= isAnsi;
+exports.isAnsiLine					= isAnsiLine;
+exports.isFormattedLine				= isFormattedLine;
+exports.splitTextAtTerms			= splitTextAtTerms;
 
 //	:TODO: create Unicode verison of this
 const VOWELS = [ 'a', 'e', 'i', 'o', 'u' ];
@@ -168,11 +174,16 @@ function pad(s, len, padChar, dir, stringSGR, padSGR, useRenderLen) {
 	return stringSGR + s;
 }
 
+function insert(s, index, substr) {
+	return `${s.slice(0, index)}${substr}${s.slice(index)}`;
+}
+
 function replaceAt(s, n, t) {
 	return s.substring(0, n) + t + s.substring(n + 1);
 }
 
-const RE_NON_PRINTABLE = /[\0-\x1F\x7F-\x9F\xAD\u0378\u0379\u037F-\u0383\u038B\u038D\u03A2\u0528-\u0530\u0557\u0558\u0560\u0588\u058B-\u058E\u0590\u05C8-\u05CF\u05EB-\u05EF\u05F5-\u0605\u061C\u061D\u06DD\u070E\u070F\u074B\u074C\u07B2-\u07BF\u07FB-\u07FF\u082E\u082F\u083F\u085C\u085D\u085F-\u089F\u08A1\u08AD-\u08E3\u08FF\u0978\u0980\u0984\u098D\u098E\u0991\u0992\u09A9\u09B1\u09B3-\u09B5\u09BA\u09BB\u09C5\u09C6\u09C9\u09CA\u09CF-\u09D6\u09D8-\u09DB\u09DE\u09E4\u09E5\u09FC-\u0A00\u0A04\u0A0B-\u0A0E\u0A11\u0A12\u0A29\u0A31\u0A34\u0A37\u0A3A\u0A3B\u0A3D\u0A43-\u0A46\u0A49\u0A4A\u0A4E-\u0A50\u0A52-\u0A58\u0A5D\u0A5F-\u0A65\u0A76-\u0A80\u0A84\u0A8E\u0A92\u0AA9\u0AB1\u0AB4\u0ABA\u0ABB\u0AC6\u0ACA\u0ACE\u0ACF\u0AD1-\u0ADF\u0AE4\u0AE5\u0AF2-\u0B00\u0B04\u0B0D\u0B0E\u0B11\u0B12\u0B29\u0B31\u0B34\u0B3A\u0B3B\u0B45\u0B46\u0B49\u0B4A\u0B4E-\u0B55\u0B58-\u0B5B\u0B5E\u0B64\u0B65\u0B78-\u0B81\u0B84\u0B8B-\u0B8D\u0B91\u0B96-\u0B98\u0B9B\u0B9D\u0BA0-\u0BA2\u0BA5-\u0BA7\u0BAB-\u0BAD\u0BBA-\u0BBD\u0BC3-\u0BC5\u0BC9\u0BCE\u0BCF\u0BD1-\u0BD6\u0BD8-\u0BE5\u0BFB-\u0C00\u0C04\u0C0D\u0C11\u0C29\u0C34\u0C3A-\u0C3C\u0C45\u0C49\u0C4E-\u0C54\u0C57\u0C5A-\u0C5F\u0C64\u0C65\u0C70-\u0C77\u0C80\u0C81\u0C84\u0C8D\u0C91\u0CA9\u0CB4\u0CBA\u0CBB\u0CC5\u0CC9\u0CCE-\u0CD4\u0CD7-\u0CDD\u0CDF\u0CE4\u0CE5\u0CF0\u0CF3-\u0D01\u0D04\u0D0D\u0D11\u0D3B\u0D3C\u0D45\u0D49\u0D4F-\u0D56\u0D58-\u0D5F\u0D64\u0D65\u0D76-\u0D78\u0D80\u0D81\u0D84\u0D97-\u0D99\u0DB2\u0DBC\u0DBE\u0DBF\u0DC7-\u0DC9\u0DCB-\u0DCE\u0DD5\u0DD7\u0DE0-\u0DF1\u0DF5-\u0E00\u0E3B-\u0E3E\u0E5C-\u0E80\u0E83\u0E85\u0E86\u0E89\u0E8B\u0E8C\u0E8E-\u0E93\u0E98\u0EA0\u0EA4\u0EA6\u0EA8\u0EA9\u0EAC\u0EBA\u0EBE\u0EBF\u0EC5\u0EC7\u0ECE\u0ECF\u0EDA\u0EDB\u0EE0-\u0EFF\u0F48\u0F6D-\u0F70\u0F98\u0FBD\u0FCD\u0FDB-\u0FFF\u10C6\u10C8-\u10CC\u10CE\u10CF\u1249\u124E\u124F\u1257\u1259\u125E\u125F\u1289\u128E\u128F\u12B1\u12B6\u12B7\u12BF\u12C1\u12C6\u12C7\u12D7\u1311\u1316\u1317\u135B\u135C\u137D-\u137F\u139A-\u139F\u13F5-\u13FF\u169D-\u169F\u16F1-\u16FF\u170D\u1715-\u171F\u1737-\u173F\u1754-\u175F\u176D\u1771\u1774-\u177F\u17DE\u17DF\u17EA-\u17EF\u17FA-\u17FF\u180F\u181A-\u181F\u1878-\u187F\u18AB-\u18AF\u18F6-\u18FF\u191D-\u191F\u192C-\u192F\u193C-\u193F\u1941-\u1943\u196E\u196F\u1975-\u197F\u19AC-\u19AF\u19CA-\u19CF\u19DB-\u19DD\u1A1C\u1A1D\u1A5F\u1A7D\u1A7E\u1A8A-\u1A8F\u1A9A-\u1A9F\u1AAE-\u1AFF\u1B4C-\u1B4F\u1B7D-\u1B7F\u1BF4-\u1BFB\u1C38-\u1C3A\u1C4A-\u1C4C\u1C80-\u1CBF\u1CC8-\u1CCF\u1CF7-\u1CFF\u1DE7-\u1DFB\u1F16\u1F17\u1F1E\u1F1F\u1F46\u1F47\u1F4E\u1F4F\u1F58\u1F5A\u1F5C\u1F5E\u1F7E\u1F7F\u1FB5\u1FC5\u1FD4\u1FD5\u1FDC\u1FF0\u1FF1\u1FF5\u1FFF\u200B-\u200F\u202A-\u202E\u2060-\u206F\u2072\u2073\u208F\u209D-\u209F\u20BB-\u20CF\u20F1-\u20FF\u218A-\u218F\u23F4-\u23FF\u2427-\u243F\u244B-\u245F\u2700\u2B4D-\u2B4F\u2B5A-\u2BFF\u2C2F\u2C5F\u2CF4-\u2CF8\u2D26\u2D28-\u2D2C\u2D2E\u2D2F\u2D68-\u2D6E\u2D71-\u2D7E\u2D97-\u2D9F\u2DA7\u2DAF\u2DB7\u2DBF\u2DC7\u2DCF\u2DD7\u2DDF\u2E3C-\u2E7F\u2E9A\u2EF4-\u2EFF\u2FD6-\u2FEF\u2FFC-\u2FFF\u3040\u3097\u3098\u3100-\u3104\u312E-\u3130\u318F\u31BB-\u31BF\u31E4-\u31EF\u321F\u32FF\u4DB6-\u4DBF\u9FCD-\u9FFF\uA48D-\uA48F\uA4C7-\uA4CF\uA62C-\uA63F\uA698-\uA69E\uA6F8-\uA6FF\uA78F\uA794-\uA79F\uA7AB-\uA7F7\uA82C-\uA82F\uA83A-\uA83F\uA878-\uA87F\uA8C5-\uA8CD\uA8DA-\uA8DF\uA8FC-\uA8FF\uA954-\uA95E\uA97D-\uA97F\uA9CE\uA9DA-\uA9DD\uA9E0-\uA9FF\uAA37-\uAA3F\uAA4E\uAA4F\uAA5A\uAA5B\uAA7C-\uAA7F\uAAC3-\uAADA\uAAF7-\uAB00\uAB07\uAB08\uAB0F\uAB10\uAB17-\uAB1F\uAB27\uAB2F-\uABBF\uABEE\uABEF\uABFA-\uABFF\uD7A4-\uD7AF\uD7C7-\uD7CA\uD7FC-\uF8FF\uFA6E\uFA6F\uFADA-\uFAFF\uFB07-\uFB12\uFB18-\uFB1C\uFB37\uFB3D\uFB3F\uFB42\uFB45\uFBC2-\uFBD2\uFD40-\uFD4F\uFD90\uFD91\uFDC8-\uFDEF\uFDFE\uFDFF\uFE1A-\uFE1F\uFE27-\uFE2F\uFE53\uFE67\uFE6C-\uFE6F\uFE75\uFEFD-\uFF00\uFFBF-\uFFC1\uFFC8\uFFC9\uFFD0\uFFD1\uFFD8\uFFD9\uFFDD-\uFFDF\uFFE7\uFFEF-\uFFFB\uFFFE\uFFFF]/;
+const RE_NON_PRINTABLE = 
+	/[\0-\x1F\x7F-\x9F\xAD\u0378\u0379\u037F-\u0383\u038B\u038D\u03A2\u0528-\u0530\u0557\u0558\u0560\u0588\u058B-\u058E\u0590\u05C8-\u05CF\u05EB-\u05EF\u05F5-\u0605\u061C\u061D\u06DD\u070E\u070F\u074B\u074C\u07B2-\u07BF\u07FB-\u07FF\u082E\u082F\u083F\u085C\u085D\u085F-\u089F\u08A1\u08AD-\u08E3\u08FF\u0978\u0980\u0984\u098D\u098E\u0991\u0992\u09A9\u09B1\u09B3-\u09B5\u09BA\u09BB\u09C5\u09C6\u09C9\u09CA\u09CF-\u09D6\u09D8-\u09DB\u09DE\u09E4\u09E5\u09FC-\u0A00\u0A04\u0A0B-\u0A0E\u0A11\u0A12\u0A29\u0A31\u0A34\u0A37\u0A3A\u0A3B\u0A3D\u0A43-\u0A46\u0A49\u0A4A\u0A4E-\u0A50\u0A52-\u0A58\u0A5D\u0A5F-\u0A65\u0A76-\u0A80\u0A84\u0A8E\u0A92\u0AA9\u0AB1\u0AB4\u0ABA\u0ABB\u0AC6\u0ACA\u0ACE\u0ACF\u0AD1-\u0ADF\u0AE4\u0AE5\u0AF2-\u0B00\u0B04\u0B0D\u0B0E\u0B11\u0B12\u0B29\u0B31\u0B34\u0B3A\u0B3B\u0B45\u0B46\u0B49\u0B4A\u0B4E-\u0B55\u0B58-\u0B5B\u0B5E\u0B64\u0B65\u0B78-\u0B81\u0B84\u0B8B-\u0B8D\u0B91\u0B96-\u0B98\u0B9B\u0B9D\u0BA0-\u0BA2\u0BA5-\u0BA7\u0BAB-\u0BAD\u0BBA-\u0BBD\u0BC3-\u0BC5\u0BC9\u0BCE\u0BCF\u0BD1-\u0BD6\u0BD8-\u0BE5\u0BFB-\u0C00\u0C04\u0C0D\u0C11\u0C29\u0C34\u0C3A-\u0C3C\u0C45\u0C49\u0C4E-\u0C54\u0C57\u0C5A-\u0C5F\u0C64\u0C65\u0C70-\u0C77\u0C80\u0C81\u0C84\u0C8D\u0C91\u0CA9\u0CB4\u0CBA\u0CBB\u0CC5\u0CC9\u0CCE-\u0CD4\u0CD7-\u0CDD\u0CDF\u0CE4\u0CE5\u0CF0\u0CF3-\u0D01\u0D04\u0D0D\u0D11\u0D3B\u0D3C\u0D45\u0D49\u0D4F-\u0D56\u0D58-\u0D5F\u0D64\u0D65\u0D76-\u0D78\u0D80\u0D81\u0D84\u0D97-\u0D99\u0DB2\u0DBC\u0DBE\u0DBF\u0DC7-\u0DC9\u0DCB-\u0DCE\u0DD5\u0DD7\u0DE0-\u0DF1\u0DF5-\u0E00\u0E3B-\u0E3E\u0E5C-\u0E80\u0E83\u0E85\u0E86\u0E89\u0E8B\u0E8C\u0E8E-\u0E93\u0E98\u0EA0\u0EA4\u0EA6\u0EA8\u0EA9\u0EAC\u0EBA\u0EBE\u0EBF\u0EC5\u0EC7\u0ECE\u0ECF\u0EDA\u0EDB\u0EE0-\u0EFF\u0F48\u0F6D-\u0F70\u0F98\u0FBD\u0FCD\u0FDB-\u0FFF\u10C6\u10C8-\u10CC\u10CE\u10CF\u1249\u124E\u124F\u1257\u1259\u125E\u125F\u1289\u128E\u128F\u12B1\u12B6\u12B7\u12BF\u12C1\u12C6\u12C7\u12D7\u1311\u1316\u1317\u135B\u135C\u137D-\u137F\u139A-\u139F\u13F5-\u13FF\u169D-\u169F\u16F1-\u16FF\u170D\u1715-\u171F\u1737-\u173F\u1754-\u175F\u176D\u1771\u1774-\u177F\u17DE\u17DF\u17EA-\u17EF\u17FA-\u17FF\u180F\u181A-\u181F\u1878-\u187F\u18AB-\u18AF\u18F6-\u18FF\u191D-\u191F\u192C-\u192F\u193C-\u193F\u1941-\u1943\u196E\u196F\u1975-\u197F\u19AC-\u19AF\u19CA-\u19CF\u19DB-\u19DD\u1A1C\u1A1D\u1A5F\u1A7D\u1A7E\u1A8A-\u1A8F\u1A9A-\u1A9F\u1AAE-\u1AFF\u1B4C-\u1B4F\u1B7D-\u1B7F\u1BF4-\u1BFB\u1C38-\u1C3A\u1C4A-\u1C4C\u1C80-\u1CBF\u1CC8-\u1CCF\u1CF7-\u1CFF\u1DE7-\u1DFB\u1F16\u1F17\u1F1E\u1F1F\u1F46\u1F47\u1F4E\u1F4F\u1F58\u1F5A\u1F5C\u1F5E\u1F7E\u1F7F\u1FB5\u1FC5\u1FD4\u1FD5\u1FDC\u1FF0\u1FF1\u1FF5\u1FFF\u200B-\u200F\u202A-\u202E\u2060-\u206F\u2072\u2073\u208F\u209D-\u209F\u20BB-\u20CF\u20F1-\u20FF\u218A-\u218F\u23F4-\u23FF\u2427-\u243F\u244B-\u245F\u2700\u2B4D-\u2B4F\u2B5A-\u2BFF\u2C2F\u2C5F\u2CF4-\u2CF8\u2D26\u2D28-\u2D2C\u2D2E\u2D2F\u2D68-\u2D6E\u2D71-\u2D7E\u2D97-\u2D9F\u2DA7\u2DAF\u2DB7\u2DBF\u2DC7\u2DCF\u2DD7\u2DDF\u2E3C-\u2E7F\u2E9A\u2EF4-\u2EFF\u2FD6-\u2FEF\u2FFC-\u2FFF\u3040\u3097\u3098\u3100-\u3104\u312E-\u3130\u318F\u31BB-\u31BF\u31E4-\u31EF\u321F\u32FF\u4DB6-\u4DBF\u9FCD-\u9FFF\uA48D-\uA48F\uA4C7-\uA4CF\uA62C-\uA63F\uA698-\uA69E\uA6F8-\uA6FF\uA78F\uA794-\uA79F\uA7AB-\uA7F7\uA82C-\uA82F\uA83A-\uA83F\uA878-\uA87F\uA8C5-\uA8CD\uA8DA-\uA8DF\uA8FC-\uA8FF\uA954-\uA95E\uA97D-\uA97F\uA9CE\uA9DA-\uA9DD\uA9E0-\uA9FF\uAA37-\uAA3F\uAA4E\uAA4F\uAA5A\uAA5B\uAA7C-\uAA7F\uAAC3-\uAADA\uAAF7-\uAB00\uAB07\uAB08\uAB0F\uAB10\uAB17-\uAB1F\uAB27\uAB2F-\uABBF\uABEE\uABEF\uABFA-\uABFF\uD7A4-\uD7AF\uD7C7-\uD7CA\uD7FC-\uF8FF\uFA6E\uFA6F\uFADA-\uFAFF\uFB07-\uFB12\uFB18-\uFB1C\uFB37\uFB3D\uFB3F\uFB42\uFB45\uFBC2-\uFBD2\uFD40-\uFD4F\uFD90\uFD91\uFDC8-\uFDEF\uFDFE\uFDFF\uFE1A-\uFE1F\uFE27-\uFE2F\uFE53\uFE67\uFE6C-\uFE6F\uFE75\uFEFD-\uFF00\uFFBF-\uFFC1\uFFC8\uFFC9\uFFD0\uFFD1\uFFD8\uFFD9\uFFDD-\uFFDF\uFFE7\uFFEF-\uFFFB\uFFFE\uFFFF]/;	//	eslint-disable-line no-control-regex
 
 function isPrintable(s) {
 	//
@@ -207,9 +218,16 @@ function stringFromNullTermBuffer(buf, encoding) {
 	return iconv.decode(buf.slice(0, nullPos), encoding || 'utf-8');
 }
 
+function stringToNullTermBuffer(s, options = { encoding : 'utf8', maxBufLen : -1 } ) {
+	let buf = iconv.encode( `${s}\0`, options.encoding ).slice(0, options.maxBufLen);
+	buf[buf.length - 1] = '\0';	//	make abs sure we null term even if truncated
+	return buf;
+}
+
 const PIPE_REGEXP			= /(\|[A-Z\d]{2})/g;
-const ANSI_REGEXP			= /[\u001b\u009b][[()#;?]*([0-9]{1,4}(?:;[0-9]{0,4})*)?([0-9A-ORZcf-npqrsuy=><])/g;
+//const ANSI_REGEXP			= /[\u001b\u009b][[()#;?]*([0-9]{1,4}(?:;[0-9]{0,4})*)?([0-9A-ORZcf-npqrsuy=><])/g;
-const ANSI_OR_PIPE_REGEXP	= new RegExp(PIPE_REGEXP.source + '|' + ANSI_REGEXP.source, 'g');
+//const ANSI_OR_PIPE_REGEXP	= new RegExp(PIPE_REGEXP.source + '|' + ANSI_REGEXP.source, 'g');
+const ANSI_OR_PIPE_REGEXP	= new RegExp(PIPE_REGEXP.source + '|' + ANSI.getFullMatchRegExp().source, 'g');
 
 //
 //	Similar to substr() but works with ANSI/Pipe code strings
@@ -322,18 +340,13 @@ function formatByteSize(byteSize, withAbbr, decimals) {
 
 //	:TODO: See notes in word_wrap.js about need to consolidate the various ANSI related RegExp's
 //const REGEXP_ANSI_CONTROL_CODES		= /(\x1b\x5b)([\?=;0-9]*?)([0-9A-ORZcf-npsu=><])/g;
-const REGEXP_ANSI_CONTROL_CODES		= /(?:\x1b\x5b)([\?=;0-9]*?)([A-ORZcf-npsu=><])/g;
+const REGEXP_ANSI_CONTROL_CODES		= /(?:\x1b\x5b)([\?=;0-9]*?)([A-ORZcf-npsu=><])/g;	//	eslint-disable-line no-control-regex
 const ANSI_OPCODES_ALLOWED_CLEAN	= [
-	'A', 'B',	//	up, down
+	//'A', 'B',	//	up, down
-	'C', 'D',	//	right, left
+	//'C', 'D',	//	right, left
 	'm',		//	color
 ];
 
-const AnsiSpecialOpCodes = {
-	positioning	: [ 'A', 'B', 'C', 'D' ],	//	up, down, right, left
-	style		: [ 'm' ]					//	color
-};
-
 function cleanControlCodes(input, options) {
 	let m;
 	let pos;
@@ -373,48 +386,42 @@ function cleanControlCodes(input, options) {
 	return cleaned;
 }
 
-function createCleanAnsi(input, options, cb) {
+function prepAnsi(input, options, cb) {
-
 	if(!input) {
-		return cb('');
+		return cb(null, '');
 	}
 
-	options.width	= options.width || 80;
+	options.termWidth			= options.termWidth 	|| 80;
-	options.height	= options.height || 25;
+	options.termHeight			= options.termHeight	|| 25;
+	options.cols				= options.cols || options.termWidth		|| 80;
+	options.rows				= options.rows || options.termHeight	|| 'auto';
+	options.startCol			= options.startCol || 1;
+	options.exportMode			= options.exportMode || false;
 
-	const canvas = new Array(options.height);
+	const canvas = Array.from( { length : 'auto' === options.rows ? 25 : options.rows }, () => Array.from( { length : options.cols}, () => new Object() ) );
-	for(let i = 0; i < options.height; ++i) {
+	const parser = new ANSIEscapeParser( { termHeight : options.termHeight, termWidth : options.termWidth } );
-		canvas[i] = new Array(options.width);
-		for(let j = 0; j < options.width; ++j) {
-			canvas[i][j] = {};
-		}
-	}
 
-	const parserOpts = {
+	const state = {
-		termHeight	: options.height,
-		termWidth	: options.width,
-	};
-
-	const parser = new ANSIEscapeParser(parserOpts);
-
-	const canvasPos = {
-		col : 0,
 		row	: 0,
+		col	: 0,
 	};
 
-	let sgr;
+	let lastRow = 0;
 
-	function ensureCell() {
+	function ensureRow(row) {
-		//	we've pre-allocated a matrix, but allow for > supplied dimens up front. They will be trimmed @ finalize
+		if(Array.isArray(canvas[row])) {
-		if(!canvas[canvasPos.row]) {
+			return;
-			canvas[canvasPos.row] = new Array(options.width);
-			for(let j = 0; j < options.width; ++j) {
-				canvas[canvasPos.row][j] = {};
 		}
+		
+		canvas[row] = Array.from( { length : options.cols}, () => new Object() );
 	}
-		canvas[canvasPos.row][canvasPos.col]	= canvas[canvasPos.row][canvasPos.col] || {};
+
-		//canvas[canvasPos.row][0].width 			= Math.max(canvas[canvasPos.row][0].width || 0, canvasPos.col); 
+	parser.on('position update', (row, col) => {
-	}
+		state.row	= row - 1;
+		state.col	= col - 1;
+
+		lastRow		= Math.max(state.row, lastRow);
+	});
 
 	parser.on('literal', literal => {
 		//
@@ -422,98 +429,206 @@ function createCleanAnsi(input, options, cb) {
 		//
 		literal = literal.replace(/\r?\n|[\r\u2028\u2029]/g, '');
 
-		for(let i = 0; i < literal.length; ++i) {
+		for(let c of literal) {
-			const c = literal.charAt(i);
+			if(state.col < options.cols && ('auto' === options.rows || state.row < options.rows)) {
+				ensureRow(state.row);
 
-			ensureCell();
+				canvas[state.row][state.col].char = c;
 
-			canvas[canvasPos.row][canvasPos.col].char	= c;
+				if(state.sgr) {
-						
+					canvas[state.row][state.col].sgr = state.sgr;
-			if(sgr) {
+					state.sgr = null;
-				canvas[canvasPos.row][canvasPos.col].sgr = sgr;
+				}
-				sgr = null;
 			}
 
-			canvasPos.col += 1;
+			state.col += 1;
 		}
 	});
 
 	parser.on('control', (match, opCode) => {
-		if('m' !== opCode) {
+		//
-			return;	//	don't care'
+		//	Movement is handled via 'position update', so we really only care about
+		//	display opCodes
+		//
+		switch(opCode) {
+			case 'm' 	: 
+				state.sgr = (state.sgr || '') + match;
+				break;
+				
+			default		: 
+				break;
 		}
-		sgr = match;
 	});
 
-	parser.on('position update', (row, col) => {
+	function getLastPopulatedColumn(row) {
-		canvasPos.row	= row - 1;
+		let col = row.length;
-		canvasPos.col	= Math.min(col - 1, options.width);
+		while(--col > 0) {
-	});
+			if(row[col].char || row[col].sgr) {
+				break;
+			}
+		}
+		return col;
+	}
 
 	parser.on('complete', () => {
-		for(let row = 0; row < options.height; ++row) {
+		let output = '';
-			let col = 0;
+		let lastSgr = '';
+		let line;
 
-			//while(col <= canvas[row][0].width) {
+		canvas.slice(0, lastRow + 1).forEach(row => {
-			while(col < options.width) {
+			const lastCol = getLastPopulatedColumn(row) + 1;
-				if(!canvas[row][col].char) {
+
-					canvas[row][col].char = ' ';
+			let i;
-					if(!canvas[row][col].sgr) {
+			line = '';
-						//	:TODO: fix duplicate SGR's in a row here - we just need one per sequence
+			for(i = 0; i < lastCol; ++i) {
-						canvas[row][col].sgr = ANSI.reset();
+				const col = row[i];
+				if(col.sgr) {
+					lastSgr = col.sgr;
 				}
+				line += `${col.sgr || ''}${col.char || ' '}`;
 			}
 
-				col += 1;
+			output += line;
+
+			if(i < row.length) {
+				output += `${ANSI.blackBG()}${row.slice(i).map( () => ' ').join('')}${lastSgr}`;
 			}
 
-			//	:TODO: end *all* with CRLF - general usage will be width : 79 - prob update defaults
+			//if(options.startCol + options.cols < options.termWidth || options.forceLineTerm) {
-
+			if(options.startCol + i < options.termWidth || options.forceLineTerm) {
-			if(col <= options.width) {
+				output += '\r\n';
-				canvas[row][col] = canvas[row][col] || {};
-
-				canvas[row][col].char = '\r\n';
-				canvas[row][col].sgr = ANSI.reset();
-
-				//	:TODO: don't splice, just reset + fill with ' ' till end
-				for(let fillCol = col; fillCol <= options.width; ++fillCol) {
-					canvas[row][fillCol].char = ' ';
 			}
+		});
 
-				//canvas[row] = canvas[row].splice(0, col + 1);
+		if(options.exportMode) {
-				//canvas[row][options.width - 1].char = '\r\n';
+			//
+			//	If we're in export mode, we do some additional hackery:
+			//	
+			//	* Hard wrap ALL lines at <= 79 *characters* (not visible columns)
+			//	  if a line must wrap early, we'll place a ESC[A ESC[<N>C where <N>
+			//	  represents chars to get back to the position we were previously at
+			//
+			//	* Replace contig spaces with ESC[<N>C as well to save... space.
+			//
+			//	:TODO: this would be better to do as part of the processing above, but this will do for now
+			const MAX_CHARS	= 79 - 8;	//	79 max, - 8 for max ESC seq's we may prefix a line with
+			let exportOutput = '';			
 			
+			let m;
+			let afterSeq;
+			let wantMore;
+			let renderStart;
 
+			splitTextAtTerms(output).forEach(fullLine => {
+				renderStart = 0;
+
+				while(fullLine.length > 0) {
+					let splitAt;
+					const ANSI_REGEXP = ANSI.getFullMatchRegExp();
+					wantMore = true;
+
+					while((m = ANSI_REGEXP.exec(fullLine))) {
+						afterSeq = m.index + m[0].length;
+
+						if(afterSeq < MAX_CHARS) {
+							//	after current seq
+							splitAt = afterSeq;
 						} else {
-				canvas[row] = canvas[row].splice(0, options.width + 1);				
+							if(m.index < MAX_CHARS) {
+								//	before last found seq
+								splitAt = m.index;
+								wantMore = false;	//	can't eat up any more
 							}
 							
+							break;	//	seq's beyond this point are >= MAX_CHARS
+						}
 					}
 
-		let out = '';
+					if(splitAt) {
-		for(let row = 0; row < options.height; ++row) {
+						if(wantMore) {
-			out += canvas[row].map( col => {
+							splitAt = Math.min(fullLine.length, MAX_CHARS - 1);
-				let c = col.sgr || '';
+						}
-				c += col.char;
+					} else {
-				return c;
+						splitAt = Math.min(fullLine.length, MAX_CHARS - 1);
-			}).join('');
-			
 					}
 
-		//	:TODO: finalize: @ any non-char cell, reset sgr & set to ' '
+					const part = fullLine.slice(0, splitAt);
-		//	:TODO: finalize: after sgr established, omit anything > supplied dimens
+					fullLine = fullLine.slice(splitAt);
-		return cb(out);
+					renderStart += renderStringLength(part);
+					exportOutput += `${part}\r\n`;
+
+					if(fullLine.length > 0) {	//	more to go for this line?
+						exportOutput += `${ANSI.up()}${ANSI.right(renderStart)}`;	
+					} else {
+						exportOutput += ANSI.up();
+					}
+				}
+			});
+
+			return cb(null, exportOutput);
+		}
+
+		return cb(null, output);
 	});
 
 	parser.parse(input);
 }
 
-/*
+function isAnsiLine(line) {
-const fs = require('graceful-fs');
+	return isAnsi(line);// || renderStringLength(line) < line.length;
-let data = fs.readFileSync('/home/nuskooler/Downloads/art3.ans');
+}
-data = iconv.decode(data, 'cp437');
+
-createCleanAnsi(data, { width : 79, height : 25 }, (out) => {
+//
-	out = iconv.encode(out, 'cp437');
+//	Returns true if the line is considered "formatted". A line is
-	fs.writeFileSync('/home/nuskooler/Downloads/art4.ans', out);
+//	considered formatted if it contains:
-});
+//	* ANSI
-*/
+//	* Pipe codes
+//	* Extended (CP437) ASCII - https://www.ascii-codes.com/
+//	* Tabs
+//	* Contigous 3+ spaces before the end of the line
+//
+function isFormattedLine(line) {
+	if(renderStringLength(line) < line.length) {
+		return true;	//	ANSI or Pipe Codes
+	}
+
+	if(line.match(/[\t\x00-\x1f\x80-\xff]/)) {	//	eslint-disable-line no-control-regex
+		return true;
+	}
+
+	if(_.trimEnd(line).match(/[ ]{3,}/)) {
+		return true;
+	}
+
+	return false;
+}
+
+function isAnsi(input) {
+	//
+	//	* ANSI found - limited, just colors
+	//	* Full ANSI art
+	//	* 
+	//	
+	//	FULL ANSI art:
+	//	* SAUCE present & reports as ANSI art
+	//	* ANSI clear screen within first 2-3 codes
+	//	* ANSI movement codes (goto, right, left, etc.)
+	//	
+	//	* 
+	/*
+	readSAUCE(input, (err, sauce) => {
+		if(!err && ('ANSi' === sauce.fileType || 'ANSiMation' === sauce.fileType)) {
+			return cb(null, 'ansi');
+		}
+	});
+	*/
+
+	//	:TODO: if a similar method is kept, use exec() until threshold
+	const ANSI_DET_REGEXP = /(?:\x1b\x5b)[\?=;0-9]*?[ABCDEFGHJKLMSTfhlmnprsu]/g;	//	eslint-disable-line no-control-regex
+	const m = input.match(ANSI_DET_REGEXP) || []; 
+	return m.length >= 4;	//	:TODO: do this reasonably, e.g. a percent or soemthing
+}
+
+function splitTextAtTerms(s) {
+	return s.split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);
+}

core/theme.js

@@ -10,6 +10,7 @@ const getFullConfig		= require('./config_util.js').getFullConfig;
 const asset				= require('./asset.js');
 const ViewController	= require('./view_controller.js').ViewController;
 const Errors			= require('./enig_error.js').Errors;
+const ErrorReasons		= require('./enig_error.js').ErrorReasons;
 
 const fs				= require('graceful-fs');
 const paths				= require('path');
@@ -80,24 +81,27 @@ function refreshThemeHelpers(theme) {
 
 function loadTheme(themeID, cb) {
 
-	var path = paths.join(Config.paths.themes, themeID, 'theme.hjson');
+	const path = paths.join(Config.paths.themes, themeID, 'theme.hjson');
 
-	configCache.getConfigWithOptions( { filePath : path, forceReCache : true }, function loaded(err, theme) {
+	configCache.getConfigWithOptions( { filePath : path, forceReCache : true }, (err, theme) => {
 		if(err) {
-			cb(err);
+			return cb(err);
-		} else {
+		}
+		
 		if(!_.isObject(theme.info) || 
 			!_.isString(theme.info.name) ||
 			!_.isString(theme.info.author))
 		{
-				cb(new Error('Invalid or missing "info" section!'));
+			return cb(Errors.Invalid('Invalid or missing "info" section'));
-				return;
+		}
+
+		if(false === _.get(theme, 'info.enabled')) {
+			return cb(Errors.General('Theme is not enalbed', ErrorReasons.ErrNotEnabled));
 		}
 
 		refreshThemeHelpers(theme);
 
-			cb(null, theme, path);
+		return cb(null, theme, path);
-		}
 	});
 }
 
@@ -261,69 +265,72 @@ function getMergedTheme(menuConfig, promptConfig, theme) {
 }
 
 function initAvailableThemes(cb) {
-	var menuConfig;
-	var promptConfig;
    
 	async.waterfall(
 		[
 			function loadMenuConfig(callback) {
-				getFullConfig(Config.general.menuFile, function gotConfig(err, mc) {
+				getFullConfig(Config.general.menuFile, (err, menuConfig) => {
-					menuConfig = mc;
+					return callback(err, menuConfig);
-					callback(err);
 				});
 			},
-			function loadPromptConfig(callback) {
+			function loadPromptConfig(menuConfig, callback) {
-				getFullConfig(Config.general.promptFile, function gotConfig(err, pc) {
+				getFullConfig(Config.general.promptFile, (err, promptConfig) => {
-					promptConfig = pc;
+					return callback(err, menuConfig, promptConfig);
-					callback(err); 
 				});
 			},
-			function getDir(callback) {
+			function getThemeDirectories(menuConfig, promptConfig, callback) {
-				fs.readdir(Config.paths.themes, function dirRead(err, files) {
+				fs.readdir(Config.paths.themes, (err, files) =>  {
-					callback(err, files);
+					if(err) {
-				});
+						return callback(err);
-			},
+					}
-			function filterFiles(files, callback) {				
+
-				var filtered = files.filter(function filter(file) {
+					return callback(
-					return fs.statSync(paths.join(Config.paths.themes, file)).isDirectory(); 
+						null, 
-				});
+						menuConfig, 
-				callback(null, filtered);
+						promptConfig, 
-			},
+						files.filter( f => {
-			function populateAvailable(filtered, callback) {
+							//	sync normally not allowed -- initAvailableThemes() is a startup-only method, however
-                //  :TODO: this is a bit broken with callback placement and configCache.on() handler
+							return fs.statSync(paths.join(Config.paths.themes, f)).isDirectory();
+						})
+					);
+				});
+			},
+			function populateAvailable(menuConfig, promptConfig, themeDirectories, callback) {
+				async.each(themeDirectories, (themeId, nextThemeDir) => {	//	theme dir = theme ID
+					loadTheme(themeId, (err, theme, themePath) => {
+						if(err) {
+							if(ErrorReasons.NotEnabled !== err.reasonCode) {
+								Log.warn( { themeId : themeId, err : err.message }, 'Failed loading theme');
+							}
+
+							return nextThemeDir(null);	//	try next
+						}
 
-				filtered.forEach(function themeEntry(themeId) {
-					loadTheme(themeId, function themeLoaded(err, theme, themePath) {
-						if(!err) {
 						availableThemes[themeId] = getMergedTheme(menuConfig, promptConfig, theme);
 
-							configCache.on('recached', function recached(path) {
+						configCache.on('recached', recachedPath => {
-								if(themePath === path) {									
+							if(themePath === recachedPath) {
-									loadTheme(themeId, function reloaded(err, reloadedTheme) {
+								loadTheme(themeId, (err, reloadedTheme) => {
+									if(!err) {
+										//	:TODO: This is still broken - Need to reapply *latest* menu config and prompt configs to theme at very least
 										Log.debug( { info : theme.info }, 'Theme recached' );
-
+										availableThemes[themeId] = getMergedTheme(menuConfig, promptConfig, reloadedTheme);
-										availableThemes[themeId] = reloadedTheme;
+									} else if(ErrorReasons.NotEnabled === err.reasonCode) {
+										//	:TODO: we need to disable this theme -- users may be using it! We'll need to re-assign them if so
+									}
 								});
 							}
 						});
 
-							Log.debug( { info : theme.info }, 'Theme loaded');
+						return nextThemeDir(null);
-						} else {
-							Log.warn( { themeId : themeId, error : err.toString() }, 'Failed to load theme');
-						}
 					});
-
+				}, err => {
+					return callback(err);
 				});
-				callback(null);
 			}
 		],
-		function onComplete(err) {
+		err => {
-			if(err) {
+			return cb(err, availableThemes ? availableThemes.length : 0);
-				cb(err);
-				return;
-			}
-
-			cb(null, availableThemes.length);
 		}
 	);
 }
@@ -340,17 +347,24 @@ function getRandomTheme() {
 }
 
 function setClientTheme(client, themeId) {
-	var desc;
+	let logMsg;
 
-	try {
+	const availThemes = getAvailableThemes();
-		client.currentTheme = getAvailableThemes()[themeId];
+
-		desc = 'Set client theme';        
+	client.currentTheme = availThemes[themeId];
-	} catch(e) {
+	if(client.currentTheme) {
-		client.currentTheme = getAvailableThemes()[Config.defaults.theme];
+		logMsg = 'Set client theme';
-		desc = 'Failed setting theme by supplied ID; Using default';
+	} else {
+		client.currentTheme = availThemes[Config.defaults.theme];
+		if(client.currentTheme) {
+			logMsg = 'Failed setting theme by supplied ID; Using default';
+		} else {
+			client.currentTheme = availThemes[Object.keys(availThemes)[0]];
+			logMsg = 'Failed setting theme by system default ID; Using the first one we can find';
+		}
 	}
 	
-	client.log.debug( { themeId : themeId, info : client.currentTheme.info }, desc);
+	client.log.debug( { themeId : themeId, info : client.currentTheme.info }, logMsg);
 }
 
 function getThemeArt(options, cb) {
@@ -375,7 +389,7 @@ function getThemeArt(options, cb) {
 	//	:TODO: Some of these options should only be set if not provided!
 	options.asAnsi		= true;	//	always convert to ANSI
 	options.readSauce	= true;	//	read SAUCE, if avail	
-	options.random		= _.isBoolean(options.random) ? options.random : true;	//	FILENAME<n>.EXT support
+	options.random		= _.get(options, 'random', true);	//	FILENAME<n>.EXT support
 
 	//
 	//	We look for themed art in the following order:
@@ -450,24 +464,11 @@ function displayThemeArt(options, cb) {
 	assert(_.isObject(options.client));
 	assert(_.isString(options.name));
 
-	getThemeArt(options, function themeArt(err, artInfo) {
+	getThemeArt(options, (err, artInfo) => {
 		if(err) {
-			cb(err);
+			return cb(err);
-		} else {
+		}
 		//	:TODO: just use simple merge of options -> displayOptions
-			/*
-			var dispOptions = {
-				art				: artInfo.data,
-				sauce			: artInfo.sauce,
-				client			: options.client,
-				font			: options.font,
-				trailingLF		: options.trailingLF,
-			};
-
-			art.display(dispOptions, function displayed(err, mciMap, extraInfo) {
-				cb(err, { mciMap : mciMap, artInfo : artInfo, extraInfo : extraInfo } );
-			});
-			*/
 		const displayOpts = {
 			sauce		: artInfo.sauce,
 			font		: options.font,
@@ -477,7 +478,6 @@ function displayThemeArt(options, cb) {
 		art.display(options.client, artInfo.data, displayOpts, (err, mciMap, extraInfo) => {
 			return cb(err, { mciMap : mciMap, artInfo : artInfo, extraInfo : extraInfo } );
 		});
-		}
 	});
 }
 
@@ -566,8 +566,10 @@ function displayThemedPrompt(name, client, options, cb) {
 
 				//
 				//	If we did *not* clear the screen, don't let the font change
-				//	as it will mess with the output of the existing art displayed in a terminal
+				//	doing so messes things up -- most terminals that support font
+				//	changing can only display a single font at at time.
 				//
+				//	:TODO: We can use term detection to do nifty things like avoid this kind of kludge:
 				const dispOptions = Object.assign( {}, promptConfig.options );
 				if(!options.clearScreen) {
 					dispOptions.font = 'not_really_a_font!';	//	kludge :)

core/tic_file_info.js

@@ -17,6 +17,7 @@ const crypto		= require('crypto');
 //
 //	Class to read and hold information from a TIC file
 //
+//	* FTS-5006.001 @ http://www.filegate.net/ftsc/FTS-5006.001
 //	* FSP-1039.001 @ http://ftsc.org/docs/old/fsp-1039.001
 //	* FSC-0087.001 @ http://ftsc.org/docs/fsc-0087.001
 //

core/user_login.js

@@ -37,14 +37,16 @@ function userLogin(client, username, password, cb) {
 		});
 
 		if(existingClientConnection) {
-			client.log.info( {
+			client.log.info(
+				{
 					existingClientId	: existingClientConnection.session.id, 
 					username			: user.username, 
-				userId				: user.userId },
+					userId				: user.userId
+				},
 				'Already logged in'
 			);
 
-			var existingConnError = new Error('Already logged in as supplied user');
+			const existingConnError = new Error('Already logged in as supplied user');
 			existingConnError.existingConn = true;
 
 			//	:TODO: We should use EnigError & pass existing connection as second param
@@ -61,24 +63,24 @@ function userLogin(client, username, password, cb) {
 			[
 				function setTheme(callback) {
 					setClientTheme(client, user.properties.theme_id);
-					callback(null);
+					return callback(null);
 				},
 				function updateSystemLoginCount(callback) {
-					StatLog.incrementSystemStat('login_count', 1, callback);
+					return StatLog.incrementSystemStat('login_count', 1, callback);
 				},
 				function recordLastLogin(callback) {
-					StatLog.setUserStat(user, 'last_login_timestamp', StatLog.now, callback);
+					return StatLog.setUserStat(user, 'last_login_timestamp', StatLog.now, callback);
 				},
 				function updateUserLoginCount(callback) {
-					StatLog.incrementUserStat(user, 'login_count', 1, callback);						
+					return StatLog.incrementUserStat(user, 'login_count', 1, callback);						
 				},
 				function recordLoginHistory(callback) {
 					const LOGIN_HISTORY_MAX	= 200;	//	history of up to last 200 callers
-					StatLog.appendSystemLogEntry('user_login_history', user.userId, LOGIN_HISTORY_MAX, StatLog.KeepType.Max, callback);
+					return StatLog.appendSystemLogEntry('user_login_history', user.userId, LOGIN_HISTORY_MAX, StatLog.KeepType.Max, callback);
 				}
 			],
-			function complete(err) {
+			err => {
-				cb(err);
+				return cb(err);
 			}
 		);
 	});

core/vertical_menu_view.js

@@ -5,10 +5,10 @@
 const MenuView		= require('./menu_view.js').MenuView;
 const ansi			= require('./ansi_term.js');
 const strUtil		= require('./string_util.js');
-const colorCodes	= require('./color_codes.js');
 
 //	deps
 const util			= require('util');
+const _				= require('lodash');
 
 exports.VerticalMenuView		= VerticalMenuView;
 
@@ -20,6 +20,14 @@ function VerticalMenuView(options) {
 
 	const self = this;
 
+	//	we want page up/page down by default
+	if(!_.isObject(options.specialKeyMap)) {
+		Object.assign(this.specialKeyMap, {
+			'page up'		: [ 'page up' ],
+			'page down'		: [ 'page down' ],
+		});
+	}
+
 	this.performAutoScale = function() {
 		if(this.autoScale.height) {
 			this.dimens.height = (self.items.length * (self.itemSpacing + 1)) - (self.itemSpacing);
@@ -99,7 +107,7 @@ VerticalMenuView.prototype.redraw = function() {
 		let row			= this.position.row + 1;
 		const endRow	= (row + this.oldDimens.height) - 2;
 		
-		while(row < endRow) {
+		while(row <= endRow) {
 			seq += ansi.goto(row, this.position.col) + blank;
 			row += 1;
 		}
@@ -160,6 +168,10 @@ VerticalMenuView.prototype.onKeyPress = function(ch, key) {
 			this.focusPrevious();
 		} else if(this.isKeyMapped('down', key.name)) {
 			this.focusNext();
+		} else if(this.isKeyMapped('page up', key.name)) {
+			this.focusPreviousPageItem();
+		} else if( this.isKeyMapped('page down', key.name)) {
+			this.focusNextPageItem();
 		}
 	}
 
@@ -243,6 +255,54 @@ VerticalMenuView.prototype.focusPrevious = function() {
 	VerticalMenuView.super_.prototype.focusPrevious.call(this);
 };
 
+VerticalMenuView.prototype.focusPreviousPageItem = function() {
+	//
+	//	Jump to current - up to page size or top
+	//	If already at the top, jump to bottom
+	//
+	if(0 === this.focusedItemIndex) {
+		return this.focusPrevious();	//	will jump to bottom
+	}
+
+	const index = Math.max(this.focusedItemIndex - this.dimens.height, 0);
+
+	if(index < this.viewWindow.top) {
+		this.oldDimens = Object.assign({}, this.dimens);
+	}
+
+	this.setFocusItemIndex(index);
+
+	return VerticalMenuView.super_.prototype.focusPreviousPageItem.call(this);
+};
+
+VerticalMenuView.prototype.focusNextPageItem = function() {
+	//
+	//	Jump to current + up to page size or bottom
+	//	If already at the bottom, jump to top
+	//	
+	if(this.items.length - 1 === this.focusedItemIndex) {
+		return this.focusNext();	//	will jump to top
+	}
+
+	const index = Math.min(this.focusedItemIndex + this.maxVisibleItems, this.items.length - 1);
+
+	if(index > this.viewWindow.bottom) {
+		this.oldDimens = Object.assign({}, this.dimens);
+
+		this.focusedItemIndex = index;
+
+		this.viewWindow = {
+			top		: this.focusedItemIndex,
+			bottom	: Math.min(this.focusedItemIndex + this.maxVisibleItems, this.items.length) - 1
+		};
+
+		this.redraw();
+	} else {
+		this.setFocusItemIndex(index);
+	}
+
+	return VerticalMenuView.super_.prototype.focusNextPageItem.call(this);
+};
 
 VerticalMenuView.prototype.setFocusItems = function(items) {
 	VerticalMenuView.super_.prototype.setFocusItems.call(this, items);

core/word_wrap.js

@@ -16,50 +16,6 @@ const SPACE_CHARS		= [
 
 const REGEXP_WORD_WRAP = new RegExp(`\t|[${SPACE_CHARS.join('')}]`, 'g'); 
 
-/*
-//
-//	ANSI & pipe codes we indend to strip
-//
-//	See also https://github.com/chalk/ansi-regex/blob/master/index.js
-//
-//	:TODO: Consolidate this, regexp's in ansi_escape_parser, and strutil. Need more complete set that includes common standads, bansi, and cterm.txt
-//	renderStringLength() from strUtil does not account for ESC[<N>C (e.g. go forward)
-const REGEXP_CONTROL_CODES = /(\|[\d]{2})|(?:\x1b\x5b)([\?=;0-9]*?)([ABCDHJKfhlmnpsu])/g;
-
-function getRenderLength(s) {
-	let m;
-	let pos;
-	let len = 0;
-
-	REGEXP_CONTROL_CODES.lastIndex = 0;	//	reset
-	
-	//
-	//	Loop counting only literal (non-control) sequences
-	//	paying special attention to ESC[<N>C which means forward <N>
-	//	
-	do {
-		pos	= REGEXP_CONTROL_CODES.lastIndex;
-		m	= REGEXP_CONTROL_CODES.exec(s);
-		
-		if(null !== m) {
-			if(m.index > pos) {
-				len += s.slice(pos, m.index).length;
-			}
-			
-			if('C' === m[3]) {	//	ESC[<N>C is foward/right
-				len += parseInt(m[2], 10) || 0;
-			}
-		}  
-	} while(0 !== REGEXP_CONTROL_CODES.lastIndex);
-	
-	if(pos < s.length) {
-		len += s.slice(pos).length;
-	}
-	
-	return len;
-}
-*/
-
 function wordWrapText2(text, options) {
 	assert(_.isObject(options));
 	assert(_.isNumber(options.width));
@@ -68,7 +24,13 @@ function wordWrapText2(text, options) {
 	options.tabWidth	= options.tabWidth || 4;
 	options.tabChar		= options.tabChar || ' ';		
 	
-	const REGEXP_GOBBLE = new RegExp(`.{0,${options.width}}`, 'g');
+	//const REGEXP_GOBBLE = new RegExp(`.{0,${options.width}}`, 'g');
+	//
+	//	For a given word, match 0->options.width chars -- alwasy include a full trailing ESC
+	//	sequence if present!
+	//
+	//	:TODO: Need to create ansi.getMatchRegex or something - this is used all over
+	const REGEXP_GOBBLE = new RegExp(`.{0,${options.width}}\\x1b\\[[\\?=;0-9]*[ABCDEFGHJKLMSTfhlmnprsu]|.{0,${options.width}}`, 'g');
 	
 	let m;
 	let word;

docs/index.md

@@ -2,7 +2,7 @@
 ENiGMA½ is a modern from scratch BBS package written in Node.js.
 
 # Quickstart
-TL;DR? This should get you started...
+Unless you have a compelling reason to do otherwise, please use **The Easy Way** below.
 
 ## The Easy Way
 Under most Linux/UNIX like environments (Linux, BSD, OS X, ...)  new users can simply execute the `install.sh` script to get everything up and running. Simply cut + paste the following into your terminal:
@@ -13,7 +13,7 @@ curl -o- https://raw.githubusercontent.com/NuSkooler/enigma-bbs/master/misc/inst
 
 For other environments such as Windows, see **The Manual Way** below.
 
-## The Manual Way
+## The Manual Way (aka Advanced)
 For Windows environments or if you simply like to do things manually, read on...
 
 ### Prerequisites
@@ -61,7 +61,7 @@ The main system configuration is handled via `~/.config/enigma-bbs/config.hjson`
 `oputil.js` can be utilized to generate your **initial** configuration. **This is the recommended way for all new users**:
 
 ```bash
-./oputil.js config --new
+./oputil.js config new
 ```
 
 (You wil be asked a series of basic questions)

docs/mci.md

@@ -74,6 +74,11 @@ There are many predefined MCI codes that can be used anywhere on the system (pla
 * `AN`: Current active node count
 * `TC`: Total login/calls to system
 * `RR`: Displays a random rumor
+* `SD`: Total downloads, system wide
+* `SO`: Total downloaded amount, system wide (formatted to appropriate bytes/megs/etc.)
+* `SU`: Total uploads, system wide
+* `SP`: Total uploaded amount, system wide (formatted to appropriate bytes/megs/etc.)
+
 
 A special `XY` MCI code may also be utilized for placement identification when creating menus.
 

misc/exodus.id_rsa

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAmpwn/vJ1CAIkVnQGDZumvEDsMyFSHioGO5RM/T2Id6XLX91r
+feJI6w48yqLV+HgKLUK7eeTOzb/l4VShH9AzOqTbAxwfZ6fgzV2cI/wZxO0S6QpS
+7IrwcVK1Bm7Wu45Kp7LcGHB66nHSb+wqIYkZobIc8Z9arClJxV4AzgaUxjJrk0wT
+hW81r5TicbTG7zm+bOMLO/mln+HA/EOtx/yfKDcfkl+mLzzbMpojor+KdwuKJUb1
++r4PhPVl6pZgOuQIl37Qh5SPY3mMjwyXW/tUe+ZmPpfOm3CKf/pTLsA45QzUbNBY
+GPLjbEcMJ4R5T3c2LXCKR+Wi9/pCkeZT7/1BbQIDAQABAoIBAQCCasrKIddahAQG
+8SPSAsQo9FLJ5oeQbj6Hr1cqHueola/yE6KCs4hyzrW08JqxVwCuoSXncnyHziGp
+a2vmnAc6pqkf/G75TwEv+pClQhiyppBXB6Bfa+vai7ury39TAnoy74r9CpSEgrLS
+OlJnq3B1lvsXTiZ8Ju/Vjq/7Gk4QyFOVPugbmjhUtuCiyRXV9V2o/HUzZGtaXDp5
+n+XOfb90mLtPhtIRC6wmgMkhlRPpGir+NN0DWQ1oBWZO+TockIFusVInOTEXY4ui
+V+JJ3KRwfaogzJMnDcqkiCck6bMT8E85ucRScsJjpENsUyEjFAoRV2grbguc/rdx
+dgG5BMx5AoGBAMgCDFGwCctHfRvRXIac5goxYuTkVYjEh4yxj8d/Y+0HmDJiH5HM
+tiUAtsgq/KYKJKM9U0PJWdPW3DPJa+wDVPQSlIqUOiXEpwLA+yhXuAvTqia9chuI
+vaP1Ze/4yfW2eQg+3Ji0vC9VEr1eoRnAwJI+fDE3fRCvoPohlT4+zOhvAoGBAMXk
+ksy5DdtUOvt0wss7R030dEtHP/Hs+qheQJOhl+GLlQt5BKP6NsdM3OKXyXYLddOc
+xrKSWdjtiWOtap0D7o7cBFv44EmgzSvM2QltYxF4phPaNn2zPC/Mkvs1EaYnMtw4
+boKNDWbwixpCapheAE+lfA96DfqU/KyVaXls9MnjAoGAaL+B2ipbBsZ7BF2imrGD
+XOU+iOf4z/c1kn7P8UiLefEXSZPQOti+sCRulejFhuQbCg8tE3xZejO2Ab1Es0eP
+b4BnoSg+R9d1LGELaLaAIlmJbF6da0QzJbJ437QpeXFGdAYQHD3TrOpeNSVhNA6a
+DD2DZ3dLHbkNktKRyhaz1CsCgYBMJbIfOK4OUZEIpVs3XK4JXyFIvjfq3aduFiZ/
+KFULIuzNJ1oTxvpBImB0iLeqxqomLVN/7zTHdk/BnT9C//pR2nOK+G9FpayNSBvT
+ttXCKUyuou8I22kzc2Kzay5JYxf9CXHspl4b2D+OcTQXQUSZYTIlum+alq3LswqN
+ANIIxQKBgHauoT79sViuB/wHcp2W/mek0p9aLkgQKt+riPJ4vKXc8DtapTgQzXkk
+6yQCOSD8T9DcVGBcap9n6T21NOyDQwM0gg+DoHVeYqBrAa93jufOi7EY3MFrkjH6
+tC0crKBcUkxu43zhY4DkHLxId5btSPH57U+lhrJGjKXdvlJrGGOM
+-----END RSA PRIVATE KEY-----

mods/file_area_filter_edit.js

@@ -61,7 +61,6 @@ exports.getModule = class FileAreaFilterEdit extends MenuModule {
 		this.menuMethods = {
 			saveFilter : (formData, extraArgs, cb) => {
 				return this.saveCurrentFilter(formData, cb);
-
 			},
 			prevFilter : (formData, extraArgs, cb) => {
 				this.currentFilterIndex -= 1;
@@ -93,7 +92,15 @@ exports.getModule = class FileAreaFilterEdit extends MenuModule {
 				return cb(null);
 			},
 			deleteFilter : (formData, extraArgs, cb) => {
-				const filterUuid = this.filtersArray[this.currentFilterIndex].uuid;
+				const selectedFilter	= this.filtersArray[this.currentFilterIndex];
+				const filterUuid		= selectedFilter.uuid;
+
+				//	cannot delete built-in/system filters
+				if(true === selectedFilter.system) {
+					this.showError('Cannot delete built in filters!');
+					return cb(null);
+				}
+
 				this.filtersArray.splice(this.currentFilterIndex, 1);	//	remove selected entry
 
 				//	remove from stored properties
@@ -144,6 +151,17 @@ exports.getModule = class FileAreaFilterEdit extends MenuModule {
 		};
 	}
 
+	showError(errMsg) {
+		const errorView = this.viewControllers.editor.getView(MciViewIds.editor.error);
+		if(errorView) {
+			if(errMsg) {
+				errorView.setText(errMsg);
+			} else {
+				errorView.clearText();
+			}
+		}
+	}
+	
 	mciReady(mciData, cb) {
 		super.mciReady(mciData, err => {
 			if(err) {

mods/file_area_list.js

@@ -8,7 +8,6 @@ const ansi				= require('../core/ansi_term.js');
 const theme				= require('../core/theme.js');
 const FileEntry			= require('../core/file_entry.js');
 const stringFormat		= require('../core/string_format.js');
-const createCleanAnsi	= require('../core/string_util.js').createCleanAnsi;
 const FileArea			= require('../core/file_base_area.js');
 const Errors			= require('../core/enig_error.js').Errors;
 const ErrNotEnabled		= require('../core/enig_error.js').ErrorReasons.NotEnabled;
@@ -18,8 +17,7 @@ const DownloadQueue		= require('../core/download_queue.js');
 const FileAreaWeb		= require('../core/file_area_web.js');
 const FileBaseFilters	= require('../core/file_base_filter.js');
 const resolveMimeType	= require('../core/mime_util.js').resolveMimeType;
-
+const isAnsi			= require('../core/string_util.js').isAnsi;
-const cleanControlCodes	= require('../core/string_util.js').cleanControlCodes;
 
 //	deps
 const async				= require('async');
@@ -74,8 +72,12 @@ exports.getModule = class FileAreaList extends MenuModule {
 	constructor(options) {
 		super(options);
 
-		if(options.extraArgs) {
+		this.filterCriteria		= _.get(options, 'extraArgs.filterCriteria');
-			this.filterCriteria	= options.extraArgs.filterCriteria;
+		this.fileList			= _.get(options, 'extraArgs.fileList');
+
+		if(this.fileList) {
+			//	we'll need to adjust position as well!
+			this.fileListPosition = 0;
 		}
 
 		this.dlQueue = new DownloadQueue(this.client);
@@ -116,7 +118,13 @@ exports.getModule = class FileAreaList extends MenuModule {
 				return this.displayDetailsPage(cb);
 			},
 			detailsQuit : (formData, extraArgs, cb) => {
-				this.viewControllers.details.setFocus(false);
+				[ 'detailsNfo', 'detailsFileList', 'details' ].forEach(n => {
+					const vc = this.viewControllers[n];
+					if(vc) {
+						vc.detachClientEvents();
+					}
+				});
+
 				return this.displayBrowsePage(true, cb);	//	true=clearScreen
 			},
 			toggleQueue : (formData, extraArgs, cb) => {
@@ -212,8 +220,8 @@ exports.getModule = class FileAreaList extends MenuModule {
 		const entryInfo = currEntry.entryInfo = {
 			fileId				: currEntry.fileId,
 			areaTag				: currEntry.areaTag,
-			areaName			: area.name || 'N/A',
+			areaName			: _.get(area, 'name') || 'N/A',
-			areaDesc			: area.desc || 'N/A',
+			areaDesc			: _.get(area, 'desc') || 'N/A',
 			fileSha256			: currEntry.fileSha256,
 			fileName			: currEntry.fileName,
 			desc				: currEntry.desc || '',
@@ -250,9 +258,9 @@ exports.getModule = class FileAreaList extends MenuModule {
 		const userRatingTicked		= config.userRatingTicked || '*';
 		const userRatingUnticked	= config.userRatingUnticked || '';					
 		entryInfo.userRating		= ~~Math.round(entryInfo.userRating) || 0;	//	be safe!
-		entryInfo.userRatingString	= new Array(entryInfo.userRating + 1).join(userRatingTicked);
+		entryInfo.userRatingString	= userRatingTicked.repeat(entryInfo.userRating);
 		if(entryInfo.userRating < 5) {
-			entryInfo.userRatingString += new Array( (5 - entryInfo.userRating) + 1).join(userRatingUnticked);
+			entryInfo.userRatingString += userRatingUnticked.repeat( (5 - entryInfo.userRating) );
 		}
 
 		FileAreaWeb.getExistingTempDownloadServeItem(this.client, this.currentFileEntry, (err, serveItem) => {
@@ -373,32 +381,35 @@ exports.getModule = class FileAreaList extends MenuModule {
 						return self.populateCurrentEntryInfo(callback);
 					});
 				},
-				function populateViews(callback) {
+				function populateDesc(callback) {
 					if(_.isString(self.currentFileEntry.desc)) {
 						const descView = self.viewControllers.browse.getView(MciViewIds.browse.desc);
 						if(descView) {
-							createCleanAnsi(
+							if(isAnsi(self.currentFileEntry.desc)) {
+								descView.setAnsi(
 									self.currentFileEntry.desc,
-								{ height : self.client.termHeight, width : descView.dimens.width },
+									{
-								cleanDesc => {
+										prepped			: false,
-									//	:TODO: use cleanDesc -- need to finish createCleanAnsi() !!
+										forceLineTerm	: true
-									//descView.setText(cleanDesc);
+									},
-									descView.setText( self.currentFileEntry.desc );
+									() => {
-
-									self.updateQueueIndicator();
-									self.populateCustomLabels('browse', MciViewIds.browse.customRangeStart);
-
 										return callback(null);
 									}
 								);
-						}
 							} else {
-						self.updateQueueIndicator();
+								descView.setText(self.currentFileEntry.desc);
-						self.populateCustomLabels('browse', MciViewIds.browse.customRangeStart);
-
 								return callback(null);
 							}
 						}
+					} else {
+						return callback(null);
+					}
+				},
+				function populateAdditionalViews(callback) {
+					self.updateQueueIndicator();
+					self.populateCustomLabels('browse', MciViewIds.browse.customRangeStart);
+					return callback(null);
+				}
 			],
 			err => {				
 				if(cb) {
@@ -618,17 +629,37 @@ exports.getModule = class FileAreaList extends MenuModule {
 						case 'nfo' :
 							{
 								const nfoView = self.viewControllers.detailsNfo.getView(MciViewIds.detailsNfo.nfo);
-								if(nfoView) {
+								if(!nfoView) {
+									return callback(null);
+								}
+
+								if(isAnsi(self.currentFileEntry.entryInfo.descLong)) {
+									nfoView.setAnsi(
+										self.currentFileEntry.entryInfo.descLong,
+										{
+											prepped			: false,
+											forceLineTerm	: true,
+										},
+										() => {
+											return callback(null);
+										}
+									);
+								} else {
 									nfoView.setText(self.currentFileEntry.entryInfo.descLong);
+									return callback(null);
 								}
 							}
 							break;
 
 						case 'fileList' :
 							self.populateFileListing();
-							break;
+							return callback(null);
-					}
 
+						default :
+							return callback(null);
+					}
+				},
+				function setLabels(callback) {
 					self.populateCustomLabels(name, MciViewIds[name].customRangeStart);
 					return callback(null);
 				}

mods/file_base_area_select.js

@@ -0,0 +1,84 @@
+/* jslint node: true */
+'use strict';
+
+//	enigma-bbs
+const MenuModule						= require('../core/menu_module.js').MenuModule;
+const Config							= require('../core/config.js').config;
+const stringFormat						= require('../core/string_format.js');
+const ViewController					= require('../core/view_controller.js').ViewController;
+const getSortedAvailableFileAreas		= require('../core/file_base_area.js').getSortedAvailableFileAreas;
+
+//	deps
+const async			= require('async');
+const _				= require('lodash');
+
+exports.moduleInfo = {
+	name	: 'File Area Selector',
+	desc	: 'Select from available file areas',
+	author	: 'NuSkooler',
+};
+
+const MciViewIds = {
+	areaList	: 1,
+};
+
+exports.getModule = class FileAreaSelectModule extends MenuModule {
+	constructor(options) {
+		super(options);
+
+		this.config = this.menuConfig.config || {};
+
+		this.loadAvailAreas();
+
+		this.menuMethods = {
+			selectArea : (formData, extraArgs, cb) => {
+				const area = this.availAreas[formData.value.areaSelect] || 0;
+
+				const filterCriteria = {
+					areaTag		: area.areaTag,
+				};
+
+				const menuOpts = {
+					extraArgs	: {
+						filterCriteria	: filterCriteria,				
+					},
+					menuFlags	: [ 'noHistory' ],
+				};
+
+				return this.gotoMenu(this.menuConfig.config.fileBaseListEntriesMenu || 'fileBaseListEntries', menuOpts, cb);
+			}
+		};
+	}
+
+	loadAvailAreas() {
+		this.availAreas = getSortedAvailableFileAreas(this.client);
+	}
+
+	mciReady(mciData, cb) {
+		super.mciReady(mciData, err => {
+			if(err) {
+				return cb(err);
+			}
+
+			this.prepViewController('allViews', 0, { mciMap : mciData.menu }, (err, vc) => {
+				if(err) {
+					return cb(err);
+				}
+
+				const areaListView = vc.getView(MciViewIds.areaList);
+
+				const areaListFormat = this.config.areaListFormat || '{name}';
+
+				areaListView.setItems(this.availAreas.map(a => stringFormat(areaListFormat, a) ) );
+
+				if(this.config.areaListFocusFormat) {
+					areaListView.setFocusItems(this.availAreas.map(a => stringFormat(this.config.areaListFocusFormat, a) ) );
+				}
+
+				areaListView.redraw();
+
+				return cb(null);
+			});
+		});
+	}
+};

mods/menu.hjson

@@ -721,7 +721,7 @@
 		}
 
 		newScanMessageList: {
-			desc: Viewing New Message List
+			desc: New Messages
 			module: msg_list
 			art: NEWMSGS
 			config: {
@@ -761,6 +761,166 @@
 			}
 		}
 
+		newScanFileBaseList: {
+			module: file_area_list
+			desc: New Files
+			config: {
+				art: {
+					browse: FNEWBRWSE
+					details: FDETAIL
+					detailsGeneral: FDETGEN
+					detailsNfo: FDETNFO
+					detailsFileList: FDETLST
+					help: FBHELP
+				}
+			}
+			form: {
+				0: {
+					mci: {
+						MT1: {
+							mode: preview
+							ansiView: true
+						}
+
+						HM2: {
+							focus: true
+							submit: true
+							argName: navSelect							
+							items: [
+								"prev", "next", "details", "toggle queue", "rate", "help", "quit"
+							]
+							focusItemIndex: 1
+						}
+
+						//	:TODO: these can be removed once the hack is not required:
+						TL10: {}
+						TL11: {}
+						TL12: {}
+						TL13: {}
+						TL14: {}
+						TL15: {}
+						TL16: {}
+						TL17: {}
+						TL18: {}
+					}
+					
+					submit: {
+						*: [
+							{
+								value: { navSelect: 0 }
+								action: @method:prevFile
+							}
+							{
+								value: { navSelect: 1 }
+								action: @method:nextFile
+							}
+							{
+								value: { navSelect: 2 }
+								action: @method:viewDetails
+							}
+							{
+								value: { navSelect: 3 }
+								action: @method:toggleQueue
+							}
+							{
+								value: { navSelect: 4 }
+								action: @menu:fileBaseGetRatingForSelectedEntry
+							}							
+							{
+								value: { navSelect: 5 }
+								action: @method:displayHelp
+							}
+							{
+								value: { navSelect: 6 }
+								action: @systemMethod:prevMenu
+							}
+						]
+					}
+
+					actionKeys: [
+						{
+							keys: [ "w", "shift + w" ]
+							action: @method:showWebDownloadLink
+						}
+						{
+							keys: [ "escape", "q", "shift + q" ]
+							action: @systemMethod:prevMenu
+						}
+						{
+							keys: [ "t", "shift + t" ]
+							action: @method:toggleQueue
+						}
+						{
+							keys: [ "v", "shift + v" ]
+							action: @method:viewDetails
+						}
+						{
+							keys: [ "r", "shift + r" ]
+							action: @menu:fileBaseGetRatingForSelectedEntry
+						}
+						{
+							keys: [ "?" ]
+							action: @method:displayHelp
+						}
+					]
+				}
+
+				1: {
+					mci: {
+						HM1: {
+							focus: true
+							submit: true
+							argName: navSelect
+							items: [
+								"general", "nfo/readme", "file listing"
+							]
+						}
+
+						//	:TODO: these can be removed once the hack is not required:
+						TL10: {}
+						TL11: {}
+						TL12: {}
+						TL13: {}
+						TL14: {}
+						TL15: {}
+						TL16: {}
+						TL17: {}
+						TL18: {}
+					}
+
+					actionKeys: [
+						{
+							keys: [ "escape", "q", "shift + q" ]
+							action: @method:detailsQuit
+						}
+					]
+				}
+
+				2: {
+					//	details - general
+					mci: {}					
+				}
+
+				3: {
+					//	details - nfo/readme
+					mci: {
+						MT1: {
+							mode: preview
+						}
+					}
+				}
+
+				4: {
+					//	details - file listing
+					mci: {
+						VM1: {
+
+						}
+					}
+				}
+			}
+		}
+
 		///////////////////////////////////////////////////////////////////////
 		//	Main Menu
 		///////////////////////////////////////////////////////////////////////
@@ -1960,7 +2120,6 @@
 						}
 					]
 				}
-				}
 
 				//	Quote builder
 				5: {
@@ -2313,9 +2472,13 @@
 			prompt: fileMenuCommand
 			submit: [
 				{
-					value: { menuOption: "B" }
+					value: { menuOption: "L" }
 					action: @menu:fileBaseListEntries
 				}
+				{
+					value: { menuOption: "B" }
+					action: @menu:fileBaseBrowseByAreaSelect
+				}
 				{
 					value: { menuOption: "F" }
 					action: @menu:fileAreaFilterEditor
@@ -2510,6 +2673,38 @@
 			}
 		}
 
+		fileBaseBrowseByAreaSelect: {
+			desc: Browsing File Areas
+			module: file_base_area_select
+			art: FAREASEL
+			form: {
+				0: {
+					mci: {
+						VM1: {
+							focus: true
+							argName: areaSelect
+						}
+					}
+
+					submit: {
+						*: [
+							{
+								value: { areaSelect: null }
+								action: @method:selectArea
+							}
+						]
+					}
+
+					actionKeys: [
+						{
+							keys: [ "escape" ]
+							action: @systemMethod:prevMenu
+						}
+					]
+				}
+			}
+		}
+
 		fileBaseGetRatingForSelectedEntry: {
 			desc: Rating a File
 			prompt: fileBaseRateEntryPrompt

mods/msg_area_view_fse.js

@@ -24,6 +24,7 @@ exports.getModule = class AreaViewFSEModule extends FullScreenEditorModule {
 		if(_.isObject(options.extraArgs)) {
 			this.messageList			= options.extraArgs.messageList;
 			this.messageIndex			= options.extraArgs.messageIndex;
+			this.lastMessageNextExit	= options.extraArgs.lastMessageNextExit;
 		}
 
 		this.messageList	= this.messageList || [];
@@ -41,6 +42,12 @@ exports.getModule = class AreaViewFSEModule extends FullScreenEditorModule {
 					return self.loadMessageByUuid(self.messageList[self.messageIndex].messageUuid, cb);
 				}
 
+				//	auto-exit if no more to go?
+				if(self.lastMessageNextExit) {
+					self.lastMessageReached = true;
+					return self.prevMenu(cb);
+				}
+
 				return cb(null);
 			},
 
@@ -120,6 +127,9 @@ exports.getModule = class AreaViewFSEModule extends FullScreenEditorModule {
 	}
 
 	getMenuResult() {
-		return this.messageIndex;
+		return {
+			messageIndex 		: this.messageIndex,
+			lastMessageReached	: this.lastMessageReached,
+		};
 	}
 };

mods/msg_list.js

@@ -49,6 +49,8 @@ exports.getModule = class MessageListModule extends MessageAreaConfTempSwitcher(
 
 		this.messageAreaTag = config.messageAreaTag;
 
+		this.lastMessageReachedExit = _.get(options, 'lastMenuResult.lastMessageReached', false);
+
 		if(options.extraArgs) {
 			//
 			//	|extraArgs| can override |messageAreaTag| provided by config
@@ -73,6 +75,7 @@ exports.getModule = class MessageListModule extends MessageAreaConfTempSwitcher(
 							messageAreaTag		: self.messageAreaTag,
 							messageList			: self.messageList,
 							messageIndex		: formData.value.message,
+							lastMessageNextExit	: true,
 						}
 					};
 
@@ -107,6 +110,10 @@ exports.getModule = class MessageListModule extends MessageAreaConfTempSwitcher(
 	}
 
 	enter() {
+		if(this.lastMessageReachedExit) {
+			return this.prevMenu();
+		}
+
 		super.enter();
 
 		//

mods/themes/luciano_blocktronics/theme.hjson

@@ -3,6 +3,7 @@
 		name: Mystery Skull
 		author: Luciano Ayres
 		group: blocktronics
+		enabled: true
 	}
 
 	customization: {
@@ -590,6 +591,124 @@
 				}
 			}
 
+			newScanFileBaseList: {
+				config: {
+					hashTagsSep: "|08, |07"
+					browseInfoFormat10: "|00|10{fileName} |08- |03{byteSize!sizeWithoutAbbr} |11{byteSize!sizeAbbr} |08- |03uploaded |11{uploadTimestamp}"
+					browseInfoFormat11: "|00|15{areaName}"				
+					browseInfoFormat12: "|00|07{hashTags}"
+					browseInfoFormat13: "|00|07{estReleaseYear}"
+					browseInfoFormat14: "|00|07{dlCount}"
+					browseInfoFormat15: "{userRatingString}"
+					browseInfoFormat16: "{isQueued}"
+					browseInfoFormat17: "{webDlLink}{webDlExpire}"
+
+					webDlExpireTimeFormat: " [|08- |07exp] ddd, MMM Do @ h:mm a"
+					webDlLinkNeedsGenerated: "|08(|07press |10W |07to generate link|08)"
+
+					isQueuedIndicator: "|00|10YES"
+					isNotQueuedIndicator: "|00|07no"
+					
+					userRatingTicked: "|00|15*"
+					userRatingUnticked: "|00|07-"
+
+					detailsGeneralInfoFormat10: "{fileName}"
+					detailsGeneralInfoFormat11: "|00|07{byteSize!sizeWithoutAbbr} |11{byteSize!sizeAbbr} |08(|03{byteSize:,} |11B|08)"
+					detailsGeneralInfoFormat12: "|00|07{hashTags}"
+					detailsGeneralInfoFormat13: "{estReleaseYear}"
+					detailsGeneralInfoFormat14: "{dlCount}"
+					detailsGeneralInfoFormat15: "{userRatingString}"
+					detailsGeneralInfoFormat16: "{fileCrc32}"
+					detailsGeneralInfoFormat17: "{fileMd5}"
+					detailsGeneralInfoFormat18: "{fileSha1}"
+					detailsGeneralInfoFormat19: "{fileSha256}"
+					detailsGeneralInfoFormat20: "{uploadByUsername}"
+					detailsGeneralInfoFormat21: "{uploadTimestamp}"
+					detailsGeneralInfoFormat22: "{archiveTypeDesc}"
+
+					fileListEntryFormat: "|00|03{fileName:<67.66} {byteSize!sizeWithoutAbbr:>7.6} |11{byteSize!sizeAbbr}"
+					focusFileListEntryFormat: "|00|19|15{fileName:<67.66} {byteSize!sizeWithoutAbbr:>7.6} {byteSize!sizeAbbr}"
+
+					notAnArchiveFormat: "|00|08( |07{fileName} is not an archive |08)"
+				}
+
+				0: {
+					mci: {
+						MT1: {
+							height: 16
+							width: 45
+						}
+						HM2: {
+							focusTextStyle: first lower
+						}
+
+						TL11: {
+							width: 21
+							textOverflow: ...
+						}
+
+						TL12: {
+							width: 21
+							textOverflow: ...
+						}
+						TL13: { width: 21 }
+						TL14: { width: 21 }
+						TL15: { width: 21 }
+						TL16: { width: 21 }						
+						TL17: { width: 73 }
+
+					}
+				}
+
+				1: {
+					mci: {
+						HM1: {
+							focusTextStyle: first lower
+						}
+					}
+				}
+
+				2: {
+
+				}
+
+				3: {
+					//	details - nfo/readme
+					mci: {
+						MT1: {
+							height: 19
+							width: 79
+						}
+					}
+				}
+
+				4: {
+					mci: {
+						VM1: {
+							height: 17
+							width: 79
+						}
+					}
+				}
+			}
+
+			fileBaseBrowseByAreaSelect: {
+				config: {
+					protListFormat: "|00|03{name}"
+					protListFocusFormat: "|00|19|15{name}"
+				}
+
+				0: {
+					mci: {
+						VM1: {
+							height: 15
+							width: 30
+							focusTextStyle: first lower
+						}
+					}
+				}
+			}			
+
 			fileBaseSearch: {
 				mci: {
 					ET1: {

mods/upload.js

@@ -15,7 +15,6 @@ const pathWithTerminatingSeparator		= require('../core/file_util.js').pathWithTe
 const Log								= require('../core/logger.js').log;
 const Errors							= require('../core/enig_error.js').Errors;
 const FileEntry							= require('../core/file_entry.js');
-const enigmaToAnsi						= require('../core/color_codes.js').enigmaToAnsi;
 
 //	deps
 const async								= require('async');

package.json

@@ -1,6 +1,6 @@
 {
   "name": "enigma-bbs",
-  "version": "0.0.6-alpha",
+  "version": "0.0.7-alpha",
   "description": "ENiGMA½ Bulletin Board System",
   "author": "Bryan Ashby <bryan@l33t.codes>",
   "license": "BSD-2-Clause",
@@ -39,13 +39,14 @@
     "ptyw.js": "NuSkooler/ptyw.js",
     "sanitize-filename": "^1.6.1",
     "sqlite3": "^3.1.1",
-    "ssh2": "^0.5.1",
+    "ssh2": "^0.5.5",
     "temptmp": "^1.0.0",
     "uuid": "^3.0.1",
     "uuid-parse": "^1.0.0",
     "ws" : "^3.0.0",
     "graceful-fs" : "^4.1.11",
-    "exiftool" : "^0.0.3"
+    "exiftool" : "^0.0.3",
+    "node-glob" : "^1.2.0"
   },
   "devDependencies": {},
   "engines": {

util/exiftool2desc.js

@@ -17,6 +17,7 @@ const FILETYPE_HANDLERS = {};
 [ 'AIFF', 'APE', 'FLAC', 'OGG', 'MP3' ].forEach(ext => FILETYPE_HANDLERS[ext] = audioFile);
 [ 'PDF', 'DOC', 'DOCX', 'DOCM', 'ODB', 'ODC', 'ODF', 'ODG', 'ODI', 'ODP', 'ODS', 'ODT' ].forEach(ext => FILETYPE_HANDLERS[ext] = documentFile);
 [ 'PNG', 'JPEG', 'GIF', 'WEBP', 'XCF' ].forEach(ext => FILETYPE_HANDLERS[ext] = imageFile);
+[ 'MP4', 'MOV', 'AVI', 'MKV', 'MPG', 'MPEG', 'M4V', 'WMV' ].forEach(ext => FILETYPE_HANDLERS[ext] = videoFile);
 
 function audioFile(metadata) {
 	//	nothing if we don't know at least the author or title
@@ -32,6 +33,10 @@ function audioFile(metadata) {
 	return desc;
 }
 
+function videoFile(metadata) {
+	return `${metadata.fileType} video(${metadata.imageSize}px, ${metadata.duration}, ${metadata.audioBitsPerSample}/${metadata.audioSampleRate} audio)`;
+}
+
 function documentFile(metadata) {
 	//	nothing if we don't know at least the author or title
 	if(!metadata.author && !metadata.title) {