* Improvements to ANSI parser

* Introduction of storage tags for file bases / areas
* Expiration for file web server items
* WIP work on clean ANSI (on hold for a bit while other file base stuff is worked on)

core/acs.js

@@ -25,6 +25,9 @@ class ACS {
 		}		
 	}
 
+	//
+	//	Message Conferences & Areas
+	//
 	hasMessageConfRead(conf) {
 		return this.check(conf.acs, 'read', ACS.Defaults.MessageConfRead);
 	}
@@ -33,10 +36,17 @@ class ACS {
 		return this.check(area.acs, 'read', ACS.Defaults.MessageAreaRead);
 	}
 
+	//
+	//	File Base / Areas
+	//
 	hasFileAreaRead(area) {
 		return this.check(area.acs, 'read', ACS.Defaults.FileAreaRead);
 	}
 
+	hasFileAreaDownload(area) {
+		return this.check(area.acs, 'download', ACS.Defaults.FileAreaDownload);
+	}
+
 	getConditionalValue(condArray, memberName) {
 		assert(_.isArray(condArray));
 		assert(_.isString(memberName));
@@ -65,6 +75,7 @@ ACS.Defaults = {
 	MessageConfRead		: 'GM[users]',
 
 	FileAreaRead		: 'GM[users]',
+	FileAreaDownload	: 'GM[users]',
 };
 
 module.exports = ACS;

core/ansi_escape_parser.js

@@ -48,8 +48,10 @@ function ANSIEscapeParser(options) {
 		self.row	= Math.max(self.row, 1);		
 		self.row	= Math.min(self.row, self.termHeight);
 
-		self.emit('move cursor', self.column, self.row);
+//		self.emit('move cursor', self.column, self.row);
-		self.rowUpdated();
+		
+		self.positionUpdated();
+		//self.rowUpdated();
 	};
 
 	self.saveCursorPosition = function() {
@@ -63,7 +65,9 @@ function ANSIEscapeParser(options) {
 		self.row	= self.savedPosition.row;
 		self.column	= self.savedPosition.column;
 		delete self.savedPosition;
-		self.rowUpdated();
+
+		self.positionUpdated();
+//		self.rowUpdated();
 	};
 
 	self.clearScreen = function() {
@@ -71,11 +75,76 @@ function ANSIEscapeParser(options) {
 		self.emit('clear screen');
 	};
 
+/*
 	self.rowUpdated = function() {
 		self.emit('row update', self.row + self.scrollBack);
+	};*/
+
+	self.positionUpdated = function() {
+		self.emit('position update', self.row, self.column);
 	};
 
 	function literal(text) {
+		let charCode;		
+		let pos;
+		let start	= 0;
+		const len	= text.length;
+
+		function emitLiteral() {
+			self.emit('literal', text.slice(start, pos));
+			start = pos;
+		}
+
+		for(pos = 0; pos < len; ++pos) {
+			charCode = text.charCodeAt(pos) & 0xff;	//	ensure 8bit clean
+
+			switch(charCode) {
+				case CR : 
+					emitLiteral();
+										
+					self.column = 1;
+
+					self.positionUpdated();
+					break;
+
+				case LF :
+					emitLiteral();
+
+					self.row += 1;
+
+					self.positionUpdated();
+					break;
+
+				default :
+					if(self.column > self.termWidth) {
+						//
+						//	Emit data up to this point so it can be drawn before the postion update
+						//
+						emitLiteral();
+
+						self.column = 1;
+						self.row	+= 1;
+						
+						self.positionUpdated();
+
+						
+					} else {
+						self.column += 1;
+					}
+					break;
+			}
+		}
+
+		self.emit('literal', text.slice(start));
+
+		if(self.column > self.termWidth) {
+			self.column = 1;
+			self.row	+= 1;
+			self.positionUpdated();
+		}
+	}
+
+	function literal2(text) {
 		var charCode;
 
 		var len = text.length;
@@ -88,29 +157,31 @@ function ANSIEscapeParser(options) {
 
 				case LF : 
 					self.row++;
-					self.rowUpdated();		
+					self.positionUpdated();
+					//self.rowUpdated();		
 					break;
 
 				default :
 					//	wrap
-					if(self.column === self.termWidth) {
+					if(self.column > self.termWidth) {
 						self.column = 1;
 						self.row++;
-						self.rowUpdated();
+						//self.rowUpdated();
+						self.positionUpdated();
 					} else {
-						self.column++;
+						self.column += 1;
 					}
 					break;
 			}
 
-			if(self.row === 26) {	//	:TODO: should be termHeight + 1 ?
+			if(self.row === self.termHeight) {
-				self.scrollBack++;
+				self.scrollBack		+= 1;
-				self.row--;
+				self.row			-= 1;
-				self.rowUpdated();
+
+				self.positionUpdated();
 			}
 		}
 
-		//self.emit('chunk', text);
 		self.emit('literal', text);
 	}
 
@@ -188,10 +259,10 @@ function ANSIEscapeParser(options) {
 		}
 	}
 
-	self.reset = function(buffer) {
+	self.reset = function(input) {
 		self.parseState = {
 			//	ignore anything past EOF marker, if any
-			buffer	: buffer.split(String.fromCharCode(0x1a), 1)[0],
+			buffer	: input.split(String.fromCharCode(0x1a), 1)[0],
 			re		: /(?:\x1b\x5b)([\?=;0-9]*?)([ABCDHJKfhlmnpsu])/g,
 			stop	: false,
 		};
@@ -201,7 +272,11 @@ function ANSIEscapeParser(options) {
 		self.parseState.stop = true;
 	};
 
-	self.parse = function() {
+	self.parse = function(input) {
+		if(input) {
+			self.reset(input);
+		}
+
 		//	:TODO: ensure this conforms to ANSI-BBS / CTerm / bansi.txt for movement/etc.
 		var pos;
 		var match;
@@ -308,40 +383,45 @@ function ANSIEscapeParser(options) {
 	*/
 
 	function escape(opCode, args) {
-		var arg;
+		let arg;
-		var i;
-		var len;
 
 		switch(opCode) {
 			//	cursor up
 			case 'A' :
-				arg = args[0] || 1;
+				//arg = args[0] || 1;
+				arg = isNaN(args[0]) ? 1 : args[0];
 				self.moveCursor(0, -arg);
 				break;
 
 			//	cursor down
 			case 'B' :
-				arg = args[0] || 1;
+				//arg = args[0] || 1;
+				arg = isNaN(args[0]) ? 1 : args[0];
 				self.moveCursor(0, arg);
 				break;
 
 			//	cursor forward/right
 			case 'C' :
-				arg = args[0] || 1;
+				//arg = args[0] || 1;
+				arg = isNaN(args[0]) ? 1 : args[0];
 				self.moveCursor(arg, 0);
 				break;
 
 			//	cursor back/left
 			case 'D' :
-				arg = args[0] || 1;
+				//arg = args[0] || 1;
+				arg = isNaN(args[0]) ? 1 : args[0];
 				self.moveCursor(-arg, 0);
 				break;
 
 			case 'f' :	//	horiz & vertical
 			case 'H' :	//	cursor position
-				self.row	= args[0] || 1;
+				//self.row	= args[0] || 1;
-				self.column	= args[1] || 1;
+				//self.column	= args[1] || 1;
-				self.rowUpdated();
+				self.row	= isNaN(args[0]) ? 1 : args[0];
+				self.column	= isNaN(args[1]) ? 1 : args[1];
+				//self.rowUpdated();
+				self.positionUpdated();
 				break;
 
 			//	save position
@@ -356,7 +436,7 @@ function ANSIEscapeParser(options) {
 
 			//	set graphic rendition
 			case 'm' :
-				for(i = 0, len = args.length; i < len; ++i) {
+				for(let i = 0, len = args.length; i < len; ++i) {
 					arg = args[i];
 
 					if(ANSIEscapeParser.foregroundColors[arg]) {
@@ -410,12 +490,13 @@ function ANSIEscapeParser(options) {
 						}
 					}
 				}
+				break;	//	m
 
-				break;
+			//	:TODO: s, u, K
 
 			//	erase display/screen
 			case 'J' :
-				//	:TODO: Handle others
+				//	:TODO: Handle other 'J' types!
 				if(2 === args[0]) {
 					self.clearScreen();
 				}

core/conf_area_util.js

@@ -16,8 +16,8 @@ function sortAreasOrConfs(areasOrConfs, type) {
 	let entryB;
 
 	areasOrConfs.sort((a, b) => {
-		entryA = a[type];
+		entryA = type ? a[type] : a;
-		entryB = b[type];
+		entryB = type ? b[type] : b;
 
 		if(_.isNumber(entryA.sort) && _.isNumber(entryB.sort)) {
 			return entryA.sort - entryB.sort;

core/config.js

@@ -361,17 +361,21 @@ function getDefaultConfig() {
 
 			fileNamePatterns: {
 				//	These are NOT case sensitive
+				//	FILE_ID.DIZ - https://en.wikipedia.org/wiki/FILE_ID.DIZ
 				shortDesc		: [ 
 					'^FILE_ID\.DIZ$', '^DESC\.SDI$', '^DESCRIPT\.ION$', '^FILE\.DES$', '$FILE\.SDI$', '^DISK\.ID$'
 				],
 
-				longDesc		: [ '^.*\.NFO$', '^README\.1ST$', '^README\.TXT$' ],
+				//	common README filename - https://en.wikipedia.org/wiki/README
+				longDesc		: [ 
+					'^.*\.NFO$', '^README\.1ST$', '^README\.TXT$', '^READ\.ME$', '^README$', '^README\.md$'
+				],
 			},
 
 			yearEstPatterns: [
 				//
-				//	Patterns should produce the year in the first submatch
+				//	Patterns should produce the year in the first submatch.
-				//	The year may be YY or YYYY
+				//	The extracted year may be YY or YYYY
 				//
 				'[0-3]?[0-9][\\-\\/\\.][0-3]?[0-9][\\-\\/\\.]((?:[0-9]{2})?[0-9]{2})',	//	m/d/yyyy, mm-dd-yyyy, etc.
 				"\\B('[1789][0-9])\\b",	//	eslint-disable-line quotes
@@ -385,17 +389,25 @@ function getDefaultConfig() {
 				expireMinutes	: 1440,	//	1 day
 			},
 
+			//
+			//	File area storage location tag/value pairs.
+			//	Non-absolute paths are relative to |areaStoragePrefix|.
+			// 
+			storageTags : {
+				sys_msg_attach	: 'msg_attach',
+			},
+
 			areas: {
-				message_attachment : {
+				systemm_message_attachment : {
-					name	: 'Message attachments',
+					name		: 'Message attachments',
-					desc	: 'File attachments to messages',
+					desc		: 'File attachments to messages',
+					storageTags	: 'sys_msg_attach',	//	may be string or array of strings
 				}
 			}
 		},
 		
 		eventScheduler : {
 			
-			
 			events : {
 				trimMessageAreas : {
 					//	may optionally use [or ]@watch:/path/to/file

core/database.js

@@ -262,6 +262,7 @@ const DB_INIT_TABLE = {
 				area_tag			VARCHAR NOT NULL,
 				file_sha1			VARCHAR NOT NULL,
 				file_name,			/* FTS @ file_fts */
+				storage_tag			VARCHAR NOT NULL,
 				desc,				/* FTS @ file_fts */
 				desc_long,			/* FTS @ file_fts */				
 				upload_timestamp	DATETIME NOT NULL

core/download_queue.js

@@ -7,7 +7,9 @@ module.exports = class DownloadQueue {
 	constructor(client) {
 		this.client	= client;
 
-		this.loadFromProperty(client);
+		if(!Array.isArray(this.client.user.downloadQueue)) {
+			this.loadFromProperty(client);
+		}
 	}
 
 	toggle(fileEntry) {

core/file_area.js

@@ -37,7 +37,8 @@ function getAvailableFileAreas(client, options) {
 	options = options || { includeSystemInternal : false };
 
 	//	perform ACS check per conf & omit system_internal if desired
-	return _.omit(Config.fileAreas.areas, (area, areaTag) => {        
+	const areasWithTags = _.map(Config.fileBase.areas, (area, areaTag) => Object.assign(area, { areaTag : areaTag } ) );
+	return _.omit(Config.fileBase.areas, (area, areaTag) => {        
 		if(!options.includeSystemInternal && WellKnownAreaTags.MessageAreaAttach === areaTag) {
 			return true;
 		}
@@ -48,21 +49,21 @@ function getAvailableFileAreas(client, options) {
 
 function getSortedAvailableFileAreas(client, options) {
 	const areas = _.map(getAvailableFileAreas(client, options), v => v);
-	sortAreasOrConfs(areas, 'area');
+	sortAreasOrConfs(areas);
 	return areas;
 }
 
 function getDefaultFileAreaTag(client, disableAcsCheck) {
-	let defaultArea = _.findKey(Config.fileAreas, o => o.default);
+	let defaultArea = _.findKey(Config.fileBase, o => o.default);
 	if(defaultArea) {
-		const area = Config.fileAreas.areas[defaultArea];
+		const area = Config.fileBase.areas[defaultArea];
 		if(true === disableAcsCheck || client.acs.hasFileAreaRead(area)) {
 			return defaultArea;
 		}
 	}
 
 	//  just use anything we can
-	defaultArea = _.findKey(Config.fileAreas.areas, (area, areaTag) => {
+	defaultArea = _.findKey(Config.fileBase.areas, (area, areaTag) => {
 		return WellKnownAreaTags.MessageAreaAttach !== areaTag && (true === disableAcsCheck || client.acs.hasFileAreaRead(area));
 	});
     
@@ -70,10 +71,10 @@ function getDefaultFileAreaTag(client, disableAcsCheck) {
 }
 
 function getFileAreaByTag(areaTag) {
-	const areaInfo = Config.fileAreas.areas[areaTag];
+	const areaInfo = Config.fileBase.areas[areaTag];
 	if(areaInfo) {
-		areaInfo.areaTag			= areaTag;	//	convienence!
+		areaInfo.areaTag	= areaTag;	//	convienence!
-		areaInfo.storageDirectory	= getAreaStorageDirectory(areaInfo);
+		areaInfo.storage	= getAreaStorageLocations(areaInfo); 
 		return areaInfo;
 	}
 }
@@ -113,8 +114,38 @@ function changeFileAreaWithOptions(client, areaTag, options, cb) {
 	);
 }
 
-function getAreaStorageDirectory(areaInfo) {
+function getAreaStorageDirectoryByTag(storageTag) {
-	return paths.join(Config.fileBase.areaStoragePrefix, areaInfo.storageDir || '');
+	const storageLocation = (storageTag && Config.fileBase.storageTags[storageTag]);
+
+	return paths.resolve(Config.fileBase.areaStoragePrefix, storageLocation || '');
+	
+	/*
+	//	absolute paths as-is
+	if(storageLocation && '/' === storageLocation.charAt(0)) {
+		return storageLocation;		
+	}
+
+	//	relative to |areaStoragePrefix|
+	return paths.join(Config.fileBase.areaStoragePrefix, storageLocation || '');
+	*/
+}
+
+function getAreaStorageLocations(areaInfo) {
+	
+	const storageTags = Array.isArray(areaInfo.storageTags) ? 
+		areaInfo.storageTags : 
+		[ areaInfo.storageTags || '' ];
+
+	const avail = Config.fileBase.storageTags;
+	
+	return _.compact(storageTags.map(storageTag => {
+		if(avail[storageTag]) {
+			return {
+				storageTag	: storageTag,
+				dir			: getAreaStorageDirectoryByTag(storageTag),
+			};
+		}
+	}));
 }
 
 function getFileEntryPath(fileEntry) {
@@ -342,29 +373,28 @@ function updateFileEntry(fileEntry, filePath, cb) {
 
 }
 
-function addOrUpdateFileEntry(areaInfo, fileName, options, cb) {
+function addOrUpdateFileEntry(areaInfo, storageLocation, fileName, options, cb) {
 	
 	const fileEntry = new FileEntry({
 		areaTag		: areaInfo.areaTag,
 		meta		: options.meta,
 		hashTags	: options.hashTags,	//	Set() or Array
 		fileName	: fileName,
+		storageTag	: storageLocation.storageTag,
 	});
 
-	const filePath	= paths.join(getAreaStorageDirectory(areaInfo), fileName);
+	const filePath	= paths.join(storageLocation.dir, fileName);
 
 	async.waterfall(
 		[
 			function processPhysicalFile(callback) {			
-				const stream = fs.createReadStream(filePath);
-
 				let byteSize	= 0;
 				const sha1		= crypto.createHash('sha1');
 				const sha256	= crypto.createHash('sha256');
 				const md5		= crypto.createHash('md5');
 				const crc32		= new CRC32();
 								
-				//	:TODO: crc32
+				const stream = fs.createReadStream(filePath);
 
 				stream.on('data', data => {
 					byteSize += data.length;
@@ -413,6 +443,58 @@ function addOrUpdateFileEntry(areaInfo, fileName, options, cb) {
 }
 
 function scanFileAreaForChanges(areaInfo, cb) {
+	const storageLocations = getAreaStorageLocations(areaInfo);
+
+	async.eachSeries(storageLocations, (storageLoc, nextLocation) => {
+		async.series(
+			[
+				function scanPhysFiles(callback) {
+					const physDir = storageLoc.dir;
+
+					fs.readdir(physDir, (err, files) => {
+						if(err) {
+							return callback(err);
+						}
+
+						async.eachSeries(files, (fileName, nextFile) => {
+							const fullPath = paths.join(physDir, fileName);
+
+							fs.stat(fullPath, (err, stats) => {
+								if(err) {
+									//	:TODO: Log me!
+									return nextFile(null);	//	always try next file
+								}
+
+								if(!stats.isFile()) {
+									return nextFile(null);
+								}
+
+								addOrUpdateFileEntry(areaInfo, storageLoc, fileName, { }, err => {
+									return nextFile(err);
+								});
+							});
+						}, err => {
+							return callback(err);
+						});
+					});
+				},
+				function scanDbEntries(callback) {
+					//	:TODO: Look @ db entries for area that were *not* processed above
+					return callback(null);
+				}
+			], 
+			err => {
+				return nextLocation(err);
+			}
+		);
+	}, 
+	err => {
+		return cb(err);
+	});
+}
+
+/*
+function scanFileAreaForChanges2(areaInfo, cb) {
 	const areaPhysDir = getAreaStorageDirectory(areaInfo);
 
 	async.series(
@@ -454,4 +536,5 @@ function scanFileAreaForChanges(areaInfo, cb) {
 			return cb(err);
 		}
 	);
-}
+}
+*/

core/file_area_web.js

@@ -28,7 +28,8 @@ const WEB_SERVER_PACKAGE_NAME	 = 'codes.l33t.enigma.web.server';
 
 class FileAreaWebAccess {
 	constructor() {
-		this.hashids	= new hashids(Config.general.boardName);
+		this.hashids		= new hashids(Config.general.boardName);
+		this.expireTimers	= {};	//	hashId->timer
 	}
 
 	startup(cb) {
@@ -37,8 +38,7 @@ class FileAreaWebAccess {
 		async.series(
 			[
 				function initFromDb(callback) {
-					//	:TODO: Init from DB & register expiration timers
+					return self.load(callback);
-					return callback(null);
 				},
 				function addWebRoute(callback) {
 					const webServer = getServer(WEB_SERVER_PACKAGE_NAME);
@@ -66,7 +66,56 @@ class FileAreaWebAccess {
 	}
 
 	load(cb) {
-		return cb(null);	//	:TODO: Load from db
+		//
+		//	Load entries, register expiration timers
+		//
+		FileDb.each(
+			`SELECT hash_id, expire_timestamp
+			FROM file_web_serve;`,
+			(err, row) => {
+				if(row) {
+					this.scheduleExpire(row.hash_id, moment(row.expire_timestamp));
+				}
+			},
+			err => {
+				return cb(err);
+			}
+		);
+	}
+
+	removeEntry(hashId) {
+		//
+		//	Delete record from DB, and our timer
+		//
+		FileDb.run(
+			`DELETE FROM file_web_serve
+			WHERE hash_id = ?;`,
+			[ hashId ]
+		);
+
+		delete this.expireTime[hashId];
+	}
+
+	scheduleExpire(hashId, expireTime) {
+
+		//	remove any previous entry for this hashId
+		const previous = this.expireTimers[hashId];
+		if(previous) {
+			clearTimeout(previous);
+			delete this.expireTimers[hashId];
+		}
+
+		const timeoutMs = expireTime.diff(moment());
+
+		if(timeoutMs <= 0) {
+			setImmediate( () => {
+				this.removeEntry(hashId);
+			});
+		} else {
+			this.expireTimers[hashId] = setTimeout( () => {
+				this.removeEntry(hashId);
+			}, timeoutMs);
+		}
 	}
 
 	loadServedHashId(hashId, cb) {
@@ -111,12 +160,9 @@ class FileAreaWebAccess {
 		//
 		//	Create a URL such as
 		//	https://l33t.codes:44512/f/qFdxyZr
-		//
-		//	:TODO: build from config
-
 		//
 		//	Prefer HTTPS over HTTP. Be explicit about the port
-		//	only if required.
+		//	only if non-standard.
 		//		
 		let schema;
 		let port;
@@ -163,8 +209,8 @@ class FileAreaWebAccess {
 					return cb(err);
 				}
 
-				//	:TODO: setup tracking of expiration time so we can clean up the entry 
+				this.scheduleExpire(hashId, options.expireTime);
-
+				
 				return cb(null, url);
 			}
 		);
@@ -173,7 +219,7 @@ class FileAreaWebAccess {
 	fileNotFound(resp) {
 		resp.writeHead(404, { 'Content-Type' : 'text/html' } );
 
-		//	:TODO: allow custom 404
+		//	:TODO: allow custom 404 - mods/<theme>/file_area_web-404.html
 		return resp.end('<html><body>Not found</html>');
 	}
 

core/file_entry.js

@@ -12,7 +12,7 @@ const _						= require('lodash');
 const paths					= require('path');
 
 const FILE_TABLE_MEMBERS	= [ 
-	'file_id', 'area_tag', 'file_sha1', 'file_name', 
+	'file_id', 'area_tag', 'file_sha1', 'file_name', 'storage_tag',
 	'desc', 'desc_long', 'upload_timestamp' 
 ];
 
@@ -44,6 +44,7 @@ module.exports = class FileEntry {
 
 		this.hashTags	= options.hashTags || new Set();
 		this.fileName	= options.fileName;
+		this.storageTag	= options.storageTag;
 	}
 
 	load(fileId, cb) {
@@ -99,9 +100,9 @@ module.exports = class FileEntry {
 				},
 				function storeEntry(callback) {
 					fileDb.run(
-						`REPLACE INTO file (area_tag, file_sha1, file_name, desc, desc_long, upload_timestamp)
+						`REPLACE INTO file (area_tag, file_sha1, file_name, storage_tag, desc, desc_long, upload_timestamp)
-						VALUES(?, ?, ?, ?, ?, ?);`,
+						VALUES(?, ?, ?, ?, ?, ?, ?);`,
-						[ self.areaTag, self.fileSha1, self.fileName, self.desc, self.descLong, getISOTimestampString() ],
+						[ self.areaTag, self.fileSha1, self.fileName, self.storageTag, self.desc, self.descLong, getISOTimestampString() ],
 						function inserted(err) {	//	use non-arrow func for 'this' scope / lastID
 							if(!err) {
 								self.fileId = this.lastID;
@@ -132,15 +133,21 @@ module.exports = class FileEntry {
 		);
 	}
 
-	get filePath() {
+	static getAreaStorageDirectoryByTag(storageTag) {
-		const areaInfo = Config.fileAreas.areas[this.areaTag];
+		const storageLocation = (storageTag && Config.fileBase.storageTags[storageTag]);
-		if(areaInfo) {
+	
-			return paths.join(
+		//	absolute paths as-is
-				Config.fileBase.areaStoragePrefix, 
+		if(storageLocation && '/' === storageLocation.charAt(0)) {
-				areaInfo.storageDir || '',
+			return storageLocation;		
-				this.fileName
-			);
 		}
+
+		//	relative to |areaStoragePrefix|
+		return paths.join(Config.fileBase.areaStoragePrefix, storageLocation || '');
+	}
+
+	get filePath() {
+		const storageDir = FileEntry.getAreaStorageDirectoryByTag(this.storageTag);
+		return paths.join(storageDir, this.fileName);
 	}
 
 	static persistMetaValue(fileId, name, value, cb) {
@@ -193,7 +200,7 @@ module.exports = class FileEntry {
 
 	static getWellKnownMetaValues() { return Object.keys(FILE_WELL_KNOWN_META); }
 
-	static findFiles(criteria, cb) {
+	static findFiles(filter, cb) {
 		//	:TODO: build search here - return [ fileid1, fileid2, ... ]
 		//	free form
 		//	areaTag
@@ -201,6 +208,8 @@ module.exports = class FileEntry {
 		//	order by
 		//	sort
 
+		filter = filter || {};
+
 		let sql = 
 			`SELECT file_id
 			FROM file`;
@@ -216,21 +225,23 @@ module.exports = class FileEntry {
 			sqlWhere += clause;
 		}
 
-		if(criteria.areaTag) {
+		if(filter.areaTag) {
-			appendWhereClause(`area_tag="${criteria.areaTag}"`);
+			appendWhereClause(`area_tag="${filter.areaTag}"`);
 		}
 
-		if(criteria.search) {
+		if(filter.terms) {
 			appendWhereClause(
 				`file_id IN (
 					SELECT rowid
 					FROM file_fts
-					WHERE file_fts MATCH "${criteria.search.replace(/"/g,'""')}"
+					WHERE file_fts MATCH "${filter.terms.replace(/"/g,'""')}"
 				)`
 			);
 		}
 		
-		if(Array.isArray(criteria.hashTags)) {
+		if(filter.tags) {
+			const tags = filter.tags.split(' ');	//	filter stores as sep separated values
+
 			appendWhereClause(
 				`file_id IN (
 					SELECT file_id
@@ -238,14 +249,14 @@ module.exports = class FileEntry {
 					WHERE hash_tag_id IN (
 						SELECT hash_tag_id
 						FROM hash_tag
-						WHERE hash_tag IN (${criteria.hashTags.join(',')})
+						WHERE hash_tag IN (${tags.join(',')})
 					)
 				)`
 			);
 		}
 
-		//	:TODO: criteria.orderBy
+		//	:TODO: filter.orderBy
-		//	:TODO: criteria.sort
+		//	:TODO: filter.sort
 
 		sql += sqlWhere + ';';
 		const matchingFileIds = [];

core/sauce.js

@@ -50,20 +50,17 @@ function readSAUCE(data, cb) {
 		.tap(function onVars(vars) {
 
 			if(!SAUCE_ID.equals(vars.id)) {
-				cb(new Error('No SAUCE record present'));
+				return cb(new Error('No SAUCE record present'));
-				return;
 			}	
 
 			var ver = iconv.decode(vars.version, 'cp437');
 
 			if('00' !== ver) {
-				cb(new Error('Unsupported SAUCE version: ' + ver));
+				return cb(new Error('Unsupported SAUCE version: ' + ver));
-				return;
 			}
 
 			if(-1 === SAUCE_VALID_DATA_TYPES.indexOf(vars.dataType)) {
-				cb(new Error('Unsupported SAUCE DataType: ' + vars.dataType));
+				return cb(new Error('Unsupported SAUCE DataType: ' + vars.dataType));
-				return;
 			}
 
 			var sauce = {

core/scanner_tossers/ftn_bso.js

@@ -1098,6 +1098,8 @@ function FTNMessageScanTossModule() {
 										
 							return nextFile();	//	unknown archive type
 						}
+
+						Log.debug( { bundleFile : bundleFile }, 'Processing bundle' );
 						
 						self.archUtil.extractTo(
 							bundleFile.path,

core/string_util.js

@@ -2,8 +2,12 @@
 'use strict';
 
 //	ENiGMA½
-const miscUtil					= require('./misc_util.js');
+const miscUtil			= require('./misc_util.js');
-const iconv						= require('iconv-lite');
+const ANSIEscapeParser	= require('./ansi_escape_parser.js').ANSIEscapeParser;
+const ANSI				= require('./ansi_term.js');
+
+//	deps
+const iconv				= require('iconv-lite');
 
 exports.stylizeString				= stylizeString;
 exports.pad							= pad;
@@ -16,6 +20,7 @@ exports.renderStringLength			= renderStringLength;
 exports.formatByteSizeAbbr			= formatByteSizeAbbr;
 exports.formatByteSize				= formatByteSize;
 exports.cleanControlCodes			= cleanControlCodes;
+exports.createCleanAnsi				= createCleanAnsi;
 
 //	:TODO: create Unicode verison of this
 const VOWELS = [ 'a', 'e', 'i', 'o', 'u' ];
@@ -310,15 +315,23 @@ function formatByteSize(byteSize, withAbbr, decimals) {
 //const REGEXP_ANSI_CONTROL_CODES		= /(\x1b\x5b)([\?=;0-9]*?)([0-9A-ORZcf-npsu=><])/g;
 const REGEXP_ANSI_CONTROL_CODES		= /(?:\x1b\x5b)([\?=;0-9]*?)([A-ORZcf-npsu=><])/g;
 const ANSI_OPCODES_ALLOWED_CLEAN	= [
-	'C', 'm' , 
+	'A', 'B',	//	up, down
-	'A', 'B', 'D'
+	'C', 'D',	//	right, left
+	'm',		//	color
 ];
 
-function cleanControlCodes(input) {
+const AnsiSpecialOpCodes = {
+	positioning	: [ 'A', 'B', 'C', 'D' ],	//	up, down, right, left
+	style		: [ 'm' ]					//	color
+};
+
+function cleanControlCodes(input, options) {
 	let m;
 	let pos;
 	let cleaned = '';
 	
+	options = options || {};
+		
 	//
 	//	Loop through |input| adding only allowed ESC
 	//	sequences and literals to |cleaned|
@@ -332,6 +345,10 @@ function cleanControlCodes(input) {
 				cleaned += input.slice(pos, m.index);
 			}
 
+			if(options.all) {
+				continue;
+			}
+
 			if(ANSI_OPCODES_ALLOWED_CLEAN.indexOf(m[2].charAt(0)) > -1) {
 				cleaned += m[0];
 			}
@@ -347,61 +364,141 @@ function cleanControlCodes(input) {
 	return cleaned;
 }
 
-function getCleanAnsi(input) {
+function createCleanAnsi(input, options, cb) {
-	//
-	//	Process |input| and produce |cleaned|, an array
-	//	of lines with "clean" ANSI.
-	//
-	//	Clean ANSI:
-	//	* Contains only color/SGR sequences
-	//	* All movement (up/down/left/right) removed but positioning
-	//	  left intact via spaces/etc.
-	//
-	//	Temporary processing will occur in a grid. Each cell
-	//	containing a character (defaulting to space) possibly a SGR
-	//
-	
-	let m;
-	let pos;
-	let grid = [];
-	let gridPos = { row : 0, col : 0 };
-	
-	function updateGrid(data, dataType) {
-		//	
-		//	Start at to grid[row][col] and populate val[0]...val[N]
-		//	creating cells as necessary
-		//
-		if(!grid[gridPos.row]) {
-			grid[gridPos.row] = [];
-		}
-		
-		if('literal' === dataType) {
-			data.forEach(c => {
-				grid[gridPos.row][gridPos.col] = (grid[gridPos.row][gridPos.col] || '') + c;	//	append to existing SGR
-				gridPos.col++; 
-			});
-		} else if('sgr' === dataType) {
-			grid[gridPos.row][gridPos.col] = (grid[gridPos.row][gridPos.col] || '') + data;
-		}
-	}
-	
-	function literal(s) {
-		let charCode;
-		const len = s.length;
-		for(let i = 0; i < len; ++i) {
-			charCode = s.charCodeAt(i) & 0xff;
 
+	options.width	= options.width || 80;
+	options.height	= options.height || 25;
+	
+	const canvas = new Array(options.height);
+	for(let i = 0; i < options.height; ++i) {
+		canvas[i] = new Array(options.width);
+		for(let j = 0; j < options.width; ++j) {
+			canvas[i][j] = {};
 		}
 	}
-	
+
-	do {
+	const parserOpts = {
-		pos	= REGEXP_ANSI_CONTROL_CODES.lastIndex;
+		termHeight	: options.height,
-		m 	= REGEXP_ANSI_CONTROL_CODES.exec(input);
+		termWidth	: options.width,
-		
+	};
-		if(null !== m) {
+
-			if(m.index > pos) {
+	const parser = new ANSIEscapeParser(parserOpts);
-				updateGrid(input.slice(pos, m.index), 'literal');
+
+	const canvasPos = {
+		col : 0,
+		row : 0,
+	};
+
+	let sgr;
+
+	function ensureCell() {
+		//	we've pre-allocated a matrix, but allow for > supplied dimens up front. They will be trimmed @ finalize
+		if(!canvas[canvasPos.row]) {
+			canvas[canvasPos.row] = new Array(options.width);
+			for(let j = 0; j < options.width; ++j) {
+				canvas[canvasPos.row][j] = {};
 			}
 		}
-	} while(0 !== REGEXP_ANSI_CONTROL_CODES.lastIndex);
+		canvas[canvasPos.row][canvasPos.col]	= canvas[canvasPos.row][canvasPos.col] || {};
+		//canvas[canvasPos.row][0].width 			= Math.max(canvas[canvasPos.row][0].width || 0, canvasPos.col); 
+	}
+
+	parser.on('literal', literal => {
+		//
+		//	CR/LF are handled for 'position update'; we don't need the chars themselves
+		//
+		literal = literal.replace(/\r?\n|[\r\u2028\u2029]/g, '');
+
+		for(let i = 0; i < literal.length; ++i) {
+			const c = literal.charAt(i);
+
+			ensureCell();
+
+			canvas[canvasPos.row][canvasPos.col].char	= c;
+						
+			if(sgr) {
+				canvas[canvasPos.row][canvasPos.col].sgr = sgr;
+				sgr = null;
+			}
+
+			canvasPos.col += 1;
+		}
+	});
+
+	parser.on('control', (match, opCode) => {
+		if('m' !== opCode) {
+			return;	//	don't care'
+		}
+		sgr = match;
+	});
+
+	parser.on('position update', (row, col) => {
+		canvasPos.row	= row - 1;
+		canvasPos.col	= Math.min(col - 1, options.width);
+	});
+
+	parser.on('complete', () => {
+		for(let row = 0; row < options.height; ++row) {
+			let col = 0;
+
+			//while(col <= canvas[row][0].width) {
+			while(col < options.width) {
+				if(!canvas[row][col].char) {
+					canvas[row][col].char = 'P';
+					if(!canvas[row][col].sgr) {
+						//	:TODO: fix duplicate SGR's in a row here - we just need one per sequence
+						canvas[row][col].sgr = ANSI.reset();
+					} 
+				}
+
+				col += 1;
+			}
+
+			//	:TODO: end *all* with CRLF - general usage will be width : 79 - prob update defaults
+
+			if(col <= options.width) {
+				canvas[row][col] = canvas[row][col] || {};
+
+				//canvas[row][col].char = '\r\n';
+				canvas[row][col].sgr = ANSI.reset();
+
+				//	:TODO: don't splice, just reset + fill with ' ' till end
+				for(let fillCol = col; fillCol <= options.width; ++fillCol) {
+					canvas[row][fillCol].char = 'X';
+				}
+				
+				//canvas[row] = canvas[row].splice(0, col + 1);
+				//canvas[row][options.width - 1].char = '\r\n';
+
+				
+			} else {
+				canvas[row] = canvas[row].splice(0, options.width + 1);				
+			}
+		
+		}
+
+		let out = '';
+		for(let row = 0; row < options.height; ++row) {
+			out += canvas[row].map( col => {
+				let c = col.sgr || '';
+				c += col.char;
+				return c;
+			}).join('');
+			
+		}
+
+		//	:TODO: finalize: @ any non-char cell, reset sgr & set to ' '
+		//	:TODO: finalize: after sgr established, omit anything > supplied dimens
+		return cb(out);
+	});
+
+	parser.parse(input);
 }
+
+const fs = require('fs');
+let data = fs.readFileSync('/home/nuskooler/Downloads/art3.ans');
+data = iconv.decode(data, 'cp437');
+createCleanAnsi(data, { width : 79, height : 25 }, (out) => {
+	out = iconv.encode(out, 'cp437');
+	fs.writeFileSync('/home/nuskooler/Downloads/art4.ans', out);
+});

core/user.js

@@ -325,6 +325,22 @@ User.prototype.persistProperty = function(propName, propValue, cb) {
 	);
 };
 
+User.prototype.removeProperty = function(propName, cb) {
+	//	update live
+	delete this.properties[propName];
+
+	userDb.run(
+		`DELETE FROM user_property
+		WHERE user_id = ? AND prop_name = ?;`,
+		[ this.userId, propName ],
+		err => {
+			if(cb) {
+				return cb(err);
+			}
+		}
+	)
+};
+
 User.prototype.persistProperties = function(properties, cb) {
 	var self = this;
 

mods/file_area_filter_edit.js

@@ -30,6 +30,7 @@ const MciViewIds = {
 
 		selectedFilterInfo	: 10,	//	{ ...filter object ... }
 		activeFilterInfo	: 11,	//	{ ...filter object ... }
+		error				: 12,	//	validation errors
 	}
 };
 
@@ -67,7 +68,6 @@ exports.getModule = class FileAreaFilterEdit extends MenuModule {
 
 				this.updateActiveLabel();
 
-				//	:TODO: Need to update %FN somehow
 				return cb(null);
 			},
 			newFilter : (formData, extraArgs, cb) => {
@@ -92,9 +92,8 @@ exports.getModule = class FileAreaFilterEdit extends MenuModule {
 						if(newActive) {
 							filters.setActive(newActive.uuid);
 						} else {
-							//	nothing to set active to
+							//	nothing to set active to						
-							//	:TODO: is this what we want?
+							this.client.user.removeProperty('file_base_filter_active_uuid');
-							this.client.user.properties.file_base_filter_active_uuid = 'none';
 						}
 					}
 
@@ -106,7 +105,23 @@ exports.getModule = class FileAreaFilterEdit extends MenuModule {
 					}
 					return cb(null);
 				});				
-			}
+			},
+
+			viewValidationListener : (err, cb) => {
+				const errorView = this.viewControllers.editor.getView(MciViewIds.editor.error);
+				let newFocusId;
+
+				if(errorView) {
+					if(err) {
+						errorView.setText(err.message);
+						err.view.clearText();	//	clear out the invalid data
+					} else {
+						errorView.clearText();
+					}
+				}
+
+				return cb(newFocusId);
+			},
 		};
 	}
 	

mods/file_area_list.js

@@ -8,12 +8,16 @@ const ansi				= require('../core/ansi_term.js');
 const theme				= require('../core/theme.js');
 const FileEntry			= require('../core/file_entry.js');
 const stringFormat		= require('../core/string_format.js');
+const createCleanAnsi	= require('../core/string_util.js').createCleanAnsi;
 const FileArea			= require('../core/file_area.js');
 const Errors			= require('../core/enig_error.js').Errors;
 const ArchiveUtil		= require('../core/archive_util.js');
 const Config			= require('../core/config.js').config;
 const DownloadQueue		= require('../core/download_queue.js');
 const FileAreaWeb		= require('../core/file_area_web.js');
+const FileBaseFilters	= require('../core/file_base_filter.js');
+
+const cleanControlCodes	= require('../core/string_util.js').cleanControlCodes;
 
 //	deps
 const async				= require('async');
@@ -328,15 +332,27 @@ exports.getModule = class FileAreaList extends MenuModule {
 				function populateViews(callback) {
 					if(_.isString(self.currentFileEntry.desc)) {
 						const descView = self.viewControllers.browse.getView(MciViewIds.browse.desc);
-						if(descView) {
+						if(descView) {					
-							descView.setText(self.currentFileEntry.desc);
+							createCleanAnsi(
+								self.currentFileEntry.desc, 
+								{ height : self.client.termHeight, width : descView.dimens.width },
+								cleanDesc => {
+									descView.setText(cleanDesc);
+
+									self.updateQueueIndicator();
+									self.populateCustomLabels('browse', 10);
+
+									return callback(null);
+								}
+							);	
+							descView.setText( self.currentFileEntry.desc );
 						}
+					} else {
+						self.updateQueueIndicator();
+						self.populateCustomLabels('browse', 10);
+
+						return callback(null);
 					}
-
-					self.updateQueueIndicator();
-					self.populateCustomLabels('browse', 10);
-
-					return callback(null);
 				}
 			],
 			err => {
@@ -442,20 +458,6 @@ exports.getModule = class FileAreaList extends MenuModule {
 		);
 
 		this.updateCustomLabelsWithFilter( 'browse', 10, [ '{isQueued}' ] );
-		/*
-		const indicatorView = this.viewControllers.browse.getView(MciViewIds.browse.queueToggle);
-	
-		if(indicatorView) {
-			const isQueuedIndicator		= this.menuConfig.config.isQueuedIndicator || 'Y';
-			const isNotQueuedIndicator	= this.menuConfig.config.isNotQueuedIndicator || 'N';
-			
-			indicatorView.setText(stringFormat(
-				this.dlQueue.isQueued(this.currentFileEntry) ? 
-					isQueuedIndicator : 
-					isNotQueuedIndicator
-				)
-			);
-		}*/
 	}
 
 	cacheArchiveEntries(cb) {
@@ -469,7 +471,7 @@ exports.getModule = class FileAreaList extends MenuModule {
 			return cb(Errors.Invalid('Invalid area tag'));
 		}
 		
-		const filePath		= paths.join(areaInfo.storageDirectory, this.currentFileEntry.fileName);
+		const filePath		= this.currentFileEntry.filePath;
 		const archiveUtil	= ArchiveUtil.getInstance();
 
 		archiveUtil.listEntries(filePath, this.currentFileEntry.entryInfo.archiveType, (err, entries) => {
@@ -574,9 +576,10 @@ exports.getModule = class FileAreaList extends MenuModule {
 	}
 
 	loadFileIds(cb) {
-		this.fileListPosition = 0;
+		this.fileListPosition	= 0;
+		const activeFilter		= FileBaseFilters.getActiveFilter(this.client);
 
-		FileEntry.findFiles(this.filterCriteria, (err, fileIds) => {
+		FileEntry.findFiles(activeFilter, (err, fileIds) => {
 			this.fileList = fileIds;
 			return cb(err);
 		});

oputil.js

@@ -435,7 +435,7 @@ function handleConfigCommand() {
 	}	
 }
 
-function fileAreaScan(areaTag) {
+function fileAreaScan() {
 	async.waterfall(
 		[
 			function init(callback) {
@@ -453,7 +453,7 @@ function fileAreaScan(areaTag) {
 			},
 			function performScan(fileAreaMod, areaInfo, callback) {
 				fileAreaMod.scanFileAreaForChanges(areaInfo, err => {
-
+					return callback(err);
 				});
 			}
 		],