* More file area utility methods/etc.

* Start adding oputil file-area --scan <areaTag> stuff
* New new ArchiveUtil.getInstance()

core/config.js

@@ -215,10 +215,24 @@ function getDefaultConfig() {
 			zip : {
 				sig				: '504b0304',
 				offset			: 0,
-				compressCmd		: '7z',
-				compressArgs	: [ 'a', '-tzip', '{archivePath}', '{fileList}' ],
-				decompressCmd	: '7z',
-				decompressArgs	: [ 'e', '-o{extractPath}', '{archivePath}' ]
+				compress		: {
+					cmd			: '7z',
+					args		: [ 'a', '-tzip', '{archivePath}', '{fileList}' ],
+				},
+				decompress		: {
+					cmd			: '7z',
+					args		: [ 'e', '-o{extractPath}', '{archivePath}' ]
+				},
+				/*
+				list			: {
+					cmd			: '7z',
+					args		: [ 'l', '{archivePath}' ],
+					match		: '...someregex...'
+				},*/
+				extract			: {
+					cmd			: '7z',
+					args		: [ 'x', '-o{extractPath}', '{archivePath}', '{fileList}' ],
+				},				
 			}
 		},
 		

core/file_area.js

@@ -5,16 +5,24 @@
 const Config			= require('./config.js').config;
 const Errors			= require('./enig_error.js').Errors;
 const sortAreasOrConfs	= require('./conf_area_util.js').sortAreasOrConfs;
+const FileEntry			= require('./file_entry.js');
+const FileDb			= require('./database.js').dbs.file;
+const ArchiveUtil		= require('./archive_util.js');
 
 //	deps
 const _				= require('lodash');
 const async			= require('async');
+const fs			= require('fs');
+const crypto		= require('crypto');
+const paths			= require('path');
 
 exports.getAvailableFileAreas			= getAvailableFileAreas;
 exports.getSortedAvailableFileAreas		= getSortedAvailableFileAreas;
 exports.getDefaultFileArea				= getDefaultFileArea;
 exports.getFileAreaByTag				= getFileAreaByTag;
 exports.changeFileAreaWithOptions		= changeFileAreaWithOptions;
+//exports.addOrUpdateFileEntry			= addOrUpdateFileEntry;
+exports.scanFileAreaForChanges			= scanFileAreaForChanges;
 
 const WellKnownAreaTags					= exports.WellKnownAreaTags = {
 	Invalid				: '',
@@ -64,7 +72,11 @@ function getDefaultFileArea(client, disableAcsCheck) {
 }
 
 function getFileAreaByTag(areaTag) {
-	return Config.fileAreas.areas[areaTag];
+	const areaInfo = Config.fileAreas.areas[areaTag];
+	if(areaInfo) {
+		areaInfo.areaTag = areaTag;	//	convienence!
+		return areaInfo;
+	}
 }
 
 function changeFileAreaWithOptions(client, areaTag, options, cb) {
@@ -101,3 +113,167 @@ function changeFileAreaWithOptions(client, areaTag, options, cb) {
 		}
 	);
 }
+
+function getAreaStorageDirectory(areaInfo) {
+	return paths.join(Config.fileBase.areaStoragePrefix, areaInfo.storageDir || '');
+}
+
+function getExistingFileEntriesBySha1(sha1, cb) {
+	const entries = [];
+
+	FileDb.each(
+		`SELECT file_id, area_tag
+		FROM file
+		WHERE file_sha1=?;`,
+		[ sha1 ],
+		(err, fileRow) => {
+			if(fileRow) {
+				entries.push({
+					fileId	: fileRow.file_id,
+					areaTag	: fileRow.area_tag,
+				});
+			}
+		},
+		err => {
+			return cb(err, entries);
+		}
+	);
+}
+
+function addNewArchiveFileEnty(fileEntry, filePath, archiveType, cb) {
+	async.series(
+		[
+			function getArchiveFileList(callback) {
+				//	:TODO: get list of files in archive
+				return callback(null);
+			}
+		],
+		err => {
+			return cb(err);
+		}
+	);
+}
+
+function addNewFileEntry(fileEntry, filePath, cb) {
+	const archiveUtil = ArchiveUtil.getInstance();
+
+	//	:TODO: Use detectTypeWithBuf() once avail - we *just* read some file data
+	archiveUtil.detectType(filePath, (err, archiveType) => {
+		if(archiveType) {
+			return addNewArchiveFileEnty(fileEntry, filePath, archiveType, cb);
+		} else {
+			//	:TODO:addNewNonArchiveFileEntry
+		}
+	});
+}
+
+function addOrUpdateFileEntry(areaInfo, fileName, options, cb) {
+	
+	const fileEntry = new FileEntry({
+		areaTag		: areaInfo.areaTag,
+		meta		: options.meta,
+		hashTags	: options.hashTags,	//	Set() or Array
+	});
+
+	const filePath	= paths.join(getAreaStorageDirectory(areaInfo), fileName);
+
+	async.waterfall(
+		[
+			function processPhysicalFile(callback) {			
+				const stream = fs.createReadStream(filePath);
+
+				let byteSize	= 0;
+				const sha1		= crypto.createHash('sha1');
+				const sha256	= crypto.createHash('sha256');
+				const md5		= crypto.createHash('md5');
+				
+				
+				//	:TODO: crc32
+
+				stream.on('data', data => {
+					byteSize += data.length;
+
+					sha1.update(data);
+					sha256.update(data);
+					md5.update(data);					
+				});
+
+				stream.on('end', () => {
+					fileEntry.meta.byte_size = byteSize;
+
+					//	sha-1 is in basic file entry
+					fileEntry.fileSha1 = sha1.digest('hex');
+
+					//	others are meta
+					fileEntry.meta.file_sha256	= sha256.digest('hex');
+					fileEntry.meta.file_md5 	= md5.digest('hex');
+
+					return callback(null);
+				});
+
+				stream.on('error', err => {
+					return callback(err);
+				});
+			},
+			function fetchExistingEntry(callback) {
+				getExistingFileEntriesBySha1(fileEntry.fileSha1, (err, existingEntries) => {
+					return callback(err, existingEntries);
+				});
+			},
+			function addOrUpdate(callback, existingEntries) {
+				if(existingEntries.length > 0) {
+
+				} else {
+					return addNewFileEntry(fileEntry, filePath, callback);
+				}
+			}, 
+		],		
+		err => {
+			return cb(err);
+		}
+	);
+}
+
+function scanFileAreaForChanges(areaInfo, cb) {
+	const areaPhysDir = getAreaStorageDirectory(areaInfo);
+
+	async.series(
+		[
+			function scanPhysFiles(callback) {
+				fs.readdir(areaPhysDir, (err, files) => {
+					if(err) {
+						return callback(err);
+					}
+
+					async.each(files, (fileName, next) => {
+						const fullPath = paths.join(areaPhysDir, fileName);
+
+						fs.stat(fullPath, (err, stats) => {
+							if(err) {
+								//	:TODO: Log me!
+								return next(null);	//	always try next file
+							}
+
+							if(!stats.isFile()) {
+								return next(null);
+							}
+
+							addOrUpdateFileEntry(areaInfo, fileName, err => {
+
+							});
+						});
+					}, err => {
+						return callback(err);
+					});
+				});
+			},
+			function scanDbEntries(callback) {
+				//	:TODO: Look @ db entries for area that were *not* processed above
+				return callback(null);
+			}
+		],
+		err => {
+			return cb(err);
+		}
+	);
+}

core/file_entry.js

@@ -10,7 +10,7 @@ const _			= require('lodash');
 
 const FILE_TABLE_MEMBERS	= [ 
 	'file_id', 'area_tag', 'file_sha1', 'file_name', 
-	'desc', 'desc_long', 'upload_by_username', 'upload_timestamp' 
+	'desc', 'desc_long', 'upload_by_username', 'upload_timestamp'	//	:TODO: remove upload_by_username -- and from database.js, etc. 
 ];
 
 const FILE_WELL_KNOWN_META = {

core/scanner_tossers/ftn_bso.js

@@ -37,6 +37,8 @@ exports.moduleInfo = {
 	* Support NetMail
 		* NetMail needs explicit isNetMail()  check
 		* NetMail filename / location / etc. is still unknown - need to post on groups & get real answers
+	* Validate packet passwords!!!!
+		=> secure vs insecure landing areas
 	
 */ 
 
@@ -49,9 +51,7 @@ function FTNMessageScanTossModule() {
 	
 	let self = this;
 
-	this.archUtil = new ArchiveUtil();
-	this.archUtil.init();
-	
+	this.archUtil = ArchiveUtil.getInstance();
 
 	if(_.has(Config, 'scannerTossers.ftn_bso')) {
 		this.moduleConfig = Config.scannerTossers.ftn_bso;	

mods/file_area_list.js

@@ -67,6 +67,8 @@ exports.getModule = class FileAreaList extends MenuModule {
 		this.filterCriteria = this.filterCriteria || { 
 			//	:TODO: set area tag - all in current area by default
 		};
+
+		this.currentFileEntry = new FileEntry();
 	}
 
 	enter() {
@@ -140,8 +142,6 @@ exports.getModule = class FileAreaList extends MenuModule {
 					return self.loadFileIds(callback);
 				},
 				function loadCurrentFileInfo(callback) {										
-					self.currentFileEntry = new FileEntry();
-					
 					self.currentFileEntry.load( self.fileList[ self.fileListPosition ], err => {
 						return callback(err);
 					});

oputil.js

@@ -38,6 +38,7 @@ global args:
 commands:
   user                  : user utilities
   config                : config file management
+  file-area             : file area management
 
 `,
 	User : 
@@ -56,11 +57,24 @@ valid args:
 
 valid args:
   --new                 : generate a new/initial configuration
+`,
+	FileArea :
+`usage: oputil.js file-area <args>
+
+valid args:
+  --scan AREA_TAG       : (re)scan area specified by AREA_TAG for new files
 `
 };
 
-function printUsage(command) {
-	console.error(USAGE_HELP[command]);
+function printUsageAndSetExitCode(command, exitCode) {
+	if(_.isUndefined(exitCode)) {
+		exitCode = ExitCodes.ERROR;
+	}
+	process.exitCode = exitCode;
+	const errMsg = USAGE_HELP[command];
+	if(errMsg) {
+		console.error(errMsg);
+	}
 }
 
 function initConfig(cb) {
@@ -144,8 +158,7 @@ function setAccountStatus(userName, active) {
 
 function handleUserCommand() {
 	if(true === argv.help || !_.isString(argv.user) || 0 === argv.user.length) {
-		process.exitCode = ExitCodes.ERROR;
-		return printUsage('User');
+		return printUsageAndSetExitCode('User', ExitCodes.ERROR);
 	}
 
 	if(_.isString(argv.password)) {
@@ -399,8 +412,7 @@ function askNewConfigQuestions(cb) {
 
 function handleConfigCommand() {
 	if(true === argv.help) {
-		process.exitCode = ExitCodes.ERROR;
-		return printUsage('Config');
+		return printUsageAndSetExitCode('Config', ExitCodes.ERROR);
 	}
 
 	if(argv.new) {
@@ -419,8 +431,48 @@ function handleConfigCommand() {
 			}
 		});		
 	} else {
+		return printUsageAndSetExitCode('Config', ExitCodes.ERROR);
+	}	
+}
+
+function fileAreaScan(areaTag) {
+	async.waterfall(
+		[
+			function init(callback) {
+				return initConfigAndDatabases(callback);
+			},
+			function getFileArea(callback) {
+				const fileAreaMod = require('./core/file_area.js');
+
+				const areaInfo = fileAreaMod.getFileAreaByTag(argv.scan);
+				if(!areaInfo) {
+					return callback(new Error('Invalid file area'));
+				}
+
+				return callback(null, fileAreaMod, areaInfo);
+			},
+			function performScan(fileAreaMod, areaInfo, callback) {
+				fileAreaMod.scanFileAreaForChanges(areaInfo, err => {
+
+				});
+			}
+		],
+		err => {
+			if(err) {
 				process.exitCode = ExitCodes.ERROR;
-		return printUsage('Config');	
+				console.error(err.message);
+			}
+		}
+	);
+}
+
+function handleFileAreaCommand() {
+	if(true === argv.help) {
+		return printUsageAndSetExitCode('FileArea', ExitCodes.ERROR);
+	}
+
+	if(argv.scan) {
+		return fileAreaScan(argv.scan);
 	}
 }
 
@@ -435,8 +487,7 @@ function main() {
 	if(0 === argv._.length ||
 		'help' === argv._[0])
 	{
-		printUsage('General');
-		process.exit(ExitCodes.SUCCESS);
+		printUsageAndSetExitCode('General', ExitCodes.SUCCESS);
 	}
 
 	switch(argv._[0]) {
@@ -448,9 +499,12 @@ function main() {
 			handleConfigCommand();
 			break;
 
+		case 'file-area' :
+			handleFileAreaCommand();
+			break;
+
 		default:
-			printUsage('');
-			process.exitCode = ExitCodes.BAD_COMMAND;
+			printUsageAndSetExitCode('', ExitCodes.BAD_COMMAND);
 	}
 }