* More file area utility methods/etc.
* Start adding oputil file-area --scan <areaTag> stuff * New new ArchiveUtil.getInstance()
This commit is contained in:
parent
35e7610670
commit
ec716fdf2c
|
@ -215,10 +215,24 @@ function getDefaultConfig() {
|
|||
zip : {
|
||||
sig : '504b0304',
|
||||
offset : 0,
|
||||
compressCmd : '7z',
|
||||
compressArgs : [ 'a', '-tzip', '{archivePath}', '{fileList}' ],
|
||||
decompressCmd : '7z',
|
||||
decompressArgs : [ 'e', '-o{extractPath}', '{archivePath}' ]
|
||||
compress : {
|
||||
cmd : '7z',
|
||||
args : [ 'a', '-tzip', '{archivePath}', '{fileList}' ],
|
||||
},
|
||||
decompress : {
|
||||
cmd : '7z',
|
||||
args : [ 'e', '-o{extractPath}', '{archivePath}' ]
|
||||
},
|
||||
/*
|
||||
list : {
|
||||
cmd : '7z',
|
||||
args : [ 'l', '{archivePath}' ],
|
||||
match : '...someregex...'
|
||||
},*/
|
||||
extract : {
|
||||
cmd : '7z',
|
||||
args : [ 'x', '-o{extractPath}', '{archivePath}', '{fileList}' ],
|
||||
},
|
||||
}
|
||||
},
|
||||
|
||||
|
|
|
@ -5,16 +5,24 @@
|
|||
const Config = require('./config.js').config;
|
||||
const Errors = require('./enig_error.js').Errors;
|
||||
const sortAreasOrConfs = require('./conf_area_util.js').sortAreasOrConfs;
|
||||
const FileEntry = require('./file_entry.js');
|
||||
const FileDb = require('./database.js').dbs.file;
|
||||
const ArchiveUtil = require('./archive_util.js');
|
||||
|
||||
// deps
|
||||
const _ = require('lodash');
|
||||
const async = require('async');
|
||||
const fs = require('fs');
|
||||
const crypto = require('crypto');
|
||||
const paths = require('path');
|
||||
|
||||
exports.getAvailableFileAreas = getAvailableFileAreas;
|
||||
exports.getSortedAvailableFileAreas = getSortedAvailableFileAreas;
|
||||
exports.getDefaultFileArea = getDefaultFileArea;
|
||||
exports.getFileAreaByTag = getFileAreaByTag;
|
||||
exports.changeFileAreaWithOptions = changeFileAreaWithOptions;
|
||||
//exports.addOrUpdateFileEntry = addOrUpdateFileEntry;
|
||||
exports.scanFileAreaForChanges = scanFileAreaForChanges;
|
||||
|
||||
const WellKnownAreaTags = exports.WellKnownAreaTags = {
|
||||
Invalid : '',
|
||||
|
@ -64,7 +72,11 @@ function getDefaultFileArea(client, disableAcsCheck) {
|
|||
}
|
||||
|
||||
function getFileAreaByTag(areaTag) {
|
||||
return Config.fileAreas.areas[areaTag];
|
||||
const areaInfo = Config.fileAreas.areas[areaTag];
|
||||
if(areaInfo) {
|
||||
areaInfo.areaTag = areaTag; // convienence!
|
||||
return areaInfo;
|
||||
}
|
||||
}
|
||||
|
||||
function changeFileAreaWithOptions(client, areaTag, options, cb) {
|
||||
|
@ -101,3 +113,167 @@ function changeFileAreaWithOptions(client, areaTag, options, cb) {
|
|||
}
|
||||
);
|
||||
}
|
||||
|
||||
function getAreaStorageDirectory(areaInfo) {
|
||||
return paths.join(Config.fileBase.areaStoragePrefix, areaInfo.storageDir || '');
|
||||
}
|
||||
|
||||
function getExistingFileEntriesBySha1(sha1, cb) {
|
||||
const entries = [];
|
||||
|
||||
FileDb.each(
|
||||
`SELECT file_id, area_tag
|
||||
FROM file
|
||||
WHERE file_sha1=?;`,
|
||||
[ sha1 ],
|
||||
(err, fileRow) => {
|
||||
if(fileRow) {
|
||||
entries.push({
|
||||
fileId : fileRow.file_id,
|
||||
areaTag : fileRow.area_tag,
|
||||
});
|
||||
}
|
||||
},
|
||||
err => {
|
||||
return cb(err, entries);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function addNewArchiveFileEnty(fileEntry, filePath, archiveType, cb) {
|
||||
async.series(
|
||||
[
|
||||
function getArchiveFileList(callback) {
|
||||
// :TODO: get list of files in archive
|
||||
return callback(null);
|
||||
}
|
||||
],
|
||||
err => {
|
||||
return cb(err);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function addNewFileEntry(fileEntry, filePath, cb) {
|
||||
const archiveUtil = ArchiveUtil.getInstance();
|
||||
|
||||
// :TODO: Use detectTypeWithBuf() once avail - we *just* read some file data
|
||||
archiveUtil.detectType(filePath, (err, archiveType) => {
|
||||
if(archiveType) {
|
||||
return addNewArchiveFileEnty(fileEntry, filePath, archiveType, cb);
|
||||
} else {
|
||||
// :TODO:addNewNonArchiveFileEntry
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function addOrUpdateFileEntry(areaInfo, fileName, options, cb) {
|
||||
|
||||
const fileEntry = new FileEntry({
|
||||
areaTag : areaInfo.areaTag,
|
||||
meta : options.meta,
|
||||
hashTags : options.hashTags, // Set() or Array
|
||||
});
|
||||
|
||||
const filePath = paths.join(getAreaStorageDirectory(areaInfo), fileName);
|
||||
|
||||
async.waterfall(
|
||||
[
|
||||
function processPhysicalFile(callback) {
|
||||
const stream = fs.createReadStream(filePath);
|
||||
|
||||
let byteSize = 0;
|
||||
const sha1 = crypto.createHash('sha1');
|
||||
const sha256 = crypto.createHash('sha256');
|
||||
const md5 = crypto.createHash('md5');
|
||||
|
||||
|
||||
// :TODO: crc32
|
||||
|
||||
stream.on('data', data => {
|
||||
byteSize += data.length;
|
||||
|
||||
sha1.update(data);
|
||||
sha256.update(data);
|
||||
md5.update(data);
|
||||
});
|
||||
|
||||
stream.on('end', () => {
|
||||
fileEntry.meta.byte_size = byteSize;
|
||||
|
||||
// sha-1 is in basic file entry
|
||||
fileEntry.fileSha1 = sha1.digest('hex');
|
||||
|
||||
// others are meta
|
||||
fileEntry.meta.file_sha256 = sha256.digest('hex');
|
||||
fileEntry.meta.file_md5 = md5.digest('hex');
|
||||
|
||||
return callback(null);
|
||||
});
|
||||
|
||||
stream.on('error', err => {
|
||||
return callback(err);
|
||||
});
|
||||
},
|
||||
function fetchExistingEntry(callback) {
|
||||
getExistingFileEntriesBySha1(fileEntry.fileSha1, (err, existingEntries) => {
|
||||
return callback(err, existingEntries);
|
||||
});
|
||||
},
|
||||
function addOrUpdate(callback, existingEntries) {
|
||||
if(existingEntries.length > 0) {
|
||||
|
||||
} else {
|
||||
return addNewFileEntry(fileEntry, filePath, callback);
|
||||
}
|
||||
},
|
||||
],
|
||||
err => {
|
||||
return cb(err);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function scanFileAreaForChanges(areaInfo, cb) {
|
||||
const areaPhysDir = getAreaStorageDirectory(areaInfo);
|
||||
|
||||
async.series(
|
||||
[
|
||||
function scanPhysFiles(callback) {
|
||||
fs.readdir(areaPhysDir, (err, files) => {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
async.each(files, (fileName, next) => {
|
||||
const fullPath = paths.join(areaPhysDir, fileName);
|
||||
|
||||
fs.stat(fullPath, (err, stats) => {
|
||||
if(err) {
|
||||
// :TODO: Log me!
|
||||
return next(null); // always try next file
|
||||
}
|
||||
|
||||
if(!stats.isFile()) {
|
||||
return next(null);
|
||||
}
|
||||
|
||||
addOrUpdateFileEntry(areaInfo, fileName, err => {
|
||||
|
||||
});
|
||||
});
|
||||
}, err => {
|
||||
return callback(err);
|
||||
});
|
||||
});
|
||||
},
|
||||
function scanDbEntries(callback) {
|
||||
// :TODO: Look @ db entries for area that were *not* processed above
|
||||
return callback(null);
|
||||
}
|
||||
],
|
||||
err => {
|
||||
return cb(err);
|
||||
}
|
||||
);
|
||||
}
|
|
@ -10,7 +10,7 @@ const _ = require('lodash');
|
|||
|
||||
const FILE_TABLE_MEMBERS = [
|
||||
'file_id', 'area_tag', 'file_sha1', 'file_name',
|
||||
'desc', 'desc_long', 'upload_by_username', 'upload_timestamp'
|
||||
'desc', 'desc_long', 'upload_by_username', 'upload_timestamp' // :TODO: remove upload_by_username -- and from database.js, etc.
|
||||
];
|
||||
|
||||
const FILE_WELL_KNOWN_META = {
|
||||
|
|
|
@ -37,6 +37,8 @@ exports.moduleInfo = {
|
|||
* Support NetMail
|
||||
* NetMail needs explicit isNetMail() check
|
||||
* NetMail filename / location / etc. is still unknown - need to post on groups & get real answers
|
||||
* Validate packet passwords!!!!
|
||||
=> secure vs insecure landing areas
|
||||
|
||||
*/
|
||||
|
||||
|
@ -49,9 +51,7 @@ function FTNMessageScanTossModule() {
|
|||
|
||||
let self = this;
|
||||
|
||||
this.archUtil = new ArchiveUtil();
|
||||
this.archUtil.init();
|
||||
|
||||
this.archUtil = ArchiveUtil.getInstance();
|
||||
|
||||
if(_.has(Config, 'scannerTossers.ftn_bso')) {
|
||||
this.moduleConfig = Config.scannerTossers.ftn_bso;
|
||||
|
|
|
@ -67,6 +67,8 @@ exports.getModule = class FileAreaList extends MenuModule {
|
|||
this.filterCriteria = this.filterCriteria || {
|
||||
// :TODO: set area tag - all in current area by default
|
||||
};
|
||||
|
||||
this.currentFileEntry = new FileEntry();
|
||||
}
|
||||
|
||||
enter() {
|
||||
|
@ -140,8 +142,6 @@ exports.getModule = class FileAreaList extends MenuModule {
|
|||
return self.loadFileIds(callback);
|
||||
},
|
||||
function loadCurrentFileInfo(callback) {
|
||||
self.currentFileEntry = new FileEntry();
|
||||
|
||||
self.currentFileEntry.load( self.fileList[ self.fileListPosition ], err => {
|
||||
return callback(err);
|
||||
});
|
||||
|
|
78
oputil.js
78
oputil.js
|
@ -38,6 +38,7 @@ global args:
|
|||
commands:
|
||||
user : user utilities
|
||||
config : config file management
|
||||
file-area : file area management
|
||||
|
||||
`,
|
||||
User :
|
||||
|
@ -56,11 +57,24 @@ valid args:
|
|||
|
||||
valid args:
|
||||
--new : generate a new/initial configuration
|
||||
`,
|
||||
FileArea :
|
||||
`usage: oputil.js file-area <args>
|
||||
|
||||
valid args:
|
||||
--scan AREA_TAG : (re)scan area specified by AREA_TAG for new files
|
||||
`
|
||||
};
|
||||
|
||||
function printUsage(command) {
|
||||
console.error(USAGE_HELP[command]);
|
||||
function printUsageAndSetExitCode(command, exitCode) {
|
||||
if(_.isUndefined(exitCode)) {
|
||||
exitCode = ExitCodes.ERROR;
|
||||
}
|
||||
process.exitCode = exitCode;
|
||||
const errMsg = USAGE_HELP[command];
|
||||
if(errMsg) {
|
||||
console.error(errMsg);
|
||||
}
|
||||
}
|
||||
|
||||
function initConfig(cb) {
|
||||
|
@ -144,8 +158,7 @@ function setAccountStatus(userName, active) {
|
|||
|
||||
function handleUserCommand() {
|
||||
if(true === argv.help || !_.isString(argv.user) || 0 === argv.user.length) {
|
||||
process.exitCode = ExitCodes.ERROR;
|
||||
return printUsage('User');
|
||||
return printUsageAndSetExitCode('User', ExitCodes.ERROR);
|
||||
}
|
||||
|
||||
if(_.isString(argv.password)) {
|
||||
|
@ -399,8 +412,7 @@ function askNewConfigQuestions(cb) {
|
|||
|
||||
function handleConfigCommand() {
|
||||
if(true === argv.help) {
|
||||
process.exitCode = ExitCodes.ERROR;
|
||||
return printUsage('Config');
|
||||
return printUsageAndSetExitCode('Config', ExitCodes.ERROR);
|
||||
}
|
||||
|
||||
if(argv.new) {
|
||||
|
@ -419,8 +431,48 @@ function handleConfigCommand() {
|
|||
}
|
||||
});
|
||||
} else {
|
||||
process.exitCode = ExitCodes.ERROR;
|
||||
return printUsage('Config');
|
||||
return printUsageAndSetExitCode('Config', ExitCodes.ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
function fileAreaScan(areaTag) {
|
||||
async.waterfall(
|
||||
[
|
||||
function init(callback) {
|
||||
return initConfigAndDatabases(callback);
|
||||
},
|
||||
function getFileArea(callback) {
|
||||
const fileAreaMod = require('./core/file_area.js');
|
||||
|
||||
const areaInfo = fileAreaMod.getFileAreaByTag(argv.scan);
|
||||
if(!areaInfo) {
|
||||
return callback(new Error('Invalid file area'));
|
||||
}
|
||||
|
||||
return callback(null, fileAreaMod, areaInfo);
|
||||
},
|
||||
function performScan(fileAreaMod, areaInfo, callback) {
|
||||
fileAreaMod.scanFileAreaForChanges(areaInfo, err => {
|
||||
|
||||
});
|
||||
}
|
||||
],
|
||||
err => {
|
||||
if(err) {
|
||||
process.exitCode = ExitCodes.ERROR;
|
||||
console.error(err.message);
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
function handleFileAreaCommand() {
|
||||
if(true === argv.help) {
|
||||
return printUsageAndSetExitCode('FileArea', ExitCodes.ERROR);
|
||||
}
|
||||
|
||||
if(argv.scan) {
|
||||
return fileAreaScan(argv.scan);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -435,8 +487,7 @@ function main() {
|
|||
if(0 === argv._.length ||
|
||||
'help' === argv._[0])
|
||||
{
|
||||
printUsage('General');
|
||||
process.exit(ExitCodes.SUCCESS);
|
||||
printUsageAndSetExitCode('General', ExitCodes.SUCCESS);
|
||||
}
|
||||
|
||||
switch(argv._[0]) {
|
||||
|
@ -448,9 +499,12 @@ function main() {
|
|||
handleConfigCommand();
|
||||
break;
|
||||
|
||||
case 'file-area' :
|
||||
handleFileAreaCommand();
|
||||
break;
|
||||
|
||||
default:
|
||||
printUsage('');
|
||||
process.exitCode = ExitCodes.BAD_COMMAND;
|
||||
printUsageAndSetExitCode('', ExitCodes.BAD_COMMAND);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue