From 5c58fd2cfabb33aa8710cbc09d08d96539de5fb2 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 18 Feb 2017 23:05:40 -0700 Subject: [PATCH 1/4] Fix loading order on fb info --- core/oputil/oputil_file_base.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/core/oputil/oputil_file_base.js b/core/oputil/oputil_file_base.js index da837180..6a2a18cb 100644 --- a/core/oputil/oputil_file_base.js +++ b/core/oputil/oputil_file_base.js @@ -139,23 +139,23 @@ function dumpFileInfo(shaOrFileId, cb) { async.waterfall( [ - function getBySha(callback) { - FileEntry.findFileBySha(shaOrFileId, (err, fileEntry) => { - return callback(null, fileEntry); + function getByFileId(callback) { + const fileId = parseInt(shaOrFileId); + if(!/^[0-9]+$/.test(shaOrFileId) || isNaN(fileId)) { + return callback(null, null); + } + + const fileEntry = new FileEntry(); + fileEntry.load(fileId, () => { + return callback(null, fileEntry); // try sha }); }, - function getByFileId(fileEntry, callback) { + function getBySha(fileEntry, callback) { if(fileEntry) { return callback(null, fileEntry); // already got it by sha } - const fileId = parseInt(shaOrFileId); - if(isNaN(fileId)) { - return callback(Errors.DoesNotExist('Not found')); - } - - fileEntry = new FileEntry(); - fileEntry.load(shaOrFileId, err => { + FileEntry.findFileBySha(shaOrFileId, (err, fileEntry) => { return callback(err, fileEntry); }); }, From 0ca2ca9bf270e321c62b616420ae5737c23dd17a Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 20 Feb 2017 11:31:24 -0700 Subject: [PATCH 2/4] * Add oputil import support for *.NA and AREAS.BBS --- core/config.js | 9 +- core/ftn_address.js | 10 +- core/message_area.js | 4 +- core/oputil/oputil_common.js | 11 +- core/oputil/oputil_config.js | 342 ++++++++++++++++++++++++++++++-- core/oputil/oputil_file_base.js | 2 + core/oputil/oputil_help.js | 11 +- core/oputil/oputil_user.js | 3 +- 8 files changed, 355 insertions(+), 37 deletions(-) diff --git a/core/config.js b/core/config.js index adf82630..f529304a 100644 --- a/core/config.js +++ b/core/config.js @@ -41,7 +41,12 @@ function hasMessageConferenceAndArea(config) { return result; } -function init(configPath, cb) { +function init(configPath, options, cb) { + if(!cb && _.isFunction(options)) { + cb = options; + options = {}; + } + async.waterfall( [ function loadUserConfig(callback) { @@ -56,7 +61,7 @@ function init(configPath, cb) { let configJson; try { - configJson = hjson.parse(configData); + configJson = hjson.parse(configData, options); } catch(e) { return callback(e); } diff --git a/core/ftn_address.js b/core/ftn_address.js index 3e849b55..616b4965 100644 --- a/core/ftn_address.js +++ b/core/ftn_address.js @@ -31,7 +31,7 @@ module.exports = class Address { this.zone === other.zone && this.point === other.point && this.domain === other.domain - ); + ); } getMatchAddr(pattern) { @@ -40,7 +40,7 @@ module.exports = class Address { let addr = { }; if(m[1]) { - addr.zone = m[1].slice(0, -1) + addr.zone = m[1].slice(0, -1); if('*' !== addr.zone) { addr.zone = parseInt(addr.zone); } @@ -116,7 +116,7 @@ module.exports = class Address { ('*' === addr.zone || this.zone === addr.zone) && ('*' === addr.point || this.point === addr.point) && ('*' === addr.domain || this.domain === addr.domain) - ); + ); } return false; @@ -193,6 +193,6 @@ module.exports = class Address { } return (left.domain || '').localeCompare(right.domain || ''); - } + }; } -} +}; diff --git a/core/message_area.js b/core/message_area.js index 8fe250a1..484d22ec 100644 --- a/core/message_area.js +++ b/core/message_area.js @@ -35,6 +35,8 @@ exports.trimMessageAreasScheduledEvent = trimMessageAreasScheduledEvent; function getAvailableMessageConferences(client, options) { options = options || { includeSystemInternal : false }; + + assert(client || true === options.noClient); // perform ACS check per conf & omit system_internal if desired return _.omitBy(Config.messageConferences, (conf, confTag) => { @@ -42,7 +44,7 @@ function getAvailableMessageConferences(client, options) { return true; } - return !client.acs.hasMessageConfRead(conf); + return client && !client.acs.hasMessageConfRead(conf); }); } diff --git a/core/oputil/oputil_common.js b/core/oputil/oputil_common.js index 3bdb5ec0..847acbf4 100644 --- a/core/oputil/oputil_common.js +++ b/core/oputil/oputil_common.js @@ -12,6 +12,7 @@ const async = require('async'); exports.printUsageAndSetExitCode = printUsageAndSetExitCode; exports.getDefaultConfigPath = getDefaultConfigPath; +exports.getConfigPath = getConfigPath; exports.initConfigAndDatabases = initConfigAndDatabases; exports.getAreaAndStorage = getAreaAndStorage; @@ -40,10 +41,14 @@ function getDefaultConfigPath() { return resolvePath('~/.config/enigma-bbs/config.hjson'); } -function initConfig(cb) { - const configPath = argv.config ? argv.config : config.getDefaultPath(); +function getConfigPath() { + return argv.config ? argv.config : config.getDefaultPath(); +} - config.init(configPath, cb); +function initConfig(cb) { + const configPath = getConfigPath(); + + config.init(configPath, { keepWsc : true }, cb); } function initConfigAndDatabases(cb) { diff --git a/core/oputil/oputil_config.js b/core/oputil/oputil_config.js index 7071f459..6158359e 100644 --- a/core/oputil/oputil_config.js +++ b/core/oputil/oputil_config.js @@ -7,8 +7,10 @@ const resolvePath = require('../../core/misc_util.js').resolvePath; const printUsageAndSetExitCode = require('./oputil_common.js').printUsageAndSetExitCode; const ExitCodes = require('./oputil_common.js').ExitCodes; const argv = require('./oputil_common.js').argv; -const getDefaultConfigPath = require('./oputil_common.js').getDefaultConfigPath; +const getConfigPath = require('./oputil_common.js').getConfigPath; const getHelpFor = require('./oputil_help.js').getHelpFor; +const initConfigAndDatabases = require('./oputil_common.js').initConfigAndDatabases; +const Errors = require('../../core/enig_error.js').Errors; // deps const async = require('async'); @@ -17,6 +19,7 @@ const mkdirsSync = require('fs-extra').mkdirsSync; const fs = require('fs'); const hjson = require('hjson'); const paths = require('path'); +const _ = require('lodash'); exports.handleConfigCommand = handleConfigCommand; @@ -38,7 +41,7 @@ const QUESTIONS = { { name : 'configPath', message : 'Configuration path:', - default : argv.config ? argv.config : getDefaultConfigPath(), + default : getConfigPath(), when : answers => answers.createNewConfig }, ], @@ -232,27 +235,326 @@ function askNewConfigQuestions(cb) { ); } +function writeConfig(config, path) { + config = hjson.stringify(config, { bracesSameLine : true, spaces : '\t', keepWsc : true, quotes : 'strings' } ); + + try { + fs.writeFileSync(path, config, 'utf8'); + return true; + } catch(e) { + return false; + } +} + +function buildNewConfig() { + askNewConfigQuestions( (err, configPath, config) => { + if(err) { + return; + } + + if(writeConfig(config, configPath)) { + console.info('Configuration generated'); + } else { + console.error('Failed writing configuration'); + } + }); +} + +function validateUplinks(uplinks) { + const ftnAddress = require('../../core/ftn_address.js'); + const valid = uplinks.every(ul => { + const addr = ftnAddress.fromString(ul); + return addr; + }); + return valid; +} + +function getMsgAreaImportType(path) { + if(argv.type) { + return argv.type.toLowerCase(); + } + + const ext = paths.extname(path).toLowerCase().substr(1); + return ext; // .bbs|.na|... +} + +function importAreas() { + const importPath = argv._[argv._.length - 1]; + if(!importPath) { + return printUsageAndSetExitCode(getHelpFor('Config'), ExitCodes.ERROR); + } + + const importType = getMsgAreaImportType(importPath); + if('na' !== importType && 'bbs' !== importType) { + return console.error(`"${importType}" is not a recognized import file type`); + } + + // optional data - we'll prompt if for anything not found + let confTag = argv.conf; + let networkName = argv.network; + let uplinks = argv.uplinks; + if(uplinks) { + uplinks = uplinks.split(/[\s,]+/); + } + + let importEntries; + + async.waterfall( + [ + function readImportFile(callback) { + fs.readFile(importPath, 'utf8', (err, importData) => { + if(err) { + return callback(err); + } + + importEntries = getImportEntries(importType, importData); + if(0 === importEntries.length) { + return callback(Errors.Invalid('Invalid or empty import file')); + } + + // We should have enough to validate uplinks + if('bbs' === importType) { + for(let i = 0; i < importEntries.length; ++i) { + if(!validateUplinks(importEntries[i].uplinks)) { + return callback(Errors.Invalid('Invalid uplink(s)')); + } + } + } else { + if(!validateUplinks(uplinks)) { + return callback(Errors.Invalid('Invalid uplink(s)')); + } + } + + return callback(null); + }); + }, + function init(callback) { + return initConfigAndDatabases(callback); + }, + function validateAndCollectInput(callback) { + const msgArea = require('../../core/message_area.js'); + const Config = require('../../core/config.js').config; + + let msgConfs = msgArea.getSortedAvailMessageConferences(null, { noClient : true } ); + if(!msgConfs) { + return callback(Errors.DoesNotExist('No conferences exist in your configuration')); + } + + msgConfs = msgConfs.map(mc => { + return { + name : mc.conf.name, + value : mc.confTag, + }; + }); + + if(confTag && !msgConfs.find(mc => { + return confTag === mc.value; + })) + { + return callback(Errors.DoesNotExist(`Conference "${confTag}" does not exist`)); + } + + let existingNetworkNames = []; + if(_.has(Config, 'messageNetworks.ftn.networks')) { + existingNetworkNames = Object.keys(Config.messageNetworks.ftn.networks); + } + + if(0 === existingNetworkNames.length) { + return callback(Errors.DoesNotExist('No FTN style networks exist in your configuration')); + } + + if(networkName && !existingNetworkNames.find(net => networkName === net)) { + return callback(Errors.DoesNotExist(`FTN style Network "${networkName}" does not exist`)); + } + + getAnswers([ + { + name : 'confTag', + message : 'Message conference:', + type : 'list', + choices : msgConfs, + pageSize : 10, + when : !confTag, + }, + { + name : 'networkName', + message : 'Network name:', + type : 'list', + choices : existingNetworkNames, + when : !networkName, + }, + { + name : 'uplinks', + message : 'Uplink(s) (comma separated):', + type : 'input', + validate : (input) => { + const inputUplinks = input.split(/[\s,]+/); + return validateUplinks(inputUplinks) ? true : 'Invalid uplink(s)'; + }, + when : !uplinks && 'bbs' !== importType, + } + ], + answers => { + confTag = confTag || answers.confTag; + networkName = networkName || answers.networkName; + uplinks = uplinks || answers.uplinks; + + importEntries.forEach(ie => { + ie.areaTag = ie.ftnTag.toLowerCase(); + }); + + return callback(null); + }); + }, + function confirmWithUser(callback) { + const Config = require('../../core/config.js').config; + + console.info(`Importing the following for "${confTag}" - (${Config.messageConferences[confTag].name} - ${Config.messageConferences[confTag].desc})`); + importEntries.forEach(ie => { + console.info(` ${ie.ftnTag} - ${ie.name}`); + }); + + console.info(''); + console.info('Importing will NOT create required FTN network configurations.'); + console.info('If you have not yet done this, you will need to complete additional steps after importing.'); + console.info('See docs/msg_networks.md for details.'); + console.info(''); + + getAnswers([ + { + name : 'proceed', + message : 'Proceed?', + type : 'confirm', + } + ], + answers => { + return callback(answers.proceed ? null : Errors.General('User canceled')); + }); + + }, + function loadConfigHjson(callback) { + const configPath = getConfigPath(); + fs.readFile(configPath, 'utf8', (err, confData) => { + if(err) { + return callback(err); + } + + let config; + try { + config = hjson.parse(confData, { keepWsc : true } ); + } catch(e) { + return callback(e); + } + return callback(null, config); + + }); + }, + function performImport(config, callback) { + const confAreas = { messageConferences : {} }; + confAreas.messageConferences[confTag] = { areas : {} }; + + const msgNetworks = { messageNetworks : { ftn : { areas : {} } } }; + + importEntries.forEach(ie => { + const specificUplinks = ie.uplinks || uplinks; // AREAS.BBS has specific uplinks per area + + confAreas.messageConferences[confTag].areas[ie.areaTag] = { + name : ie.name, + desc : ie.name, + }; + + msgNetworks.messageNetworks.ftn.areas[ie.areaTag] = { + network : networkName, + tag : ie.ftnTag, + uplinks : specificUplinks + }; + }); + + + const newConfig = _.defaultsDeep(config, confAreas, msgNetworks); + const configPath = getConfigPath(); + + if(!writeConfig(newConfig, configPath)) { + return callback(Errors.UnexpectedState('Failed writing configuration')); + } + + return callback(null); + } + ], + err => { + if(err) { + console.error(err.reason ? err.reason : err.message); + } else { + const addFieldUpd = 'bbs' === importType ? '"name" and "desc"' : '"desc"'; + console.info('Configuration generated.'); + console.info(`You may wish to validate changes made to ${getConfigPath()}`); + console.info(`as well as update ${addFieldUpd} fields, sorting, etc.`); + console.info(''); + } + } + ); + +} + +function getImportEntries(importType, importData) { + let importEntries = []; + + if('na' === importType) { + // + // parse out + // TAG DESC + // + const re = /^([^\s]+)\s+([^\r\n]+)/gm; + let m; + + while( (m = re.exec(importData) )) { + importEntries.push({ + ftnTag : m[1], + name : m[2], + }); + } + } else if ('bbs' === importType) { + // + // Various formats for AREAS.BBS seem to exist. We want to support as much as possible. + // + // SBBS http://www.synchro.net/docs/sbbsecho.html#AREAS.BBS + // CODE TAG UPLINKS + // + // VADV https://www.vadvbbs.com/products/vadv/support/docs/docs_vfido.php#AREAS.BBS + // TAG UPLINKS + // + // Misc + // PATH|OTHER TAG UPLINKS + // + // Assume the second item is TAG and 1:n UPLINKS (space and/or comma sep) after (at the end) + // + const re = /^[^\s]+\s+([^\s]+)\s+([^\n]+)$/gm; + let m; + while ( (m = re.exec(importData) )) { + const tag = m[1]; + + importEntries.push({ + ftnTag : tag, + name : `Area: ${tag}`, + uplinks : m[2].split(/[\s,]+/), + }); + } + } + + return importEntries; +} + function handleConfigCommand() { if(true === argv.help) { return printUsageAndSetExitCode(getHelpFor('Config'), ExitCodes.ERROR); } - if(argv.new) { - askNewConfigQuestions( (err, configPath, config) => { - if(err) { - return; - } - - config = hjson.stringify(config, { bracesSameLine : true, spaces : '\t' } ); - - try { - fs.writeFileSync(configPath, config, 'utf8'); - console.info('Configuration generated'); - } catch(e) { - console.error('Exception attempting to create config: ' + e.toString()); - } - }); - } else { - return printUsageAndSetExitCode(getHelpFor('Config'), ExitCodes.ERROR); - } + const action = argv._[1]; + + switch(action) { + case 'new' : return buildNewConfig(); + case 'import-areas' : return importAreas(); + + default : return printUsageAndSetExitCode(getHelpFor('Config'), ExitCodes.ERROR); + } } diff --git a/core/oputil/oputil_file_base.js b/core/oputil/oputil_file_base.js index 6a2a18cb..f84257e7 100644 --- a/core/oputil/oputil_file_base.js +++ b/core/oputil/oputil_file_base.js @@ -308,5 +308,7 @@ function handleFileBaseCommand() { switch(action) { case 'info' : return displayFileAreaInfo(); case 'scan' : return scanFileAreas(); + + default : return printUsageAndSetExitCode(getHelpFor('FileBase'), ExitCodes.ERROR); } } \ No newline at end of file diff --git a/core/oputil/oputil_help.js b/core/oputil/oputil_help.js index b04bdb38..91973295 100644 --- a/core/oputil/oputil_help.js +++ b/core/oputil/oputil_help.js @@ -13,6 +13,7 @@ const usageHelp = exports.USAGE_HELP = { global args: --config PATH : specify config path (${getDefaultConfigPath()}) + --no-prompt : assume defaults/don't prompt for input where possible where is one of: user : user utilities @@ -32,10 +33,12 @@ valid args: `, Config : -`usage: optutil.js config +`usage: optutil.js config [] -valid args: - --new : generate a new/initial configuration +where is one of: + new : generate a new/initial configuration + import-na [CONF_TAG] : import fidonet *.NA file + if CONF_TAG is not supplied, it will be prompted for `, FileBase : `usage: oputil.js fb [] [] @@ -52,8 +55,6 @@ valid scan : valid info : --show-desc : display short description, if any - - ` }; diff --git a/core/oputil/oputil_user.js b/core/oputil/oputil_user.js index 90d811f5..afe243bf 100644 --- a/core/oputil/oputil_user.js +++ b/core/oputil/oputil_user.js @@ -6,6 +6,7 @@ const printUsageAndSetExitCode = require('./oputil_common.js').printUsageAndSetE const ExitCodes = require('./oputil_common.js').ExitCodes; const argv = require('./oputil_common.js').argv; const initConfigAndDatabases = require('./oputil_common.js').initConfigAndDatabases; +const getHelpFor = require('./oputil_help.js').getHelpFor; const async = require('async'); const _ = require('lodash'); @@ -14,7 +15,7 @@ exports.handleUserCommand = handleUserCommand; function handleUserCommand() { if(true === argv.help || !_.isString(argv.user) || 0 === argv.user.length) { - return printUsageAndSetExitCode('User', ExitCodes.ERROR); + return printUsageAndSetExitCode(getHelpFor('User'), ExitCodes.ERROR); } if(_.isString(argv.password)) { From 72b0eafc7b912108ef0041bc8829d94c2fbbdfb3 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 20 Feb 2017 11:46:18 -0700 Subject: [PATCH 3/4] Minor fixes to oputil --- core/oputil/oputil_config.js | 2 +- core/oputil/oputil_help.js | 15 ++++++++++----- core/oputil/oputil_main.js | 2 +- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/core/oputil/oputil_config.js b/core/oputil/oputil_config.js index 6158359e..dc918917 100644 --- a/core/oputil/oputil_config.js +++ b/core/oputil/oputil_config.js @@ -280,7 +280,7 @@ function getMsgAreaImportType(path) { function importAreas() { const importPath = argv._[argv._.length - 1]; - if(!importPath) { + if(argv._.length < 3 || !importPath || 0 === importPath.length) { return printUsageAndSetExitCode(getHelpFor('Config'), ExitCodes.ERROR); } diff --git a/core/oputil/oputil_help.js b/core/oputil/oputil_help.js index 91973295..7bcb731d 100644 --- a/core/oputil/oputil_help.js +++ b/core/oputil/oputil_help.js @@ -36,18 +36,23 @@ valid args: `usage: optutil.js config [] where is one of: - new : generate a new/initial configuration - import-na [CONF_TAG] : import fidonet *.NA file - if CONF_TAG is not supplied, it will be prompted for + new : generate a new/initial configuration + import-areas PATH : import areas using fidonet *.NA or AREAS.BBS file from PATH + +valid import-areas : + --conf CONF_TAG : specify conference tag in which to import areas + --network NETWORK : specify network name/key to associate FTN areas + --uplinks UL1,UL2,... : specify one or more comma separated uplinks + --type TYPE : specifies area import type. valid options are "bbs" and "na" `, FileBase : `usage: oputil.js fb [] [] where is one of: - scan AREA_TAG|SHA|FILE_ID : scan specified areas + scan AREA_TAG : scan specified areas AREA_TAG may be suffixed with @STORAGE_TAG; for example: retro@bbs - info AREA_TAG|FILE_ID|SHA : display information about areas and/or files + info AREA_TAG|SHA|FILE_ID : display information about areas and/or files SHA may be a full or partial SHA-256 valid scan : diff --git a/core/oputil/oputil_main.js b/core/oputil/oputil_main.js index d27a4976..dad7e4e6 100644 --- a/core/oputil/oputil_main.js +++ b/core/oputil/oputil_main.js @@ -39,6 +39,6 @@ module.exports = function() { break; default: - return printUsageAndSetExitCode('', ExitCodes.BAD_COMMAND); + return printUsageAndSetExitCode(getHelpFor('General'), ExitCodes.BAD_COMMAND); } }; From 3af1858c39b732a8bffc250749a593e80778e87e Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 20 Feb 2017 22:31:01 -0700 Subject: [PATCH 4/4] Add 'fb move' to oputil --- core/file_entry.js | 54 +++++++++- core/menu_util.js | 2 +- core/oputil/oputil_common.js | 1 + core/oputil/oputil_file_base.js | 174 ++++++++++++++++++++++++++------ core/oputil/oputil_help.js | 6 ++ 5 files changed, 204 insertions(+), 33 deletions(-) diff --git a/core/file_entry.js b/core/file_entry.js index 50faf360..3402ddbf 100644 --- a/core/file_entry.js +++ b/core/file_entry.js @@ -4,12 +4,13 @@ const fileDb = require('./database.js').dbs.file; const Errors = require('./enig_error.js').Errors; const getISOTimestampString = require('./database.js').getISOTimestampString; -const Config = require('./config.js').config; +const Config = require('./config.js').config; // deps const async = require('async'); const _ = require('lodash'); const paths = require('path'); +const fse = require('fs-extra'); const FILE_TABLE_MEMBERS = [ 'file_id', 'area_tag', 'file_sha256', 'file_name', 'storage_tag', @@ -377,7 +378,7 @@ module.exports = class FileEntry { } else { sql = `SELECT f.file_id - FROM file`; + FROM file f`; sqlOrderBy = `${getOrderByWithCast('f.file_id')} ${sqlOrderDir}`; } @@ -387,6 +388,10 @@ module.exports = class FileEntry { appendWhereClause(`f.area_tag="${filter.areaTag}"`); } + if(filter.storageTag && filter.storageTag.length > 0) { + appendWhereClause(`f.storage_tag="${filter.storageTag}"`); + } + if(filter.terms && filter.terms.length > 0) { appendWhereClause( `f.file_id IN ( @@ -425,4 +430,49 @@ module.exports = class FileEntry { return cb(err, matchingFileIds); }); } + + static moveEntry(srcFileEntry, destAreaTag, destStorageTag, destFileName, cb) { + if(!cb && _.isFunction(destFileName)) { + cb = destFileName; + destFileName = srcFileEntry.fileName; + } + + const srcPath = srcFileEntry.filePath; + const dstDir = FileEntry.getAreaStorageDirectoryByTag(destStorageTag); + + + if(!dstDir) { + return cb(Errors.Invalid('Invalid storage tag')); + } + + const dstPath = paths.join(dstDir, destFileName); + + async.series( + [ + function movePhysFile(callback) { + if(srcPath === dstPath) { + return callback(null); // don't need to move file, but may change areas + } + + fse.move(srcPath, dstPath, err => { + return callback(err); + }); + }, + function updateDatabase(callback) { + fileDb.run( + `UPDATE file + SET area_tag = ?, file_name = ?, storage_tag = ? + WHERE file_id = ?;`, + [ destAreaTag, destFileName, destStorageTag, srcFileEntry.fileId ], + err => { + return callback(err); + } + ); + } + ], + err => { + return cb(err); + } + ); + } }; diff --git a/core/menu_util.js b/core/menu_util.js index 7e15d6da..d9e5a1a6 100644 --- a/core/menu_util.js +++ b/core/menu_util.js @@ -90,7 +90,7 @@ function loadMenu(options, cb) { }); }, function createModuleInstance(modData, callback) { - Log.debug( + Log.trace( { moduleName : modData.name, extraArgs : options.extraArgs, config : modData.config, info : modData.mod.modInfo }, 'Creating menu module instance'); diff --git a/core/oputil/oputil_common.js b/core/oputil/oputil_common.js index 847acbf4..dd17aecc 100644 --- a/core/oputil/oputil_common.js +++ b/core/oputil/oputil_common.js @@ -73,6 +73,7 @@ function getAreaAndStorage(tags) { const entry = { areaTag : parts[0], }; + entry.pattern = entry.areaTag; // handy if(parts[1]) { entry.storageTag = parts[1]; } diff --git a/core/oputil/oputil_file_base.js b/core/oputil/oputil_file_base.js index f84257e7..17cdf3d1 100644 --- a/core/oputil/oputil_file_base.js +++ b/core/oputil/oputil_file_base.js @@ -65,7 +65,7 @@ function scanFileAreaForChanges(areaInfo, options, cb) { return nextFile(null); } - process.stdout.write(`* Scanning ${fullPath}... `); + process.stdout.write(`Scanning ${fullPath}... `); fileArea.scanFile( fullPath, @@ -134,14 +134,15 @@ function dumpAreaInfo(areaInfo, areaAndStorageInfo, cb) { return cb(null); } -function dumpFileInfo(shaOrFileId, cb) { +function getSpecificFileEntry(pattern, cb) { + // spec: FILE_ID|SHA|PARTIAL_SHA const FileEntry = require('../../core/file_entry.js'); async.waterfall( [ function getByFileId(callback) { - const fileId = parseInt(shaOrFileId); - if(!/^[0-9]+$/.test(shaOrFileId) || isNaN(fileId)) { + const fileId = parseInt(pattern); + if(!/^[0-9]+$/.test(pattern) || isNaN(fileId)) { return callback(null, null); } @@ -155,7 +156,22 @@ function dumpFileInfo(shaOrFileId, cb) { return callback(null, fileEntry); // already got it by sha } - FileEntry.findFileBySha(shaOrFileId, (err, fileEntry) => { + FileEntry.findFileBySha(pattern, (err, fileEntry) => { + return callback(err, fileEntry); + }); + }, + ], + (err, fileEntry) => { + return cb(err, fileEntry); + } + ); +} + +function dumpFileInfo(shaOrFileId, cb) { + async.waterfall( + [ + function getEntry(callback) { + getSpecificFileEntry(shaOrFileId, (err, fileEntry) => { return callback(err, fileEntry); }); }, @@ -164,7 +180,8 @@ function dumpFileInfo(shaOrFileId, cb) { console.info(`file_id: ${fileEntry.fileId}`); console.info(`sha_256: ${fileEntry.fileSha256}`); - console.info(`area_tag: ${fileEntry.areaTag}`); + console.info(`area_tag: ${fileEntry.areaTag}`); + console.info(`storage_tag: ${fileEntry.storageTag}`); console.info(`path: ${fullPath}`); console.info(`hashTags: ${Array.from(fileEntry.hashTags).join(', ')}`); console.info(`uploaded: ${moment(fileEntry.uploadTimestamp).format()}`); @@ -185,30 +202,6 @@ function dumpFileInfo(shaOrFileId, cb) { return cb(err); } ); -/* - FileEntry.findFileBySha(sha, (err, fileEntry) => { - if(err) { - return cb(err); - } - - const fullPath = paths.join(fileArea.getAreaStorageDirectoryByTag(fileEntry.storageTag), fileEntry.fileName); - - console.info(`file_id: ${fileEntry.fileId}`); - console.info(`sha_256: ${fileEntry.fileSha256}`); - console.info(`area_tag: ${fileEntry.areaTag}`); - console.info(`path: ${fullPath}`); - console.info(`hashTags: ${Array.from(fileEntry.hashTags).join(', ')}`); - console.info(`uploaded: ${moment(fileEntry.uploadTimestamp).format()}`); - - _.each(fileEntry.meta, (metaValue, metaName) => { - console.info(`${metaName}: ${metaValue}`); - }); - - if(argv['show-desc']) { - console.info(`${fileEntry.desc}`); - } - }); - */ } function displayFileAreaInfo() { @@ -298,6 +291,126 @@ function scanFileAreas() { ); } +function moveFiles() { + // + // oputil fb move SRC [SRC2 ...] DST + // + // SRC: PATH|FILE_ID|SHA|AREA_TAG[@STORAGE_TAG] + // DST: AREA_TAG[@STORAGE_TAG] + // + if(argv._.length < 4) { + return printUsageAndSetExitCode(getHelpFor('FileBase'), ExitCodes.ERROR); + } + + const moveArgs = argv._.slice(2); + let src = getAreaAndStorage(moveArgs.slice(0, -1)); + let dst = getAreaAndStorage(moveArgs.slice(-1))[0]; + let FileEntry; + + async.waterfall( + [ + function init(callback) { + return initConfigAndDatabases( err => { + if(!err) { + fileArea = require('../../core/file_base_area.js'); + } + return callback(err); + }); + }, + function validateAndExpandSourceAndDest(callback) { + let srcEntries = []; + + const areaInfo = fileArea.getFileAreaByTag(dst.areaTag); + if(areaInfo) { + dst.areaInfo = areaInfo; + } else { + return callback(Errors.DoesNotExist('Invalid or unknown destination area')); + } + + // Each SRC may be PATH|FILE_ID|SHA|AREA_TAG[@STORAGE_TAG] + FileEntry = require('../../core/file_entry.js'); + + async.eachSeries(src, (areaAndStorage, next) => { + // + // If this entry represents a area tag, it means *all files* in that area + // + const areaInfo = fileArea.getFileAreaByTag(areaAndStorage.areaTag); + if(areaInfo) { + src.areaInfo = areaInfo; + + const findFilter = { + areaTag : areaAndStorage.areaTag, + }; + + if(areaAndStorage.storageTag) { + findFilter.storageTag = areaAndStorage.storageTag; + } + + FileEntry.findFiles(findFilter, (err, fileIds) => { + if(err) { + return next(err); + } + + async.each(fileIds, (fileId, nextFileId) => { + const fileEntry = new FileEntry(); + fileEntry.load(fileId, err => { + if(!err) { + srcEntries.push(fileEntry); + } + return nextFileId(err); + }); + }, + err => { + return next(err); + }); + }); + + } else { + // PATH|FILE_ID|SHA|PARTIAL_SHA + getSpecificFileEntry(areaAndStorage.pattern, (err, fileEntry) => { + if(err) { + return next(err); + } + srcEntries.push(fileEntry); + return next(null); + }); + } + }, + err => { + return callback(err, srcEntries); + }); + }, + function moveEntries(srcEntries, callback) { + + if(!dst.storageTag) { + dst.storageTag = dst.areaInfo.storageTags[0]; + } + + const destDir = FileEntry.getAreaStorageDirectoryByTag(dst.storageTag); + + async.eachSeries(srcEntries, (entry, nextEntry) => { + const srcPath = entry.filePath; + const dstPath = paths.join(destDir, entry.fileName); + + process.stdout.write(`Moving ${srcPath} => ${dstPath}... `); + + FileEntry.moveEntry(entry, dst.areaTag, dst.storageTag, err => { + if(err) { + console.info(`Failed: ${err.message}`); + } else { + console.info('Done'); + } + return nextEntry(null); // always try next + }); + }, + err => { + return callback(err); + }); + } + ] + ); +} + function handleFileBaseCommand() { if(true === argv.help) { return printUsageAndSetExitCode(getHelpFor('FileBase'), ExitCodes.ERROR); @@ -308,6 +421,7 @@ function handleFileBaseCommand() { switch(action) { case 'info' : return displayFileAreaInfo(); case 'scan' : return scanFileAreas(); + case 'move' : return moveFiles(); default : return printUsageAndSetExitCode(getHelpFor('FileBase'), ExitCodes.ERROR); } diff --git a/core/oputil/oputil_help.js b/core/oputil/oputil_help.js index 7bcb731d..bd6093f1 100644 --- a/core/oputil/oputil_help.js +++ b/core/oputil/oputil_help.js @@ -55,6 +55,12 @@ where is one of: info AREA_TAG|SHA|FILE_ID : display information about areas and/or files SHA may be a full or partial SHA-256 + move SRC DST : move entry(s) from SRC to DST where: + SRC may be FILE_ID|SHA|AREA_TAG + DST may be AREA_TAG, optionally suffixed with @STORAGE_TAG; for example: retro@bbs + SHA may be a full or partial SHA-256 + multiple instances of SRC may exist: SRC1 SRC2 ... + valid scan : --tags TAG1,TAG2,... : specify tag(s) to assign to discovered entries