From 3af1858c39b732a8bffc250749a593e80778e87e Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 20 Feb 2017 22:31:01 -0700 Subject: [PATCH] Add 'fb move' to oputil --- core/file_entry.js | 54 +++++++++- core/menu_util.js | 2 +- core/oputil/oputil_common.js | 1 + core/oputil/oputil_file_base.js | 174 ++++++++++++++++++++++++++------ core/oputil/oputil_help.js | 6 ++ 5 files changed, 204 insertions(+), 33 deletions(-) diff --git a/core/file_entry.js b/core/file_entry.js index 50faf360..3402ddbf 100644 --- a/core/file_entry.js +++ b/core/file_entry.js @@ -4,12 +4,13 @@ const fileDb = require('./database.js').dbs.file; const Errors = require('./enig_error.js').Errors; const getISOTimestampString = require('./database.js').getISOTimestampString; -const Config = require('./config.js').config; +const Config = require('./config.js').config; // deps const async = require('async'); const _ = require('lodash'); const paths = require('path'); +const fse = require('fs-extra'); const FILE_TABLE_MEMBERS = [ 'file_id', 'area_tag', 'file_sha256', 'file_name', 'storage_tag', @@ -377,7 +378,7 @@ module.exports = class FileEntry { } else { sql = `SELECT f.file_id - FROM file`; + FROM file f`; sqlOrderBy = `${getOrderByWithCast('f.file_id')} ${sqlOrderDir}`; } @@ -387,6 +388,10 @@ module.exports = class FileEntry { appendWhereClause(`f.area_tag="${filter.areaTag}"`); } + if(filter.storageTag && filter.storageTag.length > 0) { + appendWhereClause(`f.storage_tag="${filter.storageTag}"`); + } + if(filter.terms && filter.terms.length > 0) { appendWhereClause( `f.file_id IN ( @@ -425,4 +430,49 @@ module.exports = class FileEntry { return cb(err, matchingFileIds); }); } + + static moveEntry(srcFileEntry, destAreaTag, destStorageTag, destFileName, cb) { + if(!cb && _.isFunction(destFileName)) { + cb = destFileName; + destFileName = srcFileEntry.fileName; + } + + const srcPath = srcFileEntry.filePath; + const dstDir = FileEntry.getAreaStorageDirectoryByTag(destStorageTag); + + + if(!dstDir) { + return cb(Errors.Invalid('Invalid storage tag')); + } + + const dstPath = paths.join(dstDir, destFileName); + + async.series( + [ + function movePhysFile(callback) { + if(srcPath === dstPath) { + return callback(null); // don't need to move file, but may change areas + } + + fse.move(srcPath, dstPath, err => { + return callback(err); + }); + }, + function updateDatabase(callback) { + fileDb.run( + `UPDATE file + SET area_tag = ?, file_name = ?, storage_tag = ? + WHERE file_id = ?;`, + [ destAreaTag, destFileName, destStorageTag, srcFileEntry.fileId ], + err => { + return callback(err); + } + ); + } + ], + err => { + return cb(err); + } + ); + } }; diff --git a/core/menu_util.js b/core/menu_util.js index 7e15d6da..d9e5a1a6 100644 --- a/core/menu_util.js +++ b/core/menu_util.js @@ -90,7 +90,7 @@ function loadMenu(options, cb) { }); }, function createModuleInstance(modData, callback) { - Log.debug( + Log.trace( { moduleName : modData.name, extraArgs : options.extraArgs, config : modData.config, info : modData.mod.modInfo }, 'Creating menu module instance'); diff --git a/core/oputil/oputil_common.js b/core/oputil/oputil_common.js index 847acbf4..dd17aecc 100644 --- a/core/oputil/oputil_common.js +++ b/core/oputil/oputil_common.js @@ -73,6 +73,7 @@ function getAreaAndStorage(tags) { const entry = { areaTag : parts[0], }; + entry.pattern = entry.areaTag; // handy if(parts[1]) { entry.storageTag = parts[1]; } diff --git a/core/oputil/oputil_file_base.js b/core/oputil/oputil_file_base.js index f84257e7..17cdf3d1 100644 --- a/core/oputil/oputil_file_base.js +++ b/core/oputil/oputil_file_base.js @@ -65,7 +65,7 @@ function scanFileAreaForChanges(areaInfo, options, cb) { return nextFile(null); } - process.stdout.write(`* Scanning ${fullPath}... `); + process.stdout.write(`Scanning ${fullPath}... `); fileArea.scanFile( fullPath, @@ -134,14 +134,15 @@ function dumpAreaInfo(areaInfo, areaAndStorageInfo, cb) { return cb(null); } -function dumpFileInfo(shaOrFileId, cb) { +function getSpecificFileEntry(pattern, cb) { + // spec: FILE_ID|SHA|PARTIAL_SHA const FileEntry = require('../../core/file_entry.js'); async.waterfall( [ function getByFileId(callback) { - const fileId = parseInt(shaOrFileId); - if(!/^[0-9]+$/.test(shaOrFileId) || isNaN(fileId)) { + const fileId = parseInt(pattern); + if(!/^[0-9]+$/.test(pattern) || isNaN(fileId)) { return callback(null, null); } @@ -155,7 +156,22 @@ function dumpFileInfo(shaOrFileId, cb) { return callback(null, fileEntry); // already got it by sha } - FileEntry.findFileBySha(shaOrFileId, (err, fileEntry) => { + FileEntry.findFileBySha(pattern, (err, fileEntry) => { + return callback(err, fileEntry); + }); + }, + ], + (err, fileEntry) => { + return cb(err, fileEntry); + } + ); +} + +function dumpFileInfo(shaOrFileId, cb) { + async.waterfall( + [ + function getEntry(callback) { + getSpecificFileEntry(shaOrFileId, (err, fileEntry) => { return callback(err, fileEntry); }); }, @@ -164,7 +180,8 @@ function dumpFileInfo(shaOrFileId, cb) { console.info(`file_id: ${fileEntry.fileId}`); console.info(`sha_256: ${fileEntry.fileSha256}`); - console.info(`area_tag: ${fileEntry.areaTag}`); + console.info(`area_tag: ${fileEntry.areaTag}`); + console.info(`storage_tag: ${fileEntry.storageTag}`); console.info(`path: ${fullPath}`); console.info(`hashTags: ${Array.from(fileEntry.hashTags).join(', ')}`); console.info(`uploaded: ${moment(fileEntry.uploadTimestamp).format()}`); @@ -185,30 +202,6 @@ function dumpFileInfo(shaOrFileId, cb) { return cb(err); } ); -/* - FileEntry.findFileBySha(sha, (err, fileEntry) => { - if(err) { - return cb(err); - } - - const fullPath = paths.join(fileArea.getAreaStorageDirectoryByTag(fileEntry.storageTag), fileEntry.fileName); - - console.info(`file_id: ${fileEntry.fileId}`); - console.info(`sha_256: ${fileEntry.fileSha256}`); - console.info(`area_tag: ${fileEntry.areaTag}`); - console.info(`path: ${fullPath}`); - console.info(`hashTags: ${Array.from(fileEntry.hashTags).join(', ')}`); - console.info(`uploaded: ${moment(fileEntry.uploadTimestamp).format()}`); - - _.each(fileEntry.meta, (metaValue, metaName) => { - console.info(`${metaName}: ${metaValue}`); - }); - - if(argv['show-desc']) { - console.info(`${fileEntry.desc}`); - } - }); - */ } function displayFileAreaInfo() { @@ -298,6 +291,126 @@ function scanFileAreas() { ); } +function moveFiles() { + // + // oputil fb move SRC [SRC2 ...] DST + // + // SRC: PATH|FILE_ID|SHA|AREA_TAG[@STORAGE_TAG] + // DST: AREA_TAG[@STORAGE_TAG] + // + if(argv._.length < 4) { + return printUsageAndSetExitCode(getHelpFor('FileBase'), ExitCodes.ERROR); + } + + const moveArgs = argv._.slice(2); + let src = getAreaAndStorage(moveArgs.slice(0, -1)); + let dst = getAreaAndStorage(moveArgs.slice(-1))[0]; + let FileEntry; + + async.waterfall( + [ + function init(callback) { + return initConfigAndDatabases( err => { + if(!err) { + fileArea = require('../../core/file_base_area.js'); + } + return callback(err); + }); + }, + function validateAndExpandSourceAndDest(callback) { + let srcEntries = []; + + const areaInfo = fileArea.getFileAreaByTag(dst.areaTag); + if(areaInfo) { + dst.areaInfo = areaInfo; + } else { + return callback(Errors.DoesNotExist('Invalid or unknown destination area')); + } + + // Each SRC may be PATH|FILE_ID|SHA|AREA_TAG[@STORAGE_TAG] + FileEntry = require('../../core/file_entry.js'); + + async.eachSeries(src, (areaAndStorage, next) => { + // + // If this entry represents a area tag, it means *all files* in that area + // + const areaInfo = fileArea.getFileAreaByTag(areaAndStorage.areaTag); + if(areaInfo) { + src.areaInfo = areaInfo; + + const findFilter = { + areaTag : areaAndStorage.areaTag, + }; + + if(areaAndStorage.storageTag) { + findFilter.storageTag = areaAndStorage.storageTag; + } + + FileEntry.findFiles(findFilter, (err, fileIds) => { + if(err) { + return next(err); + } + + async.each(fileIds, (fileId, nextFileId) => { + const fileEntry = new FileEntry(); + fileEntry.load(fileId, err => { + if(!err) { + srcEntries.push(fileEntry); + } + return nextFileId(err); + }); + }, + err => { + return next(err); + }); + }); + + } else { + // PATH|FILE_ID|SHA|PARTIAL_SHA + getSpecificFileEntry(areaAndStorage.pattern, (err, fileEntry) => { + if(err) { + return next(err); + } + srcEntries.push(fileEntry); + return next(null); + }); + } + }, + err => { + return callback(err, srcEntries); + }); + }, + function moveEntries(srcEntries, callback) { + + if(!dst.storageTag) { + dst.storageTag = dst.areaInfo.storageTags[0]; + } + + const destDir = FileEntry.getAreaStorageDirectoryByTag(dst.storageTag); + + async.eachSeries(srcEntries, (entry, nextEntry) => { + const srcPath = entry.filePath; + const dstPath = paths.join(destDir, entry.fileName); + + process.stdout.write(`Moving ${srcPath} => ${dstPath}... `); + + FileEntry.moveEntry(entry, dst.areaTag, dst.storageTag, err => { + if(err) { + console.info(`Failed: ${err.message}`); + } else { + console.info('Done'); + } + return nextEntry(null); // always try next + }); + }, + err => { + return callback(err); + }); + } + ] + ); +} + function handleFileBaseCommand() { if(true === argv.help) { return printUsageAndSetExitCode(getHelpFor('FileBase'), ExitCodes.ERROR); @@ -308,6 +421,7 @@ function handleFileBaseCommand() { switch(action) { case 'info' : return displayFileAreaInfo(); case 'scan' : return scanFileAreas(); + case 'move' : return moveFiles(); default : return printUsageAndSetExitCode(getHelpFor('FileBase'), ExitCodes.ERROR); } diff --git a/core/oputil/oputil_help.js b/core/oputil/oputil_help.js index 7bcb731d..bd6093f1 100644 --- a/core/oputil/oputil_help.js +++ b/core/oputil/oputil_help.js @@ -55,6 +55,12 @@ where is one of: info AREA_TAG|SHA|FILE_ID : display information about areas and/or files SHA may be a full or partial SHA-256 + move SRC DST : move entry(s) from SRC to DST where: + SRC may be FILE_ID|SHA|AREA_TAG + DST may be AREA_TAG, optionally suffixed with @STORAGE_TAG; for example: retro@bbs + SHA may be a full or partial SHA-256 + multiple instances of SRC may exist: SRC1 SRC2 ... + valid scan : --tags TAG1,TAG2,... : specify tag(s) to assign to discovered entries