* Additional helpers to MenuModule
* Gzip signature (.gz) * Switch to sha-256 vs sha1 for internal file hashes * Nearly complete callback / scan update support for scanFile() * Fix data input issue after performing upload * Support 'sz' recv (uploads)
This commit is contained in:
Bryan Ashby
parent
fb176d3ab3
commit
8d51c7d47c
|
|
@ -278,6 +278,13 @@ function getDefaultConfig() {
|
|||
exts : [ 'rar' ],
|
||||
handler : '7Zip',
|
||||
desc : 'RAR Archive',
|
||||
},
|
||||
gzip : {
|
||||
sig : '1f8b',
|
||||
offset : 0,
|
||||
exts : [ 'gz' ],
|
||||
handler : '7Zip',
|
||||
desc : 'Gzip Archive',
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -294,7 +301,7 @@ function getDefaultConfig() {
|
|||
],
|
||||
recvCmd : 'rz', // Avail on Debian/Ubuntu based systems as the package "lrzsz"
|
||||
recvArgs : [
|
||||
'--zmodem', '--binary', '--restricted', // dumps to CWD which is set to {uploadDir}
|
||||
'--zmodem', '--binary', '--restricted', '--keep-uppercase', // dumps to CWD which is set to {uploadDir}
|
||||
],
|
||||
// :TODO: can we not just use --escape ?
|
||||
escapeTelnet : true, // set to true to escape Telnet codes such as IAC
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ const DB_INIT_TABLE = {
|
|||
`CREATE TABLE IF NOT EXISTS file (
|
||||
file_id INTEGER PRIMARY KEY,
|
||||
area_tag VARCHAR NOT NULL,
|
||||
file_sha1 VARCHAR NOT NULL,
|
||||
file_sha256 VARCHAR NOT NULL,
|
||||
file_name, /* FTS @ file_fts */
|
||||
storage_tag VARCHAR NOT NULL,
|
||||
desc, /* FTS @ file_fts */
|
||||
|
|
|
|||
|
|
@ -160,14 +160,14 @@ function getFileEntryPath(fileEntry) {
|
|||
}
|
||||
}
|
||||
|
||||
function getExistingFileEntriesBySha1(sha1, cb) {
|
||||
function getExistingFileEntriesBySha256(sha256, cb) {
|
||||
const entries = [];
|
||||
|
||||
FileDb.each(
|
||||
`SELECT file_id, area_tag
|
||||
FROM file
|
||||
WHERE file_sha1=?;`,
|
||||
[ sha1 ],
|
||||
WHERE file_sha256=?;`,
|
||||
[ sha256 ],
|
||||
(err, fileRow) => {
|
||||
if(fileRow) {
|
||||
entries.push({
|
||||
|
|
@ -237,14 +237,38 @@ function attemptSetEstimatedReleaseDate(fileEntry) {
|
|||
}
|
||||
}
|
||||
|
||||
function populateFileEntryWithArchive(fileEntry, filePath, archiveType, cb) {
|
||||
const archiveUtil = ArchiveUtil.getInstance();
|
||||
function populateFileEntryWithArchive(fileEntry, filePath, stepInfo, iterator, cb) {
|
||||
const archiveUtil = ArchiveUtil.getInstance();
|
||||
const archiveType = fileEntry.meta.archive_type; // we set this previous to populateFileEntryWithArchive()
|
||||
|
||||
async.waterfall(
|
||||
[
|
||||
function getArchiveFileList(callback) {
|
||||
archiveUtil.listEntries(filePath, archiveType, (err, entries) => {
|
||||
return callback(null, entries || []); // ignore any errors here
|
||||
stepInfo.step = 'archive_list_start';
|
||||
|
||||
iterator(err => {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
archiveUtil.listEntries(filePath, archiveType, (err, entries) => {
|
||||
if(err) {
|
||||
stepInfo.step = 'archive_list_failed';
|
||||
} else {
|
||||
stepInfo.step = 'archive_list_finish';
|
||||
stepInfo.archiveEntries = entries || [];
|
||||
}
|
||||
|
||||
iterator(iterErr => {
|
||||
return callback( iterErr, entries || [] ); // ignore original |err| here
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
function processDescFilesStart(entries, callback) {
|
||||
stepInfo.step = 'desc_files_start';
|
||||
iterator(err => {
|
||||
return callback(err, entries);
|
||||
});
|
||||
},
|
||||
function extractDescFiles(entries, callback) {
|
||||
|
|
@ -320,7 +344,11 @@ function populateFileEntryWithArchive(fileEntry, filePath, archiveType, cb) {
|
|||
function attemptReleaseYearEstimation(callback) {
|
||||
attemptSetEstimatedReleaseDate(fileEntry);
|
||||
return callback(null);
|
||||
}
|
||||
},
|
||||
function processDescFilesFinish(callback) {
|
||||
stepInfo.step = 'desc_files_finish';
|
||||
return iterator(callback);
|
||||
},
|
||||
],
|
||||
err => {
|
||||
return cb(err);
|
||||
|
|
@ -328,7 +356,7 @@ function populateFileEntryWithArchive(fileEntry, filePath, archiveType, cb) {
|
|||
);
|
||||
}
|
||||
|
||||
function populateFileEntryNonArchive(fileEntry, filePath, archiveType, cb) {
|
||||
function populateFileEntryNonArchive(fileEntry, filePath, stepInfo, iterator, cb) {
|
||||
// :TODO: implement me!
|
||||
return cb(null);
|
||||
}
|
||||
|
|
@ -352,11 +380,17 @@ function updateFileEntry(fileEntry, filePath, cb) {
|
|||
|
||||
}
|
||||
|
||||
function scanFile(filePath, options, cb) {
|
||||
const HASH_NAMES = [ 'sha1', 'sha256', 'md5', 'crc32' ];
|
||||
|
||||
if(_.isFunction(options) && !cb) {
|
||||
cb = options;
|
||||
options = {};
|
||||
function scanFile(filePath, options, iterator, cb) {
|
||||
|
||||
if(3 === arguments.length && _.isFunction(iterator)) {
|
||||
cb = iterator;
|
||||
iterator = null;
|
||||
} else if(2 === arguments.length && _.isFunction(options)) {
|
||||
cb = options;
|
||||
iterator = null;
|
||||
options = {};
|
||||
}
|
||||
|
||||
const fileEntry = new FileEntry({
|
||||
|
|
@ -367,42 +401,96 @@ function scanFile(filePath, options, cb) {
|
|||
storageTag : options.storageTag,
|
||||
});
|
||||
|
||||
const stepInfo = {
|
||||
filePath : filePath,
|
||||
fileName : paths.basename(filePath),
|
||||
};
|
||||
|
||||
function callIter(next) {
|
||||
if(iterator) {
|
||||
return iterator(stepInfo, next);
|
||||
} else {
|
||||
return next(null);
|
||||
}
|
||||
}
|
||||
|
||||
function readErrorCallIter(origError, next) {
|
||||
stepInfo.step = 'read_error';
|
||||
stepInfo.error = origError.message;
|
||||
|
||||
callIter( () => {
|
||||
return next(origError);
|
||||
});
|
||||
}
|
||||
|
||||
async.waterfall(
|
||||
[
|
||||
function startScan(callback) {
|
||||
fs.stat(filePath, (err, stats) => {
|
||||
if(err) {
|
||||
return readErrorCallIter(err, callback);
|
||||
}
|
||||
|
||||
stepInfo.step = 'start';
|
||||
stepInfo.byteSize = fileEntry.meta.byte_size = stats.size;
|
||||
|
||||
return callIter(callback);
|
||||
});
|
||||
},
|
||||
function processPhysicalFileGeneric(callback) {
|
||||
let byteSize = 0;
|
||||
const sha1 = crypto.createHash('sha1');
|
||||
const sha256 = crypto.createHash('sha256');
|
||||
const md5 = crypto.createHash('md5');
|
||||
const crc32 = new CRC32();
|
||||
stepInfo.bytesProcessed = 0;
|
||||
|
||||
const hashes = {
|
||||
sha1 : crypto.createHash('sha1'),
|
||||
sha256 : crypto.createHash('sha256'),
|
||||
md5 : crypto.createHash('md5'),
|
||||
crc32 : new CRC32(),
|
||||
};
|
||||
|
||||
const stream = fs.createReadStream(filePath);
|
||||
|
||||
stream.on('data', data => {
|
||||
byteSize += data.length;
|
||||
stream.pause(); // until iterator compeltes
|
||||
|
||||
sha1.update(data);
|
||||
sha256.update(data);
|
||||
md5.update(data);
|
||||
crc32.update(data);
|
||||
stepInfo.bytesProcessed += data.length;
|
||||
stepInfo.step = 'hash_update';
|
||||
|
||||
callIter(err => {
|
||||
if(err) {
|
||||
stream.destroy(); // cancel read
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
async.each( HASH_NAMES, (hashName, nextHash) => {
|
||||
hashes[hashName].update(data);
|
||||
return nextHash(null);
|
||||
}, () => {
|
||||
return stream.resume();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
stream.on('end', () => {
|
||||
fileEntry.meta.byte_size = byteSize;
|
||||
fileEntry.meta.byte_size = stepInfo.bytesProcessed;
|
||||
|
||||
// sha-1 is in basic file entry
|
||||
fileEntry.fileSha1 = sha1.digest('hex');
|
||||
async.each(HASH_NAMES, (hashName, nextHash) => {
|
||||
if('sha256' === hashName) {
|
||||
stepInfo.sha256 = fileEntry.fileSha256 = hashes.sha256.digest('hex');
|
||||
} else if('sha1' === hashName || 'md5' === hashName) {
|
||||
stepInfo[hashName] = fileEntry.meta[`file_${hashName}`] = hashes[hashName].digest('hex');
|
||||
} else if('crc32' === hashName) {
|
||||
stepInfo.crc32 = fileEntry.meta.crc32 = hashes.crc32.finalize().toString(16);
|
||||
}
|
||||
|
||||
// others are meta
|
||||
fileEntry.meta.file_sha256 = sha256.digest('hex');
|
||||
fileEntry.meta.file_md5 = md5.digest('hex');
|
||||
fileEntry.meta.file_crc32 = crc32.finalize().toString(16);
|
||||
|
||||
return callback(null);
|
||||
return nextHash(null);
|
||||
}, () => {
|
||||
stepInfo.step = 'hash_finish';
|
||||
return callIter(callback);
|
||||
});
|
||||
});
|
||||
|
||||
stream.on('error', err => {
|
||||
return callback(err);
|
||||
return readErrorCallIter(err, callback);
|
||||
});
|
||||
},
|
||||
function processPhysicalFileByType(callback) {
|
||||
|
|
@ -413,9 +501,9 @@ function scanFile(filePath, options, cb) {
|
|||
// save this off
|
||||
fileEntry.meta.archive_type = archiveType;
|
||||
|
||||
populateFileEntryWithArchive(fileEntry, filePath, archiveType, err => {
|
||||
populateFileEntryWithArchive(fileEntry, filePath, stepInfo, callIter, err => {
|
||||
if(err) {
|
||||
populateFileEntryNonArchive(fileEntry, filePath, err => {
|
||||
populateFileEntryNonArchive(fileEntry, filePath, stepInfo, callIter, err => {
|
||||
// :TODO: log err
|
||||
return callback(null); // ignore err
|
||||
});
|
||||
|
|
@ -424,7 +512,7 @@ function scanFile(filePath, options, cb) {
|
|||
}
|
||||
});
|
||||
} else {
|
||||
populateFileEntryNonArchive(fileEntry, filePath, err => {
|
||||
populateFileEntryNonArchive(fileEntry, filePath, stepInfo, callIter, err => {
|
||||
// :TODO: log err
|
||||
return callback(null); // ignore err
|
||||
});
|
||||
|
|
@ -432,92 +520,21 @@ function scanFile(filePath, options, cb) {
|
|||
});
|
||||
},
|
||||
function fetchExistingEntry(callback) {
|
||||
getExistingFileEntriesBySha1(fileEntry.fileSha1, (err, existingEntries) => {
|
||||
return callback(err, existingEntries);
|
||||
getExistingFileEntriesBySha256(fileEntry.fileSha256, (err, dupeEntries) => {
|
||||
return callback(err, dupeEntries);
|
||||
});
|
||||
}
|
||||
],
|
||||
(err, existingEntries) => {
|
||||
(err, dupeEntries) => {
|
||||
if(err) {
|
||||
return cb(err);
|
||||
}
|
||||
|
||||
return cb(null, fileEntry, existingEntries);
|
||||
return cb(null, fileEntry, dupeEntries);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/*
|
||||
function addOrUpdateFileEntry(areaInfo, storageLocation, fileName, options, cb) {
|
||||
|
||||
const fileEntry = new FileEntry({
|
||||
areaTag : areaInfo.areaTag,
|
||||
meta : options.meta,
|
||||
hashTags : options.hashTags, // Set() or Array
|
||||
fileName : fileName,
|
||||
storageTag : storageLocation.storageTag,
|
||||
});
|
||||
|
||||
const filePath = paths.join(storageLocation.dir, fileName);
|
||||
|
||||
async.waterfall(
|
||||
[
|
||||
function processPhysicalFile(callback) {
|
||||
let byteSize = 0;
|
||||
const sha1 = crypto.createHash('sha1');
|
||||
const sha256 = crypto.createHash('sha256');
|
||||
const md5 = crypto.createHash('md5');
|
||||
const crc32 = new CRC32();
|
||||
|
||||
const stream = fs.createReadStream(filePath);
|
||||
|
||||
stream.on('data', data => {
|
||||
byteSize += data.length;
|
||||
|
||||
sha1.update(data);
|
||||
sha256.update(data);
|
||||
md5.update(data);
|
||||
crc32.update(data);
|
||||
});
|
||||
|
||||
stream.on('end', () => {
|
||||
fileEntry.meta.byte_size = byteSize;
|
||||
|
||||
// sha-1 is in basic file entry
|
||||
fileEntry.fileSha1 = sha1.digest('hex');
|
||||
|
||||
// others are meta
|
||||
fileEntry.meta.file_sha256 = sha256.digest('hex');
|
||||
fileEntry.meta.file_md5 = md5.digest('hex');
|
||||
fileEntry.meta.file_crc32 = crc32.finalize().toString(16);
|
||||
|
||||
return callback(null);
|
||||
});
|
||||
|
||||
stream.on('error', err => {
|
||||
return callback(err);
|
||||
});
|
||||
},
|
||||
function fetchExistingEntry(callback) {
|
||||
getExistingFileEntriesBySha1(fileEntry.fileSha1, (err, existingEntries) => {
|
||||
return callback(err, existingEntries);
|
||||
});
|
||||
},
|
||||
function addOrUpdate(existingEntries, callback) {
|
||||
if(existingEntries.length > 0) {
|
||||
|
||||
} else {
|
||||
return addNewFileEntry(fileEntry, filePath, callback);
|
||||
}
|
||||
},
|
||||
],
|
||||
err => {
|
||||
return cb(err);
|
||||
}
|
||||
);
|
||||
}
|
||||
*/
|
||||
|
||||
function scanFileAreaForChanges(areaInfo, cb) {
|
||||
const storageLocations = getAreaStorageLocations(areaInfo);
|
||||
|
||||
|
|
@ -551,13 +568,13 @@ function scanFileAreaForChanges(areaInfo, cb) {
|
|||
areaTag : areaInfo.areaTag,
|
||||
storageTag : storageLoc.storageTag
|
||||
},
|
||||
(err, fileEntry, existingEntries) => {
|
||||
(err, fileEntry, dupeEntries) => {
|
||||
if(err) {
|
||||
// :TODO: Log me!!!
|
||||
return nextFile(null); // try next anyway
|
||||
}
|
||||
|
||||
if(existingEntries.length > 0) {
|
||||
if(dupeEntries.length > 0) {
|
||||
// :TODO: Handle duplidates -- what to do here???
|
||||
} else {
|
||||
addNewFileEntry(fileEntry, fullPath, err => {
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ const _ = require('lodash');
|
|||
const paths = require('path');
|
||||
|
||||
const FILE_TABLE_MEMBERS = [
|
||||
'file_id', 'area_tag', 'file_sha1', 'file_name', 'storage_tag',
|
||||
'file_id', 'area_tag', 'file_sha256', 'file_name', 'storage_tag',
|
||||
'desc', 'desc_long', 'upload_timestamp'
|
||||
];
|
||||
|
||||
|
|
@ -21,7 +21,7 @@ const FILE_WELL_KNOWN_META = {
|
|||
upload_by_username : null,
|
||||
upload_by_user_id : null,
|
||||
file_md5 : null,
|
||||
file_sha256 : null,
|
||||
file_sha1 : null,
|
||||
file_crc32 : null,
|
||||
est_release_year : (y) => parseInt(y) || new Date().getFullYear(),
|
||||
dl_count : (d) => parseInt(d) || 0,
|
||||
|
|
@ -100,9 +100,9 @@ module.exports = class FileEntry {
|
|||
},
|
||||
function storeEntry(callback) {
|
||||
fileDb.run(
|
||||
`REPLACE INTO file (area_tag, file_sha1, file_name, storage_tag, desc, desc_long, upload_timestamp)
|
||||
`REPLACE INTO file (area_tag, file_sha256, file_name, storage_tag, desc, desc_long, upload_timestamp)
|
||||
VALUES(?, ?, ?, ?, ?, ?, ?);`,
|
||||
[ self.areaTag, self.fileSha1, self.fileName, self.storageTag, self.desc, self.descLong, getISOTimestampString() ],
|
||||
[ self.areaTag, self.fileSha256, self.fileName, self.storageTag, self.desc, self.descLong, getISOTimestampString() ],
|
||||
function inserted(err) { // use non-arrow func for 'this' scope / lastID
|
||||
if(!err) {
|
||||
self.fileId = this.lastID;
|
||||
|
|
|
|||
|
|
@ -1,17 +1,18 @@
|
|||
/* jslint node: true */
|
||||
'use strict';
|
||||
|
||||
var PluginModule = require('./plugin_module.js').PluginModule;
|
||||
var theme = require('./theme.js');
|
||||
var ansi = require('./ansi_term.js');
|
||||
var ViewController = require('./view_controller.js').ViewController;
|
||||
var menuUtil = require('./menu_util.js');
|
||||
var Config = require('./config.js').config;
|
||||
const PluginModule = require('./plugin_module.js').PluginModule;
|
||||
const theme = require('./theme.js');
|
||||
const ansi = require('./ansi_term.js');
|
||||
const ViewController = require('./view_controller.js').ViewController;
|
||||
const menuUtil = require('./menu_util.js');
|
||||
const Config = require('./config.js').config;
|
||||
const stringFormat = require('../core/string_format.js');
|
||||
|
||||
// deps
|
||||
var async = require('async');
|
||||
var assert = require('assert');
|
||||
var _ = require('lodash');
|
||||
const async = require('async');
|
||||
const assert = require('assert');
|
||||
const _ = require('lodash');
|
||||
|
||||
exports.MenuModule = MenuModule;
|
||||
|
||||
|
|
@ -387,3 +388,27 @@ MenuModule.prototype.prepViewControllerWithArt = function(name, formId, options,
|
|||
}
|
||||
);
|
||||
};
|
||||
|
||||
MenuModule.prototype.setViewText = function(formName, mciId, text) {
|
||||
const view = this.viewControllers[formName].getView(mciId);
|
||||
if(view) {
|
||||
view.setText(text);
|
||||
}
|
||||
};
|
||||
|
||||
MenuModule.prototype.updateCustomViewTextsWithFilter = function(formName, startId, fmtObj, filter) {
|
||||
let textView;
|
||||
let customMciId = startId;
|
||||
const config = this.menuConfig.config;
|
||||
|
||||
while( (textView = this.viewControllers[formName].getView(customMciId)) ) {
|
||||
const key = `${formName}InfoFormat${customMciId}`;
|
||||
const format = config[key];
|
||||
|
||||
if(format && (!filter || filter.find(f => format.indexOf(f) > - 1))) {
|
||||
textView.setText(stringFormat(format, fmtObj));
|
||||
}
|
||||
|
||||
++customMciId;
|
||||
}
|
||||
};
|
||||
|
|
@ -195,6 +195,17 @@ function getPredefinedMCIValue(client, code) {
|
|||
//
|
||||
// :TODO: System stat log for total ul/dl, total ul/dl bytes
|
||||
|
||||
// :TODO: PT - Messages posted *today* (Obv/2)
|
||||
// :TODO: NT - New users today (Obv/2)
|
||||
// :TODO: CT - Calls *today* (Obv/2)
|
||||
// :TODO: TF - Total files on the system (Obv/2)
|
||||
// :TODO: FT - Files uploaded/added *today* (Obv/2)
|
||||
// :TODO: DD - Files downloaded *today* (iNiQUiTY)
|
||||
// :TODO: TP - total message/posts on the system (Obv/2)
|
||||
// :TODO: LC - name of last caller to system (Obv/2)
|
||||
// :TODO: TZ - Average *system* post/call ratio (iNiQUiTY)
|
||||
|
||||
|
||||
//
|
||||
// Special handling for XY
|
||||
//
|
||||
|
|
|
|||
|
|
@ -439,12 +439,12 @@ function TelnetClient(input, output) {
|
|||
};
|
||||
|
||||
this.setTemporaryDataHandler = function(handler) {
|
||||
this.input.removeAllListeners();
|
||||
this.input.removeAllListeners('data');
|
||||
this.input.on('data', handler);
|
||||
};
|
||||
|
||||
this.restoreDataHandler = function() {
|
||||
this.input.removeAllListeners();
|
||||
this.input.removeAllListeners('data');
|
||||
this.input.on('data', this.dataHandler);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -351,7 +351,13 @@ exports.getModule = class TransferFileModule extends MenuModule {
|
|||
});
|
||||
|
||||
this.client.setTemporaryDataHandler(data => {
|
||||
externalProc.write(data);
|
||||
// needed for things like sz/rz
|
||||
if(external.escapeTelnet) {
|
||||
const tmp = data.toString('binary').replace(/\xff{2}/g, '\xff'); // de-escape
|
||||
externalProc.write(new Buffer(tmp, 'binary'));
|
||||
} else {
|
||||
externalProc.write(data);
|
||||
}
|
||||
});
|
||||
|
||||
//this.client.term.output.pipe(externalProc);
|
||||
|
|
@ -359,7 +365,7 @@ exports.getModule = class TransferFileModule extends MenuModule {
|
|||
externalProc.on('data', data => {
|
||||
// needed for things like sz/rz
|
||||
if(external.escapeTelnet) {
|
||||
const tmp = data.toString('binary').replace(/\xff/g, '\xff\xff');
|
||||
const tmp = data.toString('binary').replace(/\xff/g, '\xff\xff'); // escape
|
||||
this.client.term.rawWrite(new Buffer(tmp, 'binary'));
|
||||
} else {
|
||||
this.client.term.rawWrite(data);
|
||||
|
|
@ -484,7 +490,6 @@ exports.getModule = class TransferFileModule extends MenuModule {
|
|||
StatLog.incrementSystemStat('ul_total_count', uploadCount);
|
||||
StatLog.incrementSystemStat('ul_total_bytes', uploadBytes);
|
||||
|
||||
|
||||
return cb(null);
|
||||
});
|
||||
}
|
||||
|
|
@ -556,12 +561,16 @@ exports.getModule = class TransferFileModule extends MenuModule {
|
|||
self.client.log.warn( { error : err.message }, 'File transfer error');
|
||||
}
|
||||
|
||||
return self.prevMenu();
|
||||
/*
|
||||
|
||||
// Wait for a key press - attempt to avoid issues with some terminals after xfer
|
||||
// :TODO: display ANSI if it exists else prompt -- look @ Obv/2 for filename
|
||||
self.client.term.pipeWrite('|00|07\nTransfer(s) complete. Press a key\n');
|
||||
self.client.waitForKeyPress( () => {
|
||||
return self.prevMenu();
|
||||
});
|
||||
*/
|
||||
}
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,7 @@
|
|||
/* jslint node: true */
|
||||
'use strict';
|
||||
|
||||
let uuid = require('node-uuid');
|
||||
let assert = require('assert');
|
||||
let _ = require('lodash');
|
||||
let createHash = require('crypto').createHash;
|
||||
const createHash = require('crypto').createHash;
|
||||
|
||||
exports.createNamedUUID = createNamedUUID;
|
||||
|
||||
|
|
@ -13,9 +10,9 @@ function createNamedUUID(namespaceUuid, key) {
|
|||
// v5 UUID generation code based on the work here:
|
||||
// https://github.com/download13/uuidv5/blob/master/uuid.js
|
||||
//
|
||||
if(!Buffer.isBuffer(namespaceUuid)) {
|
||||
namespaceUuid = new Buffer(namespaceUuid);
|
||||
}
|
||||
if(!Buffer.isBuffer(namespaceUuid)) {
|
||||
namespaceUuid = new Buffer(namespaceUuid);
|
||||
}
|
||||
|
||||
if(!Buffer.isBuffer(key)) {
|
||||
key = new Buffer(key);
|
||||
|
|
|
|||
|
|
@ -46,26 +46,30 @@ const FormIds = {
|
|||
|
||||
const MciViewIds = {
|
||||
browse : {
|
||||
desc : 1,
|
||||
navMenu : 2,
|
||||
// 10+ = customs
|
||||
desc : 1,
|
||||
navMenu : 2,
|
||||
|
||||
customRangeStart : 10, // 10+ = customs
|
||||
},
|
||||
details : {
|
||||
navMenu : 1,
|
||||
infoXyTop : 2, // %XY starting position for info area
|
||||
infoXyBottom : 3,
|
||||
// 10+ = customs
|
||||
navMenu : 1,
|
||||
infoXyTop : 2, // %XY starting position for info area
|
||||
infoXyBottom : 3,
|
||||
|
||||
customRangeStart : 10, // 10+ = customs
|
||||
},
|
||||
detailsGeneral : {
|
||||
// 10+ = customs
|
||||
customRangeStart : 10, // 10+ = customs
|
||||
},
|
||||
detailsNfo : {
|
||||
nfo : 1,
|
||||
// 10+ = customs
|
||||
|
||||
customRangeStart : 10, // 10+ = customs
|
||||
},
|
||||
detailsFileList : {
|
||||
fileList : 1,
|
||||
// 10+ = customs
|
||||
|
||||
customRangeStart : 10, // 10+ = customs
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -163,7 +167,7 @@ exports.getModule = class FileAreaList extends MenuModule {
|
|||
areaTag : currEntry.areaTag,
|
||||
areaName : area.name || 'N/A',
|
||||
areaDesc : area.desc || 'N/A',
|
||||
fileSha1 : currEntry.fileSha1,
|
||||
fileSha256 : currEntry.fileSha256,
|
||||
fileName : currEntry.fileName,
|
||||
desc : currEntry.desc || '',
|
||||
descLong : currEntry.descLong || '',
|
||||
|
|
@ -220,9 +224,10 @@ exports.getModule = class FileAreaList extends MenuModule {
|
|||
}
|
||||
|
||||
populateCustomLabels(category, startId) {
|
||||
return this.updateCustomLabelsWithFilter(category, startId);
|
||||
return this.updateCustomViewTextsWithFilter(category, startId, this.currentFileEntry.entryInfo);
|
||||
}
|
||||
|
||||
/*
|
||||
updateCustomLabelsWithFilter(category, startId, filter) {
|
||||
let textView;
|
||||
let customMciId = startId;
|
||||
|
|
@ -239,6 +244,7 @@ exports.getModule = class FileAreaList extends MenuModule {
|
|||
++customMciId;
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
displayArtAndPrepViewController(name, options, cb) {
|
||||
const self = this;
|
||||
|
|
@ -342,7 +348,7 @@ exports.getModule = class FileAreaList extends MenuModule {
|
|||
descView.setText( self.currentFileEntry.desc );
|
||||
|
||||
self.updateQueueIndicator();
|
||||
self.populateCustomLabels('browse', 10);
|
||||
self.populateCustomLabels('browse', MciViewIds.browse.customRangeStart);
|
||||
|
||||
return callback(null);
|
||||
}
|
||||
|
|
@ -350,7 +356,7 @@ exports.getModule = class FileAreaList extends MenuModule {
|
|||
}
|
||||
} else {
|
||||
self.updateQueueIndicator();
|
||||
self.populateCustomLabels('browse', 10);
|
||||
self.populateCustomLabels('browse', MciViewIds.browse.customRangeStart);
|
||||
|
||||
return callback(null);
|
||||
}
|
||||
|
|
@ -373,7 +379,7 @@ exports.getModule = class FileAreaList extends MenuModule {
|
|||
return self.displayArtAndPrepViewController('details', { clearScreen : true }, callback);
|
||||
},
|
||||
function populateViews(callback) {
|
||||
self.populateCustomLabels('details', 10);
|
||||
self.populateCustomLabels('details', MciViewIds.details.customRangeStart);
|
||||
return callback(null);
|
||||
},
|
||||
function prepSection(callback) {
|
||||
|
|
@ -438,7 +444,11 @@ exports.getModule = class FileAreaList extends MenuModule {
|
|||
);
|
||||
},
|
||||
function updateActiveViews(callback) {
|
||||
self.updateCustomLabelsWithFilter( 'browse', 10, [ '{webDlLink}', '{webDlExpire}' ] );
|
||||
self.updateCustomViewTextsWithFilter(
|
||||
'browse',
|
||||
MciViewIds.browse.customRangeStart, self.currentFileEntry.entryInfo,
|
||||
[ '{webDlLink}', '{webDlExpire}' ]
|
||||
);
|
||||
return callback(null);
|
||||
}
|
||||
],
|
||||
|
|
@ -458,7 +468,12 @@ exports.getModule = class FileAreaList extends MenuModule {
|
|||
isNotQueuedIndicator
|
||||
);
|
||||
|
||||
this.updateCustomLabelsWithFilter( 'browse', 10, [ '{isQueued}' ] );
|
||||
this.updateCustomViewTextsWithFilter(
|
||||
'browse',
|
||||
MciViewIds.browse.customRangeStart,
|
||||
this.currentFileEntry.entryInfo,
|
||||
[ '{isQueued}' ]
|
||||
);
|
||||
}
|
||||
|
||||
cacheArchiveEntries(cb) {
|
||||
|
|
@ -564,7 +579,7 @@ exports.getModule = class FileAreaList extends MenuModule {
|
|||
break;
|
||||
}
|
||||
|
||||
self.populateCustomLabels(name, 10);
|
||||
self.populateCustomLabels(name, MciViewIds[name].customRangeStart);
|
||||
return callback(null);
|
||||
}
|
||||
],
|
||||
|
|
|
|||
141
mods/upload.js
141
mods/upload.js
|
|
@ -40,15 +40,16 @@ const MciViewIds = {
|
|||
},
|
||||
|
||||
processing : {
|
||||
// 10+ = customs
|
||||
stepIndicator : 1,
|
||||
customRangeStart : 10, // 10+ = customs
|
||||
},
|
||||
|
||||
fileDetails : {
|
||||
desc : 1, // defaults to 'desc' (e.g. from FILE_ID.DIZ)
|
||||
tags : 2, // tag(s) for item
|
||||
estYear : 3,
|
||||
accept : 4, // accept fields & continue
|
||||
// 10+ = customs
|
||||
desc : 1, // defaults to 'desc' (e.g. from FILE_ID.DIZ)
|
||||
tags : 2, // tag(s) for item
|
||||
estYear : 3,
|
||||
accept : 4, // accept fields & continue
|
||||
customRangeStart : 10, // 10+ = customs
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -151,8 +152,66 @@ exports.getModule = class UploadModule extends MenuModule {
|
|||
if(this.isFileTransferComplete()) {
|
||||
return this.processUploadedFiles();
|
||||
}
|
||||
}
|
||||
|
||||
updateScanStepInfoViews(stepInfo) {
|
||||
// :TODO: add some blinking (e.g. toggle items) indicators - see OBV.DOC
|
||||
|
||||
const fmtObj = Object.assign( {}, stepInfo);
|
||||
let stepIndicatorFmt = '';
|
||||
|
||||
switch(stepInfo.step) {
|
||||
case 'start' :
|
||||
stepIndicatorFmt = this.menuConfig.config.scanningStartFormat || 'Scanning {fileName}';
|
||||
break;
|
||||
|
||||
case 'hash_update' :
|
||||
stepIndicatorFmt = this.menuConfig.calcHashFormat || 'Calculating hash/checksums: {calcHashPercent}%';
|
||||
|
||||
this.scanStatus.hashUpdateCount += 1;
|
||||
fmtObj.calcHashPercent = Math.round(((stepInfo.bytesProcessed / stepInfo.byteSize) * 100)).toString();
|
||||
|
||||
if(this.scanStatus.hashUpdateCount % 2) {
|
||||
fmtObj.calcHashIndicator = this.menuConfig.config.hashUpdateIndicator1Fmt || '-';
|
||||
} else {
|
||||
fmtObj.calcHashIndicator = this.menuConfig.config.hashUpdateIndicator2Fmt || '*';
|
||||
}
|
||||
break;
|
||||
|
||||
case 'hash_finish' :
|
||||
stepIndicatorFmt = this.menuConfig.calcHashCompleteFormat || 'Finished calculating hash/checksums';
|
||||
break;
|
||||
|
||||
case 'archive_list_start' :
|
||||
stepIndicatorFmt = this.menuConfig.extractArchiveListFormat || 'Extracting archive list';
|
||||
break;
|
||||
|
||||
case 'archive_list_finish' :
|
||||
fmtObj.archivedFileCount = stepInfo.archiveEntries.length;
|
||||
stepIndicatorFmt = this.menuConfig.extractArchiveListFinishFormat || 'Archive list extracted ({archivedFileCount} files)';
|
||||
break;
|
||||
|
||||
case 'archive_list_failed' :
|
||||
stepIndicatorFmt = this.menuConfig.extractArchiveListFailedFormat || 'Archive list extraction failed';
|
||||
break;
|
||||
|
||||
case 'desc_files_start' :
|
||||
stepIndicatorFmt = this.menuConfig.processingDescFilesFormat || 'Processing description files';
|
||||
break;
|
||||
|
||||
case 'desc_files_finish' :
|
||||
stepIndicatorFmt = this.menuConfig.processingDescFilesFinishFormat || 'Finished processing description files';
|
||||
break;
|
||||
}
|
||||
|
||||
const stepIndicatorText = stringFormat(stepIndicatorFmt, fmtObj);
|
||||
|
||||
if(this.hasProcessingArt) {
|
||||
this.setViewText('processing', MciViewIds.processing.stepIndicator, stepIndicatorText);
|
||||
this.updateCustomViewTextsWithFilter('processing', MciViewIds.processing.customRangeStart, fmtObj);
|
||||
} else {
|
||||
this.client.term.pipeWrite(`${stepIndicatorText}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
scanFiles(cb) {
|
||||
|
|
@ -166,35 +225,36 @@ exports.getModule = class UploadModule extends MenuModule {
|
|||
async.eachSeries(this.recvFilePaths, (filePath, nextFilePath) => {
|
||||
// :TODO: virus scanning/etc. should occur around here
|
||||
|
||||
// :TODO: update scanning status art or display line "scanning {fileName}..." type of thing
|
||||
self.scanStatus = {
|
||||
hashUpdateCount : 0,
|
||||
};
|
||||
|
||||
self.client.term.pipeWrite(`|00|07\nScanning ${paths.basename(filePath)}...`);
|
||||
const scanOpts = {
|
||||
areaTag : self.areaInfo.areaTag,
|
||||
storageTag : self.areaInfo.storageTags[0],
|
||||
};
|
||||
|
||||
scanFile(
|
||||
filePath,
|
||||
{
|
||||
areaTag : self.areaInfo.areaTag,
|
||||
storageTag : self.areaInfo.storageTags[0],
|
||||
},
|
||||
(err, fileEntry, existingEntries) => {
|
||||
if(err) {
|
||||
return nextFilePath(err);
|
||||
}
|
||||
function handleScanStep(stepInfo, nextScanStep) {
|
||||
self.updateScanStepInfoViews(stepInfo);
|
||||
return nextScanStep(null);
|
||||
}
|
||||
|
||||
self.client.term.pipeWrite(' done\n');
|
||||
|
||||
// new or dupe?
|
||||
if(existingEntries.length > 0) {
|
||||
// 1:n dupes found
|
||||
results.dupes = results.dupes.concat(existingEntries);
|
||||
} else {
|
||||
// new one
|
||||
results.newEntries.push(fileEntry);
|
||||
}
|
||||
|
||||
return nextFilePath(null);
|
||||
scanFile(filePath, scanOpts, handleScanStep, (err, fileEntry, dupeEntries) => {
|
||||
if(err) {
|
||||
return nextFilePath(err);
|
||||
}
|
||||
);
|
||||
|
||||
// new or dupe?
|
||||
if(dupeEntries.length > 0) {
|
||||
// 1:n dupes found
|
||||
results.dupes = results.dupes.concat(dupeEntries);
|
||||
} else {
|
||||
// new one
|
||||
results.newEntries.push(fileEntry);
|
||||
}
|
||||
|
||||
return nextFilePath(null);
|
||||
});
|
||||
}, err => {
|
||||
return cb(err, results);
|
||||
});
|
||||
|
|
@ -258,7 +318,11 @@ exports.getModule = class UploadModule extends MenuModule {
|
|||
}
|
||||
],
|
||||
err => {
|
||||
console.log('eh'); // :TODO: remove me :)
|
||||
if(err) {
|
||||
self.client.log.warn('File upload error encountered', { error : err.message } );
|
||||
}
|
||||
|
||||
return self.prevMenu();
|
||||
}
|
||||
);
|
||||
}
|
||||
|
|
@ -312,8 +376,17 @@ exports.getModule = class UploadModule extends MenuModule {
|
|||
}
|
||||
|
||||
displayProcessingPage(cb) {
|
||||
// :TODO: If art is supplied, display & start processing + update status/etc.; if no art, we'll just write each status update on a new line
|
||||
return cb(null);
|
||||
return this.prepViewControllerWithArt(
|
||||
'processing',
|
||||
FormIds.processing,
|
||||
{ clearScreen : true, trailingLF : false },
|
||||
err => {
|
||||
// note: this art is not required
|
||||
this.hasProcessingArt = !err;
|
||||
|
||||
return cb(null);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
fileEntryHasDetectedDesc(fileEntry) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue