* Bump version to 0.0.2-alpha: File Base alpha here

* LHA/LZH archive support via external lha command
* Nearly complete upload processor
* Set default file base filter if none is set
* Additional MenuModule common method/helpers
* MLTEV property: tabSwitchesView
This commit is contained in:
Bryan Ashby 2017-01-22 21:30:49 -07:00
parent 8d51c7d47c
commit 99036592ae
14 changed files with 269 additions and 109 deletions

View File

@ -248,9 +248,36 @@ function getDefaultConfig() {
cmd : '7za',
args : [ 'e', '-o{extractPath}', '{archivePath}', '{fileList}' ],
},
},
Lha : {
//
// 'lha' command can be obtained from:
// * apt-get: lhasa
//
// (compress not currently supported)
//
decompress : {
cmd : 'lha',
args : [ '-ew={extractPath}', '{archivePath}' ],
},
list : {
cmd : 'lha',
args : [ '-l', '{archivePath}' ],
entryMatch : '^[\\[a-z\\]]+(?:\\s+[0-9]+\\s+[0-9]+|\\s+)([0-9]+)\\s+[0-9]{2}\\.[0-9]\\%\\s+[A-Za-z]{3}\\s+[0-9]{1,2}\\s+[0-9]{4}\\s+([^\\r\\n]+)$',
},
extract : {
cmd : 'lha',
args : [ '-ew={extractPath}', '{archivePath}', '{fileList}' ]
}
}
},
formats : {
//
// Resources
// * http://www.garykessler.net/library/file_sigs.html
//
zip : {
sig : '504b0304',
offset : 0,
@ -285,6 +312,20 @@ function getDefaultConfig() {
exts : [ 'gz' ],
handler : '7Zip',
desc : 'Gzip Archive',
},
bzip : {
sig : '425a68',
offset : 0,
exts : [ 'bz2' ],
handler : '7Zip',
desc : 'BZip2 Archive',
},
lzh : {
sig : '2d6c68',
offset : 2,
exts : [ 'lzh', 'ice' ],
handler : 'Lha',
desc : 'LHArc Archive',
}
}
},
@ -399,7 +440,7 @@ function getDefaultConfig() {
'[0-3]?[0-9][\\-\\/\\.][0-3]?[0-9][\\-\\/\\.]((?:[0-9]{2})?[0-9]{2})', // m/d/yyyy, mm-dd-yyyy, etc.
"\\B('[1789][0-9])\\b", // eslint-disable-line quotes
'[0-3]?[0-9][\\-\\/\\.](?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec|january|february|march|april|may|june|july|august|september|october|november|december)[\\-\\/\\.]((?:[0-9]{2})?[0-9]{2})',
'(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec|january|february|march|april|may|june|july|august|september|october|november|december),?\\s[0-9]+(?:st|nd|rd|th)\\s((?:[0-9]{2})?[0-9]{2})', // November 29th, 1997
'(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec|january|february|march|april|may|june|july|august|september|october|november|december),?\\s[0-9]+(?:st|nd|rd|th)?,?\\s((?:[0-9]{2})?[0-9]{2})', // November 29th, 1997
// :TODO: DD/MMM/YY, DD/MMMM/YY, DD/MMM/YYYY, etc.
],

View File

@ -294,6 +294,10 @@ function populateFileEntryWithArchive(fileEntry, filePath, stepInfo, iterator, c
extractList.push(longDescFile.fileName);
}
if(0 === extractList.length) {
return callback(null, [] );
}
temp.mkdir('enigextract-', (err, tempDir) => {
if(err) {
return callback(err);
@ -423,6 +427,9 @@ function scanFile(filePath, options, iterator, cb) {
});
}
let lastCalcHashPercent;
async.waterfall(
[
function startScan(callback) {
@ -449,10 +456,28 @@ function scanFile(filePath, options, iterator, cb) {
const stream = fs.createReadStream(filePath);
function updateHashes(data) {
async.each( HASH_NAMES, (hashName, nextHash) => {
hashes[hashName].update(data);
return nextHash(null);
}, () => {
return stream.resume();
});
}
stream.on('data', data => {
stream.pause(); // until iterator compeltes
stepInfo.bytesProcessed += data.length;
stepInfo.calcHashPercent = Math.round(((stepInfo.bytesProcessed / stepInfo.byteSize) * 100));
//
// Only send 'hash_update' step update if we have a noticable percentage change in progress
//
if(stepInfo.calcHashPercent === lastCalcHashPercent) {
updateHashes(data);
} else {
lastCalcHashPercent = stepInfo.calcHashPercent;
stepInfo.step = 'hash_update';
callIter(err => {
@ -461,13 +486,9 @@ function scanFile(filePath, options, iterator, cb) {
return callback(err);
}
async.each( HASH_NAMES, (hashName, nextHash) => {
hashes[hashName].update(data);
return nextHash(null);
}, () => {
return stream.resume();
});
updateHashes(data);
});
}
});
stream.on('end', () => {

View File

@ -61,15 +61,29 @@ module.exports = class FileBaseFilters {
delete this.filters[filterUuid];
}
load(prop) {
prop = prop || this.client.user.properties.file_base_filters;
load() {
let filtersProperty = this.client.user.properties.file_base_filters;
let defaulted;
if(!filtersProperty) {
filtersProperty = JSON.stringify(FileBaseFilters.getDefaultFilters());
defaulted = true;
}
try {
this.filters = JSON.parse(prop);
this.filters = JSON.parse(filtersProperty);
} catch(e) {
this.filters = {};
this.filters = FileBaseFilters.getDefaultFilters(); // something bad happened; reset everything back to defaults :(
defaulted = true;
this.client.log.error( { error : e.message, property : filtersProperty }, 'Failed parsing file base filters property' );
}
this.client.log.error( { error : e.message, property : prop }, 'Failed parsing file base filters property' );
if(defaulted) {
this.persist( err => {
if(!err) {
const defaultActiveUuid = this.toArray()[0].uuid;
this.setActive(defaultActiveUuid);
}
});
}
}
@ -93,6 +107,23 @@ module.exports = class FileBaseFilters {
return false;
}
static getDefaultFilters() {
const filters = {};
const uuid = uuids.v4();
filters[uuid] = {
name : 'Default',
areaTag : '', // all
terms : '', // *
tags : '', // *
order : 'ascending',
sort : 'upload_timestamp',
uuid : uuid,
};
return filters;
}
static getActiveFilter(client) {
return new FileBaseFilters(client).get(client.user.properties.file_base_filter_active_uuid);
}

View File

@ -280,7 +280,7 @@ module.exports = class FileEntry {
sqlWhere += clause;
}
if(filter.sort) {
if(filter.sort && filter.sort.length > 0) {
if(Object.keys(FILE_WELL_KNOWN_META).indexOf(filter.sort) > -1) { // sorting via a meta value?
sql =
`SELECT f.file_id
@ -294,7 +294,7 @@ module.exports = class FileEntry {
`SELECT f.file_id, f.${filter.sort}
FROM file f`;
sqlOrderBy = getOrderByWithCast(`f.${filter.sort}`) + sqlOrderDir;
sqlOrderBy = getOrderByWithCast(`f.${filter.sort}`) + ' ' + sqlOrderDir;
}
} else {
sql =
@ -305,11 +305,11 @@ module.exports = class FileEntry {
}
if(filter.areaTag) {
if(filter.areaTag && filter.areaTag.length > 0) {
appendWhereClause(`f.area_tag="${filter.areaTag}"`);
}
if(filter.terms) {
if(filter.terms && filter.terms.length > 0) {
appendWhereClause(
`f.file_id IN (
SELECT rowid
@ -319,7 +319,7 @@ module.exports = class FileEntry {
);
}
if(filter.tags) {
if(filter.tags && filter.tags.length > 0) {
// build list of quoted tags; filter.tags comes in as a space separated values
const tags = filter.tags.split(' ').map( tag => `"${tag}"` ).join(',');

View File

@ -8,6 +8,7 @@ const ViewController = require('./view_controller.js').ViewController;
const menuUtil = require('./menu_util.js');
const Config = require('./config.js').config;
const stringFormat = require('../core/string_format.js');
const MultiLineEditTextView = require('../core/multi_line_edit_text_view.js').MultiLineEditTextView;
// deps
const async = require('async');
@ -183,6 +184,7 @@ MenuModule.prototype.initSequence = function() {
self.client.term.write(ansi.goto(self.afterArtPos[0], 1));
// :TODO: really need a client.term.pause() that uses the correct art/etc.
// :TODO: Use MenuModule.pausePrompt()
theme.displayThemedPause( { client : self.client }, function keyPressed() {
callback(null);
});
@ -389,24 +391,54 @@ MenuModule.prototype.prepViewControllerWithArt = function(name, formId, options,
);
};
MenuModule.prototype.setViewText = function(formName, mciId, text) {
MenuModule.prototype.pausePrompt = function(position, cb) {
if(!cb && _.isFunction(position)) {
cb = position;
position = null;
}
if(position) {
position.x = position.row || position.x || 1;
position.y = position.col || position.y || 1;
this.client.term.rawWrite(ansi.goto(position.x, position.y));
}
theme.displayThemedPause( { client : this.client }, cb);
};
MenuModule.prototype.setViewText = function(formName, mciId, text, appendMultiline) {
const view = this.viewControllers[formName].getView(mciId);
if(view) {
if(!view) {
return;
}
if(appendMultiline && (view instanceof MultiLineEditTextView)) {
view.addText(text);
} else {
view.setText(text);
}
};
MenuModule.prototype.updateCustomViewTextsWithFilter = function(formName, startId, fmtObj, filter) {
MenuModule.prototype.updateCustomViewTextsWithFilter = function(formName, startId, fmtObj, options) {
options = options || {};
let textView;
let customMciId = startId;
const config = this.menuConfig.config;
while( (textView = this.viewControllers[formName].getView(customMciId)) ) {
const key = `${formName}InfoFormat${customMciId}`;
const key = `${formName}InfoFormat${customMciId}`; // e.g. "mainInfoFormat10"
const format = config[key];
if(format && (!filter || filter.find(f => format.indexOf(f) > - 1))) {
textView.setText(stringFormat(format, fmtObj));
if(format && (!options.filter || options.filter.find(f => format.indexOf(f) > - 1))) {
const text = stringFormat(format, fmtObj);
if(options.appendMultiLine && (textView instanceof MultiLineEditTextView)) {
textView.addText(text);
} else {
textView.setText(text);
}
}
++customMciId;

View File

@ -115,8 +115,10 @@ function MultiLineEditTextView(options) {
if ('preview' === this.mode) {
this.autoScroll = options.autoScroll || true;
this.tabSwitchesView = true;
} else {
this.autoScroll = options.autoScroll || false;
this.tabSwitchesView = options.tabSwitchesView || false;
}
//
// cursorPos represents zero-based row, col positions
@ -284,7 +286,7 @@ function MultiLineEditTextView(options) {
});
return text;
}
};
this.getContiguousText = function(startIndex, endIndex, includeEol) {
var lines = self.getTextLines(startIndex, endIndex);
@ -532,7 +534,7 @@ function MultiLineEditTextView(options) {
// before and and after column
//
// :TODO: Need to clean this string (e.g. collapse tabs)
text = self.textLines
text = self.textLines;
// :TODO: Remove original line @ index
}
@ -544,15 +546,15 @@ function MultiLineEditTextView(options) {
.replace(/\b/g, '')
.split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);
var wrapped;
let wrapped;
for(var i = 0; i < text.length; ++i) {
for(let i = 0; i < text.length; ++i) {
wrapped = self.wordWrapSingleLine(
text[i], // input
'expand', // tabHandling
self.dimens.width).wrapped;
for(var j = 0; j < wrapped.length - 1; ++j) {
for(let j = 0; j < wrapped.length - 1; ++j) {
self.textLines.splice(index++, 0, { text : wrapped[j] } );
}
self.textLines.splice(index++, 0, { text : wrapped[wrapped.length - 1], eol : true } );
@ -1029,13 +1031,20 @@ MultiLineEditTextView.prototype.setPropertyValue = function(propName, value) {
this.specialKeyMap.next = [ 'tab' ];
}
break;
case 'autoScroll' : this.autoScroll = value; break;
case 'tabSwitchesView' :
this.tabSwitchesView = value;
this.specialKeyMap.next = this.specialKeyMap.next || [];
this.specialKeyMap.next.push('tab');
break;
}
MultiLineEditTextView.super_.prototype.setPropertyValue.call(this, propName, value);
};
var HANDLED_SPECIAL_KEYS = [
const HANDLED_SPECIAL_KEYS = [
'up', 'down', 'left', 'right',
'home', 'end',
'page up', 'page down',
@ -1046,13 +1055,13 @@ var HANDLED_SPECIAL_KEYS = [
'delete line',
];
var PREVIEW_MODE_KEYS = [
const PREVIEW_MODE_KEYS = [
'up', 'down', 'page up', 'page down'
];
MultiLineEditTextView.prototype.onKeyPress = function(ch, key) {
var self = this;
var handled;
const self = this;
let handled;
if(key) {
HANDLED_SPECIAL_KEYS.forEach(function aKey(specialKey) {
@ -1062,9 +1071,11 @@ MultiLineEditTextView.prototype.onKeyPress = function(ch, key) {
return;
}
if('tab' !== key.name || !self.tabSwitchesView) {
self[_.camelCase('keyPress ' + specialKey)]();
handled = true;
}
}
});
}

View File

@ -13,6 +13,7 @@ exports.stylizeString = stylizeString;
exports.pad = pad;
exports.replaceAt = replaceAt;
exports.isPrintable = isPrintable;
exports.stripAllLineFeeds = stripAllLineFeeds;
exports.debugEscapedString = debugEscapedString;
exports.stringFromNullTermBuffer = stringFromNullTermBuffer;
exports.renderSubstr = renderSubstr;
@ -189,6 +190,10 @@ function stringLength(s) {
return s.length;
}
function stripAllLineFeeds(s) {
return s.replace(/\r?\n|[\r\u2028\u2029]/g, '');
}
function debugEscapedString(s) {
return JSON.stringify(s).slice(1, -1);
}

View File

@ -10,6 +10,7 @@ const stylizeString = require('./string_util.js').stylizeString;
const renderSubstr = require('./string_util.js').renderSubstr;
const renderStringLength = require('./string_util.js').renderStringLength;
const pipeToAnsi = require('./color_codes.js').pipeToAnsi;
const stripAllLineFeeds = require('./string_util.js').stripAllLineFeeds;
// deps
const util = require('util');
@ -183,7 +184,7 @@ TextView.prototype.setText = function(text, redraw) {
text = text.toString();
}
text = pipeToAnsi(text, this.client); // expand MCI/etc.
text = pipeToAnsi(stripAllLineFeeds(text), this.client); // expand MCI/etc.
var widthDelta = 0;
if(this.text && this.text !== text) {

View File

@ -99,6 +99,7 @@ function AbracadabraModule(options) {
if(_.isString(self.config.tooManyArt)) {
theme.displayThemeArt( { client : self.client, name : self.config.tooManyArt }, function displayed() {
// :TODO: Use MenuModule.pausePrompt()
theme.displayThemedPause( { client : self.client }, function keyPressed() {
callback(new Error('Too many active instances'));
});
@ -106,6 +107,7 @@ function AbracadabraModule(options) {
} else {
self.client.term.write('\nToo many active instances. Try again later.\n');
// :TODO: Use MenuModule.pausePrompt()
theme.displayThemedPause( { client : self.client }, function keyPressed() {
callback(new Error('Too many active instances'));
});

View File

@ -227,25 +227,6 @@ exports.getModule = class FileAreaList extends MenuModule {
return this.updateCustomViewTextsWithFilter(category, startId, this.currentFileEntry.entryInfo);
}
/*
updateCustomLabelsWithFilter(category, startId, filter) {
let textView;
let customMciId = startId;
const config = this.menuConfig.config;
while( (textView = this.viewControllers[category].getView(customMciId)) ) {
const key = `${category}InfoFormat${customMciId}`;
const format = config[key];
if(format && (!filter || filter.find(f => format.indexOf(f) > - 1))) {
textView.setText(stringFormat(format, this.currentFileEntry.entryInfo));
}
++customMciId;
}
}
*/
displayArtAndPrepViewController(name, options, cb) {
const self = this;
const config = this.menuConfig.config;
@ -447,7 +428,7 @@ exports.getModule = class FileAreaList extends MenuModule {
self.updateCustomViewTextsWithFilter(
'browse',
MciViewIds.browse.customRangeStart, self.currentFileEntry.entryInfo,
[ '{webDlLink}', '{webDlExpire}' ]
{ filter : [ '{webDlLink}', '{webDlExpire}' ] }
);
return callback(null);
}
@ -472,7 +453,7 @@ exports.getModule = class FileAreaList extends MenuModule {
'browse',
MciViewIds.browse.customRangeStart,
this.currentFileEntry.entryInfo,
[ '{isQueued}' ]
{ filter : [ '{isQueued}' ] }
);
}

View File

@ -84,6 +84,7 @@ function MessageAreaListModule(options) {
if(_.has(area, 'options.pause') && false === area.options.pause) {
return self.prevMenuOnTimeout(1000, cb);
} else {
// :TODO: Use MenuModule.pausePrompt()
displayThemedPause( { client : self.client }, () => {
return self.prevMenu(cb);
});

View File

@ -71,6 +71,7 @@ function MessageConfListModule(options) {
if(_.has(conf, 'options.pause') && false === conf.options.pause) {
return self.prevMenuOnTimeout(1000, cb);
} else {
// :TODO: Use MenuModule.pausePrompt()
displayThemedPause( { client : self.client }, () => {
return self.prevMenu(cb);
});

View File

@ -3,20 +3,15 @@
// enigma-bbs
const MenuModule = require('../core/menu_module.js').MenuModule;
const ViewController = require('../core/view_controller.js').ViewController;
const theme = require('../core/theme.js');
const ansi = require('../core/ansi_term.js');
const Errors = require('../core/enig_error.js').Errors;
const stringFormat = require('../core/string_format.js');
const getSortedAvailableFileAreas = require('../core/file_area.js').getSortedAvailableFileAreas;
const getAreaDefaultStorageDirectory = require('../core/file_area.js').getAreaDefaultStorageDirectory;
const scanFile = require('../core/file_area.js').scanFile;
const getAreaStorageDirectoryByTag = require('../core/file_area.js').getAreaStorageDirectoryByTag;
const ansiGoto = require('../core/ansi_term.js').goto;
// deps
const async = require('async');
const _ = require('lodash');
const paths = require('path');
exports.moduleInfo = {
name : 'Upload',
@ -40,7 +35,10 @@ const MciViewIds = {
},
processing : {
stepIndicator : 1,
calcHashIndicator : 1,
archiveListIndicator : 2,
descFileIndicator : 3,
customRangeStart : 10, // 10+ = customs
},
@ -160,6 +158,26 @@ exports.getModule = class UploadModule extends MenuModule {
const fmtObj = Object.assign( {}, stepInfo);
let stepIndicatorFmt = '';
const indicatorStates = this.menuConfig.config.indicatorStates || [ '|', '/', '-', '\\' ];
const indicatorFinished = this.menuConfig.config.indicatorFinished || '√';
const indicator = { };
const self = this;
function updateIndicator(mci, isFinished) {
indicator.mci = mci;
if(isFinished) {
indicator.text = indicatorFinished;
} else {
self.scanStatus.indicatorPos += 1;
if(self.scanStatus.indicatorPos >= indicatorStates.length) {
self.scanStatus.indicatorPos = 0;
}
indicator.text = indicatorStates[self.scanStatus.indicatorPos];
}
}
switch(stepInfo.step) {
case 'start' :
stepIndicatorFmt = this.menuConfig.config.scanningStartFormat || 'Scanning {fileName}';
@ -167,28 +185,23 @@ exports.getModule = class UploadModule extends MenuModule {
case 'hash_update' :
stepIndicatorFmt = this.menuConfig.calcHashFormat || 'Calculating hash/checksums: {calcHashPercent}%';
this.scanStatus.hashUpdateCount += 1;
fmtObj.calcHashPercent = Math.round(((stepInfo.bytesProcessed / stepInfo.byteSize) * 100)).toString();
if(this.scanStatus.hashUpdateCount % 2) {
fmtObj.calcHashIndicator = this.menuConfig.config.hashUpdateIndicator1Fmt || '-';
} else {
fmtObj.calcHashIndicator = this.menuConfig.config.hashUpdateIndicator2Fmt || '*';
}
updateIndicator(MciViewIds.processing.calcHashIndicator);
break;
case 'hash_finish' :
stepIndicatorFmt = this.menuConfig.calcHashCompleteFormat || 'Finished calculating hash/checksums';
updateIndicator(MciViewIds.processing.calcHashIndicator, true);
break;
case 'archive_list_start' :
stepIndicatorFmt = this.menuConfig.extractArchiveListFormat || 'Extracting archive list';
updateIndicator(MciViewIds.processing.archiveListIndicator);
break;
case 'archive_list_finish' :
fmtObj.archivedFileCount = stepInfo.archiveEntries.length;
stepIndicatorFmt = this.menuConfig.extractArchiveListFinishFormat || 'Archive list extracted ({archivedFileCount} files)';
updateIndicator(MciViewIds.processing.archiveListIndicator, true);
break;
case 'archive_list_failed' :
@ -197,20 +210,25 @@ exports.getModule = class UploadModule extends MenuModule {
case 'desc_files_start' :
stepIndicatorFmt = this.menuConfig.processingDescFilesFormat || 'Processing description files';
updateIndicator(MciViewIds.processing.descFileIndicator);
break;
case 'desc_files_finish' :
stepIndicatorFmt = this.menuConfig.processingDescFilesFinishFormat || 'Finished processing description files';
updateIndicator(MciViewIds.processing.descFileIndicator, true);
break;
}
const stepIndicatorText = stringFormat(stepIndicatorFmt, fmtObj);
fmtObj.stepIndicatorText = stringFormat(stepIndicatorFmt, fmtObj);
if(this.hasProcessingArt) {
this.setViewText('processing', MciViewIds.processing.stepIndicator, stepIndicatorText);
this.updateCustomViewTextsWithFilter('processing', MciViewIds.processing.customRangeStart, fmtObj);
this.updateCustomViewTextsWithFilter('processing', MciViewIds.processing.customRangeStart, fmtObj, { appendMultiLine : true } );
if(indicator.mci && indicator.text) {
this.setViewText('processing', indicator.mci, indicator.text);
}
} else {
this.client.term.pipeWrite(`${stepIndicatorText}\n`);
this.client.term.pipeWrite(fmtObj.stepIndicatorText);
}
}
@ -226,7 +244,7 @@ exports.getModule = class UploadModule extends MenuModule {
// :TODO: virus scanning/etc. should occur around here
self.scanStatus = {
hashUpdateCount : 0,
indicatorPos : 0,
};
const scanOpts = {
@ -239,6 +257,8 @@ exports.getModule = class UploadModule extends MenuModule {
return nextScanStep(null);
}
self.client.log.debug('Scanning upload', { filePath : filePath } );
scanFile(filePath, scanOpts, handleScanStep, (err, fileEntry, dupeEntries) => {
if(err) {
return nextFilePath(err);
@ -247,6 +267,8 @@ exports.getModule = class UploadModule extends MenuModule {
// new or dupe?
if(dupeEntries.length > 0) {
// 1:n dupes found
self.client.log.debug('Duplicate(s) of upload found', { dupeEntries : dupeEntries } );
results.dupes = results.dupes.concat(dupeEntries);
} else {
// new one
@ -271,6 +293,17 @@ exports.getModule = class UploadModule extends MenuModule {
function scan(callback) {
return self.scanFiles(callback);
},
function pause(scanResults, callback) {
if(self.hasProcessingArt) {
self.client.term.rawWrite(ansiGoto(self.client.term.termHeight, 1));
} else {
self.client.term.write('\n');
}
self.pausePrompt( () => {
return callback(null, scanResults);
});
},
function displayDupes(scanResults, callback) {
if(0 === scanResults.dupes.length) {
return callback(null, scanResults);

View File

@ -1,6 +1,6 @@
{
"name": "enigma-bbs",
"version": "0.0.1-alpha",
"version": "0.0.2-alpha",
"description": "ENiGMA½ Bulletin Board System",
"author": "Bryan Ashby <bryan@l33t.codes>",
"license": "BSD-2-Clause",