enigma-bbs/core/upload.js

741 lines
28 KiB
JavaScript

/* jslint node: true */
'use strict';
// enigma-bbs
const MenuModule = require('./menu_module.js').MenuModule;
const stringFormat = require('./string_format.js');
const getSortedAvailableFileAreas = require('./file_base_area.js').getSortedAvailableFileAreas;
const getAreaDefaultStorageDirectory = require('./file_base_area.js').getAreaDefaultStorageDirectory;
const scanFile = require('./file_base_area.js').scanFile;
const getFileAreaByTag = require('./file_base_area.js').getFileAreaByTag;
const getDescFromFileName = require('./file_base_area.js').getDescFromFileName;
const ansiGoto = require('./ansi_term.js').goto;
const moveFileWithCollisionHandling = require('./file_util.js').moveFileWithCollisionHandling;
const pathWithTerminatingSeparator = require('./file_util.js').pathWithTerminatingSeparator;
const Log = require('./logger.js').log;
const Errors = require('./enig_error.js').Errors;
const FileEntry = require('./file_entry.js');
const isAnsi = require('./string_util.js').isAnsi;
const Events = require('./events.js');
// deps
const async = require('async');
const _ = require('lodash');
const temptmp = require('temptmp').createTrackedSession('upload');
const paths = require('path');
const sanatizeFilename = require('sanitize-filename');
exports.moduleInfo = {
name : 'Upload',
desc : 'Module for classic file uploads',
author : 'NuSkooler',
};
const FormIds = {
options : 0,
processing : 1,
fileDetails : 2,
dupes : 3,
};
const MciViewIds = {
options : {
area : 1, // area selection
uploadType : 2, // blind vs specify filename
fileName : 3, // for non-blind; not editable for blind
navMenu : 4, // next/cancel/etc.
errMsg : 5, // errors (e.g. filename cannot be blank)
},
processing : {
calcHashIndicator : 1,
archiveListIndicator : 2,
descFileIndicator : 3,
logStep : 4,
customRangeStart : 10, // 10+ = customs
},
fileDetails : {
desc : 1, // defaults to 'desc' (e.g. from FILE_ID.DIZ)
tags : 2, // tag(s) for item
estYear : 3,
accept : 4, // accept fields & continue
customRangeStart : 10, // 10+ = customs
},
dupes : {
dupeList : 1,
}
};
exports.getModule = class UploadModule extends MenuModule {
constructor(options) {
super(options);
this.interrupt = MenuModule.InterruptTypes.Never;
if(_.has(options, 'lastMenuResult.recvFilePaths')) {
this.recvFilePaths = options.lastMenuResult.recvFilePaths;
}
this.availAreas = getSortedAvailableFileAreas(this.client, { writeAcs : true } );
this.menuMethods = {
optionsNavContinue : (formData, extraArgs, cb) => {
return this.performUpload(cb);
},
fileDetailsContinue : (formData, extraArgs, cb) => {
// see displayFileDetailsPageForUploadEntry() for this hackery:
cb(null);
return this.fileDetailsCurrentEntrySubmitCallback(null, formData.value); // move on to the next entry, if any
},
// validation
validateNonBlindFileName : (fileName, cb) => {
if(0 === fileName.length) {
return cb(new Error('Filename cannot be empty'));
}
fileName = sanatizeFilename(fileName); // remove unsafe chars, path info, etc.
if(0 === fileName.length) { // sanatize nuked everything?
return cb(new Error('Invalid filename'));
}
// At least SEXYZ doesn't like non-blind names that start with a number - it becomes confused ;-(
if(/^[0-9].*$/.test(fileName)) {
return cb(new Error('Invalid filename'));
}
return cb(null);
},
viewValidationListener : (err, cb) => {
const errView = this.viewControllers.options.getView(MciViewIds.options.errMsg);
if(errView) {
if(err) {
errView.setText(err.message);
} else {
errView.clearText();
}
}
return cb(null);
}
};
}
getSaveState() {
// if no areas, we're falling back due to lack of access/areas avail to upload to
if(this.availAreas.length > 0) {
return {
uploadType : this.uploadType,
tempRecvDirectory : this.tempRecvDirectory,
areaInfo : this.availAreas[ this.viewControllers.options.getView(MciViewIds.options.area).getData() ],
};
}
}
restoreSavedState(savedState) {
if(savedState.areaInfo) {
this.uploadType = savedState.uploadType;
this.areaInfo = savedState.areaInfo;
this.tempRecvDirectory = savedState.tempRecvDirectory;
}
}
isBlindUpload() { return 'blind' === this.uploadType; }
isFileTransferComplete() { return !_.isUndefined(this.recvFilePaths); }
initSequence() {
const self = this;
if(0 === this.availAreas.length) {
//
return this.gotoMenu(this.menuConfig.config.noUploadAreasAvailMenu || 'fileBaseNoUploadAreasAvail');
}
async.series(
[
function before(callback) {
return self.beforeArt(callback);
},
function display(callback) {
if(self.isFileTransferComplete()) {
return self.displayProcessingPage(callback);
} else {
return self.displayOptionsPage(callback);
}
}
],
() => {
return self.finishedLoading();
}
);
}
finishedLoading() {
if(this.isFileTransferComplete()) {
return this.processUploadedFiles();
}
}
performUpload(cb) {
temptmp.mkdir( { prefix : 'enigul-' }, (err, tempRecvDirectory) => {
if(err) {
return cb(err);
}
// need a terminator for various external protocols
this.tempRecvDirectory = pathWithTerminatingSeparator(tempRecvDirectory);
const modOpts = {
extraArgs : {
recvDirectory : this.tempRecvDirectory, // we'll move files from here to their area container once processed/confirmed
direction : 'recv',
}
};
if(!this.isBlindUpload()) {
// data has been sanatized at this point
modOpts.extraArgs.recvFileName = this.viewControllers.options.getView(MciViewIds.options.fileName).getData();
}
//
// Move along to protocol selection -> file transfer
// Upon completion, we'll re-enter the module with some file paths handed to us
//
return this.gotoMenu(
this.menuConfig.config.fileTransferProtocolSelection || 'fileTransferProtocolSelection',
modOpts,
cb
);
});
}
continueNonBlindUpload(cb) {
return cb(null);
}
updateScanStepInfoViews(stepInfo) {
// :TODO: add some blinking (e.g. toggle items) indicators - see OBV.DOC
const fmtObj = Object.assign( {}, stepInfo);
let stepIndicatorFmt = '';
let logStepFmt;
const fmtConfig = this.menuConfig.config;
const indicatorStates = fmtConfig.indicatorStates || [ '|', '/', '-', '\\' ];
const indicatorFinished = fmtConfig.indicatorFinished || '√';
const indicator = { };
const self = this;
function updateIndicator(mci, isFinished) {
indicator.mci = mci;
if(isFinished) {
indicator.text = indicatorFinished;
} else {
self.scanStatus.indicatorPos += 1;
if(self.scanStatus.indicatorPos >= indicatorStates.length) {
self.scanStatus.indicatorPos = 0;
}
indicator.text = indicatorStates[self.scanStatus.indicatorPos];
}
}
switch(stepInfo.step) {
case 'start' :
logStepFmt = stepIndicatorFmt = fmtConfig.scanningStartFormat || 'Scanning {fileName}';
break;
case 'hash_update' :
stepIndicatorFmt = fmtConfig.calcHashFormat || 'Calculating hash/checksums: {calcHashPercent}%';
updateIndicator(MciViewIds.processing.calcHashIndicator);
break;
case 'hash_finish' :
stepIndicatorFmt = fmtConfig.calcHashCompleteFormat || 'Finished calculating hash/checksums';
updateIndicator(MciViewIds.processing.calcHashIndicator, true);
break;
case 'archive_list_start' :
stepIndicatorFmt = fmtConfig.extractArchiveListFormat || 'Extracting archive list';
updateIndicator(MciViewIds.processing.archiveListIndicator);
break;
case 'archive_list_finish' :
fmtObj.archivedFileCount = stepInfo.archiveEntries.length;
stepIndicatorFmt = fmtConfig.extractArchiveListFinishFormat || 'Archive list extracted ({archivedFileCount} files)';
updateIndicator(MciViewIds.processing.archiveListIndicator, true);
break;
case 'archive_list_failed' :
stepIndicatorFmt = fmtConfig.extractArchiveListFailedFormat || 'Archive list extraction failed';
break;
case 'desc_files_start' :
stepIndicatorFmt = fmtConfig.processingDescFilesFormat || 'Processing description files';
updateIndicator(MciViewIds.processing.descFileIndicator);
break;
case 'desc_files_finish' :
stepIndicatorFmt = fmtConfig.processingDescFilesFinishFormat || 'Finished processing description files';
updateIndicator(MciViewIds.processing.descFileIndicator, true);
break;
case 'finished' :
logStepFmt = stepIndicatorFmt = fmtConfig.scanningStartFormat || 'Finished';
break;
}
fmtObj.stepIndicatorText = stringFormat(stepIndicatorFmt, fmtObj);
if(this.hasProcessingArt) {
this.updateCustomViewTextsWithFilter('processing', MciViewIds.processing.customRangeStart, fmtObj, { appendMultiLine : true } );
if(indicator.mci && indicator.text) {
this.setViewText('processing', indicator.mci, indicator.text);
}
if(logStepFmt) {
this.setViewText('processing', MciViewIds.processing.logStep, stringFormat(logStepFmt, fmtObj), { appendMultiLine : true } );
}
} else {
this.client.term.pipeWrite(fmtObj.stepIndicatorText);
}
}
scanFiles(cb) {
const self = this;
const results = {
newEntries : [],
dupes : [],
};
self.client.log.debug('Scanning upload(s)', { paths : this.recvFilePaths } );
let currentFileNum = 0;
async.eachSeries(this.recvFilePaths, (filePath, nextFilePath) => {
// :TODO: virus scanning/etc. should occur around here
currentFileNum += 1;
self.scanStatus = {
indicatorPos : 0,
};
const scanOpts = {
areaTag : self.areaInfo.areaTag,
storageTag : self.areaInfo.storageTags[0],
};
function handleScanStep(stepInfo, nextScanStep) {
stepInfo.totalFileNum = self.recvFilePaths.length;
stepInfo.currentFileNum = currentFileNum;
self.updateScanStepInfoViews(stepInfo);
return nextScanStep(null);
}
self.client.log.debug('Scanning file', { filePath : filePath } );
scanFile(filePath, scanOpts, handleScanStep, (err, fileEntry, dupeEntries) => {
if(err) {
return nextFilePath(err);
}
// new or dupe?
if(dupeEntries.length > 0) {
// 1:n dupes found
self.client.log.debug('Duplicate file(s) found', { dupeEntries : dupeEntries } );
results.dupes = results.dupes.concat(dupeEntries);
} else {
// new one
results.newEntries.push(fileEntry);
}
return nextFilePath(null);
});
}, err => {
return cb(err, results);
});
}
cleanupTempFiles() {
temptmp.cleanup( paths => {
Log.debug( { paths : paths, sessionId : temptmp.sessionId }, 'Temporary files cleaned up' );
});
}
moveAndPersistUploadsToDatabase(newEntries) {
const areaStorageDir = getAreaDefaultStorageDirectory(this.areaInfo);
const self = this;
async.eachSeries(newEntries, (newEntry, nextEntry) => {
const src = paths.join(self.tempRecvDirectory, newEntry.fileName);
const dst = paths.join(areaStorageDir, newEntry.fileName);
moveFileWithCollisionHandling(src, dst, (err, finalPath) => {
if(err) {
self.client.log.error(
'Failed moving physical upload file', { error : err.message, fileName : newEntry.fileName, source : src, dest : dst }
);
if(!err && dst !== finalPath) {
// name changed; ajust before persist
newEntry.fileName = paths.basename(finalPath);
}
return nextEntry(null); // still try next file
}
self.client.log.debug('Moved upload to area', { path : finalPath } );
// persist to DB
newEntry.persist(err => {
if(err) {
self.client.log.error('Failed persisting upload to database', { path : finalPath, error : err.message } );
}
return nextEntry(null); // still try next file
});
});
}, () => {
//
// Finally, we can remove any temp files that we may have created
//
self.cleanupTempFiles();
});
}
prepDetailsForUpload(scanResults, cb) {
async.eachSeries(scanResults.newEntries, (newEntry, nextEntry) => {
newEntry.meta.upload_by_username = this.client.user.username;
newEntry.meta.upload_by_user_id = this.client.user.userId;
this.displayFileDetailsPageForUploadEntry(newEntry, (err, newValues) => {
if(err) {
return nextEntry(err);
}
if(!newEntry.descIsAnsi) {
newEntry.desc = _.trimEnd(newValues.shortDesc);
}
if(newValues.estYear.length > 0) {
newEntry.meta.est_release_year = newValues.estYear;
}
if(newValues.tags.length > 0) {
newEntry.setHashTags(newValues.tags);
}
return nextEntry(err);
});
}, err => {
delete this.fileDetailsCurrentEntrySubmitCallback;
return cb(err, scanResults);
});
}
displayDupesPage(dupes, cb) {
//
// If we have custom art to show, use it - else just dump basic info.
// Pause at the end in either case.
//
const self = this;
async.waterfall(
[
function prepArtAndViewController(callback) {
self.prepViewControllerWithArt(
'dupes',
FormIds.dupes,
{ clearScreen : true, trailingLF : false },
err => {
if(err) {
self.client.term.pipeWrite('|00|07Duplicate upload(s) found:\n');
return callback(null, null);
}
const dupeListView = self.viewControllers.dupes.getView(MciViewIds.dupes.dupeList);
return callback(null, dupeListView);
}
);
},
function prepDupeObjects(dupeListView, callback) {
// update dupe objects with additional info that can be used for formatString() and the like
async.each(dupes, (dupe, nextDupe) => {
FileEntry.loadBasicEntry(dupe.fileId, dupe, err => {
if(err) {
return nextDupe(err);
}
const areaInfo = getFileAreaByTag(dupe.areaTag);
if(areaInfo) {
dupe.areaName = areaInfo.name;
dupe.areaDesc = areaInfo.desc;
}
return nextDupe(null);
});
}, err => {
return callback(err, dupeListView);
});
},
function populateDupeInfo(dupeListView, callback) {
const dupeInfoFormat = self.menuConfig.config.dupeInfoFormat || '{fileName} @ {areaName}';
if(dupeListView) {
dupeListView.setItems(dupes.map(dupe => stringFormat(dupeInfoFormat, dupe) ) );
dupeListView.redraw();
} else {
dupes.forEach(dupe => {
self.client.term.pipeWrite(`${stringFormat(dupeInfoFormat, dupe)}\n`);
});
}
return callback(null);
},
function pause(callback) {
return self.pausePrompt( { row : self.client.term.termHeight }, callback);
}
],
err => {
return cb(err);
}
);
}
processUploadedFiles() {
//
// For each file uploaded, we need to process & gather information
//
const self = this;
async.waterfall(
[
function prepNonBlind(callback) {
if(self.isBlindUpload()) {
return callback(null);
}
//
// For non-blind uploads, batch is not supported, we expect a single file
// in |recvFilePaths|. If not, it's an error (we don't want to process the wrong thing)
//
if(self.recvFilePaths.length > 1) {
self.client.log.warn( { recvFilePaths : self.recvFilePaths }, 'Non-blind upload received 2:n files' );
return callback(Errors.UnexpectedState(`Non-blind upload expected single file but got received ${self.recvFilePaths.length}`));
}
return callback(null);
},
function scan(callback) {
return self.scanFiles(callback);
},
function pause(scanResults, callback) {
if(self.hasProcessingArt) {
self.client.term.rawWrite(ansiGoto(self.client.term.termHeight, 1));
} else {
self.client.term.write('\n');
}
self.pausePrompt( () => {
return callback(null, scanResults);
});
},
function displayDupes(scanResults, callback) {
if(0 === scanResults.dupes.length) {
return callback(null, scanResults);
}
return self.displayDupesPage(scanResults.dupes, () => {
return callback(null, scanResults);
});
},
function prepDetails(scanResults, callback) {
return self.prepDetailsForUpload(scanResults, callback);
},
function startMovingAndPersistingToDatabase(scanResults, callback) {
//
// *Start* the process of moving files from their current |tempRecvDirectory|
// locations -> their final area destinations. Don't make the user wait
// here as I/O can take quite a bit of time. Log any failures.
//
self.moveAndPersistUploadsToDatabase(scanResults.newEntries);
return callback(null, scanResults.newEntries);
},
function sendEvent(uploadedEntries, callback) {
Events.emit(
Events.getSystemEvents().UserUpload,
{
user : self.client.user,
files : uploadedEntries,
}
);
return callback(null);
}
],
err => {
if(err) {
self.client.log.warn('File upload error encountered', { error : err.message } );
self.cleanupTempFiles(); // normally called after moveAndPersistUploadsToDatabase() is completed.
}
return self.prevMenu();
}
);
}
displayOptionsPage(cb) {
const self = this;
async.series(
[
function prepArtAndViewController(callback) {
return self.prepViewControllerWithArt(
'options',
FormIds.options,
{ clearScreen : true, trailingLF : false },
callback
);
},
function populateViews(callback) {
const areaSelectView = self.viewControllers.options.getView(MciViewIds.options.area);
areaSelectView.setItems( self.availAreas.map(areaInfo => areaInfo.name ) );
const uploadTypeView = self.viewControllers.options.getView(MciViewIds.options.uploadType);
const fileNameView = self.viewControllers.options.getView(MciViewIds.options.fileName);
const blindFileNameText = self.menuConfig.config.blindFileNameText || '(blind - filename ignored)';
uploadTypeView.on('index update', idx => {
self.uploadType = (0 === idx) ? 'blind' : 'non-blind';
if(self.isBlindUpload()) {
fileNameView.setText(blindFileNameText);
fileNameView.acceptsFocus = false;
} else {
fileNameView.clearText();
fileNameView.acceptsFocus = true;
}
});
// sanatize filename for display when leaving the view
self.viewControllers.options.on('leave', prevView => {
if(prevView.id === MciViewIds.options.fileName) {
fileNameView.setText(sanatizeFilename(fileNameView.getData()));
}
});
self.uploadType = 'blind';
uploadTypeView.setFocusItemIndex(0); // default to blind
fileNameView.setText(blindFileNameText);
areaSelectView.redraw();
return callback(null);
}
],
err => {
if(cb) {
return cb(err);
}
}
);
}
displayProcessingPage(cb) {
return this.prepViewControllerWithArt(
'processing',
FormIds.processing,
{ clearScreen : true, trailingLF : false },
err => {
// note: this art is not required
this.hasProcessingArt = !err;
return cb(null);
}
);
}
fileEntryHasDetectedDesc(fileEntry) {
return (fileEntry.desc && fileEntry.desc.length > 0);
}
displayFileDetailsPageForUploadEntry(fileEntry, cb) {
const self = this;
async.waterfall(
[
function prepArtAndViewController(callback) {
return self.prepViewControllerWithArt(
'fileDetails',
FormIds.fileDetails,
{ clearScreen : true, trailingLF : false },
err => {
return callback(err);
}
);
},
function populateViews(callback) {
const descView = self.viewControllers.fileDetails.getView(MciViewIds.fileDetails.desc);
const tagsView = self.viewControllers.fileDetails.getView(MciViewIds.fileDetails.tags);
const yearView = self.viewControllers.fileDetails.getView(MciViewIds.fileDetails.estYear);
self.updateCustomViewTextsWithFilter('fileDetails', MciViewIds.fileDetails.customRangeStart, fileEntry );
tagsView.setText( Array.from(fileEntry.hashTags).join(',') ); // :TODO: optional 'hashTagsSep' like file list/browse
yearView.setText(fileEntry.meta.est_release_year || '');
if(isAnsi(fileEntry.desc)) {
fileEntry.descIsAnsi = true;
return descView.setAnsi(
fileEntry.desc,
{
prepped : false,
forceLineTerm : true,
},
() => {
return callback(null, descView, 'preview', MciViewIds.fileDetails.tags);
}
);
} else {
const hasDesc = self.fileEntryHasDetectedDesc(fileEntry);
descView.setText(
hasDesc ? fileEntry.desc : getDescFromFileName(fileEntry.fileName),
{ scrollMode : 'top' } // override scroll mode; we want to be @ top
);
return callback(null, descView, 'edit', hasDesc ? MciViewIds.fileDetails.tags : MciViewIds.fileDetails.desc);
}
},
function finalizeViews(descView, descViewMode, focusId, callback) {
descView.setPropertyValue('mode', descViewMode);
descView.acceptsFocus = 'preview' === descViewMode ? false : true;
self.viewControllers.fileDetails.switchFocus(focusId);
return callback(null);
}
],
err => {
//
// we only call |cb| here if there is an error
// else, wait for the current from to be submit - then call -
// this way we'll move on to the next file entry when ready
//
if(err) {
return cb(err);
}
self.fileDetailsCurrentEntrySubmitCallback = cb; // stash for moduleMethods.fileDetailsContinue
}
);
}
};