moon
/
enigma-bbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Temporary fix for MAJOR global temp cleanup bug: comment out node-temp .cleanup() methods 

* Don't move temp uploads to area directory until user submit/completed
* New file util module
This commit is contained in:
Bryan Ashby 2017-01-23 23:32:40 -07:00
parent 414095a9fd
commit 1c03c3021a
8 changed files with 226 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
core/file_area.js
View File

@ -339,9 +339,13 @@ function populateFileEntryWithArchive(fileEntry, filePath, stepInfo, iterator, c
});
}, () => {
// cleanup, but don't wait...
/*
:TODO: fix global temp cleanup issue!!!
temp.cleanup( err => {
// :TODO: Log me!
});
});*/
return callback(null);
});
},

62
core/file_util.js Normal file
View File

@ -0,0 +1,62 @@
/* jslint node: true */
'use strict';
// ENiGMA½
// deps
const fse = require('fs-extra');
const paths = require('path');
const async = require('async');
exports.moveFileWithCollisionHandling = moveFileWithCollisionHandling;
exports.pathWithTerminatingSeparator = pathWithTerminatingSeparator;
//
// Move |src| -> |dst| renaming to file(1).ext, file(2).ext, etc.
// in the case of collisions.
//
function moveFileWithCollisionHandling(src, dst, cb) {
const dstPath = paths.dirname(dst);
const dstFileExt = paths.extname(dst);
const dstFileSuffix = paths.basename(dst, dstFileExt);
let renameIndex = 0;
let movedOk = false;
let tryDstPath;
async.until(
() => movedOk, // until moved OK
(cb) => {
if(0 === renameIndex) {
// try originally supplied path first
tryDstPath = dst;
} else {
tryDstPath = paths.join(dstPath, `${dstFileSuffix}(${renameIndex})${dstFileExt}`);
}
fse.move(src, tryDstPath, err => {
if(err) {
if('EEXIST' === err.code) {
renameIndex += 1;
return cb(null); // keep trying
}
return cb(err);
}
movedOk = true;
return cb(null, tryDstPath);
});
},
(err, finalPath) => {
return cb(err, finalPath);
}
);
}
function pathWithTerminatingSeparator(path) {
if(path && paths.sep !== path.charAt(path.length - 1)) {
path = path + paths.sep;
}
return path;
}

6
core/menu_module.js
View File

@ -95,12 +95,6 @@ require('./mod_mixins.js').ViewControllerManagement.call(MenuModule.prototype);
MenuModule.prototype.enter = function() {
if(_.isString(this.menuConfig.desc)) {
this.client.currentStatus = this.menuConfig.desc;
} else {
this.client.currentStatus = 'Browsing menus';
}
this.initSequence();
};

1
core/predefined_mci.js
View File

@ -115,7 +115,6 @@ function getPredefinedMCIValue(client, code) {
},
MS : function accountCreated() { return moment(client.user.properties.account_created).format(client.currentTheme.helpers.getDateFormat()); },
CS : function currentStatus() { return client.currentStatus; },
PS : function userPostCount() { return userStatAsString(client, 'post_count', 0); },
PC : function userPostCallRatio() { return getRatio(client, 'post_count', 'login_count'); },

6
core/scanner_tossers/ftn_bso.js
View File

@ -1290,6 +1290,9 @@ FTNMessageScanTossModule.prototype.shutdown = function(cb) {
//
// Clean up temp dir/files we created
//
/*
:TODO: fix global temp cleanup issue!!!
temp.cleanup((err, stats) => {
const fullStats = Object.assign(stats, { exportTemp : this.exportTempDir, importTemp : this.importTempDir } );
@ -1301,6 +1304,9 @@ FTNMessageScanTossModule.prototype.shutdown = function(cb) {
FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb);
});
*/
FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb);
};
FTNMessageScanTossModule.prototype.performImport = function(cb) {

2
core/servers/login/telnet.js
View File

@ -438,7 +438,7 @@ function TelnetClient(input, output) {
newEnvironRequested : false,
};
this.setTemporaryDataHandler = function(handler) {
this.setTemporaryDirectDataHandler = function(handler) {
this.input.removeAllListeners('data');
this.input.on('data', handler);
};

143
core/transfer_file.js
View File

@ -19,6 +19,10 @@ const paths = require('path');
const fs = require('fs');
const fse = require('fs-extra');
// some consts
const SYSTEM_EOL = require('os').EOL;
const TEMP_SUFFIX = 'enigtf-'; // temp CWD/etc.
/*
Resources
@ -195,7 +199,7 @@ exports.getModule = class TransferFileModule extends MenuModule {
}
recvFiles(cb) {
this.executeExternalProtocolHandlerForRecv( (err, tempWorkingDir) => {
this.executeExternalProtocolHandlerForRecv(err => {
if(err) {
return cb(err);
}
@ -203,42 +207,39 @@ exports.getModule = class TransferFileModule extends MenuModule {
this.recvFilePaths = [];
if(this.recvFileName) {
// file name specified - we expect a single file in |tempWorkingDir|
// file name specified - we expect a single file in |this.recvDirectory|
// :TODO: support non-blind: Move file to dest path, add to recvFilePaths, etc.
return cb(null);
} else {
//
// blind recv (upload) - files in |tempWorkingDir| should be named appropriately already
// move files to |this.recvDirectory|
// Blind Upload (recv): files in |this.recvDirectory| should be named appropriately already
//
fs.readdir(tempWorkingDir, (err, files) => {
fs.readdir(this.recvDirectory, (err, files) => {
if(err) {
return cb(err);
}
async.each(files, (file, nextFile) => {
this.moveFileWithCollisionHandling(
paths.join(tempWorkingDir, file),
paths.join(this.recvDirectory, file),
(err, destPath) => {
if(err) {
this.client.log.warn(
{ tempWorkingDir : tempWorkingDir, recvDirectory : this.recvDirectory, file : file, error : err.message },
'Failed to move upload file to destination directory'
);
} else {
this.recvFilePaths.push(destPath);
}
// stat each to grab files only
async.each(files, (fileName, nextFile) => {
const recvFullPath = paths.join(this.recvDirectory, fileName);
return nextFile(null); // don't pass along err; try next
fs.stat(recvFullPath, (err, stats) => {
if(err) {
this.client.log.warn('Failed to stat file', { path : recvFullPath } );
return nextFile(null); // just try the next one
}
);
if(stats.isFile()) {
this.recvFilePaths.push(recvFullPath);
}
return nextFile(null);
});
}, () => {
return cb(null);
});
});
});
}
});
@ -254,36 +255,25 @@ exports.getModule = class TransferFileModule extends MenuModule {
prepAndBuildSendArgs(filePaths, cb) {
const external = this.protocolConfig.external;
const externalArgs = external[`${this.direction}Args`];
const self = this;
let tempWorkingDir;
async.waterfall(
[
function getTempFileListPath(callback) {
const hasFileList = externalArgs.find(ea => (ea.indexOf('{fileListPath}') > -1) );
if(!hasFileList) {
temp.mkdir('enigdl-', (err, tempDir) => {
if(err) {
return callback(err);
}
tempWorkingDir = self.pathWithTerminatingSeparator(tempDir);
return callback(null, null);
});
} else {
temp.open( { prefix : 'enigdl-', suffix : '.txt' }, (err, tempFileInfo) => {
if(err) {
return callback(err); // failed to create it
}
tempWorkingDir = self.pathWithTerminatingSeparator(paths.dirname(tempFileInfo.path));
fs.write(tempFileInfo.fd, filePaths.join('\n'));
fs.close(tempFileInfo.fd, err => {
return callback(err, tempFileInfo.path);
});
});
return callback(null, null);
}
temp.open( { prefix : TEMP_SUFFIX, suffix : '.txt' }, (err, tempFileInfo) => {
if(err) {
return callback(err); // failed to create it
}
fs.write(tempFileInfo.fd, filePaths.join(SYSTEM_EOL));
fs.close(tempFileInfo.fd, err => {
return callback(err, tempFileInfo.path);
});
});
},
function createArgs(tempFileListPath, callback) {
// initial args: ignore {filePaths} as we must break that into it's own sep array items
@ -303,54 +293,37 @@ exports.getModule = class TransferFileModule extends MenuModule {
}
],
(err, args) => {
return cb(err, args, tempWorkingDir);
return cb(err, args);
}
);
}
prepAndBuildRecvArgs(cb) {
const self = this;
const externalArgs = this.protocolConfig.external[`${this.direction}Args`];
const args = externalArgs.map(arg => stringFormat(arg, {
uploadDir : this.recvDirectory,
fileName : this.recvFileName || '',
}));
async.waterfall(
[
function getTempRecvPath(callback) {
temp.mkdir('enigrcv-', (err, tempWorkingDir) => {
tempWorkingDir = self.pathWithTerminatingSeparator(tempWorkingDir);
return callback(err, tempWorkingDir);
});
},
function createArgs(tempWorkingDir, callback) {
const externalArgs = self.protocolConfig.external[`${self.direction}Args`];
const args = externalArgs.map(arg => stringFormat(arg, {
uploadDir : tempWorkingDir,
fileName : self.recvFileName || '',
}));
return callback(null, args, tempWorkingDir);
}
],
(err, args, tempWorkingDir) => {
return cb(err, args, tempWorkingDir);
}
);
return cb(null, args);
}
executeExternalProtocolHandler(args, tempWorkingDir, cb) {
executeExternalProtocolHandler(args, cb) {
const external = this.protocolConfig.external;
const cmd = external[`${this.direction}Cmd`];
this.client.log.debug(
{ cmd : cmd, args : args, tempDir : tempWorkingDir, direction : this.direction },
{ cmd : cmd, args : args, tempDir : this.recvDirectory, direction : this.direction },
'Executing external protocol'
);
const externalProc = pty.spawn(cmd, args, {
cols : this.client.term.termWidth,
rows : this.client.term.termHeight,
cwd : tempWorkingDir,
cwd : this.recvDirectory,
});
this.client.setTemporaryDataHandler(data => {
this.client.setTemporaryDirectDataHandler(data => {
// needed for things like sz/rz
if(external.escapeTelnet) {
const tmp = data.toString('binary').replace(/\xff{2}/g, '\xff'); // de-escape
@ -360,8 +333,6 @@ exports.getModule = class TransferFileModule extends MenuModule {
}
});
//this.client.term.output.pipe(externalProc);
externalProc.on('data', data => {
// needed for things like sz/rz
if(external.escapeTelnet) {
@ -391,25 +362,25 @@ exports.getModule = class TransferFileModule extends MenuModule {
filePaths = [ filePaths ];
}
this.prepAndBuildSendArgs(filePaths, (err, args, tempWorkingDir) => {
this.prepAndBuildSendArgs(filePaths, (err, args) => {
if(err) {
return cb(err);
}
this.executeExternalProtocolHandler(args, tempWorkingDir, err => {
this.executeExternalProtocolHandler(args, err => {
return cb(err);
});
});
}
executeExternalProtocolHandlerForRecv(cb) {
this.prepAndBuildRecvArgs( (err, args, tempWorkingDir) => {
this.prepAndBuildRecvArgs( (err, args) => {
if(err) {
return cb(err);
}
this.executeExternalProtocolHandler(args, tempWorkingDir, err => {
return cb(err, tempWorkingDir);
this.executeExternalProtocolHandler(args, err => {
return cb(err);
});
});
}
@ -541,12 +512,15 @@ exports.getModule = class TransferFileModule extends MenuModule {
}
},
function cleanupTempFiles(callback) {
/* :TODO: figure out the global temp cleanup() issue!!@!
temp.cleanup( err => {
if(err) {
self.client.log.warn( { error : err.message }, 'Failed to clean up temporary file/directory(s)' );
}
return callback(null); // ignore err
});
*/
return callback(null);
},
function updateUserAndSystemStats(callback) {
if(self.isSending()) {
@ -562,15 +536,6 @@ exports.getModule = class TransferFileModule extends MenuModule {
}
return self.prevMenu();
/*
// Wait for a key press - attempt to avoid issues with some terminals after xfer
// :TODO: display ANSI if it exists else prompt -- look @ Obv/2 for filename
self.client.term.pipeWrite('|00|07\nTransfer(s) complete. Press a key\n');
self.client.waitForKeyPress( () => {
return self.prevMenu();
});
*/
}
);
}

138
mods/upload.js
View File

@ -8,10 +8,14 @@ const getSortedAvailableFileAreas = require('../core/file_area.js').getSortedAv
const getAreaDefaultStorageDirectory = require('../core/file_area.js').getAreaDefaultStorageDirectory;
const scanFile = require('../core/file_area.js').scanFile;
const ansiGoto = require('../core/ansi_term.js').goto;
const moveFileWithCollisionHandling = require('../core/file_util.js').moveFileWithCollisionHandling;
const pathWithTerminatingSeparator = require('../core/file_util.js').pathWithTerminatingSeparator;
// deps
const async = require('async');
const _ = require('lodash');
const temp = require('temp').track(); // track() cleans up temp dir/files for us
const paths = require('path');
exports.moduleInfo = {
name : 'Upload',
@ -65,21 +69,12 @@ exports.getModule = class UploadModule extends MenuModule {
this.menuMethods = {
optionsNavContinue : (formData, extraArgs, cb) => {
if(this.isBlindUpload()) {
// jump to protocol selection
const areaUploadDir = this.getSelectedAreaUploadDirectory();
const modOpts = {
extraArgs : {
recvDirectory : areaUploadDir,
direction : 'recv',
}
};
return this.gotoMenu(this.menuConfig.config.fileTransferProtocolSelection || 'fileTransferProtocolSelection', modOpts, cb);
} else {
// jump to fileDetails form
// :TODO: support non-blind: collect info/filename -> upload -> complete
return this.performBlindUpload(cb);
}
// non-blind
// jump to fileDetails form
// :TODO: support non-blind: collect info/filename -> upload -> complete
},
fileDetailsContinue : (formData, extraArgs, cb) => {
@ -94,12 +89,13 @@ exports.getModule = class UploadModule extends MenuModule {
getSaveState() {
const saveState = {
uploadType : this.uploadType,
uploadType : this.uploadType,
tempRecvDirectory : this.tempRecvDirectory
};
if(this.isBlindUpload()) {
saveState.areaInfo = this.getSelectedAreaInfo();
const areaSelectView = this.viewControllers.options.getView(MciViewIds.options.area);
saveState.areaInfo = this.availAreas[areaSelectView.getData()];
}
return saveState;
@ -107,20 +103,11 @@ exports.getModule = class UploadModule extends MenuModule {
restoreSavedState(savedState) {
if(savedState.areaInfo) {
this.areaInfo = savedState.areaInfo;
this.areaInfo = savedState.areaInfo;
this.tempRecvDirectory = savedState.tempRecvDirectory;
}
}
getSelectedAreaInfo() {
const areaSelectView = this.viewControllers.options.getView(MciViewIds.options.area);
return this.availAreas[areaSelectView.getData()];
}
getSelectedAreaUploadDirectory() {
const areaInfo = this.getSelectedAreaInfo();
return getAreaDefaultStorageDirectory(areaInfo);
}
isBlindUpload() { return 'blind' === this.uploadType; }
isFileTransferComplete() { return !_.isUndefined(this.recvFilePaths); }
@ -152,6 +139,44 @@ exports.getModule = class UploadModule extends MenuModule {
}
}
leave() {
// remove any temp files - only do this when
if(this.isFileTransferComplete()) {
// :TODO: fix global temp cleanup issue!!!
//temp.cleanup(); // remove any temp files
}
super.leave();
}
performBlindUpload(cb) {
temp.mkdir('enigul-', (err, tempRecvDirectory) => {
if(err) {
return cb(err);
}
// need a terminator for various external protocols
this.tempRecvDirectory = pathWithTerminatingSeparator(tempRecvDirectory);
const modOpts = {
extraArgs : {
recvDirectory : this.tempRecvDirectory, // we'll move files from here to their area container once processed/confirmed
direction : 'recv',
}
};
//
// Move along to protocol selection -> file transfer
// Upon completion, we'll re-enter the module with some file paths handed to us
//
return this.gotoMenu(
this.menuConfig.config.fileTransferProtocolSelection || 'fileTransferProtocolSelection',
modOpts,
cb
);
});
}
updateScanStepInfoViews(stepInfo) {
// :TODO: add some blinking (e.g. toggle items) indicators - see OBV.DOC
@ -240,6 +265,8 @@ exports.getModule = class UploadModule extends MenuModule {
dupes : [],
};
self.client.log.debug('Scanning upload(s)', { paths : this.recvFilePaths } );
async.eachSeries(this.recvFilePaths, (filePath, nextFilePath) => {
// :TODO: virus scanning/etc. should occur around here
@ -257,7 +284,7 @@ exports.getModule = class UploadModule extends MenuModule {
return nextScanStep(null);
}
self.client.log.debug('Scanning upload', { filePath : filePath } );
self.client.log.debug('Scanning file', { filePath : filePath } );
scanFile(filePath, scanOpts, handleScanStep, (err, fileEntry, dupeEntries) => {
if(err) {
@ -267,7 +294,7 @@ exports.getModule = class UploadModule extends MenuModule {
// new or dupe?
if(dupeEntries.length > 0) {
// 1:n dupes found
self.client.log.debug('Duplicate(s) of upload found', { dupeEntries : dupeEntries } );
self.client.log.debug('Duplicate file(s) found', { dupeEntries : dupeEntries } );
results.dupes = results.dupes.concat(dupeEntries);
} else {
@ -282,6 +309,38 @@ exports.getModule = class UploadModule extends MenuModule {
});
}
moveAndPersistUploadsToDatabase(newEntries) {
const areaStorageDir = getAreaDefaultStorageDirectory(this.areaInfo);
const self = this;
async.eachSeries(newEntries, (newEntry, nextEntry) => {
const src = paths.join(self.tempRecvDirectory, newEntry.fileName);
const dst = paths.join(areaStorageDir, newEntry.fileName);
moveFileWithCollisionHandling(src, dst, (err, finalPath) => {
if(err) {
self.client.log.error(
'Failed moving physical upload file', { error : err.message, fileName : newEntry.fileName, source : src, dest : dst }
);
return nextEntry(null); // still try next file
}
self.client.log.debug('Moved upload to area', { path : finalPath } );
// persist to DB
newEntry.persist(err => {
if(err) {
self.client.log.error('Failed persisting upload to database', { path : finalPath, error : err.message } );
}
return nextEntry(null); // still try next file
});
});
});
}
processUploadedFiles() {
//
// For each file uploaded, we need to process & gather information
@ -339,16 +398,15 @@ exports.getModule = class UploadModule extends MenuModule {
return callback(err, scanResults);
});
},
function persistNewEntries(scanResults, callback) {
// loop over entries again & persist to DB
async.eachSeries(scanResults.newEntries, (newEntry, nextEntry) => {
newEntry.persist(err => {
return nextEntry(err);
});
}, err => {
return callback(err);
});
}
function startMovingAndPersistingToDatabase(scanResults, callback) {
//
// *Start* the process of moving files from their current |tempRecvDirectory|
// locations -> their final area destinations. Don't make the user wait
// here as I/O can take quite a bit of time. Log any failures.
//
self.moveAndPersistUploadsToDatabase(scanResults.newEntries);
return callback(null);
},
],
err => {
if(err) {