From 1c03c3021a2ad3c327fcde7efebb039fb55b5374 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Mon, 23 Jan 2017 23:32:40 -0700 Subject: [PATCH] * Temporary fix for MAJOR global temp cleanup bug: comment out node-temp .cleanup() methods * Don't move temp uploads to area directory until user submit/completed * New file util module --- core/file_area.js | 6 +- core/file_util.js | 62 ++++++++++++++ core/menu_module.js | 6 -- core/predefined_mci.js | 1 - core/scanner_tossers/ftn_bso.js | 6 ++ core/servers/login/telnet.js | 2 +- core/transfer_file.js | 143 ++++++++++++-------------------- mods/upload.js | 138 +++++++++++++++++++++--------- 8 files changed, 226 insertions(+), 138 deletions(-) create mode 100644 core/file_util.js diff --git a/core/file_area.js b/core/file_area.js index 637847d6..99e4a80f 100644 --- a/core/file_area.js +++ b/core/file_area.js @@ -339,9 +339,13 @@ function populateFileEntryWithArchive(fileEntry, filePath, stepInfo, iterator, c }); }, () => { // cleanup, but don't wait... + /* + :TODO: fix global temp cleanup issue!!! + temp.cleanup( err => { // :TODO: Log me! - }); + });*/ + return callback(null); }); }, diff --git a/core/file_util.js b/core/file_util.js new file mode 100644 index 00000000..e2ea6e90 --- /dev/null +++ b/core/file_util.js @@ -0,0 +1,62 @@ +/* jslint node: true */ +'use strict'; + +// ENiGMA½ + +// deps +const fse = require('fs-extra'); +const paths = require('path'); +const async = require('async'); + +exports.moveFileWithCollisionHandling = moveFileWithCollisionHandling; +exports.pathWithTerminatingSeparator = pathWithTerminatingSeparator; + +// +// Move |src| -> |dst| renaming to file(1).ext, file(2).ext, etc. +// in the case of collisions. +// +function moveFileWithCollisionHandling(src, dst, cb) { + const dstPath = paths.dirname(dst); + const dstFileExt = paths.extname(dst); + const dstFileSuffix = paths.basename(dst, dstFileExt); + + let renameIndex = 0; + let movedOk = false; + let tryDstPath; + + async.until( + () => movedOk, // until moved OK + (cb) => { + if(0 === renameIndex) { + // try originally supplied path first + tryDstPath = dst; + } else { + tryDstPath = paths.join(dstPath, `${dstFileSuffix}(${renameIndex})${dstFileExt}`); + } + + fse.move(src, tryDstPath, err => { + if(err) { + if('EEXIST' === err.code) { + renameIndex += 1; + return cb(null); // keep trying + } + + return cb(err); + } + + movedOk = true; + return cb(null, tryDstPath); + }); + }, + (err, finalPath) => { + return cb(err, finalPath); + } + ); +} + +function pathWithTerminatingSeparator(path) { + if(path && paths.sep !== path.charAt(path.length - 1)) { + path = path + paths.sep; + } + return path; +} diff --git a/core/menu_module.js b/core/menu_module.js index 213debda..6cc81e4e 100644 --- a/core/menu_module.js +++ b/core/menu_module.js @@ -95,12 +95,6 @@ require('./mod_mixins.js').ViewControllerManagement.call(MenuModule.prototype); MenuModule.prototype.enter = function() { - if(_.isString(this.menuConfig.desc)) { - this.client.currentStatus = this.menuConfig.desc; - } else { - this.client.currentStatus = 'Browsing menus'; - } - this.initSequence(); }; diff --git a/core/predefined_mci.js b/core/predefined_mci.js index 5ed828cc..69463f4d 100644 --- a/core/predefined_mci.js +++ b/core/predefined_mci.js @@ -115,7 +115,6 @@ function getPredefinedMCIValue(client, code) { }, MS : function accountCreated() { return moment(client.user.properties.account_created).format(client.currentTheme.helpers.getDateFormat()); }, - CS : function currentStatus() { return client.currentStatus; }, PS : function userPostCount() { return userStatAsString(client, 'post_count', 0); }, PC : function userPostCallRatio() { return getRatio(client, 'post_count', 'login_count'); }, diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 8cd6e1a0..85fad2f7 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -1290,6 +1290,9 @@ FTNMessageScanTossModule.prototype.shutdown = function(cb) { // // Clean up temp dir/files we created // + /* + :TODO: fix global temp cleanup issue!!! + temp.cleanup((err, stats) => { const fullStats = Object.assign(stats, { exportTemp : this.exportTempDir, importTemp : this.importTempDir } ); @@ -1301,6 +1304,9 @@ FTNMessageScanTossModule.prototype.shutdown = function(cb) { FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb); }); + */ + + FTNMessageScanTossModule.super_.prototype.shutdown.call(this, cb); }; FTNMessageScanTossModule.prototype.performImport = function(cb) { diff --git a/core/servers/login/telnet.js b/core/servers/login/telnet.js index 045c9280..2e27a6f8 100644 --- a/core/servers/login/telnet.js +++ b/core/servers/login/telnet.js @@ -438,7 +438,7 @@ function TelnetClient(input, output) { newEnvironRequested : false, }; - this.setTemporaryDataHandler = function(handler) { + this.setTemporaryDirectDataHandler = function(handler) { this.input.removeAllListeners('data'); this.input.on('data', handler); }; diff --git a/core/transfer_file.js b/core/transfer_file.js index b0ee78d7..28022c06 100644 --- a/core/transfer_file.js +++ b/core/transfer_file.js @@ -19,6 +19,10 @@ const paths = require('path'); const fs = require('fs'); const fse = require('fs-extra'); +// some consts +const SYSTEM_EOL = require('os').EOL; +const TEMP_SUFFIX = 'enigtf-'; // temp CWD/etc. + /* Resources @@ -195,7 +199,7 @@ exports.getModule = class TransferFileModule extends MenuModule { } recvFiles(cb) { - this.executeExternalProtocolHandlerForRecv( (err, tempWorkingDir) => { + this.executeExternalProtocolHandlerForRecv(err => { if(err) { return cb(err); } @@ -203,42 +207,39 @@ exports.getModule = class TransferFileModule extends MenuModule { this.recvFilePaths = []; if(this.recvFileName) { - // file name specified - we expect a single file in |tempWorkingDir| + // file name specified - we expect a single file in |this.recvDirectory| // :TODO: support non-blind: Move file to dest path, add to recvFilePaths, etc. return cb(null); } else { // - // blind recv (upload) - files in |tempWorkingDir| should be named appropriately already - // move files to |this.recvDirectory| + // Blind Upload (recv): files in |this.recvDirectory| should be named appropriately already // - fs.readdir(tempWorkingDir, (err, files) => { + fs.readdir(this.recvDirectory, (err, files) => { if(err) { return cb(err); } - async.each(files, (file, nextFile) => { - this.moveFileWithCollisionHandling( - paths.join(tempWorkingDir, file), - paths.join(this.recvDirectory, file), - (err, destPath) => { - if(err) { - this.client.log.warn( - { tempWorkingDir : tempWorkingDir, recvDirectory : this.recvDirectory, file : file, error : err.message }, - 'Failed to move upload file to destination directory' - ); - } else { - this.recvFilePaths.push(destPath); - } + // stat each to grab files only + async.each(files, (fileName, nextFile) => { + const recvFullPath = paths.join(this.recvDirectory, fileName); - return nextFile(null); // don't pass along err; try next + fs.stat(recvFullPath, (err, stats) => { + if(err) { + this.client.log.warn('Failed to stat file', { path : recvFullPath } ); + return nextFile(null); // just try the next one } - ); + + if(stats.isFile()) { + this.recvFilePaths.push(recvFullPath); + } + + return nextFile(null); + }); }, () => { return cb(null); - }); - + }); }); } }); @@ -254,36 +255,25 @@ exports.getModule = class TransferFileModule extends MenuModule { prepAndBuildSendArgs(filePaths, cb) { const external = this.protocolConfig.external; const externalArgs = external[`${this.direction}Args`]; - const self = this; - let tempWorkingDir; async.waterfall( [ function getTempFileListPath(callback) { const hasFileList = externalArgs.find(ea => (ea.indexOf('{fileListPath}') > -1) ); if(!hasFileList) { - temp.mkdir('enigdl-', (err, tempDir) => { - if(err) { - return callback(err); - } - - tempWorkingDir = self.pathWithTerminatingSeparator(tempDir); - return callback(null, null); - }); - } else { - temp.open( { prefix : 'enigdl-', suffix : '.txt' }, (err, tempFileInfo) => { - if(err) { - return callback(err); // failed to create it - } - - tempWorkingDir = self.pathWithTerminatingSeparator(paths.dirname(tempFileInfo.path)); - - fs.write(tempFileInfo.fd, filePaths.join('\n')); - fs.close(tempFileInfo.fd, err => { - return callback(err, tempFileInfo.path); - }); - }); + return callback(null, null); } + + temp.open( { prefix : TEMP_SUFFIX, suffix : '.txt' }, (err, tempFileInfo) => { + if(err) { + return callback(err); // failed to create it + } + + fs.write(tempFileInfo.fd, filePaths.join(SYSTEM_EOL)); + fs.close(tempFileInfo.fd, err => { + return callback(err, tempFileInfo.path); + }); + }); }, function createArgs(tempFileListPath, callback) { // initial args: ignore {filePaths} as we must break that into it's own sep array items @@ -303,54 +293,37 @@ exports.getModule = class TransferFileModule extends MenuModule { } ], (err, args) => { - return cb(err, args, tempWorkingDir); + return cb(err, args); } ); } prepAndBuildRecvArgs(cb) { - const self = this; + const externalArgs = this.protocolConfig.external[`${this.direction}Args`]; + const args = externalArgs.map(arg => stringFormat(arg, { + uploadDir : this.recvDirectory, + fileName : this.recvFileName || '', + })); - async.waterfall( - [ - function getTempRecvPath(callback) { - temp.mkdir('enigrcv-', (err, tempWorkingDir) => { - tempWorkingDir = self.pathWithTerminatingSeparator(tempWorkingDir); - return callback(err, tempWorkingDir); - }); - }, - function createArgs(tempWorkingDir, callback) { - const externalArgs = self.protocolConfig.external[`${self.direction}Args`]; - const args = externalArgs.map(arg => stringFormat(arg, { - uploadDir : tempWorkingDir, - fileName : self.recvFileName || '', - })); - - return callback(null, args, tempWorkingDir); - } - ], - (err, args, tempWorkingDir) => { - return cb(err, args, tempWorkingDir); - } - ); + return cb(null, args); } - executeExternalProtocolHandler(args, tempWorkingDir, cb) { + executeExternalProtocolHandler(args, cb) { const external = this.protocolConfig.external; const cmd = external[`${this.direction}Cmd`]; this.client.log.debug( - { cmd : cmd, args : args, tempDir : tempWorkingDir, direction : this.direction }, + { cmd : cmd, args : args, tempDir : this.recvDirectory, direction : this.direction }, 'Executing external protocol' ); const externalProc = pty.spawn(cmd, args, { cols : this.client.term.termWidth, rows : this.client.term.termHeight, - cwd : tempWorkingDir, + cwd : this.recvDirectory, }); - this.client.setTemporaryDataHandler(data => { + this.client.setTemporaryDirectDataHandler(data => { // needed for things like sz/rz if(external.escapeTelnet) { const tmp = data.toString('binary').replace(/\xff{2}/g, '\xff'); // de-escape @@ -360,8 +333,6 @@ exports.getModule = class TransferFileModule extends MenuModule { } }); - //this.client.term.output.pipe(externalProc); - externalProc.on('data', data => { // needed for things like sz/rz if(external.escapeTelnet) { @@ -391,25 +362,25 @@ exports.getModule = class TransferFileModule extends MenuModule { filePaths = [ filePaths ]; } - this.prepAndBuildSendArgs(filePaths, (err, args, tempWorkingDir) => { + this.prepAndBuildSendArgs(filePaths, (err, args) => { if(err) { return cb(err); } - this.executeExternalProtocolHandler(args, tempWorkingDir, err => { + this.executeExternalProtocolHandler(args, err => { return cb(err); }); }); } executeExternalProtocolHandlerForRecv(cb) { - this.prepAndBuildRecvArgs( (err, args, tempWorkingDir) => { + this.prepAndBuildRecvArgs( (err, args) => { if(err) { return cb(err); } - this.executeExternalProtocolHandler(args, tempWorkingDir, err => { - return cb(err, tempWorkingDir); + this.executeExternalProtocolHandler(args, err => { + return cb(err); }); }); } @@ -541,12 +512,15 @@ exports.getModule = class TransferFileModule extends MenuModule { } }, function cleanupTempFiles(callback) { + /* :TODO: figure out the global temp cleanup() issue!!@! temp.cleanup( err => { if(err) { self.client.log.warn( { error : err.message }, 'Failed to clean up temporary file/directory(s)' ); } return callback(null); // ignore err }); + */ + return callback(null); }, function updateUserAndSystemStats(callback) { if(self.isSending()) { @@ -562,15 +536,6 @@ exports.getModule = class TransferFileModule extends MenuModule { } return self.prevMenu(); - /* - - // Wait for a key press - attempt to avoid issues with some terminals after xfer - // :TODO: display ANSI if it exists else prompt -- look @ Obv/2 for filename - self.client.term.pipeWrite('|00|07\nTransfer(s) complete. Press a key\n'); - self.client.waitForKeyPress( () => { - return self.prevMenu(); - }); - */ } ); } diff --git a/mods/upload.js b/mods/upload.js index 914856f1..c7f491e1 100644 --- a/mods/upload.js +++ b/mods/upload.js @@ -8,10 +8,14 @@ const getSortedAvailableFileAreas = require('../core/file_area.js').getSortedAv const getAreaDefaultStorageDirectory = require('../core/file_area.js').getAreaDefaultStorageDirectory; const scanFile = require('../core/file_area.js').scanFile; const ansiGoto = require('../core/ansi_term.js').goto; +const moveFileWithCollisionHandling = require('../core/file_util.js').moveFileWithCollisionHandling; +const pathWithTerminatingSeparator = require('../core/file_util.js').pathWithTerminatingSeparator; // deps const async = require('async'); const _ = require('lodash'); +const temp = require('temp').track(); // track() cleans up temp dir/files for us +const paths = require('path'); exports.moduleInfo = { name : 'Upload', @@ -65,21 +69,12 @@ exports.getModule = class UploadModule extends MenuModule { this.menuMethods = { optionsNavContinue : (formData, extraArgs, cb) => { if(this.isBlindUpload()) { - // jump to protocol selection - const areaUploadDir = this.getSelectedAreaUploadDirectory(); - - const modOpts = { - extraArgs : { - recvDirectory : areaUploadDir, - direction : 'recv', - } - }; - - return this.gotoMenu(this.menuConfig.config.fileTransferProtocolSelection || 'fileTransferProtocolSelection', modOpts, cb); - } else { - // jump to fileDetails form - // :TODO: support non-blind: collect info/filename -> upload -> complete + return this.performBlindUpload(cb); } + + // non-blind + // jump to fileDetails form + // :TODO: support non-blind: collect info/filename -> upload -> complete }, fileDetailsContinue : (formData, extraArgs, cb) => { @@ -94,12 +89,13 @@ exports.getModule = class UploadModule extends MenuModule { getSaveState() { const saveState = { - uploadType : this.uploadType, - + uploadType : this.uploadType, + tempRecvDirectory : this.tempRecvDirectory }; if(this.isBlindUpload()) { - saveState.areaInfo = this.getSelectedAreaInfo(); + const areaSelectView = this.viewControllers.options.getView(MciViewIds.options.area); + saveState.areaInfo = this.availAreas[areaSelectView.getData()]; } return saveState; @@ -107,20 +103,11 @@ exports.getModule = class UploadModule extends MenuModule { restoreSavedState(savedState) { if(savedState.areaInfo) { - this.areaInfo = savedState.areaInfo; + this.areaInfo = savedState.areaInfo; + this.tempRecvDirectory = savedState.tempRecvDirectory; } } - getSelectedAreaInfo() { - const areaSelectView = this.viewControllers.options.getView(MciViewIds.options.area); - return this.availAreas[areaSelectView.getData()]; - } - - getSelectedAreaUploadDirectory() { - const areaInfo = this.getSelectedAreaInfo(); - return getAreaDefaultStorageDirectory(areaInfo); - } - isBlindUpload() { return 'blind' === this.uploadType; } isFileTransferComplete() { return !_.isUndefined(this.recvFilePaths); } @@ -152,6 +139,44 @@ exports.getModule = class UploadModule extends MenuModule { } } + leave() { + // remove any temp files - only do this when + if(this.isFileTransferComplete()) { + // :TODO: fix global temp cleanup issue!!! + //temp.cleanup(); // remove any temp files + } + + super.leave(); + } + + performBlindUpload(cb) { + temp.mkdir('enigul-', (err, tempRecvDirectory) => { + if(err) { + return cb(err); + } + + // need a terminator for various external protocols + this.tempRecvDirectory = pathWithTerminatingSeparator(tempRecvDirectory); + + const modOpts = { + extraArgs : { + recvDirectory : this.tempRecvDirectory, // we'll move files from here to their area container once processed/confirmed + direction : 'recv', + } + }; + + // + // Move along to protocol selection -> file transfer + // Upon completion, we'll re-enter the module with some file paths handed to us + // + return this.gotoMenu( + this.menuConfig.config.fileTransferProtocolSelection || 'fileTransferProtocolSelection', + modOpts, + cb + ); + }); + } + updateScanStepInfoViews(stepInfo) { // :TODO: add some blinking (e.g. toggle items) indicators - see OBV.DOC @@ -240,6 +265,8 @@ exports.getModule = class UploadModule extends MenuModule { dupes : [], }; + self.client.log.debug('Scanning upload(s)', { paths : this.recvFilePaths } ); + async.eachSeries(this.recvFilePaths, (filePath, nextFilePath) => { // :TODO: virus scanning/etc. should occur around here @@ -257,7 +284,7 @@ exports.getModule = class UploadModule extends MenuModule { return nextScanStep(null); } - self.client.log.debug('Scanning upload', { filePath : filePath } ); + self.client.log.debug('Scanning file', { filePath : filePath } ); scanFile(filePath, scanOpts, handleScanStep, (err, fileEntry, dupeEntries) => { if(err) { @@ -267,7 +294,7 @@ exports.getModule = class UploadModule extends MenuModule { // new or dupe? if(dupeEntries.length > 0) { // 1:n dupes found - self.client.log.debug('Duplicate(s) of upload found', { dupeEntries : dupeEntries } ); + self.client.log.debug('Duplicate file(s) found', { dupeEntries : dupeEntries } ); results.dupes = results.dupes.concat(dupeEntries); } else { @@ -282,6 +309,38 @@ exports.getModule = class UploadModule extends MenuModule { }); } + moveAndPersistUploadsToDatabase(newEntries) { + + const areaStorageDir = getAreaDefaultStorageDirectory(this.areaInfo); + const self = this; + + async.eachSeries(newEntries, (newEntry, nextEntry) => { + const src = paths.join(self.tempRecvDirectory, newEntry.fileName); + const dst = paths.join(areaStorageDir, newEntry.fileName); + + moveFileWithCollisionHandling(src, dst, (err, finalPath) => { + if(err) { + self.client.log.error( + 'Failed moving physical upload file', { error : err.message, fileName : newEntry.fileName, source : src, dest : dst } + ); + + return nextEntry(null); // still try next file + } + + self.client.log.debug('Moved upload to area', { path : finalPath } ); + + // persist to DB + newEntry.persist(err => { + if(err) { + self.client.log.error('Failed persisting upload to database', { path : finalPath, error : err.message } ); + } + + return nextEntry(null); // still try next file + }); + }); + }); + } + processUploadedFiles() { // // For each file uploaded, we need to process & gather information @@ -339,16 +398,15 @@ exports.getModule = class UploadModule extends MenuModule { return callback(err, scanResults); }); }, - function persistNewEntries(scanResults, callback) { - // loop over entries again & persist to DB - async.eachSeries(scanResults.newEntries, (newEntry, nextEntry) => { - newEntry.persist(err => { - return nextEntry(err); - }); - }, err => { - return callback(err); - }); - } + function startMovingAndPersistingToDatabase(scanResults, callback) { + // + // *Start* the process of moving files from their current |tempRecvDirectory| + // locations -> their final area destinations. Don't make the user wait + // here as I/O can take quite a bit of time. Log any failures. + // + self.moveAndPersistUploadsToDatabase(scanResults.newEntries); + return callback(null); + }, ], err => { if(err) {