From dc2b3031fd95df19ea9a44090afad11ed7d3c395 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Tue, 26 Sep 2017 10:44:15 -0600 Subject: [PATCH] * Change how hashids are generated for web file area: include a 'type' * Add support for web *batch* downloads via streaming zip file creation * Add new web download manager and batch mode display * Add extra info to 'standard' downloads mod/menu --- core/file_area_web.js | 336 ++++++++++++++++++------- mods/file_base_download_manager.js | 38 ++- mods/file_base_web_download_manager.js | 287 +++++++++++++++++++++ package.json | 3 +- 4 files changed, 560 insertions(+), 104 deletions(-) create mode 100644 mods/file_base_web_download_manager.js diff --git a/core/file_area_web.js b/core/file_area_web.js index aa60974f..a8d095d6 100644 --- a/core/file_area_web.js +++ b/core/file_area_web.js @@ -22,14 +22,7 @@ const paths = require('path'); const async = require('async'); const fs = require('graceful-fs'); const mimeTypes = require('mime-types'); -const _ = require('lodash'); - - /* - :TODO: - * Load temp download URLs @ startup & set expire timers via scheduler. - * At creation, set expire timer via scheduler - * - */ +const yazl = require('yazl'); function notEnabledError() { return Errors.General('Web server is not enabled', ErrNotEnabled); @@ -59,7 +52,7 @@ class FileAreaWebAccess { const routeAdded = self.webServer.instance.addRoute({ method : 'GET', path : Config.fileBase.web.routePath, - handler : self.routeWebRequestForFile.bind(self), + handler : self.routeWebRequest.bind(self), }); return callback(routeAdded ? null : Errors.General('Failed adding route')); } else { @@ -81,6 +74,13 @@ class FileAreaWebAccess { return this.webServer.instance.isEnabled(); } + static getHashIdTypes() { + return { + SingleFile : 0, + BatchArchive : 1, + }; + } + load(cb) { // // Load entries, register expiration timers @@ -141,12 +141,14 @@ class FileAreaWebAccess { WHERE hash_id = ?`, [ hashId ], (err, result) => { - if(err) { - return cb(err); + if(err || !result) { + return cb(err ? err : Errors.DoesNotExist('Invalid or missing hash ID')); } const decoded = this.hashids.decode(hashId); - if(!result || 2 !== decoded.length) { + + // decode() should provide an array of [ userId, hashIdType, id, ... ] + if(!Array.isArray(decoded) || decoded.length < 3) { return cb(Errors.Invalid('Invalid or unknown hash ID')); } @@ -155,7 +157,8 @@ class FileAreaWebAccess { { hashId : hashId, userId : decoded[0], - fileId : decoded[1], + hashIdType : decoded[1], + fileIds : decoded.slice(2), expireTimestamp : moment(result.expire_timestamp), } ); @@ -163,46 +166,26 @@ class FileAreaWebAccess { ); } - getHashId(client, fileEntry) { - // - // Hashid is a unique combination of userId & fileId - // - return this.hashids.encode(client.user.userId, fileEntry.fileId); + getSingleFileHashId(client, fileEntry) { + return this.getHashId(client, FileAreaWebAccess.getHashIdTypes().SingleFile, [ fileEntry.fileId ] ); } - buildTempDownloadLink(client, fileEntry, hashId) { - hashId = hashId || this.getHashId(client, fileEntry); + getBatchArchiveHashId(client, batchId) { + return this.getHashId(client, FileAreaWebAccess.getHashIdTypes().BatchArchive, batchId); + } + + getHashId(client, hashIdType, identifier) { + return this.hashids.encode(client.user.userId, hashIdType, identifier); + } + + buildSingleFileTempDownloadLink(client, fileEntry, hashId) { + hashId = hashId || this.getSingleFileHashId(client, fileEntry); return this.webServer.instance.buildUrl(`${Config.fileBase.web.path}${hashId}`); - /* - - // - // Create a URL such as - // https://l33t.codes:44512/f/qFdxyZr - // - // Prefer HTTPS over HTTP. Be explicit about the port - // only if non-standard. - // - let schema; - let port; - if(_.isString(Config.contentServers.web.overrideUrlPrefix)) { - return `${Config.contentServers.web.overrideUrlPrefix}${Config.fileBase.web.path}${hashId}`; - } else { - if(Config.contentServers.web.https.enabled) { - schema = 'https://'; - port = (443 === Config.contentServers.web.https.port) ? - '' : - `:${Config.contentServers.web.https.port}`; - } else { - schema = 'http://'; - port = (80 === Config.contentServers.web.http.port) ? - '' : - `:${Config.contentServers.web.http.port}`; - } - - return `${schema}${Config.contentServers.web.domain}${port}${Config.fileBase.web.path}${hashId}`; - } - */ + } + + buildBatchArchiveTempDownloadLink(client, hashId) { + return this.webServer.instance.buildUrl(`${Config.fileBase.web.path}${hashId}`); } getExistingTempDownloadServeItem(client, fileEntry, cb) { @@ -210,49 +193,95 @@ class FileAreaWebAccess { return cb(notEnabledError()); } - const hashId = this.getHashId(client, fileEntry); + const hashId = this.getSingleFileHashId(client, fileEntry); this.loadServedHashId(hashId, (err, servedItem) => { if(err) { return cb(err); } - servedItem.url = this.buildTempDownloadLink(client, fileEntry); + servedItem.url = this.buildSingleFileTempDownloadLink(client, fileEntry); return cb(null, servedItem); }); } + _addOrUpdateHashIdRecord(dbOrTrans, hashId, expireTime, cb) { + // add/update rec with hash id and (latest) timestamp + dbOrTrans.run( + `REPLACE INTO file_web_serve (hash_id, expire_timestamp) + VALUES (?, ?);`, + [ hashId, getISOTimestampString(expireTime) ], + err => { + if(err) { + return cb(err); + } + + this.scheduleExpire(hashId, expireTime); + + return cb(null); + } + ); + } + createAndServeTempDownload(client, fileEntry, options, cb) { if(!this.isEnabled()) { return cb(notEnabledError()); } - const hashId = this.getHashId(client, fileEntry); - const url = this.buildTempDownloadLink(client, fileEntry, hashId); + const hashId = this.getSingleFileHashId(client, fileEntry); + const url = this.buildSingleFileTempDownloadLink(client, fileEntry, hashId); options.expireTime = options.expireTime || moment().add(2, 'days'); - // add/update rec with hash id and (latest) timestamp - FileDb.run( - `REPLACE INTO file_web_serve (hash_id, expire_timestamp) - VALUES (?, ?);`, - [ hashId, getISOTimestampString(options.expireTime) ], - err => { + this._addOrUpdateHashIdRecord(FileDb, hashId, options.expireTime, err => { + return cb(err, url); + }); + } + + createAndServeTempBatchDownload(client, fileEntries, options, cb) { + if(!this.isEnabled()) { + return cb(notEnabledError()); + } + + const batchId = moment().utc().unix(); + const hashId = this.getBatchArchiveHashId(client, batchId); + const url = this.buildBatchArchiveTempDownloadLink(client, hashId); + options.expireTime = options.expireTime || moment().add(2, 'days'); + + FileDb.beginTransaction( (err, trans) => { + if(err) { + return cb(err); + } + + this._addOrUpdateHashIdRecord(trans, hashId, options.expireTime, err => { if(err) { - return cb(err); + return trans.rollback( () => { + return cb(err); + }); } - this.scheduleExpire(hashId, options.expireTime); - - return cb(null, url); - } - ); + async.eachSeries(fileEntries, (entry, nextEntry) => { + trans.run( + `INSERT INTO file_web_serve_batch (hash_id, file_id) + VALUES (?, ?);`, + [ hashId, entry.fileId ], + err => { + return nextEntry(err); + } + ); + }, err => { + trans[err ? 'rollback' : 'commit']( () => { + return cb(err, url); + }); + }); + }); + }); } fileNotFound(resp) { return this.webServer.instance.fileNotFound(resp); } - routeWebRequestForFile(req, resp) { + routeWebRequest(req, resp) { const hashId = paths.basename(req.url); this.loadServedHashId(hashId, (err, servedItem) => { @@ -261,44 +290,157 @@ class FileAreaWebAccess { return this.fileNotFound(resp); } - const fileEntry = new FileEntry(); - fileEntry.load(servedItem.fileId, err => { + const hashIdTypes = FileAreaWebAccess.getHashIdTypes(); + switch(servedItem.hashIdType) { + case hashIdTypes.SingleFile : + return this.routeWebRequestForSingleFile(servedItem, req, resp); + + case hashIdTypes.BatchArchive : + return this.routeWebRequestForBatchArchive(servedItem, req, resp); + + default : + return this.fileNotFound(resp); + } + }); + } + + routeWebRequestForSingleFile(servedItem, req, resp) { + const fileEntry = new FileEntry(); + + servedItem.fileId = servedItem.fileIds[0]; + + fileEntry.load(servedItem.fileId, err => { + if(err) { + return this.fileNotFound(resp); + } + + const filePath = fileEntry.filePath; + if(!filePath) { + return this.fileNotFound(resp); + } + + fs.stat(filePath, (err, stats) => { if(err) { return this.fileNotFound(resp); } - const filePath = fileEntry.filePath; - if(!filePath) { + resp.on('close', () => { + // connection closed *before* the response was fully sent + // :TODO: Log and such + }); + + resp.on('finish', () => { + // transfer completed fully + this.updateDownloadStatsForUserIdAndSystem(servedItem.userId, stats.size); + }); + + const headers = { + 'Content-Type' : mimeTypes.contentType(filePath) || mimeTypes.contentType('.bin'), + 'Content-Length' : stats.size, + 'Content-Disposition' : `attachment; filename="${fileEntry.fileName}"`, + }; + + const readStream = fs.createReadStream(filePath); + resp.writeHead(200, headers); + return readStream.pipe(resp); + }); + }); + } + + routeWebRequestForBatchArchive(servedItem, req, resp) { + // + // We are going to build an on-the-fly zip file stream of 1:n + // files in the batch. + // + // First, collect all file IDs + // + const self = this; + + async.waterfall( + [ + function fetchFileIds(callback) { + FileDb.all( + `SELECT file_id + FROM file_web_serve_batch + WHERE hash_id = ?;`, + [ servedItem.hashId ], + (err, fileIdRows) => { + if(err || !Array.isArray(fileIdRows) || 0 === fileIdRows.length) { + return callback(Errors.DoesNotExist('Could not get file IDs for batch')); + } + + return callback(null, fileIdRows.map(r => r.file_id)); + } + ); + }, + function loadFileEntries(fileIds, callback) { + const filePaths = []; + async.eachSeries(fileIds, (fileId, nextFileId) => { + const fileEntry = new FileEntry(); + fileEntry.load(fileId, err => { + if(!err) { + filePaths.push(fileEntry.filePath); + } + return nextFileId(err); + }); + }, err => { + if(err) { + return callback(Errors.DoesNotExist('Coudl not load file IDs for batch')); + } + + return callback(null, filePaths); + }); + }, + function createAndServeStream(filePaths, callback) { + const zipFile = new yazl.ZipFile(); + + filePaths.forEach(fp => { + zipFile.addFile( + fp, // path to physical file + paths.basename(fp), // filename/path *stored in archive* + { + compress : false, // :TODO: do this smartly - if ext is in set = false, else true via isArchive() or such... mimeDB has this for us. + } + ); + }); + + zipFile.end( finalZipSize => { + if(-1 === finalZipSize) { + return callback(Errors.UnexpectedState('Unable to acquire final zip size')); + } + + resp.on('close', () => { + // connection closed *before* the response was fully sent + // :TODO: Log and such + }); + + resp.on('finish', () => { + // transfer completed fully + self.updateDownloadStatsForUserIdAndSystem(servedItem.userId, finalZipSize); + }); + + const batchFileName = `batch_${servedItem.hashId}.zip`; + + const headers = { + 'Content-Type' : mimeTypes.contentType(batchFileName) || mimeTypes.contentType('.bin'), + 'Content-Length' : finalZipSize, + 'Content-Disposition' : `attachment; filename="${batchFileName}"`, + }; + + resp.writeHead(200, headers); + return zipFile.outputStream.pipe(resp); + }); + } + ], + err => { + if(err) { + // :TODO: Log me! return this.fileNotFound(resp); } - fs.stat(filePath, (err, stats) => { - if(err) { - return this.fileNotFound(resp); - } - - resp.on('close', () => { - // connection closed *before* the response was fully sent - // :TODO: Log and such - }); - - resp.on('finish', () => { - // transfer completed fully - this.updateDownloadStatsForUserIdAndSystem(servedItem.userId, stats.size); - }); - - const headers = { - 'Content-Type' : mimeTypes.contentType(filePath) || mimeTypes.contentType('.bin'), - 'Content-Length' : stats.size, - 'Content-Disposition' : `attachment; filename="${fileEntry.fileName}"`, - }; - - const readStream = fs.createReadStream(filePath); - resp.writeHead(200, headers); - return readStream.pipe(resp); - }); - }); - }); + // ...otherwise, we would have called resp() already. + } + ); } updateDownloadStatsForUserIdAndSystem(userId, dlBytes, cb) { diff --git a/mods/file_base_download_manager.js b/mods/file_base_download_manager.js index 812a2422..382a7305 100644 --- a/mods/file_base_download_manager.js +++ b/mods/file_base_download_manager.js @@ -9,10 +9,12 @@ const theme = require('../core/theme.js'); const ansi = require('../core/ansi_term.js'); const Errors = require('../core/enig_error.js').Errors; const stringFormat = require('../core/string_format.js'); +const FileAreaWeb = require('../core/file_area_web.js'); // deps const async = require('async'); const _ = require('lodash'); +const moment = require('moment'); exports.moduleInfo = { name : 'File Base Download Queue Manager', @@ -22,17 +24,15 @@ exports.moduleInfo = { const FormIds = { queueManager : 0, - details : 1, }; const MciViewIds = { queueManager : { - queue : 1, - navMenu : 2, - }, - details : { + queue : 1, + navMenu : 2, - } + customRangeStart : 10, + }, }; exports.getModule = class FileBaseDownloadQueueManager extends MenuModule { @@ -126,6 +126,26 @@ exports.getModule = class FileBaseDownloadQueueManager extends MenuModule { return cb(null); } + displayWebDownloadLinkForFileEntry(fileEntry) { + FileAreaWeb.getExistingTempDownloadServeItem(this.client, fileEntry, (err, serveItem) => { + if(serveItem && serveItem.url) { + const webDlExpireTimeFormat = this.menuConfig.config.webDlExpireTimeFormat || 'YYYY-MMM-DD @ h:mm'; + + fileEntry.webDlLink = ansi.vtxHyperlink(this.client, serveItem.url) + serveItem.url; + fileEntry.webDlExpire = moment(serveItem.expireTimestamp).format(webDlExpireTimeFormat); + } else { + fileEntry.webDlLink = ''; + fileEntry.webDlExpire = ''; + } + + this.updateCustomViewTextsWithFilter( + 'queueManager', + MciViewIds.queueManager.customRangeStart, fileEntry, + { filter : [ '{webDlLink}', '{webDlExpire}' ] } + ); + }); + } + updateDownloadQueueView(cb) { const queueView = this.viewControllers.queueManager.getView(MciViewIds.queueManager.queue); if(!queueView) { @@ -138,7 +158,13 @@ exports.getModule = class FileBaseDownloadQueueManager extends MenuModule { queueView.setItems(this.dlQueue.items.map( queueItem => stringFormat(queueListFormat, queueItem) ) ); queueView.setFocusItems(this.dlQueue.items.map( queueItem => stringFormat(focusQueueListFormat, queueItem) ) ); + queueView.on('index update', idx => { + const fileEntry = this.dlQueue.items[idx]; + this.displayWebDownloadLinkForFileEntry(fileEntry); + }); + queueView.redraw(); + this.displayWebDownloadLinkForFileEntry(this.dlQueue.items[0]); return cb(null); } diff --git a/mods/file_base_web_download_manager.js b/mods/file_base_web_download_manager.js new file mode 100644 index 00000000..d171cfdb --- /dev/null +++ b/mods/file_base_web_download_manager.js @@ -0,0 +1,287 @@ +/* jslint node: true */ +'use strict'; + +// ENiGMA½ +const MenuModule = require('../core/menu_module.js').MenuModule; +const ViewController = require('../core/view_controller.js').ViewController; +const DownloadQueue = require('../core/download_queue.js'); +const theme = require('../core/theme.js'); +const ansi = require('../core/ansi_term.js'); +const Errors = require('../core/enig_error.js').Errors; +const stringFormat = require('../core/string_format.js'); +const FileAreaWeb = require('../core/file_area_web.js'); +const ErrNotEnabled = require('../core/enig_error.js').ErrorReasons.NotEnabled; +const Config = require('../core/config.js').config; + +// deps +const async = require('async'); +const _ = require('lodash'); +const moment = require('moment'); + +exports.moduleInfo = { + name : 'File Base Download Web Queue Manager', + desc : 'Module for interacting with web backed download queue/batch', + author : 'NuSkooler', +}; + +const FormIds = { + queueManager : 0 +}; + +const MciViewIds = { + queueManager : { + queue : 1, + navMenu : 2, + + customRangeStart : 10, + } +}; + +exports.getModule = class FileBaseWebDownloadQueueManager extends MenuModule { + + constructor(options) { + super(options); + + this.dlQueue = new DownloadQueue(this.client); + + this.menuMethods = { + removeItem : (formData, extraArgs, cb) => { + const selectedItem = this.dlQueue.items[formData.value.queueItem]; + if(!selectedItem) { + return cb(null); + } + + this.dlQueue.removeItems(selectedItem.fileId); + + // :TODO: broken: does not redraw menu properly - needs fixed! + return this.removeItemsFromDownloadQueueView(formData.value.queueItem, cb); + }, + clearQueue : (formData, extraArgs, cb) => { + this.dlQueue.clear(); + + // :TODO: broken: does not redraw menu properly - needs fixed! + return this.removeItemsFromDownloadQueueView('all', cb); + }, + getBatchLink : (formData, extraArgs, cb) => { + return this.generateAndDisplayBatchLink(cb); + } + }; + } + + initSequence() { + if(0 === this.dlQueue.items.length) { + return this.gotoMenu(this.menuConfig.config.emptyQueueMenu || 'fileBaseDownloadManagerEmptyQueue'); + } + + const self = this; + + async.series( + [ + function beforeArt(callback) { + return self.beforeArt(callback); + }, + function display(callback) { + return self.displayQueueManagerPage(false, callback); + } + ], + () => { + return self.finishedLoading(); + } + ); + } + + removeItemsFromDownloadQueueView(itemIndex, cb) { + const queueView = this.viewControllers.queueManager.getView(MciViewIds.queueManager.queue); + if(!queueView) { + return cb(Errors.DoesNotExist('Queue view does not exist')); + } + + if('all' === itemIndex) { + queueView.setItems([]); + queueView.setFocusItems([]); + } else { + queueView.removeItem(itemIndex); + } + + queueView.redraw(); + return cb(null); + } + + displayFileInfoForFileEntry(fileEntry) { + this.updateCustomViewTextsWithFilter( + 'queueManager', + MciViewIds.queueManager.customRangeStart, fileEntry, + { filter : [ '{webDlLink}', '{webDlExpire}', '{fileName}' ] } // :TODO: Others.... + ); + } + + updateDownloadQueueView(cb) { + const queueView = this.viewControllers.queueManager.getView(MciViewIds.queueManager.queue); + if(!queueView) { + return cb(Errors.DoesNotExist('Queue view does not exist')); + } + + const queueListFormat = this.menuConfig.config.queueListFormat || '{webDlLink}'; + const focusQueueListFormat = this.menuConfig.config.focusQueueListFormat || queueListFormat; + + queueView.setItems(this.dlQueue.items.map( queueItem => stringFormat(queueListFormat, queueItem) ) ); + queueView.setFocusItems(this.dlQueue.items.map( queueItem => stringFormat(focusQueueListFormat, queueItem) ) ); + + queueView.on('index update', idx => { + const fileEntry = this.dlQueue.items[idx]; + this.displayFileInfoForFileEntry(fileEntry); + }); + + queueView.redraw(); + this.displayFileInfoForFileEntry(this.dlQueue.items[0]); + + return cb(null); + } + + generateAndDisplayBatchLink(cb) { + const expireTime = moment().add(Config.fileBase.web.expireMinutes, 'minutes'); + + FileAreaWeb.createAndServeTempBatchDownload( + this.client, + this.dlQueue.items, + { + expireTime : expireTime + }, + (err, webBatchDlLink) => { + // :TODO: handle not enabled -> display such + if(err) { + return cb(err); + } + + const webDlExpireTimeFormat = this.menuConfig.config.webDlExpireTimeFormat || 'YYYY-MMM-DD @ h:mm'; + + const formatObj = { + webBatchDlLink : ansi.vtxHyperlink(this.client, webBatchDlLink) + webBatchDlLink, + webBatchDlExpire : expireTime.format(webDlExpireTimeFormat), + }; + + this.updateCustomViewTextsWithFilter( + 'queueManager', + MciViewIds.queueManager.customRangeStart, + formatObj, + { filter : Object.keys(formatObj).map(k => '{' + k + '}' ) } + ); + + return cb(null); + } + ); + } + + displayQueueManagerPage(clearScreen, cb) { + const self = this; + + async.series( + [ + function prepArtAndViewController(callback) { + return self.displayArtAndPrepViewController('queueManager', { clearScreen : clearScreen }, callback); + }, + function prepareQueueDownloadLinks(callback) { + const webDlExpireTimeFormat = self.menuConfig.config.webDlExpireTimeFormat || 'YYYY-MMM-DD @ h:mm'; + + async.each(self.dlQueue.items, (fileEntry, nextFileEntry) => { + FileAreaWeb.getExistingTempDownloadServeItem(self.client, fileEntry, (err, serveItem) => { + if(err) { + if(ErrNotEnabled === err.reasonCode) { + return nextFileEntry(err); // we should have caught this prior + } + + const expireTime = moment().add(Config.fileBase.web.expireMinutes, 'minutes'); + + FileAreaWeb.createAndServeTempDownload( + self.client, + fileEntry, + { expireTime : expireTime }, + (err, url) => { + if(err) { + return nextFileEntry(err); + } + + fileEntry.webDlLinkRaw = url; + fileEntry.webDlLink = ansi.vtxHyperlink(self.client, url) + url; + fileEntry.webDlExpire = expireTime.format(webDlExpireTimeFormat); + + return nextFileEntry(null); + } + ); + } else { + fileEntry.webDlLinkRaw = serveItem.url; + fileEntry.webDlLink = ansi.vtxHyperlink(self.client, serveItem.url) + serveItem.url; + fileEntry.webDlExpire = moment(serveItem.expireTimestamp).format(webDlExpireTimeFormat); + return nextFileEntry(null); + } + }); + }, err => { + return callback(err); + }); + }, + function populateViews(callback) { + return self.updateDownloadQueueView(callback); + } + ], + err => { + if(cb) { + return cb(err); + } + } + ); + } + + displayArtAndPrepViewController(name, options, cb) { + const self = this; + const config = this.menuConfig.config; + + async.waterfall( + [ + function readyAndDisplayArt(callback) { + if(options.clearScreen) { + self.client.term.rawWrite(ansi.resetScreen()); + } + + theme.displayThemedAsset( + config.art[name], + self.client, + { font : self.menuConfig.font, trailingLF : false }, + (err, artData) => { + return callback(err, artData); + } + ); + }, + function prepeareViewController(artData, callback) { + if(_.isUndefined(self.viewControllers[name])) { + const vcOpts = { + client : self.client, + formId : FormIds[name], + }; + + if(!_.isUndefined(options.noInput)) { + vcOpts.noInput = options.noInput; + } + + const vc = self.addViewController(name, new ViewController(vcOpts)); + + const loadOpts = { + callingMenu : self, + mciMap : artData.mciMap, + formId : FormIds[name], + }; + + return vc.loadFromMenuConfig(loadOpts, callback); + } + + self.viewControllers[name].setFocus(true); + return callback(null); + + }, + ], + err => { + return cb(err); + } + ); + } +}; + \ No newline at end of file diff --git a/package.json b/package.json index ef9da900..224fda84 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,8 @@ "temptmp": "^1.0.0", "uuid": "^3.1.0", "uuid-parse": "^1.0.0", - "ws": "^3.1.0" + "ws": "^3.1.0", + "yazl" : "^2.4.2" }, "devDependencies": {}, "engines": {