moon
/
enigma-bbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Change how hashids are generated for web file area: include a 'type' 

* Add support for web *batch* downloads via streaming zip file creation
* Add new web download manager and batch mode display
* Add extra info to 'standard' downloads mod/menu
This commit is contained in:
Bryan Ashby 2017-09-26 10:44:15 -06:00
parent e555a28160
commit dc2b3031fd
4 changed files with 560 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

336
core/file_area_web.js
View File

@ -22,14 +22,7 @@ const paths = require('path');
const async = require('async');
const fs = require('graceful-fs');
const mimeTypes = require('mime-types');
const _ = require('lodash');
/*
:TODO:
* Load temp download URLs @ startup & set expire timers via scheduler.
* At creation, set expire timer via scheduler
*
*/
const yazl = require('yazl');
function notEnabledError() {
return Errors.General('Web server is not enabled', ErrNotEnabled);
@ -59,7 +52,7 @@ class FileAreaWebAccess {
const routeAdded = self.webServer.instance.addRoute({
method : 'GET',
path : Config.fileBase.web.routePath,
handler : self.routeWebRequestForFile.bind(self),
handler : self.routeWebRequest.bind(self),
});
return callback(routeAdded ? null : Errors.General('Failed adding route'));
} else {
@ -81,6 +74,13 @@ class FileAreaWebAccess {
return this.webServer.instance.isEnabled();
}
static getHashIdTypes() {
return {
SingleFile : 0,
BatchArchive : 1,
};
}
load(cb) {
//
// Load entries, register expiration timers
@ -141,12 +141,14 @@ class FileAreaWebAccess {
WHERE hash_id = ?`,
[ hashId ],
(err, result) => {
if(err) {
return cb(err);
if(err || !result) {
return cb(err ? err : Errors.DoesNotExist('Invalid or missing hash ID'));
}
const decoded = this.hashids.decode(hashId);
if(!result || 2 !== decoded.length) {
// decode() should provide an array of [ userId, hashIdType, id, ... ]
if(!Array.isArray(decoded) || decoded.length < 3) {
return cb(Errors.Invalid('Invalid or unknown hash ID'));
}
@ -155,7 +157,8 @@ class FileAreaWebAccess {
{
hashId : hashId,
userId : decoded[0],
fileId : decoded[1],
hashIdType : decoded[1],
fileIds : decoded.slice(2),
expireTimestamp : moment(result.expire_timestamp),
}
);
@ -163,46 +166,26 @@ class FileAreaWebAccess {
);
}
getHashId(client, fileEntry) {
//
// Hashid is a unique combination of userId & fileId
//
return this.hashids.encode(client.user.userId, fileEntry.fileId);
getSingleFileHashId(client, fileEntry) {
return this.getHashId(client, FileAreaWebAccess.getHashIdTypes().SingleFile, [ fileEntry.fileId ] );
}
buildTempDownloadLink(client, fileEntry, hashId) {
hashId = hashId || this.getHashId(client, fileEntry);
getBatchArchiveHashId(client, batchId) {
return this.getHashId(client, FileAreaWebAccess.getHashIdTypes().BatchArchive, batchId);
}
getHashId(client, hashIdType, identifier) {
return this.hashids.encode(client.user.userId, hashIdType, identifier);
}
buildSingleFileTempDownloadLink(client, fileEntry, hashId) {
hashId = hashId || this.getSingleFileHashId(client, fileEntry);
return this.webServer.instance.buildUrl(`${Config.fileBase.web.path}${hashId}`);
/*
//
// Create a URL such as
// https://l33t.codes:44512/f/qFdxyZr
//
// Prefer HTTPS over HTTP. Be explicit about the port
// only if non-standard.
//
let schema;
let port;
if(_.isString(Config.contentServers.web.overrideUrlPrefix)) {
return `${Config.contentServers.web.overrideUrlPrefix}${Config.fileBase.web.path}${hashId}`;
} else {
if(Config.contentServers.web.https.enabled) {
schema = 'https://';
port = (443 === Config.contentServers.web.https.port) ?
'' :
`:${Config.contentServers.web.https.port}`;
} else {
schema = 'http://';
port = (80 === Config.contentServers.web.http.port) ?
'' :
`:${Config.contentServers.web.http.port}`;
}
return `${schema}${Config.contentServers.web.domain}${port}${Config.fileBase.web.path}${hashId}`;
}
*/
}
buildBatchArchiveTempDownloadLink(client, hashId) {
return this.webServer.instance.buildUrl(`${Config.fileBase.web.path}${hashId}`);
}
getExistingTempDownloadServeItem(client, fileEntry, cb) {
@ -210,49 +193,95 @@ class FileAreaWebAccess {
return cb(notEnabledError());
}
const hashId = this.getHashId(client, fileEntry);
const hashId = this.getSingleFileHashId(client, fileEntry);
this.loadServedHashId(hashId, (err, servedItem) => {
if(err) {
return cb(err);
}
servedItem.url = this.buildTempDownloadLink(client, fileEntry);
servedItem.url = this.buildSingleFileTempDownloadLink(client, fileEntry);
return cb(null, servedItem);
});
}
_addOrUpdateHashIdRecord(dbOrTrans, hashId, expireTime, cb) {
// add/update rec with hash id and (latest) timestamp
dbOrTrans.run(
`REPLACE INTO file_web_serve (hash_id, expire_timestamp)
VALUES (?, ?);`,
[ hashId, getISOTimestampString(expireTime) ],
err => {
if(err) {
return cb(err);
}
this.scheduleExpire(hashId, expireTime);
return cb(null);
}
);
}
createAndServeTempDownload(client, fileEntry, options, cb) {
if(!this.isEnabled()) {
return cb(notEnabledError());
}
const hashId = this.getHashId(client, fileEntry);
const url = this.buildTempDownloadLink(client, fileEntry, hashId);
const hashId = this.getSingleFileHashId(client, fileEntry);
const url = this.buildSingleFileTempDownloadLink(client, fileEntry, hashId);
options.expireTime = options.expireTime || moment().add(2, 'days');
// add/update rec with hash id and (latest) timestamp
FileDb.run(
`REPLACE INTO file_web_serve (hash_id, expire_timestamp)
VALUES (?, ?);`,
[ hashId, getISOTimestampString(options.expireTime) ],
err => {
this._addOrUpdateHashIdRecord(FileDb, hashId, options.expireTime, err => {
return cb(err, url);
});
}
createAndServeTempBatchDownload(client, fileEntries, options, cb) {
if(!this.isEnabled()) {
return cb(notEnabledError());
}
const batchId = moment().utc().unix();
const hashId = this.getBatchArchiveHashId(client, batchId);
const url = this.buildBatchArchiveTempDownloadLink(client, hashId);
options.expireTime = options.expireTime || moment().add(2, 'days');
FileDb.beginTransaction( (err, trans) => {
if(err) {
return cb(err);
}
this._addOrUpdateHashIdRecord(trans, hashId, options.expireTime, err => {
if(err) {
return cb(err);
return trans.rollback( () => {
return cb(err);
});
}
this.scheduleExpire(hashId, options.expireTime);
return cb(null, url);
}
);
async.eachSeries(fileEntries, (entry, nextEntry) => {
trans.run(
`INSERT INTO file_web_serve_batch (hash_id, file_id)
VALUES (?, ?);`,
[ hashId, entry.fileId ],
err => {
return nextEntry(err);
}
);
}, err => {
trans[err ? 'rollback' : 'commit']( () => {
return cb(err, url);
});
});
});
});
}
fileNotFound(resp) {
return this.webServer.instance.fileNotFound(resp);
}
routeWebRequestForFile(req, resp) {
routeWebRequest(req, resp) {
const hashId = paths.basename(req.url);
this.loadServedHashId(hashId, (err, servedItem) => {
@ -261,44 +290,157 @@ class FileAreaWebAccess {
return this.fileNotFound(resp);
}
const fileEntry = new FileEntry();
fileEntry.load(servedItem.fileId, err => {
const hashIdTypes = FileAreaWebAccess.getHashIdTypes();
switch(servedItem.hashIdType) {
case hashIdTypes.SingleFile :
return this.routeWebRequestForSingleFile(servedItem, req, resp);
case hashIdTypes.BatchArchive :
return this.routeWebRequestForBatchArchive(servedItem, req, resp);
default :
return this.fileNotFound(resp);
}
});
}
routeWebRequestForSingleFile(servedItem, req, resp) {
const fileEntry = new FileEntry();
servedItem.fileId = servedItem.fileIds[0];
fileEntry.load(servedItem.fileId, err => {
if(err) {
return this.fileNotFound(resp);
}
const filePath = fileEntry.filePath;
if(!filePath) {
return this.fileNotFound(resp);
}
fs.stat(filePath, (err, stats) => {
if(err) {
return this.fileNotFound(resp);
}
const filePath = fileEntry.filePath;
if(!filePath) {
resp.on('close', () => {
// connection closed *before* the response was fully sent
// :TODO: Log and such
});
resp.on('finish', () => {
// transfer completed fully
this.updateDownloadStatsForUserIdAndSystem(servedItem.userId, stats.size);
});
const headers = {
'Content-Type' : mimeTypes.contentType(filePath) || mimeTypes.contentType('.bin'),
'Content-Length' : stats.size,
'Content-Disposition' : `attachment; filename="${fileEntry.fileName}"`,
};
const readStream = fs.createReadStream(filePath);
resp.writeHead(200, headers);
return readStream.pipe(resp);
});
});
}
routeWebRequestForBatchArchive(servedItem, req, resp) {
//
// We are going to build an on-the-fly zip file stream of 1:n
// files in the batch.
//
// First, collect all file IDs
//
const self = this;
async.waterfall(
[
function fetchFileIds(callback) {
FileDb.all(
`SELECT file_id
FROM file_web_serve_batch
WHERE hash_id = ?;`,
[ servedItem.hashId ],
(err, fileIdRows) => {
if(err || !Array.isArray(fileIdRows) || 0 === fileIdRows.length) {
return callback(Errors.DoesNotExist('Could not get file IDs for batch'));
}
return callback(null, fileIdRows.map(r => r.file_id));
}
);
},
function loadFileEntries(fileIds, callback) {
const filePaths = [];
async.eachSeries(fileIds, (fileId, nextFileId) => {
const fileEntry = new FileEntry();
fileEntry.load(fileId, err => {
if(!err) {
filePaths.push(fileEntry.filePath);
}
return nextFileId(err);
});
}, err => {
if(err) {
return callback(Errors.DoesNotExist('Coudl not load file IDs for batch'));
}
return callback(null, filePaths);
});
},
function createAndServeStream(filePaths, callback) {
const zipFile = new yazl.ZipFile();
filePaths.forEach(fp => {
zipFile.addFile(
fp, // path to physical file
paths.basename(fp), // filename/path *stored in archive*
{
compress : false, // :TODO: do this smartly - if ext is in set = false, else true via isArchive() or such... mimeDB has this for us.
}
);
});
zipFile.end( finalZipSize => {
if(-1 === finalZipSize) {
return callback(Errors.UnexpectedState('Unable to acquire final zip size'));
}
resp.on('close', () => {
// connection closed *before* the response was fully sent
// :TODO: Log and such
});
resp.on('finish', () => {
// transfer completed fully
self.updateDownloadStatsForUserIdAndSystem(servedItem.userId, finalZipSize);
});
const batchFileName = `batch_${servedItem.hashId}.zip`;
const headers = {
'Content-Type' : mimeTypes.contentType(batchFileName) || mimeTypes.contentType('.bin'),
'Content-Length' : finalZipSize,
'Content-Disposition' : `attachment; filename="${batchFileName}"`,
};
resp.writeHead(200, headers);
return zipFile.outputStream.pipe(resp);
});
}
],
err => {
if(err) {
// :TODO: Log me!
return this.fileNotFound(resp);
}
fs.stat(filePath, (err, stats) => {
if(err) {
return this.fileNotFound(resp);
}
resp.on('close', () => {
// connection closed *before* the response was fully sent
// :TODO: Log and such
});
resp.on('finish', () => {
// transfer completed fully
this.updateDownloadStatsForUserIdAndSystem(servedItem.userId, stats.size);
});
const headers = {
'Content-Type' : mimeTypes.contentType(filePath) || mimeTypes.contentType('.bin'),
'Content-Length' : stats.size,
'Content-Disposition' : `attachment; filename="${fileEntry.fileName}"`,
};
const readStream = fs.createReadStream(filePath);
resp.writeHead(200, headers);
return readStream.pipe(resp);
});
});
});
// ...otherwise, we would have called resp() already.
}
);
}
updateDownloadStatsForUserIdAndSystem(userId, dlBytes, cb) {

38
mods/file_base_download_manager.js
View File

@ -9,10 +9,12 @@ const theme = require('../core/theme.js');
const ansi = require('../core/ansi_term.js');
const Errors = require('../core/enig_error.js').Errors;
const stringFormat = require('../core/string_format.js');
const FileAreaWeb = require('../core/file_area_web.js');
// deps
const async = require('async');
const _ = require('lodash');
const moment = require('moment');
exports.moduleInfo = {
name : 'File Base Download Queue Manager',
@ -22,17 +24,15 @@ exports.moduleInfo = {
const FormIds = {
queueManager : 0,
details : 1,
};
const MciViewIds = {
queueManager : {
queue : 1,
navMenu : 2,
},
details : {
queue : 1,
navMenu : 2,
}
customRangeStart : 10,
},
};
exports.getModule = class FileBaseDownloadQueueManager extends MenuModule {
@ -126,6 +126,26 @@ exports.getModule = class FileBaseDownloadQueueManager extends MenuModule {
return cb(null);
}
displayWebDownloadLinkForFileEntry(fileEntry) {
FileAreaWeb.getExistingTempDownloadServeItem(this.client, fileEntry, (err, serveItem) => {
if(serveItem && serveItem.url) {
const webDlExpireTimeFormat = this.menuConfig.config.webDlExpireTimeFormat || 'YYYY-MMM-DD @ h:mm';
fileEntry.webDlLink = ansi.vtxHyperlink(this.client, serveItem.url) + serveItem.url;
fileEntry.webDlExpire = moment(serveItem.expireTimestamp).format(webDlExpireTimeFormat);
} else {
fileEntry.webDlLink = '';
fileEntry.webDlExpire = '';
}
this.updateCustomViewTextsWithFilter(
'queueManager',
MciViewIds.queueManager.customRangeStart, fileEntry,
{ filter : [ '{webDlLink}', '{webDlExpire}' ] }
);
});
}
updateDownloadQueueView(cb) {
const queueView = this.viewControllers.queueManager.getView(MciViewIds.queueManager.queue);
if(!queueView) {
@ -138,7 +158,13 @@ exports.getModule = class FileBaseDownloadQueueManager extends MenuModule {
queueView.setItems(this.dlQueue.items.map( queueItem => stringFormat(queueListFormat, queueItem) ) );
queueView.setFocusItems(this.dlQueue.items.map( queueItem => stringFormat(focusQueueListFormat, queueItem) ) );
queueView.on('index update', idx => {
const fileEntry = this.dlQueue.items[idx];
this.displayWebDownloadLinkForFileEntry(fileEntry);
});
queueView.redraw();
this.displayWebDownloadLinkForFileEntry(this.dlQueue.items[0]);
return cb(null);
}

287
mods/file_base_web_download_manager.js Normal file
View File

@ -0,0 +1,287 @@
/* jslint node: true */
'use strict';
// ENiGMA½
const MenuModule = require('../core/menu_module.js').MenuModule;
const ViewController = require('../core/view_controller.js').ViewController;
const DownloadQueue = require('../core/download_queue.js');
const theme = require('../core/theme.js');
const ansi = require('../core/ansi_term.js');
const Errors = require('../core/enig_error.js').Errors;
const stringFormat = require('../core/string_format.js');
const FileAreaWeb = require('../core/file_area_web.js');
const ErrNotEnabled = require('../core/enig_error.js').ErrorReasons.NotEnabled;
const Config = require('../core/config.js').config;
// deps
const async = require('async');
const _ = require('lodash');
const moment = require('moment');
exports.moduleInfo = {
name : 'File Base Download Web Queue Manager',
desc : 'Module for interacting with web backed download queue/batch',
author : 'NuSkooler',
};
const FormIds = {
queueManager : 0
};
const MciViewIds = {
queueManager : {
queue : 1,
navMenu : 2,
customRangeStart : 10,
}
};
exports.getModule = class FileBaseWebDownloadQueueManager extends MenuModule {
constructor(options) {
super(options);
this.dlQueue = new DownloadQueue(this.client);
this.menuMethods = {
removeItem : (formData, extraArgs, cb) => {
const selectedItem = this.dlQueue.items[formData.value.queueItem];
if(!selectedItem) {
return cb(null);
}
this.dlQueue.removeItems(selectedItem.fileId);
// :TODO: broken: does not redraw menu properly - needs fixed!
return this.removeItemsFromDownloadQueueView(formData.value.queueItem, cb);
},
clearQueue : (formData, extraArgs, cb) => {
this.dlQueue.clear();
// :TODO: broken: does not redraw menu properly - needs fixed!
return this.removeItemsFromDownloadQueueView('all', cb);
},
getBatchLink : (formData, extraArgs, cb) => {
return this.generateAndDisplayBatchLink(cb);
}
};
}
initSequence() {
if(0 === this.dlQueue.items.length) {
return this.gotoMenu(this.menuConfig.config.emptyQueueMenu || 'fileBaseDownloadManagerEmptyQueue');
}
const self = this;
async.series(
[
function beforeArt(callback) {
return self.beforeArt(callback);
},
function display(callback) {
return self.displayQueueManagerPage(false, callback);
}
],
() => {
return self.finishedLoading();
}
);
}
removeItemsFromDownloadQueueView(itemIndex, cb) {
const queueView = this.viewControllers.queueManager.getView(MciViewIds.queueManager.queue);
if(!queueView) {
return cb(Errors.DoesNotExist('Queue view does not exist'));
}
if('all' === itemIndex) {
queueView.setItems([]);
queueView.setFocusItems([]);
} else {
queueView.removeItem(itemIndex);
}
queueView.redraw();
return cb(null);
}
displayFileInfoForFileEntry(fileEntry) {
this.updateCustomViewTextsWithFilter(
'queueManager',
MciViewIds.queueManager.customRangeStart, fileEntry,
{ filter : [ '{webDlLink}', '{webDlExpire}', '{fileName}' ] } // :TODO: Others....
);
}
updateDownloadQueueView(cb) {
const queueView = this.viewControllers.queueManager.getView(MciViewIds.queueManager.queue);
if(!queueView) {
return cb(Errors.DoesNotExist('Queue view does not exist'));
}
const queueListFormat = this.menuConfig.config.queueListFormat || '{webDlLink}';
const focusQueueListFormat = this.menuConfig.config.focusQueueListFormat || queueListFormat;
queueView.setItems(this.dlQueue.items.map( queueItem => stringFormat(queueListFormat, queueItem) ) );
queueView.setFocusItems(this.dlQueue.items.map( queueItem => stringFormat(focusQueueListFormat, queueItem) ) );
queueView.on('index update', idx => {
const fileEntry = this.dlQueue.items[idx];
this.displayFileInfoForFileEntry(fileEntry);
});
queueView.redraw();
this.displayFileInfoForFileEntry(this.dlQueue.items[0]);
return cb(null);
}
generateAndDisplayBatchLink(cb) {
const expireTime = moment().add(Config.fileBase.web.expireMinutes, 'minutes');
FileAreaWeb.createAndServeTempBatchDownload(
this.client,
this.dlQueue.items,
{
expireTime : expireTime
},
(err, webBatchDlLink) => {
// :TODO: handle not enabled -> display such
if(err) {
return cb(err);
}
const webDlExpireTimeFormat = this.menuConfig.config.webDlExpireTimeFormat || 'YYYY-MMM-DD @ h:mm';
const formatObj = {
webBatchDlLink : ansi.vtxHyperlink(this.client, webBatchDlLink) + webBatchDlLink,
webBatchDlExpire : expireTime.format(webDlExpireTimeFormat),
};
this.updateCustomViewTextsWithFilter(
'queueManager',
MciViewIds.queueManager.customRangeStart,
formatObj,
{ filter : Object.keys(formatObj).map(k => '{' + k + '}' ) }
);
return cb(null);
}
);
}
displayQueueManagerPage(clearScreen, cb) {
const self = this;
async.series(
[
function prepArtAndViewController(callback) {
return self.displayArtAndPrepViewController('queueManager', { clearScreen : clearScreen }, callback);
},
function prepareQueueDownloadLinks(callback) {
const webDlExpireTimeFormat = self.menuConfig.config.webDlExpireTimeFormat || 'YYYY-MMM-DD @ h:mm';
async.each(self.dlQueue.items, (fileEntry, nextFileEntry) => {
FileAreaWeb.getExistingTempDownloadServeItem(self.client, fileEntry, (err, serveItem) => {
if(err) {
if(ErrNotEnabled === err.reasonCode) {
return nextFileEntry(err); // we should have caught this prior
}
const expireTime = moment().add(Config.fileBase.web.expireMinutes, 'minutes');
FileAreaWeb.createAndServeTempDownload(
self.client,
fileEntry,
{ expireTime : expireTime },
(err, url) => {
if(err) {
return nextFileEntry(err);
}
fileEntry.webDlLinkRaw = url;
fileEntry.webDlLink = ansi.vtxHyperlink(self.client, url) + url;
fileEntry.webDlExpire = expireTime.format(webDlExpireTimeFormat);
return nextFileEntry(null);
}
);
} else {
fileEntry.webDlLinkRaw = serveItem.url;
fileEntry.webDlLink = ansi.vtxHyperlink(self.client, serveItem.url) + serveItem.url;
fileEntry.webDlExpire = moment(serveItem.expireTimestamp).format(webDlExpireTimeFormat);
return nextFileEntry(null);
}
});
}, err => {
return callback(err);
});
},
function populateViews(callback) {
return self.updateDownloadQueueView(callback);
}
],
err => {
if(cb) {
return cb(err);
}
}
);
}
displayArtAndPrepViewController(name, options, cb) {
const self = this;
const config = this.menuConfig.config;
async.waterfall(
[
function readyAndDisplayArt(callback) {
if(options.clearScreen) {
self.client.term.rawWrite(ansi.resetScreen());
}
theme.displayThemedAsset(
config.art[name],
self.client,
{ font : self.menuConfig.font, trailingLF : false },
(err, artData) => {
return callback(err, artData);
}
);
},
function prepeareViewController(artData, callback) {
if(_.isUndefined(self.viewControllers[name])) {
const vcOpts = {
client : self.client,
formId : FormIds[name],
};
if(!_.isUndefined(options.noInput)) {
vcOpts.noInput = options.noInput;
}
const vc = self.addViewController(name, new ViewController(vcOpts));
const loadOpts = {
callingMenu : self,
mciMap : artData.mciMap,
formId : FormIds[name],
};
return vc.loadFromMenuConfig(loadOpts, callback);
}
self.viewControllers[name].setFocus(true);
return callback(null);
},
],
err => {
return cb(err);
}
);
}
};

3
package.json
View File

@ -47,7 +47,8 @@
"temptmp": "^1.0.0",
"uuid": "^3.1.0",
"uuid-parse": "^1.0.0",
"ws": "^3.1.0"
"ws": "^3.1.0",
"yazl" : "^2.4.2"
},
"devDependencies": {},
"engines": {