551 lines
18 KiB
JavaScript
551 lines
18 KiB
JavaScript
/* jslint node: true */
|
|
'use strict';
|
|
|
|
// ENiGMA½
|
|
const Config = require('./config.js').get;
|
|
const FileDb = require('./database.js').dbs.file;
|
|
const getISOTimestampString = require('./database.js').getISOTimestampString;
|
|
const FileEntry = require('./file_entry.js');
|
|
const getServer = require('./listening_server.js').getServer;
|
|
const Errors = require('./enig_error.js').Errors;
|
|
const ErrNotEnabled = require('./enig_error.js').ErrorReasons.NotEnabled;
|
|
const StatLog = require('./stat_log.js');
|
|
const User = require('./user.js');
|
|
const Log = require('./logger.js').log;
|
|
const getConnectionByUserId = require('./client_connections.js').getConnectionByUserId;
|
|
const webServerPackageName = require('./servers/content/web.js').moduleInfo.packageName;
|
|
const Events = require('./events.js');
|
|
const UserProps = require('./user_property.js');
|
|
const SysProps = require('./system_menu_method.js');
|
|
|
|
// deps
|
|
const hashids = require('hashids/cjs');
|
|
const moment = require('moment');
|
|
const paths = require('path');
|
|
const async = require('async');
|
|
const fs = require('graceful-fs');
|
|
const mimeTypes = require('mime-types');
|
|
const yazl = require('yazl');
|
|
|
|
function notEnabledError() {
|
|
return Errors.General('Web server is not enabled', ErrNotEnabled);
|
|
}
|
|
|
|
class FileAreaWebAccess {
|
|
constructor() {
|
|
this.hashids = new hashids(Config().general.boardName);
|
|
this.expireTimers = {}; // hashId->timer
|
|
}
|
|
|
|
startup(cb) {
|
|
const self = this;
|
|
|
|
async.series(
|
|
[
|
|
function initFromDb(callback) {
|
|
return self.load(callback);
|
|
},
|
|
function addWebRoute(callback) {
|
|
self.webServer = getServer(webServerPackageName);
|
|
if (!self.webServer) {
|
|
return callback(
|
|
Errors.DoesNotExist(
|
|
`Server with package name "${webServerPackageName}" does not exist`
|
|
)
|
|
);
|
|
}
|
|
|
|
if (self.isEnabled()) {
|
|
const routeAdded = self.webServer.instance.addRoute({
|
|
method: 'GET',
|
|
path: Config().fileBase.web.routePath,
|
|
handler: self.routeWebRequest.bind(self),
|
|
});
|
|
return callback(
|
|
routeAdded ? null : Errors.General('Failed adding route')
|
|
);
|
|
} else {
|
|
return callback(null); // not enabled, but no error
|
|
}
|
|
},
|
|
],
|
|
err => {
|
|
return cb(err);
|
|
}
|
|
);
|
|
}
|
|
|
|
shutdown(cb) {
|
|
return cb(null);
|
|
}
|
|
|
|
isEnabled() {
|
|
return this.webServer.instance.isEnabled();
|
|
}
|
|
|
|
static getHashIdTypes() {
|
|
return {
|
|
SingleFile: 0,
|
|
BatchArchive: 1,
|
|
};
|
|
}
|
|
|
|
load(cb) {
|
|
//
|
|
// Load entries, register expiration timers
|
|
//
|
|
FileDb.each(
|
|
`SELECT hash_id, expire_timestamp
|
|
FROM file_web_serve;`,
|
|
(err, row) => {
|
|
if (row) {
|
|
this.scheduleExpire(row.hash_id, moment(row.expire_timestamp));
|
|
}
|
|
},
|
|
err => {
|
|
return cb(err);
|
|
}
|
|
);
|
|
}
|
|
|
|
removeEntry(hashId) {
|
|
//
|
|
// Delete record from DB, and our timer
|
|
//
|
|
FileDb.run(
|
|
`DELETE FROM file_web_serve
|
|
WHERE hash_id = ?;`,
|
|
[hashId]
|
|
);
|
|
|
|
delete this.expireTimers[hashId];
|
|
}
|
|
|
|
scheduleExpire(hashId, expireTime) {
|
|
// remove any previous entry for this hashId
|
|
const previous = this.expireTimers[hashId];
|
|
if (previous) {
|
|
clearTimeout(previous);
|
|
delete this.expireTimers[hashId];
|
|
}
|
|
|
|
const timeoutMs = expireTime.diff(moment());
|
|
|
|
if (timeoutMs <= 0) {
|
|
setImmediate(() => {
|
|
this.removeEntry(hashId);
|
|
});
|
|
} else {
|
|
this.expireTimers[hashId] = setTimeout(() => {
|
|
this.removeEntry(hashId);
|
|
}, timeoutMs);
|
|
}
|
|
}
|
|
|
|
loadServedHashId(hashId, cb) {
|
|
FileDb.get(
|
|
`SELECT expire_timestamp FROM
|
|
file_web_serve
|
|
WHERE hash_id = ?`,
|
|
[hashId],
|
|
(err, result) => {
|
|
if (err || !result) {
|
|
return cb(
|
|
err ? err : Errors.DoesNotExist('Invalid or missing hash ID')
|
|
);
|
|
}
|
|
|
|
const decoded = this.hashids.decode(hashId);
|
|
|
|
// decode() should provide an array of [ userId, hashIdType, id, ... ]
|
|
if (!Array.isArray(decoded) || decoded.length < 3) {
|
|
return cb(Errors.Invalid('Invalid or unknown hash ID'));
|
|
}
|
|
|
|
const servedItem = {
|
|
hashId: hashId,
|
|
userId: decoded[0],
|
|
hashIdType: decoded[1],
|
|
expireTimestamp: moment(result.expire_timestamp),
|
|
};
|
|
|
|
if (
|
|
FileAreaWebAccess.getHashIdTypes().SingleFile ===
|
|
servedItem.hashIdType
|
|
) {
|
|
servedItem.fileIds = decoded.slice(2);
|
|
}
|
|
|
|
return cb(null, servedItem);
|
|
}
|
|
);
|
|
}
|
|
|
|
getSingleFileHashId(client, fileEntry) {
|
|
return this.getHashId(client, FileAreaWebAccess.getHashIdTypes().SingleFile, [
|
|
fileEntry.fileId,
|
|
]);
|
|
}
|
|
|
|
getBatchArchiveHashId(client, batchId) {
|
|
return this.getHashId(
|
|
client,
|
|
FileAreaWebAccess.getHashIdTypes().BatchArchive,
|
|
batchId
|
|
);
|
|
}
|
|
|
|
getHashId(client, hashIdType, identifier) {
|
|
return this.hashids.encode(client.user.userId, hashIdType, identifier);
|
|
}
|
|
|
|
buildSingleFileTempDownloadLink(client, fileEntry, hashId) {
|
|
hashId = hashId || this.getSingleFileHashId(client, fileEntry);
|
|
|
|
return this.webServer.instance.buildUrl(`${Config().fileBase.web.path}${hashId}`);
|
|
}
|
|
|
|
buildBatchArchiveTempDownloadLink(client, hashId) {
|
|
return this.webServer.instance.buildUrl(`${Config().fileBase.web.path}${hashId}`);
|
|
}
|
|
|
|
getExistingTempDownloadServeItem(client, fileEntry, cb) {
|
|
if (!this.isEnabled()) {
|
|
return cb(notEnabledError());
|
|
}
|
|
|
|
const hashId = this.getSingleFileHashId(client, fileEntry);
|
|
this.loadServedHashId(hashId, (err, servedItem) => {
|
|
if (err) {
|
|
return cb(err);
|
|
}
|
|
|
|
servedItem.url = this.buildSingleFileTempDownloadLink(client, fileEntry);
|
|
|
|
return cb(null, servedItem);
|
|
});
|
|
}
|
|
|
|
_addOrUpdateHashIdRecord(dbOrTrans, hashId, expireTime, cb) {
|
|
// add/update rec with hash id and (latest) timestamp
|
|
dbOrTrans.run(
|
|
`REPLACE INTO file_web_serve (hash_id, expire_timestamp)
|
|
VALUES (?, ?);`,
|
|
[hashId, getISOTimestampString(expireTime)],
|
|
err => {
|
|
if (err) {
|
|
return cb(err);
|
|
}
|
|
|
|
this.scheduleExpire(hashId, expireTime);
|
|
|
|
return cb(null);
|
|
}
|
|
);
|
|
}
|
|
|
|
createAndServeTempDownload(client, fileEntry, options, cb) {
|
|
if (!this.isEnabled()) {
|
|
return cb(notEnabledError());
|
|
}
|
|
|
|
const hashId = this.getSingleFileHashId(client, fileEntry);
|
|
const url = this.buildSingleFileTempDownloadLink(client, fileEntry, hashId);
|
|
options.expireTime = options.expireTime || moment().add(2, 'days');
|
|
|
|
this._addOrUpdateHashIdRecord(FileDb, hashId, options.expireTime, err => {
|
|
return cb(err, url);
|
|
});
|
|
}
|
|
|
|
createAndServeTempBatchDownload(client, fileEntries, options, cb) {
|
|
if (!this.isEnabled()) {
|
|
return cb(notEnabledError());
|
|
}
|
|
|
|
const batchId = moment().utc().unix();
|
|
const hashId = this.getBatchArchiveHashId(client, batchId);
|
|
const url = this.buildBatchArchiveTempDownloadLink(client, hashId);
|
|
options.expireTime = options.expireTime || moment().add(2, 'days');
|
|
|
|
FileDb.beginTransaction((err, trans) => {
|
|
if (err) {
|
|
return cb(err);
|
|
}
|
|
|
|
this._addOrUpdateHashIdRecord(trans, hashId, options.expireTime, err => {
|
|
if (err) {
|
|
return trans.rollback(() => {
|
|
return cb(err);
|
|
});
|
|
}
|
|
|
|
async.eachSeries(
|
|
fileEntries,
|
|
(entry, nextEntry) => {
|
|
trans.run(
|
|
`INSERT INTO file_web_serve_batch (hash_id, file_id)
|
|
VALUES (?, ?);`,
|
|
[hashId, entry.fileId],
|
|
err => {
|
|
return nextEntry(err);
|
|
}
|
|
);
|
|
},
|
|
err => {
|
|
trans[err ? 'rollback' : 'commit'](() => {
|
|
return cb(err, url);
|
|
});
|
|
}
|
|
);
|
|
});
|
|
});
|
|
}
|
|
|
|
fileNotFound(resp) {
|
|
return this.webServer.instance.fileNotFound(resp);
|
|
}
|
|
|
|
routeWebRequest(req, resp) {
|
|
const hashId = paths.basename(req.url);
|
|
|
|
Log.debug({ hashId: hashId, url: req.url }, 'File area web request');
|
|
|
|
this.loadServedHashId(hashId, (err, servedItem) => {
|
|
if (err) {
|
|
return this.fileNotFound(resp);
|
|
}
|
|
|
|
const hashIdTypes = FileAreaWebAccess.getHashIdTypes();
|
|
switch (servedItem.hashIdType) {
|
|
case hashIdTypes.SingleFile:
|
|
return this.routeWebRequestForSingleFile(servedItem, req, resp);
|
|
|
|
case hashIdTypes.BatchArchive:
|
|
return this.routeWebRequestForBatchArchive(servedItem, req, resp);
|
|
|
|
default:
|
|
return this.fileNotFound(resp);
|
|
}
|
|
});
|
|
}
|
|
|
|
routeWebRequestForSingleFile(servedItem, req, resp) {
|
|
Log.debug({ servedItem: servedItem }, 'Single file web request');
|
|
|
|
const fileEntry = new FileEntry();
|
|
|
|
servedItem.fileId = servedItem.fileIds[0];
|
|
|
|
fileEntry.load(servedItem.fileId, err => {
|
|
if (err) {
|
|
return this.fileNotFound(resp);
|
|
}
|
|
|
|
const filePath = fileEntry.filePath;
|
|
if (!filePath) {
|
|
return this.fileNotFound(resp);
|
|
}
|
|
|
|
fs.stat(filePath, (err, stats) => {
|
|
if (err) {
|
|
return this.fileNotFound(resp);
|
|
}
|
|
|
|
resp.on('close', () => {
|
|
// connection closed *before* the response was fully sent
|
|
// :TODO: Log and such
|
|
});
|
|
|
|
resp.on('finish', () => {
|
|
// transfer completed fully
|
|
this.updateDownloadStatsForUserIdAndSystem(
|
|
servedItem.userId,
|
|
stats.size,
|
|
[fileEntry]
|
|
);
|
|
});
|
|
|
|
const headers = {
|
|
'Content-Type':
|
|
mimeTypes.contentType(filePath) || mimeTypes.contentType('.bin'),
|
|
'Content-Length': stats.size,
|
|
'Content-Disposition': `attachment; filename="${fileEntry.fileName}"`,
|
|
};
|
|
|
|
const readStream = fs.createReadStream(filePath);
|
|
resp.writeHead(200, headers);
|
|
return readStream.pipe(resp);
|
|
});
|
|
});
|
|
}
|
|
|
|
routeWebRequestForBatchArchive(servedItem, req, resp) {
|
|
Log.debug({ servedItem: servedItem }, 'Batch file web request');
|
|
|
|
//
|
|
// We are going to build an on-the-fly zip file stream of 1:n
|
|
// files in the batch.
|
|
//
|
|
// First, collect all file IDs
|
|
//
|
|
const self = this;
|
|
|
|
async.waterfall(
|
|
[
|
|
function fetchFileIds(callback) {
|
|
FileDb.all(
|
|
`SELECT file_id
|
|
FROM file_web_serve_batch
|
|
WHERE hash_id = ?;`,
|
|
[servedItem.hashId],
|
|
(err, fileIdRows) => {
|
|
if (
|
|
err ||
|
|
!Array.isArray(fileIdRows) ||
|
|
0 === fileIdRows.length
|
|
) {
|
|
return callback(
|
|
Errors.DoesNotExist(
|
|
'Could not get file IDs for batch'
|
|
)
|
|
);
|
|
}
|
|
|
|
return callback(
|
|
null,
|
|
fileIdRows.map(r => r.file_id)
|
|
);
|
|
}
|
|
);
|
|
},
|
|
function loadFileEntries(fileIds, callback) {
|
|
async.map(
|
|
fileIds,
|
|
(fileId, nextFileId) => {
|
|
const fileEntry = new FileEntry();
|
|
fileEntry.load(fileId, err => {
|
|
return nextFileId(err, fileEntry);
|
|
});
|
|
},
|
|
(err, fileEntries) => {
|
|
if (err) {
|
|
return callback(
|
|
Errors.DoesNotExist(
|
|
'Could not load file IDs for batch'
|
|
)
|
|
);
|
|
}
|
|
|
|
return callback(null, fileEntries);
|
|
}
|
|
);
|
|
},
|
|
function createAndServeStream(fileEntries, callback) {
|
|
const filePaths = fileEntries.map(fe => fe.filePath);
|
|
Log.trace(
|
|
{ filePaths: filePaths },
|
|
'Creating zip archive for batch web request'
|
|
);
|
|
|
|
const zipFile = new yazl.ZipFile();
|
|
|
|
zipFile.on('error', err => {
|
|
Log.warn(
|
|
{ error: err.message },
|
|
'Error adding file to batch web request archive'
|
|
);
|
|
});
|
|
|
|
filePaths.forEach(fp => {
|
|
zipFile.addFile(
|
|
fp, // path to physical file
|
|
paths.basename(fp), // filename/path *stored in archive*
|
|
{
|
|
compress: false, // :TODO: do this smartly - if ext is in set = false, else true via isArchive() or such... mimeDB has this for us.
|
|
}
|
|
);
|
|
});
|
|
|
|
zipFile.end(finalZipSize => {
|
|
if (-1 === finalZipSize) {
|
|
return callback(
|
|
Errors.UnexpectedState('Unable to acquire final zip size')
|
|
);
|
|
}
|
|
|
|
resp.on('close', () => {
|
|
// connection closed *before* the response was fully sent
|
|
// :TODO: Log and such
|
|
});
|
|
|
|
resp.on('finish', () => {
|
|
// transfer completed fully
|
|
self.updateDownloadStatsForUserIdAndSystem(
|
|
servedItem.userId,
|
|
finalZipSize,
|
|
fileEntries
|
|
);
|
|
});
|
|
|
|
const batchFileName = `batch_${servedItem.hashId}.zip`;
|
|
|
|
const headers = {
|
|
'Content-Type':
|
|
mimeTypes.contentType(batchFileName) ||
|
|
mimeTypes.contentType('.bin'),
|
|
'Content-Length': finalZipSize,
|
|
'Content-Disposition': `attachment; filename="${batchFileName}"`,
|
|
};
|
|
|
|
resp.writeHead(200, headers);
|
|
return zipFile.outputStream.pipe(resp);
|
|
});
|
|
},
|
|
],
|
|
err => {
|
|
if (err) {
|
|
// :TODO: Log me!
|
|
return this.fileNotFound(resp);
|
|
}
|
|
|
|
// ...otherwise, we would have called resp() already.
|
|
}
|
|
);
|
|
}
|
|
|
|
updateDownloadStatsForUserIdAndSystem(userId, dlBytes, fileEntries) {
|
|
async.waterfall([
|
|
function fetchActiveUser(callback) {
|
|
const clientForUserId = getConnectionByUserId(userId);
|
|
if (clientForUserId) {
|
|
return callback(null, clientForUserId.user);
|
|
}
|
|
|
|
// not online now - look 'em up
|
|
User.getUser(userId, (err, assocUser) => {
|
|
return callback(err, assocUser);
|
|
});
|
|
},
|
|
function updateStats(user, callback) {
|
|
StatLog.incrementUserStat(user, UserProps.FileDlTotalCount, 1);
|
|
StatLog.incrementUserStat(user, UserProps.FileDlTotalBytes, dlBytes);
|
|
|
|
StatLog.incrementSystemStat(SysProps.FileDlTotalCount, 1);
|
|
StatLog.incrementSystemStat(SysProps.FileDlTotalBytes, dlBytes);
|
|
|
|
return callback(null, user);
|
|
},
|
|
function sendEvent(user, callback) {
|
|
Events.emit(Events.getSystemEvents().UserDownload, {
|
|
user: user,
|
|
files: fileEntries,
|
|
});
|
|
return callback(null);
|
|
},
|
|
]);
|
|
}
|
|
}
|
|
|
|
module.exports = new FileAreaWebAccess();
|