enigma-bbs/core/file_area_web.js

/* jslint node: true */
'use strict';

//  ENiGMA½
const Config = require('./config.js').get;
const FileDb = require('./database.js').dbs.file;
const getISOTimestampString = require('./database.js').getISOTimestampString;
const FileEntry = require('./file_entry.js');
const getServer = require('./listening_server.js').getServer;
const Errors = require('./enig_error.js').Errors;
const ErrNotEnabled = require('./enig_error.js').ErrorReasons.NotEnabled;
const StatLog = require('./stat_log.js');
const User = require('./user.js');
const Log = require('./logger.js').log;
const getConnectionByUserId = require('./client_connections.js').getConnectionByUserId;
const webServerPackageName = require('./servers/content/web.js').moduleInfo.packageName;
const Events = require('./events.js');
const UserProps = require('./user_property.js');
const SysProps = require('./system_menu_method.js');

//  deps
const hashids = require('hashids/cjs');
const moment = require('moment');
const paths = require('path');
const async = require('async');
const fs = require('graceful-fs');
const mimeTypes = require('mime-types');
const yazl = require('yazl');

function notEnabledError() {
    return Errors.General('Web server is not enabled', ErrNotEnabled);
}

class FileAreaWebAccess {
    constructor() {
        this.hashids = new hashids(Config().general.boardName);
        this.expireTimers = {}; //  hashId->timer
    }

    startup(cb) {
        const self = this;

        async.series(
            [
                function initFromDb(callback) {
                    return self.load(callback);
                },
                function addWebRoute(callback) {
                    self.webServer = getServer(webServerPackageName);
                    if (!self.webServer) {
                        return callback(
                            Errors.DoesNotExist(
                                `Server with package name "${webServerPackageName}" does not exist`
                            )
                        );
                    }

                    if (self.isEnabled()) {
                        const routeAdded = self.webServer.instance.addRoute({
                            method: 'GET',
                            path: Config().fileBase.web.routePath,
                            handler: self.routeWebRequest.bind(self),
                        });
                        return callback(
                            routeAdded ? null : Errors.General('Failed adding route')
                        );
                    } else {
                        return callback(null); //  not enabled, but no error
                    }
                },
            ],
            err => {
                return cb(err);
            }
        );
    }

    shutdown(cb) {
        return cb(null);
    }

    isEnabled() {
        return this.webServer.instance.isEnabled();
    }

    static getHashIdTypes() {
        return {
            SingleFile: 0,
            BatchArchive: 1,
        };
    }

    load(cb) {
        //
        //  Load entries, register expiration timers
        //
        FileDb.each(
            `SELECT hash_id, expire_timestamp
            FROM file_web_serve;`,
            (err, row) => {
                if (row) {
                    this.scheduleExpire(row.hash_id, moment(row.expire_timestamp));
                }
            },
            err => {
                return cb(err);
            }
        );
    }

    removeEntry(hashId) {
        //
        //  Delete record from DB, and our timer
        //
        FileDb.run(
            `DELETE FROM file_web_serve
            WHERE hash_id = ?;`,
            [hashId]
        );

        delete this.expireTimers[hashId];
    }

    scheduleExpire(hashId, expireTime) {
        //  remove any previous entry for this hashId
        const previous = this.expireTimers[hashId];
        if (previous) {
            clearTimeout(previous);
            delete this.expireTimers[hashId];
        }

        const timeoutMs = expireTime.diff(moment());

        if (timeoutMs <= 0) {
            setImmediate(() => {
                this.removeEntry(hashId);
            });
        } else {
            this.expireTimers[hashId] = setTimeout(() => {
                this.removeEntry(hashId);
            }, timeoutMs);
        }
    }

    loadServedHashId(hashId, cb) {
        FileDb.get(
            `SELECT expire_timestamp FROM
            file_web_serve
            WHERE hash_id = ?`,
            [hashId],
            (err, result) => {
                if (err || !result) {
                    return cb(
                        err ? err : Errors.DoesNotExist('Invalid or missing hash ID')
                    );
                }

                const decoded = this.hashids.decode(hashId);

                //  decode() should provide an array of [ userId, hashIdType, id, ... ]
                if (!Array.isArray(decoded) || decoded.length < 3) {
                    return cb(Errors.Invalid('Invalid or unknown hash ID'));
                }

                const servedItem = {
                    hashId: hashId,
                    userId: decoded[0],
                    hashIdType: decoded[1],
                    expireTimestamp: moment(result.expire_timestamp),
                };

                if (
                    FileAreaWebAccess.getHashIdTypes().SingleFile ===
                    servedItem.hashIdType
                ) {
                    servedItem.fileIds = decoded.slice(2);
                }

                return cb(null, servedItem);
            }
        );
    }

    getSingleFileHashId(client, fileEntry) {
        return this.getHashId(client, FileAreaWebAccess.getHashIdTypes().SingleFile, [
            fileEntry.fileId,
        ]);
    }

    getBatchArchiveHashId(client, batchId) {
        return this.getHashId(
            client,
            FileAreaWebAccess.getHashIdTypes().BatchArchive,
            batchId
        );
    }

    getHashId(client, hashIdType, identifier) {
        return this.hashids.encode(client.user.userId, hashIdType, identifier);
    }

    buildSingleFileTempDownloadLink(client, fileEntry, hashId) {
        hashId = hashId || this.getSingleFileHashId(client, fileEntry);

        return this.webServer.instance.buildUrl(`${Config().fileBase.web.path}${hashId}`);
    }

    buildBatchArchiveTempDownloadLink(client, hashId) {
        return this.webServer.instance.buildUrl(`${Config().fileBase.web.path}${hashId}`);
    }

    getExistingTempDownloadServeItem(client, fileEntry, cb) {
        if (!this.isEnabled()) {
            return cb(notEnabledError());
        }

        const hashId = this.getSingleFileHashId(client, fileEntry);
        this.loadServedHashId(hashId, (err, servedItem) => {
            if (err) {
                return cb(err);
            }

            servedItem.url = this.buildSingleFileTempDownloadLink(client, fileEntry);

            return cb(null, servedItem);
        });
    }

    _addOrUpdateHashIdRecord(dbOrTrans, hashId, expireTime, cb) {
        //  add/update rec with hash id and (latest) timestamp
        dbOrTrans.run(
            `REPLACE INTO file_web_serve (hash_id, expire_timestamp)
            VALUES (?, ?);`,
            [hashId, getISOTimestampString(expireTime)],
            err => {
                if (err) {
                    return cb(err);
                }

                this.scheduleExpire(hashId, expireTime);

                return cb(null);
            }
        );
    }

    createAndServeTempDownload(client, fileEntry, options, cb) {
        if (!this.isEnabled()) {
            return cb(notEnabledError());
        }

        const hashId = this.getSingleFileHashId(client, fileEntry);
        const url = this.buildSingleFileTempDownloadLink(client, fileEntry, hashId);
        options.expireTime = options.expireTime || moment().add(2, 'days');

        this._addOrUpdateHashIdRecord(FileDb, hashId, options.expireTime, err => {
            return cb(err, url);
        });
    }

    createAndServeTempBatchDownload(client, fileEntries, options, cb) {
        if (!this.isEnabled()) {
            return cb(notEnabledError());
        }

        const batchId = moment().utc().unix();
        const hashId = this.getBatchArchiveHashId(client, batchId);
        const url = this.buildBatchArchiveTempDownloadLink(client, hashId);
        options.expireTime = options.expireTime || moment().add(2, 'days');

        FileDb.beginTransaction((err, trans) => {
            if (err) {
                return cb(err);
            }

            this._addOrUpdateHashIdRecord(trans, hashId, options.expireTime, err => {
                if (err) {
                    return trans.rollback(() => {
                        return cb(err);
                    });
                }

                async.eachSeries(
                    fileEntries,
                    (entry, nextEntry) => {
                        trans.run(
                            `INSERT INTO file_web_serve_batch (hash_id, file_id)
                        VALUES (?, ?);`,
                            [hashId, entry.fileId],
                            err => {
                                return nextEntry(err);
                            }
                        );
                    },
                    err => {
                        trans[err ? 'rollback' : 'commit'](() => {
                            return cb(err, url);
                        });
                    }
                );
            });
        });
    }

    fileNotFound(resp) {
        return this.webServer.instance.fileNotFound(resp);
    }

    routeWebRequest(req, resp) {
        const hashId = paths.basename(req.url);

        Log.debug({ hashId: hashId, url: req.url }, 'File area web request');

        this.loadServedHashId(hashId, (err, servedItem) => {
            if (err) {
                return this.fileNotFound(resp);
            }

            const hashIdTypes = FileAreaWebAccess.getHashIdTypes();
            switch (servedItem.hashIdType) {
                case hashIdTypes.SingleFile:
                    return this.routeWebRequestForSingleFile(servedItem, req, resp);

                case hashIdTypes.BatchArchive:
                    return this.routeWebRequestForBatchArchive(servedItem, req, resp);

                default:
                    return this.fileNotFound(resp);
            }
        });
    }

    routeWebRequestForSingleFile(servedItem, req, resp) {
        Log.debug({ servedItem: servedItem }, 'Single file web request');

        const fileEntry = new FileEntry();

        servedItem.fileId = servedItem.fileIds[0];

        fileEntry.load(servedItem.fileId, err => {
            if (err) {
                return this.fileNotFound(resp);
            }

            const filePath = fileEntry.filePath;
            if (!filePath) {
                return this.fileNotFound(resp);
            }

            fs.stat(filePath, (err, stats) => {
                if (err) {
                    return this.fileNotFound(resp);
                }

                resp.on('close', () => {
                    //  connection closed *before* the response was fully sent
                    //  :TODO: Log and such
                });

                resp.on('finish', () => {
                    //  transfer completed fully
                    this.updateDownloadStatsForUserIdAndSystem(
                        servedItem.userId,
                        stats.size,
                        [fileEntry]
                    );
                });

                const headers = {
                    'Content-Type':
                        mimeTypes.contentType(filePath) || mimeTypes.contentType('.bin'),
                    'Content-Length': stats.size,
                    'Content-Disposition': `attachment; filename="${fileEntry.fileName}"`,
                };

                const readStream = fs.createReadStream(filePath);
                resp.writeHead(200, headers);
                return readStream.pipe(resp);
            });
        });
    }

    routeWebRequestForBatchArchive(servedItem, req, resp) {
        Log.debug({ servedItem: servedItem }, 'Batch file web request');

        //
        //  We are going to build an on-the-fly zip file stream of 1:n
        //  files in the batch.
        //
        //  First, collect all file IDs
        //
        const self = this;

        async.waterfall(
            [
                function fetchFileIds(callback) {
                    FileDb.all(
                        `SELECT file_id
                        FROM file_web_serve_batch
                        WHERE hash_id = ?;`,
                        [servedItem.hashId],
                        (err, fileIdRows) => {
                            if (
                                err ||
                                !Array.isArray(fileIdRows) ||
                                0 === fileIdRows.length
                            ) {
                                return callback(
                                    Errors.DoesNotExist(
                                        'Could not get file IDs for batch'
                                    )
                                );
                            }

                            return callback(
                                null,
                                fileIdRows.map(r => r.file_id)
                            );
                        }
                    );
                },
                function loadFileEntries(fileIds, callback) {
                    async.map(
                        fileIds,
                        (fileId, nextFileId) => {
                            const fileEntry = new FileEntry();
                            fileEntry.load(fileId, err => {
                                return nextFileId(err, fileEntry);
                            });
                        },
                        (err, fileEntries) => {
                            if (err) {
                                return callback(
                                    Errors.DoesNotExist(
                                        'Could not load file IDs for batch'
                                    )
                                );
                            }

                            return callback(null, fileEntries);
                        }
                    );
                },
                function createAndServeStream(fileEntries, callback) {
                    const filePaths = fileEntries.map(fe => fe.filePath);
                    Log.trace(
                        { filePaths: filePaths },
                        'Creating zip archive for batch web request'
                    );

                    const zipFile = new yazl.ZipFile();

                    zipFile.on('error', err => {
                        Log.warn(
                            { error: err.message },
                            'Error adding file to batch web request archive'
                        );
                    });

                    filePaths.forEach(fp => {
                        zipFile.addFile(
                            fp, //  path to physical file
                            paths.basename(fp), //  filename/path *stored in archive*
                            {
                                compress: false, //  :TODO: do this smartly - if ext is in set = false, else true via isArchive() or such... mimeDB has this for us.
                            }
                        );
                    });

                    zipFile.end(finalZipSize => {
                        if (-1 === finalZipSize) {
                            return callback(
                                Errors.UnexpectedState('Unable to acquire final zip size')
                            );
                        }

                        resp.on('close', () => {
                            //  connection closed *before* the response was fully sent
                            //  :TODO: Log and such
                        });

                        resp.on('finish', () => {
                            //  transfer completed fully
                            self.updateDownloadStatsForUserIdAndSystem(
                                servedItem.userId,
                                finalZipSize,
                                fileEntries
                            );
                        });

                        const batchFileName = `batch_${servedItem.hashId}.zip`;

                        const headers = {
                            'Content-Type':
                                mimeTypes.contentType(batchFileName) ||
                                mimeTypes.contentType('.bin'),
                            'Content-Length': finalZipSize,
                            'Content-Disposition': `attachment; filename="${batchFileName}"`,
                        };

                        resp.writeHead(200, headers);
                        return zipFile.outputStream.pipe(resp);
                    });
                },
            ],
            err => {
                if (err) {
                    //  :TODO: Log me!
                    return this.fileNotFound(resp);
                }

                //  ...otherwise, we would have called resp() already.
            }
        );
    }

    updateDownloadStatsForUserIdAndSystem(userId, dlBytes, fileEntries) {
        async.waterfall([
            function fetchActiveUser(callback) {
                const clientForUserId = getConnectionByUserId(userId);
                if (clientForUserId) {
                    return callback(null, clientForUserId.user);
                }

                //  not online now - look 'em up
                User.getUser(userId, (err, assocUser) => {
                    return callback(err, assocUser);
                });
            },
            function updateStats(user, callback) {
                StatLog.incrementUserStat(user, UserProps.FileDlTotalCount, 1);
                StatLog.incrementUserStat(user, UserProps.FileDlTotalBytes, dlBytes);

                StatLog.incrementSystemStat(SysProps.FileDlTotalCount, 1);
                StatLog.incrementSystemStat(SysProps.FileDlTotalBytes, dlBytes);

                StatLog.incrementNonPersistentSystemStat(SysProps.FileDlTodayCount, 1);
                StatLog.incrementNonPersistentSystemStat(
                    SysProps.FileDlTodayBytes,
                    dlBytes
                );

                return callback(null, user);
            },
            function sendEvent(user, callback) {
                Events.emit(Events.getSystemEvents().UserDownload, {
                    user: user,
                    files: fileEntries,
                });
                return callback(null);
            },
        ]);
    }
}

module.exports = new FileAreaWebAccess();