enigma-bbs/core/upload.js

/* jslint node: true */
'use strict';

//  enigma-bbs
const { MenuModule, MenuFlags } = require('./menu_module');
const stringFormat = require('./string_format.js');
const getSortedAvailableFileAreas =
    require('./file_base_area.js').getSortedAvailableFileAreas;
const getAreaDefaultStorageDirectory =
    require('./file_base_area.js').getAreaDefaultStorageDirectory;
const scanFile = require('./file_base_area.js').scanFile;
const getFileAreaByTag = require('./file_base_area.js').getFileAreaByTag;
const getDescFromFileName = require('./file_base_area.js').getDescFromFileName;
const ansiGoto = require('./ansi_term.js').goto;
const moveFileWithCollisionHandling =
    require('./file_util.js').moveFileWithCollisionHandling;
const pathWithTerminatingSeparator =
    require('./file_util.js').pathWithTerminatingSeparator;
const Log = require('./logger.js').log;
const Errors = require('./enig_error.js').Errors;
const FileEntry = require('./file_entry.js');
const isAnsi = require('./string_util.js').isAnsi;
const Events = require('./events.js');

//  deps
const async = require('async');
const _ = require('lodash');
const temptmp = require('temptmp').createTrackedSession('upload');
const paths = require('path');
const sanatizeFilename = require('sanitize-filename');

exports.moduleInfo = {
    name: 'Upload',
    desc: 'Module for classic file uploads',
    author: 'NuSkooler',
};

const FormIds = {
    options: 0,
    processing: 1,
    fileDetails: 2,
    dupes: 3,
};

const MciViewIds = {
    options: {
        area: 1, //  area selection
        uploadType: 2, //  blind vs specify filename
        fileName: 3, //  for non-blind; not editable for blind
        navMenu: 4, //  next/cancel/etc.
        errMsg: 5, //  errors (e.g. filename cannot be blank)
    },

    processing: {
        calcHashIndicator: 1,
        archiveListIndicator: 2,
        descFileIndicator: 3,
        logStep: 4,
        customRangeStart: 10, //  10+ = customs
    },

    fileDetails: {
        desc: 1, //  defaults to 'desc' (e.g. from FILE_ID.DIZ)
        tags: 2, //  tag(s) for item
        estYear: 3,
        accept: 4, //  accept fields & continue
        customRangeStart: 10, //  10+ = customs
    },

    dupes: {
        dupeList: 1,
    },
};

exports.getModule = class UploadModule extends MenuModule {
    constructor(options) {
        super(options);

        this.setMergedFlag(MenuFlags.NoHistory);

        this.interrupt = MenuModule.InterruptTypes.Never;

        if (_.has(options, 'lastMenuResult.recvFilePaths')) {
            this.recvFilePaths = options.lastMenuResult.recvFilePaths;
        }

        this.availAreas = getSortedAvailableFileAreas(this.client, { writeAcs: true });

        this.menuMethods = {
            optionsNavContinue: (formData, extraArgs, cb) => {
                return this.performUpload(cb);
            },

            fileDetailsContinue: (formData, extraArgs, cb) => {
                //  see displayFileDetailsPageForUploadEntry() for this hackery:
                cb(null);
                return this.fileDetailsCurrentEntrySubmitCallback(null, formData.value); //  move on to the next entry, if any
            },

            //  validation
            validateNonBlindFileName: (fileName, cb) => {
                if (0 === fileName.length) {
                    return cb(new Error('Filename cannot be empty'));
                }

                fileName = sanatizeFilename(fileName); //  remove unsafe chars, path info, etc.
                if (0 === fileName.length) {
                    //  sanatize nuked everything?
                    return cb(new Error('Invalid filename'));
                }

                //  At least SEXYZ doesn't like non-blind names that start with a number - it becomes confused ;-(
                if (/^[0-9].*$/.test(fileName)) {
                    return cb(new Error('Invalid filename'));
                }

                return cb(null);
            },
            viewValidationListener: (err, cb) => {
                const errView = this.viewControllers.options.getView(
                    MciViewIds.options.errMsg
                );
                if (errView) {
                    if (err) {
                        errView.setText(err.message);
                    } else {
                        errView.clearText();
                    }
                }

                return cb(null);
            },
        };
    }

    getSaveState() {
        //  if no areas, we're falling back due to lack of access/areas avail to upload to
        if (this.availAreas.length > 0 && this.viewControllers.options) {
            return {
                uploadType: this.uploadType,
                tempRecvDirectory: this.tempRecvDirectory,
                areaInfo:
                    this.availAreas[
                        this.viewControllers.options
                            .getView(MciViewIds.options.area)
                            .getData()
                    ],
            };
        }
    }

    restoreSavedState(savedState) {
        if (savedState.areaInfo) {
            this.uploadType = savedState.uploadType;
            this.areaInfo = savedState.areaInfo;
            this.tempRecvDirectory = savedState.tempRecvDirectory;
        }
    }

    isBlindUpload() {
        return 'blind' === this.uploadType;
    }
    isFileTransferComplete() {
        return !_.isUndefined(this.recvFilePaths);
    }

    initSequence() {
        const self = this;

        if (0 === this.availAreas.length) {
            //
            return this.gotoMenu(
                this.menuConfig.config.noUploadAreasAvailMenu ||
                    'fileBaseNoUploadAreasAvail'
            );
        }

        async.series(
            [
                function before(callback) {
                    return self.beforeArt(callback);
                },
                function display(callback) {
                    if (self.isFileTransferComplete()) {
                        return self.displayProcessingPage(callback);
                    } else {
                        return self.displayOptionsPage(callback);
                    }
                },
            ],
            () => {
                return self.finishedLoading();
            }
        );
    }

    finishedLoading() {
        if (this.isFileTransferComplete()) {
            return this.processUploadedFiles();
        }
    }

    performUpload(cb) {
        temptmp.mkdir({ prefix: 'enigul-' }, (err, tempRecvDirectory) => {
            if (err) {
                return cb(err);
            }

            //  need a terminator for various external protocols
            this.tempRecvDirectory = pathWithTerminatingSeparator(tempRecvDirectory);

            const modOpts = {
                extraArgs: {
                    recvDirectory: this.tempRecvDirectory, //  we'll move files from here to their area container once processed/confirmed
                    direction: 'recv',
                },
            };

            if (!this.isBlindUpload()) {
                //  data has been sanatized at this point
                modOpts.extraArgs.recvFileName = this.viewControllers.options
                    .getView(MciViewIds.options.fileName)
                    .getData();
            }

            //
            //  Move along to protocol selection -> file transfer
            //  Upon completion, we'll re-enter the module with some file paths handed to us
            //
            return this.gotoMenu(
                this.menuConfig.config.fileTransferProtocolSelection ||
                    'fileTransferProtocolSelection',
                modOpts,
                cb
            );
        });
    }

    continueNonBlindUpload(cb) {
        return cb(null);
    }

    updateScanStepInfoViews(stepInfo) {
        //  :TODO: add some blinking (e.g. toggle items) indicators - see OBV.DOC

        const fmtObj = Object.assign({}, stepInfo);
        let stepIndicatorFmt = '';
        let logStepFmt;

        const fmtConfig = this.menuConfig.config;

        const indicatorStates = fmtConfig.indicatorStates || ['|', '/', '-', '\\'];
        const indicatorFinished = fmtConfig.indicatorFinished || '√';

        const indicator = {};
        const self = this;

        function updateIndicator(mci, isFinished) {
            indicator.mci = mci;

            if (isFinished) {
                indicator.text = indicatorFinished;
            } else {
                self.scanStatus.indicatorPos += 1;
                if (self.scanStatus.indicatorPos >= indicatorStates.length) {
                    self.scanStatus.indicatorPos = 0;
                }
                indicator.text = indicatorStates[self.scanStatus.indicatorPos];
            }
        }

        switch (stepInfo.step) {
            case 'start':
                logStepFmt = stepIndicatorFmt =
                    fmtConfig.scanningStartFormat || 'Scanning {fileName}';
                break;

            case 'hash_update':
                stepIndicatorFmt =
                    fmtConfig.calcHashFormat ||
                    'Calculating hash/checksums: {calcHashPercent}%';
                updateIndicator(MciViewIds.processing.calcHashIndicator);
                break;

            case 'hash_finish':
                stepIndicatorFmt =
                    fmtConfig.calcHashCompleteFormat ||
                    'Finished calculating hash/checksums';
                updateIndicator(MciViewIds.processing.calcHashIndicator, true);
                break;

            case 'archive_list_start':
                stepIndicatorFmt =
                    fmtConfig.extractArchiveListFormat || 'Extracting archive list';
                updateIndicator(MciViewIds.processing.archiveListIndicator);
                break;

            case 'archive_list_finish':
                fmtObj.archivedFileCount = stepInfo.archiveEntries.length;
                stepIndicatorFmt =
                    fmtConfig.extractArchiveListFinishFormat ||
                    'Archive list extracted ({archivedFileCount} files)';
                updateIndicator(MciViewIds.processing.archiveListIndicator, true);
                break;

            case 'archive_list_failed':
                stepIndicatorFmt =
                    fmtConfig.extractArchiveListFailedFormat ||
                    'Archive list extraction failed';
                break;

            case 'desc_files_start':
                stepIndicatorFmt =
                    fmtConfig.processingDescFilesFormat || 'Processing description files';
                updateIndicator(MciViewIds.processing.descFileIndicator);
                break;

            case 'desc_files_finish':
                stepIndicatorFmt =
                    fmtConfig.processingDescFilesFinishFormat ||
                    'Finished processing description files';
                updateIndicator(MciViewIds.processing.descFileIndicator, true);
                break;

            case 'finished':
                logStepFmt = stepIndicatorFmt =
                    fmtConfig.scanningStartFormat || 'Finished';
                break;
        }

        fmtObj.stepIndicatorText = stringFormat(stepIndicatorFmt, fmtObj);

        if (this.hasProcessingArt) {
            this.updateCustomViewTextsWithFilter(
                'processing',
                MciViewIds.processing.customRangeStart,
                fmtObj,
                { appendMultiLine: true }
            );

            if (indicator.mci && indicator.text) {
                this.setViewText('processing', indicator.mci, indicator.text);
            }

            if (logStepFmt) {
                this.setViewText(
                    'processing',
                    MciViewIds.processing.logStep,
                    stringFormat(logStepFmt, fmtObj),
                    { appendMultiLine: true }
                );
            }
        } else {
            this.client.term.pipeWrite(fmtObj.stepIndicatorText);
        }
    }

    scanFiles(cb) {
        const self = this;

        const results = {
            newEntries: [],
            dupes: [],
        };

        self.client.log.debug('Scanning upload(s)', { paths: this.recvFilePaths });

        let currentFileNum = 0;

        async.eachSeries(
            this.recvFilePaths,
            (filePath, nextFilePath) => {
                //  :TODO: virus scanning/etc. should occur around here

                currentFileNum += 1;

                self.scanStatus = {
                    indicatorPos: 0,
                };

                const scanOpts = {
                    areaTag: self.areaInfo.areaTag,
                    storageTag: self.areaInfo.storageTags[0],
                    hashTags: self.areaInfo.hashTags,
                };

                function handleScanStep(stepInfo, nextScanStep) {
                    stepInfo.totalFileNum = self.recvFilePaths.length;
                    stepInfo.currentFileNum = currentFileNum;

                    self.updateScanStepInfoViews(stepInfo);
                    return nextScanStep(null);
                }

                self.client.log.debug('Scanning file', { filePath: filePath });

                scanFile(
                    filePath,
                    scanOpts,
                    handleScanStep,
                    (err, fileEntry, dupeEntries) => {
                        if (err) {
                            return nextFilePath(err);
                        }

                        //  new or dupe?
                        if (dupeEntries.length > 0) {
                            //  1:n dupes found
                            self.client.log.debug('Duplicate file(s) found', {
                                dupeEntries: dupeEntries,
                            });

                            results.dupes = results.dupes.concat(dupeEntries);
                        } else {
                            //  new one
                            results.newEntries.push(fileEntry);
                        }

                        return nextFilePath(null);
                    }
                );
            },
            err => {
                return cb(err, results);
            }
        );
    }

    cleanupTempFiles() {
        temptmp.cleanup(paths => {
            Log.debug(
                { paths: paths, sessionId: temptmp.sessionId },
                'Temporary files cleaned up'
            );
        });
    }

    moveAndPersistUploadsToDatabase(newEntries) {
        const areaStorageDir = getAreaDefaultStorageDirectory(this.areaInfo);
        const self = this;

        async.eachSeries(
            newEntries,
            (newEntry, nextEntry) => {
                const src = paths.join(self.tempRecvDirectory, newEntry.fileName);
                const dst = paths.join(areaStorageDir, newEntry.fileName);

                moveFileWithCollisionHandling(src, dst, (err, finalPath) => {
                    if (err) {
                        self.client.log.error('Failed moving physical upload file', {
                            error: err.message,
                            fileName: newEntry.fileName,
                            source: src,
                            dest: dst,
                        });
                        return nextEntry(null); //  still try next file
                    } else if (dst !== finalPath) {
                        //  name changed; adjust before persist
                        newEntry.fileName = paths.basename(finalPath);
                    }

                    self.client.log.debug('Moved upload to area', { path: finalPath });

                    //  persist to DB
                    newEntry.persist(err => {
                        if (err) {
                            self.client.log.error(
                                'Failed persisting upload to database',
                                { path: finalPath, error: err.message }
                            );
                        }

                        return nextEntry(null); //  still try next file
                    });
                });
            },
            () => {
                //
                //  Finally, we can remove any temp files that we may have created
                //
                self.cleanupTempFiles();
            }
        );
    }

    prepDetailsForUpload(scanResults, cb) {
        async.eachSeries(
            scanResults.newEntries,
            (newEntry, nextEntry) => {
                newEntry.meta.upload_by_username = this.client.user.username;
                newEntry.meta.upload_by_user_id = this.client.user.userId;

                this.displayFileDetailsPageForUploadEntry(newEntry, (err, newValues) => {
                    if (err) {
                        return nextEntry(err);
                    }

                    if (!newEntry.descIsAnsi) {
                        newEntry.desc = _.trimEnd(newValues.shortDesc);
                    }

                    if (newValues.estYear.length > 0) {
                        newEntry.meta.est_release_year = newValues.estYear;
                    }

                    if (newValues.tags.length > 0) {
                        newEntry.setHashTags(newValues.tags);
                    }

                    return nextEntry(err);
                });
            },
            err => {
                delete this.fileDetailsCurrentEntrySubmitCallback;
                return cb(err, scanResults);
            }
        );
    }

    displayDupesPage(dupes, cb) {
        //
        //  If we have custom art to show, use it - else just dump basic info.
        //  Pause at the end in either case.
        //
        const self = this;

        async.waterfall(
            [
                function prepArtAndViewController(callback) {
                    self.prepViewControllerWithArt(
                        'dupes',
                        FormIds.dupes,
                        { clearScreen: true, trailingLF: false },
                        err => {
                            if (err) {
                                self.client.term.pipeWrite(
                                    '|00|07Duplicate upload(s) found:\n'
                                );
                                return callback(null, null);
                            }

                            const dupeListView = self.viewControllers.dupes.getView(
                                MciViewIds.dupes.dupeList
                            );
                            return callback(null, dupeListView);
                        }
                    );
                },
                function prepDupeObjects(dupeListView, callback) {
                    //  update dupe objects with additional info that can be used for formatString() and the like
                    async.each(
                        dupes,
                        (dupe, nextDupe) => {
                            FileEntry.loadBasicEntry(dupe.fileId, dupe, err => {
                                if (err) {
                                    return nextDupe(err);
                                }

                                const areaInfo = getFileAreaByTag(dupe.areaTag);
                                if (areaInfo) {
                                    dupe.areaName = areaInfo.name;
                                    dupe.areaDesc = areaInfo.desc;
                                }
                                return nextDupe(null);
                            });
                        },
                        err => {
                            return callback(err, dupeListView);
                        }
                    );
                },
                function populateDupeInfo(dupeListView, callback) {
                    const dupeInfoFormat =
                        self.menuConfig.config.dupeInfoFormat ||
                        '{fileName} @ {areaName}';

                    if (dupeListView) {
                        dupeListView.setItems(
                            dupes.map(dupe => stringFormat(dupeInfoFormat, dupe))
                        );
                        dupeListView.redraw();
                    } else {
                        dupes.forEach(dupe => {
                            self.client.term.pipeWrite(
                                `${stringFormat(dupeInfoFormat, dupe)}\n`
                            );
                        });
                    }

                    return callback(null);
                },
                function pause(callback) {
                    return self.pausePrompt(
                        { row: self.client.term.termHeight },
                        callback
                    );
                },
            ],
            err => {
                return cb(err);
            }
        );
    }

    processUploadedFiles() {
        //
        //  For each file uploaded, we need to process & gather information
        //
        const self = this;

        async.waterfall(
            [
                function prepNonBlind(callback) {
                    if (self.isBlindUpload()) {
                        return callback(null);
                    }

                    //
                    //  For non-blind uploads, batch is not supported, we expect a single file
                    //  in |recvFilePaths|. If not, it's an error (we don't want to process the wrong thing)
                    //
                    if (self.recvFilePaths.length > 1) {
                        self.client.log.warn(
                            { recvFilePaths: self.recvFilePaths },
                            'Non-blind upload received 2:n files'
                        );
                        return callback(
                            Errors.UnexpectedState(
                                `Non-blind upload expected single file but got received ${self.recvFilePaths.length}`
                            )
                        );
                    }

                    return callback(null);
                },
                function scan(callback) {
                    return self.scanFiles(callback);
                },
                function pause(scanResults, callback) {
                    if (self.hasProcessingArt) {
                        self.client.term.rawWrite(
                            ansiGoto(self.client.term.termHeight, 1)
                        );
                    } else {
                        self.client.term.write('\n');
                    }

                    self.pausePrompt(() => {
                        return callback(null, scanResults);
                    });
                },
                function displayDupes(scanResults, callback) {
                    if (0 === scanResults.dupes.length) {
                        return callback(null, scanResults);
                    }

                    return self.displayDupesPage(scanResults.dupes, () => {
                        return callback(null, scanResults);
                    });
                },
                function prepDetails(scanResults, callback) {
                    return self.prepDetailsForUpload(scanResults, callback);
                },
                function startMovingAndPersistingToDatabase(scanResults, callback) {
                    //
                    //  *Start* the process of moving files from their current |tempRecvDirectory|
                    //  locations -> their final area destinations. Don't make the user wait
                    //  here as I/O can take quite a bit of time. Log any failures.
                    //
                    self.moveAndPersistUploadsToDatabase(scanResults.newEntries);
                    return callback(null, scanResults.newEntries);
                },
                function sendEvent(uploadedEntries, callback) {
                    Events.emit(Events.getSystemEvents().UserUpload, {
                        user: self.client.user,
                        files: uploadedEntries,
                    });
                    return callback(null);
                },
            ],
            err => {
                if (err) {
                    self.client.log.warn('File upload error encountered', {
                        error: err.message,
                    });
                    self.cleanupTempFiles(); //  normally called after moveAndPersistUploadsToDatabase() is completed.
                }

                return self.prevMenu();
            }
        );
    }

    displayOptionsPage(cb) {
        const self = this;

        async.series(
            [
                function prepArtAndViewController(callback) {
                    return self.prepViewControllerWithArt(
                        'options',
                        FormIds.options,
                        { clearScreen: true, trailingLF: false },
                        callback
                    );
                },
                function populateViews(callback) {
                    const areaSelectView = self.viewControllers.options.getView(
                        MciViewIds.options.area
                    );
                    areaSelectView.setItems(
                        self.availAreas.map(areaInfo => areaInfo.name)
                    );

                    const uploadTypeView = self.viewControllers.options.getView(
                        MciViewIds.options.uploadType
                    );
                    const fileNameView = self.viewControllers.options.getView(
                        MciViewIds.options.fileName
                    );

                    const blindFileNameText =
                        self.menuConfig.config.blindFileNameText ||
                        '(blind - filename ignored)';

                    uploadTypeView.on('index update', idx => {
                        self.uploadType = 0 === idx ? 'blind' : 'non-blind';

                        if (self.isBlindUpload()) {
                            fileNameView.setText(blindFileNameText);
                            fileNameView.acceptsFocus = false;
                        } else {
                            fileNameView.clearText();
                            fileNameView.acceptsFocus = true;
                        }
                    });

                    //  sanatize filename for display when leaving the view
                    self.viewControllers.options.on('leave', prevView => {
                        if (prevView.id === MciViewIds.options.fileName) {
                            fileNameView.setText(
                                sanatizeFilename(fileNameView.getData())
                            );
                        }
                    });

                    self.uploadType = 'blind';
                    uploadTypeView.setFocusItemIndex(0); //  default to blind
                    fileNameView.setText(blindFileNameText);
                    areaSelectView.redraw();

                    return callback(null);
                },
            ],
            err => {
                if (cb) {
                    return cb(err);
                }
            }
        );
    }

    displayProcessingPage(cb) {
        return this.prepViewControllerWithArt(
            'processing',
            FormIds.processing,
            { clearScreen: true, trailingLF: false },
            err => {
                //  note: this art is not required
                this.hasProcessingArt = !err;

                return cb(null);
            }
        );
    }

    fileEntryHasDetectedDesc(fileEntry) {
        return fileEntry.desc && fileEntry.desc.length > 0;
    }

    displayFileDetailsPageForUploadEntry(fileEntry, cb) {
        const self = this;

        async.waterfall(
            [
                function prepArtAndViewController(callback) {
                    return self.prepViewControllerWithArt(
                        'fileDetails',
                        FormIds.fileDetails,
                        { clearScreen: true, trailingLF: false },
                        err => {
                            return callback(err);
                        }
                    );
                },
                function populateViews(callback) {
                    const descView = self.viewControllers.fileDetails.getView(
                        MciViewIds.fileDetails.desc
                    );
                    const tagsView = self.viewControllers.fileDetails.getView(
                        MciViewIds.fileDetails.tags
                    );
                    const yearView = self.viewControllers.fileDetails.getView(
                        MciViewIds.fileDetails.estYear
                    );

                    self.updateCustomViewTextsWithFilter(
                        'fileDetails',
                        MciViewIds.fileDetails.customRangeStart,
                        fileEntry
                    );

                    tagsView.setText(Array.from(fileEntry.hashTags).join(',')); //  :TODO: optional 'hashTagsSep' like file list/browse
                    yearView.setText(fileEntry.meta.est_release_year || '');

                    if (isAnsi(fileEntry.desc)) {
                        fileEntry.descIsAnsi = true;

                        return descView.setAnsi(
                            fileEntry.desc,
                            {
                                prepped: false,
                                forceLineTerm: true,
                            },
                            () => {
                                return callback(
                                    null,
                                    descView,
                                    'preview',
                                    MciViewIds.fileDetails.tags
                                );
                            }
                        );
                    } else {
                        const hasDesc = self.fileEntryHasDetectedDesc(fileEntry);
                        descView.setText(
                            hasDesc
                                ? fileEntry.desc
                                : getDescFromFileName(fileEntry.fileName),
                            { scrollMode: 'top' } //  override scroll mode; we want to be @ top
                        );
                        return callback(
                            null,
                            descView,
                            'edit',
                            hasDesc
                                ? MciViewIds.fileDetails.tags
                                : MciViewIds.fileDetails.desc
                        );
                    }
                },
                function finalizeViews(descView, descViewMode, focusId, callback) {
                    descView.setPropertyValue('mode', descViewMode);
                    descView.acceptsFocus = 'preview' === descViewMode ? false : true;
                    self.viewControllers.fileDetails.switchFocus(focusId);
                    return callback(null);
                },
            ],
            err => {
                //
                //  we only call |cb| here if there is an error
                //  else, wait for the current from to be submit - then call -
                //  this way we'll move on to the next file entry when ready
                //
                if (err) {
                    return cb(err);
                }

                self.fileDetailsCurrentEntrySubmitCallback = cb; //  stash for moduleMethods.fileDetailsContinue
            }
        );
    }
};