/* jslint node: true */ 'use strict'; // ENiGMA½ const stringFormat = require('./string_format.js'); const FileEntry = require('./file_entry.js'); const FileArea = require('./file_base_area.js'); const Config = require('./config.js').get; const { Errors } = require('./enig_error.js'); const { splitTextAtTerms, isAnsi, } = require('./string_util.js'); const AnsiPrep = require('./ansi_prep.js'); const Log = require('./logger.js').log; // deps const _ = require('lodash'); const async = require('async'); const fs = require('graceful-fs'); const paths = require('path'); const iconv = require('iconv-lite'); const moment = require('moment'); exports.exportFileList = exportFileList; exports.updateFileBaseDescFilesScheduledEvent = updateFileBaseDescFilesScheduledEvent; function exportFileList(filterCriteria, options, cb) { options.templateEncoding = options.templateEncoding || 'utf8'; options.entryTemplate = options.entryTemplate || 'descript_ion_export_entry_template.asc'; options.tsFormat = options.tsFormat || 'YYYY-MM-DD'; options.descWidth = options.descWidth || 45; // FILE_ID.DIZ spec options.escapeDesc = _.get(options, 'escapeDesc', false); // escape \r and \n in desc? if(true === options.escapeDesc) { options.escapeDesc = '\\n'; } const state = { total : 0, current : 0, step : 'preparing', status : 'Preparing', }; const updateProgress = _.isFunction(options.progress) ? progCb => { return options.progress(state, progCb); } : progCb => { return progCb(null); } ; async.waterfall( [ function readTemplateFiles(callback) { updateProgress(err => { if(err) { return callback(err); } const templateFiles = [ { name : options.headerTemplate, req : false }, { name : options.entryTemplate, req : true } ]; const config = Config(); async.map(templateFiles, (template, nextTemplate) => { if(!template.name && !template.req) { return nextTemplate(null, Buffer.from([])); } template.name = paths.isAbsolute(template.name) ? template.name : paths.join(config.paths.misc, template.name); fs.readFile(template.name, (err, data) => { return nextTemplate(err, data); }); }, (err, templates) => { if(err) { return callback(Errors.General(err.message)); } // decode + ensure DOS style CRLF templates = templates.map(tmp => iconv.decode(tmp, options.templateEncoding).replace(/\r?\n/g, '\r\n') ); // Look for the first {fileDesc} (if any) in 'entry' template & find indentation requirements let descIndent = 0; if(!options.escapeDesc) { splitTextAtTerms(templates[1]).some(line => { const pos = line.indexOf('{fileDesc}'); if(pos > -1) { descIndent = pos; return true; // found it! } return false; // keep looking }); } return callback(null, templates[0], templates[1], descIndent); }); }); }, function findFiles(headerTemplate, entryTemplate, descIndent, callback) { state.step = 'gathering'; state.status = 'Gathering files for supplied criteria'; updateProgress(err => { if(err) { return callback(err); } FileEntry.findFiles(filterCriteria, (err, fileIds) => { if(0 === fileIds.length) { return callback(Errors.General('No results for criteria', 'NORESULTS')); } return callback(err, headerTemplate, entryTemplate, descIndent, fileIds); }); }); }, function buildListEntries(headerTemplate, entryTemplate, descIndent, fileIds, callback) { const formatObj = { totalFileCount : fileIds.length, }; let current = 0; let listBody = ''; const totals = { fileCount : fileIds.length, bytes : 0 }; state.total = fileIds.length; state.step = 'file'; async.eachSeries(fileIds, (fileId, nextFileId) => { const fileInfo = new FileEntry(); current += 1; fileInfo.load(fileId, err => { if(err) { return nextFileId(null); // failed, but try the next } totals.bytes += fileInfo.meta.byte_size; const appendFileInfo = () => { if(options.escapeDesc) { formatObj.fileDesc = formatObj.fileDesc.replace(/\r?\n/g, options.escapeDesc); } if(options.maxDescLen) { formatObj.fileDesc = formatObj.fileDesc.slice(0, options.maxDescLen); } listBody += stringFormat(entryTemplate, formatObj); state.current = current; state.status = `Processing ${fileInfo.fileName}`; state.fileInfo = formatObj; updateProgress(err => { return nextFileId(err); }); }; const area = FileArea.getFileAreaByTag(fileInfo.areaTag); formatObj.fileId = fileId; formatObj.areaName = _.get(area, 'name') || 'N/A'; formatObj.areaDesc = _.get(area, 'desc') || 'N/A'; formatObj.userRating = fileInfo.userRating || 0; formatObj.fileName = fileInfo.fileName; formatObj.fileSize = fileInfo.meta.byte_size; formatObj.fileDesc = fileInfo.desc || ''; formatObj.fileDescShort = formatObj.fileDesc.slice(0, options.descWidth); formatObj.fileSha256 = fileInfo.fileSha256; formatObj.fileCrc32 = fileInfo.meta.file_crc32; formatObj.fileMd5 = fileInfo.meta.file_md5; formatObj.fileSha1 = fileInfo.meta.file_sha1; formatObj.uploadBy = fileInfo.meta.upload_by_username || 'N/A'; formatObj.fileUploadTs = moment(fileInfo.uploadTimestamp).format(options.tsFormat); formatObj.fileHashTags = fileInfo.hashTags.size > 0 ? Array.from(fileInfo.hashTags).join(', ') : 'N/A'; formatObj.currentFile = current; formatObj.progress = Math.floor( (current / fileIds.length) * 100 ); if(isAnsi(fileInfo.desc)) { AnsiPrep( fileInfo.desc, { cols : Math.min(options.descWidth, 79 - descIndent), forceLineTerm : true, // ensure each line is term'd asciiMode : true, // export to ASCII fillLines : false, // don't fill up to |cols| indent : descIndent, }, (err, desc) => { if(desc) { formatObj.fileDesc = desc; } return appendFileInfo(); } ); } else { const indentSpc = descIndent > 0 ? ' '.repeat(descIndent) : ''; formatObj.fileDesc = splitTextAtTerms(formatObj.fileDesc).join(`\r\n${indentSpc}`) + '\r\n'; return appendFileInfo(); } }); }, err => { return callback(err, listBody, headerTemplate, totals); }); }, function buildHeader(listBody, headerTemplate, totals, callback) { // header is built last such that we can have totals/etc. let filterAreaName; let filterAreaDesc; if(filterCriteria.areaTag) { const area = FileArea.getFileAreaByTag(filterCriteria.areaTag); filterAreaName = _.get(area, 'name') || 'N/A'; filterAreaDesc = _.get(area, 'desc') || 'N/A'; } else { filterAreaName = '-ALL-'; filterAreaDesc = 'All areas'; } const headerFormatObj = { nowTs : moment().format(options.tsFormat), boardName : Config().general.boardName, totalFileCount : totals.fileCount, totalFileSize : totals.bytes, filterAreaTag : filterCriteria.areaTag || '-ALL-', filterAreaName : filterAreaName, filterAreaDesc : filterAreaDesc, filterTerms : filterCriteria.terms || '(none)', filterHashTags : filterCriteria.tags || '(none)', }; listBody = stringFormat(headerTemplate, headerFormatObj) + listBody; return callback(null, listBody); }, function done(listBody, callback) { delete state.fileInfo; state.step = 'finished'; state.status = 'Finished processing'; updateProgress( () => { return callback(null, listBody); }); } ], (err, listBody) => { return cb(err, listBody); } ); } function updateFileBaseDescFilesScheduledEvent(args, cb) { // // For each area, loop over storage locations and build // DESCRIPT.ION file to store in the same directory. // // Standard-ish 4DOS spec is as such: // * Entry: [0x04]\r\n // * Multi line descriptions are stored with *escaped* \r\n pairs // * Default template uses 0x2c for as per https://stackoverflow.com/questions/1810398/descript-ion-file-spec // const entryTemplate = args[0]; const headerTemplate = args[1]; const areas = FileArea.getAvailableFileAreas(null, { skipAcsCheck : true }); async.each(areas, (area, nextArea) => { const storageLocations = FileArea.getAreaStorageLocations(area); async.each(storageLocations, (storageLoc, nextStorageLoc) => { const filterCriteria = { areaTag : area.areaTag, storageTag : storageLoc.storageTag, }; const exportOpts = { headerTemplate : headerTemplate, entryTemplate : entryTemplate, escapeDesc : true, // escape CRLF's maxDescLen : 4096, // DESCRIPT.ION: "The line length limit is 4096 bytes" }; exportFileList(filterCriteria, exportOpts, (err, listBody) => { const descIonPath = paths.join(storageLoc.dir, 'DESCRIPT.ION'); fs.writeFile(descIonPath, iconv.encode(listBody, 'cp437'), err => { if(err) { Log.warn( { error : err.message, path : descIonPath }, 'Failed (re)creating DESCRIPT.ION'); } else { Log.debug( { path : descIonPath }, '(Re)generated DESCRIPT.ION'); } return nextStorageLoc(null); }); }); }, () => { return nextArea(null); }); }, () => { return cb(null); }); }