302 lines
13 KiB
JavaScript
302 lines
13 KiB
JavaScript
/* jslint node: true */
|
|
'use strict';
|
|
|
|
// ENiGMA½
|
|
const stringFormat = require('./string_format.js');
|
|
const FileEntry = require('./file_entry.js');
|
|
const FileArea = require('./file_base_area.js');
|
|
const Config = require('./config.js').get;
|
|
const { Errors } = require('./enig_error.js');
|
|
const {
|
|
splitTextAtTerms,
|
|
isAnsi,
|
|
} = require('./string_util.js');
|
|
const AnsiPrep = require('./ansi_prep.js');
|
|
const Log = require('./logger.js').log;
|
|
|
|
// deps
|
|
const _ = require('lodash');
|
|
const async = require('async');
|
|
const fs = require('graceful-fs');
|
|
const paths = require('path');
|
|
const iconv = require('iconv-lite');
|
|
const moment = require('moment');
|
|
|
|
exports.exportFileList = exportFileList;
|
|
exports.updateFileBaseDescFilesScheduledEvent = updateFileBaseDescFilesScheduledEvent;
|
|
|
|
function exportFileList(filterCriteria, options, cb) {
|
|
options.templateEncoding = options.templateEncoding || 'utf8';
|
|
options.entryTemplate = options.entryTemplate || 'descript_ion_export_entry_template.asc';
|
|
options.tsFormat = options.tsFormat || 'YYYY-MM-DD';
|
|
options.descWidth = options.descWidth || 45; // FILE_ID.DIZ spec
|
|
options.escapeDesc = _.get(options, 'escapeDesc', false); // escape \r and \n in desc?
|
|
|
|
if(true === options.escapeDesc) {
|
|
options.escapeDesc = '\\n';
|
|
}
|
|
|
|
const state = {
|
|
total : 0,
|
|
current : 0,
|
|
step : 'preparing',
|
|
status : 'Preparing',
|
|
};
|
|
|
|
const updateProgress = _.isFunction(options.progress) ?
|
|
progCb => {
|
|
return options.progress(state, progCb);
|
|
} :
|
|
progCb => {
|
|
return progCb(null);
|
|
}
|
|
;
|
|
|
|
async.waterfall(
|
|
[
|
|
function readTemplateFiles(callback) {
|
|
updateProgress(err => {
|
|
if(err) {
|
|
return callback(err);
|
|
}
|
|
|
|
const templateFiles = [
|
|
{ name : options.headerTemplate, req : false },
|
|
{ name : options.entryTemplate, req : true }
|
|
];
|
|
|
|
const config = Config();
|
|
async.map(templateFiles, (template, nextTemplate) => {
|
|
if(!template.name && !template.req) {
|
|
return nextTemplate(null, Buffer.from([]));
|
|
}
|
|
|
|
template.name = paths.isAbsolute(template.name) ? template.name : paths.join(config.paths.misc, template.name);
|
|
fs.readFile(template.name, (err, data) => {
|
|
return nextTemplate(err, data);
|
|
});
|
|
}, (err, templates) => {
|
|
if(err) {
|
|
return callback(Errors.General(err.message));
|
|
}
|
|
|
|
// decode + ensure DOS style CRLF
|
|
templates = templates.map(tmp => iconv.decode(tmp, options.templateEncoding).replace(/\r?\n/g, '\r\n') );
|
|
|
|
// Look for the first {fileDesc} (if any) in 'entry' template & find indentation requirements
|
|
let descIndent = 0;
|
|
if(!options.escapeDesc) {
|
|
splitTextAtTerms(templates[1]).some(line => {
|
|
const pos = line.indexOf('{fileDesc}');
|
|
if(pos > -1) {
|
|
descIndent = pos;
|
|
return true; // found it!
|
|
}
|
|
return false; // keep looking
|
|
});
|
|
}
|
|
|
|
return callback(null, templates[0], templates[1], descIndent);
|
|
});
|
|
});
|
|
},
|
|
function findFiles(headerTemplate, entryTemplate, descIndent, callback) {
|
|
state.step = 'gathering';
|
|
state.status = 'Gathering files for supplied criteria';
|
|
updateProgress(err => {
|
|
if(err) {
|
|
return callback(err);
|
|
}
|
|
|
|
FileEntry.findFiles(filterCriteria, (err, fileIds) => {
|
|
if(0 === fileIds.length) {
|
|
return callback(Errors.General('No results for criteria', 'NORESULTS'));
|
|
}
|
|
|
|
return callback(err, headerTemplate, entryTemplate, descIndent, fileIds);
|
|
});
|
|
});
|
|
},
|
|
function buildListEntries(headerTemplate, entryTemplate, descIndent, fileIds, callback) {
|
|
const formatObj = {
|
|
totalFileCount : fileIds.length,
|
|
};
|
|
|
|
let current = 0;
|
|
let listBody = '';
|
|
const totals = { fileCount : fileIds.length, bytes : 0 };
|
|
state.total = fileIds.length;
|
|
|
|
state.step = 'file';
|
|
|
|
async.eachSeries(fileIds, (fileId, nextFileId) => {
|
|
const fileInfo = new FileEntry();
|
|
current += 1;
|
|
|
|
fileInfo.load(fileId, err => {
|
|
if(err) {
|
|
return nextFileId(null); // failed, but try the next
|
|
}
|
|
|
|
totals.bytes += fileInfo.meta.byte_size;
|
|
|
|
const appendFileInfo = () => {
|
|
if(options.escapeDesc) {
|
|
formatObj.fileDesc = formatObj.fileDesc.replace(/\r?\n/g, options.escapeDesc);
|
|
}
|
|
|
|
if(options.maxDescLen) {
|
|
formatObj.fileDesc = formatObj.fileDesc.slice(0, options.maxDescLen);
|
|
}
|
|
|
|
listBody += stringFormat(entryTemplate, formatObj);
|
|
|
|
state.current = current;
|
|
state.status = `Processing ${fileInfo.fileName}`;
|
|
state.fileInfo = formatObj;
|
|
|
|
updateProgress(err => {
|
|
return nextFileId(err);
|
|
});
|
|
};
|
|
|
|
const area = FileArea.getFileAreaByTag(fileInfo.areaTag);
|
|
|
|
formatObj.fileId = fileId;
|
|
formatObj.areaName = _.get(area, 'name') || 'N/A';
|
|
formatObj.areaDesc = _.get(area, 'desc') || 'N/A';
|
|
formatObj.userRating = fileInfo.userRating || 0;
|
|
formatObj.fileName = fileInfo.fileName;
|
|
formatObj.fileSize = fileInfo.meta.byte_size;
|
|
formatObj.fileDesc = fileInfo.desc || '';
|
|
formatObj.fileDescShort = formatObj.fileDesc.slice(0, options.descWidth);
|
|
formatObj.fileSha256 = fileInfo.fileSha256;
|
|
formatObj.fileCrc32 = fileInfo.meta.file_crc32;
|
|
formatObj.fileMd5 = fileInfo.meta.file_md5;
|
|
formatObj.fileSha1 = fileInfo.meta.file_sha1;
|
|
formatObj.uploadBy = fileInfo.meta.upload_by_username || 'N/A';
|
|
formatObj.fileUploadTs = moment(fileInfo.uploadTimestamp).format(options.tsFormat);
|
|
formatObj.fileHashTags = fileInfo.hashTags.size > 0 ? Array.from(fileInfo.hashTags).join(', ') : 'N/A';
|
|
formatObj.currentFile = current;
|
|
formatObj.progress = Math.floor( (current / fileIds.length) * 100 );
|
|
|
|
if(isAnsi(fileInfo.desc)) {
|
|
AnsiPrep(
|
|
fileInfo.desc,
|
|
{
|
|
cols : Math.min(options.descWidth, 79 - descIndent),
|
|
forceLineTerm : true, // ensure each line is term'd
|
|
asciiMode : true, // export to ASCII
|
|
fillLines : false, // don't fill up to |cols|
|
|
indent : descIndent,
|
|
},
|
|
(err, desc) => {
|
|
if(desc) {
|
|
formatObj.fileDesc = desc;
|
|
}
|
|
return appendFileInfo();
|
|
}
|
|
);
|
|
} else {
|
|
const indentSpc = descIndent > 0 ? ' '.repeat(descIndent) : '';
|
|
formatObj.fileDesc = splitTextAtTerms(formatObj.fileDesc).join(`\r\n${indentSpc}`) + '\r\n';
|
|
return appendFileInfo();
|
|
}
|
|
});
|
|
}, err => {
|
|
return callback(err, listBody, headerTemplate, totals);
|
|
});
|
|
},
|
|
function buildHeader(listBody, headerTemplate, totals, callback) {
|
|
// header is built last such that we can have totals/etc.
|
|
|
|
let filterAreaName;
|
|
let filterAreaDesc;
|
|
if(filterCriteria.areaTag) {
|
|
const area = FileArea.getFileAreaByTag(filterCriteria.areaTag);
|
|
filterAreaName = _.get(area, 'name') || 'N/A';
|
|
filterAreaDesc = _.get(area, 'desc') || 'N/A';
|
|
} else {
|
|
filterAreaName = '-ALL-';
|
|
filterAreaDesc = 'All areas';
|
|
}
|
|
|
|
const headerFormatObj = {
|
|
nowTs : moment().format(options.tsFormat),
|
|
boardName : Config().general.boardName,
|
|
totalFileCount : totals.fileCount,
|
|
totalFileSize : totals.bytes,
|
|
filterAreaTag : filterCriteria.areaTag || '-ALL-',
|
|
filterAreaName : filterAreaName,
|
|
filterAreaDesc : filterAreaDesc,
|
|
filterTerms : filterCriteria.terms || '(none)',
|
|
filterHashTags : filterCriteria.tags || '(none)',
|
|
};
|
|
|
|
listBody = stringFormat(headerTemplate, headerFormatObj) + listBody;
|
|
return callback(null, listBody);
|
|
},
|
|
function done(listBody, callback) {
|
|
delete state.fileInfo;
|
|
state.step = 'finished';
|
|
state.status = 'Finished processing';
|
|
updateProgress( () => {
|
|
return callback(null, listBody);
|
|
});
|
|
}
|
|
], (err, listBody) => {
|
|
return cb(err, listBody);
|
|
}
|
|
);
|
|
}
|
|
|
|
function updateFileBaseDescFilesScheduledEvent(args, cb) {
|
|
//
|
|
// For each area, loop over storage locations and build
|
|
// DESCRIPT.ION file to store in the same directory.
|
|
//
|
|
// Standard-ish 4DOS spec is as such:
|
|
// * Entry: <QUOTED_LFN> <DESC>[0x04<AppData>]\r\n
|
|
// * Multi line descriptions are stored with *escaped* \r\n pairs
|
|
// * Default template uses 0x2c for <AppData> as per https://stackoverflow.com/questions/1810398/descript-ion-file-spec
|
|
//
|
|
const entryTemplate = args[0];
|
|
const headerTemplate = args[1];
|
|
|
|
const areas = FileArea.getAvailableFileAreas(null, { skipAcsCheck : true });
|
|
async.each(areas, (area, nextArea) => {
|
|
const storageLocations = FileArea.getAreaStorageLocations(area);
|
|
|
|
async.each(storageLocations, (storageLoc, nextStorageLoc) => {
|
|
const filterCriteria = {
|
|
areaTag : area.areaTag,
|
|
storageTag : storageLoc.storageTag,
|
|
};
|
|
|
|
const exportOpts = {
|
|
headerTemplate : headerTemplate,
|
|
entryTemplate : entryTemplate,
|
|
escapeDesc : true, // escape CRLF's
|
|
maxDescLen : 4096, // DESCRIPT.ION: "The line length limit is 4096 bytes"
|
|
};
|
|
|
|
exportFileList(filterCriteria, exportOpts, (err, listBody) => {
|
|
|
|
const descIonPath = paths.join(storageLoc.dir, 'DESCRIPT.ION');
|
|
fs.writeFile(descIonPath, iconv.encode(listBody, 'cp437'), err => {
|
|
if(err) {
|
|
Log.warn( { error : err.message, path : descIonPath }, 'Failed (re)creating DESCRIPT.ION');
|
|
} else {
|
|
Log.debug( { path : descIonPath }, '(Re)generated DESCRIPT.ION');
|
|
}
|
|
return nextStorageLoc(null);
|
|
});
|
|
});
|
|
}, () => {
|
|
return nextArea(null);
|
|
});
|
|
}, () => {
|
|
return cb(null);
|
|
});
|
|
}
|