diff --git a/core/descript_ion_file.js b/core/descript_ion_file.js index a5d68e1d..d34551ba 100644 --- a/core/descript_ion_file.js +++ b/core/descript_ion_file.js @@ -1,6 +1,8 @@ /* jslint node: true */ 'use strict'; +const { Errors } = require('./enig_error.js'); + // deps const fs = require('graceful-fs'); const iconv = require('iconv-lite'); @@ -64,7 +66,10 @@ module.exports = class DescriptIonFile { return nextLine(null); }, () => { - return cb(null, descIonFile); + return cb( + descIonFile.entries.size > 0 ? null : Errors.Invalid('Invalid or unrecognized DESCRIPT.ION format'), + descIonFile + ); }); }); } diff --git a/core/files_bbs_file.js b/core/files_bbs_file.js new file mode 100644 index 00000000..989068a6 --- /dev/null +++ b/core/files_bbs_file.js @@ -0,0 +1,158 @@ +/* jslint node: true */ +'use strict'; + +const { Errors } = require('./enig_error.js'); + +// deps +const fs = require('graceful-fs'); +const iconv = require('iconv-lite'); +const moment = require('moment'); + +module.exports = class FilesBBSFile { + constructor() { + this.entries = new Map(); + } + + get(fileName) { + return this.entries.get(fileName); + } + + getDescription(fileName) { + const entry = this.get(fileName); + if(entry) { + return entry.desc; + } + } + + static createFromFile(path, cb) { + fs.readFile(path, (err, descData) => { + if(err) { + return cb(err); + } + + // :TODO: encoding should be default to CP437, but allowed to change - ie for Amiga/etc. + const lines = iconv.decode(descData, 'cp437').split(/\r?\n/g); + const filesBbs = new FilesBBSFile(); + + // + // Contrary to popular belief, there is not a FILES.BBS standard. Instead, + // many formats have been used over the years. We'll try to support as much + // as we can within reason. + // + // Resources: + // - Great info from Mystic @ http://wiki.mysticbbs.com/doku.php?id=mutil_import_files.bbs + // - https://alt.bbs.synchronet.narkive.com/I6Vrxq6q/format-of-files-bbs + // + // Example files: + // - https://github.com/NuSkooler/ansi-bbs/tree/master/ancient_formats/files_bbs + // + const detectDecoder = () => { + // + // Try to figure out which decoder to use + // + const decoders = [ + { + // I've been told this is what Syncrhonet uses + tester : /^([^ ]{1,12})\s{1,11}([0-3][0-9]\/[0-3][0-9]\/[1789][0-9]) ([^\r\n]+)$/, + extract : function() { + for(let i = 0; i < lines.length; ++i) { + let line = lines[i]; + const hdr = line.match(this.tester); + if(!hdr) { + continue; + } + const long = []; + for(let j = i + 1; j < lines.length; ++j) { + line = lines[j]; + if(!line.startsWith(' ')) { + break; + } + long.push(line.trim()); + ++i; + } + const desc = long.join('\r\n') || hdr[3] || ''; + const fileName = hdr[1]; + const timestamp = moment(hdr[2], 'MM/DD/YY'); + + filesBbs.entries.set(fileName, { timestamp, desc } ); + } + } + }, + + { + // + // Aminet Amiga CDROM, March 1994. Walnut Creek CDROM. + // CP/M CDROM, Sep. 1994. Walnut Creek CDROM. + // ...and many others. Basically: <8.3 filename> + // + // May contain headers, but we'll just skip 'em. + // + tester : /^([^ ]{1,12})\s{1,11}([^\r\n]+)$/, + extract : function() { + lines.forEach(line => { + const hdr = line.match(this.tester); + if(!hdr) { + return; // forEach + } + + const fileName = hdr[1].trim(); + const desc = hdr[2].trim(); + + if(desc) { + filesBbs.entries.set(fileName, { desc } ); + } + }); + } + }, + + { + // Found on AMINET CD's & similar + tester : /^(.{1,22}) ([0-9]+)K ([^\r\n]+)$/, + extract : function() { + lines.forEach(line => { + const hdr = line.match(this.tester); + if(!hdr) { + return; // forEach + } + + const fileName = hdr[1].trim(); + let size = parseInt(hdr[2]); + const desc = hdr[3].trim(); + + if(!isNaN(size)) { + size *= 1024; // K->bytes. + } + + if(desc) { // omit empty entries + filesBbs.entries.set(fileName, { size, desc } ); + } + }); + } + }, + ]; + + const decoder = decoders.find(d => { + return lines + .slice(0, 10) // 10 lines in should be enough to detect - skipping headers/etc. + .some(l => d.tester.test(l)); + }); + + return decoder; + }; + + const decoder = detectDecoder(); + if(!decoder) { + return cb(Errors.Invalid('Invalid or unrecognized FILES.BBS format')); + } + + decoder.extract(decoder); + + return cb( + filesBbs.entries.size > 0 ? null : Errors.Invalid('Invalid or unrecognized FILES.BBS format'), + filesBbs + ); + }); + } + + +}; diff --git a/core/oputil/oputil_file_base.js b/core/oputil/oputil_file_base.js index 2077b385..18113cc0 100644 --- a/core/oputil/oputil_file_base.js +++ b/core/oputil/oputil_file_base.js @@ -48,7 +48,7 @@ function finalizeEntryAndPersist(isUpdate, fileEntry, descHandler, cb) { async.series( [ function getDescFromHandlerIfNeeded(callback) { - if((fileEntry.desc && fileEntry.desc.length > 0 ) && !argv['desc-file']) { + if((fileEntry.desc && fileEntry.descSrc != 'fileName' && fileEntry.desc.length > 0 ) && !argv['desc-file']) { return callback(null); // we have a desc already and are NOT overriding with desc file } @@ -101,18 +101,47 @@ function finalizeEntryAndPersist(isUpdate, fileEntry, descHandler, cb) { ); } -const SCAN_EXCLUDE_FILENAMES = [ 'DESCRIPT.ION', 'FILES.BBS' ]; +const SCAN_EXCLUDE_FILENAMES = [ + 'DESCRIPT.ION', + 'FILES.BBS', + 'ALLFILES.TXT', +]; function loadDescHandler(path, cb) { - const DescIon = require('../../core/descript_ion_file.js'); + const handlerClassFromFileName = { + 'descript.ion' : require('../../core/descript_ion_file.js'), + 'files.bbs' : require('../../core/files_bbs_file.js'), + }[paths.basename(path).toLowerCase()]; - // :TODO: support FILES.BBS also + if(!handlerClassFromFileName) { + return cb(Errors.DoesNotExist(`No handlers registered for ${paths.basename(path)}`)); + } - DescIon.createFromFile(path, (err, descHandler) => { + handlerClassFromFileName.createFromFile(path, (err, descHandler) => { return cb(err, descHandler); }); } +// +// Try to find a suitable description handler by +// checking for common filenames. +// +function findSuitableDescHandler(basePath, cb) { + const commonFiles = [ 'FILES.BBS', 'DESCRIPT.ION' ]; + + async.eachSeries(commonFiles, (fileName, nextFileName) => { + loadDescHandler(paths.join(basePath, fileName), (err, handler) => { + if(!err && handler) { + return cb(null, handler); + } + return nextFileName(null); + }); + }, + () => { + return cb(Errors.DoesNotExist('No suitable description handler available')); + }); +} + function scanFileAreaForChanges(areaInfo, options, cb) { const storageLocations = fileArea.getAreaStorageLocations(areaInfo).filter(sl => { @@ -145,7 +174,7 @@ function scanFileAreaForChanges(areaInfo, options, cb) { return callback(null, options.descFileHandler); // we're going to use the global handler } - loadDescHandler(paths.join(storageLoc.dir, 'DESCRIPT.ION'), (err, descHandler) => { + findSuitableDescHandler(storageLoc.dir, (err, descHandler) => { return callback(null, descHandler); }); },