+ Initial FILES.BBS support during file scan: A few formats supported so far, more to come...

* Detect DESCRIPT.ION, FILES.BBS, etc. during scans
This commit is contained in:
Bryan Ashby 2018-12-17 12:08:06 -07:00
parent 8356f00ba6
commit 832e04cdf0
3 changed files with 199 additions and 7 deletions

View File

@ -1,6 +1,8 @@
/* jslint node: true */ /* jslint node: true */
'use strict'; 'use strict';
const { Errors } = require('./enig_error.js');
// deps // deps
const fs = require('graceful-fs'); const fs = require('graceful-fs');
const iconv = require('iconv-lite'); const iconv = require('iconv-lite');
@ -64,7 +66,10 @@ module.exports = class DescriptIonFile {
return nextLine(null); return nextLine(null);
}, },
() => { () => {
return cb(null, descIonFile); return cb(
descIonFile.entries.size > 0 ? null : Errors.Invalid('Invalid or unrecognized DESCRIPT.ION format'),
descIonFile
);
}); });
}); });
} }

158
core/files_bbs_file.js Normal file
View File

@ -0,0 +1,158 @@
/* jslint node: true */
'use strict';
const { Errors } = require('./enig_error.js');
// deps
const fs = require('graceful-fs');
const iconv = require('iconv-lite');
const moment = require('moment');
module.exports = class FilesBBSFile {
constructor() {
this.entries = new Map();
}
get(fileName) {
return this.entries.get(fileName);
}
getDescription(fileName) {
const entry = this.get(fileName);
if(entry) {
return entry.desc;
}
}
static createFromFile(path, cb) {
fs.readFile(path, (err, descData) => {
if(err) {
return cb(err);
}
// :TODO: encoding should be default to CP437, but allowed to change - ie for Amiga/etc.
const lines = iconv.decode(descData, 'cp437').split(/\r?\n/g);
const filesBbs = new FilesBBSFile();
//
// Contrary to popular belief, there is not a FILES.BBS standard. Instead,
// many formats have been used over the years. We'll try to support as much
// as we can within reason.
//
// Resources:
// - Great info from Mystic @ http://wiki.mysticbbs.com/doku.php?id=mutil_import_files.bbs
// - https://alt.bbs.synchronet.narkive.com/I6Vrxq6q/format-of-files-bbs
//
// Example files:
// - https://github.com/NuSkooler/ansi-bbs/tree/master/ancient_formats/files_bbs
//
const detectDecoder = () => {
//
// Try to figure out which decoder to use
//
const decoders = [
{
// I've been told this is what Syncrhonet uses
tester : /^([^ ]{1,12})\s{1,11}([0-3][0-9]\/[0-3][0-9]\/[1789][0-9]) ([^\r\n]+)$/,
extract : function() {
for(let i = 0; i < lines.length; ++i) {
let line = lines[i];
const hdr = line.match(this.tester);
if(!hdr) {
continue;
}
const long = [];
for(let j = i + 1; j < lines.length; ++j) {
line = lines[j];
if(!line.startsWith(' ')) {
break;
}
long.push(line.trim());
++i;
}
const desc = long.join('\r\n') || hdr[3] || '';
const fileName = hdr[1];
const timestamp = moment(hdr[2], 'MM/DD/YY');
filesBbs.entries.set(fileName, { timestamp, desc } );
}
}
},
{
//
// Aminet Amiga CDROM, March 1994. Walnut Creek CDROM.
// CP/M CDROM, Sep. 1994. Walnut Creek CDROM.
// ...and many others. Basically: <8.3 filename> <description>
//
// May contain headers, but we'll just skip 'em.
//
tester : /^([^ ]{1,12})\s{1,11}([^\r\n]+)$/,
extract : function() {
lines.forEach(line => {
const hdr = line.match(this.tester);
if(!hdr) {
return; // forEach
}
const fileName = hdr[1].trim();
const desc = hdr[2].trim();
if(desc) {
filesBbs.entries.set(fileName, { desc } );
}
});
}
},
{
// Found on AMINET CD's & similar
tester : /^(.{1,22}) ([0-9]+)K ([^\r\n]+)$/,
extract : function() {
lines.forEach(line => {
const hdr = line.match(this.tester);
if(!hdr) {
return; // forEach
}
const fileName = hdr[1].trim();
let size = parseInt(hdr[2]);
const desc = hdr[3].trim();
if(!isNaN(size)) {
size *= 1024; // K->bytes.
}
if(desc) { // omit empty entries
filesBbs.entries.set(fileName, { size, desc } );
}
});
}
},
];
const decoder = decoders.find(d => {
return lines
.slice(0, 10) // 10 lines in should be enough to detect - skipping headers/etc.
.some(l => d.tester.test(l));
});
return decoder;
};
const decoder = detectDecoder();
if(!decoder) {
return cb(Errors.Invalid('Invalid or unrecognized FILES.BBS format'));
}
decoder.extract(decoder);
return cb(
filesBbs.entries.size > 0 ? null : Errors.Invalid('Invalid or unrecognized FILES.BBS format'),
filesBbs
);
});
}
};

View File

@ -48,7 +48,7 @@ function finalizeEntryAndPersist(isUpdate, fileEntry, descHandler, cb) {
async.series( async.series(
[ [
function getDescFromHandlerIfNeeded(callback) { function getDescFromHandlerIfNeeded(callback) {
if((fileEntry.desc && fileEntry.desc.length > 0 ) && !argv['desc-file']) { if((fileEntry.desc && fileEntry.descSrc != 'fileName' && fileEntry.desc.length > 0 ) && !argv['desc-file']) {
return callback(null); // we have a desc already and are NOT overriding with desc file return callback(null); // we have a desc already and are NOT overriding with desc file
} }
@ -101,18 +101,47 @@ function finalizeEntryAndPersist(isUpdate, fileEntry, descHandler, cb) {
); );
} }
const SCAN_EXCLUDE_FILENAMES = [ 'DESCRIPT.ION', 'FILES.BBS' ]; const SCAN_EXCLUDE_FILENAMES = [
'DESCRIPT.ION',
'FILES.BBS',
'ALLFILES.TXT',
];
function loadDescHandler(path, cb) { function loadDescHandler(path, cb) {
const DescIon = require('../../core/descript_ion_file.js'); const handlerClassFromFileName = {
'descript.ion' : require('../../core/descript_ion_file.js'),
'files.bbs' : require('../../core/files_bbs_file.js'),
}[paths.basename(path).toLowerCase()];
// :TODO: support FILES.BBS also if(!handlerClassFromFileName) {
return cb(Errors.DoesNotExist(`No handlers registered for ${paths.basename(path)}`));
}
DescIon.createFromFile(path, (err, descHandler) => { handlerClassFromFileName.createFromFile(path, (err, descHandler) => {
return cb(err, descHandler); return cb(err, descHandler);
}); });
} }
//
// Try to find a suitable description handler by
// checking for common filenames.
//
function findSuitableDescHandler(basePath, cb) {
const commonFiles = [ 'FILES.BBS', 'DESCRIPT.ION' ];
async.eachSeries(commonFiles, (fileName, nextFileName) => {
loadDescHandler(paths.join(basePath, fileName), (err, handler) => {
if(!err && handler) {
return cb(null, handler);
}
return nextFileName(null);
});
},
() => {
return cb(Errors.DoesNotExist('No suitable description handler available'));
});
}
function scanFileAreaForChanges(areaInfo, options, cb) { function scanFileAreaForChanges(areaInfo, options, cb) {
const storageLocations = fileArea.getAreaStorageLocations(areaInfo).filter(sl => { const storageLocations = fileArea.getAreaStorageLocations(areaInfo).filter(sl => {
@ -145,7 +174,7 @@ function scanFileAreaForChanges(areaInfo, options, cb) {
return callback(null, options.descFileHandler); // we're going to use the global handler return callback(null, options.descFileHandler); // we're going to use the global handler
} }
loadDescHandler(paths.join(storageLoc.dir, 'DESCRIPT.ION'), (err, descHandler) => { findSuitableDescHandler(storageLoc.dir, (err, descHandler) => {
return callback(null, descHandler); return callback(null, descHandler);
}); });
}, },