+ Initial FILES.BBS support during file scan: A few formats supported so far, more to come...
* Detect DESCRIPT.ION, FILES.BBS, etc. during scans
This commit is contained in:
Bryan Ashby
parent
8356f00ba6
commit
832e04cdf0
|
|
@ -1,6 +1,8 @@
|
|||
/* jslint node: true */
|
||||
'use strict';
|
||||
|
||||
const { Errors } = require('./enig_error.js');
|
||||
|
||||
// deps
|
||||
const fs = require('graceful-fs');
|
||||
const iconv = require('iconv-lite');
|
||||
|
|
@ -64,7 +66,10 @@ module.exports = class DescriptIonFile {
|
|||
return nextLine(null);
|
||||
},
|
||||
() => {
|
||||
return cb(null, descIonFile);
|
||||
return cb(
|
||||
descIonFile.entries.size > 0 ? null : Errors.Invalid('Invalid or unrecognized DESCRIPT.ION format'),
|
||||
descIonFile
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,158 @@
|
|||
/* jslint node: true */
|
||||
'use strict';
|
||||
|
||||
const { Errors } = require('./enig_error.js');
|
||||
|
||||
// deps
|
||||
const fs = require('graceful-fs');
|
||||
const iconv = require('iconv-lite');
|
||||
const moment = require('moment');
|
||||
|
||||
module.exports = class FilesBBSFile {
|
||||
constructor() {
|
||||
this.entries = new Map();
|
||||
}
|
||||
|
||||
get(fileName) {
|
||||
return this.entries.get(fileName);
|
||||
}
|
||||
|
||||
getDescription(fileName) {
|
||||
const entry = this.get(fileName);
|
||||
if(entry) {
|
||||
return entry.desc;
|
||||
}
|
||||
}
|
||||
|
||||
static createFromFile(path, cb) {
|
||||
fs.readFile(path, (err, descData) => {
|
||||
if(err) {
|
||||
return cb(err);
|
||||
}
|
||||
|
||||
// :TODO: encoding should be default to CP437, but allowed to change - ie for Amiga/etc.
|
||||
const lines = iconv.decode(descData, 'cp437').split(/\r?\n/g);
|
||||
const filesBbs = new FilesBBSFile();
|
||||
|
||||
//
|
||||
// Contrary to popular belief, there is not a FILES.BBS standard. Instead,
|
||||
// many formats have been used over the years. We'll try to support as much
|
||||
// as we can within reason.
|
||||
//
|
||||
// Resources:
|
||||
// - Great info from Mystic @ http://wiki.mysticbbs.com/doku.php?id=mutil_import_files.bbs
|
||||
// - https://alt.bbs.synchronet.narkive.com/I6Vrxq6q/format-of-files-bbs
|
||||
//
|
||||
// Example files:
|
||||
// - https://github.com/NuSkooler/ansi-bbs/tree/master/ancient_formats/files_bbs
|
||||
//
|
||||
const detectDecoder = () => {
|
||||
//
|
||||
// Try to figure out which decoder to use
|
||||
//
|
||||
const decoders = [
|
||||
{
|
||||
// I've been told this is what Syncrhonet uses
|
||||
tester : /^([^ ]{1,12})\s{1,11}([0-3][0-9]\/[0-3][0-9]\/[1789][0-9]) ([^\r\n]+)$/,
|
||||
extract : function() {
|
||||
for(let i = 0; i < lines.length; ++i) {
|
||||
let line = lines[i];
|
||||
const hdr = line.match(this.tester);
|
||||
if(!hdr) {
|
||||
continue;
|
||||
}
|
||||
const long = [];
|
||||
for(let j = i + 1; j < lines.length; ++j) {
|
||||
line = lines[j];
|
||||
if(!line.startsWith(' ')) {
|
||||
break;
|
||||
}
|
||||
long.push(line.trim());
|
||||
++i;
|
||||
}
|
||||
const desc = long.join('\r\n') || hdr[3] || '';
|
||||
const fileName = hdr[1];
|
||||
const timestamp = moment(hdr[2], 'MM/DD/YY');
|
||||
|
||||
filesBbs.entries.set(fileName, { timestamp, desc } );
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
//
|
||||
// Aminet Amiga CDROM, March 1994. Walnut Creek CDROM.
|
||||
// CP/M CDROM, Sep. 1994. Walnut Creek CDROM.
|
||||
// ...and many others. Basically: <8.3 filename> <description>
|
||||
//
|
||||
// May contain headers, but we'll just skip 'em.
|
||||
//
|
||||
tester : /^([^ ]{1,12})\s{1,11}([^\r\n]+)$/,
|
||||
extract : function() {
|
||||
lines.forEach(line => {
|
||||
const hdr = line.match(this.tester);
|
||||
if(!hdr) {
|
||||
return; // forEach
|
||||
}
|
||||
|
||||
const fileName = hdr[1].trim();
|
||||
const desc = hdr[2].trim();
|
||||
|
||||
if(desc) {
|
||||
filesBbs.entries.set(fileName, { desc } );
|
||||
}
|
||||
});
|
||||
}
|
||||
},
|
||||
|
||||
{
|
||||
// Found on AMINET CD's & similar
|
||||
tester : /^(.{1,22}) ([0-9]+)K ([^\r\n]+)$/,
|
||||
extract : function() {
|
||||
lines.forEach(line => {
|
||||
const hdr = line.match(this.tester);
|
||||
if(!hdr) {
|
||||
return; // forEach
|
||||
}
|
||||
|
||||
const fileName = hdr[1].trim();
|
||||
let size = parseInt(hdr[2]);
|
||||
const desc = hdr[3].trim();
|
||||
|
||||
if(!isNaN(size)) {
|
||||
size *= 1024; // K->bytes.
|
||||
}
|
||||
|
||||
if(desc) { // omit empty entries
|
||||
filesBbs.entries.set(fileName, { size, desc } );
|
||||
}
|
||||
});
|
||||
}
|
||||
},
|
||||
];
|
||||
|
||||
const decoder = decoders.find(d => {
|
||||
return lines
|
||||
.slice(0, 10) // 10 lines in should be enough to detect - skipping headers/etc.
|
||||
.some(l => d.tester.test(l));
|
||||
});
|
||||
|
||||
return decoder;
|
||||
};
|
||||
|
||||
const decoder = detectDecoder();
|
||||
if(!decoder) {
|
||||
return cb(Errors.Invalid('Invalid or unrecognized FILES.BBS format'));
|
||||
}
|
||||
|
||||
decoder.extract(decoder);
|
||||
|
||||
return cb(
|
||||
filesBbs.entries.size > 0 ? null : Errors.Invalid('Invalid or unrecognized FILES.BBS format'),
|
||||
filesBbs
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
};
|
||||
|
|
@ -48,7 +48,7 @@ function finalizeEntryAndPersist(isUpdate, fileEntry, descHandler, cb) {
|
|||
async.series(
|
||||
[
|
||||
function getDescFromHandlerIfNeeded(callback) {
|
||||
if((fileEntry.desc && fileEntry.desc.length > 0 ) && !argv['desc-file']) {
|
||||
if((fileEntry.desc && fileEntry.descSrc != 'fileName' && fileEntry.desc.length > 0 ) && !argv['desc-file']) {
|
||||
return callback(null); // we have a desc already and are NOT overriding with desc file
|
||||
}
|
||||
|
||||
|
|
@ -101,18 +101,47 @@ function finalizeEntryAndPersist(isUpdate, fileEntry, descHandler, cb) {
|
|||
);
|
||||
}
|
||||
|
||||
const SCAN_EXCLUDE_FILENAMES = [ 'DESCRIPT.ION', 'FILES.BBS' ];
|
||||
const SCAN_EXCLUDE_FILENAMES = [
|
||||
'DESCRIPT.ION',
|
||||
'FILES.BBS',
|
||||
'ALLFILES.TXT',
|
||||
];
|
||||
|
||||
function loadDescHandler(path, cb) {
|
||||
const DescIon = require('../../core/descript_ion_file.js');
|
||||
const handlerClassFromFileName = {
|
||||
'descript.ion' : require('../../core/descript_ion_file.js'),
|
||||
'files.bbs' : require('../../core/files_bbs_file.js'),
|
||||
}[paths.basename(path).toLowerCase()];
|
||||
|
||||
// :TODO: support FILES.BBS also
|
||||
if(!handlerClassFromFileName) {
|
||||
return cb(Errors.DoesNotExist(`No handlers registered for ${paths.basename(path)}`));
|
||||
}
|
||||
|
||||
DescIon.createFromFile(path, (err, descHandler) => {
|
||||
handlerClassFromFileName.createFromFile(path, (err, descHandler) => {
|
||||
return cb(err, descHandler);
|
||||
});
|
||||
}
|
||||
|
||||
//
|
||||
// Try to find a suitable description handler by
|
||||
// checking for common filenames.
|
||||
//
|
||||
function findSuitableDescHandler(basePath, cb) {
|
||||
const commonFiles = [ 'FILES.BBS', 'DESCRIPT.ION' ];
|
||||
|
||||
async.eachSeries(commonFiles, (fileName, nextFileName) => {
|
||||
loadDescHandler(paths.join(basePath, fileName), (err, handler) => {
|
||||
if(!err && handler) {
|
||||
return cb(null, handler);
|
||||
}
|
||||
return nextFileName(null);
|
||||
});
|
||||
},
|
||||
() => {
|
||||
return cb(Errors.DoesNotExist('No suitable description handler available'));
|
||||
});
|
||||
}
|
||||
|
||||
function scanFileAreaForChanges(areaInfo, options, cb) {
|
||||
|
||||
const storageLocations = fileArea.getAreaStorageLocations(areaInfo).filter(sl => {
|
||||
|
|
@ -145,7 +174,7 @@ function scanFileAreaForChanges(areaInfo, options, cb) {
|
|||
return callback(null, options.descFileHandler); // we're going to use the global handler
|
||||
}
|
||||
|
||||
loadDescHandler(paths.join(storageLoc.dir, 'DESCRIPT.ION'), (err, descHandler) => {
|
||||
findSuitableDescHandler(storageLoc.dir, (err, descHandler) => {
|
||||
return callback(null, descHandler);
|
||||
});
|
||||
},
|
||||
|
|
|
|||
Loading…
Reference in New Issue