From 0e5d0c53d29d8e49424c38191b672869ff6a0ad3 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Wed, 10 May 2017 21:21:07 -0600 Subject: [PATCH] Use Config.fileTypes with mime-db backed MIME types; remove old archives.formats{} --- core/archive_util.js | 90 ++++++++++++---------------- core/config.js | 133 ++++++++++++++++++++++++----------------- core/mime_util.js | 10 ++++ mods/file_area_list.js | 6 +- 4 files changed, 128 insertions(+), 111 deletions(-) create mode 100644 core/mime_util.js diff --git a/core/archive_util.js b/core/archive_util.js index 02a2e283..62f81059 100644 --- a/core/archive_util.js +++ b/core/archive_util.js @@ -2,9 +2,10 @@ 'use strict'; // ENiGMA½ -const Config = require('./config.js').config; -const stringFormat = require('./string_format.js'); -const Errors = require('./enig_error.js').Errors; +const Config = require('./config.js').config; +const stringFormat = require('./string_format.js'); +const Errors = require('./enig_error.js').Errors; +const resolveMimeType = require('./mime_util.js').resolveMimeType; // base/modules const fs = require('fs'); @@ -19,9 +20,6 @@ class Archiver { this.decompress = config.decompress; this.list = config.list; this.extract = config.extract; - - /*this.sig = new Buffer(config.sig, 'hex'); - this.offset = config.offset || 0;*/ } ok() { @@ -76,39 +74,33 @@ module.exports = class ArchiveUtil { }); } - if(_.has(Config, 'archives.formats')) { - Object.keys(Config.archives.formats).forEach(fmtKey => { + if(_.isObject(Config.fileTypes)) { + Object.keys(Config.fileTypes).forEach(mimeType => { + const fileType = Config.fileTypes[mimeType]; + if(fileType.sig) { + fileType.sig = new Buffer(fileType.sig, 'hex'); + fileType.offset = fileType.offset || 0; - Config.archives.formats[fmtKey].sig = new Buffer(Config.archives.formats[fmtKey].sig, 'hex'); - Config.archives.formats[fmtKey].offset = Config.archives.formats[fmtKey].offset || 0; - - const sigLen = Config.archives.formats[fmtKey].offset + Config.archives.formats[fmtKey].sig.length; - if(sigLen > this.longestSignature) { - this.longestSignature = sigLen; - } + // :TODO: this is broken: sig is NOT this long, it's sig.length long; offset needs to allow for -negative values as well + const sigLen =fileType.offset + fileType.sig.length; + if(sigLen > this.longestSignature) { + this.longestSignature = sigLen; + } + } }); } } - - /* - getArchiver(archType) { - if(!archType || 0 === archType.length) { - return; - } + + getArchiver(mimeTypeOrExtension) { + mimeTypeOrExtension = resolveMimeType(mimeTypeOrExtension); - archType = archType.toLowerCase(); - return this.archivers[archType]; - }*/ - - getArchiver(archType) { - if(!archType || 0 === archType.length) { + if(!mimeTypeOrExtension) { // lookup returns false on failure return; } - if(_.has(Config, [ 'archives', 'formats', archType, 'handler' ] ) && - _.has(Config, [ 'archives', 'archivers', Config.archives.formats[archType].handler ] )) - { - return Config.archives.archivers[ Config.archives.formats[archType].handler ]; + const archiveHandler = _.get( Config, [ 'fileTypes', mimeTypeOrExtension, 'archiveHandler'] ); + if(archiveHandler) { + return _.get( Config, [ 'archives', 'archivers', archiveHandler ] ); } } @@ -121,10 +113,6 @@ module.exports = class ArchiveUtil { } detectType(path, cb) { - if(!_.has(Config, 'archives.formats')) { - return cb(Errors.DoesNotExist('No formats configured')); - } - fs.open(path, 'r', (err, fd) => { if(err) { return cb(err); @@ -136,15 +124,19 @@ module.exports = class ArchiveUtil { return cb(err); } - const archFormat = _.findKey(Config.archives.formats, archFormat => { - const lenNeeded = archFormat.offset + archFormat.sig.length; + const archFormat = _.findKey(Config.fileTypes, fileTypeInfo => { + if(!fileTypeInfo.sig) { + return false; + } + + const lenNeeded = fileTypeInfo.offset + fileTypeInfo.sig.length; if(bytesRead < lenNeeded) { return false; } - const comp = buf.slice(archFormat.offset, archFormat.offset + archFormat.sig.length); - return (archFormat.sig.equals(comp)); + const comp = buf.slice(fileTypeInfo.offset, fileTypeInfo.offset + fileTypeInfo.sig.length); + return (fileTypeInfo.sig.equals(comp)); }); return cb(archFormat ? null : Errors.General('Unknown type'), archFormat); @@ -157,19 +149,13 @@ module.exports = class ArchiveUtil { // so we have this horrible, horrible hack: let err; proc.once('data', d => { - if(_.isString(d) && d.startsWith('execvp(3) failed.: No such file or directory')) { - err = new Error(`${action} failed: ${d.trim()}`); + if(_.isString(d) && d.startsWith('execvp(3) failed.')) { + err = Errors.ExternalProcess(`${action} failed: ${d.trim()}`); } }); proc.once('exit', exitCode => { - if(exitCode) { - return cb(new Error(`${action} failed with exit code: ${exitCode}`)); - } - if(err) { - return cb(err); - } - return cb(null); + return cb(exitCode ? Errors.ExternalProcess(`${action} failed with exit code: ${exitCode}`) : err); }); } @@ -177,7 +163,7 @@ module.exports = class ArchiveUtil { const archiver = this.getArchiver(archType); if(!archiver) { - return cb(new Error(`Unknown archive type: ${archType}`)); + return cb(Errors.Invalid(`Unknown archive type: ${archType}`)); } const fmtObj = { @@ -205,7 +191,7 @@ module.exports = class ArchiveUtil { const archiver = this.getArchiver(archType); if(!archiver) { - return cb(new Error(`Unknown archive type: ${archType}`)); + return cb(Errors.Invalid(`Unknown archive type: ${archType}`)); } const fmtObj = { @@ -235,7 +221,7 @@ module.exports = class ArchiveUtil { const archiver = this.getArchiver(archType); if(!archiver) { - return cb(new Error(`Unknown archive type: ${archType}`)); + return cb(Errors.Invalid(`Unknown archive type: ${archType}`)); } const fmtObj = { @@ -254,7 +240,7 @@ module.exports = class ArchiveUtil { proc.once('exit', exitCode => { if(exitCode) { - return cb(new Error(`List failed with exit code: ${exitCode}`)); + return cb(Errors.ExternalProcess(`List failed with exit code: ${exitCode}`)); } const entryGroupOrder = archiver.list.entryGroupOrder || { byteSize : 1, fileName : 2 }; diff --git a/core/config.js b/core/config.js index 854d842f..2993ec48 100644 --- a/core/config.js +++ b/core/config.js @@ -263,6 +263,83 @@ function getDefaultConfig() { } } }, + + fileTypes : { + // + // File types explicitly known to the system. Here we can configure + // information extraction, archive treatment, etc. + // + // MIME types can be found in mime-db: https://github.com/jshttp/mime-db + // + // Resources for signature/magic bytes: + // * http://www.garykessler.net/library/file_sigs.html + // + // + // :TODO: text/x-ansi -> SAUCE extraction for .ans uploads + + // + // Audio + // + 'audio/mpeg' : { + desc : 'MP3 Audio', + infoExtract : { + cmd : `${__dirname}./../util/exif2desc.js`, // ensure chmod +x + args : [ '{filePath}' ], + }, + }, + // + // Archives + // + 'application/zip' : { + desc : 'ZIP Archive', + sig : '504b0304', + offset : 0, + archiveHandler : '7Zip', + }, + 'application/x-arj' : { + desc : 'ARJ Archive', + sig : '60ea', + offset : 0, + archiveHandler : 'Arj', + }, + 'application/x-rar-compressed' : { + desc : 'RAR Archive', + sig : '526172211a0700', + offset : 0, + archiveHandler : 'Rar', + }, + 'application/gzip' : { + desc : 'Gzip Archive', + sig : '1f8b', + offset : 0, + archiveHandler : '7Zip', + }, + // :TODO: application/x-bzip + 'application/x-bzip2' : { + desc : 'BZip2 Archive', + sig : '425a68', + offset : 0, + archiveHandler : '7Zip', + }, + 'application/x-lzh-compressed' : { + desc : 'LHArc Archive', + sig : '2d6c68', + offset : 2, + archiveHandler : 'Lha', + }, + 'application/x-7z-compressed' : { + desc : '7-Zip Archive', + sig : '377abcaf271c', + offset : 0, + archiveHandler : '7Zip', + } + + // :TODO: update archives::formats to fall here + // * archive handler -> archiveHandler (consider archive if archiveHandler present) + // * sig, offset, ... + // * mime-db -> exts lookup + // * + }, archives : { archivers : { @@ -348,62 +425,6 @@ function getDefaultConfig() { } } }, - - formats : { - // - // Resources - // * http://www.garykessler.net/library/file_sigs.html - // - zip : { - sig : '504b0304', - offset : 0, - exts : [ 'zip' ], - handler : '7Zip', - desc : 'ZIP Archive', - }, - '7z' : { - sig : '377abcaf271c', - offset : 0, - exts : [ '7z' ], - handler : '7Zip', - desc : '7-Zip Archive', - }, - arj : { - sig : '60ea', - offset : 0, - exts : [ 'arj' ], - handler : 'Arj', - desc : 'ARJ Archive', - }, - rar : { - sig : '526172211a0700', - offset : 0, - exts : [ 'rar' ], - handler : 'Rar', - desc : 'RAR Archive', - }, - gzip : { - sig : '1f8b', - offset : 0, - exts : [ 'gz' ], - handler : '7Zip', - desc : 'Gzip Archive', - }, - bzip : { - sig : '425a68', - offset : 0, - exts : [ 'bz2' ], - handler : '7Zip', - desc : 'BZip2 Archive', - }, - lzh : { - sig : '2d6c68', - offset : 2, - exts : [ 'lzh', 'ice' ], - handler : 'Lha', - desc : 'LHArc Archive', - } - } }, fileTransferProtocols : { diff --git a/core/mime_util.js b/core/mime_util.js new file mode 100644 index 00000000..93e0b40e --- /dev/null +++ b/core/mime_util.js @@ -0,0 +1,10 @@ +/* jslint node: true */ +'use strict'; + +const mimeTypes = require('mime-types'); + +exports.resolveMimeType = resolveMimeType; + +function resolveMimeType(query) { + return mimeTypes.extension(query) || mimeTypes.lookup(query) || undefined; // lookup() returns false; we want undefined +} \ No newline at end of file diff --git a/mods/file_area_list.js b/mods/file_area_list.js index 0a0e2134..9dacc23d 100644 --- a/mods/file_area_list.js +++ b/mods/file_area_list.js @@ -17,6 +17,7 @@ const Config = require('../core/config.js').config; const DownloadQueue = require('../core/download_queue.js'); const FileAreaWeb = require('../core/file_area_web.js'); const FileBaseFilters = require('../core/file_base_filter.js'); +const resolveMimeType = require('../core/mime_util.js').resolveMimeType; const cleanControlCodes = require('../core/string_util.js').cleanControlCodes; @@ -236,9 +237,8 @@ exports.getModule = class FileAreaList extends MenuModule { }); if(entryInfo.archiveType) { - entryInfo.archiveTypeDesc = _.has(Config, [ 'archives', 'formats', entryInfo.archiveType, 'desc' ]) ? - Config.archives.formats[entryInfo.archiveType].desc : - entryInfo.archiveType; + const mimeType = resolveMimeType(entryInfo.archiveType); + entryInfo.archiveTypeDesc = mimeType ? _.get(Config, [ 'fileTypes', mimeType, 'desc' ] ) || mimeType : entryInfo.archiveType; } else { entryInfo.archiveTypeDesc = 'N/A'; }