moon
/
enigma-bbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Improvements to ANSI parser 

* Introduction of storage tags for file bases / areas
* Expiration for file web server items
* WIP work on clean ANSI (on hold for a bit while other file base stuff is worked on)
This commit is contained in:
Bryan Ashby 2016-12-06 18:58:56 -07:00
parent a7c0f2b7b0
commit 6da7d557f9
16 changed files with 557 additions and 180 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
core/acs.js
View File

@ -25,6 +25,9 @@ class ACS {
}
}
//
// Message Conferences & Areas
//
hasMessageConfRead(conf) {
return this.check(conf.acs, 'read', ACS.Defaults.MessageConfRead);
}
@ -33,10 +36,17 @@ class ACS {
return this.check(area.acs, 'read', ACS.Defaults.MessageAreaRead);
}
//
// File Base / Areas
//
hasFileAreaRead(area) {
return this.check(area.acs, 'read', ACS.Defaults.FileAreaRead);
}
hasFileAreaDownload(area) {
return this.check(area.acs, 'download', ACS.Defaults.FileAreaDownload);
}
getConditionalValue(condArray, memberName) {
assert(_.isArray(condArray));
assert(_.isString(memberName));
@ -65,6 +75,7 @@ ACS.Defaults = {
MessageConfRead : 'GM[users]',
FileAreaRead : 'GM[users]',
FileAreaDownload : 'GM[users]',
};
module.exports = ACS;

137
core/ansi_escape_parser.js
View File

@ -48,8 +48,10 @@ function ANSIEscapeParser(options) {
self.row = Math.max(self.row, 1);
self.row = Math.min(self.row, self.termHeight);
self.emit('move cursor', self.column, self.row);
self.rowUpdated();
// self.emit('move cursor', self.column, self.row);
self.positionUpdated();
//self.rowUpdated();
};
self.saveCursorPosition = function() {
@ -63,7 +65,9 @@ function ANSIEscapeParser(options) {
self.row = self.savedPosition.row;
self.column = self.savedPosition.column;
delete self.savedPosition;
self.rowUpdated();
self.positionUpdated();
// self.rowUpdated();
};
self.clearScreen = function() {
@ -71,11 +75,76 @@ function ANSIEscapeParser(options) {
self.emit('clear screen');
};
/*
self.rowUpdated = function() {
self.emit('row update', self.row + self.scrollBack);
};*/
self.positionUpdated = function() {
self.emit('position update', self.row, self.column);
};
function literal(text) {
let charCode;
let pos;
let start = 0;
const len = text.length;
function emitLiteral() {
self.emit('literal', text.slice(start, pos));
start = pos;
}
for(pos = 0; pos < len; ++pos) {
charCode = text.charCodeAt(pos) & 0xff; // ensure 8bit clean
switch(charCode) {
case CR :
emitLiteral();
self.column = 1;
self.positionUpdated();
break;
case LF :
emitLiteral();
self.row += 1;
self.positionUpdated();
break;
default :
if(self.column > self.termWidth) {
//
// Emit data up to this point so it can be drawn before the postion update
//
emitLiteral();
self.column = 1;
self.row += 1;
self.positionUpdated();
} else {
self.column += 1;
}
break;
}
}
self.emit('literal', text.slice(start));
if(self.column > self.termWidth) {
self.column = 1;
self.row += 1;
self.positionUpdated();
}
}
function literal2(text) {
var charCode;
var len = text.length;
@ -88,29 +157,31 @@ function ANSIEscapeParser(options) {
case LF :
self.row++;
self.rowUpdated();
self.positionUpdated();
//self.rowUpdated();
break;
default :
// wrap
if(self.column === self.termWidth) {
if(self.column > self.termWidth) {
self.column = 1;
self.row++;
self.rowUpdated();
//self.rowUpdated();
self.positionUpdated();
} else {
self.column++;
self.column += 1;
}
break;
}
if(self.row === 26) { // :TODO: should be termHeight + 1 ?
self.scrollBack++;
self.row--;
self.rowUpdated();
if(self.row === self.termHeight) {
self.scrollBack += 1;
self.row -= 1;
self.positionUpdated();
}
}
//self.emit('chunk', text);
self.emit('literal', text);
}
@ -188,10 +259,10 @@ function ANSIEscapeParser(options) {
}
}
self.reset = function(buffer) {
self.reset = function(input) {
self.parseState = {
// ignore anything past EOF marker, if any
buffer : buffer.split(String.fromCharCode(0x1a), 1)[0],
buffer : input.split(String.fromCharCode(0x1a), 1)[0],
re : /(?:\x1b\x5b)([\?=;0-9]*?)([ABCDHJKfhlmnpsu])/g,
stop : false,
};
@ -201,7 +272,11 @@ function ANSIEscapeParser(options) {
self.parseState.stop = true;
};
self.parse = function() {
self.parse = function(input) {
if(input) {
self.reset(input);
}
// :TODO: ensure this conforms to ANSI-BBS / CTerm / bansi.txt for movement/etc.
var pos;
var match;
@ -308,40 +383,45 @@ function ANSIEscapeParser(options) {
*/
function escape(opCode, args) {
var arg;
var i;
var len;
let arg;
switch(opCode) {
// cursor up
case 'A' :
arg = args[0] || 1;
//arg = args[0] || 1;
arg = isNaN(args[0]) ? 1 : args[0];
self.moveCursor(0, -arg);
break;
// cursor down
case 'B' :
arg = args[0] || 1;
//arg = args[0] || 1;
arg = isNaN(args[0]) ? 1 : args[0];
self.moveCursor(0, arg);
break;
// cursor forward/right
case 'C' :
arg = args[0] || 1;
//arg = args[0] || 1;
arg = isNaN(args[0]) ? 1 : args[0];
self.moveCursor(arg, 0);
break;
// cursor back/left
case 'D' :
arg = args[0] || 1;
//arg = args[0] || 1;
arg = isNaN(args[0]) ? 1 : args[0];
self.moveCursor(-arg, 0);
break;
case 'f' : // horiz & vertical
case 'H' : // cursor position
self.row = args[0] || 1;
self.column = args[1] || 1;
self.rowUpdated();
//self.row = args[0] || 1;
//self.column = args[1] || 1;
self.row = isNaN(args[0]) ? 1 : args[0];
self.column = isNaN(args[1]) ? 1 : args[1];
//self.rowUpdated();
self.positionUpdated();
break;
// save position
@ -356,7 +436,7 @@ function ANSIEscapeParser(options) {
// set graphic rendition
case 'm' :
for(i = 0, len = args.length; i < len; ++i) {
for(let i = 0, len = args.length; i < len; ++i) {
arg = args[i];
if(ANSIEscapeParser.foregroundColors[arg]) {
@ -410,12 +490,13 @@ function ANSIEscapeParser(options) {
}
}
}
break; // m
break;
// :TODO: s, u, K
// erase display/screen
case 'J' :
// :TODO: Handle others
// :TODO: Handle other 'J' types!
if(2 === args[0]) {
self.clearScreen();
}

4
core/conf_area_util.js
View File

@ -16,8 +16,8 @@ function sortAreasOrConfs(areasOrConfs, type) {
let entryB;
areasOrConfs.sort((a, b) => {
entryA = a[type];
entryB = b[type];
entryA = type ? a[type] : a;
entryB = type ? b[type] : b;
if(_.isNumber(entryA.sort) && _.isNumber(entryB.sort)) {
return entryA.sort - entryB.sort;

22
core/config.js
View File

@ -361,17 +361,21 @@ function getDefaultConfig() {
fileNamePatterns: {
// These are NOT case sensitive
// FILE_ID.DIZ - https://en.wikipedia.org/wiki/FILE_ID.DIZ
shortDesc : [
'^FILE_ID\.DIZ$', '^DESC\.SDI$', '^DESCRIPT\.ION$', '^FILE\.DES$', '$FILE\.SDI$', '^DISK\.ID$'
],
longDesc : [ '^.*\.NFO$', '^README\.1ST$', '^README\.TXT$' ],
// common README filename - https://en.wikipedia.org/wiki/README
longDesc : [
'^.*\.NFO$', '^README\.1ST$', '^README\.TXT$', '^READ\.ME$', '^README$', '^README\.md$'
],
},
yearEstPatterns: [
//
// Patterns should produce the year in the first submatch
// The year may be YY or YYYY
// Patterns should produce the year in the first submatch.
// The extracted year may be YY or YYYY
//
'[0-3]?[0-9][\\-\\/\\.][0-3]?[0-9][\\-\\/\\.]((?:[0-9]{2})?[0-9]{2})', // m/d/yyyy, mm-dd-yyyy, etc.
"\\B('[1789][0-9])\\b", // eslint-disable-line quotes
@ -385,17 +389,25 @@ function getDefaultConfig() {
expireMinutes : 1440, // 1 day
},
//
// File area storage location tag/value pairs.
// Non-absolute paths are relative to |areaStoragePrefix|.
//
storageTags : {
sys_msg_attach : 'msg_attach',
},
areas: {
message_attachment : {
systemm_message_attachment : {
name : 'Message attachments',
desc : 'File attachments to messages',
storageTags : 'sys_msg_attach', // may be string or array of strings
}
}
},
eventScheduler : {
events : {
trimMessageAreas : {
// may optionally use [or ]@watch:/path/to/file

1
core/database.js
View File

@ -262,6 +262,7 @@ const DB_INIT_TABLE = {
area_tag VARCHAR NOT NULL,
file_sha1 VARCHAR NOT NULL,
file_name, /* FTS @ file_fts */
storage_tag VARCHAR NOT NULL,
desc, /* FTS @ file_fts */
desc_long, /* FTS @ file_fts */
upload_timestamp DATETIME NOT NULL

2
core/download_queue.js
View File

@ -7,8 +7,10 @@ module.exports = class DownloadQueue {
constructor(client) {
this.client = client;
if(!Array.isArray(this.client.user.downloadQueue)) {
this.loadFromProperty(client);
}
}
toggle(fileEntry) {
if(this.isQueued(fileEntry)) {

111
core/file_area.js
View File

@ -37,7 +37,8 @@ function getAvailableFileAreas(client, options) {
options = options || { includeSystemInternal : false };
// perform ACS check per conf & omit system_internal if desired
return _.omit(Config.fileAreas.areas, (area, areaTag) => {
const areasWithTags = _.map(Config.fileBase.areas, (area, areaTag) => Object.assign(area, { areaTag : areaTag } ) );
return _.omit(Config.fileBase.areas, (area, areaTag) => {
if(!options.includeSystemInternal && WellKnownAreaTags.MessageAreaAttach === areaTag) {
return true;
}
@ -48,21 +49,21 @@ function getAvailableFileAreas(client, options) {
function getSortedAvailableFileAreas(client, options) {
const areas = _.map(getAvailableFileAreas(client, options), v => v);
sortAreasOrConfs(areas, 'area');
sortAreasOrConfs(areas);
return areas;
}
function getDefaultFileAreaTag(client, disableAcsCheck) {
let defaultArea = _.findKey(Config.fileAreas, o => o.default);
let defaultArea = _.findKey(Config.fileBase, o => o.default);
if(defaultArea) {
const area = Config.fileAreas.areas[defaultArea];
const area = Config.fileBase.areas[defaultArea];
if(true === disableAcsCheck || client.acs.hasFileAreaRead(area)) {
return defaultArea;
}
}
// just use anything we can
defaultArea = _.findKey(Config.fileAreas.areas, (area, areaTag) => {
defaultArea = _.findKey(Config.fileBase.areas, (area, areaTag) => {
return WellKnownAreaTags.MessageAreaAttach !== areaTag && (true === disableAcsCheck || client.acs.hasFileAreaRead(area));
});
@ -70,10 +71,10 @@ function getDefaultFileAreaTag(client, disableAcsCheck) {
}
function getFileAreaByTag(areaTag) {
const areaInfo = Config.fileAreas.areas[areaTag];
const areaInfo = Config.fileBase.areas[areaTag];
if(areaInfo) {
areaInfo.areaTag = areaTag; // convienence!
areaInfo.storageDirectory = getAreaStorageDirectory(areaInfo);
areaInfo.storage = getAreaStorageLocations(areaInfo);
return areaInfo;
}
}
@ -113,8 +114,38 @@ function changeFileAreaWithOptions(client, areaTag, options, cb) {
);
}
function getAreaStorageDirectory(areaInfo) {
return paths.join(Config.fileBase.areaStoragePrefix, areaInfo.storageDir || '');
function getAreaStorageDirectoryByTag(storageTag) {
const storageLocation = (storageTag && Config.fileBase.storageTags[storageTag]);
return paths.resolve(Config.fileBase.areaStoragePrefix, storageLocation || '');
/*
// absolute paths as-is
if(storageLocation && '/' === storageLocation.charAt(0)) {
return storageLocation;
}
// relative to |areaStoragePrefix|
return paths.join(Config.fileBase.areaStoragePrefix, storageLocation || '');
*/
}
function getAreaStorageLocations(areaInfo) {
const storageTags = Array.isArray(areaInfo.storageTags) ?
areaInfo.storageTags :
[ areaInfo.storageTags || '' ];
const avail = Config.fileBase.storageTags;
return _.compact(storageTags.map(storageTag => {
if(avail[storageTag]) {
return {
storageTag : storageTag,
dir : getAreaStorageDirectoryByTag(storageTag),
};
}
}));
}
function getFileEntryPath(fileEntry) {
@ -342,29 +373,28 @@ function updateFileEntry(fileEntry, filePath, cb) {
}
function addOrUpdateFileEntry(areaInfo, fileName, options, cb) {
function addOrUpdateFileEntry(areaInfo, storageLocation, fileName, options, cb) {
const fileEntry = new FileEntry({
areaTag : areaInfo.areaTag,
meta : options.meta,
hashTags : options.hashTags, // Set() or Array
fileName : fileName,
storageTag : storageLocation.storageTag,
});
const filePath = paths.join(getAreaStorageDirectory(areaInfo), fileName);
const filePath = paths.join(storageLocation.dir, fileName);
async.waterfall(
[
function processPhysicalFile(callback) {
const stream = fs.createReadStream(filePath);
let byteSize = 0;
const sha1 = crypto.createHash('sha1');
const sha256 = crypto.createHash('sha256');
const md5 = crypto.createHash('md5');
const crc32 = new CRC32();
// :TODO: crc32
const stream = fs.createReadStream(filePath);
stream.on('data', data => {
byteSize += data.length;
@ -413,6 +443,58 @@ function addOrUpdateFileEntry(areaInfo, fileName, options, cb) {
}
function scanFileAreaForChanges(areaInfo, cb) {
const storageLocations = getAreaStorageLocations(areaInfo);
async.eachSeries(storageLocations, (storageLoc, nextLocation) => {
async.series(
[
function scanPhysFiles(callback) {
const physDir = storageLoc.dir;
fs.readdir(physDir, (err, files) => {
if(err) {
return callback(err);
}
async.eachSeries(files, (fileName, nextFile) => {
const fullPath = paths.join(physDir, fileName);
fs.stat(fullPath, (err, stats) => {
if(err) {
// :TODO: Log me!
return nextFile(null); // always try next file
}
if(!stats.isFile()) {
return nextFile(null);
}
addOrUpdateFileEntry(areaInfo, storageLoc, fileName, { }, err => {
return nextFile(err);
});
});
}, err => {
return callback(err);
});
});
},
function scanDbEntries(callback) {
// :TODO: Look @ db entries for area that were *not* processed above
return callback(null);
}
],
err => {
return nextLocation(err);
}
);
},
err => {
return cb(err);
});
}
/*
function scanFileAreaForChanges2(areaInfo, cb) {
const areaPhysDir = getAreaStorageDirectory(areaInfo);
async.series(
@ -455,3 +537,4 @@ function scanFileAreaForChanges(areaInfo, cb) {
}
);
}
*/

64
core/file_area_web.js
View File

@ -29,6 +29,7 @@ const WEB_SERVER_PACKAGE_NAME = 'codes.l33t.enigma.web.server';
class FileAreaWebAccess {
constructor() {
this.hashids = new hashids(Config.general.boardName);
this.expireTimers = {}; // hashId->timer
}
startup(cb) {
@ -37,8 +38,7 @@ class FileAreaWebAccess {
async.series(
[
function initFromDb(callback) {
// :TODO: Init from DB & register expiration timers
return callback(null);
return self.load(callback);
},
function addWebRoute(callback) {
const webServer = getServer(WEB_SERVER_PACKAGE_NAME);
@ -66,7 +66,56 @@ class FileAreaWebAccess {
}
load(cb) {
return cb(null); // :TODO: Load from db
//
// Load entries, register expiration timers
//
FileDb.each(
`SELECT hash_id, expire_timestamp
FROM file_web_serve;`,
(err, row) => {
if(row) {
this.scheduleExpire(row.hash_id, moment(row.expire_timestamp));
}
},
err => {
return cb(err);
}
);
}
removeEntry(hashId) {
//
// Delete record from DB, and our timer
//
FileDb.run(
`DELETE FROM file_web_serve
WHERE hash_id = ?;`,
[ hashId ]
);
delete this.expireTime[hashId];
}
scheduleExpire(hashId, expireTime) {
// remove any previous entry for this hashId
const previous = this.expireTimers[hashId];
if(previous) {
clearTimeout(previous);
delete this.expireTimers[hashId];
}
const timeoutMs = expireTime.diff(moment());
if(timeoutMs <= 0) {
setImmediate( () => {
this.removeEntry(hashId);
});
} else {
this.expireTimers[hashId] = setTimeout( () => {
this.removeEntry(hashId);
}, timeoutMs);
}
}
loadServedHashId(hashId, cb) {
@ -111,12 +160,9 @@ class FileAreaWebAccess {
//
// Create a URL such as
// https://l33t.codes:44512/f/qFdxyZr
//
// :TODO: build from config
//
// Prefer HTTPS over HTTP. Be explicit about the port
// only if required.
// only if non-standard.
//
let schema;
let port;
@ -163,7 +209,7 @@ class FileAreaWebAccess {
return cb(err);
}
// :TODO: setup tracking of expiration time so we can clean up the entry
this.scheduleExpire(hashId, options.expireTime);
return cb(null, url);
}
@ -173,7 +219,7 @@ class FileAreaWebAccess {
fileNotFound(resp) {
resp.writeHead(404, { 'Content-Type' : 'text/html' } );
// :TODO: allow custom 404
// :TODO: allow custom 404 - mods/<theme>/file_area_web-404.html
return resp.end('<html><body>Not found</html>');
}

53
core/file_entry.js
View File

@ -12,7 +12,7 @@ const _ = require('lodash');
const paths = require('path');
const FILE_TABLE_MEMBERS = [
'file_id', 'area_tag', 'file_sha1', 'file_name',
'file_id', 'area_tag', 'file_sha1', 'file_name', 'storage_tag',
'desc', 'desc_long', 'upload_timestamp'
];
@ -44,6 +44,7 @@ module.exports = class FileEntry {
this.hashTags = options.hashTags || new Set();
this.fileName = options.fileName;
this.storageTag = options.storageTag;
}
load(fileId, cb) {
@ -99,9 +100,9 @@ module.exports = class FileEntry {
},
function storeEntry(callback) {
fileDb.run(
`REPLACE INTO file (area_tag, file_sha1, file_name, desc, desc_long, upload_timestamp)
VALUES(?, ?, ?, ?, ?, ?);`,
[ self.areaTag, self.fileSha1, self.fileName, self.desc, self.descLong, getISOTimestampString() ],
`REPLACE INTO file (area_tag, file_sha1, file_name, storage_tag, desc, desc_long, upload_timestamp)
VALUES(?, ?, ?, ?, ?, ?, ?);`,
[ self.areaTag, self.fileSha1, self.fileName, self.storageTag, self.desc, self.descLong, getISOTimestampString() ],
function inserted(err) { // use non-arrow func for 'this' scope / lastID
if(!err) {
self.fileId = this.lastID;
@ -132,15 +133,21 @@ module.exports = class FileEntry {
);
}
get filePath() {
const areaInfo = Config.fileAreas.areas[this.areaTag];
if(areaInfo) {
return paths.join(
Config.fileBase.areaStoragePrefix,
areaInfo.storageDir || '',
this.fileName
);
static getAreaStorageDirectoryByTag(storageTag) {
const storageLocation = (storageTag && Config.fileBase.storageTags[storageTag]);
// absolute paths as-is
if(storageLocation && '/' === storageLocation.charAt(0)) {
return storageLocation;
}
// relative to |areaStoragePrefix|
return paths.join(Config.fileBase.areaStoragePrefix, storageLocation || '');
}
get filePath() {
const storageDir = FileEntry.getAreaStorageDirectoryByTag(this.storageTag);
return paths.join(storageDir, this.fileName);
}
static persistMetaValue(fileId, name, value, cb) {
@ -193,7 +200,7 @@ module.exports = class FileEntry {
static getWellKnownMetaValues() { return Object.keys(FILE_WELL_KNOWN_META); }
static findFiles(criteria, cb) {
static findFiles(filter, cb) {
// :TODO: build search here - return [ fileid1, fileid2, ... ]
// free form
// areaTag
@ -201,6 +208,8 @@ module.exports = class FileEntry {
// order by
// sort
filter = filter || {};
let sql =
`SELECT file_id
FROM file`;
@ -216,21 +225,23 @@ module.exports = class FileEntry {
sqlWhere += clause;
}
if(criteria.areaTag) {
appendWhereClause(`area_tag="${criteria.areaTag}"`);
if(filter.areaTag) {
appendWhereClause(`area_tag="${filter.areaTag}"`);
}
if(criteria.search) {
if(filter.terms) {
appendWhereClause(
`file_id IN (
SELECT rowid
FROM file_fts
WHERE file_fts MATCH "${criteria.search.replace(/"/g,'""')}"
WHERE file_fts MATCH "${filter.terms.replace(/"/g,'""')}"
)`
);
}
if(Array.isArray(criteria.hashTags)) {
if(filter.tags) {
const tags = filter.tags.split(' '); // filter stores as sep separated values
appendWhereClause(
`file_id IN (
SELECT file_id
@ -238,14 +249,14 @@ module.exports = class FileEntry {
WHERE hash_tag_id IN (
SELECT hash_tag_id
FROM hash_tag
WHERE hash_tag IN (${criteria.hashTags.join(',')})
WHERE hash_tag IN (${tags.join(',')})
)
)`
);
}
// :TODO: criteria.orderBy
// :TODO: criteria.sort
// :TODO: filter.orderBy
// :TODO: filter.sort
sql += sqlWhere + ';';
const matchingFileIds = [];

9
core/sauce.js
View File

@ -50,20 +50,17 @@ function readSAUCE(data, cb) {
.tap(function onVars(vars) {
if(!SAUCE_ID.equals(vars.id)) {
cb(new Error('No SAUCE record present'));
return;
return cb(new Error('No SAUCE record present'));
}
var ver = iconv.decode(vars.version, 'cp437');
if('00' !== ver) {
cb(new Error('Unsupported SAUCE version: ' + ver));
return;
return cb(new Error('Unsupported SAUCE version: ' + ver));
}
if(-1 === SAUCE_VALID_DATA_TYPES.indexOf(vars.dataType)) {
cb(new Error('Unsupported SAUCE DataType: ' + vars.dataType));
return;
return cb(new Error('Unsupported SAUCE DataType: ' + vars.dataType));
}
var sauce = {

2
core/scanner_tossers/ftn_bso.js
View File

@ -1099,6 +1099,8 @@ function FTNMessageScanTossModule() {
return nextFile(); // unknown archive type
}
Log.debug( { bundleFile : bundleFile }, 'Processing bundle' );
self.archUtil.extractTo(
bundleFile.path,
self.importTempDir,

191
core/string_util.js
View File

@ -3,6 +3,10 @@
// ENiGMA½
const miscUtil = require('./misc_util.js');
const ANSIEscapeParser = require('./ansi_escape_parser.js').ANSIEscapeParser;
const ANSI = require('./ansi_term.js');
// deps
const iconv = require('iconv-lite');
exports.stylizeString = stylizeString;
@ -16,6 +20,7 @@ exports.renderStringLength = renderStringLength;
exports.formatByteSizeAbbr = formatByteSizeAbbr;
exports.formatByteSize = formatByteSize;
exports.cleanControlCodes = cleanControlCodes;
exports.createCleanAnsi = createCleanAnsi;
// :TODO: create Unicode verison of this
const VOWELS = [ 'a', 'e', 'i', 'o', 'u' ];
@ -310,15 +315,23 @@ function formatByteSize(byteSize, withAbbr, decimals) {
//const REGEXP_ANSI_CONTROL_CODES = /(\x1b\x5b)([\?=;0-9]*?)([0-9A-ORZcf-npsu=><])/g;
const REGEXP_ANSI_CONTROL_CODES = /(?:\x1b\x5b)([\?=;0-9]*?)([A-ORZcf-npsu=><])/g;
const ANSI_OPCODES_ALLOWED_CLEAN = [
'C', 'm' ,
'A', 'B', 'D'
'A', 'B', // up, down
'C', 'D', // right, left
'm', // color
];
function cleanControlCodes(input) {
const AnsiSpecialOpCodes = {
positioning : [ 'A', 'B', 'C', 'D' ], // up, down, right, left
style : [ 'm' ] // color
};
function cleanControlCodes(input, options) {
let m;
let pos;
let cleaned = '';
options = options || {};
//
// Loop through |input| adding only allowed ESC
// sequences and literals to |cleaned|
@ -332,6 +345,10 @@ function cleanControlCodes(input) {
cleaned += input.slice(pos, m.index);
}
if(options.all) {
continue;
}
if(ANSI_OPCODES_ALLOWED_CLEAN.indexOf(m[2].charAt(0)) > -1) {
cleaned += m[0];
}
@ -347,61 +364,141 @@ function cleanControlCodes(input) {
return cleaned;
}
function getCleanAnsi(input) {
//
// Process |input| and produce |cleaned|, an array
// of lines with "clean" ANSI.
//
// Clean ANSI:
// * Contains only color/SGR sequences
// * All movement (up/down/left/right) removed but positioning
// left intact via spaces/etc.
//
// Temporary processing will occur in a grid. Each cell
// containing a character (defaulting to space) possibly a SGR
//
function createCleanAnsi(input, options, cb) {
let m;
let pos;
let grid = [];
let gridPos = { row : 0, col : 0 };
options.width = options.width || 80;
options.height = options.height || 25;
function updateGrid(data, dataType) {
//
// Start at to grid[row][col] and populate val[0]...val[N]
// creating cells as necessary
//
if(!grid[gridPos.row]) {
grid[gridPos.row] = [];
const canvas = new Array(options.height);
for(let i = 0; i < options.height; ++i) {
canvas[i] = new Array(options.width);
for(let j = 0; j < options.width; ++j) {
canvas[i][j] = {};
}
}
if('literal' === dataType) {
data.forEach(c => {
grid[gridPos.row][gridPos.col] = (grid[gridPos.row][gridPos.col] || '') + c; // append to existing SGR
gridPos.col++;
const parserOpts = {
termHeight : options.height,
termWidth : options.width,
};
const parser = new ANSIEscapeParser(parserOpts);
const canvasPos = {
col : 0,
row : 0,
};
let sgr;
function ensureCell() {
// we've pre-allocated a matrix, but allow for > supplied dimens up front. They will be trimmed @ finalize
if(!canvas[canvasPos.row]) {
canvas[canvasPos.row] = new Array(options.width);
for(let j = 0; j < options.width; ++j) {
canvas[canvasPos.row][j] = {};
}
}
canvas[canvasPos.row][canvasPos.col] = canvas[canvasPos.row][canvasPos.col] || {};
//canvas[canvasPos.row][0].width = Math.max(canvas[canvasPos.row][0].width || 0, canvasPos.col);
}
parser.on('literal', literal => {
//
// CR/LF are handled for 'position update'; we don't need the chars themselves
//
literal = literal.replace(/\r?\n|[\r\u2028\u2029]/g, '');
for(let i = 0; i < literal.length; ++i) {
const c = literal.charAt(i);
ensureCell();
canvas[canvasPos.row][canvasPos.col].char = c;
if(sgr) {
canvas[canvasPos.row][canvasPos.col].sgr = sgr;
sgr = null;
}
canvasPos.col += 1;
}
});
} else if('sgr' === dataType) {
grid[gridPos.row][gridPos.col] = (grid[gridPos.row][gridPos.col] || '') + data;
parser.on('control', (match, opCode) => {
if('m' !== opCode) {
return; // don't care'
}
sgr = match;
});
parser.on('position update', (row, col) => {
canvasPos.row = row - 1;
canvasPos.col = Math.min(col - 1, options.width);
});
parser.on('complete', () => {
for(let row = 0; row < options.height; ++row) {
let col = 0;
//while(col <= canvas[row][0].width) {
while(col < options.width) {
if(!canvas[row][col].char) {
canvas[row][col].char = 'P';
if(!canvas[row][col].sgr) {
// :TODO: fix duplicate SGR's in a row here - we just need one per sequence
canvas[row][col].sgr = ANSI.reset();
}
}
function literal(s) {
let charCode;
const len = s.length;
for(let i = 0; i < len; ++i) {
charCode = s.charCodeAt(i) & 0xff;
}
col += 1;
}
do {
pos = REGEXP_ANSI_CONTROL_CODES.lastIndex;
m = REGEXP_ANSI_CONTROL_CODES.exec(input);
// :TODO: end *all* with CRLF - general usage will be width : 79 - prob update defaults
if(null !== m) {
if(m.index > pos) {
updateGrid(input.slice(pos, m.index), 'literal');
if(col <= options.width) {
canvas[row][col] = canvas[row][col] || {};
//canvas[row][col].char = '\r\n';
canvas[row][col].sgr = ANSI.reset();
// :TODO: don't splice, just reset + fill with ' ' till end
for(let fillCol = col; fillCol <= options.width; ++fillCol) {
canvas[row][fillCol].char = 'X';
}
//canvas[row] = canvas[row].splice(0, col + 1);
//canvas[row][options.width - 1].char = '\r\n';
} else {
canvas[row] = canvas[row].splice(0, options.width + 1);
}
} while(0 !== REGEXP_ANSI_CONTROL_CODES.lastIndex);
}
let out = '';
for(let row = 0; row < options.height; ++row) {
out += canvas[row].map( col => {
let c = col.sgr || '';
c += col.char;
return c;
}).join('');
}
// :TODO: finalize: @ any non-char cell, reset sgr & set to ' '
// :TODO: finalize: after sgr established, omit anything > supplied dimens
return cb(out);
});
parser.parse(input);
}
const fs = require('fs');
let data = fs.readFileSync('/home/nuskooler/Downloads/art3.ans');
data = iconv.decode(data, 'cp437');
createCleanAnsi(data, { width : 79, height : 25 }, (out) => {
out = iconv.encode(out, 'cp437');
fs.writeFileSync('/home/nuskooler/Downloads/art4.ans', out);
});

16
core/user.js
View File

@ -325,6 +325,22 @@ User.prototype.persistProperty = function(propName, propValue, cb) {
);
};
User.prototype.removeProperty = function(propName, cb) {
// update live
delete this.properties[propName];
userDb.run(
`DELETE FROM user_property
WHERE user_id = ? AND prop_name = ?;`,
[ this.userId, propName ],
err => {
if(cb) {
return cb(err);
}
}
)
};
User.prototype.persistProperties = function(properties, cb) {
var self = this;

21
mods/file_area_filter_edit.js
View File

@ -30,6 +30,7 @@ const MciViewIds = {
selectedFilterInfo : 10, // { ...filter object ... }
activeFilterInfo : 11, // { ...filter object ... }
error : 12, // validation errors
}
};
@ -67,7 +68,6 @@ exports.getModule = class FileAreaFilterEdit extends MenuModule {
this.updateActiveLabel();
// :TODO: Need to update %FN somehow
return cb(null);
},
newFilter : (formData, extraArgs, cb) => {
@ -93,8 +93,7 @@ exports.getModule = class FileAreaFilterEdit extends MenuModule {
filters.setActive(newActive.uuid);
} else {
// nothing to set active to
// :TODO: is this what we want?
this.client.user.properties.file_base_filter_active_uuid = 'none';
this.client.user.removeProperty('file_base_filter_active_uuid');
}
}
@ -106,7 +105,23 @@ exports.getModule = class FileAreaFilterEdit extends MenuModule {
}
return cb(null);
});
},
viewValidationListener : (err, cb) => {
const errorView = this.viewControllers.editor.getView(MciViewIds.editor.error);
let newFocusId;
if(errorView) {
if(err) {
errorView.setText(err.message);
err.view.clearText(); // clear out the invalid data
} else {
errorView.clearText();
}
}
return cb(newFocusId);
},
};
}

41
mods/file_area_list.js
View File

@ -8,12 +8,16 @@ const ansi = require('../core/ansi_term.js');
const theme = require('../core/theme.js');
const FileEntry = require('../core/file_entry.js');
const stringFormat = require('../core/string_format.js');
const createCleanAnsi = require('../core/string_util.js').createCleanAnsi;
const FileArea = require('../core/file_area.js');
const Errors = require('../core/enig_error.js').Errors;
const ArchiveUtil = require('../core/archive_util.js');
const Config = require('../core/config.js').config;
const DownloadQueue = require('../core/download_queue.js');
const FileAreaWeb = require('../core/file_area_web.js');
const FileBaseFilters = require('../core/file_base_filter.js');
const cleanControlCodes = require('../core/string_util.js').cleanControlCodes;
// deps
const async = require('async');
@ -329,15 +333,27 @@ exports.getModule = class FileAreaList extends MenuModule {
if(_.isString(self.currentFileEntry.desc)) {
const descView = self.viewControllers.browse.getView(MciViewIds.browse.desc);
if(descView) {
descView.setText(self.currentFileEntry.desc);
}
}
createCleanAnsi(
self.currentFileEntry.desc,
{ height : self.client.termHeight, width : descView.dimens.width },
cleanDesc => {
descView.setText(cleanDesc);
self.updateQueueIndicator();
self.populateCustomLabels('browse', 10);
return callback(null);
}
);
descView.setText( self.currentFileEntry.desc );
}
} else {
self.updateQueueIndicator();
self.populateCustomLabels('browse', 10);
return callback(null);
}
}
],
err => {
if(cb) {
@ -442,20 +458,6 @@ exports.getModule = class FileAreaList extends MenuModule {
);
this.updateCustomLabelsWithFilter( 'browse', 10, [ '{isQueued}' ] );
/*
const indicatorView = this.viewControllers.browse.getView(MciViewIds.browse.queueToggle);
if(indicatorView) {
const isQueuedIndicator = this.menuConfig.config.isQueuedIndicator || 'Y';
const isNotQueuedIndicator = this.menuConfig.config.isNotQueuedIndicator || 'N';
indicatorView.setText(stringFormat(
this.dlQueue.isQueued(this.currentFileEntry) ?
isQueuedIndicator :
isNotQueuedIndicator
)
);
}*/
}
cacheArchiveEntries(cb) {
@ -469,7 +471,7 @@ exports.getModule = class FileAreaList extends MenuModule {
return cb(Errors.Invalid('Invalid area tag'));
}
const filePath = paths.join(areaInfo.storageDirectory, this.currentFileEntry.fileName);
const filePath = this.currentFileEntry.filePath;
const archiveUtil = ArchiveUtil.getInstance();
archiveUtil.listEntries(filePath, this.currentFileEntry.entryInfo.archiveType, (err, entries) => {
@ -575,8 +577,9 @@ exports.getModule = class FileAreaList extends MenuModule {
loadFileIds(cb) {
this.fileListPosition = 0;
const activeFilter = FileBaseFilters.getActiveFilter(this.client);
FileEntry.findFiles(this.filterCriteria, (err, fileIds) => {
FileEntry.findFiles(activeFilter, (err, fileIds) => {
this.fileList = fileIds;
return cb(err);
});

4
oputil.js
View File

@ -435,7 +435,7 @@ function handleConfigCommand() {
}
}
function fileAreaScan(areaTag) {
function fileAreaScan() {
async.waterfall(
[
function init(callback) {
@ -453,7 +453,7 @@ function fileAreaScan(areaTag) {
},
function performScan(fileAreaMod, areaInfo, callback) {
fileAreaMod.scanFileAreaForChanges(areaInfo, err => {
return callback(err);
});
}
],