From d132f3932aa31db6648d560e3afcbd1cb92e6f76 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sat, 26 Aug 2017 20:37:20 -0600 Subject: [PATCH] Prepare exported ANSI messages by ensuring they are < 79 characters in length, using ESC[A ESC[C to adjust long lines --- core/ftn_mail_packet.js | 216 +++++++++++++++++++------------- core/scanner_tossers/ftn_bso.js | 24 ++-- core/string_util.js | 89 +++++++++++-- 3 files changed, 221 insertions(+), 108 deletions(-) diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index e517c49a..9fb8347b 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -636,108 +636,144 @@ function Packet(options) { }); }; - this.getMessageEntryBuffer = function(message, options) { - let basicHeader = new Buffer(34); + this.getMessageEntryBuffer = function(message, options, cb) { - basicHeader.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_node, 2); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_node, 4); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_network, 8); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_attr_flags, 10); - basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12); - - const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0'); - dateTimeBuffer.copy(basicHeader, 14); - - // toUserName & fromUserName: up to 36 bytes in length, NULL term'd - // :TODO: DRY... - let toUserNameBuf = iconv.encode(message.toUserName + '\0', 'CP437').slice(0, 36); - toUserNameBuf[toUserNameBuf.length - 1] = '\0'; // ensure it's null term'd - - let fromUserNameBuf = iconv.encode(message.fromUserName + '\0', 'CP437').slice(0, 36); - fromUserNameBuf[fromUserNameBuf.length - 1] = '\0'; // ensure it's null term'd - - // subject: up to 72 bytes in length, NULL term'd - let subjectBuf = iconv.encode(message.subject + '\0', 'CP437').slice(0, 72); - subjectBuf[subjectBuf.length - 1] = '\0'; // ensure it's null term'd - - // - // message: unbound length, NULL term'd - // - // We need to build in various special lines - kludges, area, - // seen-by, etc. - // - // :TODO: Put this in it's own method - let msgBody = ''; - - function appendMeta(k, m) { + function getAppendMeta(k, m) { + let append = ''; if(m) { let a = m; if(!_.isArray(a)) { a = [ a ]; } a.forEach(v => { - msgBody += `${k}: ${v}\r`; + append += `${k}: ${v}\r`; }); } + return append; } + + async.waterfall( + [ + function prepareHeaderAndKludges(callback) { + const basicHeader = new Buffer(34); - // - // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 - // AREA:CONFERENCE - // Should be first line in a message - // - if(message.meta.FtnProperty.ftn_area) { - msgBody += `AREA:${message.meta.FtnProperty.ftn_area}\r`; // note: no ^A (0x01) - } - - Object.keys(message.meta.FtnKludge).forEach(k => { - // we want PATH to be last - if('PATH' !== k) { - appendMeta(`\x01${k}`, message.meta.FtnKludge[k]); + basicHeader.writeUInt16LE(FTN_PACKET_MESSAGE_TYPE, 0); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_node, 2); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_node, 4); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_orig_network, 6); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_dest_network, 8); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_attr_flags, 10); + basicHeader.writeUInt16LE(message.meta.FtnProperty.ftn_cost, 12); + + const dateTimeBuffer = new Buffer(ftn.getDateTimeString(message.modTimestamp) + '\0'); + dateTimeBuffer.copy(basicHeader, 14); + + // toUserName & fromUserName: up to 36 bytes in length, NULL term'd + // :TODO: DRY... + let toUserNameBuf = iconv.encode(message.toUserName + '\0', 'CP437').slice(0, 36); + toUserNameBuf[toUserNameBuf.length - 1] = '\0'; // ensure it's null term'd + + let fromUserNameBuf = iconv.encode(message.fromUserName + '\0', 'CP437').slice(0, 36); + fromUserNameBuf[fromUserNameBuf.length - 1] = '\0'; // ensure it's null term'd + + // subject: up to 72 bytes in length, NULL term'd + let subjectBuf = iconv.encode(message.subject + '\0', 'CP437').slice(0, 72); + subjectBuf[subjectBuf.length - 1] = '\0'; // ensure it's null term'd + + // + // message: unbound length, NULL term'd + // + // We need to build in various special lines - kludges, area, + // seen-by, etc. + // + // :TODO: Put this in it's own method + let msgBody = ''; + + // + // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 + // AREA:CONFERENCE + // Should be first line in a message + // + if(message.meta.FtnProperty.ftn_area) { + msgBody += `AREA:${message.meta.FtnProperty.ftn_area}\r`; // note: no ^A (0x01) + } + + Object.keys(message.meta.FtnKludge).forEach(k => { + // we want PATH to be last + if('PATH' !== k) { + msgBody += getAppendMeta(`\x01${k}`, message.meta.FtnKludge[k]); + } + }); + + return callback(null, basicHeader, toUserNameBuf, fromUserNameBuf, subjectBuf, msgBody); + }, + function prepareAnsiMessageBody(basicHeader, toUserNameBuf, fromUserNameBuf, subjectBuf, msgBody, callback) { + if(!strUtil.isAnsi(message.message)) { + return callback(null, basicHeader, toUserNameBuf, fromUserNameBuf, subjectBuf, msgBody, message.message); + } + + strUtil.prepAnsi( + message.message, + { + cols : 80, + rows : 'auto', + preserveTextLines : true, + forceLineTerm : true, + exportMode : true, + }, + (err, preppedMsg) => { + return callback(null, basicHeader, toUserNameBuf, fromUserNameBuf, subjectBuf, msgBody, preppedMsg || message.message); + } + ); + }, + function addMessageBody(basicHeader, toUserNameBuf, fromUserNameBuf, subjectBuf, msgBody, preppedMsg, callback) { + msgBody += preppedMsg + '\r'; + + // + // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 + // Tear line should be near the bottom of a message + // + if(message.meta.FtnProperty.ftn_tear_line) { + msgBody += `${message.meta.FtnProperty.ftn_tear_line}\r`; + } + + // + // Origin line should be near the bottom of a message + // + if(message.meta.FtnProperty.ftn_origin) { + msgBody += `${message.meta.FtnProperty.ftn_origin}\r`; + } + + // + // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 + // SEEN-BY and PATH should be the last lines of a message + // + msgBody += getAppendMeta('SEEN-BY', message.meta.FtnProperty.ftn_seen_by); // note: no ^A (0x01) + msgBody += getAppendMeta('\x01PATH', message.meta.FtnKludge['PATH']); + + let msgBodyEncoded; + try { + msgBodyEncoded = iconv.encode(msgBody + '\0', options.encoding); + } catch(e) { + msgBodyEncoded = iconv.encode(msgBody + '\0', 'ascii'); + } + + return callback( + null, + Buffer.concat( [ + basicHeader, + toUserNameBuf, + fromUserNameBuf, + subjectBuf, + msgBodyEncoded + ]) + ); + } + ], + (err, msgEntryBuffer) => { + return cb(err, msgEntryBuffer); } - }); - - msgBody += message.message + '\r'; - - // - // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 - // Tear line should be near the bottom of a message - // - if(message.meta.FtnProperty.ftn_tear_line) { - msgBody += `${message.meta.FtnProperty.ftn_tear_line}\r`; - } - - // - // Origin line should be near the bottom of a message - // - if(message.meta.FtnProperty.ftn_origin) { - msgBody += `${message.meta.FtnProperty.ftn_origin}\r`; - } - - // - // FTN-0004.001 @ http://ftsc.org/docs/fts-0004.001 - // SEEN-BY and PATH should be the last lines of a message - // - appendMeta('SEEN-BY', message.meta.FtnProperty.ftn_seen_by); // note: no ^A (0x01) - - appendMeta('\x01PATH', message.meta.FtnKludge['PATH']); - - let msgBodyEncoded; - try { - msgBodyEncoded = iconv.encode(msgBody + '\0', options.encoding); - } catch(e) { - msgBodyEncoded = iconv.encode(msgBody + '\0', 'ascii'); - } - - return Buffer.concat( [ - basicHeader, - toUserNameBuf, - fromUserNameBuf, - subjectBuf, - msgBodyEncoded - ]); + ); }; this.writeMessage = function(message, ws, options) { diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 7c36a3df..039caac7 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -608,16 +608,22 @@ function FTNMessageScanTossModule() { callback(null); }, function appendMessage(callback) { - const msgBuf = packet.getMessageEntryBuffer(message, exportOpts); - currPacketSize += msgBuf.length; + packet.getMessageEntryBuffer(message, exportOpts, (err, msgBuf) => { + if(err) { + return callback(err); + } + + currPacketSize += msgBuf.length; - if(currPacketSize >= self.moduleConfig.packetTargetByteSize) { - remainMessageBuf = msgBuf; // save for next packet - remainMessageId = message.messageId; - } else { - ws.write(msgBuf); - } - callback(null); + if(currPacketSize >= self.moduleConfig.packetTargetByteSize) { + remainMessageBuf = msgBuf; // save for next packet + remainMessageId = message.messageId; + } else { + ws.write(msgBuf); + } + + return callback(null); + }); }, function storeStateFlags0Meta(callback) { message.persistMetaValue('System', 'state_flags0', Message.StateFlags0.Exported.toString(), err => { diff --git a/core/string_util.js b/core/string_util.js index b7cdb081..98b691d9 100644 --- a/core/string_util.js +++ b/core/string_util.js @@ -217,8 +217,9 @@ function stringFromNullTermBuffer(buf, encoding) { } const PIPE_REGEXP = /(\|[A-Z\d]{2})/g; -const ANSI_REGEXP = /[\u001b\u009b][[()#;?]*([0-9]{1,4}(?:;[0-9]{0,4})*)?([0-9A-ORZcf-npqrsuy=><])/g; -const ANSI_OR_PIPE_REGEXP = new RegExp(PIPE_REGEXP.source + '|' + ANSI_REGEXP.source, 'g'); +//const ANSI_REGEXP = /[\u001b\u009b][[()#;?]*([0-9]{1,4}(?:;[0-9]{0,4})*)?([0-9A-ORZcf-npqrsuy=><])/g; +//const ANSI_OR_PIPE_REGEXP = new RegExp(PIPE_REGEXP.source + '|' + ANSI_REGEXP.source, 'g'); +const ANSI_OR_PIPE_REGEXP = new RegExp(PIPE_REGEXP.source + '|' + ANSI.getFullMatchRegExp().source, 'g'); // // Similar to substr() but works with ANSI/Pipe code strings @@ -393,6 +394,7 @@ function prepAnsi(input, options, cb) { options.rows = options.rows || options.termHeight || 'auto'; options.startCol = options.startCol || 1; options.preserveTextLines = options.preserveTextLines || false; + options.exportMode = options.exportMode || false; const canvas = Array.from( { length : 'auto' === options.rows ? 25 : options.rows }, () => Array.from( { length : options.cols}, () => new Object() ) ); const parser = new ANSIEscapeParser( { termHeight : options.termHeight, termWidth : options.termWidth } ); @@ -516,12 +518,81 @@ function prepAnsi(input, options, cb) { output += `${ANSI.blackBG()}${row.slice(i).map( () => ' ').join('')}${lastSgr}`; } - if(options.startCol + options.cols < options.termWidth || options.forceLineTerm) { + //if(options.startCol + options.cols < options.termWidth || options.forceLineTerm) { + if(options.startCol + i < options.termWidth || options.forceLineTerm) { output += '\r\n'; } } }); + if(options.exportMode) { + // + // If we're in export mode, we do some additional hackery: + // + // * Hard wrap ALL lines at <= 79 *characters* (not visible columns) + // if a line must wrap early, we'll place a ESC[A ESC[C where + // represents chars to get back to the position we were previously at + // + // * Replace contig spaces with ESC[C as well to save... space. + // + // :TODO: this would be better to do as part of the processing above, but this will do for now + const MAX_CHARS = 79 - 8; // 79 max, - 8 for max ESC seq's we may prefix a line with + let exportOutput = ''; + + let m; + let afterSeq; + let wantMore; + let renderStart; + + splitTextAtTerms(output).forEach(fullLine => { + renderStart = 0; + + while(fullLine.length > 0) { + let splitAt; + const ANSI_REGEXP = ANSI.getFullMatchRegExp(); + wantMore = true; + + while((m = ANSI_REGEXP.exec(fullLine))) { + afterSeq = m.index + m[0].length; + + if(afterSeq < MAX_CHARS) { + // after current seq + splitAt = afterSeq; + } else { + if(m.index < MAX_CHARS) { + // before last found seq + splitAt = m.index; + wantMore = false; // can't eat up any more + } + + break; // seq's beyond this point are >= MAX_CHARS + } + } + + if(splitAt) { + if(wantMore) { + splitAt = Math.min(fullLine.length, MAX_CHARS - 1); + } + } else { + splitAt = Math.min(fullLine.length, MAX_CHARS - 1); + } + + const part = fullLine.slice(0, splitAt); + fullLine = fullLine.slice(splitAt); + renderStart += renderStringLength(part); + exportOutput += `${part}\r\n`; + + if(fullLine.length > 0) { // more to go for this line? + exportOutput += `${ANSI.up()}${ANSI.right(renderStart)}`; + } else { + exportOutput += ANSI.up(); + } + } + }); + + return cb(null, exportOutput); + } + return cb(null, output); }); @@ -561,17 +632,17 @@ function isAnsi(input) { function splitTextAtTerms(s) { return s.split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g); } + + /* const fs = require('graceful-fs'); -//let data = fs.readFileSync('/home/nuskooler/Downloads/art3.ans'); +let data = fs.readFileSync('/home/nuskooler/Downloads/art3.ans'); //let data = fs.readFileSync('/home/nuskooler/dev/enigma-bbs/mods/themes/nu-xibalba/MATRIX1.ANS'); //let data = fs.readFileSync('/home/nuskooler/Downloads/ansi_diz_test/file_id.diz.2.ans'); -let data = fs.readFileSync('/home/nuskooler/Downloads/acidunder.ans'); -data = data.toString().replace(/\n/g,'\r\n'); -//data = iconv.decode(data, 'cp437'); -prepAnsi(data, { cols : 80, rows : 50 }, (err, out) => { +data = fs.readFileSync('/home/nuskooler/ownCloud/temp/BS-AUW.ANS'); +data = iconv.decode(data, 'cp437'); +prepAnsi(data, { cols : 80, rows : 50, exportMode : true, forceLineTerm : true }, (err, out) => { out = iconv.encode(out, 'cp437'); fs.writeFileSync('/home/nuskooler/Downloads/art4.ans', out); }); - */ \ No newline at end of file