ANSI improvements

* ANSI in FSE
* ANSI vs standard quote builder
* ANSI handling methods/helpers

core/fse.js

@@ -10,10 +10,10 @@ const Message						= require('./message.js');
 const updateMessageAreaLastReadId	= require('./message_area.js').updateMessageAreaLastReadId;
 const getMessageAreaByTag			= require('./message_area.js').getMessageAreaByTag;
 const User							= require('./user.js');
-const cleanControlCodes				= require('./string_util.js').cleanControlCodes;
 const StatLog						= require('./stat_log.js');
 const stringFormat					= require('./string_format.js');
 const MessageAreaConfTempSwitcher	= require('./mod_mixins.js').MessageAreaConfTempSwitcher;
+const { isAnsi, cleanControlCodes }	= require('./string_util.js');
 
 //	deps
 const async							= require('async');
@@ -344,9 +344,24 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul
 					this.initHeaderViewMode();
 					this.initFooterViewMode();
 
-					var bodyMessageView = this.viewControllers.body.getView(1);
+					const bodyMessageView = this.viewControllers.body.getView(1);
 					if(bodyMessageView && _.has(this, 'message.message')) {
-						bodyMessageView.setText(cleanControlCodes(this.message.message));
+						//
+						//	We handle ANSI messages differently than standard messages -- this is required as
+						//	we don't want to do things like word wrap ANSI, but instead, trust that it's formatted
+						//	how the author wanted it
+						//
+						if(isAnsi(this.message.message)) {
+							bodyMessageView.setAnsi(
+								this.message.message.replace(/\r?\n/g, '\r\n'),	//	messages are stored with CRLF -> LF
+								{
+									prepped			: false,
+									forceLineTerm	: true
+								}
+							);
+						} else {
+							bodyMessageView.setText(cleanControlCodes(this.message.message));
+						}
 					}
 				}
 			}
@@ -848,9 +863,29 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul
 					}
 				},
 				function loadQuoteLines(callback) {
-					var quoteView = self.viewControllers.quoteBuilder.getView(3);
+					const quoteView = self.viewControllers.quoteBuilder.getView(3);
-					quoteView.setItems(self.replyToMessage.getQuoteLines(quoteView.dimens.width));
+					const bodyView	= self.viewControllers.body.getView(1);
-					callback(null);
+
+					self.replyToMessage.getQuoteLines(
+						{
+							termWidth			: self.client.term.termWidth,
+							termHeight			: self.client.term.termHeight,
+							cols				: quoteView.dimens.width,
+							startCol			: quoteView.position.col,
+							ansiResetSgr		: bodyView.styleSGR1,
+							ansiFocusPrefixSgr	: quoteView.styleSGR2,
+						},
+						(err, quoteLines, focusQuoteLines) => {
+							if(err) {
+								return callback(err);
+							}
+
+							quoteView.setItems(quoteLines);
+							quoteView.setFocusItems(focusQuoteLines);
+
+							return callback(null);
+						}
+					);
 				},
 				function setViewFocus(callback) {
 					self.viewControllers.quoteBuilder.getView(1).setFocus(false);

core/ftn_mail_packet.js

@@ -480,8 +480,8 @@ function Packet(options) {
 						Log.debug( { encoding : encoding, error : e.toString() }, 'Error decoding. Falling back to ASCII');
 						decoded = iconv.decode(messageBodyBuffer, 'ascii');
 					}
-					//const messageLines = iconv.decode(messageBodyBuffer, encoding).replace(/\xec/g, '').split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);
+					
-					const messageLines	= decoded.replace(/\xec/g, '').split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);
+					const messageLines	= strUtil.splitTextAtTerms(decoded.replace(/\xec/g, ''));
 					let endOfMessage	= false;
 
 					messageLines.forEach(line => {

core/message.js

@@ -6,6 +6,13 @@ const wordWrapText			= require('./word_wrap.js').wordWrapText;
 const ftnUtil				= require('./ftn_util.js');
 const createNamedUUID		= require('./uuid_util.js').createNamedUUID;
 const getISOTimestampString	= require('./database.js').getISOTimestampString;
+const Errors				= require('./enig_error.js').Errors;
+const ANSI					= require('./ansi_term.js');
+
+const { 
+	prepAnsi, isAnsi, 
+	splitTextAtTerms
+}							= require('./string_util.js');
 
 //	deps
 const uuidParse				= require('uuid-parse');
@@ -429,33 +436,167 @@ Message.prototype.getFTNQuotePrefix = function(source) {
 	return ftnUtil.getQuotePrefix(this[source]);
 };
 
-Message.prototype.getQuoteLines = function(width, options) {
+Message.prototype.getQuoteLines = function(options, cb) {
-	//	:TODO: options.maxBlankLines = 1
+	if(!options.termWidth || !options.termHeight || !options.cols) {
-
+		return cb(Errors.MissingParam());
-	options = options || {};
+	}
 	
-	//
+	options.startCol			= options.startCol || 1;
-	//	Include FSC-0032 style quote prefixes?
+	options.includePrefix 		= _.get(options, 'includePrefix', true);
-	//
+	options.ansiResetSgr		= options.ansiResetSgr || ANSI.getSGRFromGraphicRendition( { fg : 39, bg : 49 }, true);
-	//	See http://ftsc.org/docs/fsc-0032.001
+	options.ansiFocusPrefixSgr	= options.ansiFocusPrefixSgr || ANSI.getSGRFromGraphicRendition( { intensity : 'bold', fg : 39, bg : 49 } );
-	//
+	
-	if(!_.isBoolean(options.includePrefix)) {
+	/*
-		options.includePrefix = true;
+		Some long text that needs to be wrapped and quoted should look right after
+		doing so, don't ya think? yeah I think so		
+
+		Nu> Some long text that needs to be wrapped and quoted should look right 
+		Nu> after doing so, don't ya think? yeah I think so
+
+		Ot> Nu> Some long text that needs to be wrapped and quoted should look 
+		Ot> Nu> right after doing so, don't ya think? yeah I think so
+
+	*/
+	const quotePrefix = options.includePrefix ? this.getFTNQuotePrefix(options.prefixSource || 'fromUserName') : '';
+
+	function getWrapped(text, extraPrefix) {
+		extraPrefix = extraPrefix ? ` ${extraPrefix}` : '';
+
+		const wrapOpts = {
+			width		: options.cols - (quotePrefix.length + extraPrefix.length),
+			tabHandling	: 'expand',
+			tabWidth	: 4,
+		};
+		
+		return wordWrapText(text, wrapOpts).wrapped.map( (w, i) => {
+			return i === 0 ? `${quotePrefix}${w}` : `${quotePrefix}${extraPrefix}${w}`;
+		});
 	}
 
-	var quoteLines = [];
+	if(isAnsi(this.message)) {
+		prepAnsi(
+			this.message.replace(/\r?\n/g, '\r\n'),	//	normalized LF -> CRLF
+			{
+				termWidth		: options.termWidth,
+				termHeight		: options.termHeight,
+				cols			: options.cols - quotePrefix.length,
+				rows			: 5000,	//	:TODO: Need 'auto'
+				startCol		: options.startCol,
+				forceLineTerm	: true,				
+			},
+			(err, prepped) => {
+				prepped = prepped || this.message;
+				
+				//const reset 	= ANSI.reset() + ANSI.white();	//	:TODO: this is quite borked...
+				let lastSgr = '';
+				const split = splitTextAtTerms(prepped);
+				
+				const quoteLines		= [];
+				const focusQuoteLines	= [];
 
-	var origLines = this.message
+				//
+				//	Create items (standard) and inverted items for focus views
+				//
+				split.forEach(l => {
+					quoteLines.push(`${options.ansiResetSgr}${quotePrefix}${lastSgr}${l}`);
+					focusQuoteLines.push(`${options.ansiFocusPrefixSgr}${quotePrefix}${lastSgr}${l}`);
+				
+					lastSgr = (l.match(/(?:\x1b\x5b)[0-9]{1,3}[m](?!.*(?:\x1b\x5b)[0-9]{1,3}[m])/) || [])[0] || '';	//	eslint-disable-line no-control-regex
+				});				
+
+				quoteLines[quoteLines.length - 1] += options.ansiResetSgr;//ANSI.getSGRFromGraphicRendition( { fg : 39, bg : 49 }, true );
+				
+				return cb(null, quoteLines, focusQuoteLines);
+			}
+		);
+	} else {
+		const QUOTE_RE	= /^ ((?:[A-Za-z0-9]{2}\> )+(?:[A-Za-z0-9]{2}\>)*) */;
+		const quoted	= [];
+		const input		= this.message.trim().replace(/\b/g, '');
+	
+		//	find *last* tearline
+		let tearLinePos = input.match(/^--- .+$(?![\s\S]*^--- .+$)/m);
+		tearLinePos = tearLinePos ? tearLinePos.index : input.length;	//	we just want the index or the entire string
+		
+		input.slice(0, tearLinePos).split(/\r\n\r\n|\n\n/).forEach(paragraph => {
+			//
+			//	For each paragraph, a state machine:
+			//	- New line - line
+			//	- New (pre)quoted line - quote_line
+			//	- Continuation of new/quoted line
+			//
+			//	:TODO: fix extra space in quoted quotes, e.g. "Nu>  Su> blah blah"
+			let state;
+			let buf = '';
+			let quoteMatch;
+			paragraph.split(/\r?\n/).forEach(line => {
+				quoteMatch = line.match(QUOTE_RE);
+
+				switch(state) {
+					case 'line' :
+						if(quoteMatch) {
+							quoted.push(...getWrapped(buf, quoteMatch[1]));
+							state = 'quote_line';
+							buf = line;
+						} else {
+							buf += ` ${line}`;
+						}
+						break;
+						
+					case 'quote_line' :
+						if(quoteMatch) {
+							const rem = line.slice(quoteMatch[0].length);
+							if(!buf.startsWith(quoteMatch[0])) {
+								quoted.push(...getWrapped(buf, quoteMatch[1]));
+								buf = rem;
+							} else {
+								buf += ` ${rem}`;
+							}
+						} else {
+							quoted.push(...getWrapped(buf));
+							buf = line;
+							state = 'line';
+						}
+						break;
+							
+					default :
+						state	= quoteMatch ? 'quote_line' : 'line';
+						buf		= 'line' === state ? line : _.trimStart(line);
+						break;
+				}		
+			});
+			
+			quoted.push(...getWrapped(buf, quoteMatch ? quoteMatch[1] : null));
+		});
+		
+		input.slice(tearLinePos).split(/\r?\n/).forEach(l => {
+			quoted.push(...getWrapped(l));
+		});
+
+		return cb(null, quoted);
+	}
+};
+
+Message.prototype.getQuoteLines2 = function(width, options = { includePrefix : true } ) {
+	//	:TODO: options.maxBlankLines = 1
+
+	//
+	//	See FSC-0032 for quote prefix/spec @ http://ftsc.org/docs/fsc-0032.001
+	//
+	const quoteLines = [];
+
+	const origLines = this.message
 		.trim()
 		.replace(/\b/g, '')
-		.split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);	
+		.split(/(?:\r\n|[\n\v\f\r\x85\u2028\u2029])(?:\r\n|[\n\v\f\r\x85\u2028\u2029])/);
+	//	.split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);
 	
-	var quotePrefix = '';	//	we need this init even if blank
+	let quotePrefix = '';	//	we need this init even if blank
 	if(options.includePrefix) {
 		quotePrefix = this.getFTNQuotePrefix(options.prefixSource || 'fromUserName');
 	}
 
-	var wrapOpts = {
+	const wrapOpts = {
 		width		: width - quotePrefix.length,
 		tabHandling	: 'expand',
 		tabWidth	: 4,
@@ -465,7 +606,7 @@ Message.prototype.getQuoteLines = function(width, options) {
 		return quotePrefix + l;
 	}
 
-	var wrapped;
+	let wrapped;
 	for(var i = 0; i < origLines.length; ++i) {
 		wrapped = wordWrapText(origLines[i], wrapOpts).wrapped;
 		Array.prototype.push.apply(quoteLines, _.map(wrapped, addPrefix));

core/multi_line_edit_text_view.js

@@ -165,43 +165,49 @@ function MultiLineEditTextView(options) {
 		return self.textLines.length;
 	};
 
+	this.toggleTextCursor = function(action) {
+		self.client.term.rawWrite(`${self.getSGRFor('text')}${'hide' === action ? ansi.hideCursor() : ansi.showCursor()}`);
+	};
+
 	this.redrawRows = function(startRow, endRow) {
-		self.client.term.rawWrite(self.getSGRFor('text') + ansi.hideCursor());
+		self.toggleTextCursor('hide');
 
-		var startIndex	= self.getTextLinesIndex(startRow);
+		const startIndex	= self.getTextLinesIndex(startRow);
-		var endIndex	= Math.min(self.getTextLinesIndex(endRow), self.textLines.length);
+		const endIndex		= Math.min(self.getTextLinesIndex(endRow), self.textLines.length);
-		var absPos		= self.getAbsolutePosition(startRow, 0);
+		const absPos		= self.getAbsolutePosition(startRow, 0);
 
-		for(var i = startIndex; i < endIndex; ++i) {
+		for(let i = startIndex; i < endIndex; ++i) {
 			self.client.term.write(
-				ansi.goto(absPos.row++, absPos.col) +
+				`${self.getSGRFor('text')}${ansi.goto(absPos.row++, absPos.col)}${self.getRenderText(i)}`, 
-				self.getRenderText(i), false);
+				false	//	convertLineFeeds
+			);
 		}
 
-		self.client.term.rawWrite(ansi.showCursor());
+		self.toggleTextCursor('show');
 
 		return absPos.row - self.position.row;	//	row we ended on
 	};
 
 	this.eraseRows = function(startRow, endRow) {
-		self.client.term.rawWrite(self.getSGRFor('text') + ansi.hideCursor());
+		self.toggleTextCursor('hide');
 
-		var absPos		= self.getAbsolutePosition(startRow, 0);
+		const absPos		= self.getAbsolutePosition(startRow, 0);
-		var absPosEnd	= self.getAbsolutePosition(endRow, 0);
+		const absPosEnd		= self.getAbsolutePosition(endRow, 0);
-		var eraseFiller	= new Array(self.dimens.width).join(' ');
+		const eraseFiller	= ' '.repeat(self.dimens.width);//new Array(self.dimens.width).join(' ');
 
 		while(absPos.row < absPosEnd.row) {
 			self.client.term.write(
-				ansi.goto(absPos.row++, absPos.col) +
+				`${ansi.goto(absPos.row++, absPos.col)}${eraseFiller}`,
-				eraseFiller, false);
+				false	//	convertLineFeeds
+			);
 		}
 
-		self.client.term.rawWrite(ansi.showCursor());
+		self.toggleTextCursor('show');
 	};
 
 	this.redrawVisibleArea = function() {
 		assert(self.topVisibleIndex <= self.textLines.length);
-		var lastRow = self.redrawRows(0, self.dimens.height);
+		const lastRow = self.redrawRows(0, self.dimens.height);
 
 		self.eraseRows(lastRow, self.dimens.height);
 		/*
@@ -255,11 +261,14 @@ function MultiLineEditTextView(options) {
 	};
 
 	this.getRenderText = function(index) {
-		var text = self.getVisibleText(index);
+		let text 		= self.getVisibleText(index);
-		var remain	= self.dimens.width - text.length;
+		const remain	= self.dimens.width - text.length;
+
 		if(remain > 0) {
-			text += new Array(remain + 1).join(' ');
+			text += ' '.repeat(remain + 1);
+//			text += new Array(remain + 1).join(' ');
 		}
+
 		return text;
 	};
 
@@ -509,10 +518,6 @@ function MultiLineEditTextView(options) {
 			});
 	};
 
-	this.splitText = function(text) {
-		return text.replace(/\b/g, '').split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);
-	};
-
 	this.setTextLines = function(lines, index, termWithEol) {
 		if(0 === index && (0 === self.textLines.length || (self.textLines.length === 1 && '' === self.textLines[0].text) )) {
 			//	quick path: just set the things
@@ -545,7 +550,7 @@ function MultiLineEditTextView(options) {
 	this.setAnsiWithOptions = function(ansi, options, cb) {
 
 		function setLines(text) {
-			self.setTextLines( self.splitText(text), 0 );
+			self.setTextLines( strUtil.splitTextAtTerms(text), 0 );
 			self.cursorStartOfDocument();
 
 			if(cb) {
@@ -605,7 +610,7 @@ function MultiLineEditTextView(options) {
 			index = self.textLines.length;
 		}
 
-		text = self.splitText(text);
+		text = strUtil.splitTextAtTerms(text);
 
 		let wrapped;
 		text.forEach(line => {

core/string_util.js

@@ -22,6 +22,8 @@ exports.formatByteSizeAbbr			= formatByteSizeAbbr;
 exports.formatByteSize				= formatByteSize;
 exports.cleanControlCodes			= cleanControlCodes;
 exports.prepAnsi					= prepAnsi;
+exports.isAnsi						= isAnsi;
+exports.splitTextAtTerms			= splitTextAtTerms;
 
 //	:TODO: create Unicode verison of this
 const VOWELS = [ 'a', 'e', 'i', 'o', 'u' ];
@@ -324,8 +326,8 @@ function formatByteSize(byteSize, withAbbr, decimals) {
 //const REGEXP_ANSI_CONTROL_CODES		= /(\x1b\x5b)([\?=;0-9]*?)([0-9A-ORZcf-npsu=><])/g;
 const REGEXP_ANSI_CONTROL_CODES		= /(?:\x1b\x5b)([\?=;0-9]*?)([A-ORZcf-npsu=><])/g;
 const ANSI_OPCODES_ALLOWED_CLEAN	= [
-	'A', 'B',	//	up, down
+	//'A', 'B',	//	up, down
-	'C', 'D',	//	right, left
+	//'C', 'D',	//	right, left
 	'm',		//	color
 ];
 
@@ -434,9 +436,7 @@ function prepAnsi(input, options, cb) {
 				break;
 				
 			default		: 
-				if(-1 === [ 'C' ].indexOf(opCode)) {
+				break;
-					console.log(`ignore opCode: ${opCode}`);	//	:TODO: REMOVE ME
-				}
 		}
 	});
 
@@ -479,14 +479,45 @@ function prepAnsi(input, options, cb) {
 
 	parser.parse(input);
 }
+
+function isAnsi(input) {
+	//
+	//	* ANSI found - limited, just colors
+	//	* Full ANSI art
+	//	* 
+	//	
+	//	FULL ANSI art:
+	//	* SAUCE present & reports as ANSI art
+	//	* ANSI clear screen within first 2-3 codes
+	//	* ANSI movement codes (goto, right, left, etc.)
+	//	
+	//	* 
+	/*
+	readSAUCE(input, (err, sauce) => {
+		if(!err && ('ANSi' === sauce.fileType || 'ANSiMation' === sauce.fileType)) {
+			return cb(null, 'ansi');
+		}
+	});
+	*/
+
+	const ANSI_DET_REGEXP = /(?:\x1b\x5b)[0-9]{1,3}[ABCDEFGJKLMSTrsuHfhlm]/g;
+	return ( input.match(ANSI_DET_REGEXP) || [] ).length > 4;	//	:TODO: do this reasonably, e.g. a percent or soemthing
+}
+
+function splitTextAtTerms(s) {
+	return s.split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g);
+}
 /*
 const fs = require('graceful-fs');
 //let data = fs.readFileSync('/home/nuskooler/Downloads/art3.ans');
 //let data = fs.readFileSync('/home/nuskooler/dev/enigma-bbs/mods/themes/nu-xibalba/MATRIX1.ANS');
-let data = fs.readFileSync('/home/nuskooler/Downloads/ansi_diz_test/file_id.diz.2.ans');
+//let data = fs.readFileSync('/home/nuskooler/Downloads/ansi_diz_test/file_id.diz.2.ans');
-data = iconv.decode(data, 'cp437');
+let data = fs.readFileSync('/home/nuskooler/Downloads/acidunder.ans');
-prepAnsi(data, { cols : 45, rows : 25 }, (err, out) => {
+data = data.toString().replace(/\n/g,'\r\n');
+//data = iconv.decode(data, 'cp437');
+prepAnsi(data, { cols : 80, rows : 50 }, (err, out) => {
 	out = iconv.encode(out, 'cp437');
 	fs.writeFileSync('/home/nuskooler/Downloads/art4.ans', out);
 });
+
 */