From 2c082ceedda2ae539f93e131385b65988e118311 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 16 Jul 2015 17:13:48 -0600 Subject: [PATCH] * WIP on FidoNet stuff --- core/database.js | 2 +- core/ftn_mail_packet.js | 70 ++++++++++++++++++++++++++++++++--------- core/message.js | 30 ++++++++++++++---- 3 files changed, 81 insertions(+), 21 deletions(-) diff --git a/core/database.js b/core/database.js index 7f8c4f71..b5c20ef8 100644 --- a/core/database.js +++ b/core/database.js @@ -134,7 +134,7 @@ function createMessageBaseTables() { ' message_id INTEGER NOT NULL,' + ' meta_name VARCHAR NOT NULL,' + ' meta_value VARCHAR NOT NULL,' + - ' UNIQUE(message_id, meta_name),' + + ' UNIQUE(message_id, meta_name, meta_value),' + ' FOREIGN KEY(message_id) REFERENCES message(message_id)' + ');' ); diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index 09ff6f52..b70a5b14 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -11,6 +11,7 @@ var binary = require('binary'); var fs = require('fs'); var util = require('util'); var async = require('async'); +var iconv = require('iconv-lite'); // // References @@ -117,21 +118,66 @@ function FTNMailPacket(options) { }); }; - self.getMessageMeta = function(msgData) { - Object.keys(msgData.kludgeLines).forEach(function kludgeName(kn) { - + self.getKludgeLineMessageMeta = function(msgBody) { + // + // For all well known |msgBody.kludgeLines|, create metadata entries. + // + // Example: 'MSGID' = [ ... ] -> 'fidonet_msg_id' = [ ... ] + // + var kludgeMeta = {}; + + var mapName; + // :TODO: there is probably a nicer way: + Object.keys(msgBody.kludgeLines).forEach(function kludgeName(kn) { + mapName = Message.FidoNetMetaNameMap[kn]; + if(mapName) { + kludgeMeta[mapName] = msgBody.kludgeLines[kn]; + } }); + + return kludgeMeta; + }; + + self.getMessageMeta = function(msgBody) { - return {}; // :TODO: convert msgData kludges/etc. -> Message meta + var meta = self.getKludgeLineMessageMeta(msgBody); + + if(msgBody.seenBy) { + meta[Message.MetaNames.FidoNetSeenBy] = msgBody.seenBy; + } + + return meta; }; this.parseFtnMessageBody = function(msgBodyBuffer, cb) { - // :TODO: Look at spec about line endings/etc. Prob \r\n|[\r\n]| + // + // From FTS-0001.16: + // "Message text is unbounded and null terminated (note exception below). + // + // A 'hard' carriage return, 0DH, marks the end of a paragraph, and must + // be preserved. + // + // So called 'soft' carriage returns, 8DH, may mark a previous + // processor's automatic line wrap, and should be ignored. Beware that + // they may be followed by linefeeds, or may not. + // + // All linefeeds, 0AH, should be ignored. Systems which display message + // text should wrap long lines to suit their application." + // + // This is a bit tricky. Decoding the buffer to CP437 converts all 0x8d -> 0xec, so we'll + // have to replace those characters if the buffer is left as CP437. + // After decoding, we'll need to peek at the buffer for the various kludge lines + // for charsets & possibly re-decode. Uggh! + // // :TODO: Use the proper encoding here. There appear to be multiple specs and/or // stuff people do with this... some specs kludge lines, which is kinda durpy since // to get to that point, one must read the file (and decode) to find said kludge... - var msgLines = msgBodyBuffer.toString().split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g); + + + //var msgLines = msgBodyBuffer.toString().split(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g); + + var msgLines = iconv.decode(msgBodyBuffer, 'CP437').replace(/\xec/g, '').split(/\r\n|[\r\n]/g); var msgBody = { message : [], @@ -151,9 +197,7 @@ function FTNMailPacket(options) { } else { msgBody.kludgeLines[kludgeParts[0]].push(kludgeParts[1]); } - }; - - // :TODO: should 0x8d "soft line feeds" be removed? + } msgLines.forEach(function nextLine(line) { if(0 === line.length) { @@ -214,10 +258,8 @@ function FTNMailPacket(options) { } // buffer to string conversion - // :TODO: What is the real encoding here? ... like messages, may need to look at various - // specs for encoding but default to ASCII? [ 'modDateTime', 'toUserName', 'fromUserName', 'subject', ].forEach(function field(f) { - msgData[f] = msgData[f].toString(); + msgData[f] = iconv.decode(msgData[f], 'CP437'); }); self.parseFtnMessageBody(msgData.message, function msgBodyParsed(err, msgBody) { @@ -231,7 +273,7 @@ function FTNMailPacket(options) { subject : msgData.subject, message : msgBody.message.join('\n'), // :TODO: \r\n is better? modTimestamp : ftn.getDateFromFtnDateTime(msgData.modDateTime), - meta : self.getMessageMeta(msgData), + meta : self.getMessageMeta(msgBody), }); self.emit('message', msg); // :TODO: Placeholder @@ -416,7 +458,7 @@ mailPacket.on('message', function msgParsed(msg) { console.log(msg); }); -mailPacket.read( { packetPath : '/home/nuskooler/ownCloud/Projects/ENiGMA½ BBS/FTNPackets/BAD_BNDL.007' } ); +mailPacket.read( { packetPath : '/home/bashby/ownCloud/Projects/ENiGMA½ BBS/FTNPackets/BAD_BNDL.007' } ); /* mailPacket.parse('/home/nuskooler/ownCloud/Projects/ENiGMA½ BBS/FTNPackets/BAD_BNDL.007', function parsed(err, messages) { diff --git a/core/message.js b/core/message.js index 329e718d..811229d6 100644 --- a/core/message.js +++ b/core/message.js @@ -20,7 +20,14 @@ function Message(options) { this.fromUserName = options.fromUserName || ''; this.subject = options.subject || ''; this.message = options.message || ''; - this.modTimestamp = options.modTimestamp || ''; // blank = set @ persist + + if(_.isDate(options.modTimestamp)) { + this.modTimestamp = options.modTimestamp; + } else if(_.isString(options.modTimestamp)) { + this.modTimestamp = new Date(options.modTimestamp); + } + + //this.modTimestamp = options.modTimestamp || ''; // blank = set @ persist this.viewCount = options.viewCount || 0; this.meta = options.meta || {}; this.hashTags = options.hashTags || []; @@ -32,8 +39,9 @@ function Message(options) { return true; }; - this.createMessageTimestamp = function() { - return new Date().toISOString(); + this.getMessageTimestampString = function(ts) { + ts = ts || new Date(); + return ts.toISOString(); }; /* @@ -62,6 +70,9 @@ Message.MetaNames = { // // FidoNet: http://ftsc.org/docs/fts-0001.016 // + // Note that we do not store even kludge line identifiers as-is + // here for a) consistency, b) in case of implementation conflicts, etc. + // FidoNetCost : 'fidonet_cost', FidoNetOrigNode : 'fidonet_orig_node', FidoNetDestNode : 'fidonet_dest_node', @@ -93,6 +104,15 @@ Message.MetaNames = { }; +Message.FidoNetMetaNameMap = { + 'PID' : Message.MetaNames.FidoNetProgramID, + 'MSGID' : Message.MetaNames.FidoNetMsgID, + 'MESSAGE-ID' : Message.MetaNames.FidoNetMessageID, + 'IN-REPLY-TO' : Message.MetaNames.FidoNetInReplyTo, + 'SEEN-BY' : Message.MetaNames.FidoNetSeenBy, + 'PATH' : Message.MetaNames.FidoNetPath, +}; + Message.prototype.setLocalToUserId = function(userId) { this.meta.LocalToUserID = userId; }; @@ -118,11 +138,9 @@ Message.prototype.persist = function(cb) { }); }, function storeMessage(callback) { - var modTs = self.modTimestamp || self.createMessageTimestamp(); - msgDb.run( 'INSERT INTO message (area_id, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp) ' + - 'VALUES (?, ?, ?, ?, ?, ?, ?, ?);', [ self.areaId, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, modTs ], + 'VALUES (?, ?, ?, ?, ?, ?, ?, ?);', [ self.areaId, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ], function msgInsert(err) { if(!err) { self.messageId = this.lastID;