From 801771d7b1532d2eb5bd197f0b7663fc65464a2c Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Thu, 16 Jul 2015 22:57:08 -0600 Subject: [PATCH] * Metadata now has a category. That is, meta = {message_id, cat, name, value} * Kludges are now stored as-is, so any kludge known or not can be captured --- core/database.js | 13 ++--- core/ftn_mail_packet.js | 50 +++++++----------- core/message.js | 113 ++++++++++++++++++++-------------------- package.json | 2 +- 4 files changed, 83 insertions(+), 95 deletions(-) diff --git a/core/database.js b/core/database.js index b5c20ef8..a77836dd 100644 --- a/core/database.js +++ b/core/database.js @@ -130,12 +130,13 @@ function createMessageBaseTables() { ); dbs.message.run( - 'CREATE TABLE IF NOT EXISTS message_meta (' + - ' message_id INTEGER NOT NULL,' + - ' meta_name VARCHAR NOT NULL,' + - ' meta_value VARCHAR NOT NULL,' + - ' UNIQUE(message_id, meta_name, meta_value),' + - ' FOREIGN KEY(message_id) REFERENCES message(message_id)' + + 'CREATE TABLE IF NOT EXISTS message_meta (' + + ' message_id INTEGER NOT NULL,' + + ' meta_category VARCHAR NOT NULL,' + + ' meta_name VARCHAR NOT NULL,' + + ' meta_value VARCHAR NOT NULL,' + + ' UNIQUE(message_id, meta_category, meta_name, meta_value),' + + ' FOREIGN KEY(message_id) REFERENCES message(message_id)' + ');' ); diff --git a/core/ftn_mail_packet.js b/core/ftn_mail_packet.js index b70a5b14..0777650d 100644 --- a/core/ftn_mail_packet.js +++ b/core/ftn_mail_packet.js @@ -118,32 +118,24 @@ function FTNMailPacket(options) { }); }; - self.getKludgeLineMessageMeta = function(msgBody) { - // - // For all well known |msgBody.kludgeLines|, create metadata entries. - // - // Example: 'MSGID' = [ ... ] -> 'fidonet_msg_id' = [ ... ] - // - var kludgeMeta = {}; - - var mapName; - // :TODO: there is probably a nicer way: - Object.keys(msgBody.kludgeLines).forEach(function kludgeName(kn) { - mapName = Message.FidoNetMetaNameMap[kn]; - if(mapName) { - kludgeMeta[mapName] = msgBody.kludgeLines[kn]; - } - }); - - return kludgeMeta; - }; self.getMessageMeta = function(msgBody) { - - var meta = self.getKludgeLineMessageMeta(msgBody); + var meta = { + ftn_kludge : msgBody.kludgeLines, + ftn_control : {}, + }; - if(msgBody.seenBy) { - meta[Message.MetaNames.FidoNetSeenBy] = msgBody.seenBy; + if(msgBody.tearLine) { + meta.ftn_control.ftn_tear_line = [ msgBody.tearLine ]; + } + if(msgBody.seenBy.length > 0) { + meta.ftn_control.ftn_seen_by = msgBody.seenBy; + } + if(msgBody.area) { + meta.ftn_control.ftn_area = [ msgBody.area ]; + } + if(msgBody.originLine) { + meta.ftn_control.ftn_origin = [ msgBody.originLine ]; } return meta; @@ -181,8 +173,8 @@ function FTNMailPacket(options) { var msgBody = { message : [], + kludgeLines : {}, // -> [ value1, value2, ... ] seenBy : [], - kludgeLines : {}, // upper(kludge) -> [ value, ... ] }; var preOrigin = true; @@ -192,11 +184,7 @@ function FTNMailPacket(options) { kludgeParts[0] = kludgeParts[0].toUpperCase(); kludgeParts[1] = kludgeParts[1].trim(); - if(!(kludgeParts[0] in msgBody.kludgeLines)) { - msgBody.kludgeLines[kludgeParts[0]] = [ kludgeParts[1] ]; - } else { - msgBody.kludgeLines[kludgeParts[0]].push(kludgeParts[1]); - } + (msgBody.kludgeLines[kludgeParts[0]] = msgBody.kludgeLines[kludgeParts[0]] || []).push(kludgeParts[1]); } msgLines.forEach(function nextLine(line) { @@ -210,7 +198,7 @@ function FTNMailPacket(options) { msgBody.area = line.substring(line.indexOf(':') + 1).trim(); } else if(_.startsWith(line, '--- ')) { // Tag lines are tracked allowing for specialized display/etc. - msgBody.tagLine = line; + msgBody.tearLine = line; } else if(/[ ]{1,2}(\* )?Origin\: /.test(line)) { // To spec is " * Origin: ..." msgBody.originLine = line; preOrigin = false; @@ -458,7 +446,7 @@ mailPacket.on('message', function msgParsed(msg) { console.log(msg); }); -mailPacket.read( { packetPath : '/home/bashby/ownCloud/Projects/ENiGMA½ BBS/FTNPackets/BAD_BNDL.007' } ); +mailPacket.read( { packetPath : '/home/nuskooler/ownCloud/Projects/ENiGMA½ BBS/FTNPackets/BAD_BNDL.007' } ); /* mailPacket.parse('/home/nuskooler/ownCloud/Projects/ENiGMA½ BBS/FTNPackets/BAD_BNDL.007', function parsed(err, messages) { diff --git a/core/message.js b/core/message.js index 811229d6..7c344845 100644 --- a/core/message.js +++ b/core/message.js @@ -27,8 +27,16 @@ function Message(options) { this.modTimestamp = new Date(options.modTimestamp); } - //this.modTimestamp = options.modTimestamp || ''; // blank = set @ persist this.viewCount = options.viewCount || 0; + + this.meta = { + system : {}, // we'll always have this one + }; + + if(_.isObject(options.meta)) { + _.defaultsDeep(this.meta, options.meta); + } + this.meta = options.meta || {}; this.hashTags = options.hashTags || []; @@ -66,59 +74,48 @@ Message.WellKnownAreaIds = { Private : 1, }; -Message.MetaNames = { - // - // FidoNet: http://ftsc.org/docs/fts-0001.016 - // - // Note that we do not store even kludge line identifiers as-is - // here for a) consistency, b) in case of implementation conflicts, etc. - // - FidoNetCost : 'fidonet_cost', - FidoNetOrigNode : 'fidonet_orig_node', - FidoNetDestNode : 'fidonet_dest_node', - FidoNetOrigNetwork : 'fidonet_orig_network', - FidoNetDestNetwork : 'fidonet_dest_network', - FidoNetOrigZone : 'fidonet_orig_zone', - FidoNetDestZone : 'fidonet_dest_zone', - FidoNetOrigPoint : 'fidonet_orig_point', - FidoNetDestPoint : 'fidonet_dest_point', - FidoNetAttribute : 'fidonet_attribute', - - FidoNetProgramID : 'fidonet_program_id', // "PID" http://ftsc.org/docs/fsc-0046.005 - - FidoNetMsgID : 'fidonet_msg_id', // "MSGID" http://ftsc.org/docs/fsc-0070.002 - - FidoNetMessageID : 'fidonet_message_id', // "MESSAGE-ID" http://ftsc.org/docs/fsc-0030.001 - FidoNetInReplyTo : 'fidonet_in_reply_to', // "IN-REPLY-TO" http://ftsc.org/docs/fsc-0030.001 - - FidoNetTearLineBanner : 'fidonet_tear_line_banner', // FTN style tear line http://ftsc.org/docs/fts-0004.001 - FidoNetOrigin : 'fidonet_origin', // FTN style "* Origin..." http://ftsc.org/docs/fts-0004.001 - FidoNetSeenBy : 'fidonet_seen_by', // FTN style "SEEN-BY" http://ftsc.org/docs/fts-0004.001 - FidoNetPath : 'fidonet_path', // FTN style "PATH" http://ftsc.org/docs/fts-0004.001 +Message.MetaCategories = { + System : 'system', + FtnProperty : 'ftn_prop', // + FtnKludge : 'ftn_kludge', // PATH, MSGID, ... + FtnControl : 'ftn_control' // ftn_area, ftn_origin, ... +}; +Message.SystemMetaNames = { LocalToUserID : 'local_to_user_id', LocalFromUserID : 'local_from_user_id', - - // :TODO: Search further: - // https://www.npmjs.com/package/fidonet-jam - }; -Message.FidoNetMetaNameMap = { - 'PID' : Message.MetaNames.FidoNetProgramID, - 'MSGID' : Message.MetaNames.FidoNetMsgID, - 'MESSAGE-ID' : Message.MetaNames.FidoNetMessageID, - 'IN-REPLY-TO' : Message.MetaNames.FidoNetInReplyTo, - 'SEEN-BY' : Message.MetaNames.FidoNetSeenBy, - 'PATH' : Message.MetaNames.FidoNetPath, +Message.FtnPropertyNames = { + FtnCost : 'ftn_cost', + FtnOrigNode : 'ftn_orig_node', + FtnDestNode : 'ftn_dest_node', + FtnOrigNetwork : 'ftn_orig_network', + FtnDestNetwork : 'ftn_dest_network', + FtnOrigZone : 'ftn_orig_zone', + FtnDestZone : 'ftn_dest_zone', + FtnOrigPoint : 'ftn_orig_point', + FtnDestPoint : 'ftn_dest_point', + FtnAttribute : 'ftn_attribute', }; +// Note: kludges are stored with their names as-is + +Message.FtnControlNames = { + FtnTearLine : 'ftn_tear_line', // http://ftsc.org/docs/fts-0004.001 + FtnOrigin : 'ftn_origin', // http://ftsc.org/docs/fts-0004.001 + FtnArea : 'ftn_area', // http://ftsc.org/docs/fts-0004.001 + FtnSeenBy : 'ftn_seen_by', // http://ftsc.org/docs/fts-0004.001 +}; + +// meta: { 'categoryName' : { name : value, name : value, ... } } + Message.prototype.setLocalToUserId = function(userId) { - this.meta.LocalToUserID = userId; + this.meta.system.local_to_user_id = userId; }; Message.prototype.setLocalFromUserId = function(userId) { - this.meta.LocalFromUserID = userId; + this.meta.system.local_from_user_id = userId; }; Message.prototype.persist = function(cb) { @@ -156,22 +153,24 @@ Message.prototype.persist = function(cb) { } else { // :TODO: this should be it's own method such that meta can be updated var metaStmt = msgDb.prepare( - 'INSERT INTO message_meta (message_id, meta_name, meta_value) ' + - 'VALUES (?, ?, ?);'); + 'INSERT INTO message_meta (message_id, meta_category, meta_name, meta_value) ' + + 'VALUES (?, ?, ?, ?);'); - async.each(Object.keys(self.meta), function meta(metaName, next) { - metaStmt.run(self.messageId, metaName, self.meta[metaName], function inserted(err) { - next(err); - }); - }, function complete(err) { - if(!err) { - metaStmt.finalize(function finalized() { - callback(null); + for(var metaCategroy in self.meta) { + async.each(Object.keys(self.meta[metaCategroy]), function meta(metaName, next) { + metaStmt.run(self.messageId, metaCategroy, metaName, self.meta[metaCategroy][metaName], function inserted(err) { + next(err); }); - } else { - callback(err); - } - }); + }, function complete(err) { + if(!err) { + metaStmt.finalize(function finalized() { + callback(null); + }); + } else { + callback(err); + } + }); + } } }, function storeHashTags(callback) { diff --git a/package.json b/package.json index 8ee433c6..1fa2439e 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "buffers" : "0.1.x", "bunyan" : "1.3.x", "iconv-lite" : "0.4.x", - "lodash" : "3.7.x", + "lodash" : "3.10.x", "sqlite3" : "3.0.x", "ssh2" : "0.4.x", "strip-json-comments" : "1.0.x",