moon
/
enigma-bbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Changed scan check to use new System state_flags0 meta to skip already imported/exported msgs 

* Use moment.js for Message modTimestamp
* Remove user_message_status stuff
* Add REPLY kludge support @ export
* Use TID vs PID kludge @ export (spec)
* Start work on @immediate - nearly complete
This commit is contained in:
Bryan Ashby 2016-03-14 22:29:41 -06:00
parent a787a2eab3
commit 964c53ea9f
4 changed files with 165 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
core/database.js
View File

@ -204,17 +204,6 @@ function createMessageBaseTables() {
');'
);
// :TODO: Not currently used
dbs.message.run(
'CREATE TABLE IF NOT EXISTS user_message_status (' +
' user_id INTEGER NOT NULL,' +
' message_id INTEGER NOT NULL,' +
' status INTEGER NOT NULL,' +
' UNIQUE(user_id, message_id, status),' +
' FOREIGN KEY(user_id) REFERENCES user(id)' +
');'
);
dbs.message.run(
`CREATE TABLE IF NOT EXISTS message_area_last_scan (
scan_toss VARCHAR NOT NULL,

3
core/ftn_util.js
View File

@ -68,7 +68,8 @@ function getDateFromFtnDateTime(dateTime) {
// "27 Feb 15 00:00:03"
//
// :TODO: Use moment.js here
return (new Date(Date.parse(dateTime))).toISOString();
return moment(Date.parse(dateTime)); // Date.parse() allows funky formats
// return (new Date(Date.parse(dateTime))).toISOString();
}
function getDateTimeString(m) {

105
core/message.js
View File

@ -1,14 +1,15 @@
/* jslint node: true */
'use strict';
var msgDb = require('./database.js').dbs.message;
var wordWrapText = require('./word_wrap.js').wordWrapText;
var ftnUtil = require('./ftn_util.js');
let msgDb = require('./database.js').dbs.message;
let wordWrapText = require('./word_wrap.js').wordWrapText;
let ftnUtil = require('./ftn_util.js');
var uuid = require('node-uuid');
var async = require('async');
var _ = require('lodash');
var assert = require('assert');
let uuid = require('node-uuid');
let async = require('async');
let _ = require('lodash');
let assert = require('assert');
let moment = require('moment');
module.exports = Message;
@ -24,10 +25,10 @@ function Message(options) {
this.subject = options.subject || '';
this.message = options.message || '';
if(_.isDate(options.modTimestamp)) {
this.modTimestamp = options.modTimestamp;
if(_.isDate(options.modTimestamp) || moment.isMoment(options.modTimestamp)) {
this.modTimestamp = moment(options.modTimestamp);
} else if(_.isString(options.modTimestamp)) {
this.modTimestamp = new Date(options.modTimestamp);
this.modTimestamp = moment(options.modTimestamp);
}
this.viewCount = options.viewCount || 0;
@ -44,11 +45,8 @@ function Message(options) {
this.meta = options.meta;
}
// this.meta = options.meta || {};
this.hashTags = options.hashTags || [];
var self = this;
this.isValid = function() {
// :TODO: validate as much as possible
return true;
@ -59,8 +57,8 @@ function Message(options) {
};
this.getMessageTimestampString = function(ts) {
ts = ts || new Date();
return ts.toISOString();
ts = ts || moment();
return ts.format('YYYY-MM-DDTHH:mm:ss.SSSZ');
};
}
@ -70,13 +68,7 @@ Message.WellKnownAreaTags = {
Bulletin : 'local_bulletin',
};
// :TODO: This doesn't seem like a good way to go -- perhaps only for local/user2user, or just use
// a system similar to the "last read" for general areas
Message.Status = {
New : 0,
Read : 1,
};
// :TODO: FTN stuff really doesn't belong here - move it elsewhere and/or just use the names directly when needed
Message.MetaCategories = {
System : 1, // ENiGMA1/2 stuff
FtnProperty : 2, // Various FTN network properties, ftn_cost, ftn_origin, ...
@ -86,6 +78,13 @@ Message.MetaCategories = {
Message.SystemMetaNames = {
LocalToUserID : 'local_to_user_id',
LocalFromUserID : 'local_from_user_id',
StateFlags0 : 'state_flags0', // See Message.StateFlags0
};
Message.StateFlags0 = {
None : 0x00000000,
Imported : 0x00000001, // imported from foreign system
Exported : 0x00000002, // exported to foreign system
};
Message.FtnPropertyNames = {
@ -152,7 +151,31 @@ Message.getMessageIdsByMetaValue = function(category, name, value, cb) {
);
};
Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name, cb) {
Message.getMetaValuesByMessageId = function(messageId, category, name, cb) {
const sql =
`SELECT meta_value
FROM message_meta
WHERE message_id = ? AND meta_category = ? AND meta_name = ?;`;
msgDb.all(sql, [ messageId, category, name ], (err, rows) => {
if(err) {
return cb(err);
}
if(0 === rows.length) {
return cb(new Error('No value for category/name'));
}
// single values are returned without an array
if(1 === rows.length) {
return cb(null, rows[0].meta_value);
}
cb(null, rows.map(r => r.meta_value)); // map to array of values only
});
};
Message.getMetaValuesByMessageUuid = function(uuid, category, name, cb) {
async.waterfall(
[
function getMessageId(callback) {
@ -160,32 +183,14 @@ Message.loadMetaValueForCategegoryByMessageUuid = function(uuid, category, name,
callback(err, messageId);
});
},
function getMetaValue(messageId, callback) {
const sql =
`SELECT meta_value
FROM message_meta
WHERE message_id = ? AND message_category = ? AND meta_name = ?;`;
msgDb.all(sql, [ messageId, category, name ], (err, rows) => {
if(err) {
return callback(err);
}
if(0 === rows.length) {
return callback(new Error('No value for category/name'));
}
// single values are returned without an array
if(1 === rows.length) {
return callback(null, rows[0].meta_value);
}
callback(null, rows.map(r => r.meta_value));
function getMetaValues(messageId, callback) {
Message.getMetaValuesByMessageId(messageId, category, name, (err, values) => {
callback(err, values);
});
}
],
(err, value) => {
cb(err, value);
(err, values) => {
cb(err, values);
}
);
};
@ -254,7 +259,7 @@ Message.prototype.load = function(options, cb) {
self.fromUserName = msgRow.from_user_name;
self.subject = msgRow.subject;
self.message = msgRow.message;
self.modTimestamp = msgRow.modified_timestamp;
self.modTimestamp = moment(msgRow.modified_timestamp);
self.viewCount = msgRow.view_count;
callback(err);
@ -269,12 +274,6 @@ Message.prototype.load = function(options, cb) {
function loadHashTags(callback) {
// :TODO:
callback(null);
},
function loadMessageStatus(callback) {
if(options.user) {
// :TODO: Load from user_message_status
}
callback(null);
}
],
function complete(err) {

127
core/scanner_tossers/ftn_bso.js
View File

@ -30,12 +30,11 @@ exports.moduleInfo = {
/*
:TODO:
* Add bundle timer (arcmail)
* Queue until time elapses / fixed time interval
* Pakcets append until >= max byte size
* [if arch type is not empty): Packets -> bundle until max byte size -> repeat process
* Support (approx) max bundle size
* Support NetMail
* NetMail needs explicit isNetMail() check
* NetMail filename / location / etc. is still unknown - need to post on groups & get real answers
*/
exports.getModule = FTNMessageScanTossModule;
@ -242,9 +241,7 @@ function FTNMessageScanTossModule() {
message.meta.FtnProperty.ftn_dest_node = options.destAddress.node;
message.meta.FtnProperty.ftn_orig_network = options.network.localAddress.net;
message.meta.FtnProperty.ftn_dest_network = options.destAddress.net;
// :TODO: attr1 & 2
message.meta.FtnProperty.ftn_cost = 0;
message.meta.FtnProperty.ftn_tear_line = ftnUtil.getTearLine();
// :TODO: Need an explicit isNetMail() check
@ -309,14 +306,14 @@ function FTNMessageScanTossModule() {
message.meta.FtnKludge.TZUTC = ftnUtil.getUTCTimeZoneOffset();
if(!message.meta.FtnKludge.PID) {
message.meta.FtnKludge.PID = ftnUtil.getProductIdentifier();
}
if(!message.meta.FtnKludge.TID) {
// :TODO: Create TID!!
//message.meta.FtnKludge.TID =
}
//
// According to FSC-0046:
//
// "When a Conference Mail processor adds a TID to a message, it may not
// add a PID. An existing TID should, however, be replaced. TIDs follow
// the same format used for PIDs, as explained above."
//
message.meta.FtnKludge.TID = ftnUtil.getProductIdentifier();
//
// Determine CHRS and actual internal encoding name
@ -332,12 +329,31 @@ function FTNMessageScanTossModule() {
options.encoding = encoding; // save for later
message.meta.FtnKludge.CHRS = ftnUtil.getCharacterSetIdentifierByEncoding(encoding);
// :TODO: FLAGS kludge?
// :TODO: Add REPLY kludge if appropriate
};
this.setReplyKludgeFromReplyToMsgId = function(message, cb) {
//
// Look up MSGID kludge for |message.replyToMsgId|, if any.
// If found, we can create a REPLY kludge with the previously
// discovered MSGID.
//
if(0 === message.replyToMsgId) {
return cb(null); // nothing to do
}
Message.getMetaValuesByMessageId(message.replyToMsgId, 'FtnKludge', 'MSGID', (err, msgIdVal) => {
assert(_.isString(msgIdVal));
if(!err) {
// got a MSGID - create a REPLY
message.meta.FtnKludge.REPLY = msgIdVal;
}
cb(null); // this method always passes
});
};
// :TODO: change to something like isAreaConfigValid
// check paths, Addresses, etc.
this.isAreaConfigValid = function(areaConfig) {
if(!_.isString(areaConfig.tag) || !_.isString(areaConfig.network)) {
@ -415,7 +431,7 @@ function FTNMessageScanTossModule() {
});
};
this.getNodeConfigKeyForUplink = function(uplink) {
this.getNodeConfigKeyByAddress = function(uplink) {
// :TODO: sort by least # of '*' & take top?
const nodeKey = _.filter(Object.keys(this.moduleConfig.nodes), addr => {
return Address.fromString(addr).isPatternMatch(uplink);
@ -457,11 +473,17 @@ function FTNMessageScanTossModule() {
},
function loadMessage(callback) {
message.load( { uuid : msgUuid }, err => {
if(!err) {
self.prepareMessage(message, exportOpts);
if(err) {
return callback(err);
}
// General preperation
self.prepareMessage(message, exportOpts);
self.setReplyKludgeFromReplyToMsgId(message, err => {
callback(err);
});
});
},
function createNewPacket(callback) {
if(currPacketSize >= self.moduleConfig.packetTargetByteSize) {
@ -502,7 +524,12 @@ function FTNMessageScanTossModule() {
}
callback(null);
},
function updateStoredMeta(callback) {
function storeStateFlags0Meta(callback) {
message.persistMetaValue('System', 'state_flags0', Message.StateFlags0.Exported.toString(), err => {
callback(err);
});
},
function storeMsgIdMeta(callback) {
//
// We want to store some meta as if we had imported
// this message for later reference
@ -577,7 +604,7 @@ function FTNMessageScanTossModule() {
this.exportMessagesToUplinks = function(messageUuids, areaConfig, cb) {
async.each(areaConfig.uplinks, (uplink, nextUplink) => {
const nodeConfigKey = self.getNodeConfigKeyForUplink(uplink);
const nodeConfigKey = self.getNodeConfigKeyByAddress(uplink);
if(!nodeConfigKey) {
return nextUplink();
}
@ -657,7 +684,6 @@ function FTNMessageScanTossModule() {
const newPath = paths.join(outgoingDir, paths.basename(oldPath));
fs.rename(oldPath, newPath, err => {
if(err) {
// :TODO: Log this - but move on to the next file
Log.warn(
{ oldPath : oldPath, newPath : newPath },
'Failed moving temporary bundle file!');
@ -756,6 +782,10 @@ function FTNMessageScanTossModule() {
});
},
function persistImport(callback) {
// mark as imported
message.meta.System.StateFlags0 = Message.StateFlags0.Imported.toString();
// save to disc
message.persist(err => {
callback(err);
});
@ -783,6 +813,8 @@ function FTNMessageScanTossModule() {
if(!_.isString(localNetworkName)) {
next(new Error('No configuration for this packet'));
} else {
// :TODO: password needs validated - need to determine if it will use the same node config (which can have wildcards) or something else?!
next(null);
}
@ -1011,6 +1043,10 @@ FTNMessageScanTossModule.prototype.startup = function(cb) {
if(exportSchedule.watchFile) {
// :TODO: monitor file for changes/existance with gaze
}
if(_.isBoolean(exportSchedule.immediate)) {
this.exportImmediate = exportSchedule.immediate;
}
}
const importSchedule = this.parseScheduleString(this.moduleConfig.schedule.import);
@ -1043,6 +1079,10 @@ FTNMessageScanTossModule.prototype.shutdown = function(cb) {
this.exportTimer.clear();
}
if(this.importTimer) {
this.importTimer.clear();
}
//
// Clean up temp dir/files we created
//
@ -1088,6 +1128,7 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) {
// Additionally exclude messages that have a ftn_attr_flags FtnProperty meta
// as those came via import!
//
/*
const getNewUuidsSql =
`SELECT message_id, message_uuid
FROM message m
@ -1096,6 +1137,23 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) {
FROM message_meta
WHERE message_id = m.message_id AND meta_category = 'FtnProperty' AND meta_name = 'ftn_attr_flags') = 0
ORDER BY message_id;`;
*/
//
// Select all messages with a |message_id| > |lastScanId|.
// Additionally exclude messages with the System state_flags0 which will be present for
// imported or already exported messages
//
// NOTE: If StateFlags0 starts to use additional bits, we'll likely need to check them here!
//
const getNewUuidsSql =
`SELECT message_id, message_uuid
FROM message m
WHERE area_tag = ? AND message_id > ? AND
(SELECT COUNT(message_id)
FROM message_meta
WHERE message_id = m.message_id AND meta_category = 'System' AND meta_name = 'state_flags0') = 0
ORDER BY message_id;`;
var self = this;
@ -1156,5 +1214,26 @@ FTNMessageScanTossModule.prototype.performExport = function(cb) {
FTNMessageScanTossModule.prototype.record = function(message) {
//
// :TODO: If @immediate, we should do something here!
// This module works off schedules, but we do support @immediate for export
//
if(true !== this.exportImmediate || !this.hasValidConfiguration()) {
return;
}
if(message.isPrivate()) {
// :TODO: support NetMail
} else if(message.areaTag) {
const areaConfig = Config.messageNetworks.ftn.areas[message.areaTag];
if(!this.isAreaConfigValid(areaConfig)) {
return;
}
// :TODO: We must share a check to block export with schedule/timer when this is exporting...
// :TODO: Messages must be marked as "exported" else we will export this particular message again later @ schedule/timer
// ...if getNewUuidsSql in performExport checks for MSGID existence also we can omit already-exported messages
this.exportMessagesToUplinks( [ message.uuid ], areaConfig, err => {
});
}
};