moon
/
enigma-bbs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

* Resolve issue #59: Better message UUID generation and dupe checks

This commit is contained in:
Bryan Ashby 2016-07-05 22:18:43 -06:00
parent 9e573e6810
commit f87e9917a0
4 changed files with 85 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
core/database.js
View File

@ -205,7 +205,6 @@ function createMessageBaseTables() {
END;`
);
// :TODO: need SQL to ensure cleaned up if delete from message?
dbs.message.run(
`CREATE TABLE IF NOT EXISTS message_meta (
message_id INTEGER NOT NULL,
@ -213,11 +212,12 @@ function createMessageBaseTables() {
meta_name VARCHAR NOT NULL,
meta_value VARCHAR NOT NULL,
UNIQUE(message_id, meta_category, meta_name, meta_value),
FOREIGN KEY(message_id) REFERENCES message(message_id)
FOREIGN KEY(message_id) REFERENCES message(message_id) ON DELETE CASCADE
);`
);
// :TODO: need SQL to ensure cleaned up if delete from message?
/*
dbs.message.run(
`CREATE TABLE IF NOT EXISTS hash_tag (
hash_tag_id INTEGER PRIMARY KEY,
@ -230,9 +230,10 @@ function createMessageBaseTables() {
dbs.message.run(
`CREATE TABLE IF NOT EXISTS message_hash_tag (
hash_tag_id INTEGER NOT NULL,
message_id INTEGER NOT NULL
message_id INTEGER NOT NULL,
);`
);
*/
dbs.message.run(
`CREATE TABLE IF NOT EXISTS user_message_area_last_read (

43
core/ftn_util.js
View File

@ -4,10 +4,8 @@
let Config = require('./config.js').config;
let Address = require('./ftn_address.js');
let FNV1a = require('./fnv1a.js');
let createNamedUUID = require('./uuid_util.js').createNamedUUID;
let _ = require('lodash');
let assert = require('assert');
let iconv = require('iconv-lite');
let moment = require('moment');
let uuid = require('node-uuid');
@ -18,8 +16,6 @@ let packageJson = require('../package.json');
// :TODO: Remove "Ftn" from most of these -- it's implied in the module
exports.stringToNullPaddedBuffer = stringToNullPaddedBuffer;
exports.getMessageSerialNumber = getMessageSerialNumber;
exports.createMessageUuid = createMessageUuid;
exports.createMessageUuidAlternate = createMessageUuidAlternate;
exports.getDateFromFtnDateTime = getDateFromFtnDateTime;
exports.getDateTimeString = getDateTimeString;
@ -96,45 +92,6 @@ function getDateTimeString(m) {
return m.format('DD MMM YY HH:mm:ss');
}
//
// Create a v5 named UUID given a message ID ("MSGID") and
// FTN area tag ("AREA").
//
// This is similar to CrashMail
// See https://github.com/larsks/crashmail/blob/master/crashmail/dupe.c
//
function createMessageUuid(ftnMsgId, ftnArea) {
assert(_.isString(ftnMsgId));
assert(_.isString(ftnArea));
ftnMsgId = iconv.encode(ftnMsgId, 'CP437');
ftnArea = iconv.encode(ftnArea.toUpperCase(), 'CP437');
return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnMsgId, ftnArea ] )));
};
//
// Create a v5 named UUID given a FTN area tag ("AREA"),
// create/modified date, subject, and message body
//
// This method should be used as a backup for when a MSGID is
// not available in which createMessageUuid() above should be
// used instead.
//
function createMessageUuidAlternate(ftnArea, modTimestamp, subject, msgBody) {
assert(_.isString(ftnArea));
assert(_.isDate(modTimestamp) || moment.isMoment(modTimestamp));
assert(_.isString(subject));
assert(_.isString(msgBody));
ftnArea = iconv.encode(ftnArea.toUpperCase(), 'CP437');
modTimestamp = iconv.encode(getDateTimeString(modTimestamp), 'CP437');
subject = iconv.encode(subject.toUpperCase().trim(), 'CP437');
msgBody = iconv.encode(msgBody.replace(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g, '').trim(), 'CP437');
return uuid.unparse(createNamedUUID(ENIGMA_FTN_MSGID_NAMESPACE, Buffer.concat( [ ftnArea, modTimestamp, subject, msgBody ] )));
}
function getMessageSerialNumber(messageId) {
const msSinceEnigmaEpoc = (Date.now() - Date.UTC(2016, 1, 1));
const hash = Math.abs(new FNV1a(msSinceEnigmaEpoc + messageId).value).toString(16);

41
core/message.js
View File

@ -4,21 +4,30 @@
let msgDb = require('./database.js').dbs.message;
let wordWrapText = require('./word_wrap.js').wordWrapText;
let ftnUtil = require('./ftn_util.js');
let createNamedUUID = require('./uuid_util.js').createNamedUUID;
let uuid = require('node-uuid');
let async = require('async');
let _ = require('lodash');
let assert = require('assert');
let moment = require('moment');
const iconvEncode = require('iconv-lite').encode;
module.exports = Message;
const ENIGMA_MESSAGE_UUID_NAMESPACE = uuid.parse('154506df-1df8-46b9-98f8-ebb5815baaf8');
function Message(options) {
options = options || {};
this.messageId = options.messageId || 0; // always generated @ persist
this.areaTag = options.areaTag || Message.WellKnownAreaTags.Invalid;
this.uuid = options.uuid || uuid.v1();
if(options.uuid) {
// note: new messages have UUID generated @ time of persist. See also Message.createMessageUUID()
this.uuid = options.uuid;
}
this.replyToMsgId = options.replyToMsgId || 0;
this.toUserName = options.toUserName || '';
this.fromUserName = options.fromUserName || '';
@ -110,6 +119,24 @@ Message.prototype.setLocalFromUserId = function(userId) {
this.meta.System.local_from_user_id = userId;
};
Message.createMessageUUID = function(areaTag, modTimestamp, subject, body) {
assert(_.isString(areaTag));
assert(_.isDate(modTimestamp) || moment.isMoment(modTimestamp));
assert(_.isString(subject));
assert(_.isString(body));
if(!moment.isMoment(modTimestamp)) {
modTimestamp = moment(modTimestamp);
}
areaTag = iconvEncode(areaTag.toUpperCase(), 'CP437');
modTimestamp = iconvEncode(modTimestamp.format('DD MMM YY HH:mm:ss'), 'CP437');
subject = iconvEncode(subject.toUpperCase().trim(), 'CP437');
body = iconvEncode(body.replace(/\r\n|[\n\v\f\r\x85\u2028\u2029]/g, '').trim(), 'CP437');
return uuid.unparse(createNamedUUID(ENIGMA_MESSAGE_UUID_NAMESPACE, Buffer.concat( [ areaTag, modTimestamp, subject, body ] )));
}
Message.getMessageIdByUuid = function(uuid, cb) {
msgDb.get(
`SELECT message_id
@ -330,10 +357,20 @@ Message.prototype.persist = function(cb) {
});
},
function storeMessage(callback) {
// generate a UUID for this message if required (general case)
const msgTimestamp = moment();
if(!self.uuid) {
self.uuid = Message.createMessageUUID(
self.areaTag,
msgTimestamp,
self.subject,
self.message);
}
msgDb.run(
`INSERT INTO message (area_tag, message_uuid, reply_to_message_id, to_user_name, from_user_name, subject, message, modified_timestamp)
VALUES (?, ?, ?, ?, ?, ?, ?, ?);`,
[ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(self.modTimestamp) ],
[ self.areaTag, self.uuid, self.replyToMsgId, self.toUserName, self.fromUserName, self.subject, self.message, self.getMessageTimestampString(msgTimestamp) ],
function inserted(err) { // use for this scope
if(!err) {
self.messageId = this.lastID;

72
core/scanner_tossers/ftn_bso.js
View File

@ -23,6 +23,7 @@ const assert = require('assert');
const gaze = require('gaze');
const fse = require('fs-extra');
const iconv = require('iconv-lite');
const uuid = require('node-uuid');
exports.moduleInfo = {
name : 'FTN BSO',
@ -192,11 +193,11 @@ function FTNMessageScanTossModule() {
let ext;
switch(flowType) {
case 'mail' : ext = `${exportType.toLowerCase()[0]}ut`; break;
case 'ref' : ext = `${exportType.toLowerCase()[0]}lo`; break;
case 'busy' : ext = 'bsy'; break;
case 'request' : ext = 'req'; break;
case 'requests' : ext = 'hrq'; break;
case 'mail' : ext = `${exportType.toLowerCase()[0]}ut`; break;
case 'ref' : ext = `${exportType.toLowerCase()[0]}lo`; break;
case 'busy' : ext = 'bsy'; break;
case 'request' : ext = 'req'; break;
case 'requests' : ext = 'hrq'; break;
}
return ext;
@ -307,8 +308,8 @@ function FTNMessageScanTossModule() {
// Set appropriate attribute flag for export type
//
switch(this.getExportType(options.nodeConfig)) {
case 'crash' : ftnAttribute |= ftnMailPacket.Packet.Attribute.Crash; break;
case 'hold' : ftnAttribute |= ftnMailPacket.Packet.Attribute.Hold; break;
case 'crash' : ftnAttribute |= ftnMailPacket.Packet.Attribute.Crash; break;
case 'hold' : ftnAttribute |= ftnMailPacket.Packet.Attribute.Hold; break;
// :TODO: Others?
}
@ -783,9 +784,13 @@ function FTNMessageScanTossModule() {
}
Message.getMessageIdsByMetaValue('FtnKludge', 'MSGID', message.meta.FtnKludge.REPLY, (err, msgIds) => {
if(msgIds && msgIds.length > 0) {
assert(1 === msgIds.length);
message.replyToMsgId = msgIds[0];
if(msgIds) {
// expect a single match, but dupe checking is not perfect - warn otherwise
if(1 === msgIds.length) {
message.replyToMsgId = msgIds[0];
} else {
Log.warn( { msgIds : msgIds, replyKludge : message.meta.FtnKludge.REPLY }, 'Found 2:n MSGIDs matching REPLY kludge!');
}
}
cb();
});
@ -804,24 +809,13 @@ function FTNMessageScanTossModule() {
message.areaTag = localAreaTag;
//
// If duplicates are NOT allowed in the area (the default), we need to update
// the message UUID using data available to us. Duplicate UUIDs are internally
// not allowed in our local database.
//
if(!Config.messageNetworks.ftn.areas[localAreaTag].allowDupes) {
if(self.messageHasValidMSGID(message)) {
// Update UUID with our preferred generation method
message.uuid = ftnUtil.createMessageUuid(
message.meta.FtnKludge.MSGID,
message.meta.FtnProperty.ftn_area);
} else {
// Update UUID with alternate/backup generation method
message.uuid = ftnUtil.createMessageUuidAlternate(
message.meta.FtnProperty.ftn_area,
message.modTimestamp,
message.subject,
message.message);
}
// If we *allow* dupes (disabled by default), then just generate
// a random UUID. Otherwise, don't assign the UUID just yet. It will be
// generated at persist() time and should be consistent across import/exports
//
if(Config.messageNetworks.ftn.areas[localAreaTag].allowDupes) {
// just generate a UUID & therefor always allow for dupes
message.uuid = uuid.v1();
}
callback(null);
@ -846,6 +840,16 @@ function FTNMessageScanTossModule() {
}
);
};
this.appendTearAndOrigin = function(message) {
if(message.meta.FtnProperty.ftn_tear_line) {
message.message += `\r\n${message.meta.FtnProperty.ftn_tear_line}\r\n`;
}
if(message.meta.FtnProperty.ftn_origin) {
message.message += `${message.meta.FtnProperty.ftn_origin}\r\n`;
}
};
//
// Ref. implementations on import:
@ -855,7 +859,7 @@ function FTNMessageScanTossModule() {
this.importMessagesFromPacketFile = function(packetPath, password, cb) {
let packetHeader;
const packetOpts = { keepTearAndOrigin : true };
const packetOpts = { keepTearAndOrigin : false }; // needed so we can calc message UUID without these; we'll add later
let importStats = {
areaSuccess : {}, // areaTag->count
@ -879,13 +883,21 @@ function FTNMessageScanTossModule() {
} else if('message' === entryType) {
const message = entryData;
const areaTag = message.meta.FtnProperty.ftn_area;
if(areaTag) {
//
// EchoMail
//
const localAreaTag = self.getLocalAreaTagByFtnAreaTag(areaTag);
if(localAreaTag) {
message.uuid = Message.createMessageUUID(
localAreaTag,
message.modTimestamp,
message.subject,
message.message);
self.appendTearAndOrigin(message);
self.importEchoMailToArea(localAreaTag, packetHeader, message, err => {
if(err) {
// bump area fail stats