From e7b0e4af30dfe8a709670c82bc07f03ae6c2a982 Mon Sep 17 00:00:00 2001 From: Bryan Ashby Date: Sun, 14 Jan 2018 13:52:40 -0700 Subject: [PATCH] Add private exported + sent mail cleanup to trimMessageAreasScheduledEvent() scheduled event --- core/config.js | 5 +- core/fse.js | 2 +- core/message_area.js | 87 ++++++++++++++++++++++++--------- core/scanner_tossers/ftn_bso.js | 8 ++- 4 files changed, 75 insertions(+), 27 deletions(-) diff --git a/core/config.js b/core/config.js index f9f277a8..62ce6032 100644 --- a/core/config.js +++ b/core/config.js @@ -598,8 +598,9 @@ function getDefaultConfig() { areas : { private_mail : { - name : 'Private Mail', - desc : 'Private user to user mail/email', + name : 'Private Mail', + desc : 'Private user to user mail/email', + maxExternalSentAgeDays : 30, // max external "outbox" item age }, local_bulletin : { diff --git a/core/fse.js b/core/fse.js index 51b8dc5c..b266a9c9 100644 --- a/core/fse.js +++ b/core/fse.js @@ -990,7 +990,7 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul this.viewControllers.body.switchFocus(1); this.observeEditorEvents(); - }; + } switchToFooter() { this.viewControllers.header.setFocus(false); diff --git a/core/message_area.js b/core/message_area.js index e81c85e1..53dd3086 100644 --- a/core/message_area.js +++ b/core/message_area.js @@ -598,11 +598,11 @@ function trimMessageAreasScheduledEvent(args, cb) { LIMIT -1 OFFSET ${areaInfo.maxMessages} );`, [ areaInfo.areaTag.toLowerCase() ], - err => { + function result(err) { // no arrow func; need this if(err) { - Log.error( { areaInfo : areaInfo, err : err, type : 'maxMessages' }, 'Error trimming message area'); + Log.error( { areaInfo : areaInfo, error : err.message, type : 'maxMessages' }, 'Error trimming message area'); } else { - Log.debug( { areaInfo : areaInfo, type : 'maxMessages' }, 'Area trimmed successfully'); + Log.debug( { areaInfo : areaInfo, type : 'maxMessages', count : this.changes }, 'Area trimmed successfully'); } return cb(err); } @@ -618,21 +618,25 @@ function trimMessageAreasScheduledEvent(args, cb) { `DELETE FROM message WHERE area_tag = ? AND modified_timestamp < date('now', '-${areaInfo.maxAgeDays} days');`, [ areaInfo.areaTag ], - err => { + function result(err) { // no arrow func; need this if(err) { - Log.warn( { areaInfo : areaInfo, err : err, type : 'maxAgeDays' }, 'Error trimming message area'); + Log.warn( { areaInfo : areaInfo, error : err.message, type : 'maxAgeDays' }, 'Error trimming message area'); } else { - Log.debug( { areaInfo : areaInfo, type : 'maxAgeDays' }, 'Area trimmed successfully'); + Log.debug( { areaInfo : areaInfo, type : 'maxAgeDays', count : this.changes }, 'Area trimmed successfully'); } return cb(err); } ); } - + async.waterfall( - [ + [ function getAreaTags(callback) { - let areaTags = []; + const areaTags = []; + + // + // We use SQL here vs API such that no-longer-used tags are picked up + // msgDb.each( `SELECT DISTINCT area_tag FROM message;`, @@ -640,7 +644,11 @@ function trimMessageAreasScheduledEvent(args, cb) { if(err) { return callback(err); } - areaTags.push(row.area_tag); + + // We treat private mail special + if(!Message.isPrivateAreaTag(row.area_tag)) { + areaTags.push(row.area_tag); + } }, err => { return callback(err, areaTags); @@ -652,30 +660,26 @@ function trimMessageAreasScheduledEvent(args, cb) { // determine maxMessages & maxAgeDays per area areaTags.forEach(areaTag => { - + let maxMessages = Config.messageAreaDefaults.maxMessages; let maxAgeDays = Config.messageAreaDefaults.maxAgeDays; - + const area = getMessageAreaByTag(areaTag); // note: we don't know the conf here if(area) { - if(area.maxMessages) { - maxMessages = area.maxMessages; - } - if(area.maxAgeDays) { - maxAgeDays = area.maxAgeDays; - } + maxMessages = area.maxMessages || maxMessages; + maxAgeDays = area.maxAgeDays || maxAgeDays; } areaInfos.push( { areaTag : areaTag, maxMessages : maxMessages, maxAgeDays : maxAgeDays, - } ); + } ); }); return callback(null, areaInfos); }, - function trimAreas(areaInfos, callback) { + function trimGeneralAreas(areaInfos, callback) { async.each( areaInfos, (areaInfo, next) => { @@ -691,11 +695,50 @@ function trimMessageAreasScheduledEvent(args, cb) { }, callback ); - } + }, + function trimExternalPrivateSentMail(callback) { + // + // *External* (FTN, email, ...) outgoing is cleaned up *after export* + // if it is older than the configured |maxExternalSentAgeDays| days + // + // Outgoing externally exported private mail is: + // - In the 'private_mail' area + // - Marked exported (state_flags0 exported bit set) + // - Marked with any external flavor (we don't mark local) + // + const maxExternalSentAgeDays = _.get( + Config, + 'messageConferences.system_internal.areas.private_mail.maxExternalSentAgeDays', + 30 + ); + + msgDb.run( + `DELETE FROM message + WHERE message_id IN ( + SELECT m.message_id + FROM message m + JOIN message_meta mms + ON m.message_id = mms.message_id AND + (mms.meta_category='System' AND mms.meta_name='${Message.SystemMetaNames.StateFlags0}' AND (mms.meta_value & ${Message.StateFlags0.Exported} = ${Message.StateFlags0.Exported})) + JOIN message_meta mmf + ON m.message_id = mmf.message_id AND + (mmf.meta_category='System' AND mmf.meta_name='${Message.SystemMetaNames.ExternalFlavor}') + WHERE m.area_tag='${Message.WellKnownAreaTags.Private}' AND DATETIME('now') > DATETIME(m.modified_timestamp, '+${maxExternalSentAgeDays} days') + );`, + function results(err) { // no arrow func; need this + if(err) { + Log.warn( { error : err.message }, 'Error trimming private externally sent messages'); + } else { + Log.debug( { count : this.changes }, 'Private externally sent messages trimmed successfully'); + } + } + ); + + return callback(null); + } ], err => { return cb(err); } ); - } \ No newline at end of file diff --git a/core/scanner_tossers/ftn_bso.js b/core/scanner_tossers/ftn_bso.js index 4285884e..830e9daa 100644 --- a/core/scanner_tossers/ftn_bso.js +++ b/core/scanner_tossers/ftn_bso.js @@ -1884,7 +1884,11 @@ function FTNMessageScanTossModule() { ORDER BY message_id;` ; - async.each(Object.keys(Config.messageNetworks.ftn.areas), (areaTag, nextArea) => { + // we shouldn't, but be sure we don't try to pick up private mail here + const areaTags = Object.keys(Config.messageNetworks.ftn.areas) + .filter(areaTag => Message.WellKnownAreaTags.Private !== areaTag); + + async.each(areaTags, (areaTag, nextArea) => { const areaConfig = Config.messageNetworks.ftn.areas[areaTag]; if(!this.isAreaConfigValid(areaConfig)) { return nextArea(); @@ -1948,10 +1952,10 @@ function FTNMessageScanTossModule() { // Just like EchoMail, we additionally exclude messages with the System state_flags0 // which will be present for imported or already exported messages // - // NOTE: If StateFlags0 starts to use additional bits, we'll likely need to check them here! // // :TODO: fill out the rest of the consts here // :TODO: this statement is crazy ugly + // :TODO: this should really check for extenral "flavor" and not-already-exported state_flags0 const getNewUuidsSql = `SELECT message_id, message_uuid FROM message m