Major changes around events, event log, etc.

* User event log is now functional & attached to various events
* Add additional missing system events
* Completely re-write last_callers to have new functionality, etc.
* Events.addListenerMultipleEvents()
* New 'moduleInitialize' export for module init vs Event specific registerEvents
* Add docs on last_callers mod
This commit is contained in:
Bryan Ashby 2018-07-21 14:32:06 -06:00
parent c1ae3d88ba
commit 52585c78f0
16 changed files with 392 additions and 171 deletions

View File

@ -57,6 +57,8 @@ webSocket: {
proxied: true // X-Forwarded-Proto: https support
}
```
* The module export `registerEvents` has been deprecated. If you have a module that depends on this, use the new more generic `moduleInitialize` export instead.
* The `system.db` `user_event_log` table has been updated to include a unique session ID. Previously this table was not used, but you will need to perform a slight maintenance task before it can be properly used. After updating to `0.0.9-alpha`, please run the following: `sqlite3 db/system.db DROP TABLE user_event_log;`. The new table format will be created and used at startup.
# 0.0.7-alpha to 0.0.8-alpha

View File

@ -16,6 +16,8 @@ This document attempts to track **major** changes and additions in ENiGMA½. For
* Ability to delete from personal mailbox (finally!)
* Add ability to skip file and/or message areas during newscan. Set config.omitFileAreaTags and config.omitMessageAreaTags in new_scan configuration of your menu.hjson
* `{userName}` (sanatized) and `{userNameRaw}` as well as `{cwd}` have been added to param options when launching a door.
* Any module may now register for a system startup intiialization via the `initializeModules(initInfo, cb)` export.
* User event log is now functional. Various events a user performs will be persisted to the `system.db` `user_event_log` table for up to 90 days. An example usage can be found in the updated `last_callers` module where events are turned into Ami/X style actions. Please see `UPGRADE.md`!
## 0.0.8-alpha

View File

@ -1,11 +1,12 @@
/* jslint node: true */
'use strict';
const MenuModule = require('./menu_module.js').MenuModule;
const { MenuModule } = require('./menu_module.js');
const DropFile = require('./dropfile.js');
const Door = require('./door.js');
const theme = require('./theme.js');
const ansi = require('./ansi_term.js');
const Events = require('./events.js');
const async = require('async');
const assert = require('assert');
@ -145,6 +146,8 @@ exports.getModule = class AbracadabraModule extends MenuModule {
}
runDoor() {
Events.emit(Events.getSystemEvents().UserRunDoor, { user : this.client.user } );
this.client.term.write(ansi.resetScreen());
const exeInfo = {

View File

@ -267,6 +267,9 @@ function initialize(cb) {
function readyEvents(callback) {
return require('./events.js').startup(callback);
},
function genericModulesInit(callback) {
return require('./module_util.js').initializeModules(callback);
},
function listenConnections(callback) {
return require('./listening_server.js').startup(callback);
},
@ -286,6 +289,9 @@ function initialize(cb) {
initServices.eventScheduler = modInst;
return callback(err);
});
},
function listenUserEventsForStatLog(callback) {
return require('./stat_log.js').initUserEvents(callback);
}
],
function onComplete(err) {

View File

@ -147,7 +147,7 @@ function getDefaultConfig() {
users : {
usernameMin : 2,
usernameMax : 16, // Note that FidoNet wants 36 max
usernamePattern : '^[A-Za-z0-9~!@#$%^&*()\\-\\_+ ]+$',
usernamePattern : '^[A-Za-z0-9~!@#$%^&*()\\-\\_+ .]+$',
passwordMin : 6,
passwordMax : 128,

View File

@ -18,6 +18,7 @@ const dbs = {};
exports.getTransactionDatabase = getTransactionDatabase;
exports.getModDatabasePath = getModDatabasePath;
exports.loadDatabaseForMod = loadDatabaseForMod;
exports.getISOTimestampString = getISOTimestampString;
exports.sanatizeString = sanatizeString;
exports.initializeDatabases = initializeDatabases;
@ -55,6 +56,15 @@ function getModDatabasePath(moduleInfo, suffix) {
return paths.join(conf.config.paths.modsDb, `${full}.sqlite3`);
}
function loadDatabaseForMod(modInfo, cb) {
const db = getTransactionDatabase(new sqlite3.Database(
getModDatabasePath(modInfo),
err => {
return cb(err, db);
}
));
}
function getISOTimestampString(ts) {
ts = ts || moment();
return ts.format('YYYY-MM-DDTHH:mm:ss.SSSZ');
@ -131,10 +141,11 @@ const DB_INIT_TABLE = {
id INTEGER PRIMARY KEY,
timestamp DATETIME NOT NULL,
user_id INTEGER NOT NULL,
session_id VARCHAR NOT NULL,
log_name VARCHAR NOT NULL,
log_value VARCHAR NOT NULL,
UNIQUE(timestamp, user_id, log_name)
UNIQUE(timestamp, user_id, session_id, log_name)
);`
);

View File

@ -1,20 +1,14 @@
/* jslint node: true */
'use strict';
const paths = require('path');
const events = require('events');
const Log = require('./logger.js').log;
const SystemEvents = require('./system_events.js');
// deps
const _ = require('lodash');
const async = require('async');
const glob = require('glob');
module.exports = new class Events extends events.EventEmitter {
constructor() {
super();
this.setMaxListeners(32); // :TODO: play with this...
this.setMaxListeners(64); // :TODO: play with this...
}
getSystemEvents() {
@ -41,39 +35,21 @@ module.exports = new class Events extends events.EventEmitter {
return super.once(event, listener);
}
addListenerMultipleEvents(events, listener) {
Log.trace( { events }, 'Registring event listeners');
events.forEach(eventName => {
this.on(eventName, event => {
listener(eventName, event);
});
});
}
removeListener(event, listener) {
Log.trace( { event : event }, 'Removing listener');
return super.removeListener(event, listener);
}
startup(cb) {
async.each(require('./module_util.js').getModulePaths(), (modulePath, nextPath) => {
glob('*{.js,/*.js}', { cwd : modulePath }, (err, files) => {
if(err) {
return nextPath(err);
}
async.each(files, (moduleName, nextModule) => {
const fullModulePath = paths.join(modulePath, moduleName);
try {
const mod = require(fullModulePath);
if(_.isFunction(mod.registerEvents)) {
// :TODO: ... or just systemInit() / systemShutdown() & mods could call Events.on() / Events.removeListener() ?
mod.registerEvents(this);
}
} catch(e) {
Log.warn( { error : e }, 'Exception during module "registerEvents"');
}
return nextModule(null);
}, err => {
return nextPath(err);
});
});
}, err => {
return cb(err);
});
return cb(null);
}
};

View File

@ -2,20 +2,28 @@
'use strict';
// ENiGMA½
const MenuModule = require('./menu_module.js').MenuModule;
const ViewController = require('./view_controller.js').ViewController;
const { MenuModule } = require('./menu_module.js');
const { ViewController } = require('./view_controller.js');
const ansi = require('./ansi_term.js');
const theme = require('./theme.js');
const Message = require('./message.js');
const updateMessageAreaLastReadId = require('./message_area.js').updateMessageAreaLastReadId;
const getMessageAreaByTag = require('./message_area.js').getMessageAreaByTag;
const {
updateMessageAreaLastReadId
} = require('./message_area.js');
const { getMessageAreaByTag } = require('./message_area.js');
const User = require('./user.js');
const StatLog = require('./stat_log.js');
const stringFormat = require('./string_format.js');
const MessageAreaConfTempSwitcher = require('./mod_mixins.js').MessageAreaConfTempSwitcher;
const { isAnsi, cleanControlCodes, insert } = require('./string_util.js');
const {
MessageAreaConfTempSwitcher
} = require('./mod_mixins.js');
const {
isAnsi, cleanControlCodes,
insert
} = require('./string_util.js');
const Config = require('./config.js').get;
const { getAddressedToInfo } = require('./mail_util.js');
const Events = require('./events.js');
// deps
const async = require('async');
@ -463,12 +471,14 @@ exports.FullScreenEditorModule = exports.getModule = class FullScreenEditorModul
updateUserStats(cb) {
if(Message.isPrivateAreaTag(this.message.areaTag)) {
Events.emit(Events.getSystemEvents().UserSendMail, { user : this.client.user });
if(cb) {
cb(null);
}
return; // don't inc stats for private messages
}
Events.emit(Events.getSystemEvents().UserPostMessage, { user : this.client.user, areaTag : this.message.areaTag });
return StatLog.incrementUserStat(this.client.user, 'post_count', 1, cb);
}

View File

@ -2,27 +2,16 @@
'use strict';
// ENiGMA½
const MenuModule = require('./menu_module.js').MenuModule;
const ViewController = require('./view_controller.js').ViewController;
const { MenuModule } = require('./menu_module.js');
const StatLog = require('./stat_log.js');
const User = require('./user.js');
const stringFormat = require('./string_format.js');
const sysDb = require('./database.js').dbs.system;
// deps
const moment = require('moment');
const async = require('async');
const _ = require('lodash');
/*
Available listFormat object members:
userId
userName
location
affiliation
ts
*/
exports.moduleInfo = {
name : 'Last Callers',
desc : 'Last callers to the system',
@ -37,6 +26,9 @@ const MciCodeIds = {
exports.getModule = class LastCallersModule extends MenuModule {
constructor(options) {
super(options);
this.actionIndicators = _.get(options, 'menuConfig.config.actionIndicators', {});
this.actionIndicatorDefault = _.get(options, 'menuConfig.config.actionIndicatorDefault', '-');
}
mciReady(mciData, cb) {
@ -45,107 +37,191 @@ exports.getModule = class LastCallersModule extends MenuModule {
return cb(err);
}
const self = this;
const vc = self.viewControllers.allViews = new ViewController( { client : self.client } );
let loginHistory;
let callersView;
async.series(
async.waterfall(
[
function loadFromConfig(callback) {
const loadOpts = {
callingMenu : self,
mciMap : mciData.menu,
noInput : true,
};
vc.loadFromMenuConfig(loadOpts, callback);
},
function fetchHistory(callback) {
callersView = vc.getView(MciCodeIds.CallerList);
// fetch up
StatLog.getSystemLogEntries('user_login_history', StatLog.Order.TimestampDesc, 200, (err, lh) => {
loginHistory = lh;
if(self.menuConfig.config.hideSysOpLogin) {
const noOpLoginHistory = loginHistory.filter(lh => {
return false === User.isRootUserId(parseInt(lh.log_value)); // log_value=userId
});
//
// If we have enough items to display, or hideSysOpLogin is set to 'always',
// then set loginHistory to our filtered list. Else, we'll leave it be.
//
if(noOpLoginHistory.length >= callersView.dimens.height || 'always' === self.menuConfig.config.hideSysOpLogin) {
loginHistory = noOpLoginHistory;
}
}
//
// Finally, we need to trim up the list to the needed size
//
loginHistory = loginHistory.slice(0, callersView.dimens.height);
return callback(err);
(next) => {
this.prepViewController('callers', 0, mciData.menu, err => {
return next(err);
});
},
function getUserNamesAndProperties(callback) {
const getPropOpts = {
names : [ 'location', 'affiliation' ]
};
const dateTimeFormat = self.menuConfig.config.dateTimeFormat || 'ddd MMM DD';
async.each(
loginHistory,
(item, next) => {
item.userId = parseInt(item.log_value);
item.ts = moment(item.timestamp).format(dateTimeFormat);
User.getUserName(item.userId, (err, userName) => {
if(err) {
item.deleted = true;
return next(null);
} else {
item.userName = userName || 'N/A';
User.loadProperties(item.userId, getPropOpts, (err, props) => {
if(!err && props) {
item.location = props.location || 'N/A';
item.affiliation = item.affils = (props.affiliation || 'N/A');
} else {
item.location = 'N/A';
item.affiliation = item.affils = 'N/A';
}
return next(null);
});
}
(next) => {
this.fetchHistory( (err, loginHistory) => {
return next(err, loginHistory);
});
},
err => {
loginHistory = loginHistory.filter(lh => true !== lh.deleted);
return callback(err);
}
);
(loginHistory, next) => {
this.loadUserForHistoryItems(loginHistory, (err, updatedHistory) => {
return next(err, updatedHistory);
});
},
function populateList(callback) {
const listFormat = self.menuConfig.config.listFormat || '{userName} - {location} - {affiliation} - {ts}';
callersView.setItems(_.map(loginHistory, ce => stringFormat(listFormat, ce) ) );
(loginHistory, next) => {
const callersView = this.viewControllers.callers.getView(MciCodeIds.CallerList);
callersView.setItems(loginHistory);
callersView.redraw();
return callback(null);
return next(null);
}
],
(err) => {
err => {
if(err) {
self.client.log.error( { error : err.toString() }, 'Error loading last callers');
this.client.log.warn( { error : err.message }, 'Error loading last callers');
}
cb(err);
return cb(err);
}
);
});
}
getCollapse(conf) {
let collapse = _.get(this, conf);
collapse = collapse && collapse.match(/^([0-9]+)\s*(minutes|seconds|hours|days|months)$/);
if(collapse) {
return moment.duration(parseInt(collapse[1]), collapse[2]);
}
}
fetchHistory(cb) {
const callersView = this.viewControllers.callers.getView(MciCodeIds.CallerList);
if(!callersView || 0 === callersView.dimens.height) {
return cb(null);
}
StatLog.getSystemLogEntries(
'user_login_history',
StatLog.Order.TimestampDesc,
200, // max items to fetch - we need more than max displayed for filtering/etc.
(err, loginHistory) => {
if(err) {
return cb(err);
}
const dateTimeFormat = _.get(
this, 'menuConfig.config.dateTimeFormat', this.client.currentTheme.helpers.getDateFormat('short'));
loginHistory = loginHistory.map(item => {
try {
const historyItem = JSON.parse(item.log_value);
if(_.isObject(historyItem)) {
item.userId = historyItem.userId;
item.sessionId = historyItem.sessionId;
} else {
item.userId = historyItem; // older format
item.sessionId = '-none-';
}
} catch(e) {
return null; // we'll filter this out
}
item.timestamp = moment(item.timestamp);
return Object.assign(
item,
{
ts : moment(item.timestamp).format(dateTimeFormat)
}
);
});
const hideSysOp = _.get(this, 'menuConfig.config.sysop.hide');
const sysOpCollapse = this.getCollapse('menuConfig.config.sysop.collapse');
if(hideSysOp) {
loginHistory = loginHistory.filter(item => false === User.isRootUserId(item.userId));
} else if(sysOpCollapse) {
// :TODO: DRY op & user collapse code
const maxAge = sysOpCollapse.asSeconds();
let lastUserId;
let lastTimestamp;
loginHistory = loginHistory.filter(item => {
const op = User.isRootUserId(item.userId);
const repeat = lastUserId === item.userId;
const recent = lastTimestamp ? moment.duration(lastTimestamp.diff(item.timestamp)).seconds() < maxAge : false;
lastUserId = item.userId;
lastTimestamp = item.timestamp;
return !op || !repeat || !recent;
});
}
const userCollapse = this.getCollapse('menuConfig.config.user.collapse');
if(userCollapse) {
const maxAge = userCollapse.asSeconds();
let lastUserId;
let lastTimestamp;
loginHistory = loginHistory.filter(item => {
const repeat = lastUserId === item.userId;
const recent = lastTimestamp ? moment.duration(lastTimestamp.diff(item.timestamp)).seconds() < maxAge : false;
lastUserId = item.userId;
lastTimestamp = item.timestamp;
return !repeat || !recent;
});
}
return cb(
null,
loginHistory.slice(0, callersView.dimens.height) // trim the fat
);
}
);
}
loadUserForHistoryItems(loginHistory, cb) {
const getPropOpts = {
names : [ 'real_name', 'location', 'affiliation' ]
};
const actionIndicatorNames = _.map(this.actionIndicators, (v, k) => k);
let indicatorSumsSql;
if(actionIndicatorNames.length > 0) {
indicatorSumsSql = actionIndicatorNames.map(i => {
return `SUM(CASE WHEN log_value='${_.snakeCase(i)}' THEN 1 ELSE 0 END) AS ${i}`;
});
}
async.map(loginHistory, (item, next) => {
User.getUserName(item.userId, (err, userName) => {
if(err) {
return cb(null, null);
}
item.userName = item.text = (userName || 'N/A');
User.loadProperties(item.userId, getPropOpts, (err, props) => {
item.location = (props && props.location) || 'N/A';
item.affiliation = item.affils = (props && props.affiliation) || 'N/A';
item.realName = (props && props.real_name) || 'N/A';
if(!indicatorSumsSql) {
return next(null, item);
}
sysDb.get(
`SELECT ${indicatorSumsSql.join(', ')}
FROM user_event_log
WHERE user_id=? AND session_id=?
LIMIT 1;`,
[ item.userId, item.sessionId ],
(err, results) => {
if(_.isObject(results)) {
item.actions = '';
Object.keys(results).forEach(n => {
const indicator = results[n] > 0 ? this.actionIndicators[n] || this.actionIndicatorDefault : this.actionIndicatorDefault;
item[n] = indicator;
item.actions += indicator;
});
}
return next(null, item);
}
);
});
});
},
(err, mapped) => {
return cb(err, mapped.filter(item => item)); // remove deleted
});
}
};

View File

@ -25,7 +25,8 @@ exports.MaskEditTextView = MaskEditTextView;
// :TODO:
// * Hint, e.g. YYYY/MM/DD
// * Return values with literals in place
//
// * Tab in/out results in oddities such as cursor placement & ability to type in non-pattern chars
// * There exists some sort of condition that allows pattern position to get out of sync
function MaskEditTextView(options) {
options.acceptsFocus = miscUtil.valueWithDefault(options.acceptsFocus, true);

View File

@ -3,6 +3,7 @@
// ENiGMA½
const Config = require('./config.js').get;
const Log = require('./logger.js').log;
// deps
const fs = require('graceful-fs');
@ -10,12 +11,14 @@ const paths = require('path');
const _ = require('lodash');
const assert = require('assert');
const async = require('async');
const glob = require('glob');
// exports
exports.loadModuleEx = loadModuleEx;
exports.loadModule = loadModule;
exports.loadModulesForCategory = loadModulesForCategory;
exports.getModulePaths = getModulePaths;
exports.initializeModules = initializeModules;
function loadModuleEx(options, cb) {
assert(_.isObject(options));
@ -108,3 +111,54 @@ function getModulePaths() {
config.paths.scannerTossers,
];
}
function initializeModules(cb) {
const Events = require('./events.js');
const modulePaths = getModulePaths().concat(__dirname);
async.each(modulePaths, (modulePath, nextPath) => {
glob('*{.js,/*.js}', { cwd : modulePath }, (err, files) => {
if(err) {
return nextPath(err);
}
const ourPath = paths.join(__dirname, __filename);
async.each(files, (moduleName, nextModule) => {
const fullModulePath = paths.join(modulePath, moduleName);
if(ourPath === fullModulePath) {
return nextModule(null);
}
try {
const mod = require(fullModulePath);
if(_.isFunction(mod.moduleInitialize)) {
const initInfo = {
events : Events,
};
mod.moduleInitialize(initInfo, err => {
if(err) {
Log.warn( { error : err.message, modulePath : fullModulePath }, 'Error during "moduleInitialize"');
}
return nextModule(null);
});
} else {
return nextModule(null);
}
} catch(e) {
Log.warn( { error : e }, 'Exception during "moduleInitialize"');
return nextModule(null);
}
},
err => {
return nextPath(err);
});
});
},
err => {
return cb(err);
});
}

View File

@ -103,7 +103,11 @@ exports.getModule = class NewUserAppModule extends MenuModule {
}
// :TODO: User.create() should validate email uniqueness!
newUser.create(formData.value.password, err => {
const createUserInfo = {
password : formData.value.password,
sessionId : self.client.session.uniqueId, // used for events/etc.
};
newUser.create(createUserInfo, err => {
if(err) {
self.client.log.info( { error : err, username : formData.value.username }, 'New user creation failed');

View File

@ -252,10 +252,16 @@ class StatLog {
appendUserLogEntry(user, logName, logValue, keepDays, cb) {
sysDb.run(
`INSERT INTO user_event_log (timestamp, user_id, log_name, log_value)
VALUES (?, ?, ?, ?);`,
[ this.now, user.userId, logName, logValue ],
() => {
`INSERT INTO user_event_log (timestamp, user_id, session_id, log_name, log_value)
VALUES (?, ?, ?, ?, ?);`,
[ this.now, user.userId, user.sessionId, logName, logValue ],
err => {
if(err) {
if(cb) {
cb(err);
}
return;
}
//
// Handle keepDays
//
@ -280,6 +286,33 @@ class StatLog {
}
);
}
initUserEvents(cb) {
//
// We map some user events directly to user stat log entries such that they
// are persisted for a time.
//
const Events = require('./events.js');
const systemEvents = Events.getSystemEvents();
const interestedEvents = [
systemEvents.NewUser,
systemEvents.UserUpload, systemEvents.UserDownload,
systemEvents.UserPostMessage, systemEvents.UserSendMail,
systemEvents.UserRunDoor,
];
Events.addListenerMultipleEvents(interestedEvents, (eventName, event) => {
this.appendUserLogEntry(
event.user,
'system_event',
eventName.replace(/^codes\.l33t\.enigma\.system\./, ''), // strip package name prefix
90
);
});
return cb(null);
}
}
module.exports = new StatLog();

View File

@ -179,7 +179,7 @@ module.exports = class User {
);
}
create(password, cb) {
create(createUserInfo , cb) {
assert(0 === this.userId);
const config = Config();
@ -219,7 +219,7 @@ module.exports = class User {
);
},
function genAuthCredentials(trans, callback) {
User.generatePasswordDerivedKeyAndSalt(password, (err, info) => {
User.generatePasswordDerivedKeyAndSalt(createUserInfo.password, (err, info) => {
if(err) {
return callback(err);
}
@ -244,7 +244,12 @@ module.exports = class User {
});
},
function sendEvent(trans, callback) {
Events.emit(Events.getSystemEvents().NewUser, { user : self });
Events.emit(
Events.getSystemEvents().NewUser,
{
user : Object.assign({}, self, { sessionId : createUserInfo.sessionId } )
}
);
return callback(null, trans);
}
],

View File

@ -87,7 +87,11 @@ function userLogin(client, username, password, cb) {
},
function recordLoginHistory(callback) {
const LOGIN_HISTORY_MAX = 200; // history of up to last 200 callers
return StatLog.appendSystemLogEntry('user_login_history', user.userId, LOGIN_HISTORY_MAX, StatLog.KeepType.Max, callback);
const historyItem = JSON.stringify({
userId : user.userId,
sessionId : user.sessionId,
});
return StatLog.appendSystemLogEntry('user_login_history', historyItem, LOGIN_HISTORY_MAX, StatLog.KeepType.Max, callback);
}
],
err => {

View File

@ -0,0 +1,34 @@
---
layout: page
title: Last Callers
---
## The Last Callers Module
The built in `last_callers` module provides flexible retro last callers mod.
## Configuration
### Config Block
Available `config` block entries:
* `dateTimeFormat`: [moment.js](https://momentjs.com) style format. Defaults to current theme → system `short` format.
* `user`: User options:
* `collapse`: Collapse or roll up entries that fall within the period specified. May be a string in the form of `30 minutes`, `3 weeks`, `1 hour`, etc.
* `sysop`: Sysop options:
* `collapse`: Collapse or roll up entries that fall within the period specified. May be a string in the form of `30 minutes`, `3 weeks`, `1 hour`, etc.
* `hide`: Hide all +op logins
* `actionIndicators`: Maps user actions to indicators. For example: `userDownload` to "D". Available indicators:
* `userDownload`
* `userUpload`
* `userPostMsg`
* `userSendMail`
* `userRunDoor`
* `actionIndicatorDefault`: Default indicator when an action is not set. Defaults to "-".
### Theming
When in a list view, the following `itemFormat` object is provided:
* `userId`: User ID.
* `realName`: User's real name or "N/A".
* `ts`: Timestamp in `dateTimeFormat` format.
* `location`: User's location or "N/A".
* `affiliation` or `affils`: Users affiliations or "N/A".
* `actions`: A string built by concatenating action indicators for a users logged in session. For example, given a indincator of `userDownload` mapped to "D", the string may be "-D----". The format was made popular on Amiga style boards.