/* jslint node: true */ 'use strict'; // ENiGMA½ const conf = require('./config'); // deps const sqlite3 = require('sqlite3'); const sqlite3Trans = require('sqlite3-trans'); const paths = require('path'); const async = require('async'); const _ = require('lodash'); const assert = require('assert'); const moment = require('moment'); // database handles const dbs = {}; exports.getTransactionDatabase = getTransactionDatabase; exports.getModDatabasePath = getModDatabasePath; exports.loadDatabaseForMod = loadDatabaseForMod; exports.getISOTimestampString = getISOTimestampString; exports.sanitizeString = sanitizeString; exports.initializeDatabases = initializeDatabases; exports.scheduledEventOptimizeDatabases = scheduledEventOptimizeDatabases; exports.dbs = dbs; function getTransactionDatabase(db) { return sqlite3Trans.wrap(db); } function getDatabasePath(name) { const Config = conf.get(); return paths.join(Config.paths.db, `${name}.sqlite3`); } function getModDatabasePath(moduleInfo, suffix) { // // Mods that use a database are stored in Config.paths.modsDb (e.g. enigma-bbs/db/mods) // We expect that moduleInfo defines packageName which will be the base of the modules // filename. An optional suffix may be supplied as well. // const HOST_RE = /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]*[A-Za-z0-9])$/; assert(_.isObject(moduleInfo)); assert(_.isString(moduleInfo.packageName), 'moduleInfo must define "packageName"!'); let full = moduleInfo.packageName; if (suffix) { full += `.${suffix}`; } assert( full.split('.').length > 1 && HOST_RE.test(full), 'packageName must follow Reverse Domain Name Notation - https://en.wikipedia.org/wiki/Reverse_domain_name_notation' ); const Config = conf.get(); return paths.join(Config.paths.modsDb, `${full}.sqlite3`); } function loadDatabaseForMod(modInfo, cb) { const db = getTransactionDatabase( new sqlite3.Database(getModDatabasePath(modInfo), err => { return cb(err, db); }) ); } function getISOTimestampString(ts) { ts = ts || moment(); if (!moment.isMoment(ts)) { if (_.isString(ts)) { ts = ts.replace(/\//g, '-'); } ts = moment(ts); } return ts.format('YYYY-MM-DDTHH:mm:ss.SSSZ'); } function sanitizeString(s) { return s.replace(/[\0\x08\x09\x1a\n\r"'\\%]/g, c => { // eslint-disable-line no-control-regex switch (c) { case '\0': return '\\0'; case '\x08': return '\\b'; case '\x09': return '\\t'; case '\x1a': return '\\z'; case '\n': return '\\n'; case '\r': return '\\r'; case '"': case "'": return `${c}${c}`; case '\\': case '%': return `\\${c}`; } }); } function initializeDatabases(cb) { async.eachSeries( ['system', 'user', 'message', 'file', 'activitypub'], (dbName, next) => { dbs[dbName] = sqlite3Trans.wrap( new sqlite3.Database(getDatabasePath(dbName), err => { if (err) { return cb(err); } dbs[dbName].serialize(() => { DB_INIT_TABLE[dbName](() => { return next(null); }); }); }) ); }, err => { return cb(err); } ); } function enableForeignKeys(db) { db.run('PRAGMA foreign_keys = ON;'); } const DB_INIT_TABLE = { system: cb => { enableForeignKeys(dbs.system); // Various stat/event logging - see stat_log.js dbs.system.run( `CREATE TABLE IF NOT EXISTS system_stat ( stat_name VARCHAR PRIMARY KEY NOT NULL, stat_value VARCHAR NOT NULL );` ); dbs.system.run( `CREATE TABLE IF NOT EXISTS system_event_log ( id INTEGER PRIMARY KEY, timestamp DATETIME NOT NULL, log_name VARCHAR NOT NULL, log_value VARCHAR NOT NULL, UNIQUE(timestamp, log_name) );` ); dbs.system.run( `CREATE TABLE IF NOT EXISTS user_event_log ( id INTEGER PRIMARY KEY, timestamp DATETIME NOT NULL, user_id INTEGER NOT NULL, session_id VARCHAR NOT NULL, log_name VARCHAR NOT NULL, log_value VARCHAR NOT NULL, UNIQUE(timestamp, user_id, session_id, log_name) );` ); return cb(null); }, user: cb => { enableForeignKeys(dbs.user); dbs.user.run( `CREATE TABLE IF NOT EXISTS user ( id INTEGER PRIMARY KEY, user_name VARCHAR NOT NULL, UNIQUE(user_name) );` ); // :TODO: create FK on delete/etc. dbs.user.run( `CREATE TABLE IF NOT EXISTS user_property ( user_id INTEGER NOT NULL, prop_name VARCHAR NOT NULL, prop_value VARCHAR, UNIQUE(user_id, prop_name), FOREIGN KEY(user_id) REFERENCES user(id) ON DELETE CASCADE );` ); dbs.user.run( `CREATE INDEX IF NOT EXISTS user_property_id_and_name_index0 ON user_property (user_id, prop_name);` ); dbs.user.run( `CREATE TABLE IF NOT EXISTS user_group_member ( group_name VARCHAR NOT NULL, user_id INTEGER NOT NULL, UNIQUE(group_name, user_id) );` ); dbs.user.run( `CREATE TABLE IF NOT EXISTS user_achievement ( user_id INTEGER NOT NULL, achievement_tag VARCHAR NOT NULL, timestamp DATETIME NOT NULL, match VARCHAR NOT NULL, title VARCHAR NOT NULL, text VARCHAR NOT NULL, points INTEGER NOT NULL, UNIQUE(user_id, achievement_tag, match), FOREIGN KEY(user_id) REFERENCES user(id) ON DELETE CASCADE );` ); // // Table for temporary tokens, generally used for e.g. 'outside' // access such as email links. // Examples: PW reset, enabling of 2FA/OTP, etc. // dbs.user.run( `CREATE TABLE IF NOT EXISTS user_temporary_token ( user_id INTEGER NOT NULL, token VARCHAR NOT NULL, token_type VARCHAR NOT NULL, timestamp DATETIME NOT NULL, UNIQUE(user_id, token_type), FOREIGN KEY(user_id) REFERENCES user(id) ON DELETE CASCADE );` ); return cb(null); }, message: cb => { enableForeignKeys(dbs.message); dbs.message.run( `CREATE TABLE IF NOT EXISTS message ( message_id INTEGER PRIMARY KEY, area_tag VARCHAR NOT NULL, message_uuid VARCHAR(36) NOT NULL, reply_to_message_id INTEGER, to_user_name VARCHAR NOT NULL, from_user_name VARCHAR NOT NULL, subject, /* FTS @ message_fts */ message, /* FTS @ message_fts */ modified_timestamp DATETIME NOT NULL, view_count INTEGER NOT NULL DEFAULT 0, UNIQUE(message_uuid) );` ); dbs.message.run( `CREATE INDEX IF NOT EXISTS message_by_area_tag_index ON message (area_tag);` ); dbs.message.run( `CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts4 ( content="message", subject, message );` ); dbs.message.run( `CREATE TRIGGER IF NOT EXISTS message_before_update BEFORE UPDATE ON message BEGIN DELETE FROM message_fts WHERE docid=old.rowid; END;` ); dbs.message.run( `CREATE TRIGGER IF NOT EXISTS message_before_delete BEFORE DELETE ON message BEGIN DELETE FROM message_fts WHERE docid=old.rowid; END;` ); dbs.message.run( `CREATE TRIGGER IF NOT EXISTS message_after_update AFTER UPDATE ON message BEGIN INSERT INTO message_fts(docid, subject, message) VALUES(new.rowid, new.subject, new.message); END;` ); dbs.message.run( `CREATE TRIGGER IF NOT EXISTS message_after_insert AFTER INSERT ON message BEGIN INSERT INTO message_fts(docid, subject, message) VALUES(new.rowid, new.subject, new.message); END;` ); dbs.message.run( `CREATE TABLE IF NOT EXISTS message_meta ( message_id INTEGER NOT NULL, meta_category INTEGER NOT NULL, meta_name VARCHAR NOT NULL, meta_value VARCHAR NOT NULL, UNIQUE(message_id, meta_category, meta_name, meta_value), FOREIGN KEY(message_id) REFERENCES message(message_id) ON DELETE CASCADE );` ); // :TODO: need SQL to ensure cleaned up if delete from message? /* dbs.message.run( `CREATE TABLE IF NOT EXISTS hash_tag ( hash_tag_id INTEGER PRIMARY KEY, hash_tag_name VARCHAR NOT NULL, UNIQUE(hash_tag_name) );` ); // :TODO: need SQL to ensure cleaned up if delete from message? dbs.message.run( `CREATE TABLE IF NOT EXISTS message_hash_tag ( hash_tag_id INTEGER NOT NULL, message_id INTEGER NOT NULL, );` ); */ dbs.message.run( `CREATE TABLE IF NOT EXISTS user_message_area_last_read ( user_id INTEGER NOT NULL, area_tag VARCHAR NOT NULL, message_id INTEGER NOT NULL, UNIQUE(user_id, area_tag) );` ); dbs.message.run( `CREATE TABLE IF NOT EXISTS message_area_last_scan ( scan_toss VARCHAR NOT NULL, area_tag VARCHAR NOT NULL, message_id INTEGER NOT NULL, UNIQUE(scan_toss, area_tag) );` ); return cb(null); }, file: cb => { enableForeignKeys(dbs.file); dbs.file.run( // :TODO: should any of this be unique -- file_sha256 unless dupes are allowed on the system `CREATE TABLE IF NOT EXISTS file ( file_id INTEGER PRIMARY KEY, area_tag VARCHAR NOT NULL, file_sha256 VARCHAR NOT NULL, file_name, /* FTS @ file_fts */ storage_tag VARCHAR NOT NULL, desc, /* FTS @ file_fts */ desc_long, /* FTS @ file_fts */ upload_timestamp DATETIME NOT NULL );` ); dbs.file.run( `CREATE INDEX IF NOT EXISTS file_by_area_tag_index ON file (area_tag);` ); dbs.file.run( `CREATE INDEX IF NOT EXISTS file_by_sha256_index ON file (file_sha256);` ); dbs.file.run( `CREATE VIRTUAL TABLE IF NOT EXISTS file_fts USING fts4 ( content="file", file_name, desc, desc_long );` ); dbs.file.run( `CREATE TRIGGER IF NOT EXISTS file_before_update BEFORE UPDATE ON file BEGIN DELETE FROM file_fts WHERE docid=old.rowid; END;` ); dbs.file.run( `CREATE TRIGGER IF NOT EXISTS file_before_delete BEFORE DELETE ON file BEGIN DELETE FROM file_fts WHERE docid=old.rowid; END;` ); dbs.file.run( `CREATE TRIGGER IF NOT EXISTS file_after_update AFTER UPDATE ON file BEGIN INSERT INTO file_fts(docid, file_name, desc, desc_long) VALUES(new.rowid, new.file_name, new.desc, new.desc_long); END;` ); dbs.file.run( `CREATE TRIGGER IF NOT EXISTS file_after_insert AFTER INSERT ON file BEGIN INSERT INTO file_fts(docid, file_name, desc, desc_long) VALUES(new.rowid, new.file_name, new.desc, new.desc_long); END;` ); dbs.file.run( `CREATE TABLE IF NOT EXISTS file_meta ( file_id INTEGER NOT NULL, meta_name VARCHAR NOT NULL, meta_value VARCHAR NOT NULL, UNIQUE(file_id, meta_name, meta_value), FOREIGN KEY(file_id) REFERENCES file(file_id) ON DELETE CASCADE );` ); dbs.file.run( `CREATE TABLE IF NOT EXISTS hash_tag ( hash_tag_id INTEGER PRIMARY KEY, hash_tag VARCHAR NOT NULL, UNIQUE(hash_tag) );` ); dbs.file.run( `CREATE TABLE IF NOT EXISTS file_hash_tag ( hash_tag_id INTEGER NOT NULL, file_id INTEGER NOT NULL, UNIQUE(hash_tag_id, file_id), FOREIGN KEY(file_id) REFERENCES file(file_id) ON DELETE CASCADE );` ); dbs.file.run( `CREATE TABLE IF NOT EXISTS file_user_rating ( file_id INTEGER NOT NULL, user_id INTEGER NOT NULL, rating INTEGER NOT NULL, UNIQUE(file_id, user_id), FOREIGN KEY(file_id) REFERENCES file(file_id) ON DELETE CASCADE -- Note that we cannot CASCADE if user_id is removed from user.db -- See processing in oputil's removeUser() );` ); dbs.file.run( `CREATE TABLE IF NOT EXISTS file_web_serve ( hash_id VARCHAR NOT NULL PRIMARY KEY, expire_timestamp DATETIME NOT NULL );` ); dbs.file.run( `CREATE TABLE IF NOT EXISTS file_web_serve_batch ( hash_id VARCHAR NOT NULL, file_id INTEGER NOT NULL, UNIQUE(hash_id, file_id), FOREIGN KEY(file_id) REFERENCES file(file_id) ON DELETE CASCADE );` ); return cb(null); }, activitypub: cb => { enableForeignKeys(dbs.activitypub); // ActivityPub Collections of various types such as followers, following, likes, ... dbs.activitypub.run( `CREATE TABLE IF NOT EXISTS collection ( collection_id VARCHAR NOT NULL, -- ie: http://somewhere.com/_enig/ap/users/NuSkooler/followers name VARCHAR NOT NULL, -- examples: followers, follows, ... timestamp DATETIME NOT NULL, -- Timestamp in which this entry was created owner_actor_id VARCHAR NOT NULL, -- Local, owning Actor ID, or the #Public magic collection ID object_id VARCHAR NOT NULL, -- Object ID from obj_json.id object_json VARCHAR NOT NULL, -- Object varies by collection (obj_json.type) is_private INTEGER NOT NULL, -- Is this object private to |owner_actor_id|? UNIQUE(name, collection_id, object_id) );` ); dbs.activitypub.run( `CREATE INDEX IF NOT EXISTS collection_entry_by_name_actor_id_index0 ON collection (name, owner_actor_id);` ); dbs.activitypub.run( `CREATE INDEX IF NOT EXISTS collection_entry_by_name_collection_id_index0 ON collection (name, collection_id);` ); // Collection meta contains 0:N additional metadata records for a object_id in a collection dbs.activitypub.run( `CREATE TABLE IF NOT EXISTS collection_object_meta ( collection_id VARCHAR NOT NULL, name VARCHAR NOT NULL, object_id VARCHAR NOT NULL, meta_name VARCHAR NOT NULL, meta_value VARCHAR NOT NULL, UNIQUE(collection_id, object_id, meta_name), FOREIGN KEY(name, collection_id, object_id) REFERENCES collection(name, collection_id, object_id) ON DELETE CASCADE );` ); return cb(null); }, }; function scheduledEventOptimizeDatabases(args, cb) { async.forEachSeries( Object.keys(dbs), (db, nextDb) => { return db.run('PRAGMA OPTIMIZE', nextDb); }, err => { return cb(err); } ); }