enigma-bbs/core/database.js

521 lines
17 KiB
JavaScript
Raw Normal View History

2014-10-20 05:30:44 +00:00
/* jslint node: true */
'use strict';
// ENiGMA½
2020-06-08 03:21:31 +00:00
const conf = require('./config');
// deps
const sqlite3 = require('sqlite3');
const sqlite3Trans = require('sqlite3-trans');
const paths = require('path');
const async = require('async');
const _ = require('lodash');
const assert = require('assert');
const moment = require('moment');
// database handles
const dbs = {};
2014-10-21 04:47:13 +00:00
exports.getTransactionDatabase = getTransactionDatabase;
exports.getModDatabasePath = getModDatabasePath;
exports.loadDatabaseForMod = loadDatabaseForMod;
exports.getISOTimestampString = getISOTimestampString;
exports.sanitizeString = sanitizeString;
exports.initializeDatabases = initializeDatabases;
2014-10-20 05:30:44 +00:00
exports.dbs = dbs;
2014-10-20 05:30:44 +00:00
function getTransactionDatabase(db) {
return sqlite3Trans.wrap(db);
}
2014-10-20 05:30:44 +00:00
function getDatabasePath(name) {
2020-06-08 03:21:31 +00:00
const Config = conf.get();
return paths.join(Config.paths.db, `${name}.sqlite3`);
2016-06-26 04:37:28 +00:00
}
function getModDatabasePath(moduleInfo, suffix) {
//
// Mods that use a database are stored in Config.paths.modsDb (e.g. enigma-bbs/db/mods)
// We expect that moduleInfo defines packageName which will be the base of the modules
// filename. An optional suffix may be supplied as well.
//
const HOST_RE =
/^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]*[A-Za-z0-9])$/;
assert(_.isObject(moduleInfo));
assert(_.isString(moduleInfo.packageName), 'moduleInfo must define "packageName"!');
let full = moduleInfo.packageName;
if (suffix) {
full += `.${suffix}`;
}
assert(
full.split('.').length > 1 && HOST_RE.test(full),
'packageName must follow Reverse Domain Name Notation - https://en.wikipedia.org/wiki/Reverse_domain_name_notation'
);
2020-06-08 03:21:31 +00:00
const Config = conf.get();
return paths.join(Config.paths.modsDb, `${full}.sqlite3`);
2014-10-20 05:30:44 +00:00
}
function loadDatabaseForMod(modInfo, cb) {
const db = getTransactionDatabase(
new sqlite3.Database(getModDatabasePath(modInfo), err => {
return cb(err, db);
})
);
}
function getISOTimestampString(ts) {
ts = ts || moment();
if (!moment.isMoment(ts)) {
if (_.isString(ts)) {
ts = ts.replace(/\//g, '-');
}
ts = moment(ts);
}
2018-11-26 03:31:25 +00:00
return ts.format('YYYY-MM-DDTHH:mm:ss.SSSZ');
}
function sanitizeString(s) {
return s.replace(/[\0\x08\x09\x1a\n\r"'\\%]/g, c => {
// eslint-disable-line no-control-regex
switch (c) {
case '\0':
return '\\0';
case '\x08':
return '\\b';
case '\x09':
return '\\t';
case '\x1a':
return '\\z';
case '\n':
return '\\n';
case '\r':
return '\\r';
case '"':
case "'":
return `${c}${c}`;
case '\\':
case '%':
return `\\${c}`;
}
});
2018-01-27 04:34:32 +00:00
}
2015-11-07 01:25:07 +00:00
function initializeDatabases(cb) {
async.eachSeries(
['system', 'user', 'message', 'file', 'activitypub'],
(dbName, next) => {
dbs[dbName] = sqlite3Trans.wrap(
new sqlite3.Database(getDatabasePath(dbName), err => {
if (err) {
return cb(err);
}
dbs[dbName].serialize(() => {
DB_INIT_TABLE[dbName](() => {
return next(null);
});
});
})
);
},
err => {
return cb(err);
}
);
2015-05-11 03:39:39 +00:00
}
function enableForeignKeys(db) {
db.run('PRAGMA foreign_keys = ON;');
}
const DB_INIT_TABLE = {
system: cb => {
enableForeignKeys(dbs.system);
// Various stat/event logging - see stat_log.js
dbs.system.run(
`CREATE TABLE IF NOT EXISTS system_stat (
stat_name VARCHAR PRIMARY KEY NOT NULL,
stat_value VARCHAR NOT NULL
);`
);
dbs.system.run(
`CREATE TABLE IF NOT EXISTS system_event_log (
id INTEGER PRIMARY KEY,
timestamp DATETIME NOT NULL,
log_name VARCHAR NOT NULL,
log_value VARCHAR NOT NULL,
UNIQUE(timestamp, log_name)
);`
);
dbs.system.run(
`CREATE TABLE IF NOT EXISTS user_event_log (
id INTEGER PRIMARY KEY,
timestamp DATETIME NOT NULL,
user_id INTEGER NOT NULL,
session_id VARCHAR NOT NULL,
log_name VARCHAR NOT NULL,
log_value VARCHAR NOT NULL,
UNIQUE(timestamp, user_id, session_id, log_name)
);`
);
return cb(null);
},
user: cb => {
enableForeignKeys(dbs.user);
dbs.user.run(
2020-06-08 03:21:31 +00:00
`CREATE TABLE IF NOT EXISTS user (
id INTEGER PRIMARY KEY,
2020-06-08 03:21:31 +00:00
user_name VARCHAR NOT NULL,
UNIQUE(user_name)
);`
);
// :TODO: create FK on delete/etc.
dbs.user.run(
`CREATE TABLE IF NOT EXISTS user_property (
user_id INTEGER NOT NULL,
prop_name VARCHAR NOT NULL,
prop_value VARCHAR,
UNIQUE(user_id, prop_name),
2020-06-08 03:21:31 +00:00
FOREIGN KEY(user_id) REFERENCES user(id) ON DELETE CASCADE
);`
);
2020-07-14 04:40:49 +00:00
dbs.user.run(
`CREATE INDEX IF NOT EXISTS user_property_id_and_name_index0
ON user_property (user_id, prop_name);`
);
dbs.user.run(
2020-06-08 03:21:31 +00:00
`CREATE TABLE IF NOT EXISTS user_group_member (
group_name VARCHAR NOT NULL,
user_id INTEGER NOT NULL,
UNIQUE(group_name, user_id)
);`
);
dbs.user.run(
`CREATE TABLE IF NOT EXISTS user_achievement (
user_id INTEGER NOT NULL,
achievement_tag VARCHAR NOT NULL,
timestamp DATETIME NOT NULL,
2019-01-14 01:19:00 +00:00
match VARCHAR NOT NULL,
title VARCHAR NOT NULL,
text VARCHAR NOT NULL,
points INTEGER NOT NULL,
2019-01-14 01:19:00 +00:00
UNIQUE(user_id, achievement_tag, match),
FOREIGN KEY(user_id) REFERENCES user(id) ON DELETE CASCADE
);`
);
//
// Table for temporary tokens, generally used for e.g. 'outside'
// access such as email links.
// Examples: PW reset, enabling of 2FA/OTP, etc.
//
dbs.user.run(
`CREATE TABLE IF NOT EXISTS user_temporary_token (
user_id INTEGER NOT NULL,
token VARCHAR NOT NULL,
2019-06-12 01:54:57 +00:00
token_type VARCHAR NOT NULL,
timestamp DATETIME NOT NULL,
2019-06-12 01:54:57 +00:00
UNIQUE(user_id, token_type),
FOREIGN KEY(user_id) REFERENCES user(id) ON DELETE CASCADE
);`
);
return cb(null);
},
message: cb => {
enableForeignKeys(dbs.message);
dbs.message.run(
`CREATE TABLE IF NOT EXISTS message (
2020-06-08 03:21:31 +00:00
message_id INTEGER PRIMARY KEY,
area_tag VARCHAR NOT NULL,
2020-06-08 03:21:31 +00:00
message_uuid VARCHAR(36) NOT NULL,
reply_to_message_id INTEGER,
to_user_name VARCHAR NOT NULL,
from_user_name VARCHAR NOT NULL,
subject, /* FTS @ message_fts */
message, /* FTS @ message_fts */
modified_timestamp DATETIME NOT NULL,
view_count INTEGER NOT NULL DEFAULT 0,
2020-06-08 03:21:31 +00:00
UNIQUE(message_uuid)
);`
);
dbs.message.run(
`CREATE INDEX IF NOT EXISTS message_by_area_tag_index
ON message (area_tag);`
);
dbs.message.run(
`CREATE VIRTUAL TABLE IF NOT EXISTS message_fts USING fts4 (
content="message",
subject,
message
);`
);
dbs.message.run(
`CREATE TRIGGER IF NOT EXISTS message_before_update BEFORE UPDATE ON message BEGIN
DELETE FROM message_fts WHERE docid=old.rowid;
END;`
);
dbs.message.run(
`CREATE TRIGGER IF NOT EXISTS message_before_delete BEFORE DELETE ON message BEGIN
DELETE FROM message_fts WHERE docid=old.rowid;
END;`
);
dbs.message.run(
`CREATE TRIGGER IF NOT EXISTS message_after_update AFTER UPDATE ON message BEGIN
INSERT INTO message_fts(docid, subject, message) VALUES(new.rowid, new.subject, new.message);
END;`
);
dbs.message.run(
`CREATE TRIGGER IF NOT EXISTS message_after_insert AFTER INSERT ON message BEGIN
INSERT INTO message_fts(docid, subject, message) VALUES(new.rowid, new.subject, new.message);
END;`
);
dbs.message.run(
`CREATE TABLE IF NOT EXISTS message_meta (
message_id INTEGER NOT NULL,
meta_category INTEGER NOT NULL,
meta_name VARCHAR NOT NULL,
meta_value VARCHAR NOT NULL,
2020-06-08 03:21:31 +00:00
UNIQUE(message_id, meta_category, meta_name, meta_value),
FOREIGN KEY(message_id) REFERENCES message(message_id) ON DELETE CASCADE
);`
);
// :TODO: need SQL to ensure cleaned up if delete from message?
/*
dbs.message.run(
`CREATE TABLE IF NOT EXISTS hash_tag (
hash_tag_id INTEGER PRIMARY KEY,
hash_tag_name VARCHAR NOT NULL,
UNIQUE(hash_tag_name)
);`
);
// :TODO: need SQL to ensure cleaned up if delete from message?
dbs.message.run(
`CREATE TABLE IF NOT EXISTS message_hash_tag (
hash_tag_id INTEGER NOT NULL,
message_id INTEGER NOT NULL,
);`
);
*/
dbs.message.run(
`CREATE TABLE IF NOT EXISTS user_message_area_last_read (
user_id INTEGER NOT NULL,
area_tag VARCHAR NOT NULL,
message_id INTEGER NOT NULL,
UNIQUE(user_id, area_tag)
);`
);
dbs.message.run(
`CREATE TABLE IF NOT EXISTS message_area_last_scan (
scan_toss VARCHAR NOT NULL,
area_tag VARCHAR NOT NULL,
message_id INTEGER NOT NULL,
UNIQUE(scan_toss, area_tag)
);`
);
return cb(null);
},
file: cb => {
enableForeignKeys(dbs.file);
dbs.file.run(
// :TODO: should any of this be unique -- file_sha256 unless dupes are allowed on the system
`CREATE TABLE IF NOT EXISTS file (
file_id INTEGER PRIMARY KEY,
area_tag VARCHAR NOT NULL,
file_sha256 VARCHAR NOT NULL,
file_name, /* FTS @ file_fts */
storage_tag VARCHAR NOT NULL,
desc, /* FTS @ file_fts */
2020-06-08 03:21:31 +00:00
desc_long, /* FTS @ file_fts */
upload_timestamp DATETIME NOT NULL
);`
);
dbs.file.run(
`CREATE INDEX IF NOT EXISTS file_by_area_tag_index
ON file (area_tag);`
);
dbs.file.run(
`CREATE INDEX IF NOT EXISTS file_by_sha256_index
ON file (file_sha256);`
);
dbs.file.run(
`CREATE VIRTUAL TABLE IF NOT EXISTS file_fts USING fts4 (
content="file",
file_name,
desc,
desc_long
);`
);
dbs.file.run(
`CREATE TRIGGER IF NOT EXISTS file_before_update BEFORE UPDATE ON file BEGIN
DELETE FROM file_fts WHERE docid=old.rowid;
END;`
);
dbs.file.run(
`CREATE TRIGGER IF NOT EXISTS file_before_delete BEFORE DELETE ON file BEGIN
DELETE FROM file_fts WHERE docid=old.rowid;
END;`
);
dbs.file.run(
`CREATE TRIGGER IF NOT EXISTS file_after_update AFTER UPDATE ON file BEGIN
INSERT INTO file_fts(docid, file_name, desc, desc_long) VALUES(new.rowid, new.file_name, new.desc, new.desc_long);
END;`
);
dbs.file.run(
`CREATE TRIGGER IF NOT EXISTS file_after_insert AFTER INSERT ON file BEGIN
INSERT INTO file_fts(docid, file_name, desc, desc_long) VALUES(new.rowid, new.file_name, new.desc, new.desc_long);
END;`
);
dbs.file.run(
`CREATE TABLE IF NOT EXISTS file_meta (
file_id INTEGER NOT NULL,
meta_name VARCHAR NOT NULL,
meta_value VARCHAR NOT NULL,
2020-06-08 03:21:31 +00:00
UNIQUE(file_id, meta_name, meta_value),
FOREIGN KEY(file_id) REFERENCES file(file_id) ON DELETE CASCADE
);`
);
dbs.file.run(
`CREATE TABLE IF NOT EXISTS hash_tag (
hash_tag_id INTEGER PRIMARY KEY,
hash_tag VARCHAR NOT NULL,
2020-06-08 03:21:31 +00:00
UNIQUE(hash_tag)
);`
);
dbs.file.run(
`CREATE TABLE IF NOT EXISTS file_hash_tag (
hash_tag_id INTEGER NOT NULL,
file_id INTEGER NOT NULL,
2020-06-08 03:21:31 +00:00
2020-11-29 23:24:51 +00:00
UNIQUE(hash_tag_id, file_id),
FOREIGN KEY(file_id) REFERENCES file(file_id) ON DELETE CASCADE
);`
);
dbs.file.run(
`CREATE TABLE IF NOT EXISTS file_user_rating (
file_id INTEGER NOT NULL,
user_id INTEGER NOT NULL,
rating INTEGER NOT NULL,
2020-11-29 23:24:51 +00:00
UNIQUE(file_id, user_id),
FOREIGN KEY(file_id) REFERENCES file(file_id) ON DELETE CASCADE
-- Note that we cannot CASCADE if user_id is removed from user.db
-- See processing in oputil's removeUser()
);`
);
dbs.file.run(
`CREATE TABLE IF NOT EXISTS file_web_serve (
hash_id VARCHAR NOT NULL PRIMARY KEY,
expire_timestamp DATETIME NOT NULL
);`
);
dbs.file.run(
`CREATE TABLE IF NOT EXISTS file_web_serve_batch (
hash_id VARCHAR NOT NULL,
file_id INTEGER NOT NULL,
2017-09-26 16:39:23 +00:00
2020-11-29 23:24:51 +00:00
UNIQUE(hash_id, file_id),
FOREIGN KEY(file_id) REFERENCES file(file_id) ON DELETE CASCADE
);`
);
2017-09-26 16:39:23 +00:00
return cb(null);
},
activitypub: cb => {
2023-01-28 19:59:12 +00:00
enableForeignKeys(dbs.activitypub);
2023-01-22 01:51:54 +00:00
// Actors we know about and have cached
dbs.activitypub.run(
2023-01-22 01:51:54 +00:00
`CREATE TABLE IF NOT EXISTS actor_cache (
actor_id VARCHAR NOT NULL, -- Fully qualified Actor ID/URL
actor_json VARCHAR NOT NULL, -- Actor document
subject VARCHAR, -- Subject obtained from WebFinger, e.g. @Username@some.domain
2023-01-22 01:51:54 +00:00
timestamp DATETIME NOT NULL, -- Timestamp in which this Actor was cached
2023-01-22 01:51:54 +00:00
UNIQUE(actor_id)
);`
);
dbs.activitypub.run(
2023-01-22 01:51:54 +00:00
`CREATE INDEX IF NOT EXISTS actor_cache_actor_id_index0
ON actor_cache (actor_id);`
2023-01-13 06:19:52 +00:00
);
// ActivityPub Collections of various types such as followers, following, likes, ...
dbs.activitypub.run(
2023-01-21 08:19:19 +00:00
`CREATE TABLE IF NOT EXISTS collection (
collection_id VARCHAR NOT NULL, -- ie: http://somewhere.com/_enig/ap/users/NuSkooler/followers
name VARCHAR NOT NULL, -- examples: followers, follows, ...
timestamp DATETIME NOT NULL, -- Timestamp in which this entry was created
owner_actor_id VARCHAR NOT NULL, -- Local, owning Actor ID, or the #Public magic collection ID
object_id VARCHAR NOT NULL, -- Object ID from obj_json.id
object_json VARCHAR NOT NULL, -- Object varies by collection (obj_json.type)
is_private INTEGER NOT NULL, -- Is this object private to |owner_actor_id|?
2023-01-21 08:19:19 +00:00
UNIQUE(name, collection_id, object_id)
);`
);
dbs.activitypub.run(
`CREATE INDEX IF NOT EXISTS collection_entry_by_name_actor_id_index0
ON collection (name, owner_actor_id);`
);
dbs.activitypub.run(
`CREATE INDEX IF NOT EXISTS collection_entry_by_name_collection_id_index0
ON collection (name, collection_id);`
);
return cb(null);
},
};