From dc57415df3cb87cd448c411ff540ae48d79b2535 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 2 Mar 2024 17:40:29 -0600 Subject: [PATCH 1/9] SqliteWorker: log amount of time each query takes --- src/workers/sqlite.worker.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/workers/sqlite.worker.ts b/src/workers/sqlite.worker.ts index 5ea1dd5..eb283a9 100644 --- a/src/workers/sqlite.worker.ts +++ b/src/workers/sqlite.worker.ts @@ -1,5 +1,5 @@ /// - +import { ScopedPerformance } from 'https://deno.land/x/scoped_performance@v2.0.0/mod.ts'; import { Comlink, type CompiledQuery, Debug, DenoSqlite3, type QueryResult } from '@/deps.ts'; import '@/sentry.ts'; @@ -12,12 +12,20 @@ export const SqliteWorker = { }, executeQuery({ sql, parameters }: CompiledQuery): QueryResult { if (!db) throw new Error('Database not open'); - debug(sql); - return { + + const perf = new ScopedPerformance(); + perf.mark('start'); + + const result = { rows: db!.prepare(sql).all(...parameters as any[]) as R[], numAffectedRows: BigInt(db!.changes), insertId: BigInt(db!.lastInsertRowId), }; + + const { duration } = perf.measure('end', 'start'); + debug(`${sql} \x1b[90m(${(duration / 1000).toFixed(2)}s)\x1b[0m`); + + return result; }, destroy() { db?.close(); From ad59c24f7793c3b922f17f7c1bc0b7af30dbc3df Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 2 Mar 2024 17:53:43 -0600 Subject: [PATCH 2/9] docs: add debugging SQLite information --- docs/debugging.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/debugging.md b/docs/debugging.md index 84e057a..6abc513 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -14,4 +14,14 @@ To access the debugger remotely, you can use SSH port forwarding. Run this comma ssh -L 9229:localhost:9229 @ ``` -Then, in Chromium, go to `chrome://inspect` and the Ditto server should be available. \ No newline at end of file +Then, in Chromium, go to `chrome://inspect` and the Ditto server should be available. + +## SQLite performance + +To track slow queries, first set `DEBUG=ditto:sqlite.worker` in the environment so only SQLite logs are shown. + +Then, grep for any logs above 0.001s: + +```sh +journalctl -fu ditto | grep -v '(0.00s)' +``` \ No newline at end of file From dffa70e2fe02f6ce5c97aedc23e32fd806008e85 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 2 Mar 2024 17:57:43 -0600 Subject: [PATCH 3/9] Add missing index on kind-pubkey-created_at --- src/db/migrations/011_kind_author_index.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 src/db/migrations/011_kind_author_index.ts diff --git a/src/db/migrations/011_kind_author_index.ts b/src/db/migrations/011_kind_author_index.ts new file mode 100644 index 0000000..e894aa3 --- /dev/null +++ b/src/db/migrations/011_kind_author_index.ts @@ -0,0 +1,16 @@ +import { Kysely } from '@/deps.ts'; + +export async function up(db: Kysely): Promise { + await db.schema + .createIndex('idx_events_kind_pubkey_created_at') + .on('events') + .columns(['kind', 'pubkey', 'created_at']) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .dropIndex('idx_events_kind_pubkey_created_at') + .on('events') + .execute(); +} From 9c4301e7915498774fb92c4e7997e603c07cba56 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 2 Mar 2024 18:08:37 -0600 Subject: [PATCH 4/9] getConfigs: wrap in try-catch --- src/controllers/api/pleroma.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/pleroma.ts b/src/controllers/api/pleroma.ts index f76df01..9425878 100644 --- a/src/controllers/api/pleroma.ts +++ b/src/controllers/api/pleroma.ts @@ -72,9 +72,12 @@ async function getConfigs(signal: AbortSignal): Promise { limit: 1, }], { signal }); - return jsonSchema.pipe(configSchema.array()).catch([]).parse( - await new AdminSigner().nip44.decrypt(Conf.pubkey, event.content).catch(() => ''), - ); + try { + const decrypted = await new AdminSigner().nip44.decrypt(Conf.pubkey, event.content); + return jsonSchema.pipe(configSchema.array()).catch([]).parse(decrypted); + } catch (_e) { + return []; + } } export { configController, frontendConfigController, pleromaAdminDeleteStatusController, updateConfigController }; From 19adb3ab4438ba0754a807d9cd106c3644b188c9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 2 Mar 2024 19:21:48 -0600 Subject: [PATCH 5/9] Use composite index for tags --- src/db/migrations/012_tags_composite_index.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/db/migrations/012_tags_composite_index.ts diff --git a/src/db/migrations/012_tags_composite_index.ts b/src/db/migrations/012_tags_composite_index.ts new file mode 100644 index 0000000..418df28 --- /dev/null +++ b/src/db/migrations/012_tags_composite_index.ts @@ -0,0 +1,31 @@ +import { Kysely } from '@/deps.ts'; + +export async function up(db: Kysely): Promise { + await db.schema.dropIndex('idx_tags_tag').on('tags').execute(); + await db.schema.dropIndex('idx_tags_value').on('tags').execute(); + + await db.schema + .createIndex('idx_tags_tag_value') + .on('tags') + .columns(['tag', 'value']) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .dropIndex('idx_tags_tag_value') + .on('tags') + .execute(); + + await db.schema + .createIndex('idx_tags_tag') + .on('tags') + .column('tag') + .execute(); + + await db.schema + .createIndex('idx_tags_value') + .on('tags') + .column('value') + .execute(); +} From 96b5ecc4357558a06e19630094d1710b92d816d7 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 2 Mar 2024 19:23:28 -0600 Subject: [PATCH 6/9] Fix dropIndex calls --- src/db/migrations/011_kind_author_index.ts | 5 +---- src/db/migrations/012_tags_composite_index.ts | 9 +++------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/db/migrations/011_kind_author_index.ts b/src/db/migrations/011_kind_author_index.ts index e894aa3..da21988 100644 --- a/src/db/migrations/011_kind_author_index.ts +++ b/src/db/migrations/011_kind_author_index.ts @@ -9,8 +9,5 @@ export async function up(db: Kysely): Promise { } export async function down(db: Kysely): Promise { - await db.schema - .dropIndex('idx_events_kind_pubkey_created_at') - .on('events') - .execute(); + await db.schema.dropIndex('idx_events_kind_pubkey_created_at').execute(); } diff --git a/src/db/migrations/012_tags_composite_index.ts b/src/db/migrations/012_tags_composite_index.ts index 418df28..8769289 100644 --- a/src/db/migrations/012_tags_composite_index.ts +++ b/src/db/migrations/012_tags_composite_index.ts @@ -1,8 +1,8 @@ import { Kysely } from '@/deps.ts'; export async function up(db: Kysely): Promise { - await db.schema.dropIndex('idx_tags_tag').on('tags').execute(); - await db.schema.dropIndex('idx_tags_value').on('tags').execute(); + await db.schema.dropIndex('idx_tags_tag').execute(); + await db.schema.dropIndex('idx_tags_value').execute(); await db.schema .createIndex('idx_tags_tag_value') @@ -12,10 +12,7 @@ export async function up(db: Kysely): Promise { } export async function down(db: Kysely): Promise { - await db.schema - .dropIndex('idx_tags_tag_value') - .on('tags') - .execute(); + await db.schema.dropIndex('idx_tags_tag_value').execute(); await db.schema .createIndex('idx_tags_tag') From e4f53b3936450de0f90df3bdbf29b734095ef4f1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 2 Mar 2024 21:11:45 -0600 Subject: [PATCH 7/9] Soft-delete events --- src/db.ts | 1 + src/db/migrations/013_soft_deletion.ts | 9 +++++++++ src/storages/events-db.ts | 8 +++----- 3 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 src/db/migrations/013_soft_deletion.ts diff --git a/src/db.ts b/src/db.ts index 253fed9..e1039dc 100644 --- a/src/db.ts +++ b/src/db.ts @@ -38,6 +38,7 @@ interface EventRow { created_at: number; tags: string; sig: string; + deleted_at: number | null; } interface EventFTSRow { diff --git a/src/db/migrations/013_soft_deletion.ts b/src/db/migrations/013_soft_deletion.ts new file mode 100644 index 0000000..3856ca0 --- /dev/null +++ b/src/db/migrations/013_soft_deletion.ts @@ -0,0 +1,9 @@ +import { Kysely } from '@/deps.ts'; + +export async function up(db: Kysely): Promise { + await db.schema.alterTable('events').addColumn('deleted_at', 'integer').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable('events').dropColumn('deleted_at').execute(); +} diff --git a/src/storages/events-db.ts b/src/storages/events-db.ts index b5c8ee7..304f283 100644 --- a/src/storages/events-db.ts +++ b/src/storages/events-db.ts @@ -155,6 +155,7 @@ class EventsDB implements NStore { 'events.created_at', 'events.sig', ]) + .where('events.deleted_at', 'is', null) .orderBy('events.created_at', 'desc'); for (const [key, value] of Object.entries(filter)) { @@ -329,12 +330,9 @@ class EventsDB implements NStore { const query = this.getEventsQuery(filters).clearSelect().select('id'); - await db.deleteFrom('events_fts') - .where('id', 'in', () => query) - .execute(); - - return db.deleteFrom('events') + return await db.updateTable('events') .where('id', 'in', () => query) + .set({ deleted_at: Math.floor(Date.now() / 1000) }) .execute(); } From 2972cb4b6df58991d3343c8e2f7c2380f1ce2cd3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 2 Mar 2024 21:20:43 -0600 Subject: [PATCH 8/9] Add event stats indexes --- src/db/migrations/014_stats_indexes.ts.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/db/migrations/014_stats_indexes.ts.ts diff --git a/src/db/migrations/014_stats_indexes.ts.ts b/src/db/migrations/014_stats_indexes.ts.ts new file mode 100644 index 0000000..d9071c6 --- /dev/null +++ b/src/db/migrations/014_stats_indexes.ts.ts @@ -0,0 +1,11 @@ +import { Kysely } from '@/deps.ts'; + +export async function up(db: Kysely): Promise { + await db.schema.createIndex('idx_author_stats_pubkey').on('author_stats').column('pubkey').execute(); + await db.schema.createIndex('idx_event_stats_event_id').on('event_stats').column('event_id').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('idx_author_stats_pubkey').on('author_stats').execute(); + await db.schema.dropIndex('idx_event_stats_event_id').on('event_stats').execute(); +} From 7eaa652ae9a17559e7feca95e2039e71cba960fd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 2 Mar 2024 21:46:23 -0600 Subject: [PATCH 9/9] Fix EventsDB tests --- src/storages/events-db.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/storages/events-db.test.ts b/src/storages/events-db.test.ts index 32aee80..744935b 100644 --- a/src/storages/events-db.test.ts +++ b/src/storages/events-db.test.ts @@ -28,13 +28,6 @@ Deno.test('insert and filter events', async () => { ); }); -Deno.test('delete events', async () => { - await eventsDB.event(event1); - assertEquals(await eventsDB.query([{ kinds: [1] }]), [event1]); - await eventsDB.remove([{ kinds: [1] }]); - assertEquals(await eventsDB.query([{ kinds: [1] }]), []); -}); - Deno.test('query events with local filter', async () => { await eventsDB.event(event1); @@ -54,6 +47,13 @@ Deno.test('query events with local filter', async () => { assertEquals(await eventsDB.query([{ kinds: [1], local: false }]), []); }); +Deno.test('delete events', async () => { + await eventsDB.event(event1); + assertEquals(await eventsDB.query([{ kinds: [1] }]), [event1]); + await eventsDB.remove([{ kinds: [1] }]); + assertEquals(await eventsDB.query([{ kinds: [1] }]), []); +}); + Deno.test('inserting replaceable events', async () => { assertEquals((await eventsDB.count([{ kinds: [0], authors: [event0.pubkey] }])).count, 0);