From 9ccf6bbea0874a718e9b83e08070170384e45c76 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 19 Apr 2024 20:52:42 -0500 Subject: [PATCH] Mold the things to work with Postgres --- deno.json | 4 +++- src/db/DittoTables.ts | 2 +- src/db/adapters/DittoSQLite.ts | 3 +++ src/db/migrations/000_create_events.ts | 9 +-------- src/db/migrations/002_events_fts.ts | 7 +++++-- src/db/migrations/003_events_admin.ts | 6 +----- src/db/migrations/004_add_user_indexes.ts | 13 +------------ src/db/migrations/006_pragma.ts | 11 +++-------- src/db/migrations/007_unattached_media.ts | 4 ++-- src/db/migrations/008_wal.ts | 8 +++----- src/db/migrations/010_drop_users.ts | 2 +- src/db/unattached-media.ts | 6 +++--- src/storages/events-db.ts | 5 ++++- src/workers/trends.worker.ts | 2 +- 14 files changed, 32 insertions(+), 50 deletions(-) diff --git a/deno.json b/deno.json index 001337a..433878b 100644 --- a/deno.json +++ b/deno.json @@ -15,7 +15,9 @@ "@/": "./src/", "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.9.7", "~/fixtures/": "./fixtures/", - "kysely": "npm:kysely@^0.26.3" + "kysely": "npm:kysely@^0.26.3", + "pg": "npm:pg@^8.11.5", + "pg-pool": "npm:pg-pool@^3.6.2" }, "lint": { "include": ["src/", "scripts/"], diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index 4d8ce2b..79fec5d 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -56,7 +56,7 @@ interface UnattachedMediaRow { pubkey: string; url: string; data: string; - uploaded_at: Date; + uploaded_at: number; } interface PubkeyDomainRow { diff --git a/src/db/adapters/DittoSQLite.ts b/src/db/adapters/DittoSQLite.ts index 47c7ae8..c91407a 100644 --- a/src/db/adapters/DittoSQLite.ts +++ b/src/db/adapters/DittoSQLite.ts @@ -21,6 +21,9 @@ export class DittoSQLite { await Promise.all([ sql`PRAGMA synchronous = normal`.execute(this.db), sql`PRAGMA temp_store = memory`.execute(this.db), + sql`PRAGMA foreign_keys = ON`.execute(this.db), + sql`PRAGMA auto_vacuum = FULL`.execute(this.db), + sql`PRAGMA journal_mode = WAL`.execute(this.db), sql.raw(`PRAGMA mmap_size = ${Conf.sqlite.mmapSize}`).execute(this.db), ]); } diff --git a/src/db/migrations/000_create_events.ts b/src/db/migrations/000_create_events.ts index 8422071..158551b 100644 --- a/src/db/migrations/000_create_events.ts +++ b/src/db/migrations/000_create_events.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from '@/deps.ts'; +import { Kysely } from '@/deps.ts'; export async function up(db: Kysely): Promise { await db.schema @@ -21,13 +21,6 @@ export async function up(db: Kysely): Promise { .addColumn('event_id', 'text', (col) => col.notNull()) .execute(); - await db.schema - .createTable('users') - .addColumn('pubkey', 'text', (col) => col.primaryKey()) - .addColumn('username', 'text', (col) => col.notNull().unique()) - .addColumn('inserted_at', 'datetime', (col) => col.notNull().defaultTo(sql`CURRENT_TIMESTAMP`)) - .execute(); - await db.schema .createIndex('idx_events_kind') .on('events') diff --git a/src/db/migrations/002_events_fts.ts b/src/db/migrations/002_events_fts.ts index c0341c1..9324195 100644 --- a/src/db/migrations/002_events_fts.ts +++ b/src/db/migrations/002_events_fts.ts @@ -1,9 +1,12 @@ +import { Conf } from '@/config.ts'; import { Kysely, sql } from '@/deps.ts'; export async function up(db: Kysely): Promise { - await sql`CREATE VIRTUAL TABLE events_fts USING fts5(id, content)`.execute(db); + if (Conf.databaseUrl.protocol === 'sqlite:') { + await sql`CREATE VIRTUAL TABLE events_fts USING fts5(id, content)`.execute(db); + } } export async function down(db: Kysely): Promise { - await db.schema.dropTable('events_fts').execute(); + await db.schema.dropTable('events_fts').ifExists().execute(); } diff --git a/src/db/migrations/003_events_admin.ts b/src/db/migrations/003_events_admin.ts index e481a59..8469fc2 100644 --- a/src/db/migrations/003_events_admin.ts +++ b/src/db/migrations/003_events_admin.ts @@ -1,10 +1,6 @@ import { Kysely } from '@/deps.ts'; -export async function up(db: Kysely): Promise { - await db.schema - .alterTable('users') - .addColumn('admin', 'boolean', (col) => col.defaultTo(false)) - .execute(); +export async function up(_db: Kysely): Promise { } export async function down(db: Kysely): Promise { diff --git a/src/db/migrations/004_add_user_indexes.ts b/src/db/migrations/004_add_user_indexes.ts index e77693b..929181c 100644 --- a/src/db/migrations/004_add_user_indexes.ts +++ b/src/db/migrations/004_add_user_indexes.ts @@ -1,17 +1,6 @@ import { Kysely } from '@/deps.ts'; -export async function up(db: Kysely): Promise { - await db.schema - .createIndex('idx_users_pubkey') - .on('users') - .column('pubkey') - .execute(); - - await db.schema - .createIndex('idx_users_username') - .on('users') - .column('username') - .execute(); +export async function up(_db: Kysely): Promise { } export async function down(db: Kysely): Promise { diff --git a/src/db/migrations/006_pragma.ts b/src/db/migrations/006_pragma.ts index e9dbad1..2639e81 100644 --- a/src/db/migrations/006_pragma.ts +++ b/src/db/migrations/006_pragma.ts @@ -1,12 +1,7 @@ -import { Kysely, sql } from '@/deps.ts'; +import { Kysely } from '@/deps.ts'; -export async function up(db: Kysely): Promise { - await sql`PRAGMA foreign_keys = ON`.execute(db); - await sql`PRAGMA auto_vacuum = FULL`.execute(db); - await sql`VACUUM`.execute(db); +export async function up(_db: Kysely): Promise { } -export async function down(db: Kysely): Promise { - await sql`PRAGMA foreign_keys = OFF`.execute(db); - await sql`PRAGMA auto_vacuum = NONE`.execute(db); +export async function down(_db: Kysely): Promise { } diff --git a/src/db/migrations/007_unattached_media.ts b/src/db/migrations/007_unattached_media.ts index a2b36a2..1887111 100644 --- a/src/db/migrations/007_unattached_media.ts +++ b/src/db/migrations/007_unattached_media.ts @@ -1,4 +1,4 @@ -import { Kysely, sql } from '@/deps.ts'; +import { Kysely } from '@/deps.ts'; export async function up(db: Kysely): Promise { await db.schema @@ -7,7 +7,7 @@ export async function up(db: Kysely): Promise { .addColumn('pubkey', 'text', (c) => c.notNull()) .addColumn('url', 'text', (c) => c.notNull()) .addColumn('data', 'text', (c) => c.notNull()) - .addColumn('uploaded_at', 'datetime', (c) => c.notNull().defaultTo(sql`CURRENT_TIMESTAMP`)) + .addColumn('uploaded_at', 'bigint', (c) => c.notNull()) .execute(); await db.schema diff --git a/src/db/migrations/008_wal.ts b/src/db/migrations/008_wal.ts index 7f96226..2639e81 100644 --- a/src/db/migrations/008_wal.ts +++ b/src/db/migrations/008_wal.ts @@ -1,9 +1,7 @@ -import { Kysely, sql } from '@/deps.ts'; +import { Kysely } from '@/deps.ts'; -export async function up(db: Kysely): Promise { - await sql`PRAGMA journal_mode = WAL`.execute(db); +export async function up(_db: Kysely): Promise { } -export async function down(db: Kysely): Promise { - await sql`PRAGMA journal_mode = DELETE`.execute(db); +export async function down(_db: Kysely): Promise { } diff --git a/src/db/migrations/010_drop_users.ts b/src/db/migrations/010_drop_users.ts index 9649b64..6cd83c0 100644 --- a/src/db/migrations/010_drop_users.ts +++ b/src/db/migrations/010_drop_users.ts @@ -1,7 +1,7 @@ import { Kysely } from '@/deps.ts'; export async function up(db: Kysely): Promise { - await db.schema.dropTable('users').execute(); + await db.schema.dropTable('users').ifExists().execute(); } export async function down(_db: Kysely): Promise { diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index ae9d882..3761947 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -7,14 +7,14 @@ interface UnattachedMedia { pubkey: string; url: string; data: MediaData; - uploaded_at: Date; + uploaded_at: number; } /** Add unattached media into the database. */ async function insertUnattachedMedia(media: Omit) { const result = { id: uuid62.v4(), - uploaded_at: new Date(), + uploaded_at: Date.now(), ...media, }; @@ -41,7 +41,7 @@ function selectUnattachedMediaQuery() { function getUnattachedMedia(until: Date) { return selectUnattachedMediaQuery() .leftJoin('tags', 'unattached_media.url', 'tags.value') - .where('uploaded_at', '<', until) + .where('uploaded_at', '<', until.getTime()) .execute(); } diff --git a/src/storages/events-db.ts b/src/storages/events-db.ts index 84a865e..6d80f70 100644 --- a/src/storages/events-db.ts +++ b/src/storages/events-db.ts @@ -60,6 +60,7 @@ type EventQuery = SelectQueryBuilder; #debug = Debug('ditto:db:events'); + private protocol = Conf.databaseUrl.protocol; constructor(db: Kysely) { this.#db = db; @@ -82,8 +83,10 @@ class EventsDB implements NStore { .execute(); } + const protocol = this.protocol; /** Add search data to the FTS table. */ async function indexSearch() { + if (protocol !== 'sqlite:') return; const searchContent = buildSearchContent(event); if (!searchContent) return; await trx.insertInto('events_fts') @@ -194,7 +197,7 @@ class EventsDB implements NStore { } } - if (filter.search) { + if (filter.search && this.protocol === 'sqlite:') { query = query .innerJoin('events_fts', 'events_fts.id', 'events.id') .where('events_fts.content', 'match', JSON.stringify(filter.search)); diff --git a/src/workers/trends.worker.ts b/src/workers/trends.worker.ts index fd9e8fe..df06fbb 100644 --- a/src/workers/trends.worker.ts +++ b/src/workers/trends.worker.ts @@ -28,7 +28,7 @@ export const TrendsWorker = { CREATE TABLE IF NOT EXISTS tag_usages ( tag TEXT NOT NULL COLLATE NOCASE, pubkey8 TEXT NOT NULL, - inserted_at DATETIME NOT NULL + inserted_at INTEGER NOT NULL ); CREATE INDEX IF NOT EXISTS idx_time_tag ON tag_usages(inserted_at, tag);