From 87980bbba1fbe7b3f7208dee4dc5d4058a75a944 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 15 Nov 2023 16:22:57 -0600 Subject: [PATCH 01/10] Add an SqliteWorker --- src/workers.ts | 5 +++++ src/workers/sqlite.ts | 37 ++++++++++++++++++++++++++++++++++++ src/workers/sqlite.worker.ts | 32 +++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 src/workers.ts create mode 100644 src/workers/sqlite.ts create mode 100644 src/workers/sqlite.worker.ts diff --git a/src/workers.ts b/src/workers.ts new file mode 100644 index 0000000..f00c43a --- /dev/null +++ b/src/workers.ts @@ -0,0 +1,5 @@ +import SqliteWorker from './workers/sqlite.ts'; + +const sqliteWorker = new SqliteWorker('./data/db.sqlite3'); + +export { sqliteWorker }; \ No newline at end of file diff --git a/src/workers/sqlite.ts b/src/workers/sqlite.ts new file mode 100644 index 0000000..aaab650 --- /dev/null +++ b/src/workers/sqlite.ts @@ -0,0 +1,37 @@ +class SqliteWorker { + #path: string; + #worker: Worker; + + constructor(path: string) { + this.#path = path; + this.#worker = new Worker(new URL('./sqlite.worker.ts', import.meta.url).href, { type: 'module' }); + } + + open(): Promise { + return this.#call(['open', [this.#path]]); + } + + query(sql: string, params?: any): Promise { + return this.#call(['query', [sql, params]]); + } + + #call(msg: [string, unknown[]]): Promise { + const id = crypto.randomUUID(); + + this.#worker.postMessage([id, msg]); + + // TODO: use a hashmap instead of an event listener for better performance. + return new Promise((resolve) => { + const handleEvent = (event: MessageEvent<[string, T]>) => { + const [_id, result] = event.data; + if (_id === id) { + this.#worker.removeEventListener('message', handleEvent); + resolve(result); + } + }; + this.#worker.addEventListener('message', handleEvent); + }); + } +} + +export default SqliteWorker; diff --git a/src/workers/sqlite.worker.ts b/src/workers/sqlite.worker.ts new file mode 100644 index 0000000..ea024dc --- /dev/null +++ b/src/workers/sqlite.worker.ts @@ -0,0 +1,32 @@ +/// + +import { DenoSqlite3 } from '@/deps.ts'; + +let db: DenoSqlite3; + +type Msg = + | ['open', [string]] + | ['query', [string, unknown[]]]; + +function call([cmd, args]: Msg) { + switch(cmd) { + case 'open': + return handleOpen(args[0]); + case 'query': + return handleQuery(args[0], args[1]); + } +} + +function handleOpen(path: string): void { + db = new DenoSqlite3(path); +} + +function handleQuery(sql: string, params: any[] = []) { + return db.prepare(sql).all(...params); +} + +self.addEventListener('message', (event: MessageEvent<[string, Msg]>) => { + const [id, msg] = event.data; + const result = call(msg); + self.postMessage([id, result]); +}); \ No newline at end of file From 71e8d26195054c383c5848490c1d2bce31929cab Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 15 Nov 2023 17:09:30 -0600 Subject: [PATCH 02/10] SqliteWorker: wait until worker is ready --- src/workers/sqlite.ts | 17 +++++++++++++++-- src/workers/sqlite.worker.ts | 4 +++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/workers/sqlite.ts b/src/workers/sqlite.ts index aaab650..6e71823 100644 --- a/src/workers/sqlite.ts +++ b/src/workers/sqlite.ts @@ -1,17 +1,30 @@ class SqliteWorker { #path: string; #worker: Worker; + ready: Promise; constructor(path: string) { this.#path = path; this.#worker = new Worker(new URL('./sqlite.worker.ts', import.meta.url).href, { type: 'module' }); + + this.ready = new Promise((resolve) => { + const handleEvent = (event: MessageEvent) => { + if (event.data[0] === 'ready') { + this.#worker.removeEventListener('message', handleEvent); + resolve(); + } + }; + this.#worker.addEventListener('message', handleEvent); + }); } - open(): Promise { + async open(): Promise { + await this.ready; return this.#call(['open', [this.#path]]); } - query(sql: string, params?: any): Promise { + async query(sql: string, params?: any): Promise { + await this.ready; return this.#call(['query', [sql, params]]); } diff --git a/src/workers/sqlite.worker.ts b/src/workers/sqlite.worker.ts index ea024dc..b21f467 100644 --- a/src/workers/sqlite.worker.ts +++ b/src/workers/sqlite.worker.ts @@ -29,4 +29,6 @@ self.addEventListener('message', (event: MessageEvent<[string, Msg]>) => { const [id, msg] = event.data; const result = call(msg); self.postMessage([id, result]); -}); \ No newline at end of file +}); + +self.postMessage(['ready']); \ No newline at end of file From e601c4319715ba530614a46aef6d5b89f482c7b5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 15 Nov 2023 17:09:53 -0600 Subject: [PATCH 03/10] deno fmt --- src/workers.ts | 2 +- src/workers/sqlite.worker.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/workers.ts b/src/workers.ts index f00c43a..ffd3e96 100644 --- a/src/workers.ts +++ b/src/workers.ts @@ -2,4 +2,4 @@ import SqliteWorker from './workers/sqlite.ts'; const sqliteWorker = new SqliteWorker('./data/db.sqlite3'); -export { sqliteWorker }; \ No newline at end of file +export { sqliteWorker }; diff --git a/src/workers/sqlite.worker.ts b/src/workers/sqlite.worker.ts index b21f467..e271d67 100644 --- a/src/workers/sqlite.worker.ts +++ b/src/workers/sqlite.worker.ts @@ -9,7 +9,7 @@ type Msg = | ['query', [string, unknown[]]]; function call([cmd, args]: Msg) { - switch(cmd) { + switch (cmd) { case 'open': return handleOpen(args[0]); case 'query': @@ -31,4 +31,4 @@ self.addEventListener('message', (event: MessageEvent<[string, Msg]>) => { self.postMessage([id, result]); }); -self.postMessage(['ready']); \ No newline at end of file +self.postMessage(['ready']); From ae56d059b1494b7ef82587db71fd137c11de89bd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 15 Nov 2023 19:23:24 -0600 Subject: [PATCH 04/10] SqliteWorker: return query result instead of rows for query --- src/workers/sqlite.ts | 8 +++++++- src/workers/sqlite.worker.ts | 6 +++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/workers/sqlite.ts b/src/workers/sqlite.ts index 6e71823..22f39e3 100644 --- a/src/workers/sqlite.ts +++ b/src/workers/sqlite.ts @@ -1,3 +1,9 @@ +interface QueryResult { + rows: unknown[]; + numAffectedRows: bigint; + insertId: bigint; +} + class SqliteWorker { #path: string; #worker: Worker; @@ -23,7 +29,7 @@ class SqliteWorker { return this.#call(['open', [this.#path]]); } - async query(sql: string, params?: any): Promise { + async query(sql: string, params?: any): Promise { await this.ready; return this.#call(['query', [sql, params]]); } diff --git a/src/workers/sqlite.worker.ts b/src/workers/sqlite.worker.ts index e271d67..69acf6f 100644 --- a/src/workers/sqlite.worker.ts +++ b/src/workers/sqlite.worker.ts @@ -22,7 +22,11 @@ function handleOpen(path: string): void { } function handleQuery(sql: string, params: any[] = []) { - return db.prepare(sql).all(...params); + return { + rows: db.prepare(sql).all(...params), + numAffectedRows: BigInt(db.changes), + insertId: BigInt(db.lastInsertRowId), + }; } self.addEventListener('message', (event: MessageEvent<[string, Msg]>) => { From 01839fbcbf5659881d1b14ecd43ffedf8bbea9f3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 1 Dec 2023 17:57:01 -0600 Subject: [PATCH 05/10] Upgrade kysely and kysely-deno-sqlite --- src/db.ts | 4 ++-- src/deps.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/db.ts b/src/db.ts index 99339a8..6a7778f 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,7 +1,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; -import { DenoSqlite3, DenoSqliteDialect, FileMigrationProvider, Kysely, Migrator } from '@/deps.ts'; +import { DenoSqlite3, DenoSqlite3Dialect, FileMigrationProvider, Kysely, Migrator } from '@/deps.ts'; import { Conf } from '@/config.ts'; import { getPragma, setPragma } from '@/pragma.ts'; @@ -57,7 +57,7 @@ interface UnattachedMediaRow { } const db = new Kysely({ - dialect: new DenoSqliteDialect({ + dialect: new DenoSqlite3Dialect({ database: new DenoSqlite3(Conf.dbPath), }), }); diff --git a/src/deps.ts b/src/deps.ts index a02ad20..4afd54d 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -66,8 +66,8 @@ export { Migrator, type NullableInsertKeys, sql, -} from 'npm:kysely@^0.25.0'; -export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/v1.1.0/mod.ts'; +} from 'npm:kysely@^0.26.3'; +export { DenoSqlite3Dialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/v2.0.0/mod.ts'; export { default as tldts } from 'npm:tldts@^6.0.14'; export * as cron from 'https://deno.land/x/deno_cron@v1.0.0/cron.ts'; export { S3Client } from 'https://deno.land/x/s3_lite_client@0.6.1/mod.ts'; From 89b74217b62338d67d328333ec5aaa2fd116876d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 1 Dec 2023 18:15:39 -0600 Subject: [PATCH 06/10] Use the SqliteWorker with our new Kysely dialect --- src/db.ts | 10 +++++++--- src/deps.ts | 4 +++- src/workers/sqlite.ts | 14 +++++++------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/db.ts b/src/db.ts index 6a7778f..c05abe6 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,9 +1,10 @@ import fs from 'node:fs/promises'; import path from 'node:path'; -import { DenoSqlite3, DenoSqlite3Dialect, FileMigrationProvider, Kysely, Migrator } from '@/deps.ts'; +import { PolySqliteDialect, FileMigrationProvider, Kysely, Migrator } from '@/deps.ts'; import { Conf } from '@/config.ts'; import { getPragma, setPragma } from '@/pragma.ts'; +import { sqliteWorker } from '@/workers.ts'; interface DittoDB { events: EventRow; @@ -56,9 +57,12 @@ interface UnattachedMediaRow { uploaded_at: Date; } +await sqliteWorker.ready; +await sqliteWorker.open(); + const db = new Kysely({ - dialect: new DenoSqlite3Dialect({ - database: new DenoSqlite3(Conf.dbPath), + dialect: new PolySqliteDialect({ + database: sqliteWorker, }), }); diff --git a/src/deps.ts b/src/deps.ts index 4afd54d..6cbfa9c 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -63,11 +63,13 @@ export { FileMigrationProvider, type Insertable, Kysely, + type QueryResult, + type CompiledQuery, Migrator, type NullableInsertKeys, sql, } from 'npm:kysely@^0.26.3'; -export { DenoSqlite3Dialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/v2.0.0/mod.ts'; +export { PolySqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/v2.0.0/mod.ts'; export { default as tldts } from 'npm:tldts@^6.0.14'; export * as cron from 'https://deno.land/x/deno_cron@v1.0.0/cron.ts'; export { S3Client } from 'https://deno.land/x/s3_lite_client@0.6.1/mod.ts'; diff --git a/src/workers/sqlite.ts b/src/workers/sqlite.ts index 22f39e3..8d23f20 100644 --- a/src/workers/sqlite.ts +++ b/src/workers/sqlite.ts @@ -1,8 +1,4 @@ -interface QueryResult { - rows: unknown[]; - numAffectedRows: bigint; - insertId: bigint; -} +import type { CompiledQuery, QueryResult } from '@/deps.ts'; class SqliteWorker { #path: string; @@ -29,9 +25,9 @@ class SqliteWorker { return this.#call(['open', [this.#path]]); } - async query(sql: string, params?: any): Promise { + async executeQuery({ sql, parameters }: CompiledQuery): Promise> { await this.ready; - return this.#call(['query', [sql, params]]); + return this.#call(['query', [sql, parameters]]); } #call(msg: [string, unknown[]]): Promise { @@ -51,6 +47,10 @@ class SqliteWorker { this.#worker.addEventListener('message', handleEvent); }); } + + async destroy() { + this.#worker.terminate(); + } } export default SqliteWorker; From 1ad7eeb96199681fc2da89a2dd22bfc2a391f319 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 1 Dec 2023 19:28:33 -0600 Subject: [PATCH 07/10] deno fmt, deno lint --- src/db.ts | 2 +- src/deps.ts | 4 ++-- src/workers/sqlite.ts | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/db.ts b/src/db.ts index c05abe6..c5a3e17 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,7 +1,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; -import { PolySqliteDialect, FileMigrationProvider, Kysely, Migrator } from '@/deps.ts'; +import { FileMigrationProvider, Kysely, Migrator, PolySqliteDialect } from '@/deps.ts'; import { Conf } from '@/config.ts'; import { getPragma, setPragma } from '@/pragma.ts'; import { sqliteWorker } from '@/workers.ts'; diff --git a/src/deps.ts b/src/deps.ts index 6cbfa9c..8b98e17 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -60,13 +60,13 @@ export { export { Database as DenoSqlite3 } from 'https://deno.land/x/sqlite3@0.9.1/mod.ts'; export * as dotenv from 'https://deno.land/std@0.198.0/dotenv/mod.ts'; export { + type CompiledQuery, FileMigrationProvider, type Insertable, Kysely, - type QueryResult, - type CompiledQuery, Migrator, type NullableInsertKeys, + type QueryResult, sql, } from 'npm:kysely@^0.26.3'; export { PolySqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/v2.0.0/mod.ts'; diff --git a/src/workers/sqlite.ts b/src/workers/sqlite.ts index 8d23f20..0a6f35f 100644 --- a/src/workers/sqlite.ts +++ b/src/workers/sqlite.ts @@ -48,6 +48,7 @@ class SqliteWorker { }); } + // deno-lint-ignore require-await async destroy() { this.#worker.terminate(); } From b168175d147d1538425583012a4ea8d068cabdcf Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 2 Dec 2023 13:18:20 -0600 Subject: [PATCH 08/10] Upgrade Deno to v1.38.4 --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dbdee01..bcf4615 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.37.1 +image: denoland/deno:1.38.4 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index 04e52ae..1252389 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.38.3 +deno 1.38.4 From 455752e656efde65e9d9725bff0b0a9fc43e0d27 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 2 Dec 2023 13:18:35 -0600 Subject: [PATCH 09/10] SqliteWorker: use SQLite path from config --- src/workers.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/workers.ts b/src/workers.ts index ffd3e96..291217e 100644 --- a/src/workers.ts +++ b/src/workers.ts @@ -1,5 +1,6 @@ -import SqliteWorker from './workers/sqlite.ts'; +import { Conf } from '@/config.ts'; +import SqliteWorker from '@/workers/sqlite.ts'; -const sqliteWorker = new SqliteWorker('./data/db.sqlite3'); +const sqliteWorker = new SqliteWorker(Conf.dbPath); export { sqliteWorker }; From a4e7c241d0e858c77468ea47c1695848e77817d3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 3 Dec 2023 14:58:35 -0600 Subject: [PATCH 10/10] Rewrite SqliteWorker with Comlink --- src/db.ts | 6 ++--- src/workers.ts | 6 ----- src/workers/sqlite.ts | 48 ++++++++++++---------------------- src/workers/sqlite.worker.ts | 50 ++++++++++++++---------------------- 4 files changed, 38 insertions(+), 72 deletions(-) delete mode 100644 src/workers.ts diff --git a/src/db.ts b/src/db.ts index c5a3e17..a722d4e 100644 --- a/src/db.ts +++ b/src/db.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import { FileMigrationProvider, Kysely, Migrator, PolySqliteDialect } from '@/deps.ts'; import { Conf } from '@/config.ts'; import { getPragma, setPragma } from '@/pragma.ts'; -import { sqliteWorker } from '@/workers.ts'; +import SqliteWorker from '@/workers/sqlite.ts'; interface DittoDB { events: EventRow; @@ -57,8 +57,8 @@ interface UnattachedMediaRow { uploaded_at: Date; } -await sqliteWorker.ready; -await sqliteWorker.open(); +const sqliteWorker = new SqliteWorker(); +await sqliteWorker.open(Conf.dbPath); const db = new Kysely({ dialect: new PolySqliteDialect({ diff --git a/src/workers.ts b/src/workers.ts deleted file mode 100644 index 291217e..0000000 --- a/src/workers.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { Conf } from '@/config.ts'; -import SqliteWorker from '@/workers/sqlite.ts'; - -const sqliteWorker = new SqliteWorker(Conf.dbPath); - -export { sqliteWorker }; diff --git a/src/workers/sqlite.ts b/src/workers/sqlite.ts index 0a6f35f..a6d2fac 100644 --- a/src/workers/sqlite.ts +++ b/src/workers/sqlite.ts @@ -1,15 +1,18 @@ +import { Comlink } from '@/deps.ts'; + +import type { SqliteWorker as _SqliteWorker } from './sqlite.worker.ts'; import type { CompiledQuery, QueryResult } from '@/deps.ts'; class SqliteWorker { - #path: string; #worker: Worker; - ready: Promise; + #client: ReturnType>; + #ready: Promise; - constructor(path: string) { - this.#path = path; + constructor() { this.#worker = new Worker(new URL('./sqlite.worker.ts', import.meta.url).href, { type: 'module' }); + this.#client = Comlink.wrap(this.#worker); - this.ready = new Promise((resolve) => { + this.#ready = new Promise((resolve) => { const handleEvent = (event: MessageEvent) => { if (event.data[0] === 'ready') { this.#worker.removeEventListener('message', handleEvent); @@ -20,37 +23,18 @@ class SqliteWorker { }); } - async open(): Promise { - await this.ready; - return this.#call(['open', [this.#path]]); + async open(path: string): Promise { + await this.#ready; + return this.#client.open(path); } - async executeQuery({ sql, parameters }: CompiledQuery): Promise> { - await this.ready; - return this.#call(['query', [sql, parameters]]); + async executeQuery(query: CompiledQuery): Promise> { + await this.#ready; + return this.#client.executeQuery(query) as Promise>; } - #call(msg: [string, unknown[]]): Promise { - const id = crypto.randomUUID(); - - this.#worker.postMessage([id, msg]); - - // TODO: use a hashmap instead of an event listener for better performance. - return new Promise((resolve) => { - const handleEvent = (event: MessageEvent<[string, T]>) => { - const [_id, result] = event.data; - if (_id === id) { - this.#worker.removeEventListener('message', handleEvent); - resolve(result); - } - }; - this.#worker.addEventListener('message', handleEvent); - }); - } - - // deno-lint-ignore require-await - async destroy() { - this.#worker.terminate(); + destroy(): Promise { + return this.#client.destroy(); } } diff --git a/src/workers/sqlite.worker.ts b/src/workers/sqlite.worker.ts index 69acf6f..c79f078 100644 --- a/src/workers/sqlite.worker.ts +++ b/src/workers/sqlite.worker.ts @@ -1,38 +1,26 @@ /// -import { DenoSqlite3 } from '@/deps.ts'; +import { Comlink, type CompiledQuery, DenoSqlite3, type QueryResult } from '@/deps.ts'; -let db: DenoSqlite3; +let db: DenoSqlite3 | undefined; -type Msg = - | ['open', [string]] - | ['query', [string, unknown[]]]; +export const SqliteWorker = { + open(path: string): void { + db = new DenoSqlite3(path); + }, + executeQuery({ sql, parameters }: CompiledQuery): QueryResult { + if (!db) throw new Error('Database not open'); + return { + rows: db.prepare(sql).all(...parameters as any[]) as R[], + numAffectedRows: BigInt(db.changes), + insertId: BigInt(db.lastInsertRowId), + }; + }, + destroy() { + db?.close(); + }, +}; -function call([cmd, args]: Msg) { - switch (cmd) { - case 'open': - return handleOpen(args[0]); - case 'query': - return handleQuery(args[0], args[1]); - } -} - -function handleOpen(path: string): void { - db = new DenoSqlite3(path); -} - -function handleQuery(sql: string, params: any[] = []) { - return { - rows: db.prepare(sql).all(...params), - numAffectedRows: BigInt(db.changes), - insertId: BigInt(db.lastInsertRowId), - }; -} - -self.addEventListener('message', (event: MessageEvent<[string, Msg]>) => { - const [id, msg] = event.data; - const result = call(msg); - self.postMessage([id, result]); -}); +Comlink.expose(SqliteWorker); self.postMessage(['ready']);