From 87264eeef10ac091e4ca222e79c62074fb3e6bcd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 1 May 2024 19:15:20 -0500 Subject: [PATCH 1/3] Remove `relays` table from the database, track them with a NIP-65 admin event --- deno.json | 1 - scripts/relays.ts | 23 ------------------- src/db/DittoTables.ts | 7 ------ src/db/migrations/017_rm_relays.ts | 14 +++++++++++ src/db/relays.ts | 37 ------------------------------ src/pipeline.ts | 20 +--------------- src/pool.ts | 16 +++++++++++-- src/utils.ts | 8 ------- 8 files changed, 29 insertions(+), 97 deletions(-) delete mode 100644 scripts/relays.ts create mode 100644 src/db/migrations/017_rm_relays.ts delete mode 100644 src/db/relays.ts diff --git a/deno.json b/deno.json index 167176f..b4e834a 100644 --- a/deno.json +++ b/deno.json @@ -7,7 +7,6 @@ "debug": "deno run -A --inspect src/server.ts", "test": "DATABASE_URL=\"sqlite://:memory:\" deno test -A", "check": "deno check src/server.ts", - "relays:sync": "deno run -A scripts/relays.ts sync", "nsec": "deno run scripts/nsec.ts", "admin:event": "deno run -A scripts/admin-event.ts", "admin:role": "deno run -A scripts/admin-role.ts" diff --git a/scripts/relays.ts b/scripts/relays.ts deleted file mode 100644 index 84f8a7e..0000000 --- a/scripts/relays.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { addRelays } from '@/db/relays.ts'; -import { filteredArray } from '@/schema.ts'; -import { relaySchema } from '@/utils.ts'; - -switch (Deno.args[0]) { - case 'sync': - await sync(Deno.args.slice(1)); - break; - default: - console.log('Usage: deno run -A scripts/relays.ts sync '); -} - -async function sync([url]: string[]) { - if (!url) { - console.error('Error: please provide a URL'); - Deno.exit(1); - } - const response = await fetch(url); - const data = await response.json(); - const values = filteredArray(relaySchema).parse(data) as `wss://${string}`[]; - await addRelays(values, { active: true }); - console.log(`Done: added ${values.length} relays.`); -} diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index 79fec5d..d71f48a 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -2,7 +2,6 @@ export interface DittoTables { events: EventRow; events_fts: EventFTSRow; tags: TagRow; - relays: RelayRow; unattached_media: UnattachedMediaRow; author_stats: AuthorStatsRow; event_stats: EventStatsRow; @@ -45,12 +44,6 @@ interface TagRow { event_id: string; } -interface RelayRow { - url: string; - domain: string; - active: boolean; -} - interface UnattachedMediaRow { id: string; pubkey: string; diff --git a/src/db/migrations/017_rm_relays.ts b/src/db/migrations/017_rm_relays.ts new file mode 100644 index 0000000..70a274d --- /dev/null +++ b/src/db/migrations/017_rm_relays.ts @@ -0,0 +1,14 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema.dropTable('relays').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .createTable('relays') + .addColumn('url', 'text', (col) => col.primaryKey()) + .addColumn('domain', 'text', (col) => col.notNull()) + .addColumn('active', 'boolean', (col) => col.notNull()) + .execute(); +} diff --git a/src/db/relays.ts b/src/db/relays.ts deleted file mode 100644 index da29b79..0000000 --- a/src/db/relays.ts +++ /dev/null @@ -1,37 +0,0 @@ -import tldts from 'tldts'; - -import { db } from '@/db.ts'; - -interface AddRelaysOpts { - active?: boolean; -} - -/** Inserts relays into the database, skipping duplicates. */ -function addRelays(relays: `wss://${string}`[], opts: AddRelaysOpts = {}) { - if (!relays.length) return Promise.resolve(); - const { active = false } = opts; - - const values = relays.map((url) => ({ - url: new URL(url).toString(), - domain: tldts.getDomain(url)!, - active, - })); - - return db.insertInto('relays') - .values(values) - .onConflict((oc) => oc.column('url').doNothing()) - .execute(); -} - -/** Get a list of all known active relay URLs. */ -async function getActiveRelays(): Promise { - const rows = await db - .selectFrom('relays') - .select('relays.url') - .where('relays.active', '=', true) - .execute(); - - return rows.map((row) => row.url); -} - -export { addRelays, getActiveRelays }; diff --git a/src/pipeline.ts b/src/pipeline.ts index f91626d..16876f2 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -5,7 +5,6 @@ import { sql } from 'kysely'; import { Conf } from '@/config.ts'; import { db } from '@/db.ts'; -import { addRelays } from '@/db/relays.ts'; import { deleteAttachedMedia } from '@/db/unattached-media.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { isEphemeralKind } from '@/kinds.ts'; @@ -14,7 +13,7 @@ import { updateStats } from '@/stats.ts'; import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; import { getTagSet } from '@/tags.ts'; -import { eventAge, isRelay, nostrDate, nostrNow, parseNip05, Time } from '@/utils.ts'; +import { eventAge, nostrDate, nostrNow, parseNip05, Time } from '@/utils.ts'; import { fetchWorker } from '@/workers/fetch.ts'; import { TrendsWorker } from '@/workers/trends.ts'; import { verifyEventWorker } from '@/workers/verify.ts'; @@ -59,7 +58,6 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { } } -/** Tracks known relays in the database. */ -function trackRelays(event: NostrEvent) { - const relays = new Set<`wss://${string}`>(); - - event.tags.forEach((tag) => { - if (['p', 'e', 'a'].includes(tag[0]) && isRelay(tag[2])) { - relays.add(tag[2]); - } - if (event.kind === 10002 && tag[0] === 'r' && isRelay(tag[1])) { - relays.add(tag[1]); - } - }); - - return addRelays([...relays]); -} - /** Queue related events to fetch. */ async function fetchRelatedEvents(event: DittoEvent, signal: AbortSignal) { if (!event.user) { diff --git a/src/pool.ts b/src/pool.ts index 48d5e1f..3ac1a1d 100644 --- a/src/pool.ts +++ b/src/pool.ts @@ -1,8 +1,20 @@ import { RelayPoolWorker } from 'nostr-relaypool'; -import { getActiveRelays } from '@/db/relays.ts'; +import { Storages } from '@/storages.ts'; +import { Conf } from '@/config.ts'; -const activeRelays = await getActiveRelays(); +const [relayList] = await Storages.db.query([ + { kinds: [10002], authors: [Conf.pubkey], limit: 1 }, +]); + +const tags = relayList?.tags ?? []; + +const activeRelays = tags.reduce((acc, [name, url, marker]) => { + if (name === 'r' && !marker) { + acc.push(url); + } + return acc; +}, []); console.log(`pool: connecting to ${activeRelays.length} relays.`); diff --git a/src/utils.ts b/src/utils.ts index 085d8af..65e4d59 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -82,12 +82,6 @@ async function sha256(message: string): Promise { return hashHex; } -/** Schema to parse a relay URL. */ -const relaySchema = z.string().max(255).startsWith('wss://').url(); - -/** Check whether the value is a valid relay URL. */ -const isRelay = (relay: string): relay is `wss://${string}` => relaySchema.safeParse(relay).success; - /** Deduplicate events by ID. */ function dedupeEvents(events: NostrEvent[]): NostrEvent[] { return [...new Map(events.map((event) => [event.id, event])).values()]; @@ -143,13 +137,11 @@ export { eventMatchesTemplate, findTag, isNostrId, - isRelay, isURL, type Nip05, nostrDate, nostrNow, parseNip05, - relaySchema, sha256, }; From 439dfca311fae8559b8cb4ce39d76ec906bbf458 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 1 May 2024 19:16:12 -0500 Subject: [PATCH 2/3] Fix kysely imports in migrations --- src/db/migrations/000_create_events.ts | 2 +- src/db/migrations/001_add_relays.ts | 2 +- src/db/migrations/003_events_admin.ts | 2 +- src/db/migrations/004_add_user_indexes.ts | 2 +- src/db/migrations/006_pragma.ts | 2 +- src/db/migrations/007_unattached_media.ts | 2 +- src/db/migrations/008_wal.ts | 2 +- src/db/migrations/009_add_stats.ts | 2 +- src/db/migrations/010_drop_users.ts | 2 +- src/db/migrations/011_kind_author_index.ts | 2 +- src/db/migrations/012_tags_composite_index.ts | 2 +- src/db/migrations/013_soft_deletion.ts | 2 +- src/db/migrations/014_stats_indexes.ts.ts | 2 +- src/db/migrations/015_add_pubkey_domains.ts | 2 +- src/db/migrations/016_pubkey_domains_updated_at.ts | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/db/migrations/000_create_events.ts b/src/db/migrations/000_create_events.ts index 158551b..f08a614 100644 --- a/src/db/migrations/000_create_events.ts +++ b/src/db/migrations/000_create_events.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/src/db/migrations/001_add_relays.ts b/src/db/migrations/001_add_relays.ts index 1415f5f..11c6884 100644 --- a/src/db/migrations/001_add_relays.ts +++ b/src/db/migrations/001_add_relays.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/src/db/migrations/003_events_admin.ts b/src/db/migrations/003_events_admin.ts index 8469fc2..388a3a4 100644 --- a/src/db/migrations/003_events_admin.ts +++ b/src/db/migrations/003_events_admin.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(_db: Kysely): Promise { } diff --git a/src/db/migrations/004_add_user_indexes.ts b/src/db/migrations/004_add_user_indexes.ts index 929181c..fca9c5f 100644 --- a/src/db/migrations/004_add_user_indexes.ts +++ b/src/db/migrations/004_add_user_indexes.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(_db: Kysely): Promise { } diff --git a/src/db/migrations/006_pragma.ts b/src/db/migrations/006_pragma.ts index 2639e81..f20ee9b 100644 --- a/src/db/migrations/006_pragma.ts +++ b/src/db/migrations/006_pragma.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(_db: Kysely): Promise { } diff --git a/src/db/migrations/007_unattached_media.ts b/src/db/migrations/007_unattached_media.ts index 1887111..a36c5d3 100644 --- a/src/db/migrations/007_unattached_media.ts +++ b/src/db/migrations/007_unattached_media.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/src/db/migrations/008_wal.ts b/src/db/migrations/008_wal.ts index 2639e81..f20ee9b 100644 --- a/src/db/migrations/008_wal.ts +++ b/src/db/migrations/008_wal.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(_db: Kysely): Promise { } diff --git a/src/db/migrations/009_add_stats.ts b/src/db/migrations/009_add_stats.ts index 60d9447..ef1c443 100644 --- a/src/db/migrations/009_add_stats.ts +++ b/src/db/migrations/009_add_stats.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/src/db/migrations/010_drop_users.ts b/src/db/migrations/010_drop_users.ts index 6cd83c0..c36f2fa 100644 --- a/src/db/migrations/010_drop_users.ts +++ b/src/db/migrations/010_drop_users.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.dropTable('users').ifExists().execute(); diff --git a/src/db/migrations/011_kind_author_index.ts b/src/db/migrations/011_kind_author_index.ts index da21988..3e7d010 100644 --- a/src/db/migrations/011_kind_author_index.ts +++ b/src/db/migrations/011_kind_author_index.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/src/db/migrations/012_tags_composite_index.ts b/src/db/migrations/012_tags_composite_index.ts index 8769289..412fa59 100644 --- a/src/db/migrations/012_tags_composite_index.ts +++ b/src/db/migrations/012_tags_composite_index.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.dropIndex('idx_tags_tag').execute(); diff --git a/src/db/migrations/013_soft_deletion.ts b/src/db/migrations/013_soft_deletion.ts index 3856ca0..df19da5 100644 --- a/src/db/migrations/013_soft_deletion.ts +++ b/src/db/migrations/013_soft_deletion.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.alterTable('events').addColumn('deleted_at', 'integer').execute(); diff --git a/src/db/migrations/014_stats_indexes.ts.ts b/src/db/migrations/014_stats_indexes.ts.ts index d9071c6..0f27a7f 100644 --- a/src/db/migrations/014_stats_indexes.ts.ts +++ b/src/db/migrations/014_stats_indexes.ts.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.createIndex('idx_author_stats_pubkey').on('author_stats').column('pubkey').execute(); diff --git a/src/db/migrations/015_add_pubkey_domains.ts b/src/db/migrations/015_add_pubkey_domains.ts index 0b5fe29..4b7e23c 100644 --- a/src/db/migrations/015_add_pubkey_domains.ts +++ b/src/db/migrations/015_add_pubkey_domains.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema diff --git a/src/db/migrations/016_pubkey_domains_updated_at.ts b/src/db/migrations/016_pubkey_domains_updated_at.ts index 3a000c1..8b1f75d 100644 --- a/src/db/migrations/016_pubkey_domains_updated_at.ts +++ b/src/db/migrations/016_pubkey_domains_updated_at.ts @@ -1,4 +1,4 @@ -import { Kysely } from '@/deps.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema From 76f30f5cc76f58d923070f17fcddf56b0499fc5b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 1 May 2024 19:19:52 -0500 Subject: [PATCH 3/3] utils: remove unused import of zod --- src/utils.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils.ts b/src/utils.ts index 65e4d59..c56abb8 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,5 @@ import { NostrEvent } from '@nostrify/nostrify'; import { EventTemplate, getEventHash, nip19 } from 'nostr-tools'; -import { z } from 'zod'; import { nostrIdSchema } from '@/schemas/nostr.ts';