From 2ede439005fab791af01ebe36d63a0755b5c68d3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 May 2024 09:27:22 -0500 Subject: [PATCH 1/3] Upgrade Nostrify to v0.19.1, fix phantom deletions --- deno.json | 2 +- src/db/DittoDB.ts | 2 +- src/pipeline.ts | 21 ------- src/storages/EventsDB.test.ts | 106 +++++++++++++++++++++++++++++++--- src/storages/EventsDB.ts | 10 ++++ 5 files changed, 110 insertions(+), 31 deletions(-) diff --git a/deno.json b/deno.json index c8eba2a..25ec2ed 100644 --- a/deno.json +++ b/deno.json @@ -21,7 +21,7 @@ "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.19.0", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.19.1", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", diff --git a/src/db/DittoDB.ts b/src/db/DittoDB.ts index 9c3b280..68fdc62 100644 --- a/src/db/DittoDB.ts +++ b/src/db/DittoDB.ts @@ -41,7 +41,7 @@ export class DittoDB { } /** Migrate the database to the latest version. */ - private static async migrate(kysely: Kysely) { + static async migrate(kysely: Kysely) { const migrator = new Migrator({ db: kysely, provider: new FileMigrationProvider({ diff --git a/src/pipeline.ts b/src/pipeline.ts index 6f487ed..6e4ebb1 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -45,7 +45,6 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { - if (event.kind === 5) { - const ids = getTagSet(event.tags, 'e'); - const store = await Storages.db(); - - if (event.pubkey === Conf.pubkey) { - await store.remove([{ ids: [...ids] }], { signal }); - } else { - const events = await store.query( - [{ ids: [...ids], authors: [event.pubkey] }], - { signal }, - ); - - const deleteIds = events.map(({ id }) => id); - await store.remove([{ ids: deleteIds }], { signal }); - } - } -} - /** Track whenever a hashtag is used, for processing trending tags. */ async function trackHashtags(event: NostrEvent): Promise { const date = nostrDate(event.created_at); diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 10f0d11..95c446c 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -1,22 +1,39 @@ +import { Database as Sqlite } from '@db/sqlite'; +import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; import { assertEquals, assertRejects } from '@std/assert'; +import { Kysely } from 'kysely'; +import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; +import { EventsDB } from '@/storages/EventsDB.ts'; import event0 from '~/fixtures/events/event-0.json' with { type: 'json' }; import event1 from '~/fixtures/events/event-1.json' with { type: 'json' }; -import { EventsDB } from '@/storages/EventsDB.ts'; - -const kysely = await DittoDB.getInstance(); -const eventsDB = new EventsDB(kysely); +/** Create in-memory database for testing. */ +const createDB = async () => { + const kysely = new Kysely({ + dialect: new DenoSqlite3Dialect({ + database: new Sqlite(':memory:'), + }), + }); + const eventsDB = new EventsDB(kysely); + await DittoDB.migrate(kysely); + return { eventsDB, kysely }; +}; Deno.test('count filters', async () => { + const { eventsDB } = await createDB(); + assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 0); await eventsDB.event(event1); assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 1); }); Deno.test('insert and filter events', async () => { + const { eventsDB } = await createDB(); + await eventsDB.event(event1); assertEquals(await eventsDB.query([{ kinds: [1] }]), [event1]); @@ -30,6 +47,8 @@ Deno.test('insert and filter events', async () => { }); Deno.test('query events with domain search filter', async () => { + const { eventsDB, kysely } = await createDB(); + await eventsDB.event(event1); assertEquals(await eventsDB.query([{}]), [event1]); @@ -46,13 +65,84 @@ Deno.test('query events with domain search filter', async () => { }); Deno.test('delete events', async () => { - await eventsDB.event(event1); - assertEquals(await eventsDB.query([{ kinds: [1] }]), [event1]); - await eventsDB.remove([{ kinds: [1] }]); - assertEquals(await eventsDB.query([{ kinds: [1] }]), []); + const { eventsDB } = await createDB(); + + const [one, two] = [ + { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }, + { id: '2', kind: 1, pubkey: 'abc', content: 'yolo fam', created_at: 2, sig: '', tags: [] }, + ]; + + await eventsDB.event(one); + await eventsDB.event(two); + + // Sanity check + assertEquals(await eventsDB.query([{ kinds: [1] }]), [two, one]); + + await eventsDB.event({ + kind: 5, + pubkey: one.pubkey, + tags: [['e', one.id]], + created_at: 0, + content: '', + id: '', + sig: '', + }); + + assertEquals(await eventsDB.query([{ kinds: [1] }]), [two]); +}); + +Deno.test("user cannot delete another user's event", async () => { + const { eventsDB } = await createDB(); + + const event = { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }; + await eventsDB.event(event); + + // Sanity check + assertEquals(await eventsDB.query([{ kinds: [1] }]), [event]); + + await eventsDB.event({ + kind: 5, + pubkey: 'def', // different pubkey + tags: [['e', event.id]], + created_at: 0, + content: '', + id: '', + sig: '', + }); + + assertEquals(await eventsDB.query([{ kinds: [1] }]), [event]); +}); + +Deno.test('admin can delete any event', async () => { + const { eventsDB } = await createDB(); + + const [one, two] = [ + { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] }, + { id: '2', kind: 1, pubkey: 'abc', content: 'yolo fam', created_at: 2, sig: '', tags: [] }, + ]; + + await eventsDB.event(one); + await eventsDB.event(two); + + // Sanity check + assertEquals(await eventsDB.query([{ kinds: [1] }]), [two, one]); + + await eventsDB.event({ + kind: 5, + pubkey: Conf.pubkey, // Admin pubkey + tags: [['e', one.id]], + created_at: 0, + content: '', + id: '', + sig: '', + }); + + assertEquals(await eventsDB.query([{ kinds: [1] }]), [two]); }); Deno.test('inserting replaceable events', async () => { + const { eventsDB } = await createDB(); + assertEquals((await eventsDB.count([{ kinds: [0], authors: [event0.pubkey] }])).count, 0); await eventsDB.event(event0); diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 5f1acbb..f2789d3 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -8,6 +8,7 @@ import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { normalizeFilters } from '@/filter.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; +import { getTagSet } from '@/tags.ts'; import { isNostrId, isURL } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; @@ -51,9 +52,18 @@ class EventsDB implements NStore { async event(event: NostrEvent, _opts?: { signal?: AbortSignal }): Promise { event = purifyEvent(event); this.console.debug('EVENT', JSON.stringify(event)); + await this.deleteEventsAdmin(event); return this.store.event(event); } + /** The DITTO_NSEC can delete any event from the database. NDatabase already handles user deletions. */ + async deleteEventsAdmin(event: NostrEvent): Promise { + if (event.kind === 5 && event.pubkey === Conf.pubkey) { + const ids = getTagSet(event.tags, 'e'); + await this.remove([{ ids: [...ids] }]); + } + } + /** Get events for filters from the database. */ async query(filters: NostrFilter[], opts: { signal?: AbortSignal; limit?: number } = {}): Promise { filters = await this.expandFilters(filters); From 4df2c7ba9c69b259d2a98e3e46b2ac6652efb45e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 May 2024 10:29:14 -0500 Subject: [PATCH 2/3] Improve EventsDB error handling --- src/pipeline.ts | 13 ++------- src/storages/EventsDB.test.ts | 52 ++++++++++++++++++++++++++++++----- src/storages/EventsDB.ts | 29 +++++++++++++++++-- src/test.ts | 16 +++++++++++ 4 files changed, 90 insertions(+), 20 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 6e4ebb1..15d495e 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -122,17 +122,8 @@ async function storeEvent(event: DittoEvent, signal?: AbortSignal): Promise { @@ -140,16 +143,51 @@ Deno.test('admin can delete any event', async () => { assertEquals(await eventsDB.query([{ kinds: [1] }]), [two]); }); +Deno.test('throws a RelayError when inserting an event deleted by the admin', async () => { + const { eventsDB } = await createDB(); + + const event = genEvent(); + await eventsDB.event(event); + + const deletion = genEvent({ kind: 5, tags: [['e', event.id]] }, Conf.seckey); + await eventsDB.event(deletion); + + await assertRejects( + () => eventsDB.event(event), + RelayError, + 'event deleted by admin', + ); +}); + +Deno.test('throws a RelayError when inserting an event deleted by a user', async () => { + const { eventsDB } = await createDB(); + + const sk = generateSecretKey(); + + const event = genEvent({}, sk); + await eventsDB.event(event); + + const deletion = genEvent({ kind: 5, tags: [['e', event.id]] }, sk); + await eventsDB.event(deletion); + + await assertRejects( + () => eventsDB.event(event), + RelayError, + 'event deleted by user', + ); +}); + Deno.test('inserting replaceable events', async () => { const { eventsDB } = await createDB(); - assertEquals((await eventsDB.count([{ kinds: [0], authors: [event0.pubkey] }])).count, 0); + const event = event0; + await eventsDB.event(event); - await eventsDB.event(event0); - await assertRejects(() => eventsDB.event(event0)); - assertEquals((await eventsDB.count([{ kinds: [0], authors: [event0.pubkey] }])).count, 1); + const olderEvent = { ...event, id: '123', created_at: event.created_at - 1 }; + await eventsDB.event(olderEvent); + assertEquals(await eventsDB.query([{ kinds: [0], authors: [event.pubkey] }]), [event]); - const changeEvent = { ...event0, id: '123', created_at: event0.created_at + 1 }; - await eventsDB.event(changeEvent); - assertEquals(await eventsDB.query([{ kinds: [0] }]), [changeEvent]); + const newerEvent = { ...event, id: '123', created_at: event.created_at + 1 }; + await eventsDB.event(newerEvent); + assertEquals(await eventsDB.query([{ kinds: [0] }]), [newerEvent]); }); diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index f2789d3..aac8e52 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -11,6 +11,7 @@ import { purifyEvent } from '@/storages/hydrate.ts'; import { getTagSet } from '@/tags.ts'; import { isNostrId, isURL } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; +import { RelayError } from '@/RelayError.ts'; /** Function to decide whether or not to index a tag. */ type TagCondition = ({ event, count, value }: { @@ -52,12 +53,36 @@ class EventsDB implements NStore { async event(event: NostrEvent, _opts?: { signal?: AbortSignal }): Promise { event = purifyEvent(event); this.console.debug('EVENT', JSON.stringify(event)); + + if (await this.isDeletedAdmin(event)) { + throw new RelayError('blocked', 'event deleted by admin'); + } + await this.deleteEventsAdmin(event); - return this.store.event(event); + + try { + await this.store.event(event); + } catch (e) { + if (e.message === 'Cannot add a deleted event') { + throw new RelayError('blocked', 'event deleted by user'); + } else if (e.message === 'Cannot replace an event with an older event') { + return; + } else { + this.console.debug('ERROR', e.message); + } + } + } + + /** Check if an event has been deleted by the admin. */ + private async isDeletedAdmin(event: NostrEvent): Promise { + const [deletion] = await this.query([ + { kinds: [5], authors: [Conf.pubkey], '#e': [event.id], limit: 1 }, + ]); + return !!deletion; } /** The DITTO_NSEC can delete any event from the database. NDatabase already handles user deletions. */ - async deleteEventsAdmin(event: NostrEvent): Promise { + private async deleteEventsAdmin(event: NostrEvent): Promise { if (event.kind === 5 && event.pubkey === Conf.pubkey) { const ids = getTagSet(event.tags, 'e'); await this.remove([{ ids: [...ids] }]); diff --git a/src/test.ts b/src/test.ts index 4586282..ea9c8fa 100644 --- a/src/test.ts +++ b/src/test.ts @@ -1,7 +1,23 @@ import { NostrEvent } from '@nostrify/nostrify'; +import { finalizeEvent, generateSecretKey } from 'nostr-tools'; + +import { purifyEvent } from '@/storages/hydrate.ts'; /** Import an event fixture by name in tests. */ export async function eventFixture(name: string): Promise { const result = await import(`~/fixtures/events/${name}.json`, { with: { type: 'json' } }); return structuredClone(result.default); } + +/** Generate an event for use in tests. */ +export function genEvent(t: Partial = {}, sk: Uint8Array = generateSecretKey()): NostrEvent { + const event = finalizeEvent({ + kind: 255, + created_at: 0, + content: '', + tags: [], + ...t, + }, sk); + + return purifyEvent(event); +} From 031a3eac04b686dc53202fdc0177c90d1b82d1de Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 May 2024 10:30:54 -0500 Subject: [PATCH 3/3] EventsDB.test: import order --- src/storages/EventsDB.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 939406a..2f34379 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -2,6 +2,7 @@ import { Database as Sqlite } from '@db/sqlite'; import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; import { assertEquals, assertRejects } from '@std/assert'; import { Kysely } from 'kysely'; +import { generateSecretKey } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; @@ -12,7 +13,6 @@ import { genEvent } from '@/test.ts'; import event0 from '~/fixtures/events/event-0.json' with { type: 'json' }; import event1 from '~/fixtures/events/event-1.json' with { type: 'json' }; -import { generateSecretKey } from 'nostr-tools'; /** Create in-memory database for testing. */ const createDB = async () => {