Merge branch 'phantom-deletions' into 'main'

Upgrade Nostrify to v0.19.1, fix phantom deletions

See merge request soapbox-pub/ditto!267

deno.json

@@ -21,7 +21,7 @@
     "@db/sqlite": "jsr:@db/sqlite@^0.11.1",
     "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1",
     "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0",
-    "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.19.0",
+    "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.19.1",
     "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs",
     "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0",
     "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0",

src/db/DittoDB.ts

@@ -41,7 +41,7 @@ export class DittoDB {
   }
 
   /** Migrate the database to the latest version. */
-  private static async migrate(kysely: Kysely<DittoTables>) {
+  static async migrate(kysely: Kysely<DittoTables>) {
     const migrator = new Migrator({
       db: kysely,
       provider: new FileMigrationProvider({

src/pipeline.ts

@@ -45,7 +45,6 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise<void
   await Promise.all([
     storeEvent(event, signal),
     parseMetadata(event, signal),
-    processDeletions(event, signal),
     DVM.event(event),
     trackHashtags(event),
     fetchRelatedEvents(event),
@@ -123,17 +122,8 @@ async function storeEvent(event: DittoEvent, signal?: AbortSignal): Promise<void
   if (NKinds.ephemeral(event.kind)) return;
   const store = await Storages.db();
 
-  const [deletion] = await store.query(
-    [{ kinds: [5], authors: [Conf.pubkey, event.pubkey], '#e': [event.id], limit: 1 }],
-    { signal },
-  );
-
-  if (deletion) {
-    return Promise.reject(new RelayError('blocked', 'event was deleted'));
-  } else {
   await updateStats(event).catch(debug);
-    await store.event(event, { signal }).catch(debug);
+  await store.event(event, { signal });
-  }
 }
 
 /** Parse kind 0 metadata and track indexes in the database. */
@@ -174,26 +164,6 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise<vo
   }
 }
 
-/** Query to-be-deleted events, ensure their pubkey matches, then delete them from the database. */
-async function processDeletions(event: NostrEvent, signal: AbortSignal): Promise<void> {
-  if (event.kind === 5) {
-    const ids = getTagSet(event.tags, 'e');
-    const store = await Storages.db();
-
-    if (event.pubkey === Conf.pubkey) {
-      await store.remove([{ ids: [...ids] }], { signal });
-    } else {
-      const events = await store.query(
-        [{ ids: [...ids], authors: [event.pubkey] }],
-        { signal },
-      );
-
-      const deleteIds = events.map(({ id }) => id);
-      await store.remove([{ ids: deleteIds }], { signal });
-    }
-  }
-}
-
 /** Track whenever a hashtag is used, for processing trending tags. */
 async function trackHashtags(event: NostrEvent): Promise<void> {
   const date = nostrDate(event.created_at);

src/storages/EventsDB.test.ts

@@ -1,22 +1,42 @@
+import { Database as Sqlite } from '@db/sqlite';
+import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite';
 import { assertEquals, assertRejects } from '@std/assert';
+import { Kysely } from 'kysely';
+import { generateSecretKey } from 'nostr-tools';
 
+import { Conf } from '@/config.ts';
 import { DittoDB } from '@/db/DittoDB.ts';
+import { DittoTables } from '@/db/DittoTables.ts';
+import { RelayError } from '@/RelayError.ts';
+import { EventsDB } from '@/storages/EventsDB.ts';
+import { genEvent } from '@/test.ts';
 
 import event0 from '~/fixtures/events/event-0.json' with { type: 'json' };
 import event1 from '~/fixtures/events/event-1.json' with { type: 'json' };
 
-import { EventsDB } from '@/storages/EventsDB.ts';
+/** Create in-memory database for testing. */
-
+const createDB = async () => {
-const kysely = await DittoDB.getInstance();
+  const kysely = new Kysely<DittoTables>({
+    dialect: new DenoSqlite3Dialect({
+      database: new Sqlite(':memory:'),
+    }),
+  });
   const eventsDB = new EventsDB(kysely);
+  await DittoDB.migrate(kysely);
+  return { eventsDB, kysely };
+};
 
 Deno.test('count filters', async () => {
+  const { eventsDB } = await createDB();
+
   assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 0);
   await eventsDB.event(event1);
   assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 1);
 });
 
 Deno.test('insert and filter events', async () => {
+  const { eventsDB } = await createDB();
+
   await eventsDB.event(event1);
 
   assertEquals(await eventsDB.query([{ kinds: [1] }]), [event1]);
@@ -30,6 +50,8 @@ Deno.test('insert and filter events', async () => {
 });
 
 Deno.test('query events with domain search filter', async () => {
+  const { eventsDB, kysely } = await createDB();
+
   await eventsDB.event(event1);
 
   assertEquals(await eventsDB.query([{}]), [event1]);
@@ -46,20 +68,126 @@ Deno.test('query events with domain search filter', async () => {
 });
 
 Deno.test('delete events', async () => {
-  await eventsDB.event(event1);
+  const { eventsDB } = await createDB();
-  assertEquals(await eventsDB.query([{ kinds: [1] }]), [event1]);
+
-  await eventsDB.remove([{ kinds: [1] }]);
+  const [one, two] = [
-  assertEquals(await eventsDB.query([{ kinds: [1] }]), []);
+    { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] },
+    { id: '2', kind: 1, pubkey: 'abc', content: 'yolo fam', created_at: 2, sig: '', tags: [] },
+  ];
+
+  await eventsDB.event(one);
+  await eventsDB.event(two);
+
+  // Sanity check
+  assertEquals(await eventsDB.query([{ kinds: [1] }]), [two, one]);
+
+  await eventsDB.event({
+    kind: 5,
+    pubkey: one.pubkey,
+    tags: [['e', one.id]],
+    created_at: 0,
+    content: '',
+    id: '',
+    sig: '',
+  });
+
+  assertEquals(await eventsDB.query([{ kinds: [1] }]), [two]);
+});
+
+Deno.test("user cannot delete another user's event", async () => {
+  const { eventsDB } = await createDB();
+
+  const event = { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] };
+  await eventsDB.event(event);
+
+  // Sanity check
+  assertEquals(await eventsDB.query([{ kinds: [1] }]), [event]);
+
+  await eventsDB.event({
+    kind: 5,
+    pubkey: 'def', // different pubkey
+    tags: [['e', event.id]],
+    created_at: 0,
+    content: '',
+    id: '',
+    sig: '',
+  });
+
+  assertEquals(await eventsDB.query([{ kinds: [1] }]), [event]);
+});
+
+Deno.test('admin can delete any event', async () => {
+  const { eventsDB } = await createDB();
+
+  const [one, two] = [
+    { id: '1', kind: 1, pubkey: 'abc', content: 'hello world', created_at: 1, sig: '', tags: [] },
+    { id: '2', kind: 1, pubkey: 'abc', content: 'yolo fam', created_at: 2, sig: '', tags: [] },
+  ];
+
+  await eventsDB.event(one);
+  await eventsDB.event(two);
+
+  // Sanity check
+  assertEquals(await eventsDB.query([{ kinds: [1] }]), [two, one]);
+
+  await eventsDB.event({
+    kind: 5,
+    pubkey: Conf.pubkey, // Admin pubkey
+    tags: [['e', one.id]],
+    created_at: 0,
+    content: '',
+    id: '',
+    sig: '',
+  });
+
+  assertEquals(await eventsDB.query([{ kinds: [1] }]), [two]);
+});
+
+Deno.test('throws a RelayError when inserting an event deleted by the admin', async () => {
+  const { eventsDB } = await createDB();
+
+  const event = genEvent();
+  await eventsDB.event(event);
+
+  const deletion = genEvent({ kind: 5, tags: [['e', event.id]] }, Conf.seckey);
+  await eventsDB.event(deletion);
+
+  await assertRejects(
+    () => eventsDB.event(event),
+    RelayError,
+    'event deleted by admin',
+  );
+});
+
+Deno.test('throws a RelayError when inserting an event deleted by a user', async () => {
+  const { eventsDB } = await createDB();
+
+  const sk = generateSecretKey();
+
+  const event = genEvent({}, sk);
+  await eventsDB.event(event);
+
+  const deletion = genEvent({ kind: 5, tags: [['e', event.id]] }, sk);
+  await eventsDB.event(deletion);
+
+  await assertRejects(
+    () => eventsDB.event(event),
+    RelayError,
+    'event deleted by user',
+  );
 });
 
 Deno.test('inserting replaceable events', async () => {
-  assertEquals((await eventsDB.count([{ kinds: [0], authors: [event0.pubkey] }])).count, 0);
+  const { eventsDB } = await createDB();
 
-  await eventsDB.event(event0);
+  const event = event0;
-  await assertRejects(() => eventsDB.event(event0));
+  await eventsDB.event(event);
-  assertEquals((await eventsDB.count([{ kinds: [0], authors: [event0.pubkey] }])).count, 1);
 
-  const changeEvent = { ...event0, id: '123', created_at: event0.created_at + 1 };
+  const olderEvent = { ...event, id: '123', created_at: event.created_at - 1 };
-  await eventsDB.event(changeEvent);
+  await eventsDB.event(olderEvent);
-  assertEquals(await eventsDB.query([{ kinds: [0] }]), [changeEvent]);
+  assertEquals(await eventsDB.query([{ kinds: [0], authors: [event.pubkey] }]), [event]);
+
+  const newerEvent = { ...event, id: '123', created_at: event.created_at + 1 };
+  await eventsDB.event(newerEvent);
+  assertEquals(await eventsDB.query([{ kinds: [0] }]), [newerEvent]);
 });

src/storages/EventsDB.ts

@@ -8,8 +8,10 @@ import { Conf } from '@/config.ts';
 import { DittoTables } from '@/db/DittoTables.ts';
 import { normalizeFilters } from '@/filter.ts';
 import { purifyEvent } from '@/storages/hydrate.ts';
+import { getTagSet } from '@/tags.ts';
 import { isNostrId, isURL } from '@/utils.ts';
 import { abortError } from '@/utils/abort.ts';
+import { RelayError } from '@/RelayError.ts';
 
 /** Function to decide whether or not to index a tag. */
 type TagCondition = ({ event, count, value }: {
@@ -51,7 +53,40 @@ class EventsDB implements NStore {
   async event(event: NostrEvent, _opts?: { signal?: AbortSignal }): Promise<void> {
     event = purifyEvent(event);
     this.console.debug('EVENT', JSON.stringify(event));
-    return this.store.event(event);
+
+    if (await this.isDeletedAdmin(event)) {
+      throw new RelayError('blocked', 'event deleted by admin');
+    }
+
+    await this.deleteEventsAdmin(event);
+
+    try {
+      await this.store.event(event);
+    } catch (e) {
+      if (e.message === 'Cannot add a deleted event') {
+        throw new RelayError('blocked', 'event deleted by user');
+      } else if (e.message === 'Cannot replace an event with an older event') {
+        return;
+      } else {
+        this.console.debug('ERROR', e.message);
+      }
+    }
+  }
+
+  /** Check if an event has been deleted by the admin. */
+  private async isDeletedAdmin(event: NostrEvent): Promise<boolean> {
+    const [deletion] = await this.query([
+      { kinds: [5], authors: [Conf.pubkey], '#e': [event.id], limit: 1 },
+    ]);
+    return !!deletion;
+  }
+
+  /** The DITTO_NSEC can delete any event from the database. NDatabase already handles user deletions. */
+  private async deleteEventsAdmin(event: NostrEvent): Promise<void> {
+    if (event.kind === 5 && event.pubkey === Conf.pubkey) {
+      const ids = getTagSet(event.tags, 'e');
+      await this.remove([{ ids: [...ids] }]);
+    }
   }
 
   /** Get events for filters from the database. */

src/test.ts

@@ -1,7 +1,23 @@
 import { NostrEvent } from '@nostrify/nostrify';
+import { finalizeEvent, generateSecretKey } from 'nostr-tools';
+
+import { purifyEvent } from '@/storages/hydrate.ts';
 
 /** Import an event fixture by name in tests. */
 export async function eventFixture(name: string): Promise<NostrEvent> {
   const result = await import(`~/fixtures/events/${name}.json`, { with: { type: 'json' } });
   return structuredClone(result.default);
 }
+
+/** Generate an event for use in tests. */
+export function genEvent(t: Partial<NostrEvent> = {}, sk: Uint8Array = generateSecretKey()): NostrEvent {
+  const event = finalizeEvent({
+    kind: 255,
+    created_at: 0,
+    content: '',
+    tags: [],
+    ...t,
+  }, sk);
+
+  return purifyEvent(event);
+}