Merge branch 'users-to-events' into 'main'

Convert users to events

See merge request soapbox-pub/ditto!89

.gitlab-ci.yml

@@ -16,4 +16,6 @@ lint:
 
 test:
   stage: test
-  script: deno task test
+  script: deno task test
+  variables:
+    DITTO_NSEC: nsec1zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs4rm7hz

docs/events.md

@@ -1,13 +1,25 @@
-# Ditto NIP-78 events
+# Ditto custom events
+
+## Users
+
+Ditto user events describe a pubkey's relationship with the Ditto server. They are parameterized replaceable events of kind `30361` where the `d` tag is a pubkey. These events are published by Ditto's internal admin keypair.
+
+User events have the following tags:
+
+- `d` - pubkey of the user.
+- `name` - NIP-05 username granted to the user, without the domain.
+- `role` - one of `admin` or `user`.
+
+## NIP-78
 
 [NIP-78](https://github.com/nostr-protocol/nips/blob/master/78.md) defines events of kind `30078` with a globally unique `d` tag. These events are queried by the `d` tag, which allows Ditto to store custom data on relays. Ditto uses reverse DNS names like `pub.ditto.<thing>` for `d` tags.
 
 The sections below describe the `content` field. Some are encrypted and some are not, depending on whether the data should be public. Also, some events are user events, and some are admin events.
 
-## `pub.ditto.blocks`
+### `pub.ditto.blocks`
 
-An encrypted array of blocked pubkeys, JSON stringified and encrypted with `nip07.encrypt`.
+An encrypted array of blocked pubkeys, JSON stringified in `content` and encrypted with `nip04.encrypt`.
 
-## `pub.ditto.frontendConfig`
+### `pub.ditto.frontendConfig`
 
 JSON data for Pleroma frontends served on `/api/pleroma/frontend_configurations`. Each key contains arbitrary data used by a different frontend.

scripts/db.ts

@@ -0,0 +1,44 @@
+import { Conf } from '@/config.ts';
+import { db } from '@/db.ts';
+import { eventsDB } from '@/db/events.ts';
+import { type Kysely } from '@/deps.ts';
+import { signAdminEvent } from '@/sign.ts';
+
+interface DB {
+  users: {
+    pubkey: string;
+    username: string;
+    inserted_at: Date;
+    admin: 0 | 1;
+  };
+}
+
+switch (Deno.args[0]) {
+  case 'users-to-events':
+    await usersToEvents();
+    break;
+  default:
+    console.log('Usage: deno run -A scripts/db.ts <command>');
+}
+
+async function usersToEvents() {
+  const { origin, host } = Conf.url;
+
+  for (const row of await (db as unknown as Kysely<DB>).selectFrom('users').selectAll().execute()) {
+    const event = await signAdminEvent({
+      kind: 30361,
+      tags: [
+        ['d', row.pubkey],
+        ['name', row.username],
+        ['role', row.admin ? 'admin' : 'user'],
+        ['origin', origin],
+        // NIP-31: https://github.com/nostr-protocol/nips/blob/master/31.md
+        ['alt', `@${row.username}@${host}'s account was updated by the admins of ${host}`],
+      ],
+      content: '',
+      created_at: Math.floor(new Date(row.inserted_at).getTime() / 1000),
+    });
+
+    await eventsDB.storeEvent(event);
+  }
+}

src/controllers/api/accounts.ts

@@ -42,7 +42,7 @@ const createAccountController: AppController = async (c) => {
       pubkey,
       username: result.data.username,
       inserted_at: new Date(),
-      admin: 0,
+      admin: false,
     });
 
     return c.json({

src/controllers/nostr/relay-info.ts

@@ -7,7 +7,7 @@ const relayInfoController: AppController = (c) => {
     description: 'Nostr and the Fediverse.',
     pubkey: Conf.pubkey,
     contact: `mailto:${Conf.adminEmail}`,
-    supported_nips: [1, 5, 9, 11, 45, 46, 98],
+    supported_nips: [1, 5, 9, 11, 16, 45, 46, 98],
     software: 'Ditto',
     version: '0.0.0',
     limitation: {

src/controllers/nostr/relay.ts

@@ -96,7 +96,7 @@ function connectStream(socket: WebSocket) {
 
   /** Handle COUNT. Return the number of events matching the filters. */
   async function handleCount([_, subId, ...rest]: ClientCOUNT): Promise<void> {
-    const count = await eventsDB.countFilters(prepareFilters(rest));
+    const count = await eventsDB.countEvents(prepareFilters(rest));
     send(['COUNT', subId, { count, approximate: false }]);
   }
 

src/db.ts

@@ -10,7 +10,6 @@ interface DittoDB {
   events: EventRow;
   events_fts: EventFTSRow;
   tags: TagRow;
-  users: UserRow;
   relays: RelayRow;
   unattached_media: UnattachedMediaRow;
   author_stats: AuthorStatsRow;
@@ -52,13 +51,6 @@ interface TagRow {
   event_id: string;
 }
 
-interface UserRow {
-  pubkey: string;
-  username: string;
-  inserted_at: Date;
-  admin: 0 | 1;
-}
-
 interface RelayRow {
   url: string;
   domain: string;
@@ -120,4 +112,4 @@ async function migrate() {
 
 await migrate();
 
-export { type AuthorStatsRow, db, type DittoDB, type EventRow, type EventStatsRow, type TagRow, type UserRow };
+export { type AuthorStatsRow, db, type DittoDB, type EventRow, type EventStatsRow, type TagRow };

src/db/events.test.ts

@@ -1,6 +1,7 @@
-import { assertEquals } from '@/deps-test.ts';
+import { assertEquals, assertRejects } from '@/deps-test.ts';
-import { insertUser } from '@/db/users.ts';
+import { buildUserEvent } from '@/db/users.ts';
 
+import event0 from '~/fixtures/events/event-0.json' assert { type: 'json' };
 import event1 from '~/fixtures/events/event-1.json' assert { type: 'json' };
 
 import { eventsDB as db } from './events.ts';
@@ -38,13 +39,26 @@ Deno.test('query events with local filter', async () => {
   assertEquals(await db.getEvents([{ local: true }]), []);
   assertEquals(await db.getEvents([{ local: false }]), [event1]);
 
-  await insertUser({
+  const userEvent = await buildUserEvent({
     username: 'alex',
     pubkey: event1.pubkey,
     inserted_at: new Date(),
-    admin: 0,
+    admin: false,
   });
+  await db.storeEvent(userEvent);
 
-  assertEquals(await db.getEvents([{ local: true }]), [event1]);
+  assertEquals(await db.getEvents([{ kinds: [1], local: true }]), [event1]);
-  assertEquals(await db.getEvents([{ local: false }]), []);
+  assertEquals(await db.getEvents([{ kinds: [1], local: false }]), []);
+});
+
+Deno.test('inserting replaceable events', async () => {
+  assertEquals(await db.countEvents([{ kinds: [0], authors: [event0.pubkey] }]), 0);
+
+  await db.storeEvent(event0);
+  await assertRejects(() => db.storeEvent(event0));
+  assertEquals(await db.countEvents([{ kinds: [0], authors: [event0.pubkey] }]), 1);
+
+  const changeEvent = { ...event0, id: '123', created_at: event0.created_at + 1 };
+  await db.storeEvent(changeEvent);
+  assertEquals(await db.getEvents([{ kinds: [0] }]), [changeEvent]);
 });

src/db/events.ts

@@ -1,7 +1,8 @@
+import { Conf } from '@/config.ts';
 import { db, type DittoDB } from '@/db.ts';
-import { Debug, type Event, type SelectQueryBuilder } from '@/deps.ts';
+import { Debug, type Event, Kysely, type SelectQueryBuilder } from '@/deps.ts';
 import { type DittoFilter } from '@/filter.ts';
-import { isParameterizedReplaceableKind } from '@/kinds.ts';
+import { isDittoInternalKind, isParameterizedReplaceableKind, isReplaceableKind } from '@/kinds.ts';
 import { jsonMetaContentSchema } from '@/schemas/nostr.ts';
 import { type DittoEvent, EventStore, type GetEventsOpts, type StoreEventOpts } from '@/store.ts';
 import { isNostrId, isURL } from '@/utils.ts';
@@ -25,13 +26,19 @@ const tagConditions: Record<string, TagCondition> = {
   'proxy': ({ count, value }) => count === 0 && isURL(value),
   'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value),
   't': ({ count, value }) => count < 5 && value.length < 50,
+  'name': ({ event, count }) => event.kind === 30361 && count === 0,
+  'role': ({ event, count }) => event.kind === 30361 && count === 0,
 };
 
 /** Insert an event (and its tags) into the database. */
-function storeEvent(event: Event, opts: StoreEventOpts = {}): Promise<void> {
+async function storeEvent(event: Event, opts: StoreEventOpts = {}): Promise<void> {
   debug('EVENT', JSON.stringify(event));
 
-  return db.transaction().execute(async (trx) => {
+  if (isDittoInternalKind(event.kind) && event.pubkey !== Conf.pubkey) {
+    throw new Error('Internal events can only be stored by the server keypair');
+  }
+
+  return await db.transaction().execute(async (trx) => {
     /** Insert the event into the database. */
     async function addEvent() {
       await trx.insertInto('events')
@@ -59,6 +66,30 @@ function storeEvent(event: Event, opts: StoreEventOpts = {}): Promise<void> {
         .execute();
     }
 
+    if (isReplaceableKind(event.kind)) {
+      const prevEvents = await getFilterQuery(trx, { kinds: [event.kind], authors: [event.pubkey] }).execute();
+      for (const prevEvent of prevEvents) {
+        if (prevEvent.created_at >= event.created_at) {
+          throw new Error('Cannot replace an event with an older event');
+        }
+      }
+      await deleteEventsTrx(trx, [{ kinds: [event.kind], authors: [event.pubkey] }]);
+    }
+
+    if (isParameterizedReplaceableKind(event.kind)) {
+      const d = event.tags.find(([tag]) => tag === 'd')?.[1];
+      if (d) {
+        const prevEvents = await getFilterQuery(trx, { kinds: [event.kind], authors: [event.pubkey], '#d': [d] })
+          .execute();
+        for (const prevEvent of prevEvents) {
+          if (prevEvent.created_at >= event.created_at) {
+            throw new Error('Cannot replace an event with an older event');
+          }
+        }
+        await deleteEventsTrx(trx, [{ kinds: [event.kind], authors: [event.pubkey], '#d': [d] }]);
+      }
+    }
+
     // Run the queries.
     await Promise.all([
       addEvent(),
@@ -99,7 +130,7 @@ type EventQuery = SelectQueryBuilder<DittoDB, 'events', {
 }>;
 
 /** Build the query for a filter. */
-function getFilterQuery(filter: DittoFilter): EventQuery {
+function getFilterQuery(db: Kysely<DittoDB>, filter: DittoFilter): EventQuery {
   let query = db
     .selectFrom('events')
     .select([
@@ -148,9 +179,9 @@ function getFilterQuery(filter: DittoFilter): EventQuery {
   }
 
   if (typeof filter.local === 'boolean') {
-    query = filter.local
+    query = query
-      ? query.innerJoin('users', 'users.pubkey', 'events.pubkey') as typeof query
+      .leftJoin(usersQuery, (join) => join.onRef('users.d_tag', '=', 'events.pubkey'))
-      : query.leftJoin('users', 'users.pubkey', 'events.pubkey').where('users.pubkey', 'is', null) as typeof query;
+      .where('users.d_tag', filter.local ? 'is not' : 'is', null);
   }
 
   if (filter.relations?.includes('author')) {
@@ -208,10 +239,19 @@ function getFilterQuery(filter: DittoFilter): EventQuery {
 /** Combine filter queries into a single union query. */
 function getEventsQuery(filters: DittoFilter[]) {
   return filters
-    .map((filter) => db.selectFrom(() => getFilterQuery(filter).as('events')).selectAll())
+    .map((filter) => db.selectFrom(() => getFilterQuery(db, filter).as('events')).selectAll())
     .reduce((result, query) => result.unionAll(query));
 }
 
+/** Query to get user events, joined by tags. */
+function usersQuery() {
+  return getFilterQuery(db, { kinds: [30361], authors: [Conf.pubkey] })
+    .leftJoin('tags', 'tags.event_id', 'events.id')
+    .where('tags.tag', '=', 'd')
+    .select('tags.value as d_tag')
+    .as('users');
+}
+
 /** Get events for filters from the database. */
 async function getEvents<K extends number>(
   filters: DittoFilter<K>[],
@@ -268,22 +308,28 @@ async function getEvents<K extends number>(
   });
 }
 
+/** Delete events from each table. Should be run in a transaction! */
+async function deleteEventsTrx(db: Kysely<DittoDB>, filters: DittoFilter[]) {
+  if (!filters.length) return Promise.resolve();
+  debug('DELETE', JSON.stringify(filters));
+
+  const query = getEventsQuery(filters).clearSelect().select('id');
+
+  await db.deleteFrom('events_fts')
+    .where('id', 'in', () => query)
+    .execute();
+
+  return db.deleteFrom('events')
+    .where('id', 'in', () => query)
+    .execute();
+}
+
 /** Delete events based on filters from the database. */
 async function deleteEvents<K extends number>(filters: DittoFilter<K>[]): Promise<void> {
   if (!filters.length) return Promise.resolve();
   debug('DELETE', JSON.stringify(filters));
 
-  await db.transaction().execute(async (trx) => {
+  await db.transaction().execute((trx) => deleteEventsTrx(trx, filters));
-    const query = getEventsQuery(filters).clearSelect().select('id');
-
-    await trx.deleteFrom('events_fts')
-      .where('id', 'in', () => query)
-      .execute();
-
-    return trx.deleteFrom('events')
-      .where('id', 'in', () => query)
-      .execute();
-  });
 }
 
 /** Get number of events that would be returned by filters. */

src/db/migrations/010_drop_users.ts

@@ -0,0 +1,8 @@
+import { Kysely } from '@/deps.ts';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema.dropTable('users').execute();
+}
+
+export async function down(_db: Kysely<any>): Promise<void> {
+}

src/db/users.ts

@@ -1,6 +1,8 @@
-import { Debug, type Insertable } from '@/deps.ts';
+import { Conf } from '@/config.ts';
-
+import { Debug, type Filter } from '@/deps.ts';
-import { db, type UserRow } from '../db.ts';
+import { eventsDB } from '@/db/events.ts';
+import * as pipeline from '@/pipeline.ts';
+import { signAdminEvent } from '@/sign.ts';
 
 const debug = Debug('ditto:users');
 
@@ -11,10 +13,30 @@ interface User {
   admin: boolean;
 }
 
-/** Adds a user to the database. */
+function buildUserEvent(user: User) {
-function insertUser(user: Insertable<UserRow>) {
   debug('insertUser', JSON.stringify(user));
-  return db.insertInto('users').values(user).execute();
+  const { origin, host } = Conf.url;
+
+  return signAdminEvent({
+    kind: 30361,
+    tags: [
+      ['d', user.pubkey],
+      ['name', user.username],
+      ['role', user.admin ? 'admin' : 'user'],
+      ['origin', origin],
+      // NIP-31: https://github.com/nostr-protocol/nips/blob/master/31.md
+      ['alt', `@${user.username}@${host}'s account was updated by the admins of ${host}`],
+    ],
+    content: '',
+    created_at: Math.floor(user.inserted_at.getTime() / 1000),
+  });
+}
+
+/** Adds a user to the database. */
+async function insertUser(user: User) {
+  debug('insertUser', JSON.stringify(user));
+  const event = await buildUserEvent(user);
+  return pipeline.handleEvent(event);
 }
 
 /**
@@ -24,21 +46,33 @@ function insertUser(user: Insertable<UserRow>) {
  * await findUser({ username: 'alex' });
  * ```
  */
-async function findUser(user: Partial<Insertable<UserRow>>): Promise<User | undefined> {
+async function findUser(user: Partial<User>): Promise<User | undefined> {
-  let query = db.selectFrom('users').selectAll();
+  const filter: Filter = { kinds: [30361], authors: [Conf.pubkey], limit: 1 };
 
   for (const [key, value] of Object.entries(user)) {
-    query = query.where(key as keyof UserRow, '=', value);
+    switch (key) {
+      case 'pubkey':
+        filter['#d'] = [String(value)];
+        break;
+      case 'username':
+        filter['#name'] = [String(value)];
+        break;
+      case 'admin':
+        filter['#role'] = [value ? 'admin' : 'user'];
+        break;
+    }
   }
 
-  const row = await query.executeTakeFirst();
+  const [event] = await eventsDB.getEvents([filter]);
 
-  if (row) {
+  if (event) {
     return {
-      ...row,
+      pubkey: event.tags.find(([name]) => name === 'd')?.[1]!,
-      admin: row.admin === 1,
+      username: event.tags.find(([name]) => name === 'name')?.[1]!,
+      inserted_at: new Date(event.created_at * 1000),
+      admin: event.tags.find(([name]) => name === 'role')?.[1] === 'admin',
     };
   }
 }
 
-export { findUser, insertUser, type User };
+export { buildUserEvent, findUser, insertUser, type User };

src/kinds.ts

@@ -18,6 +18,11 @@ function isParameterizedReplaceableKind(kind: number) {
   return 30000 <= kind && kind < 40000;
 }
 
+/** These events are only valid if published by the server keypair. */
+function isDittoInternalKind(kind: number) {
+  return kind === 30361;
+}
+
 /** Classification of the event kind. */
 type KindClassification = 'regular' | 'replaceable' | 'ephemeral' | 'parameterized' | 'unknown';
 
@@ -32,6 +37,7 @@ function classifyKind(kind: number): KindClassification {
 
 export {
   classifyKind,
+  isDittoInternalKind,
   isEphemeralKind,
   isParameterizedReplaceableKind,
   isRegularKind,

src/pipeline.ts

@@ -71,7 +71,7 @@ async function storeEvent(event: Event, data: EventData, opts: StoreEventOpts =
   if (force || data.user || isAdminEvent(event) || await isLocallyFollowed(event.pubkey)) {
     const [deletion] = await eventsDB.getEvents(
       [{ kinds: [5], authors: [event.pubkey], '#e': [event.id], limit: 1 }],
-      { limit: 1, signal: AbortSignal.timeout(Time.seconds(1)) },
+      { limit: 1 },
     );
 
     if (deletion) {

src/views/mastodon/accounts.ts

@@ -1,9 +1,9 @@
 import { Conf } from '@/config.ts';
-import { type DittoEvent } from '@/db/events.ts';
 import { findUser } from '@/db/users.ts';
 import { lodash, nip19, type UnsignedEvent } from '@/deps.ts';
 import { jsonMetaContentSchema } from '@/schemas/nostr.ts';
 import { verifyNip05Cached } from '@/utils/nip05.ts';
+import { type DittoEvent } from '@/store.ts';
 import { Nip05, nostrDate, nostrNow, parseNip05 } from '@/utils.ts';
 import { renderEmojis } from '@/views/mastodon/emojis.ts';
 

src/views/mastodon/statuses.ts

@@ -6,13 +6,14 @@ import { findReplyTag, nip19 } from '@/deps.ts';
 import { getMediaLinks, parseNoteContent } from '@/note.ts';
 import { getAuthor } from '@/queries.ts';
 import { jsonMediaDataSchema } from '@/schemas/nostr.ts';
+import { DittoEvent } from '@/store.ts';
 import { nostrDate } from '@/utils.ts';
 import { unfurlCardCached } from '@/utils/unfurl.ts';
 import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
 import { DittoAttachment, renderAttachment } from '@/views/mastodon/attachments.ts';
 import { renderEmojis } from '@/views/mastodon/emojis.ts';
 
-async function renderStatus(event: eventsDB.DittoEvent<1>, viewerPubkey?: string) {
+async function renderStatus(event: DittoEvent<1>, viewerPubkey?: string) {
   const account = event.author
     ? await renderAccount({ ...event.author, author_stats: event.author_stats })
     : await accountFromPubkey(event.pubkey);