diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab6c748..dba5873 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,4 +16,6 @@ lint: test: stage: test - script: deno task test \ No newline at end of file + script: deno task test + variables: + DITTO_NSEC: nsec1zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs4rm7hz \ No newline at end of file diff --git a/docs/nip78.md b/docs/events.md similarity index 51% rename from docs/nip78.md rename to docs/events.md index caa6111..8f011e4 100644 --- a/docs/nip78.md +++ b/docs/events.md @@ -1,13 +1,25 @@ -# Ditto NIP-78 events +# Ditto custom events + +## Users + +Ditto user events describe a pubkey's relationship with the Ditto server. They are parameterized replaceable events of kind `30361` where the `d` tag is a pubkey. These events are published by Ditto's internal admin keypair. + +User events have the following tags: + +- `d` - pubkey of the user. +- `name` - NIP-05 username granted to the user, without the domain. +- `role` - one of `admin` or `user`. + +## NIP-78 [NIP-78](https://github.com/nostr-protocol/nips/blob/master/78.md) defines events of kind `30078` with a globally unique `d` tag. These events are queried by the `d` tag, which allows Ditto to store custom data on relays. Ditto uses reverse DNS names like `pub.ditto.` for `d` tags. The sections below describe the `content` field. Some are encrypted and some are not, depending on whether the data should be public. Also, some events are user events, and some are admin events. -## `pub.ditto.blocks` +### `pub.ditto.blocks` -An encrypted array of blocked pubkeys, JSON stringified and encrypted with `nip07.encrypt`. +An encrypted array of blocked pubkeys, JSON stringified in `content` and encrypted with `nip04.encrypt`. -## `pub.ditto.frontendConfig` +### `pub.ditto.frontendConfig` JSON data for Pleroma frontends served on `/api/pleroma/frontend_configurations`. Each key contains arbitrary data used by a different frontend. \ No newline at end of file diff --git a/scripts/db.ts b/scripts/db.ts new file mode 100644 index 0000000..cdc5174 --- /dev/null +++ b/scripts/db.ts @@ -0,0 +1,44 @@ +import { Conf } from '@/config.ts'; +import { db } from '@/db.ts'; +import { eventsDB } from '@/db/events.ts'; +import { type Kysely } from '@/deps.ts'; +import { signAdminEvent } from '@/sign.ts'; + +interface DB { + users: { + pubkey: string; + username: string; + inserted_at: Date; + admin: 0 | 1; + }; +} + +switch (Deno.args[0]) { + case 'users-to-events': + await usersToEvents(); + break; + default: + console.log('Usage: deno run -A scripts/db.ts '); +} + +async function usersToEvents() { + const { origin, host } = Conf.url; + + for (const row of await (db as unknown as Kysely).selectFrom('users').selectAll().execute()) { + const event = await signAdminEvent({ + kind: 30361, + tags: [ + ['d', row.pubkey], + ['name', row.username], + ['role', row.admin ? 'admin' : 'user'], + ['origin', origin], + // NIP-31: https://github.com/nostr-protocol/nips/blob/master/31.md + ['alt', `@${row.username}@${host}'s account was updated by the admins of ${host}`], + ], + content: '', + created_at: Math.floor(new Date(row.inserted_at).getTime() / 1000), + }); + + await eventsDB.storeEvent(event); + } +} diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index ddd8170..5e345ae 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -42,7 +42,7 @@ const createAccountController: AppController = async (c) => { pubkey, username: result.data.username, inserted_at: new Date(), - admin: 0, + admin: false, }); return c.json({ diff --git a/src/controllers/nostr/relay-info.ts b/src/controllers/nostr/relay-info.ts index 0edafbd..9c16cdd 100644 --- a/src/controllers/nostr/relay-info.ts +++ b/src/controllers/nostr/relay-info.ts @@ -7,7 +7,7 @@ const relayInfoController: AppController = (c) => { description: 'Nostr and the Fediverse.', pubkey: Conf.pubkey, contact: `mailto:${Conf.adminEmail}`, - supported_nips: [1, 5, 9, 11, 45, 46, 98], + supported_nips: [1, 5, 9, 11, 16, 45, 46, 98], software: 'Ditto', version: '0.0.0', limitation: { diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index fe1cc31..a1f4de0 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -96,7 +96,7 @@ function connectStream(socket: WebSocket) { /** Handle COUNT. Return the number of events matching the filters. */ async function handleCount([_, subId, ...rest]: ClientCOUNT): Promise { - const count = await eventsDB.countFilters(prepareFilters(rest)); + const count = await eventsDB.countEvents(prepareFilters(rest)); send(['COUNT', subId, { count, approximate: false }]); } diff --git a/src/db.ts b/src/db.ts index 25d6d78..253fed9 100644 --- a/src/db.ts +++ b/src/db.ts @@ -10,7 +10,6 @@ interface DittoDB { events: EventRow; events_fts: EventFTSRow; tags: TagRow; - users: UserRow; relays: RelayRow; unattached_media: UnattachedMediaRow; author_stats: AuthorStatsRow; @@ -52,13 +51,6 @@ interface TagRow { event_id: string; } -interface UserRow { - pubkey: string; - username: string; - inserted_at: Date; - admin: 0 | 1; -} - interface RelayRow { url: string; domain: string; @@ -120,4 +112,4 @@ async function migrate() { await migrate(); -export { type AuthorStatsRow, db, type DittoDB, type EventRow, type EventStatsRow, type TagRow, type UserRow }; +export { type AuthorStatsRow, db, type DittoDB, type EventRow, type EventStatsRow, type TagRow }; diff --git a/src/db/events.test.ts b/src/db/events.test.ts index ab689d5..7a5537e 100644 --- a/src/db/events.test.ts +++ b/src/db/events.test.ts @@ -1,6 +1,7 @@ -import { assertEquals } from '@/deps-test.ts'; -import { insertUser } from '@/db/users.ts'; +import { assertEquals, assertRejects } from '@/deps-test.ts'; +import { buildUserEvent } from '@/db/users.ts'; +import event0 from '~/fixtures/events/event-0.json' assert { type: 'json' }; import event1 from '~/fixtures/events/event-1.json' assert { type: 'json' }; import { eventsDB as db } from './events.ts'; @@ -38,13 +39,26 @@ Deno.test('query events with local filter', async () => { assertEquals(await db.getEvents([{ local: true }]), []); assertEquals(await db.getEvents([{ local: false }]), [event1]); - await insertUser({ + const userEvent = await buildUserEvent({ username: 'alex', pubkey: event1.pubkey, inserted_at: new Date(), - admin: 0, + admin: false, }); + await db.storeEvent(userEvent); - assertEquals(await db.getEvents([{ local: true }]), [event1]); - assertEquals(await db.getEvents([{ local: false }]), []); + assertEquals(await db.getEvents([{ kinds: [1], local: true }]), [event1]); + assertEquals(await db.getEvents([{ kinds: [1], local: false }]), []); +}); + +Deno.test('inserting replaceable events', async () => { + assertEquals(await db.countEvents([{ kinds: [0], authors: [event0.pubkey] }]), 0); + + await db.storeEvent(event0); + await assertRejects(() => db.storeEvent(event0)); + assertEquals(await db.countEvents([{ kinds: [0], authors: [event0.pubkey] }]), 1); + + const changeEvent = { ...event0, id: '123', created_at: event0.created_at + 1 }; + await db.storeEvent(changeEvent); + assertEquals(await db.getEvents([{ kinds: [0] }]), [changeEvent]); }); diff --git a/src/db/events.ts b/src/db/events.ts index 96cf845..a20c650 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -1,7 +1,8 @@ +import { Conf } from '@/config.ts'; import { db, type DittoDB } from '@/db.ts'; -import { Debug, type Event, type SelectQueryBuilder } from '@/deps.ts'; +import { Debug, type Event, Kysely, type SelectQueryBuilder } from '@/deps.ts'; import { type DittoFilter } from '@/filter.ts'; -import { isParameterizedReplaceableKind } from '@/kinds.ts'; +import { isDittoInternalKind, isParameterizedReplaceableKind, isReplaceableKind } from '@/kinds.ts'; import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; import { type DittoEvent, EventStore, type GetEventsOpts, type StoreEventOpts } from '@/store.ts'; import { isNostrId, isURL } from '@/utils.ts'; @@ -25,13 +26,19 @@ const tagConditions: Record = { 'proxy': ({ count, value }) => count === 0 && isURL(value), 'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value), 't': ({ count, value }) => count < 5 && value.length < 50, + 'name': ({ event, count }) => event.kind === 30361 && count === 0, + 'role': ({ event, count }) => event.kind === 30361 && count === 0, }; /** Insert an event (and its tags) into the database. */ -function storeEvent(event: Event, opts: StoreEventOpts = {}): Promise { +async function storeEvent(event: Event, opts: StoreEventOpts = {}): Promise { debug('EVENT', JSON.stringify(event)); - return db.transaction().execute(async (trx) => { + if (isDittoInternalKind(event.kind) && event.pubkey !== Conf.pubkey) { + throw new Error('Internal events can only be stored by the server keypair'); + } + + return await db.transaction().execute(async (trx) => { /** Insert the event into the database. */ async function addEvent() { await trx.insertInto('events') @@ -59,6 +66,30 @@ function storeEvent(event: Event, opts: StoreEventOpts = {}): Promise { .execute(); } + if (isReplaceableKind(event.kind)) { + const prevEvents = await getFilterQuery(trx, { kinds: [event.kind], authors: [event.pubkey] }).execute(); + for (const prevEvent of prevEvents) { + if (prevEvent.created_at >= event.created_at) { + throw new Error('Cannot replace an event with an older event'); + } + } + await deleteEventsTrx(trx, [{ kinds: [event.kind], authors: [event.pubkey] }]); + } + + if (isParameterizedReplaceableKind(event.kind)) { + const d = event.tags.find(([tag]) => tag === 'd')?.[1]; + if (d) { + const prevEvents = await getFilterQuery(trx, { kinds: [event.kind], authors: [event.pubkey], '#d': [d] }) + .execute(); + for (const prevEvent of prevEvents) { + if (prevEvent.created_at >= event.created_at) { + throw new Error('Cannot replace an event with an older event'); + } + } + await deleteEventsTrx(trx, [{ kinds: [event.kind], authors: [event.pubkey], '#d': [d] }]); + } + } + // Run the queries. await Promise.all([ addEvent(), @@ -99,7 +130,7 @@ type EventQuery = SelectQueryBuilder; /** Build the query for a filter. */ -function getFilterQuery(filter: DittoFilter): EventQuery { +function getFilterQuery(db: Kysely, filter: DittoFilter): EventQuery { let query = db .selectFrom('events') .select([ @@ -148,9 +179,9 @@ function getFilterQuery(filter: DittoFilter): EventQuery { } if (typeof filter.local === 'boolean') { - query = filter.local - ? query.innerJoin('users', 'users.pubkey', 'events.pubkey') as typeof query - : query.leftJoin('users', 'users.pubkey', 'events.pubkey').where('users.pubkey', 'is', null) as typeof query; + query = query + .leftJoin(usersQuery, (join) => join.onRef('users.d_tag', '=', 'events.pubkey')) + .where('users.d_tag', filter.local ? 'is not' : 'is', null); } if (filter.relations?.includes('author')) { @@ -208,10 +239,19 @@ function getFilterQuery(filter: DittoFilter): EventQuery { /** Combine filter queries into a single union query. */ function getEventsQuery(filters: DittoFilter[]) { return filters - .map((filter) => db.selectFrom(() => getFilterQuery(filter).as('events')).selectAll()) + .map((filter) => db.selectFrom(() => getFilterQuery(db, filter).as('events')).selectAll()) .reduce((result, query) => result.unionAll(query)); } +/** Query to get user events, joined by tags. */ +function usersQuery() { + return getFilterQuery(db, { kinds: [30361], authors: [Conf.pubkey] }) + .leftJoin('tags', 'tags.event_id', 'events.id') + .where('tags.tag', '=', 'd') + .select('tags.value as d_tag') + .as('users'); +} + /** Get events for filters from the database. */ async function getEvents( filters: DittoFilter[], @@ -268,22 +308,28 @@ async function getEvents( }); } +/** Delete events from each table. Should be run in a transaction! */ +async function deleteEventsTrx(db: Kysely, filters: DittoFilter[]) { + if (!filters.length) return Promise.resolve(); + debug('DELETE', JSON.stringify(filters)); + + const query = getEventsQuery(filters).clearSelect().select('id'); + + await db.deleteFrom('events_fts') + .where('id', 'in', () => query) + .execute(); + + return db.deleteFrom('events') + .where('id', 'in', () => query) + .execute(); +} + /** Delete events based on filters from the database. */ async function deleteEvents(filters: DittoFilter[]): Promise { if (!filters.length) return Promise.resolve(); debug('DELETE', JSON.stringify(filters)); - await db.transaction().execute(async (trx) => { - const query = getEventsQuery(filters).clearSelect().select('id'); - - await trx.deleteFrom('events_fts') - .where('id', 'in', () => query) - .execute(); - - return trx.deleteFrom('events') - .where('id', 'in', () => query) - .execute(); - }); + await db.transaction().execute((trx) => deleteEventsTrx(trx, filters)); } /** Get number of events that would be returned by filters. */ diff --git a/src/db/migrations/010_drop_users.ts b/src/db/migrations/010_drop_users.ts new file mode 100644 index 0000000..9649b64 --- /dev/null +++ b/src/db/migrations/010_drop_users.ts @@ -0,0 +1,8 @@ +import { Kysely } from '@/deps.ts'; + +export async function up(db: Kysely): Promise { + await db.schema.dropTable('users').execute(); +} + +export async function down(_db: Kysely): Promise { +} diff --git a/src/db/users.ts b/src/db/users.ts index 7d56e1c..d223b1c 100644 --- a/src/db/users.ts +++ b/src/db/users.ts @@ -1,6 +1,8 @@ -import { Debug, type Insertable } from '@/deps.ts'; - -import { db, type UserRow } from '../db.ts'; +import { Conf } from '@/config.ts'; +import { Debug, type Filter } from '@/deps.ts'; +import { eventsDB } from '@/db/events.ts'; +import * as pipeline from '@/pipeline.ts'; +import { signAdminEvent } from '@/sign.ts'; const debug = Debug('ditto:users'); @@ -11,10 +13,30 @@ interface User { admin: boolean; } -/** Adds a user to the database. */ -function insertUser(user: Insertable) { +function buildUserEvent(user: User) { debug('insertUser', JSON.stringify(user)); - return db.insertInto('users').values(user).execute(); + const { origin, host } = Conf.url; + + return signAdminEvent({ + kind: 30361, + tags: [ + ['d', user.pubkey], + ['name', user.username], + ['role', user.admin ? 'admin' : 'user'], + ['origin', origin], + // NIP-31: https://github.com/nostr-protocol/nips/blob/master/31.md + ['alt', `@${user.username}@${host}'s account was updated by the admins of ${host}`], + ], + content: '', + created_at: Math.floor(user.inserted_at.getTime() / 1000), + }); +} + +/** Adds a user to the database. */ +async function insertUser(user: User) { + debug('insertUser', JSON.stringify(user)); + const event = await buildUserEvent(user); + return pipeline.handleEvent(event); } /** @@ -24,21 +46,33 @@ function insertUser(user: Insertable) { * await findUser({ username: 'alex' }); * ``` */ -async function findUser(user: Partial>): Promise { - let query = db.selectFrom('users').selectAll(); +async function findUser(user: Partial): Promise { + const filter: Filter = { kinds: [30361], authors: [Conf.pubkey], limit: 1 }; for (const [key, value] of Object.entries(user)) { - query = query.where(key as keyof UserRow, '=', value); + switch (key) { + case 'pubkey': + filter['#d'] = [String(value)]; + break; + case 'username': + filter['#name'] = [String(value)]; + break; + case 'admin': + filter['#role'] = [value ? 'admin' : 'user']; + break; + } } - const row = await query.executeTakeFirst(); + const [event] = await eventsDB.getEvents([filter]); - if (row) { + if (event) { return { - ...row, - admin: row.admin === 1, + pubkey: event.tags.find(([name]) => name === 'd')?.[1]!, + username: event.tags.find(([name]) => name === 'name')?.[1]!, + inserted_at: new Date(event.created_at * 1000), + admin: event.tags.find(([name]) => name === 'role')?.[1] === 'admin', }; } } -export { findUser, insertUser, type User }; +export { buildUserEvent, findUser, insertUser, type User }; diff --git a/src/kinds.ts b/src/kinds.ts index 45c9729..7953837 100644 --- a/src/kinds.ts +++ b/src/kinds.ts @@ -18,6 +18,11 @@ function isParameterizedReplaceableKind(kind: number) { return 30000 <= kind && kind < 40000; } +/** These events are only valid if published by the server keypair. */ +function isDittoInternalKind(kind: number) { + return kind === 30361; +} + /** Classification of the event kind. */ type KindClassification = 'regular' | 'replaceable' | 'ephemeral' | 'parameterized' | 'unknown'; @@ -32,6 +37,7 @@ function classifyKind(kind: number): KindClassification { export { classifyKind, + isDittoInternalKind, isEphemeralKind, isParameterizedReplaceableKind, isRegularKind, diff --git a/src/pipeline.ts b/src/pipeline.ts index f4e0abe..69ae941 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -71,7 +71,7 @@ async function storeEvent(event: Event, data: EventData, opts: StoreEventOpts = if (force || data.user || isAdminEvent(event) || await isLocallyFollowed(event.pubkey)) { const [deletion] = await eventsDB.getEvents( [{ kinds: [5], authors: [event.pubkey], '#e': [event.id], limit: 1 }], - { limit: 1, signal: AbortSignal.timeout(Time.seconds(1)) }, + { limit: 1 }, ); if (deletion) { diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index bfcd7ef..eae3dd7 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -1,9 +1,9 @@ import { Conf } from '@/config.ts'; -import { type DittoEvent } from '@/db/events.ts'; import { findUser } from '@/db/users.ts'; import { lodash, nip19, type UnsignedEvent } from '@/deps.ts'; import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; import { verifyNip05Cached } from '@/utils/nip05.ts'; +import { type DittoEvent } from '@/store.ts'; import { Nip05, nostrDate, nostrNow, parseNip05 } from '@/utils.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 2363ef9..b49be49 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -6,13 +6,14 @@ import { findReplyTag, nip19 } from '@/deps.ts'; import { getMediaLinks, parseNoteContent } from '@/note.ts'; import { getAuthor } from '@/queries.ts'; import { jsonMediaDataSchema } from '@/schemas/nostr.ts'; +import { DittoEvent } from '@/store.ts'; import { nostrDate } from '@/utils.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { DittoAttachment, renderAttachment } from '@/views/mastodon/attachments.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; -async function renderStatus(event: eventsDB.DittoEvent<1>, viewerPubkey?: string) { +async function renderStatus(event: DittoEvent<1>, viewerPubkey?: string) { const account = event.author ? await renderAccount({ ...event.author, author_stats: event.author_stats }) : await accountFromPubkey(event.pubkey);