From 13bf936088165a73aa510095f8f53f1d8ebb3678 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 29 Dec 2023 16:37:18 -0600 Subject: [PATCH 01/10] Convert users to Events --- docs/nip78.md | 20 ++++++++++++++---- scripts/db.ts | 34 ++++++++++++++++++++++++++++++ src/db/events.ts | 9 +++++++- src/db/users.ts | 54 ++++++++++++++++++++++++++++++++++++++---------- src/kinds.ts | 6 ++++++ 5 files changed, 107 insertions(+), 16 deletions(-) create mode 100644 scripts/db.ts diff --git a/docs/nip78.md b/docs/nip78.md index caa6111..8f011e4 100644 --- a/docs/nip78.md +++ b/docs/nip78.md @@ -1,13 +1,25 @@ -# Ditto NIP-78 events +# Ditto custom events + +## Users + +Ditto user events describe a pubkey's relationship with the Ditto server. They are parameterized replaceable events of kind `30361` where the `d` tag is a pubkey. These events are published by Ditto's internal admin keypair. + +User events have the following tags: + +- `d` - pubkey of the user. +- `name` - NIP-05 username granted to the user, without the domain. +- `role` - one of `admin` or `user`. + +## NIP-78 [NIP-78](https://github.com/nostr-protocol/nips/blob/master/78.md) defines events of kind `30078` with a globally unique `d` tag. These events are queried by the `d` tag, which allows Ditto to store custom data on relays. Ditto uses reverse DNS names like `pub.ditto.` for `d` tags. The sections below describe the `content` field. Some are encrypted and some are not, depending on whether the data should be public. Also, some events are user events, and some are admin events. -## `pub.ditto.blocks` +### `pub.ditto.blocks` -An encrypted array of blocked pubkeys, JSON stringified and encrypted with `nip07.encrypt`. +An encrypted array of blocked pubkeys, JSON stringified in `content` and encrypted with `nip04.encrypt`. -## `pub.ditto.frontendConfig` +### `pub.ditto.frontendConfig` JSON data for Pleroma frontends served on `/api/pleroma/frontend_configurations`. Each key contains arbitrary data used by a different frontend. \ No newline at end of file diff --git a/scripts/db.ts b/scripts/db.ts new file mode 100644 index 0000000..f1b7989 --- /dev/null +++ b/scripts/db.ts @@ -0,0 +1,34 @@ +import { Conf } from '@/config.ts'; +import { db } from '@/db.ts'; +import { eventsDB } from '@/db/events.ts'; +import { signAdminEvent } from '@/sign.ts'; + +switch (Deno.args[0]) { + case 'users-to-events': + await usersToEvents(); + break; + default: + console.log('Usage: deno run -A scripts/db.ts '); +} + +async function usersToEvents() { + const { origin, host } = Conf.url; + + for (const row of await db.selectFrom('users').selectAll().execute()) { + const event = await signAdminEvent({ + kind: 30361, + tags: [ + ['d', row.pubkey], + ['name', row.username], + ['role', row.admin ? 'admin' : 'user'], + ['origin', origin], + // NIP-31: https://github.com/nostr-protocol/nips/blob/master/31.md + ['alt', `@${row.username}@${host}'s account was updated by the admins of ${host}`], + ], + content: '', + created_at: Math.floor(new Date(row.inserted_at).getTime() / 1000), + }); + + await eventsDB.storeEvent(event); + } +} diff --git a/src/db/events.ts b/src/db/events.ts index 96cf845..2d18af2 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -1,7 +1,8 @@ +import { Conf } from '@/config.ts'; import { db, type DittoDB } from '@/db.ts'; import { Debug, type Event, type SelectQueryBuilder } from '@/deps.ts'; import { type DittoFilter } from '@/filter.ts'; -import { isParameterizedReplaceableKind } from '@/kinds.ts'; +import { isDittoInternalKind, isParameterizedReplaceableKind } from '@/kinds.ts'; import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; import { type DittoEvent, EventStore, type GetEventsOpts, type StoreEventOpts } from '@/store.ts'; import { isNostrId, isURL } from '@/utils.ts'; @@ -25,12 +26,18 @@ const tagConditions: Record = { 'proxy': ({ count, value }) => count === 0 && isURL(value), 'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value), 't': ({ count, value }) => count < 5 && value.length < 50, + 'name': ({ event, count }) => event.kind === 30361 && count === 0, + 'role': ({ event, count }) => event.kind === 30361 && count === 0, }; /** Insert an event (and its tags) into the database. */ function storeEvent(event: Event, opts: StoreEventOpts = {}): Promise { debug('EVENT', JSON.stringify(event)); + if (isDittoInternalKind(event.kind) && event.pubkey !== Conf.pubkey) { + throw new Error('Internal events can only be stored by the server keypair'); + } + return db.transaction().execute(async (trx) => { /** Insert the event into the database. */ async function addEvent() { diff --git a/src/db/users.ts b/src/db/users.ts index 7d56e1c..8f0a78a 100644 --- a/src/db/users.ts +++ b/src/db/users.ts @@ -1,6 +1,10 @@ -import { Debug, type Insertable } from '@/deps.ts'; - -import { db, type UserRow } from '../db.ts'; +import { Conf } from '@/config.ts'; +import { Debug, type Filter, type Insertable } from '@/deps.ts'; +import { type UserRow } from '@/db.ts'; +import { eventsDB } from '@/db/events.ts'; +import * as pipeline from '@/pipeline.ts'; +import { signAdminEvent } from '@/sign.ts'; +import { nostrNow } from '@/utils.ts'; const debug = Debug('ditto:users'); @@ -12,9 +16,25 @@ interface User { } /** Adds a user to the database. */ -function insertUser(user: Insertable) { +async function insertUser(user: Insertable) { debug('insertUser', JSON.stringify(user)); - return db.insertInto('users').values(user).execute(); + const { origin, host } = Conf.url; + + const event = await signAdminEvent({ + kind: 30361, + tags: [ + ['d', user.pubkey], + ['name', user.username], + ['role', user.admin ? 'admin' : 'user'], + ['origin', origin], + // NIP-31: https://github.com/nostr-protocol/nips/blob/master/31.md + ['alt', `@${user.username}@${host}'s account was updated by the admins of ${host}`], + ], + content: '', + created_at: nostrNow(), + }); + + return pipeline.handleEvent(event); } /** @@ -25,18 +45,30 @@ function insertUser(user: Insertable) { * ``` */ async function findUser(user: Partial>): Promise { - let query = db.selectFrom('users').selectAll(); + const filter: Filter = { kinds: [30361], authors: [Conf.pubkey], limit: 1 }; for (const [key, value] of Object.entries(user)) { - query = query.where(key as keyof UserRow, '=', value); + switch (key) { + case 'pubkey': + filter['#d'] = [String(value)]; + break; + case 'username': + filter['#name'] = [String(value)]; + break; + case 'admin': + filter['#role'] = [value ? 'admin' : 'user']; + break; + } } - const row = await query.executeTakeFirst(); + const [event] = await eventsDB.getEvents([filter]); - if (row) { + if (event) { return { - ...row, - admin: row.admin === 1, + pubkey: event.tags.find(([name]) => name === 'd')?.[1]!, + username: event.tags.find(([name]) => name === 'name')?.[1]!, + inserted_at: new Date(event.created_at * 1000), + admin: event.tags.find(([name]) => name === 'role')?.[1] === 'admin', }; } } diff --git a/src/kinds.ts b/src/kinds.ts index 45c9729..7953837 100644 --- a/src/kinds.ts +++ b/src/kinds.ts @@ -18,6 +18,11 @@ function isParameterizedReplaceableKind(kind: number) { return 30000 <= kind && kind < 40000; } +/** These events are only valid if published by the server keypair. */ +function isDittoInternalKind(kind: number) { + return kind === 30361; +} + /** Classification of the event kind. */ type KindClassification = 'regular' | 'replaceable' | 'ephemeral' | 'parameterized' | 'unknown'; @@ -32,6 +37,7 @@ function classifyKind(kind: number): KindClassification { export { classifyKind, + isDittoInternalKind, isEphemeralKind, isParameterizedReplaceableKind, isRegularKind, From 358396fdac9c7ce71b5ef7a06f6630baa34898f1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 29 Dec 2023 17:01:23 -0600 Subject: [PATCH 02/10] Drop users table --- scripts/db.ts | 12 +++++++++++- src/controllers/api/accounts.ts | 2 +- src/db.ts | 10 +--------- src/db/events.ts | 1 + src/db/migrations/010_drop_users.ts | 8 ++++++++ src/db/users.ts | 10 ++++------ src/views/mastodon/accounts.ts | 2 +- src/views/mastodon/statuses.ts | 3 ++- 8 files changed, 29 insertions(+), 19 deletions(-) create mode 100644 src/db/migrations/010_drop_users.ts diff --git a/scripts/db.ts b/scripts/db.ts index f1b7989..cdc5174 100644 --- a/scripts/db.ts +++ b/scripts/db.ts @@ -1,8 +1,18 @@ import { Conf } from '@/config.ts'; import { db } from '@/db.ts'; import { eventsDB } from '@/db/events.ts'; +import { type Kysely } from '@/deps.ts'; import { signAdminEvent } from '@/sign.ts'; +interface DB { + users: { + pubkey: string; + username: string; + inserted_at: Date; + admin: 0 | 1; + }; +} + switch (Deno.args[0]) { case 'users-to-events': await usersToEvents(); @@ -14,7 +24,7 @@ switch (Deno.args[0]) { async function usersToEvents() { const { origin, host } = Conf.url; - for (const row of await db.selectFrom('users').selectAll().execute()) { + for (const row of await (db as unknown as Kysely).selectFrom('users').selectAll().execute()) { const event = await signAdminEvent({ kind: 30361, tags: [ diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index ddd8170..5e345ae 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -42,7 +42,7 @@ const createAccountController: AppController = async (c) => { pubkey, username: result.data.username, inserted_at: new Date(), - admin: 0, + admin: false, }); return c.json({ diff --git a/src/db.ts b/src/db.ts index 25d6d78..253fed9 100644 --- a/src/db.ts +++ b/src/db.ts @@ -10,7 +10,6 @@ interface DittoDB { events: EventRow; events_fts: EventFTSRow; tags: TagRow; - users: UserRow; relays: RelayRow; unattached_media: UnattachedMediaRow; author_stats: AuthorStatsRow; @@ -52,13 +51,6 @@ interface TagRow { event_id: string; } -interface UserRow { - pubkey: string; - username: string; - inserted_at: Date; - admin: 0 | 1; -} - interface RelayRow { url: string; domain: string; @@ -120,4 +112,4 @@ async function migrate() { await migrate(); -export { type AuthorStatsRow, db, type DittoDB, type EventRow, type EventStatsRow, type TagRow, type UserRow }; +export { type AuthorStatsRow, db, type DittoDB, type EventRow, type EventStatsRow, type TagRow }; diff --git a/src/db/events.ts b/src/db/events.ts index 2d18af2..0c03157 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -154,6 +154,7 @@ function getFilterQuery(filter: DittoFilter): EventQuery { } } + // FIXME: local filtering is broken. if (typeof filter.local === 'boolean') { query = filter.local ? query.innerJoin('users', 'users.pubkey', 'events.pubkey') as typeof query diff --git a/src/db/migrations/010_drop_users.ts b/src/db/migrations/010_drop_users.ts new file mode 100644 index 0000000..9649b64 --- /dev/null +++ b/src/db/migrations/010_drop_users.ts @@ -0,0 +1,8 @@ +import { Kysely } from '@/deps.ts'; + +export async function up(db: Kysely): Promise { + await db.schema.dropTable('users').execute(); +} + +export async function down(_db: Kysely): Promise { +} diff --git a/src/db/users.ts b/src/db/users.ts index 8f0a78a..195dfe2 100644 --- a/src/db/users.ts +++ b/src/db/users.ts @@ -1,10 +1,8 @@ import { Conf } from '@/config.ts'; -import { Debug, type Filter, type Insertable } from '@/deps.ts'; -import { type UserRow } from '@/db.ts'; +import { Debug, type Filter } from '@/deps.ts'; import { eventsDB } from '@/db/events.ts'; import * as pipeline from '@/pipeline.ts'; import { signAdminEvent } from '@/sign.ts'; -import { nostrNow } from '@/utils.ts'; const debug = Debug('ditto:users'); @@ -16,7 +14,7 @@ interface User { } /** Adds a user to the database. */ -async function insertUser(user: Insertable) { +async function insertUser(user: User) { debug('insertUser', JSON.stringify(user)); const { origin, host } = Conf.url; @@ -31,7 +29,7 @@ async function insertUser(user: Insertable) { ['alt', `@${user.username}@${host}'s account was updated by the admins of ${host}`], ], content: '', - created_at: nostrNow(), + created_at: Math.floor(user.inserted_at.getTime() / 1000), }); return pipeline.handleEvent(event); @@ -44,7 +42,7 @@ async function insertUser(user: Insertable) { * await findUser({ username: 'alex' }); * ``` */ -async function findUser(user: Partial>): Promise { +async function findUser(user: Partial): Promise { const filter: Filter = { kinds: [30361], authors: [Conf.pubkey], limit: 1 }; for (const [key, value] of Object.entries(user)) { diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index bfcd7ef..eae3dd7 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -1,9 +1,9 @@ import { Conf } from '@/config.ts'; -import { type DittoEvent } from '@/db/events.ts'; import { findUser } from '@/db/users.ts'; import { lodash, nip19, type UnsignedEvent } from '@/deps.ts'; import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; import { verifyNip05Cached } from '@/utils/nip05.ts'; +import { type DittoEvent } from '@/store.ts'; import { Nip05, nostrDate, nostrNow, parseNip05 } from '@/utils.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 2363ef9..b49be49 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -6,13 +6,14 @@ import { findReplyTag, nip19 } from '@/deps.ts'; import { getMediaLinks, parseNoteContent } from '@/note.ts'; import { getAuthor } from '@/queries.ts'; import { jsonMediaDataSchema } from '@/schemas/nostr.ts'; +import { DittoEvent } from '@/store.ts'; import { nostrDate } from '@/utils.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { DittoAttachment, renderAttachment } from '@/views/mastodon/attachments.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; -async function renderStatus(event: eventsDB.DittoEvent<1>, viewerPubkey?: string) { +async function renderStatus(event: DittoEvent<1>, viewerPubkey?: string) { const account = event.author ? await renderAccount({ ...event.author, author_stats: event.author_stats }) : await accountFromPubkey(event.pubkey); From 9492b7654f19fe700878f383512c7b673e87536c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 29 Dec 2023 20:53:01 -0600 Subject: [PATCH 03/10] db/events: fix the `local` filter --- src/db/events.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index 0c03157..00187a4 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -154,11 +154,11 @@ function getFilterQuery(filter: DittoFilter): EventQuery { } } - // FIXME: local filtering is broken. if (typeof filter.local === 'boolean') { query = filter.local - ? query.innerJoin('users', 'users.pubkey', 'events.pubkey') as typeof query - : query.leftJoin('users', 'users.pubkey', 'events.pubkey').where('users.pubkey', 'is', null) as typeof query; + ? query.innerJoin(usersQuery, (join) => join.onRef('users.d_tag', '=', 'events.pubkey')) + : query.leftJoin(usersQuery, (join) => join.onRef('users.d_tag', '=', 'events.pubkey')) + .where('users.d_tag', 'is', null); } if (filter.relations?.includes('author')) { @@ -220,6 +220,15 @@ function getEventsQuery(filters: DittoFilter[]) { .reduce((result, query) => result.unionAll(query)); } +/** Query to get user events, joined by tags. */ +function usersQuery() { + return getFilterQuery({ kinds: [30361], authors: [Conf.pubkey] }) + .leftJoin('tags', 'tags.event_id', 'events.id') + .where('tags.tag', '=', 'd') + .select('tags.value as d_tag') + .as('users'); +} + /** Get events for filters from the database. */ async function getEvents( filters: DittoFilter[], From b39902d322e0dc22106d63a5d4361eeb33cf62da Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 29 Dec 2023 21:05:31 -0600 Subject: [PATCH 04/10] Load local timeline a lot faster by using leftJoin instead of innerJoin, wow --- src/db/events.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/db/events.ts b/src/db/events.ts index 00187a4..d9602ed 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -156,7 +156,8 @@ function getFilterQuery(filter: DittoFilter): EventQuery { if (typeof filter.local === 'boolean') { query = filter.local - ? query.innerJoin(usersQuery, (join) => join.onRef('users.d_tag', '=', 'events.pubkey')) + ? query.leftJoin(usersQuery, (join) => join.onRef('users.d_tag', '=', 'events.pubkey')) + .where('users.d_tag', 'is not', null) : query.leftJoin(usersQuery, (join) => join.onRef('users.d_tag', '=', 'events.pubkey')) .where('users.d_tag', 'is', null); } From f6a8ab570f953e9711754f1571f4faa2f88c386c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 29 Dec 2023 21:08:59 -0600 Subject: [PATCH 05/10] db/events: simplify it further --- src/db/events.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index d9602ed..1847d6b 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -155,11 +155,9 @@ function getFilterQuery(filter: DittoFilter): EventQuery { } if (typeof filter.local === 'boolean') { - query = filter.local - ? query.leftJoin(usersQuery, (join) => join.onRef('users.d_tag', '=', 'events.pubkey')) - .where('users.d_tag', 'is not', null) - : query.leftJoin(usersQuery, (join) => join.onRef('users.d_tag', '=', 'events.pubkey')) - .where('users.d_tag', 'is', null); + query = query + .leftJoin(usersQuery, (join) => join.onRef('users.d_tag', '=', 'events.pubkey')) + .where('users.d_tag', filter.local ? 'is not' : 'is', null); } if (filter.relations?.includes('author')) { From 3472428da875a843607570c8ece206d31f594a8f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 29 Dec 2023 21:12:57 -0600 Subject: [PATCH 06/10] relay: fix wrong function name --- src/controllers/nostr/relay.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index fe1cc31..a1f4de0 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -96,7 +96,7 @@ function connectStream(socket: WebSocket) { /** Handle COUNT. Return the number of events matching the filters. */ async function handleCount([_, subId, ...rest]: ClientCOUNT): Promise { - const count = await eventsDB.countFilters(prepareFilters(rest)); + const count = await eventsDB.countEvents(prepareFilters(rest)); send(['COUNT', subId, { count, approximate: false }]); } From 38238cc4a8699a053cb18cd7aa973a63a16d819f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 29 Dec 2023 21:21:56 -0600 Subject: [PATCH 07/10] Fix db/events test, remove unnecessary AbortSignal from pipeline --- src/db/events.test.ts | 6 +++--- src/pipeline.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/db/events.test.ts b/src/db/events.test.ts index ab689d5..bb3f6f4 100644 --- a/src/db/events.test.ts +++ b/src/db/events.test.ts @@ -42,9 +42,9 @@ Deno.test('query events with local filter', async () => { username: 'alex', pubkey: event1.pubkey, inserted_at: new Date(), - admin: 0, + admin: false, }); - assertEquals(await db.getEvents([{ local: true }]), [event1]); - assertEquals(await db.getEvents([{ local: false }]), []); + assertEquals(await db.getEvents([{ kinds: [1], local: true }]), [event1]); + assertEquals(await db.getEvents([{ kinds: [1], local: false }]), []); }); diff --git a/src/pipeline.ts b/src/pipeline.ts index f4e0abe..69ae941 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -71,7 +71,7 @@ async function storeEvent(event: Event, data: EventData, opts: StoreEventOpts = if (force || data.user || isAdminEvent(event) || await isLocallyFollowed(event.pubkey)) { const [deletion] = await eventsDB.getEvents( [{ kinds: [5], authors: [event.pubkey], '#e': [event.id], limit: 1 }], - { limit: 1, signal: AbortSignal.timeout(Time.seconds(1)) }, + { limit: 1 }, ); if (deletion) { From 08059f6b40e1131d0ec61e469c0ddfc9304a7d0c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 29 Dec 2023 21:22:59 -0600 Subject: [PATCH 08/10] docs: nip78.md -> events.md --- docs/{nip78.md => events.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{nip78.md => events.md} (100%) diff --git a/docs/nip78.md b/docs/events.md similarity index 100% rename from docs/nip78.md rename to docs/events.md From 529bc11da1767fa482834779ea8c53f936ee23fe Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 29 Dec 2023 23:21:05 -0600 Subject: [PATCH 09/10] Support replaceable events and parameterized replaceable events (delete old versions upon insert) --- .gitlab-ci.yml | 4 +- src/controllers/nostr/relay-info.ts | 2 +- src/db/events.test.ts | 20 +++++++-- src/db/events.ts | 66 +++++++++++++++++++++-------- src/db/users.ts | 12 ++++-- 5 files changed, 77 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ab6c748..d065493 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,4 +16,6 @@ lint: test: stage: test - script: deno task test \ No newline at end of file + script: deno task test + variables: + DITTO_NSEC: nsec1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqwkhnav \ No newline at end of file diff --git a/src/controllers/nostr/relay-info.ts b/src/controllers/nostr/relay-info.ts index 0edafbd..9c16cdd 100644 --- a/src/controllers/nostr/relay-info.ts +++ b/src/controllers/nostr/relay-info.ts @@ -7,7 +7,7 @@ const relayInfoController: AppController = (c) => { description: 'Nostr and the Fediverse.', pubkey: Conf.pubkey, contact: `mailto:${Conf.adminEmail}`, - supported_nips: [1, 5, 9, 11, 45, 46, 98], + supported_nips: [1, 5, 9, 11, 16, 45, 46, 98], software: 'Ditto', version: '0.0.0', limitation: { diff --git a/src/db/events.test.ts b/src/db/events.test.ts index bb3f6f4..7a5537e 100644 --- a/src/db/events.test.ts +++ b/src/db/events.test.ts @@ -1,6 +1,7 @@ -import { assertEquals } from '@/deps-test.ts'; -import { insertUser } from '@/db/users.ts'; +import { assertEquals, assertRejects } from '@/deps-test.ts'; +import { buildUserEvent } from '@/db/users.ts'; +import event0 from '~/fixtures/events/event-0.json' assert { type: 'json' }; import event1 from '~/fixtures/events/event-1.json' assert { type: 'json' }; import { eventsDB as db } from './events.ts'; @@ -38,13 +39,26 @@ Deno.test('query events with local filter', async () => { assertEquals(await db.getEvents([{ local: true }]), []); assertEquals(await db.getEvents([{ local: false }]), [event1]); - await insertUser({ + const userEvent = await buildUserEvent({ username: 'alex', pubkey: event1.pubkey, inserted_at: new Date(), admin: false, }); + await db.storeEvent(userEvent); assertEquals(await db.getEvents([{ kinds: [1], local: true }]), [event1]); assertEquals(await db.getEvents([{ kinds: [1], local: false }]), []); }); + +Deno.test('inserting replaceable events', async () => { + assertEquals(await db.countEvents([{ kinds: [0], authors: [event0.pubkey] }]), 0); + + await db.storeEvent(event0); + await assertRejects(() => db.storeEvent(event0)); + assertEquals(await db.countEvents([{ kinds: [0], authors: [event0.pubkey] }]), 1); + + const changeEvent = { ...event0, id: '123', created_at: event0.created_at + 1 }; + await db.storeEvent(changeEvent); + assertEquals(await db.getEvents([{ kinds: [0] }]), [changeEvent]); +}); diff --git a/src/db/events.ts b/src/db/events.ts index 1847d6b..a20c650 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -1,8 +1,8 @@ import { Conf } from '@/config.ts'; import { db, type DittoDB } from '@/db.ts'; -import { Debug, type Event, type SelectQueryBuilder } from '@/deps.ts'; +import { Debug, type Event, Kysely, type SelectQueryBuilder } from '@/deps.ts'; import { type DittoFilter } from '@/filter.ts'; -import { isDittoInternalKind, isParameterizedReplaceableKind } from '@/kinds.ts'; +import { isDittoInternalKind, isParameterizedReplaceableKind, isReplaceableKind } from '@/kinds.ts'; import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; import { type DittoEvent, EventStore, type GetEventsOpts, type StoreEventOpts } from '@/store.ts'; import { isNostrId, isURL } from '@/utils.ts'; @@ -31,14 +31,14 @@ const tagConditions: Record = { }; /** Insert an event (and its tags) into the database. */ -function storeEvent(event: Event, opts: StoreEventOpts = {}): Promise { +async function storeEvent(event: Event, opts: StoreEventOpts = {}): Promise { debug('EVENT', JSON.stringify(event)); if (isDittoInternalKind(event.kind) && event.pubkey !== Conf.pubkey) { throw new Error('Internal events can only be stored by the server keypair'); } - return db.transaction().execute(async (trx) => { + return await db.transaction().execute(async (trx) => { /** Insert the event into the database. */ async function addEvent() { await trx.insertInto('events') @@ -66,6 +66,30 @@ function storeEvent(event: Event, opts: StoreEventOpts = {}): Promise { .execute(); } + if (isReplaceableKind(event.kind)) { + const prevEvents = await getFilterQuery(trx, { kinds: [event.kind], authors: [event.pubkey] }).execute(); + for (const prevEvent of prevEvents) { + if (prevEvent.created_at >= event.created_at) { + throw new Error('Cannot replace an event with an older event'); + } + } + await deleteEventsTrx(trx, [{ kinds: [event.kind], authors: [event.pubkey] }]); + } + + if (isParameterizedReplaceableKind(event.kind)) { + const d = event.tags.find(([tag]) => tag === 'd')?.[1]; + if (d) { + const prevEvents = await getFilterQuery(trx, { kinds: [event.kind], authors: [event.pubkey], '#d': [d] }) + .execute(); + for (const prevEvent of prevEvents) { + if (prevEvent.created_at >= event.created_at) { + throw new Error('Cannot replace an event with an older event'); + } + } + await deleteEventsTrx(trx, [{ kinds: [event.kind], authors: [event.pubkey], '#d': [d] }]); + } + } + // Run the queries. await Promise.all([ addEvent(), @@ -106,7 +130,7 @@ type EventQuery = SelectQueryBuilder; /** Build the query for a filter. */ -function getFilterQuery(filter: DittoFilter): EventQuery { +function getFilterQuery(db: Kysely, filter: DittoFilter): EventQuery { let query = db .selectFrom('events') .select([ @@ -215,13 +239,13 @@ function getFilterQuery(filter: DittoFilter): EventQuery { /** Combine filter queries into a single union query. */ function getEventsQuery(filters: DittoFilter[]) { return filters - .map((filter) => db.selectFrom(() => getFilterQuery(filter).as('events')).selectAll()) + .map((filter) => db.selectFrom(() => getFilterQuery(db, filter).as('events')).selectAll()) .reduce((result, query) => result.unionAll(query)); } /** Query to get user events, joined by tags. */ function usersQuery() { - return getFilterQuery({ kinds: [30361], authors: [Conf.pubkey] }) + return getFilterQuery(db, { kinds: [30361], authors: [Conf.pubkey] }) .leftJoin('tags', 'tags.event_id', 'events.id') .where('tags.tag', '=', 'd') .select('tags.value as d_tag') @@ -284,22 +308,28 @@ async function getEvents( }); } +/** Delete events from each table. Should be run in a transaction! */ +async function deleteEventsTrx(db: Kysely, filters: DittoFilter[]) { + if (!filters.length) return Promise.resolve(); + debug('DELETE', JSON.stringify(filters)); + + const query = getEventsQuery(filters).clearSelect().select('id'); + + await db.deleteFrom('events_fts') + .where('id', 'in', () => query) + .execute(); + + return db.deleteFrom('events') + .where('id', 'in', () => query) + .execute(); +} + /** Delete events based on filters from the database. */ async function deleteEvents(filters: DittoFilter[]): Promise { if (!filters.length) return Promise.resolve(); debug('DELETE', JSON.stringify(filters)); - await db.transaction().execute(async (trx) => { - const query = getEventsQuery(filters).clearSelect().select('id'); - - await trx.deleteFrom('events_fts') - .where('id', 'in', () => query) - .execute(); - - return trx.deleteFrom('events') - .where('id', 'in', () => query) - .execute(); - }); + await db.transaction().execute((trx) => deleteEventsTrx(trx, filters)); } /** Get number of events that would be returned by filters. */ diff --git a/src/db/users.ts b/src/db/users.ts index 195dfe2..d223b1c 100644 --- a/src/db/users.ts +++ b/src/db/users.ts @@ -13,12 +13,11 @@ interface User { admin: boolean; } -/** Adds a user to the database. */ -async function insertUser(user: User) { +function buildUserEvent(user: User) { debug('insertUser', JSON.stringify(user)); const { origin, host } = Conf.url; - const event = await signAdminEvent({ + return signAdminEvent({ kind: 30361, tags: [ ['d', user.pubkey], @@ -31,7 +30,12 @@ async function insertUser(user: User) { content: '', created_at: Math.floor(user.inserted_at.getTime() / 1000), }); +} +/** Adds a user to the database. */ +async function insertUser(user: User) { + debug('insertUser', JSON.stringify(user)); + const event = await buildUserEvent(user); return pipeline.handleEvent(event); } @@ -71,4 +75,4 @@ async function findUser(user: Partial): Promise { } } -export { findUser, insertUser, type User }; +export { buildUserEvent, findUser, insertUser, type User }; From 47e5696a1bba29b15e6ec7e165e24d2f2e6db2e2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 29 Dec 2023 23:29:38 -0600 Subject: [PATCH 10/10] gitlab-ci: fix DITTO_NSEC in tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d065493..dba5873 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,4 +18,4 @@ test: stage: test script: deno task test variables: - DITTO_NSEC: nsec1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqwkhnav \ No newline at end of file + DITTO_NSEC: nsec1zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs4rm7hz \ No newline at end of file