From 887c68f052db1b66aa38e4102b4d36bea9f9eb7e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 11:55:16 -0500 Subject: [PATCH 01/19] config: add comments to all config options --- src/config.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/config.ts b/src/config.ts index f13a407..7add7d8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,4 @@ -import { dotenv, nip19, secp } from '@/deps.ts'; +import { dotenv, getPublicKey, nip19, secp } from '@/deps.ts'; /** Load environment config from `.env` */ await dotenv.load({ @@ -9,6 +9,7 @@ await dotenv.load({ /** Application-wide configuration. */ const Conf = { + /** Ditto admin secret key in nip19 format. This is the way it's configured by an admin. */ get nsec() { const value = Deno.env.get('DITTO_NSEC'); if (!value) { @@ -19,13 +20,15 @@ const Conf = { } return value as `nsec1${string}`; }, + /** Ditto admin secret key in hex format. */ get seckey() { - const result = nip19.decode(Conf.nsec); - if (result.type !== 'nsec') { - throw new Error('Invalid DITTO_NSEC'); - } - return result.data; + return nip19.decode(Conf.nsec).data; }, + /** Ditto admin public key in hex format. */ + get pubkey() { + return getPublicKey(Conf.seckey); + }, + /** Ditto admin secret key as a Web Crypto key. */ get cryptoKey() { return crypto.subtle.importKey( 'raw', @@ -39,24 +42,31 @@ const Conf = { const { protocol, host } = Conf.url; return `${protocol === 'https:' ? 'wss:' : 'ws:'}//${host}/relay`; }, + /** Domain of the Ditto server, including the protocol. */ get localDomain() { return Deno.env.get('LOCAL_DOMAIN') || 'http://localhost:8000'; }, + /** Path to the main SQLite database which stores users, events, and more. */ get dbPath() { return Deno.env.get('DB_PATH') || 'data/db.sqlite3'; }, + /** Character limit to enforce for posts made through Mastodon API. */ get postCharLimit() { return Number(Deno.env.get('POST_CHAR_LIMIT') || 5000); }, + /** Admin contact to expose through various endpoints. This information is public. */ get adminEmail() { return Deno.env.get('ADMIN_EMAIL') || 'webmaster@localhost'; }, + /** @deprecated Use relays from the database instead. */ get poolRelays() { return (Deno.env.get('RELAY_POOL') || '').split(',').filter(Boolean); }, + /** @deprecated Publish only to the local relay unless users are mentioned, then try to also send to the relay of those users. Deletions should also be fanned out. */ get publishRelays() { return ['wss://relay.mostr.pub']; }, + /** Domain of the Ditto server as a `URL` object, for easily grabbing the `hostname`, etc. */ get url() { return new URL(Conf.localDomain); }, From baf4c00fee99eff8dae8d790bd8c0c4349c19318 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 12:01:46 -0500 Subject: [PATCH 02/19] instance: add urls.nostr_relay and pubkey properties to instance --- src/controllers/api/instance.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index d0235f5..0b5706a 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -5,6 +5,9 @@ import type { Context } from '@/deps.ts'; function instanceController(c: Context) { const { host, protocol } = Conf.url; + /** Protocol to use for WebSocket URLs, depending on the protocol of the `LOCAL_DOMAIN`. */ + const wsProtocol = protocol === 'http:' ? 'ws:' : 'wss:'; + return c.json({ uri: host, title: 'Ditto', @@ -35,10 +38,15 @@ function instanceController(c: Context) { user_count: 0, }, urls: { - streaming_api: `${protocol === 'http:' ? 'ws:' : 'wss:'}//${host}`, + /** Base URL for the streaming API, so it can be hosted on another domain. Clients will add `/api/v1/streaming` to it. */ + streaming_api: `${wsProtocol}//${host}`, + /** Full URL to the Nostr relay. */ + nostr_relay: `${wsProtocol}//${host}/relay`, }, version: '0.0.0 (compatible; Ditto 0.0.1)', email: Conf.adminEmail, + /** Ditto admin pubkey, so clients can query and validate Nostr events from the server. */ + pubkey: Conf.pubkey, rules: [], }); } From 3c279175bc3799146341e10fbd75e023d050b163 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 12:28:25 -0500 Subject: [PATCH 03/19] instance: actually, put all this under a nostr key --- src/controllers/api/instance.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index 0b5706a..6ffb85c 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -38,15 +38,14 @@ function instanceController(c: Context) { user_count: 0, }, urls: { - /** Base URL for the streaming API, so it can be hosted on another domain. Clients will add `/api/v1/streaming` to it. */ streaming_api: `${wsProtocol}//${host}`, - /** Full URL to the Nostr relay. */ - nostr_relay: `${wsProtocol}//${host}/relay`, }, version: '0.0.0 (compatible; Ditto 0.0.1)', email: Conf.adminEmail, - /** Ditto admin pubkey, so clients can query and validate Nostr events from the server. */ - pubkey: Conf.pubkey, + nostr: { + pubkey: Conf.pubkey, + relay: `${wsProtocol}//${host}/relay`, + }, rules: [], }); } From 60cecafdb56b7bcea4120e89be2b2fce44289c5e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 12:48:08 -0500 Subject: [PATCH 04/19] Remove admin.ts, move to sign.ts, add createAdminEvent function --- src/admin.ts | 15 --------------- src/sign.ts | 17 ++++++++++------- src/utils/web.ts | 39 ++++++++++++++++++++++++++++++++------- 3 files changed, 42 insertions(+), 29 deletions(-) delete mode 100644 src/admin.ts diff --git a/src/admin.ts b/src/admin.ts deleted file mode 100644 index 85e2fd8..0000000 --- a/src/admin.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Conf } from '@/config.ts'; -import { type Event, type EventTemplate, finishEvent, nip19 } from '@/deps.ts'; - -// deno-lint-ignore require-await -async function signAdminEvent(event: EventTemplate): Promise> { - if (!Conf.nsec) throw new Error('No secret key. Set one with DITTO_NSEC.'); - - const result = nip19.decode(Conf.nsec); - - if (result.type !== 'nsec') throw new Error('Invalid DITTO_NSEC. It should start with "nsec1..."'); - - return finishEvent(event, result.data); -} - -export { signAdminEvent }; diff --git a/src/sign.ts b/src/sign.ts index d78aa4c..732c769 100644 --- a/src/sign.ts +++ b/src/sign.ts @@ -1,5 +1,6 @@ import { type AppContext } from '@/app.ts'; -import { type Event, type EventTemplate, getEventHash, getPublicKey, getSignature, HTTPException, z } from '@/deps.ts'; +import { Conf } from '@/config.ts'; +import { type Event, type EventTemplate, finishEvent, HTTPException, z } from '@/deps.ts'; import { signedEventSchema } from '@/schemas/nostr.ts'; import { ws } from '@/stream.ts'; @@ -61,11 +62,13 @@ async function signEvent(event: EventTemplate, c: }); } - (event as Event).pubkey = getPublicKey(seckey); - (event as Event).id = getEventHash(event as Event); - (event as Event).sig = getSignature(event as Event, seckey); - - return event as Event; + return finishEvent(event, seckey); } -export { signEvent }; +/** Sign event as the Ditto server. */ +// deno-lint-ignore require-await +async function signAdminEvent(event: EventTemplate): Promise> { + return finishEvent(event, Conf.seckey); +} + +export { signAdminEvent, signEvent }; diff --git a/src/utils/web.ts b/src/utils/web.ts index 1504614..91eaacd 100644 --- a/src/utils/web.ts +++ b/src/utils/web.ts @@ -1,16 +1,18 @@ import { Conf } from '@/config.ts'; import { type Context, type Event, EventTemplate, HTTPException, parseFormData, z } from '@/deps.ts'; import * as pipeline from '@/pipeline.ts'; -import { signEvent } from '@/sign.ts'; +import { signAdminEvent, signEvent } from '@/sign.ts'; import { nostrNow } from '@/utils.ts'; import type { AppContext } from '@/app.ts'; -/** Publish an event through the API, throwing a Hono exception on failure. */ -async function createEvent( - t: Omit, 'created_at'>, - c: AppContext, -): Promise> { +/** EventTemplate with or without a timestamp. If no timestamp is given, it will be generated. */ +interface PendingEvent extends Omit, 'created_at'> { + created_at?: number; +} + +/** Publish an event through the pipeline. */ +async function createEvent(t: PendingEvent, c: AppContext): Promise> { const pubkey = c.get('pubkey'); if (!pubkey) { @@ -22,6 +24,21 @@ async function createEvent( ...t, }, c); + return publishEvent(event, c); +} + +/** Publish an admin event through the pipeline. */ +async function createAdminEvent(t: PendingEvent, c: AppContext): Promise> { + const event = await signAdminEvent({ + created_at: nostrNow(), + ...t, + }); + + return publishEvent(event, c); +} + +/** Push the event through the pipeline, rethrowing any RelayError. */ +async function publishEvent(event: Event, c: AppContext): Promise> { try { await pipeline.handleEvent(event); } catch (e) { @@ -90,4 +107,12 @@ function activityJson(c: Context, object: T) { return response; } -export { activityJson, buildLinkHeader, createEvent, type PaginationParams, paginationSchema, parseBody }; +export { + activityJson, + buildLinkHeader, + createAdminEvent, + createEvent, + type PaginationParams, + paginationSchema, + parseBody, +}; From e17111a85943c686b52146971e1fea2782f6d5c5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 12:52:24 -0500 Subject: [PATCH 05/19] utils/web: PendingEvent --> EventStub --- src/utils/web.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/web.ts b/src/utils/web.ts index 91eaacd..5df0060 100644 --- a/src/utils/web.ts +++ b/src/utils/web.ts @@ -7,12 +7,12 @@ import { nostrNow } from '@/utils.ts'; import type { AppContext } from '@/app.ts'; /** EventTemplate with or without a timestamp. If no timestamp is given, it will be generated. */ -interface PendingEvent extends Omit, 'created_at'> { +interface EventStub extends Omit, 'created_at'> { created_at?: number; } /** Publish an event through the pipeline. */ -async function createEvent(t: PendingEvent, c: AppContext): Promise> { +async function createEvent(t: EventStub, c: AppContext): Promise> { const pubkey = c.get('pubkey'); if (!pubkey) { @@ -28,7 +28,7 @@ async function createEvent(t: PendingEvent, c: AppContext): } /** Publish an admin event through the pipeline. */ -async function createAdminEvent(t: PendingEvent, c: AppContext): Promise> { +async function createAdminEvent(t: EventStub, c: AppContext): Promise> { const event = await signAdminEvent({ created_at: nostrNow(), ...t, From 7570b0dee4a43ac5fa5a5c2cff15ece81bec23f8 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 12:56:07 -0500 Subject: [PATCH 06/19] utils: new Date().getTime() --> Date.now() --- src/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.ts b/src/utils.ts index 653c824..44dff18 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -3,7 +3,7 @@ import { lookupNip05Cached } from '@/nip05.ts'; import { getAuthor } from '@/queries.ts'; /** Get the current time in Nostr format. */ -const nostrNow = () => Math.floor(new Date().getTime() / 1000); +const nostrNow = () => Math.floor(Date.now() / 1000); /** Convenience function to convert Nostr dates into native Date objects. */ const nostrDate = (seconds: number) => new Date(seconds * 1000); From 63def1d62c5106c004768e29a6fb5416e02a94ed Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 12:58:17 -0500 Subject: [PATCH 07/19] utils: add return types (to improve readability) --- src/utils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 44dff18..7f47f31 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -3,12 +3,12 @@ import { lookupNip05Cached } from '@/nip05.ts'; import { getAuthor } from '@/queries.ts'; /** Get the current time in Nostr format. */ -const nostrNow = () => Math.floor(Date.now() / 1000); +const nostrNow = (): number => Math.floor(Date.now() / 1000); /** Convenience function to convert Nostr dates into native Date objects. */ -const nostrDate = (seconds: number) => new Date(seconds * 1000); +const nostrDate = (seconds: number): Date => new Date(seconds * 1000); /** Pass to sort() to sort events by date. */ -const eventDateComparator = (a: Event, b: Event) => b.created_at - a.created_at; +const eventDateComparator = (a: Event, b: Event): number => b.created_at - a.created_at; /** Get pubkey from bech32 string, if applicable. */ function bech32ToPubkey(bech32: string): string | undefined { From fdc39297fdba5e9c7510c8ed9cc00a1183da9c1f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 13:18:58 -0500 Subject: [PATCH 08/19] Add kinds module to classify events by kind --- src/kinds.ts | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 src/kinds.ts diff --git a/src/kinds.ts b/src/kinds.ts new file mode 100644 index 0000000..33558cb --- /dev/null +++ b/src/kinds.ts @@ -0,0 +1,40 @@ +/** Events are **regular**, which means they're all expected to be stored by relays. */ +function isRegularKind(kind: number) { + return 1000 <= kind && kind < 10000; +} + +/** Events are **replaceable**, which means that, for each combination of `pubkey` and `kind`, only the latest event is expected to (SHOULD) be stored by relays, older versions are expected to be discarded. */ +function isReplaceableKind(kind: number) { + return (10000 <= kind && kind < 20000) || kind == 0 || kind == 3; +} + +/** Events are **ephemeral**, which means they are not expected to be stored by relays. */ +function isEphemeralKind(kind: number) { + return 20000 <= kind && kind < 30000; +} + +/** Events are **parameterized replaceable**, which means that, for each combination of `pubkey`, `kind` and the `d` tag, only the latest event is expected to be stored by relays, older versions are expected to be discarded. */ +function isParameterizedReplaceableKind(kind: number) { + return 30000 <= kind && kind < 40000; +} + +/** Classification of the event kind. */ +type KindClassification = 'regular' | 'replaceable' | 'ephemeral' | 'parameterized' | 'unknown'; + +/** Determine the classification of this kind of event if known, or `unknown`. */ +function classifyKind(kind: number): KindClassification { + if (isRegularKind(kind)) return 'regular'; + if (isReplaceableKind(kind)) return 'replaceable'; + if (isEphemeralKind(kind)) return 'ephemeral'; + if (isParameterizedReplaceableKind(kind)) return 'parameterized'; + return 'unknown'; +} + +export { + classifyKind, + isEphemeralKind, + isParameterizedReplaceableKind, + isRegularKind, + isReplaceableKind, + type KindClassification, +}; From 2aefdc4bd170e4db40663ec8a85ac0003d7580fd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 13:25:32 -0500 Subject: [PATCH 09/19] kinds: improve detection of legacy kinds --- src/kinds.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kinds.ts b/src/kinds.ts index 33558cb..45c9729 100644 --- a/src/kinds.ts +++ b/src/kinds.ts @@ -1,11 +1,11 @@ /** Events are **regular**, which means they're all expected to be stored by relays. */ function isRegularKind(kind: number) { - return 1000 <= kind && kind < 10000; + return (1000 <= kind && kind < 10000) || [1, 2, 4, 5, 6, 7, 8, 16, 40, 41, 42, 43, 44].includes(kind); } /** Events are **replaceable**, which means that, for each combination of `pubkey` and `kind`, only the latest event is expected to (SHOULD) be stored by relays, older versions are expected to be discarded. */ function isReplaceableKind(kind: number) { - return (10000 <= kind && kind < 20000) || kind == 0 || kind == 3; + return (10000 <= kind && kind < 20000) || [0, 3].includes(kind); } /** Events are **ephemeral**, which means they are not expected to be stored by relays. */ From 1b2a486c659d23877af46a115c8b4d826f14e418 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 13:40:10 -0500 Subject: [PATCH 10/19] pipeline: don't store ephemeral events --- src/pipeline.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipeline.ts b/src/pipeline.ts index e1a9c3a..14a364a 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -2,6 +2,7 @@ import * as eventsDB from '@/db/events.ts'; import { addRelays } from '@/db/relays.ts'; import { findUser } from '@/db/users.ts'; import { type Event, LRUCache } from '@/deps.ts'; +import { isEphemeralKind } from '@/kinds.ts'; import { isLocallyFollowed } from '@/queries.ts'; import { Sub } from '@/subs.ts'; import { trends } from '@/trends.ts'; @@ -43,6 +44,7 @@ async function getEventData({ pubkey }: Event): Promise { /** Maybe store the event, if eligible. */ async function storeEvent(event: Event, data: EventData): Promise { + if (isEphemeralKind(event.kind)) return; if (data.user || await isLocallyFollowed(event.pubkey)) { await eventsDB.insertEvent(event).catch(console.warn); } else { From 67bba508aff5cb5ed39e0884a922ba940ed86f06 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 15:21:33 -0500 Subject: [PATCH 11/19] utils/web: make `tags` optional --- src/deps.ts | 2 ++ src/utils/web.ts | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/deps.ts b/src/deps.ts index d8c9c8f..bcc3f42 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -63,3 +63,5 @@ export { } from 'npm:kysely@^0.25.0'; export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/v1.0.0/mod.ts'; export { default as tldts } from 'npm:tldts@^6.0.14'; + +export type * as TypeFest from 'npm:type-fest@^4.3.0'; diff --git a/src/utils/web.ts b/src/utils/web.ts index 5df0060..7044d04 100644 --- a/src/utils/web.ts +++ b/src/utils/web.ts @@ -1,15 +1,13 @@ import { Conf } from '@/config.ts'; -import { type Context, type Event, EventTemplate, HTTPException, parseFormData, z } from '@/deps.ts'; +import { type Context, type Event, EventTemplate, HTTPException, parseFormData, type TypeFest, z } from '@/deps.ts'; import * as pipeline from '@/pipeline.ts'; import { signAdminEvent, signEvent } from '@/sign.ts'; import { nostrNow } from '@/utils.ts'; import type { AppContext } from '@/app.ts'; -/** EventTemplate with or without a timestamp. If no timestamp is given, it will be generated. */ -interface EventStub extends Omit, 'created_at'> { - created_at?: number; -} +/** EventTemplate with defaults. */ +type EventStub = TypeFest.SetOptional, 'created_at' | 'tags'>; /** Publish an event through the pipeline. */ async function createEvent(t: EventStub, c: AppContext): Promise> { @@ -21,6 +19,7 @@ async function createEvent(t: EventStub, c: AppContext): Pr const event = await signEvent({ created_at: nostrNow(), + tags: [], ...t, }, c); @@ -31,6 +30,7 @@ async function createEvent(t: EventStub, c: AppContext): Pr async function createAdminEvent(t: EventStub, c: AppContext): Promise> { const event = await signAdminEvent({ created_at: nostrNow(), + tags: [], ...t, }); From c13b7f4af7d55f717b8147c2debb0faef704c6cb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 15:55:16 -0500 Subject: [PATCH 12/19] subs: allow any object in place of the socket --- src/subs.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/subs.ts b/src/subs.ts index eebfac1..f9d6606 100644 --- a/src/subs.ts +++ b/src/subs.ts @@ -9,7 +9,7 @@ import type { EventData } from '@/types.ts'; * Subscriptions can be added, removed, and matched against events. */ class SubscriptionStore { - #store = new Map>(); + #store = new Map>(); /** * Add a subscription to the store, and then iterate over it. @@ -20,7 +20,7 @@ class SubscriptionStore { * } * ``` */ - sub(socket: WebSocket, id: string, filters: DittoFilter[]): Subscription { + sub(socket: unknown, id: string, filters: DittoFilter[]): Subscription { let subs = this.#store.get(socket); if (!subs) { @@ -37,13 +37,13 @@ class SubscriptionStore { } /** Remove a subscription from the store. */ - unsub(socket: WebSocket, id: string): void { + unsub(socket: unknown, id: string): void { this.#store.get(socket)?.get(id)?.close(); this.#store.get(socket)?.delete(id); } /** Remove an entire socket. */ - close(socket: WebSocket): void { + close(socket: unknown): void { const subs = this.#store.get(socket); if (subs) { From 9cd1ca18610aa838b2938c4790db8a1d89d5c4f3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 15:57:51 -0500 Subject: [PATCH 13/19] streaming: don't forget to close the subscription when the socket closes --- src/controllers/api/streaming.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 8b655f9..1c85259 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -61,6 +61,7 @@ const streamingController: AppController = (c) => { socket.onclose = () => { ws.unsubscribeAll(socket); + Sub.close(socket); }; return response; From 1806cf2286de8a8f8655b623347c1c20622b2bb5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 17:31:52 -0500 Subject: [PATCH 14/19] filter: treat the admin pubkey as local --- src/filter.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/filter.ts b/src/filter.ts index 605d4cb..4bc806a 100644 --- a/src/filter.ts +++ b/src/filter.ts @@ -1,3 +1,4 @@ +import { Conf } from '@/config.ts'; import { type Event, type Filter, matchFilters } from '@/deps.ts'; import type { EventData } from '@/types.ts'; @@ -16,7 +17,7 @@ interface GetFiltersOpts { } function matchDittoFilter(filter: DittoFilter, event: Event, data: EventData): boolean { - if (filter.local && !data.user) { + if (filter.local && !(data.user || event.pubkey === Conf.pubkey)) { return false; } From 655004e77554dc5dd6047206418e5d9e323258fc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 18:03:59 -0500 Subject: [PATCH 15/19] Rework web signer to use NIP-46 events --- src/crypto.ts | 14 +++++ src/deps.ts | 1 + src/schemas/nostr.ts | 7 +++ src/sign.ts | 136 ++++++++++++++++++++++++++----------------- 4 files changed, 105 insertions(+), 53 deletions(-) create mode 100644 src/crypto.ts diff --git a/src/crypto.ts b/src/crypto.ts new file mode 100644 index 0000000..9cafb51 --- /dev/null +++ b/src/crypto.ts @@ -0,0 +1,14 @@ +import { Conf } from '@/config.ts'; +import { nip04 } from '@/deps.ts'; + +/** Encrypt a message as the Ditto server account. */ +function encryptAdmin(targetPubkey: string, message: string): Promise { + return nip04.encrypt(Conf.seckey, targetPubkey, message); +} + +/** Decrypt a message as the Ditto server account. */ +function decryptAdmin(targetPubkey: string, message: string): Promise { + return nip04.decrypt(Conf.seckey, targetPubkey, message); +} + +export { decryptAdmin, encryptAdmin }; diff --git a/src/deps.ts b/src/deps.ts index bcc3f42..dd887b6 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -20,6 +20,7 @@ export { getSignature, Kind, matchFilters, + nip04, nip05, nip19, nip21, diff --git a/src/schemas/nostr.ts b/src/schemas/nostr.ts index cbc6816..4147c3c 100644 --- a/src/schemas/nostr.ts +++ b/src/schemas/nostr.ts @@ -80,12 +80,19 @@ const relayInfoDocSchema = z.object({ icon: safeUrlSchema.optional().catch(undefined), }); +/** NIP-46 signer response. */ +const connectResponseSchema = z.object({ + id: z.string(), + result: signedEventSchema, +}); + export { type ClientCLOSE, type ClientEVENT, type ClientMsg, clientMsgSchema, type ClientREQ, + connectResponseSchema, filterSchema, jsonMetaContentSchema, metaContentSchema, diff --git a/src/sign.ts b/src/sign.ts index 732c769..20a0350 100644 --- a/src/sign.ts +++ b/src/sign.ts @@ -1,68 +1,98 @@ import { type AppContext } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { type Event, type EventTemplate, finishEvent, HTTPException, z } from '@/deps.ts'; -import { signedEventSchema } from '@/schemas/nostr.ts'; -import { ws } from '@/stream.ts'; - -/** Get signing WebSocket from app context. */ -function getSignStream(c: AppContext): WebSocket | undefined { - const pubkey = c.get('pubkey'); - const session = c.get('session'); - - if (pubkey && session) { - const [socket] = ws.getSockets(`nostr:${pubkey}:${session}`); - return socket; - } -} - -const nostrStreamingEventSchema = z.object({ - type: z.literal('nostr.sign'), - data: signedEventSchema, -}); +import { decryptAdmin, encryptAdmin } from '@/crypto.ts'; +import { type Event, type EventTemplate, finishEvent, HTTPException } from '@/deps.ts'; +import { connectResponseSchema } from '@/schemas/nostr.ts'; +import { Sub } from '@/subs.ts'; +import { Time } from '@/utils.ts'; +import { createAdminEvent } from '@/utils/web.ts'; /** * Sign Nostr event using the app context. * * - If a secret key is provided, it will be used to sign the event. - * - If a signing WebSocket is provided, it will be used to sign the event. + * - If `X-Nostr-Sign` is passed, it will use a NIP-46 to sign the event. */ async function signEvent(event: EventTemplate, c: AppContext): Promise> { const seckey = c.get('seckey'); - const stream = getSignStream(c); + const header = c.req.headers.get('x-nostr-sign'); - if (!seckey && stream) { - try { - return await new Promise>((resolve, reject) => { - const handleMessage = (e: MessageEvent) => { - try { - const { data: event } = nostrStreamingEventSchema.parse(JSON.parse(e.data)); - stream.removeEventListener('message', handleMessage); - resolve(event as Event); - } catch (_e) { - // + if (seckey) { + return finishEvent(event, seckey); + } + + if (header) { + return await signNostrConnect(event, c); + } + + throw new HTTPException(400, { + res: c.json({ id: 'ditto.sign', error: 'Unable to sign event' }, 400), + }); +} + +/** Sign event with NIP-46, waiting in the background for the signed event. */ +async function signNostrConnect(event: EventTemplate, c: AppContext): Promise> { + const pubkey = c.get('pubkey'); + + if (!pubkey) { + throw new HTTPException(401); + } + + const messageId = crypto.randomUUID(); + + createAdminEvent({ + kind: 24133, + content: await encryptAdmin( + pubkey, + JSON.stringify({ + id: messageId, + method: 'sign_event', + params: [event], + }), + ), + tags: [['p', pubkey]], + }, c); + + return awaitSignedEvent(pubkey, messageId, c); +} + +/** Wait for signed event to be sent through Nostr relay. */ +function awaitSignedEvent( + pubkey: string, + messageId: string, + c: AppContext, +): Promise> { + const sub = Sub.sub(messageId, '1', [{ kinds: [24133], authors: [pubkey], '#p': [Conf.pubkey] }]); + + function close(): void { + Sub.close(messageId); + } + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + close(); + reject( + new HTTPException(408, { + res: c.json({ id: 'ditto.timeout', error: 'Signing timeout' }), + }), + ); + }, Time.minutes(1)); + + (async () => { + for await (const event of sub) { + if (event.kind === 24133) { + const decrypted = await decryptAdmin(event.pubkey, event.content); + const msg = connectResponseSchema.parse(decrypted); + + if (msg.id === messageId) { + close(); + clearTimeout(timeout); + resolve(msg.result as Event); } - }; - stream.addEventListener('message', handleMessage); - stream.send(JSON.stringify({ event: 'nostr.sign', payload: JSON.stringify(event) })); - setTimeout(() => { - stream.removeEventListener('message', handleMessage); - reject(); - }, 60000); - }); - } catch (_e) { - throw new HTTPException(408, { - res: c.json({ id: 'ditto.timeout', error: 'Signing timeout' }, 408), - }); - } - } - - if (!seckey) { - throw new HTTPException(400, { - res: c.json({ id: 'ditto.private_key', error: 'No private key' }, 400), - }); - } - - return finishEvent(event, seckey); + } + } + })(); + }); } /** Sign event as the Ditto server. */ From 0227eb3b345fa06cd79db507f950d90675e29a25 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 19:14:29 -0500 Subject: [PATCH 16/19] deno.json: fix imports warning --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index a2faa60..72a9518 100644 --- a/deno.json +++ b/deno.json @@ -11,7 +11,7 @@ }, "imports": { "@/": "./src/", - "~/": "./" + "~/fixtures/": "./fixtures/" }, "lint": { "include": ["src/", "scripts/"], From 0a9b53bbc13beb2a206c221aaebcb3f1ecbe68d9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 22:49:32 -0500 Subject: [PATCH 17/19] sign: fix parsing connect response from string to JSON --- src/sign.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sign.ts b/src/sign.ts index 20a0350..2a91a06 100644 --- a/src/sign.ts +++ b/src/sign.ts @@ -3,6 +3,7 @@ import { Conf } from '@/config.ts'; import { decryptAdmin, encryptAdmin } from '@/crypto.ts'; import { type Event, type EventTemplate, finishEvent, HTTPException } from '@/deps.ts'; import { connectResponseSchema } from '@/schemas/nostr.ts'; +import { jsonSchema } from '@/schema.ts'; import { Sub } from '@/subs.ts'; import { Time } from '@/utils.ts'; import { createAdminEvent } from '@/utils/web.ts'; @@ -82,7 +83,7 @@ function awaitSignedEvent( for await (const event of sub) { if (event.kind === 24133) { const decrypted = await decryptAdmin(event.pubkey, event.content); - const msg = connectResponseSchema.parse(decrypted); + const msg = jsonSchema.pipe(connectResponseSchema).parse(decrypted); if (msg.id === messageId) { close(); From 320d2f493ee43dfcdbb82ec78a590a77e151d169 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 26 Aug 2023 23:13:48 -0500 Subject: [PATCH 18/19] change .positive() to .nonnegative() !!! --- src/controllers/api/accounts.ts | 2 +- src/schemas/nostr.ts | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 95fe0c7..9244562 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -82,7 +82,7 @@ const relationshipsController: AppController = async (c) => { const accountStatusesQuerySchema = z.object({ pinned: booleanParamSchema.optional(), - limit: z.coerce.number().positive().transform((v) => Math.min(v, 40)).catch(20), + limit: z.coerce.number().nonnegative().transform((v) => Math.min(v, 40)).catch(20), exclude_replies: booleanParamSchema.optional(), tagged: z.string().optional(), }); diff --git a/src/schemas/nostr.ts b/src/schemas/nostr.ts index 4147c3c..272981f 100644 --- a/src/schemas/nostr.ts +++ b/src/schemas/nostr.ts @@ -5,7 +5,7 @@ import { jsonSchema, safeUrlSchema } from '../schema.ts'; /** Schema to validate Nostr hex IDs such as event IDs and pubkeys. */ const nostrIdSchema = z.string().regex(/^[0-9a-f]{64}$/); /** Nostr kinds are positive integers. */ -const kindSchema = z.number().int().positive(); +const kindSchema = z.number().int().nonnegative(); /** Nostr event schema. */ const eventSchema = z.object({ @@ -26,9 +26,9 @@ const filterSchema = z.object({ kinds: kindSchema.array().optional(), ids: nostrIdSchema.array().optional(), authors: nostrIdSchema.array().optional(), - since: z.number().int().positive().optional(), - until: z.number().int().positive().optional(), - limit: z.number().int().positive().optional(), + since: z.number().int().nonnegative().optional(), + until: z.number().int().nonnegative().optional(), + limit: z.number().int().nonnegative().optional(), }).passthrough().and( z.record( z.custom<`#${string}`>((val) => typeof val === 'string' && val.startsWith('#')), @@ -75,7 +75,7 @@ const relayInfoDocSchema = z.object({ description: z.string().transform((val) => val.slice(0, 3000)).optional().catch(undefined), pubkey: nostrIdSchema.optional().catch(undefined), contact: safeUrlSchema.optional().catch(undefined), - supported_nips: z.number().int().positive().array().optional().catch(undefined), + supported_nips: z.number().int().nonnegative().array().optional().catch(undefined), software: safeUrlSchema.optional().catch(undefined), icon: safeUrlSchema.optional().catch(undefined), }); From 2e8b26cf4f9c817f7e217eaa6feebaa05cf95c3f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 27 Aug 2023 10:07:13 -0500 Subject: [PATCH 19/19] Remove legacy stream.ts module --- src/controllers/api/streaming.ts | 24 ++--- src/stream.ts | 158 ------------------------------- src/utils.ts | 1 - 3 files changed, 12 insertions(+), 171 deletions(-) delete mode 100644 src/stream.ts diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 1c85259..ca8cafc 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -1,10 +1,20 @@ import { AppController } from '@/app.ts'; +import { z } from '@/deps.ts'; import { type DittoFilter } from '@/filter.ts'; import { TOKEN_REGEX } from '@/middleware/auth19.ts'; -import { streamSchema, ws } from '@/stream.ts'; import { Sub } from '@/subs.ts'; import { toStatus } from '@/transformers/nostr-to-mastoapi.ts'; -import { bech32ToPubkey } from '@/utils.ts'; + +/** + * Streaming timelines/categories. + * https://docs.joinmastodon.org/methods/streaming/#streams + */ +const streamSchema = z.enum([ + 'nostr', + 'public', + 'public:local', + 'user', +]); const streamingController: AppController = (c) => { const upgrade = c.req.headers.get('upgrade'); @@ -26,12 +36,6 @@ const streamingController: AppController = (c) => { const { socket, response } = Deno.upgradeWebSocket(c.req.raw, { protocol: token }); - const conn = { - socket, - session: match[2], - pubkey: bech32ToPubkey(match[1]), - }; - function send(name: string, payload: object) { if (socket.readyState === WebSocket.OPEN) { socket.send(JSON.stringify({ @@ -44,9 +48,6 @@ const streamingController: AppController = (c) => { socket.onopen = async () => { if (!stream) return; - - ws.subscribe(conn, { stream }); - const filter = topicToFilter(stream); if (filter) { @@ -60,7 +61,6 @@ const streamingController: AppController = (c) => { }; socket.onclose = () => { - ws.unsubscribeAll(socket); Sub.close(socket); }; diff --git a/src/stream.ts b/src/stream.ts deleted file mode 100644 index fa352ee..0000000 --- a/src/stream.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { z } from '@/deps.ts'; - -/** Internal key for event subscriptions. */ -type Topic = - | `nostr:${string}:${string}` - | 'public' - | 'public:local'; - -/** - * Streaming timelines/categories. - * https://docs.joinmastodon.org/methods/streaming/#streams - */ -const streamSchema = z.enum([ - 'nostr', - 'public', - 'public:local', - 'user', -]); - -type Stream = z.infer; - -/** Only the necessary metadata needed from the request. */ -interface StreamConn { - /** Hex pubkey parsed from the `Sec-Websocket-Protocol` header. */ - pubkey?: string; - /** Base62 session UUID parsed from the `Sec-Websocket-Protocol` header. */ - session?: string; - /** The WebSocket stream. */ - socket: WebSocket; -} - -/** Requested streaming channel, eg `user`, `notifications`. Some channels like `hashtag` have additional params. */ -// TODO: Make this a discriminated union (needed for hashtags). -interface StreamSub { - /** Name of the channel, eg `user`. */ - stream: Stream; - /** Additional query params, eg `tag`. */ - params?: Record; -} - -/** Class to organize WebSocket connections by topic. */ -class WebSocketConnections { - /** Set of WebSockets by topic. */ - #sockets = new Map>(); - /** Set of topics by WebSocket. We need to track this so we can unsubscribe properly. */ - #topics = new WeakMap>(); - - /** Add the WebSocket to the streaming channel. */ - subscribe(conn: StreamConn, sub: StreamSub): void { - const topic = getTopic(conn, sub); - - if (topic) { - this.#addSocket(conn.socket, topic); - this.#addTopic(conn.socket, topic); - } - } - - /** Remove the WebSocket from the streaming channel. */ - unsubscribe(conn: StreamConn, sub: StreamSub): void { - const topic = getTopic(conn, sub); - - if (topic) { - this.#removeSocket(conn.socket, topic); - this.#removeTopic(conn.socket, topic); - } - } - - /** Remove the WebSocket from all its streaming channels. */ - unsubscribeAll(socket: WebSocket): void { - const topics = this.#topics.get(socket); - - if (topics) { - for (const topic of topics) { - this.#removeSocket(socket, topic); - } - } - - this.#topics.delete(socket); - } - - /** Get WebSockets for the given topic. */ - getSockets(topic: Topic): Set { - return this.#sockets.get(topic) ?? new Set(); - } - - /** Add a WebSocket to a topics set in the state. */ - #addSocket(socket: WebSocket, topic: Topic): void { - let subscribers = this.#sockets.get(topic); - - if (!subscribers) { - subscribers = new Set(); - this.#sockets.set(topic, subscribers); - } - - subscribers.add(socket); - } - - /** Remove a WebSocket from a topics set in the state. */ - #removeSocket(socket: WebSocket, topic: Topic): void { - const subscribers = this.#sockets.get(topic); - - if (subscribers) { - subscribers.delete(socket); - - if (subscribers.size === 0) { - this.#sockets.delete(topic); - } - } - } - - /** Add a topic to a WebSocket set in the state. */ - #addTopic(socket: WebSocket, topic: Topic): void { - let topics = this.#topics.get(socket); - - if (!topics) { - topics = new Set(); - this.#topics.set(socket, topics); - } - - topics.add(topic); - } - - /** Remove a topic from a WebSocket set in the state. */ - #removeTopic(socket: WebSocket, topic: Topic): void { - const topics = this.#topics.get(socket); - - if (topics) { - topics.delete(topic); - - if (topics.size === 0) { - this.#topics.delete(socket); - } - } - } -} - -/** - * Convert the "stream" parameter into a "topic". - * The stream parameter is part of the public-facing API, while the topic is internal. - */ -function getTopic(conn: StreamConn, sub: StreamSub): Topic | undefined { - switch (sub.stream) { - case 'public': - case 'public:local': - return sub.stream; - default: - if (!conn.pubkey) { - return; - // Nostr signing topics contain the session ID for privacy reasons. - } else if (sub.stream === 'nostr') { - return conn.session ? `nostr:${conn.pubkey}:${conn.session}` : undefined; - } - } -} - -const ws = new WebSocketConnections(); - -export { type Stream, streamSchema, ws }; diff --git a/src/utils.ts b/src/utils.ts index 7f47f31..e202982 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -102,7 +102,6 @@ function isFollowing(source: Event<3>, targetPubkey: string): boolean { } export { - bech32ToPubkey, eventAge, eventDateComparator, findTag,