From 801e68c6c46d06bb8b59f9d5d87ba51194da21c4 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sat, 11 May 2024 12:03:41 -0300 Subject: [PATCH 01/39] fix: add error prefix according to NIP-01 --- src/policies/MuteListPolicy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/policies/MuteListPolicy.ts b/src/policies/MuteListPolicy.ts index 1db8556..cae08eb 100644 --- a/src/policies/MuteListPolicy.ts +++ b/src/policies/MuteListPolicy.ts @@ -10,7 +10,7 @@ export class MuteListPolicy implements NPolicy { const pubkeys = getTagSet(muteList?.tags ?? [], 'p'); if (pubkeys.has(event.pubkey)) { - return ['OK', event.id, false, 'You are banned in this server.']; + return ['OK', event.id, false, 'blocked: Your account has been deactivated.']; } return ['OK', event.id, true, '']; From fe66937bba49d6aa00679ed5da3bbf4b16c29069 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sat, 11 May 2024 12:04:44 -0300 Subject: [PATCH 02/39] feat: do not allow deactivated accounts to post --- src/pipeline.ts | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 3eb8913..d05b09a 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -21,18 +21,10 @@ import { AdminSigner } from '@/signers/AdminSigner.ts'; import { lnurlCache } from '@/utils/lnurl.ts'; import { nip05Cache } from '@/utils/nip05.ts'; +import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; + const debug = Debug('ditto:pipeline'); -let UserPolicy: any; - -try { - UserPolicy = (await import('../data/policy.ts')).default; - debug('policy loaded from data/policy.ts'); -} catch (_e) { - // do nothing - debug('policy not found'); -} - /** * Common pipeline function to process (and maybe store) events. * It is idempotent, so it can be called multiple times for the same event. @@ -43,17 +35,8 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise ${event.id}`); await hydrateEvent(event, signal); - if (UserPolicy) { - const result = await new UserPolicy().call(event, signal); - debug(JSON.stringify(result)); - const [_, _eventId, ok, reason] = result; - if (!ok) { - const [prefix, ...rest] = reason.split(': '); - throw new RelayError(prefix, rest.join(': ')); - } - } - await Promise.all([ + policyFilter(event), storeEvent(event, signal), parseMetadata(event, signal), processDeletions(event, signal), @@ -66,6 +49,25 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { + const UserPolicy = new MuteListPolicy(Conf.pubkey, Storages.admin); + const result = await UserPolicy.call(event); + + debug(JSON.stringify(result)); + + const [_, _eventId, ok, reason] = result; + if (!ok) { + const [prefix, ...rest] = reason.split(': '); + if (['duplicate', 'pow', 'blocked', 'rate-limited', 'invalid'].includes(prefix)) { + const error = new RelayError(prefix as any, rest.join(': ')); + return Promise.reject(error); + } else { + const error = new RelayError('error', rest.join(': ')); + return Promise.reject(error); + } + } +} + /** Encounter the event, and return whether it has already been encountered. */ async function encounterEvent(event: NostrEvent, signal: AbortSignal): Promise { const [existing] = await Storages.cache.query([{ ids: [event.id], limit: 1 }]); From 04968fefaa9abd57e24c0609d45392644e8e50a3 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sat, 11 May 2024 14:02:24 -0300 Subject: [PATCH 03/39] test(MuteListPolicy): update error msg --- src/policies/MuteListPolicy.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/policies/MuteListPolicy.test.ts b/src/policies/MuteListPolicy.test.ts index 69561f8..2c3baa3 100644 --- a/src/policies/MuteListPolicy.test.ts +++ b/src/policies/MuteListPolicy.test.ts @@ -27,7 +27,7 @@ Deno.test('block event: muted user cannot post', async () => { const ok = await policy.call(event1authorUserMeCopy); - assertEquals(ok, ['OK', event1authorUserMeCopy.id, false, 'You are banned in this server.']); + assertEquals(ok, ['OK', event1authorUserMeCopy.id, false, 'blocked: Your account has been deactivated.']); }); Deno.test('allow event: user is NOT muted because there is no muted event', async () => { From 9bff7a5086b62a0f61d6ff2a84f891e591c18eb9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 13 May 2024 12:30:56 -0500 Subject: [PATCH 04/39] Fix some issues in pipeline and utils/api.ts --- deno.json | 2 +- src/pipeline.ts | 17 +++++++++-------- src/utils/api.ts | 6 ++---- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/deno.json b/deno.json index 5d45588..0225510 100644 --- a/deno.json +++ b/deno.json @@ -20,7 +20,7 @@ "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.17.1", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.19.0", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", diff --git a/src/pipeline.ts b/src/pipeline.ts index d05b09a..c086c92 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -35,8 +35,9 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise ${event.id}`); await hydrateEvent(event, signal); + await policyFilter(event); + await Promise.all([ - policyFilter(event), storeEvent(event, signal), parseMetadata(event, signal), processDeletions(event, signal), @@ -50,8 +51,8 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { - const UserPolicy = new MuteListPolicy(Conf.pubkey, Storages.admin); - const result = await UserPolicy.call(event); + const policy = new MuteListPolicy(Conf.pubkey, Storages.admin); + const result = await policy.call(event); debug(JSON.stringify(result)); @@ -59,11 +60,9 @@ async function policyFilter(event: NostrEvent): Promise { if (!ok) { const [prefix, ...rest] = reason.split(': '); if (['duplicate', 'pow', 'blocked', 'rate-limited', 'invalid'].includes(prefix)) { - const error = new RelayError(prefix as any, rest.join(': ')); - return Promise.reject(error); + throw new RelayError(prefix as RelayErrorPrefix, rest.join(': ')); } else { - const error = new RelayError('error', rest.join(': ')); - return Promise.reject(error); + throw new RelayError('error', rest.join(': ')); } } } @@ -272,9 +271,11 @@ async function streamOut(event: NostrEvent): Promise { } } +type RelayErrorPrefix = 'duplicate' | 'pow' | 'blocked' | 'rate-limited' | 'invalid' | 'error'; + /** NIP-20 command line result. */ class RelayError extends Error { - constructor(prefix: 'duplicate' | 'pow' | 'blocked' | 'rate-limited' | 'invalid' | 'error', message: string) { + constructor(prefix: RelayErrorPrefix, message: string) { super(`${prefix}: ${message}`); } } diff --git a/src/utils/api.ts b/src/utils/api.ts index cba7c66..8da87fb 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -103,10 +103,8 @@ async function updateAdminEvent( async function publishEvent(event: NostrEvent, c: AppContext): Promise { debug('EVENT', event); try { - await Promise.all([ - pipeline.handleEvent(event, c.req.raw.signal), - Storages.client.event(event), - ]); + await pipeline.handleEvent(event, c.req.raw.signal); + await Storages.client.event(event); } catch (e) { if (e instanceof pipeline.RelayError) { throw new HTTPException(422, { From 6105e00c808ebc5a5b2ff2a764aa121ed95d2be8 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 13 May 2024 12:43:01 -0500 Subject: [PATCH 05/39] pipeline: add a placeholder for custom policy --- src/pipeline.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index c086c92..7da026f 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,5 +1,6 @@ import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; import { LNURL } from '@nostrify/nostrify/ln'; +import { PipePolicy } from '@nostrify/nostrify/policies'; import Debug from '@soapbox/stickynotes/debug'; import { sql } from 'kysely'; @@ -33,9 +34,12 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise ${event.id}`); - await hydrateEvent(event, signal); - await policyFilter(event); + if (event.kind !== 24133) { + await policyFilter(event); + } + + await hydrateEvent(event, signal); await Promise.all([ storeEvent(event, signal), @@ -51,9 +55,12 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { - const policy = new MuteListPolicy(Conf.pubkey, Storages.admin); - const result = await policy.call(event); + const policy = new PipePolicy([ + new MuteListPolicy(Conf.pubkey, Storages.admin), + // put custom policy here + ]); + const result = await policy.call(event); debug(JSON.stringify(result)); const [_, _eventId, ok, reason] = result; From 4029971407165173cacf65105fbc4a04dc682816 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 13 May 2024 17:44:33 -0300 Subject: [PATCH 06/39] fix(pipeline): load custom policy if available --- src/pipeline.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 7da026f..a47a2da 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; +import { NostrEvent, NPolicy, NSchema as n } from '@nostrify/nostrify'; import { LNURL } from '@nostrify/nostrify/ln'; import { PipePolicy } from '@nostrify/nostrify/policies'; import Debug from '@soapbox/stickynotes/debug'; @@ -55,10 +55,18 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { - const policy = new PipePolicy([ + const policies: NPolicy[] = [ new MuteListPolicy(Conf.pubkey, Storages.admin), - // put custom policy here - ]); + ]; + + try { + const customPolicy = (await import('../data/policy.ts')).default; + policies.push(new customPolicy()); + } catch (_e) { + debug('policy not found - https://docs.soapbox.pub/ditto/policies/'); + } + + const policy = new PipePolicy(policies.reverse()); const result = await policy.call(event); debug(JSON.stringify(result)); From 9cb63a99183ab2e09adf5ba90846f505f6f52fb2 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 13 May 2024 20:18:11 -0300 Subject: [PATCH 07/39] test: kind 0(user 'dictator') fixture --- fixtures/events/kind-0-dictator.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 fixtures/events/kind-0-dictator.json diff --git a/fixtures/events/kind-0-dictator.json b/fixtures/events/kind-0-dictator.json new file mode 100644 index 0000000..a547332 --- /dev/null +++ b/fixtures/events/kind-0-dictator.json @@ -0,0 +1,9 @@ +{ + "id": "2238893aee54bbe9188498a5aa124d62870d5757894bf52cdb362d1a0874ed18", + "pubkey": "c9f5508526e213c3bc5468161f1b738a86063a2ece540730f9412e7becd5f0b2", + "created_at": 1715517440, + "kind": 0, + "tags": [], + "content": "{\"name\":\"dictator\",\"about\":\"\",\"nip05\":\"\"}", + "sig": "a630ba158833eea10289fe077087ccad22c71ddfbe475153958cfc158ae94fb0a5f7b7626e62da6a3ef8bfbe67321e8f993517ed7f1578a45aff11bc2bec484c" +} From aec0afd731d6da6247d6ee2cd272f00a5ba9a583 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 13 May 2024 20:18:42 -0300 Subject: [PATCH 08/39] test: kind 0(user 'george orwell') fixture --- fixtures/events/kind-0-george-orwell.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 fixtures/events/kind-0-george-orwell.json diff --git a/fixtures/events/kind-0-george-orwell.json b/fixtures/events/kind-0-george-orwell.json new file mode 100644 index 0000000..d835447 --- /dev/null +++ b/fixtures/events/kind-0-george-orwell.json @@ -0,0 +1,9 @@ +{ + "id": "da4e1e727c6456cee2b0341a1d7a2356e4263523374a2570a7dd318ab5d73f93", + "pubkey": "e4d96e951739787e62ada74ee06a9a185af22791a899a6166ec23aab58c5d700", + "created_at": 1715517565, + "kind": 0, + "tags": [], + "content": "{\"name\":\"george orwell\",\"about\":\"\",\"nip05\":\"\"}", + "sig": "cd375e2065cf452d3bfefa9951b04ab63018ab7c253803256cca1d89d03b38e454c71ed36fdd3c28a8ff2723cc19b21371ce0f9bbd39a92b1d1aa946137237bd" +} From f225f7498df6383ee8978d1c7d31468192327535 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 13 May 2024 20:19:30 -0300 Subject: [PATCH 09/39] test: kind 1 (author is 'george orwell') fixture --- fixtures/events/kind-1-author-george-orwell.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 fixtures/events/kind-1-author-george-orwell.json diff --git a/fixtures/events/kind-1-author-george-orwell.json b/fixtures/events/kind-1-author-george-orwell.json new file mode 100644 index 0000000..d1bd4ab --- /dev/null +++ b/fixtures/events/kind-1-author-george-orwell.json @@ -0,0 +1,9 @@ +{ + "id": "44f19148f5af60b0f43ed8c737fbda31b165e05bb55562003c45d9a9f02e8228", + "pubkey": "e4d96e951739787e62ada74ee06a9a185af22791a899a6166ec23aab58c5d700", + "created_at": 1715636249, + "kind": 1, + "tags": [], + "content": "I like free speech", + "sig": "6b50db9c1c02bd8b0e64512e71d53a0058569f44e8dcff65ad17fce544d6ae79f8f79fa0f9a615446fa8cbc2375709bf835751843b0cd10e62ae5d505fe106d4" +} From c67cb034b08d10e867dbc4a1a6b3c6fc53d48f10 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 13 May 2024 20:20:18 -0300 Subject: [PATCH 10/39] test: kind 1984 (author 'dictator') fixture --- ...d-1984-dictator-reports-george-orwell.json | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 fixtures/events/kind-1984-dictator-reports-george-orwell.json diff --git a/fixtures/events/kind-1984-dictator-reports-george-orwell.json b/fixtures/events/kind-1984-dictator-reports-george-orwell.json new file mode 100644 index 0000000..7280c59 --- /dev/null +++ b/fixtures/events/kind-1984-dictator-reports-george-orwell.json @@ -0,0 +1,24 @@ +{ + "id": "129b2749330a7f1189d3e74c6764a955851f1e4017a818dfd51ab8e24192b0f3", + "pubkey": "c9f5508526e213c3bc5468161f1b738a86063a2ece540730f9412e7becd5f0b2", + "created_at": 1715636348, + "kind": 1984, + "tags": [ + [ + "p", + "e4d96e951739787e62ada74ee06a9a185af22791a899a6166ec23aab58c5d700", + "other" + ], + [ + "P", + "e724b1c1b90eab9cc0f5976b380b80dda050de1820dc143e62d9e4f27a9a0b2c" + ], + [ + "e", + "44f19148f5af60b0f43ed8c737fbda31b165e05bb55562003c45d9a9f02e8228", + "other" + ] + ], + "content": "freedom of speech not freedom of reach", + "sig": "cd05a14749cdf0c7664d056e2c02518740000387732218dacd0c71de5b96c0c3c99a0b927b0cd0778f25a211525fa03b4ed4f4f537bb1221c73467780d4ee1bc" +} From 78137f373f7e14c7b228b0b8a630df532817c1e4 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 13 May 2024 20:22:27 -0300 Subject: [PATCH 11/39] test(hydrate): kind 1984 --- src/storages/hydrate.test.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index 4e38d8a..1664a6d 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -129,3 +129,31 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () }; assertEquals(event6copy, expectedEvent6); }); + +Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stats', async () => { + const db = new MockRelay(); + + const authorDictator = await eventFixture('kind-0-dictator'); + const authorVictim = await eventFixture('kind-0-george-orwell'); + const reportEvent = await eventFixture('kind-1984-dictator-reports-george-orwell'); + const event1 = await eventFixture('kind-1-author-george-orwell'); + + // Save events to database + await db.event(authorDictator); + await db.event(authorVictim); + await db.event(reportEvent); + await db.event(event1); + + await hydrateEvents({ + events: [reportEvent], + storage: db, + }); + + const expectedEvent: DittoEvent = { + ...reportEvent, + author: authorDictator, + reported_notes: [event1], + reported_profile: authorVictim, + }; + assertEquals(reportEvent, expectedEvent); +}); From caaa7016f0feb875105166704974755398b2c063 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 13 May 2024 21:56:29 -0300 Subject: [PATCH 12/39] test(hydrate): refactor to import fixtures with 'eventFixture' function --- src/storages/hydrate.test.ts | 102 ++++++++++++++--------------------- 1 file changed, 39 insertions(+), 63 deletions(-) diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index 1664a6d..b55cd2b 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -5,129 +5,105 @@ import { MockRelay } from '@nostrify/nostrify/test'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { eventFixture } from '@/test.ts'; -import event0madePost from '~/fixtures/events/event-0-the-one-who-post-and-users-repost.json' with { type: 'json' }; -import event0madeRepost from '~/fixtures/events/event-0-the-one-who-repost.json' with { type: 'json' }; -import event0madeQuoteRepost from '~/fixtures/events/event-0-the-one-who-quote-repost.json' with { type: 'json' }; -import event1 from '~/fixtures/events/event-1.json' with { type: 'json' }; -import event1quoteRepost from '~/fixtures/events/event-1-quote-repost.json' with { type: 'json' }; -import event1futureIsMine from '~/fixtures/events/event-1-will-be-reposted-with-quote-repost.json' with { - type: 'json', -}; -import event1quoteRepostLatin from '~/fixtures/events/event-1-quote-repost-will-be-reposted.json' with { type: 'json' }; -import event1willBeQuoteReposted from '~/fixtures/events/event-1-that-will-be-quote-reposted.json' with { - type: 'json', -}; -import event1reposted from '~/fixtures/events/event-1-reposted.json' with { type: 'json' }; -import event6 from '~/fixtures/events/event-6.json' with { type: 'json' }; -import event6ofQuoteRepost from '~/fixtures/events/event-6-of-quote-repost.json' with { type: 'json' }; - Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { const db = new MockRelay(); const event0 = await eventFixture('event-0'); - const event1copy = structuredClone(event1); + const event1 = await eventFixture('event-1'); // Save events to database await db.event(event0); - await db.event(event1copy); - - assertEquals((event1copy as DittoEvent).author, undefined, "Event hasn't been hydrated yet"); + await db.event(event1); await hydrateEvents({ - events: [event1copy], + events: [event1], storage: db, }); - const expectedEvent = { ...event1copy, author: event0 }; - assertEquals(event1copy, expectedEvent); + const expectedEvent = { ...event1, author: event0 }; + assertEquals(event1, expectedEvent); }); Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { const db = new MockRelay(); - const event0madePostCopy = structuredClone(event0madePost); - const event0madeRepostCopy = structuredClone(event0madeRepost); - const event1repostedCopy = structuredClone(event1reposted); - const event6copy = structuredClone(event6); + const event0madePost = await eventFixture('event-0-the-one-who-post-and-users-repost'); + const event0madeRepost = await eventFixture('event-0-the-one-who-repost'); + const event1reposted = await eventFixture('event-1-reposted'); + const event6 = await eventFixture('event-6'); // Save events to database - await db.event(event0madePostCopy); - await db.event(event0madeRepostCopy); - await db.event(event1repostedCopy); - await db.event(event6copy); - - assertEquals((event6copy as DittoEvent).author, undefined, "Event hasn't hydrated author yet"); - assertEquals((event6copy as DittoEvent).repost, undefined, "Event hasn't hydrated repost yet"); + await db.event(event0madePost); + await db.event(event0madeRepost); + await db.event(event1reposted); + await db.event(event6); await hydrateEvents({ - events: [event6copy], + events: [event6], storage: db, }); const expectedEvent6 = { - ...event6copy, - author: event0madeRepostCopy, - repost: { ...event1repostedCopy, author: event0madePostCopy }, + ...event6, + author: event0madeRepost, + repost: { ...event1reposted, author: event0madePost }, }; - assertEquals(event6copy, expectedEvent6); + assertEquals(event6, expectedEvent6); }); Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { const db = new MockRelay(); - const event0madeQuoteRepostCopy = structuredClone(event0madeQuoteRepost); + const event0madeQuoteRepost = await eventFixture('event-0-the-one-who-quote-repost'); const event0 = await eventFixture('event-0'); - const event1quoteRepostCopy = structuredClone(event1quoteRepost); - const event1willBeQuoteRepostedCopy = structuredClone(event1willBeQuoteReposted); + const event1quoteRepost = await eventFixture('event-1-quote-repost'); + const event1willBeQuoteReposted = await eventFixture('event-1-that-will-be-quote-reposted'); // Save events to database - await db.event(event0madeQuoteRepostCopy); + await db.event(event0madeQuoteRepost); await db.event(event0); - await db.event(event1quoteRepostCopy); - await db.event(event1willBeQuoteRepostedCopy); + await db.event(event1quoteRepost); + await db.event(event1willBeQuoteReposted); await hydrateEvents({ - events: [event1quoteRepostCopy], + events: [event1quoteRepost], storage: db, }); const expectedEvent1quoteRepost = { - ...event1quoteRepostCopy, - author: event0madeQuoteRepostCopy, - quote: { ...event1willBeQuoteRepostedCopy, author: event0 }, + ...event1quoteRepost, + author: event0madeQuoteRepost, + quote: { ...event1willBeQuoteReposted, author: event0 }, }; - assertEquals(event1quoteRepostCopy, expectedEvent1quoteRepost); + assertEquals(event1quoteRepost, expectedEvent1quoteRepost); }); Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () => { const db = new MockRelay(); const author = await eventFixture('event-0-makes-repost-with-quote-repost'); - const event1copy = structuredClone(event1futureIsMine); - const event1quoteCopy = structuredClone(event1quoteRepostLatin); - const event6copy = structuredClone(event6ofQuoteRepost); + const event1 = await eventFixture('event-1-will-be-reposted-with-quote-repost'); + const event6 = await eventFixture('event-6-of-quote-repost'); + const event1quote = await eventFixture('event-1-quote-repost-will-be-reposted'); // Save events to database await db.event(author); - await db.event(event1copy); - await db.event(event1quoteCopy); - await db.event(event6copy); - - assertEquals((event6copy as DittoEvent).author, undefined, "Event hasn't hydrated author yet"); - assertEquals((event6copy as DittoEvent).repost, undefined, "Event hasn't hydrated repost yet"); + await db.event(event1); + await db.event(event1quote); + await db.event(event6); await hydrateEvents({ - events: [event6copy], + events: [event6], storage: db, }); const expectedEvent6 = { - ...event6copy, + ...event6, author, - repost: { ...event1quoteCopy, author, quote: { author, ...event1copy } }, + repost: { ...event1quote, author, quote: { author, ...event1 } }, }; - assertEquals(event6copy, expectedEvent6); + assertEquals(event6, expectedEvent6); }); Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stats', async () => { From ee7864da8c191a626e5b579aa6bf2ef86d9576cc Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 11:30:45 -0500 Subject: [PATCH 13/39] Add a signerMiddleware --- src/app.ts | 17 ++++++++++++++--- src/middleware/signerMiddleware.ts | 13 +++++++++++++ src/signers/APISigner.ts | 1 + 3 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 src/middleware/signerMiddleware.ts diff --git a/src/app.ts b/src/app.ts index 05a1c81..4f93608 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NStore } from '@nostrify/nostrify'; +import { NostrEvent, NostrSigner, NStore } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { type Context, Env as HonoEnv, type Handler, Hono, Input as HonoInput, type MiddlewareHandler } from 'hono'; import { cors, logger, serveStatic } from 'hono/middleware'; @@ -29,6 +29,7 @@ import { adminAccountAction, adminAccountsController } from '@/controllers/api/a import { appCredentialsController, createAppController } from '@/controllers/api/apps.ts'; import { blocksController } from '@/controllers/api/blocks.ts'; import { bookmarksController } from '@/controllers/api/bookmarks.ts'; +import { adminRelaysController, adminSetRelaysController } from '@/controllers/api/ditto.ts'; import { emptyArrayController, emptyObjectController, notImplementedController } from '@/controllers/api/fallback.ts'; import { instanceController } from '@/controllers/api/instance.ts'; import { markersController, updateMarkersController } from '@/controllers/api/markers.ts'; @@ -84,13 +85,15 @@ import { auth19, requirePubkey } from '@/middleware/auth19.ts'; import { auth98, requireProof, requireRole } from '@/middleware/auth98.ts'; import { cache } from '@/middleware/cache.ts'; import { csp } from '@/middleware/csp.ts'; -import { adminRelaysController, adminSetRelaysController } from '@/controllers/api/ditto.ts'; +import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; import { storeMiddleware } from '@/middleware/store.ts'; import { blockController } from '@/controllers/api/accounts.ts'; import { unblockController } from '@/controllers/api/accounts.ts'; interface AppEnv extends HonoEnv { Variables: { + /** Signer to get the logged-in user's pubkey, relays, and to sign events, or `undefined` if the user isn't logged in. */ + signer?: NostrSigner; /** Hex pubkey for the current user. If provided, the user is considered "logged in." */ pubkey?: string; /** Hex secret key for the current user. Optional, but easiest way to use legacy Mastodon apps. */ @@ -123,7 +126,15 @@ app.get('/api/v1/streaming', streamingController); app.get('/api/v1/streaming/', streamingController); app.get('/relay', relayController); -app.use('*', csp(), cors({ origin: '*', exposeHeaders: ['link'] }), auth19, auth98(), storeMiddleware); +app.use( + '*', + csp(), + cors({ origin: '*', exposeHeaders: ['link'] }), + auth19, + auth98(), + storeMiddleware, + signerMiddleware, +); app.get('/.well-known/webfinger', webfingerController); app.get('/.well-known/host-meta', hostMetaController); diff --git a/src/middleware/signerMiddleware.ts b/src/middleware/signerMiddleware.ts new file mode 100644 index 0000000..8e7eb7a --- /dev/null +++ b/src/middleware/signerMiddleware.ts @@ -0,0 +1,13 @@ +import { AppMiddleware } from '@/app.ts'; +import { APISigner } from '@/signers/APISigner.ts'; + +/** Make a `signer` object available to all controllers, or unset if the user isn't logged in. */ +export const signerMiddleware: AppMiddleware = async (c, next) => { + try { + c.set('signer', new APISigner(c)); + } catch { + // do nothing + } + + await next(); +}; diff --git a/src/signers/APISigner.ts b/src/signers/APISigner.ts index e9914b1..0a9317b 100644 --- a/src/signers/APISigner.ts +++ b/src/signers/APISigner.ts @@ -2,6 +2,7 @@ import { NConnectSigner, NostrEvent, NostrSigner, NSecSigner } from '@nostrify/nostrify'; import { HTTPException } from 'hono'; + import { type AppContext } from '@/app.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; From 5a2b1b7de7aec546abed2c1f8fd02e4dabcbe4df Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 11:40:20 -0500 Subject: [PATCH 14/39] Destroy everything --- src/app.ts | 7 +--- src/middleware/auth19.ts | 49 ---------------------- src/middleware/signerMiddleware.ts | 41 ++++++++++++++++--- src/signers/APISigner.ts | 66 ------------------------------ 4 files changed, 37 insertions(+), 126 deletions(-) delete mode 100644 src/middleware/auth19.ts delete mode 100644 src/signers/APISigner.ts diff --git a/src/app.ts b/src/app.ts index 4f93608..057bded 100644 --- a/src/app.ts +++ b/src/app.ts @@ -81,7 +81,7 @@ import { hostMetaController } from '@/controllers/well-known/host-meta.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; import { webfingerController } from '@/controllers/well-known/webfinger.ts'; -import { auth19, requirePubkey } from '@/middleware/auth19.ts'; +import { requirePubkey } from '@/middleware/auth19.ts'; import { auth98, requireProof, requireRole } from '@/middleware/auth98.ts'; import { cache } from '@/middleware/cache.ts'; import { csp } from '@/middleware/csp.ts'; @@ -94,10 +94,6 @@ interface AppEnv extends HonoEnv { Variables: { /** Signer to get the logged-in user's pubkey, relays, and to sign events, or `undefined` if the user isn't logged in. */ signer?: NostrSigner; - /** Hex pubkey for the current user. If provided, the user is considered "logged in." */ - pubkey?: string; - /** Hex secret key for the current user. Optional, but easiest way to use legacy Mastodon apps. */ - seckey?: Uint8Array; /** NIP-98 signed event proving the pubkey is owned by the user. */ proof?: NostrEvent; /** User associated with the pubkey, if any. */ @@ -130,7 +126,6 @@ app.use( '*', csp(), cors({ origin: '*', exposeHeaders: ['link'] }), - auth19, auth98(), storeMiddleware, signerMiddleware, diff --git a/src/middleware/auth19.ts b/src/middleware/auth19.ts deleted file mode 100644 index 90fc444..0000000 --- a/src/middleware/auth19.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { HTTPException } from 'hono'; -import { getPublicKey, nip19 } from 'nostr-tools'; - -import { type AppMiddleware } from '@/app.ts'; - -/** We only accept "Bearer" type. */ -const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); - -/** NIP-19 auth middleware. */ -const auth19: AppMiddleware = async (c, next) => { - const authHeader = c.req.header('authorization'); - const match = authHeader?.match(BEARER_REGEX); - - if (match) { - const [_, bech32] = match; - - try { - const decoded = nip19.decode(bech32!); - - switch (decoded.type) { - case 'npub': - c.set('pubkey', decoded.data); - break; - case 'nprofile': - c.set('pubkey', decoded.data.pubkey); - break; - case 'nsec': - c.set('pubkey', getPublicKey(decoded.data)); - c.set('seckey', decoded.data); - break; - } - } catch (_e) { - // - } - } - - await next(); -}; - -/** Throw a 401 if the pubkey isn't set. */ -const requirePubkey: AppMiddleware = async (c, next) => { - if (!c.get('pubkey')) { - throw new HTTPException(401, { message: 'No pubkey provided' }); - } - - await next(); -}; - -export { auth19, requirePubkey }; diff --git a/src/middleware/signerMiddleware.ts b/src/middleware/signerMiddleware.ts index 8e7eb7a..d056391 100644 --- a/src/middleware/signerMiddleware.ts +++ b/src/middleware/signerMiddleware.ts @@ -1,12 +1,43 @@ +import { NConnectSigner, NSecSigner } from '@nostrify/nostrify'; +import { nip19 } from 'nostr-tools'; + import { AppMiddleware } from '@/app.ts'; -import { APISigner } from '@/signers/APISigner.ts'; +import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { Storages } from '@/storages.ts'; + +/** We only accept "Bearer" type. */ +const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); /** Make a `signer` object available to all controllers, or unset if the user isn't logged in. */ export const signerMiddleware: AppMiddleware = async (c, next) => { - try { - c.set('signer', new APISigner(c)); - } catch { - // do nothing + const header = c.req.header('authorization'); + const match = header?.match(BEARER_REGEX); + + if (match) { + const [_, bech32] = match; + + try { + const decoded = nip19.decode(bech32!); + + switch (decoded.type) { + case 'npub': + c.set( + 'signer', + new NConnectSigner({ + pubkey: decoded.data, + relay: Storages.pubsub, + signer: new AdminSigner(), + timeout: 60000, + }), + ); + break; + case 'nsec': + c.set('signer', new NSecSigner(decoded.data)); + break; + } + } catch { + // the user is not logged in + } } await next(); diff --git a/src/signers/APISigner.ts b/src/signers/APISigner.ts deleted file mode 100644 index 0a9317b..0000000 --- a/src/signers/APISigner.ts +++ /dev/null @@ -1,66 +0,0 @@ -// deno-lint-ignore-file require-await - -import { NConnectSigner, NostrEvent, NostrSigner, NSecSigner } from '@nostrify/nostrify'; -import { HTTPException } from 'hono'; - -import { type AppContext } from '@/app.ts'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { Storages } from '@/storages.ts'; - -/** - * Sign Nostr event using the app context. - * - * - If a secret key is provided, it will be used to sign the event. - * - Otherwise, it will use NIP-46 to sign the event. - */ -export class APISigner implements NostrSigner { - private signer: NostrSigner; - - constructor(c: AppContext) { - const seckey = c.get('seckey'); - const pubkey = c.get('pubkey'); - - if (!pubkey) { - throw new HTTPException(401, { message: 'Missing pubkey' }); - } - - if (seckey) { - this.signer = new NSecSigner(seckey); - } else { - this.signer = new NConnectSigner({ - pubkey, - relay: Storages.pubsub, - signer: new AdminSigner(), - timeout: 60000, - }); - } - } - - async getPublicKey(): Promise { - return this.signer.getPublicKey(); - } - - async signEvent(event: Omit): Promise { - return this.signer.signEvent(event); - } - - readonly nip04 = { - encrypt: async (pubkey: string, plaintext: string): Promise => { - return this.signer.nip04!.encrypt(pubkey, plaintext); - }, - - decrypt: async (pubkey: string, ciphertext: string): Promise => { - return this.signer.nip04!.decrypt(pubkey, ciphertext); - }, - }; - - readonly nip44 = { - encrypt: async (pubkey: string, plaintext: string): Promise => { - return this.signer.nip44!.encrypt(pubkey, plaintext); - }, - - decrypt: async (pubkey: string, ciphertext: string): Promise => { - return this.signer.nip44!.decrypt(pubkey, ciphertext); - }, - }; -} From c5fbe69b80c2de965614e404742da8b96a1e6836 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 11:42:53 -0500 Subject: [PATCH 15/39] requirePubkey -> requireSigner --- src/app.ts | 62 ++++++++++++++++----------------- src/middleware/requireSigner.ts | 12 +++++++ 2 files changed, 43 insertions(+), 31 deletions(-) create mode 100644 src/middleware/requireSigner.ts diff --git a/src/app.ts b/src/app.ts index 057bded..5ef5056 100644 --- a/src/app.ts +++ b/src/app.ts @@ -81,10 +81,10 @@ import { hostMetaController } from '@/controllers/well-known/host-meta.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; import { webfingerController } from '@/controllers/well-known/webfinger.ts'; -import { requirePubkey } from '@/middleware/auth19.ts'; import { auth98, requireProof, requireRole } from '@/middleware/auth98.ts'; import { cache } from '@/middleware/cache.ts'; import { csp } from '@/middleware/csp.ts'; +import { requireSigner } from '@/middleware/requireSigner.ts'; import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; import { storeMiddleware } from '@/middleware/store.ts'; import { blockController } from '@/controllers/api/accounts.ts'; @@ -151,17 +151,17 @@ app.post('/oauth/authorize', oauthAuthorizeController); app.get('/oauth/authorize', oauthController); app.post('/api/v1/accounts', requireProof({ pow: 20 }), createAccountController); -app.get('/api/v1/accounts/verify_credentials', requirePubkey, verifyCredentialsController); -app.patch('/api/v1/accounts/update_credentials', requirePubkey, updateCredentialsController); +app.get('/api/v1/accounts/verify_credentials', requireSigner, verifyCredentialsController); +app.patch('/api/v1/accounts/update_credentials', requireSigner, updateCredentialsController); app.get('/api/v1/accounts/search', accountSearchController); app.get('/api/v1/accounts/lookup', accountLookupController); -app.get('/api/v1/accounts/relationships', requirePubkey, relationshipsController); -app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/block', requirePubkey, blockController); -app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/unblock', requirePubkey, unblockController); -app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/mute', requirePubkey, muteController); -app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/unmute', requirePubkey, unmuteController); -app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/follow', requirePubkey, followController); -app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/unfollow', requirePubkey, unfollowController); +app.get('/api/v1/accounts/relationships', requireSigner, relationshipsController); +app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/block', requireSigner, blockController); +app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/unblock', requireSigner, unblockController); +app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/mute', requireSigner, muteController); +app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/unmute', requireSigner, unmuteController); +app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/follow', requireSigner, followController); +app.post('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/unfollow', requireSigner, unfollowController); app.get('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/followers', followersController); app.get('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/following', followingController); app.get('/api/v1/accounts/:pubkey{[0-9a-f]{64}}/statuses', accountStatusesController); @@ -171,21 +171,21 @@ app.get('/api/v1/statuses/:id{[0-9a-f]{64}}/favourited_by', favouritedByControll app.get('/api/v1/statuses/:id{[0-9a-f]{64}}/reblogged_by', rebloggedByController); app.get('/api/v1/statuses/:id{[0-9a-f]{64}}/context', contextController); app.get('/api/v1/statuses/:id{[0-9a-f]{64}}', statusController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/favourite', requirePubkey, favouriteController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/bookmark', requirePubkey, bookmarkController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unbookmark', requirePubkey, unbookmarkController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/pin', requirePubkey, pinController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unpin', requirePubkey, unpinController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/zap', requirePubkey, zapController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/reblog', requirePubkey, reblogStatusController); -app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unreblog', requirePubkey, unreblogStatusController); -app.post('/api/v1/statuses', requirePubkey, createStatusController); -app.delete('/api/v1/statuses/:id{[0-9a-f]{64}}', requirePubkey, deleteStatusController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/favourite', requireSigner, favouriteController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/bookmark', requireSigner, bookmarkController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unbookmark', requireSigner, unbookmarkController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/pin', requireSigner, pinController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unpin', requireSigner, unpinController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/zap', requireSigner, zapController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/reblog', requireSigner, reblogStatusController); +app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/unreblog', requireSigner, unreblogStatusController); +app.post('/api/v1/statuses', requireSigner, createStatusController); +app.delete('/api/v1/statuses/:id{[0-9a-f]{64}}', requireSigner, deleteStatusController); app.post('/api/v1/media', mediaController); app.post('/api/v2/media', mediaController); -app.get('/api/v1/timelines/home', requirePubkey, homeTimelineController); +app.get('/api/v1/timelines/home', requireSigner, homeTimelineController); app.get('/api/v1/timelines/public', publicTimelineController); app.get('/api/v1/timelines/tag/:hashtag', hashtagTimelineController); @@ -201,11 +201,11 @@ app.get('/api/v1/trends', cache({ cacheName: 'web', expires: Time.minutes(15) }) app.get('/api/v1/suggestions', suggestionsV1Controller); app.get('/api/v2/suggestions', suggestionsV2Controller); -app.get('/api/v1/notifications', requirePubkey, notificationsController); -app.get('/api/v1/favourites', requirePubkey, favouritesController); -app.get('/api/v1/bookmarks', requirePubkey, bookmarksController); -app.get('/api/v1/blocks', requirePubkey, blocksController); -app.get('/api/v1/mutes', requirePubkey, mutesController); +app.get('/api/v1/notifications', requireSigner, notificationsController); +app.get('/api/v1/favourites', requireSigner, favouritesController); +app.get('/api/v1/bookmarks', requireSigner, bookmarksController); +app.get('/api/v1/blocks', requireSigner, blocksController); +app.get('/api/v1/mutes', requireSigner, mutesController); app.get('/api/v1/markers', requireProof(), markersController); app.post('/api/v1/markers', requireProof(), updateMarkersController); @@ -218,17 +218,17 @@ app.delete('/api/v1/pleroma/admin/statuses/:id', requireRole('admin'), pleromaAd app.get('/api/v1/admin/ditto/relays', requireRole('admin'), adminRelaysController); app.put('/api/v1/admin/ditto/relays', requireRole('admin'), adminSetRelaysController); -app.post('/api/v1/reports', requirePubkey, reportController); -app.get('/api/v1/admin/reports', requirePubkey, requireRole('admin'), adminReportsController); -app.get('/api/v1/admin/reports/:id{[0-9a-f]{64}}', requirePubkey, requireRole('admin'), adminReportController); +app.post('/api/v1/reports', requireSigner, reportController); +app.get('/api/v1/admin/reports', requireSigner, requireRole('admin'), adminReportsController); +app.get('/api/v1/admin/reports/:id{[0-9a-f]{64}}', requireSigner, requireRole('admin'), adminReportController); app.post( '/api/v1/admin/reports/:id{[0-9a-f]{64}}/resolve', - requirePubkey, + requireSigner, requireRole('admin'), adminReportResolveController, ); -app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', requirePubkey, requireRole('admin'), adminAccountAction); +app.post('/api/v1/admin/accounts/:id{[0-9a-f]{64}}/action', requireSigner, requireRole('admin'), adminAccountAction); // Not (yet) implemented. app.get('/api/v1/custom_emojis', emptyArrayController); diff --git a/src/middleware/requireSigner.ts b/src/middleware/requireSigner.ts new file mode 100644 index 0000000..6e337c2 --- /dev/null +++ b/src/middleware/requireSigner.ts @@ -0,0 +1,12 @@ +import { HTTPException } from 'hono'; + +import { AppMiddleware } from '@/app.ts'; + +/** Throw a 401 if a signer isn't set. */ +export const requireSigner: AppMiddleware = async (c, next) => { + if (!c.get('signer')) { + throw new HTTPException(401, { message: 'No pubkey provided' }); + } + + await next(); +}; From c715827c81a9e85d8b68de655b865b9665267f0e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 11:57:03 -0500 Subject: [PATCH 16/39] c.get('pubkey') -> await c.get('signer')?.getPublicKey() --- src/controllers/api/accounts.ts | 30 +++++++++++++++++---------- src/controllers/api/bookmarks.ts | 2 +- src/controllers/api/markers.ts | 4 ++-- src/controllers/api/media.ts | 2 +- src/controllers/api/mutes.ts | 2 +- src/controllers/api/notifications.ts | 6 +++--- src/controllers/api/reports.ts | 12 ++++++++--- src/controllers/api/search.ts | 3 ++- src/controllers/api/statuses.ts | 31 ++++++++++++++++------------ src/controllers/api/timelines.ts | 8 ++++--- src/middleware/auth98.ts | 12 ++++++++--- src/middleware/store.ts | 2 +- src/utils/api.ts | 9 ++++++-- src/views.ts | 4 +++- 14 files changed, 81 insertions(+), 46 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 68edfbc..88d19b1 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -29,7 +29,7 @@ const createAccountSchema = z.object({ }); const createAccountController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const result = createAccountSchema.safeParse(await c.req.json()); if (!result.success) { @@ -45,7 +45,7 @@ const createAccountController: AppController = async (c) => { }; const verifyCredentialsController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const event = await getAuthor(pubkey, { relations: ['author_stats'] }); if (event) { @@ -122,7 +122,7 @@ const accountSearchController: AppController = async (c) => { }; const relationshipsController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const ids = z.array(z.string()).safeParse(c.req.queries('id[]')); if (!ids.success) { @@ -178,7 +178,11 @@ const accountStatusesController: AppController = async (c) => { return events; }); - const statuses = await Promise.all(events.map((event) => renderStatus(event, { viewerPubkey: c.get('pubkey') }))); + const viewerPubkey = await c.get('signer')?.getPublicKey(); + + const statuses = await Promise.all( + events.map((event) => renderStatus(event, { viewerPubkey })), + ); return paginated(c, events, statuses); }; @@ -194,7 +198,7 @@ const updateCredentialsSchema = z.object({ }); const updateCredentialsController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const body = await parseBody(c.req.raw); const result = updateCredentialsSchema.safeParse(body); @@ -236,7 +240,7 @@ const updateCredentialsController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/accounts/#follow */ const followController: AppController = async (c) => { - const sourcePubkey = c.get('pubkey')!; + const sourcePubkey = await c.get('signer')?.getPublicKey()!; const targetPubkey = c.req.param('pubkey'); await updateListEvent( @@ -253,7 +257,7 @@ const followController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/accounts/#unfollow */ const unfollowController: AppController = async (c) => { - const sourcePubkey = c.get('pubkey')!; + const sourcePubkey = await c.get('signer')?.getPublicKey()!; const targetPubkey = c.req.param('pubkey'); await updateListEvent( @@ -290,7 +294,7 @@ const unblockController: AppController = (c) => { /** https://docs.joinmastodon.org/methods/accounts/#mute */ const muteController: AppController = async (c) => { - const sourcePubkey = c.get('pubkey')!; + const sourcePubkey = await c.get('signer')?.getPublicKey()!; const targetPubkey = c.req.param('pubkey'); await updateListEvent( @@ -305,7 +309,7 @@ const muteController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/accounts/#unmute */ const unmuteController: AppController = async (c) => { - const sourcePubkey = c.get('pubkey')!; + const sourcePubkey = await c.get('signer')?.getPublicKey()!; const targetPubkey = c.req.param('pubkey'); await updateListEvent( @@ -319,7 +323,7 @@ const unmuteController: AppController = async (c) => { }; const favouritesController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const params = paginationSchema.parse(c.req.query()); const { signal } = c.req.raw; @@ -335,7 +339,11 @@ const favouritesController: AppController = async (c) => { const events1 = await Storages.db.query([{ kinds: [1], ids }], { signal }) .then((events) => hydrateEvents({ events, storage: Storages.db, signal })); - const statuses = await Promise.all(events1.map((event) => renderStatus(event, { viewerPubkey: c.get('pubkey') }))); + const viewerPubkey = await c.get('signer')?.getPublicKey(); + + const statuses = await Promise.all( + events1.map((event) => renderStatus(event, { viewerPubkey })), + ); return paginated(c, events1, statuses); }; diff --git a/src/controllers/api/bookmarks.ts b/src/controllers/api/bookmarks.ts index 8d44f95..1616fa2 100644 --- a/src/controllers/api/bookmarks.ts +++ b/src/controllers/api/bookmarks.ts @@ -5,7 +5,7 @@ import { renderStatuses } from '@/views.ts'; /** https://docs.joinmastodon.org/methods/bookmarks/#get */ const bookmarksController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const { signal } = c.req.raw; const [event10003] = await Storages.db.query( diff --git a/src/controllers/api/markers.ts b/src/controllers/api/markers.ts index ce1c4ec..005ebbe 100644 --- a/src/controllers/api/markers.ts +++ b/src/controllers/api/markers.ts @@ -14,7 +14,7 @@ interface Marker { } export const markersController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const timelines = c.req.queries('timeline[]') ?? []; const results = await kv.getMany( @@ -37,7 +37,7 @@ const markerDataSchema = z.object({ }); export const updateMarkersController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const record = z.record(z.enum(['home', 'notifications']), markerDataSchema).parse(await parseBody(c.req.raw)); const timelines = Object.keys(record) as Timeline[]; diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index dd36a53..33b7981 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -14,7 +14,7 @@ const mediaBodySchema = z.object({ }); const mediaController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const result = mediaBodySchema.safeParse(await parseBody(c.req.raw)); const { signal } = c.req.raw; diff --git a/src/controllers/api/mutes.ts b/src/controllers/api/mutes.ts index 77b60e3..fe048e9 100644 --- a/src/controllers/api/mutes.ts +++ b/src/controllers/api/mutes.ts @@ -5,7 +5,7 @@ import { renderAccounts } from '@/views.ts'; /** https://docs.joinmastodon.org/methods/mutes/#get */ const mutesController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const { signal } = c.req.raw; const [event10000] = await Storages.db.query( diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index b2fa15e..857f2a3 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -5,8 +5,8 @@ import { hydrateEvents } from '@/storages/hydrate.ts'; import { paginated, paginationSchema } from '@/utils/api.ts'; import { renderNotification } from '@/views/mastodon/notifications.ts'; -const notificationsController: AppController = (c) => { - const pubkey = c.get('pubkey')!; +const notificationsController: AppController = async (c) => { + const pubkey = await c.get('signer')?.getPublicKey()!; const { since, until } = paginationSchema.parse(c.req.query()); return renderNotifications(c, [{ kinds: [1, 6, 7], '#p': [pubkey], since, until }]); @@ -14,7 +14,7 @@ const notificationsController: AppController = (c) => { async function renderNotifications(c: AppContext, filters: NostrFilter[]) { const store = c.get('store'); - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const { signal } = c.req.raw; const events = await store diff --git a/src/controllers/api/reports.ts b/src/controllers/api/reports.ts index eb85bd2..55fb601 100644 --- a/src/controllers/api/reports.ts +++ b/src/controllers/api/reports.ts @@ -55,9 +55,15 @@ const reportController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/admin/reports/#get */ const adminReportsController: AppController = async (c) => { const store = c.get('store'); + const viewerPubkey = await c.get('signer')?.getPublicKey(); + const reports = await store.query([{ kinds: [1984], '#P': [Conf.pubkey] }]) .then((events) => hydrateEvents({ storage: store, events: events, signal: c.req.raw.signal })) - .then((events) => Promise.all(events.map((event) => renderAdminReport(event, { viewerPubkey: c.get('pubkey') })))); + .then((events) => + Promise.all( + events.map((event) => renderAdminReport(event, { viewerPubkey })), + ) + ); return c.json(reports); }; @@ -67,7 +73,7 @@ const adminReportController: AppController = async (c) => { const eventId = c.req.param('id'); const { signal } = c.req.raw; const store = c.get('store'); - const pubkey = c.get('pubkey'); + const pubkey = await c.get('signer')?.getPublicKey(); const [event] = await store.query([{ kinds: [1984], @@ -89,7 +95,7 @@ const adminReportResolveController: AppController = async (c) => { const eventId = c.req.param('id'); const { signal } = c.req.raw; const store = c.get('store'); - const pubkey = c.get('pubkey'); + const pubkey = await c.get('signer')?.getPublicKey(); const [event] = await store.query([{ kinds: [1984], diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index e674d0e..fe08ace 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -43,6 +43,7 @@ const searchController: AppController = async (c) => { } const results = dedupeEvents(events); + const viewerPubkey = await c.get('signer')?.getPublicKey(); const [accounts, statuses] = await Promise.all([ Promise.all( @@ -54,7 +55,7 @@ const searchController: AppController = async (c) => { Promise.all( results .filter((event) => event.kind === 1) - .map((event) => renderStatus(event, { viewerPubkey: c.get('pubkey') })) + .map((event) => renderStatus(event, { viewerPubkey })) .filter(Boolean), ), ]); diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 3420383..1138c0a 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -47,7 +47,7 @@ const statusController: AppController = async (c) => { }); if (event) { - return c.json(await renderStatus(event, { viewerPubkey: c.get('pubkey') })); + return c.json(await renderStatus(event, { viewerPubkey: await c.get('signer')?.getPublicKey() })); } return c.json({ error: 'Event not found.' }, 404); @@ -89,9 +89,11 @@ const createStatusController: AppController = async (c) => { tags.push(['subject', data.spoiler_text]); } + const viewerPubkey = await c.get('signer')?.getPublicKey(); + if (data.media_ids?.length) { const media = await getUnattachedMediaByIds(data.media_ids) - .then((media) => media.filter(({ pubkey }) => pubkey === c.get('pubkey'))) + .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey)) .then((media) => media.map(({ url, data }) => ['media', url, data])); tags.push(...media); @@ -143,12 +145,12 @@ const createStatusController: AppController = async (c) => { }); } - return c.json(await renderStatus({ ...event, author }, { viewerPubkey: c.get('pubkey') })); + return c.json(await renderStatus({ ...event, author }, { viewerPubkey: await c.get('signer')?.getPublicKey() })); }; const deleteStatusController: AppController = async (c) => { const id = c.req.param('id'); - const pubkey = c.get('pubkey'); + const pubkey = await c.get('signer')?.getPublicKey(); const event = await getEvent(id, { signal: c.req.raw.signal }); @@ -172,9 +174,12 @@ const deleteStatusController: AppController = async (c) => { const contextController: AppController = async (c) => { const id = c.req.param('id'); const event = await getEvent(id, { kind: 1, relations: ['author', 'event_stats', 'author_stats'] }); + const viewerPubkey = await c.get('signer')?.getPublicKey(); async function renderStatuses(events: NostrEvent[]) { - const statuses = await Promise.all(events.map((event) => renderStatus(event, { viewerPubkey: c.get('pubkey') }))); + const statuses = await Promise.all( + events.map((event) => renderStatus(event, { viewerPubkey })), + ); return statuses.filter(Boolean); } @@ -204,7 +209,7 @@ const favouriteController: AppController = async (c) => { ], }, c); - const status = await renderStatus(target, { viewerPubkey: c.get('pubkey') }); + const status = await renderStatus(target, { viewerPubkey: await c.get('signer')?.getPublicKey() }); if (status) { status.favourited = true; @@ -247,7 +252,7 @@ const reblogStatusController: AppController = async (c) => { signal: signal, }); - const status = await renderReblog(reblogEvent, { viewerPubkey: c.get('pubkey') }); + const status = await renderReblog(reblogEvent, { viewerPubkey: await c.get('signer')?.getPublicKey() }); return c.json(status); }; @@ -255,7 +260,7 @@ const reblogStatusController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#unreblog */ const unreblogStatusController: AppController = async (c) => { const eventId = c.req.param('id'); - const pubkey = c.get('pubkey') as string; + const pubkey = await c.get('signer')?.getPublicKey() as string; const event = await getEvent(eventId, { kind: 1, @@ -282,7 +287,7 @@ const rebloggedByController: AppController = (c) => { /** https://docs.joinmastodon.org/methods/statuses/#bookmark */ const bookmarkController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const eventId = c.req.param('id'); const event = await getEvent(eventId, { @@ -309,7 +314,7 @@ const bookmarkController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#unbookmark */ const unbookmarkController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const eventId = c.req.param('id'); const event = await getEvent(eventId, { @@ -336,7 +341,7 @@ const unbookmarkController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#pin */ const pinController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const eventId = c.req.param('id'); const event = await getEvent(eventId, { @@ -363,7 +368,7 @@ const pinController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#unpin */ const unpinController: AppController = async (c) => { - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const eventId = c.req.param('id'); const { signal } = c.req.raw; @@ -423,7 +428,7 @@ const zapController: AppController = async (c) => { ], }, c); - const status = await renderStatus(target, { viewerPubkey: c.get('pubkey') }); + const status = await renderStatus(target, { viewerPubkey: await c.get('signer')?.getPublicKey() }); status.zapped = true; return c.json(status); diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index 27459a8..0880d84 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -11,7 +11,7 @@ import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; const homeTimelineController: AppController = async (c) => { const params = paginationSchema.parse(c.req.query()); - const pubkey = c.get('pubkey')!; + const pubkey = await c.get('signer')?.getPublicKey()!; const authors = await getFeedPubkeys(pubkey); return renderStatuses(c, [{ authors, kinds: [1, 6], ...params }]); }; @@ -61,11 +61,13 @@ async function renderStatuses(c: AppContext, filters: NostrFilter[]) { return c.json([]); } + const viewerPubkey = await c.get('signer')?.getPublicKey(); + const statuses = (await Promise.all(events.map((event) => { if (event.kind === 6) { - return renderReblog(event, { viewerPubkey: c.get('pubkey') }); + return renderReblog(event, { viewerPubkey }); } - return renderStatus(event, { viewerPubkey: c.get('pubkey') }); + return renderStatus(event, { viewerPubkey }); }))).filter((boolean) => boolean); if (!statuses.length) { diff --git a/src/middleware/auth98.ts b/src/middleware/auth98.ts index db025ae..d761b95 100644 --- a/src/middleware/auth98.ts +++ b/src/middleware/auth98.ts @@ -8,7 +8,6 @@ import { validateAuthEvent, } from '@/utils/nip98.ts'; import { localRequest } from '@/utils/api.ts'; -import { APISigner } from '@/signers/APISigner.ts'; import { findUser, User } from '@/db/users.ts'; /** @@ -70,7 +69,7 @@ function withProof( opts?: ParseAuthRequestOpts, ): AppMiddleware { return async (c, next) => { - const pubkey = c.get('pubkey'); + const pubkey = await c.get('signer')?.getPublicKey(); const proof = c.get('proof') || await obtainProof(c, opts); // Prevent people from accidentally using the wrong account. This has no other security implications. @@ -90,9 +89,16 @@ function withProof( /** Get the proof over Nostr Connect. */ async function obtainProof(c: AppContext, opts?: ParseAuthRequestOpts) { + const signer = c.get('signer'); + if (!signer) { + throw new HTTPException(401, { + res: c.json({ error: 'No way to sign Nostr event' }, 401), + }); + } + const req = localRequest(c); const reqEvent = await buildAuthEventTemplate(req, opts); - const resEvent = await new APISigner(c).signEvent(reqEvent); + const resEvent = await signer.signEvent(reqEvent); const result = await validateAuthEvent(req, resEvent, opts); if (result.success) { diff --git a/src/middleware/store.ts b/src/middleware/store.ts index 60055b3..40b7c59 100644 --- a/src/middleware/store.ts +++ b/src/middleware/store.ts @@ -4,7 +4,7 @@ import { Storages } from '@/storages.ts'; /** Store middleware. */ const storeMiddleware: AppMiddleware = async (c, next) => { - const pubkey = c.get('pubkey'); + const pubkey = await c.get('signer')?.getPublicKey(); if (pubkey) { const store = new UserStore(pubkey, Storages.admin); diff --git a/src/utils/api.ts b/src/utils/api.ts index cba7c66..3dfdfa3 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -10,7 +10,6 @@ import { type AppContext } from '@/app.ts'; import { Conf } from '@/config.ts'; import * as pipeline from '@/pipeline.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { APISigner } from '@/signers/APISigner.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; @@ -21,7 +20,13 @@ type EventStub = TypeFest.SetOptional { - const signer = new APISigner(c); + const signer = c.get('signer'); + + if (!signer) { + throw new HTTPException(401, { + res: c.json({ error: 'No way to sign Nostr event' }, 401), + }); + } const event = await signer.signEvent({ content: '', diff --git a/src/views.ts b/src/views.ts index 9c31dfc..451ce14 100644 --- a/src/views.ts +++ b/src/views.ts @@ -59,8 +59,10 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal const sortedEvents = [...events].sort((a, b) => ids.indexOf(a.id) - ids.indexOf(b.id)); + const viewerPubkey = await c.get('signer')?.getPublicKey(); + const statuses = await Promise.all( - sortedEvents.map((event) => renderStatus(event, { viewerPubkey: c.get('pubkey') })), + sortedEvents.map((event) => renderStatus(event, { viewerPubkey })), ); // TODO: pagination with min_id and max_id based on the order of `ids`. From 1accae2222a5a7c7571e4e72c8d0fce93f576b97 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 12:04:31 -0500 Subject: [PATCH 17/39] Add a ConnectSigner to wrap our default opts to NConnectSigner, add c.set('signer') calls to nip98 middleware --- src/app.ts | 2 +- src/middleware/auth98.ts | 10 ++++++---- src/middleware/signerMiddleware.ts | 15 +++------------ src/signers/ConnectSigner.ts | 20 ++++++++++++++++++++ 4 files changed, 30 insertions(+), 17 deletions(-) create mode 100644 src/signers/ConnectSigner.ts diff --git a/src/app.ts b/src/app.ts index 5ef5056..62da64c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -126,9 +126,9 @@ app.use( '*', csp(), cors({ origin: '*', exposeHeaders: ['link'] }), + signerMiddleware, auth98(), storeMiddleware, - signerMiddleware, ); app.get('/.well-known/webfinger', webfingerController); diff --git a/src/middleware/auth98.ts b/src/middleware/auth98.ts index d761b95..fbadb2c 100644 --- a/src/middleware/auth98.ts +++ b/src/middleware/auth98.ts @@ -1,14 +1,16 @@ import { NostrEvent } from '@nostrify/nostrify'; import { HTTPException } from 'hono'; + import { type AppContext, type AppMiddleware } from '@/app.ts'; +import { findUser, User } from '@/db/users.ts'; +import { ConnectSigner } from '@/signers/ConnectSigner.ts'; +import { localRequest } from '@/utils/api.ts'; import { buildAuthEventTemplate, parseAuthRequest, type ParseAuthRequestOpts, validateAuthEvent, } from '@/utils/nip98.ts'; -import { localRequest } from '@/utils/api.ts'; -import { findUser, User } from '@/db/users.ts'; /** * NIP-98 auth. @@ -20,7 +22,7 @@ function auth98(opts: ParseAuthRequestOpts = {}): AppMiddleware { const result = await parseAuthRequest(req, opts); if (result.success) { - c.set('pubkey', result.data.pubkey); + c.set('signer', new ConnectSigner(result.data.pubkey)); c.set('proof', result.data); } @@ -78,7 +80,7 @@ function withProof( } if (proof) { - c.set('pubkey', proof.pubkey); + c.set('signer', new ConnectSigner(proof.pubkey)); c.set('proof', proof); await handler(c, proof, next); } else { diff --git a/src/middleware/signerMiddleware.ts b/src/middleware/signerMiddleware.ts index d056391..85a2ff1 100644 --- a/src/middleware/signerMiddleware.ts +++ b/src/middleware/signerMiddleware.ts @@ -1,9 +1,8 @@ -import { NConnectSigner, NSecSigner } from '@nostrify/nostrify'; +import { NSecSigner } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; import { AppMiddleware } from '@/app.ts'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { Storages } from '@/storages.ts'; +import { ConnectSigner } from '@/signers/ConnectSigner.ts'; /** We only accept "Bearer" type. */ const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); @@ -21,15 +20,7 @@ export const signerMiddleware: AppMiddleware = async (c, next) => { switch (decoded.type) { case 'npub': - c.set( - 'signer', - new NConnectSigner({ - pubkey: decoded.data, - relay: Storages.pubsub, - signer: new AdminSigner(), - timeout: 60000, - }), - ); + c.set('signer', new ConnectSigner(decoded.data)); break; case 'nsec': c.set('signer', new NSecSigner(decoded.data)); diff --git a/src/signers/ConnectSigner.ts b/src/signers/ConnectSigner.ts new file mode 100644 index 0000000..b98a59f --- /dev/null +++ b/src/signers/ConnectSigner.ts @@ -0,0 +1,20 @@ +import { NConnectSigner } from '@nostrify/nostrify'; + +import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { Storages } from '@/storages.ts'; + +/** + * NIP-46 signer. + * + * Simple extension of nostrify's `NConnectSigner`, with our options to keep it DRY. + */ +export class ConnectSigner extends NConnectSigner { + constructor(pubkey: string) { + super({ + pubkey, + relay: Storages.pubsub, + signer: new AdminSigner(), + timeout: 60000, + }); + } +} From 084143c5c8d9fd8a61c90cc3319005a6d8a3841c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 12:07:54 -0500 Subject: [PATCH 18/39] Rename all middleware to thingMiddleware --- src/app.ts | 22 +++++++++++-------- .../{auth98.ts => auth98Middleware.ts} | 4 ++-- .../{cache.ts => cacheMiddleware.ts} | 2 +- src/middleware/{csp.ts => cspMiddleware.ts} | 4 +--- .../{store.ts => storeMiddleware.ts} | 4 +--- 5 files changed, 18 insertions(+), 18 deletions(-) rename src/middleware/{auth98.ts => auth98Middleware.ts} (95%) rename src/middleware/{cache.ts => cacheMiddleware.ts} (95%) rename src/middleware/{csp.ts => cspMiddleware.ts} (93%) rename src/middleware/{store.ts => storeMiddleware.ts} (81%) diff --git a/src/app.ts b/src/app.ts index 62da64c..8c8cbc0 100644 --- a/src/app.ts +++ b/src/app.ts @@ -81,12 +81,12 @@ import { hostMetaController } from '@/controllers/well-known/host-meta.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; import { webfingerController } from '@/controllers/well-known/webfinger.ts'; -import { auth98, requireProof, requireRole } from '@/middleware/auth98.ts'; -import { cache } from '@/middleware/cache.ts'; -import { csp } from '@/middleware/csp.ts'; +import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; +import { cacheMiddleware } from '@/middleware/cacheMiddleware.ts'; +import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; import { requireSigner } from '@/middleware/requireSigner.ts'; import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; -import { storeMiddleware } from '@/middleware/store.ts'; +import { storeMiddleware } from '@/middleware/storeMiddleware.ts'; import { blockController } from '@/controllers/api/accounts.ts'; import { unblockController } from '@/controllers/api/accounts.ts'; @@ -124,10 +124,10 @@ app.get('/relay', relayController); app.use( '*', - csp(), + cspMiddleware(), cors({ origin: '*', exposeHeaders: ['link'] }), signerMiddleware, - auth98(), + auth98Middleware(), storeMiddleware, ); @@ -140,7 +140,7 @@ app.get('/users/:username', actorController); app.get('/nodeinfo/:version', nodeInfoSchemaController); -app.get('/api/v1/instance', cache({ cacheName: 'web', expires: Time.minutes(5) }), instanceController); +app.get('/api/v1/instance', cacheMiddleware({ cacheName: 'web', expires: Time.minutes(5) }), instanceController); app.get('/api/v1/apps/verify_credentials', appCredentialsController); app.post('/api/v1/apps', createAppController); @@ -195,8 +195,12 @@ app.get('/api/v2/search', searchController); app.get('/api/pleroma/frontend_configurations', frontendConfigController); -app.get('/api/v1/trends/tags', cache({ cacheName: 'web', expires: Time.minutes(15) }), trendingTagsController); -app.get('/api/v1/trends', cache({ cacheName: 'web', expires: Time.minutes(15) }), trendingTagsController); +app.get( + '/api/v1/trends/tags', + cacheMiddleware({ cacheName: 'web', expires: Time.minutes(15) }), + trendingTagsController, +); +app.get('/api/v1/trends', cacheMiddleware({ cacheName: 'web', expires: Time.minutes(15) }), trendingTagsController); app.get('/api/v1/suggestions', suggestionsV1Controller); app.get('/api/v2/suggestions', suggestionsV2Controller); diff --git a/src/middleware/auth98.ts b/src/middleware/auth98Middleware.ts similarity index 95% rename from src/middleware/auth98.ts rename to src/middleware/auth98Middleware.ts index fbadb2c..7cd7059 100644 --- a/src/middleware/auth98.ts +++ b/src/middleware/auth98Middleware.ts @@ -16,7 +16,7 @@ import { * NIP-98 auth. * https://github.com/nostr-protocol/nips/blob/master/98.md */ -function auth98(opts: ParseAuthRequestOpts = {}): AppMiddleware { +function auth98Middleware(opts: ParseAuthRequestOpts = {}): AppMiddleware { return async (c, next) => { const req = localRequest(c); const result = await parseAuthRequest(req, opts); @@ -108,4 +108,4 @@ async function obtainProof(c: AppContext, opts?: ParseAuthRequestOpts) { } } -export { auth98, requireProof, requireRole }; +export { auth98Middleware, requireProof, requireRole }; diff --git a/src/middleware/cache.ts b/src/middleware/cacheMiddleware.ts similarity index 95% rename from src/middleware/cache.ts rename to src/middleware/cacheMiddleware.ts index 181623f..baa4976 100644 --- a/src/middleware/cache.ts +++ b/src/middleware/cacheMiddleware.ts @@ -5,7 +5,7 @@ import ExpiringCache from '@/utils/expiring-cache.ts'; const debug = Debug('ditto:middleware:cache'); -export const cache = (options: { +export const cacheMiddleware = (options: { cacheName: string; expires?: number; }): MiddlewareHandler => { diff --git a/src/middleware/csp.ts b/src/middleware/cspMiddleware.ts similarity index 93% rename from src/middleware/csp.ts rename to src/middleware/cspMiddleware.ts index fdce5c7..00c4ecc 100644 --- a/src/middleware/csp.ts +++ b/src/middleware/cspMiddleware.ts @@ -1,7 +1,7 @@ import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; -const csp = (): AppMiddleware => { +export const cspMiddleware = (): AppMiddleware => { return async (c, next) => { const { host, protocol, origin } = Conf.url; const wsProtocol = protocol === 'http:' ? 'ws:' : 'wss:'; @@ -26,5 +26,3 @@ const csp = (): AppMiddleware => { await next(); }; }; - -export { csp }; diff --git a/src/middleware/store.ts b/src/middleware/storeMiddleware.ts similarity index 81% rename from src/middleware/store.ts rename to src/middleware/storeMiddleware.ts index 40b7c59..efb65ed 100644 --- a/src/middleware/store.ts +++ b/src/middleware/storeMiddleware.ts @@ -3,7 +3,7 @@ import { UserStore } from '@/storages/UserStore.ts'; import { Storages } from '@/storages.ts'; /** Store middleware. */ -const storeMiddleware: AppMiddleware = async (c, next) => { +export const storeMiddleware: AppMiddleware = async (c, next) => { const pubkey = await c.get('signer')?.getPublicKey(); if (pubkey) { @@ -14,5 +14,3 @@ const storeMiddleware: AppMiddleware = async (c, next) => { } await next(); }; - -export { storeMiddleware }; From 03182f8a5a260447173c0e40d9f6c388002ffabd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 12:14:27 -0500 Subject: [PATCH 19/39] ConnectSigner: implement getRelays, support nprofile auth again --- src/middleware/signerMiddleware.ts | 3 +++ src/signers/ConnectSigner.ts | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/middleware/signerMiddleware.ts b/src/middleware/signerMiddleware.ts index 85a2ff1..8779937 100644 --- a/src/middleware/signerMiddleware.ts +++ b/src/middleware/signerMiddleware.ts @@ -22,6 +22,9 @@ export const signerMiddleware: AppMiddleware = async (c, next) => { case 'npub': c.set('signer', new ConnectSigner(decoded.data)); break; + case 'nprofile': + c.set('signer', new ConnectSigner(decoded.data.pubkey, decoded.data.relays)); + break; case 'nsec': c.set('signer', new NSecSigner(decoded.data)); break; diff --git a/src/signers/ConnectSigner.ts b/src/signers/ConnectSigner.ts index b98a59f..e7d61a8 100644 --- a/src/signers/ConnectSigner.ts +++ b/src/signers/ConnectSigner.ts @@ -9,12 +9,22 @@ import { Storages } from '@/storages.ts'; * Simple extension of nostrify's `NConnectSigner`, with our options to keep it DRY. */ export class ConnectSigner extends NConnectSigner { - constructor(pubkey: string) { + constructor(pubkey: string, private relays?: string[]) { super({ pubkey, + // TODO: use a remote relay for `nprofile` signing, if present and Conf.relay isn't already in the list relay: Storages.pubsub, signer: new AdminSigner(), timeout: 60000, }); } + + /** Get the user's relays if they passed in an `nprofile` auth token. */ + // deno-lint-ignore require-await + async getRelays(): Promise> { + return this.relays?.reduce>((acc, relay) => { + acc[relay] = { read: true, write: true }; + return acc; + }, {}) ?? {}; + } } From cd2a35d951e6db472e9fd36b496de8748e340376 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 12:20:36 -0500 Subject: [PATCH 20/39] ConnectSigner: make getPublicKey used the stored value instead of actually hitting the relay --- src/signers/ConnectSigner.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/signers/ConnectSigner.ts b/src/signers/ConnectSigner.ts index e7d61a8..4dda0b1 100644 --- a/src/signers/ConnectSigner.ts +++ b/src/signers/ConnectSigner.ts @@ -1,3 +1,4 @@ +// deno-lint-ignore-file require-await import { NConnectSigner } from '@nostrify/nostrify'; import { AdminSigner } from '@/signers/AdminSigner.ts'; @@ -9,18 +10,26 @@ import { Storages } from '@/storages.ts'; * Simple extension of nostrify's `NConnectSigner`, with our options to keep it DRY. */ export class ConnectSigner extends NConnectSigner { + private _pubkey: string; + constructor(pubkey: string, private relays?: string[]) { super({ pubkey, - // TODO: use a remote relay for `nprofile` signing, if present and Conf.relay isn't already in the list + // TODO: use a remote relay for `nprofile` signing (if present and `Conf.relay` isn't already in the list) relay: Storages.pubsub, signer: new AdminSigner(), timeout: 60000, }); + + this._pubkey = pubkey; + } + + // Prevent unnecessary NIP-46 round-trips. + async getPublicKey(): Promise { + return this._pubkey; } /** Get the user's relays if they passed in an `nprofile` auth token. */ - // deno-lint-ignore require-await async getRelays(): Promise> { return this.relays?.reduce>((acc, relay) => { acc[relay] = { read: true, write: true }; From dd3c64ef849e3dfeb5e1cd4847e4a1c954369edb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 13:09:16 -0500 Subject: [PATCH 21/39] Remove accidentally checked in file --- ..env.swp | Bin 1024 -> 0 bytes .gitignore | 1 + 2 files changed, 1 insertion(+) delete mode 100644 ..env.swp diff --git a/..env.swp b/..env.swp deleted file mode 100644 index 2ab134254314be032359c6894717262e417e7684..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1024 zcmYc?$V<%2S1{8vVn6|1lPwq$b5bi%1aWW*@(XnHi*ZOI3G1cil_7CQnWG^v8Uh0x F0sw&A3CREe diff --git a/.gitignore b/.gitignore index d816f9d..39dbfbb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .env *.cpuprofile +*.swp deno-test.xml \ No newline at end of file From b5f0d2f0e60c23bb7025727a13ad59c409ce5212 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 14:12:43 -0500 Subject: [PATCH 22/39] docs/events: clean up User event kind --- docs/events.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/events.md b/docs/events.md index e850fcb..1674239 100644 --- a/docs/events.md +++ b/docs/events.md @@ -9,9 +9,7 @@ The Ditto server publishes kind `30361` events to represent users. These events User events have the following tags: - `d` - pubkey of the user. -- `name` - NIP-05 username granted to the user, without the domain. - `role` - one of `admin` or `user`. -- `origin` - the origin of the user's NIP-05, at the time the event was published. Example: @@ -25,7 +23,6 @@ Example: "tags": [ ["d", "79c2cae114ea28a981e7559b4fe7854a473521a8d22a66bbab9fa248eb820ff6"], ["role", "user"], - ["origin", "https://ditto.ngrok.app"], ["alt", "User's account was updated by the admins of ditto.ngrok.app"] ], "sig": "fc12db77b1c8f8aa86c73b617f0cd4af1e6ba244239eaf3164a292de6d39363f32d6b817ffff796ace7a103d75e1d8e6a0fb7f618819b32d81a953b4a75d7507" @@ -40,4 +37,4 @@ The sections below describe the `content` field. Some are encrypted and some are ### `pub.ditto.pleroma.config` -NIP-04 encrypted JSON array of Pleroma ConfigDB objects. Pleroma admin API endpoints set this config, and Ditto reads from it. \ No newline at end of file +NIP-04 encrypted JSON array of Pleroma ConfigDB objects. Pleroma admin API endpoints set this config, and Ditto reads from it. From a061c248bd5d288a27305c3a502add4296d6d999 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 14:18:44 -0500 Subject: [PATCH 23/39] signerMiddleware: add debug log --- src/middleware/signerMiddleware.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/middleware/signerMiddleware.ts b/src/middleware/signerMiddleware.ts index 8779937..1d35708 100644 --- a/src/middleware/signerMiddleware.ts +++ b/src/middleware/signerMiddleware.ts @@ -1,9 +1,12 @@ import { NSecSigner } from '@nostrify/nostrify'; +import { Stickynotes } from '@soapbox/stickynotes'; import { nip19 } from 'nostr-tools'; import { AppMiddleware } from '@/app.ts'; import { ConnectSigner } from '@/signers/ConnectSigner.ts'; +const console = new Stickynotes('ditto:signerMiddleware'); + /** We only accept "Bearer" type. */ const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); @@ -30,7 +33,7 @@ export const signerMiddleware: AppMiddleware = async (c, next) => { break; } } catch { - // the user is not logged in + console.debug('The user is not logged in'); } } From 45b766af4d768853c6236eeeadfeb7aee16f47a7 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 14:24:48 -0500 Subject: [PATCH 24/39] Remove 'user' from AppContext --- src/app.ts | 2 -- src/middleware/auth98Middleware.ts | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/app.ts b/src/app.ts index 8c8cbc0..447499f 100644 --- a/src/app.ts +++ b/src/app.ts @@ -96,8 +96,6 @@ interface AppEnv extends HonoEnv { signer?: NostrSigner; /** NIP-98 signed event proving the pubkey is owned by the user. */ proof?: NostrEvent; - /** User associated with the pubkey, if any. */ - user?: User; /** Store */ store: NStore; }; diff --git a/src/middleware/auth98Middleware.ts b/src/middleware/auth98Middleware.ts index 7cd7059..abecea7 100644 --- a/src/middleware/auth98Middleware.ts +++ b/src/middleware/auth98Middleware.ts @@ -34,9 +34,8 @@ type UserRole = 'user' | 'admin'; /** Require the user to prove their role before invoking the controller. */ function requireRole(role: UserRole, opts?: ParseAuthRequestOpts): AppMiddleware { - return withProof(async (c, proof, next) => { + return withProof(async (_c, proof, next) => { const user = await findUser({ pubkey: proof.pubkey }); - c.set('user', user); if (user && matchesRole(user, role)) { await next(); From ecfea827e1a3b5a06f6fad43f6e2a1806e683ffb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 14:38:05 -0500 Subject: [PATCH 25/39] Move RelayError into its own file, add helper methods --- src/RelayError.ts | 24 ++++++++++++++++++++++++ src/controllers/nostr/relay.ts | 3 ++- src/pipeline.ts | 22 +++------------------- src/utils/api.ts | 3 ++- 4 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 src/RelayError.ts diff --git a/src/RelayError.ts b/src/RelayError.ts new file mode 100644 index 0000000..1d275f6 --- /dev/null +++ b/src/RelayError.ts @@ -0,0 +1,24 @@ +import { NostrRelayOK } from '@nostrify/nostrify'; + +export type RelayErrorPrefix = 'duplicate' | 'pow' | 'blocked' | 'rate-limited' | 'invalid' | 'error'; + +/** NIP-01 command line result. */ +export class RelayError extends Error { + constructor(prefix: RelayErrorPrefix, message: string) { + super(`${prefix}: ${message}`); + } + + /** Construct a RelayError from the reason message. */ + static fromReason(reason: string): RelayError { + const [prefix, ...rest] = reason.split(': '); + return new RelayError(prefix as RelayErrorPrefix, rest.join(': ')); + } + + /** Throw a new RelayError if the OK message is false. */ + static assert(msg: NostrRelayOK): void { + const [_, _eventId, ok, reason] = msg; + if (!ok) { + throw RelayError.fromReason(reason); + } + } +} diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 7d70ad9..c0fa026 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -10,6 +10,7 @@ import { } from '@nostrify/nostrify'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; import * as pipeline from '@/pipeline.ts'; +import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; import type { AppController } from '@/app.ts'; @@ -95,7 +96,7 @@ function connectStream(socket: WebSocket) { await pipeline.handleEvent(event, AbortSignal.timeout(1000)); send(['OK', event.id, true, '']); } catch (e) { - if (e instanceof pipeline.RelayError) { + if (e instanceof RelayError) { send(['OK', event.id, false, e.message]); } else { send(['OK', event.id, false, 'error: something went wrong']); diff --git a/src/pipeline.ts b/src/pipeline.ts index a47a2da..995abf2 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -10,6 +10,7 @@ import { deleteAttachedMedia } from '@/db/unattached-media.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { isEphemeralKind } from '@/kinds.ts'; import { DVM } from '@/pipeline/DVM.ts'; +import { RelayError } from '@/RelayError.ts'; import { updateStats } from '@/stats.ts'; import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; @@ -71,15 +72,7 @@ async function policyFilter(event: NostrEvent): Promise { const result = await policy.call(event); debug(JSON.stringify(result)); - const [_, _eventId, ok, reason] = result; - if (!ok) { - const [prefix, ...rest] = reason.split(': '); - if (['duplicate', 'pow', 'blocked', 'rate-limited', 'invalid'].includes(prefix)) { - throw new RelayError(prefix as RelayErrorPrefix, rest.join(': ')); - } else { - throw new RelayError('error', rest.join(': ')); - } - } + RelayError.assert(result); } /** Encounter the event, and return whether it has already been encountered. */ @@ -286,13 +279,4 @@ async function streamOut(event: NostrEvent): Promise { } } -type RelayErrorPrefix = 'duplicate' | 'pow' | 'blocked' | 'rate-limited' | 'invalid' | 'error'; - -/** NIP-20 command line result. */ -class RelayError extends Error { - constructor(prefix: RelayErrorPrefix, message: string) { - super(`${prefix}: ${message}`); - } -} - -export { handleEvent, RelayError }; +export { handleEvent }; diff --git a/src/utils/api.ts b/src/utils/api.ts index 8da87fb..70fd995 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -9,6 +9,7 @@ import { z } from 'zod'; import { type AppContext } from '@/app.ts'; import { Conf } from '@/config.ts'; import * as pipeline from '@/pipeline.ts'; +import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { APISigner } from '@/signers/APISigner.ts'; import { Storages } from '@/storages.ts'; @@ -106,7 +107,7 @@ async function publishEvent(event: NostrEvent, c: AppContext): Promise Date: Tue, 14 May 2024 14:39:48 -0500 Subject: [PATCH 26/39] Uppercase CustomPolicy --- src/pipeline.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 995abf2..b3eea25 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -61,8 +61,8 @@ async function policyFilter(event: NostrEvent): Promise { ]; try { - const customPolicy = (await import('../data/policy.ts')).default; - policies.push(new customPolicy()); + const CustomPolicy = (await import('../data/policy.ts')).default; + policies.push(new CustomPolicy()); } catch (_e) { debug('policy not found - https://docs.soapbox.pub/ditto/policies/'); } @@ -71,7 +71,6 @@ async function policyFilter(event: NostrEvent): Promise { const result = await policy.call(event); debug(JSON.stringify(result)); - RelayError.assert(result); } From e53ea222742ec0d7074988caa5b73884d140d469 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 14:48:37 -0500 Subject: [PATCH 27/39] Remove unused import --- src/app.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/app.ts b/src/app.ts index 447499f..84ad780 100644 --- a/src/app.ts +++ b/src/app.ts @@ -3,7 +3,6 @@ import Debug from '@soapbox/stickynotes/debug'; import { type Context, Env as HonoEnv, type Handler, Hono, Input as HonoInput, type MiddlewareHandler } from 'hono'; import { cors, logger, serveStatic } from 'hono/middleware'; -import { type User } from '@/db/users.ts'; import '@/firehose.ts'; import { Time } from '@/utils.ts'; From eef349f1e9aa19d3b1dd80489531f69361b1e55f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 15:05:59 -0500 Subject: [PATCH 28/39] Update stats before storing event --- src/pipeline.ts | 6 ++---- src/stats.ts | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index b3eea25..25cadb5 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -107,10 +107,8 @@ async function storeEvent(event: DittoEvent, signal?: AbortSignal): Promise[] = []; // Kind 3 is a special case - replace the count with the new list. if (event.kind === 3) { - prev = await maybeGetPrev(event); + prev = await getPrevEvent(event); if (!prev || event.created_at >= prev.created_at) { queries.push(updateFollowingCountQuery(event)); } @@ -153,12 +153,14 @@ function eventStatsQuery(diffs: EventStatDiff[]) { } /** Get the last version of the event, if any. */ -async function maybeGetPrev(event: NostrEvent): Promise { - const [prev] = await Storages.db.query([ - { kinds: [event.kind], authors: [event.pubkey], limit: 1 }, - ]); +async function getPrevEvent(event: NostrEvent): Promise { + if (NKinds.replaceable(event.kind) || NKinds.parameterizedReplaceable(event.kind)) { + const [prev] = await Storages.db.query([ + { kinds: [event.kind], authors: [event.pubkey], limit: 1 }, + ]); - return prev; + return prev; + } } /** Set the following count to the total number of unique "p" tags in the follow list. */ From 7feecab7232d284e42f70c0e0b942d8c1fa4346c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 15:25:56 -0500 Subject: [PATCH 29/39] stats: fix ambiguous column name error in Postgres? --- src/stats.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/stats.ts b/src/stats.ts index 1e95901..21a4d97 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -119,9 +119,9 @@ function authorStatsQuery(diffs: AuthorStatDiff[]) { oc .column('pubkey') .doUpdateSet((eb) => ({ - followers_count: eb('followers_count', '+', eb.ref('excluded.followers_count')), - following_count: eb('following_count', '+', eb.ref('excluded.following_count')), - notes_count: eb('notes_count', '+', eb.ref('excluded.notes_count')), + followers_count: eb('author_stats.followers_count', '+', eb.ref('excluded.followers_count')), + following_count: eb('author_stats.following_count', '+', eb.ref('excluded.following_count')), + notes_count: eb('author_stats.notes_count', '+', eb.ref('excluded.notes_count')), })) ); } @@ -145,9 +145,9 @@ function eventStatsQuery(diffs: EventStatDiff[]) { oc .column('event_id') .doUpdateSet((eb) => ({ - replies_count: eb('replies_count', '+', eb.ref('excluded.replies_count')), - reposts_count: eb('reposts_count', '+', eb.ref('excluded.reposts_count')), - reactions_count: eb('reactions_count', '+', eb.ref('excluded.reactions_count')), + replies_count: eb('event_stats.replies_count', '+', eb.ref('excluded.replies_count')), + reposts_count: eb('event_stats.reposts_count', '+', eb.ref('excluded.reposts_count')), + reactions_count: eb('event_stats.reactions_count', '+', eb.ref('excluded.reactions_count')), })) ); } From 3c706dc81be4e883505ccd8bfdb648df18d4a0f6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 16:10:50 -0500 Subject: [PATCH 30/39] Storages: make all methods async (total chaos and destruction) --- src/storages.ts | 85 +++++++++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/src/storages.ts b/src/storages.ts index a51cbd1..056228b 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -1,3 +1,4 @@ +// deno-lint-ignore-file require-await import { NCache } from '@nostrify/nostrify'; import { Conf } from '@/config.ts'; import { db } from '@/db.ts'; @@ -12,89 +13,97 @@ import { UserStore } from '@/storages/UserStore.ts'; import { Time } from '@/utils/time.ts'; export class Storages { - private static _db: EventsDB | undefined; - private static _admin: UserStore | undefined; - private static _cache: NCache | undefined; - private static _client: PoolStore | undefined; - private static _optimizer: Optimizer | undefined; - private static _reqmeister: Reqmeister | undefined; - private static _pubsub: InternalRelay | undefined; - private static _search: SearchStore | undefined; + private static _db: Promise | undefined; + private static _admin: Promise | undefined; + private static _cache: Promise | undefined; + private static _client: Promise | undefined; + private static _optimizer: Promise | undefined; + private static _reqmeister: Promise | undefined; + private static _pubsub: Promise | undefined; + private static _search: Promise | undefined; /** SQLite database to store events this Ditto server cares about. */ - public static get db(): EventsDB { + public static async db(): Promise { if (!this._db) { - this._db = new EventsDB(db); + this._db = Promise.resolve(new EventsDB(db)); } return this._db; } /** Admin user storage. */ - public static get admin(): UserStore { + public static async admin(): Promise { if (!this._admin) { - this._admin = new UserStore(Conf.pubkey, this.db); + this._admin = Promise.resolve(new UserStore(Conf.pubkey, await this.db())); } return this._admin; } /** Internal pubsub relay between controllers and the pipeline. */ - public static get pubsub(): InternalRelay { + public static async pubsub(): Promise { if (!this._pubsub) { - this._pubsub = new InternalRelay(); + this._pubsub = Promise.resolve(new InternalRelay()); } return this._pubsub; } /** Relay pool storage. */ - public static get client(): PoolStore { + public static async client(): Promise { if (!this._client) { - this._client = new PoolStore({ - pool, - relays: activeRelays, - }); + this._client = Promise.resolve( + new PoolStore({ + pool, + relays: activeRelays, + }), + ); } return this._client; } /** In-memory data store for cached events. */ - public static get cache(): NCache { + public static async cache(): Promise { if (!this._cache) { - this._cache = new NCache({ max: 3000 }); + this._cache = Promise.resolve(new NCache({ max: 3000 })); } return this._cache; } /** Batches requests for single events. */ - public static get reqmeister(): Reqmeister { + public static async reqmeister(): Promise { if (!this._reqmeister) { - this._reqmeister = new Reqmeister({ - client: this.client, - delay: Time.seconds(1), - timeout: Time.seconds(1), - }); + this._reqmeister = Promise.resolve( + new Reqmeister({ + client: await this.client(), + delay: Time.seconds(1), + timeout: Time.seconds(1), + }), + ); } return this._reqmeister; } /** Main Ditto storage adapter */ - public static get optimizer(): Optimizer { + public static async optimizer(): Promise { if (!this._optimizer) { - this._optimizer = new Optimizer({ - db: this.db, - cache: this.cache, - client: this.reqmeister, - }); + this._optimizer = Promise.resolve( + new Optimizer({ + db: await this.db(), + cache: await this.cache(), + client: await this.reqmeister(), + }), + ); } return this._optimizer; } /** Storage to use for remote search. */ - public static get search(): SearchStore { + public static async search(): Promise { if (!this._search) { - this._search = new SearchStore({ - relay: Conf.searchRelay, - fallback: this.optimizer, - }); + this._search = Promise.resolve( + new SearchStore({ + relay: Conf.searchRelay, + fallback: await this.optimizer(), + }), + ); } return this._search; } From 08c9ee0670f3d583af56e24271153006ef01b523 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 16:25:24 -0500 Subject: [PATCH 31/39] Refactor client and firehose --- src/app.ts | 7 ++++++- src/config.ts | 4 ++++ src/firehose.ts | 39 +++++++++++++++++++-------------------- src/pool.ts | 34 ---------------------------------- src/stats.ts | 9 ++++++--- src/storages.ts | 39 ++++++++++++++++++++++++++++++++++----- 6 files changed, 69 insertions(+), 63 deletions(-) delete mode 100644 src/pool.ts diff --git a/src/app.ts b/src/app.ts index 84ad780..059e883 100644 --- a/src/app.ts +++ b/src/app.ts @@ -3,7 +3,8 @@ import Debug from '@soapbox/stickynotes/debug'; import { type Context, Env as HonoEnv, type Handler, Hono, Input as HonoInput, type MiddlewareHandler } from 'hono'; import { cors, logger, serveStatic } from 'hono/middleware'; -import '@/firehose.ts'; +import { Conf } from '@/config.ts'; +import { startFirehose } from '@/firehose.ts'; import { Time } from '@/utils.ts'; import { actorController } from '@/controllers/activitypub/actor.ts'; @@ -108,6 +109,10 @@ const app = new Hono(); const debug = Debug('ditto:http'); +if (Conf.firehoseEnabled) { + startFirehose(); +} + app.use('/api/*', logger(debug)); app.use('/relay/*', logger(debug)); app.use('/.well-known/*', logger(debug)); diff --git a/src/config.ts b/src/config.ts index 4266033..3d3e51b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -215,6 +215,10 @@ class Conf { return Number(Deno.env.get('PG_POOL_SIZE') ?? 10); }, }; + /** Whether to enable requesting events from known relays. */ + static get firehoseEnabled(): boolean { + return optionalBooleanSchema.parse(Deno.env.get('FIREHOSE_ENABLED')) ?? true; + } } const optionalBooleanSchema = z diff --git a/src/firehose.ts b/src/firehose.ts index d7aaab3..2c776fe 100644 --- a/src/firehose.ts +++ b/src/firehose.ts @@ -1,29 +1,28 @@ -import { NostrEvent } from '@nostrify/nostrify'; -import Debug from '@soapbox/stickynotes/debug'; +import { Stickynotes } from '@soapbox/stickynotes'; -import { activeRelays, pool } from '@/pool.ts'; +import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; import * as pipeline from './pipeline.ts'; -const debug = Debug('ditto:firehose'); +const console = new Stickynotes('ditto:firehose'); -// This file watches events on all known relays and performs -// side-effects based on them, such as trending hashtag tracking -// and storing events for notifications and the home feed. -pool.subscribe( - [{ kinds: [0, 1, 3, 5, 6, 7, 9735, 10002], limit: 0, since: nostrNow() }], - activeRelays, - handleEvent, - undefined, - undefined, -); +/** + * This function watches events on all known relays and performs + * side-effects based on them, such as trending hashtag tracking + * and storing events for notifications and the home feed. + */ +export async function startFirehose() { + const store = await Storages.client(); -/** Handle events through the firehose pipeline. */ -function handleEvent(event: NostrEvent): Promise { - debug(`NostrEvent<${event.kind}> ${event.id}`); + for await (const msg of store.req([{ kinds: [0, 1, 3, 5, 6, 7, 9735, 10002], limit: 0, since: nostrNow() }])) { + if (msg[0] === 'EVENT') { + const event = msg[2]; + console.debug(`NostrEvent<${event.kind}> ${event.id}`); - return pipeline - .handleEvent(event, AbortSignal.timeout(5000)) - .catch(() => {}); + pipeline + .handleEvent(event, AbortSignal.timeout(5000)) + .catch(() => {}); + } + } } diff --git a/src/pool.ts b/src/pool.ts deleted file mode 100644 index 3ac1a1d..0000000 --- a/src/pool.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { RelayPoolWorker } from 'nostr-relaypool'; - -import { Storages } from '@/storages.ts'; -import { Conf } from '@/config.ts'; - -const [relayList] = await Storages.db.query([ - { kinds: [10002], authors: [Conf.pubkey], limit: 1 }, -]); - -const tags = relayList?.tags ?? []; - -const activeRelays = tags.reduce((acc, [name, url, marker]) => { - if (name === 'r' && !marker) { - acc.push(url); - } - return acc; -}, []); - -console.log(`pool: connecting to ${activeRelays.length} relays.`); - -const worker = new Worker('https://unpkg.com/nostr-relaypool2@0.6.34/lib/nostr-relaypool.worker.js', { - type: 'module', -}); - -// @ts-ignore Wrong types. -const pool = new RelayPoolWorker(worker, activeRelays, { - autoReconnect: true, - // The pipeline verifies events. - skipVerification: true, - // The logging feature overwhelms the CPU and creates too many logs. - logErrorsAndNotices: false, -}); - -export { activeRelays, pool }; diff --git a/src/stats.ts b/src/stats.ts index 21a4d97..bff2aeb 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -47,6 +47,7 @@ async function updateStats(event: NostrEvent) { /** Calculate stats changes ahead of time so we can build an efficient query. */ async function getStatsDiff(event: NostrEvent, prev: NostrEvent | undefined): Promise { + const store = await Storages.db(); const statDiffs: StatDiff[] = []; const firstTaggedId = event.tags.find(([name]) => name === 'e')?.[1]; @@ -65,7 +66,7 @@ async function getStatsDiff(event: NostrEvent, prev: NostrEvent | undefined): Pr case 5: { if (!firstTaggedId) break; - const [repostedEvent] = await Storages.db.query( + const [repostedEvent] = await store.query( [{ kinds: [6], ids: [firstTaggedId], authors: [event.pubkey] }], { limit: 1 }, ); @@ -77,7 +78,7 @@ async function getStatsDiff(event: NostrEvent, prev: NostrEvent | undefined): Pr const eventBeingRepostedPubkey = repostedEvent.tags.find(([name]) => name === 'p')?.[1]; if (!eventBeingRepostedId || !eventBeingRepostedPubkey) break; - const [eventBeingReposted] = await Storages.db.query( + const [eventBeingReposted] = await store.query( [{ kinds: [1], ids: [eventBeingRepostedId], authors: [eventBeingRepostedPubkey] }], { limit: 1 }, ); @@ -155,7 +156,9 @@ function eventStatsQuery(diffs: EventStatDiff[]) { /** Get the last version of the event, if any. */ async function getPrevEvent(event: NostrEvent): Promise { if (NKinds.replaceable(event.kind) || NKinds.parameterizedReplaceable(event.kind)) { - const [prev] = await Storages.db.query([ + const store = await Storages.db(); + + const [prev] = await store.query([ { kinds: [event.kind], authors: [event.pubkey], limit: 1 }, ]); diff --git a/src/storages.ts b/src/storages.ts index 056228b..cbda925 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -2,7 +2,6 @@ import { NCache } from '@nostrify/nostrify'; import { Conf } from '@/config.ts'; import { db } from '@/db.ts'; -import { activeRelays, pool } from '@/pool.ts'; import { EventsDB } from '@/storages/events-db.ts'; import { Optimizer } from '@/storages/optimizer.ts'; import { PoolStore } from '@/storages/pool-store.ts'; @@ -49,12 +48,42 @@ export class Storages { /** Relay pool storage. */ public static async client(): Promise { if (!this._client) { - this._client = Promise.resolve( - new PoolStore({ + this._client = (async () => { + const db = await this.db(); + + const [relayList] = await db.query([ + { kinds: [10002], authors: [Conf.pubkey], limit: 1 }, + ]); + + const tags = relayList?.tags ?? []; + + const activeRelays = tags.reduce((acc, [name, url, marker]) => { + if (name === 'r' && !marker) { + acc.push(url); + } + return acc; + }, []); + + console.log(`pool: connecting to ${activeRelays.length} relays.`); + + const worker = new Worker('https://unpkg.com/nostr-relaypool2@0.6.34/lib/nostr-relaypool.worker.js', { + type: 'module', + }); + + // @ts-ignore Wrong types. + const pool = new RelayPoolWorker(worker, activeRelays, { + autoReconnect: true, + // The pipeline verifies events. + skipVerification: true, + // The logging feature overwhelms the CPU and creates too many logs. + logErrorsAndNotices: false, + }); + + return new PoolStore({ pool, relays: activeRelays, - }), - ); + }); + })(); } return this._client; } From 68b5887ed01ac64badec805439e359cef44d7aa2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 18:23:41 -0500 Subject: [PATCH 32/39] Don't let your memes be dreams --- src/controllers/activitypub/actor.ts | 2 +- src/controllers/api/accounts.ts | 29 ++++++++------ src/controllers/api/admin.ts | 7 ++-- src/controllers/api/bookmarks.ts | 3 +- src/controllers/api/ditto.ts | 7 +++- src/controllers/api/instance.ts | 3 +- src/controllers/api/mutes.ts | 3 +- src/controllers/api/notifications.ts | 2 +- src/controllers/api/pleroma.ts | 15 +++++--- src/controllers/api/reports.ts | 8 ++-- src/controllers/api/search.ts | 13 ++++--- src/controllers/api/statuses.ts | 37 ++++++++++-------- src/controllers/api/streaming.ts | 6 ++- src/controllers/api/suggestions.ts | 2 +- src/controllers/api/timelines.ts | 8 +--- src/controllers/nostr/relay-info.ts | 4 +- src/controllers/nostr/relay.ts | 10 +++-- src/controllers/well-known/nostr.ts | 2 +- src/controllers/well-known/webfinger.ts | 2 +- src/db/users.ts | 3 +- src/middleware/storeMiddleware.ts | 4 +- src/pipeline.ts | 39 ++++++++++++------- src/pipeline/DVM.ts | 4 +- src/queries.ts | 22 +++++++---- src/signers/ConnectSigner.ts | 51 ++++++++++++++++++++----- src/storages/UserStore.ts | 17 +++------ src/storages/hydrate.test.ts | 10 ++--- src/storages/hydrate.ts | 46 +++++++++++----------- src/storages/pool-store.ts | 3 +- src/storages/search-store.ts | 2 +- src/utils/api.ts | 11 ++++-- src/utils/connect.ts | 3 +- src/utils/instance.ts | 7 ++-- src/utils/nip05.ts | 9 +++-- src/utils/outbox.ts | 9 +++-- src/views.ts | 18 +++++---- src/views/mastodon/relationships.ts | 4 +- src/views/mastodon/statuses.ts | 9 +++-- 38 files changed, 260 insertions(+), 174 deletions(-) diff --git a/src/controllers/activitypub/actor.ts b/src/controllers/activitypub/actor.ts index e82a88a..19f5f10 100644 --- a/src/controllers/activitypub/actor.ts +++ b/src/controllers/activitypub/actor.ts @@ -9,7 +9,7 @@ const actorController: AppController = async (c) => { const username = c.req.param('username'); const { signal } = c.req.raw; - const pointer = await localNip05Lookup(username); + const pointer = await localNip05Lookup(c.get('store'), username); if (!pointer) return notFound(c); const event = await getAuthor(pointer.pubkey, { signal }); diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 88d19b1..5c26ba5 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -94,15 +94,16 @@ const accountSearchController: AppController = async (c) => { } const query = decodeURIComponent(q); + const store = await Storages.search(); const [event, events] = await Promise.all([ lookupAccount(query), - Storages.search.query([{ kinds: [0], search: query, limit: 20 }], { signal: c.req.raw.signal }), + store.query([{ kinds: [0], search: query, limit: 20 }], { signal: c.req.raw.signal }), ]); const results = await hydrateEvents({ events: event ? [event, ...events] : events, - storage: Storages.db, + store, signal: c.req.raw.signal, }); @@ -147,8 +148,10 @@ const accountStatusesController: AppController = async (c) => { const { pinned, limit, exclude_replies, tagged } = accountStatusesQuerySchema.parse(c.req.query()); const { signal } = c.req.raw; + const store = await Storages.db(); + if (pinned) { - const [pinEvent] = await Storages.db.query([{ kinds: [10001], authors: [pubkey], limit: 1 }], { signal }); + const [pinEvent] = await store.query([{ kinds: [10001], authors: [pubkey], limit: 1 }], { signal }); if (pinEvent) { const pinnedEventIds = getTagSet(pinEvent.tags, 'e'); return renderStatuses(c, [...pinnedEventIds].reverse()); @@ -169,8 +172,8 @@ const accountStatusesController: AppController = async (c) => { filter['#t'] = [tagged]; } - const events = await Storages.db.query([filter], { signal }) - .then((events) => hydrateEvents({ events, storage: Storages.db, signal })) + const events = await store.query([filter], { signal }) + .then((events) => hydrateEvents({ events, store, signal })) .then((events) => { if (exclude_replies) { return events.filter((event) => !findReplyTag(event.tags)); @@ -244,7 +247,7 @@ const followController: AppController = async (c) => { const targetPubkey = c.req.param('pubkey'); await updateListEvent( - { kinds: [3], authors: [sourcePubkey] }, + { kinds: [3], authors: [sourcePubkey], limit: 1 }, (tags) => addTag(tags, ['p', targetPubkey]), c, ); @@ -261,7 +264,7 @@ const unfollowController: AppController = async (c) => { const targetPubkey = c.req.param('pubkey'); await updateListEvent( - { kinds: [3], authors: [sourcePubkey] }, + { kinds: [3], authors: [sourcePubkey], limit: 1 }, (tags) => deleteTag(tags, ['p', targetPubkey]), c, ); @@ -298,7 +301,7 @@ const muteController: AppController = async (c) => { const targetPubkey = c.req.param('pubkey'); await updateListEvent( - { kinds: [10000], authors: [sourcePubkey] }, + { kinds: [10000], authors: [sourcePubkey], limit: 1 }, (tags) => addTag(tags, ['p', targetPubkey]), c, ); @@ -313,7 +316,7 @@ const unmuteController: AppController = async (c) => { const targetPubkey = c.req.param('pubkey'); await updateListEvent( - { kinds: [10000], authors: [sourcePubkey] }, + { kinds: [10000], authors: [sourcePubkey], limit: 1 }, (tags) => deleteTag(tags, ['p', targetPubkey]), c, ); @@ -327,7 +330,9 @@ const favouritesController: AppController = async (c) => { const params = paginationSchema.parse(c.req.query()); const { signal } = c.req.raw; - const events7 = await Storages.db.query( + const store = await Storages.db(); + + const events7 = await store.query( [{ kinds: [7], authors: [pubkey], ...params }], { signal }, ); @@ -336,8 +341,8 @@ const favouritesController: AppController = async (c) => { .map((event) => event.tags.find((tag) => tag[0] === 'e')?.[1]) .filter((id): id is string => !!id); - const events1 = await Storages.db.query([{ kinds: [1], ids }], { signal }) - .then((events) => hydrateEvents({ events, storage: Storages.db, signal })); + const events1 = await store.query([{ kinds: [1], ids }], { signal }) + .then((events) => hydrateEvents({ events, store, signal })); const viewerPubkey = await c.get('signer')?.getPublicKey(); diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index 99a8e5b..b9464a3 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -39,12 +39,13 @@ const adminAccountsController: AppController = async (c) => { return c.json([]); } + const store = await Storages.db(); const { since, until, limit } = paginationSchema.parse(c.req.query()); const { signal } = c.req.raw; - const events = await Storages.db.query([{ kinds: [30361], authors: [Conf.pubkey], since, until, limit }], { signal }); + const events = await store.query([{ kinds: [30361], authors: [Conf.pubkey], since, until, limit }], { signal }); const pubkeys = events.map((event) => event.tags.find(([name]) => name === 'd')?.[1]!); - const authors = await Storages.db.query([{ kinds: [0], authors: pubkeys }], { signal }); + const authors = await store.query([{ kinds: [0], authors: pubkeys }], { signal }); for (const event of events) { const d = event.tags.find(([name]) => name === 'd')?.[1]; @@ -78,7 +79,7 @@ const adminAccountAction: AppController = async (c) => { } await updateListAdminEvent( - { kinds: [10000], authors: [Conf.pubkey] }, + { kinds: [10000], authors: [Conf.pubkey], limit: 1 }, (tags) => addTag(tags, ['p', authorId]), c, ); diff --git a/src/controllers/api/bookmarks.ts b/src/controllers/api/bookmarks.ts index 1616fa2..7655182 100644 --- a/src/controllers/api/bookmarks.ts +++ b/src/controllers/api/bookmarks.ts @@ -5,10 +5,11 @@ import { renderStatuses } from '@/views.ts'; /** https://docs.joinmastodon.org/methods/bookmarks/#get */ const bookmarksController: AppController = async (c) => { + const store = await Storages.db(); const pubkey = await c.get('signer')?.getPublicKey()!; const { signal } = c.req.raw; - const [event10003] = await Storages.db.query( + const [event10003] = await store.query( [{ kinds: [10003], authors: [pubkey], limit: 1 }], { signal }, ); diff --git a/src/controllers/api/ditto.ts b/src/controllers/api/ditto.ts index f0f7036..df4f210 100644 --- a/src/controllers/api/ditto.ts +++ b/src/controllers/api/ditto.ts @@ -16,7 +16,9 @@ const relaySchema = z.object({ type RelayEntity = z.infer; export const adminRelaysController: AppController = async (c) => { - const [event] = await Storages.db.query([ + const store = await Storages.db(); + + const [event] = await store.query([ { kinds: [10002], authors: [Conf.pubkey], limit: 1 }, ]); @@ -28,6 +30,7 @@ export const adminRelaysController: AppController = async (c) => { }; export const adminSetRelaysController: AppController = async (c) => { + const store = await Storages.db(); const relays = relaySchema.array().parse(await c.req.json()); const event = await new AdminSigner().signEvent({ @@ -37,7 +40,7 @@ export const adminSetRelaysController: AppController = async (c) => { created_at: Math.floor(Date.now() / 1000), }); - await Storages.db.event(event); + await store.event(event); return c.json(renderRelays(event)); }; diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index cc71b1f..5f949b0 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -1,10 +1,11 @@ import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; +import { Storages } from '@/storages.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; const instanceController: AppController = async (c) => { const { host, protocol } = Conf.url; - const meta = await getInstanceMetadata(c.req.raw.signal); + const meta = await getInstanceMetadata(await Storages.db(), c.req.raw.signal); /** Protocol to use for WebSocket URLs, depending on the protocol of the `LOCAL_DOMAIN`. */ const wsProtocol = protocol === 'http:' ? 'ws:' : 'wss:'; diff --git a/src/controllers/api/mutes.ts b/src/controllers/api/mutes.ts index fe048e9..4afb6c4 100644 --- a/src/controllers/api/mutes.ts +++ b/src/controllers/api/mutes.ts @@ -5,10 +5,11 @@ import { renderAccounts } from '@/views.ts'; /** https://docs.joinmastodon.org/methods/mutes/#get */ const mutesController: AppController = async (c) => { + const store = await Storages.db(); const pubkey = await c.get('signer')?.getPublicKey()!; const { signal } = c.req.raw; - const [event10000] = await Storages.db.query( + const [event10000] = await store.query( [{ kinds: [10000], authors: [pubkey], limit: 1 }], { signal }, ); diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index 857f2a3..ba15bd0 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -20,7 +20,7 @@ async function renderNotifications(c: AppContext, filters: NostrFilter[]) { const events = await store .query(filters, { signal }) .then((events) => events.filter((event) => event.pubkey !== pubkey)) - .then((events) => hydrateEvents({ events, storage: store, signal })); + .then((events) => hydrateEvents({ events, store, signal })); if (!events.length) { return c.json([]); diff --git a/src/controllers/api/pleroma.ts b/src/controllers/api/pleroma.ts index 4b693c4..3bbdd70 100644 --- a/src/controllers/api/pleroma.ts +++ b/src/controllers/api/pleroma.ts @@ -1,4 +1,4 @@ -import { NSchema as n } from '@nostrify/nostrify'; +import { NSchema as n, NStore } from '@nostrify/nostrify'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; @@ -9,7 +9,8 @@ import { Storages } from '@/storages.ts'; import { createAdminEvent } from '@/utils/api.ts'; const frontendConfigController: AppController = async (c) => { - const configs = await getConfigs(c.req.raw.signal); + const store = await Storages.db(); + const configs = await getConfigs(store, c.req.raw.signal); const frontendConfig = configs.find(({ group, key }) => group === ':pleroma' && key === ':frontend_configurations'); if (frontendConfig) { @@ -25,7 +26,8 @@ const frontendConfigController: AppController = async (c) => { }; const configController: AppController = async (c) => { - const configs = await getConfigs(c.req.raw.signal); + const store = await Storages.db(); + const configs = await getConfigs(store, c.req.raw.signal); return c.json({ configs, need_reboot: false }); }; @@ -33,7 +35,8 @@ const configController: AppController = async (c) => { const updateConfigController: AppController = async (c) => { const { pubkey } = Conf; - const configs = await getConfigs(c.req.raw.signal); + const store = await Storages.db(); + const configs = await getConfigs(store, c.req.raw.signal); const { configs: newConfigs } = z.object({ configs: z.array(configSchema) }).parse(await c.req.json()); for (const { group, key, value } of newConfigs) { @@ -63,10 +66,10 @@ const pleromaAdminDeleteStatusController: AppController = async (c) => { return c.json({}); }; -async function getConfigs(signal: AbortSignal): Promise { +async function getConfigs(store: NStore, signal: AbortSignal): Promise { const { pubkey } = Conf; - const [event] = await Storages.db.query([{ + const [event] = await store.query([{ kinds: [30078], authors: [pubkey], '#d': ['pub.ditto.pleroma.config'], diff --git a/src/controllers/api/reports.ts b/src/controllers/api/reports.ts index 55fb601..9cb2627 100644 --- a/src/controllers/api/reports.ts +++ b/src/controllers/api/reports.ts @@ -48,7 +48,7 @@ const reportController: AppController = async (c) => { tags, }, c); - await hydrateEvents({ events: [event], storage: store }); + await hydrateEvents({ events: [event], store }); return c.json(await renderReport(event)); }; @@ -58,7 +58,7 @@ const adminReportsController: AppController = async (c) => { const viewerPubkey = await c.get('signer')?.getPublicKey(); const reports = await store.query([{ kinds: [1984], '#P': [Conf.pubkey] }]) - .then((events) => hydrateEvents({ storage: store, events: events, signal: c.req.raw.signal })) + .then((events) => hydrateEvents({ store, events: events, signal: c.req.raw.signal })) .then((events) => Promise.all( events.map((event) => renderAdminReport(event, { viewerPubkey })), @@ -85,7 +85,7 @@ const adminReportController: AppController = async (c) => { return c.json({ error: 'This action is not allowed' }, 403); } - await hydrateEvents({ events: [event], storage: store, signal }); + await hydrateEvents({ events: [event], store, signal }); return c.json(await renderAdminReport(event, { viewerPubkey: pubkey })); }; @@ -107,7 +107,7 @@ const adminReportResolveController: AppController = async (c) => { return c.json({ error: 'This action is not allowed' }, 403); } - await hydrateEvents({ events: [event], storage: store, signal }); + await hydrateEvents({ events: [event], store, signal }); await createAdminEvent({ kind: 5, diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index fe08ace..0151f7d 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -78,7 +78,7 @@ const searchController: AppController = async (c) => { }; /** Get events for the search params. */ -function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: AbortSignal): Promise { +async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: AbortSignal): Promise { if (type === 'hashtags') return Promise.resolve([]); const filter: NostrFilter = { @@ -91,8 +91,10 @@ function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: Abort filter.authors = [account_id]; } - return Storages.search.query([filter], { signal }) - .then((events) => hydrateEvents({ events, storage: Storages.search, signal })); + const store = await Storages.search(); + + return store.query([filter], { signal }) + .then((events) => hydrateEvents({ events, store, signal })); } /** Get event kinds to search from `type` query param. */ @@ -110,9 +112,10 @@ function typeToKinds(type: SearchQuery['type']): number[] { /** Resolve a searched value into an event, if applicable. */ async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise { const filters = await getLookupFilters(query, signal); + const store = await Storages.search(); - return Storages.search.query(filters, { limit: 1, signal }) - .then((events) => hydrateEvents({ events, storage: Storages.search, signal })) + return store.query(filters, { limit: 1, signal }) + .then((events) => hydrateEvents({ events, store, signal })) .then(([event]) => event); } diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 1138c0a..a52a408 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -140,7 +140,7 @@ const createStatusController: AppController = async (c) => { if (data.quote_id) { await hydrateEvents({ events: [event], - storage: Storages.db, + store: await Storages.db(), signal: c.req.raw.signal, }); } @@ -248,7 +248,7 @@ const reblogStatusController: AppController = async (c) => { await hydrateEvents({ events: [reblogEvent], - storage: Storages.db, + store: await Storages.db(), signal: signal, }); @@ -260,23 +260,30 @@ const reblogStatusController: AppController = async (c) => { /** https://docs.joinmastodon.org/methods/statuses/#unreblog */ const unreblogStatusController: AppController = async (c) => { const eventId = c.req.param('id'); - const pubkey = await c.get('signer')?.getPublicKey() as string; + const pubkey = await c.get('signer')?.getPublicKey()!; - const event = await getEvent(eventId, { - kind: 1, - }); - if (!event) return c.json({ error: 'Event not found.' }, 404); + const event = await getEvent(eventId, { kind: 1 }); - const filters: NostrFilter[] = [{ kinds: [6], authors: [pubkey], '#e': [event.id] }]; - const [repostedEvent] = await Storages.db.query(filters, { limit: 1 }); - if (!repostedEvent) return c.json({ error: 'Event not found.' }, 404); + if (!event) { + return c.json({ error: 'Event not found.' }, 404); + } + + const store = await Storages.db(); + + const [repostedEvent] = await store.query( + [{ kinds: [6], authors: [pubkey], '#e': [event.id], limit: 1 }], + ); + + if (!repostedEvent) { + return c.json({ error: 'Event not found.' }, 404); + } await createEvent({ kind: 5, tags: [['e', repostedEvent.id]], }, c); - return c.json(await renderStatus(event, {})); + return c.json(await renderStatus(event, { viewerPubkey: pubkey })); }; const rebloggedByController: AppController = (c) => { @@ -297,7 +304,7 @@ const bookmarkController: AppController = async (c) => { if (event) { await updateListEvent( - { kinds: [10003], authors: [pubkey] }, + { kinds: [10003], authors: [pubkey], limit: 1 }, (tags) => addTag(tags, ['e', eventId]), c, ); @@ -324,7 +331,7 @@ const unbookmarkController: AppController = async (c) => { if (event) { await updateListEvent( - { kinds: [10003], authors: [pubkey] }, + { kinds: [10003], authors: [pubkey], limit: 1 }, (tags) => deleteTag(tags, ['e', eventId]), c, ); @@ -351,7 +358,7 @@ const pinController: AppController = async (c) => { if (event) { await updateListEvent( - { kinds: [10001], authors: [pubkey] }, + { kinds: [10001], authors: [pubkey], limit: 1 }, (tags) => addTag(tags, ['e', eventId]), c, ); @@ -380,7 +387,7 @@ const unpinController: AppController = async (c) => { if (event) { await updateListEvent( - { kinds: [10001], authors: [pubkey] }, + { kinds: [10001], authors: [pubkey], limit: 1 }, (tags) => deleteTag(tags, ['e', eventId]), c, ); diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 8d22d5c..e79c51e 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -68,13 +68,15 @@ const streamingController: AppController = (c) => { if (!filter) return; try { - for await (const msg of Storages.pubsub.req([filter], { signal: controller.signal })) { + const store = await Storages.pubsub(); + + for await (const msg of store.req([filter], { signal: controller.signal })) { if (msg[0] === 'EVENT') { const event = msg[2]; await hydrateEvents({ events: [event], - storage: Storages.admin, + store, signal: AbortSignal.timeout(1000), }); diff --git a/src/controllers/api/suggestions.ts b/src/controllers/api/suggestions.ts index bde0916..6377bd4 100644 --- a/src/controllers/api/suggestions.ts +++ b/src/controllers/api/suggestions.ts @@ -40,7 +40,7 @@ async function renderSuggestedAccounts(store: NStore, signal?: AbortSignal) { [{ kinds: [0], authors: pubkeys, limit: pubkeys.length }], { signal }, ) - .then((events) => hydrateEvents({ events, storage: store, signal })); + .then((events) => hydrateEvents({ events, store, signal })); const accounts = await Promise.all(pubkeys.map((pubkey) => { const profile = profiles.find((event) => event.pubkey === pubkey); diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index 0880d84..e83c50c 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -49,13 +49,7 @@ async function renderStatuses(c: AppContext, filters: NostrFilter[]) { const events = await store .query(filters, { signal }) - .then((events) => - hydrateEvents({ - events, - storage: store, - signal, - }) - ); + .then((events) => hydrateEvents({ events, store, signal })); if (!events.length) { return c.json([]); diff --git a/src/controllers/nostr/relay-info.ts b/src/controllers/nostr/relay-info.ts index 192cab2..bbce7d3 100644 --- a/src/controllers/nostr/relay-info.ts +++ b/src/controllers/nostr/relay-info.ts @@ -1,9 +1,11 @@ import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; +import { Storages } from '@/storages.ts'; import { getInstanceMetadata } from '@/utils/instance.ts'; const relayInfoController: AppController = async (c) => { - const meta = await getInstanceMetadata(c.req.raw.signal); + const store = await Storages.db(); + const meta = await getInstanceMetadata(store, c.req.raw.signal); return c.json({ name: meta.name, diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index c0fa026..259f5e9 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -72,14 +72,17 @@ function connectStream(socket: WebSocket) { controllers.get(subId)?.abort(); controllers.set(subId, controller); - for (const event of await Storages.db.query(filters, { limit: FILTER_LIMIT })) { + const db = await Storages.db(); + const pubsub = await Storages.pubsub(); + + for (const event of await db.query(filters, { limit: FILTER_LIMIT })) { send(['EVENT', subId, event]); } send(['EOSE', subId]); try { - for await (const msg of Storages.pubsub.req(filters, { signal: controller.signal })) { + for await (const msg of pubsub.req(filters, { signal: controller.signal })) { if (msg[0] === 'EVENT') { send(['EVENT', subId, msg[2]]); } @@ -116,7 +119,8 @@ function connectStream(socket: WebSocket) { /** Handle COUNT. Return the number of events matching the filters. */ async function handleCount([_, subId, ...rest]: NostrClientCOUNT): Promise { - const { count } = await Storages.db.count(prepareFilters(rest)); + const store = await Storages.db(); + const { count } = await store.count(prepareFilters(rest)); send(['COUNT', subId, { count, approximate: false }]); } diff --git a/src/controllers/well-known/nostr.ts b/src/controllers/well-known/nostr.ts index f1ebb6b..0669888 100644 --- a/src/controllers/well-known/nostr.ts +++ b/src/controllers/well-known/nostr.ts @@ -12,7 +12,7 @@ const nameSchema = z.string().min(1).regex(/^\w+$/); const nostrController: AppController = async (c) => { const result = nameSchema.safeParse(c.req.query('name')); const name = result.success ? result.data : undefined; - const pointer = name ? await localNip05Lookup(name) : undefined; + const pointer = name ? await localNip05Lookup(c.get('store'), name) : undefined; if (!name || !pointer) { return c.json({ names: {}, relays: {} }); diff --git a/src/controllers/well-known/webfinger.ts b/src/controllers/well-known/webfinger.ts index 38bc994..c1c8b81 100644 --- a/src/controllers/well-known/webfinger.ts +++ b/src/controllers/well-known/webfinger.ts @@ -45,7 +45,7 @@ async function handleAcct(c: AppContext, resource: URL): Promise { } const [username, host] = result.data; - const pointer = await localNip05Lookup(username); + const pointer = await localNip05Lookup(c.get('store'), username); if (!pointer) { return c.json({ error: 'Not found' }, 404); diff --git a/src/db/users.ts b/src/db/users.ts index c7659e4..bf0cab7 100644 --- a/src/db/users.ts +++ b/src/db/users.ts @@ -60,7 +60,8 @@ async function findUser(user: Partial, signal?: AbortSignal): Promise { const pubkey = await c.get('signer')?.getPublicKey(); if (pubkey) { - const store = new UserStore(pubkey, Storages.admin); + const store = new UserStore(pubkey, await Storages.admin()); c.set('store', store); } else { - c.set('store', Storages.admin); + c.set('store', await Storages.admin()); } await next(); }; diff --git a/src/pipeline.ts b/src/pipeline.ts index 25cadb5..9742b49 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -57,7 +57,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { const policies: NPolicy[] = [ - new MuteListPolicy(Conf.pubkey, Storages.admin), + new MuteListPolicy(Conf.pubkey, await Storages.admin()), ]; try { @@ -76,15 +76,20 @@ async function policyFilter(event: NostrEvent): Promise { /** Encounter the event, and return whether it has already been encountered. */ async function encounterEvent(event: NostrEvent, signal: AbortSignal): Promise { - const [existing] = await Storages.cache.query([{ ids: [event.id], limit: 1 }]); - Storages.cache.event(event); - Storages.reqmeister.event(event, { signal }); + const cache = await Storages.cache(); + const reqmeister = await Storages.reqmeister(); + + const [existing] = await cache.query([{ ids: [event.id], limit: 1 }]); + + cache.event(event); + reqmeister.event(event, { signal }); + return !!existing; } /** Hydrate the event with the user, if applicable. */ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { - await hydrateEvents({ events: [event], storage: Storages.db, signal }); + await hydrateEvents({ events: [event], store: await Storages.db(), signal }); const domain = await db .selectFrom('pubkey_domains') @@ -98,8 +103,9 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { if (isEphemeralKind(event.kind)) return; + const store = await Storages.db(); - const [deletion] = await Storages.db.query( + const [deletion] = await store.query( [{ kinds: [5], authors: [Conf.pubkey, event.pubkey], '#e': [event.id], limit: 1 }], { signal }, ); @@ -108,7 +114,7 @@ async function storeEvent(event: DittoEvent, signal?: AbortSignal): Promise { if (event.kind === 5) { const ids = getTagSet(event.tags, 'e'); + const store = await Storages.db(); if (event.pubkey === Conf.pubkey) { - await Storages.db.remove([{ ids: [...ids] }], { signal }); + await store.remove([{ ids: [...ids] }], { signal }); } else { - const events = await Storages.db.query( + const events = await store.query( [{ ids: [...ids], authors: [event.pubkey] }], { signal }, ); const deleteIds = events.map(({ id }) => id); - await Storages.db.remove([{ ids: deleteIds }], { signal }); + await store.remove([{ ids: deleteIds }], { signal }); } } } @@ -189,19 +196,22 @@ async function trackHashtags(event: NostrEvent): Promise { /** Queue related events to fetch. */ async function fetchRelatedEvents(event: DittoEvent) { + const cache = await Storages.cache(); + const reqmeister = await Storages.reqmeister(); + if (!event.author) { const signal = AbortSignal.timeout(3000); - Storages.reqmeister.query([{ kinds: [0], authors: [event.pubkey] }], { signal }) + reqmeister.query([{ kinds: [0], authors: [event.pubkey] }], { signal }) .then((events) => Promise.allSettled(events.map((event) => handleEvent(event, signal)))) .catch(() => {}); } for (const [name, id] of event.tags) { if (name === 'e') { - const { count } = await Storages.cache.count([{ ids: [id] }]); + const { count } = await cache.count([{ ids: [id] }]); if (!count) { const signal = AbortSignal.timeout(3000); - Storages.reqmeister.query([{ ids: [id] }], { signal }) + reqmeister.query([{ ids: [id] }], { signal }) .then((events) => Promise.allSettled(events.map((event) => handleEvent(event, signal)))) .catch(() => {}); } @@ -272,7 +282,8 @@ function isFresh(event: NostrEvent): boolean { /** Distribute the event through active subscriptions. */ async function streamOut(event: NostrEvent): Promise { if (isFresh(event)) { - await Storages.pubsub.event(event); + const pubsub = await Storages.pubsub(); + await pubsub.event(event); } } diff --git a/src/pipeline/DVM.ts b/src/pipeline/DVM.ts index 953e9be..a811067 100644 --- a/src/pipeline/DVM.ts +++ b/src/pipeline/DVM.ts @@ -34,7 +34,9 @@ export class DVM { return DVM.feedback(event, 'error', `Forbidden user: ${user}`); } - const [label] = await Storages.db.query([{ + const store = await Storages.db(); + + const [label] = await store.query([{ kinds: [1985], authors: [admin], '#L': ['nip05'], diff --git a/src/queries.ts b/src/queries.ts index 5626c45..76fabfd 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -25,6 +25,7 @@ const getEvent = async ( opts: GetEventOpts = {}, ): Promise => { debug(`getEvent: ${id}`); + const store = await Storages.optimizer(); const { kind, signal = AbortSignal.timeout(1000) } = opts; const filter: NostrFilter = { ids: [id], limit: 1 }; @@ -32,23 +33,25 @@ const getEvent = async ( filter.kinds = [kind]; } - return await Storages.optimizer.query([filter], { limit: 1, signal }) - .then((events) => hydrateEvents({ events, storage: Storages.optimizer, signal })) + return await store.query([filter], { limit: 1, signal }) + .then((events) => hydrateEvents({ events, store, signal })) .then(([event]) => event); }; /** Get a Nostr `set_medatadata` event for a user's pubkey. */ const getAuthor = async (pubkey: string, opts: GetEventOpts = {}): Promise => { + const store = await Storages.optimizer(); const { signal = AbortSignal.timeout(1000) } = opts; - return await Storages.optimizer.query([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, signal }) - .then((events) => hydrateEvents({ events, storage: Storages.optimizer, signal })) + return await store.query([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, signal }) + .then((events) => hydrateEvents({ events, store, signal })) .then(([event]) => event); }; /** Get users the given pubkey follows. */ const getFollows = async (pubkey: string, signal?: AbortSignal): Promise => { - const [event] = await Storages.db.query([{ authors: [pubkey], kinds: [3], limit: 1 }], { limit: 1, signal }); + const store = await Storages.db(); + const [event] = await store.query([{ authors: [pubkey], kinds: [3], limit: 1 }], { limit: 1, signal }); return event; }; @@ -84,15 +87,18 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi } async function getDescendants(eventId: string, signal = AbortSignal.timeout(2000)): Promise { - const events = await Storages.db.query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal }); - return hydrateEvents({ events, storage: Storages.db, signal }); + const store = await Storages.db(); + const events = await store.query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal }); + return hydrateEvents({ events, store, signal }); } /** Returns whether the pubkey is followed by a local user. */ async function isLocallyFollowed(pubkey: string): Promise { const { host } = Conf.url; - const [event] = await Storages.db.query( + const store = await Storages.db(); + + const [event] = await store.query( [{ kinds: [3], '#p': [pubkey], search: `domain:${host}`, limit: 1 }], { limit: 1 }, ); diff --git a/src/signers/ConnectSigner.ts b/src/signers/ConnectSigner.ts index 4dda0b1..f482413 100644 --- a/src/signers/ConnectSigner.ts +++ b/src/signers/ConnectSigner.ts @@ -1,5 +1,5 @@ // deno-lint-ignore-file require-await -import { NConnectSigner } from '@nostrify/nostrify'; +import { NConnectSigner, NostrEvent, NostrSigner } from '@nostrify/nostrify'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; @@ -9,24 +9,55 @@ import { Storages } from '@/storages.ts'; * * Simple extension of nostrify's `NConnectSigner`, with our options to keep it DRY. */ -export class ConnectSigner extends NConnectSigner { - private _pubkey: string; +export class ConnectSigner implements NostrSigner { + private signer: Promise; - constructor(pubkey: string, private relays?: string[]) { - super({ - pubkey, + constructor(private pubkey: string, private relays?: string[]) { + this.signer = this.init(); + } + + async init(): Promise { + return new NConnectSigner({ + pubkey: this.pubkey, // TODO: use a remote relay for `nprofile` signing (if present and `Conf.relay` isn't already in the list) - relay: Storages.pubsub, + relay: await Storages.pubsub(), signer: new AdminSigner(), timeout: 60000, }); - - this._pubkey = pubkey; } + async signEvent(event: Omit): Promise { + const signer = await this.signer; + return signer.signEvent(event); + } + + readonly nip04 = { + encrypt: async (pubkey: string, plaintext: string): Promise => { + const signer = await this.signer; + return signer.nip04.encrypt(pubkey, plaintext); + }, + + decrypt: async (pubkey: string, ciphertext: string): Promise => { + const signer = await this.signer; + return signer.nip04.decrypt(pubkey, ciphertext); + }, + }; + + readonly nip44 = { + encrypt: async (pubkey: string, plaintext: string): Promise => { + const signer = await this.signer; + return signer.nip44.encrypt(pubkey, plaintext); + }, + + decrypt: async (pubkey: string, ciphertext: string): Promise => { + const signer = await this.signer; + return signer.nip44.decrypt(pubkey, ciphertext); + }, + }; + // Prevent unnecessary NIP-46 round-trips. async getPublicKey(): Promise { - return this._pubkey; + return this.pubkey; } /** Get the user's relays if they passed in an `nprofile` auth token. */ diff --git a/src/storages/UserStore.ts b/src/storages/UserStore.ts index 1c7aaee..c5657b6 100644 --- a/src/storages/UserStore.ts +++ b/src/storages/UserStore.ts @@ -4,13 +4,7 @@ import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getTagSet } from '@/tags.ts'; export class UserStore implements NStore { - private store: NStore; - private pubkey: string; - - constructor(pubkey: string, store: NStore) { - this.pubkey = pubkey; - this.store = store; - } + constructor(private pubkey: string, private store: NStore) {} async event(event: NostrEvent, opts?: { signal?: AbortSignal }): Promise { return await this.store.event(event, opts); @@ -21,12 +15,11 @@ export class UserStore implements NStore { * https://github.com/nostr-protocol/nips/blob/master/51.md#standard-lists */ async query(filters: NostrFilter[], opts: { signal?: AbortSignal; limit?: number } = {}): Promise { - const allEvents = await this.store.query(filters, opts); + const events = await this.store.query(filters, opts); + const pubkeys = await this.getMutedPubkeys(); - const mutedPubkeys = await this.getMutedPubkeys(); - - return allEvents.filter((event) => { - return event.kind === 0 || mutedPubkeys.has(event.pubkey) === false; + return events.filter((event) => { + return event.kind === 0 || !pubkeys.has(event.pubkey); }); } diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index b55cd2b..f5c70af 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -17,7 +17,7 @@ Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { await hydrateEvents({ events: [event1], - storage: db, + store: db, }); const expectedEvent = { ...event1, author: event0 }; @@ -40,7 +40,7 @@ Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { await hydrateEvents({ events: [event6], - storage: db, + store: db, }); const expectedEvent6 = { @@ -67,7 +67,7 @@ Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { await hydrateEvents({ events: [event1quoteRepost], - storage: db, + store: db, }); const expectedEvent1quoteRepost = { @@ -95,7 +95,7 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () await hydrateEvents({ events: [event6], - storage: db, + store: db, }); const expectedEvent6 = { @@ -122,7 +122,7 @@ Deno.test('hydrateEvents(): report pubkey and post // kind 1984 --- WITHOUT stat await hydrateEvents({ events: [reportEvent], - storage: db, + store: db, }); const expectedEvent: DittoEvent = { diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index e197ca8..3109ac6 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -8,13 +8,13 @@ import { Conf } from '@/config.ts'; interface HydrateOpts { events: DittoEvent[]; - storage: NStore; + store: NStore; signal?: AbortSignal; } /** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateOpts): Promise { - const { events, storage, signal } = opts; + const { events, store, signal } = opts; if (!events.length) { return events; @@ -22,31 +22,31 @@ async function hydrateEvents(opts: HydrateOpts): Promise { const cache = [...events]; - for (const event of await gatherReposts({ events: cache, storage, signal })) { + for (const event of await gatherReposts({ events: cache, store, signal })) { cache.push(event); } - for (const event of await gatherReacted({ events: cache, storage, signal })) { + for (const event of await gatherReacted({ events: cache, store, signal })) { cache.push(event); } - for (const event of await gatherQuotes({ events: cache, storage, signal })) { + for (const event of await gatherQuotes({ events: cache, store, signal })) { cache.push(event); } - for (const event of await gatherAuthors({ events: cache, storage, signal })) { + for (const event of await gatherAuthors({ events: cache, store, signal })) { cache.push(event); } - for (const event of await gatherUsers({ events: cache, storage, signal })) { + for (const event of await gatherUsers({ events: cache, store, signal })) { cache.push(event); } - for (const event of await gatherReportedProfiles({ events: cache, storage, signal })) { + for (const event of await gatherReportedProfiles({ events: cache, store, signal })) { cache.push(event); } - for (const event of await gatherReportedNotes({ events: cache, storage, signal })) { + for (const event of await gatherReportedNotes({ events: cache, store, signal })) { cache.push(event); } @@ -123,7 +123,7 @@ function assembleEvents( } /** Collect reposts from the events. */ -function gatherReposts({ events, storage, signal }: HydrateOpts): Promise { +function gatherReposts({ events, store, signal }: HydrateOpts): Promise { const ids = new Set(); for (const event of events) { @@ -135,14 +135,14 @@ function gatherReposts({ events, storage, signal }: HydrateOpts): Promise { +function gatherReacted({ events, store, signal }: HydrateOpts): Promise { const ids = new Set(); for (const event of events) { @@ -154,14 +154,14 @@ function gatherReacted({ events, storage, signal }: HydrateOpts): Promise { +function gatherQuotes({ events, store, signal }: HydrateOpts): Promise { const ids = new Set(); for (const event of events) { @@ -173,34 +173,34 @@ function gatherQuotes({ events, storage, signal }: HydrateOpts): Promise { +function gatherAuthors({ events, store, signal }: HydrateOpts): Promise { const pubkeys = new Set(events.map((event) => event.pubkey)); - return storage.query( + return store.query( [{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }], { signal }, ); } /** Collect users from the events. */ -function gatherUsers({ events, storage, signal }: HydrateOpts): Promise { +function gatherUsers({ events, store, signal }: HydrateOpts): Promise { const pubkeys = new Set(events.map((event) => event.pubkey)); - return storage.query( + return store.query( [{ kinds: [30361], authors: [Conf.pubkey], '#d': [...pubkeys], limit: pubkeys.size }], { signal }, ); } /** Collect reported notes from the events. */ -function gatherReportedNotes({ events, storage, signal }: HydrateOpts): Promise { +function gatherReportedNotes({ events, store, signal }: HydrateOpts): Promise { const ids = new Set(); for (const event of events) { if (event.kind === 1984) { @@ -213,14 +213,14 @@ function gatherReportedNotes({ events, storage, signal }: HydrateOpts): Promise< } } - return storage.query( + return store.query( [{ kinds: [1], ids: [...ids], limit: ids.size }], { signal }, ); } /** Collect reported profiles from the events. */ -function gatherReportedProfiles({ events, storage, signal }: HydrateOpts): Promise { +function gatherReportedProfiles({ events, store, signal }: HydrateOpts): Promise { const pubkeys = new Set(); for (const event of events) { @@ -232,7 +232,7 @@ function gatherReportedProfiles({ events, storage, signal }: HydrateOpts): Promi } } - return storage.query( + return store.query( [{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }], { signal }, ); diff --git a/src/storages/pool-store.ts b/src/storages/pool-store.ts index 9f45205..5456509 100644 --- a/src/storages/pool-store.ts +++ b/src/storages/pool-store.ts @@ -13,6 +13,7 @@ import { RelayPoolWorker } from 'nostr-relaypool'; import { getFilterLimit, matchFilters } from 'nostr-tools'; import { Conf } from '@/config.ts'; +import { Storages } from '@/storages.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; import { abortError } from '@/utils/abort.ts'; import { getRelays } from '@/utils/outbox.ts'; @@ -35,7 +36,7 @@ class PoolStore implements NRelay { async event(event: NostrEvent, opts: { signal?: AbortSignal } = {}): Promise { if (opts.signal?.aborted) return Promise.reject(abortError()); - const relaySet = await getRelays(event.pubkey); + const relaySet = await getRelays(await Storages.db(), event.pubkey); relaySet.delete(Conf.relay); const relays = [...relaySet].slice(0, 4); diff --git a/src/storages/search-store.ts b/src/storages/search-store.ts index be6e2b4..4951c72 100644 --- a/src/storages/search-store.ts +++ b/src/storages/search-store.ts @@ -48,7 +48,7 @@ class SearchStore implements NStore { return hydrateEvents({ events, - storage: this.#hydrator, + store: this.#hydrator, signal: opts?.signal, }); } else { diff --git a/src/utils/api.ts b/src/utils/api.ts index 81b157c..dceede7 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -42,7 +42,7 @@ async function createEvent(t: EventStub, c: AppContext): Promise { /** Filter for fetching an existing event to update. */ interface UpdateEventFilter extends NostrFilter { kinds: [number]; - limit?: 1; + limit: 1; } /** Fetch existing event, update it, then publish the new event. */ @@ -51,7 +51,8 @@ async function updateEvent( fn: (prev: NostrEvent | undefined) => E, c: AppContext, ): Promise { - const [prev] = await Storages.db.query([filter], { limit: 1, signal: c.req.raw.signal }); + const store = await Storages.db(); + const [prev] = await store.query([filter], { signal: c.req.raw.signal }); return createEvent(fn(prev), c); } @@ -101,7 +102,8 @@ async function updateAdminEvent( fn: (prev: NostrEvent | undefined) => E, c: AppContext, ): Promise { - const [prev] = await Storages.db.query([filter], { limit: 1, signal: c.req.raw.signal }); + const store = await Storages.db(); + const [prev] = await store.query([filter], { limit: 1, signal: c.req.raw.signal }); return createAdminEvent(fn(prev), c); } @@ -110,7 +112,8 @@ async function publishEvent(event: NostrEvent, c: AppContext): Promise { const uri = new URL('nostrconnect://'); - const { name, tagline } = await getInstanceMetadata(signal); + const { name, tagline } = await getInstanceMetadata(await Storages.db(), signal); const metadata: ConnectMetadata = { name, diff --git a/src/utils/instance.ts b/src/utils/instance.ts index 004e4cf..386c796 100644 --- a/src/utils/instance.ts +++ b/src/utils/instance.ts @@ -1,8 +1,7 @@ -import { NostrEvent, NostrMetadata, NSchema as n } from '@nostrify/nostrify'; +import { NostrEvent, NostrMetadata, NSchema as n, NStore } from '@nostrify/nostrify'; import { Conf } from '@/config.ts'; import { serverMetaSchema } from '@/schemas/nostr.ts'; -import { Storages } from '@/storages.ts'; /** Like NostrMetadata, but some fields are required and also contains some extra fields. */ export interface InstanceMetadata extends NostrMetadata { @@ -14,8 +13,8 @@ export interface InstanceMetadata extends NostrMetadata { } /** Get and parse instance metadata from the kind 0 of the admin user. */ -export async function getInstanceMetadata(signal?: AbortSignal): Promise { - const [event] = await Storages.db.query( +export async function getInstanceMetadata(store: NStore, signal?: AbortSignal): Promise { + const [event] = await store.query( [{ kinds: [0], authors: [Conf.pubkey], limit: 1 }], { signal }, ); diff --git a/src/utils/nip05.ts b/src/utils/nip05.ts index 0b4c6e3..eaab6ed 100644 --- a/src/utils/nip05.ts +++ b/src/utils/nip05.ts @@ -1,4 +1,4 @@ -import { NIP05 } from '@nostrify/nostrify'; +import { NIP05, NStore } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { nip19 } from 'nostr-tools'; @@ -16,7 +16,8 @@ const nip05Cache = new SimpleLRU( const [name, domain] = key.split('@'); try { if (domain === Conf.url.host) { - const pointer = await localNip05Lookup(name); + const store = await Storages.db(); + const pointer = await localNip05Lookup(store, name); if (pointer) { debug(`Found: ${key} is ${pointer.pubkey}`); return pointer; @@ -36,8 +37,8 @@ const nip05Cache = new SimpleLRU( { max: 500, ttl: Time.hours(1) }, ); -async function localNip05Lookup(name: string): Promise { - const [label] = await Storages.db.query([{ +async function localNip05Lookup(store: NStore, name: string): Promise { + const [label] = await store.query([{ kinds: [1985], authors: [Conf.pubkey], '#L': ['nip05'], diff --git a/src/utils/outbox.ts b/src/utils/outbox.ts index 13edaf6..72b8338 100644 --- a/src/utils/outbox.ts +++ b/src/utils/outbox.ts @@ -1,10 +1,11 @@ -import { Conf } from '@/config.ts'; -import { Storages } from '@/storages.ts'; +import { NStore } from '@nostrify/nostrify'; -export async function getRelays(pubkey: string): Promise> { +import { Conf } from '@/config.ts'; + +export async function getRelays(store: NStore, pubkey: string): Promise> { const relays = new Set<`wss://${string}`>(); - const events = await Storages.db.query([ + const events = await store.query([ { kinds: [10002], authors: [pubkey, Conf.pubkey], limit: 2 }, ]); diff --git a/src/views.ts b/src/views.ts index 451ce14..a737542 100644 --- a/src/views.ts +++ b/src/views.ts @@ -12,15 +12,16 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal return c.json([]); } - const events = await Storages.db.query(filters, { signal }); + const store = await Storages.db(); + const events = await store.query(filters, { signal }); const pubkeys = new Set(events.map(({ pubkey }) => pubkey)); if (!pubkeys.size) { return c.json([]); } - const authors = await Storages.db.query([{ kinds: [0], authors: [...pubkeys] }], { signal }) - .then((events) => hydrateEvents({ events, storage: Storages.db, signal })); + const authors = await store.query([{ kinds: [0], authors: [...pubkeys] }], { signal }) + .then((events) => hydrateEvents({ events, store, signal })); const accounts = await Promise.all( authors.map((event) => renderAccount(event)), @@ -32,8 +33,10 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal async function renderAccounts(c: AppContext, authors: string[], signal = AbortSignal.timeout(1000)) { const { since, until, limit } = paginationSchema.parse(c.req.query()); - const events = await Storages.db.query([{ kinds: [0], authors, since, until, limit }], { signal }) - .then((events) => hydrateEvents({ events, storage: Storages.db, signal })); + const store = await Storages.db(); + + const events = await store.query([{ kinds: [0], authors, since, until, limit }], { signal }) + .then((events) => hydrateEvents({ events, store, signal })); const accounts = await Promise.all( events.map((event) => renderAccount(event)), @@ -48,10 +51,11 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal return c.json([]); } + const store = await Storages.db(); const { limit } = paginationSchema.parse(c.req.query()); - const events = await Storages.db.query([{ kinds: [1], ids, limit }], { signal }) - .then((events) => hydrateEvents({ events, storage: Storages.db, signal })); + const events = await store.query([{ kinds: [1], ids, limit }], { signal }) + .then((events) => hydrateEvents({ events, store, signal })); if (!events.length) { return c.json([]); diff --git a/src/views/mastodon/relationships.ts b/src/views/mastodon/relationships.ts index d358024..2f8ffdd 100644 --- a/src/views/mastodon/relationships.ts +++ b/src/views/mastodon/relationships.ts @@ -2,7 +2,9 @@ import { Storages } from '@/storages.ts'; import { hasTag } from '@/tags.ts'; async function renderRelationship(sourcePubkey: string, targetPubkey: string) { - const events = await Storages.db.query([ + const db = await Storages.db(); + + const events = await db.query([ { kinds: [3], authors: [sourcePubkey], limit: 1 }, { kinds: [3], authors: [targetPubkey], limit: 1 }, { kinds: [10000], authors: [sourcePubkey], limit: 1 }, diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index d00759d..776f016 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -22,7 +22,7 @@ interface RenderStatusOpts { async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise { const { viewerPubkey, depth = 1 } = opts; - if (depth > 2 || depth < 0) return null; + if (depth > 2 || depth < 0) return; const note = nip19.noteEncode(event.id); @@ -40,7 +40,10 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< ), ]; - const mentionedProfiles = await Storages.optimizer.query( + const db = await Storages.db(); + const optimizer = await Storages.optimizer(); + + const mentionedProfiles = await optimizer.query( [{ kinds: [0], authors: mentionedPubkeys, limit: mentionedPubkeys.length }], ); @@ -53,7 +56,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< ), firstUrl ? unfurlCardCached(firstUrl) : null, viewerPubkey - ? await Storages.db.query([ + ? await db.query([ { kinds: [6], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, { kinds: [7], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, { kinds: [9734], '#e': [event.id], authors: [viewerPubkey], limit: 1 }, From a4226a963f4a398c6db9ee00b43d0b56f2d0b857 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 18:44:42 -0500 Subject: [PATCH 33/39] Rework Kysely db to be async --- deno.json | 1 + src/db.ts | 41 ------------------------- src/db/DittoDB.ts | 55 ++++++++++++++++++++++++++++++++-- src/db/unattached-media.ts | 27 ++++++++++------- src/pipeline.ts | 8 +++-- src/stats.ts | 23 +++++++------- src/storages.ts | 8 +++-- src/storages/events-db.test.ts | 10 ++++--- src/storages/hydrate.ts | 12 ++++---- 9 files changed, 106 insertions(+), 79 deletions(-) delete mode 100644 src/db.ts diff --git a/deno.json b/deno.json index 0225510..12fbb57 100644 --- a/deno.json +++ b/deno.json @@ -24,6 +24,7 @@ "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", + "@std/assert": "jsr:@std/assert@^0.225.1", "@std/cli": "jsr:@std/cli@^0.223.0", "@std/crypto": "jsr:@std/crypto@^0.224.0", "@std/dotenv": "jsr:@std/dotenv@^0.224.0", diff --git a/src/db.ts b/src/db.ts deleted file mode 100644 index 1a5f06d..0000000 --- a/src/db.ts +++ /dev/null @@ -1,41 +0,0 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; - -import { FileMigrationProvider, Migrator } from 'kysely'; - -import { DittoDB } from '@/db/DittoDB.ts'; - -const db = await DittoDB.getInstance(); - -const migrator = new Migrator({ - db, - provider: new FileMigrationProvider({ - fs, - path, - migrationFolder: new URL(import.meta.resolve('./db/migrations')).pathname, - }), -}); - -/** Migrate the database to the latest version. */ -async function migrate() { - console.info('Running migrations...'); - const results = await migrator.migrateToLatest(); - - if (results.error) { - console.error(results.error); - Deno.exit(1); - } else { - if (!results.results?.length) { - console.info('Everything up-to-date.'); - } else { - console.info('Migrations finished!'); - for (const { migrationName, status } of results.results!) { - console.info(` - ${migrationName}: ${status}`); - } - } - } -} - -await migrate(); - -export { db }; diff --git a/src/db/DittoDB.ts b/src/db/DittoDB.ts index abe068b..9c3b280 100644 --- a/src/db/DittoDB.ts +++ b/src/db/DittoDB.ts @@ -1,4 +1,7 @@ -import { Kysely } from 'kysely'; +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; import { Conf } from '@/config.ts'; import { DittoPostgres } from '@/db/adapters/DittoPostgres.ts'; @@ -6,17 +9,63 @@ import { DittoSQLite } from '@/db/adapters/DittoSQLite.ts'; import { DittoTables } from '@/db/DittoTables.ts'; export class DittoDB { + private static kysely: Promise> | undefined; + static getInstance(): Promise> { + if (!this.kysely) { + this.kysely = this._getInstance(); + } + return this.kysely; + } + + static async _getInstance(): Promise> { const { databaseUrl } = Conf; + let kysely: Kysely; + switch (databaseUrl.protocol) { case 'sqlite:': - return DittoSQLite.getInstance(); + kysely = await DittoSQLite.getInstance(); + break; case 'postgres:': case 'postgresql:': - return DittoPostgres.getInstance(); + kysely = await DittoPostgres.getInstance(); + break; default: throw new Error('Unsupported database URL.'); } + + await this.migrate(kysely); + + return kysely; + } + + /** Migrate the database to the latest version. */ + private static async migrate(kysely: Kysely) { + const migrator = new Migrator({ + db: kysely, + provider: new FileMigrationProvider({ + fs, + path, + migrationFolder: new URL(import.meta.resolve('../db/migrations')).pathname, + }), + }); + + console.info('Running migrations...'); + const results = await migrator.migrateToLatest(); + + if (results.error) { + console.error(results.error); + Deno.exit(1); + } else { + if (!results.results?.length) { + console.info('Everything up-to-date.'); + } else { + console.info('Migrations finished!'); + for (const { migrationName, status } of results.results!) { + console.info(` - ${migrationName}: ${status}`); + } + } + } } } diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index 960abe8..708e2f9 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -1,6 +1,6 @@ import uuid62 from 'uuid62'; -import { db } from '@/db.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; import { type MediaData } from '@/schemas/nostr.ts'; interface UnattachedMedia { @@ -19,7 +19,8 @@ async function insertUnattachedMedia(media: Omit { if (!urls.length) return; - await db.deleteFrom('unattached_media') + const kysely = await DittoDB.getInstance(); + await kysely.deleteFrom('unattached_media') .where('pubkey', '=', pubkey) .where('url', 'in', urls) .execute(); diff --git a/src/pipeline.ts b/src/pipeline.ts index 9742b49..48e5c38 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -5,7 +5,7 @@ import Debug from '@soapbox/stickynotes/debug'; import { sql } from 'kysely'; import { Conf } from '@/config.ts'; -import { db } from '@/db.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; import { deleteAttachedMedia } from '@/db/unattached-media.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { isEphemeralKind } from '@/kinds.ts'; @@ -91,7 +91,8 @@ async function encounterEvent(event: NostrEvent, signal: AbortSignal): Promise { await hydrateEvents({ events: [event], store: await Storages.db(), signal }); - const domain = await db + const kysely = await DittoDB.getInstance(); + const domain = await kysely .selectFrom('pubkey_domains') .select('domain') .where('pubkey', '=', event.pubkey) @@ -140,6 +141,7 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise pubkey_domains.last_updated_at - `.execute(db); + `.execute(kysely); } catch (_e) { // do nothing } diff --git a/src/stats.ts b/src/stats.ts index bff2aeb..f8efe16 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -2,7 +2,7 @@ import { NKinds, NostrEvent } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { InsertQueryBuilder } from 'kysely'; -import { db } from '@/db.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Storages } from '@/storages.ts'; import { findReplyTag } from '@/tags.ts'; @@ -25,7 +25,7 @@ async function updateStats(event: NostrEvent) { if (event.kind === 3) { prev = await getPrevEvent(event); if (!prev || event.created_at >= prev.created_at) { - queries.push(updateFollowingCountQuery(event)); + queries.push(await updateFollowingCountQuery(event)); } } @@ -37,8 +37,8 @@ async function updateStats(event: NostrEvent) { debug(JSON.stringify({ id: event.id, pubkey: event.pubkey, kind: event.kind, tags: event.tags, statDiffs })); } - if (pubkeyDiffs.length) queries.push(authorStatsQuery(pubkeyDiffs)); - if (eventDiffs.length) queries.push(eventStatsQuery(eventDiffs)); + if (pubkeyDiffs.length) queries.push(await authorStatsQuery(pubkeyDiffs)); + if (eventDiffs.length) queries.push(await eventStatsQuery(eventDiffs)); if (queries.length) { await Promise.all(queries.map((query) => query.execute())); @@ -102,7 +102,7 @@ async function getStatsDiff(event: NostrEvent, prev: NostrEvent | undefined): Pr } /** Create an author stats query from the list of diffs. */ -function authorStatsQuery(diffs: AuthorStatDiff[]) { +async function authorStatsQuery(diffs: AuthorStatDiff[]) { const values: DittoTables['author_stats'][] = diffs.map(([_, pubkey, stat, diff]) => { const row: DittoTables['author_stats'] = { pubkey, @@ -114,7 +114,8 @@ function authorStatsQuery(diffs: AuthorStatDiff[]) { return row; }); - return db.insertInto('author_stats') + const kysely = await DittoDB.getInstance(); + return kysely.insertInto('author_stats') .values(values) .onConflict((oc) => oc @@ -128,7 +129,7 @@ function authorStatsQuery(diffs: AuthorStatDiff[]) { } /** Create an event stats query from the list of diffs. */ -function eventStatsQuery(diffs: EventStatDiff[]) { +async function eventStatsQuery(diffs: EventStatDiff[]) { const values: DittoTables['event_stats'][] = diffs.map(([_, event_id, stat, diff]) => { const row: DittoTables['event_stats'] = { event_id, @@ -140,7 +141,8 @@ function eventStatsQuery(diffs: EventStatDiff[]) { return row; }); - return db.insertInto('event_stats') + const kysely = await DittoDB.getInstance(); + return kysely.insertInto('event_stats') .values(values) .onConflict((oc) => oc @@ -167,14 +169,15 @@ async function getPrevEvent(event: NostrEvent): Promise } /** Set the following count to the total number of unique "p" tags in the follow list. */ -function updateFollowingCountQuery({ pubkey, tags }: NostrEvent) { +async function updateFollowingCountQuery({ pubkey, tags }: NostrEvent) { const following_count = new Set( tags .filter(([name]) => name === 'p') .map(([_, value]) => value), ).size; - return db.insertInto('author_stats') + const kysely = await DittoDB.getInstance(); + return kysely.insertInto('author_stats') .values({ pubkey, following_count, diff --git a/src/storages.ts b/src/storages.ts index cbda925..f591e11 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -1,7 +1,8 @@ // deno-lint-ignore-file require-await import { NCache } from '@nostrify/nostrify'; + import { Conf } from '@/config.ts'; -import { db } from '@/db.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/events-db.ts'; import { Optimizer } from '@/storages/optimizer.ts'; import { PoolStore } from '@/storages/pool-store.ts'; @@ -24,7 +25,10 @@ export class Storages { /** SQLite database to store events this Ditto server cares about. */ public static async db(): Promise { if (!this._db) { - this._db = Promise.resolve(new EventsDB(db)); + this._db = (async () => { + const kysely = await DittoDB.getInstance(); + return new EventsDB(kysely); + })(); } return this._db; } diff --git a/src/storages/events-db.test.ts b/src/storages/events-db.test.ts index dd92c1b..d935e6b 100644 --- a/src/storages/events-db.test.ts +++ b/src/storages/events-db.test.ts @@ -1,12 +1,14 @@ -import { db } from '@/db.ts'; -import { assertEquals, assertRejects } from '@/deps-test.ts'; +import { assertEquals, assertRejects } from '@std/assert'; + +import { DittoDB } from '@/db/DittoDB.ts'; import event0 from '~/fixtures/events/event-0.json' with { type: 'json' }; import event1 from '~/fixtures/events/event-1.json' with { type: 'json' }; import { EventsDB } from '@/storages/events-db.ts'; -const eventsDB = new EventsDB(db); +const kysely = await DittoDB.getInstance(); +const eventsDB = new EventsDB(kysely); Deno.test('count filters', async () => { assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 0); @@ -34,7 +36,7 @@ Deno.test('query events with domain search filter', async () => { assertEquals(await eventsDB.query([{ search: 'domain:localhost:8000' }]), []); assertEquals(await eventsDB.query([{ search: '' }]), [event1]); - await db + await kysely .insertInto('pubkey_domains') .values({ pubkey: event1.pubkey, domain: 'localhost:8000', last_updated_at: event1.created_at }) .execute(); diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 3109ac6..8d2d302 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,7 +1,7 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; import { matchFilter } from 'nostr-tools'; -import { db } from '@/db.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; @@ -239,7 +239,7 @@ function gatherReportedProfiles({ events, store, signal }: HydrateOpts): Promise } /** Collect author stats from the events. */ -function gatherAuthorStats(events: DittoEvent[]): Promise { +async function gatherAuthorStats(events: DittoEvent[]): Promise { const pubkeys = new Set( events .filter((event) => event.kind === 0) @@ -250,7 +250,8 @@ function gatherAuthorStats(events: DittoEvent[]): Promise { +async function gatherEventStats(events: DittoEvent[]): Promise { const ids = new Set( events .filter((event) => event.kind === 1) @@ -269,7 +270,8 @@ function gatherEventStats(events: DittoEvent[]): Promise Date: Tue, 14 May 2024 18:46:55 -0500 Subject: [PATCH 34/39] Add missing nostr-relaypool import --- src/storages.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storages.ts b/src/storages.ts index f591e11..1cf06c1 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -1,5 +1,6 @@ // deno-lint-ignore-file require-await import { NCache } from '@nostrify/nostrify'; +import { RelayPoolWorker } from 'nostr-relaypool'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; From 3d1d56355d9b511be1031a136d5c90989db12df2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 18:51:35 -0500 Subject: [PATCH 35/39] Update scripts for async db --- scripts/admin-event.ts | 5 +++-- scripts/admin-role.ts | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/admin-event.ts b/scripts/admin-event.ts index a9939ad..29d3ae2 100644 --- a/scripts/admin-event.ts +++ b/scripts/admin-event.ts @@ -1,7 +1,7 @@ import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { db } from '@/db.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { EventsDB } from '@/storages/events-db.ts'; import { type EventStub } from '@/utils/api.ts'; @@ -9,7 +9,8 @@ import { nostrNow } from '@/utils.ts'; const signer = new AdminSigner(); -const eventsDB = new EventsDB(db); +const kysely = await DittoDB.getInstance(); +const eventsDB = new EventsDB(kysely); const readable = Deno.stdin.readable .pipeThrough(new TextDecoderStream()) diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 4fa212e..57a17e3 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -1,12 +1,13 @@ import { NSchema } from '@nostrify/nostrify'; -import { db } from '@/db.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; import { Conf } from '@/config.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { EventsDB } from '@/storages/events-db.ts'; import { nostrNow } from '@/utils.ts'; -const eventsDB = new EventsDB(db); +const kysely = await DittoDB.getInstance(); +const eventsDB = new EventsDB(kysely); const [pubkey, role] = Deno.args; From d3a7f0849f0026ed40f18b28bee145d01fa98c6e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 19:02:15 -0500 Subject: [PATCH 36/39] deno lint --- src/controllers/api/statuses.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index a52a408..98173b0 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; +import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; import ISO6391 from 'iso-639-1'; import { z } from 'zod'; From 477ee8b124f2fddd12e3e0d8c44cd453837ad350 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 19:09:25 -0500 Subject: [PATCH 37/39] Fix hydrateEvents in streaming --- src/controllers/api/streaming.ts | 7 ++++--- src/controllers/api/timelines.ts | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index e79c51e..1de7bbf 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -68,15 +68,16 @@ const streamingController: AppController = (c) => { if (!filter) return; try { - const store = await Storages.pubsub(); + const db = await Storages.db(); + const pubsub = await Storages.pubsub(); - for await (const msg of store.req([filter], { signal: controller.signal })) { + for await (const msg of pubsub.req([filter], { signal: controller.signal })) { if (msg[0] === 'EVENT') { const event = msg[2]; await hydrateEvents({ events: [event], - store, + store: db, signal: AbortSignal.timeout(1000), }); diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index e83c50c..8ea66ba 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -62,7 +62,7 @@ async function renderStatuses(c: AppContext, filters: NostrFilter[]) { return renderReblog(event, { viewerPubkey }); } return renderStatus(event, { viewerPubkey }); - }))).filter((boolean) => boolean); + }))).filter(Boolean); if (!statuses.length) { return c.json([]); From 2fd50261f9b1032ae64614d49a410302eb9c8074 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 19:11:38 -0500 Subject: [PATCH 38/39] streaming: actually hydrate with optimizer --- src/controllers/api/streaming.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 1de7bbf..ff47c07 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -68,8 +68,8 @@ const streamingController: AppController = (c) => { if (!filter) return; try { - const db = await Storages.db(); const pubsub = await Storages.pubsub(); + const optimizer = await Storages.optimizer(); for await (const msg of pubsub.req([filter], { signal: controller.signal })) { if (msg[0] === 'EVENT') { @@ -77,7 +77,7 @@ const streamingController: AppController = (c) => { await hydrateEvents({ events: [event], - store: db, + store: optimizer, signal: AbortSignal.timeout(1000), }); From f163af55d86f00f19f5de47b4c5090bd5cd700a1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 14 May 2024 19:19:57 -0500 Subject: [PATCH 39/39] Remove deps-test.ts --- src/deps-test.ts | 1 - src/filter.test.ts | 2 +- src/policies/MuteListPolicy.test.ts | 2 +- src/storages/UserStore.test.ts | 2 +- src/storages/hydrate.test.ts | 2 +- src/tags.test.ts | 2 +- src/utils/expiring-cache.test.ts | 2 +- src/utils/time.test.ts | 2 +- src/workers/fetch.test.ts | 2 +- src/workers/trends.test.ts | 2 +- 10 files changed, 9 insertions(+), 10 deletions(-) delete mode 100644 src/deps-test.ts diff --git a/src/deps-test.ts b/src/deps-test.ts deleted file mode 100644 index 3e6da88..0000000 --- a/src/deps-test.ts +++ /dev/null @@ -1 +0,0 @@ -export { assert, assertEquals, assertRejects, assertThrows } from 'https://deno.land/std@0.198.0/assert/mod.ts'; diff --git a/src/filter.test.ts b/src/filter.test.ts index 9983b5a..9379208 100644 --- a/src/filter.test.ts +++ b/src/filter.test.ts @@ -1,4 +1,4 @@ -import { assertEquals } from '@/deps-test.ts'; +import { assertEquals } from '@std/assert'; import event0 from '~/fixtures/events/event-0.json' with { type: 'json' }; import event1 from '~/fixtures/events/event-1.json' with { type: 'json' }; diff --git a/src/policies/MuteListPolicy.test.ts b/src/policies/MuteListPolicy.test.ts index 2c3baa3..89d7d99 100644 --- a/src/policies/MuteListPolicy.test.ts +++ b/src/policies/MuteListPolicy.test.ts @@ -1,6 +1,6 @@ import { MockRelay } from '@nostrify/nostrify/test'; -import { assertEquals } from '@/deps-test.ts'; +import { assertEquals } from '@std/assert'; import { UserStore } from '@/storages/UserStore.ts'; import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; diff --git a/src/storages/UserStore.test.ts b/src/storages/UserStore.test.ts index b1955bd..d04ece0 100644 --- a/src/storages/UserStore.test.ts +++ b/src/storages/UserStore.test.ts @@ -1,6 +1,6 @@ import { MockRelay } from '@nostrify/nostrify/test'; -import { assertEquals } from '@/deps-test.ts'; +import { assertEquals } from '@std/assert'; import { UserStore } from '@/storages/UserStore.ts'; import userBlack from '~/fixtures/events/kind-0-black.json' with { type: 'json' }; diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index f5c70af..1edafd7 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -1,4 +1,4 @@ -import { assertEquals } from '@/deps-test.ts'; +import { assertEquals } from '@std/assert'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { MockRelay } from '@nostrify/nostrify/test'; diff --git a/src/tags.test.ts b/src/tags.test.ts index c4d3214..e49d31a 100644 --- a/src/tags.test.ts +++ b/src/tags.test.ts @@ -1,4 +1,4 @@ -import { assertEquals } from '@/deps-test.ts'; +import { assertEquals } from '@std/assert'; import { addTag, deleteTag, getTagSet } from './tags.ts'; diff --git a/src/utils/expiring-cache.test.ts b/src/utils/expiring-cache.test.ts index 9827de8..8c6d7b1 100644 --- a/src/utils/expiring-cache.test.ts +++ b/src/utils/expiring-cache.test.ts @@ -1,4 +1,4 @@ -import { assert } from '@/deps-test.ts'; +import { assert } from '@std/assert'; import ExpiringCache from './expiring-cache.ts'; diff --git a/src/utils/time.test.ts b/src/utils/time.test.ts index c167caf..f820a1b 100644 --- a/src/utils/time.test.ts +++ b/src/utils/time.test.ts @@ -1,4 +1,4 @@ -import { assertEquals } from '@/deps-test.ts'; +import { assertEquals } from '@std/assert'; import { generateDateRange } from './time.ts'; diff --git a/src/workers/fetch.test.ts b/src/workers/fetch.test.ts index e7283b7..d657d1f 100644 --- a/src/workers/fetch.test.ts +++ b/src/workers/fetch.test.ts @@ -1,4 +1,4 @@ -import { assertEquals, assertRejects } from '@/deps-test.ts'; +import { assertEquals, assertRejects } from '@std/assert'; import { fetchWorker } from '@/workers/fetch.ts'; diff --git a/src/workers/trends.test.ts b/src/workers/trends.test.ts index ef51f23..ca1646e 100644 --- a/src/workers/trends.test.ts +++ b/src/workers/trends.test.ts @@ -1,4 +1,4 @@ -import { assertEquals } from '@/deps-test.ts'; +import { assertEquals } from '@std/assert'; import { TrendsWorker } from './trends.ts';