From 0aab3eb775a2d34e90a3d5bfa692359c4f487611 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Apr 2024 19:31:40 -0500 Subject: [PATCH 1/6] Rewrite hydrateEvents --- src/controllers/nostr/relay.ts | 3 +- src/storages/hydrate.test.ts | 8 +- src/storages/hydrate.ts | 222 ++++++++++++++++++--------------- 3 files changed, 130 insertions(+), 103 deletions(-) diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 2921aaa..c0e905b 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -11,6 +11,7 @@ import { clientMsgSchema, type ClientREQ, } from '@/schemas/nostr.ts'; +import { purifyEvent } from '@/storages/hydrate.ts'; import { Sub } from '@/subs.ts'; import type { AppController } from '@/app.ts'; @@ -70,7 +71,7 @@ function connectStream(socket: WebSocket) { send(['EOSE', subId]); for await (const event of Sub.sub(socket, subId, filters)) { - send(['EVENT', subId, event]); + send(['EVENT', subId, purifyEvent(event)]); } } diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index d9acfa1..1c16021 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -67,8 +67,8 @@ Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { await db.event(event1repostedCopy); await db.event(event6copy); - assertEquals((event6copy as DittoEvent).author, undefined, "Event hasn't been hydrated author yet"); - assertEquals((event6copy as DittoEvent).repost, undefined, "Event hasn't been hydrated repost yet"); + assertEquals((event6copy as DittoEvent).author, undefined, "Event hasn't hydrated author yet"); + assertEquals((event6copy as DittoEvent).repost, undefined, "Event hasn't hydrated repost yet"); const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 1000); @@ -143,8 +143,8 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () await db.event(event1quoteCopy); await db.event(event6copy); - assertEquals((event6copy as DittoEvent).author, undefined, "Event hasn't been hydrated author yet"); - assertEquals((event6copy as DittoEvent).repost, undefined, "Event hasn't been hydrated repost yet"); + assertEquals((event6copy as DittoEvent).author, undefined, "Event hasn't hydrated author yet"); + assertEquals((event6copy as DittoEvent).repost, undefined, "Event hasn't hydrated repost yet"); const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), 1000); diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index f65641f..7c21071 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,137 +1,163 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; + import { db } from '@/db.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; +import { Conf } from '@/config.ts'; -interface HydrateEventOpts { +interface HydrateOpts { events: DittoEvent[]; storage: NStore; signal?: AbortSignal; } /** Hydrate events using the provided storage. */ -async function hydrateEvents(opts: HydrateEventOpts): Promise { +async function hydrateEvents(opts: HydrateOpts): Promise { const { events, storage, signal } = opts; if (!events.length) { return events; } - const allEventsMap: Map = new Map(events.map((event) => { - return [event.id, structuredClone(event)]; - })); + const cache = [...events]; - const childrenEventsIds = (events.map((event) => { - if (event.kind === 1) return event.tags.find(([name]) => name === 'q')?.[1]; // possible quote repost - if (event.kind === 6) return event.tags.find(([name]) => name === 'e')?.[1]; // possible repost - return; - }).filter(Boolean)) as string[]; - - if (childrenEventsIds.length > 0) { - const childrenEvents = await storage.query([{ ids: childrenEventsIds }], { signal }); - childrenEvents.forEach((event) => { - allEventsMap.set(event.id, structuredClone(event)); - }); - - if (childrenEvents.length > 0) { - const grandChildrenEventsIds = (childrenEvents.map((event) => { - if (event.kind === 1) return event.tags.find(([name]) => name === 'q')?.[1]; // possible quote repost - return; - }).filter(Boolean)) as string[]; - if (grandChildrenEventsIds.length > 0) { - const grandChildrenEvents = await storage.query([{ ids: grandChildrenEventsIds }], { signal }); - grandChildrenEvents.forEach((event) => { - allEventsMap.set(event.id, structuredClone(event)); - }); - } - } + for (const event of await gatherReposts({ events, storage, signal })) { + cache.push(event); } - await hydrateAuthors({ events: [...allEventsMap.values()], storage, signal }); - await hydrateAuthorStats([...allEventsMap.values()].filter((e) => e.kind === 0)); - await hydrateEventStats([...allEventsMap.values()].filter((e) => e.kind === 1)); - events.forEach((event) => { - const correspondingEvent = allEventsMap.get(event.id); - if (correspondingEvent?.author) event.author = correspondingEvent.author; - if (correspondingEvent?.author_stats) event.author_stats = correspondingEvent.author_stats; - if (correspondingEvent?.event_stats) event.event_stats = correspondingEvent.event_stats; + for (const event of await gatherQuotes({ events, storage, signal })) { + cache.push(event); + } + + for (const event of await gatherAuthors({ events, storage, signal })) { + cache.push(event); + } + + for (const event of await gatherUsers({ events, storage, signal })) { + cache.push(event); + } + + const stats = { + authors: await gatherAuthorStats(cache), + events: await gatherEventStats(cache), + }; + + assembleEvents(cache, cache, stats); + assembleEvents(events, cache, stats); + + return events; +} + +function assembleEvents( + a: DittoEvent[], + b: DittoEvent[], + stats: { authors: DittoTables['author_stats'][]; events: DittoTables['event_stats'][] }, +): DittoEvent[] { + const admin = Conf.pubkey; + + for (const event of a) { + if (event.kind === 6) { + const id = event.tags.find(([name]) => name === 'e')?.[1]; + event.repost = b.find((e) => e.kind === 1 && id === e.id); + } if (event.kind === 1) { - const quoteId = event.tags.find(([name]) => name === 'q')?.[1]; - if (quoteId) { - event.quote_repost = allEventsMap.get(quoteId); - } - } else if (event.kind === 6) { - const repostedId = event.tags.find(([name]) => name === 'e')?.[1]; - if (repostedId) { - const repostedEvent = allEventsMap.get(repostedId); - if (repostedEvent && repostedEvent.tags.find(([name]) => name === 'q')?.[1]) { // The repost is a repost of a quote repost - const postBeingQuoteRepostedId = repostedEvent.tags.find(([name]) => name === 'q')?.[1]; - event.repost = { - quote_repost: allEventsMap.get(postBeingQuoteRepostedId!), - ...allEventsMap.get(repostedId)!, - }; - } else { // The repost is a repost of a normal post - event.repost = allEventsMap.get(repostedId); - } - } + const id = event.tags.find(([name]) => name === 'q')?.[1]; + event.quote_repost = b.find((e) => e.kind === 1 && id === e.id); } - }); - return events; -} -async function hydrateAuthors(opts: Omit): Promise { - const { events, storage, signal } = opts; + event.author = b.find((e) => e.kind === 0 && e.pubkey === event.pubkey); + event.author_stats = stats.authors.find((stats) => stats.pubkey === event.pubkey); + event.event_stats = stats.events.find((stats) => stats.event_id === event.id); - const pubkeys = new Set([...events].map((event) => event.pubkey)); - const authors = await storage.query([{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }], { signal }); - - for (const event of events) { - event.author = authors.find((author) => author.pubkey === event.pubkey); + event.user = b.find((e) => + e.kind === 30361 && e.pubkey === admin && e.tags.find(([name]) => name === 'd')?.[1] === event.pubkey + ); } - return events; + return a; } -async function hydrateAuthorStats(events: DittoEvent[]): Promise { - const results = await db +function gatherReposts({ events, storage, signal }: HydrateOpts): Promise { + const ids = new Set(); + + for (const event of events) { + if (event.kind === 6) { + const id = event.tags.find(([name]) => name === 'e')?.[1]; + if (id) { + ids.add(id); + } + } + } + + return storage.query( + [{ ids: [...ids], limit: ids.size }], + { signal }, + ); +} + +function gatherQuotes({ events, storage, signal }: HydrateOpts): Promise { + const ids = new Set(); + + for (const event of events) { + if (event.kind === 1) { + const id = event.tags.find(([name]) => name === 'q')?.[1]; + if (id) { + ids.add(id); + } + } + } + + return storage.query( + [{ ids: [...ids], limit: ids.size }], + { signal }, + ); +} + +function gatherAuthors({ events, storage, signal }: HydrateOpts): Promise { + const pubkeys = new Set(events.map((event) => event.pubkey)); + + return storage.query( + [{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }], + { signal }, + ); +} + +function gatherUsers({ events, storage, signal }: HydrateOpts): Promise { + const pubkeys = new Set(events.map((event) => event.pubkey)); + + return storage.query( + [{ kinds: [30361], authors: [Conf.pubkey], '#d': [...pubkeys], limit: pubkeys.size }], + { signal }, + ); +} + +function gatherAuthorStats(events: DittoEvent[]): Promise { + const pubkeys = new Set( + events + .filter((event) => event.kind === 0) + .map((event) => event.pubkey), + ); + + return db .selectFrom('author_stats') .selectAll() - .where('pubkey', 'in', events.map((event) => event.pubkey)) + .where('pubkey', 'in', [...pubkeys]) .execute(); - - for (const event of events) { - const stat = results.find((result) => result.pubkey === event.pubkey); - if (stat) { - event.author_stats = { - followers_count: Math.max(stat.followers_count, 0) || 0, - following_count: Math.max(stat.following_count, 0) || 0, - notes_count: Math.max(stat.notes_count, 0) || 0, - }; - } - } - - return events; } -async function hydrateEventStats(events: DittoEvent[]): Promise { - const results = await db +function gatherEventStats(events: DittoEvent[]): Promise { + const ids = new Set( + events + .filter((event) => event.kind === 1) + .map((event) => event.id), + ); + + return db .selectFrom('event_stats') .selectAll() - .where('event_id', 'in', events.map((event) => event.id)) + .where('event_id', 'in', [...ids]) .execute(); - - for (const event of events) { - const stat = results.find((result) => result.event_id === event.id); - if (stat) { - event.event_stats = { - replies_count: Math.max(stat.replies_count, 0) || 0, - reposts_count: Math.max(stat.reposts_count, 0) || 0, - reactions_count: Math.max(stat.reactions_count, 0) || 0, - }; - } - } - - return events; } /** Return a normalized event without any non-standard keys. */ From 191b370c85dffe664303637af699941755fef740 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Apr 2024 19:50:23 -0500 Subject: [PATCH 2/6] hydrate.test: remove unnecessary boilerplate --- src/storages/hydrate.test.ts | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/src/storages/hydrate.test.ts b/src/storages/hydrate.test.ts index 1c16021..de69008 100644 --- a/src/storages/hydrate.test.ts +++ b/src/storages/hydrate.test.ts @@ -35,22 +35,13 @@ Deno.test('hydrateEvents(): author --- WITHOUT stats', async () => { assertEquals((event1copy as DittoEvent).author, undefined, "Event hasn't been hydrated yet"); - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 1000); - await hydrateEvents({ events: [event1copy], storage: db, - signal: controller.signal, }); const expectedEvent = { ...event1copy, author: event0copy }; assertEquals(event1copy, expectedEvent); - - await db.remove([{ kinds: [0, 1] }]); - assertEquals(await db.query([{ kinds: [0, 1] }]), []); - - clearTimeout(timeoutId); }); Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { @@ -70,13 +61,9 @@ Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { assertEquals((event6copy as DittoEvent).author, undefined, "Event hasn't hydrated author yet"); assertEquals((event6copy as DittoEvent).repost, undefined, "Event hasn't hydrated repost yet"); - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 1000); - await hydrateEvents({ events: [event6copy], storage: db, - signal: controller.signal, }); const expectedEvent6 = { @@ -85,11 +72,6 @@ Deno.test('hydrateEvents(): repost --- WITHOUT stats', async () => { repost: { ...event1repostedCopy, author: event0madePostCopy }, }; assertEquals(event6copy, expectedEvent6); - - await db.remove([{ kinds: [0, 1, 6] }]); - assertEquals(await db.query([{ kinds: [0, 1, 6] }]), []); - - clearTimeout(timeoutId); }); Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { @@ -106,13 +88,9 @@ Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { await db.event(event1quoteRepostCopy); await db.event(event1willBeQuoteRepostedCopy); - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 1000); - await hydrateEvents({ events: [event1quoteRepostCopy], storage: db, - signal: controller.signal, }); const expectedEvent1quoteRepost = { @@ -122,11 +100,6 @@ Deno.test('hydrateEvents(): quote repost --- WITHOUT stats', async () => { }; assertEquals(event1quoteRepostCopy, expectedEvent1quoteRepost); - - await db.remove([{ kinds: [0, 1] }]); - assertEquals(await db.query([{ kinds: [0, 1] }]), []); - - clearTimeout(timeoutId); }); Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () => { @@ -146,13 +119,9 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () assertEquals((event6copy as DittoEvent).author, undefined, "Event hasn't hydrated author yet"); assertEquals((event6copy as DittoEvent).repost, undefined, "Event hasn't hydrated repost yet"); - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 1000); - await hydrateEvents({ events: [event6copy], storage: db, - signal: controller.signal, }); const expectedEvent6 = { @@ -161,9 +130,4 @@ Deno.test('hydrateEvents(): repost of quote repost --- WITHOUT stats', async () repost: { ...event1quoteCopy, author: event0copy, quote_repost: { author: event0copy, ...event1copy } }, }; assertEquals(event6copy, expectedEvent6); - - await db.remove([{ kinds: [0, 1, 6] }]); - assertEquals(await db.query([{ kinds: [0, 1, 6] }]), []); - - clearTimeout(timeoutId); }); From a1423bbf6536402f13ce9a4b5c0dceb818a7bec3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Apr 2024 20:11:07 -0500 Subject: [PATCH 3/6] Fix hydrateEvents, lol --- src/storages/hydrate.ts | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 7c21071..369f39f 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -21,33 +21,43 @@ async function hydrateEvents(opts: HydrateOpts): Promise { const cache = [...events]; - for (const event of await gatherReposts({ events, storage, signal })) { + for (const event of await gatherReposts({ events: cache, storage, signal })) { cache.push(event); } - for (const event of await gatherQuotes({ events, storage, signal })) { + for (const event of await gatherQuotes({ events: cache, storage, signal })) { cache.push(event); } - for (const event of await gatherAuthors({ events, storage, signal })) { + for (const event of await gatherAuthors({ events: cache, storage, signal })) { cache.push(event); } - for (const event of await gatherUsers({ events, storage, signal })) { + for (const event of await gatherUsers({ events: cache, storage, signal })) { cache.push(event); } + const [authorStats, eventStats] = await Promise.all([ + gatherAuthorStats(cache), + gatherEventStats(cache), + ]); + const stats = { - authors: await gatherAuthorStats(cache), - events: await gatherEventStats(cache), + authors: authorStats, + events: eventStats, }; - assembleEvents(cache, cache, stats); - assembleEvents(events, cache, stats); + // Dedupe events. + const results = [...new Map(cache.map((event) => [event.id, event])).values()]; + + // First connect all the events to each-other, then connect the connected events to the original list. + assembleEvents(results, results, stats); + assembleEvents(events, results, stats); return events; } +/** Connect the events in list `b` to the DittoEvent fields in list `a`. */ function assembleEvents( a: DittoEvent[], b: DittoEvent[], @@ -78,6 +88,7 @@ function assembleEvents( return a; } +/** Collect reposts from the events. */ function gatherReposts({ events, storage, signal }: HydrateOpts): Promise { const ids = new Set(); @@ -96,6 +107,7 @@ function gatherReposts({ events, storage, signal }: HydrateOpts): Promise { const ids = new Set(); @@ -114,6 +126,7 @@ function gatherQuotes({ events, storage, signal }: HydrateOpts): Promise { const pubkeys = new Set(events.map((event) => event.pubkey)); @@ -123,6 +136,7 @@ function gatherAuthors({ events, storage, signal }: HydrateOpts): Promise { const pubkeys = new Set(events.map((event) => event.pubkey)); @@ -132,6 +146,7 @@ function gatherUsers({ events, storage, signal }: HydrateOpts): Promise { const pubkeys = new Set( events @@ -146,6 +161,7 @@ function gatherAuthorStats(events: DittoEvent[]): Promise { const ids = new Set( events From b8d01ea3de26c2d08559ee9f4e1900a3c587627f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Apr 2024 21:10:21 -0500 Subject: [PATCH 4/6] hydrateEvents: use filters to find events in memory --- src/storages/hydrate.ts | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 369f39f..0ef84e0 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,6 +1,7 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; import { db } from '@/db.ts'; +import { matchFilter } from '@/deps.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; @@ -37,14 +38,9 @@ async function hydrateEvents(opts: HydrateOpts): Promise { cache.push(event); } - const [authorStats, eventStats] = await Promise.all([ - gatherAuthorStats(cache), - gatherEventStats(cache), - ]); - const stats = { - authors: authorStats, - events: eventStats, + authors: await gatherAuthorStats(cache), + events: await gatherEventStats(cache), }; // Dedupe events. @@ -66,23 +62,25 @@ function assembleEvents( const admin = Conf.pubkey; for (const event of a) { + event.author = b.find((e) => matchFilter({ kinds: [0], authors: [event.pubkey] }, e)); + event.user = b.find((e) => matchFilter({ kinds: [30361], authors: [admin], '#d': [event.pubkey] }, e)); + if (event.kind === 6) { const id = event.tags.find(([name]) => name === 'e')?.[1]; - event.repost = b.find((e) => e.kind === 1 && id === e.id); + if (id) { + event.repost = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); + } } if (event.kind === 1) { const id = event.tags.find(([name]) => name === 'q')?.[1]; - event.quote_repost = b.find((e) => e.kind === 1 && id === e.id); + if (id) { + event.quote_repost = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); + } } - event.author = b.find((e) => e.kind === 0 && e.pubkey === event.pubkey); event.author_stats = stats.authors.find((stats) => stats.pubkey === event.pubkey); event.event_stats = stats.events.find((stats) => stats.event_id === event.id); - - event.user = b.find((e) => - e.kind === 30361 && e.pubkey === admin && e.tags.find(([name]) => name === 'd')?.[1] === event.pubkey - ); } return a; From a2c3daade71f60aff49823f6c3efdc863057b6be Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Apr 2024 21:32:48 -0500 Subject: [PATCH 5/6] hydrateEvents: return early if the results would be empty --- src/storages/hydrate.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 0ef84e0..1535571 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -99,6 +99,10 @@ function gatherReposts({ events, storage, signal }: HydrateOpts): Promise { const pubkeys = new Set(events.map((event) => event.pubkey)); + if (!pubkeys.size) { + return Promise.resolve([]); + } + return storage.query( [{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }], { signal }, @@ -138,6 +150,10 @@ function gatherAuthors({ events, storage, signal }: HydrateOpts): Promise { const pubkeys = new Set(events.map((event) => event.pubkey)); + if (!pubkeys.size) { + return Promise.resolve([]); + } + return storage.query( [{ kinds: [30361], authors: [Conf.pubkey], '#d': [...pubkeys], limit: pubkeys.size }], { signal }, @@ -152,6 +168,10 @@ function gatherAuthorStats(events: DittoEvent[]): Promise event.pubkey), ); + if (!pubkeys.size) { + return Promise.resolve([]); + } + return db .selectFrom('author_stats') .selectAll() @@ -167,6 +187,10 @@ function gatherEventStats(events: DittoEvent[]): Promise event.id), ); + if (!ids.size) { + return Promise.resolve([]); + } + return db .selectFrom('event_stats') .selectAll() From 1f5ba81e9848dfa3aa751aa4f7ab40b84535767d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 23 Apr 2024 21:40:02 -0500 Subject: [PATCH 6/6] hydrateEvents: early return is only needed for stats? --- src/storages/hydrate.ts | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 1535571..619b798 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -99,10 +99,6 @@ function gatherReposts({ events, storage, signal }: HydrateOpts): Promise { const pubkeys = new Set(events.map((event) => event.pubkey)); - if (!pubkeys.size) { - return Promise.resolve([]); - } - return storage.query( [{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }], { signal }, @@ -150,10 +138,6 @@ function gatherAuthors({ events, storage, signal }: HydrateOpts): Promise { const pubkeys = new Set(events.map((event) => event.pubkey)); - if (!pubkeys.size) { - return Promise.resolve([]); - } - return storage.query( [{ kinds: [30361], authors: [Conf.pubkey], '#d': [...pubkeys], limit: pubkeys.size }], { signal },