From b77c8a00cd4b624764658f10804070df61582d98 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 22 Apr 2024 19:51:29 -0300 Subject: [PATCH] perf: make up to 5 calls to database in hydrateEvents & remove old hydrate functions --- src/controllers/api/accounts.ts | 9 +- src/controllers/api/search.ts | 8 +- src/controllers/api/statuses.ts | 3 - src/controllers/api/streaming.ts | 1 - src/controllers/api/timelines.ts | 1 - src/interfaces/DittoEvent.ts | 4 +- src/pipeline.ts | 2 +- src/queries.ts | 14 ++- src/storages/hydrate.ts | 176 ++++++++++--------------------- src/storages/search-store.ts | 1 - src/views.ts | 8 +- 11 files changed, 71 insertions(+), 156 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 472c9e5..b6cd735 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -95,7 +95,6 @@ const accountSearchController: AppController = async (c) => { const results = await hydrateEvents({ events: event ? [event, ...events] : events, - relations: ['author_stats'], storage: eventsDB, signal: c.req.raw.signal, }); @@ -164,9 +163,7 @@ const accountStatusesController: AppController = async (c) => { } const events = await eventsDB.query([filter], { signal }) - .then((events) => - hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal }) - ) + .then((events) => hydrateEvents({ events, storage: eventsDB, signal })) .then((events) => { if (exclude_replies) { return events.filter((event) => !findReplyTag(event.tags)); @@ -317,9 +314,7 @@ const favouritesController: AppController = async (c) => { .filter((id): id is string => !!id); const events1 = await eventsDB.query([{ kinds: [1], ids }], { signal }) - .then((events) => - hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal }) - ); + .then((events) => hydrateEvents({ events, storage: eventsDB, signal })); const statuses = await Promise.all(events1.map((event) => renderStatus(event, { viewerPubkey: c.get('pubkey') }))); return paginated(c, events1, statuses); diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 628bb3a..6c1c972 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -90,9 +90,7 @@ function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: Abort } return searchStore.query([filter], { signal }) - .then((events) => - hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: searchStore, signal }) - ); + .then((events) => hydrateEvents({ events, storage: searchStore, signal })); } /** Get event kinds to search from `type` query param. */ @@ -112,9 +110,7 @@ async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise - hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: searchStore, signal }) - ) + .then((events) => hydrateEvents({ events, storage: searchStore, signal })) .then(([event]) => event); } diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index b063abd..a26ce08 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -41,7 +41,6 @@ const statusController: AppController = async (c) => { const event = await getEvent(id, { kind: 1, - relations: ['author', 'event_stats', 'author_stats', 'quote_repost'], signal: AbortSignal.timeout(1500), }); @@ -135,7 +134,6 @@ const createStatusController: AppController = async (c) => { if (data.quote_id) { await hydrateEvents({ events: [event], - relations: ['quote_repost'], storage: eventsDB, signal: c.req.raw.signal, }); @@ -241,7 +239,6 @@ const reblogStatusController: AppController = async (c) => { await hydrateEvents({ events: [reblogEvent], - relations: ['repost', 'author'], storage: eventsDB, signal: signal, }); diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 4ad878e..3855b9b 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -68,7 +68,6 @@ const streamingController: AppController = (c) => { if (event.kind === 6) { await hydrateEvents({ events: [event], - relations: ['repost', 'author'], storage: eventsDB, signal: AbortSignal.timeout(1000), }); diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index ebf1912..8bd4c52 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -51,7 +51,6 @@ async function renderStatuses(c: AppContext, filters: NostrFilter[]) { .then((events) => hydrateEvents({ events, - relations: ['author', 'author_stats', 'event_stats', 'repost', 'quote_repost'], storage: eventsDB, signal, }) diff --git a/src/interfaces/DittoEvent.ts b/src/interfaces/DittoEvent.ts index 84c8a3e..08879f8 100644 --- a/src/interfaces/DittoEvent.ts +++ b/src/interfaces/DittoEvent.ts @@ -22,6 +22,6 @@ export interface DittoEvent extends NostrEvent { event_stats?: EventStats; d_author?: DittoEvent; user?: DittoEvent; - repost?: NostrEvent; - quote_repost?: NostrEvent; + repost?: DittoEvent; + quote_repost?: DittoEvent; } diff --git a/src/pipeline.ts b/src/pipeline.ts index 56d7bb2..000a105 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -56,7 +56,7 @@ async function encounterEvent(event: NostrEvent, signal: AbortSignal): Promise { - await hydrateEvents({ events: [event], relations: ['author', 'user'], storage: eventsDB, signal }); + await hydrateEvents({ events: [event], storage: eventsDB, signal }); const domain = await db .selectFrom('pubkey_domains') diff --git a/src/queries.ts b/src/queries.ts index 61887b2..cf61b84 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -24,7 +24,7 @@ const getEvent = async ( opts: GetEventOpts = {}, ): Promise => { debug(`getEvent: ${id}`); - const { kind, relations = [], signal = AbortSignal.timeout(1000) } = opts; + const { kind, signal = AbortSignal.timeout(1000) } = opts; const filter: NostrFilter = { ids: [id], limit: 1 }; if (kind) { @@ -32,16 +32,16 @@ const getEvent = async ( } return await optimizer.query([filter], { limit: 1, signal }) - .then((events) => hydrateEvents({ events, relations, storage: optimizer, signal })) + .then((events) => hydrateEvents({ events, storage: optimizer, signal })) .then(([event]) => event); }; /** Get a Nostr `set_medatadata` event for a user's pubkey. */ const getAuthor = async (pubkey: string, opts: GetEventOpts = {}): Promise => { - const { relations = [], signal = AbortSignal.timeout(1000) } = opts; + const { signal = AbortSignal.timeout(1000) } = opts; return await optimizer.query([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, signal }) - .then((events) => hydrateEvents({ events, relations, storage: optimizer, signal })) + .then((events) => hydrateEvents({ events, storage: optimizer, signal })) .then(([event]) => event); }; @@ -70,7 +70,7 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi const inReplyTo = replyTag ? replyTag[1] : undefined; if (inReplyTo) { - const parentEvent = await getEvent(inReplyTo, { kind: 1, relations: ['author', 'event_stats', 'author_stats'] }); + const parentEvent = await getEvent(inReplyTo, { kind: 1 }); if (parentEvent) { result.push(parentEvent); @@ -84,9 +84,7 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi function getDescendants(eventId: string, signal = AbortSignal.timeout(2000)): Promise { return eventsDB.query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal }) - .then((events) => - hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal }) - ); + .then((events) => hydrateEvents({ events, storage: eventsDB, signal })); } /** Returns whether the pubkey is followed by a local user. */ diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 56518e2..8785a9a 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,46 +1,75 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; -import { Conf } from '@/config.ts'; import { db } from '@/db.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { type DittoRelation } from '@/interfaces/DittoFilter.ts'; interface HydrateEventOpts { events: DittoEvent[]; - relations: DittoRelation[]; storage: NStore; signal?: AbortSignal; } -/** Hydrate event relationships using the provided storage. */ +/** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateEventOpts): Promise { - const { events, relations, storage, signal } = opts; + const { events, storage, signal } = opts; - if (!events.length || !relations.length) { + if (!events.length) { return events; } - for (const relation of relations) { - switch (relation) { - case 'author': - await hydrateAuthors({ events, storage, signal }); - break; - case 'author_stats': - await hydrateAuthorStats(events); - break; - case 'event_stats': - await hydrateEventStats(events); - break; - case 'user': - await hydrateUsers({ events, storage, signal }); - break; - case 'repost': - await hydrateRepostEvents({ events, storage, signal }); - break; - case 'quote_repost': - await hydrateQuoteRepostEvents({ events, storage, signal }); - break; + const allEvents: DittoEvent[] = structuredClone(events); + + const childrenEventsIds = (events.map((event) => { + if (event.kind === 1) return event.tags.find(([name]) => name === 'q')?.[1]; // possible quote repost + if (event.kind === 6) return event.tags.find(([name]) => name === 'e')?.[1]; // possible repost + return; + }).filter(Boolean)) as string[]; + + if (childrenEventsIds.length > 0) { + const childrenEvents = await storage.query([{ ids: childrenEventsIds }], { signal }); + allEvents.push(...childrenEvents); + + if (childrenEvents.length > 0) { + const grandChildrenEventsIds = (childrenEvents.map((event) => { + if (event.kind === 1) return event.tags.find(([name]) => name === 'q')?.[1]; // possible quote repost + return; + }).filter(Boolean)) as string[]; + if (grandChildrenEventsIds.length > 0) { + const grandChildrenEvents = await storage.query([{ ids: grandChildrenEventsIds }], { signal }); + allEvents.push(...grandChildrenEvents); + } } } + await hydrateAuthors({ events: allEvents, storage, signal }); + await hydrateAuthorStats(allEvents); + await hydrateEventStats(allEvents); + + events.forEach((event) => { + const correspondingEvent = allEvents.find((element) => element.id === event.id); + if (correspondingEvent?.author) event.author = correspondingEvent.author; + if (correspondingEvent?.author_stats) event.author_stats = correspondingEvent.author_stats; + if (correspondingEvent?.event_stats) event.event_stats = correspondingEvent.event_stats; + + if (event.kind === 1) { + const quoteId = event.tags.find(([name]) => name === 'q')?.[1]; + if (quoteId) { + event.quote_repost = allEvents.find((element) => element.id === quoteId); + } + } else if (event.kind === 6) { + const repostedId = event.tags.find(([name]) => name === 'e')?.[1]; + if (repostedId) { + const repostedEvent = allEvents.find((element) => element.id === repostedId); + if (repostedEvent && repostedEvent.tags.find(([name]) => name === 'q')?.[1]) { // The repost is a repost of a quote repost + const postBeingQuoteRepostedId = repostedEvent.tags.find(([name]) => name === 'q')?.[1]; + event.repost = { + quote_repost: allEvents.find((element) => element.id === postBeingQuoteRepostedId), + ...allEvents.find((element) => element.id === repostedId) as DittoEvent, + }; + } else { // The repost is a repost of a normal post + event.repost = allEvents.find((element) => element.id === repostedId); + } + } + } + }); return events; } @@ -58,23 +87,6 @@ async function hydrateAuthors(opts: Omit): Promis return events; } -async function hydrateUsers(opts: Omit): Promise { - const { events, storage, signal } = opts; - - const pubkeys = new Set([...events].map((event) => event.pubkey)); - - const users = await storage.query( - [{ kinds: [30361], authors: [Conf.pubkey], '#d': [...pubkeys], limit: pubkeys.size }], - { signal }, - ); - - for (const event of events) { - event.user = users.find((user) => user.tags.find(([name]) => name === 'd')?.[1] === event.pubkey); - } - - return events; -} - async function hydrateAuthorStats(events: DittoEvent[]): Promise { const results = await db .selectFrom('author_stats') @@ -117,84 +129,6 @@ async function hydrateEventStats(events: DittoEvent[]): Promise { return events; } -async function hydrateRepostEvents(opts: Omit): Promise { - const { events, storage, signal } = opts; - const results = await storage.query([{ - kinds: [1], - ids: events.map((event) => { - if (event.kind === 6) { - const originalPostId = event.tags.find(([name]) => name === 'e')?.[1]; - if (!originalPostId) return event.id; - else return originalPostId; - } - return event.id; - }), - }], { signal }); - - for (const event of events) { - if (event.kind === 6) { - const originalPostId = event.tags.find(([name]) => name === 'e')?.[1]; - if (!originalPostId) continue; - - const originalPostEvent = results.find((event) => event.id === originalPostId); - if (!originalPostEvent) continue; - - await hydrateEvents({ - events: [originalPostEvent], - storage: storage, - signal: signal, - relations: ['author', 'event_stats'], - }); - event.repost = originalPostEvent; - } - } - - return events; -} - -async function hydrateQuoteRepostEvents(opts: Omit): Promise { - const { events, storage, signal } = opts; - - const results = await storage.query([{ - kinds: [1], - ids: events.map((event) => { - if (event.kind === 1) { - const originalPostId = event.tags.find(([name]) => name === 'q')?.[1]; - if (!originalPostId) return event.id; - else return originalPostId; - } - return event.id; - }), - }], { signal }); - - for (const event of events) { - if (event.kind === 1) { - const originalPostId = event.tags.find(([name]) => name === 'q')?.[1]; - if (!originalPostId) continue; - - const originalPostEvent = events.find((event) => event.id === originalPostId); - if (!originalPostEvent) { - const originalPostEvent = results.find((event) => event.id === originalPostId); - if (!originalPostEvent) continue; - - await hydrateEvents({ events: [originalPostEvent], storage: storage, signal: signal, relations: ['author'] }); - - event.quote_repost = originalPostEvent; - continue; - } - if (!originalPostEvent.author) { - await hydrateEvents({ events: [originalPostEvent], storage: storage, signal: signal, relations: ['author'] }); - - event.quote_repost = originalPostEvent; - continue; - } - event.quote_repost = originalPostEvent; - } - } - - return events; -} - /** Return a normalized event without any non-standard keys. */ function purifyEvent(event: NostrEvent): NostrEvent { return { diff --git a/src/storages/search-store.ts b/src/storages/search-store.ts index 5164108..6150896 100644 --- a/src/storages/search-store.ts +++ b/src/storages/search-store.ts @@ -47,7 +47,6 @@ class SearchStore implements NStore { return hydrateEvents({ events, - relations: ['author', 'event_stats', 'author_stats'], storage: this.#hydrator, signal: opts?.signal, }); diff --git a/src/views.ts b/src/views.ts index 618bd9f..94bb8df 100644 --- a/src/views.ts +++ b/src/views.ts @@ -20,7 +20,7 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal } const authors = await eventsDB.query([{ kinds: [0], authors: [...pubkeys] }], { signal }) - .then((events) => hydrateEvents({ events, relations: ['author_stats'], storage: eventsDB, signal })); + .then((events) => hydrateEvents({ events, storage: eventsDB, signal })); const accounts = await Promise.all( authors.map((event) => renderAccount(event)), @@ -33,7 +33,7 @@ async function renderAccounts(c: AppContext, authors: string[], signal = AbortSi const { since, until, limit } = paginationSchema.parse(c.req.query()); const events = await eventsDB.query([{ kinds: [0], authors, since, until, limit }], { signal }) - .then((events) => hydrateEvents({ events, relations: ['author_stats'], storage: eventsDB, signal })); + .then((events) => hydrateEvents({ events, storage: eventsDB, signal })); const accounts = await Promise.all( events.map((event) => renderAccount(event)), @@ -51,9 +51,7 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal const { limit } = paginationSchema.parse(c.req.query()); const events = await eventsDB.query([{ kinds: [1], ids, limit }], { signal }) - .then((events) => - hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal }) - ); + .then((events) => hydrateEvents({ events, storage: eventsDB, signal })); if (!events.length) { return c.json([]);