From 8b9566d79bac8f2b03cfc5da169c3fe9914c30cf Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 6 Mar 2024 12:55:02 -0600 Subject: [PATCH] Remove relations filters, switch some stuff to use optimizer (requires bravery) --- src/controllers/api/accounts.ts | 25 ++++++---- src/controllers/api/search.ts | 25 ++++++---- src/interfaces/DittoFilter.ts | 2 - src/queries.ts | 83 +++++++-------------------------- src/storages/events-db.ts | 43 ----------------- src/storages/hydrate.ts | 4 +- src/storages/optimizer.ts | 2 +- src/views.ts | 21 ++++----- 8 files changed, 59 insertions(+), 146 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 79934fe..2152208 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -15,6 +15,7 @@ import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderRelationship } from '@/views/mastodon/relationships.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { DittoFilter } from '@/interfaces/DittoFilter.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; const usernameSchema = z .string().min(1).max(30) @@ -147,7 +148,6 @@ const accountStatusesController: AppController = async (c) => { const filter: DittoFilter = { authors: [pubkey], kinds: [1], - relations: ['author', 'event_stats', 'author_stats'], since, until, limit, @@ -157,11 +157,16 @@ const accountStatusesController: AppController = async (c) => { filter['#t'] = [tagged]; } - let events = await eventsDB.query([filter], { signal }); - - if (exclude_replies) { - events = events.filter((event) => !findReplyTag(event.tags)); - } + const events = await eventsDB.query([filter], { signal }) + .then((events) => + hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal }) + ) + .then((events) => { + if (exclude_replies) { + return events.filter((event) => !findReplyTag(event.tags)); + } + return events; + }); const statuses = await Promise.all(events.map((event) => renderStatus(event, c.get('pubkey')))); return paginated(c, events, statuses); @@ -304,10 +309,10 @@ const favouritesController: AppController = async (c) => { .map((event) => event.tags.find((tag) => tag[0] === 'e')?.[1]) .filter((id): id is string => !!id); - const events1 = await eventsDB.query( - [{ kinds: [1], ids, relations: ['author', 'event_stats', 'author_stats'] }], - { signal }, - ); + const events1 = await eventsDB.query([{ kinds: [1], ids }], { signal }) + .then((events) => + hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal }) + ); const statuses = await Promise.all(events1.map((event) => renderStatus(event, c.get('pubkey')))); return paginated(c, events1, statuses); diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index f785f34..8d680fb 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -8,6 +8,7 @@ import { dedupeEvents } from '@/utils.ts'; import { nip05Cache } from '@/utils/nip05.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; /** Matches NIP-05 names with or without an @ in front. */ const ACCT_REGEX = /^@?(?:([\w.+-]+)@)?([\w.-]+)$/; @@ -69,7 +70,6 @@ function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: Abort const filter: DittoFilter = { kinds: typeToKinds(type), search: q, - relations: ['author', 'event_stats', 'author_stats'], limit, }; @@ -77,7 +77,10 @@ function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: Abort filter.authors = [account_id]; } - return searchStore.query([filter], { signal }); + return searchStore.query([filter], { signal }) + .then((events) => + hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: searchStore, signal }) + ); } /** Get event kinds to search from `type` query param. */ @@ -95,8 +98,12 @@ function typeToKinds(type: SearchQuery['type']): number[] { /** Resolve a searched value into an event, if applicable. */ async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise { const filters = await getLookupFilters(query, signal); - const [event] = await searchStore.query(filters, { limit: 1, signal }); - return event; + + return searchStore.query(filters, { limit: 1, signal }) + .then((events) => + hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: searchStore, signal }) + ) + .then(([event]) => event); } /** Get filters to lookup the input value. */ @@ -115,19 +122,19 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort const result = nip19.decode(q); switch (result.type) { case 'npub': - if (accounts) filters.push({ kinds: [0], authors: [result.data], relations: ['author_stats'] }); + if (accounts) filters.push({ kinds: [0], authors: [result.data] }); break; case 'nprofile': - if (accounts) filters.push({ kinds: [0], authors: [result.data.pubkey], relations: ['author_stats'] }); + if (accounts) filters.push({ kinds: [0], authors: [result.data.pubkey] }); break; case 'note': if (statuses) { - filters.push({ kinds: [1], ids: [result.data], relations: ['author', 'event_stats', 'author_stats'] }); + filters.push({ kinds: [1], ids: [result.data] }); } break; case 'nevent': if (statuses) { - filters.push({ kinds: [1], ids: [result.data.id], relations: ['author', 'event_stats', 'author_stats'] }); + filters.push({ kinds: [1], ids: [result.data.id] }); } break; } @@ -141,7 +148,7 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort try { const { pubkey } = await nip05Cache.fetch(q, { signal }); if (pubkey) { - filters.push({ kinds: [0], authors: [pubkey], relations: ['author_stats'] }); + filters.push({ kinds: [0], authors: [pubkey] }); } } catch (_e) { // do nothing diff --git a/src/interfaces/DittoFilter.ts b/src/interfaces/DittoFilter.ts index 4ecda96..bcc1719 100644 --- a/src/interfaces/DittoFilter.ts +++ b/src/interfaces/DittoFilter.ts @@ -9,6 +9,4 @@ export type DittoRelation = Exclude; export interface DittoFilter extends NostrFilter { /** Whether the event was authored by a local user. */ local?: boolean; - /** Additional fields to add to the returned event. */ - relations?: DittoRelation[]; } diff --git a/src/queries.ts b/src/queries.ts index 92d2df4..c6be412 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -1,9 +1,9 @@ -import { cache, eventsDB, reqmeister } from '@/storages.ts'; -import { Debug, type NostrEvent } from '@/deps.ts'; -import { type AuthorMicrofilter, type IdMicrofilter } from '@/filter.ts'; +import { eventsDB, optimizer } from '@/storages.ts'; +import { Debug, type NostrEvent, type NostrFilter } from '@/deps.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { type DittoFilter, type DittoRelation } from '@/interfaces/DittoFilter.ts'; +import { type DittoRelation } from '@/interfaces/DittoFilter.ts'; import { findReplyTag, getTagSet } from '@/tags.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; const debug = Debug('ditto:queries'); @@ -22,76 +22,25 @@ const getEvent = async ( opts: GetEventOpts = {}, ): Promise => { debug(`getEvent: ${id}`); - const { kind, relations, signal = AbortSignal.timeout(1000) } = opts; - const microfilter: IdMicrofilter = { ids: [id] }; + const { kind, relations = [], signal = AbortSignal.timeout(1000) } = opts; - const [memoryEvent] = await cache.query([microfilter]) as DittoEvent[]; - - if (memoryEvent && !relations) { - debug(`getEvent: ${id.slice(0, 8)} found in memory`); - return memoryEvent; - } - - const filter: DittoFilter = { ids: [id], relations, limit: 1 }; + const filter: NostrFilter = { ids: [id], limit: 1 }; if (kind) { filter.kinds = [kind]; } - const dbEvent = await eventsDB.query([filter], { limit: 1, signal }) + return await optimizer.query([filter], { limit: 1, signal }) + .then(([event]) => hydrateEvents({ events: [event], relations, storage: optimizer, signal })) .then(([event]) => event); - - // TODO: make this DRY-er. - - if (dbEvent && !dbEvent.author) { - const [author] = await cache.query([{ kinds: [0], authors: [dbEvent.pubkey] }]); - dbEvent.author = author; - } - - if (dbEvent) { - debug(`getEvent: ${id.slice(0, 8)} found in db`); - return dbEvent; - } - - if (memoryEvent && !memoryEvent.author) { - const [author] = await cache.query([{ kinds: [0], authors: [memoryEvent.pubkey] }]); - memoryEvent.author = author; - } - - if (memoryEvent) { - debug(`getEvent: ${id.slice(0, 8)} found in memory`); - return memoryEvent; - } - - const reqEvent = await reqmeister.req(microfilter, opts).catch(() => undefined); - - if (reqEvent) { - debug(`getEvent: ${id.slice(0, 8)} found by reqmeister`); - return reqEvent; - } - - debug(`getEvent: ${id.slice(0, 8)} not found`); }; /** Get a Nostr `set_medatadata` event for a user's pubkey. */ const getAuthor = async (pubkey: string, opts: GetEventOpts = {}): Promise => { - const { relations, signal = AbortSignal.timeout(1000) } = opts; - const microfilter: AuthorMicrofilter = { kinds: [0], authors: [pubkey] }; + const { relations = [], signal = AbortSignal.timeout(1000) } = opts; - const [memoryEvent] = await cache.query([microfilter]); - - if (memoryEvent && !relations) { - return memoryEvent; - } - - const dbEvent = await eventsDB.query( - [{ authors: [pubkey], relations, kinds: [0], limit: 1 }], - { limit: 1, signal }, - ).then(([event]) => event); - - if (dbEvent) return dbEvent; - if (memoryEvent) return memoryEvent; - - return reqmeister.req(microfilter, opts).catch(() => undefined); + return await optimizer.query([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, signal }) + .then(([event]) => hydrateEvents({ events: [event], relations, storage: optimizer, signal })) + .then(([event]) => event); }; /** Get users the given pubkey follows. */ @@ -132,10 +81,10 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi } function getDescendants(eventId: string, signal = AbortSignal.timeout(2000)): Promise { - return eventsDB.query( - [{ kinds: [1], '#e': [eventId], relations: ['author', 'event_stats', 'author_stats'] }], - { limit: 200, signal }, - ); + return eventsDB.query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal }) + .then((events) => + hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal }) + ); } /** Returns whether the pubkey is followed by a local user. */ diff --git a/src/storages/events-db.ts b/src/storages/events-db.ts index 304f283..37573aa 100644 --- a/src/storages/events-db.ts +++ b/src/storages/events-db.ts @@ -198,49 +198,6 @@ class EventsDB implements NStore { .where('users.d_tag', filter.local ? 'is not' : 'is', null); } - if (filter.relations?.includes('author')) { - query = query - .leftJoin( - (eb) => - eb - .selectFrom('events') - .selectAll() - .where('kind', '=', 0) - .groupBy('pubkey') - .as('authors'), - (join) => join.onRef('authors.pubkey', '=', 'events.pubkey'), - ) - .select([ - 'authors.id as author_id', - 'authors.kind as author_kind', - 'authors.pubkey as author_pubkey', - 'authors.content as author_content', - 'authors.tags as author_tags', - 'authors.created_at as author_created_at', - 'authors.sig as author_sig', - ]); - } - - if (filter.relations?.includes('author_stats')) { - query = query - .leftJoin('author_stats', 'author_stats.pubkey', 'events.pubkey') - .select((eb) => [ - eb.fn.coalesce('author_stats.followers_count', eb.val(0)).as('author_stats_followers_count'), - eb.fn.coalesce('author_stats.following_count', eb.val(0)).as('author_stats_following_count'), - eb.fn.coalesce('author_stats.notes_count', eb.val(0)).as('author_stats_notes_count'), - ]); - } - - if (filter.relations?.includes('event_stats')) { - query = query - .leftJoin('event_stats', 'event_stats.event_id', 'events.id') - .select((eb) => [ - eb.fn.coalesce('event_stats.replies_count', eb.val(0)).as('stats_replies_count'), - eb.fn.coalesce('event_stats.reposts_count', eb.val(0)).as('stats_reposts_count'), - eb.fn.coalesce('event_stats.reactions_count', eb.val(0)).as('stats_reactions_count'), - ]); - } - if (filter.search) { query = query .innerJoin('events_fts', 'events_fts.id', 'events.id') diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 718b7a1..403a74a 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -13,13 +13,13 @@ interface HydrateEventOpts { async function hydrateEvents(opts: HydrateEventOpts): Promise { const { events, relations, storage, signal } = opts; - if (events.length === 0) { + if (!events.length || !relations.length) { return events; } if (relations.includes('author')) { const pubkeys = new Set([...events].map((event) => event.pubkey)); - const authors = await storage.query([{ kinds: [0], authors: [...pubkeys] }], { signal }); + const authors = await storage.query([{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }], { signal }); for (const event of events) { event.author = authors.find((author) => author.pubkey === event.pubkey); diff --git a/src/storages/optimizer.ts b/src/storages/optimizer.ts index dcf7ad9..2c029cf 100644 --- a/src/storages/optimizer.ts +++ b/src/storages/optimizer.ts @@ -23,7 +23,7 @@ class Optimizer implements NStore { this.#client = opts.client; } - async event(event: DittoEvent, opts?: NStoreOpts | undefined): Promise { + async event(event: DittoEvent, opts?: NStoreOpts): Promise { if (opts?.signal?.aborted) return Promise.reject(abortError()); await Promise.all([ diff --git a/src/views.ts b/src/views.ts index d631da4..f9e84fc 100644 --- a/src/views.ts +++ b/src/views.ts @@ -4,6 +4,7 @@ import { eventsDB } from '@/storages.ts'; import { renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { paginated, paginationSchema } from '@/utils/api.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; /** Render account objects for the author of each event. */ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal = AbortSignal.timeout(1000)) { @@ -18,10 +19,8 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal return c.json([]); } - const authors = await eventsDB.query( - [{ kinds: [0], authors: [...pubkeys], relations: ['author_stats'] }], - { signal }, - ); + const authors = await eventsDB.query([{ kinds: [0], authors: [...pubkeys] }], { signal }) + .then((events) => hydrateEvents({ events, relations: ['author_stats'], storage: eventsDB, signal })); const accounts = await Promise.all( authors.map((event) => renderAccount(event)), @@ -33,10 +32,8 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal async function renderAccounts(c: AppContext, authors: string[], signal = AbortSignal.timeout(1000)) { const { since, until, limit } = paginationSchema.parse(c.req.query()); - const events = await eventsDB.query( - [{ kinds: [0], authors, relations: ['author_stats'], since, until, limit }], - { signal }, - ); + const events = await eventsDB.query([{ kinds: [0], authors, since, until, limit }], { signal }) + .then((events) => hydrateEvents({ events, relations: ['author_stats'], storage: eventsDB, signal })); const accounts = await Promise.all( events.map((event) => renderAccount(event)), @@ -53,10 +50,10 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal const { limit } = paginationSchema.parse(c.req.query()); - const events = await eventsDB.query( - [{ kinds: [1], ids, relations: ['author', 'event_stats', 'author_stats'], limit }], - { signal }, - ); + const events = await eventsDB.query([{ kinds: [1], ids, limit }], { signal }) + .then((events) => + hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal }) + ); if (!events.length) { return c.json([]);