Remove relations filters, switch some stuff to use optimizer (requires bravery)

src/controllers/api/accounts.ts

@@ -15,6 +15,7 @@ import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
 import { renderRelationship } from '@/views/mastodon/relationships.ts';
 import { renderStatus } from '@/views/mastodon/statuses.ts';
 import { DittoFilter } from '@/interfaces/DittoFilter.ts';
+import { hydrateEvents } from '@/storages/hydrate.ts';
 
 const usernameSchema = z
   .string().min(1).max(30)
@@ -147,7 +148,6 @@ const accountStatusesController: AppController = async (c) => {
   const filter: DittoFilter = {
     authors: [pubkey],
     kinds: [1],
-    relations: ['author', 'event_stats', 'author_stats'],
     since,
     until,
     limit,
@@ -157,11 +157,16 @@ const accountStatusesController: AppController = async (c) => {
     filter['#t'] = [tagged];
   }
 
-  let events = await eventsDB.query([filter], { signal });
+  const events = await eventsDB.query([filter], { signal })
-
+    .then((events) =>
-  if (exclude_replies) {
+      hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal })
-    events = events.filter((event) => !findReplyTag(event.tags));
+    )
-  }
+    .then((events) => {
+      if (exclude_replies) {
+        return events.filter((event) => !findReplyTag(event.tags));
+      }
+      return events;
+    });
 
   const statuses = await Promise.all(events.map((event) => renderStatus(event, c.get('pubkey'))));
   return paginated(c, events, statuses);
@@ -304,10 +309,10 @@ const favouritesController: AppController = async (c) => {
     .map((event) => event.tags.find((tag) => tag[0] === 'e')?.[1])
     .filter((id): id is string => !!id);
 
-  const events1 = await eventsDB.query(
+  const events1 = await eventsDB.query([{ kinds: [1], ids }], { signal })
-    [{ kinds: [1], ids, relations: ['author', 'event_stats', 'author_stats'] }],
+    .then((events) =>
-    { signal },
+      hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal })
-  );
+    );
 
   const statuses = await Promise.all(events1.map((event) => renderStatus(event, c.get('pubkey'))));
   return paginated(c, events1, statuses);

src/controllers/api/search.ts

@@ -8,6 +8,7 @@ import { dedupeEvents } from '@/utils.ts';
 import { nip05Cache } from '@/utils/nip05.ts';
 import { renderAccount } from '@/views/mastodon/accounts.ts';
 import { renderStatus } from '@/views/mastodon/statuses.ts';
+import { hydrateEvents } from '@/storages/hydrate.ts';
 
 /** Matches NIP-05 names with or without an @ in front. */
 const ACCT_REGEX = /^@?(?:([\w.+-]+)@)?([\w.-]+)$/;
@@ -69,7 +70,6 @@ function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: Abort
   const filter: DittoFilter = {
     kinds: typeToKinds(type),
     search: q,
-    relations: ['author', 'event_stats', 'author_stats'],
     limit,
   };
 
@@ -77,7 +77,10 @@ function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: Abort
     filter.authors = [account_id];
   }
 
-  return searchStore.query([filter], { signal });
+  return searchStore.query([filter], { signal })
+    .then((events) =>
+      hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: searchStore, signal })
+    );
 }
 
 /** Get event kinds to search from `type` query param. */
@@ -95,8 +98,12 @@ function typeToKinds(type: SearchQuery['type']): number[] {
 /** Resolve a searched value into an event, if applicable. */
 async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise<NostrEvent | undefined> {
   const filters = await getLookupFilters(query, signal);
-  const [event] = await searchStore.query(filters, { limit: 1, signal });
+
-  return event;
+  return searchStore.query(filters, { limit: 1, signal })
+    .then((events) =>
+      hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: searchStore, signal })
+    )
+    .then(([event]) => event);
 }
 
 /** Get filters to lookup the input value. */
@@ -115,19 +122,19 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
       const result = nip19.decode(q);
       switch (result.type) {
         case 'npub':
-          if (accounts) filters.push({ kinds: [0], authors: [result.data], relations: ['author_stats'] });
+          if (accounts) filters.push({ kinds: [0], authors: [result.data] });
           break;
         case 'nprofile':
-          if (accounts) filters.push({ kinds: [0], authors: [result.data.pubkey], relations: ['author_stats'] });
+          if (accounts) filters.push({ kinds: [0], authors: [result.data.pubkey] });
           break;
         case 'note':
           if (statuses) {
-            filters.push({ kinds: [1], ids: [result.data], relations: ['author', 'event_stats', 'author_stats'] });
+            filters.push({ kinds: [1], ids: [result.data] });
           }
           break;
         case 'nevent':
           if (statuses) {
-            filters.push({ kinds: [1], ids: [result.data.id], relations: ['author', 'event_stats', 'author_stats'] });
+            filters.push({ kinds: [1], ids: [result.data.id] });
           }
           break;
       }
@@ -141,7 +148,7 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
     try {
       const { pubkey } = await nip05Cache.fetch(q, { signal });
       if (pubkey) {
-        filters.push({ kinds: [0], authors: [pubkey], relations: ['author_stats'] });
+        filters.push({ kinds: [0], authors: [pubkey] });
       }
     } catch (_e) {
       // do nothing

src/interfaces/DittoFilter.ts

@@ -9,6 +9,4 @@ export type DittoRelation = Exclude<keyof DittoEvent, keyof NostrEvent>;
 export interface DittoFilter extends NostrFilter {
   /** Whether the event was authored by a local user. */
   local?: boolean;
-  /** Additional fields to add to the returned event. */
-  relations?: DittoRelation[];
 }

src/queries.ts

@@ -1,9 +1,9 @@
-import { cache, eventsDB, reqmeister } from '@/storages.ts';
+import { eventsDB, optimizer } from '@/storages.ts';
-import { Debug, type NostrEvent } from '@/deps.ts';
+import { Debug, type NostrEvent, type NostrFilter } from '@/deps.ts';
-import { type AuthorMicrofilter, type IdMicrofilter } from '@/filter.ts';
 import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
-import { type DittoFilter, type DittoRelation } from '@/interfaces/DittoFilter.ts';
+import { type DittoRelation } from '@/interfaces/DittoFilter.ts';
 import { findReplyTag, getTagSet } from '@/tags.ts';
+import { hydrateEvents } from '@/storages/hydrate.ts';
 
 const debug = Debug('ditto:queries');
 
@@ -22,76 +22,25 @@ const getEvent = async (
   opts: GetEventOpts = {},
 ): Promise<DittoEvent | undefined> => {
   debug(`getEvent: ${id}`);
-  const { kind, relations, signal = AbortSignal.timeout(1000) } = opts;
+  const { kind, relations = [], signal = AbortSignal.timeout(1000) } = opts;
-  const microfilter: IdMicrofilter = { ids: [id] };
 
-  const [memoryEvent] = await cache.query([microfilter]) as DittoEvent[];
+  const filter: NostrFilter = { ids: [id], limit: 1 };
-
-  if (memoryEvent && !relations) {
-    debug(`getEvent: ${id.slice(0, 8)} found in memory`);
-    return memoryEvent;
-  }
-
-  const filter: DittoFilter = { ids: [id], relations, limit: 1 };
   if (kind) {
     filter.kinds = [kind];
   }
 
-  const dbEvent = await eventsDB.query([filter], { limit: 1, signal })
+  return await optimizer.query([filter], { limit: 1, signal })
+    .then(([event]) => hydrateEvents({ events: [event], relations, storage: optimizer, signal }))
     .then(([event]) => event);
-
-  // TODO: make this DRY-er.
-
-  if (dbEvent && !dbEvent.author) {
-    const [author] = await cache.query([{ kinds: [0], authors: [dbEvent.pubkey] }]);
-    dbEvent.author = author;
-  }
-
-  if (dbEvent) {
-    debug(`getEvent: ${id.slice(0, 8)} found in db`);
-    return dbEvent;
-  }
-
-  if (memoryEvent && !memoryEvent.author) {
-    const [author] = await cache.query([{ kinds: [0], authors: [memoryEvent.pubkey] }]);
-    memoryEvent.author = author;
-  }
-
-  if (memoryEvent) {
-    debug(`getEvent: ${id.slice(0, 8)} found in memory`);
-    return memoryEvent;
-  }
-
-  const reqEvent = await reqmeister.req(microfilter, opts).catch(() => undefined);
-
-  if (reqEvent) {
-    debug(`getEvent: ${id.slice(0, 8)} found by reqmeister`);
-    return reqEvent;
-  }
-
-  debug(`getEvent: ${id.slice(0, 8)} not found`);
 };
 
 /** Get a Nostr `set_medatadata` event for a user's pubkey. */
 const getAuthor = async (pubkey: string, opts: GetEventOpts = {}): Promise<NostrEvent | undefined> => {
-  const { relations, signal = AbortSignal.timeout(1000) } = opts;
+  const { relations = [], signal = AbortSignal.timeout(1000) } = opts;
-  const microfilter: AuthorMicrofilter = { kinds: [0], authors: [pubkey] };
 
-  const [memoryEvent] = await cache.query([microfilter]);
+  return await optimizer.query([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, signal })
-
+    .then(([event]) => hydrateEvents({ events: [event], relations, storage: optimizer, signal }))
-  if (memoryEvent && !relations) {
+    .then(([event]) => event);
-    return memoryEvent;
-  }
-
-  const dbEvent = await eventsDB.query(
-    [{ authors: [pubkey], relations, kinds: [0], limit: 1 }],
-    { limit: 1, signal },
-  ).then(([event]) => event);
-
-  if (dbEvent) return dbEvent;
-  if (memoryEvent) return memoryEvent;
-
-  return reqmeister.req(microfilter, opts).catch(() => undefined);
 };
 
 /** Get users the given pubkey follows. */
@@ -132,10 +81,10 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi
 }
 
 function getDescendants(eventId: string, signal = AbortSignal.timeout(2000)): Promise<NostrEvent[]> {
-  return eventsDB.query(
+  return eventsDB.query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal })
-    [{ kinds: [1], '#e': [eventId], relations: ['author', 'event_stats', 'author_stats'] }],
+    .then((events) =>
-    { limit: 200, signal },
+      hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal })
-  );
+    );
 }
 
 /** Returns whether the pubkey is followed by a local user. */

src/storages/events-db.ts

@@ -198,49 +198,6 @@ class EventsDB implements NStore {
         .where('users.d_tag', filter.local ? 'is not' : 'is', null);
     }
 
-    if (filter.relations?.includes('author')) {
-      query = query
-        .leftJoin(
-          (eb) =>
-            eb
-              .selectFrom('events')
-              .selectAll()
-              .where('kind', '=', 0)
-              .groupBy('pubkey')
-              .as('authors'),
-          (join) => join.onRef('authors.pubkey', '=', 'events.pubkey'),
-        )
-        .select([
-          'authors.id as author_id',
-          'authors.kind as author_kind',
-          'authors.pubkey as author_pubkey',
-          'authors.content as author_content',
-          'authors.tags as author_tags',
-          'authors.created_at as author_created_at',
-          'authors.sig as author_sig',
-        ]);
-    }
-
-    if (filter.relations?.includes('author_stats')) {
-      query = query
-        .leftJoin('author_stats', 'author_stats.pubkey', 'events.pubkey')
-        .select((eb) => [
-          eb.fn.coalesce('author_stats.followers_count', eb.val(0)).as('author_stats_followers_count'),
-          eb.fn.coalesce('author_stats.following_count', eb.val(0)).as('author_stats_following_count'),
-          eb.fn.coalesce('author_stats.notes_count', eb.val(0)).as('author_stats_notes_count'),
-        ]);
-    }
-
-    if (filter.relations?.includes('event_stats')) {
-      query = query
-        .leftJoin('event_stats', 'event_stats.event_id', 'events.id')
-        .select((eb) => [
-          eb.fn.coalesce('event_stats.replies_count', eb.val(0)).as('stats_replies_count'),
-          eb.fn.coalesce('event_stats.reposts_count', eb.val(0)).as('stats_reposts_count'),
-          eb.fn.coalesce('event_stats.reactions_count', eb.val(0)).as('stats_reactions_count'),
-        ]);
-    }
-
     if (filter.search) {
       query = query
         .innerJoin('events_fts', 'events_fts.id', 'events.id')

src/storages/hydrate.ts

@@ -13,13 +13,13 @@ interface HydrateEventOpts {
 async function hydrateEvents(opts: HydrateEventOpts): Promise<DittoEvent[]> {
   const { events, relations, storage, signal } = opts;
 
-  if (events.length === 0) {
+  if (!events.length || !relations.length) {
     return events;
   }
 
   if (relations.includes('author')) {
     const pubkeys = new Set([...events].map((event) => event.pubkey));
-    const authors = await storage.query([{ kinds: [0], authors: [...pubkeys] }], { signal });
+    const authors = await storage.query([{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }], { signal });
 
     for (const event of events) {
       event.author = authors.find((author) => author.pubkey === event.pubkey);

src/storages/optimizer.ts

@@ -23,7 +23,7 @@ class Optimizer implements NStore {
     this.#client = opts.client;
   }
 
-  async event(event: DittoEvent, opts?: NStoreOpts | undefined): Promise<void> {
+  async event(event: DittoEvent, opts?: NStoreOpts): Promise<void> {
     if (opts?.signal?.aborted) return Promise.reject(abortError());
 
     await Promise.all([

src/views.ts

@@ -4,6 +4,7 @@ import { eventsDB } from '@/storages.ts';
 import { renderAccount } from '@/views/mastodon/accounts.ts';
 import { renderStatus } from '@/views/mastodon/statuses.ts';
 import { paginated, paginationSchema } from '@/utils/api.ts';
+import { hydrateEvents } from '@/storages/hydrate.ts';
 
 /** Render account objects for the author of each event. */
 async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal = AbortSignal.timeout(1000)) {
@@ -18,10 +19,8 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal
     return c.json([]);
   }
 
-  const authors = await eventsDB.query(
+  const authors = await eventsDB.query([{ kinds: [0], authors: [...pubkeys] }], { signal })
-    [{ kinds: [0], authors: [...pubkeys], relations: ['author_stats'] }],
+    .then((events) => hydrateEvents({ events, relations: ['author_stats'], storage: eventsDB, signal }));
-    { signal },
-  );
 
   const accounts = await Promise.all(
     authors.map((event) => renderAccount(event)),
@@ -33,10 +32,8 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal
 async function renderAccounts(c: AppContext, authors: string[], signal = AbortSignal.timeout(1000)) {
   const { since, until, limit } = paginationSchema.parse(c.req.query());
 
-  const events = await eventsDB.query(
+  const events = await eventsDB.query([{ kinds: [0], authors, since, until, limit }], { signal })
-    [{ kinds: [0], authors, relations: ['author_stats'], since, until, limit }],
+    .then((events) => hydrateEvents({ events, relations: ['author_stats'], storage: eventsDB, signal }));
-    { signal },
-  );
 
   const accounts = await Promise.all(
     events.map((event) => renderAccount(event)),
@@ -53,10 +50,10 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal
 
   const { limit } = paginationSchema.parse(c.req.query());
 
-  const events = await eventsDB.query(
+  const events = await eventsDB.query([{ kinds: [1], ids, limit }], { signal })
-    [{ kinds: [1], ids, relations: ['author', 'event_stats', 'author_stats'], limit }],
+    .then((events) =>
-    { signal },
+      hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal })
-  );
+    );
 
   if (!events.length) {
     return c.json([]);