Merge branch 'queries' into 'develop'

Add Queries module

See merge request soapbox-pub/ditto!13

.vscode/settings.json

@@ -1,7 +1,6 @@
 {
   "deno.enable": true,
   "deno.lint": true,
-  "deno.unstable": true,
   "editor.defaultFormatter": "denoland.vscode-deno",
   "path-intellisense.extensionOnImport": true
 }

deno.json

@@ -2,12 +2,12 @@
   "$schema": "https://deno.land/x/deno@v1.32.3/cli/schemas/config-file.v1.json",
   "lock": false,
   "tasks": {
-    "start": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable src/server.ts",
+    "start": "deno run --allow-read --allow-write=data --allow-env --allow-net src/server.ts",
-    "dev": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --watch src/server.ts",
+    "dev": "deno run --allow-read --allow-write=data --allow-env --allow-net --watch src/server.ts",
-    "debug": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --inspect src/server.ts",
+    "debug": "deno run --allow-read --allow-write=data --allow-env --allow-net --inspect src/server.ts",
-    "test": "DB_PATH=\":memory:\" deno test --allow-read --allow-write=data --allow-env --unstable src",
+    "test": "DB_PATH=\":memory:\" deno test --allow-read --allow-write=data --allow-env src",
-    "check": "deno check --unstable src/server.ts",
+    "check": "deno check src/server.ts",
-    "relays:sync": "deno run -A --unstable scripts/relays.ts sync",
+    "relays:sync": "deno run -A scripts/relays.ts sync"
   },
   "imports": {
     "@/": "./src/",

src/client.ts

@@ -1,11 +1,9 @@
 import { Conf } from '@/config.ts';
-import { Author, type Event, type Filter, findReplyTag, matchFilters, RelayPool, TTLCache } from '@/deps.ts';
+import { type Event, type Filter, matchFilters, RelayPool, TTLCache } from '@/deps.ts';
-import { eventDateComparator, type PaginationParams, Time } from '@/utils.ts';
+import { Time } from '@/utils.ts';
 
 import type { GetFiltersOpts } from '@/types.ts';
 
-const db = await Deno.openKv();
-
 type Pool = InstanceType<typeof RelayPool>;
 
 /** HACK: Websockets in Deno are finnicky... get a new pool every 30 minutes. */
@@ -72,88 +70,6 @@ function getFilters<K extends number>(filters: Filter<K>[], opts: GetFiltersOpts
   });
 }
 
-/** Get a Nostr event by its ID. */
-const getEvent = async <K extends number = number>(id: string, kind?: K): Promise<Event<K> | undefined> => {
-  const event = await (getPool().getEventById(id, Conf.poolRelays, 0) as Promise<Event>);
-  if (event) {
-    if (event.id !== id) return undefined;
-    if (kind && event.kind !== kind) return undefined;
-    return event as Event<K>;
-  }
-};
-
-/** Get a Nostr `set_medatadata` event for a user's pubkey. */
-const getAuthor = async (pubkey: string, timeout = 1000): Promise<Event<0> | undefined> => {
-  const author = new Author(getPool(), Conf.poolRelays, pubkey);
-
-  const event: Event<0> | null = await new Promise((resolve) => {
-    setTimeout(resolve, timeout, null);
-    return author.metaData(resolve, 0);
-  });
-
-  return event?.pubkey === pubkey ? event : undefined;
-};
-
-/** Get users the given pubkey follows. */
-const getFollows = async (pubkey: string): Promise<Event<3> | undefined> => {
-  const [event] = await getFilters([{ authors: [pubkey], kinds: [3] }], { timeout: 5000 });
-
-  // TODO: figure out a better, more generic & flexible way to handle event cache (and timeouts?)
-  // Prewarm cache in GET `/api/v1/accounts/verify_credentials`
-  if (event) {
-    await db.set(['event3', pubkey], event);
-    return event;
-  } else {
-    return (await db.get<Event<3>>(['event3', pubkey])).value || undefined;
-  }
-};
-
-/** Get events from people the user follows. */
-async function getFeed(event3: Event<3>, params: PaginationParams): Promise<Event<1>[]> {
-  const authors = event3.tags
-    .filter((tag) => tag[0] === 'p')
-    .map((tag) => tag[1]);
-
-  authors.push(event3.pubkey); // see own events in feed
-
-  const filter: Filter = {
-    authors,
-    kinds: [1],
-    ...params,
-  };
-
-  const results = await getFilters([filter], { timeout: 5000 }) as Event<1>[];
-  return results.sort(eventDateComparator);
-}
-
-/** Get a feed of all known text notes. */
-async function getPublicFeed(params: PaginationParams): Promise<Event<1>[]> {
-  const results = await getFilters([{ kinds: [1], ...params }], { timeout: 5000 });
-  return results.sort(eventDateComparator);
-}
-
-async function getAncestors(event: Event<1>, result = [] as Event<1>[]): Promise<Event<1>[]> {
-  if (result.length < 100) {
-    const replyTag = findReplyTag(event);
-    const inReplyTo = replyTag ? replyTag[1] : undefined;
-
-    if (inReplyTo) {
-      const parentEvent = await getEvent(inReplyTo, 1);
-
-      if (parentEvent) {
-        result.push(parentEvent);
-        return getAncestors(parentEvent, result);
-      }
-    }
-  }
-
-  return result.reverse();
-}
-
-function getDescendants(eventId: string): Promise<Event<1>[]> {
-  return getFilters([{ kinds: [1], '#e': [eventId] }], { limit: 200, timeout: 2000 }) as Promise<Event<1>[]>;
-}
-
 /** Publish an event to the Nostr relay. */
 function publish(event: Event, relays = Conf.publishRelays): void {
   console.log('Publishing event', event, relays);
@@ -164,4 +80,4 @@ function publish(event: Event, relays = Conf.publishRelays): void {
   }
 }
 
-export { getAncestors, getAuthor, getDescendants, getEvent, getFeed, getFilters, getFollows, getPublicFeed, publish };
+export { getFilters, publish };

src/controllers/activitypub/actor.ts

@@ -1,5 +1,5 @@
-import { getAuthor } from '@/client.ts';
 import { findUser } from '@/db/users.ts';
+import { getAuthor } from '@/queries.ts';
 import { toActor } from '@/transformers/nostr-to-activitypub.ts';
 import { activityJson } from '@/utils.ts';
 

src/controllers/api/accounts.ts

@@ -1,7 +1,8 @@
 import { type AppController } from '@/app.ts';
 import { type Filter, findReplyTag, z } from '@/deps.ts';
-import { getAuthor, getFollows, publish } from '@/client.ts';
+import { publish } from '@/client.ts';
-import { getFilters } from '@/mixer.ts';
+import * as mixer from '@/mixer.ts';
+import { getAuthor, getFollows } from '@/queries.ts';
 import { jsonMetaContentSchema } from '@/schemas/nostr.ts';
 import { signEvent } from '@/sign.ts';
 import { toAccount, toStatus } from '@/transformers/nostr-to-mastoapi.ts';
@@ -116,7 +117,7 @@ const accountStatusesController: AppController = async (c) => {
     filter['#t'] = [tagged];
   }
 
-  let events = await getFilters([filter]);
+  let events = await mixer.getFilters([filter]);
   events.sort(eventDateComparator);
 
   if (exclude_replies) {

src/controllers/api/statuses.ts

@@ -1,6 +1,7 @@
 import { type AppController } from '@/app.ts';
-import { getAncestors, getDescendants, getEvent, publish } from '@/client.ts';
+import { publish } from '@/client.ts';
-import { type Event, ISO6391, Kind, z } from '@/deps.ts';
+import { ISO6391, Kind, z } from '@/deps.ts';
+import { getAncestors, getDescendants, getEvent } from '@/queries.ts';
 import { signEvent } from '@/sign.ts';
 import { toStatus } from '@/transformers/nostr-to-mastoapi.ts';
 import { nostrNow, parseBody } from '@/utils.ts';
@@ -28,9 +29,9 @@ const createStatusSchema = z.object({
 const statusController: AppController = async (c) => {
   const id = c.req.param('id');
 
-  const event = await getEvent(id, 1);
+  const event = await getEvent(id, { kind: 1 });
   if (event) {
-    return c.json(await toStatus(event as Event<1>));
+    return c.json(await toStatus(event));
   }
 
   return c.json({ error: 'Event not found.' }, 404);
@@ -87,7 +88,7 @@ const createStatusController: AppController = async (c) => {
 const contextController: AppController = async (c) => {
   const id = c.req.param('id');
 
-  const event = await getEvent(id, 1);
+  const event = await getEvent(id, { kind: 1 });
 
   if (event) {
     const ancestorEvents = await getAncestors(event);
@@ -104,7 +105,7 @@ const contextController: AppController = async (c) => {
 
 const favouriteController: AppController = async (c) => {
   const id = c.req.param('id');
-  const target = await getEvent(id, 1);
+  const target = await getEvent(id, { kind: 1 });
 
   if (target) {
     const event = await signEvent({

src/controllers/api/timelines.ts

@@ -1,4 +1,4 @@
-import { getFeed, getFollows, getPublicFeed } from '@/client.ts';
+import { getFeed, getPublicFeed } from '@/queries.ts';
 import { toStatus } from '@/transformers/nostr-to-mastoapi.ts';
 import { buildLinkHeader, paginationSchema } from '@/utils.ts';
 
@@ -8,12 +8,7 @@ const homeController: AppController = async (c) => {
   const params = paginationSchema.parse(c.req.query());
   const pubkey = c.get('pubkey')!;
 
-  const follows = await getFollows(pubkey);
+  const events = await getFeed(pubkey, params);
-  if (!follows) {
-    return c.json([]);
-  }
-
-  const events = await getFeed(follows, params);
   if (!events.length) {
     return c.json([]);
   }

src/controllers/nostr/relay.ts

@@ -1,4 +1,4 @@
-import { getFilters, insertEvent } from '@/db/events.ts';
+import * as eventsDB from '@/db/events.ts';
 import { findUser } from '@/db/users.ts';
 import { jsonSchema } from '@/schema.ts';
 import {
@@ -46,7 +46,7 @@ function connectStream(socket: WebSocket) {
   }
 
   async function handleReq([_, sub, ...filters]: ClientREQ) {
-    for (const event of await getFilters(prepareFilters(filters))) {
+    for (const event of await eventsDB.getFilters(prepareFilters(filters))) {
       send(['EVENT', sub, event]);
     }
     send(['EOSE', sub]);
@@ -54,7 +54,7 @@ function connectStream(socket: WebSocket) {
 
   async function handleEvent([_, event]: ClientEVENT) {
     if (await findUser({ pubkey: event.pubkey })) {
-      insertEvent(event);
+      eventsDB.insertEvent(event);
       send(['OK', event.id, true, '']);
     } else {
       send(['OK', event.id, false, 'blocked: only registered users can post']);

src/mixer.ts

@@ -1,7 +1,7 @@
 import { type Event, matchFilters } from '@/deps.ts';
 
-import { getFilters as getFiltersClient } from '@/client.ts';
+import * as client from '@/client.ts';
-import { getFilters as getFiltersDB } from '@/db/events.ts';
+import * as eventsDB from '@/db/events.ts';
 import { eventDateComparator } from '@/utils.ts';
 
 import type { DittoFilter, GetFiltersOpts } from '@/types.ts';
@@ -12,8 +12,8 @@ async function getFilters<K extends number>(
   opts?: GetFiltersOpts,
 ): Promise<Event<K>[]> {
   const results = await Promise.allSettled([
-    getFiltersClient(filters, opts),
+    client.getFilters(filters, opts),
-    getFiltersDB(filters, opts),
+    eventsDB.getFilters(filters, opts),
   ]);
 
   const events = results

src/queries.ts

@@ -0,0 +1,86 @@
+import { type Event, type Filter, findReplyTag } from '@/deps.ts';
+import { type PaginationParams } from '@/utils.ts';
+
+import * as mixer from './mixer.ts';
+
+interface GetEventOpts<K extends number> {
+  /** Timeout in milliseconds. */
+  timeout?: number;
+  /** Event kind. */
+  kind?: K;
+}
+
+/** Get a Nostr event by its ID. */
+const getEvent = async <K extends number = number>(
+  id: string,
+  opts: GetEventOpts<K> = {},
+): Promise<Event<K> | undefined> => {
+  const { kind, timeout = 1000 } = opts;
+  const filter: Filter<K> = { ids: [id], limit: 1 };
+  if (kind) {
+    filter.kinds = [kind];
+  }
+  const [event] = await mixer.getFilters([filter], { limit: 1, timeout });
+  return event;
+};
+
+/** Get a Nostr `set_medatadata` event for a user's pubkey. */
+const getAuthor = async (pubkey: string, timeout = 1000): Promise<Event<0> | undefined> => {
+  const [event] = await mixer.getFilters([{ authors: [pubkey], kinds: [0] }], { timeout });
+  return event;
+};
+
+/** Get users the given pubkey follows. */
+const getFollows = async (pubkey: string, timeout = 1000): Promise<Event<3> | undefined> => {
+  const [event] = await mixer.getFilters([{ authors: [pubkey], kinds: [3] }], { timeout });
+  return event;
+};
+
+/** Get events from people the user follows. */
+async function getFeed(pubkey: string, params: PaginationParams): Promise<Event<1>[]> {
+  const event3 = await getFollows(pubkey);
+  if (!event3) return [];
+
+  const authors = event3.tags
+    .filter((tag) => tag[0] === 'p')
+    .map((tag) => tag[1]);
+
+  authors.push(event3.pubkey); // see own events in feed
+
+  const filter: Filter<1> = {
+    authors,
+    kinds: [1],
+    ...params,
+  };
+
+  return mixer.getFilters([filter], { timeout: 5000 });
+}
+
+/** Get a feed of all known text notes. */
+function getPublicFeed(params: PaginationParams): Promise<Event<1>[]> {
+  return mixer.getFilters([{ kinds: [1], ...params }], { timeout: 5000 });
+}
+
+async function getAncestors(event: Event<1>, result = [] as Event<1>[]): Promise<Event<1>[]> {
+  if (result.length < 100) {
+    const replyTag = findReplyTag(event);
+    const inReplyTo = replyTag ? replyTag[1] : undefined;
+
+    if (inReplyTo) {
+      const parentEvent = await getEvent(inReplyTo, { kind: 1 });
+
+      if (parentEvent) {
+        result.push(parentEvent);
+        return getAncestors(parentEvent, result);
+      }
+    }
+  }
+
+  return result.reverse();
+}
+
+function getDescendants(eventId: string): Promise<Event<1>[]> {
+  return mixer.getFilters([{ kinds: [1], '#e': [eventId] }], { limit: 200, timeout: 2000 });
+}
+
+export { getAncestors, getAuthor, getDescendants, getEvent, getFeed, getFollows, getPublicFeed };

src/transformers/nostr-to-mastoapi.ts

@@ -1,10 +1,10 @@
 import { isCWTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts';
 
-import { getAuthor } from '@/client.ts';
 import { Conf } from '@/config.ts';
 import { type Event, findReplyTag, lodash, nip19, sanitizeHtml, TTLCache, unfurl, z } from '@/deps.ts';
 import { verifyNip05Cached } from '@/nip05.ts';
 import { getMediaLinks, type MediaLink, parseNoteContent } from '@/note.ts';
+import { getAuthor } from '@/queries.ts';
 import { emojiTagSchema, filteredArray } from '@/schema.ts';
 import { jsonMetaContentSchema } from '@/schemas/nostr.ts';
 import { type Nip05, nostrDate, parseNip05, Time } from '@/utils.ts';

src/utils.ts

@@ -1,7 +1,7 @@
-import { getAuthor } from '@/client.ts';
 import { Conf } from '@/config.ts';
 import { type Context, type Event, nip19, parseFormData, z } from '@/deps.ts';
 import { lookupNip05Cached } from '@/nip05.ts';
+import { getAuthor } from '@/queries.ts';
 
 /** Get the current time in Nostr format. */
 const nostrNow = () => Math.floor(new Date().getTime() / 1000);