Merge branch 'counters' into 'develop'

Make status counters work

See merge request soapbox-pub/ditto!27

src/controllers/api/accounts.ts

@@ -5,7 +5,7 @@ import { getAuthor, getFollows, syncUser } from '@/queries.ts';
 import { booleanParamSchema } from '@/schema.ts';
 import { jsonMetaContentSchema } from '@/schemas/nostr.ts';
 import { toAccount, toRelationship, toStatus } from '@/transformers/nostr-to-mastoapi.ts';
-import { eventDateComparator, isFollowing, lookupAccount } from '@/utils.ts';
+import { isFollowing, lookupAccount } from '@/utils.ts';
 import { paginated, paginationSchema, parseBody } from '@/utils/web.ts';
 import { createEvent } from '@/utils/web.ts';
 
@@ -103,13 +103,12 @@ const accountStatusesController: AppController = async (c) => {
   }
 
   let events = await mixer.getFilters([filter]);
-  events.sort(eventDateComparator);
 
   if (exclude_replies) {
     events = events.filter((event) => !findReplyTag(event));
   }
 
-  const statuses = await Promise.all(events.map(toStatus));
+  const statuses = await Promise.all(events.map((event) => toStatus(event, c.get('pubkey'))));
   return paginated(c, events, statuses);
 };
 

src/controllers/api/notifications.ts

@@ -13,7 +13,7 @@ const notificationsController: AppController = async (c) => {
     { timeout: Time.seconds(3) },
   );
 
-  const statuses = await Promise.all(events.map(toNotification));
+  const statuses = await Promise.all(events.map((event) => toNotification(event, pubkey)));
   return paginated(c, events, statuses);
 };
 

src/controllers/api/statuses.ts

@@ -1,5 +1,5 @@
 import { type AppController } from '@/app.ts';
-import { ISO6391, Kind, z } from '@/deps.ts';
+import { type Event, ISO6391, z } from '@/deps.ts';
 import { getAncestors, getDescendants, getEvent } from '@/queries.ts';
 import { toStatus } from '@/transformers/nostr-to-mastoapi.ts';
 import { createEvent, parseBody } from '@/utils/web.ts';
@@ -29,7 +29,7 @@ const statusController: AppController = async (c) => {
 
   const event = await getEvent(id, { kind: 1 });
   if (event) {
-    return c.json(await toStatus(event));
+    return c.json(await toStatus(event, c.get('pubkey')));
   }
 
   return c.json({ error: 'Event not found.' }, 404);
@@ -69,12 +69,12 @@ const createStatusController: AppController = async (c) => {
     }
 
     const event = await createEvent({
-      kind: Kind.Text,
+      kind: 1,
       content: data.status ?? '',
       tags,
     }, c);
 
-    return c.json(await toStatus(event));
+    return c.json(await toStatus(event, c.get('pubkey')));
   } else {
     return c.json({ error: 'Bad request', schema: result.error }, 400);
   }
@@ -82,17 +82,20 @@ const createStatusController: AppController = async (c) => {
 
 const contextController: AppController = async (c) => {
   const id = c.req.param('id');
-
   const event = await getEvent(id, { kind: 1 });
 
-  if (event) {
-    const ancestorEvents = await getAncestors(event);
-    const descendantEvents = await getDescendants(event.id);
+  async function renderStatuses(events: Event<1>[]) {
+    const statuses = await Promise.all(events.map((event) => toStatus(event, c.get('pubkey'))));
+    return statuses.filter(Boolean);
+  }
 
-    return c.json({
-      ancestors: (await Promise.all(ancestorEvents.map(toStatus))).filter(Boolean),
-      descendants: (await Promise.all(descendantEvents.map(toStatus))).filter(Boolean),
-    });
+  if (event) {
+    const [ancestors, descendants] = await Promise.all([
+      getAncestors(event).then(renderStatuses),
+      getDescendants(event.id).then(renderStatuses),
+    ]);
+
+    return c.json({ ancestors, descendants });
   }
 
   return c.json({ error: 'Event not found.' }, 404);
@@ -104,7 +107,7 @@ const favouriteController: AppController = async (c) => {
 
   if (target) {
     await createEvent({
-      kind: Kind.Reaction,
+      kind: 7,
       content: '+',
       tags: [
         ['e', target.id],
@@ -112,7 +115,7 @@ const favouriteController: AppController = async (c) => {
       ],
     }, c);
 
-    const status = await toStatus(target);
+    const status = await toStatus(target, c.get('pubkey'));
 
     if (status) {
       status.favourited = true;

src/controllers/api/streaming.ts

@@ -63,7 +63,7 @@ const streamingController: AppController = (c) => {
 
     if (filter) {
       for await (const event of Sub.sub(socket, '1', [filter])) {
-        const status = await toStatus(event);
+        const status = await toStatus(event, pubkey);
         if (status) {
           send('update', status);
         }

src/controllers/api/timelines.ts

@@ -40,7 +40,7 @@ async function renderStatuses(c: AppContext, filters: DittoFilter<1>[]) {
     return c.json([]);
   }
 
-  const statuses = await Promise.all(events.map(toStatus));
+  const statuses = await Promise.all(events.map((event) => toStatus(event, c.get('pubkey'))));
   return paginated(c, events, statuses);
 }
 

src/controllers/nostr/relay.ts

@@ -3,6 +3,7 @@ import * as pipeline from '@/pipeline.ts';
 import { jsonSchema } from '@/schema.ts';
 import {
   type ClientCLOSE,
+  type ClientCOUNT,
   type ClientEVENT,
   type ClientMsg,
   clientMsgSchema,
@@ -13,7 +14,7 @@ import { Sub } from '@/subs.ts';
 import type { AppController } from '@/app.ts';
 import type { Event, Filter } from '@/deps.ts';
 
-/** Limit of events returned per-filter. */
+/** Limit of initial events returned for a subscription. */
 const FILTER_LIMIT = 100;
 
 /** NIP-01 relay to client message. */
@@ -21,7 +22,8 @@ type RelayMsg =
   | ['EVENT', string, Event]
   | ['NOTICE', string]
   | ['EOSE', string]
-  | ['OK', string, boolean, string];
+  | ['OK', string, boolean, string]
+  | ['COUNT', string, { count: number; approximate?: boolean }];
 
 /** Set up the Websocket connection. */
 function connectStream(socket: WebSocket) {
@@ -50,6 +52,9 @@ function connectStream(socket: WebSocket) {
       case 'CLOSE':
         handleClose(msg);
         return;
+      case 'COUNT':
+        handleCount(msg);
+        return;
     }
   }
 
@@ -57,7 +62,7 @@ function connectStream(socket: WebSocket) {
   async function handleReq([_, subId, ...rest]: ClientREQ): Promise<void> {
     const filters = prepareFilters(rest);
 
-    for (const event of await eventsDB.getFilters(filters)) {
+    for (const event of await eventsDB.getFilters(filters, { limit: FILTER_LIMIT })) {
       send(['EVENT', subId, event]);
     }
 
@@ -88,6 +93,12 @@ function connectStream(socket: WebSocket) {
     Sub.unsub(socket, subId);
   }
 
+  /** Handle COUNT. Return the number of events matching the filters. */
+  async function handleCount([_, subId, ...rest]: ClientCOUNT): Promise<void> {
+    const count = await eventsDB.countFilters(prepareFilters(rest));
+    send(['COUNT', subId, { count, approximate: false }]);
+  }
+
   /** Send a message back to the client. */
   function send(msg: RelayMsg): void {
     if (socket.readyState === WebSocket.OPEN) {
@@ -100,8 +111,6 @@ function connectStream(socket: WebSocket) {
 function prepareFilters(filters: ClientREQ[2][]): Filter[] {
   return filters.map((filter) => ({
     ...filter,
-    // Limit the number of events returned per-filter.
-    limit: Math.min(filter.limit || FILTER_LIMIT, FILTER_LIMIT),
     // Return only local events unless the query is already narrow.
     local: !filter.ids?.length && !filter.authors?.length,
   }));

src/db/events.ts

@@ -131,4 +131,16 @@ async function getFilters<K extends number>(
   ));
 }
 
-export { getFilters, insertEvent };
+async function countFilters<K extends number>(filters: DittoFilter<K>[]): Promise<number> {
+  if (!filters.length) return Promise.resolve(0);
+  const query = filters.map(getFilterQuery).reduce((acc, curr) => acc.union(curr));
+
+  const [{ count }] = await query
+    .clearSelect()
+    .select((eb) => eb.fn.count('id').as('count'))
+    .execute();
+
+  return Number(count);
+}
+
+export { countFilters, getFilters, insertEvent };

src/deps.ts

@@ -18,7 +18,6 @@ export {
   getEventHash,
   getPublicKey,
   getSignature,
-  Kind,
   matchFilters,
   nip04,
   nip05,

src/firehose.ts

@@ -1,6 +1,5 @@
 import { getActiveRelays } from '@/db/relays.ts';
 import { type Event, RelayPool } from '@/deps.ts';
-import { nostrNow } from '@/utils.ts';
 
 import * as pipeline from './pipeline.ts';
 
@@ -11,7 +10,7 @@ const pool = new RelayPool(relays);
 // side-effects based on them, such as trending hashtag tracking
 // and storing events for notifications and the home feed.
 pool.subscribe(
-  [{ kinds: [0, 1, 3, 5, 6, 7, 10002], since: nostrNow() }],
+  [{ kinds: [0, 1, 3, 5, 6, 7, 10002], limit: 0 }],
   relays,
   handleEvent,
   undefined,

src/schemas/nostr.ts

@@ -39,12 +39,14 @@ const filterSchema = z.object({
 const clientReqSchema = z.tuple([z.literal('REQ'), z.string().min(1)]).rest(filterSchema);
 const clientEventSchema = z.tuple([z.literal('EVENT'), signedEventSchema]);
 const clientCloseSchema = z.tuple([z.literal('CLOSE'), z.string().min(1)]);
+const clientCountSchema = z.tuple([z.literal('COUNT'), z.string().min(1)]).rest(filterSchema);
 
 /** Client message to a Nostr relay. */
 const clientMsgSchema = z.union([
   clientReqSchema,
   clientEventSchema,
   clientCloseSchema,
+  clientCountSchema,
 ]);
 
 /** REQ message from client to relay. */
@@ -53,6 +55,8 @@ type ClientREQ = z.infer<typeof clientReqSchema>;
 type ClientEVENT = z.infer<typeof clientEventSchema>;
 /** CLOSE message from client to relay. */
 type ClientCLOSE = z.infer<typeof clientCloseSchema>;
+/** COUNT message from client to relay. */
+type ClientCOUNT = z.infer<typeof clientCountSchema>;
 /** Client message to a Nostr relay. */
 type ClientMsg = z.infer<typeof clientMsgSchema>;
 
@@ -88,6 +92,7 @@ const connectResponseSchema = z.object({
 
 export {
   type ClientCLOSE,
+  type ClientCOUNT,
   type ClientEVENT,
   type ClientMsg,
   clientMsgSchema,

src/transformers/nostr-to-mastoapi.ts

@@ -1,6 +1,7 @@
 import { isCWTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts';
 
 import { Conf } from '@/config.ts';
+import * as eventsDB from '@/db/events.ts';
 import { type Event, findReplyTag, lodash, nip19, sanitizeHtml, TTLCache, unfurl, z } from '@/deps.ts';
 import { verifyNip05Cached } from '@/nip05.ts';
 import { getMediaLinks, type MediaLink, parseNoteContent } from '@/note.ts';
@@ -91,7 +92,7 @@ async function toMention(pubkey: string) {
   }
 }
 
-async function toStatus(event: Event<1>) {
+async function toStatus(event: Event<1>, viewerPubkey?: string) {
   const profile = await getAuthor(event.pubkey);
   const account = profile ? await toAccount(profile) : undefined;
   if (!account) return;
@@ -109,9 +110,19 @@ async function toStatus(event: Event<1>) {
   const { html, links, firstUrl } = parseNoteContent(event.content);
   const mediaLinks = getMediaLinks(links);
 
-  const [mentions, card] = await Promise.all([
+  const [mentions, card, repliesCount, reblogsCount, favouritesCount, [repostEvent], [reactionEvent]] = await Promise
+    .all([
       Promise.all(mentionedPubkeys.map(toMention)),
-    firstUrl ? await unfurlCardCached(firstUrl) : null,
+      firstUrl ? unfurlCardCached(firstUrl) : null,
+      eventsDB.countFilters([{ kinds: [1], '#e': [event.id] }]),
+      eventsDB.countFilters([{ kinds: [6], '#e': [event.id] }]),
+      eventsDB.countFilters([{ kinds: [7], '#e': [event.id] }]),
+      viewerPubkey
+        ? eventsDB.getFilters([{ kinds: [6], '#e': [event.id], authors: [viewerPubkey] }], { limit: 1 })
+        : [],
+      viewerPubkey
+        ? eventsDB.getFilters([{ kinds: [7], '#e': [event.id], authors: [viewerPubkey] }], { limit: 1 })
+        : [],
     ]);
 
   const content = buildInlineRecipients(mentions) + html;
@@ -131,11 +142,11 @@ async function toStatus(event: Event<1>) {
     spoiler_text: (cw ? cw[1] : subject?.[1]) || '',
     visibility: 'public',
     language: event.tags.find((tag) => tag[0] === 'lang')?.[1] || null,
-    replies_count: 0,
-    reblogs_count: 0,
-    favourites_count: 0,
-    favourited: false,
-    reblogged: false,
+    replies_count: repliesCount,
+    reblogs_count: reblogsCount,
+    favourites_count: favouritesCount,
+    favourited: reactionEvent?.content === '+',
+    reblogged: Boolean(repostEvent),
     muted: false,
     bookmarked: false,
     reblog: null,
@@ -276,15 +287,15 @@ async function toRelationship(sourcePubkey: string, targetPubkey: string) {
   };
 }
 
-function toNotification(event: Event) {
+function toNotification(event: Event, viewerPubkey?: string) {
   switch (event.kind) {
     case 1:
-      return toNotificationMention(event as Event<1>);
+      return toNotificationMention(event as Event<1>, viewerPubkey);
   }
 }
 
-async function toNotificationMention(event: Event<1>) {
-  const status = await toStatus(event);
+async function toNotificationMention(event: Event<1>, viewerPubkey?: string) {
+  const status = await toStatus(event, viewerPubkey);
   if (!status) return;
 
   return {