Merge branch 'recompute-stats' into 'main'

Recompute author stats

See merge request soapbox-pub/ditto!273

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: denoland/deno:1.41.3
+image: denoland/deno:1.43.3
 
 default:
   interruptible: true

.tool-versions

@@ -1 +1 @@
-deno 1.41.3
+deno 1.43.3

scripts/stats-recompute.ts

@@ -1,8 +1,6 @@
 import { nip19 } from 'nostr-tools';
 
-import { DittoDB } from '@/db/DittoDB.ts';
+import { refreshAuthorStats } from '@/stats.ts';
-import { DittoTables } from '@/db/DittoTables.ts';
-import { Storages } from '@/storages.ts';
 
 let pubkey: string;
 try {
@@ -17,23 +15,4 @@ try {
   Deno.exit(1);
 }
 
-const store = await Storages.db();
+await refreshAuthorStats(pubkey);
-const kysely = await DittoDB.getInstance();
-
-const [followList] = await store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]);
-
-const authorStats: DittoTables['author_stats'] = {
-  pubkey,
-  followers_count: (await store.count([{ kinds: [3], '#p': [pubkey] }])).count,
-  following_count: followList?.tags.filter(([name]) => name === 'p')?.length ?? 0,
-  notes_count: (await store.count([{ kinds: [1], authors: [pubkey] }])).count,
-};
-
-await kysely.insertInto('author_stats')
-  .values(authorStats)
-  .onConflict((oc) =>
-    oc
-      .column('pubkey')
-      .doUpdateSet(authorStats)
-  )
-  .execute();

src/stats.ts

@@ -1,11 +1,13 @@
-import { NKinds, NostrEvent } from '@nostrify/nostrify';
+import { NKinds, NostrEvent, NStore } from '@nostrify/nostrify';
 import Debug from '@soapbox/stickynotes/debug';
 import { InsertQueryBuilder, Kysely } from 'kysely';
+import { LRUCache } from 'lru-cache';
+import { SetRequired } from 'type-fest';
 
 import { DittoDB } from '@/db/DittoDB.ts';
 import { DittoTables } from '@/db/DittoTables.ts';
 import { Storages } from '@/storages.ts';
-import { findReplyTag } from '@/tags.ts';
+import { findReplyTag, getTagSet } from '@/tags.ts';
 
 type AuthorStat = keyof Omit<DittoTables['author_stats'], 'pubkey'>;
 type EventStat = keyof Omit<DittoTables['event_stats'], 'event_id'>;
@@ -216,4 +218,46 @@ function getFollowDiff(event: NostrEvent, prev?: NostrEvent): AuthorStatDiff[] {
   ];
 }
 
-export { updateStats };
+/** Refresh the author's stats in the database. */
+async function refreshAuthorStats(pubkey: string): Promise<DittoTables['author_stats']> {
+  const store = await Storages.db();
+  const stats = await countAuthorStats(store, pubkey);
+
+  const kysely = await DittoDB.getInstance();
+  await kysely.insertInto('author_stats')
+    .values(stats)
+    .onConflict((oc) => oc.column('pubkey').doUpdateSet(stats))
+    .execute();
+
+  return stats;
+}
+
+/** Calculate author stats from the database. */
+async function countAuthorStats(
+  store: SetRequired<NStore, 'count'>,
+  pubkey: string,
+): Promise<DittoTables['author_stats']> {
+  const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([
+    store.count([{ kinds: [3], '#p': [pubkey] }]),
+    store.count([{ kinds: [1], authors: [pubkey] }]),
+    store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]),
+  ]);
+
+  return {
+    pubkey,
+    followers_count,
+    following_count: getTagSet(followList?.tags ?? [], 'p').size,
+    notes_count,
+  };
+}
+
+const lru = new LRUCache<string, true>({ max: 1000 });
+
+/** Calls `refreshAuthorStats` only once per author. */
+function refreshAuthorStatsDebounced(pubkey: string): void {
+  if (lru.get(pubkey)) return;
+  lru.set(pubkey, true);
+  refreshAuthorStats(pubkey).catch(() => {});
+}
+
+export { refreshAuthorStats, refreshAuthorStatsDebounced, updateStats };

src/storages/hydrate.ts

@@ -5,6 +5,7 @@ import { DittoDB } from '@/db/DittoDB.ts';
 import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
 import { DittoTables } from '@/db/DittoTables.ts';
 import { Conf } from '@/config.ts';
+import { refreshAuthorStatsDebounced } from '@/stats.ts';
 
 interface HydrateOpts {
   events: DittoEvent[];
@@ -55,6 +56,8 @@ async function hydrateEvents(opts: HydrateOpts): Promise<DittoEvent[]> {
     events: await gatherEventStats(cache),
   };
 
+  refreshMissingAuthorStats(events, stats.authors);
+
   // Dedupe events.
   const results = [...new Map(cache.map((event) => [event.id, event])).values()];
 
@@ -266,6 +269,22 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise<DittoTables['aut
   }));
 }
 
+function refreshMissingAuthorStats(events: NostrEvent[], stats: DittoTables['author_stats'][]) {
+  const pubkeys = new Set<string>(
+    events
+      .filter((event) => event.kind === 0)
+      .map((event) => event.pubkey),
+  );
+
+  const missing = pubkeys.difference(
+    new Set(stats.map((stat) => stat.pubkey)),
+  );
+
+  for (const pubkey of missing) {
+    refreshAuthorStatsDebounced(pubkey);
+  }
+}
+
 /** Collect event stats from the events. */
 async function gatherEventStats(events: DittoEvent[]): Promise<DittoTables['event_stats'][]> {
   const ids = new Set<string>(