Improve TrendsDB, add loopback script, almost ready to do something?

data/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

deno.json

@@ -3,7 +3,7 @@
   "lock": false,
   "tasks": {
     "dev": "deno run --allow-read --allow-env --allow-net --allow-ffi --unstable --watch src/server.ts",
-    "test": "deno test"
+    "test": "deno test -A --unstable src"
   },
   "imports": {
     "@/": "./src/"

src/app.ts

@@ -1,5 +1,6 @@
 import { type Context, cors, type Handler, Hono, type HonoEnv, logger, type MiddlewareHandler } from '@/deps.ts';
 import { type Event } from '@/event.ts';
+import '@/loopback.ts';
 
 import {
   accountController,

src/config.ts

@@ -4,7 +4,11 @@ const Conf = {
     return Deno.env.get('DITTO_NSEC');
   },
   get relay() {
-    return Deno.env.get('DITTO_RELAY');
+    const value = Deno.env.get('DITTO_RELAY');
+    if (!value) {
+      throw new Error('Missing DITTO_RELAY');
+    }
+    return value;
   },
   get localDomain() {
     return Deno.env.get('LOCAL_DOMAIN') || 'http://localhost:8000';

src/db.ts

@@ -1,9 +1,10 @@
 import { createPentagon, z } from '@/deps.ts';
+import { hexIdSchema } from '@/schema.ts';
 
 const kv = await Deno.openKv();
 
 const userSchema = z.object({
-  pubkey: z.string().regex(/^[0-9a-f]{64}$/).describe('primary'),
+  pubkey: hexIdSchema.describe('primary'),
   username: z.string().regex(/^\w{1,30}$/).describe('unique'),
   createdAt: z.date(),
 });

src/deps-test.ts

@@ -0,0 +1 @@
+export { assert, assertEquals, assertThrows } from 'https://deno.land/std@0.177.0/testing/asserts.ts';

src/deps.ts

@@ -20,6 +20,7 @@ export {
   nip05,
   nip19,
   nip21,
+  relayInit,
   verifySignature,
 } from 'npm:nostr-tools@^1.11.2';
 export { findReplyTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts';
@@ -40,3 +41,4 @@ export { default as ISO6391 } from 'npm:iso-639-1@2.1.15';
 export { Dongoose } from 'https://raw.githubusercontent.com/alexgleason/dongoose/68b7ad9dd7b6ec0615e246a9f1603123c1709793/mod.ts';
 export { createPentagon } from 'https://deno.land/x/pentagon@v0.1.1/mod.ts';
 export { DB as Sqlite } from 'https://deno.land/x/sqlite@v3.7.0/mod.ts';
+export { serve } from 'https://deno.land/std@0.177.0/http/server.ts';

src/loopback.ts

@@ -0,0 +1,24 @@
+import { Conf } from '@/config.ts';
+import { relayInit, Sqlite } from '@/deps.ts';
+import { TrendsDB } from '@/trends.ts';
+
+const db = new Sqlite('data/trends.sqlite3');
+const trends = new TrendsDB(db);
+
+const relay = relayInit(Conf.relay);
+await relay.connect();
+
+const sub = relay.sub([{ kinds: [1] }]);
+
+sub.on('eose', sub.unsub);
+sub.on('event', (event) => {
+  const tags = event.tags
+    .filter((tag) => tag[0] === 't')
+    .map((tag) => tag[1]);
+
+  try {
+    trends.addTagUsages(event.pubkey, tags);
+  } catch (_e) {
+    // do nothing
+  }
+});

src/schema.ts

@@ -67,15 +67,15 @@ const relaySchema = z.custom<URL>((relay) => {
   }
 });
 
-const nostrIdSchema = z.string().regex(/^[0-9a-f]{64}$/);
+const hexIdSchema = z.string().regex(/^[0-9a-f]{64}$/);
 
 const eventSchema = z.object({
-  id: nostrIdSchema,
+  id: hexIdSchema,
   kind: z.number(),
   tags: z.array(z.array(z.string())),
   content: z.string(),
   created_at: z.number(),
-  pubkey: nostrIdSchema,
+  pubkey: hexIdSchema,
   sig: z.string(),
 });
 
@@ -95,10 +95,14 @@ const decode64Schema = z.string().transform((value, ctx) => {
   }
 });
 
+const hashtagSchema = z.string().regex(/^\w{1,30}$/);
+
 export {
   decode64Schema,
   emojiTagSchema,
   filteredArray,
+  hashtagSchema,
+  hexIdSchema,
   jsonSchema,
   type MetaContent,
   metaContentSchema,

src/server.ts

@@ -1,5 +1,5 @@
 import 'https://deno.land/std@0.177.0/dotenv/load.ts';
-import { serve } from 'https://deno.land/std@0.177.0/http/server.ts';
+import { serve } from '@/deps.ts';
 
 import app from './app.ts';
 

src/trends.test.ts

@@ -0,0 +1,24 @@
+import { assertEquals } from '@/deps-test.ts';
+import { Sqlite } from '@/deps.ts';
+
+import { TrendsDB } from './trends.ts';
+
+const db = new Sqlite(':memory:');
+const trends = new TrendsDB(db);
+
+const p8 = (pubkey8: string) => `${pubkey8}00000000000000000000000000000000000000000000000000000000`;
+
+Deno.test('getTrendingTags', () => {
+  trends.addTagUsages(p8('00000000'), ['ditto', 'hello', 'yolo']);
+  trends.addTagUsages(p8('00000001'), ['Ditto', 'hello']);
+  trends.addTagUsages(p8('00000010'), ['DITTO']);
+
+  const result = trends.getTrendingTags(
+    new Date('1999-01-01T00:00:00'),
+    new Date('2999-01-01T00:00:00'),
+  );
+
+  assertEquals(result, ['ditto', 'hello', 'yolo']);
+
+  trends.cleanupTagUsages(new Date('2999-01-01T00:00:00'));
+});

src/trends.ts

@@ -1,4 +1,5 @@
 import { Sqlite } from '@/deps.ts';
+import { hashtagSchema, hexIdSchema } from '@/schema.ts';
 
 class TrendsDB {
   #db: Sqlite;
@@ -8,9 +9,9 @@ class TrendsDB {
 
     this.#db.execute(`
       CREATE TABLE IF NOT EXISTS tag_usages (
-        tag TEXT NOT NULL,
+        tag TEXT NOT NULL COLLATE NOCASE,
         pubkey8 TEXT NOT NULL,
-        inserted_at DATETIME NOT NULL,
+        inserted_at DATETIME NOT NULL
       );
 
       CREATE INDEX IF NOT EXISTS idx_time_tag ON tag_usages(inserted_at, tag);
@@ -31,10 +32,14 @@ class TrendsDB {
     ).map((row) => row[0]);
   }
 
-  addTagUsage(tag: string, pubkey8: string): void {
+  addTagUsages(pubkey: string, hashtags: string[]): void {
+    const pubkey8 = hexIdSchema.parse(pubkey).substring(0, 8);
+    const tags = hashtagSchema.array().parse(hashtags);
+    const now = new Date();
+
     this.#db.query(
-      'INSERT INTO tag_usages (tag, pubkey8, inserted_at) VALUES (?, ?, ?)',
+      'INSERT INTO tag_usages (tag, pubkey8, inserted_at) VALUES ' + tags.map(() => '(?, ?, ?)').join(', '),
-      [tag, pubkey8, new Date()],
+      tags.map((tag) => [tag, pubkey8, now]).flat(),
     );
   }