From 9ebf83af5dd0cab383ac74fa778eb723c28388ed Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Mon, 24 Jul 2023 22:38:36 -0500
Subject: [PATCH 01/18] Add preliminary TrendsDB module

---
 src/deps.ts   |  1 +
 src/trends.ts | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 src/trends.ts

diff --git a/src/deps.ts b/src/deps.ts
index 91833db..c930339 100644
--- a/src/deps.ts
+++ b/src/deps.ts
@@ -39,3 +39,4 @@ export { default as sanitizeHtml } from 'npm:sanitize-html@^2.10.0';
 export { default as ISO6391 } from 'npm:iso-639-1@2.1.15';
 export { Dongoose } from 'https://raw.githubusercontent.com/alexgleason/dongoose/68b7ad9dd7b6ec0615e246a9f1603123c1709793/mod.ts';
 export { createPentagon } from 'https://deno.land/x/pentagon@v0.1.1/mod.ts';
+export { DB as Sqlite } from 'https://deno.land/x/sqlite@v3.7.0/mod.ts';
diff --git a/src/trends.ts b/src/trends.ts
new file mode 100644
index 0000000..15b80d7
--- /dev/null
+++ b/src/trends.ts
@@ -0,0 +1,46 @@
+import { Sqlite } from '@/deps.ts';
+
+class TrendsDB {
+  #db: Sqlite;
+
+  constructor(db: Sqlite) {
+    this.#db = db;
+
+    this.#db.execute(`
+      CREATE TABLE IF NOT EXISTS tag_usages (
+        tag TEXT NOT NULL,
+        pubkey8 TEXT NOT NULL,
+        inserted_at DATETIME NOT NULL,
+      );
+
+      CREATE INDEX IF NOT EXISTS idx_time_tag ON tag_usages(inserted_at, tag);
+    `);
+  }
+
+  getTrendingTags(): string[] {
+    return this.#db.query<string[]>(`
+      SELECT tag, COUNT(DISTINCT pubkey8)
+        FROM tag_usages
+        WHERE inserted_at >= $1 AND inserted_at < $2
+        GROUP BY tag
+        ORDER BY COUNT(DISTINCT pubkey8)
+        DESC LIMIT 10;
+    `).map((row) => row[0]);
+  }
+
+  addTagUsage(tag: string, pubkey8: string): void {
+    this.#db.query(
+      'INSERT INTO tag_usages (tag, pubkey8, inserted_at) VALUES (?, ?, ?)',
+      [tag, pubkey8, new Date()],
+    );
+  }
+
+  cleanupTagUsages(): void {
+    this.#db.query(
+      'DELETE FROM tag_usages WHERE inserted_at < ?',
+      [new Date(Date.now() - 1000 * 60 * 60 * 24 * 7)],
+    );
+  }
+}
+
+export { TrendsDB };

From 3bdde98f8fa5638b11b15eb72e3428e5f384940c Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 12:10:37 -0500
Subject: [PATCH 02/18] TrendsDB: fix parameterized queries

---
 src/trends.ts | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/trends.ts b/src/trends.ts
index 15b80d7..f35ddea 100644
--- a/src/trends.ts
+++ b/src/trends.ts
@@ -17,15 +17,18 @@ class TrendsDB {
     `);
   }
 
-  getTrendingTags(): string[] {
-    return this.#db.query<string[]>(`
+  getTrendingTags(since: Date, until: Date): string[] {
+    return this.#db.query<string[]>(
+      `
       SELECT tag, COUNT(DISTINCT pubkey8)
         FROM tag_usages
-        WHERE inserted_at >= $1 AND inserted_at < $2
+        WHERE inserted_at >= ? AND inserted_at < ?
         GROUP BY tag
         ORDER BY COUNT(DISTINCT pubkey8)
         DESC LIMIT 10;
-    `).map((row) => row[0]);
+    `,
+      [since, until],
+    ).map((row) => row[0]);
   }
 
   addTagUsage(tag: string, pubkey8: string): void {
@@ -35,10 +38,10 @@ class TrendsDB {
     );
   }
 
-  cleanupTagUsages(): void {
+  cleanupTagUsages(until: Date): void {
     this.#db.query(
       'DELETE FROM tag_usages WHERE inserted_at < ?',
-      [new Date(Date.now() - 1000 * 60 * 60 * 24 * 7)],
+      [until],
     );
   }
 }

From 11f21e392285ef183096372cc6cce2b18f41d7ab Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 15:30:58 -0500
Subject: [PATCH 03/18] Improve TrendsDB, add loopback script, almost ready to
 do something?

---
 data/.gitignore    |  2 ++
 deno.json          |  2 +-
 src/app.ts         |  1 +
 src/config.ts      |  6 +++++-
 src/db.ts          |  3 ++-
 src/deps-test.ts   |  1 +
 src/deps.ts        |  2 ++
 src/loopback.ts    | 24 ++++++++++++++++++++++++
 src/schema.ts      | 10 +++++++---
 src/server.ts      |  2 +-
 src/trends.test.ts | 24 ++++++++++++++++++++++++
 src/trends.ts      | 15 ++++++++++-----
 12 files changed, 80 insertions(+), 12 deletions(-)
 create mode 100644 data/.gitignore
 create mode 100644 src/deps-test.ts
 create mode 100644 src/loopback.ts
 create mode 100644 src/trends.test.ts

diff --git a/data/.gitignore b/data/.gitignore
new file mode 100644
index 0000000..c96a04f
--- /dev/null
+++ b/data/.gitignore
@@ -0,0 +1,2 @@
+*
+!.gitignore
\ No newline at end of file
diff --git a/deno.json b/deno.json
index 8b2a01d..bf7bc4f 100644
--- a/deno.json
+++ b/deno.json
@@ -3,7 +3,7 @@
   "lock": false,
   "tasks": {
     "dev": "deno run --allow-read --allow-env --allow-net --allow-ffi --unstable --watch src/server.ts",
-    "test": "deno test"
+    "test": "deno test -A --unstable src"
   },
   "imports": {
     "@/": "./src/"
diff --git a/src/app.ts b/src/app.ts
index 8a90aba..a2f43c3 100644
--- a/src/app.ts
+++ b/src/app.ts
@@ -1,5 +1,6 @@
 import { type Context, cors, type Handler, Hono, type HonoEnv, logger, type MiddlewareHandler } from '@/deps.ts';
 import { type Event } from '@/event.ts';
+import '@/loopback.ts';
 
 import {
   accountController,
diff --git a/src/config.ts b/src/config.ts
index 4183d96..ac6a296 100644
--- a/src/config.ts
+++ b/src/config.ts
@@ -4,7 +4,11 @@ const Conf = {
     return Deno.env.get('DITTO_NSEC');
   },
   get relay() {
-    return Deno.env.get('DITTO_RELAY');
+    const value = Deno.env.get('DITTO_RELAY');
+    if (!value) {
+      throw new Error('Missing DITTO_RELAY');
+    }
+    return value;
   },
   get localDomain() {
     return Deno.env.get('LOCAL_DOMAIN') || 'http://localhost:8000';
diff --git a/src/db.ts b/src/db.ts
index 5f13af7..8a22426 100644
--- a/src/db.ts
+++ b/src/db.ts
@@ -1,9 +1,10 @@
 import { createPentagon, z } from '@/deps.ts';
+import { hexIdSchema } from '@/schema.ts';
 
 const kv = await Deno.openKv();
 
 const userSchema = z.object({
-  pubkey: z.string().regex(/^[0-9a-f]{64}$/).describe('primary'),
+  pubkey: hexIdSchema.describe('primary'),
   username: z.string().regex(/^\w{1,30}$/).describe('unique'),
   createdAt: z.date(),
 });
diff --git a/src/deps-test.ts b/src/deps-test.ts
new file mode 100644
index 0000000..e57b4ad
--- /dev/null
+++ b/src/deps-test.ts
@@ -0,0 +1 @@
+export { assert, assertEquals, assertThrows } from 'https://deno.land/std@0.177.0/testing/asserts.ts';
diff --git a/src/deps.ts b/src/deps.ts
index c930339..52ce68c 100644
--- a/src/deps.ts
+++ b/src/deps.ts
@@ -20,6 +20,7 @@ export {
   nip05,
   nip19,
   nip21,
+  relayInit,
   verifySignature,
 } from 'npm:nostr-tools@^1.11.2';
 export { findReplyTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts';
@@ -40,3 +41,4 @@ export { default as ISO6391 } from 'npm:iso-639-1@2.1.15';
 export { Dongoose } from 'https://raw.githubusercontent.com/alexgleason/dongoose/68b7ad9dd7b6ec0615e246a9f1603123c1709793/mod.ts';
 export { createPentagon } from 'https://deno.land/x/pentagon@v0.1.1/mod.ts';
 export { DB as Sqlite } from 'https://deno.land/x/sqlite@v3.7.0/mod.ts';
+export { serve } from 'https://deno.land/std@0.177.0/http/server.ts';
diff --git a/src/loopback.ts b/src/loopback.ts
new file mode 100644
index 0000000..30256de
--- /dev/null
+++ b/src/loopback.ts
@@ -0,0 +1,24 @@
+import { Conf } from '@/config.ts';
+import { relayInit, Sqlite } from '@/deps.ts';
+import { TrendsDB } from '@/trends.ts';
+
+const db = new Sqlite('data/trends.sqlite3');
+const trends = new TrendsDB(db);
+
+const relay = relayInit(Conf.relay);
+await relay.connect();
+
+const sub = relay.sub([{ kinds: [1] }]);
+
+sub.on('eose', sub.unsub);
+sub.on('event', (event) => {
+  const tags = event.tags
+    .filter((tag) => tag[0] === 't')
+    .map((tag) => tag[1]);
+
+  try {
+    trends.addTagUsages(event.pubkey, tags);
+  } catch (_e) {
+    // do nothing
+  }
+});
diff --git a/src/schema.ts b/src/schema.ts
index 8027023..635dff8 100644
--- a/src/schema.ts
+++ b/src/schema.ts
@@ -67,15 +67,15 @@ const relaySchema = z.custom<URL>((relay) => {
   }
 });
 
-const nostrIdSchema = z.string().regex(/^[0-9a-f]{64}$/);
+const hexIdSchema = z.string().regex(/^[0-9a-f]{64}$/);
 
 const eventSchema = z.object({
-  id: nostrIdSchema,
+  id: hexIdSchema,
   kind: z.number(),
   tags: z.array(z.array(z.string())),
   content: z.string(),
   created_at: z.number(),
-  pubkey: nostrIdSchema,
+  pubkey: hexIdSchema,
   sig: z.string(),
 });
 
@@ -95,10 +95,14 @@ const decode64Schema = z.string().transform((value, ctx) => {
   }
 });
 
+const hashtagSchema = z.string().regex(/^\w{1,30}$/);
+
 export {
   decode64Schema,
   emojiTagSchema,
   filteredArray,
+  hashtagSchema,
+  hexIdSchema,
   jsonSchema,
   type MetaContent,
   metaContentSchema,
diff --git a/src/server.ts b/src/server.ts
index a7bd083..c9d3f4f 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -1,5 +1,5 @@
 import 'https://deno.land/std@0.177.0/dotenv/load.ts';
-import { serve } from 'https://deno.land/std@0.177.0/http/server.ts';
+import { serve } from '@/deps.ts';
 
 import app from './app.ts';
 
diff --git a/src/trends.test.ts b/src/trends.test.ts
new file mode 100644
index 0000000..76de876
--- /dev/null
+++ b/src/trends.test.ts
@@ -0,0 +1,24 @@
+import { assertEquals } from '@/deps-test.ts';
+import { Sqlite } from '@/deps.ts';
+
+import { TrendsDB } from './trends.ts';
+
+const db = new Sqlite(':memory:');
+const trends = new TrendsDB(db);
+
+const p8 = (pubkey8: string) => `${pubkey8}00000000000000000000000000000000000000000000000000000000`;
+
+Deno.test('getTrendingTags', () => {
+  trends.addTagUsages(p8('00000000'), ['ditto', 'hello', 'yolo']);
+  trends.addTagUsages(p8('00000001'), ['Ditto', 'hello']);
+  trends.addTagUsages(p8('00000010'), ['DITTO']);
+
+  const result = trends.getTrendingTags(
+    new Date('1999-01-01T00:00:00'),
+    new Date('2999-01-01T00:00:00'),
+  );
+
+  assertEquals(result, ['ditto', 'hello', 'yolo']);
+
+  trends.cleanupTagUsages(new Date('2999-01-01T00:00:00'));
+});
diff --git a/src/trends.ts b/src/trends.ts
index f35ddea..2575851 100644
--- a/src/trends.ts
+++ b/src/trends.ts
@@ -1,4 +1,5 @@
 import { Sqlite } from '@/deps.ts';
+import { hashtagSchema, hexIdSchema } from '@/schema.ts';
 
 class TrendsDB {
   #db: Sqlite;
@@ -8,9 +9,9 @@ class TrendsDB {
 
     this.#db.execute(`
       CREATE TABLE IF NOT EXISTS tag_usages (
-        tag TEXT NOT NULL,
+        tag TEXT NOT NULL COLLATE NOCASE,
         pubkey8 TEXT NOT NULL,
-        inserted_at DATETIME NOT NULL,
+        inserted_at DATETIME NOT NULL
       );
 
       CREATE INDEX IF NOT EXISTS idx_time_tag ON tag_usages(inserted_at, tag);
@@ -31,10 +32,14 @@ class TrendsDB {
     ).map((row) => row[0]);
   }
 
-  addTagUsage(tag: string, pubkey8: string): void {
+  addTagUsages(pubkey: string, hashtags: string[]): void {
+    const pubkey8 = hexIdSchema.parse(pubkey).substring(0, 8);
+    const tags = hashtagSchema.array().parse(hashtags);
+    const now = new Date();
+
     this.#db.query(
-      'INSERT INTO tag_usages (tag, pubkey8, inserted_at) VALUES (?, ?, ?)',
-      [tag, pubkey8, new Date()],
+      'INSERT INTO tag_usages (tag, pubkey8, inserted_at) VALUES ' + tags.map(() => '(?, ?, ?)').join(', '),
+      tags.map((tag) => [tag, pubkey8, now]).flat(),
     );
   }
 

From 1d67181e5299537fff3706e134510821eaf3db7b Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 17:07:09 -0500
Subject: [PATCH 04/18] Add trends controller... and it kind of works!

---
 deno.json                     |  2 +-
 src/app.ts                    |  4 ++++
 src/controllers/api/trends.ts | 23 +++++++++++++++++++++++
 src/loopback.ts               | 12 +++++-------
 src/trends.test.ts            |  8 +++++++-
 src/trends.ts                 | 15 +++++++++++----
 6 files changed, 51 insertions(+), 13 deletions(-)
 create mode 100644 src/controllers/api/trends.ts

diff --git a/deno.json b/deno.json
index bf7bc4f..f0c10e6 100644
--- a/deno.json
+++ b/deno.json
@@ -2,7 +2,7 @@
   "$schema": "https://deno.land/x/deno@v1.32.3/cli/schemas/config-file.v1.json",
   "lock": false,
   "tasks": {
-    "dev": "deno run --allow-read --allow-env --allow-net --allow-ffi --unstable --watch src/server.ts",
+    "dev": "deno run --allow-read --allow-write --allow-env --allow-net --allow-ffi --unstable --watch src/server.ts",
     "test": "deno test -A --unstable src"
   },
   "imports": {
diff --git a/src/app.ts b/src/app.ts
index a2f43c3..af95656 100644
--- a/src/app.ts
+++ b/src/app.ts
@@ -27,6 +27,7 @@ import {
   statusController,
 } from './controllers/api/statuses.ts';
 import { streamingController } from './controllers/api/streaming.ts';
+import { trendingTagsController } from './controllers/api/trends.ts';
 import { indexController } from './controllers/site.ts';
 import { hostMetaController } from './controllers/well-known/host-meta.ts';
 import { nodeInfoController, nodeInfoSchemaController } from './controllers/well-known/nodeinfo.ts';
@@ -101,6 +102,9 @@ app.get('/api/v2/search', searchController);
 
 app.get('/api/pleroma/frontend_configurations', frontendConfigController);
 
+app.get('/api/v1/trends/tags', trendingTagsController);
+app.get('/api/v1/trends', trendingTagsController);
+
 // Not (yet) implemented.
 app.get('/api/v1/notifications', emptyArrayController);
 app.get('/api/v1/bookmarks', emptyArrayController);
diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts
new file mode 100644
index 0000000..f88b486
--- /dev/null
+++ b/src/controllers/api/trends.ts
@@ -0,0 +1,23 @@
+import { type AppController } from '@/app.ts';
+import { Conf } from '@/config.ts';
+import { trends } from '@/trends.ts';
+import { Time } from '@/utils.ts';
+
+const trendingTagsController: AppController = (c) => {
+  const yesterday = new Date(new Date().getTime() - Time.days(1));
+  const now = new Date();
+
+  const tags = trends.getTrendingTags(yesterday, now);
+
+  return c.json(tags.map(({ name, accounts }) => ({
+    name,
+    url: Conf.local(`/tags/${name}`),
+    history: [{
+      day: String(Math.floor(yesterday.getTime() / 1000)),
+      uses: String(accounts), // Not actually true - we don't collect this
+      accounts: String(accounts),
+    }],
+  })));
+};
+
+export { trendingTagsController };
diff --git a/src/loopback.ts b/src/loopback.ts
index 30256de..b971501 100644
--- a/src/loopback.ts
+++ b/src/loopback.ts
@@ -1,17 +1,15 @@
 import { Conf } from '@/config.ts';
-import { relayInit, Sqlite } from '@/deps.ts';
-import { TrendsDB } from '@/trends.ts';
-
-const db = new Sqlite('data/trends.sqlite3');
-const trends = new TrendsDB(db);
+import { relayInit } from '@/deps.ts';
+import { trends } from '@/trends.ts';
+import { nostrNow } from '@/utils.ts';
 
 const relay = relayInit(Conf.relay);
 await relay.connect();
 
-const sub = relay.sub([{ kinds: [1] }]);
+const sub = relay.sub([{ kinds: [1], since: nostrNow() }]);
 
-sub.on('eose', sub.unsub);
 sub.on('event', (event) => {
+  console.info('loopback event:', event.id);
   const tags = event.tags
     .filter((tag) => tag[0] === 't')
     .map((tag) => tag[1]);
diff --git a/src/trends.test.ts b/src/trends.test.ts
index 76de876..3f7eaeb 100644
--- a/src/trends.test.ts
+++ b/src/trends.test.ts
@@ -18,7 +18,13 @@ Deno.test('getTrendingTags', () => {
     new Date('2999-01-01T00:00:00'),
   );
 
-  assertEquals(result, ['ditto', 'hello', 'yolo']);
+  const expected = [
+    { name: 'ditto', accounts: 3 },
+    { name: 'hello', accounts: 2 },
+    { name: 'yolo', accounts: 1 },
+  ];
+
+  assertEquals(result, expected);
 
   trends.cleanupTagUsages(new Date('2999-01-01T00:00:00'));
 });
diff --git a/src/trends.ts b/src/trends.ts
index 2575851..4831168 100644
--- a/src/trends.ts
+++ b/src/trends.ts
@@ -18,7 +18,7 @@ class TrendsDB {
     `);
   }
 
-  getTrendingTags(since: Date, until: Date): string[] {
+  getTrendingTags(since: Date, until: Date) {
     return this.#db.query<string[]>(
       `
       SELECT tag, COUNT(DISTINCT pubkey8)
@@ -29,12 +29,15 @@ class TrendsDB {
         DESC LIMIT 10;
     `,
       [since, until],
-    ).map((row) => row[0]);
+    ).map((row) => ({
+      name: row[0],
+      accounts: Number(row[1]),
+    }));
   }
 
   addTagUsages(pubkey: string, hashtags: string[]): void {
     const pubkey8 = hexIdSchema.parse(pubkey).substring(0, 8);
-    const tags = hashtagSchema.array().parse(hashtags);
+    const tags = hashtagSchema.array().min(1).parse(hashtags);
     const now = new Date();
 
     this.#db.query(
@@ -51,4 +54,8 @@ class TrendsDB {
   }
 }
 
-export { TrendsDB };
+const trends = new TrendsDB(
+  new Sqlite('data/trends.sqlite3'),
+);
+
+export { trends, TrendsDB };

From e8df41183402c257e1ee24a850fdc194afe983eb Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 17:19:31 -0500
Subject: [PATCH 05/18] Trends: also track total tag usages

---
 src/controllers/api/trends.ts | 6 +++---
 src/trends.ts                 | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts
index f88b486..d4a4f7e 100644
--- a/src/controllers/api/trends.ts
+++ b/src/controllers/api/trends.ts
@@ -4,17 +4,17 @@ import { trends } from '@/trends.ts';
 import { Time } from '@/utils.ts';
 
 const trendingTagsController: AppController = (c) => {
-  const yesterday = new Date(new Date().getTime() - Time.days(1));
   const now = new Date();
+  const yesterday = new Date(now.getTime() - Time.days(1));
 
   const tags = trends.getTrendingTags(yesterday, now);
 
-  return c.json(tags.map(({ name, accounts }) => ({
+  return c.json(tags.map(({ name, accounts, uses }) => ({
     name,
     url: Conf.local(`/tags/${name}`),
     history: [{
       day: String(Math.floor(yesterday.getTime() / 1000)),
-      uses: String(accounts), // Not actually true - we don't collect this
+      uses: String(uses),
       accounts: String(accounts),
     }],
   })));
diff --git a/src/trends.ts b/src/trends.ts
index 4831168..e112673 100644
--- a/src/trends.ts
+++ b/src/trends.ts
@@ -21,7 +21,7 @@ class TrendsDB {
   getTrendingTags(since: Date, until: Date) {
     return this.#db.query<string[]>(
       `
-      SELECT tag, COUNT(DISTINCT pubkey8)
+      SELECT tag, COUNT(DISTINCT pubkey8), COUNT(*)
         FROM tag_usages
         WHERE inserted_at >= ? AND inserted_at < ?
         GROUP BY tag
@@ -32,6 +32,7 @@ class TrendsDB {
     ).map((row) => ({
       name: row[0],
       accounts: Number(row[1]),
+      uses: Number(row[2]),
     }));
   }
 

From 48b7310d52727a5f29c63a290a4f193909926516 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 17:33:16 -0500
Subject: [PATCH 06/18] Trends: clean up old tag usages

---
 src/trends.ts | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/trends.ts b/src/trends.ts
index e112673..86a2873 100644
--- a/src/trends.ts
+++ b/src/trends.ts
@@ -1,5 +1,6 @@
 import { Sqlite } from '@/deps.ts';
 import { hashtagSchema, hexIdSchema } from '@/schema.ts';
+import { Time } from './utils.ts';
 
 class TrendsDB {
   #db: Sqlite;
@@ -16,6 +17,12 @@ class TrendsDB {
 
       CREATE INDEX IF NOT EXISTS idx_time_tag ON tag_usages(inserted_at, tag);
     `);
+
+    setInterval(() => {
+      console.info('Cleaning up old tag usages...');
+      const lastWeek = new Date(new Date().getTime() - Time.days(7));
+      this.cleanupTagUsages(lastWeek);
+    }, Time.days(1));
   }
 
   getTrendingTags(since: Date, until: Date) {

From cbb294dbc21dc6128d3cc7221f3a4ee928d7d332 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 17:44:38 -0500
Subject: [PATCH 07/18] Trends: support `limit` param

---
 src/controllers/api/trends.ts | 8 +++++++-
 src/trends.ts                 | 6 +++---
 src/utils.ts                  | 2 +-
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts
index d4a4f7e..a8d74bc 100644
--- a/src/controllers/api/trends.ts
+++ b/src/controllers/api/trends.ts
@@ -1,13 +1,19 @@
 import { type AppController } from '@/app.ts';
 import { Conf } from '@/config.ts';
+import { z } from '@/deps.ts';
 import { trends } from '@/trends.ts';
 import { Time } from '@/utils.ts';
 
+const limitSchema = z.coerce.number().catch(10).transform((value) => Math.min(Math.max(value, 0), 20));
+
 const trendingTagsController: AppController = (c) => {
+  const limit = limitSchema.parse(c.req.query('limit'));
+  if (limit < 1) return c.json([]);
+
   const now = new Date();
   const yesterday = new Date(now.getTime() - Time.days(1));
 
-  const tags = trends.getTrendingTags(yesterday, now);
+  const tags = trends.getTrendingTags(yesterday, now, limit);
 
   return c.json(tags.map(({ name, accounts, uses }) => ({
     name,
diff --git a/src/trends.ts b/src/trends.ts
index 86a2873..2c28396 100644
--- a/src/trends.ts
+++ b/src/trends.ts
@@ -25,7 +25,7 @@ class TrendsDB {
     }, Time.days(1));
   }
 
-  getTrendingTags(since: Date, until: Date) {
+  getTrendingTags(since: Date, until: Date, limit = 10) {
     return this.#db.query<string[]>(
       `
       SELECT tag, COUNT(DISTINCT pubkey8), COUNT(*)
@@ -33,9 +33,9 @@ class TrendsDB {
         WHERE inserted_at >= ? AND inserted_at < ?
         GROUP BY tag
         ORDER BY COUNT(DISTINCT pubkey8)
-        DESC LIMIT 10;
+        DESC LIMIT ?;
     `,
-      [since, until],
+      [since, until, limit],
     ).map((row) => ({
       name: row[0],
       accounts: Number(row[1]),
diff --git a/src/utils.ts b/src/utils.ts
index 7a91fc2..a81c0fd 100644
--- a/src/utils.ts
+++ b/src/utils.ts
@@ -81,7 +81,7 @@ async function parseBody(req: Request): Promise<unknown> {
 const paginationSchema = z.object({
   since: z.coerce.number().optional().catch(undefined),
   until: z.lazy(() => z.coerce.number().catch(nostrNow())),
-  limit: z.coerce.number().min(0).max(40).catch(20),
+  limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)),
 });
 
 type PaginationParams = z.infer<typeof paginationSchema>;

From 969bc1fcbaa14a64537eb4246d48021fa332e95a Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 17:49:42 -0500
Subject: [PATCH 08/18] TrendsDB: cleanup on construction

---
 src/trends.ts | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/trends.ts b/src/trends.ts
index 2c28396..9a04fa1 100644
--- a/src/trends.ts
+++ b/src/trends.ts
@@ -18,11 +18,14 @@ class TrendsDB {
       CREATE INDEX IF NOT EXISTS idx_time_tag ON tag_usages(inserted_at, tag);
     `);
 
-    setInterval(() => {
+    const cleanup = () => {
       console.info('Cleaning up old tag usages...');
       const lastWeek = new Date(new Date().getTime() - Time.days(7));
       this.cleanupTagUsages(lastWeek);
-    }, Time.days(1));
+    };
+
+    setInterval(cleanup, Time.hours(1));
+    cleanup();
   }
 
   getTrendingTags(since: Date, until: Date, limit = 10) {

From 33f87822d4ceb6fea6c07a761df4788d45949bd2 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 18:04:00 -0500
Subject: [PATCH 09/18] Trends: support author threshold (default: 3)

---
 src/controllers/api/trends.ts |  6 +++++-
 src/trends.test.ts            | 16 +++++++++-------
 src/trends.ts                 | 12 ++++++++++--
 3 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts
index a8d74bc..99c2d7b 100644
--- a/src/controllers/api/trends.ts
+++ b/src/controllers/api/trends.ts
@@ -13,7 +13,11 @@ const trendingTagsController: AppController = (c) => {
   const now = new Date();
   const yesterday = new Date(now.getTime() - Time.days(1));
 
-  const tags = trends.getTrendingTags(yesterday, now, limit);
+  const tags = trends.getTrendingTags({
+    since: yesterday,
+    until: now,
+    limit,
+  });
 
   return c.json(tags.map(({ name, accounts, uses }) => ({
     name,
diff --git a/src/trends.test.ts b/src/trends.test.ts
index 3f7eaeb..dbc6d7c 100644
--- a/src/trends.test.ts
+++ b/src/trends.test.ts
@@ -10,18 +10,20 @@ const p8 = (pubkey8: string) => `${pubkey8}0000000000000000000000000000000000000
 
 Deno.test('getTrendingTags', () => {
   trends.addTagUsages(p8('00000000'), ['ditto', 'hello', 'yolo']);
+  trends.addTagUsages(p8('00000000'), ['hello']);
   trends.addTagUsages(p8('00000001'), ['Ditto', 'hello']);
   trends.addTagUsages(p8('00000010'), ['DITTO']);
 
-  const result = trends.getTrendingTags(
-    new Date('1999-01-01T00:00:00'),
-    new Date('2999-01-01T00:00:00'),
-  );
+  const result = trends.getTrendingTags({
+    since: new Date('1999-01-01T00:00:00'),
+    until: new Date('2999-01-01T00:00:00'),
+    threshold: 1,
+  });
 
   const expected = [
-    { name: 'ditto', accounts: 3 },
-    { name: 'hello', accounts: 2 },
-    { name: 'yolo', accounts: 1 },
+    { name: 'ditto', accounts: 3, uses: 3 },
+    { name: 'hello', accounts: 2, uses: 3 },
+    { name: 'yolo', accounts: 1, uses: 1 },
   ];
 
   assertEquals(result, expected);
diff --git a/src/trends.ts b/src/trends.ts
index 9a04fa1..4a471ce 100644
--- a/src/trends.ts
+++ b/src/trends.ts
@@ -2,6 +2,13 @@ import { Sqlite } from '@/deps.ts';
 import { hashtagSchema, hexIdSchema } from '@/schema.ts';
 import { Time } from './utils.ts';
 
+interface GetTrendingTagsOpts {
+  since: Date;
+  until: Date;
+  limit?: number;
+  threshold?: number;
+}
+
 class TrendsDB {
   #db: Sqlite;
 
@@ -28,17 +35,18 @@ class TrendsDB {
     cleanup();
   }
 
-  getTrendingTags(since: Date, until: Date, limit = 10) {
+  getTrendingTags({ since, until, limit = 10, threshold = 3 }: GetTrendingTagsOpts) {
     return this.#db.query<string[]>(
       `
       SELECT tag, COUNT(DISTINCT pubkey8), COUNT(*)
         FROM tag_usages
         WHERE inserted_at >= ? AND inserted_at < ?
         GROUP BY tag
+        HAVING COUNT(DISTINCT pubkey8) >= ?
         ORDER BY COUNT(DISTINCT pubkey8)
         DESC LIMIT ?;
     `,
-      [since, until, limit],
+      [since, until, threshold, limit],
     ).map((row) => ({
       name: row[0],
       accounts: Number(row[1]),

From 79ec5dd4e0677197d8d074def52e6cb1b2cabf9b Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 18:22:05 -0500
Subject: [PATCH 10/18] Refactor loopback.ts

---
 src/loopback.ts | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/loopback.ts b/src/loopback.ts
index b971501..d7b8069 100644
--- a/src/loopback.ts
+++ b/src/loopback.ts
@@ -3,13 +3,26 @@ import { relayInit } from '@/deps.ts';
 import { trends } from '@/trends.ts';
 import { nostrNow } from '@/utils.ts';
 
+import type { Event } from '@/event.ts';
+
 const relay = relayInit(Conf.relay);
 await relay.connect();
 
-const sub = relay.sub([{ kinds: [1], since: nostrNow() }]);
+// This file watches all events on your Ditto relay and triggers
+// side-effects based on them. This can be used for things like
+// notifications, trending hashtag tracking, etc.
+relay
+  .sub([{ kinds: [1], since: nostrNow() }])
+  .on('event', handleEvent);
 
-sub.on('event', (event) => {
+/** Handle events through the loopback pipeline. */
+function handleEvent(event: Event): void {
   console.info('loopback event:', event.id);
+  trackHashtags(event);
+}
+
+/** Track whenever a hashtag is used, for processing trending tags. */
+function trackHashtags(event: Event): void {
   const tags = event.tags
     .filter((tag) => tag[0] === 't')
     .map((tag) => tag[1]);
@@ -19,4 +32,4 @@ sub.on('event', (event) => {
   } catch (_e) {
     // do nothing
   }
-});
+}

From 7c8aa8806993e903eede11abb0066a1a357bb069 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 18:35:07 -0500
Subject: [PATCH 11/18] Trends: track based on the event's created_at date,
 instead of whenever the row was inserted

---
 src/loopback.ts | 6 ++++--
 src/trends.ts   | 5 ++---
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/loopback.ts b/src/loopback.ts
index d7b8069..013c951 100644
--- a/src/loopback.ts
+++ b/src/loopback.ts
@@ -1,7 +1,7 @@
 import { Conf } from '@/config.ts';
 import { relayInit } from '@/deps.ts';
 import { trends } from '@/trends.ts';
-import { nostrNow } from '@/utils.ts';
+import { nostrDate, nostrNow } from '@/utils.ts';
 
 import type { Event } from '@/event.ts';
 
@@ -23,12 +23,14 @@ function handleEvent(event: Event): void {
 
 /** Track whenever a hashtag is used, for processing trending tags. */
 function trackHashtags(event: Event): void {
+  const date = nostrDate(event.created_at);
+
   const tags = event.tags
     .filter((tag) => tag[0] === 't')
     .map((tag) => tag[1]);
 
   try {
-    trends.addTagUsages(event.pubkey, tags);
+    trends.addTagUsages(event.pubkey, tags, date);
   } catch (_e) {
     // do nothing
   }
diff --git a/src/trends.ts b/src/trends.ts
index 4a471ce..a74fd46 100644
--- a/src/trends.ts
+++ b/src/trends.ts
@@ -54,14 +54,13 @@ class TrendsDB {
     }));
   }
 
-  addTagUsages(pubkey: string, hashtags: string[]): void {
+  addTagUsages(pubkey: string, hashtags: string[], date = new Date()): void {
     const pubkey8 = hexIdSchema.parse(pubkey).substring(0, 8);
     const tags = hashtagSchema.array().min(1).parse(hashtags);
-    const now = new Date();
 
     this.#db.query(
       'INSERT INTO tag_usages (tag, pubkey8, inserted_at) VALUES ' + tags.map(() => '(?, ?, ?)').join(', '),
-      tags.map((tag) => [tag, pubkey8, now]).flat(),
+      tags.map((tag) => [tag, pubkey8, date]).flat(),
     );
   }
 

From 1a860adde7cf9c30702c6f20a2f04029c2d2f4ce Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 20:26:49 -0500
Subject: [PATCH 12/18] Support tag history

---
 src/controllers/api/trends.ts | 13 +++++++------
 src/trends.ts                 | 17 +++++++++++++++++
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts
index 99c2d7b..2590d4c 100644
--- a/src/controllers/api/trends.ts
+++ b/src/controllers/api/trends.ts
@@ -12,6 +12,7 @@ const trendingTagsController: AppController = (c) => {
 
   const now = new Date();
   const yesterday = new Date(now.getTime() - Time.days(1));
+  const lastWeek = new Date(now.getTime() - Time.days(7));
 
   const tags = trends.getTrendingTags({
     since: yesterday,
@@ -19,14 +20,14 @@ const trendingTagsController: AppController = (c) => {
     limit,
   });
 
-  return c.json(tags.map(({ name, accounts, uses }) => ({
+  return c.json(tags.map(({ name }) => ({
     name,
     url: Conf.local(`/tags/${name}`),
-    history: [{
-      day: String(Math.floor(yesterday.getTime() / 1000)),
-      uses: String(uses),
-      accounts: String(accounts),
-    }],
+    history: trends.getTagHistory(name, lastWeek, now).map((history) => ({
+      day: String(Math.floor(history.day.getTime() / 1000)),
+      accounts: String(history.accounts),
+      uses: String(history.uses),
+    })),
   })));
 };
 
diff --git a/src/trends.ts b/src/trends.ts
index a74fd46..f266a1f 100644
--- a/src/trends.ts
+++ b/src/trends.ts
@@ -54,6 +54,23 @@ class TrendsDB {
     }));
   }
 
+  getTagHistory(tag: string, since: Date, until: Date) {
+    return this.#db.query<string[]>(
+      `
+      SELECT inserted_at, COUNT(DISTINCT pubkey8), COUNT(*)
+        FROM tag_usages
+        WHERE tag = ? AND inserted_at >= ? AND inserted_at < ?
+        GROUP BY date(inserted_at)
+        ORDER BY date(inserted_at);
+    `,
+      [tag, since, until],
+    ).map((row) => ({
+      day: new Date(row[0]),
+      accounts: Number(row[1]),
+      uses: Number(row[2]),
+    }));
+  }
+
   addTagUsages(pubkey: string, hashtags: string[], date = new Date()): void {
     const pubkey8 = hexIdSchema.parse(pubkey).substring(0, 8);
     const tags = hashtagSchema.array().min(1).parse(hashtags);

From ad48e4a787e744e04d62cbeb6c38fca2a34ad09d Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 20:40:31 -0500
Subject: [PATCH 13/18] Trends: add limit param, fix order

---
 src/trends.ts | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/trends.ts b/src/trends.ts
index f266a1f..959d40a 100644
--- a/src/trends.ts
+++ b/src/trends.ts
@@ -54,16 +54,17 @@ class TrendsDB {
     }));
   }
 
-  getTagHistory(tag: string, since: Date, until: Date) {
+  getTagHistory(tag: string, since: Date, until: Date, limit = 7) {
     return this.#db.query<string[]>(
       `
       SELECT inserted_at, COUNT(DISTINCT pubkey8), COUNT(*)
         FROM tag_usages
         WHERE tag = ? AND inserted_at >= ? AND inserted_at < ?
         GROUP BY date(inserted_at)
-        ORDER BY date(inserted_at);
+        ORDER BY date(inserted_at) DESC
+        LIMIT ?;
     `,
-      [tag, since, until],
+      [tag, since, until, limit],
     ).map((row) => ({
       day: new Date(row[0]),
       accounts: Number(row[1]),

From 5b536040ced2cd4b69e297cb8894edc0e49d405c Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 20:55:12 -0500
Subject: [PATCH 14/18] Minor logging tweaks

---
 src/client.ts   | 2 +-
 src/loopback.ts | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/client.ts b/src/client.ts
index e922060..378dc9c 100644
--- a/src/client.ts
+++ b/src/client.ts
@@ -172,7 +172,7 @@ function getDescendants(eventId: string): Promise<SignedEvent<1>[]> {
 
 /** Publish an event to the Nostr relay. */
 function publish(event: SignedEvent, relays = Conf.publishRelays): void {
-  console.log('Publishing event', event);
+  console.log('Publishing event', event, relays);
   try {
     getPool().publish(event, relays);
   } catch (e) {
diff --git a/src/loopback.ts b/src/loopback.ts
index 013c951..3359420 100644
--- a/src/loopback.ts
+++ b/src/loopback.ts
@@ -29,7 +29,10 @@ function trackHashtags(event: Event): void {
     .filter((tag) => tag[0] === 't')
     .map((tag) => tag[1]);
 
+  if (!tags.length) return;
+
   try {
+    console.info('tracking tags:', tags);
     trends.addTagUsages(event.pubkey, tags, date);
   } catch (_e) {
     // do nothing

From d7316c5eeb9ff3171ebe40affb01ce9bfac3b7e0 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 20:55:43 -0500
Subject: [PATCH 15/18] GitLab CI: enable test job

---
 .gitlab-ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b9889ba..9b2675e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,6 +14,6 @@ lint:
   stage: test
   script: deno lint
 
-# test:
-#   stage: test
-#   script: deno task test
\ No newline at end of file
+test:
+  stage: test
+  script: deno task test
\ No newline at end of file

From c88e58344ffc8adec9bb7c2e6a05691eac127d9a Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 25 Jul 2023 23:21:42 -0500
Subject: [PATCH 16/18] Improve tag history

---
 src/controllers/api/trends.ts | 25 +++++++++++++++++++------
 src/trends.ts                 | 15 ++++++++++++---
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts
index 2590d4c..cadfecf 100644
--- a/src/controllers/api/trends.ts
+++ b/src/controllers/api/trends.ts
@@ -20,14 +20,27 @@ const trendingTagsController: AppController = (c) => {
     limit,
   });
 
-  return c.json(tags.map(({ name }) => ({
+  return c.json(tags.map(({ name, uses, accounts }) => ({
     name,
     url: Conf.local(`/tags/${name}`),
-    history: trends.getTagHistory(name, lastWeek, now).map((history) => ({
-      day: String(Math.floor(history.day.getTime() / 1000)),
-      accounts: String(history.accounts),
-      uses: String(history.uses),
-    })),
+    history: [
+      {
+        day: String(Math.floor(now.getTime() / 1000)),
+        accounts: String(accounts),
+        uses: String(uses),
+      },
+      ...trends.getTagHistory({
+        tag: name,
+        since: lastWeek,
+        until: now,
+        limit: 6,
+        offset: 1,
+      }).map((history) => ({
+        day: String(Math.floor(history.day.getTime() / 1000)),
+        accounts: String(history.accounts),
+        uses: String(history.uses),
+      })),
+    ],
   })));
 };
 
diff --git a/src/trends.ts b/src/trends.ts
index 959d40a..9ae531b 100644
--- a/src/trends.ts
+++ b/src/trends.ts
@@ -9,6 +9,14 @@ interface GetTrendingTagsOpts {
   threshold?: number;
 }
 
+interface GetTagHistoryOpts {
+  tag: string;
+  since: Date;
+  until: Date;
+  limit?: number;
+  offset?: number;
+}
+
 class TrendsDB {
   #db: Sqlite;
 
@@ -54,7 +62,7 @@ class TrendsDB {
     }));
   }
 
-  getTagHistory(tag: string, since: Date, until: Date, limit = 7) {
+  getTagHistory({ tag, since, until, limit = 7, offset = 0 }: GetTagHistoryOpts) {
     return this.#db.query<string[]>(
       `
       SELECT inserted_at, COUNT(DISTINCT pubkey8), COUNT(*)
@@ -62,9 +70,10 @@ class TrendsDB {
         WHERE tag = ? AND inserted_at >= ? AND inserted_at < ?
         GROUP BY date(inserted_at)
         ORDER BY date(inserted_at) DESC
-        LIMIT ?;
+        LIMIT ?
+        OFFSET ?;
     `,
-      [tag, since, until, limit],
+      [tag, since, until, limit, offset],
     ).map((row) => ({
       day: new Date(row[0]),
       accounts: Number(row[1]),

From 7eedeef2b46b387ff64ea8d41870833d370d5619 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Wed, 26 Jul 2023 09:40:52 -0500
Subject: [PATCH 17/18] Track up to 5 tags max per post

---
 src/loopback.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/loopback.ts b/src/loopback.ts
index 3359420..bddc66e 100644
--- a/src/loopback.ts
+++ b/src/loopback.ts
@@ -27,7 +27,8 @@ function trackHashtags(event: Event): void {
 
   const tags = event.tags
     .filter((tag) => tag[0] === 't')
-    .map((tag) => tag[1]);
+    .map((tag) => tag[1])
+    .slice(0, 5);
 
   if (!tags.length) return;
 

From 14e60048a68a65b773e6298c5ce0fb640ce08f46 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Wed, 26 Jul 2023 12:54:06 -0500
Subject: [PATCH 18/18] loopback: use RelayPool instead of nostr-tools just
 because it doesn't die so often

---
 src/deps.ts     |  1 -
 src/loopback.ts | 15 +++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/deps.ts b/src/deps.ts
index 52ce68c..1f0bdfe 100644
--- a/src/deps.ts
+++ b/src/deps.ts
@@ -20,7 +20,6 @@ export {
   nip05,
   nip19,
   nip21,
-  relayInit,
   verifySignature,
 } from 'npm:nostr-tools@^1.11.2';
 export { findReplyTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts';
diff --git a/src/loopback.ts b/src/loopback.ts
index bddc66e..e9f13fc 100644
--- a/src/loopback.ts
+++ b/src/loopback.ts
@@ -1,19 +1,22 @@
 import { Conf } from '@/config.ts';
-import { relayInit } from '@/deps.ts';
+import { RelayPool } from '@/deps.ts';
 import { trends } from '@/trends.ts';
 import { nostrDate, nostrNow } from '@/utils.ts';
 
 import type { Event } from '@/event.ts';
 
-const relay = relayInit(Conf.relay);
-await relay.connect();
+const relay = new RelayPool([Conf.relay]);
 
 // This file watches all events on your Ditto relay and triggers
 // side-effects based on them. This can be used for things like
 // notifications, trending hashtag tracking, etc.
-relay
-  .sub([{ kinds: [1], since: nostrNow() }])
-  .on('event', handleEvent);
+relay.subscribe(
+  [{ kinds: [1], since: nostrNow() }],
+  [Conf.relay],
+  handleEvent,
+  undefined,
+  undefined,
+);
 
 /** Handle events through the loopback pipeline. */
 function handleEvent(event: Event): void {