From c969738736186bd00efeaf6dab2b8a5871d8f9e2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 13:15:05 -0500 Subject: [PATCH 01/16] Save trending hashtags as labels --- src/cron.ts | 26 ++++++++++++++++++++++ src/trends/trending-hashtags.ts | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 src/trends/trending-hashtags.ts diff --git a/src/cron.ts b/src/cron.ts index 707f738..7f8f46a 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -5,6 +5,7 @@ import { getTrendingNotes } from '@/trends/trending-notes.ts'; import { Time } from '@/utils/time.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { handleEvent } from '@/pipeline.ts'; +import { getTrendingHashtags } from '@/trends/trending-hashtags.ts'; const console = new Stickynotes('ditto:trends'); @@ -32,7 +33,32 @@ async function updateTrendingNotesCache() { console.info('Trending notes cache updated.'); } +async function updateTrendingHashtagsCache() { + console.info('Updating trending hashtags cache...'); + const kysely = await DittoDB.getInstance(); + const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); + const signal = AbortSignal.timeout(1000); + + const hashtags = await getTrendingHashtags(kysely, { since: yesterday, limit: 20, threshold: 3 }); + const signer = new AdminSigner(); + + const label = await signer.signEvent({ + kind: 1985, + content: '', + tags: [ + ['L', 'pub.ditto.trends'], + ['l', 'hashtags', 'pub.ditto.trends'], + ...hashtags.map(({ tag }) => ['t', tag]), + ], + created_at: Math.floor(Date.now() / 1000), + }); + + await handleEvent(label, signal); + console.info('Trending hashtags cache updated.'); +} + /** Start cron jobs for the application. */ export function cron() { Deno.cron('update trending notes cache', { minute: { every: 15 } }, updateTrendingNotesCache); + Deno.cron('update trending hashtags cache', { dayOfMonth: { every: 1 } }, updateTrendingHashtagsCache); } diff --git a/src/trends/trending-hashtags.ts b/src/trends/trending-hashtags.ts new file mode 100644 index 0000000..a70acb9 --- /dev/null +++ b/src/trends/trending-hashtags.ts @@ -0,0 +1,38 @@ +import { Kysely } from 'kysely'; + +import { DittoTables } from '@/db/DittoTables.ts'; + +interface GetTrendingHashtagsOpts { + /** Unix timestamp in _seconds_ for the starting point of this query. */ + since: number; + /** Maximum number of trending hashtags to return. */ + limit: number; + /** Minimum number of unique accounts that have used a hashtag to be considered trending. */ + threshold: number; +} + +/** Get the trending hashtags in the given time frame. */ +export async function getTrendingHashtags( + /** Kysely instance to execute queries on. */ + kysely: Kysely, + /** Options for this query. */ + opts: GetTrendingHashtagsOpts, +): Promise<{ tag: string; accounts: number; uses: number }[]> { + const { since, limit, threshold } = opts; + + return await kysely + .selectFrom('nostr_tags') + .innerJoin('nostr_events', 'nostr_events.id', 'nostr_tags.event_id') + .select(({ fn }) => [ + 'nostr_tags.value as tag', + fn.agg('count', ['nostr_events.pubkey']).distinct().as('accounts'), + fn.countAll().as('uses'), + ]) + .where('nostr_tags.name', '=', 't') + .where('nostr_events.created_at', '>', since) + .groupBy('nostr_tags.value') + .having((c) => c(c.fn.agg('count', ['nostr_events.pubkey']).distinct(), '>=', threshold)) + .orderBy((c) => c.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc') + .limit(limit) + .execute(); +} From b5545ddb601fc3fc17f24aa4a7f24e945b8f750a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 16:07:25 -0500 Subject: [PATCH 02/16] getTrendingNotes -> getTrendingEvents --- src/cron.ts | 14 ++++++++--- src/trends/trending-events.ts | 47 +++++++++++++++++++++++++++++++++++ src/trends/trending-notes.ts | 37 --------------------------- 3 files changed, 58 insertions(+), 40 deletions(-) create mode 100644 src/trends/trending-events.ts delete mode 100644 src/trends/trending-notes.ts diff --git a/src/cron.ts b/src/cron.ts index 7f8f46a..53714e8 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -1,7 +1,7 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { DittoDB } from '@/db/DittoDB.ts'; -import { getTrendingNotes } from '@/trends/trending-notes.ts'; +import { getTrendingEvents } from '@/trends/trending-events.ts'; import { Time } from '@/utils/time.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { handleEvent } from '@/pipeline.ts'; @@ -12,10 +12,18 @@ const console = new Stickynotes('ditto:trends'); async function updateTrendingNotesCache() { console.info('Updating trending notes cache...'); const kysely = await DittoDB.getInstance(); - const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); const signal = AbortSignal.timeout(1000); - const events = await getTrendingNotes(kysely, yesterday, 20); + const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); + const now = Math.floor(Date.now() / 1000); + + const events = await getTrendingEvents(kysely, { + kinds: [1], + since: yesterday, + until: now, + limit: 20, + }); + const signer = new AdminSigner(); const label = await signer.signEvent({ diff --git a/src/trends/trending-events.ts b/src/trends/trending-events.ts new file mode 100644 index 0000000..3f0e8ef --- /dev/null +++ b/src/trends/trending-events.ts @@ -0,0 +1,47 @@ +import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; +import { Kysely, sql } from 'kysely'; + +import { DittoTables } from '@/db/DittoTables.ts'; + +/** + * Make a direct query to the database to get trending events within the specified timeframe. + * Trending events are determined by the number of reposts, replies, and reactions. + * + * This query makes use of cached stats (in the `event_stats` table). + * The query is SLOW so it needs to be run on a schedule and cached. + */ +export async function getTrendingEvents( + /** Kysely instance to execute queries on. */ + kysely: Kysely, + /** Filter of eligible events. */ + filter: NostrFilter, +): Promise { + let query = kysely + .selectFrom('nostr_events') + .selectAll('nostr_events') + .innerJoin('event_stats', 'event_stats.event_id', 'nostr_events.id') + .orderBy( + sql`(event_stats.reposts_count * 2) + (event_stats.replies_count) + (event_stats.reactions_count)`, + 'desc', + ); + + if (filter.kinds) { + query = query.where('nostr_events.kind', 'in', filter.kinds); + } + if (typeof filter.since === 'number') { + query = query.where('nostr_events.created_at', '>=', filter.since); + } + if (typeof filter.until === 'number') { + query = query.where('nostr_events.created_at', '<=', filter.until); + } + if (typeof filter.limit === 'number') { + query = query.limit(filter.limit); + } + + const rows = await query.execute(); + + return rows.map((row) => ({ + ...row, + tags: JSON.parse(row.tags), + })); +} diff --git a/src/trends/trending-notes.ts b/src/trends/trending-notes.ts deleted file mode 100644 index a469aa9..0000000 --- a/src/trends/trending-notes.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { NostrEvent } from '@nostrify/nostrify'; -import { Kysely, sql } from 'kysely'; - -import { DittoTables } from '@/db/DittoTables.ts'; - -/** - * Make a direct query to the database to get trending kind 1 notes within the specified timeframe. - * - * This query makes use of cached stats (in the `event_stats` table). - * The query is SLOW so it needs to be run on a schedule and cached. - */ -export async function getTrendingNotes( - /** Kysely instance to execute queries on. */ - kysely: Kysely, - /** Unix timestamp in _seconds_ for the starting point of this query. */ - since: number, - /** Maximum number of trending notes to return. */ - limit: number, -): Promise { - const rows = await kysely - .selectFrom('nostr_events') - .selectAll('nostr_events') - .innerJoin('event_stats', 'event_stats.event_id', 'nostr_events.id') - .where('nostr_events.kind', '=', 1) - .where('nostr_events.created_at', '>', since) - .orderBy( - sql`(event_stats.reposts_count * 2) + (event_stats.replies_count) + (event_stats.reactions_count)`, - 'desc', - ) - .limit(limit) - .execute(); - - return rows.map((row) => ({ - ...row, - tags: JSON.parse(row.tags), - })); -} From cd46d63999549889f1772506ef719ee06087dec1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 16:17:16 -0500 Subject: [PATCH 03/16] getTrendingHashtags -> getTrendingTagValues --- src/cron.ts | 21 +++++++++----- src/trends/trending-hashtags.ts | 38 ------------------------- src/trends/trending-tag-values.ts | 47 +++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 45 deletions(-) delete mode 100644 src/trends/trending-hashtags.ts create mode 100644 src/trends/trending-tag-values.ts diff --git a/src/cron.ts b/src/cron.ts index 53714e8..b0d08f9 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -1,11 +1,11 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { DittoDB } from '@/db/DittoDB.ts'; -import { getTrendingEvents } from '@/trends/trending-events.ts'; -import { Time } from '@/utils/time.ts'; -import { AdminSigner } from '@/signers/AdminSigner.ts'; import { handleEvent } from '@/pipeline.ts'; -import { getTrendingHashtags } from '@/trends/trending-hashtags.ts'; +import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { getTrendingEvents } from '@/trends/trending-events.ts'; +import { getTrendingTagValues } from '@/trends/trending-tag-values.ts'; +import { Time } from '@/utils/time.ts'; const console = new Stickynotes('ditto:trends'); @@ -44,10 +44,17 @@ async function updateTrendingNotesCache() { async function updateTrendingHashtagsCache() { console.info('Updating trending hashtags cache...'); const kysely = await DittoDB.getInstance(); - const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); const signal = AbortSignal.timeout(1000); - const hashtags = await getTrendingHashtags(kysely, { since: yesterday, limit: 20, threshold: 3 }); + const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); + const now = Math.floor(Date.now() / 1000); + + const hashtags = await getTrendingTagValues(kysely, 't', { + since: yesterday, + until: now, + limit: 20, + }); + const signer = new AdminSigner(); const label = await signer.signEvent({ @@ -56,7 +63,7 @@ async function updateTrendingHashtagsCache() { tags: [ ['L', 'pub.ditto.trends'], ['l', 'hashtags', 'pub.ditto.trends'], - ...hashtags.map(({ tag }) => ['t', tag]), + ...hashtags.map(({ value }) => ['t', value]), ], created_at: Math.floor(Date.now() / 1000), }); diff --git a/src/trends/trending-hashtags.ts b/src/trends/trending-hashtags.ts deleted file mode 100644 index a70acb9..0000000 --- a/src/trends/trending-hashtags.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Kysely } from 'kysely'; - -import { DittoTables } from '@/db/DittoTables.ts'; - -interface GetTrendingHashtagsOpts { - /** Unix timestamp in _seconds_ for the starting point of this query. */ - since: number; - /** Maximum number of trending hashtags to return. */ - limit: number; - /** Minimum number of unique accounts that have used a hashtag to be considered trending. */ - threshold: number; -} - -/** Get the trending hashtags in the given time frame. */ -export async function getTrendingHashtags( - /** Kysely instance to execute queries on. */ - kysely: Kysely, - /** Options for this query. */ - opts: GetTrendingHashtagsOpts, -): Promise<{ tag: string; accounts: number; uses: number }[]> { - const { since, limit, threshold } = opts; - - return await kysely - .selectFrom('nostr_tags') - .innerJoin('nostr_events', 'nostr_events.id', 'nostr_tags.event_id') - .select(({ fn }) => [ - 'nostr_tags.value as tag', - fn.agg('count', ['nostr_events.pubkey']).distinct().as('accounts'), - fn.countAll().as('uses'), - ]) - .where('nostr_tags.name', '=', 't') - .where('nostr_events.created_at', '>', since) - .groupBy('nostr_tags.value') - .having((c) => c(c.fn.agg('count', ['nostr_events.pubkey']).distinct(), '>=', threshold)) - .orderBy((c) => c.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc') - .limit(limit) - .execute(); -} diff --git a/src/trends/trending-tag-values.ts b/src/trends/trending-tag-values.ts new file mode 100644 index 0000000..aec0ddf --- /dev/null +++ b/src/trends/trending-tag-values.ts @@ -0,0 +1,47 @@ +import { NostrFilter } from '@nostrify/nostrify'; +import { Kysely } from 'kysely'; + +import { DittoTables } from '@/db/DittoTables.ts'; + +/** Get trending tag values for a given tag in the given time frame. */ +export async function getTrendingTagValues( + /** Kysely instance to execute queries on. */ + kysely: Kysely, + /** Tag name to filter by, eg `t` or `r`. */ + tagName: string, + /** Filter of eligible events. */ + filter: NostrFilter, +): Promise<{ value: string; authors: number; uses: number }[]> { + let query = kysely + .selectFrom('nostr_tags') + .innerJoin('nostr_events', 'nostr_events.id', 'nostr_tags.event_id') + .select(({ fn }) => [ + 'nostr_tags.value', + fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), + fn.countAll().as('uses'), + ]) + .where('nostr_tags.name', '=', tagName) + .groupBy('nostr_tags.value') + .orderBy((c) => c.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); + + if (filter.kinds) { + query = query.where('nostr_events.kind', 'in', filter.kinds); + } + if (typeof filter.since === 'number') { + query = query.where('nostr_events.created_at', '>=', filter.since); + } + if (typeof filter.until === 'number') { + query = query.where('nostr_events.created_at', '<=', filter.until); + } + if (typeof filter.limit === 'number') { + query = query.limit(filter.limit); + } + + const rows = await query.execute(); + + return rows.map((row) => ({ + value: row.value, + authors: Number(row.authors), + uses: Number(row.uses), + })); +} From 7f16a8d556b6b8251cc0f40998812eca70dd7499 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 16:42:25 -0500 Subject: [PATCH 04/16] Support trending links --- src/cron.ts | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/src/cron.ts b/src/cron.ts index b0d08f9..7c934a7 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -1,5 +1,6 @@ import { Stickynotes } from '@soapbox/stickynotes'; +import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { handleEvent } from '@/pipeline.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; @@ -21,7 +22,7 @@ async function updateTrendingNotesCache() { kinds: [1], since: yesterday, until: now, - limit: 20, + limit: 40, }); const signer = new AdminSigner(); @@ -32,7 +33,7 @@ async function updateTrendingNotesCache() { tags: [ ['L', 'pub.ditto.trends'], ['l', 'notes', 'pub.ditto.trends'], - ...events.map(({ id }) => ['e', id]), + ...events.map(({ id }) => ['e', id, Conf.relay]), ], created_at: Math.floor(Date.now() / 1000), }); @@ -63,7 +64,7 @@ async function updateTrendingHashtagsCache() { tags: [ ['L', 'pub.ditto.trends'], ['l', 'hashtags', 'pub.ditto.trends'], - ...hashtags.map(({ value }) => ['t', value]), + ...hashtags.map(({ value, authors, uses }) => ['t', value, authors.toString(), uses.toString()]), ], created_at: Math.floor(Date.now() / 1000), }); @@ -72,8 +73,40 @@ async function updateTrendingHashtagsCache() { console.info('Trending hashtags cache updated.'); } +async function updateTrendingLinksCache() { + console.info('Updating trending links cache...'); + const kysely = await DittoDB.getInstance(); + const signal = AbortSignal.timeout(1000); + + const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); + const now = Math.floor(Date.now() / 1000); + + const links = await getTrendingTagValues(kysely, 'r', { + since: yesterday, + until: now, + limit: 20, + }); + + const signer = new AdminSigner(); + + const label = await signer.signEvent({ + kind: 1985, + content: '', + tags: [ + ['L', 'pub.ditto.trends'], + ['l', 'links', 'pub.ditto.trends'], + ...links.map(({ value, authors, uses }) => ['r', value, authors.toString(), uses.toString()]), + ], + created_at: Math.floor(Date.now() / 1000), + }); + + await handleEvent(label, signal); + console.info('Trending links cache updated.'); +} + /** Start cron jobs for the application. */ export function cron() { - Deno.cron('update trending notes cache', { minute: { every: 15 } }, updateTrendingNotesCache); - Deno.cron('update trending hashtags cache', { dayOfMonth: { every: 1 } }, updateTrendingHashtagsCache); + Deno.cron('update trending notes cache', '15 * * * *', updateTrendingNotesCache); + Deno.cron('update trending hashtags cache', '30 * * * *', updateTrendingHashtagsCache); + Deno.cron('update trending links cache', '45 * * * *', updateTrendingLinksCache); } From e236e98259d3ee7b8a8b762e73acda48af0eef4b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 16:45:20 -0500 Subject: [PATCH 05/16] cron: remove the unnecessary word "cache" from all the things --- src/cron.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/cron.ts b/src/cron.ts index 7c934a7..dc697be 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -10,8 +10,8 @@ import { Time } from '@/utils/time.ts'; const console = new Stickynotes('ditto:trends'); -async function updateTrendingNotesCache() { - console.info('Updating trending notes cache...'); +async function updateTrendingNotes() { + console.info('Updating trending notes...'); const kysely = await DittoDB.getInstance(); const signal = AbortSignal.timeout(1000); @@ -39,11 +39,11 @@ async function updateTrendingNotesCache() { }); await handleEvent(label, signal); - console.info('Trending notes cache updated.'); + console.info('Trending notes updated.'); } -async function updateTrendingHashtagsCache() { - console.info('Updating trending hashtags cache...'); +async function updateTrendingHashtags() { + console.info('Updating trending hashtags...'); const kysely = await DittoDB.getInstance(); const signal = AbortSignal.timeout(1000); @@ -70,11 +70,11 @@ async function updateTrendingHashtagsCache() { }); await handleEvent(label, signal); - console.info('Trending hashtags cache updated.'); + console.info('Trending hashtags updated.'); } -async function updateTrendingLinksCache() { - console.info('Updating trending links cache...'); +async function updateTrendingLinks() { + console.info('Updating trending links...'); const kysely = await DittoDB.getInstance(); const signal = AbortSignal.timeout(1000); @@ -101,12 +101,12 @@ async function updateTrendingLinksCache() { }); await handleEvent(label, signal); - console.info('Trending links cache updated.'); + console.info('Trending links updated.'); } /** Start cron jobs for the application. */ export function cron() { - Deno.cron('update trending notes cache', '15 * * * *', updateTrendingNotesCache); - Deno.cron('update trending hashtags cache', '30 * * * *', updateTrendingHashtagsCache); - Deno.cron('update trending links cache', '45 * * * *', updateTrendingLinksCache); + Deno.cron('update trending notes', '15 * * * *', updateTrendingNotes); + Deno.cron('update trending hashtags', '30 * * * *', updateTrendingHashtags); + Deno.cron('update trending links', '45 * * * *', updateTrendingLinks); } From b46bcc559e3d4f07365472ea55e1a13cf8307ad3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 16:59:11 -0500 Subject: [PATCH 06/16] cron: hashtags -> #t, links -> #r --- src/cron.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cron.ts b/src/cron.ts index dc697be..c28e7a3 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -63,7 +63,7 @@ async function updateTrendingHashtags() { content: '', tags: [ ['L', 'pub.ditto.trends'], - ['l', 'hashtags', 'pub.ditto.trends'], + ['l', '#t', 'pub.ditto.trends'], ...hashtags.map(({ value, authors, uses }) => ['t', value, authors.toString(), uses.toString()]), ], created_at: Math.floor(Date.now() / 1000), @@ -94,7 +94,7 @@ async function updateTrendingLinks() { content: '', tags: [ ['L', 'pub.ditto.trends'], - ['l', 'links', 'pub.ditto.trends'], + ['l', '#r', 'pub.ditto.trends'], ...links.map(({ value, authors, uses }) => ['r', value, authors.toString(), uses.toString()]), ], created_at: Math.floor(Date.now() / 1000), From 76c882d8362d26646a6b07396a6cda2536820b02 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 17:47:36 -0500 Subject: [PATCH 07/16] Crunch the tag history in the controller --- src/controllers/api/trends.ts | 90 +++++++++++++++++++++-------------- src/storages/EventsDB.ts | 2 +- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index 688e3e8..7a91a70 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -5,12 +5,8 @@ import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; -import { Time } from '@/utils.ts'; -import { stripTime } from '@/utils/time.ts'; +import { generateDateRange, Time } from '@/utils/time.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; -import { TrendsWorker } from '@/workers/trends.ts'; - -await TrendsWorker.open('data/trends.sqlite3'); let trendingHashtagsCache = getTrendingHashtags(); @@ -31,42 +27,62 @@ const trendingTagsController: AppController = async (c) => { }; async function getTrendingHashtags() { + const store = await Storages.db(); + + const [label] = await store.query([{ + kinds: [1985], + '#L': ['pub.ditto.trends'], + '#l': ['#t'], + authors: [Conf.pubkey], + limit: 1, + }]); + + if (!label) { + return []; + } + + const tags = label.tags.filter(([name]) => name === 't'); + const now = new Date(); - const yesterday = new Date(now.getTime() - Time.days(1)); const lastWeek = new Date(now.getTime() - Time.days(7)); + const dates = generateDateRange(lastWeek, now); - /** Most used hashtags within the past 24h. */ - const tags = await TrendsWorker.getTrendingTags({ - since: yesterday, - until: now, - limit: 20, - }); + return Promise.all(tags.map(async ([_, hashtag]) => { + const filters = dates.map((date) => ({ + kinds: [1985], + '#L': ['pub.ditto.trends'], + '#l': ['#t'], + '#t': [hashtag], + authors: [Conf.pubkey], + since: Math.floor(date.getTime() / 1000), + until: Math.floor((date.getTime() + Time.days(1)) / 1000), + limit: 1, + })); - return Promise.all(tags.map(async ({ tag, uses, accounts }) => ({ - name: tag, - url: Conf.local(`/tags/${tag}`), - history: [ - // Use the full 24h query for the current day. Then use `offset: 1` to adjust for this below. - // This result is more accurate than what Mastodon returns. - { - day: String(Math.floor(stripTime(now).getTime() / 1000)), - accounts: String(accounts), - uses: String(uses), - }, - ...(await TrendsWorker.getTagHistory({ - tag, - since: lastWeek, - until: now, - limit: 6, - offset: 1, - })).map((history) => ({ - // For some reason, Mastodon wants these to be strings... oh well. - day: String(Math.floor(history.day.getTime() / 1000)), - accounts: String(history.accounts), - uses: String(history.uses), - })), - ], - }))); + const labels = await store.query(filters); + + const history = dates.map((date) => { + const label = labels.find((label) => { + const since = Math.floor(date.getTime() / 1000); + const until = Math.floor((date.getTime() + Time.days(1)) / 1000); + return label.created_at >= since && label.created_at < until; + }); + + const [, , accounts, uses] = label?.tags.find(([name, value]) => name === 't' && value === hashtag) ?? []; + + return { + day: String(date.getTime() / 1000), + accounts: accounts || '0', + uses: uses || '0', + }; + }); + + return { + name: hashtag, + url: Conf.local(`/tags/${hashtag}`), + history, + }; + })); } const trendingStatusesQuerySchema = z.object({ diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 0f7d080..67daca9 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -36,7 +36,7 @@ class EventsDB implements NStore { 'p': ({ event, count, value }) => (count < 15 || event.kind === 3) && isNostrId(value), 'proxy': ({ count, value }) => count === 0 && isURL(value), 'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value), - 't': ({ count, value }) => count < 5 && value.length < 50, + 't': ({ event, count, value }) => (event.kind === 1985 ? count < 20 : count < 5) && value.length < 50, 'name': ({ event, count }) => event.kind === 30361 && count === 0, 'role': ({ event, count }) => event.kind === 30361 && count === 0, }; From fc11e3449913c36c3fc0d1ee7fd678eec6f82a8e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 18:28:43 -0500 Subject: [PATCH 08/16] trends: notes -> #e, normalize --- src/controllers/api/trends.ts | 4 ++-- src/cron.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index 7a91a70..ec5e792 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -68,7 +68,7 @@ async function getTrendingHashtags() { return label.created_at >= since && label.created_at < until; }); - const [, , accounts, uses] = label?.tags.find(([name, value]) => name === 't' && value === hashtag) ?? []; + const [, , , accounts, uses] = label?.tags.find(([name, value]) => name === 't' && value === hashtag) ?? []; return { day: String(date.getTime() / 1000), @@ -97,7 +97,7 @@ const trendingStatusesController: AppController = async (c) => { const [label] = await store.query([{ kinds: [1985], '#L': ['pub.ditto.trends'], - '#l': ['notes'], + '#l': ['#e'], authors: [Conf.pubkey], limit: 1, }]); diff --git a/src/cron.ts b/src/cron.ts index c28e7a3..64eddfd 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -32,7 +32,7 @@ async function updateTrendingNotes() { content: '', tags: [ ['L', 'pub.ditto.trends'], - ['l', 'notes', 'pub.ditto.trends'], + ['l', '#e', 'pub.ditto.trends'], ...events.map(({ id }) => ['e', id, Conf.relay]), ], created_at: Math.floor(Date.now() / 1000), @@ -64,7 +64,7 @@ async function updateTrendingHashtags() { tags: [ ['L', 'pub.ditto.trends'], ['l', '#t', 'pub.ditto.trends'], - ...hashtags.map(({ value, authors, uses }) => ['t', value, authors.toString(), uses.toString()]), + ...hashtags.map(({ value, authors, uses }) => ['t', '', value, authors.toString(), uses.toString()]), ], created_at: Math.floor(Date.now() / 1000), }); @@ -95,7 +95,7 @@ async function updateTrendingLinks() { tags: [ ['L', 'pub.ditto.trends'], ['l', '#r', 'pub.ditto.trends'], - ...links.map(({ value, authors, uses }) => ['r', value, authors.toString(), uses.toString()]), + ...links.map(({ value, authors, uses }) => ['r', '', value, authors.toString(), uses.toString()]), ], created_at: Math.floor(Date.now() / 1000), }); From 133a684d3207a9ac2128ae4911bde8cda60a74a4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 19:02:31 -0500 Subject: [PATCH 09/16] Fix tag order --- src/controllers/api/trends.ts | 125 +++++++++++++++++++++------------- src/cron.ts | 16 ++++- 2 files changed, 90 insertions(+), 51 deletions(-) diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index ec5e792..0a9210e 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -1,7 +1,7 @@ -import { NostrEvent } from '@nostrify/nostrify'; +import { NostrEvent, NStore } from '@nostrify/nostrify'; import { z } from 'zod'; -import { type AppController } from '@/app.ts'; +import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; @@ -10,7 +10,7 @@ import { renderStatus } from '@/views/mastodon/statuses.ts'; let trendingHashtagsCache = getTrendingHashtags(); -Deno.cron('update trends cache', { minute: { every: 15 } }, async () => { +Deno.cron('update trending hashtags cache', { minute: { every: 15 } }, async () => { const trends = await getTrendingHashtags(); trendingHashtagsCache = Promise.resolve(trends); }); @@ -28,61 +28,23 @@ const trendingTagsController: AppController = async (c) => { async function getTrendingHashtags() { const store = await Storages.db(); + const trends = await getTrendingTags(store, 't'); - const [label] = await store.query([{ - kinds: [1985], - '#L': ['pub.ditto.trends'], - '#l': ['#t'], - authors: [Conf.pubkey], - limit: 1, - }]); + return trends.map((trend) => { + const hashtag = trend.value; - if (!label) { - return []; - } - - const tags = label.tags.filter(([name]) => name === 't'); - - const now = new Date(); - const lastWeek = new Date(now.getTime() - Time.days(7)); - const dates = generateDateRange(lastWeek, now); - - return Promise.all(tags.map(async ([_, hashtag]) => { - const filters = dates.map((date) => ({ - kinds: [1985], - '#L': ['pub.ditto.trends'], - '#l': ['#t'], - '#t': [hashtag], - authors: [Conf.pubkey], - since: Math.floor(date.getTime() / 1000), - until: Math.floor((date.getTime() + Time.days(1)) / 1000), - limit: 1, + const history = trend.history.map(({ day, authors, uses }) => ({ + day: String(day), + accounts: String(authors), + uses: String(uses), })); - const labels = await store.query(filters); - - const history = dates.map((date) => { - const label = labels.find((label) => { - const since = Math.floor(date.getTime() / 1000); - const until = Math.floor((date.getTime() + Time.days(1)) / 1000); - return label.created_at >= since && label.created_at < until; - }); - - const [, , , accounts, uses] = label?.tags.find(([name, value]) => name === 't' && value === hashtag) ?? []; - - return { - day: String(date.getTime() / 1000), - accounts: accounts || '0', - uses: uses || '0', - }; - }); - return { name: hashtag, url: Conf.local(`/tags/${hashtag}`), history, }; - })); + }); } const trendingStatusesQuerySchema = z.object({ @@ -126,4 +88,69 @@ const trendingStatusesController: AppController = async (c) => { return c.json(statuses.filter(Boolean)); }; +interface TrendingTag { + name: string; + value: string; + history: { + day: number; + authors: number; + uses: number; + }[]; +} + +export async function getTrendingTags(store: NStore, tagName: string): Promise { + const filter = { + kinds: [1985], + '#L': ['pub.ditto.trends'], + '#l': [`#${tagName}`], + authors: [Conf.pubkey], + limit: 1, + }; + + const [label] = await store.query([filter]); + + if (!label) { + return []; + } + + const tags = label.tags.filter(([name]) => name === tagName); + + const now = new Date(); + const lastWeek = new Date(now.getTime() - Time.days(7)); + const dates = generateDateRange(lastWeek, now); + + return Promise.all(tags.map(async ([_, value]) => { + const filters = dates.map((date) => ({ + ...filter, + [`#${tagName}`]: [value], + since: Math.floor(date.getTime() / 1000), + until: Math.floor((date.getTime() + Time.days(1)) / 1000), + })); + + const labels = await store.query(filters); + + const history = dates.map((date) => { + const label = labels.find((label) => { + const since = Math.floor(date.getTime() / 1000); + const until = Math.floor((date.getTime() + Time.days(1)) / 1000); + return label.created_at >= since && label.created_at < until; + }); + + const [, , , accounts, uses] = label?.tags.find((tag) => tag[0] === tagName && tag[1] === value) ?? []; + + return { + day: Math.floor(date.getTime() / 1000), + authors: Number(accounts || 0), + uses: Number(uses || 0), + }; + }); + + return { + name: tagName, + value, + history, + }; + })); +} + export { trendingStatusesController, trendingTagsController }; diff --git a/src/cron.ts b/src/cron.ts index 64eddfd..4823e16 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -25,6 +25,10 @@ async function updateTrendingNotes() { limit: 40, }); + if (!events.length) { + return; + } + const signer = new AdminSigner(); const label = await signer.signEvent({ @@ -56,6 +60,10 @@ async function updateTrendingHashtags() { limit: 20, }); + if (!hashtags.length) { + return; + } + const signer = new AdminSigner(); const label = await signer.signEvent({ @@ -64,7 +72,7 @@ async function updateTrendingHashtags() { tags: [ ['L', 'pub.ditto.trends'], ['l', '#t', 'pub.ditto.trends'], - ...hashtags.map(({ value, authors, uses }) => ['t', '', value, authors.toString(), uses.toString()]), + ...hashtags.map(({ value, authors, uses }) => ['t', value, '', authors.toString(), uses.toString()]), ], created_at: Math.floor(Date.now() / 1000), }); @@ -87,6 +95,10 @@ async function updateTrendingLinks() { limit: 20, }); + if (!links.length) { + return; + } + const signer = new AdminSigner(); const label = await signer.signEvent({ @@ -95,7 +107,7 @@ async function updateTrendingLinks() { tags: [ ['L', 'pub.ditto.trends'], ['l', '#r', 'pub.ditto.trends'], - ...links.map(({ value, authors, uses }) => ['r', '', value, authors.toString(), uses.toString()]), + ...links.map(({ value, authors, uses }) => ['r', value, '', authors.toString(), uses.toString()]), ], created_at: Math.floor(Date.now() / 1000), }); From 7e44368c080db0a28b1d0d51b7ad229cfafd9ffb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 19:07:10 -0500 Subject: [PATCH 10/16] trends: reverse the history dates --- src/controllers/api/trends.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index 0a9210e..9e2c3a7 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -117,7 +117,7 @@ export async function getTrendingTags(store: NStore, tagName: string): Promise { const filters = dates.map((date) => ({ From b35090ef1056989e4730c3fc1c11163267414803 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 19:18:42 -0500 Subject: [PATCH 11/16] cron: DRY updateTrendingTags --- src/cron.ts | 53 +++++++++-------------------------------------------- 1 file changed, 9 insertions(+), 44 deletions(-) diff --git a/src/cron.ts b/src/cron.ts index 4823e16..aa19caf 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -46,21 +46,21 @@ async function updateTrendingNotes() { console.info('Trending notes updated.'); } -async function updateTrendingHashtags() { - console.info('Updating trending hashtags...'); +async function updateTrendingTags(tagName: string, extra = '') { + console.info(`Updating trending #${tagName}...`); const kysely = await DittoDB.getInstance(); const signal = AbortSignal.timeout(1000); const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); const now = Math.floor(Date.now() / 1000); - const hashtags = await getTrendingTagValues(kysely, 't', { + const trends = await getTrendingTagValues(kysely, tagName, { since: yesterday, until: now, limit: 20, }); - if (!hashtags.length) { + if (!trends.length) { return; } @@ -71,54 +71,19 @@ async function updateTrendingHashtags() { content: '', tags: [ ['L', 'pub.ditto.trends'], - ['l', '#t', 'pub.ditto.trends'], - ...hashtags.map(({ value, authors, uses }) => ['t', value, '', authors.toString(), uses.toString()]), + ['l', `#${tagName}`, 'pub.ditto.trends'], + ...trends.map(({ value, authors, uses }) => [tagName, value, extra, authors.toString(), uses.toString()]), ], created_at: Math.floor(Date.now() / 1000), }); await handleEvent(label, signal); - console.info('Trending hashtags updated.'); -} - -async function updateTrendingLinks() { - console.info('Updating trending links...'); - const kysely = await DittoDB.getInstance(); - const signal = AbortSignal.timeout(1000); - - const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); - const now = Math.floor(Date.now() / 1000); - - const links = await getTrendingTagValues(kysely, 'r', { - since: yesterday, - until: now, - limit: 20, - }); - - if (!links.length) { - return; - } - - const signer = new AdminSigner(); - - const label = await signer.signEvent({ - kind: 1985, - content: '', - tags: [ - ['L', 'pub.ditto.trends'], - ['l', '#r', 'pub.ditto.trends'], - ...links.map(({ value, authors, uses }) => ['r', value, '', authors.toString(), uses.toString()]), - ], - created_at: Math.floor(Date.now() / 1000), - }); - - await handleEvent(label, signal); - console.info('Trending links updated.'); + console.info(`Trending #${tagName} updated.`); } /** Start cron jobs for the application. */ export function cron() { Deno.cron('update trending notes', '15 * * * *', updateTrendingNotes); - Deno.cron('update trending hashtags', '30 * * * *', updateTrendingHashtags); - Deno.cron('update trending links', '45 * * * *', updateTrendingLinks); + Deno.cron('update trending hashtags', '30 * * * *', () => updateTrendingTags('t')); + Deno.cron('update trending links', '45 * * * *', () => updateTrendingTags('r')); } From 3373673706e4df880286d4e17aef11f8ebf55136 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 19:27:44 -0500 Subject: [PATCH 12/16] updateTrendingTags: accept limit param --- src/cron.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cron.ts b/src/cron.ts index aa19caf..0313e7d 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -46,7 +46,7 @@ async function updateTrendingNotes() { console.info('Trending notes updated.'); } -async function updateTrendingTags(tagName: string, extra = '') { +async function updateTrendingTags(tagName: string, limit: number, extra = '') { console.info(`Updating trending #${tagName}...`); const kysely = await DittoDB.getInstance(); const signal = AbortSignal.timeout(1000); @@ -57,7 +57,7 @@ async function updateTrendingTags(tagName: string, extra = '') { const trends = await getTrendingTagValues(kysely, tagName, { since: yesterday, until: now, - limit: 20, + limit, }); if (!trends.length) { @@ -84,6 +84,6 @@ async function updateTrendingTags(tagName: string, extra = '') { /** Start cron jobs for the application. */ export function cron() { Deno.cron('update trending notes', '15 * * * *', updateTrendingNotes); - Deno.cron('update trending hashtags', '30 * * * *', () => updateTrendingTags('t')); - Deno.cron('update trending links', '45 * * * *', () => updateTrendingTags('r')); + Deno.cron('update trending hashtags', '30 * * * *', () => updateTrendingTags('t', 20)); + Deno.cron('update trending links', '45 * * * *', () => updateTrendingTags('r', 20)); } From c7c75a714757ac6b15e9167ab337ea06b996795f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 19:40:42 -0500 Subject: [PATCH 13/16] Get rid if trending-events.ts, do everything with tag values (brave) --- src/cron.ts | 50 +++++-------------------------- src/trends/trending-events.ts | 47 ----------------------------- src/trends/trending-tag-values.ts | 4 +-- 3 files changed, 10 insertions(+), 91 deletions(-) delete mode 100644 src/trends/trending-events.ts diff --git a/src/cron.ts b/src/cron.ts index 0313e7d..f450e8d 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -4,49 +4,12 @@ import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { handleEvent } from '@/pipeline.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { getTrendingEvents } from '@/trends/trending-events.ts'; import { getTrendingTagValues } from '@/trends/trending-tag-values.ts'; import { Time } from '@/utils/time.ts'; const console = new Stickynotes('ditto:trends'); -async function updateTrendingNotes() { - console.info('Updating trending notes...'); - const kysely = await DittoDB.getInstance(); - const signal = AbortSignal.timeout(1000); - - const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); - const now = Math.floor(Date.now() / 1000); - - const events = await getTrendingEvents(kysely, { - kinds: [1], - since: yesterday, - until: now, - limit: 40, - }); - - if (!events.length) { - return; - } - - const signer = new AdminSigner(); - - const label = await signer.signEvent({ - kind: 1985, - content: '', - tags: [ - ['L', 'pub.ditto.trends'], - ['l', '#e', 'pub.ditto.trends'], - ...events.map(({ id }) => ['e', id, Conf.relay]), - ], - created_at: Math.floor(Date.now() / 1000), - }); - - await handleEvent(label, signal); - console.info('Trending notes updated.'); -} - -async function updateTrendingTags(tagName: string, limit: number, extra = '') { +async function updateTrendingTags(tagName: string, kinds: number[], limit: number, extra = '', aliases?: string[]) { console.info(`Updating trending #${tagName}...`); const kysely = await DittoDB.getInstance(); const signal = AbortSignal.timeout(1000); @@ -54,7 +17,10 @@ async function updateTrendingTags(tagName: string, limit: number, extra = '') { const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); const now = Math.floor(Date.now() / 1000); - const trends = await getTrendingTagValues(kysely, tagName, { + const tagNames = aliases ? [tagName, ...aliases] : [tagName]; + + const trends = await getTrendingTagValues(kysely, tagNames, { + kinds, since: yesterday, until: now, limit, @@ -83,7 +49,7 @@ async function updateTrendingTags(tagName: string, limit: number, extra = '') { /** Start cron jobs for the application. */ export function cron() { - Deno.cron('update trending notes', '15 * * * *', updateTrendingNotes); - Deno.cron('update trending hashtags', '30 * * * *', () => updateTrendingTags('t', 20)); - Deno.cron('update trending links', '45 * * * *', () => updateTrendingTags('r', 20)); + Deno.cron('update trending notes', '15 * * * *', () => updateTrendingTags('e', [1, 6, 7], 40, Conf.relay, ['q'])); + Deno.cron('update trending hashtags', '30 * * * *', () => updateTrendingTags('t', [1], 20)); + Deno.cron('update trending links', '45 * * * *', () => updateTrendingTags('r', [1], 20)); } diff --git a/src/trends/trending-events.ts b/src/trends/trending-events.ts deleted file mode 100644 index 3f0e8ef..0000000 --- a/src/trends/trending-events.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { NostrEvent, NostrFilter } from '@nostrify/nostrify'; -import { Kysely, sql } from 'kysely'; - -import { DittoTables } from '@/db/DittoTables.ts'; - -/** - * Make a direct query to the database to get trending events within the specified timeframe. - * Trending events are determined by the number of reposts, replies, and reactions. - * - * This query makes use of cached stats (in the `event_stats` table). - * The query is SLOW so it needs to be run on a schedule and cached. - */ -export async function getTrendingEvents( - /** Kysely instance to execute queries on. */ - kysely: Kysely, - /** Filter of eligible events. */ - filter: NostrFilter, -): Promise { - let query = kysely - .selectFrom('nostr_events') - .selectAll('nostr_events') - .innerJoin('event_stats', 'event_stats.event_id', 'nostr_events.id') - .orderBy( - sql`(event_stats.reposts_count * 2) + (event_stats.replies_count) + (event_stats.reactions_count)`, - 'desc', - ); - - if (filter.kinds) { - query = query.where('nostr_events.kind', 'in', filter.kinds); - } - if (typeof filter.since === 'number') { - query = query.where('nostr_events.created_at', '>=', filter.since); - } - if (typeof filter.until === 'number') { - query = query.where('nostr_events.created_at', '<=', filter.until); - } - if (typeof filter.limit === 'number') { - query = query.limit(filter.limit); - } - - const rows = await query.execute(); - - return rows.map((row) => ({ - ...row, - tags: JSON.parse(row.tags), - })); -} diff --git a/src/trends/trending-tag-values.ts b/src/trends/trending-tag-values.ts index aec0ddf..17ec53d 100644 --- a/src/trends/trending-tag-values.ts +++ b/src/trends/trending-tag-values.ts @@ -8,7 +8,7 @@ export async function getTrendingTagValues( /** Kysely instance to execute queries on. */ kysely: Kysely, /** Tag name to filter by, eg `t` or `r`. */ - tagName: string, + tagNames: string[], /** Filter of eligible events. */ filter: NostrFilter, ): Promise<{ value: string; authors: number; uses: number }[]> { @@ -20,7 +20,7 @@ export async function getTrendingTagValues( fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), fn.countAll().as('uses'), ]) - .where('nostr_tags.name', '=', tagName) + .where('nostr_tags.name', 'in', tagNames) .groupBy('nostr_tags.value') .orderBy((c) => c.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); From 051f23d90883d93e09d26662f6d65ec294fa8d42 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 19:44:20 -0500 Subject: [PATCH 14/16] Remove TrendsWorker --- deno.lock | 6 ++ src/pipeline.ts | 23 +----- src/workers/trends.test.ts | 30 ------- src/workers/trends.ts | 19 ----- src/workers/trends.worker.ts | 152 ----------------------------------- 5 files changed, 7 insertions(+), 223 deletions(-) delete mode 100644 src/workers/trends.test.ts delete mode 100644 src/workers/trends.ts delete mode 100644 src/workers/trends.worker.ts diff --git a/deno.lock b/deno.lock index 56277db..f064971 100644 --- a/deno.lock +++ b/deno.lock @@ -1091,15 +1091,18 @@ "https://deno.land/x/hono@v3.10.1/adapter/deno/serve-static.ts": "ba10cf6aaf39da942b0d49c3b9877ddba69d41d414c6551d890beb1085f58eea", "https://deno.land/x/hono@v3.10.1/client/client.ts": "ff340f58041203879972dd368b011ed130c66914f789826610869a90603406bf", "https://deno.land/x/hono@v3.10.1/client/index.ts": "3ff4cf246f3543f827a85a2c84d66a025ac350ee927613629bda47e854bfb7ba", + "https://deno.land/x/hono@v3.10.1/client/types.ts": "52c66cbe74540e1811259a48c30622ac915666196eb978092d166435cbc15213", "https://deno.land/x/hono@v3.10.1/client/utils.ts": "053273c002963b549d38268a1b423ac8ca211a8028bdab1ed0b781a62aa5e661", "https://deno.land/x/hono@v3.10.1/compose.ts": "e8ab4b345aa367f2dd65f221c9fe829dd885326a613f4215b654f93a4066bb5c", "https://deno.land/x/hono@v3.10.1/context.ts": "261cc8b8b1e8f04b98beab1cca6692f317b7dc6d2b75b4f84c982e54cf1db730", + "https://deno.land/x/hono@v3.10.1/helper/adapter/index.ts": "eea9b4caedbfa3a3b4a020bf46c88c0171a00008cd6c10708cd85a3e39d86e62", "https://deno.land/x/hono@v3.10.1/helper/cookie/index.ts": "55ccd20bbd8d9a8bb2ecd998e90845c1d306c19027f54b3d1b89a5be35968b80", "https://deno.land/x/hono@v3.10.1/helper/html/index.ts": "aba19e8d29f217c7fffa5719cf606c4e259b540d51296e82bbea3c992e2ecbc6", "https://deno.land/x/hono@v3.10.1/hono-base.ts": "cc55e0a4c63a7bdf44df3e804ea4737d5399eeb6606b45d102f8e48c3ff1e925", "https://deno.land/x/hono@v3.10.1/hono.ts": "2cc4c292e541463a4d6f83edbcea58048d203e9564ae62ec430a3d466b49a865", "https://deno.land/x/hono@v3.10.1/http-exception.ts": "6071df078b5f76d279684d52fe82a590f447a64ffe1b75eb5064d0c8a8d2d676", "https://deno.land/x/hono@v3.10.1/jsx/index.ts": "019512d3a9b3897b879e87fa5fb179cd34f3d326f8ff8b93379c2bb707ec168a", + "https://deno.land/x/hono@v3.10.1/jsx/intrinsic-elements.ts": "03250beb610bda1c72017bc0912c2505ff764b7a8d869e7e4add40eb4cfec096", "https://deno.land/x/hono@v3.10.1/jsx/streaming.ts": "5d03b4d02eaa396c8f0f33c3f6e8c7ed3afb7598283c2d4a7ddea0ada8c212a7", "https://deno.land/x/hono@v3.10.1/middleware.ts": "57b2047c4b9d775a052a9c44a3b805802c1d1cb477ab9c4bb6185d27382d1b96", "https://deno.land/x/hono@v3.10.1/middleware/basic-auth/index.ts": "5505288ccf9364f56f7be2dfac841543b72e20656e54ac646a1a73a0aa853261", @@ -1131,6 +1134,7 @@ "https://deno.land/x/hono@v3.10.1/router/trie-router/index.ts": "3eb75e7f71ba81801631b30de6b1f5cefb2c7239c03797e2b2cbab5085911b41", "https://deno.land/x/hono@v3.10.1/router/trie-router/node.ts": "3af15fa9c9994a8664a2b7a7c11233504b5bb9d4fcf7bb34cf30d7199052c39f", "https://deno.land/x/hono@v3.10.1/router/trie-router/router.ts": "54ced78d35676302c8fcdda4204f7bdf5a7cc907fbf9967c75674b1e394f830d", + "https://deno.land/x/hono@v3.10.1/types.ts": "edc414a92383f9deb82f5f7a09e95bcf76f6100c23457c27d041986768f5345c", "https://deno.land/x/hono@v3.10.1/utils/body.ts": "7a16a6656331a96bcae57642f8d5e3912bd361cbbcc2c0d2157ecc3f218f7a92", "https://deno.land/x/hono@v3.10.1/utils/buffer.ts": "9066a973e64498cb262c7e932f47eed525a51677b17f90893862b7279dc0773e", "https://deno.land/x/hono@v3.10.1/utils/cookie.ts": "19920ba6756944aae1ad8585c3ddeaa9df479733f59d05359db096f7361e5e4b", @@ -1138,11 +1142,13 @@ "https://deno.land/x/hono@v3.10.1/utils/encode.ts": "3b7c7d736123b5073542b34321700d4dbf5ff129c138f434bb2144a4d425ee89", "https://deno.land/x/hono@v3.10.1/utils/filepath.ts": "18461b055a914d6da85077f453051b516281bb17cf64fa74bf5ef604dc9d2861", "https://deno.land/x/hono@v3.10.1/utils/html.ts": "01c1520a4256f899da1954357cf63ae11c348eda141a505f72d7090cf5481aba", + "https://deno.land/x/hono@v3.10.1/utils/http-status.ts": "e0c4343ea7717c314dc600131e16b636c29d61cfdaf9df93b267258d1729d1a0", "https://deno.land/x/hono@v3.10.1/utils/jwt/index.ts": "5e4b82a42eb3603351dfce726cd781ca41cb57437395409d227131aec348d2d5", "https://deno.land/x/hono@v3.10.1/utils/jwt/jwt.ts": "02ff7bbf1298ffcc7a40266842f8eac44b6c136453e32d4441e24d0cbfba3a95", "https://deno.land/x/hono@v3.10.1/utils/jwt/types.ts": "58ddf908f76ba18d9c62ddfc2d1e40cc2e306bf987409a6169287efa81ce2546", "https://deno.land/x/hono@v3.10.1/utils/mime.ts": "0105d2b5e8e91f07acc70f5d06b388313995d62af23c802fcfba251f5a744d95", "https://deno.land/x/hono@v3.10.1/utils/stream.ts": "1789dcc73c5b0ede28f83d7d34e47ae432c20e680907cb3275a9c9187f293983", + "https://deno.land/x/hono@v3.10.1/utils/types.ts": "ddff055e6d35066232efdfbd42c8954e855c04279c27dcd735d929b6b4f319b3", "https://deno.land/x/hono@v3.10.1/utils/url.ts": "5fc3307ef3cb2e6f34ec2a03e3d7f2126c6a9f5f0eab677222df3f0e40bd7567", "https://deno.land/x/hono@v3.10.1/validator/index.ts": "6c986e8b91dcf857ecc8164a506ae8eea8665792a4ff7215471df669c632ae7c", "https://deno.land/x/hono@v3.10.1/validator/validator.ts": "afa5e52495e0996fbba61996736fab5c486590d72d376f809e9f9ff4e0c463e9", diff --git a/src/pipeline.ts b/src/pipeline.ts index 9aa16fd..5005ec1 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -13,9 +13,8 @@ import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; import { RelayError } from '@/RelayError.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; -import { eventAge, nostrDate, parseNip05, Time } from '@/utils.ts'; +import { eventAge, parseNip05, Time } from '@/utils.ts'; import { policyWorker } from '@/workers/policy.ts'; -import { TrendsWorker } from '@/workers/trends.ts'; import { verifyEventWorker } from '@/workers/verify.ts'; import { nip05Cache } from '@/utils/nip05.ts'; import { updateStats } from '@/utils/stats.ts'; @@ -49,7 +48,6 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { - const date = nostrDate(event.created_at); - - const tags = event.tags - .filter((tag) => tag[0] === 't') - .map((tag) => tag[1]) - .slice(0, 5); - - if (!tags.length) return; - - try { - debug('tracking tags:', JSON.stringify(tags)); - await TrendsWorker.addTagUsages(event.pubkey, tags, date); - } catch (_e) { - // do nothing - } -} - /** Delete unattached media entries that are attached to the event. */ function processMedia({ tags, pubkey, user }: DittoEvent) { if (user) { diff --git a/src/workers/trends.test.ts b/src/workers/trends.test.ts deleted file mode 100644 index da33820..0000000 --- a/src/workers/trends.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { assertEquals } from '@std/assert'; - -import { TrendsWorker } from './trends.ts'; - -await TrendsWorker.open(':memory:'); - -const p8 = (pubkey8: string) => `${pubkey8}00000000000000000000000000000000000000000000000000000000`; - -Deno.test('getTrendingTags', async () => { - await TrendsWorker.addTagUsages(p8('00000000'), ['ditto', 'hello', 'yolo']); - await TrendsWorker.addTagUsages(p8('00000000'), ['hello']); - await TrendsWorker.addTagUsages(p8('00000001'), ['Ditto', 'hello']); - await TrendsWorker.addTagUsages(p8('00000010'), ['DITTO']); - - const result = await TrendsWorker.getTrendingTags({ - since: new Date('1999-01-01T00:00:00'), - until: new Date('2999-01-01T00:00:00'), - threshold: 1, - }); - - const expected = [ - { tag: 'ditto', accounts: 3, uses: 3 }, - { tag: 'hello', accounts: 2, uses: 3 }, - { tag: 'yolo', accounts: 1, uses: 1 }, - ]; - - assertEquals(result, expected); - - await TrendsWorker.cleanupTagUsages(new Date('2999-01-01T00:00:00')); -}); diff --git a/src/workers/trends.ts b/src/workers/trends.ts deleted file mode 100644 index 31db381..0000000 --- a/src/workers/trends.ts +++ /dev/null @@ -1,19 +0,0 @@ -import * as Comlink from 'comlink'; - -import type { TrendsWorker as _TrendsWorker } from '@/workers/trends.worker.ts'; - -const worker = new Worker(new URL('./trends.worker.ts', import.meta.url), { type: 'module' }); - -const TrendsWorker = Comlink.wrap(worker); - -await new Promise((resolve) => { - const handleEvent = ({ data }: MessageEvent) => { - if (data === 'ready') { - worker.removeEventListener('message', handleEvent); - resolve(); - } - }; - worker.addEventListener('message', handleEvent); -}); - -export { TrendsWorker }; diff --git a/src/workers/trends.worker.ts b/src/workers/trends.worker.ts deleted file mode 100644 index 90d1b57..0000000 --- a/src/workers/trends.worker.ts +++ /dev/null @@ -1,152 +0,0 @@ -/// -import { Database as Sqlite } from '@db/sqlite'; -import { NSchema } from '@nostrify/nostrify'; -import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; -import { Kysely, sql } from 'kysely'; -import * as Comlink from 'comlink'; - -import { hashtagSchema } from '@/schema.ts'; -import { generateDateRange, Time } from '@/utils/time.ts'; - -interface GetTrendingTagsOpts { - since: Date; - until: Date; - limit?: number; - threshold?: number; -} - -interface GetTagHistoryOpts { - tag: string; - since: Date; - until: Date; - limit?: number; - offset?: number; -} - -interface TagsDB { - tag_usages: { - tag: string; - pubkey8: string; - inserted_at: Date; - }; -} - -let kysely: Kysely; - -export const TrendsWorker = { - async open(path: string) { - kysely = new Kysely({ - dialect: new DenoSqlite3Dialect({ - database: new Sqlite(path), - }), - }); - - await sql`PRAGMA synchronous = normal`.execute(kysely); - await sql`PRAGMA temp_store = memory`.execute(kysely); - await sql`PRAGMA foreign_keys = ON`.execute(kysely); - await sql`PRAGMA auto_vacuum = FULL`.execute(kysely); - await sql`PRAGMA journal_mode = WAL`.execute(kysely); - await sql`PRAGMA mmap_size = 50000000`.execute(kysely); - - await kysely.schema - .createTable('tag_usages') - .ifNotExists() - .addColumn('tag', 'text', (c) => c.notNull().modifyEnd(sql`collate nocase`)) - .addColumn('pubkey8', 'text', (c) => c.notNull()) - .addColumn('inserted_at', 'integer', (c) => c.notNull()) - .execute(); - - await kysely.schema - .createIndex('idx_time_tag') - .ifNotExists() - .on('tag_usages') - .column('inserted_at') - .column('tag') - .execute(); - - Deno.cron('cleanup tag usages older than a week', { hour: { every: 1 } }, async () => { - const lastWeek = new Date(new Date().getTime() - Time.days(7)); - await this.cleanupTagUsages(lastWeek); - }); - }, - - /** Gets the most used hashtags between the date range. */ - getTrendingTags({ since, until, limit = 10, threshold = 3 }: GetTrendingTagsOpts): Promise<{ - tag: string; - accounts: number; - uses: number; - }[]> { - return kysely.selectFrom('tag_usages') - .select(({ fn }) => [ - 'tag', - fn.agg('count', ['pubkey8']).distinct().as('accounts'), - fn.countAll().as('uses'), - ]) - .where('inserted_at', '>=', since) - .where('inserted_at', '<', until) - .groupBy('tag') - .having((c) => c(c.fn.agg('count', ['pubkey8']).distinct(), '>=', threshold)) - .orderBy((c) => c.fn.agg('count', ['pubkey8']).distinct(), 'desc') - .limit(limit) - .execute(); - }, - - /** - * Gets the tag usage count for a specific tag. - * It returns an array with counts for each date between the range. - */ - async getTagHistory({ tag, since, until, limit = 7, offset = 0 }: GetTagHistoryOpts) { - const result = await kysely - .selectFrom('tag_usages') - .select(({ fn }) => [ - sql`date(inserted_at)`.as('day'), - fn.agg('count', ['pubkey8']).distinct().as('accounts'), - fn.countAll().as('uses'), - ]) - .where('tag', '=', tag) - .where('inserted_at', '>=', since) - .where('inserted_at', '<', until) - .groupBy(sql`date(inserted_at)`) - .orderBy(sql`date(inserted_at)`, 'desc') - .limit(limit) - .offset(offset) - .execute(); - - /** Full date range between `since` and `until`. */ - const dateRange = generateDateRange( - new Date(since.getTime() + Time.days(1)), - new Date(until.getTime() - Time.days(offset)), - ).reverse(); - - // Fill in missing dates with 0 usages. - return dateRange.map((day) => { - const data = result.find((item) => new Date(item.day).getTime() === day.getTime()); - if (data) { - return { ...data, day: new Date(data.day) }; - } else { - return { day, accounts: 0, uses: 0 }; - } - }); - }, - - async addTagUsages(pubkey: string, hashtags: string[], inserted_at = new Date()): Promise { - const pubkey8 = NSchema.id().parse(pubkey).substring(0, 8); - const tags = hashtagSchema.array().min(1).parse(hashtags); - - await kysely - .insertInto('tag_usages') - .values(tags.map((tag) => ({ tag, pubkey8, inserted_at }))) - .execute(); - }, - - async cleanupTagUsages(until: Date): Promise { - await kysely - .deleteFrom('tag_usages') - .where('inserted_at', '<', until) - .execute(); - }, -}; - -Comlink.expose(TrendsWorker); - -self.postMessage('ready'); From 6a4fc7b6ba7f69301c1b2e7eedd0429e301c53ae Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 19:51:04 -0500 Subject: [PATCH 15/16] Track trending pubkeys too, why not --- src/cron.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cron.ts b/src/cron.ts index f450e8d..af7c4c4 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -49,6 +49,7 @@ async function updateTrendingTags(tagName: string, kinds: number[], limit: numbe /** Start cron jobs for the application. */ export function cron() { + Deno.cron('update trending pubkeys', '0 * * * *', () => updateTrendingTags('p', [1, 3], 40, Conf.relay)); Deno.cron('update trending notes', '15 * * * *', () => updateTrendingTags('e', [1, 6, 7], 40, Conf.relay, ['q'])); Deno.cron('update trending hashtags', '30 * * * *', () => updateTrendingTags('t', [1], 20)); Deno.cron('update trending links', '45 * * * *', () => updateTrendingTags('r', [1], 20)); From 9d2194a92868e22728abf4d8fd9f7600a6dd1985 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Jun 2024 19:54:48 -0500 Subject: [PATCH 16/16] trendingStatusesController: enforce kind 1 events --- src/controllers/api/trends.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index 9e2c3a7..13075d1 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -73,7 +73,7 @@ const trendingStatusesController: AppController = async (c) => { return c.json([]); } - const results = await store.query([{ ids }]) + const results = await store.query([{ kinds: [1], ids }]) .then((events) => hydrateEvents({ events, store })); // Sort events in the order they appear in the label.