From 89b98ae77aa1153b1452c6a80df9a8714a36e142 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 26 Jul 2023 14:51:57 -0500 Subject: [PATCH 1/4] Fix tag history date --- src/trends.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/trends.ts b/src/trends.ts index 9ae531b..e079e7d 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -65,7 +65,7 @@ class TrendsDB { getTagHistory({ tag, since, until, limit = 7, offset = 0 }: GetTagHistoryOpts) { return this.#db.query( ` - SELECT inserted_at, COUNT(DISTINCT pubkey8), COUNT(*) + SELECT date(inserted_at), COUNT(DISTINCT pubkey8), COUNT(*) FROM tag_usages WHERE tag = ? AND inserted_at >= ? AND inserted_at < ? GROUP BY date(inserted_at) From f8d46cae585d28c9a8dec0e738f987c6e8a50484 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 26 Jul 2023 15:40:29 -0500 Subject: [PATCH 2/4] Trends: fill in empty tag histories Fixes https://gitlab.com/soapbox-pub/ditto/-/issues/29 --- src/controllers/api/trends.ts | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index cadfecf..916bf0a 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -29,7 +29,7 @@ const trendingTagsController: AppController = (c) => { accounts: String(accounts), uses: String(uses), }, - ...trends.getTagHistory({ + ...getTagHistoryWithGapsFilled({ tag: name, since: lastWeek, until: now, @@ -44,4 +44,28 @@ const trendingTagsController: AppController = (c) => { }))); }; +function generateDateRange(since: Date, until: Date): Date[] { + const dates = []; + + const sinceDate = new Date(Date.UTC(since.getUTCFullYear(), since.getUTCMonth(), since.getUTCDate() + 1)); + const untilDate = new Date(Date.UTC(until.getUTCFullYear(), until.getUTCMonth(), until.getUTCDate())); + + while (sinceDate < untilDate) { + dates.push(new Date(sinceDate)); + sinceDate.setUTCDate(sinceDate.getUTCDate() + 1); + } + + return dates.reverse(); +} + +function getTagHistoryWithGapsFilled(params: Parameters[0]) { + const history = trends.getTagHistory(params); + const dateRange = generateDateRange(params.since, params.until); + + return dateRange.map((day) => { + const data = history.find((item) => item.day.getTime() === day.getTime()); + return data || { day, accounts: 0, uses: 0 }; + }); +} + export { trendingTagsController }; From bc3a96eeedf82c4671d4f5d038548c74f67ec94f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 26 Jul 2023 17:18:21 -0500 Subject: [PATCH 3/4] Refactor trending tags history, make it more correct --- src/controllers/api/trends.ts | 29 +++-------------------------- src/trends.ts | 15 +++++++++++++-- src/utils/time.test.ts | 23 +++++++++++++++++++++++ src/utils/time.ts | 22 +++++++++++++++++++++- 4 files changed, 60 insertions(+), 29 deletions(-) create mode 100644 src/utils/time.test.ts diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index 916bf0a..28132f6 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -3,6 +3,7 @@ import { Conf } from '@/config.ts'; import { z } from '@/deps.ts'; import { trends } from '@/trends.ts'; import { Time } from '@/utils.ts'; +import { stripTime } from '@/utils/time.ts'; const limitSchema = z.coerce.number().catch(10).transform((value) => Math.min(Math.max(value, 0), 20)); @@ -25,11 +26,11 @@ const trendingTagsController: AppController = (c) => { url: Conf.local(`/tags/${name}`), history: [ { - day: String(Math.floor(now.getTime() / 1000)), + day: String(Math.floor(stripTime(now).getTime() / 1000)), accounts: String(accounts), uses: String(uses), }, - ...getTagHistoryWithGapsFilled({ + ...trends.getTagHistory({ tag: name, since: lastWeek, until: now, @@ -44,28 +45,4 @@ const trendingTagsController: AppController = (c) => { }))); }; -function generateDateRange(since: Date, until: Date): Date[] { - const dates = []; - - const sinceDate = new Date(Date.UTC(since.getUTCFullYear(), since.getUTCMonth(), since.getUTCDate() + 1)); - const untilDate = new Date(Date.UTC(until.getUTCFullYear(), until.getUTCMonth(), until.getUTCDate())); - - while (sinceDate < untilDate) { - dates.push(new Date(sinceDate)); - sinceDate.setUTCDate(sinceDate.getUTCDate() + 1); - } - - return dates.reverse(); -} - -function getTagHistoryWithGapsFilled(params: Parameters[0]) { - const history = trends.getTagHistory(params); - const dateRange = generateDateRange(params.since, params.until); - - return dateRange.map((day) => { - const data = history.find((item) => item.day.getTime() === day.getTime()); - return data || { day, accounts: 0, uses: 0 }; - }); -} - export { trendingTagsController }; diff --git a/src/trends.ts b/src/trends.ts index e079e7d..8e941b0 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -1,6 +1,7 @@ import { Sqlite } from '@/deps.ts'; import { hashtagSchema, hexIdSchema } from '@/schema.ts'; -import { Time } from './utils.ts'; +import { Time } from '@/utils.ts'; +import { generateDateRange } from '@/utils/time.ts'; interface GetTrendingTagsOpts { since: Date; @@ -63,7 +64,7 @@ class TrendsDB { } getTagHistory({ tag, since, until, limit = 7, offset = 0 }: GetTagHistoryOpts) { - return this.#db.query( + const result = this.#db.query( ` SELECT date(inserted_at), COUNT(DISTINCT pubkey8), COUNT(*) FROM tag_usages @@ -79,6 +80,16 @@ class TrendsDB { accounts: Number(row[1]), uses: Number(row[2]), })); + + const dateRange = generateDateRange( + new Date(since.getTime() + Time.days(1)), + new Date(until.getTime() - Time.days(offset)), + ).reverse(); + + return dateRange.map((day) => { + const data = result.find((item) => item.day.getTime() === day.getTime()); + return data || { day, accounts: 0, uses: 0 }; + }); } addTagUsages(pubkey: string, hashtags: string[], date = new Date()): void { diff --git a/src/utils/time.test.ts b/src/utils/time.test.ts new file mode 100644 index 0000000..c167caf --- /dev/null +++ b/src/utils/time.test.ts @@ -0,0 +1,23 @@ +import { assertEquals } from '@/deps-test.ts'; + +import { generateDateRange } from './time.ts'; + +Deno.test('generateDateRange', () => { + const since = new Date('2023-07-03T16:30:00.000Z'); + const until = new Date('2023-07-07T09:01:00.000Z'); + + const expected = [ + new Date('2023-07-03T00:00:00.000Z'), + new Date('2023-07-04T00:00:00.000Z'), + new Date('2023-07-05T00:00:00.000Z'), + new Date('2023-07-06T00:00:00.000Z'), + new Date('2023-07-07T00:00:00.000Z'), + ]; + + const result = generateDateRange(since, until); + + assertEquals( + result.map((d) => d.getTime()), + expected.map((d) => d.getTime()), + ); +}); diff --git a/src/utils/time.ts b/src/utils/time.ts index 1c2444e..2266eb4 100644 --- a/src/utils/time.ts +++ b/src/utils/time.ts @@ -9,4 +9,24 @@ const Time = { years: (y: number) => y * Time.days(365), }; -export { Time }; +/** Strips the time off the date, giving 12am UTC. */ +function stripTime(date: Date): Date { + return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())); +} + +/** Strips times off the dates and generates all 24h intervals between them, inclusive of both inputs. */ +function generateDateRange(since: Date, until: Date): Date[] { + const dates = []; + + const sinceDate = stripTime(since); + const untilDate = stripTime(until); + + while (sinceDate <= untilDate) { + dates.push(new Date(sinceDate)); + sinceDate.setUTCDate(sinceDate.getUTCDate() + 1); + } + + return dates; +} + +export { generateDateRange, stripTime, Time }; From c251b51803543c445ee38cfc17b417b87d6d03d2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 26 Jul 2023 17:30:17 -0500 Subject: [PATCH 4/4] Trends: add some useful comments for later --- src/controllers/api/trends.ts | 4 ++++ src/trends.ts | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/src/controllers/api/trends.ts b/src/controllers/api/trends.ts index 28132f6..7fc98e3 100644 --- a/src/controllers/api/trends.ts +++ b/src/controllers/api/trends.ts @@ -15,6 +15,7 @@ const trendingTagsController: AppController = (c) => { const yesterday = new Date(now.getTime() - Time.days(1)); const lastWeek = new Date(now.getTime() - Time.days(7)); + /** Most used hashtags within the past 24h. */ const tags = trends.getTrendingTags({ since: yesterday, until: now, @@ -25,6 +26,8 @@ const trendingTagsController: AppController = (c) => { name, url: Conf.local(`/tags/${name}`), history: [ + // Use the full 24h query for the current day. Then use `offset: 1` to adjust for this below. + // This result is more accurate than what Mastodon returns. { day: String(Math.floor(stripTime(now).getTime() / 1000)), accounts: String(accounts), @@ -37,6 +40,7 @@ const trendingTagsController: AppController = (c) => { limit: 6, offset: 1, }).map((history) => ({ + // For some reason, Mastodon wants these to be strings... oh well. day: String(Math.floor(history.day.getTime() / 1000)), accounts: String(history.accounts), uses: String(history.uses), diff --git a/src/trends.ts b/src/trends.ts index 8e941b0..3bb19d4 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -44,6 +44,7 @@ class TrendsDB { cleanup(); } + /** Gets the most used hashtags between the date range. */ getTrendingTags({ since, until, limit = 10, threshold = 3 }: GetTrendingTagsOpts) { return this.#db.query( ` @@ -63,6 +64,10 @@ class TrendsDB { })); } + /** + * Gets the tag usage count for a specific tag. + * It returns an array with counts for each date between the range. + */ getTagHistory({ tag, since, until, limit = 7, offset = 0 }: GetTagHistoryOpts) { const result = this.#db.query( ` @@ -81,11 +86,13 @@ class TrendsDB { uses: Number(row[2]), })); + /** Full date range between `since` and `until`. */ const dateRange = generateDateRange( new Date(since.getTime() + Time.days(1)), new Date(until.getTime() - Time.days(offset)), ).reverse(); + // Fill in missing dates with 0 usages. return dateRange.map((day) => { const data = result.find((item) => item.day.getTime() === day.getTime()); return data || { day, accounts: 0, uses: 0 };