From ce4a33081224686231e32c0857d024f2c0ca33ca Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 29 Aug 2023 12:41:14 -0500 Subject: [PATCH 1/8] Rename timeline controllers, homeController --> homeTimelineController, etc --- src/app.ts | 10 +++++++--- src/controllers/api/timelines.ts | 6 +++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/app.ts b/src/app.ts index 6d2305e..346831f 100644 --- a/src/app.ts +++ b/src/app.ts @@ -38,7 +38,11 @@ import { statusController, } from './controllers/api/statuses.ts'; import { streamingController } from './controllers/api/streaming.ts'; -import { hashtagTimelineController, homeController, publicController } from './controllers/api/timelines.ts'; +import { + hashtagTimelineController, + homeTimelineController, + publicTimelineController, +} from './controllers/api/timelines.ts'; import { trendingTagsController } from './controllers/api/trends.ts'; import { indexController } from './controllers/site.ts'; import { hostMetaController } from './controllers/well-known/host-meta.ts'; @@ -107,8 +111,8 @@ app.get('/api/v1/statuses/:id{[0-9a-f]{64}}', statusController); app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/favourite', favouriteController); app.post('/api/v1/statuses', requirePubkey, createStatusController); -app.get('/api/v1/timelines/home', requirePubkey, homeController); -app.get('/api/v1/timelines/public', publicController); +app.get('/api/v1/timelines/home', requirePubkey, homeTimelineController); +app.get('/api/v1/timelines/public', publicTimelineController); app.get('/api/v1/timelines/tag/:hashtag', hashtagTimelineController); app.get('/api/v1/preferences', preferencesController); diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index 801ee10..1b7c52b 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -8,7 +8,7 @@ import { Time } from '@/utils.ts'; import type { AppController } from '@/app.ts'; -const homeController: AppController = async (c) => { +const homeTimelineController: AppController = async (c) => { const params = paginationSchema.parse(c.req.query()); const pubkey = c.get('pubkey')!; @@ -27,7 +27,7 @@ const publicQuerySchema = z.object({ local: booleanParamSchema.catch(false), }); -const publicController: AppController = async (c) => { +const publicTimelineController: AppController = async (c) => { const params = paginationSchema.parse(c.req.query()); const { local } = publicQuerySchema.parse(c.req.query()); @@ -61,4 +61,4 @@ const hashtagTimelineController: AppController = async (c) => { return c.json(statuses, 200, link ? { link } : undefined); }; -export { hashtagTimelineController, homeController, publicController }; +export { hashtagTimelineController, homeTimelineController, publicTimelineController }; From 4216a7931aad0bbe2f990e228592d84648641c22 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 29 Aug 2023 12:55:00 -0500 Subject: [PATCH 2/8] Add `paginated` helper function, DRY pagination code --- src/controllers/api/accounts.ts | 6 ++---- src/controllers/api/notifications.ts | 6 ++---- src/controllers/api/timelines.ts | 20 +++++++------------- src/utils/web.ts | 26 +++++++++++++++++--------- 4 files changed, 28 insertions(+), 30 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 9244562..d84382a 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -6,7 +6,7 @@ import { booleanParamSchema } from '@/schema.ts'; import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; import { toAccount, toRelationship, toStatus } from '@/transformers/nostr-to-mastoapi.ts'; import { eventDateComparator, isFollowing, lookupAccount } from '@/utils.ts'; -import { buildLinkHeader, paginationSchema, parseBody } from '@/utils/web.ts'; +import { paginated, paginationSchema, parseBody } from '@/utils/web.ts'; import { createEvent } from '@/utils/web.ts'; const createAccountController: AppController = (c) => { @@ -110,9 +110,7 @@ const accountStatusesController: AppController = async (c) => { } const statuses = await Promise.all(events.map(toStatus)); - - const link = buildLinkHeader(c.req.url, events); - return c.json(statuses, 200, link ? { link } : undefined); + return paginated(c, events, statuses); }; const fileSchema = z.custom((value) => value instanceof File); diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index 30c3859..dce75e4 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -1,6 +1,6 @@ import { type AppController } from '@/app.ts'; import * as mixer from '@/mixer.ts'; -import { buildLinkHeader, paginationSchema } from '@/utils/web.ts'; +import { paginated, paginationSchema } from '@/utils/web.ts'; import { toNotification } from '@/transformers/nostr-to-mastoapi.ts'; import { Time } from '@/utils.ts'; @@ -14,9 +14,7 @@ const notificationsController: AppController = async (c) => { ); const statuses = await Promise.all(events.map(toNotification)); - - const link = buildLinkHeader(c.req.url, events); - return c.json(statuses, 200, link ? { link } : undefined); + return paginated(c, events, statuses); }; export { notificationsController }; diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index 1b7c52b..6f43034 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -3,7 +3,7 @@ import * as mixer from '@/mixer.ts'; import { getFeed, getPublicFeed } from '@/queries.ts'; import { booleanParamSchema } from '@/schema.ts'; import { toStatus } from '@/transformers/nostr-to-mastoapi.ts'; -import { buildLinkHeader, paginationSchema } from '@/utils/web.ts'; +import { paginated, paginationSchema } from '@/utils/web.ts'; import { Time } from '@/utils.ts'; import type { AppController } from '@/app.ts'; @@ -17,10 +17,8 @@ const homeTimelineController: AppController = async (c) => { return c.json([]); } - const statuses = (await Promise.all(events.map(toStatus))).filter(Boolean); - - const link = buildLinkHeader(c.req.url, events); - return c.json(statuses, 200, link ? { link } : undefined); + const statuses = await Promise.all(events.map(toStatus)); + return paginated(c, events, statuses); }; const publicQuerySchema = z.object({ @@ -36,10 +34,8 @@ const publicTimelineController: AppController = async (c) => { return c.json([]); } - const statuses = (await Promise.all(events.map(toStatus))).filter(Boolean); - - const link = buildLinkHeader(c.req.url, events); - return c.json(statuses, 200, link ? { link } : undefined); + const statuses = await Promise.all(events.map(toStatus)); + return paginated(c, events, statuses); }; const hashtagTimelineController: AppController = async (c) => { @@ -55,10 +51,8 @@ const hashtagTimelineController: AppController = async (c) => { return c.json([]); } - const statuses = (await Promise.all(events.map(toStatus))).filter(Boolean); - - const link = buildLinkHeader(c.req.url, events); - return c.json(statuses, 200, link ? { link } : undefined); + const statuses = await Promise.all(events.map(toStatus)); + return paginated(c, events, statuses); }; export { hashtagTimelineController, homeTimelineController, publicTimelineController }; diff --git a/src/utils/web.ts b/src/utils/web.ts index 7044d04..6d89523 100644 --- a/src/utils/web.ts +++ b/src/utils/web.ts @@ -89,6 +89,22 @@ function buildLinkHeader(url: string, events: Event[]): string | undefined { return `<${next}>; rel="next", <${prev}>; rel="prev"`; } +type Entity = { id: string }; +type HeaderRecord = Record; + +/** Return results with pagination headers. */ +function paginated(c: AppContext, events: Event[], entities: (Entity | undefined)[], headers: HeaderRecord = {}) { + const link = buildLinkHeader(c.req.url, events); + + if (link) { + headers.link = link; + } + + // Filter out undefined entities. + const results = entities.filter((entity): entity is Entity => Boolean(entity)); + return c.json(results, 200, headers); +} + /** JSON-LD context. */ type LDContext = (string | Record>)[]; @@ -107,12 +123,4 @@ function activityJson(c: Context, object: T) { return response; } -export { - activityJson, - buildLinkHeader, - createAdminEvent, - createEvent, - type PaginationParams, - paginationSchema, - parseBody, -}; +export { activityJson, createAdminEvent, createEvent, paginated, type PaginationParams, paginationSchema, parseBody }; From 4d211d637e6e2e08c77e44e8c8aed41b611e4d05 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 29 Aug 2023 13:01:47 -0500 Subject: [PATCH 3/8] timelines: add DRY renderStatuses function --- src/controllers/api/timelines.ts | 39 ++++++++++++-------------------- src/queries.ts | 31 +------------------------ 2 files changed, 15 insertions(+), 55 deletions(-) diff --git a/src/controllers/api/timelines.ts b/src/controllers/api/timelines.ts index 6f43034..d5833ca 100644 --- a/src/controllers/api/timelines.ts +++ b/src/controllers/api/timelines.ts @@ -1,51 +1,40 @@ import { z } from '@/deps.ts'; +import { type DittoFilter } from '@/filter.ts'; import * as mixer from '@/mixer.ts'; -import { getFeed, getPublicFeed } from '@/queries.ts'; +import { getFeedPubkeys } from '@/queries.ts'; import { booleanParamSchema } from '@/schema.ts'; import { toStatus } from '@/transformers/nostr-to-mastoapi.ts'; import { paginated, paginationSchema } from '@/utils/web.ts'; import { Time } from '@/utils.ts'; -import type { AppController } from '@/app.ts'; +import type { AppContext, AppController } from '@/app.ts'; const homeTimelineController: AppController = async (c) => { const params = paginationSchema.parse(c.req.query()); const pubkey = c.get('pubkey')!; - - const events = await getFeed(pubkey, params); - if (!events.length) { - return c.json([]); - } - - const statuses = await Promise.all(events.map(toStatus)); - return paginated(c, events, statuses); + const authors = await getFeedPubkeys(pubkey); + return renderStatuses(c, [{ authors, kinds: [1], ...params }]); }; const publicQuerySchema = z.object({ local: booleanParamSchema.catch(false), }); -const publicTimelineController: AppController = async (c) => { +const publicTimelineController: AppController = (c) => { const params = paginationSchema.parse(c.req.query()); const { local } = publicQuerySchema.parse(c.req.query()); - - const events = await getPublicFeed(params, local); - if (!events.length) { - return c.json([]); - } - - const statuses = await Promise.all(events.map(toStatus)); - return paginated(c, events, statuses); + return renderStatuses(c, [{ kinds: [1], local, ...params }]); }; -const hashtagTimelineController: AppController = async (c) => { +const hashtagTimelineController: AppController = (c) => { const hashtag = c.req.param('hashtag')!; const params = paginationSchema.parse(c.req.query()); + return renderStatuses(c, [{ kinds: [1], '#t': [hashtag], ...params }]); +}; - const events = await mixer.getFilters( - [{ kinds: [1], '#t': [hashtag], ...params }], - { timeout: Time.seconds(3) }, - ); +/** Render statuses for timelines. */ +async function renderStatuses(c: AppContext, filters: DittoFilter<1>[]) { + const events = await mixer.getFilters(filters, { timeout: Time.seconds(3) }); if (!events.length) { return c.json([]); @@ -53,6 +42,6 @@ const hashtagTimelineController: AppController = async (c) => { const statuses = await Promise.all(events.map(toStatus)); return paginated(c, events, statuses); -}; +} export { hashtagTimelineController, homeTimelineController, publicTimelineController }; diff --git a/src/queries.ts b/src/queries.ts index f990a8e..06a501a 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -53,24 +53,6 @@ async function getFeedPubkeys(pubkey: string): Promise { return [...authors, pubkey]; } -/** Get events from people the user follows. */ -async function getFeed(pubkey: string, params: PaginationParams): Promise[]> { - const authors = await getFeedPubkeys(pubkey); - - const filter: Filter<1> = { - authors, - kinds: [1], - ...params, - }; - - return mixer.getFilters([filter], { timeout: 5000 }); -} - -/** Get a feed of all known text notes. */ -function getPublicFeed(params: PaginationParams, local: boolean): Promise[]> { - return mixer.getFilters([{ kinds: [1], local, ...params }], { timeout: 5000 }); -} - async function getAncestors(event: Event<1>, result = [] as Event<1>[]): Promise[]> { if (result.length < 100) { const replyTag = findReplyTag(event); @@ -106,15 +88,4 @@ async function syncUser(pubkey: string): Promise { ], { timeout: 5000 }); } -export { - getAncestors, - getAuthor, - getDescendants, - getEvent, - getFeed, - getFeedPubkeys, - getFollows, - getPublicFeed, - isLocallyFollowed, - syncUser, -}; +export { getAncestors, getAuthor, getDescendants, getEvent, getFeedPubkeys, getFollows, isLocallyFollowed, syncUser }; From d21ec6d241a2b5ba8c1d593114a8f2f4f3d9dee3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 29 Aug 2023 13:04:38 -0500 Subject: [PATCH 4/8] apps: use AppController type --- src/controllers/api/apps.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/controllers/api/apps.ts b/src/controllers/api/apps.ts index c3ebee3..4f60a19 100644 --- a/src/controllers/api/apps.ts +++ b/src/controllers/api/apps.ts @@ -1,4 +1,4 @@ -import type { Context } from '@/deps.ts'; +import type { AppController } from '@/app.ts'; /** * Apps are unnecessary cruft in Mastodon API, but necessary to make clients work. @@ -14,7 +14,7 @@ const FAKE_APP = { vapid_key: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', }; -async function createAppController(c: Context) { +const createAppController: AppController = async (c) => { // TODO: Handle both formData and json. 422 on parsing error. try { const { redirect_uris } = await c.req.json(); @@ -26,10 +26,10 @@ async function createAppController(c: Context) { } catch (_e) { return c.json(FAKE_APP); } -} +}; -function appCredentialsController(c: Context) { +const appCredentialsController: AppController = (c) => { return c.json(FAKE_APP); -} +}; export { appCredentialsController, createAppController }; From ebd933126a585d6b8cad55237e3eee33f4fed5c3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 29 Aug 2023 13:14:03 -0500 Subject: [PATCH 5/8] webfinger: fix import order --- src/controllers/well-known/webfinger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/well-known/webfinger.ts b/src/controllers/well-known/webfinger.ts index b3a14af..fa45aaa 100644 --- a/src/controllers/well-known/webfinger.ts +++ b/src/controllers/well-known/webfinger.ts @@ -1,9 +1,9 @@ import { Conf } from '@/config.ts'; +import { findUser } from '@/db/users.ts'; import { nip19, z } from '@/deps.ts'; import type { AppContext, AppController } from '@/app.ts'; import type { Webfinger } from '@/schemas/webfinger.ts'; -import { findUser } from '@/db/users.ts'; const webfingerQuerySchema = z.object({ resource: z.string().url(), From 77b09baa8c2f2b46cfd6e250eadbf490ebc4b27f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 29 Aug 2023 13:20:21 -0500 Subject: [PATCH 6/8] db/events: don't throw on duplicate events --- src/db/events.ts | 11 +++++++++-- src/deps.ts | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index 48b9666..0404834 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -1,5 +1,5 @@ import { db, type TagRow } from '@/db.ts'; -import { type Event, type Insertable } from '@/deps.ts'; +import { type Event, type Insertable, SqliteError } from '@/deps.ts'; import type { DittoFilter, GetFiltersOpts } from '@/filter.ts'; @@ -23,7 +23,7 @@ function insertEvent(event: Event): Promise { ...event, tags: JSON.stringify(event.tags), }) - .executeTakeFirst(); + .execute(); const tagCounts: Record = {}; const tags = event.tags.reduce[]>((results, tag) => { @@ -48,6 +48,13 @@ function insertEvent(event: Event): Promise { .values(tags) .execute(); } + }).catch((error) => { + // Don't throw for duplicate events. + if (error instanceof SqliteError && error.code === 19) { + return; + } else { + throw error; + } }); } diff --git a/src/deps.ts b/src/deps.ts index 1ac6d9f..b361c5b 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -51,7 +51,7 @@ export { export { generateSeededRsa } from 'https://gitlab.com/soapbox-pub/seeded-rsa/-/raw/v1.0.0/mod.ts'; export * as secp from 'npm:@noble/secp256k1@^2.0.0'; export { LRUCache } from 'npm:lru-cache@^10.0.0'; -export { DB as Sqlite } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; +export { DB as Sqlite, SqliteError } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; export * as dotenv from 'https://deno.land/std@0.198.0/dotenv/mod.ts'; export { FileMigrationProvider, From 2841d4f399d49959a7ac2e0f8c87f3f6a8299023 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 29 Aug 2023 13:25:58 -0500 Subject: [PATCH 7/8] queries: remove unused import --- src/queries.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/queries.ts b/src/queries.ts index 06a501a..5429abd 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -2,7 +2,6 @@ import * as client from '@/client.ts'; import * as eventsDB from '@/db/events.ts'; import { type Event, type Filter, findReplyTag } from '@/deps.ts'; import * as mixer from '@/mixer.ts'; -import { type PaginationParams } from '@/utils/web.ts'; interface GetEventOpts { /** Timeout in milliseconds. */ From 95761e2eefdfb20144780de5c8970f8923821af7 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 29 Aug 2023 13:28:16 -0500 Subject: [PATCH 8/8] schema: add comments --- src/schema.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/schema.ts b/src/schema.ts index 3ecc76a..d32251b 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -11,6 +11,7 @@ function filteredArray(schema: T) { )); } +/** Parses a JSON string into its native type. */ const jsonSchema = z.string().transform((value, ctx) => { try { return JSON.parse(value) as unknown; @@ -20,6 +21,7 @@ const jsonSchema = z.string().transform((value, ctx) => { } }); +/** Parses a Nostr emoji tag. */ const emojiTagSchema = z.tuple([z.literal('emoji'), z.string(), z.string().url()]); /** https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem */ @@ -34,6 +36,7 @@ const decode64Schema = z.string().transform((value, ctx) => { } }); +/** Parses a hashtag, eg `#yolo`. */ const hashtagSchema = z.string().regex(/^\w{1,30}$/); /**