From 66277041143630c382fcf5d9ab585151c0c1ec58 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 28 Apr 2024 17:07:41 -0300 Subject: [PATCH 01/65] feat: save user preferences (kind 30078) & encrypt it --- src/controllers/api/accounts.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 0289799..f6440fe 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -18,6 +18,7 @@ import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderRelationship } from '@/views/mastodon/relationships.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { APISigner } from '@/signers/APISigner.ts'; const usernameSchema = z .string().min(1).max(30) @@ -186,6 +187,7 @@ const updateCredentialsSchema = z.object({ bot: z.boolean().optional(), discoverable: z.boolean().optional(), nip05: z.string().optional(), + pleroma_settings_store: z.object({ soapbox_fe: z.object({ themeMode: z.string() }).passthrough() }).optional(), }); const updateCredentialsController: AppController = async (c) => { @@ -225,6 +227,16 @@ const updateCredentialsController: AppController = async (c) => { tags: [], }, c); + const soapbox_fe = result.data.pleroma_settings_store?.soapbox_fe; + if (soapbox_fe) { + const signer = new APISigner(c); + await createEvent({ + kind: 30078, + tags: [['d', 'pub.ditto.preferences']], + content: await signer.nip44.encrypt(pubkey, JSON.stringify(soapbox_fe)), + }, c); + } + const account = await renderAccount(event, { withSource: true }); return c.json(account); }; From bb82df14c6c4d0cf9612a6556fbcdb994517ddbb Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 28 Apr 2024 21:42:57 -0300 Subject: [PATCH 02/65] refactor: user preference in create & verify credentials --- src/controllers/api/accounts.ts | 39 ++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index f6440fe..5f0840e 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -50,7 +50,21 @@ const verifyCredentialsController: AppController = async (c) => { const event = await getAuthor(pubkey, { relations: ['author_stats'] }); if (event) { - return c.json(await renderAccount(event, { withSource: true })); + const account = await renderAccount(event, { withSource: true }); + + const [userPreferencesEvent] = await eventsDB.query([{ + authors: [pubkey], + kinds: [30078], + '#d': ['pub.ditto.pleroma_settings_store'], + limit: 1, + }]); + if (userPreferencesEvent) { + const signer = new APISigner(c); + const userPreference = JSON.parse(await signer.nip44.decrypt(pubkey, userPreferencesEvent.content)); + (account.pleroma as any).settings_store = userPreference; + } + + return c.json(account); } else { return c.json(await accountFromPubkey(pubkey, { withSource: true })); } @@ -187,7 +201,7 @@ const updateCredentialsSchema = z.object({ bot: z.boolean().optional(), discoverable: z.boolean().optional(), nip05: z.string().optional(), - pleroma_settings_store: z.object({ soapbox_fe: z.object({ themeMode: z.string() }).passthrough() }).optional(), + pleroma_settings_store: z.object({ soapbox_fe: z.record(z.string(), z.unknown()) }).optional(), }); const updateCredentialsController: AppController = async (c) => { @@ -227,17 +241,30 @@ const updateCredentialsController: AppController = async (c) => { tags: [], }, c); - const soapbox_fe = result.data.pleroma_settings_store?.soapbox_fe; - if (soapbox_fe) { + const pleroma_frontend = result.data.pleroma_settings_store; + if (pleroma_frontend) { const signer = new APISigner(c); await createEvent({ kind: 30078, - tags: [['d', 'pub.ditto.preferences']], - content: await signer.nip44.encrypt(pubkey, JSON.stringify(soapbox_fe)), + tags: [['d', 'pub.ditto.pleroma_settings_store']], + content: await signer.nip44.encrypt(pubkey, JSON.stringify(pleroma_frontend)), }, c); } const account = await renderAccount(event, { withSource: true }); + + const [userPreferencesEvent] = await eventsDB.query([{ + authors: [pubkey], + kinds: [30078], + '#d': ['pub.ditto.pleroma_settings_store'], + limit: 1, + }]); + if (userPreferencesEvent) { + const signer = new APISigner(c); + const userPreference = JSON.parse(await signer.nip44.decrypt(pubkey, userPreferencesEvent.content)); + (account.pleroma as any).settings_store = userPreference; + } + return c.json(account); }; From e61cbecb3e840100d6a4561a795f6ccab17d5cb4 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 16 May 2024 10:29:14 -0300 Subject: [PATCH 03/65] refactor(unreblog): update error messages and query with Storages.db() --- src/controllers/api/statuses.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 98173b0..984dfdb 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -261,21 +261,19 @@ const reblogStatusController: AppController = async (c) => { const unreblogStatusController: AppController = async (c) => { const eventId = c.req.param('id'); const pubkey = await c.get('signer')?.getPublicKey()!; - - const event = await getEvent(eventId, { kind: 1 }); - - if (!event) { - return c.json({ error: 'Event not found.' }, 404); - } - const store = await Storages.db(); + const [event] = await store.query([{ ids: [eventId], kinds: [1] }]); + if (!event) { + return c.json({ error: 'Record not found' }, 404); + } + const [repostedEvent] = await store.query( [{ kinds: [6], authors: [pubkey], '#e': [event.id], limit: 1 }], ); if (!repostedEvent) { - return c.json({ error: 'Event not found.' }, 404); + return c.json({ error: 'Record not found' }, 404); } await createEvent({ From 6c3f0849b231400dea4b78f0ea9b2a0fbb943d85 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 May 2024 12:57:01 -0500 Subject: [PATCH 04/65] Upgrade Nostrify to v0.19.2, fix crash on mixed filters --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 25ec2ed..b783f64 100644 --- a/deno.json +++ b/deno.json @@ -21,7 +21,7 @@ "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.19.1", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.19.2", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", From 00d4bf23448826fffdcacdc449270ba86a4e911e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 May 2024 15:42:32 -0500 Subject: [PATCH 05/65] Upgrade Nostrify to v0.20.0, enable Postgres FTS --- deno.json | 2 +- src/db/migrations/020_pgfts.ts | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 src/db/migrations/020_pgfts.ts diff --git a/deno.json b/deno.json index b783f64..e8719a4 100644 --- a/deno.json +++ b/deno.json @@ -21,7 +21,7 @@ "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.19.2", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.20.0", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", diff --git a/src/db/migrations/020_pgfts.ts b/src/db/migrations/020_pgfts.ts new file mode 100644 index 0000000..8b3cfa0 --- /dev/null +++ b/src/db/migrations/020_pgfts.ts @@ -0,0 +1,19 @@ +import { Kysely, sql } from 'kysely'; + +import { Conf } from '@/config.ts'; + +export async function up(db: Kysely): Promise { + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + await db.schema.createTable('nostr_pgfts') + .ifNotExists() + .addColumn('event_id', 'text', (c) => c.primaryKey().references('nostr_events.id').onDelete('cascade')) + .addColumn('search_vec', sql`tsvector`, (c) => c.notNull()) + .execute(); + } +} + +export async function down(db: Kysely): Promise { + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + await db.schema.dropTable('nostr_pgfts').ifExists().execute(); + } +} From baa698688094195ed3533c50b0eb2450a06180ae Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 May 2024 15:48:22 -0500 Subject: [PATCH 06/65] EventsDB: enable fts conditionally based on DATABASE_URL --- src/storages/EventsDB.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index aac8e52..ef51a89 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -42,8 +42,17 @@ class EventsDB implements NStore { }; constructor(private kysely: Kysely) { + let fts: 'sqlite' | 'postgres' | undefined; + + if (Conf.databaseUrl.protocol === 'sqlite:') { + fts = 'sqlite'; + } + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + fts = 'postgres'; + } + this.store = new NDatabase(kysely, { - fts5: Conf.databaseUrl.protocol === 'sqlite:', + fts, indexTags: EventsDB.indexTags, searchText: EventsDB.searchText, }); From 5aacbe7af5717b2d032064dbd18fd333e55cf8fe Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 16 May 2024 18:53:04 -0500 Subject: [PATCH 07/65] Fix media uploads due to 'awaiting' a query builder instance --- src/controllers/api/statuses.ts | 4 +++- src/db/unattached-media.ts | 15 +++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 984dfdb..00f9a98 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -4,6 +4,7 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; import { addTag, deleteTag } from '@/tags.ts'; @@ -56,6 +57,7 @@ const statusController: AppController = async (c) => { const createStatusController: AppController = async (c) => { const body = await parseBody(c.req.raw); const result = createStatusSchema.safeParse(body); + const kysely = await DittoDB.getInstance(); if (!result.success) { return c.json({ error: 'Bad request', schema: result.error }, 400); @@ -92,7 +94,7 @@ const createStatusController: AppController = async (c) => { const viewerPubkey = await c.get('signer')?.getPublicKey(); if (data.media_ids?.length) { - const media = await getUnattachedMediaByIds(data.media_ids) + const media = await getUnattachedMediaByIds(kysely, data.media_ids) .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey)) .then((media) => media.map(({ url, data }) => ['media', url, data])); diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index 8063628..cee1e3a 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -1,6 +1,8 @@ +import { Kysely } from 'kysely'; import uuid62 from 'uuid62'; import { DittoDB } from '@/db/DittoDB.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { type MediaData } from '@/schemas/nostr.ts'; interface UnattachedMedia { @@ -28,8 +30,7 @@ async function insertUnattachedMedia(media: Omit) { return kysely.selectFrom('unattached_media') .select([ 'unattached_media.id', @@ -41,9 +42,8 @@ async function selectUnattachedMediaQuery() { } /** Find attachments that exist but aren't attached to any events. */ -async function getUnattachedMedia(until: Date) { - const query = await selectUnattachedMediaQuery(); - return query +function getUnattachedMedia(kysely: Kysely, until: Date) { + return selectUnattachedMediaQuery(kysely) .leftJoin('nostr_tags', 'unattached_media.url', 'nostr_tags.value') .where('uploaded_at', '<', until.getTime()) .execute(); @@ -58,10 +58,9 @@ async function deleteUnattachedMediaByUrl(url: string) { } /** Get unattached media by IDs. */ -async function getUnattachedMediaByIds(ids: string[]) { +async function getUnattachedMediaByIds(kysely: Kysely, ids: string[]) { if (!ids.length) return []; - const query = await selectUnattachedMediaQuery(); - return query + return await selectUnattachedMediaQuery(kysely) .where('id', 'in', ids) .execute(); } From 2aee2e6bf6facc883b1364af8f630ec88a962cfa Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 17 May 2024 09:45:19 -0300 Subject: [PATCH 08/65] fix(renderReblog): render account from pubkey if there is no kind 0 --- src/views/mastodon/statuses.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 776f016..aefe258 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -123,8 +123,6 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< async function renderReblog(event: DittoEvent, opts: RenderStatusOpts) { const { viewerPubkey } = opts; - if (!event.author) return; - const repostId = event.tags.find(([name]) => name === 'e')?.[1]; if (!repostId) return; @@ -134,7 +132,7 @@ async function renderReblog(event: DittoEvent, opts: RenderStatusOpts) { return { id: event.id, - account: await renderAccount(event.author), + account: event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey), reblogged: true, reblog, }; From 4cc1d13d44b50dc31ad9af93f050eb19ab9d3b83 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 17 May 2024 11:25:17 -0300 Subject: [PATCH 09/65] fix: render followers & following list when no kind 0 --- src/views.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/views.ts b/src/views.ts index a737542..863dd30 100644 --- a/src/views.ts +++ b/src/views.ts @@ -5,6 +5,7 @@ import { renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { paginated, paginationSchema } from '@/utils/api.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { accountFromPubkey } from '@/views/mastodon/accounts.ts'; /** Render account objects for the author of each event. */ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal = AbortSignal.timeout(1000)) { @@ -24,7 +25,13 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal .then((events) => hydrateEvents({ events, store, signal })); const accounts = await Promise.all( - authors.map((event) => renderAccount(event)), + Array.from(pubkeys).map(async (pubkey) => { + const event = authors.find((event) => event.pubkey === pubkey); + if (event) { + return await renderAccount(event); + } + return await accountFromPubkey(pubkey); + }), ); return paginated(c, events, accounts); @@ -39,7 +46,13 @@ async function renderAccounts(c: AppContext, authors: string[], signal = AbortSi .then((events) => hydrateEvents({ events, store, signal })); const accounts = await Promise.all( - events.map((event) => renderAccount(event)), + authors.map(async (pubkey) => { + const event = events.find((event) => event.pubkey === pubkey); + if (event) { + return await renderAccount(event); + } + return await accountFromPubkey(pubkey); + }), ); return paginated(c, events, accounts); From 251500fba1b82de21098776c4d98bfd0d6a40660 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 11:39:21 -0500 Subject: [PATCH 10/65] Never let stats be less than 0 --- src/storages/hydrate.ts | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 8d2d302..9b95841 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -251,11 +251,19 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise ({ + pubkey: row.pubkey, + followers_count: Math.max(0, row.followers_count), + following_count: Math.max(0, row.following_count), + notes_count: Math.max(0, row.notes_count), + })); } /** Collect event stats from the events. */ @@ -271,11 +279,19 @@ async function gatherEventStats(events: DittoEvent[]): Promise ({ + event_id: row.event_id, + reposts_count: Math.max(0, row.reposts_count), + reactions_count: Math.max(0, row.reactions_count), + replies_count: Math.max(0, row.replies_count), + })); } /** Return a normalized event without any non-standard keys. */ From a39910fa984e5f4896aa860143af07c1044e0271 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 13:12:40 -0500 Subject: [PATCH 11/65] Add a function to recalculate author stats --- scripts/stats-recompute.ts | 25 ++----------------------- src/stats.ts | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index dcb0bc0..4037a85 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -1,8 +1,6 @@ import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; -import { Storages } from '@/storages.ts'; +import { refreshAuthorStats } from '@/stats.ts'; let pubkey: string; try { @@ -17,23 +15,4 @@ try { Deno.exit(1); } -const store = await Storages.db(); -const kysely = await DittoDB.getInstance(); - -const [followList] = await store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]); - -const authorStats: DittoTables['author_stats'] = { - pubkey, - followers_count: (await store.count([{ kinds: [3], '#p': [pubkey] }])).count, - following_count: followList?.tags.filter(([name]) => name === 'p')?.length ?? 0, - notes_count: (await store.count([{ kinds: [1], authors: [pubkey] }])).count, -}; - -await kysely.insertInto('author_stats') - .values(authorStats) - .onConflict((oc) => - oc - .column('pubkey') - .doUpdateSet(authorStats) - ) - .execute(); +await refreshAuthorStats(pubkey); diff --git a/src/stats.ts b/src/stats.ts index 9204071..74242f7 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,11 +1,12 @@ -import { NKinds, NostrEvent } from '@nostrify/nostrify'; +import { NKinds, NostrEvent, NStore } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { InsertQueryBuilder, Kysely } from 'kysely'; +import { SetRequired } from 'type-fest'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Storages } from '@/storages.ts'; -import { findReplyTag } from '@/tags.ts'; +import { findReplyTag, getTagSet } from '@/tags.ts'; type AuthorStat = keyof Omit; type EventStat = keyof Omit; @@ -216,4 +217,35 @@ function getFollowDiff(event: NostrEvent, prev?: NostrEvent): AuthorStatDiff[] { ]; } -export { updateStats }; +/** Refresh the author's stats in the database. */ +async function refreshAuthorStats(pubkey: string): Promise { + const store = await Storages.db(); + const stats = await countAuthorStats(store, pubkey); + + const kysely = await DittoDB.getInstance(); + await kysely.insertInto('author_stats') + .values(stats) + .onConflict((oc) => oc.column('pubkey').doUpdateSet(stats)) + .execute(); +} + +/** Calculate author stats from the database. */ +async function countAuthorStats( + store: SetRequired, + pubkey: string, +): Promise { + const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([ + store.count([{ kinds: [3], '#p': [pubkey] }]), + store.count([{ kinds: [1], authors: [pubkey] }]), + store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), + ]); + + return { + pubkey, + followers_count, + following_count: getTagSet(followList?.tags ?? [], 'p').size, + notes_count, + }; +} + +export { refreshAuthorStats, updateStats }; From 6995bd2b292810913c6c8d5877ea8f76b91e6b42 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 16:23:23 -0500 Subject: [PATCH 12/65] Upgrade Deno to the latest version --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9dee45..8e72888 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.41.3 +image: denoland/deno:1.43.4 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index a13fd5f..9bbaf96 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.41.3 \ No newline at end of file +deno 1.43.4 \ No newline at end of file From ae9516b445332df74aa817a26e5be522e243a923 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 16:23:38 -0500 Subject: [PATCH 13/65] refreshAuthorStats: return the stats --- src/stats.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/stats.ts b/src/stats.ts index 74242f7..08d4bb9 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -218,7 +218,7 @@ function getFollowDiff(event: NostrEvent, prev?: NostrEvent): AuthorStatDiff[] { } /** Refresh the author's stats in the database. */ -async function refreshAuthorStats(pubkey: string): Promise { +async function refreshAuthorStats(pubkey: string): Promise { const store = await Storages.db(); const stats = await countAuthorStats(store, pubkey); @@ -227,6 +227,8 @@ async function refreshAuthorStats(pubkey: string): Promise { .values(stats) .onConflict((oc) => oc.column('pubkey').doUpdateSet(stats)) .execute(); + + return stats; } /** Calculate author stats from the database. */ From 17b633019339506d2414bc27056531088544499a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 16:59:45 -0500 Subject: [PATCH 14/65] Downgrade Deno to v1.43.3 due to TypeScript issues --- .gitlab-ci.yml | 2 +- .tool-versions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e72888..b2140db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.43.4 +image: denoland/deno:1.43.3 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index 9bbaf96..b3e19cd 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.43.4 \ No newline at end of file +deno 1.43.3 \ No newline at end of file From 5c2e3450a9ae2b06645f570ef2059109120a648f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 17:50:30 -0500 Subject: [PATCH 15/65] Refresh author stats: less naive way --- src/storages/hydrate.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 9b95841..7b964d7 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,10 +1,12 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; +import { LRUCache } from 'lru-cache'; import { matchFilter } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; +import { refreshAuthorStats } from '@/stats.ts'; interface HydrateOpts { events: DittoEvent[]; @@ -55,6 +57,8 @@ async function hydrateEvents(opts: HydrateOpts): Promise { events: await gatherEventStats(cache), }; + requestMissingAuthorStats(events, stats.authors); + // Dedupe events. const results = [...new Map(cache.map((event) => [event.id, event])).values()]; @@ -266,6 +270,31 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise( + events + .filter((event) => event.kind === 0) + .map((event) => event.pubkey), + ); + + const missing = pubkeys.difference( + new Set(stats.map((stat) => stat.pubkey)), + ); + + for (const pubkey of missing) { + refreshAuthorStatsDebounced(pubkey); + } +} + +const lru = new LRUCache({ max: 1000 }); + +/** Calls `refreshAuthorStats` only once per author. */ +function refreshAuthorStatsDebounced(pubkey: string): void { + if (lru.get(pubkey)) return; + lru.set(pubkey, true); + refreshAuthorStats(pubkey).catch(() => {}); +} + /** Collect event stats from the events. */ async function gatherEventStats(events: DittoEvent[]): Promise { const ids = new Set( From bf479d01625497eac9190286eaf383f381ca606b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 18:26:55 -0500 Subject: [PATCH 16/65] Move refreshAuthorStatsDebounced to stats.ts --- src/stats.ts | 12 +++++++++++- src/storages/hydrate.ts | 16 +++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/stats.ts b/src/stats.ts index 08d4bb9..4364781 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,6 +1,7 @@ import { NKinds, NostrEvent, NStore } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { InsertQueryBuilder, Kysely } from 'kysely'; +import { LRUCache } from 'lru-cache'; import { SetRequired } from 'type-fest'; import { DittoDB } from '@/db/DittoDB.ts'; @@ -250,4 +251,13 @@ async function countAuthorStats( }; } -export { refreshAuthorStats, updateStats }; +const lru = new LRUCache({ max: 1000 }); + +/** Calls `refreshAuthorStats` only once per author. */ +function refreshAuthorStatsDebounced(pubkey: string): void { + if (lru.get(pubkey)) return; + lru.set(pubkey, true); + refreshAuthorStats(pubkey).catch(() => {}); +} + +export { refreshAuthorStats, refreshAuthorStatsDebounced, updateStats }; diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 7b964d7..e5c488e 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,12 +1,11 @@ import { NostrEvent, NStore } from '@nostrify/nostrify'; -import { LRUCache } from 'lru-cache'; import { matchFilter } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; -import { refreshAuthorStats } from '@/stats.ts'; +import { refreshAuthorStatsDebounced } from '@/stats.ts'; interface HydrateOpts { events: DittoEvent[]; @@ -57,7 +56,7 @@ async function hydrateEvents(opts: HydrateOpts): Promise { events: await gatherEventStats(cache), }; - requestMissingAuthorStats(events, stats.authors); + refreshMissingAuthorStats(events, stats.authors); // Dedupe events. const results = [...new Map(cache.map((event) => [event.id, event])).values()]; @@ -270,7 +269,7 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise( events .filter((event) => event.kind === 0) @@ -286,15 +285,6 @@ function requestMissingAuthorStats(events: NostrEvent[], stats: DittoTables['aut } } -const lru = new LRUCache({ max: 1000 }); - -/** Calls `refreshAuthorStats` only once per author. */ -function refreshAuthorStatsDebounced(pubkey: string): void { - if (lru.get(pubkey)) return; - lru.set(pubkey, true); - refreshAuthorStats(pubkey).catch(() => {}); -} - /** Collect event stats from the events. */ async function gatherEventStats(events: DittoEvent[]): Promise { const ids = new Set( From 23a366081fa19e4e75d411641b34ec4ff97e2fb4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 18:42:45 -0500 Subject: [PATCH 17/65] stats: maybe refresh stats when updating --- src/stats.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/stats.ts b/src/stats.ts index 4364781..9f0d257 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -39,6 +39,8 @@ async function updateStats(event: NostrEvent) { debug(JSON.stringify({ id: event.id, pubkey: event.pubkey, kind: event.kind, tags: event.tags, statDiffs })); } + pubkeyDiffs.forEach(([_, pubkey]) => refreshAuthorStatsDebounced(pubkey)); + const kysely = await DittoDB.getInstance(); if (pubkeyDiffs.length) queries.push(authorStatsQuery(kysely, pubkeyDiffs)); From f9a0055e78a582e26e88458bdc8ba1f5eaa6c798 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 17 May 2024 19:00:56 -0500 Subject: [PATCH 18/65] stats: add a Semaphore when refreshing author stats --- deno.json | 1 + src/stats.ts | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/deno.json b/deno.json index e8719a4..84d5351 100644 --- a/deno.json +++ b/deno.json @@ -20,6 +20,7 @@ "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", + "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.20.0", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", diff --git a/src/stats.ts b/src/stats.ts index 9f0d257..7112496 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,3 +1,4 @@ +import { Semaphore } from '@lambdalisue/async'; import { NKinds, NostrEvent, NStore } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { InsertQueryBuilder, Kysely } from 'kysely'; @@ -253,13 +254,19 @@ async function countAuthorStats( }; } -const lru = new LRUCache({ max: 1000 }); +const authorStatsSemaphore = new Semaphore(10); +const refreshedAuthors = new LRUCache({ max: 1000 }); /** Calls `refreshAuthorStats` only once per author. */ function refreshAuthorStatsDebounced(pubkey: string): void { - if (lru.get(pubkey)) return; - lru.set(pubkey, true); - refreshAuthorStats(pubkey).catch(() => {}); + if (refreshedAuthors.get(pubkey)) { + return; + } + + refreshedAuthors.set(pubkey, true); + + authorStatsSemaphore + .lock(() => refreshAuthorStats(pubkey).catch(() => {})); } export { refreshAuthorStats, refreshAuthorStatsDebounced, updateStats }; From 3e93b42251db66e24ef25ba7bd3d2cc6aa91d0cb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 08:13:37 -0500 Subject: [PATCH 19/65] stats: add a debug call --- src/stats.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stats.ts b/src/stats.ts index 7112496..256c570 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -264,6 +264,7 @@ function refreshAuthorStatsDebounced(pubkey: string): void { } refreshedAuthors.set(pubkey, true); + debug('refreshing author stats:', pubkey); authorStatsSemaphore .lock(() => refreshAuthorStats(pubkey).catch(() => {})); From 6ac4c072a6caeb9216c9c818e53e70d1249d69eb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 08:20:43 -0500 Subject: [PATCH 20/65] Fix crash decoding url --- src/note.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/note.ts b/src/note.ts index 5603c53..1c5c70c 100644 --- a/src/note.ts +++ b/src/note.ts @@ -18,7 +18,7 @@ const linkifyOpts: linkify.Opts = { return `#${tag}`; }, url: ({ content }) => { - if (nip21.test(content)) { + try { const { decoded } = nip21.parse(content); const pubkey = getDecodedPubkey(decoded); if (pubkey) { @@ -28,7 +28,7 @@ const linkifyOpts: linkify.Opts = { } else { return ''; } - } else { + } catch { return `${content}`; } }, From 4c87e723c06c3df9bc059282c790d59bd72d53b6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 10:56:34 -0500 Subject: [PATCH 21/65] Bump nostrify to v0.21.1 --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 84d5351..e40dd24 100644 --- a/deno.json +++ b/deno.json @@ -22,7 +22,7 @@ "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.20.0", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.21.1", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", From f97064afb4bc1a6c05e430f4d4315ca94ed4f179 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 11:35:29 -0500 Subject: [PATCH 22/65] Remove dependency on npm:mime, switch to @std/media-types --- deno.json | 2 +- src/deps.ts | 2 -- src/note.ts | 9 +++++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/deno.json b/deno.json index e40dd24..54ba468 100644 --- a/deno.json +++ b/deno.json @@ -32,7 +32,7 @@ "@std/dotenv": "jsr:@std/dotenv@^0.224.0", "@std/encoding": "jsr:@std/encoding@^0.224.0", "@std/json": "jsr:@std/json@^0.223.0", - "@std/media-types": "jsr:@std/media-types@^0.224.0", + "@std/media-types": "jsr:@std/media-types@^0.224.1", "@std/streams": "jsr:@std/streams@^0.223.0", "comlink": "npm:comlink@^4.4.1", "deno-safe-fetch": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", diff --git a/src/deps.ts b/src/deps.ts index 46d8fec..12be07f 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,8 +1,6 @@ import 'deno-safe-fetch'; // @deno-types="npm:@types/lodash@4.14.194" export { default as lodash } from 'https://esm.sh/lodash@4.17.21'; -// @deno-types="npm:@types/mime@3.0.0" -export { default as mime } from 'npm:mime@^3.0.0'; // @deno-types="npm:@types/sanitize-html@2.9.0" export { default as sanitizeHtml } from 'npm:sanitize-html@^2.11.0'; export { diff --git a/src/note.ts b/src/note.ts index 1c5c70c..7dc39f5 100644 --- a/src/note.ts +++ b/src/note.ts @@ -1,10 +1,10 @@ +import { typeByExtension } from '@std/media-types'; import 'linkify-plugin-hashtag'; import linkifyStr from 'linkify-string'; import linkify from 'linkifyjs'; import { nip19, nip21 } from 'nostr-tools'; import { Conf } from '@/config.ts'; -import { mime } from '@/deps.ts'; import { type DittoAttachment } from '@/views/mastodon/attachments.ts'; linkify.registerCustomProtocol('nostr', true); @@ -87,12 +87,13 @@ function isLinkURL(link: Link): boolean { return link.type === 'url'; } -/** `npm:mime` treats `.com` as a file extension, so parse the full URL to get its path first. */ +/** Get the extension from the URL, then get its type. */ function getUrlMimeType(url: string): string | undefined { try { const { pathname } = new URL(url); - return mime.getType(pathname) || undefined; - } catch (_e) { + const ext = pathname.split('.').pop() ?? ''; + return typeByExtension(ext); + } catch { return undefined; } } From 5997ff0fff38bf79486818a7e0f9ceae2182e4b6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 11:52:33 -0500 Subject: [PATCH 23/65] Create utils/media.ts, move some code from note.ts there --- src/note.ts | 25 ++++++------------------- src/utils/media.test.ts | 17 +++++++++++++++++ src/utils/media.ts | 24 ++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 19 deletions(-) create mode 100644 src/utils/media.test.ts create mode 100644 src/utils/media.ts diff --git a/src/note.ts b/src/note.ts index 7dc39f5..71dc017 100644 --- a/src/note.ts +++ b/src/note.ts @@ -1,10 +1,10 @@ -import { typeByExtension } from '@std/media-types'; import 'linkify-plugin-hashtag'; import linkifyStr from 'linkify-string'; import linkify from 'linkifyjs'; import { nip19, nip21 } from 'nostr-tools'; import { Conf } from '@/config.ts'; +import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts'; import { type DittoAttachment } from '@/views/mastodon/attachments.ts'; linkify.registerCustomProtocol('nostr', true); @@ -60,16 +60,14 @@ function parseNoteContent(content: string): ParsedNoteContent { function getMediaLinks(links: Link[]): DittoAttachment[] { return links.reduce((acc, link) => { - const mimeType = getUrlMimeType(link.href); - if (!mimeType) return acc; + const mediaType = getUrlMediaType(link.href); + if (!mediaType) return acc; - const [baseType, _subType] = mimeType.split('/'); - - if (['audio', 'image', 'video'].includes(baseType)) { + if (isPermittedMediaType(mediaType, ['audio', 'image', 'video'])) { acc.push({ url: link.href, data: { - mime: mimeType, + mime: mediaType, }, }); } @@ -79,7 +77,7 @@ function getMediaLinks(links: Link[]): DittoAttachment[] { } function isNonMediaLink({ href }: Link): boolean { - return /^https?:\/\//.test(href) && !getUrlMimeType(href); + return /^https?:\/\//.test(href) && !getUrlMediaType(href); } /** Ensures the Link is a URL so it can be parsed. */ @@ -87,17 +85,6 @@ function isLinkURL(link: Link): boolean { return link.type === 'url'; } -/** Get the extension from the URL, then get its type. */ -function getUrlMimeType(url: string): string | undefined { - try { - const { pathname } = new URL(url); - const ext = pathname.split('.').pop() ?? ''; - return typeByExtension(ext); - } catch { - return undefined; - } -} - /** Get pubkey from decoded bech32 entity, or undefined if not applicable. */ function getDecodedPubkey(decoded: nip19.DecodeResult): string | undefined { switch (decoded.type) { diff --git a/src/utils/media.test.ts b/src/utils/media.test.ts new file mode 100644 index 0000000..e88e97d --- /dev/null +++ b/src/utils/media.test.ts @@ -0,0 +1,17 @@ +import { assertEquals } from '@std/assert'; + +import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts'; + +Deno.test('getUrlMediaType', () => { + assertEquals(getUrlMediaType('https://example.com/image.png'), 'image/png'); + assertEquals(getUrlMediaType('https://example.com/index.html'), 'text/html'); + assertEquals(getUrlMediaType('https://example.com/yolo'), undefined); + assertEquals(getUrlMediaType('https://example.com/'), undefined); +}); + +Deno.test('isPermittedMediaType', () => { + assertEquals(isPermittedMediaType('image/png', ['image', 'video']), true); + assertEquals(isPermittedMediaType('video/webm', ['image', 'video']), true); + assertEquals(isPermittedMediaType('audio/ogg', ['image', 'video']), false); + assertEquals(isPermittedMediaType('application/json', ['image', 'video']), false); +}); diff --git a/src/utils/media.ts b/src/utils/media.ts new file mode 100644 index 0000000..9c0ea9e --- /dev/null +++ b/src/utils/media.ts @@ -0,0 +1,24 @@ +import { typeByExtension } from '@std/media-types'; + +/** Get media type of the filename in the URL by its extension, if any. */ +export function getUrlMediaType(url: string): string | undefined { + try { + const { pathname } = new URL(url); + const ext = pathname.split('.').pop() ?? ''; + return typeByExtension(ext); + } catch { + return undefined; + } +} + +/** + * Check if the base type matches any of the permitted types. + * + * ```ts + * isPermittedMediaType('image/png', ['image', 'video']); // true + * ``` + */ +export function isPermittedMediaType(mediaType: string, permitted: string[]): boolean { + const [baseType, _subType] = mediaType.split('/'); + return permitted.includes(baseType); +} From 942260aa54f61a96a43492f886328df4cfa29dc4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 11:53:33 -0500 Subject: [PATCH 24/65] note.ts -> utils/note.ts --- src/{ => utils}/note.ts | 0 src/views/mastodon/statuses.ts | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{ => utils}/note.ts (100%) diff --git a/src/note.ts b/src/utils/note.ts similarity index 100% rename from src/note.ts rename to src/utils/note.ts diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index aefe258..1b0ebe8 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -4,10 +4,10 @@ import { nip19 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { getMediaLinks, parseNoteContent } from '@/note.ts'; import { Storages } from '@/storages.ts'; import { findReplyTag } from '@/tags.ts'; import { nostrDate } from '@/utils.ts'; +import { getMediaLinks, parseNoteContent } from '@/utils/note.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { DittoAttachment, renderAttachment } from '@/views/mastodon/attachments.ts'; From c8f9483795f12d8e79470c6142be88f8879f398d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 11:56:22 -0500 Subject: [PATCH 25/65] Add note.test.ts --- src/utils/note.test.ts | 28 ++++++++++++++++++++++++++++ src/utils/note.ts | 2 +- 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 src/utils/note.test.ts diff --git a/src/utils/note.test.ts b/src/utils/note.test.ts new file mode 100644 index 0000000..d123050 --- /dev/null +++ b/src/utils/note.test.ts @@ -0,0 +1,28 @@ +import { assertEquals } from '@std/assert'; + +import { getMediaLinks, parseNoteContent } from '@/utils/note.ts'; + +Deno.test('parseNoteContent', () => { + const { html, links, firstUrl } = parseNoteContent('Hello, world!'); + assertEquals(html, 'Hello, world!'); + assertEquals(links, []); + assertEquals(firstUrl, undefined); +}); + +Deno.test('getMediaLinks', () => { + const links = [ + { href: 'https://example.com/image.png' }, + { href: 'https://example.com/index.html' }, + { href: 'https://example.com/yolo' }, + { href: 'https://example.com/' }, + ]; + const mediaLinks = getMediaLinks(links); + assertEquals(mediaLinks, [ + { + url: 'https://example.com/image.png', + data: { + mime: 'image/png', + }, + }, + ]); +}); diff --git a/src/utils/note.ts b/src/utils/note.ts index 71dc017..580c207 100644 --- a/src/utils/note.ts +++ b/src/utils/note.ts @@ -58,7 +58,7 @@ function parseNoteContent(content: string): ParsedNoteContent { }; } -function getMediaLinks(links: Link[]): DittoAttachment[] { +function getMediaLinks(links: Pick[]): DittoAttachment[] { return links.reduce((acc, link) => { const mediaType = getUrlMediaType(link.href); if (!mediaType) return acc; From 7d34b9401e8b191e91801d205c85c3685a876769 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 13:22:20 -0500 Subject: [PATCH 26/65] Support imeta tags --- src/controllers/api/statuses.ts | 2 +- src/db/unattached-media.ts | 3 +-- src/schemas/nostr.ts | 25 +-------------------- src/upload.ts | 37 ++++++++++++++++++++----------- src/views/mastodon/attachments.ts | 16 ++++++++----- src/views/mastodon/statuses.ts | 12 ++++++---- 6 files changed, 46 insertions(+), 49 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 00f9a98..8904b2b 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -96,7 +96,7 @@ const createStatusController: AppController = async (c) => { if (data.media_ids?.length) { const media = await getUnattachedMediaByIds(kysely, data.media_ids) .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey)) - .then((media) => media.map(({ url, data }) => ['media', url, data])); + .then((media) => media.map(({ data }) => ['imeta', ...data])); tags.push(...media); } diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index cee1e3a..0628278 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -3,13 +3,12 @@ import uuid62 from 'uuid62'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; -import { type MediaData } from '@/schemas/nostr.ts'; interface UnattachedMedia { id: string; pubkey: string; url: string; - data: MediaData; + data: string[][]; // NIP-94 tags uploaded_at: number; } diff --git a/src/schemas/nostr.ts b/src/schemas/nostr.ts index a42b9f0..d8aa29a 100644 --- a/src/schemas/nostr.ts +++ b/src/schemas/nostr.ts @@ -9,27 +9,12 @@ const signedEventSchema = n.event() .refine((event) => event.id === getEventHash(event), 'Event ID does not match hash') .refine(verifyEvent, 'Event signature is invalid'); -/** Media data schema from `"media"` tags. */ -const mediaDataSchema = z.object({ - blurhash: z.string().optional().catch(undefined), - cid: z.string().optional().catch(undefined), - description: z.string().max(200).optional().catch(undefined), - height: z.number().int().positive().optional().catch(undefined), - mime: z.string().optional().catch(undefined), - name: z.string().optional().catch(undefined), - size: z.number().int().positive().optional().catch(undefined), - width: z.number().int().positive().optional().catch(undefined), -}); - /** Kind 0 content schema for the Ditto server admin user. */ const serverMetaSchema = n.metadata().and(z.object({ tagline: z.string().optional().catch(undefined), email: z.string().optional().catch(undefined), })); -/** Media data from `"media"` tags. */ -type MediaData = z.infer; - /** NIP-11 Relay Information Document. */ const relayInfoDocSchema = z.object({ name: z.string().transform((val) => val.slice(0, 30)).optional().catch(undefined), @@ -47,12 +32,4 @@ const emojiTagSchema = z.tuple([z.literal('emoji'), z.string(), z.string().url() /** NIP-30 custom emoji tag. */ type EmojiTag = z.infer; -export { - type EmojiTag, - emojiTagSchema, - type MediaData, - mediaDataSchema, - relayInfoDocSchema, - serverMetaSchema, - signedEventSchema, -}; +export { type EmojiTag, emojiTagSchema, relayInfoDocSchema, serverMetaSchema, signedEventSchema }; diff --git a/src/upload.ts b/src/upload.ts index 632dbab..4f5fd14 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -8,26 +8,37 @@ interface FileMeta { } /** Upload a file, track it in the database, and return the resulting media object. */ -async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal) { - const { name, type, size } = file; +async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Promise { + const { type, size } = file; const { pubkey, description } = meta; if (file.size > Conf.maxUploadSize) { throw new Error('File size is too large.'); } - const { url } = await uploader.upload(file, { signal }); + const { url, sha256, cid } = await uploader.upload(file, { signal }); - return insertUnattachedMedia({ - pubkey, - url, - data: { - name, - size, - description, - mime: type, - }, - }); + const data: string[][] = [ + ['url', url], + ['m', type], + ['size', size.toString()], + ]; + + if (sha256) { + data.push(['x', sha256]); + } + + if (cid) { + data.push(['cid', cid]); + } + + if (description) { + data.push(['alt', description]); + } + + await insertUnattachedMedia({ pubkey, url, data }); + + return data; } export { uploadFile }; diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 3ea989e..18fe031 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -6,15 +6,21 @@ type DittoAttachment = TypeFest.SetOptional name === 'm')?.[1]; + const alt = data.find(([name]) => name === 'alt')?.[1]; + const cid = data.find(([name]) => name === 'cid')?.[1]; + const blurhash = data.find(([name]) => name === 'blurhash')?.[1]; + return { - id: id ?? url ?? data.cid, - type: getAttachmentType(data.mime ?? ''), + id: id ?? url, + type: getAttachmentType(m ?? ''), url, preview_url: url, remote_url: null, - description: data.description ?? '', - blurhash: data.blurhash || null, - cid: data.cid, + description: alt ?? '', + blurhash: blurhash || null, + cid: cid, }; } diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 1b0ebe8..8428e9a 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; +import { NostrEvent } from '@nostrify/nostrify'; import { isCWTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts'; import { nip19 } from 'nostr-tools'; @@ -12,7 +12,6 @@ import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { DittoAttachment, renderAttachment } from '@/views/mastodon/attachments.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; -import { mediaDataSchema } from '@/schemas/nostr.ts'; interface RenderStatusOpts { viewerPubkey?: string; @@ -80,8 +79,13 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const mediaLinks = getMediaLinks(links); const mediaTags: DittoAttachment[] = event.tags - .filter((tag) => tag[0] === 'media') - .map(([_, url, json]) => ({ url, data: n.json().pipe(mediaDataSchema).parse(json) })); + .filter(([name]) => name === 'imeta') + .map(([_, ...entries]) => { + const data = entries.map((entry) => entry.split(' ')); + const url = data.find(([name]) => name === 'url')?.[1]; + return { url, data }; + }) + .filter((media): media is DittoAttachment => !!media.url); const media = [...mediaLinks, ...mediaTags]; From 611a94bdcf640efcd22615ff12d1f911fbf61cc4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 14:32:50 -0500 Subject: [PATCH 27/65] Fix uploading (almost) --- src/controllers/api/statuses.ts | 12 +++++++++--- src/db/unattached-media.ts | 9 +++++++++ src/utils/api.ts | 2 ++ src/utils/note.ts | 16 +++++++--------- src/views/mastodon/attachments.ts | 14 +++++++------- src/views/mastodon/statuses.ts | 13 ++++--------- 6 files changed, 38 insertions(+), 28 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 8904b2b..d8186f5 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -5,7 +5,7 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; +import { getUnattachedMediaByUrls } from '@/db/unattached-media.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; import { addTag, deleteTag } from '@/tags.ts'; import { createEvent, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; @@ -94,9 +94,15 @@ const createStatusController: AppController = async (c) => { const viewerPubkey = await c.get('signer')?.getPublicKey(); if (data.media_ids?.length) { - const media = await getUnattachedMediaByIds(kysely, data.media_ids) + const media = await getUnattachedMediaByUrls(kysely, data.media_ids) .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey)) - .then((media) => media.map(({ data }) => ['imeta', ...data])); + .then((media) => + media.map(({ data }) => { + const tags: string[][] = JSON.parse(data); + const values: string[] = tags.map((tag) => tag.join(' ')); + return ['imeta', ...values]; + }) + ); tags.push(...media); } diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index 0628278..397a1f7 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -64,6 +64,14 @@ async function getUnattachedMediaByIds(kysely: Kysely, ids: string[ .execute(); } +/** Get unattached media by URLs. */ +async function getUnattachedMediaByUrls(kysely: Kysely, urls: string[]) { + if (!urls.length) return []; + return await selectUnattachedMediaQuery(kysely) + .where('url', 'in', urls) + .execute(); +} + /** Delete rows as an event with media is being created. */ async function deleteAttachedMedia(pubkey: string, urls: string[]): Promise { if (!urls.length) return; @@ -79,6 +87,7 @@ export { deleteUnattachedMediaByUrl, getUnattachedMedia, getUnattachedMediaByIds, + getUnattachedMediaByUrls, insertUnattachedMedia, type UnattachedMedia, }; diff --git a/src/utils/api.ts b/src/utils/api.ts index dceede7..c54f5aa 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -29,6 +29,8 @@ async function createEvent(t: EventStub, c: AppContext): Promise { }); } + console.log(t); + const event = await signer.signEvent({ content: '', created_at: nostrNow(), diff --git a/src/utils/note.ts b/src/utils/note.ts index 580c207..20cb83a 100644 --- a/src/utils/note.ts +++ b/src/utils/note.ts @@ -5,7 +5,6 @@ import { nip19, nip21 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts'; -import { type DittoAttachment } from '@/views/mastodon/attachments.ts'; linkify.registerCustomProtocol('nostr', true); linkify.registerCustomProtocol('wss'); @@ -58,18 +57,17 @@ function parseNoteContent(content: string): ParsedNoteContent { }; } -function getMediaLinks(links: Pick[]): DittoAttachment[] { - return links.reduce((acc, link) => { +/** Returns a matrix of tags. Each item is a list of NIP-94 tags representing a file. */ +function getMediaLinks(links: Pick[]): string[][][] { + return links.reduce((acc, link) => { const mediaType = getUrlMediaType(link.href); if (!mediaType) return acc; if (isPermittedMediaType(mediaType, ['audio', 'image', 'video'])) { - acc.push({ - url: link.href, - data: { - mime: mediaType, - }, - }); + acc.push([ + ['url', link.href], + ['m', mediaType], + ]); } return acc; diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 18fe031..8922985 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -4,16 +4,16 @@ import { UnattachedMedia } from '@/db/unattached-media.ts'; type DittoAttachment = TypeFest.SetOptional; -function renderAttachment(media: DittoAttachment) { - const { id, data, url } = media; +function renderAttachment(tags: string[][]) { + const url = tags.find(([name]) => name === 'url')?.[1]; - const m = data.find(([name]) => name === 'm')?.[1]; - const alt = data.find(([name]) => name === 'alt')?.[1]; - const cid = data.find(([name]) => name === 'cid')?.[1]; - const blurhash = data.find(([name]) => name === 'blurhash')?.[1]; + const m = tags.find(([name]) => name === 'm')?.[1]; + const alt = tags.find(([name]) => name === 'alt')?.[1]; + const cid = tags.find(([name]) => name === 'cid')?.[1]; + const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; return { - id: id ?? url, + id: url, type: getAttachmentType(m ?? ''), url, preview_url: url, diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 8428e9a..889c23d 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -10,7 +10,7 @@ import { nostrDate } from '@/utils.ts'; import { getMediaLinks, parseNoteContent } from '@/utils/note.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; -import { DittoAttachment, renderAttachment } from '@/views/mastodon/attachments.ts'; +import { renderAttachment } from '@/views/mastodon/attachments.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; interface RenderStatusOpts { @@ -78,16 +78,11 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const mediaLinks = getMediaLinks(links); - const mediaTags: DittoAttachment[] = event.tags + const imeta: string[][][] = event.tags .filter(([name]) => name === 'imeta') - .map(([_, ...entries]) => { - const data = entries.map((entry) => entry.split(' ')); - const url = data.find(([name]) => name === 'url')?.[1]; - return { url, data }; - }) - .filter((media): media is DittoAttachment => !!media.url); + .map(([_, ...entries]) => entries.map((entry) => entry.split(' '))); - const media = [...mediaLinks, ...mediaTags]; + const media = [...mediaLinks, ...imeta]; return { id: event.id, From e7d350a0e305a58191331b6664866419d215802f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 14:54:10 -0500 Subject: [PATCH 28/65] Fix uploading by URL --- deno.json | 1 - src/controllers/api/statuses.ts | 4 ++-- src/db/unattached-media.ts | 25 +++++-------------------- src/upload.ts | 11 ++++++++++- src/utils/api.ts | 2 -- src/views/mastodon/attachments.ts | 6 +++--- 6 files changed, 20 insertions(+), 29 deletions(-) diff --git a/deno.json b/deno.json index 54ba468..946b4e0 100644 --- a/deno.json +++ b/deno.json @@ -54,7 +54,6 @@ "tseep": "npm:tseep@^1.2.1", "type-fest": "npm:type-fest@^4.3.0", "unfurl.js": "npm:unfurl.js@^6.4.0", - "uuid62": "npm:uuid62@^1.0.2", "zod": "npm:zod@^3.23.5", "~/fixtures/": "./fixtures/" }, diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index d8186f5..f7a603f 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -5,7 +5,7 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { getUnattachedMediaByUrls } from '@/db/unattached-media.ts'; +import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; import { addTag, deleteTag } from '@/tags.ts'; import { createEvent, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; @@ -94,7 +94,7 @@ const createStatusController: AppController = async (c) => { const viewerPubkey = await c.get('signer')?.getPublicKey(); if (data.media_ids?.length) { - const media = await getUnattachedMediaByUrls(kysely, data.media_ids) + const media = await getUnattachedMediaByIds(kysely, data.media_ids) .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey)) .then((media) => media.map(({ data }) => { diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index 397a1f7..0ab46b6 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -1,5 +1,4 @@ import { Kysely } from 'kysely'; -import uuid62 from 'uuid62'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; @@ -8,24 +7,19 @@ interface UnattachedMedia { id: string; pubkey: string; url: string; - data: string[][]; // NIP-94 tags + /** NIP-94 tags. */ + data: string[][]; uploaded_at: number; } /** Add unattached media into the database. */ -async function insertUnattachedMedia(media: Omit) { - const result = { - id: uuid62.v4(), - uploaded_at: Date.now(), - ...media, - }; - +async function insertUnattachedMedia(media: UnattachedMedia) { const kysely = await DittoDB.getInstance(); await kysely.insertInto('unattached_media') - .values({ ...result, data: JSON.stringify(media.data) }) + .values({ ...media, data: JSON.stringify(media.data) }) .execute(); - return result; + return media; } /** Select query for unattached media. */ @@ -64,14 +58,6 @@ async function getUnattachedMediaByIds(kysely: Kysely, ids: string[ .execute(); } -/** Get unattached media by URLs. */ -async function getUnattachedMediaByUrls(kysely: Kysely, urls: string[]) { - if (!urls.length) return []; - return await selectUnattachedMediaQuery(kysely) - .where('url', 'in', urls) - .execute(); -} - /** Delete rows as an event with media is being created. */ async function deleteAttachedMedia(pubkey: string, urls: string[]): Promise { if (!urls.length) return; @@ -87,7 +73,6 @@ export { deleteUnattachedMediaByUrl, getUnattachedMedia, getUnattachedMediaByIds, - getUnattachedMediaByUrls, insertUnattachedMedia, type UnattachedMedia, }; diff --git a/src/upload.ts b/src/upload.ts index 4f5fd14..d815bd8 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -36,7 +36,16 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro data.push(['alt', description]); } - await insertUnattachedMedia({ pubkey, url, data }); + const uuid = crypto.randomUUID(); + data.push(['uuid', uuid]); + + await insertUnattachedMedia({ + id: uuid, + pubkey, + url, + data, + uploaded_at: Date.now(), + }); return data; } diff --git a/src/utils/api.ts b/src/utils/api.ts index c54f5aa..dceede7 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -29,8 +29,6 @@ async function createEvent(t: EventStub, c: AppContext): Promise { }); } - console.log(t); - const event = await signer.signEvent({ content: '', created_at: nostrNow(), diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 8922985..1dbcda9 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -5,15 +5,15 @@ import { UnattachedMedia } from '@/db/unattached-media.ts'; type DittoAttachment = TypeFest.SetOptional; function renderAttachment(tags: string[][]) { - const url = tags.find(([name]) => name === 'url')?.[1]; - const m = tags.find(([name]) => name === 'm')?.[1]; + const url = tags.find(([name]) => name === 'url')?.[1]; const alt = tags.find(([name]) => name === 'alt')?.[1]; const cid = tags.find(([name]) => name === 'cid')?.[1]; + const uuid = tags.find(([name]) => name === 'uuid')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; return { - id: url, + id: uuid, type: getAttachmentType(m ?? ''), url, preview_url: url, From 91ea4577f1549be74b8f036293ca3fa0a56d7bd5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 14:58:48 -0500 Subject: [PATCH 29/65] Filter out attachments with no url --- src/views/mastodon/attachments.ts | 4 +++- src/views/mastodon/statuses.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 1dbcda9..9f8e5c3 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -12,8 +12,10 @@ function renderAttachment(tags: string[][]) { const uuid = tags.find(([name]) => name === 'uuid')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; + if (!url) return; + return { - id: uuid, + id: uuid ?? url, type: getAttachmentType(m ?? ''), url, preview_url: url, diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 889c23d..16ba482 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -106,7 +106,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< pinned: Boolean(pinEvent), reblog: null, application: null, - media_attachments: media.map(renderAttachment), + media_attachments: media.map(renderAttachment).filter(Boolean), mentions, tags: [], emojis: renderEmojis(event), From b1b341d3b8a947b2a07325bfa07f2060c01cadcd Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 15:29:12 -0500 Subject: [PATCH 30/65] Insert media URL into text --- src/controllers/api/statuses.ts | 27 +++++++++++++-------------- src/db/unattached-media.ts | 10 ++++++++-- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index f7a603f..e620b93 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -91,21 +91,14 @@ const createStatusController: AppController = async (c) => { tags.push(['subject', data.spoiler_text]); } - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const media = data.media_ids?.length ? await getUnattachedMediaByIds(kysely, data.media_ids) : []; - if (data.media_ids?.length) { - const media = await getUnattachedMediaByIds(kysely, data.media_ids) - .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey)) - .then((media) => - media.map(({ data }) => { - const tags: string[][] = JSON.parse(data); - const values: string[] = tags.map((tag) => tag.join(' ')); - return ['imeta', ...values]; - }) - ); + const imeta: string[][] = media.map(({ data }) => { + const values: string[] = data.map((tag) => tag.join(' ')); + return ['imeta', ...values]; + }); - tags.push(...media); - } + tags.push(...imeta); const pubkeys = new Set(); @@ -137,9 +130,15 @@ const createStatusController: AppController = async (c) => { tags.push(['t', match[1]]); } + const mediaUrls: string[] = media + .map(({ data }) => data.find(([name]) => name === 'url')?.[1]) + .filter((url): url is string => Boolean(url)); + + const mediaCompat: string = mediaUrls.length ? ['', '', ...mediaUrls].join('\n') : ''; + const event = await createEvent({ kind: 1, - content, + content: content + mediaCompat, tags, }, c); diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index 0ab46b6..0e0aeea 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -51,11 +51,17 @@ async function deleteUnattachedMediaByUrl(url: string) { } /** Get unattached media by IDs. */ -async function getUnattachedMediaByIds(kysely: Kysely, ids: string[]) { +async function getUnattachedMediaByIds(kysely: Kysely, ids: string[]): Promise { if (!ids.length) return []; - return await selectUnattachedMediaQuery(kysely) + + const results = await selectUnattachedMediaQuery(kysely) .where('id', 'in', ids) .execute(); + + return results.map((row) => ({ + ...row, + data: JSON.parse(row.data), + })); } /** Delete rows as an event with media is being created. */ From c8b999a1f7a808e89aab968c24ae2bf467fb4a8f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 15:36:17 -0500 Subject: [PATCH 31/65] imeta: don't get attachment ID from a tag --- src/upload.ts | 13 ++++--------- src/views/mastodon/attachments.ts | 14 +++++--------- src/views/mastodon/statuses.ts | 2 +- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/src/upload.ts b/src/upload.ts index d815bd8..40184f0 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -1,5 +1,5 @@ import { Conf } from '@/config.ts'; -import { insertUnattachedMedia } from '@/db/unattached-media.ts'; +import { insertUnattachedMedia, UnattachedMedia } from '@/db/unattached-media.ts'; import { configUploader as uploader } from '@/uploaders/config.ts'; interface FileMeta { @@ -8,7 +8,7 @@ interface FileMeta { } /** Upload a file, track it in the database, and return the resulting media object. */ -async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Promise { +async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Promise { const { type, size } = file; const { pubkey, description } = meta; @@ -36,18 +36,13 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro data.push(['alt', description]); } - const uuid = crypto.randomUUID(); - data.push(['uuid', uuid]); - - await insertUnattachedMedia({ - id: uuid, + return insertUnattachedMedia({ + id: crypto.randomUUID(), pubkey, url, data, uploaded_at: Date.now(), }); - - return data; } export { uploadFile }; diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 9f8e5c3..273b460 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -1,21 +1,17 @@ -import * as TypeFest from 'type-fest'; +/** Render Mastodon media attachment. */ +function renderAttachment(media: { id?: string; data: string[][] }) { + const { id, data: tags } = media; -import { UnattachedMedia } from '@/db/unattached-media.ts'; - -type DittoAttachment = TypeFest.SetOptional; - -function renderAttachment(tags: string[][]) { const m = tags.find(([name]) => name === 'm')?.[1]; const url = tags.find(([name]) => name === 'url')?.[1]; const alt = tags.find(([name]) => name === 'alt')?.[1]; const cid = tags.find(([name]) => name === 'cid')?.[1]; - const uuid = tags.find(([name]) => name === 'uuid')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; if (!url) return; return { - id: uuid ?? url, + id: id ?? url, type: getAttachmentType(m ?? ''), url, preview_url: url, @@ -40,4 +36,4 @@ function getAttachmentType(mime: string): string { } } -export { type DittoAttachment, renderAttachment }; +export { renderAttachment }; diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 16ba482..c674e16 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -106,7 +106,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< pinned: Boolean(pinEvent), reblog: null, application: null, - media_attachments: media.map(renderAttachment).filter(Boolean), + media_attachments: media.map((m) => renderAttachment({ data: m })).filter(Boolean), mentions, tags: [], emojis: renderEmojis(event), From cbf0bc35940b87c84d30feb1a91d0c900ac47cd5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 15:46:28 -0500 Subject: [PATCH 32/65] Fix note test --- src/utils/note.test.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/utils/note.test.ts b/src/utils/note.test.ts index d123050..9c8fad7 100644 --- a/src/utils/note.test.ts +++ b/src/utils/note.test.ts @@ -17,12 +17,8 @@ Deno.test('getMediaLinks', () => { { href: 'https://example.com/' }, ]; const mediaLinks = getMediaLinks(links); - assertEquals(mediaLinks, [ - { - url: 'https://example.com/image.png', - data: { - mime: 'image/png', - }, - }, - ]); + assertEquals(mediaLinks, [[ + ['url', 'https://example.com/image.png'], + ['m', 'image/png'], + ]]); }); From c89be75e5b9f240c8eba9fa180cad7ebc6301d76 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 16:22:24 -0500 Subject: [PATCH 33/65] Add a nostr.build uploader --- fixtures/nostrbuild-gif.json | 34 ++++++++++++++++++++++++++++++++++ fixtures/nostrbuild-mp3.json | 29 +++++++++++++++++++++++++++++ src/config.ts | 4 ++++ src/schemas/nostrbuild.ts | 18 ++++++++++++++++++ src/upload.ts | 6 +++++- src/uploaders/config.ts | 3 +++ src/uploaders/nostrbuild.ts | 33 +++++++++++++++++++++++++++++++++ src/uploaders/types.ts | 2 ++ 8 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 fixtures/nostrbuild-gif.json create mode 100644 fixtures/nostrbuild-mp3.json create mode 100644 src/schemas/nostrbuild.ts create mode 100644 src/uploaders/nostrbuild.ts diff --git a/fixtures/nostrbuild-gif.json b/fixtures/nostrbuild-gif.json new file mode 100644 index 0000000..49a969a --- /dev/null +++ b/fixtures/nostrbuild-gif.json @@ -0,0 +1,34 @@ +{ + "status": "success", + "message": "Upload successful.", + "data": [ + { + "input_name": "APIv2", + "name": "e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "sha256": "0a71f1c9dd982079bc52e96403368209cbf9507c5f6956134686f56e684b6377", + "original_sha256": "e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3", + "type": "picture", + "mime": "image/gif", + "size": 1796276, + "blurhash": "LGH-S^Vwm]x]04kX-qR-R]SL5FxZ", + "dimensions": { + "width": 360, + "height": 216 + }, + "dimensionsString": "360x216", + "url": "https://image.nostr.build/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "thumbnail": "https://image.nostr.build/thumb/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "responsive": { + "240p": "https://image.nostr.build/resp/240p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "360p": "https://image.nostr.build/resp/360p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "480p": "https://image.nostr.build/resp/480p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "720p": "https://image.nostr.build/resp/720p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "1080p": "https://image.nostr.build/resp/1080p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif" + }, + "metadata": { + "date:create": "2024-05-18T02:11:39+00:00", + "date:modify": "2024-05-18T02:11:39+00:00" + } + } + ] +} \ No newline at end of file diff --git a/fixtures/nostrbuild-mp3.json b/fixtures/nostrbuild-mp3.json new file mode 100644 index 0000000..42a60b4 --- /dev/null +++ b/fixtures/nostrbuild-mp3.json @@ -0,0 +1,29 @@ +{ + "status": "success", + "message": "Upload successful.", + "data": [ + { + "id": 0, + "input_name": "APIv2", + "name": "f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "url": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "thumbnail": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "responsive": { + "240p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "360p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "480p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "720p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "1080p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3" + }, + "blurhash": "", + "sha256": "f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725", + "original_sha256": "f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725", + "type": "video", + "mime": "audio/mpeg", + "size": 1519616, + "metadata": [], + "dimensions": [], + "dimensionsString": "0x0" + } + ] +} \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index 6fe62b9..f3b472e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -136,6 +136,10 @@ class Conf { return Deno.env.get('IPFS_API_URL') || 'http://localhost:5001'; }, }; + /** nostr.build API endpoint when the `nostrbuild` uploader is used. */ + static get nostrbuildEndpoint(): string { + return Deno.env.get('NOSTRBUILD_ENDPOINT') || 'https://nostr.build/api/v2/upload/files'; + } /** Module to upload files with. */ static get uploader() { return Deno.env.get('DITTO_UPLOADER'); diff --git a/src/schemas/nostrbuild.ts b/src/schemas/nostrbuild.ts new file mode 100644 index 0000000..c9fd680 --- /dev/null +++ b/src/schemas/nostrbuild.ts @@ -0,0 +1,18 @@ +import { z } from 'zod'; + +export const nostrbuildFileSchema = z.object({ + name: z.string(), + url: z.string().url(), + thumbnail: z.string(), + blurhash: z.string(), + sha256: z.string(), + mime: z.string(), + dimensions: z.object({ + width: z.number(), + height: z.number(), + }), +}); + +export const nostrbuildSchema = z.object({ + data: nostrbuildFileSchema.array().min(1), +}); diff --git a/src/upload.ts b/src/upload.ts index 40184f0..5b43f39 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -16,7 +16,7 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro throw new Error('File size is too large.'); } - const { url, sha256, cid } = await uploader.upload(file, { signal }); + const { url, sha256, cid, blurhash } = await uploader.upload(file, { signal }); const data: string[][] = [ ['url', url], @@ -32,6 +32,10 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro data.push(['cid', cid]); } + if (blurhash) { + data.push(['blurhash', blurhash]); + } + if (description) { data.push(['alt', description]); } diff --git a/src/uploaders/config.ts b/src/uploaders/config.ts index 5b4c7af..8ce22b6 100644 --- a/src/uploaders/config.ts +++ b/src/uploaders/config.ts @@ -2,6 +2,7 @@ import { Conf } from '@/config.ts'; import { ipfsUploader } from '@/uploaders/ipfs.ts'; import { localUploader } from '@/uploaders/local.ts'; +import { nostrbuildUploader } from '@/uploaders/nostrbuild.ts'; import { s3Uploader } from '@/uploaders/s3.ts'; import type { Uploader } from './types.ts'; @@ -25,6 +26,8 @@ function uploader() { return ipfsUploader; case 'local': return localUploader; + case 'nostrbuild': + return nostrbuildUploader; default: throw new Error('No `DITTO_UPLOADER` configured. Uploads are disabled.'); } diff --git a/src/uploaders/nostrbuild.ts b/src/uploaders/nostrbuild.ts new file mode 100644 index 0000000..d9eed24 --- /dev/null +++ b/src/uploaders/nostrbuild.ts @@ -0,0 +1,33 @@ +import { Conf } from '@/config.ts'; +import { nostrbuildSchema } from '@/schemas/nostrbuild.ts'; + +import type { Uploader } from './types.ts'; + +/** nostr.build uploader. */ +export const nostrbuildUploader: Uploader = { + async upload(file) { + const formData = new FormData(); + formData.append('fileToUpload', file); + + const response = await fetch(Conf.nostrbuildEndpoint, { + method: 'POST', + body: formData, + }); + + const json = await response.json(); + console.log(JSON.stringify(json)); + + const [data] = nostrbuildSchema.parse(json).data; + + return { + id: data.url, + sha256: data.sha256, + url: data.url, + blurhash: data.blurhash, + }; + }, + // deno-lint-ignore require-await + async delete(): Promise { + return; + }, +}; diff --git a/src/uploaders/types.ts b/src/uploaders/types.ts index c514ad1..ac5bf05 100644 --- a/src/uploaders/types.ts +++ b/src/uploaders/types.ts @@ -14,6 +14,8 @@ interface UploadResult { url: string; /** SHA-256 hash of the file. */ sha256?: string; + /** Blurhash of the file. */ + blurhash?: string; /** IPFS CID of the file. */ cid?: string; } From ce49c500ae2fe62f6a83607055803aa18034c714 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 16:47:47 -0500 Subject: [PATCH 34/65] renderStatus: fix duplicated attachments --- src/views/mastodon/statuses.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index c674e16..c707ebb 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -76,13 +76,11 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const cw = event.tags.find(isCWTag); const subject = event.tags.find((tag) => tag[0] === 'subject'); - const mediaLinks = getMediaLinks(links); - const imeta: string[][][] = event.tags .filter(([name]) => name === 'imeta') .map(([_, ...entries]) => entries.map((entry) => entry.split(' '))); - const media = [...mediaLinks, ...imeta]; + const media = imeta.length ? imeta : getMediaLinks(links); return { id: event.id, From 353111051a79a37aa3b7be1c44b4c54ffbeb9904 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 16:53:17 -0500 Subject: [PATCH 35/65] Use dimensions from nostr.build --- src/schemas/nostrbuild.ts | 3 ++- src/upload.ts | 6 +++++- src/uploaders/nostrbuild.ts | 4 ++-- src/uploaders/types.ts | 4 ++++ src/views/mastodon/attachments.ts | 14 ++++++++++++++ 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/schemas/nostrbuild.ts b/src/schemas/nostrbuild.ts index c9fd680..db9f607 100644 --- a/src/schemas/nostrbuild.ts +++ b/src/schemas/nostrbuild.ts @@ -6,11 +6,12 @@ export const nostrbuildFileSchema = z.object({ thumbnail: z.string(), blurhash: z.string(), sha256: z.string(), + original_sha256: z.string(), mime: z.string(), dimensions: z.object({ width: z.number(), height: z.number(), - }), + }).optional().catch(undefined), }); export const nostrbuildSchema = z.object({ diff --git a/src/upload.ts b/src/upload.ts index 5b43f39..1da5a7d 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -16,7 +16,7 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro throw new Error('File size is too large.'); } - const { url, sha256, cid, blurhash } = await uploader.upload(file, { signal }); + const { url, sha256, cid, blurhash, width, height } = await uploader.upload(file, { signal }); const data: string[][] = [ ['url', url], @@ -24,6 +24,10 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro ['size', size.toString()], ]; + if (typeof width === 'number' && typeof height === 'number') { + data.push(['dim', `${width}x${height}`]); + } + if (sha256) { data.push(['x', sha256]); } diff --git a/src/uploaders/nostrbuild.ts b/src/uploaders/nostrbuild.ts index d9eed24..d9d0865 100644 --- a/src/uploaders/nostrbuild.ts +++ b/src/uploaders/nostrbuild.ts @@ -15,8 +15,6 @@ export const nostrbuildUploader: Uploader = { }); const json = await response.json(); - console.log(JSON.stringify(json)); - const [data] = nostrbuildSchema.parse(json).data; return { @@ -24,6 +22,8 @@ export const nostrbuildUploader: Uploader = { sha256: data.sha256, url: data.url, blurhash: data.blurhash, + width: data.dimensions?.width, + height: data.dimensions?.height, }; }, // deno-lint-ignore require-await diff --git a/src/uploaders/types.ts b/src/uploaders/types.ts index ac5bf05..e423028 100644 --- a/src/uploaders/types.ts +++ b/src/uploaders/types.ts @@ -18,6 +18,10 @@ interface UploadResult { blurhash?: string; /** IPFS CID of the file. */ cid?: string; + /** Width of the file, if applicable. */ + width?: number; + /** Height of the file, if applicable. */ + height?: number; } export type { Uploader }; diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 273b460..0b1b8eb 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -6,10 +6,23 @@ function renderAttachment(media: { id?: string; data: string[][] }) { const url = tags.find(([name]) => name === 'url')?.[1]; const alt = tags.find(([name]) => name === 'alt')?.[1]; const cid = tags.find(([name]) => name === 'cid')?.[1]; + const dim = tags.find(([name]) => name === 'dim')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; if (!url) return; + const [width, height] = dim?.split('x').map(Number) ?? [null, null]; + + const meta = (typeof width === 'number' && typeof height === 'number') + ? { + original: { + width, + height, + aspect: width / height, + }, + } + : undefined; + return { id: id ?? url, type: getAttachmentType(m ?? ''), @@ -18,6 +31,7 @@ function renderAttachment(media: { id?: string; data: string[][] }) { remote_url: null, description: alt ?? '', blurhash: blurhash || null, + meta, cid: cid, }; } From e5595d34be546a99d5594eb0af185cd4119f4c88 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 17:08:30 -0500 Subject: [PATCH 36/65] Strip imeta links from the end of the content --- src/utils/note.ts | 28 +++++++++++++++++++++++++++- src/views/mastodon/statuses.ts | 4 ++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/utils/note.ts b/src/utils/note.ts index 20cb83a..03da2de 100644 --- a/src/utils/note.ts +++ b/src/utils/note.ts @@ -57,6 +57,32 @@ function parseNoteContent(content: string): ParsedNoteContent { }; } +/** Remove imeta links. */ +function stripimeta(content: string, tags: string[][]): string { + const imeta = tags.filter(([name]) => name === 'imeta'); + + if (!imeta.length) { + return content; + } + + const urls = new Set( + imeta.map(([, ...values]) => values.map((v) => v.split(' ')).find(([name]) => name === 'url')?.[1]), + ); + + const lines = content.split('\n').reverse(); + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (line === '' || urls.has(line)) { + lines.splice(i, 1); + } else { + break; + } + } + + return lines.reverse().join('\n'); +} + /** Returns a matrix of tags. Each item is a list of NIP-94 tags representing a file. */ function getMediaLinks(links: Pick[]): string[][][] { return links.reduce((acc, link) => { @@ -93,4 +119,4 @@ function getDecodedPubkey(decoded: nip19.DecodeResult): string | undefined { } } -export { getMediaLinks, parseNoteContent }; +export { getMediaLinks, parseNoteContent, stripimeta }; diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index c707ebb..a06aac2 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -7,7 +7,7 @@ import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { Storages } from '@/storages.ts'; import { findReplyTag } from '@/tags.ts'; import { nostrDate } from '@/utils.ts'; -import { getMediaLinks, parseNoteContent } from '@/utils/note.ts'; +import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderAttachment } from '@/views/mastodon/attachments.ts'; @@ -46,7 +46,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< [{ kinds: [0], authors: mentionedPubkeys, limit: mentionedPubkeys.length }], ); - const { html, links, firstUrl } = parseNoteContent(event.content); + const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags)); const [mentions, card, relatedEvents] = await Promise .all([ From 6090c4a6d9a0de134015ab8a02aaf8884631dc7f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 18:23:04 -0500 Subject: [PATCH 37/65] Make Uploaders return NIP-94 tags --- src/upload.ts | 30 ++++-------------------------- src/uploaders/config.ts | 4 ++-- src/uploaders/ipfs.ts | 11 ++++++----- src/uploaders/local.ts | 11 ++++++----- src/uploaders/nostrbuild.ts | 26 ++++++++++++++------------ src/uploaders/s3.ts | 12 +++++++----- src/uploaders/types.ts | 22 ++-------------------- 7 files changed, 41 insertions(+), 75 deletions(-) diff --git a/src/upload.ts b/src/upload.ts index 1da5a7d..cd9d2bb 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -9,46 +9,24 @@ interface FileMeta { /** Upload a file, track it in the database, and return the resulting media object. */ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Promise { - const { type, size } = file; const { pubkey, description } = meta; if (file.size > Conf.maxUploadSize) { throw new Error('File size is too large.'); } - const { url, sha256, cid, blurhash, width, height } = await uploader.upload(file, { signal }); - - const data: string[][] = [ - ['url', url], - ['m', type], - ['size', size.toString()], - ]; - - if (typeof width === 'number' && typeof height === 'number') { - data.push(['dim', `${width}x${height}`]); - } - - if (sha256) { - data.push(['x', sha256]); - } - - if (cid) { - data.push(['cid', cid]); - } - - if (blurhash) { - data.push(['blurhash', blurhash]); - } + const tags = await uploader.upload(file, { signal }); + const url = tags[0][1]; if (description) { - data.push(['alt', description]); + tags.push(['alt', description]); } return insertUnattachedMedia({ id: crypto.randomUUID(), pubkey, url, - data, + data: tags, uploaded_at: Date.now(), }); } diff --git a/src/uploaders/config.ts b/src/uploaders/config.ts index 8ce22b6..3f3aac7 100644 --- a/src/uploaders/config.ts +++ b/src/uploaders/config.ts @@ -12,8 +12,8 @@ const configUploader: Uploader = { upload(file, opts) { return uploader().upload(file, opts); }, - delete(id, opts) { - return uploader().delete(id, opts); + async delete(id, opts) { + return await uploader().delete?.(id, opts); }, }; diff --git a/src/uploaders/ipfs.ts b/src/uploaders/ipfs.ts index 21619b5..b83dc2e 100644 --- a/src/uploaders/ipfs.ts +++ b/src/uploaders/ipfs.ts @@ -32,11 +32,12 @@ const ipfsUploader: Uploader = { const { Hash: cid } = ipfsAddResponseSchema.parse(await response.json()); - return { - id: cid, - cid, - url: new URL(`/ipfs/${cid}`, Conf.mediaDomain).toString(), - }; + return [ + ['url', new URL(`/ipfs/${cid}`, Conf.mediaDomain).toString()], + ['m', file.type], + ['cid', cid], + ['size', file.size.toString()], + ]; }, async delete(cid, opts) { const url = new URL('/api/v0/pin/rm', Conf.ipfs.apiUrl); diff --git a/src/uploaders/local.ts b/src/uploaders/local.ts index a2381a3..d5cd46a 100644 --- a/src/uploaders/local.ts +++ b/src/uploaders/local.ts @@ -22,11 +22,12 @@ const localUploader: Uploader = { const url = new URL(mediaDomain); const path = url.pathname === '/' ? filename : join(url.pathname, filename); - return { - id: filename, - sha256, - url: new URL(path, url).toString(), - }; + return [ + ['url', new URL(path, url).toString()], + ['m', file.type], + ['x', sha256], + ['size', file.size.toString()], + ]; }, async delete(id) { await Deno.remove(join(Conf.uploadsDir, id)); diff --git a/src/uploaders/nostrbuild.ts b/src/uploaders/nostrbuild.ts index d9d0865..8bca331 100644 --- a/src/uploaders/nostrbuild.ts +++ b/src/uploaders/nostrbuild.ts @@ -17,17 +17,19 @@ export const nostrbuildUploader: Uploader = { const json = await response.json(); const [data] = nostrbuildSchema.parse(json).data; - return { - id: data.url, - sha256: data.sha256, - url: data.url, - blurhash: data.blurhash, - width: data.dimensions?.width, - height: data.dimensions?.height, - }; - }, - // deno-lint-ignore require-await - async delete(): Promise { - return; + const tags: [['url', string], ...string[][]] = [ + ['url', data.url], + ['m', data.mime], + ['x', data.sha256], + ['ox', data.original_sha256], + ['size', file.size.toString()], + ['blurhash', data.blurhash], + ]; + + if (data.dimensions) { + tags.push(['dim', `${data.dimensions.width}x${data.dimensions.height}`]); + } + + return tags; }, }; diff --git a/src/uploaders/s3.ts b/src/uploaders/s3.ts index 267d817..aaff8c8 100644 --- a/src/uploaders/s3.ts +++ b/src/uploaders/s3.ts @@ -25,12 +25,14 @@ const s3Uploader: Uploader = { const { pathStyle, bucket } = Conf.s3; const path = (pathStyle && bucket) ? join(bucket, filename) : filename; + const url = new URL(path, Conf.mediaDomain).toString(); - return { - id: filename, - sha256, - url: new URL(path, Conf.mediaDomain).toString(), - }; + return [ + ['url', url], + ['m', file.type], + ['x', sha256], + ['size', file.size.toString()], + ]; }, async delete(id) { await client().deleteObject(id); diff --git a/src/uploaders/types.ts b/src/uploaders/types.ts index e423028..81b8a0a 100644 --- a/src/uploaders/types.ts +++ b/src/uploaders/types.ts @@ -1,27 +1,9 @@ /** Modular uploader interface, to support uploading to different backends. */ interface Uploader { /** Upload the file to the backend. */ - upload(file: File, opts?: { signal?: AbortSignal }): Promise; + upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]>; /** Delete the file from the backend. */ - delete(cid: string, opts?: { signal?: AbortSignal }): Promise; -} - -/** Return value from the uploader after uploading a file. */ -interface UploadResult { - /** File ID specific to the uploader, so it can later be referenced or deleted. */ - id: string; - /** URL where the file can be accessed. */ - url: string; - /** SHA-256 hash of the file. */ - sha256?: string; - /** Blurhash of the file. */ - blurhash?: string; - /** IPFS CID of the file. */ - cid?: string; - /** Width of the file, if applicable. */ - width?: number; - /** Height of the file, if applicable. */ - height?: number; + delete?(cid: string, opts?: { signal?: AbortSignal }): Promise; } export type { Uploader }; From 82c03dcb56f9d9235a4d15d1ebe41a696bd92546 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 22:00:24 -0500 Subject: [PATCH 38/65] Rewrite all the uploaders --- src/app.ts | 5 ++ src/controllers/api/accounts.ts | 9 +++- src/controllers/api/media.ts | 7 ++- src/interfaces/DittoUploader.ts | 3 ++ src/middleware/uploaderMiddleware.ts | 27 ++++++++++ src/schemas/nostrbuild.ts | 19 ------- src/upload.ts | 12 +++-- src/uploaders/DenoUploader.ts | 44 ++++++++++++++++ src/uploaders/IPFSUploader.ts | 70 ++++++++++++++++++++++++++ src/uploaders/NostrBuildUploader.ts | 65 ++++++++++++++++++++++++ src/uploaders/{s3.ts => S3Uploader.ts} | 40 +++++++++------ src/uploaders/config.ts | 36 ------------- src/uploaders/ipfs.ts | 57 --------------------- src/uploaders/local.ts | 37 -------------- src/uploaders/nostrbuild.ts | 35 ------------- src/uploaders/types.ts | 9 ---- 16 files changed, 260 insertions(+), 215 deletions(-) create mode 100644 src/interfaces/DittoUploader.ts create mode 100644 src/middleware/uploaderMiddleware.ts delete mode 100644 src/schemas/nostrbuild.ts create mode 100644 src/uploaders/DenoUploader.ts create mode 100644 src/uploaders/IPFSUploader.ts create mode 100644 src/uploaders/NostrBuildUploader.ts rename src/uploaders/{s3.ts => S3Uploader.ts} (58%) delete mode 100644 src/uploaders/config.ts delete mode 100644 src/uploaders/ipfs.ts delete mode 100644 src/uploaders/local.ts delete mode 100644 src/uploaders/nostrbuild.ts delete mode 100644 src/uploaders/types.ts diff --git a/src/app.ts b/src/app.ts index 059e883..ddc9990 100644 --- a/src/app.ts +++ b/src/app.ts @@ -81,6 +81,7 @@ import { hostMetaController } from '@/controllers/well-known/host-meta.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; import { webfingerController } from '@/controllers/well-known/webfinger.ts'; +import { DittoUploader } from '@/interfaces/DittoUploader.ts'; import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cacheMiddleware } from '@/middleware/cacheMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; @@ -89,11 +90,14 @@ import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; import { storeMiddleware } from '@/middleware/storeMiddleware.ts'; import { blockController } from '@/controllers/api/accounts.ts'; import { unblockController } from '@/controllers/api/accounts.ts'; +import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts'; interface AppEnv extends HonoEnv { Variables: { /** Signer to get the logged-in user's pubkey, relays, and to sign events, or `undefined` if the user isn't logged in. */ signer?: NostrSigner; + /** Uploader for the user to upload files. */ + uploader?: DittoUploader; /** NIP-98 signed event proving the pubkey is owned by the user. */ proof?: NostrEvent; /** Store */ @@ -129,6 +133,7 @@ app.use( cspMiddleware(), cors({ origin: '*', exposeHeaders: ['link'] }), signerMiddleware, + uploaderMiddleware, auth98Middleware(), storeMiddleware, ); diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 5c26ba5..d67fd88 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -202,6 +202,7 @@ const updateCredentialsSchema = z.object({ const updateCredentialsController: AppController = async (c) => { const pubkey = await c.get('signer')?.getPublicKey()!; + const uploader = c.get('uploader'); const body = await parseBody(c.req.raw); const result = updateCredentialsSchema.safeParse(body); @@ -220,9 +221,13 @@ const updateCredentialsController: AppController = async (c) => { nip05, } = result.data; + if ((avatarFile || headerFile) && !uploader) { + return c.json({ error: 'No uploader configured.' }, 500); + } + const [avatar, header] = await Promise.all([ - avatarFile ? uploadFile(avatarFile, { pubkey }) : undefined, - headerFile ? uploadFile(headerFile, { pubkey }) : undefined, + (avatarFile && uploader) ? uploadFile(uploader, avatarFile, { pubkey }) : undefined, + (headerFile && uploader) ? uploadFile(uploader, headerFile, { pubkey }) : undefined, ]); meta.name = display_name ?? meta.name; diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index 33b7981..101b776 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -14,6 +14,11 @@ const mediaBodySchema = z.object({ }); const mediaController: AppController = async (c) => { + const uploader = c.get('uploader'); + if (!uploader) { + return c.json({ error: 'No uploader configured.' }, 500); + } + const pubkey = await c.get('signer')?.getPublicKey()!; const result = mediaBodySchema.safeParse(await parseBody(c.req.raw)); const { signal } = c.req.raw; @@ -24,7 +29,7 @@ const mediaController: AppController = async (c) => { try { const { file, description } = result.data; - const media = await uploadFile(file, { pubkey, description }, signal); + const media = await uploadFile(uploader, file, { pubkey, description }, signal); return c.json(renderAttachment(media)); } catch (e) { console.error(e); diff --git a/src/interfaces/DittoUploader.ts b/src/interfaces/DittoUploader.ts new file mode 100644 index 0000000..08cbf50 --- /dev/null +++ b/src/interfaces/DittoUploader.ts @@ -0,0 +1,3 @@ +export interface DittoUploader { + upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]>; +} diff --git a/src/middleware/uploaderMiddleware.ts b/src/middleware/uploaderMiddleware.ts new file mode 100644 index 0000000..8279a12 --- /dev/null +++ b/src/middleware/uploaderMiddleware.ts @@ -0,0 +1,27 @@ +import { AppMiddleware } from '@/app.ts'; +import { Conf } from '@/config.ts'; +import { DenoUploader } from '@/uploaders/DenoUploader.ts'; +import { IPFSUploader } from '@/uploaders/IPFSUploader.ts'; +import { NostrBuildUploader } from '@/uploaders/NostrBuildUploader.ts'; +import { S3Uploader } from '@/uploaders/S3Uploader.ts'; +import { fetchWorker } from '@/workers/fetch.ts'; + +/** Set an uploader for the user. */ +export const uploaderMiddleware: AppMiddleware = async (c, next) => { + switch (Conf.uploader) { + case 's3': + c.set('uploader', new S3Uploader(Conf.s3)); + break; + case 'ipfs': + c.set('uploader', new IPFSUploader({ baseUrl: Conf.mediaDomain, apiUrl: Conf.ipfs.apiUrl, fetch: fetchWorker })); + break; + case 'local': + c.set('uploader', new DenoUploader({ baseUrl: Conf.mediaDomain, dir: Conf.uploadsDir })); + break; + case 'nostrbuild': + c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, fetch: fetchWorker })); + break; + } + + await next(); +}; diff --git a/src/schemas/nostrbuild.ts b/src/schemas/nostrbuild.ts deleted file mode 100644 index db9f607..0000000 --- a/src/schemas/nostrbuild.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { z } from 'zod'; - -export const nostrbuildFileSchema = z.object({ - name: z.string(), - url: z.string().url(), - thumbnail: z.string(), - blurhash: z.string(), - sha256: z.string(), - original_sha256: z.string(), - mime: z.string(), - dimensions: z.object({ - width: z.number(), - height: z.number(), - }).optional().catch(undefined), -}); - -export const nostrbuildSchema = z.object({ - data: nostrbuildFileSchema.array().min(1), -}); diff --git a/src/upload.ts b/src/upload.ts index cd9d2bb..0d1a085 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -1,14 +1,18 @@ import { Conf } from '@/config.ts'; import { insertUnattachedMedia, UnattachedMedia } from '@/db/unattached-media.ts'; -import { configUploader as uploader } from '@/uploaders/config.ts'; - +import { DittoUploader } from '@/interfaces/DittoUploader.ts'; interface FileMeta { pubkey: string; description?: string; } /** Upload a file, track it in the database, and return the resulting media object. */ -async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Promise { +export async function uploadFile( + uploader: DittoUploader, + file: File, + meta: FileMeta, + signal?: AbortSignal, +): Promise { const { pubkey, description } = meta; if (file.size > Conf.maxUploadSize) { @@ -30,5 +34,3 @@ async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Pro uploaded_at: Date.now(), }); } - -export { uploadFile }; diff --git a/src/uploaders/DenoUploader.ts b/src/uploaders/DenoUploader.ts new file mode 100644 index 0000000..6c2e6d4 --- /dev/null +++ b/src/uploaders/DenoUploader.ts @@ -0,0 +1,44 @@ +import { join } from 'node:path'; + +import { crypto } from '@std/crypto'; +import { encodeHex } from '@std/encoding/hex'; +import { extensionsByType } from '@std/media-types'; + +import { DittoUploader } from '@/interfaces/DittoUploader.ts'; + +export interface DenoUploaderOpts { + baseUrl: string; + dir: string; +} + +/** Local Deno filesystem uploader. */ +export class DenoUploader implements DittoUploader { + constructor(private opts: DenoUploaderOpts) {} + + async upload(file: File): Promise<[['url', string], ...string[][]]> { + const { dir, baseUrl } = this.opts; + + const sha256 = encodeHex(await crypto.subtle.digest('SHA-256', file.stream())); + const ext = extensionsByType(file.type)?.[0] ?? 'bin'; + const filename = `${sha256}.${ext}`; + + await Deno.mkdir(dir, { recursive: true }); + await Deno.writeFile(join(dir, filename), file.stream()); + + const url = new URL(baseUrl); + const path = url.pathname === '/' ? filename : join(url.pathname, filename); + + return [ + ['url', new URL(path, url).toString()], + ['m', file.type], + ['x', sha256], + ['size', file.size.toString()], + ]; + } + + async delete(filename: string) { + const { dir } = this.opts; + const path = join(dir, filename); + await Deno.remove(path); + } +} diff --git a/src/uploaders/IPFSUploader.ts b/src/uploaders/IPFSUploader.ts new file mode 100644 index 0000000..ceb4e82 --- /dev/null +++ b/src/uploaders/IPFSUploader.ts @@ -0,0 +1,70 @@ +import { z } from 'zod'; + +import { DittoUploader } from '@/interfaces/DittoUploader.ts'; + +export interface IPFSUploaderOpts { + baseUrl: string; + apiUrl?: string; + fetch?: typeof fetch; +} + +/** + * IPFS uploader. It expects an IPFS node up and running. + * It will try to connect to `http://localhost:5001` by default, + * and upload the file using the REST API. + */ +export class IPFSUploader implements DittoUploader { + private baseUrl: string; + private apiUrl: string; + private fetch: typeof fetch; + + constructor(opts: IPFSUploaderOpts) { + this.baseUrl = opts.baseUrl; + this.apiUrl = opts.apiUrl ?? 'http://localhost:5001'; + this.fetch = opts.fetch ?? globalThis.fetch; + } + + async upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]> { + const url = new URL('/api/v0/add', this.apiUrl); + + const formData = new FormData(); + formData.append('file', file); + + const response = await this.fetch(url, { + method: 'POST', + body: formData, + signal: opts?.signal, + }); + + const { Hash: cid } = IPFSUploader.schema().parse(await response.json()); + + return [ + ['url', new URL(`/ipfs/${cid}`, this.baseUrl).toString()], + ['m', file.type], + ['cid', cid], + ['size', file.size.toString()], + ]; + } + + async delete(cid: string, opts?: { signal?: AbortSignal }): Promise { + const url = new URL('/api/v0/pin/rm', this.apiUrl); + + const query = new URLSearchParams(); + query.set('arg', cid); + url.search = query.toString(); + + await this.fetch(url, { + method: 'POST', + signal: opts?.signal, + }); + } + + /** Response schema for POST `/api/v0/add`. */ + static schema() { + return z.object({ + Name: z.string(), + Hash: z.string(), + Size: z.string(), + }); + } +} diff --git a/src/uploaders/NostrBuildUploader.ts b/src/uploaders/NostrBuildUploader.ts new file mode 100644 index 0000000..7e16448 --- /dev/null +++ b/src/uploaders/NostrBuildUploader.ts @@ -0,0 +1,65 @@ +import { z } from 'zod'; + +import { DittoUploader } from '@/interfaces/DittoUploader.ts'; + +export interface NostrBuildUploaderOpts { + endpoint?: string; + fetch?: typeof fetch; +} + +/** Upload files to nostr.build or another compatible server. */ +export class NostrBuildUploader implements DittoUploader { + constructor(private opts: NostrBuildUploaderOpts) {} + + async upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]> { + const { endpoint = 'https://nostr.build/api/v2/upload/files', fetch = globalThis.fetch } = this.opts; + + const formData = new FormData(); + formData.append('fileToUpload', file); + + const response = await fetch(endpoint, { + method: 'POST', + body: formData, + signal: opts?.signal, + }); + + const json = await response.json(); + const [data] = NostrBuildUploader.schema().parse(json).data; + + const tags: [['url', string], ...string[][]] = [ + ['url', data.url], + ['m', data.mime], + ['x', data.sha256], + ['ox', data.original_sha256], + ['size', data.size.toString()], + ]; + + if (data.dimensions) { + tags.push(['dim', `${data.dimensions.width}x${data.dimensions.height}`]); + } + + if (data.blurhash) { + tags.push(['blurhash', data.blurhash]); + } + + return tags; + } + + /** nostr.build API response schema. */ + private static schema() { + return z.object({ + data: z.object({ + url: z.string().url(), + blurhash: z.string().optional().catch(undefined), + sha256: z.string(), + original_sha256: z.string(), + mime: z.string(), + size: z.number(), + dimensions: z.object({ + width: z.number(), + height: z.number(), + }).optional().catch(undefined), + }).array().min(1), + }); + } +} diff --git a/src/uploaders/s3.ts b/src/uploaders/S3Uploader.ts similarity index 58% rename from src/uploaders/s3.ts rename to src/uploaders/S3Uploader.ts index aaff8c8..f210ce8 100644 --- a/src/uploaders/s3.ts +++ b/src/uploaders/S3Uploader.ts @@ -6,17 +6,34 @@ import { encodeHex } from '@std/encoding/hex'; import { extensionsByType } from '@std/media-types'; import { Conf } from '@/config.ts'; +import { DittoUploader } from '@/interfaces/DittoUploader.ts'; -import type { Uploader } from './types.ts'; +export interface S3UploaderOpts { + endPoint: string; + region: string; + accessKey?: string; + secretKey?: string; + bucket?: string; + pathStyle?: boolean; + port?: number; + sessionToken?: string; + useSSL?: boolean; +} /** S3-compatible uploader for AWS, Wasabi, DigitalOcean Spaces, and more. */ -const s3Uploader: Uploader = { - async upload(file) { +export class S3Uploader implements DittoUploader { + private client: S3Client; + + constructor(opts: S3UploaderOpts) { + this.client = new S3Client(opts); + } + + async upload(file: File): Promise<[['url', string], ...string[][]]> { const sha256 = encodeHex(await crypto.subtle.digest('SHA-256', file.stream())); const ext = extensionsByType(file.type)?.[0] ?? 'bin'; const filename = `${sha256}.${ext}`; - await client().putObject(filename, file.stream(), { + await this.client.putObject(filename, file.stream(), { metadata: { 'Content-Type': file.type, 'x-amz-acl': 'public-read', @@ -24,6 +41,7 @@ const s3Uploader: Uploader = { }); const { pathStyle, bucket } = Conf.s3; + const path = (pathStyle && bucket) ? join(bucket, filename) : filename; const url = new URL(path, Conf.mediaDomain).toString(); @@ -33,15 +51,9 @@ const s3Uploader: Uploader = { ['x', sha256], ['size', file.size.toString()], ]; - }, - async delete(id) { - await client().deleteObject(id); - }, -}; + } -/** Build S3 client from config. */ -function client() { - return new S3Client({ ...Conf.s3 }); + async delete(objectName: string) { + await this.client.deleteObject(objectName); + } } - -export { s3Uploader }; diff --git a/src/uploaders/config.ts b/src/uploaders/config.ts deleted file mode 100644 index 3f3aac7..0000000 --- a/src/uploaders/config.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Conf } from '@/config.ts'; - -import { ipfsUploader } from '@/uploaders/ipfs.ts'; -import { localUploader } from '@/uploaders/local.ts'; -import { nostrbuildUploader } from '@/uploaders/nostrbuild.ts'; -import { s3Uploader } from '@/uploaders/s3.ts'; - -import type { Uploader } from './types.ts'; - -/** Meta-uploader determined from configuration. */ -const configUploader: Uploader = { - upload(file, opts) { - return uploader().upload(file, opts); - }, - async delete(id, opts) { - return await uploader().delete?.(id, opts); - }, -}; - -/** Get the uploader module based on configuration. */ -function uploader() { - switch (Conf.uploader) { - case 's3': - return s3Uploader; - case 'ipfs': - return ipfsUploader; - case 'local': - return localUploader; - case 'nostrbuild': - return nostrbuildUploader; - default: - throw new Error('No `DITTO_UPLOADER` configured. Uploads are disabled.'); - } -} - -export { configUploader }; diff --git a/src/uploaders/ipfs.ts b/src/uploaders/ipfs.ts deleted file mode 100644 index b83dc2e..0000000 --- a/src/uploaders/ipfs.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { z } from 'zod'; - -import { Conf } from '@/config.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; - -import type { Uploader } from './types.ts'; - -/** Response schema for POST `/api/v0/add`. */ -const ipfsAddResponseSchema = z.object({ - Name: z.string(), - Hash: z.string(), - Size: z.string(), -}); - -/** - * IPFS uploader. It expects an IPFS node up and running. - * It will try to connect to `http://localhost:5001` by default, - * and upload the file using the REST API. - */ -const ipfsUploader: Uploader = { - async upload(file, opts) { - const url = new URL('/api/v0/add', Conf.ipfs.apiUrl); - - const formData = new FormData(); - formData.append('file', file); - - const response = await fetchWorker(url, { - method: 'POST', - body: formData, - signal: opts?.signal, - }); - - const { Hash: cid } = ipfsAddResponseSchema.parse(await response.json()); - - return [ - ['url', new URL(`/ipfs/${cid}`, Conf.mediaDomain).toString()], - ['m', file.type], - ['cid', cid], - ['size', file.size.toString()], - ]; - }, - async delete(cid, opts) { - const url = new URL('/api/v0/pin/rm', Conf.ipfs.apiUrl); - - const query = new URLSearchParams(); - query.set('arg', cid); - - url.search = query.toString(); - - await fetchWorker(url, { - method: 'POST', - signal: opts?.signal, - }); - }, -}; - -export { ipfsUploader }; diff --git a/src/uploaders/local.ts b/src/uploaders/local.ts deleted file mode 100644 index d5cd46a..0000000 --- a/src/uploaders/local.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { join } from 'node:path'; - -import { crypto } from '@std/crypto'; -import { encodeHex } from '@std/encoding/hex'; -import { extensionsByType } from '@std/media-types'; - -import { Conf } from '@/config.ts'; - -import type { Uploader } from './types.ts'; - -/** Local filesystem uploader. */ -const localUploader: Uploader = { - async upload(file) { - const sha256 = encodeHex(await crypto.subtle.digest('SHA-256', file.stream())); - const ext = extensionsByType(file.type)?.[0] ?? 'bin'; - const filename = `${sha256}.${ext}`; - - await Deno.mkdir(Conf.uploadsDir, { recursive: true }); - await Deno.writeFile(join(Conf.uploadsDir, filename), file.stream()); - - const { mediaDomain } = Conf; - const url = new URL(mediaDomain); - const path = url.pathname === '/' ? filename : join(url.pathname, filename); - - return [ - ['url', new URL(path, url).toString()], - ['m', file.type], - ['x', sha256], - ['size', file.size.toString()], - ]; - }, - async delete(id) { - await Deno.remove(join(Conf.uploadsDir, id)); - }, -}; - -export { localUploader }; diff --git a/src/uploaders/nostrbuild.ts b/src/uploaders/nostrbuild.ts deleted file mode 100644 index 8bca331..0000000 --- a/src/uploaders/nostrbuild.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { Conf } from '@/config.ts'; -import { nostrbuildSchema } from '@/schemas/nostrbuild.ts'; - -import type { Uploader } from './types.ts'; - -/** nostr.build uploader. */ -export const nostrbuildUploader: Uploader = { - async upload(file) { - const formData = new FormData(); - formData.append('fileToUpload', file); - - const response = await fetch(Conf.nostrbuildEndpoint, { - method: 'POST', - body: formData, - }); - - const json = await response.json(); - const [data] = nostrbuildSchema.parse(json).data; - - const tags: [['url', string], ...string[][]] = [ - ['url', data.url], - ['m', data.mime], - ['x', data.sha256], - ['ox', data.original_sha256], - ['size', file.size.toString()], - ['blurhash', data.blurhash], - ]; - - if (data.dimensions) { - tags.push(['dim', `${data.dimensions.width}x${data.dimensions.height}`]); - } - - return tags; - }, -}; diff --git a/src/uploaders/types.ts b/src/uploaders/types.ts deleted file mode 100644 index 81b8a0a..0000000 --- a/src/uploaders/types.ts +++ /dev/null @@ -1,9 +0,0 @@ -/** Modular uploader interface, to support uploading to different backends. */ -interface Uploader { - /** Upload the file to the backend. */ - upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]>; - /** Delete the file from the backend. */ - delete?(cid: string, opts?: { signal?: AbortSignal }): Promise; -} - -export type { Uploader }; From 6542d6a77789dd1a662696a7142497334fe9df5a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 22:04:43 -0500 Subject: [PATCH 39/65] Move uploader.ts to utils, make it kind of like api.ts --- src/controllers/api/accounts.ts | 11 +++-------- src/controllers/api/media.ts | 9 ++------- src/{ => utils}/upload.ts | 12 ++++++++++-- 3 files changed, 15 insertions(+), 17 deletions(-) rename src/{ => utils}/upload.ts (75%) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index d67fd88..4777f56 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -8,7 +8,7 @@ import { getAuthor, getFollowedPubkeys } from '@/queries.ts'; import { booleanParamSchema, fileSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; import { addTag, deleteTag, findReplyTag, getTagSet } from '@/tags.ts'; -import { uploadFile } from '@/upload.ts'; +import { uploadFile } from '@/utils/upload.ts'; import { nostrNow } from '@/utils.ts'; import { createEvent, paginated, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; import { lookupAccount } from '@/utils/lookup.ts'; @@ -202,7 +202,6 @@ const updateCredentialsSchema = z.object({ const updateCredentialsController: AppController = async (c) => { const pubkey = await c.get('signer')?.getPublicKey()!; - const uploader = c.get('uploader'); const body = await parseBody(c.req.raw); const result = updateCredentialsSchema.safeParse(body); @@ -221,13 +220,9 @@ const updateCredentialsController: AppController = async (c) => { nip05, } = result.data; - if ((avatarFile || headerFile) && !uploader) { - return c.json({ error: 'No uploader configured.' }, 500); - } - const [avatar, header] = await Promise.all([ - (avatarFile && uploader) ? uploadFile(uploader, avatarFile, { pubkey }) : undefined, - (headerFile && uploader) ? uploadFile(uploader, headerFile, { pubkey }) : undefined, + avatarFile ? uploadFile(c, avatarFile, { pubkey }) : undefined, + headerFile ? uploadFile(c, headerFile, { pubkey }) : undefined, ]); meta.name = display_name ?? meta.name; diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index 101b776..71b3e78 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -4,7 +4,7 @@ import { AppController } from '@/app.ts'; import { fileSchema } from '@/schema.ts'; import { parseBody } from '@/utils/api.ts'; import { renderAttachment } from '@/views/mastodon/attachments.ts'; -import { uploadFile } from '@/upload.ts'; +import { uploadFile } from '@/utils/upload.ts'; const mediaBodySchema = z.object({ file: fileSchema, @@ -14,11 +14,6 @@ const mediaBodySchema = z.object({ }); const mediaController: AppController = async (c) => { - const uploader = c.get('uploader'); - if (!uploader) { - return c.json({ error: 'No uploader configured.' }, 500); - } - const pubkey = await c.get('signer')?.getPublicKey()!; const result = mediaBodySchema.safeParse(await parseBody(c.req.raw)); const { signal } = c.req.raw; @@ -29,7 +24,7 @@ const mediaController: AppController = async (c) => { try { const { file, description } = result.data; - const media = await uploadFile(uploader, file, { pubkey, description }, signal); + const media = await uploadFile(c, file, { pubkey, description }, signal); return c.json(renderAttachment(media)); } catch (e) { console.error(e); diff --git a/src/upload.ts b/src/utils/upload.ts similarity index 75% rename from src/upload.ts rename to src/utils/upload.ts index 0d1a085..c4f2fc5 100644 --- a/src/upload.ts +++ b/src/utils/upload.ts @@ -1,6 +1,7 @@ +import { AppContext } from '@/app.ts'; import { Conf } from '@/config.ts'; import { insertUnattachedMedia, UnattachedMedia } from '@/db/unattached-media.ts'; -import { DittoUploader } from '@/interfaces/DittoUploader.ts'; +import { HTTPException } from 'hono'; interface FileMeta { pubkey: string; description?: string; @@ -8,11 +9,18 @@ interface FileMeta { /** Upload a file, track it in the database, and return the resulting media object. */ export async function uploadFile( - uploader: DittoUploader, + c: AppContext, file: File, meta: FileMeta, signal?: AbortSignal, ): Promise { + const uploader = c.get('uploader'); + if (!uploader) { + throw new HTTPException(500, { + res: c.json({ error: 'No uploader configured.' }), + }); + } + const { pubkey, description } = meta; if (file.size > Conf.maxUploadSize) { From 24659d8edb0cf1daf9c20090382c50a59994c561 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 22:11:54 -0500 Subject: [PATCH 40/65] IPFSUploader: make schema private --- src/uploaders/IPFSUploader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uploaders/IPFSUploader.ts b/src/uploaders/IPFSUploader.ts index ceb4e82..9141e78 100644 --- a/src/uploaders/IPFSUploader.ts +++ b/src/uploaders/IPFSUploader.ts @@ -60,7 +60,7 @@ export class IPFSUploader implements DittoUploader { } /** Response schema for POST `/api/v0/add`. */ - static schema() { + private static schema() { return z.object({ Name: z.string(), Hash: z.string(), From acef173ac465c40b60919dbcdcb842ede7e0ff39 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 22:15:33 -0500 Subject: [PATCH 41/65] Do things the boilerplatey way just for consistency --- src/uploaders/DenoUploader.ts | 19 +++++++++++-------- src/uploaders/NostrBuildUploader.ts | 12 ++++++++---- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/uploaders/DenoUploader.ts b/src/uploaders/DenoUploader.ts index 6c2e6d4..e2224ab 100644 --- a/src/uploaders/DenoUploader.ts +++ b/src/uploaders/DenoUploader.ts @@ -13,19 +13,23 @@ export interface DenoUploaderOpts { /** Local Deno filesystem uploader. */ export class DenoUploader implements DittoUploader { - constructor(private opts: DenoUploaderOpts) {} + baseUrl: string; + dir: string; + + constructor(opts: DenoUploaderOpts) { + this.baseUrl = opts.baseUrl; + this.dir = opts.dir; + } async upload(file: File): Promise<[['url', string], ...string[][]]> { - const { dir, baseUrl } = this.opts; - const sha256 = encodeHex(await crypto.subtle.digest('SHA-256', file.stream())); const ext = extensionsByType(file.type)?.[0] ?? 'bin'; const filename = `${sha256}.${ext}`; - await Deno.mkdir(dir, { recursive: true }); - await Deno.writeFile(join(dir, filename), file.stream()); + await Deno.mkdir(this.dir, { recursive: true }); + await Deno.writeFile(join(this.dir, filename), file.stream()); - const url = new URL(baseUrl); + const url = new URL(this.baseUrl); const path = url.pathname === '/' ? filename : join(url.pathname, filename); return [ @@ -37,8 +41,7 @@ export class DenoUploader implements DittoUploader { } async delete(filename: string) { - const { dir } = this.opts; - const path = join(dir, filename); + const path = join(this.dir, filename); await Deno.remove(path); } } diff --git a/src/uploaders/NostrBuildUploader.ts b/src/uploaders/NostrBuildUploader.ts index 7e16448..ff4a4f0 100644 --- a/src/uploaders/NostrBuildUploader.ts +++ b/src/uploaders/NostrBuildUploader.ts @@ -9,15 +9,19 @@ export interface NostrBuildUploaderOpts { /** Upload files to nostr.build or another compatible server. */ export class NostrBuildUploader implements DittoUploader { - constructor(private opts: NostrBuildUploaderOpts) {} + private endpoint: string; + private fetch: typeof fetch; + + constructor(opts: NostrBuildUploaderOpts) { + this.endpoint = opts.endpoint ?? 'https://nostr.build/api/v2/upload/files'; + this.fetch = opts.fetch ?? globalThis.fetch; + } async upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]> { - const { endpoint = 'https://nostr.build/api/v2/upload/files', fetch = globalThis.fetch } = this.opts; - const formData = new FormData(); formData.append('fileToUpload', file); - const response = await fetch(endpoint, { + const response = await this.fetch(this.endpoint, { method: 'POST', body: formData, signal: opts?.signal, From 5523c3fc0eeca605cdbf5691a37b7581001b0928 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 18 May 2024 23:02:59 -0500 Subject: [PATCH 42/65] verifyCredentials: wait up to 5 seconds --- src/controllers/api/accounts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 4777f56..f66e0ac 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -47,7 +47,7 @@ const createAccountController: AppController = async (c) => { const verifyCredentialsController: AppController = async (c) => { const pubkey = await c.get('signer')?.getPublicKey()!; - const event = await getAuthor(pubkey, { relations: ['author_stats'] }); + const event = await getAuthor(pubkey, { signal: AbortSignal.timeout(5000) }); if (event) { return c.json(await renderAccount(event, { withSource: true })); } else { From 7f5179efcac065ac902a1b84f03d610a9f9e5f63 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 09:13:53 -0500 Subject: [PATCH 43/65] renderAttachment: guess mime from url --- src/views/mastodon/attachments.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 0b1b8eb..2d65804 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -1,9 +1,12 @@ +import { getUrlMediaType } from '@/utils/media.ts'; + /** Render Mastodon media attachment. */ function renderAttachment(media: { id?: string; data: string[][] }) { const { id, data: tags } = media; - const m = tags.find(([name]) => name === 'm')?.[1]; const url = tags.find(([name]) => name === 'url')?.[1]; + + const m = tags.find(([name]) => name === 'm')?.[1] ?? getUrlMediaType(url!); const alt = tags.find(([name]) => name === 'alt')?.[1]; const cid = tags.find(([name]) => name === 'cid')?.[1]; const dim = tags.find(([name]) => name === 'dim')?.[1]; From 540bd058a2b727df2c8e6e6af4e04ad7057f8a11 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 11:33:59 -0500 Subject: [PATCH 44/65] Fix NIP-27 mentions --- src/controllers/api/statuses.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index e620b93..291d970 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -1,5 +1,6 @@ import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; import ISO6391 from 'iso-639-1'; +import { nip19 } from 'nostr-tools'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; @@ -111,7 +112,11 @@ const createStatusController: AppController = async (c) => { pubkeys.add(pubkey); } - return `nostr:${pubkey}`; + try { + return `nostr:${nip19.npubEncode(pubkey)}`; + } catch { + return match; + } }); // Explicit addressing From 9754e29603d404d20d6db3a2c435570266f507bb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 11:45:42 -0500 Subject: [PATCH 45/65] accountSearchController: respect the `limit` param --- src/controllers/api/accounts.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index f66e0ac..39161ef 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -86,25 +86,35 @@ const accountLookupController: AppController = async (c) => { } }; -const accountSearchController: AppController = async (c) => { - const q = c.req.query('q'); +const accountSearchQuerySchema = z.object({ + q: z.string().transform(decodeURIComponent), + resolve: booleanParamSchema.optional().transform(Boolean), + following: z.boolean().default(false), + limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)), +}); - if (!q) { - return c.json({ error: 'Missing `q` query parameter.' }, 422); +const accountSearchController: AppController = async (c) => { + const result = accountSearchQuerySchema.safeParse(c.req.query()); + const { signal } = c.req.raw; + + if (!result.success) { + return c.json({ error: 'Bad request', schema: result.error }, 422); } + const { q, limit } = result.data; + const query = decodeURIComponent(q); const store = await Storages.search(); const [event, events] = await Promise.all([ lookupAccount(query), - store.query([{ kinds: [0], search: query, limit: 20 }], { signal: c.req.raw.signal }), + store.query([{ kinds: [0], search: query, limit }], { signal }), ]); const results = await hydrateEvents({ events: event ? [event, ...events] : events, store, - signal: c.req.raw.signal, + signal, }); if ((results.length < 1) && query.match(/npub1\w+/)) { From 7c5b7c5d835e23cd5fc2d2354f2d9a970e6e2d3b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 15:38:42 -0500 Subject: [PATCH 46/65] Upgrade Nostrify to v0.22.0 --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 946b4e0..5567cd2 100644 --- a/deno.json +++ b/deno.json @@ -22,7 +22,7 @@ "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.21.1", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.22.0", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", From f0b247130f46875275c814da6386952d08038071 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 15:42:45 -0500 Subject: [PATCH 47/65] Add support for Blossom uploader --- src/config.ts | 4 ++++ src/middleware/uploaderMiddleware.ts | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/src/config.ts b/src/config.ts index f3b472e..cc14998 100644 --- a/src/config.ts +++ b/src/config.ts @@ -140,6 +140,10 @@ class Conf { static get nostrbuildEndpoint(): string { return Deno.env.get('NOSTRBUILD_ENDPOINT') || 'https://nostr.build/api/v2/upload/files'; } + /** Default Blossom servers to use when the `blossom` uploader is set. */ + static get blossomServers(): string[] { + return Deno.env.get('BLOSSOM_SERVERS')?.split(',') || ['https://blossom.primal.net/']; + } /** Module to upload files with. */ static get uploader() { return Deno.env.get('DITTO_UPLOADER'); diff --git a/src/middleware/uploaderMiddleware.ts b/src/middleware/uploaderMiddleware.ts index 8279a12..b0ee570 100644 --- a/src/middleware/uploaderMiddleware.ts +++ b/src/middleware/uploaderMiddleware.ts @@ -1,3 +1,5 @@ +import { BlossomUploader } from '@nostrify/nostrify/uploaders'; + import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DenoUploader } from '@/uploaders/DenoUploader.ts'; @@ -8,6 +10,8 @@ import { fetchWorker } from '@/workers/fetch.ts'; /** Set an uploader for the user. */ export const uploaderMiddleware: AppMiddleware = async (c, next) => { + const signer = c.get('signer'); + switch (Conf.uploader) { case 's3': c.set('uploader', new S3Uploader(Conf.s3)); @@ -21,6 +25,11 @@ export const uploaderMiddleware: AppMiddleware = async (c, next) => { case 'nostrbuild': c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, fetch: fetchWorker })); break; + case 'blossom': + if (signer) { + c.set('uploader', new BlossomUploader({ servers: Conf.blossomServers, signer, fetch: fetchWorker })); + } + break; } await next(); From 0541287f0e0b6a5a2c8b10a2835e66b26f88d26c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 15:43:24 -0500 Subject: [PATCH 48/65] Replace our NostrBuildUploader with the one from Nostrify --- src/middleware/uploaderMiddleware.ts | 5 +- src/uploaders/NostrBuildUploader.ts | 69 ---------------------------- 2 files changed, 2 insertions(+), 72 deletions(-) delete mode 100644 src/uploaders/NostrBuildUploader.ts diff --git a/src/middleware/uploaderMiddleware.ts b/src/middleware/uploaderMiddleware.ts index b0ee570..38e8ace 100644 --- a/src/middleware/uploaderMiddleware.ts +++ b/src/middleware/uploaderMiddleware.ts @@ -1,10 +1,9 @@ -import { BlossomUploader } from '@nostrify/nostrify/uploaders'; +import { BlossomUploader, NostrBuildUploader } from '@nostrify/nostrify/uploaders'; import { AppMiddleware } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DenoUploader } from '@/uploaders/DenoUploader.ts'; import { IPFSUploader } from '@/uploaders/IPFSUploader.ts'; -import { NostrBuildUploader } from '@/uploaders/NostrBuildUploader.ts'; import { S3Uploader } from '@/uploaders/S3Uploader.ts'; import { fetchWorker } from '@/workers/fetch.ts'; @@ -23,7 +22,7 @@ export const uploaderMiddleware: AppMiddleware = async (c, next) => { c.set('uploader', new DenoUploader({ baseUrl: Conf.mediaDomain, dir: Conf.uploadsDir })); break; case 'nostrbuild': - c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, fetch: fetchWorker })); + c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, signer, fetch: fetchWorker })); break; case 'blossom': if (signer) { diff --git a/src/uploaders/NostrBuildUploader.ts b/src/uploaders/NostrBuildUploader.ts deleted file mode 100644 index ff4a4f0..0000000 --- a/src/uploaders/NostrBuildUploader.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { z } from 'zod'; - -import { DittoUploader } from '@/interfaces/DittoUploader.ts'; - -export interface NostrBuildUploaderOpts { - endpoint?: string; - fetch?: typeof fetch; -} - -/** Upload files to nostr.build or another compatible server. */ -export class NostrBuildUploader implements DittoUploader { - private endpoint: string; - private fetch: typeof fetch; - - constructor(opts: NostrBuildUploaderOpts) { - this.endpoint = opts.endpoint ?? 'https://nostr.build/api/v2/upload/files'; - this.fetch = opts.fetch ?? globalThis.fetch; - } - - async upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]> { - const formData = new FormData(); - formData.append('fileToUpload', file); - - const response = await this.fetch(this.endpoint, { - method: 'POST', - body: formData, - signal: opts?.signal, - }); - - const json = await response.json(); - const [data] = NostrBuildUploader.schema().parse(json).data; - - const tags: [['url', string], ...string[][]] = [ - ['url', data.url], - ['m', data.mime], - ['x', data.sha256], - ['ox', data.original_sha256], - ['size', data.size.toString()], - ]; - - if (data.dimensions) { - tags.push(['dim', `${data.dimensions.width}x${data.dimensions.height}`]); - } - - if (data.blurhash) { - tags.push(['blurhash', data.blurhash]); - } - - return tags; - } - - /** nostr.build API response schema. */ - private static schema() { - return z.object({ - data: z.object({ - url: z.string().url(), - blurhash: z.string().optional().catch(undefined), - sha256: z.string(), - original_sha256: z.string(), - mime: z.string(), - size: z.number(), - dimensions: z.object({ - width: z.number(), - height: z.number(), - }).optional().catch(undefined), - }).array().min(1), - }); - } -} From 6f6e87525e9b7124e52af8fc2451e57cdbd62828 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 19 May 2024 15:57:04 -0500 Subject: [PATCH 49/65] Remove DittoUploader interface in favor of NUploader --- src/app.ts | 5 ++--- src/interfaces/DittoUploader.ts | 3 --- src/uploaders/DenoUploader.ts | 5 ++--- src/uploaders/IPFSUploader.ts | 5 ++--- src/uploaders/S3Uploader.ts | 4 ++-- 5 files changed, 8 insertions(+), 14 deletions(-) delete mode 100644 src/interfaces/DittoUploader.ts diff --git a/src/app.ts b/src/app.ts index ddc9990..5300b48 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NostrSigner, NStore } from '@nostrify/nostrify'; +import { NostrEvent, NostrSigner, NStore, NUploader } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { type Context, Env as HonoEnv, type Handler, Hono, Input as HonoInput, type MiddlewareHandler } from 'hono'; import { cors, logger, serveStatic } from 'hono/middleware'; @@ -81,7 +81,6 @@ import { hostMetaController } from '@/controllers/well-known/host-meta.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; import { webfingerController } from '@/controllers/well-known/webfinger.ts'; -import { DittoUploader } from '@/interfaces/DittoUploader.ts'; import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cacheMiddleware } from '@/middleware/cacheMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; @@ -97,7 +96,7 @@ interface AppEnv extends HonoEnv { /** Signer to get the logged-in user's pubkey, relays, and to sign events, or `undefined` if the user isn't logged in. */ signer?: NostrSigner; /** Uploader for the user to upload files. */ - uploader?: DittoUploader; + uploader?: NUploader; /** NIP-98 signed event proving the pubkey is owned by the user. */ proof?: NostrEvent; /** Store */ diff --git a/src/interfaces/DittoUploader.ts b/src/interfaces/DittoUploader.ts deleted file mode 100644 index 08cbf50..0000000 --- a/src/interfaces/DittoUploader.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface DittoUploader { - upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]>; -} diff --git a/src/uploaders/DenoUploader.ts b/src/uploaders/DenoUploader.ts index e2224ab..fd30d8c 100644 --- a/src/uploaders/DenoUploader.ts +++ b/src/uploaders/DenoUploader.ts @@ -1,18 +1,17 @@ import { join } from 'node:path'; +import { NUploader } from '@nostrify/nostrify'; import { crypto } from '@std/crypto'; import { encodeHex } from '@std/encoding/hex'; import { extensionsByType } from '@std/media-types'; -import { DittoUploader } from '@/interfaces/DittoUploader.ts'; - export interface DenoUploaderOpts { baseUrl: string; dir: string; } /** Local Deno filesystem uploader. */ -export class DenoUploader implements DittoUploader { +export class DenoUploader implements NUploader { baseUrl: string; dir: string; diff --git a/src/uploaders/IPFSUploader.ts b/src/uploaders/IPFSUploader.ts index 9141e78..7bf5165 100644 --- a/src/uploaders/IPFSUploader.ts +++ b/src/uploaders/IPFSUploader.ts @@ -1,7 +1,6 @@ +import { NUploader } from '@nostrify/nostrify'; import { z } from 'zod'; -import { DittoUploader } from '@/interfaces/DittoUploader.ts'; - export interface IPFSUploaderOpts { baseUrl: string; apiUrl?: string; @@ -13,7 +12,7 @@ export interface IPFSUploaderOpts { * It will try to connect to `http://localhost:5001` by default, * and upload the file using the REST API. */ -export class IPFSUploader implements DittoUploader { +export class IPFSUploader implements NUploader { private baseUrl: string; private apiUrl: string; private fetch: typeof fetch; diff --git a/src/uploaders/S3Uploader.ts b/src/uploaders/S3Uploader.ts index f210ce8..b74796a 100644 --- a/src/uploaders/S3Uploader.ts +++ b/src/uploaders/S3Uploader.ts @@ -1,12 +1,12 @@ import { join } from 'node:path'; import { S3Client } from '@bradenmacdonald/s3-lite-client'; +import { NUploader } from '@nostrify/nostrify'; import { crypto } from '@std/crypto'; import { encodeHex } from '@std/encoding/hex'; import { extensionsByType } from '@std/media-types'; import { Conf } from '@/config.ts'; -import { DittoUploader } from '@/interfaces/DittoUploader.ts'; export interface S3UploaderOpts { endPoint: string; @@ -21,7 +21,7 @@ export interface S3UploaderOpts { } /** S3-compatible uploader for AWS, Wasabi, DigitalOcean Spaces, and more. */ -export class S3Uploader implements DittoUploader { +export class S3Uploader implements NUploader { private client: S3Client; constructor(opts: S3UploaderOpts) { From 7b099ee5659f278dfe49802e79c9e878ae4d69a1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 20 May 2024 11:39:31 -0500 Subject: [PATCH 50/65] EventsDB: don't index the user's bio for kind 0 events --- src/storages/EventsDB.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index ef51a89..5a3839a 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -180,8 +180,8 @@ class EventsDB implements NStore { /** Build search content for a user. */ static buildUserSearchContent(event: NostrEvent): string { - const { name, nip05, about } = n.json().pipe(n.metadata()).catch({}).parse(event.content); - return [name, nip05, about].filter(Boolean).join('\n'); + const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); + return [name, nip05].filter(Boolean).join('\n'); } /** Build search content from tag values. */ From 98fd4babcebc2d61506be13a370d32758e5e49c4 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 20 May 2024 14:46:53 -0300 Subject: [PATCH 51/65] test(EventsDB): use eventFixture() --- src/storages/EventsDB.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 2f34379..16b429d 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -9,10 +9,7 @@ import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { RelayError } from '@/RelayError.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { genEvent } from '@/test.ts'; - -import event0 from '~/fixtures/events/event-0.json' with { type: 'json' }; -import event1 from '~/fixtures/events/event-1.json' with { type: 'json' }; +import { eventFixture, genEvent } from '@/test.ts'; /** Create in-memory database for testing. */ const createDB = async () => { @@ -28,6 +25,7 @@ const createDB = async () => { Deno.test('count filters', async () => { const { eventsDB } = await createDB(); + const event1 = await eventFixture('event-1'); assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 0); await eventsDB.event(event1); @@ -37,6 +35,7 @@ Deno.test('count filters', async () => { Deno.test('insert and filter events', async () => { const { eventsDB } = await createDB(); + const event1 = await eventFixture('event-1'); await eventsDB.event(event1); assertEquals(await eventsDB.query([{ kinds: [1] }]), [event1]); @@ -52,6 +51,7 @@ Deno.test('insert and filter events', async () => { Deno.test('query events with domain search filter', async () => { const { eventsDB, kysely } = await createDB(); + const event1 = await eventFixture('event-1'); await eventsDB.event(event1); assertEquals(await eventsDB.query([{}]), [event1]); @@ -180,7 +180,7 @@ Deno.test('throws a RelayError when inserting an event deleted by a user', async Deno.test('inserting replaceable events', async () => { const { eventsDB } = await createDB(); - const event = event0; + const event = await eventFixture('event-0'); await eventsDB.event(event); const olderEvent = { ...event, id: '123', created_at: event.created_at - 1 }; From 6861dc1d57c8c9822a1b86a1e251643b6b8e8293 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 20 May 2024 12:49:01 -0500 Subject: [PATCH 52/65] Fix crash parsing Lightning URL Fixes https://gitlab.com/soapbox-pub/ditto/-/issues/139 --- src/utils/lnurl.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/utils/lnurl.ts b/src/utils/lnurl.ts index ea5ce8a..af344f2 100644 --- a/src/utils/lnurl.ts +++ b/src/utils/lnurl.ts @@ -28,8 +28,12 @@ function getLnurl({ lud06, lud16 }: { lud06?: string; lud16?: string }, limit?: if (lud16) { const [name, host] = lud16.split('@'); if (name && host) { - const url = new URL(`/.well-known/lnurlp/${name}`, `https://${host}`); - return LNURL.encode(url, limit); + try { + const url = new URL(`/.well-known/lnurlp/${name}`, `https://${host}`); + return LNURL.encode(url, limit); + } catch { + return; + } } } } From 7d786d731e05cf48d26fdec56d7c27a6c9bfee86 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 21 May 2024 10:09:06 -0300 Subject: [PATCH 53/65] feat: add script that installs latest soapbox --- deno.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 5567cd2..1d9ad6d 100644 --- a/deno.json +++ b/deno.json @@ -11,7 +11,8 @@ "nsec": "deno run scripts/nsec.ts", "admin:event": "deno run -A scripts/admin-event.ts", "admin:role": "deno run -A scripts/admin-role.ts", - "stats:recompute": "deno run -A scripts/stats-recompute.ts" + "stats:recompute": "deno run -A scripts/stats-recompute.ts", + "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip" }, "unstable": ["ffi", "kv", "worker-options"], "exclude": ["./public"], From 83e51ad67eb313b730d30ad3af8a6c355a555df1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 12:21:35 -0500 Subject: [PATCH 54/65] lodash -> entities --- deno.json | 1 + src/controllers/api/oauth.ts | 6 +++--- src/deps.ts | 2 -- src/views/mastodon/accounts.ts | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/deno.json b/deno.json index 1d9ad6d..1ebc35e 100644 --- a/deno.json +++ b/deno.json @@ -37,6 +37,7 @@ "@std/streams": "jsr:@std/streams@^0.223.0", "comlink": "npm:comlink@^4.4.1", "deno-safe-fetch": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", + "entities": "npm:entities@^4.5.0", "fast-stable-stringify": "npm:fast-stable-stringify@^1.0.0", "formdata-helper": "npm:formdata-helper@^0.3.0", "hono": "https://deno.land/x/hono@v3.10.1/mod.ts", diff --git a/src/controllers/api/oauth.ts b/src/controllers/api/oauth.ts index a755a4d..c1407f4 100644 --- a/src/controllers/api/oauth.ts +++ b/src/controllers/api/oauth.ts @@ -1,9 +1,9 @@ import { encodeBase64 } from '@std/encoding/base64'; +import { escape } from 'entities'; import { nip19 } from 'nostr-tools'; import { z } from 'zod'; import { AppController } from '@/app.ts'; -import { lodash } from '@/deps.ts'; import { nostrNow } from '@/utils.ts'; import { parseBody } from '@/utils/api.ts'; import { getClientConnectUri } from '@/utils/connect.ts'; @@ -100,11 +100,11 @@ const oauthController: AppController = async (c) => {
- +

- Nostr Connect + Nostr Connect `); diff --git a/src/deps.ts b/src/deps.ts index 12be07f..dc9d912 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,6 +1,4 @@ import 'deno-safe-fetch'; -// @deno-types="npm:@types/lodash@4.14.194" -export { default as lodash } from 'https://esm.sh/lodash@4.17.21'; // @deno-types="npm:@types/sanitize-html@2.9.0" export { default as sanitizeHtml } from 'npm:sanitize-html@^2.11.0'; export { diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index b337167..69cf1db 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -1,8 +1,8 @@ import { NSchema as n } from '@nostrify/nostrify'; +import { escape } from 'entities'; import { nip19, UnsignedEvent } from 'nostr-tools'; import { Conf } from '@/config.ts'; -import { lodash } from '@/deps.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getLnurl } from '@/utils/lnurl.ts'; import { nip05Cache } from '@/utils/nip05.ts'; @@ -53,7 +53,7 @@ async function renderAccount( header_static: banner, last_status_at: null, locked: false, - note: lodash.escape(about), + note: about ? escape(about) : '', roles: [], source: withSource ? { From d4a029b35b26e513715c9c0dedd8d297f7bbd528 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 12:44:54 -0500 Subject: [PATCH 55/65] sanitize-html -> isomorphic-dompurify --- deno.json | 1 + src/deps.ts | 2 -- src/utils/unfurl.ts | 10 ++++------ 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/deno.json b/deno.json index 1ebc35e..316c65b 100644 --- a/deno.json +++ b/deno.json @@ -43,6 +43,7 @@ "hono": "https://deno.land/x/hono@v3.10.1/mod.ts", "hono/middleware": "https://deno.land/x/hono@v3.10.1/middleware.ts", "iso-639-1": "npm:iso-639-1@2.1.15", + "isomorphic-dompurify": "npm:isomorphic-dompurify@^2.11.0", "kysely": "npm:kysely@^0.27.3", "kysely_deno_postgres": "https://deno.land/x/kysely_deno_postgres@v0.4.0/mod.ts", "linkify-plugin-hashtag": "npm:linkify-plugin-hashtag@^4.1.1", diff --git a/src/deps.ts b/src/deps.ts index dc9d912..ae96034 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,6 +1,4 @@ import 'deno-safe-fetch'; -// @deno-types="npm:@types/sanitize-html@2.9.0" -export { default as sanitizeHtml } from 'npm:sanitize-html@^2.11.0'; export { type ParsedSignature, pemToPublicKey, diff --git a/src/utils/unfurl.ts b/src/utils/unfurl.ts index b028be5..e5ae428 100644 --- a/src/utils/unfurl.ts +++ b/src/utils/unfurl.ts @@ -1,8 +1,8 @@ import TTLCache from '@isaacs/ttlcache'; import Debug from '@soapbox/stickynotes/debug'; +import DOMPurify from 'isomorphic-dompurify'; import { unfurl } from 'unfurl.js'; -import { sanitizeHtml } from '@/deps.ts'; import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; @@ -44,11 +44,9 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise Date: Tue, 21 May 2024 12:51:19 -0500 Subject: [PATCH 56/65] Remove ActivityPub actor endpoint, remove deps.ts --- deno.json | 3 +- src/app.ts | 3 -- src/controllers/activitypub/actor.ts | 28 ---------------- src/deps.ts | 12 ------- src/server.ts | 2 ++ src/utils/rsa.ts | 32 ------------------- src/views/activitypub/actor.ts | 48 ---------------------------- src/workers/policy.worker.ts | 2 +- src/workers/trends.worker.ts | 2 +- 9 files changed, 6 insertions(+), 126 deletions(-) delete mode 100644 src/controllers/activitypub/actor.ts delete mode 100644 src/deps.ts delete mode 100644 src/utils/rsa.ts delete mode 100644 src/views/activitypub/actor.ts diff --git a/deno.json b/deno.json index 316c65b..75ea333 100644 --- a/deno.json +++ b/deno.json @@ -36,7 +36,8 @@ "@std/media-types": "jsr:@std/media-types@^0.224.1", "@std/streams": "jsr:@std/streams@^0.223.0", "comlink": "npm:comlink@^4.4.1", - "deno-safe-fetch": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", + "deno-safe-fetch/load": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", + "deno-sqlite": "https://raw.githubusercontent.com/alexgleason/deno-sqlite/325f66d8c395e7f6f5ee78ebfa42a0eeea4a942b/mod.ts", "entities": "npm:entities@^4.5.0", "fast-stable-stringify": "npm:fast-stable-stringify@^1.0.0", "formdata-helper": "npm:formdata-helper@^0.3.0", diff --git a/src/app.ts b/src/app.ts index 5300b48..e7862ab 100644 --- a/src/app.ts +++ b/src/app.ts @@ -7,7 +7,6 @@ import { Conf } from '@/config.ts'; import { startFirehose } from '@/firehose.ts'; import { Time } from '@/utils.ts'; -import { actorController } from '@/controllers/activitypub/actor.ts'; import { accountController, accountLookupController, @@ -142,8 +141,6 @@ app.get('/.well-known/host-meta', hostMetaController); app.get('/.well-known/nodeinfo', nodeInfoController); app.get('/.well-known/nostr.json', nostrController); -app.get('/users/:username', actorController); - app.get('/nodeinfo/:version', nodeInfoSchemaController); app.get('/api/v1/instance', cacheMiddleware({ cacheName: 'web', expires: Time.minutes(5) }), instanceController); diff --git a/src/controllers/activitypub/actor.ts b/src/controllers/activitypub/actor.ts deleted file mode 100644 index 19f5f10..0000000 --- a/src/controllers/activitypub/actor.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { getAuthor } from '@/queries.ts'; -import { activityJson } from '@/utils/api.ts'; -import { renderActor } from '@/views/activitypub/actor.ts'; -import { localNip05Lookup } from '@/utils/nip05.ts'; - -import type { AppContext, AppController } from '@/app.ts'; - -const actorController: AppController = async (c) => { - const username = c.req.param('username'); - const { signal } = c.req.raw; - - const pointer = await localNip05Lookup(c.get('store'), username); - if (!pointer) return notFound(c); - - const event = await getAuthor(pointer.pubkey, { signal }); - if (!event) return notFound(c); - - const actor = await renderActor(event, username); - if (!actor) return notFound(c); - - return activityJson(c, actor); -}; - -function notFound(c: AppContext) { - return c.json({ error: 'Not found' }, 404); -} - -export { actorController }; diff --git a/src/deps.ts b/src/deps.ts deleted file mode 100644 index ae96034..0000000 --- a/src/deps.ts +++ /dev/null @@ -1,12 +0,0 @@ -import 'deno-safe-fetch'; -export { - type ParsedSignature, - pemToPublicKey, - publicKeyToPem, - signRequest, - verifyRequest, -} from 'https://gitlab.com/soapbox-pub/fedisign/-/raw/v0.2.1/mod.ts'; -export { generateSeededRsa } from 'https://gitlab.com/soapbox-pub/seeded-rsa/-/raw/v1.0.0/mod.ts'; -export { - DB as Sqlite, -} from 'https://raw.githubusercontent.com/alexgleason/deno-sqlite/325f66d8c395e7f6f5ee78ebfa42a0eeea4a942b/mod.ts'; diff --git a/src/server.ts b/src/server.ts index 4825e99..f7a33dc 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,3 +1,5 @@ +import 'deno-safe-fetch/load'; + import '@/precheck.ts'; import '@/sentry.ts'; import '@/nostr-wasm.ts'; diff --git a/src/utils/rsa.ts b/src/utils/rsa.ts deleted file mode 100644 index 6942c43..0000000 --- a/src/utils/rsa.ts +++ /dev/null @@ -1,32 +0,0 @@ -import * as secp from '@noble/secp256k1'; -import { LRUCache } from 'lru-cache'; - -import { Conf } from '@/config.ts'; -import { generateSeededRsa, publicKeyToPem } from '@/deps.ts'; - -const opts = { - bits: 2048, -}; - -const rsaCache = new LRUCache>({ max: 1000 }); - -async function buildSeed(pubkey: string): Promise { - const key = await Conf.cryptoKey; - const data = new TextEncoder().encode(pubkey); - const signature = await window.crypto.subtle.sign('HMAC', key, data); - return secp.etc.bytesToHex(new Uint8Array(signature)); -} - -async function getPublicKeyPem(pubkey: string): Promise { - const cached = await rsaCache.get(pubkey); - if (cached) return cached; - - const seed = await buildSeed(pubkey); - const { publicKey } = await generateSeededRsa(seed, opts); - const promise = publicKeyToPem(publicKey); - - rsaCache.set(pubkey, promise); - return promise; -} - -export { getPublicKeyPem }; diff --git a/src/views/activitypub/actor.ts b/src/views/activitypub/actor.ts deleted file mode 100644 index cfd40ba..0000000 --- a/src/views/activitypub/actor.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { NSchema as n } from '@nostrify/nostrify'; - -import { Conf } from '@/config.ts'; -import { getPublicKeyPem } from '@/utils/rsa.ts'; - -import type { NostrEvent } from '@nostrify/nostrify'; -import type { Actor } from '@/schemas/activitypub.ts'; - -/** Nostr metadata event to ActivityPub actor. */ -async function renderActor(event: NostrEvent, username: string): Promise { - const content = n.json().pipe(n.metadata()).catch({}).parse(event.content); - - return { - type: 'Person', - id: Conf.local(`/users/${username}`), - name: content?.name || '', - preferredUsername: username, - inbox: Conf.local(`/users/${username}/inbox`), - followers: Conf.local(`/users/${username}/followers`), - following: Conf.local(`/users/${username}/following`), - outbox: Conf.local(`/users/${username}/outbox`), - icon: content.picture - ? { - type: 'Image', - url: content.picture, - } - : undefined, - image: content.banner - ? { - type: 'Image', - url: content.banner, - } - : undefined, - summary: content.about ?? '', - attachment: [], - tag: [], - publicKey: { - id: Conf.local(`/users/${username}#main-key`), - owner: Conf.local(`/users/${username}`), - publicKeyPem: await getPublicKeyPem(event.pubkey), - }, - endpoints: { - sharedInbox: Conf.local('/inbox'), - }, - }; -} - -export { renderActor }; diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 4e4bcae..d1368bc 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -1,4 +1,4 @@ -import 'deno-safe-fetch'; +import 'deno-safe-fetch/load'; import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { ReadOnlyPolicy } from '@nostrify/nostrify/policies'; import * as Comlink from 'comlink'; diff --git a/src/workers/trends.worker.ts b/src/workers/trends.worker.ts index 33fd1a1..74a256b 100644 --- a/src/workers/trends.worker.ts +++ b/src/workers/trends.worker.ts @@ -1,7 +1,7 @@ import { NSchema } from '@nostrify/nostrify'; import * as Comlink from 'comlink'; +import { DB as Sqlite } from 'deno-sqlite'; -import { Sqlite } from '@/deps.ts'; import { hashtagSchema } from '@/schema.ts'; import { generateDateRange, Time } from '@/utils/time.ts'; From 938e26e2a0eedb45606f8c972f3eb7e979eb904d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 12:54:10 -0500 Subject: [PATCH 57/65] Remove webfinger and host-meta --- src/app.ts | 4 - src/controllers/well-known/host-meta.ts | 20 ----- src/controllers/well-known/webfinger.ts | 97 ------------------------- src/schemas/webfinger.ts | 19 ----- 4 files changed, 140 deletions(-) delete mode 100644 src/controllers/well-known/host-meta.ts delete mode 100644 src/controllers/well-known/webfinger.ts delete mode 100644 src/schemas/webfinger.ts diff --git a/src/app.ts b/src/app.ts index e7862ab..be2ce84 100644 --- a/src/app.ts +++ b/src/app.ts @@ -76,10 +76,8 @@ import { } from '@/controllers/api/timelines.ts'; import { trendingTagsController } from '@/controllers/api/trends.ts'; import { indexController } from '@/controllers/site.ts'; -import { hostMetaController } from '@/controllers/well-known/host-meta.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; -import { webfingerController } from '@/controllers/well-known/webfinger.ts'; import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cacheMiddleware } from '@/middleware/cacheMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; @@ -136,8 +134,6 @@ app.use( storeMiddleware, ); -app.get('/.well-known/webfinger', webfingerController); -app.get('/.well-known/host-meta', hostMetaController); app.get('/.well-known/nodeinfo', nodeInfoController); app.get('/.well-known/nostr.json', nostrController); diff --git a/src/controllers/well-known/host-meta.ts b/src/controllers/well-known/host-meta.ts deleted file mode 100644 index 85da11b..0000000 --- a/src/controllers/well-known/host-meta.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Conf } from '@/config.ts'; - -import type { AppController } from '@/app.ts'; - -/** https://datatracker.ietf.org/doc/html/rfc6415 */ -const hostMetaController: AppController = (c) => { - const template = Conf.local('/.well-known/webfinger?resource={uri}'); - - c.header('content-type', 'application/xrd+xml'); - - return c.body( - ` - - - -`, - ); -}; - -export { hostMetaController }; diff --git a/src/controllers/well-known/webfinger.ts b/src/controllers/well-known/webfinger.ts deleted file mode 100644 index c1c8b81..0000000 --- a/src/controllers/well-known/webfinger.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { nip19 } from 'nostr-tools'; -import { z } from 'zod'; - -import { Conf } from '@/config.ts'; -import { localNip05Lookup } from '@/utils/nip05.ts'; - -import type { AppContext, AppController } from '@/app.ts'; -import type { Webfinger } from '@/schemas/webfinger.ts'; - -const webfingerQuerySchema = z.object({ - resource: z.string().url(), -}); - -const webfingerController: AppController = (c) => { - const query = webfingerQuerySchema.safeParse(c.req.query()); - if (!query.success) { - return c.json({ error: 'Bad request', schema: query.error }, 400); - } - - const resource = new URL(query.data.resource); - - switch (resource.protocol) { - case 'acct:': { - return handleAcct(c, resource); - } - default: - return c.json({ error: 'Unsupported URI scheme' }, 400); - } -}; - -/** Transforms the resource URI into a `[username, domain]` tuple. */ -const acctSchema = z.custom((value) => value instanceof URL) - .transform((uri) => uri.pathname) - .pipe(z.string().email('Invalid acct')) - .transform((acct) => acct.split('@') as [username: string, host: string]) - .refine(([_username, host]) => host === Conf.url.hostname, { - message: 'Host must be local', - path: ['resource', 'acct'], - }); - -async function handleAcct(c: AppContext, resource: URL): Promise { - const result = acctSchema.safeParse(resource); - if (!result.success) { - return c.json({ error: 'Invalid acct URI', schema: result.error }, 400); - } - - const [username, host] = result.data; - const pointer = await localNip05Lookup(c.get('store'), username); - - if (!pointer) { - return c.json({ error: 'Not found' }, 404); - } - - const json = renderWebfinger({ - pubkey: pointer.pubkey, - username, - subject: `acct:${username}@${host}`, - }); - - c.header('content-type', 'application/jrd+json'); - return c.body(JSON.stringify(json)); -} - -interface RenderWebfingerOpts { - pubkey: string; - username: string; - subject: string; -} - -/** Present Nostr user on Webfinger. */ -function renderWebfinger({ pubkey, username, subject }: RenderWebfingerOpts): Webfinger { - const apId = Conf.local(`/users/${username}`); - - return { - subject, - aliases: [apId], - links: [ - { - rel: 'self', - type: 'application/activity+json', - href: apId, - }, - { - rel: 'self', - type: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - href: apId, - }, - { - rel: 'self', - type: 'application/nostr+json', - href: `nostr:${nip19.npubEncode(pubkey)}`, - }, - ], - }; -} - -export { webfingerController }; diff --git a/src/schemas/webfinger.ts b/src/schemas/webfinger.ts deleted file mode 100644 index 8c9cf57..0000000 --- a/src/schemas/webfinger.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { z } from 'zod'; - -const linkSchema = z.object({ - rel: z.string().optional(), - type: z.string().optional(), - href: z.string().optional(), - template: z.string().optional(), -}); - -const webfingerSchema = z.object({ - subject: z.string(), - aliases: z.array(z.string()).catch([]), - links: z.array(linkSchema), -}); - -type Webfinger = z.infer; - -export { webfingerSchema }; -export type { Webfinger }; From f30aad11a5281e7599434486818f62ddc6177ac3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 13:04:23 -0500 Subject: [PATCH 58/65] Fix legacy quote posts --- src/storages/hydrate.ts | 5 +++-- src/tags.ts | 11 +++++++++-- src/views/mastodon/statuses.ts | 12 ++++++------ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index e5c488e..f55d741 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -6,6 +6,7 @@ import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; import { refreshAuthorStatsDebounced } from '@/stats.ts'; +import { findQuoteTag } from '@/tags.ts'; interface HydrateOpts { events: DittoEvent[]; @@ -81,7 +82,7 @@ function assembleEvents( event.user = b.find((e) => matchFilter({ kinds: [30361], authors: [admin], '#d': [event.pubkey] }, e)); if (event.kind === 1) { - const id = event.tags.find(([name]) => name === 'q')?.[1]; + const id = findQuoteTag(event.tags)?.[1]; if (id) { event.quote = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); } @@ -169,7 +170,7 @@ function gatherQuotes({ events, store, signal }: HydrateOpts): Promise name === 'q')?.[1]; + const id = findQuoteTag(event.tags)?.[1]; if (id) { ids.add(id); } diff --git a/src/tags.ts b/src/tags.ts index a683393..ecddaf4 100644 --- a/src/tags.ts +++ b/src/tags.ts @@ -35,8 +35,15 @@ const isReplyTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'reply'; const isRootTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'root'; const isLegacyReplyTag = (tag: string[]) => tag[0] === 'e' && !tag[3]; -function findReplyTag(tags: string[][]) { +const isQuoteTag = (tag: string[]) => tag[0] === 'q'; +const isLegacyQuoteTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'mention'; + +function findReplyTag(tags: string[][]): string[] | undefined { return tags.find(isReplyTag) || tags.find(isRootTag) || tags.findLast(isLegacyReplyTag); } -export { addTag, deleteTag, findReplyTag, getTagSet, hasTag }; +function findQuoteTag(tags: string[][]): string[] | undefined { + return tags.find(isQuoteTag) || tags.find(isLegacyQuoteTag); +} + +export { addTag, deleteTag, findQuoteTag, findReplyTag, getTagSet, hasTag }; diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index a06aac2..1571977 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -1,11 +1,10 @@ import { NostrEvent } from '@nostrify/nostrify'; -import { isCWTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts'; import { nip19 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { Storages } from '@/storages.ts'; -import { findReplyTag } from '@/tags.ts'; +import { findQuoteTag, findReplyTag } from '@/tags.ts'; import { nostrDate } from '@/utils.ts'; import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; @@ -30,6 +29,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< : await accountFromPubkey(event.pubkey); const replyTag = findReplyTag(event.tags); + const quoteTag = findQuoteTag(event.tags); const mentionedPubkeys = [ ...new Set( @@ -73,8 +73,8 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const content = buildInlineRecipients(mentions) + html; - const cw = event.tags.find(isCWTag); - const subject = event.tags.find((tag) => tag[0] === 'subject'); + const cw = event.tags.find(([name]) => name === 'content-warning'); + const subject = event.tags.find(([name]) => name === 'subject'); const imeta: string[][][] = event.tags .filter(([name]) => name === 'imeta') @@ -88,7 +88,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< card, content, created_at: nostrDate(event.created_at).toISOString(), - in_reply_to_id: replyTag ? replyTag[1] : null, + in_reply_to_id: replyTag?.[1] ?? null, in_reply_to_account_id: null, sensitive: !!cw, spoiler_text: (cw ? cw[1] : subject?.[1]) || '', @@ -110,7 +110,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< emojis: renderEmojis(event), poll: null, quote: !event.quote ? null : await renderStatus(event.quote, { depth: depth + 1 }), - quote_id: event.tags.find(([name]) => name === 'q')?.[1] ?? null, + quote_id: quoteTag?.[1] ?? null, uri: Conf.external(note), url: Conf.external(note), zapped: Boolean(zapEvent), From 9839b8138f36baa55240a38748049f67bc1b0c08 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 13:08:08 -0500 Subject: [PATCH 59/65] tags.ts -> utils/tags.ts --- src/controllers/api/accounts.ts | 2 +- src/controllers/api/admin.ts | 2 +- src/controllers/api/bookmarks.ts | 2 +- src/controllers/api/mutes.ts | 2 +- src/controllers/api/statuses.ts | 8 ++++---- src/controllers/api/suggestions.ts | 2 +- src/pipeline.ts | 2 +- src/policies/MuteListPolicy.ts | 2 +- src/queries.ts | 2 +- src/stats.ts | 2 +- src/storages/EventsDB.ts | 4 ++-- src/storages/UserStore.ts | 2 +- src/storages/hydrate.ts | 2 +- src/{ => utils}/tags.ts | 0 src/views/mastodon/relationships.ts | 2 +- src/views/mastodon/statuses.ts | 2 +- 16 files changed, 19 insertions(+), 19 deletions(-) rename src/{ => utils}/tags.ts (100%) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index f717be3..2d61adc 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -7,7 +7,6 @@ import { Conf } from '@/config.ts'; import { getAuthor, getFollowedPubkeys } from '@/queries.ts'; import { booleanParamSchema, fileSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; -import { addTag, deleteTag, findReplyTag, getTagSet } from '@/tags.ts'; import { uploadFile } from '@/utils/upload.ts'; import { nostrNow } from '@/utils.ts'; import { createEvent, paginated, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; @@ -18,6 +17,7 @@ import { renderRelationship } from '@/views/mastodon/relationships.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { bech32ToPubkey } from '@/utils.ts'; +import { addTag, deleteTag, findReplyTag, getTagSet } from '@/utils/tags.ts'; const usernameSchema = z .string().min(1).max(30) diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index 77571aa..d7cd365 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -5,8 +5,8 @@ import { Conf } from '@/config.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { booleanParamSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; -import { addTag } from '@/tags.ts'; import { paginated, paginationSchema, parseBody, updateListAdminEvent } from '@/utils/api.ts'; +import { addTag } from '@/utils/tags.ts'; import { renderAdminAccount } from '@/views/mastodon/admin-accounts.ts'; const adminAccountQuerySchema = z.object({ diff --git a/src/controllers/api/bookmarks.ts b/src/controllers/api/bookmarks.ts index 7655182..6d80b50 100644 --- a/src/controllers/api/bookmarks.ts +++ b/src/controllers/api/bookmarks.ts @@ -1,6 +1,6 @@ import { type AppController } from '@/app.ts'; import { Storages } from '@/storages.ts'; -import { getTagSet } from '@/tags.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { renderStatuses } from '@/views.ts'; /** https://docs.joinmastodon.org/methods/bookmarks/#get */ diff --git a/src/controllers/api/mutes.ts b/src/controllers/api/mutes.ts index 4afb6c4..31f54ee 100644 --- a/src/controllers/api/mutes.ts +++ b/src/controllers/api/mutes.ts @@ -1,6 +1,6 @@ import { type AppController } from '@/app.ts'; import { Storages } from '@/storages.ts'; -import { getTagSet } from '@/tags.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { renderAccounts } from '@/views.ts'; /** https://docs.joinmastodon.org/methods/mutes/#get */ diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 291d970..e9c872f 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -8,15 +8,15 @@ import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; -import { addTag, deleteTag } from '@/tags.ts'; -import { createEvent, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; import { renderEventAccounts } from '@/views.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; -import { getLnurl } from '@/utils/lnurl.ts'; -import { asyncReplaceAll } from '@/utils/text.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { createEvent, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; +import { getLnurl } from '@/utils/lnurl.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; +import { addTag, deleteTag } from '@/utils/tags.ts'; +import { asyncReplaceAll } from '@/utils/text.ts'; const createStatusSchema = z.object({ in_reply_to_id: z.string().regex(/[0-9a-f]{64}/).nullish(), diff --git a/src/controllers/api/suggestions.ts b/src/controllers/api/suggestions.ts index 6377bd4..012244a 100644 --- a/src/controllers/api/suggestions.ts +++ b/src/controllers/api/suggestions.ts @@ -2,8 +2,8 @@ import { NStore } from '@nostrify/nostrify'; import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { getTagSet } from '@/tags.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; export const suggestionsV1Controller: AppController = async (c) => { diff --git a/src/pipeline.ts b/src/pipeline.ts index 15d495e..bfb0577 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -13,7 +13,6 @@ import { RelayError } from '@/RelayError.ts'; import { updateStats } from '@/stats.ts'; import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; -import { getTagSet } from '@/tags.ts'; import { eventAge, nostrDate, nostrNow, parseNip05, Time } from '@/utils.ts'; import { fetchWorker } from '@/workers/fetch.ts'; import { policyWorker } from '@/workers/policy.ts'; @@ -22,6 +21,7 @@ import { verifyEventWorker } from '@/workers/verify.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { lnurlCache } from '@/utils/lnurl.ts'; import { nip05Cache } from '@/utils/nip05.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; diff --git a/src/policies/MuteListPolicy.ts b/src/policies/MuteListPolicy.ts index cae08eb..130d10d 100644 --- a/src/policies/MuteListPolicy.ts +++ b/src/policies/MuteListPolicy.ts @@ -1,6 +1,6 @@ import { NostrEvent, NostrRelayOK, NPolicy, NStore } from '@nostrify/nostrify'; -import { getTagSet } from '@/tags.ts'; +import { getTagSet } from '@/utils/tags.ts'; export class MuteListPolicy implements NPolicy { constructor(private pubkey: string, private store: NStore) {} diff --git a/src/queries.ts b/src/queries.ts index 76fabfd..6a197ea 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -5,8 +5,8 @@ import { Conf } from '@/config.ts'; import { Storages } from '@/storages.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { type DittoRelation } from '@/interfaces/DittoFilter.ts'; -import { findReplyTag, getTagSet } from '@/tags.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { findReplyTag, getTagSet } from '@/utils/tags.ts'; const debug = Debug('ditto:queries'); diff --git a/src/stats.ts b/src/stats.ts index 256c570..6ffe5f7 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -8,7 +8,7 @@ import { SetRequired } from 'type-fest'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Storages } from '@/storages.ts'; -import { findReplyTag, getTagSet } from '@/tags.ts'; +import { findReplyTag, getTagSet } from '@/utils/tags.ts'; type AuthorStat = keyof Omit; type EventStat = keyof Omit; diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 5a3839a..a550f39 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -7,11 +7,11 @@ import { Kysely } from 'kysely'; import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { normalizeFilters } from '@/filter.ts'; +import { RelayError } from '@/RelayError.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; -import { getTagSet } from '@/tags.ts'; import { isNostrId, isURL } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; -import { RelayError } from '@/RelayError.ts'; +import { getTagSet } from '@/utils/tags.ts'; /** Function to decide whether or not to index a tag. */ type TagCondition = ({ event, count, value }: { diff --git a/src/storages/UserStore.ts b/src/storages/UserStore.ts index c5657b6..43c1771 100644 --- a/src/storages/UserStore.ts +++ b/src/storages/UserStore.ts @@ -1,7 +1,7 @@ import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { getTagSet } from '@/tags.ts'; +import { getTagSet } from '@/utils/tags.ts'; export class UserStore implements NStore { constructor(private pubkey: string, private store: NStore) {} diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index f55d741..68dc0bd 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -6,7 +6,7 @@ import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; import { refreshAuthorStatsDebounced } from '@/stats.ts'; -import { findQuoteTag } from '@/tags.ts'; +import { findQuoteTag } from '@/utils/tags.ts'; interface HydrateOpts { events: DittoEvent[]; diff --git a/src/tags.ts b/src/utils/tags.ts similarity index 100% rename from src/tags.ts rename to src/utils/tags.ts diff --git a/src/views/mastodon/relationships.ts b/src/views/mastodon/relationships.ts index 2f8ffdd..425ea56 100644 --- a/src/views/mastodon/relationships.ts +++ b/src/views/mastodon/relationships.ts @@ -1,5 +1,5 @@ import { Storages } from '@/storages.ts'; -import { hasTag } from '@/tags.ts'; +import { hasTag } from '@/utils/tags.ts'; async function renderRelationship(sourcePubkey: string, targetPubkey: string) { const db = await Storages.db(); diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 1571977..cc7cc36 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -4,9 +4,9 @@ import { nip19 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { Storages } from '@/storages.ts'; -import { findQuoteTag, findReplyTag } from '@/tags.ts'; import { nostrDate } from '@/utils.ts'; import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; +import { findQuoteTag, findReplyTag } from '@/utils/tags.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderAttachment } from '@/views/mastodon/attachments.ts'; From 5e607f664e2009ed8d25172609af6ad54caf8b0b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 13:32:27 -0500 Subject: [PATCH 60/65] Add tags test --- src/utils/tags.test.ts | 45 ++++++++++++++++++++++++++++++++++++++++++ src/utils/tags.ts | 36 ++++++++++++++++++++++++++------- 2 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 src/utils/tags.test.ts diff --git a/src/utils/tags.test.ts b/src/utils/tags.test.ts new file mode 100644 index 0000000..43e1235 --- /dev/null +++ b/src/utils/tags.test.ts @@ -0,0 +1,45 @@ +import { assertEquals } from '@std/assert'; + +import { addTag, deleteTag, findQuoteTag, findReplyTag, getTagSet, hasTag } from './tags.ts'; + +Deno.test('addTag', () => { + const tags = [['p', 'alex']]; + assertEquals(addTag(tags, ['p', 'alex']), [['p', 'alex']]); + assertEquals(addTag(tags, ['p', 'fiatjaf']), [['p', 'alex'], ['p', 'fiatjaf']]); +}); + +Deno.test('deleteTag', () => { + const tags = [['p', 'alex'], ['p', 'fiatjaf']]; + assertEquals(deleteTag(tags, ['p', 'alex']), [['p', 'fiatjaf']]); + assertEquals(deleteTag(tags, ['p', 'fiatjaf']), [['p', 'alex']]); +}); + +Deno.test('findQuoteTag', () => { + assertEquals(findQuoteTag([['q', '123']]), ['q', '123']); + assertEquals(findQuoteTag([['e', '', '', 'mention', '456']]), ['e', '', '', 'mention', '456']); + assertEquals(findQuoteTag([['e', '', '', 'mention', '456'], ['q', '123']]), ['q', '123']); + assertEquals(findQuoteTag([['q', '123'], ['e', '', '', 'mention', '456']]), ['q', '123']); +}); + +Deno.test('findReplyTag', () => { + const root = ['e', '123', '', 'root']; + const reply = ['e', '456', '', 'reply']; + + assertEquals(findReplyTag([root]), root); + assertEquals(findReplyTag([reply]), reply); + assertEquals(findReplyTag([root, reply]), reply); + assertEquals(findReplyTag([reply, root]), reply); + assertEquals(findReplyTag([['e', '321'], ['e', '789']]), ['e', '789']); + assertEquals(findReplyTag([reply, ['e', '789']]), reply); +}); + +Deno.test('getTagSet', () => { + const tags = [['p', 'alex'], ['p', 'fiatjaf'], ['p', 'alex']]; + assertEquals(getTagSet(tags, 'p'), new Set(['alex', 'fiatjaf'])); +}); + +Deno.test('hasTag', () => { + const tags = [['p', 'alex']]; + assertEquals(hasTag(tags, ['p', 'alex']), true); + assertEquals(hasTag(tags, ['p', 'fiatjaf']), false); +}); diff --git a/src/utils/tags.ts b/src/utils/tags.ts index ecddaf4..6375e81 100644 --- a/src/utils/tags.ts +++ b/src/utils/tags.ts @@ -31,18 +31,40 @@ function addTag(tags: readonly string[][], tag: string[]): string[][] { } } -const isReplyTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'reply'; -const isRootTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'root'; -const isLegacyReplyTag = (tag: string[]) => tag[0] === 'e' && !tag[3]; +/** Tag is a NIP-10 root tag. */ +function isRootTag(tag: string[]): tag is ['e', string, string, 'root', ...string[]] { + return tag[0] === 'e' && tag[3] === 'root'; +} -const isQuoteTag = (tag: string[]) => tag[0] === 'q'; -const isLegacyQuoteTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'mention'; +/** Tag is a NIP-10 reply tag. */ +function isReplyTag(tag: string[]): tag is ['e', string, string, 'reply', ...string[]] { + return tag[0] === 'e' && tag[3] === 'reply'; +} -function findReplyTag(tags: string[][]): string[] | undefined { +/** Tag is a legacy "e" tag with a "mention" marker. */ +function isLegacyQuoteTag(tag: string[]): tag is ['e', string, string, 'mention', ...string[]] { + return tag[0] === 'e' && tag[3] === 'mention'; +} + +/** Tag is an "e" tag without a NIP-10 marker. */ +function isLegacyReplyTag(tag: string[]): tag is ['e', string, string] { + return tag[0] === 'e' && !tag[3]; +} + +/** Tag is a "q" tag. */ +function isQuoteTag(tag: string[]): tag is ['q', ...string[]] { + return tag[0] === 'q'; +} + +/** Get the "e" tag for the event being replied to, first according to the NIPs then falling back to the legacy way. */ +function findReplyTag(tags: string[][]): ['e', ...string[]] | undefined { return tags.find(isReplyTag) || tags.find(isRootTag) || tags.findLast(isLegacyReplyTag); } -function findQuoteTag(tags: string[][]): string[] | undefined { +/** Get the "q" tag, falling back to the legacy "e" tag with a "mention" marker. */ +function findQuoteTag( + tags: string[][], +): ['q', ...string[]] | ['e', string, string, 'mention', ...string[]] | undefined { return tags.find(isQuoteTag) || tags.find(isLegacyQuoteTag); } From 9873afab699b6c6ef24ce253abcf38e668bcbd7f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 13:37:24 -0500 Subject: [PATCH 61/65] Remove old tags.test.ts --- src/tags.test.ts | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 src/tags.test.ts diff --git a/src/tags.test.ts b/src/tags.test.ts deleted file mode 100644 index e49d31a..0000000 --- a/src/tags.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { assertEquals } from '@std/assert'; - -import { addTag, deleteTag, getTagSet } from './tags.ts'; - -Deno.test('getTagSet', () => { - assertEquals(getTagSet([], 'p'), new Set()); - assertEquals(getTagSet([['p', '123']], 'p'), new Set(['123'])); - assertEquals(getTagSet([['p', '123'], ['p', '456']], 'p'), new Set(['123', '456'])); - assertEquals(getTagSet([['p', '123'], ['p', '456'], ['q', '789']], 'p'), new Set(['123', '456'])); -}); - -Deno.test('addTag', () => { - assertEquals(addTag([], ['p', '123']), [['p', '123']]); - assertEquals(addTag([['p', '123']], ['p', '123']), [['p', '123']]); - assertEquals(addTag([['p', '123'], ['p', '456']], ['p', '123']), [['p', '123'], ['p', '456']]); - assertEquals(addTag([['p', '123'], ['p', '456']], ['p', '789']), [['p', '123'], ['p', '456'], ['p', '789']]); -}); - -Deno.test('deleteTag', () => { - assertEquals(deleteTag([], ['p', '123']), []); - assertEquals(deleteTag([['p', '123']], ['p', '123']), []); - assertEquals(deleteTag([['p', '123']], ['p', '456']), [['p', '123']]); - assertEquals(deleteTag([['p', '123'], ['p', '123']], ['p', '123']), []); - assertEquals(deleteTag([['p', '123'], ['p', '456']], ['p', '456']), [['p', '123']]); -}); From bf6dc22c5a6d059e19393490e115344b5ee8829f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 14:05:48 -0500 Subject: [PATCH 62/65] Fix imeta stripping so it doesn't remove a bunch of newlines --- fixtures/events/event-imeta.json | 20 ++++++++++++++++++++ src/utils/note.test.ts | 13 ++++++++++++- src/utils/note.ts | 5 ++--- 3 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 fixtures/events/event-imeta.json diff --git a/fixtures/events/event-imeta.json b/fixtures/events/event-imeta.json new file mode 100644 index 0000000..4952ca8 --- /dev/null +++ b/fixtures/events/event-imeta.json @@ -0,0 +1,20 @@ +{ + "id": "1264cc4051db59af9a21f7fd001fdf5213424f558ea9ab16a1b014fca2250af5", + "pubkey": "6be38f8c63df7dbf84db7ec4a6e6fbbd8d19dca3b980efad18585c46f04b26f9", + "created_at": 1716306470, + "kind": 1, + "tags": [ + [ + "imeta", + "url https://image.nostr.build/258d978b91e7424cfa43b31f3cfc077d7172ae10b3b45ac956feff9e72175126.png", + "m image/png", + "x b1ceee58405ef05a41190a0946ca6b6511dff426c68013cdd165514c1ef301f9", + "ox 258d978b91e7424cfa43b31f3cfc077d7172ae10b3b45ac956feff9e72175126", + "size 114350", + "dim 1414x594", + "blurhash LDRfkC.8_4_N_3NGR*t8%gIVWBxt" + ] + ], + "content": "Today we were made aware of multiple Fediverse blog posts incorrectly attributing “vote Trump” spam on Bluesky to the Mostr.pub Bridge. \n\nThis spam is NOT coming from Mostr. From the screenshots used in these blogs, it's clear the spam is coming from an entirely different bridge called momostr.pink. This bridge is not affiliated with Mostr, and is not even a fork of Mostr. We appreciate that the authors of these posts responded quickly to us and have since corrected the blogs. \n\nMostr.pub uses stirfry policies for anti-spam filtering. This includes an anti-duplication policy that prevents spam like the recent “vote Trump” posts we’ve seen repeated over and over. \n\nIt is important to note WHY there are multiple bridges, though. \n\nWhen Mostr.pub launched, multiple major servers immediately blocked Mostr, including Mastodon.social. The moderators of Mastodon.social claimed that this was because Nostr was unregulated, and suggested to one user that if they want to bridge their account they should host their own bridge.\n\nThat is exactly what momostr.pink, the source of this spam, has done. \n\nThe obvious response to the censorship of the Mostr Bridge is to build more bridges. \n\nWhile we have opted for pro-social policies that aim to reduce spam and build better connections between decentralized platforms, other bridges built to get around censorship of the Mostr Bridge may not — as we’re already seeing.\n\nThere will inevitably be multiple bridges, and we’re working on creating solutions to the problems that arise from that. In the meantime, if the Fediverse could do itself a favor and chill with the censorship for two seconds, we might not have so many problems. \n\n\nhttps://image.nostr.build/258d978b91e7424cfa43b31f3cfc077d7172ae10b3b45ac956feff9e72175126.png", + "sig": "b950e6e2ff1dc786ef344e7dad3edf8aa315a1053ede146725bde181acf7c2c1a5fcf1e0c796552b743607d6ae161a3ff4eb3af5033ffbfd314e68213d315215" +} diff --git a/src/utils/note.test.ts b/src/utils/note.test.ts index 9c8fad7..b351dbf 100644 --- a/src/utils/note.test.ts +++ b/src/utils/note.test.ts @@ -1,6 +1,7 @@ import { assertEquals } from '@std/assert'; -import { getMediaLinks, parseNoteContent } from '@/utils/note.ts'; +import { eventFixture } from '@/test.ts'; +import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; Deno.test('parseNoteContent', () => { const { html, links, firstUrl } = parseNoteContent('Hello, world!'); @@ -22,3 +23,13 @@ Deno.test('getMediaLinks', () => { ['m', 'image/png'], ]]); }); + +Deno.test('stripimeta', async () => { + const { content, tags } = await eventFixture('event-imeta'); + + const stripped = stripimeta(content, tags); + const expected = + `Today we were made aware of multiple Fediverse blog posts incorrectly attributing “vote Trump” spam on Bluesky to the Mostr.pub Bridge. \n\nThis spam is NOT coming from Mostr. From the screenshots used in these blogs, it's clear the spam is coming from an entirely different bridge called momostr.pink. This bridge is not affiliated with Mostr, and is not even a fork of Mostr. We appreciate that the authors of these posts responded quickly to us and have since corrected the blogs. \n\nMostr.pub uses stirfry policies for anti-spam filtering. This includes an anti-duplication policy that prevents spam like the recent “vote Trump” posts we’ve seen repeated over and over. \n\nIt is important to note WHY there are multiple bridges, though. \n\nWhen Mostr.pub launched, multiple major servers immediately blocked Mostr, including Mastodon.social. The moderators of Mastodon.social claimed that this was because Nostr was unregulated, and suggested to one user that if they want to bridge their account they should host their own bridge.\n\nThat is exactly what momostr.pink, the source of this spam, has done. \n\nThe obvious response to the censorship of the Mostr Bridge is to build more bridges. \n\nWhile we have opted for pro-social policies that aim to reduce spam and build better connections between decentralized platforms, other bridges built to get around censorship of the Mostr Bridge may not — as we’re already seeing.\n\nThere will inevitably be multiple bridges, and we’re working on creating solutions to the problems that arise from that. In the meantime, if the Fediverse could do itself a favor and chill with the censorship for two seconds, we might not have so many problems. `; + + assertEquals(stripped, expected); +}); diff --git a/src/utils/note.ts b/src/utils/note.ts index 03da2de..0d1e3b2 100644 --- a/src/utils/note.ts +++ b/src/utils/note.ts @@ -71,10 +71,9 @@ function stripimeta(content: string, tags: string[][]): string { const lines = content.split('\n').reverse(); - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; + for (const line of [...lines]) { if (line === '' || urls.has(line)) { - lines.splice(i, 1); + lines.splice(0, 1); } else { break; } From fc325880d2270eaf4f779087502b5cc1b2b85fc2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 14:27:49 -0500 Subject: [PATCH 63/65] Add root tags to replies --- src/controllers/api/statuses.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index e9c872f..58526cb 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -76,12 +76,21 @@ const createStatusController: AppController = async (c) => { const tags: string[][] = []; - if (data.quote_id) { - tags.push(['q', data.quote_id]); + if (data.in_reply_to_id) { + const ancestor = await getEvent(data.in_reply_to_id); + + if (!ancestor) { + return c.json({ error: 'Original post not found.' }, 404); + } + + const root = ancestor.tags.find((tag) => tag[0] === 'e' && tag[3] === 'root')?.[1] ?? ancestor.id; + + tags.push(['e', root, 'root']); + tags.push(['e', data.in_reply_to_id, 'reply']); } - if (data.in_reply_to_id) { - tags.push(['e', data.in_reply_to_id, 'reply']); + if (data.quote_id) { + tags.push(['q', data.quote_id]); } if (data.sensitive && data.spoiler_text) { From 9ab38203dfa921c002715baed74386aaec061ddb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 14:37:18 -0500 Subject: [PATCH 64/65] getDescendants: filter out non-replies --- src/queries.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/queries.ts b/src/queries.ts index 6a197ea..7407077 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -88,7 +88,11 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi async function getDescendants(eventId: string, signal = AbortSignal.timeout(2000)): Promise { const store = await Storages.db(); - const events = await store.query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal }); + + const events = await store + .query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal }) + .then((events) => events.filter(({ tags }) => findReplyTag(tags)?.[1] === eventId)); + return hydrateEvents({ events, store, signal }); } From 101a16bc1200f74b395934ac7053aaacd3642314 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 21 May 2024 16:29:36 -0500 Subject: [PATCH 65/65] Index the Postgres FTS column --- src/db/migrations/021_pgfts_index.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/db/migrations/021_pgfts_index.ts diff --git a/src/db/migrations/021_pgfts_index.ts b/src/db/migrations/021_pgfts_index.ts new file mode 100644 index 0000000..d18d110 --- /dev/null +++ b/src/db/migrations/021_pgfts_index.ts @@ -0,0 +1,21 @@ +import { Kysely } from 'kysely'; + +import { Conf } from '@/config.ts'; + +export async function up(db: Kysely): Promise { + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + await db.schema + .createIndex('nostr_pgfts_gin_search_vec') + .ifNotExists() + .on('nostr_pgfts') + .using('gin') + .column('search_vec') + .execute(); + } +} + +export async function down(db: Kysely): Promise { + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + await db.schema.dropIndex('nostr_pgfts_gin_search_vec').ifExists().execute(); + } +}