diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9dee45..b2140db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.41.3 +image: denoland/deno:1.43.3 default: interruptible: true diff --git a/.tool-versions b/.tool-versions index a13fd5f..b3e19cd 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.41.3 \ No newline at end of file +deno 1.43.3 \ No newline at end of file diff --git a/deno.json b/deno.json index 25ec2ed..75ea333 100644 --- a/deno.json +++ b/deno.json @@ -11,7 +11,8 @@ "nsec": "deno run scripts/nsec.ts", "admin:event": "deno run -A scripts/admin-event.ts", "admin:role": "deno run -A scripts/admin-role.ts", - "stats:recompute": "deno run -A scripts/stats-recompute.ts" + "stats:recompute": "deno run -A scripts/stats-recompute.ts", + "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip" }, "unstable": ["ffi", "kv", "worker-options"], "exclude": ["./public"], @@ -20,8 +21,9 @@ "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", + "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.19.1", + "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.22.0", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", @@ -31,15 +33,18 @@ "@std/dotenv": "jsr:@std/dotenv@^0.224.0", "@std/encoding": "jsr:@std/encoding@^0.224.0", "@std/json": "jsr:@std/json@^0.223.0", - "@std/media-types": "jsr:@std/media-types@^0.224.0", + "@std/media-types": "jsr:@std/media-types@^0.224.1", "@std/streams": "jsr:@std/streams@^0.223.0", "comlink": "npm:comlink@^4.4.1", - "deno-safe-fetch": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", + "deno-safe-fetch/load": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", + "deno-sqlite": "https://raw.githubusercontent.com/alexgleason/deno-sqlite/325f66d8c395e7f6f5ee78ebfa42a0eeea4a942b/mod.ts", + "entities": "npm:entities@^4.5.0", "fast-stable-stringify": "npm:fast-stable-stringify@^1.0.0", "formdata-helper": "npm:formdata-helper@^0.3.0", "hono": "https://deno.land/x/hono@v3.10.1/mod.ts", "hono/middleware": "https://deno.land/x/hono@v3.10.1/middleware.ts", "iso-639-1": "npm:iso-639-1@2.1.15", + "isomorphic-dompurify": "npm:isomorphic-dompurify@^2.11.0", "kysely": "npm:kysely@^0.27.3", "kysely_deno_postgres": "https://deno.land/x/kysely_deno_postgres@v0.4.0/mod.ts", "linkify-plugin-hashtag": "npm:linkify-plugin-hashtag@^4.1.1", @@ -53,7 +58,6 @@ "tseep": "npm:tseep@^1.2.1", "type-fest": "npm:type-fest@^4.3.0", "unfurl.js": "npm:unfurl.js@^6.4.0", - "uuid62": "npm:uuid62@^1.0.2", "zod": "npm:zod@^3.23.5", "~/fixtures/": "./fixtures/" }, diff --git a/fixtures/events/event-imeta.json b/fixtures/events/event-imeta.json new file mode 100644 index 0000000..4952ca8 --- /dev/null +++ b/fixtures/events/event-imeta.json @@ -0,0 +1,20 @@ +{ + "id": "1264cc4051db59af9a21f7fd001fdf5213424f558ea9ab16a1b014fca2250af5", + "pubkey": "6be38f8c63df7dbf84db7ec4a6e6fbbd8d19dca3b980efad18585c46f04b26f9", + "created_at": 1716306470, + "kind": 1, + "tags": [ + [ + "imeta", + "url https://image.nostr.build/258d978b91e7424cfa43b31f3cfc077d7172ae10b3b45ac956feff9e72175126.png", + "m image/png", + "x b1ceee58405ef05a41190a0946ca6b6511dff426c68013cdd165514c1ef301f9", + "ox 258d978b91e7424cfa43b31f3cfc077d7172ae10b3b45ac956feff9e72175126", + "size 114350", + "dim 1414x594", + "blurhash LDRfkC.8_4_N_3NGR*t8%gIVWBxt" + ] + ], + "content": "Today we were made aware of multiple Fediverse blog posts incorrectly attributing “vote Trump” spam on Bluesky to the Mostr.pub Bridge. \n\nThis spam is NOT coming from Mostr. From the screenshots used in these blogs, it's clear the spam is coming from an entirely different bridge called momostr.pink. This bridge is not affiliated with Mostr, and is not even a fork of Mostr. We appreciate that the authors of these posts responded quickly to us and have since corrected the blogs. \n\nMostr.pub uses stirfry policies for anti-spam filtering. This includes an anti-duplication policy that prevents spam like the recent “vote Trump” posts we’ve seen repeated over and over. \n\nIt is important to note WHY there are multiple bridges, though. \n\nWhen Mostr.pub launched, multiple major servers immediately blocked Mostr, including Mastodon.social. The moderators of Mastodon.social claimed that this was because Nostr was unregulated, and suggested to one user that if they want to bridge their account they should host their own bridge.\n\nThat is exactly what momostr.pink, the source of this spam, has done. \n\nThe obvious response to the censorship of the Mostr Bridge is to build more bridges. \n\nWhile we have opted for pro-social policies that aim to reduce spam and build better connections between decentralized platforms, other bridges built to get around censorship of the Mostr Bridge may not — as we’re already seeing.\n\nThere will inevitably be multiple bridges, and we’re working on creating solutions to the problems that arise from that. In the meantime, if the Fediverse could do itself a favor and chill with the censorship for two seconds, we might not have so many problems. \n\n\nhttps://image.nostr.build/258d978b91e7424cfa43b31f3cfc077d7172ae10b3b45ac956feff9e72175126.png", + "sig": "b950e6e2ff1dc786ef344e7dad3edf8aa315a1053ede146725bde181acf7c2c1a5fcf1e0c796552b743607d6ae161a3ff4eb3af5033ffbfd314e68213d315215" +} diff --git a/fixtures/nostrbuild-gif.json b/fixtures/nostrbuild-gif.json new file mode 100644 index 0000000..49a969a --- /dev/null +++ b/fixtures/nostrbuild-gif.json @@ -0,0 +1,34 @@ +{ + "status": "success", + "message": "Upload successful.", + "data": [ + { + "input_name": "APIv2", + "name": "e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "sha256": "0a71f1c9dd982079bc52e96403368209cbf9507c5f6956134686f56e684b6377", + "original_sha256": "e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3", + "type": "picture", + "mime": "image/gif", + "size": 1796276, + "blurhash": "LGH-S^Vwm]x]04kX-qR-R]SL5FxZ", + "dimensions": { + "width": 360, + "height": 216 + }, + "dimensionsString": "360x216", + "url": "https://image.nostr.build/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "thumbnail": "https://image.nostr.build/thumb/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "responsive": { + "240p": "https://image.nostr.build/resp/240p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "360p": "https://image.nostr.build/resp/360p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "480p": "https://image.nostr.build/resp/480p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "720p": "https://image.nostr.build/resp/720p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif", + "1080p": "https://image.nostr.build/resp/1080p/e5f6e0e380536780efa774e8d3c8a5a040e3f9f99dbb48910b261c32872ee3a3.gif" + }, + "metadata": { + "date:create": "2024-05-18T02:11:39+00:00", + "date:modify": "2024-05-18T02:11:39+00:00" + } + } + ] +} \ No newline at end of file diff --git a/fixtures/nostrbuild-mp3.json b/fixtures/nostrbuild-mp3.json new file mode 100644 index 0000000..42a60b4 --- /dev/null +++ b/fixtures/nostrbuild-mp3.json @@ -0,0 +1,29 @@ +{ + "status": "success", + "message": "Upload successful.", + "data": [ + { + "id": 0, + "input_name": "APIv2", + "name": "f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "url": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "thumbnail": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "responsive": { + "240p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "360p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "480p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "720p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3", + "1080p": "https://media.nostr.build/av/f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725.mp3" + }, + "blurhash": "", + "sha256": "f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725", + "original_sha256": "f94665e6877741feb3fa3031342f95ae2ee00caae1cc651ce31ed6d524e05725", + "type": "video", + "mime": "audio/mpeg", + "size": 1519616, + "metadata": [], + "dimensions": [], + "dimensionsString": "0x0" + } + ] +} \ No newline at end of file diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index dcb0bc0..4037a85 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -1,8 +1,6 @@ import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; -import { Storages } from '@/storages.ts'; +import { refreshAuthorStats } from '@/stats.ts'; let pubkey: string; try { @@ -17,23 +15,4 @@ try { Deno.exit(1); } -const store = await Storages.db(); -const kysely = await DittoDB.getInstance(); - -const [followList] = await store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]); - -const authorStats: DittoTables['author_stats'] = { - pubkey, - followers_count: (await store.count([{ kinds: [3], '#p': [pubkey] }])).count, - following_count: followList?.tags.filter(([name]) => name === 'p')?.length ?? 0, - notes_count: (await store.count([{ kinds: [1], authors: [pubkey] }])).count, -}; - -await kysely.insertInto('author_stats') - .values(authorStats) - .onConflict((oc) => - oc - .column('pubkey') - .doUpdateSet(authorStats) - ) - .execute(); +await refreshAuthorStats(pubkey); diff --git a/src/app.ts b/src/app.ts index 059e883..be2ce84 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NostrSigner, NStore } from '@nostrify/nostrify'; +import { NostrEvent, NostrSigner, NStore, NUploader } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { type Context, Env as HonoEnv, type Handler, Hono, Input as HonoInput, type MiddlewareHandler } from 'hono'; import { cors, logger, serveStatic } from 'hono/middleware'; @@ -7,7 +7,6 @@ import { Conf } from '@/config.ts'; import { startFirehose } from '@/firehose.ts'; import { Time } from '@/utils.ts'; -import { actorController } from '@/controllers/activitypub/actor.ts'; import { accountController, accountLookupController, @@ -77,10 +76,8 @@ import { } from '@/controllers/api/timelines.ts'; import { trendingTagsController } from '@/controllers/api/trends.ts'; import { indexController } from '@/controllers/site.ts'; -import { hostMetaController } from '@/controllers/well-known/host-meta.ts'; import { nodeInfoController, nodeInfoSchemaController } from '@/controllers/well-known/nodeinfo.ts'; import { nostrController } from '@/controllers/well-known/nostr.ts'; -import { webfingerController } from '@/controllers/well-known/webfinger.ts'; import { auth98Middleware, requireProof, requireRole } from '@/middleware/auth98Middleware.ts'; import { cacheMiddleware } from '@/middleware/cacheMiddleware.ts'; import { cspMiddleware } from '@/middleware/cspMiddleware.ts'; @@ -89,11 +86,14 @@ import { signerMiddleware } from '@/middleware/signerMiddleware.ts'; import { storeMiddleware } from '@/middleware/storeMiddleware.ts'; import { blockController } from '@/controllers/api/accounts.ts'; import { unblockController } from '@/controllers/api/accounts.ts'; +import { uploaderMiddleware } from '@/middleware/uploaderMiddleware.ts'; interface AppEnv extends HonoEnv { Variables: { /** Signer to get the logged-in user's pubkey, relays, and to sign events, or `undefined` if the user isn't logged in. */ signer?: NostrSigner; + /** Uploader for the user to upload files. */ + uploader?: NUploader; /** NIP-98 signed event proving the pubkey is owned by the user. */ proof?: NostrEvent; /** Store */ @@ -129,17 +129,14 @@ app.use( cspMiddleware(), cors({ origin: '*', exposeHeaders: ['link'] }), signerMiddleware, + uploaderMiddleware, auth98Middleware(), storeMiddleware, ); -app.get('/.well-known/webfinger', webfingerController); -app.get('/.well-known/host-meta', hostMetaController); app.get('/.well-known/nodeinfo', nodeInfoController); app.get('/.well-known/nostr.json', nostrController); -app.get('/users/:username', actorController); - app.get('/nodeinfo/:version', nodeInfoSchemaController); app.get('/api/v1/instance', cacheMiddleware({ cacheName: 'web', expires: Time.minutes(5) }), instanceController); diff --git a/src/config.ts b/src/config.ts index 6fe62b9..cc14998 100644 --- a/src/config.ts +++ b/src/config.ts @@ -136,6 +136,14 @@ class Conf { return Deno.env.get('IPFS_API_URL') || 'http://localhost:5001'; }, }; + /** nostr.build API endpoint when the `nostrbuild` uploader is used. */ + static get nostrbuildEndpoint(): string { + return Deno.env.get('NOSTRBUILD_ENDPOINT') || 'https://nostr.build/api/v2/upload/files'; + } + /** Default Blossom servers to use when the `blossom` uploader is set. */ + static get blossomServers(): string[] { + return Deno.env.get('BLOSSOM_SERVERS')?.split(',') || ['https://blossom.primal.net/']; + } /** Module to upload files with. */ static get uploader() { return Deno.env.get('DITTO_UPLOADER'); diff --git a/src/controllers/activitypub/actor.ts b/src/controllers/activitypub/actor.ts deleted file mode 100644 index 19f5f10..0000000 --- a/src/controllers/activitypub/actor.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { getAuthor } from '@/queries.ts'; -import { activityJson } from '@/utils/api.ts'; -import { renderActor } from '@/views/activitypub/actor.ts'; -import { localNip05Lookup } from '@/utils/nip05.ts'; - -import type { AppContext, AppController } from '@/app.ts'; - -const actorController: AppController = async (c) => { - const username = c.req.param('username'); - const { signal } = c.req.raw; - - const pointer = await localNip05Lookup(c.get('store'), username); - if (!pointer) return notFound(c); - - const event = await getAuthor(pointer.pubkey, { signal }); - if (!event) return notFound(c); - - const actor = await renderActor(event, username); - if (!actor) return notFound(c); - - return activityJson(c, actor); -}; - -function notFound(c: AppContext) { - return c.json({ error: 'Not found' }, 404); -} - -export { actorController }; diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 5c26ba5..2d61adc 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -7,8 +7,7 @@ import { Conf } from '@/config.ts'; import { getAuthor, getFollowedPubkeys } from '@/queries.ts'; import { booleanParamSchema, fileSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; -import { addTag, deleteTag, findReplyTag, getTagSet } from '@/tags.ts'; -import { uploadFile } from '@/upload.ts'; +import { uploadFile } from '@/utils/upload.ts'; import { nostrNow } from '@/utils.ts'; import { createEvent, paginated, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; import { lookupAccount } from '@/utils/lookup.ts'; @@ -18,6 +17,7 @@ import { renderRelationship } from '@/views/mastodon/relationships.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { bech32ToPubkey } from '@/utils.ts'; +import { addTag, deleteTag, findReplyTag, getTagSet } from '@/utils/tags.ts'; const usernameSchema = z .string().min(1).max(30) @@ -45,14 +45,32 @@ const createAccountController: AppController = async (c) => { }; const verifyCredentialsController: AppController = async (c) => { - const pubkey = await c.get('signer')?.getPublicKey()!; + const signer = c.get('signer')!; + const pubkey = await signer.getPublicKey(); - const event = await getAuthor(pubkey, { relations: ['author_stats'] }); - if (event) { - return c.json(await renderAccount(event, { withSource: true })); - } else { - return c.json(await accountFromPubkey(pubkey, { withSource: true })); + const eventsDB = await Storages.db(); + + const [author, [settingsStore]] = await Promise.all([ + getAuthor(pubkey, { signal: AbortSignal.timeout(5000) }), + + eventsDB.query([{ + authors: [pubkey], + kinds: [30078], + '#d': ['pub.ditto.pleroma_settings_store'], + limit: 1, + }]), + ]); + + const account = author + ? await renderAccount(author, { withSource: true }) + : await accountFromPubkey(pubkey, { withSource: true }); + + if (settingsStore) { + const data = await signer.nip44!.decrypt(pubkey, settingsStore.content); + account.pleroma.settings_store = JSON.parse(data); } + + return c.json(account); }; const accountController: AppController = async (c) => { @@ -86,25 +104,35 @@ const accountLookupController: AppController = async (c) => { } }; -const accountSearchController: AppController = async (c) => { - const q = c.req.query('q'); +const accountSearchQuerySchema = z.object({ + q: z.string().transform(decodeURIComponent), + resolve: booleanParamSchema.optional().transform(Boolean), + following: z.boolean().default(false), + limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)), +}); - if (!q) { - return c.json({ error: 'Missing `q` query parameter.' }, 422); +const accountSearchController: AppController = async (c) => { + const result = accountSearchQuerySchema.safeParse(c.req.query()); + const { signal } = c.req.raw; + + if (!result.success) { + return c.json({ error: 'Bad request', schema: result.error }, 422); } + const { q, limit } = result.data; + const query = decodeURIComponent(q); const store = await Storages.search(); const [event, events] = await Promise.all([ lookupAccount(query), - store.query([{ kinds: [0], search: query, limit: 20 }], { signal: c.req.raw.signal }), + store.query([{ kinds: [0], search: query, limit }], { signal }), ]); const results = await hydrateEvents({ events: event ? [event, ...events] : events, store, - signal: c.req.raw.signal, + signal, }); if ((results.length < 1) && query.match(/npub1\w+/)) { @@ -198,10 +226,12 @@ const updateCredentialsSchema = z.object({ bot: z.boolean().optional(), discoverable: z.boolean().optional(), nip05: z.string().optional(), + pleroma_settings_store: z.unknown().optional(), }); const updateCredentialsController: AppController = async (c) => { - const pubkey = await c.get('signer')?.getPublicKey()!; + const signer = c.get('signer')!; + const pubkey = await signer.getPublicKey(); const body = await parseBody(c.req.raw); const result = updateCredentialsSchema.safeParse(body); @@ -221,8 +251,8 @@ const updateCredentialsController: AppController = async (c) => { } = result.data; const [avatar, header] = await Promise.all([ - avatarFile ? uploadFile(avatarFile, { pubkey }) : undefined, - headerFile ? uploadFile(headerFile, { pubkey }) : undefined, + avatarFile ? uploadFile(c, avatarFile, { pubkey }) : undefined, + headerFile ? uploadFile(c, headerFile, { pubkey }) : undefined, ]); meta.name = display_name ?? meta.name; @@ -238,6 +268,18 @@ const updateCredentialsController: AppController = async (c) => { }, c); const account = await renderAccount(event, { withSource: true }); + const settingsStore = result.data.pleroma_settings_store; + + if (settingsStore) { + await createEvent({ + kind: 30078, + tags: [['d', 'pub.ditto.pleroma_settings_store']], + content: await signer.nip44!.encrypt(pubkey, JSON.stringify(settingsStore)), + }, c); + } + + account.pleroma.settings_store = settingsStore; + return c.json(account); }; diff --git a/src/controllers/api/admin.ts b/src/controllers/api/admin.ts index 77571aa..d7cd365 100644 --- a/src/controllers/api/admin.ts +++ b/src/controllers/api/admin.ts @@ -5,8 +5,8 @@ import { Conf } from '@/config.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { booleanParamSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; -import { addTag } from '@/tags.ts'; import { paginated, paginationSchema, parseBody, updateListAdminEvent } from '@/utils/api.ts'; +import { addTag } from '@/utils/tags.ts'; import { renderAdminAccount } from '@/views/mastodon/admin-accounts.ts'; const adminAccountQuerySchema = z.object({ diff --git a/src/controllers/api/bookmarks.ts b/src/controllers/api/bookmarks.ts index 7655182..6d80b50 100644 --- a/src/controllers/api/bookmarks.ts +++ b/src/controllers/api/bookmarks.ts @@ -1,6 +1,6 @@ import { type AppController } from '@/app.ts'; import { Storages } from '@/storages.ts'; -import { getTagSet } from '@/tags.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { renderStatuses } from '@/views.ts'; /** https://docs.joinmastodon.org/methods/bookmarks/#get */ diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index 33b7981..71b3e78 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -4,7 +4,7 @@ import { AppController } from '@/app.ts'; import { fileSchema } from '@/schema.ts'; import { parseBody } from '@/utils/api.ts'; import { renderAttachment } from '@/views/mastodon/attachments.ts'; -import { uploadFile } from '@/upload.ts'; +import { uploadFile } from '@/utils/upload.ts'; const mediaBodySchema = z.object({ file: fileSchema, @@ -24,7 +24,7 @@ const mediaController: AppController = async (c) => { try { const { file, description } = result.data; - const media = await uploadFile(file, { pubkey, description }, signal); + const media = await uploadFile(c, file, { pubkey, description }, signal); return c.json(renderAttachment(media)); } catch (e) { console.error(e); diff --git a/src/controllers/api/mutes.ts b/src/controllers/api/mutes.ts index 4afb6c4..31f54ee 100644 --- a/src/controllers/api/mutes.ts +++ b/src/controllers/api/mutes.ts @@ -1,6 +1,6 @@ import { type AppController } from '@/app.ts'; import { Storages } from '@/storages.ts'; -import { getTagSet } from '@/tags.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { renderAccounts } from '@/views.ts'; /** https://docs.joinmastodon.org/methods/mutes/#get */ diff --git a/src/controllers/api/oauth.ts b/src/controllers/api/oauth.ts index a755a4d..c1407f4 100644 --- a/src/controllers/api/oauth.ts +++ b/src/controllers/api/oauth.ts @@ -1,9 +1,9 @@ import { encodeBase64 } from '@std/encoding/base64'; +import { escape } from 'entities'; import { nip19 } from 'nostr-tools'; import { z } from 'zod'; import { AppController } from '@/app.ts'; -import { lodash } from '@/deps.ts'; import { nostrNow } from '@/utils.ts'; import { parseBody } from '@/utils/api.ts'; import { getClientConnectUri } from '@/utils/connect.ts'; @@ -100,11 +100,11 @@ const oauthController: AppController = async (c) => {
- +

- Nostr Connect + Nostr Connect `); diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 98173b0..58526cb 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -1,20 +1,22 @@ import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; import ISO6391 from 'iso-639-1'; +import { nip19 } from 'nostr-tools'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; -import { addTag, deleteTag } from '@/tags.ts'; -import { createEvent, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; import { renderEventAccounts } from '@/views.ts'; import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; -import { getLnurl } from '@/utils/lnurl.ts'; -import { asyncReplaceAll } from '@/utils/text.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { createEvent, paginationSchema, parseBody, updateListEvent } from '@/utils/api.ts'; +import { getLnurl } from '@/utils/lnurl.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; +import { addTag, deleteTag } from '@/utils/tags.ts'; +import { asyncReplaceAll } from '@/utils/text.ts'; const createStatusSchema = z.object({ in_reply_to_id: z.string().regex(/[0-9a-f]{64}/).nullish(), @@ -56,6 +58,7 @@ const statusController: AppController = async (c) => { const createStatusController: AppController = async (c) => { const body = await parseBody(c.req.raw); const result = createStatusSchema.safeParse(body); + const kysely = await DittoDB.getInstance(); if (!result.success) { return c.json({ error: 'Bad request', schema: result.error }, 400); @@ -73,12 +76,21 @@ const createStatusController: AppController = async (c) => { const tags: string[][] = []; - if (data.quote_id) { - tags.push(['q', data.quote_id]); + if (data.in_reply_to_id) { + const ancestor = await getEvent(data.in_reply_to_id); + + if (!ancestor) { + return c.json({ error: 'Original post not found.' }, 404); + } + + const root = ancestor.tags.find((tag) => tag[0] === 'e' && tag[3] === 'root')?.[1] ?? ancestor.id; + + tags.push(['e', root, 'root']); + tags.push(['e', data.in_reply_to_id, 'reply']); } - if (data.in_reply_to_id) { - tags.push(['e', data.in_reply_to_id, 'reply']); + if (data.quote_id) { + tags.push(['q', data.quote_id]); } if (data.sensitive && data.spoiler_text) { @@ -89,15 +101,14 @@ const createStatusController: AppController = async (c) => { tags.push(['subject', data.spoiler_text]); } - const viewerPubkey = await c.get('signer')?.getPublicKey(); + const media = data.media_ids?.length ? await getUnattachedMediaByIds(kysely, data.media_ids) : []; - if (data.media_ids?.length) { - const media = await getUnattachedMediaByIds(data.media_ids) - .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey)) - .then((media) => media.map(({ url, data }) => ['media', url, data])); + const imeta: string[][] = media.map(({ data }) => { + const values: string[] = data.map((tag) => tag.join(' ')); + return ['imeta', ...values]; + }); - tags.push(...media); - } + tags.push(...imeta); const pubkeys = new Set(); @@ -110,7 +121,11 @@ const createStatusController: AppController = async (c) => { pubkeys.add(pubkey); } - return `nostr:${pubkey}`; + try { + return `nostr:${nip19.npubEncode(pubkey)}`; + } catch { + return match; + } }); // Explicit addressing @@ -129,9 +144,15 @@ const createStatusController: AppController = async (c) => { tags.push(['t', match[1]]); } + const mediaUrls: string[] = media + .map(({ data }) => data.find(([name]) => name === 'url')?.[1]) + .filter((url): url is string => Boolean(url)); + + const mediaCompat: string = mediaUrls.length ? ['', '', ...mediaUrls].join('\n') : ''; + const event = await createEvent({ kind: 1, - content, + content: content + mediaCompat, tags, }, c); @@ -261,21 +282,19 @@ const reblogStatusController: AppController = async (c) => { const unreblogStatusController: AppController = async (c) => { const eventId = c.req.param('id'); const pubkey = await c.get('signer')?.getPublicKey()!; - - const event = await getEvent(eventId, { kind: 1 }); - - if (!event) { - return c.json({ error: 'Event not found.' }, 404); - } - const store = await Storages.db(); + const [event] = await store.query([{ ids: [eventId], kinds: [1] }]); + if (!event) { + return c.json({ error: 'Record not found' }, 404); + } + const [repostedEvent] = await store.query( [{ kinds: [6], authors: [pubkey], '#e': [event.id], limit: 1 }], ); if (!repostedEvent) { - return c.json({ error: 'Event not found.' }, 404); + return c.json({ error: 'Record not found' }, 404); } await createEvent({ diff --git a/src/controllers/api/suggestions.ts b/src/controllers/api/suggestions.ts index 6377bd4..012244a 100644 --- a/src/controllers/api/suggestions.ts +++ b/src/controllers/api/suggestions.ts @@ -2,8 +2,8 @@ import { NStore } from '@nostrify/nostrify'; import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { getTagSet } from '@/tags.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; export const suggestionsV1Controller: AppController = async (c) => { diff --git a/src/controllers/well-known/host-meta.ts b/src/controllers/well-known/host-meta.ts deleted file mode 100644 index 85da11b..0000000 --- a/src/controllers/well-known/host-meta.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Conf } from '@/config.ts'; - -import type { AppController } from '@/app.ts'; - -/** https://datatracker.ietf.org/doc/html/rfc6415 */ -const hostMetaController: AppController = (c) => { - const template = Conf.local('/.well-known/webfinger?resource={uri}'); - - c.header('content-type', 'application/xrd+xml'); - - return c.body( - ` - - - -`, - ); -}; - -export { hostMetaController }; diff --git a/src/controllers/well-known/webfinger.ts b/src/controllers/well-known/webfinger.ts deleted file mode 100644 index c1c8b81..0000000 --- a/src/controllers/well-known/webfinger.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { nip19 } from 'nostr-tools'; -import { z } from 'zod'; - -import { Conf } from '@/config.ts'; -import { localNip05Lookup } from '@/utils/nip05.ts'; - -import type { AppContext, AppController } from '@/app.ts'; -import type { Webfinger } from '@/schemas/webfinger.ts'; - -const webfingerQuerySchema = z.object({ - resource: z.string().url(), -}); - -const webfingerController: AppController = (c) => { - const query = webfingerQuerySchema.safeParse(c.req.query()); - if (!query.success) { - return c.json({ error: 'Bad request', schema: query.error }, 400); - } - - const resource = new URL(query.data.resource); - - switch (resource.protocol) { - case 'acct:': { - return handleAcct(c, resource); - } - default: - return c.json({ error: 'Unsupported URI scheme' }, 400); - } -}; - -/** Transforms the resource URI into a `[username, domain]` tuple. */ -const acctSchema = z.custom((value) => value instanceof URL) - .transform((uri) => uri.pathname) - .pipe(z.string().email('Invalid acct')) - .transform((acct) => acct.split('@') as [username: string, host: string]) - .refine(([_username, host]) => host === Conf.url.hostname, { - message: 'Host must be local', - path: ['resource', 'acct'], - }); - -async function handleAcct(c: AppContext, resource: URL): Promise { - const result = acctSchema.safeParse(resource); - if (!result.success) { - return c.json({ error: 'Invalid acct URI', schema: result.error }, 400); - } - - const [username, host] = result.data; - const pointer = await localNip05Lookup(c.get('store'), username); - - if (!pointer) { - return c.json({ error: 'Not found' }, 404); - } - - const json = renderWebfinger({ - pubkey: pointer.pubkey, - username, - subject: `acct:${username}@${host}`, - }); - - c.header('content-type', 'application/jrd+json'); - return c.body(JSON.stringify(json)); -} - -interface RenderWebfingerOpts { - pubkey: string; - username: string; - subject: string; -} - -/** Present Nostr user on Webfinger. */ -function renderWebfinger({ pubkey, username, subject }: RenderWebfingerOpts): Webfinger { - const apId = Conf.local(`/users/${username}`); - - return { - subject, - aliases: [apId], - links: [ - { - rel: 'self', - type: 'application/activity+json', - href: apId, - }, - { - rel: 'self', - type: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', - href: apId, - }, - { - rel: 'self', - type: 'application/nostr+json', - href: `nostr:${nip19.npubEncode(pubkey)}`, - }, - ], - }; -} - -export { webfingerController }; diff --git a/src/db/migrations/020_pgfts.ts b/src/db/migrations/020_pgfts.ts new file mode 100644 index 0000000..8b3cfa0 --- /dev/null +++ b/src/db/migrations/020_pgfts.ts @@ -0,0 +1,19 @@ +import { Kysely, sql } from 'kysely'; + +import { Conf } from '@/config.ts'; + +export async function up(db: Kysely): Promise { + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + await db.schema.createTable('nostr_pgfts') + .ifNotExists() + .addColumn('event_id', 'text', (c) => c.primaryKey().references('nostr_events.id').onDelete('cascade')) + .addColumn('search_vec', sql`tsvector`, (c) => c.notNull()) + .execute(); + } +} + +export async function down(db: Kysely): Promise { + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + await db.schema.dropTable('nostr_pgfts').ifExists().execute(); + } +} diff --git a/src/db/migrations/021_pgfts_index.ts b/src/db/migrations/021_pgfts_index.ts new file mode 100644 index 0000000..d18d110 --- /dev/null +++ b/src/db/migrations/021_pgfts_index.ts @@ -0,0 +1,21 @@ +import { Kysely } from 'kysely'; + +import { Conf } from '@/config.ts'; + +export async function up(db: Kysely): Promise { + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + await db.schema + .createIndex('nostr_pgfts_gin_search_vec') + .ifNotExists() + .on('nostr_pgfts') + .using('gin') + .column('search_vec') + .execute(); + } +} + +export async function down(db: Kysely): Promise { + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + await db.schema.dropIndex('nostr_pgfts_gin_search_vec').ifExists().execute(); + } +} diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index 8063628..0e0aeea 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -1,35 +1,29 @@ -import uuid62 from 'uuid62'; +import { Kysely } from 'kysely'; import { DittoDB } from '@/db/DittoDB.ts'; -import { type MediaData } from '@/schemas/nostr.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; interface UnattachedMedia { id: string; pubkey: string; url: string; - data: MediaData; + /** NIP-94 tags. */ + data: string[][]; uploaded_at: number; } /** Add unattached media into the database. */ -async function insertUnattachedMedia(media: Omit) { - const result = { - id: uuid62.v4(), - uploaded_at: Date.now(), - ...media, - }; - +async function insertUnattachedMedia(media: UnattachedMedia) { const kysely = await DittoDB.getInstance(); await kysely.insertInto('unattached_media') - .values({ ...result, data: JSON.stringify(media.data) }) + .values({ ...media, data: JSON.stringify(media.data) }) .execute(); - return result; + return media; } /** Select query for unattached media. */ -async function selectUnattachedMediaQuery() { - const kysely = await DittoDB.getInstance(); +function selectUnattachedMediaQuery(kysely: Kysely) { return kysely.selectFrom('unattached_media') .select([ 'unattached_media.id', @@ -41,9 +35,8 @@ async function selectUnattachedMediaQuery() { } /** Find attachments that exist but aren't attached to any events. */ -async function getUnattachedMedia(until: Date) { - const query = await selectUnattachedMediaQuery(); - return query +function getUnattachedMedia(kysely: Kysely, until: Date) { + return selectUnattachedMediaQuery(kysely) .leftJoin('nostr_tags', 'unattached_media.url', 'nostr_tags.value') .where('uploaded_at', '<', until.getTime()) .execute(); @@ -58,12 +51,17 @@ async function deleteUnattachedMediaByUrl(url: string) { } /** Get unattached media by IDs. */ -async function getUnattachedMediaByIds(ids: string[]) { +async function getUnattachedMediaByIds(kysely: Kysely, ids: string[]): Promise { if (!ids.length) return []; - const query = await selectUnattachedMediaQuery(); - return query + + const results = await selectUnattachedMediaQuery(kysely) .where('id', 'in', ids) .execute(); + + return results.map((row) => ({ + ...row, + data: JSON.parse(row.data), + })); } /** Delete rows as an event with media is being created. */ diff --git a/src/deps.ts b/src/deps.ts deleted file mode 100644 index 46d8fec..0000000 --- a/src/deps.ts +++ /dev/null @@ -1,18 +0,0 @@ -import 'deno-safe-fetch'; -// @deno-types="npm:@types/lodash@4.14.194" -export { default as lodash } from 'https://esm.sh/lodash@4.17.21'; -// @deno-types="npm:@types/mime@3.0.0" -export { default as mime } from 'npm:mime@^3.0.0'; -// @deno-types="npm:@types/sanitize-html@2.9.0" -export { default as sanitizeHtml } from 'npm:sanitize-html@^2.11.0'; -export { - type ParsedSignature, - pemToPublicKey, - publicKeyToPem, - signRequest, - verifyRequest, -} from 'https://gitlab.com/soapbox-pub/fedisign/-/raw/v0.2.1/mod.ts'; -export { generateSeededRsa } from 'https://gitlab.com/soapbox-pub/seeded-rsa/-/raw/v1.0.0/mod.ts'; -export { - DB as Sqlite, -} from 'https://raw.githubusercontent.com/alexgleason/deno-sqlite/325f66d8c395e7f6f5ee78ebfa42a0eeea4a942b/mod.ts'; diff --git a/src/middleware/uploaderMiddleware.ts b/src/middleware/uploaderMiddleware.ts new file mode 100644 index 0000000..38e8ace --- /dev/null +++ b/src/middleware/uploaderMiddleware.ts @@ -0,0 +1,35 @@ +import { BlossomUploader, NostrBuildUploader } from '@nostrify/nostrify/uploaders'; + +import { AppMiddleware } from '@/app.ts'; +import { Conf } from '@/config.ts'; +import { DenoUploader } from '@/uploaders/DenoUploader.ts'; +import { IPFSUploader } from '@/uploaders/IPFSUploader.ts'; +import { S3Uploader } from '@/uploaders/S3Uploader.ts'; +import { fetchWorker } from '@/workers/fetch.ts'; + +/** Set an uploader for the user. */ +export const uploaderMiddleware: AppMiddleware = async (c, next) => { + const signer = c.get('signer'); + + switch (Conf.uploader) { + case 's3': + c.set('uploader', new S3Uploader(Conf.s3)); + break; + case 'ipfs': + c.set('uploader', new IPFSUploader({ baseUrl: Conf.mediaDomain, apiUrl: Conf.ipfs.apiUrl, fetch: fetchWorker })); + break; + case 'local': + c.set('uploader', new DenoUploader({ baseUrl: Conf.mediaDomain, dir: Conf.uploadsDir })); + break; + case 'nostrbuild': + c.set('uploader', new NostrBuildUploader({ endpoint: Conf.nostrbuildEndpoint, signer, fetch: fetchWorker })); + break; + case 'blossom': + if (signer) { + c.set('uploader', new BlossomUploader({ servers: Conf.blossomServers, signer, fetch: fetchWorker })); + } + break; + } + + await next(); +}; diff --git a/src/pipeline.ts b/src/pipeline.ts index 0672c88..36ec54e 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -13,7 +13,6 @@ import { RelayError } from '@/RelayError.ts'; import { updateStats } from '@/stats.ts'; import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts'; import { Storages } from '@/storages.ts'; -import { getTagSet } from '@/tags.ts'; import { eventAge, nostrDate, nostrNow, parseNip05, Time } from '@/utils.ts'; import { fetchWorker } from '@/workers/fetch.ts'; import { policyWorker } from '@/workers/policy.ts'; @@ -22,6 +21,7 @@ import { verifyEventWorker } from '@/workers/verify.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { lnurlCache } from '@/utils/lnurl.ts'; import { nip05Cache } from '@/utils/nip05.ts'; +import { getTagSet } from '@/utils/tags.ts'; import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; diff --git a/src/policies/MuteListPolicy.ts b/src/policies/MuteListPolicy.ts index cae08eb..130d10d 100644 --- a/src/policies/MuteListPolicy.ts +++ b/src/policies/MuteListPolicy.ts @@ -1,6 +1,6 @@ import { NostrEvent, NostrRelayOK, NPolicy, NStore } from '@nostrify/nostrify'; -import { getTagSet } from '@/tags.ts'; +import { getTagSet } from '@/utils/tags.ts'; export class MuteListPolicy implements NPolicy { constructor(private pubkey: string, private store: NStore) {} diff --git a/src/queries.ts b/src/queries.ts index 76fabfd..7407077 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -5,8 +5,8 @@ import { Conf } from '@/config.ts'; import { Storages } from '@/storages.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { type DittoRelation } from '@/interfaces/DittoFilter.ts'; -import { findReplyTag, getTagSet } from '@/tags.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { findReplyTag, getTagSet } from '@/utils/tags.ts'; const debug = Debug('ditto:queries'); @@ -88,7 +88,11 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi async function getDescendants(eventId: string, signal = AbortSignal.timeout(2000)): Promise { const store = await Storages.db(); - const events = await store.query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal }); + + const events = await store + .query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal }) + .then((events) => events.filter(({ tags }) => findReplyTag(tags)?.[1] === eventId)); + return hydrateEvents({ events, store, signal }); } diff --git a/src/schemas/nostr.ts b/src/schemas/nostr.ts index a42b9f0..d8aa29a 100644 --- a/src/schemas/nostr.ts +++ b/src/schemas/nostr.ts @@ -9,27 +9,12 @@ const signedEventSchema = n.event() .refine((event) => event.id === getEventHash(event), 'Event ID does not match hash') .refine(verifyEvent, 'Event signature is invalid'); -/** Media data schema from `"media"` tags. */ -const mediaDataSchema = z.object({ - blurhash: z.string().optional().catch(undefined), - cid: z.string().optional().catch(undefined), - description: z.string().max(200).optional().catch(undefined), - height: z.number().int().positive().optional().catch(undefined), - mime: z.string().optional().catch(undefined), - name: z.string().optional().catch(undefined), - size: z.number().int().positive().optional().catch(undefined), - width: z.number().int().positive().optional().catch(undefined), -}); - /** Kind 0 content schema for the Ditto server admin user. */ const serverMetaSchema = n.metadata().and(z.object({ tagline: z.string().optional().catch(undefined), email: z.string().optional().catch(undefined), })); -/** Media data from `"media"` tags. */ -type MediaData = z.infer; - /** NIP-11 Relay Information Document. */ const relayInfoDocSchema = z.object({ name: z.string().transform((val) => val.slice(0, 30)).optional().catch(undefined), @@ -47,12 +32,4 @@ const emojiTagSchema = z.tuple([z.literal('emoji'), z.string(), z.string().url() /** NIP-30 custom emoji tag. */ type EmojiTag = z.infer; -export { - type EmojiTag, - emojiTagSchema, - type MediaData, - mediaDataSchema, - relayInfoDocSchema, - serverMetaSchema, - signedEventSchema, -}; +export { type EmojiTag, emojiTagSchema, relayInfoDocSchema, serverMetaSchema, signedEventSchema }; diff --git a/src/schemas/webfinger.ts b/src/schemas/webfinger.ts deleted file mode 100644 index 8c9cf57..0000000 --- a/src/schemas/webfinger.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { z } from 'zod'; - -const linkSchema = z.object({ - rel: z.string().optional(), - type: z.string().optional(), - href: z.string().optional(), - template: z.string().optional(), -}); - -const webfingerSchema = z.object({ - subject: z.string(), - aliases: z.array(z.string()).catch([]), - links: z.array(linkSchema), -}); - -type Webfinger = z.infer; - -export { webfingerSchema }; -export type { Webfinger }; diff --git a/src/server.ts b/src/server.ts index 4825e99..f7a33dc 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,3 +1,5 @@ +import 'deno-safe-fetch/load'; + import '@/precheck.ts'; import '@/sentry.ts'; import '@/nostr-wasm.ts'; diff --git a/src/stats.ts b/src/stats.ts index 9204071..6ffe5f7 100644 --- a/src/stats.ts +++ b/src/stats.ts @@ -1,11 +1,14 @@ -import { NKinds, NostrEvent } from '@nostrify/nostrify'; +import { Semaphore } from '@lambdalisue/async'; +import { NKinds, NostrEvent, NStore } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; import { InsertQueryBuilder, Kysely } from 'kysely'; +import { LRUCache } from 'lru-cache'; +import { SetRequired } from 'type-fest'; import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Storages } from '@/storages.ts'; -import { findReplyTag } from '@/tags.ts'; +import { findReplyTag, getTagSet } from '@/utils/tags.ts'; type AuthorStat = keyof Omit; type EventStat = keyof Omit; @@ -37,6 +40,8 @@ async function updateStats(event: NostrEvent) { debug(JSON.stringify({ id: event.id, pubkey: event.pubkey, kind: event.kind, tags: event.tags, statDiffs })); } + pubkeyDiffs.forEach(([_, pubkey]) => refreshAuthorStatsDebounced(pubkey)); + const kysely = await DittoDB.getInstance(); if (pubkeyDiffs.length) queries.push(authorStatsQuery(kysely, pubkeyDiffs)); @@ -216,4 +221,53 @@ function getFollowDiff(event: NostrEvent, prev?: NostrEvent): AuthorStatDiff[] { ]; } -export { updateStats }; +/** Refresh the author's stats in the database. */ +async function refreshAuthorStats(pubkey: string): Promise { + const store = await Storages.db(); + const stats = await countAuthorStats(store, pubkey); + + const kysely = await DittoDB.getInstance(); + await kysely.insertInto('author_stats') + .values(stats) + .onConflict((oc) => oc.column('pubkey').doUpdateSet(stats)) + .execute(); + + return stats; +} + +/** Calculate author stats from the database. */ +async function countAuthorStats( + store: SetRequired, + pubkey: string, +): Promise { + const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([ + store.count([{ kinds: [3], '#p': [pubkey] }]), + store.count([{ kinds: [1], authors: [pubkey] }]), + store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]), + ]); + + return { + pubkey, + followers_count, + following_count: getTagSet(followList?.tags ?? [], 'p').size, + notes_count, + }; +} + +const authorStatsSemaphore = new Semaphore(10); +const refreshedAuthors = new LRUCache({ max: 1000 }); + +/** Calls `refreshAuthorStats` only once per author. */ +function refreshAuthorStatsDebounced(pubkey: string): void { + if (refreshedAuthors.get(pubkey)) { + return; + } + + refreshedAuthors.set(pubkey, true); + debug('refreshing author stats:', pubkey); + + authorStatsSemaphore + .lock(() => refreshAuthorStats(pubkey).catch(() => {})); +} + +export { refreshAuthorStats, refreshAuthorStatsDebounced, updateStats }; diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 2f34379..16b429d 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -9,10 +9,7 @@ import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { RelayError } from '@/RelayError.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { genEvent } from '@/test.ts'; - -import event0 from '~/fixtures/events/event-0.json' with { type: 'json' }; -import event1 from '~/fixtures/events/event-1.json' with { type: 'json' }; +import { eventFixture, genEvent } from '@/test.ts'; /** Create in-memory database for testing. */ const createDB = async () => { @@ -28,6 +25,7 @@ const createDB = async () => { Deno.test('count filters', async () => { const { eventsDB } = await createDB(); + const event1 = await eventFixture('event-1'); assertEquals((await eventsDB.count([{ kinds: [1] }])).count, 0); await eventsDB.event(event1); @@ -37,6 +35,7 @@ Deno.test('count filters', async () => { Deno.test('insert and filter events', async () => { const { eventsDB } = await createDB(); + const event1 = await eventFixture('event-1'); await eventsDB.event(event1); assertEquals(await eventsDB.query([{ kinds: [1] }]), [event1]); @@ -52,6 +51,7 @@ Deno.test('insert and filter events', async () => { Deno.test('query events with domain search filter', async () => { const { eventsDB, kysely } = await createDB(); + const event1 = await eventFixture('event-1'); await eventsDB.event(event1); assertEquals(await eventsDB.query([{}]), [event1]); @@ -180,7 +180,7 @@ Deno.test('throws a RelayError when inserting an event deleted by a user', async Deno.test('inserting replaceable events', async () => { const { eventsDB } = await createDB(); - const event = event0; + const event = await eventFixture('event-0'); await eventsDB.event(event); const olderEvent = { ...event, id: '123', created_at: event.created_at - 1 }; diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index aac8e52..a550f39 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -7,11 +7,11 @@ import { Kysely } from 'kysely'; import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { normalizeFilters } from '@/filter.ts'; +import { RelayError } from '@/RelayError.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; -import { getTagSet } from '@/tags.ts'; import { isNostrId, isURL } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; -import { RelayError } from '@/RelayError.ts'; +import { getTagSet } from '@/utils/tags.ts'; /** Function to decide whether or not to index a tag. */ type TagCondition = ({ event, count, value }: { @@ -42,8 +42,17 @@ class EventsDB implements NStore { }; constructor(private kysely: Kysely) { + let fts: 'sqlite' | 'postgres' | undefined; + + if (Conf.databaseUrl.protocol === 'sqlite:') { + fts = 'sqlite'; + } + if (['postgres:', 'postgresql:'].includes(Conf.databaseUrl.protocol!)) { + fts = 'postgres'; + } + this.store = new NDatabase(kysely, { - fts5: Conf.databaseUrl.protocol === 'sqlite:', + fts, indexTags: EventsDB.indexTags, searchText: EventsDB.searchText, }); @@ -171,8 +180,8 @@ class EventsDB implements NStore { /** Build search content for a user. */ static buildUserSearchContent(event: NostrEvent): string { - const { name, nip05, about } = n.json().pipe(n.metadata()).catch({}).parse(event.content); - return [name, nip05, about].filter(Boolean).join('\n'); + const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); + return [name, nip05].filter(Boolean).join('\n'); } /** Build search content from tag values. */ diff --git a/src/storages/UserStore.ts b/src/storages/UserStore.ts index c5657b6..43c1771 100644 --- a/src/storages/UserStore.ts +++ b/src/storages/UserStore.ts @@ -1,7 +1,7 @@ import { NostrEvent, NostrFilter, NStore } from '@nostrify/nostrify'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { getTagSet } from '@/tags.ts'; +import { getTagSet } from '@/utils/tags.ts'; export class UserStore implements NStore { constructor(private pubkey: string, private store: NStore) {} diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 8d2d302..68dc0bd 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -5,6 +5,8 @@ import { DittoDB } from '@/db/DittoDB.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; +import { refreshAuthorStatsDebounced } from '@/stats.ts'; +import { findQuoteTag } from '@/utils/tags.ts'; interface HydrateOpts { events: DittoEvent[]; @@ -55,6 +57,8 @@ async function hydrateEvents(opts: HydrateOpts): Promise { events: await gatherEventStats(cache), }; + refreshMissingAuthorStats(events, stats.authors); + // Dedupe events. const results = [...new Map(cache.map((event) => [event.id, event])).values()]; @@ -78,7 +82,7 @@ function assembleEvents( event.user = b.find((e) => matchFilter({ kinds: [30361], authors: [admin], '#d': [event.pubkey] }, e)); if (event.kind === 1) { - const id = event.tags.find(([name]) => name === 'q')?.[1]; + const id = findQuoteTag(event.tags)?.[1]; if (id) { event.quote = b.find((e) => matchFilter({ kinds: [1], ids: [id] }, e)); } @@ -166,7 +170,7 @@ function gatherQuotes({ events, store, signal }: HydrateOpts): Promise name === 'q')?.[1]; + const id = findQuoteTag(event.tags)?.[1]; if (id) { ids.add(id); } @@ -251,11 +255,35 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise ({ + pubkey: row.pubkey, + followers_count: Math.max(0, row.followers_count), + following_count: Math.max(0, row.following_count), + notes_count: Math.max(0, row.notes_count), + })); +} + +function refreshMissingAuthorStats(events: NostrEvent[], stats: DittoTables['author_stats'][]) { + const pubkeys = new Set( + events + .filter((event) => event.kind === 0) + .map((event) => event.pubkey), + ); + + const missing = pubkeys.difference( + new Set(stats.map((stat) => stat.pubkey)), + ); + + for (const pubkey of missing) { + refreshAuthorStatsDebounced(pubkey); + } } /** Collect event stats from the events. */ @@ -271,11 +299,19 @@ async function gatherEventStats(events: DittoEvent[]): Promise ({ + event_id: row.event_id, + reposts_count: Math.max(0, row.reposts_count), + reactions_count: Math.max(0, row.reactions_count), + replies_count: Math.max(0, row.replies_count), + })); } /** Return a normalized event without any non-standard keys. */ diff --git a/src/tags.test.ts b/src/tags.test.ts deleted file mode 100644 index e49d31a..0000000 --- a/src/tags.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { assertEquals } from '@std/assert'; - -import { addTag, deleteTag, getTagSet } from './tags.ts'; - -Deno.test('getTagSet', () => { - assertEquals(getTagSet([], 'p'), new Set()); - assertEquals(getTagSet([['p', '123']], 'p'), new Set(['123'])); - assertEquals(getTagSet([['p', '123'], ['p', '456']], 'p'), new Set(['123', '456'])); - assertEquals(getTagSet([['p', '123'], ['p', '456'], ['q', '789']], 'p'), new Set(['123', '456'])); -}); - -Deno.test('addTag', () => { - assertEquals(addTag([], ['p', '123']), [['p', '123']]); - assertEquals(addTag([['p', '123']], ['p', '123']), [['p', '123']]); - assertEquals(addTag([['p', '123'], ['p', '456']], ['p', '123']), [['p', '123'], ['p', '456']]); - assertEquals(addTag([['p', '123'], ['p', '456']], ['p', '789']), [['p', '123'], ['p', '456'], ['p', '789']]); -}); - -Deno.test('deleteTag', () => { - assertEquals(deleteTag([], ['p', '123']), []); - assertEquals(deleteTag([['p', '123']], ['p', '123']), []); - assertEquals(deleteTag([['p', '123']], ['p', '456']), [['p', '123']]); - assertEquals(deleteTag([['p', '123'], ['p', '123']], ['p', '123']), []); - assertEquals(deleteTag([['p', '123'], ['p', '456']], ['p', '456']), [['p', '123']]); -}); diff --git a/src/tags.ts b/src/tags.ts deleted file mode 100644 index a683393..0000000 --- a/src/tags.ts +++ /dev/null @@ -1,42 +0,0 @@ -/** Get the values for a tag in a `Set`. */ -function getTagSet(tags: string[][], tagName: string): Set { - const set = new Set(); - - tags.forEach((tag) => { - if (tag[0] === tagName) { - set.add(tag[1]); - } - }); - - return set; -} - -/** Check if the tag exists by its name and value. */ -function hasTag(tags: string[][], tag: string[]): boolean { - return tags.some(([name, value]) => name === tag[0] && value === tag[1]); -} - -/** Delete all occurences of the tag by its name/value pair. */ -function deleteTag(tags: readonly string[][], tag: string[]): string[][] { - return tags.filter(([name, value]) => !(name === tag[0] && value === tag[1])); -} - -/** Add a tag to the list, replacing the name/value pair if it already exists. */ -function addTag(tags: readonly string[][], tag: string[]): string[][] { - const tagIndex = tags.findIndex(([name, value]) => name === tag[0] && value === tag[1]); - if (tagIndex === -1) { - return [...tags, tag]; - } else { - return [...tags.slice(0, tagIndex), tag, ...tags.slice(tagIndex + 1)]; - } -} - -const isReplyTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'reply'; -const isRootTag = (tag: string[]) => tag[0] === 'e' && tag[3] === 'root'; -const isLegacyReplyTag = (tag: string[]) => tag[0] === 'e' && !tag[3]; - -function findReplyTag(tags: string[][]) { - return tags.find(isReplyTag) || tags.find(isRootTag) || tags.findLast(isLegacyReplyTag); -} - -export { addTag, deleteTag, findReplyTag, getTagSet, hasTag }; diff --git a/src/upload.ts b/src/upload.ts deleted file mode 100644 index 632dbab..0000000 --- a/src/upload.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Conf } from '@/config.ts'; -import { insertUnattachedMedia } from '@/db/unattached-media.ts'; -import { configUploader as uploader } from '@/uploaders/config.ts'; - -interface FileMeta { - pubkey: string; - description?: string; -} - -/** Upload a file, track it in the database, and return the resulting media object. */ -async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal) { - const { name, type, size } = file; - const { pubkey, description } = meta; - - if (file.size > Conf.maxUploadSize) { - throw new Error('File size is too large.'); - } - - const { url } = await uploader.upload(file, { signal }); - - return insertUnattachedMedia({ - pubkey, - url, - data: { - name, - size, - description, - mime: type, - }, - }); -} - -export { uploadFile }; diff --git a/src/uploaders/DenoUploader.ts b/src/uploaders/DenoUploader.ts new file mode 100644 index 0000000..fd30d8c --- /dev/null +++ b/src/uploaders/DenoUploader.ts @@ -0,0 +1,46 @@ +import { join } from 'node:path'; + +import { NUploader } from '@nostrify/nostrify'; +import { crypto } from '@std/crypto'; +import { encodeHex } from '@std/encoding/hex'; +import { extensionsByType } from '@std/media-types'; + +export interface DenoUploaderOpts { + baseUrl: string; + dir: string; +} + +/** Local Deno filesystem uploader. */ +export class DenoUploader implements NUploader { + baseUrl: string; + dir: string; + + constructor(opts: DenoUploaderOpts) { + this.baseUrl = opts.baseUrl; + this.dir = opts.dir; + } + + async upload(file: File): Promise<[['url', string], ...string[][]]> { + const sha256 = encodeHex(await crypto.subtle.digest('SHA-256', file.stream())); + const ext = extensionsByType(file.type)?.[0] ?? 'bin'; + const filename = `${sha256}.${ext}`; + + await Deno.mkdir(this.dir, { recursive: true }); + await Deno.writeFile(join(this.dir, filename), file.stream()); + + const url = new URL(this.baseUrl); + const path = url.pathname === '/' ? filename : join(url.pathname, filename); + + return [ + ['url', new URL(path, url).toString()], + ['m', file.type], + ['x', sha256], + ['size', file.size.toString()], + ]; + } + + async delete(filename: string) { + const path = join(this.dir, filename); + await Deno.remove(path); + } +} diff --git a/src/uploaders/IPFSUploader.ts b/src/uploaders/IPFSUploader.ts new file mode 100644 index 0000000..7bf5165 --- /dev/null +++ b/src/uploaders/IPFSUploader.ts @@ -0,0 +1,69 @@ +import { NUploader } from '@nostrify/nostrify'; +import { z } from 'zod'; + +export interface IPFSUploaderOpts { + baseUrl: string; + apiUrl?: string; + fetch?: typeof fetch; +} + +/** + * IPFS uploader. It expects an IPFS node up and running. + * It will try to connect to `http://localhost:5001` by default, + * and upload the file using the REST API. + */ +export class IPFSUploader implements NUploader { + private baseUrl: string; + private apiUrl: string; + private fetch: typeof fetch; + + constructor(opts: IPFSUploaderOpts) { + this.baseUrl = opts.baseUrl; + this.apiUrl = opts.apiUrl ?? 'http://localhost:5001'; + this.fetch = opts.fetch ?? globalThis.fetch; + } + + async upload(file: File, opts?: { signal?: AbortSignal }): Promise<[['url', string], ...string[][]]> { + const url = new URL('/api/v0/add', this.apiUrl); + + const formData = new FormData(); + formData.append('file', file); + + const response = await this.fetch(url, { + method: 'POST', + body: formData, + signal: opts?.signal, + }); + + const { Hash: cid } = IPFSUploader.schema().parse(await response.json()); + + return [ + ['url', new URL(`/ipfs/${cid}`, this.baseUrl).toString()], + ['m', file.type], + ['cid', cid], + ['size', file.size.toString()], + ]; + } + + async delete(cid: string, opts?: { signal?: AbortSignal }): Promise { + const url = new URL('/api/v0/pin/rm', this.apiUrl); + + const query = new URLSearchParams(); + query.set('arg', cid); + url.search = query.toString(); + + await this.fetch(url, { + method: 'POST', + signal: opts?.signal, + }); + } + + /** Response schema for POST `/api/v0/add`. */ + private static schema() { + return z.object({ + Name: z.string(), + Hash: z.string(), + Size: z.string(), + }); + } +} diff --git a/src/uploaders/S3Uploader.ts b/src/uploaders/S3Uploader.ts new file mode 100644 index 0000000..b74796a --- /dev/null +++ b/src/uploaders/S3Uploader.ts @@ -0,0 +1,59 @@ +import { join } from 'node:path'; + +import { S3Client } from '@bradenmacdonald/s3-lite-client'; +import { NUploader } from '@nostrify/nostrify'; +import { crypto } from '@std/crypto'; +import { encodeHex } from '@std/encoding/hex'; +import { extensionsByType } from '@std/media-types'; + +import { Conf } from '@/config.ts'; + +export interface S3UploaderOpts { + endPoint: string; + region: string; + accessKey?: string; + secretKey?: string; + bucket?: string; + pathStyle?: boolean; + port?: number; + sessionToken?: string; + useSSL?: boolean; +} + +/** S3-compatible uploader for AWS, Wasabi, DigitalOcean Spaces, and more. */ +export class S3Uploader implements NUploader { + private client: S3Client; + + constructor(opts: S3UploaderOpts) { + this.client = new S3Client(opts); + } + + async upload(file: File): Promise<[['url', string], ...string[][]]> { + const sha256 = encodeHex(await crypto.subtle.digest('SHA-256', file.stream())); + const ext = extensionsByType(file.type)?.[0] ?? 'bin'; + const filename = `${sha256}.${ext}`; + + await this.client.putObject(filename, file.stream(), { + metadata: { + 'Content-Type': file.type, + 'x-amz-acl': 'public-read', + }, + }); + + const { pathStyle, bucket } = Conf.s3; + + const path = (pathStyle && bucket) ? join(bucket, filename) : filename; + const url = new URL(path, Conf.mediaDomain).toString(); + + return [ + ['url', url], + ['m', file.type], + ['x', sha256], + ['size', file.size.toString()], + ]; + } + + async delete(objectName: string) { + await this.client.deleteObject(objectName); + } +} diff --git a/src/uploaders/config.ts b/src/uploaders/config.ts deleted file mode 100644 index 5b4c7af..0000000 --- a/src/uploaders/config.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Conf } from '@/config.ts'; - -import { ipfsUploader } from '@/uploaders/ipfs.ts'; -import { localUploader } from '@/uploaders/local.ts'; -import { s3Uploader } from '@/uploaders/s3.ts'; - -import type { Uploader } from './types.ts'; - -/** Meta-uploader determined from configuration. */ -const configUploader: Uploader = { - upload(file, opts) { - return uploader().upload(file, opts); - }, - delete(id, opts) { - return uploader().delete(id, opts); - }, -}; - -/** Get the uploader module based on configuration. */ -function uploader() { - switch (Conf.uploader) { - case 's3': - return s3Uploader; - case 'ipfs': - return ipfsUploader; - case 'local': - return localUploader; - default: - throw new Error('No `DITTO_UPLOADER` configured. Uploads are disabled.'); - } -} - -export { configUploader }; diff --git a/src/uploaders/ipfs.ts b/src/uploaders/ipfs.ts deleted file mode 100644 index 21619b5..0000000 --- a/src/uploaders/ipfs.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { z } from 'zod'; - -import { Conf } from '@/config.ts'; -import { fetchWorker } from '@/workers/fetch.ts'; - -import type { Uploader } from './types.ts'; - -/** Response schema for POST `/api/v0/add`. */ -const ipfsAddResponseSchema = z.object({ - Name: z.string(), - Hash: z.string(), - Size: z.string(), -}); - -/** - * IPFS uploader. It expects an IPFS node up and running. - * It will try to connect to `http://localhost:5001` by default, - * and upload the file using the REST API. - */ -const ipfsUploader: Uploader = { - async upload(file, opts) { - const url = new URL('/api/v0/add', Conf.ipfs.apiUrl); - - const formData = new FormData(); - formData.append('file', file); - - const response = await fetchWorker(url, { - method: 'POST', - body: formData, - signal: opts?.signal, - }); - - const { Hash: cid } = ipfsAddResponseSchema.parse(await response.json()); - - return { - id: cid, - cid, - url: new URL(`/ipfs/${cid}`, Conf.mediaDomain).toString(), - }; - }, - async delete(cid, opts) { - const url = new URL('/api/v0/pin/rm', Conf.ipfs.apiUrl); - - const query = new URLSearchParams(); - query.set('arg', cid); - - url.search = query.toString(); - - await fetchWorker(url, { - method: 'POST', - signal: opts?.signal, - }); - }, -}; - -export { ipfsUploader }; diff --git a/src/uploaders/local.ts b/src/uploaders/local.ts deleted file mode 100644 index a2381a3..0000000 --- a/src/uploaders/local.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { join } from 'node:path'; - -import { crypto } from '@std/crypto'; -import { encodeHex } from '@std/encoding/hex'; -import { extensionsByType } from '@std/media-types'; - -import { Conf } from '@/config.ts'; - -import type { Uploader } from './types.ts'; - -/** Local filesystem uploader. */ -const localUploader: Uploader = { - async upload(file) { - const sha256 = encodeHex(await crypto.subtle.digest('SHA-256', file.stream())); - const ext = extensionsByType(file.type)?.[0] ?? 'bin'; - const filename = `${sha256}.${ext}`; - - await Deno.mkdir(Conf.uploadsDir, { recursive: true }); - await Deno.writeFile(join(Conf.uploadsDir, filename), file.stream()); - - const { mediaDomain } = Conf; - const url = new URL(mediaDomain); - const path = url.pathname === '/' ? filename : join(url.pathname, filename); - - return { - id: filename, - sha256, - url: new URL(path, url).toString(), - }; - }, - async delete(id) { - await Deno.remove(join(Conf.uploadsDir, id)); - }, -}; - -export { localUploader }; diff --git a/src/uploaders/s3.ts b/src/uploaders/s3.ts deleted file mode 100644 index 267d817..0000000 --- a/src/uploaders/s3.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { join } from 'node:path'; - -import { S3Client } from '@bradenmacdonald/s3-lite-client'; -import { crypto } from '@std/crypto'; -import { encodeHex } from '@std/encoding/hex'; -import { extensionsByType } from '@std/media-types'; - -import { Conf } from '@/config.ts'; - -import type { Uploader } from './types.ts'; - -/** S3-compatible uploader for AWS, Wasabi, DigitalOcean Spaces, and more. */ -const s3Uploader: Uploader = { - async upload(file) { - const sha256 = encodeHex(await crypto.subtle.digest('SHA-256', file.stream())); - const ext = extensionsByType(file.type)?.[0] ?? 'bin'; - const filename = `${sha256}.${ext}`; - - await client().putObject(filename, file.stream(), { - metadata: { - 'Content-Type': file.type, - 'x-amz-acl': 'public-read', - }, - }); - - const { pathStyle, bucket } = Conf.s3; - const path = (pathStyle && bucket) ? join(bucket, filename) : filename; - - return { - id: filename, - sha256, - url: new URL(path, Conf.mediaDomain).toString(), - }; - }, - async delete(id) { - await client().deleteObject(id); - }, -}; - -/** Build S3 client from config. */ -function client() { - return new S3Client({ ...Conf.s3 }); -} - -export { s3Uploader }; diff --git a/src/uploaders/types.ts b/src/uploaders/types.ts deleted file mode 100644 index c514ad1..0000000 --- a/src/uploaders/types.ts +++ /dev/null @@ -1,21 +0,0 @@ -/** Modular uploader interface, to support uploading to different backends. */ -interface Uploader { - /** Upload the file to the backend. */ - upload(file: File, opts?: { signal?: AbortSignal }): Promise; - /** Delete the file from the backend. */ - delete(cid: string, opts?: { signal?: AbortSignal }): Promise; -} - -/** Return value from the uploader after uploading a file. */ -interface UploadResult { - /** File ID specific to the uploader, so it can later be referenced or deleted. */ - id: string; - /** URL where the file can be accessed. */ - url: string; - /** SHA-256 hash of the file. */ - sha256?: string; - /** IPFS CID of the file. */ - cid?: string; -} - -export type { Uploader }; diff --git a/src/utils/lnurl.ts b/src/utils/lnurl.ts index ea5ce8a..af344f2 100644 --- a/src/utils/lnurl.ts +++ b/src/utils/lnurl.ts @@ -28,8 +28,12 @@ function getLnurl({ lud06, lud16 }: { lud06?: string; lud16?: string }, limit?: if (lud16) { const [name, host] = lud16.split('@'); if (name && host) { - const url = new URL(`/.well-known/lnurlp/${name}`, `https://${host}`); - return LNURL.encode(url, limit); + try { + const url = new URL(`/.well-known/lnurlp/${name}`, `https://${host}`); + return LNURL.encode(url, limit); + } catch { + return; + } } } } diff --git a/src/utils/media.test.ts b/src/utils/media.test.ts new file mode 100644 index 0000000..e88e97d --- /dev/null +++ b/src/utils/media.test.ts @@ -0,0 +1,17 @@ +import { assertEquals } from '@std/assert'; + +import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts'; + +Deno.test('getUrlMediaType', () => { + assertEquals(getUrlMediaType('https://example.com/image.png'), 'image/png'); + assertEquals(getUrlMediaType('https://example.com/index.html'), 'text/html'); + assertEquals(getUrlMediaType('https://example.com/yolo'), undefined); + assertEquals(getUrlMediaType('https://example.com/'), undefined); +}); + +Deno.test('isPermittedMediaType', () => { + assertEquals(isPermittedMediaType('image/png', ['image', 'video']), true); + assertEquals(isPermittedMediaType('video/webm', ['image', 'video']), true); + assertEquals(isPermittedMediaType('audio/ogg', ['image', 'video']), false); + assertEquals(isPermittedMediaType('application/json', ['image', 'video']), false); +}); diff --git a/src/utils/media.ts b/src/utils/media.ts new file mode 100644 index 0000000..9c0ea9e --- /dev/null +++ b/src/utils/media.ts @@ -0,0 +1,24 @@ +import { typeByExtension } from '@std/media-types'; + +/** Get media type of the filename in the URL by its extension, if any. */ +export function getUrlMediaType(url: string): string | undefined { + try { + const { pathname } = new URL(url); + const ext = pathname.split('.').pop() ?? ''; + return typeByExtension(ext); + } catch { + return undefined; + } +} + +/** + * Check if the base type matches any of the permitted types. + * + * ```ts + * isPermittedMediaType('image/png', ['image', 'video']); // true + * ``` + */ +export function isPermittedMediaType(mediaType: string, permitted: string[]): boolean { + const [baseType, _subType] = mediaType.split('/'); + return permitted.includes(baseType); +} diff --git a/src/utils/note.test.ts b/src/utils/note.test.ts new file mode 100644 index 0000000..b351dbf --- /dev/null +++ b/src/utils/note.test.ts @@ -0,0 +1,35 @@ +import { assertEquals } from '@std/assert'; + +import { eventFixture } from '@/test.ts'; +import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; + +Deno.test('parseNoteContent', () => { + const { html, links, firstUrl } = parseNoteContent('Hello, world!'); + assertEquals(html, 'Hello, world!'); + assertEquals(links, []); + assertEquals(firstUrl, undefined); +}); + +Deno.test('getMediaLinks', () => { + const links = [ + { href: 'https://example.com/image.png' }, + { href: 'https://example.com/index.html' }, + { href: 'https://example.com/yolo' }, + { href: 'https://example.com/' }, + ]; + const mediaLinks = getMediaLinks(links); + assertEquals(mediaLinks, [[ + ['url', 'https://example.com/image.png'], + ['m', 'image/png'], + ]]); +}); + +Deno.test('stripimeta', async () => { + const { content, tags } = await eventFixture('event-imeta'); + + const stripped = stripimeta(content, tags); + const expected = + `Today we were made aware of multiple Fediverse blog posts incorrectly attributing “vote Trump” spam on Bluesky to the Mostr.pub Bridge. \n\nThis spam is NOT coming from Mostr. From the screenshots used in these blogs, it's clear the spam is coming from an entirely different bridge called momostr.pink. This bridge is not affiliated with Mostr, and is not even a fork of Mostr. We appreciate that the authors of these posts responded quickly to us and have since corrected the blogs. \n\nMostr.pub uses stirfry policies for anti-spam filtering. This includes an anti-duplication policy that prevents spam like the recent “vote Trump” posts we’ve seen repeated over and over. \n\nIt is important to note WHY there are multiple bridges, though. \n\nWhen Mostr.pub launched, multiple major servers immediately blocked Mostr, including Mastodon.social. The moderators of Mastodon.social claimed that this was because Nostr was unregulated, and suggested to one user that if they want to bridge their account they should host their own bridge.\n\nThat is exactly what momostr.pink, the source of this spam, has done. \n\nThe obvious response to the censorship of the Mostr Bridge is to build more bridges. \n\nWhile we have opted for pro-social policies that aim to reduce spam and build better connections between decentralized platforms, other bridges built to get around censorship of the Mostr Bridge may not — as we’re already seeing.\n\nThere will inevitably be multiple bridges, and we’re working on creating solutions to the problems that arise from that. In the meantime, if the Fediverse could do itself a favor and chill with the censorship for two seconds, we might not have so many problems. `; + + assertEquals(stripped, expected); +}); diff --git a/src/note.ts b/src/utils/note.ts similarity index 64% rename from src/note.ts rename to src/utils/note.ts index 5603c53..0d1e3b2 100644 --- a/src/note.ts +++ b/src/utils/note.ts @@ -4,8 +4,7 @@ import linkify from 'linkifyjs'; import { nip19, nip21 } from 'nostr-tools'; import { Conf } from '@/config.ts'; -import { mime } from '@/deps.ts'; -import { type DittoAttachment } from '@/views/mastodon/attachments.ts'; +import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts'; linkify.registerCustomProtocol('nostr', true); linkify.registerCustomProtocol('wss'); @@ -18,7 +17,7 @@ const linkifyOpts: linkify.Opts = { return `#${tag}`; }, url: ({ content }) => { - if (nip21.test(content)) { + try { const { decoded } = nip21.parse(content); const pubkey = getDecodedPubkey(decoded); if (pubkey) { @@ -28,7 +27,7 @@ const linkifyOpts: linkify.Opts = { } else { return ''; } - } else { + } catch { return `${content}`; } }, @@ -58,20 +57,42 @@ function parseNoteContent(content: string): ParsedNoteContent { }; } -function getMediaLinks(links: Link[]): DittoAttachment[] { - return links.reduce((acc, link) => { - const mimeType = getUrlMimeType(link.href); - if (!mimeType) return acc; +/** Remove imeta links. */ +function stripimeta(content: string, tags: string[][]): string { + const imeta = tags.filter(([name]) => name === 'imeta'); - const [baseType, _subType] = mimeType.split('/'); + if (!imeta.length) { + return content; + } - if (['audio', 'image', 'video'].includes(baseType)) { - acc.push({ - url: link.href, - data: { - mime: mimeType, - }, - }); + const urls = new Set( + imeta.map(([, ...values]) => values.map((v) => v.split(' ')).find(([name]) => name === 'url')?.[1]), + ); + + const lines = content.split('\n').reverse(); + + for (const line of [...lines]) { + if (line === '' || urls.has(line)) { + lines.splice(0, 1); + } else { + break; + } + } + + return lines.reverse().join('\n'); +} + +/** Returns a matrix of tags. Each item is a list of NIP-94 tags representing a file. */ +function getMediaLinks(links: Pick[]): string[][][] { + return links.reduce((acc, link) => { + const mediaType = getUrlMediaType(link.href); + if (!mediaType) return acc; + + if (isPermittedMediaType(mediaType, ['audio', 'image', 'video'])) { + acc.push([ + ['url', link.href], + ['m', mediaType], + ]); } return acc; @@ -79,7 +100,7 @@ function getMediaLinks(links: Link[]): DittoAttachment[] { } function isNonMediaLink({ href }: Link): boolean { - return /^https?:\/\//.test(href) && !getUrlMimeType(href); + return /^https?:\/\//.test(href) && !getUrlMediaType(href); } /** Ensures the Link is a URL so it can be parsed. */ @@ -87,16 +108,6 @@ function isLinkURL(link: Link): boolean { return link.type === 'url'; } -/** `npm:mime` treats `.com` as a file extension, so parse the full URL to get its path first. */ -function getUrlMimeType(url: string): string | undefined { - try { - const { pathname } = new URL(url); - return mime.getType(pathname) || undefined; - } catch (_e) { - return undefined; - } -} - /** Get pubkey from decoded bech32 entity, or undefined if not applicable. */ function getDecodedPubkey(decoded: nip19.DecodeResult): string | undefined { switch (decoded.type) { @@ -107,4 +118,4 @@ function getDecodedPubkey(decoded: nip19.DecodeResult): string | undefined { } } -export { getMediaLinks, parseNoteContent }; +export { getMediaLinks, parseNoteContent, stripimeta }; diff --git a/src/utils/rsa.ts b/src/utils/rsa.ts deleted file mode 100644 index 6942c43..0000000 --- a/src/utils/rsa.ts +++ /dev/null @@ -1,32 +0,0 @@ -import * as secp from '@noble/secp256k1'; -import { LRUCache } from 'lru-cache'; - -import { Conf } from '@/config.ts'; -import { generateSeededRsa, publicKeyToPem } from '@/deps.ts'; - -const opts = { - bits: 2048, -}; - -const rsaCache = new LRUCache>({ max: 1000 }); - -async function buildSeed(pubkey: string): Promise { - const key = await Conf.cryptoKey; - const data = new TextEncoder().encode(pubkey); - const signature = await window.crypto.subtle.sign('HMAC', key, data); - return secp.etc.bytesToHex(new Uint8Array(signature)); -} - -async function getPublicKeyPem(pubkey: string): Promise { - const cached = await rsaCache.get(pubkey); - if (cached) return cached; - - const seed = await buildSeed(pubkey); - const { publicKey } = await generateSeededRsa(seed, opts); - const promise = publicKeyToPem(publicKey); - - rsaCache.set(pubkey, promise); - return promise; -} - -export { getPublicKeyPem }; diff --git a/src/utils/tags.test.ts b/src/utils/tags.test.ts new file mode 100644 index 0000000..43e1235 --- /dev/null +++ b/src/utils/tags.test.ts @@ -0,0 +1,45 @@ +import { assertEquals } from '@std/assert'; + +import { addTag, deleteTag, findQuoteTag, findReplyTag, getTagSet, hasTag } from './tags.ts'; + +Deno.test('addTag', () => { + const tags = [['p', 'alex']]; + assertEquals(addTag(tags, ['p', 'alex']), [['p', 'alex']]); + assertEquals(addTag(tags, ['p', 'fiatjaf']), [['p', 'alex'], ['p', 'fiatjaf']]); +}); + +Deno.test('deleteTag', () => { + const tags = [['p', 'alex'], ['p', 'fiatjaf']]; + assertEquals(deleteTag(tags, ['p', 'alex']), [['p', 'fiatjaf']]); + assertEquals(deleteTag(tags, ['p', 'fiatjaf']), [['p', 'alex']]); +}); + +Deno.test('findQuoteTag', () => { + assertEquals(findQuoteTag([['q', '123']]), ['q', '123']); + assertEquals(findQuoteTag([['e', '', '', 'mention', '456']]), ['e', '', '', 'mention', '456']); + assertEquals(findQuoteTag([['e', '', '', 'mention', '456'], ['q', '123']]), ['q', '123']); + assertEquals(findQuoteTag([['q', '123'], ['e', '', '', 'mention', '456']]), ['q', '123']); +}); + +Deno.test('findReplyTag', () => { + const root = ['e', '123', '', 'root']; + const reply = ['e', '456', '', 'reply']; + + assertEquals(findReplyTag([root]), root); + assertEquals(findReplyTag([reply]), reply); + assertEquals(findReplyTag([root, reply]), reply); + assertEquals(findReplyTag([reply, root]), reply); + assertEquals(findReplyTag([['e', '321'], ['e', '789']]), ['e', '789']); + assertEquals(findReplyTag([reply, ['e', '789']]), reply); +}); + +Deno.test('getTagSet', () => { + const tags = [['p', 'alex'], ['p', 'fiatjaf'], ['p', 'alex']]; + assertEquals(getTagSet(tags, 'p'), new Set(['alex', 'fiatjaf'])); +}); + +Deno.test('hasTag', () => { + const tags = [['p', 'alex']]; + assertEquals(hasTag(tags, ['p', 'alex']), true); + assertEquals(hasTag(tags, ['p', 'fiatjaf']), false); +}); diff --git a/src/utils/tags.ts b/src/utils/tags.ts new file mode 100644 index 0000000..6375e81 --- /dev/null +++ b/src/utils/tags.ts @@ -0,0 +1,71 @@ +/** Get the values for a tag in a `Set`. */ +function getTagSet(tags: string[][], tagName: string): Set { + const set = new Set(); + + tags.forEach((tag) => { + if (tag[0] === tagName) { + set.add(tag[1]); + } + }); + + return set; +} + +/** Check if the tag exists by its name and value. */ +function hasTag(tags: string[][], tag: string[]): boolean { + return tags.some(([name, value]) => name === tag[0] && value === tag[1]); +} + +/** Delete all occurences of the tag by its name/value pair. */ +function deleteTag(tags: readonly string[][], tag: string[]): string[][] { + return tags.filter(([name, value]) => !(name === tag[0] && value === tag[1])); +} + +/** Add a tag to the list, replacing the name/value pair if it already exists. */ +function addTag(tags: readonly string[][], tag: string[]): string[][] { + const tagIndex = tags.findIndex(([name, value]) => name === tag[0] && value === tag[1]); + if (tagIndex === -1) { + return [...tags, tag]; + } else { + return [...tags.slice(0, tagIndex), tag, ...tags.slice(tagIndex + 1)]; + } +} + +/** Tag is a NIP-10 root tag. */ +function isRootTag(tag: string[]): tag is ['e', string, string, 'root', ...string[]] { + return tag[0] === 'e' && tag[3] === 'root'; +} + +/** Tag is a NIP-10 reply tag. */ +function isReplyTag(tag: string[]): tag is ['e', string, string, 'reply', ...string[]] { + return tag[0] === 'e' && tag[3] === 'reply'; +} + +/** Tag is a legacy "e" tag with a "mention" marker. */ +function isLegacyQuoteTag(tag: string[]): tag is ['e', string, string, 'mention', ...string[]] { + return tag[0] === 'e' && tag[3] === 'mention'; +} + +/** Tag is an "e" tag without a NIP-10 marker. */ +function isLegacyReplyTag(tag: string[]): tag is ['e', string, string] { + return tag[0] === 'e' && !tag[3]; +} + +/** Tag is a "q" tag. */ +function isQuoteTag(tag: string[]): tag is ['q', ...string[]] { + return tag[0] === 'q'; +} + +/** Get the "e" tag for the event being replied to, first according to the NIPs then falling back to the legacy way. */ +function findReplyTag(tags: string[][]): ['e', ...string[]] | undefined { + return tags.find(isReplyTag) || tags.find(isRootTag) || tags.findLast(isLegacyReplyTag); +} + +/** Get the "q" tag, falling back to the legacy "e" tag with a "mention" marker. */ +function findQuoteTag( + tags: string[][], +): ['q', ...string[]] | ['e', string, string, 'mention', ...string[]] | undefined { + return tags.find(isQuoteTag) || tags.find(isLegacyQuoteTag); +} + +export { addTag, deleteTag, findQuoteTag, findReplyTag, getTagSet, hasTag }; diff --git a/src/utils/unfurl.ts b/src/utils/unfurl.ts index b028be5..e5ae428 100644 --- a/src/utils/unfurl.ts +++ b/src/utils/unfurl.ts @@ -1,8 +1,8 @@ import TTLCache from '@isaacs/ttlcache'; import Debug from '@soapbox/stickynotes/debug'; +import DOMPurify from 'isomorphic-dompurify'; import { unfurl } from 'unfurl.js'; -import { sanitizeHtml } from '@/deps.ts'; import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; @@ -44,11 +44,9 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise { + const uploader = c.get('uploader'); + if (!uploader) { + throw new HTTPException(500, { + res: c.json({ error: 'No uploader configured.' }), + }); + } + + const { pubkey, description } = meta; + + if (file.size > Conf.maxUploadSize) { + throw new Error('File size is too large.'); + } + + const tags = await uploader.upload(file, { signal }); + const url = tags[0][1]; + + if (description) { + tags.push(['alt', description]); + } + + return insertUnattachedMedia({ + id: crypto.randomUUID(), + pubkey, + url, + data: tags, + uploaded_at: Date.now(), + }); +} diff --git a/src/views.ts b/src/views.ts index a737542..863dd30 100644 --- a/src/views.ts +++ b/src/views.ts @@ -5,6 +5,7 @@ import { renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; import { paginated, paginationSchema } from '@/utils/api.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; +import { accountFromPubkey } from '@/views/mastodon/accounts.ts'; /** Render account objects for the author of each event. */ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal = AbortSignal.timeout(1000)) { @@ -24,7 +25,13 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal .then((events) => hydrateEvents({ events, store, signal })); const accounts = await Promise.all( - authors.map((event) => renderAccount(event)), + Array.from(pubkeys).map(async (pubkey) => { + const event = authors.find((event) => event.pubkey === pubkey); + if (event) { + return await renderAccount(event); + } + return await accountFromPubkey(pubkey); + }), ); return paginated(c, events, accounts); @@ -39,7 +46,13 @@ async function renderAccounts(c: AppContext, authors: string[], signal = AbortSi .then((events) => hydrateEvents({ events, store, signal })); const accounts = await Promise.all( - events.map((event) => renderAccount(event)), + authors.map(async (pubkey) => { + const event = events.find((event) => event.pubkey === pubkey); + if (event) { + return await renderAccount(event); + } + return await accountFromPubkey(pubkey); + }), ); return paginated(c, events, accounts); diff --git a/src/views/activitypub/actor.ts b/src/views/activitypub/actor.ts deleted file mode 100644 index cfd40ba..0000000 --- a/src/views/activitypub/actor.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { NSchema as n } from '@nostrify/nostrify'; - -import { Conf } from '@/config.ts'; -import { getPublicKeyPem } from '@/utils/rsa.ts'; - -import type { NostrEvent } from '@nostrify/nostrify'; -import type { Actor } from '@/schemas/activitypub.ts'; - -/** Nostr metadata event to ActivityPub actor. */ -async function renderActor(event: NostrEvent, username: string): Promise { - const content = n.json().pipe(n.metadata()).catch({}).parse(event.content); - - return { - type: 'Person', - id: Conf.local(`/users/${username}`), - name: content?.name || '', - preferredUsername: username, - inbox: Conf.local(`/users/${username}/inbox`), - followers: Conf.local(`/users/${username}/followers`), - following: Conf.local(`/users/${username}/following`), - outbox: Conf.local(`/users/${username}/outbox`), - icon: content.picture - ? { - type: 'Image', - url: content.picture, - } - : undefined, - image: content.banner - ? { - type: 'Image', - url: content.banner, - } - : undefined, - summary: content.about ?? '', - attachment: [], - tag: [], - publicKey: { - id: Conf.local(`/users/${username}#main-key`), - owner: Conf.local(`/users/${username}`), - publicKeyPem: await getPublicKeyPem(event.pubkey), - }, - endpoints: { - sharedInbox: Conf.local('/inbox'), - }, - }; -} - -export { renderActor }; diff --git a/src/views/mastodon/accounts.ts b/src/views/mastodon/accounts.ts index e00856e..69cf1db 100644 --- a/src/views/mastodon/accounts.ts +++ b/src/views/mastodon/accounts.ts @@ -1,8 +1,8 @@ import { NSchema as n } from '@nostrify/nostrify'; +import { escape } from 'entities'; import { nip19, UnsignedEvent } from 'nostr-tools'; import { Conf } from '@/config.ts'; -import { lodash } from '@/deps.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getLnurl } from '@/utils/lnurl.ts'; import { nip05Cache } from '@/utils/nip05.ts'; @@ -53,7 +53,7 @@ async function renderAccount( header_static: banner, last_status_at: null, locked: false, - note: lodash.escape(about), + note: about ? escape(about) : '', roles: [], source: withSource ? { @@ -79,6 +79,7 @@ async function renderAccount( is_admin: role === 'admin', is_moderator: ['admin', 'moderator'].includes(role), is_local: parsed05?.domain === Conf.url.host, + settings_store: undefined as unknown, }, nostr: { pubkey, diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 3ea989e..2d65804 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -1,20 +1,41 @@ -import * as TypeFest from 'type-fest'; +import { getUrlMediaType } from '@/utils/media.ts'; -import { UnattachedMedia } from '@/db/unattached-media.ts'; +/** Render Mastodon media attachment. */ +function renderAttachment(media: { id?: string; data: string[][] }) { + const { id, data: tags } = media; -type DittoAttachment = TypeFest.SetOptional; + const url = tags.find(([name]) => name === 'url')?.[1]; + + const m = tags.find(([name]) => name === 'm')?.[1] ?? getUrlMediaType(url!); + const alt = tags.find(([name]) => name === 'alt')?.[1]; + const cid = tags.find(([name]) => name === 'cid')?.[1]; + const dim = tags.find(([name]) => name === 'dim')?.[1]; + const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; + + if (!url) return; + + const [width, height] = dim?.split('x').map(Number) ?? [null, null]; + + const meta = (typeof width === 'number' && typeof height === 'number') + ? { + original: { + width, + height, + aspect: width / height, + }, + } + : undefined; -function renderAttachment(media: DittoAttachment) { - const { id, data, url } = media; return { - id: id ?? url ?? data.cid, - type: getAttachmentType(data.mime ?? ''), + id: id ?? url, + type: getAttachmentType(m ?? ''), url, preview_url: url, remote_url: null, - description: data.description ?? '', - blurhash: data.blurhash || null, - cid: data.cid, + description: alt ?? '', + blurhash: blurhash || null, + meta, + cid: cid, }; } @@ -32,4 +53,4 @@ function getAttachmentType(mime: string): string { } } -export { type DittoAttachment, renderAttachment }; +export { renderAttachment }; diff --git a/src/views/mastodon/relationships.ts b/src/views/mastodon/relationships.ts index 2f8ffdd..425ea56 100644 --- a/src/views/mastodon/relationships.ts +++ b/src/views/mastodon/relationships.ts @@ -1,5 +1,5 @@ import { Storages } from '@/storages.ts'; -import { hasTag } from '@/tags.ts'; +import { hasTag } from '@/utils/tags.ts'; async function renderRelationship(sourcePubkey: string, targetPubkey: string) { const db = await Storages.db(); diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index 776f016..cc7cc36 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -1,18 +1,16 @@ -import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; -import { isCWTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts'; +import { NostrEvent } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { getMediaLinks, parseNoteContent } from '@/note.ts'; import { Storages } from '@/storages.ts'; -import { findReplyTag } from '@/tags.ts'; import { nostrDate } from '@/utils.ts'; +import { getMediaLinks, parseNoteContent, stripimeta } from '@/utils/note.ts'; +import { findQuoteTag, findReplyTag } from '@/utils/tags.ts'; import { unfurlCardCached } from '@/utils/unfurl.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; -import { DittoAttachment, renderAttachment } from '@/views/mastodon/attachments.ts'; +import { renderAttachment } from '@/views/mastodon/attachments.ts'; import { renderEmojis } from '@/views/mastodon/emojis.ts'; -import { mediaDataSchema } from '@/schemas/nostr.ts'; interface RenderStatusOpts { viewerPubkey?: string; @@ -31,6 +29,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< : await accountFromPubkey(event.pubkey); const replyTag = findReplyTag(event.tags); + const quoteTag = findQuoteTag(event.tags); const mentionedPubkeys = [ ...new Set( @@ -47,7 +46,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< [{ kinds: [0], authors: mentionedPubkeys, limit: mentionedPubkeys.length }], ); - const { html, links, firstUrl } = parseNoteContent(event.content); + const { html, links, firstUrl } = parseNoteContent(stripimeta(event.content, event.tags)); const [mentions, card, relatedEvents] = await Promise .all([ @@ -74,16 +73,14 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const content = buildInlineRecipients(mentions) + html; - const cw = event.tags.find(isCWTag); - const subject = event.tags.find((tag) => tag[0] === 'subject'); + const cw = event.tags.find(([name]) => name === 'content-warning'); + const subject = event.tags.find(([name]) => name === 'subject'); - const mediaLinks = getMediaLinks(links); + const imeta: string[][][] = event.tags + .filter(([name]) => name === 'imeta') + .map(([_, ...entries]) => entries.map((entry) => entry.split(' '))); - const mediaTags: DittoAttachment[] = event.tags - .filter((tag) => tag[0] === 'media') - .map(([_, url, json]) => ({ url, data: n.json().pipe(mediaDataSchema).parse(json) })); - - const media = [...mediaLinks, ...mediaTags]; + const media = imeta.length ? imeta : getMediaLinks(links); return { id: event.id, @@ -91,7 +88,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< card, content, created_at: nostrDate(event.created_at).toISOString(), - in_reply_to_id: replyTag ? replyTag[1] : null, + in_reply_to_id: replyTag?.[1] ?? null, in_reply_to_account_id: null, sensitive: !!cw, spoiler_text: (cw ? cw[1] : subject?.[1]) || '', @@ -107,13 +104,13 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< pinned: Boolean(pinEvent), reblog: null, application: null, - media_attachments: media.map(renderAttachment), + media_attachments: media.map((m) => renderAttachment({ data: m })).filter(Boolean), mentions, tags: [], emojis: renderEmojis(event), poll: null, quote: !event.quote ? null : await renderStatus(event.quote, { depth: depth + 1 }), - quote_id: event.tags.find(([name]) => name === 'q')?.[1] ?? null, + quote_id: quoteTag?.[1] ?? null, uri: Conf.external(note), url: Conf.external(note), zapped: Boolean(zapEvent), @@ -123,8 +120,6 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< async function renderReblog(event: DittoEvent, opts: RenderStatusOpts) { const { viewerPubkey } = opts; - if (!event.author) return; - const repostId = event.tags.find(([name]) => name === 'e')?.[1]; if (!repostId) return; @@ -134,7 +129,7 @@ async function renderReblog(event: DittoEvent, opts: RenderStatusOpts) { return { id: event.id, - account: await renderAccount(event.author), + account: event.author ? await renderAccount(event.author) : await accountFromPubkey(event.pubkey), reblogged: true, reblog, }; diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 4e4bcae..d1368bc 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -1,4 +1,4 @@ -import 'deno-safe-fetch'; +import 'deno-safe-fetch/load'; import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { ReadOnlyPolicy } from '@nostrify/nostrify/policies'; import * as Comlink from 'comlink'; diff --git a/src/workers/trends.worker.ts b/src/workers/trends.worker.ts index b9f5f80..5a93fa3 100644 --- a/src/workers/trends.worker.ts +++ b/src/workers/trends.worker.ts @@ -1,5 +1,6 @@ import { NSchema } from '@nostrify/nostrify'; import * as Comlink from 'comlink'; +import { DB as Sqlite } from 'deno-sqlite'; import { hashtagSchema } from '@/schema.ts'; import { generateDateRange, Time } from '@/utils/time.ts';