diff --git a/src/app.ts b/src/app.ts index 2a36a83..7d9750a 100644 --- a/src/app.ts +++ b/src/app.ts @@ -29,6 +29,7 @@ import { import { appCredentialsController, createAppController } from './controllers/api/apps.ts'; import { emptyArrayController, emptyObjectController } from './controllers/api/fallback.ts'; import { instanceController } from './controllers/api/instance.ts'; +import { mediaController } from './controllers/api/media.ts'; import { notificationsController } from './controllers/api/notifications.ts'; import { createTokenController, oauthAuthorizeController, oauthController } from './controllers/api/oauth.ts'; import { frontendConfigController, updateConfigController } from './controllers/api/pleroma.ts'; @@ -56,7 +57,7 @@ import { nodeInfoController, nodeInfoSchemaController } from './controllers/well import { nostrController } from './controllers/well-known/nostr.ts'; import { webfingerController } from './controllers/well-known/webfinger.ts'; import { auth19, requirePubkey } from './middleware/auth19.ts'; -import { auth98, requireAdmin } from './middleware/auth98.ts'; +import { auth98, requireRole } from './middleware/auth98.ts'; interface AppEnv extends HonoEnv { Variables: { @@ -121,6 +122,9 @@ app.get('/api/v1/statuses/:id{[0-9a-f]{64}}', statusController); app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/favourite', favouriteController); app.post('/api/v1/statuses', requirePubkey, createStatusController); +app.post('/api/v1/media', requireRole('user', { validatePayload: false }), mediaController); +app.post('/api/v2/media', requireRole('user', { validatePayload: false }), mediaController); + app.get('/api/v1/timelines/home', requirePubkey, homeTimelineController); app.get('/api/v1/timelines/public', publicTimelineController); app.get('/api/v1/timelines/tag/:hashtag', hashtagTimelineController); @@ -137,7 +141,7 @@ app.get('/api/v1/trends', trendingTagsController); app.get('/api/v1/notifications', requirePubkey, notificationsController); app.get('/api/v1/favourites', requirePubkey, favouritesController); -app.post('/api/v1/pleroma/admin/config', requireAdmin, updateConfigController); +app.post('/api/v1/pleroma/admin/config', requireRole('admin'), updateConfigController); // Not (yet) implemented. app.get('/api/v1/bookmarks', emptyArrayController); diff --git a/src/config.ts b/src/config.ts index dc870b1..c92ad05 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,4 +1,4 @@ -import { dotenv, getPublicKey, nip19, secp } from '@/deps.ts'; +import { dotenv, getPublicKey, nip19, secp, z } from '@/deps.ts'; /** Load environment config from `.env` */ await dotenv.load({ @@ -42,7 +42,7 @@ const Conf = { const { protocol, host } = Conf.url; return `${protocol === 'https:' ? 'wss:' : 'ws:'}//${host}/relay`; }, - /** Domain of the Ditto server, including the protocol. */ + /** Origin of the Ditto server, including the protocol and port. */ get localDomain() { return Deno.env.get('LOCAL_DOMAIN') || 'http://localhost:8000'; }, @@ -58,22 +58,96 @@ const Conf = { get adminEmail() { return Deno.env.get('ADMIN_EMAIL') || 'webmaster@localhost'; }, + /** S3 media storage configuration. */ + s3: { + get endPoint() { + return Deno.env.get('S3_ENDPOINT')!; + }, + get region() { + return Deno.env.get('S3_REGION')!; + }, + get accessKey() { + return Deno.env.get('S3_ACCESS_KEY'); + }, + get secretKey() { + return Deno.env.get('S3_SECRET_KEY'); + }, + get bucket() { + return Deno.env.get('S3_BUCKET'); + }, + get pathStyle() { + return optionalBooleanSchema.parse(Deno.env.get('S3_PATH_STYLE')); + }, + get port() { + return optionalNumberSchema.parse(Deno.env.get('S3_PORT')); + }, + get sessionToken() { + return Deno.env.get('S3_SESSION_TOKEN'); + }, + get useSSL() { + return optionalBooleanSchema.parse(Deno.env.get('S3_USE_SSL')); + }, + }, + /** IPFS uploader configuration. */ + ipfs: { + /** Base URL for private IPFS API calls. */ + get apiUrl() { + return Deno.env.get('IPFS_API_URL') || 'http://localhost:5001'; + }, + }, + /** Module to upload files with. */ + get uploader() { + return Deno.env.get('DITTO_UPLOADER'); + }, + /** Media base URL for uploads. */ + get mediaDomain() { + const value = Deno.env.get('MEDIA_DOMAIN'); + + if (!value) { + const url = Conf.url; + url.host = `media.${url.host}`; + return url.toString(); + } + + return value; + }, + /** Max upload size for files in number of bytes. Default 100MiB. */ + get maxUploadSize() { + return Number(Deno.env.get('MAX_UPLOAD_SIZE') || 100 * 1024 * 1024); + }, /** Domain of the Ditto server as a `URL` object, for easily grabbing the `hostname`, etc. */ get url() { return new URL(Conf.localDomain); }, /** Merges the path with the localDomain. */ local(path: string): string { - const url = new URL(path.startsWith('/') ? path : new URL(path).pathname, Conf.localDomain); - - if (!path.startsWith('/')) { - // Copy query parameters from the original URL to the new URL - const originalUrl = new URL(path); - url.search = originalUrl.search; - } - - return url.toString(); + return mergePaths(Conf.localDomain, path); }, }; +const optionalBooleanSchema = z + .enum(['true', 'false']) + .optional() + .transform((value) => value !== undefined ? value === 'true' : undefined); + +const optionalNumberSchema = z + .string() + .optional() + .transform((value) => value !== undefined ? Number(value) : undefined); + +function mergePaths(base: string, path: string) { + const url = new URL( + path.startsWith('/') ? path : new URL(path).pathname, + base, + ); + + if (!path.startsWith('/')) { + // Copy query parameters from the original URL to the new URL + const originalUrl = new URL(path); + url.search = originalUrl.search; + } + + return url.toString(); +} + export { Conf }; diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index e7e7a70..1147c6c 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -2,7 +2,7 @@ import { type AppController } from '@/app.ts'; import { type Filter, findReplyTag, z } from '@/deps.ts'; import * as mixer from '@/mixer.ts'; import { getAuthor, getFollowedPubkeys, getFollows, syncUser } from '@/queries.ts'; -import { booleanParamSchema } from '@/schema.ts'; +import { booleanParamSchema, fileSchema } from '@/schema.ts'; import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; import { toAccount, toRelationship, toStatus } from '@/transformers/nostr-to-mastoapi.ts'; import { isFollowing, lookupAccount, Time } from '@/utils.ts'; @@ -113,8 +113,6 @@ const accountStatusesController: AppController = async (c) => { return paginated(c, events, statuses); }; -const fileSchema = z.custom((value) => value instanceof File); - const updateCredentialsSchema = z.object({ display_name: z.string().optional(), note: z.string().optional(), diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts new file mode 100644 index 0000000..1e310e1 --- /dev/null +++ b/src/controllers/api/media.ts @@ -0,0 +1,52 @@ +import { AppController } from '@/app.ts'; +import { Conf } from '@/config.ts'; +import { insertUnattachedMedia } from '@/db/unattached-media.ts'; +import { z } from '@/deps.ts'; +import { fileSchema } from '@/schema.ts'; +import { configUploader as uploader } from '@/uploaders/config.ts'; +import { parseBody } from '@/utils/web.ts'; +import { renderAttachment } from '@/views/attachment.ts'; + +const uploadSchema = fileSchema + .refine((file) => !!file.type, 'File type is required.') + .refine((file) => file.size <= Conf.maxUploadSize, 'File size is too large.'); + +const mediaBodySchema = z.object({ + file: uploadSchema, + thumbnail: uploadSchema.optional(), + description: z.string().optional(), + focus: z.string().optional(), +}); + +const mediaController: AppController = async (c) => { + const result = mediaBodySchema.safeParse(await parseBody(c.req.raw)); + + if (!result.success) { + return c.json({ error: 'Bad request.', schema: result.error }, 422); + } + + try { + const { file, description } = result.data; + const { cid } = await uploader.upload(file); + + const url = new URL(`/ipfs/${cid}`, Conf.mediaDomain).toString(); + + const media = await insertUnattachedMedia({ + pubkey: c.get('pubkey')!, + url, + data: { + name: file.name, + mime: file.type, + size: file.size, + description, + }, + }); + + return c.json(renderAttachment(media)); + } catch (e) { + console.error(e); + return c.json({ error: 'Failed to upload file.' }, 500); + } +}; + +export { mediaController }; diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index f7839f8..4756df0 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -4,6 +4,7 @@ import { getAncestors, getDescendants, getEvent } from '@/queries.ts'; import { toStatus } from '@/transformers/nostr-to-mastoapi.ts'; import { createEvent, paginationSchema, parseBody } from '@/utils/web.ts'; import { renderEventAccounts } from '@/views.ts'; +import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; const createStatusSchema = z.object({ in_reply_to_id: z.string().regex(/[0-9a-f]{64}/).nullish(), @@ -40,45 +41,49 @@ const createStatusController: AppController = async (c) => { const body = await parseBody(c.req.raw); const result = createStatusSchema.safeParse(body); - if (result.success) { - const { data } = result; - - if (data.visibility !== 'public') { - return c.json({ error: 'Only posting publicly is supported.' }, 422); - } - - if (data.poll) { - return c.json({ error: 'Polls are not yet supported.' }, 422); - } - - if (data.media_ids?.length) { - return c.json({ error: 'Media uploads are not yet supported.' }, 422); - } - - const tags: string[][] = []; - - if (data.in_reply_to_id) { - tags.push(['e', data.in_reply_to_id, 'reply']); - } - - if (data.sensitive && data.spoiler_text) { - tags.push(['content-warning', data.spoiler_text]); - } else if (data.sensitive) { - tags.push(['content-warning']); - } else if (data.spoiler_text) { - tags.push(['subject', data.spoiler_text]); - } - - const event = await createEvent({ - kind: 1, - content: data.status ?? '', - tags, - }, c); - - return c.json(await toStatus(event, c.get('pubkey'))); - } else { + if (!result.success) { return c.json({ error: 'Bad request', schema: result.error }, 400); } + + const { data } = result; + + if (data.visibility !== 'public') { + return c.json({ error: 'Only posting publicly is supported.' }, 422); + } + + if (data.poll) { + return c.json({ error: 'Polls are not yet supported.' }, 422); + } + + const tags: string[][] = []; + + if (data.in_reply_to_id) { + tags.push(['e', data.in_reply_to_id, 'reply']); + } + + if (data.sensitive && data.spoiler_text) { + tags.push(['content-warning', data.spoiler_text]); + } else if (data.sensitive) { + tags.push(['content-warning']); + } else if (data.spoiler_text) { + tags.push(['subject', data.spoiler_text]); + } + + if (data.media_ids?.length) { + const media = await getUnattachedMediaByIds(data.media_ids) + .then((media) => media.filter(({ pubkey }) => pubkey === c.get('pubkey'))) + .then((media) => media.map(({ url, data }) => ['media', url, data])); + + tags.push(...media); + } + + const event = await createEvent({ + kind: 1, + content: data.status ?? '', + tags, + }, c); + + return c.json(await toStatus(event, c.get('pubkey'))); }; const contextController: AppController = async (c) => { diff --git a/src/cron.ts b/src/cron.ts index a08fdf6..b29ab19 100644 --- a/src/cron.ts +++ b/src/cron.ts @@ -1,6 +1,9 @@ import * as eventsDB from '@/db/events.ts'; +import { deleteUnattachedMediaByUrl, getUnattachedMedia } from '@/db/unattached-media.ts'; import { cron } from '@/deps.ts'; import { Time } from '@/utils/time.ts'; +import { configUploader as uploader } from '@/uploaders/config.ts'; +import { cidFromUrl } from '@/utils/ipfs.ts'; /** Clean up old remote events. */ async function cleanupEvents() { @@ -14,6 +17,29 @@ async function cleanupEvents() { console.log(`Cleaned up ${result?.numDeletedRows ?? 0} old remote events.`); } +/** Delete files that aren't attached to any events. */ +async function cleanupMedia() { + console.log('Deleting orphaned media files...'); + + const until = new Date(Date.now() - Time.minutes(15)); + const media = await getUnattachedMedia(until); + + for (const { url } of media) { + const cid = cidFromUrl(new URL(url))!; + try { + await uploader.delete(cid); + await deleteUnattachedMediaByUrl(url); + } catch (e) { + console.error(`Failed to delete file ${url}`); + console.error(e); + } + } + + console.log(`Removed ${media?.length ?? 0} orphaned media files.`); +} + await cleanupEvents(); +await cleanupMedia(); cron.every15Minute(cleanupEvents); +cron.every15Minute(cleanupMedia); diff --git a/src/db.ts b/src/db.ts index e371ac9..abbe982 100644 --- a/src/db.ts +++ b/src/db.ts @@ -10,6 +10,7 @@ interface DittoDB { tags: TagRow; users: UserRow; relays: RelayRow; + unattached_media: UnattachedMediaRow; } interface EventRow { @@ -46,6 +47,14 @@ interface RelayRow { active: boolean; } +interface UnattachedMediaRow { + id: string; + pubkey: string; + url: string; + data: string; + uploaded_at: Date; +} + const db = new Kysely({ dialect: new DenoSqliteDialect({ database: new Sqlite(Conf.dbPath), diff --git a/src/db/events.test.ts b/src/db/events.test.ts index a9ab016..297813a 100644 --- a/src/db/events.test.ts +++ b/src/db/events.test.ts @@ -6,12 +6,12 @@ import { insertUser } from '@/db/users.ts'; Deno.test('count filters', async () => { assertEquals(await countFilters([{ kinds: [1] }]), 0); - await insertEvent(event55920b75); + await insertEvent(event55920b75, { user: undefined }); assertEquals(await countFilters([{ kinds: [1] }]), 1); }); Deno.test('insert and filter events', async () => { - await insertEvent(event55920b75); + await insertEvent(event55920b75, { user: undefined }); assertEquals(await getFilters([{ kinds: [1] }]), [event55920b75]); assertEquals(await getFilters([{ kinds: [3] }]), []); @@ -24,14 +24,14 @@ Deno.test('insert and filter events', async () => { }); Deno.test('delete events', async () => { - await insertEvent(event55920b75); + await insertEvent(event55920b75, { user: undefined }); assertEquals(await getFilters([{ kinds: [1] }]), [event55920b75]); await deleteFilters([{ kinds: [1] }]); assertEquals(await getFilters([{ kinds: [1] }]), []); }); Deno.test('query events with local filter', async () => { - await insertEvent(event55920b75); + await insertEvent(event55920b75, { user: undefined }); assertEquals(await getFilters([{}]), [event55920b75]); assertEquals(await getFilters([{ local: true }]), []); diff --git a/src/db/events.ts b/src/db/events.ts index b86e70d..6dade28 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -1,58 +1,67 @@ -import { db, type TagRow } from '@/db.ts'; -import { type Event, type Insertable, SqliteError } from '@/deps.ts'; +import { db } from '@/db.ts'; +import { type Event, SqliteError } from '@/deps.ts'; +import { isParameterizedReplaceableKind } from '@/kinds.ts'; +import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; +import { EventData } from '@/types.ts'; +import { isNostrId, isURL } from '@/utils.ts'; import type { DittoFilter, GetFiltersOpts } from '@/filter.ts'; -import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; -type TagCondition = ({ event, count }: { event: Event; count: number }) => boolean; +/** Function to decide whether or not to index a tag. */ +type TagCondition = ({ event, count, value }: { + event: Event; + data: EventData; + count: number; + value: string; +}) => boolean; /** Conditions for when to index certain tags. */ const tagConditions: Record = { - 'd': ({ event, count }) => 30000 <= event.kind && event.kind < 40000 && count === 0, - 'e': ({ count }) => count < 15, - 'p': ({ event, count }) => event.kind === 3 || count < 15, - 'proxy': ({ count }) => count === 0, - 'q': ({ event, count }) => event.kind === 1 && count === 0, - 't': ({ count }) => count < 5, + 'd': ({ event, count }) => count === 0 && isParameterizedReplaceableKind(event.kind), + 'e': ({ count, value }) => count < 15 && isNostrId(value), + 'media': ({ count, value, data }) => (data.user || count < 4) && isURL(value), + 'p': ({ event, count, value }) => (count < 15 || event.kind === 3) && isNostrId(value), + 'proxy': ({ count, value }) => count === 0 && isURL(value), + 'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value), + 't': ({ count, value }) => count < 5 && value.length < 50, }; /** Insert an event (and its tags) into the database. */ -function insertEvent(event: Event): Promise { +function insertEvent(event: Event, data: EventData): Promise { return db.transaction().execute(async (trx) => { - await trx.insertInto('events') - .values({ - ...event, - tags: JSON.stringify(event.tags), - }) - .execute(); + /** Insert the event into the database. */ + async function addEvent() { + await trx.insertInto('events') + .values({ ...event, tags: JSON.stringify(event.tags) }) + .execute(); + } - const searchContent = buildSearchContent(event); - if (searchContent) { + /** Add search data to the FTS table. */ + async function indexSearch() { + const searchContent = buildSearchContent(event); + if (!searchContent) return; await trx.insertInto('events_fts') .values({ id: event.id, content: searchContent.substring(0, 1000) }) .execute(); } - const tagCounts: Record = {}; - const tags = event.tags.reduce[]>((results, [name, value]) => { - tagCounts[name] = (tagCounts[name] || 0) + 1; + /** Index event tags depending on the conditions defined above. */ + async function indexTags() { + const tags = filterIndexableTags(event, data); + const rows = tags.map(([tag, value]) => ({ event_id: event.id, tag, value })); - if (value && tagConditions[name]?.({ event, count: tagCounts[name] - 1 })) { - results.push({ - event_id: event.id, - tag: name, - value, - }); - } - - return results; - }, []); - - if (tags.length) { + if (!tags.length) return; await trx.insertInto('tags') - .values(tags) + .values(rows) .execute(); } + + // Run the queries. + await Promise.all([ + addEvent(), + indexTags(), + indexSearch(), + ]); }).catch((error) => { // Don't throw for duplicate events. if (error instanceof SqliteError && error.code === 19) { @@ -181,6 +190,40 @@ async function countFilters(filters: DittoFilter[]): Promis return Number(count); } +/** Return only the tags that should be indexed. */ +function filterIndexableTags(event: Event, data: EventData): string[][] { + const tagCounts: Record = {}; + + function getCount(name: string) { + return tagCounts[name] || 0; + } + + function incrementCount(name: string) { + tagCounts[name] = getCount(name) + 1; + } + + function checkCondition(name: string, value: string, condition: TagCondition) { + return condition({ + event, + data, + count: getCount(name), + value, + }); + } + + return event.tags.reduce((results, tag) => { + const [name, value] = tag; + const condition = tagConditions[name] as TagCondition | undefined; + + if (value && condition && value.length < 200 && checkCondition(name, value, condition)) { + results.push(tag); + } + + incrementCount(name); + return results; + }, []); +} + /** Build a search index from the event. */ function buildSearchContent(event: Event): string { switch (event.kind) { diff --git a/src/db/migrations/007_unattached_media.ts b/src/db/migrations/007_unattached_media.ts new file mode 100644 index 0000000..a2b36a2 --- /dev/null +++ b/src/db/migrations/007_unattached_media.ts @@ -0,0 +1,34 @@ +import { Kysely, sql } from '@/deps.ts'; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('unattached_media') + .addColumn('id', 'text', (c) => c.primaryKey()) + .addColumn('pubkey', 'text', (c) => c.notNull()) + .addColumn('url', 'text', (c) => c.notNull()) + .addColumn('data', 'text', (c) => c.notNull()) + .addColumn('uploaded_at', 'datetime', (c) => c.notNull().defaultTo(sql`CURRENT_TIMESTAMP`)) + .execute(); + + await db.schema + .createIndex('unattached_media_id') + .on('unattached_media') + .column('id') + .execute(); + + await db.schema + .createIndex('unattached_media_pubkey') + .on('unattached_media') + .column('pubkey') + .execute(); + + await db.schema + .createIndex('unattached_media_url') + .on('unattached_media') + .column('url') + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('unattached_media').execute(); +} diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts new file mode 100644 index 0000000..ae9d882 --- /dev/null +++ b/src/db/unattached-media.ts @@ -0,0 +1,77 @@ +import { db } from '@/db.ts'; +import { uuid62 } from '@/deps.ts'; +import { type MediaData } from '@/schemas/nostr.ts'; + +interface UnattachedMedia { + id: string; + pubkey: string; + url: string; + data: MediaData; + uploaded_at: Date; +} + +/** Add unattached media into the database. */ +async function insertUnattachedMedia(media: Omit) { + const result = { + id: uuid62.v4(), + uploaded_at: new Date(), + ...media, + }; + + await db.insertInto('unattached_media') + .values({ ...result, data: JSON.stringify(media.data) }) + .execute(); + + return result; +} + +/** Select query for unattached media. */ +function selectUnattachedMediaQuery() { + return db.selectFrom('unattached_media') + .select([ + 'unattached_media.id', + 'unattached_media.pubkey', + 'unattached_media.url', + 'unattached_media.data', + 'unattached_media.uploaded_at', + ]); +} + +/** Find attachments that exist but aren't attached to any events. */ +function getUnattachedMedia(until: Date) { + return selectUnattachedMediaQuery() + .leftJoin('tags', 'unattached_media.url', 'tags.value') + .where('uploaded_at', '<', until) + .execute(); +} + +/** Delete unattached media by URL. */ +function deleteUnattachedMediaByUrl(url: string) { + return db.deleteFrom('unattached_media') + .where('url', '=', url) + .execute(); +} + +/** Get unattached media by IDs. */ +function getUnattachedMediaByIds(ids: string[]) { + return selectUnattachedMediaQuery() + .where('id', 'in', ids) + .execute(); +} + +/** Delete rows as an event with media is being created. */ +function deleteAttachedMedia(pubkey: string, urls: string[]) { + return db.deleteFrom('unattached_media') + .where('pubkey', '=', pubkey) + .where('url', 'in', urls) + .execute(); +} + +export { + deleteAttachedMedia, + deleteUnattachedMediaByUrl, + getUnattachedMedia, + getUnattachedMediaByIds, + insertUnattachedMedia, + type UnattachedMedia, +}; diff --git a/src/deps.ts b/src/deps.ts index f4efc62..727bba1 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -66,5 +66,8 @@ export { export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/v1.0.1/mod.ts'; export { default as tldts } from 'npm:tldts@^6.0.14'; export * as cron from 'https://deno.land/x/deno_cron@v1.0.0/cron.ts'; +export { S3Client } from 'https://deno.land/x/s3_lite_client@0.6.1/mod.ts'; +export { default as IpfsHash } from 'npm:ipfs-only-hash@^4.0.0'; +export { default as uuid62 } from 'npm:uuid62@^1.0.2'; export type * as TypeFest from 'npm:type-fest@^4.3.0'; diff --git a/src/middleware/auth98.ts b/src/middleware/auth98.ts index dad0d0a..be97320 100644 --- a/src/middleware/auth98.ts +++ b/src/middleware/auth98.ts @@ -1,9 +1,14 @@ import { type AppContext, type AppMiddleware } from '@/app.ts'; import { HTTPException } from '@/deps.ts'; -import { buildAuthEventTemplate, parseAuthRequest, type ParseAuthRequestOpts } from '@/utils/nip98.ts'; +import { + buildAuthEventTemplate, + parseAuthRequest, + type ParseAuthRequestOpts, + validateAuthEvent, +} from '@/utils/nip98.ts'; import { localRequest } from '@/utils/web.ts'; -import { signNostrConnect } from '@/sign.ts'; -import { findUser } from '@/db/users.ts'; +import { signEvent } from '@/sign.ts'; +import { findUser, User } from '@/db/users.ts'; /** * NIP-98 auth. @@ -23,26 +28,47 @@ function auth98(opts: ParseAuthRequestOpts = {}): AppMiddleware { }; } -/** Require the user to prove they're an admin before invoking the controller. */ -const requireAdmin: AppMiddleware = async (c, next) => { - const header = c.req.headers.get('x-nostr-sign'); - const proof = c.get('proof') || header ? await obtainProof(c) : undefined; - const user = proof ? await findUser({ pubkey: proof.pubkey }) : undefined; +type UserRole = 'user' | 'admin'; - if (proof && user?.admin) { - c.set('pubkey', proof.pubkey); - c.set('proof', proof); - await next(); - } else { - throw new HTTPException(401); - } -}; +/** Require the user to prove their role before invoking the controller. */ +function requireRole(role: UserRole, opts?: ParseAuthRequestOpts): AppMiddleware { + return async (c, next) => { + const header = c.req.headers.get('x-nostr-sign'); + const proof = c.get('proof') || header ? await obtainProof(c, opts) : undefined; + const user = proof ? await findUser({ pubkey: proof.pubkey }) : undefined; -/** Get the proof over Nostr Connect. */ -async function obtainProof(c: AppContext) { - const req = localRequest(c); - const event = await buildAuthEventTemplate(req); - return signNostrConnect(event, c); + if (proof && user && matchesRole(user, role)) { + c.set('pubkey', proof.pubkey); + c.set('proof', proof); + await next(); + } else { + throw new HTTPException(401); + } + }; } -export { auth98, requireAdmin }; +/** Check whether the user fulfills the role. */ +function matchesRole(user: User, role: UserRole): boolean { + switch (role) { + case 'user': + return true; + case 'admin': + return user.admin; + default: + return false; + } +} + +/** Get the proof over Nostr Connect. */ +async function obtainProof(c: AppContext, opts?: ParseAuthRequestOpts) { + const req = localRequest(c); + const reqEvent = await buildAuthEventTemplate(req, opts); + const resEvent = await signEvent(reqEvent, c); + const result = await validateAuthEvent(req, resEvent, opts); + + if (result.success) { + return result.data; + } +} + +export { auth98, requireRole }; diff --git a/src/note.ts b/src/note.ts index 82688b6..92baf52 100644 --- a/src/note.ts +++ b/src/note.ts @@ -1,5 +1,6 @@ import { Conf } from '@/config.ts'; import { linkify, linkifyStr, mime, nip19, nip21 } from '@/deps.ts'; +import { type DittoAttachment } from '@/views/attachment.ts'; linkify.registerCustomProtocol('nostr', true); linkify.registerCustomProtocol('wss'); @@ -52,13 +53,8 @@ function parseNoteContent(content: string): ParsedNoteContent { }; } -interface MediaLink { - url: string; - mimeType: string; -} - -function getMediaLinks(links: Link[]): MediaLink[] { - return links.reduce((acc, link) => { +function getMediaLinks(links: Link[]): DittoAttachment[] { + return links.reduce((acc, link) => { const mimeType = getUrlMimeType(link.href); if (!mimeType) return acc; @@ -67,7 +63,9 @@ function getMediaLinks(links: Link[]): MediaLink[] { if (['audio', 'image', 'video'].includes(baseType)) { acc.push({ url: link.href, - mimeType, + data: { + mime: mimeType, + }, }); } @@ -110,4 +108,4 @@ function getDecodedPubkey(decoded: nip19.DecodeResult): string | undefined { } } -export { getMediaLinks, type MediaLink, parseNoteContent }; +export { getMediaLinks, parseNoteContent }; diff --git a/src/pipeline.ts b/src/pipeline.ts index 38e3214..5a80b00 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,6 +1,7 @@ import { Conf } from '@/config.ts'; import * as eventsDB from '@/db/events.ts'; import { addRelays } from '@/db/relays.ts'; +import { deleteAttachedMedia } from '@/db/unattached-media.ts'; import { findUser } from '@/db/users.ts'; import { type Event, LRUCache } from '@/deps.ts'; import { isEphemeralKind } from '@/kinds.ts'; @@ -27,6 +28,7 @@ async function handleEvent(event: Event): Promise { processDeletions(event), trackRelays(event), trackHashtags(event), + processMedia(event, data), streamOut(event, data), broadcast(event, data), ]); @@ -64,7 +66,7 @@ async function storeEvent(event: Event, data: EventData): Promise { if (deletion) { return Promise.reject(new RelayError('blocked', 'event was deleted')); } else { - await eventsDB.insertEvent(event).catch(console.warn); + await eventsDB.insertEvent(event, data).catch(console.warn); } } else { return Promise.reject(new RelayError('blocked', 'only registered users can post')); @@ -120,6 +122,14 @@ function trackRelays(event: Event) { return addRelays([...relays]); } +/** Delete unattached media entries that are attached to the event. */ +function processMedia({ tags, pubkey }: Event, { user }: EventData) { + if (user) { + const urls = getTagSet(tags, 'media'); + return deleteAttachedMedia(pubkey, [...urls]); + } +} + /** Determine if the event is being received in a timely manner. */ const isFresh = (event: Event): boolean => eventAge(event) < Time.seconds(10); diff --git a/src/precheck.ts b/src/precheck.ts new file mode 100644 index 0000000..40ab2fd --- /dev/null +++ b/src/precheck.ts @@ -0,0 +1,22 @@ +import { Conf } from '@/config.ts'; + +/** Ensure the media URL is not on the same host as the local domain. */ +function checkMediaHost() { + const { url, mediaDomain } = Conf; + const mediaUrl = new URL(mediaDomain); + + if (url.host === mediaUrl.host) { + throw new PrecheckError('For security reasons, MEDIA_DOMAIN cannot be on the same host as LOCAL_DOMAIN.'); + } +} + +/** Error class for precheck errors. */ +class PrecheckError extends Error { + constructor(message: string) { + super(`${message}\nTo disable this check, set DITTO_PRECHECK="false"`); + } +} + +if (Deno.env.get('DITTO_PRECHECK') !== 'false') { + checkMediaHost(); +} diff --git a/src/schema.ts b/src/schema.ts index d32251b..a29191f 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -48,4 +48,16 @@ const safeUrlSchema = z.string().max(2048).url(); /** https://github.com/colinhacks/zod/issues/1630#issuecomment-1365983831 */ const booleanParamSchema = z.enum(['true', 'false']).transform((value) => value === 'true'); -export { booleanParamSchema, decode64Schema, emojiTagSchema, filteredArray, hashtagSchema, jsonSchema, safeUrlSchema }; +/** Schema for `File` objects. */ +const fileSchema = z.custom((value) => value instanceof File); + +export { + booleanParamSchema, + decode64Schema, + emojiTagSchema, + fileSchema, + filteredArray, + hashtagSchema, + jsonSchema, + safeUrlSchema, +}; diff --git a/src/schemas/nostr.ts b/src/schemas/nostr.ts index 1294804..c097935 100644 --- a/src/schemas/nostr.ts +++ b/src/schemas/nostr.ts @@ -73,9 +73,27 @@ const metaContentSchema = z.object({ lud16: z.string().optional().catch(undefined), }).partial().passthrough(); +/** Media data schema from `"media"` tags. */ +const mediaDataSchema = z.object({ + blurhash: z.string().optional().catch(undefined), + cid: z.string().optional().catch(undefined), + description: z.string().max(200).optional().catch(undefined), + height: z.number().int().positive().optional().catch(undefined), + mime: z.string().optional().catch(undefined), + name: z.string().optional().catch(undefined), + size: z.number().int().positive().optional().catch(undefined), + width: z.number().int().positive().optional().catch(undefined), +}); + +/** Media data from `"media"` tags. */ +type MediaData = z.infer; + /** Parses kind 0 content from a JSON string. */ const jsonMetaContentSchema = jsonSchema.pipe(metaContentSchema).catch({}); +/** Parses media data from a JSON string. */ +const jsonMediaDataSchema = jsonSchema.pipe(mediaDataSchema).catch({}); + /** NIP-11 Relay Information Document. */ const relayInfoDocSchema = z.object({ name: z.string().transform((val) => val.slice(0, 30)).optional().catch(undefined), @@ -102,7 +120,10 @@ export { type ClientREQ, connectResponseSchema, filterSchema, + jsonMediaDataSchema, jsonMetaContentSchema, + type MediaData, + mediaDataSchema, metaContentSchema, nostrIdSchema, relayInfoDocSchema, diff --git a/src/server.ts b/src/server.ts index 76cbc9f..e3cf7ca 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,3 +1,4 @@ +import './precheck.ts'; import app from './app.ts'; Deno.serve(app.fetch); diff --git a/src/sign.ts b/src/sign.ts index e260af7..a97478a 100644 --- a/src/sign.ts +++ b/src/sign.ts @@ -99,4 +99,4 @@ async function signAdminEvent(event: EventTemplate return finishEvent(event, Conf.seckey); } -export { signAdminEvent, signEvent, signNostrConnect }; +export { signAdminEvent, signEvent }; diff --git a/src/transformers/nostr-to-mastoapi.ts b/src/transformers/nostr-to-mastoapi.ts index aa0b231..5fc02a8 100644 --- a/src/transformers/nostr-to-mastoapi.ts +++ b/src/transformers/nostr-to-mastoapi.ts @@ -2,14 +2,15 @@ import { isCWTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ad import { Conf } from '@/config.ts'; import * as eventsDB from '@/db/events.ts'; -import { type Event, findReplyTag, lodash, nip19, sanitizeHtml, TTLCache, unfurl, z } from '@/deps.ts'; -import { getMediaLinks, type MediaLink, parseNoteContent } from '@/note.ts'; +import { type Event, findReplyTag, lodash, nip19, sanitizeHtml, TTLCache, unfurl } from '@/deps.ts'; +import { getMediaLinks, parseNoteContent } from '@/note.ts'; import { getAuthor, getFollowedPubkeys, getFollows } from '@/queries.ts'; import { emojiTagSchema, filteredArray } from '@/schema.ts'; -import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; +import { jsonMediaDataSchema, jsonMetaContentSchema } from '@/schemas/nostr.ts'; import { isFollowing, type Nip05, nostrDate, parseNip05, Time } from '@/utils.ts'; import { verifyNip05Cached } from '@/utils/nip05.ts'; import { findUser } from '@/db/users.ts'; +import { DittoAttachment, renderAttachment } from '@/views/attachment.ts'; const DEFAULT_AVATAR = 'https://gleasonator.com/images/avi.png'; const DEFAULT_BANNER = 'https://gleasonator.com/images/banner.png'; @@ -118,7 +119,6 @@ async function toStatus(event: Event<1>, viewerPubkey?: string) { ]; const { html, links, firstUrl } = parseNoteContent(event.content); - const mediaLinks = getMediaLinks(links); const [mentions, card, repliesCount, reblogsCount, favouritesCount, [repostEvent], [reactionEvent]] = await Promise .all([ @@ -140,6 +140,14 @@ async function toStatus(event: Event<1>, viewerPubkey?: string) { const cw = event.tags.find(isCWTag); const subject = event.tags.find((tag) => tag[0] === 'subject'); + const mediaLinks = getMediaLinks(links); + + const mediaTags: DittoAttachment[] = event.tags + .filter((tag) => tag[0] === 'media') + .map(([_, url, json]) => ({ url, data: jsonMediaDataSchema.parse(json) })); + + const media = [...mediaLinks, ...mediaTags]; + return { id: event.id, account, @@ -161,7 +169,7 @@ async function toStatus(event: Event<1>, viewerPubkey?: string) { bookmarked: false, reblog: null, application: null, - media_attachments: mediaLinks.map(renderAttachment), + media_attachments: media.map(renderAttachment), mentions, tags: [], emojis: toEmojis(event), @@ -185,24 +193,6 @@ function buildInlineRecipients(mentions: Mention[]): string { return `${elements.join(' ')} `; } -const attachmentTypeSchema = z.enum(['image', 'video', 'gifv', 'audio', 'unknown']).catch('unknown'); - -function renderAttachment({ url, mimeType }: MediaLink) { - const [baseType, _subType] = mimeType.split('/'); - const type = attachmentTypeSchema.parse(baseType); - - return { - id: url, - type, - url, - preview_url: url, - remote_url: null, - meta: {}, - description: '', - blurhash: null, - }; -} - interface PreviewCard { url: string; title: string; diff --git a/src/uploaders/config.ts b/src/uploaders/config.ts new file mode 100644 index 0000000..b0adece --- /dev/null +++ b/src/uploaders/config.ts @@ -0,0 +1,30 @@ +import { Conf } from '@/config.ts'; + +import { ipfsUploader } from './ipfs.ts'; +import { s3Uploader } from './s3.ts'; + +import type { Uploader } from './types.ts'; + +/** Meta-uploader determined from configuration. */ +const configUploader: Uploader = { + upload(file) { + return uploader().upload(file); + }, + delete(cid) { + return uploader().delete(cid); + }, +}; + +/** Get the uploader module based on configuration. */ +function uploader() { + switch (Conf.uploader) { + case 's3': + return s3Uploader; + case 'ipfs': + return ipfsUploader; + default: + return ipfsUploader; + } +} + +export { configUploader }; diff --git a/src/uploaders/ipfs.ts b/src/uploaders/ipfs.ts new file mode 100644 index 0000000..e6a33cd --- /dev/null +++ b/src/uploaders/ipfs.ts @@ -0,0 +1,50 @@ +import { Conf } from '@/config.ts'; +import { z } from '@/deps.ts'; + +import type { Uploader } from './types.ts'; + +/** Response schema for POST `/api/v0/add`. */ +const ipfsAddResponseSchema = z.object({ + Name: z.string(), + Hash: z.string(), + Size: z.string(), +}); + +/** + * IPFS uploader. It expects an IPFS node up and running. + * It will try to connect to `http://localhost:5001` by default, + * and upload the file using the REST API. + */ +const ipfsUploader: Uploader = { + async upload(file) { + const url = new URL('/api/v0/add', Conf.ipfs.apiUrl); + + const formData = new FormData(); + formData.append('file', file); + + const response = await fetch(url, { + method: 'POST', + body: formData, + }); + + const { Hash } = ipfsAddResponseSchema.parse(await response.json()); + + return { + cid: Hash, + }; + }, + async delete(cid) { + const url = new URL('/api/v0/pin/rm', Conf.ipfs.apiUrl); + + const query = new URLSearchParams(); + query.set('arg', cid); + + url.search = query.toString(); + + await fetch(url, { + method: 'POST', + }); + }, +}; + +export { ipfsUploader }; diff --git a/src/uploaders/s3.ts b/src/uploaders/s3.ts new file mode 100644 index 0000000..9242bee --- /dev/null +++ b/src/uploaders/s3.ts @@ -0,0 +1,33 @@ +import { Conf } from '@/config.ts'; +import { IpfsHash, S3Client } from '@/deps.ts'; + +import type { Uploader } from './types.ts'; + +const s3 = new S3Client({ ...Conf.s3 }); + +/** + * S3-compatible uploader for AWS, Wasabi, DigitalOcean Spaces, and more. + * Files are named by their IPFS CID and exposed at `/ipfs/`, letting it + * take advantage of IPFS features while not really using IPFS. + */ +const s3Uploader: Uploader = { + async upload(file) { + const cid = await IpfsHash.of(file.stream()) as string; + + await s3.putObject(`ipfs/${cid}`, file.stream(), { + metadata: { + 'Content-Type': file.type, + 'x-amz-acl': 'public-read', + }, + }); + + return { + cid, + }; + }, + async delete(cid) { + await s3.deleteObject(`ipfs/${cid}`); + }, +}; + +export { s3Uploader }; diff --git a/src/uploaders/types.ts b/src/uploaders/types.ts new file mode 100644 index 0000000..80bf431 --- /dev/null +++ b/src/uploaders/types.ts @@ -0,0 +1,15 @@ +/** Modular uploader interface, to support uploading to different backends. */ +interface Uploader { + /** Upload the file to the backend. */ + upload(file: File): Promise; + /** Delete the file from the backend. */ + delete(cid: string): Promise; +} + +/** Return value from the uploader after uploading a file. */ +interface UploadResult { + /** IPFS CID for the file. */ + cid: string; +} + +export type { Uploader }; diff --git a/src/utils.ts b/src/utils.ts index 1836298..11ef0ea 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,7 @@ import { type Event, type EventTemplate, getEventHash, nip19, z } from '@/deps.ts'; import { getAuthor } from '@/queries.ts'; import { lookupNip05Cached } from '@/utils/nip05.ts'; +import { nostrIdSchema } from '@/schemas/nostr.ts'; /** Get the current time in Nostr format. */ const nostrNow = (): number => Math.floor(Date.now() / 1000); @@ -111,6 +112,21 @@ function eventMatchesTemplate(event: Event, template: EventTemplate): boolean { return getEventHash(event) === getEventHash({ pubkey: event.pubkey, ...template }); } +/** Test whether the value is a Nostr ID. */ +function isNostrId(value: unknown): boolean { + return nostrIdSchema.safeParse(value).success; +} + +/** Test whether the value is a URL. */ +function isURL(value: unknown): boolean { + try { + new URL(value as string); + return true; + } catch (_) { + return false; + } +} + export { bech32ToPubkey, dedupeEvents, @@ -119,7 +135,9 @@ export { eventMatchesTemplate, findTag, isFollowing, + isNostrId, isRelay, + isURL, lookupAccount, type Nip05, nostrDate, diff --git a/src/utils/ipfs.ts b/src/utils/ipfs.ts new file mode 100644 index 0000000..b4f9444 --- /dev/null +++ b/src/utils/ipfs.ts @@ -0,0 +1,27 @@ +/** https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway */ +const IPFS_PATH_REGEX = /^\/ipfs\/([^/]+)/; +/** https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway */ +const IPFS_HOST_REGEX = /^([^/]+)\.ipfs\./; + +/** Get IPFS CID out of a path. */ +function cidFromPath(path: string) { + return path.match(IPFS_PATH_REGEX)?.[1]; +} + +/** Get IPFS CID out of a host. */ +function cidFromHost(host: string) { + return host.match(IPFS_HOST_REGEX)?.[1]; +} + +/** Get IPFS CID out of a URL. */ +function cidFromUrl({ protocol, hostname, pathname }: URL) { + switch (protocol) { + case 'ipfs:': + return hostname; + case 'http:': + case 'https:': + return cidFromPath(pathname) || cidFromHost(hostname); + } +} + +export { cidFromUrl }; diff --git a/src/utils/nip98.ts b/src/utils/nip98.ts index e1cae5d..5606d8b 100644 --- a/src/utils/nip98.ts +++ b/src/utils/nip98.ts @@ -15,13 +15,21 @@ interface ParseAuthRequestOpts { } /** Parse the auth event from a Request, returning a zod SafeParse type. */ -function parseAuthRequest(req: Request, opts: ParseAuthRequestOpts = {}) { - const { maxAge = Time.minutes(1), validatePayload = true } = opts; - +// deno-lint-ignore require-await +async function parseAuthRequest(req: Request, opts: ParseAuthRequestOpts = {}) { const header = req.headers.get('authorization'); const base64 = header?.match(/^Nostr (.+)$/)?.[1]; + const result = decode64EventSchema.safeParse(base64); - const schema = decode64EventSchema + if (!result.success) return result; + return validateAuthEvent(req, result.data, opts); +} + +/** Compare the auth event with the request, returning a zod SafeParse type. */ +function validateAuthEvent(req: Request, event: Event, opts: ParseAuthRequestOpts = {}) { + const { maxAge = Time.minutes(1), validatePayload = true } = opts; + + const schema = signedEventSchema .refine((event): event is Event<27235> => event.kind === 27235, 'Event must be kind 27235') .refine((event) => eventAge(event) < maxAge, 'Event expired') .refine((event) => tagValue(event, 'method') === req.method, 'Event method does not match HTTP request method') @@ -35,22 +43,28 @@ function parseAuthRequest(req: Request, opts: ParseAuthRequestOpts = {}) { .then((hash) => hash === tagValue(event, 'payload')); } - return schema.safeParseAsync(base64); + return schema.safeParseAsync(event); } /** Create an auth EventTemplate from a Request. */ -async function buildAuthEventTemplate(req: Request): Promise> { +async function buildAuthEventTemplate(req: Request, opts: ParseAuthRequestOpts = {}): Promise> { + const { validatePayload = true } = opts; const { method, url } = req; - const payload = await req.clone().text().then(sha256); + + const tags = [ + ['method', method], + ['u', url], + ]; + + if (validatePayload) { + const payload = await req.clone().text().then(sha256); + tags.push(['payload', payload]); + } return { kind: 27235, content: '', - tags: [ - ['method', method], - ['u', url], - ['payload', payload], - ], + tags, created_at: nostrNow(), }; } @@ -60,4 +74,4 @@ function tagValue(event: Event, tagName: string): string | undefined { return findTag(event.tags, tagName)?.[1]; } -export { buildAuthEventTemplate, parseAuthRequest, type ParseAuthRequestOpts }; +export { buildAuthEventTemplate, parseAuthRequest, type ParseAuthRequestOpts, validateAuthEvent }; diff --git a/src/views/attachment.ts b/src/views/attachment.ts new file mode 100644 index 0000000..38ddb37 --- /dev/null +++ b/src/views/attachment.ts @@ -0,0 +1,34 @@ +import { UnattachedMedia } from '@/db/unattached-media.ts'; +import { type TypeFest } from '@/deps.ts'; + +type DittoAttachment = TypeFest.SetOptional; + +function renderAttachment(media: DittoAttachment) { + const { id, data, url } = media; + return { + id: id ?? url ?? data.cid, + type: getAttachmentType(data.mime ?? ''), + url, + preview_url: url, + remote_url: null, + description: data.description ?? '', + blurhash: data.blurhash || null, + cid: data.cid, + }; +} + +/** MIME to Mastodon API `Attachment` type. */ +function getAttachmentType(mime: string): string { + const [type] = mime.split('/'); + + switch (type) { + case 'image': + case 'video': + case 'audio': + return type; + default: + return 'unknown'; + } +} + +export { type DittoAttachment, renderAttachment };