Merge branch 'imeta' into 'main'

Support imeta media attachments

See merge request soapbox-pub/ditto!274

deno.json

@@ -32,7 +32,7 @@
     "@std/dotenv": "jsr:@std/dotenv@^0.224.0",
     "@std/encoding": "jsr:@std/encoding@^0.224.0",
     "@std/json": "jsr:@std/json@^0.223.0",
-    "@std/media-types": "jsr:@std/media-types@^0.224.0",
+    "@std/media-types": "jsr:@std/media-types@^0.224.1",
     "@std/streams": "jsr:@std/streams@^0.223.0",
     "comlink": "npm:comlink@^4.4.1",
     "deno-safe-fetch": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts",
@@ -54,7 +54,6 @@
     "tseep": "npm:tseep@^1.2.1",
     "type-fest": "npm:type-fest@^4.3.0",
     "unfurl.js": "npm:unfurl.js@^6.4.0",
-    "uuid62": "npm:uuid62@^1.0.2",
     "zod": "npm:zod@^3.23.5",
     "~/fixtures/": "./fixtures/"
   },

src/controllers/api/statuses.ts

@@ -91,15 +91,14 @@ const createStatusController: AppController = async (c) => {
     tags.push(['subject', data.spoiler_text]);
   }
 
-  const viewerPubkey = await c.get('signer')?.getPublicKey();
+  const media = data.media_ids?.length ? await getUnattachedMediaByIds(kysely, data.media_ids) : [];
 
-  if (data.media_ids?.length) {
+  const imeta: string[][] = media.map(({ data }) => {
-    const media = await getUnattachedMediaByIds(kysely, data.media_ids)
+    const values: string[] = data.map((tag) => tag.join(' '));
-      .then((media) => media.filter(({ pubkey }) => pubkey === viewerPubkey))
+    return ['imeta', ...values];
-      .then((media) => media.map(({ url, data }) => ['media', url, data]));
+  });
 
-    tags.push(...media);
+  tags.push(...imeta);
-  }
 
   const pubkeys = new Set<string>();
 
@@ -131,9 +130,15 @@ const createStatusController: AppController = async (c) => {
     tags.push(['t', match[1]]);
   }
 
+  const mediaUrls: string[] = media
+    .map(({ data }) => data.find(([name]) => name === 'url')?.[1])
+    .filter((url): url is string => Boolean(url));
+
+  const mediaCompat: string = mediaUrls.length ? ['', '', ...mediaUrls].join('\n') : '';
+
   const event = await createEvent({
     kind: 1,
-    content,
+    content: content + mediaCompat,
     tags,
   }, c);
 

src/db/unattached-media.ts

@@ -1,32 +1,25 @@
 import { Kysely } from 'kysely';
-import uuid62 from 'uuid62';
 
 import { DittoDB } from '@/db/DittoDB.ts';
 import { DittoTables } from '@/db/DittoTables.ts';
-import { type MediaData } from '@/schemas/nostr.ts';
 
 interface UnattachedMedia {
   id: string;
   pubkey: string;
   url: string;
-  data: MediaData;
+  /** NIP-94 tags. */
+  data: string[][];
   uploaded_at: number;
 }
 
 /** Add unattached media into the database. */
-async function insertUnattachedMedia(media: Omit<UnattachedMedia, 'id' | 'uploaded_at'>) {
+async function insertUnattachedMedia(media: UnattachedMedia) {
-  const result = {
-    id: uuid62.v4(),
-    uploaded_at: Date.now(),
-    ...media,
-  };
-
   const kysely = await DittoDB.getInstance();
   await kysely.insertInto('unattached_media')
-    .values({ ...result, data: JSON.stringify(media.data) })
+    .values({ ...media, data: JSON.stringify(media.data) })
     .execute();
 
-  return result;
+  return media;
 }
 
 /** Select query for unattached media. */
@@ -58,11 +51,17 @@ async function deleteUnattachedMediaByUrl(url: string) {
 }
 
 /** Get unattached media by IDs. */
-async function getUnattachedMediaByIds(kysely: Kysely<DittoTables>, ids: string[]) {
+async function getUnattachedMediaByIds(kysely: Kysely<DittoTables>, ids: string[]): Promise<UnattachedMedia[]> {
   if (!ids.length) return [];
-  return await selectUnattachedMediaQuery(kysely)
+
+  const results = await selectUnattachedMediaQuery(kysely)
     .where('id', 'in', ids)
     .execute();
+
+  return results.map((row) => ({
+    ...row,
+    data: JSON.parse(row.data),
+  }));
 }
 
 /** Delete rows as an event with media is being created. */

src/deps.ts

@@ -1,8 +1,6 @@
 import 'deno-safe-fetch';
 // @deno-types="npm:@types/lodash@4.14.194"
 export { default as lodash } from 'https://esm.sh/lodash@4.17.21';
-// @deno-types="npm:@types/mime@3.0.0"
-export { default as mime } from 'npm:mime@^3.0.0';
 // @deno-types="npm:@types/sanitize-html@2.9.0"
 export { default as sanitizeHtml } from 'npm:sanitize-html@^2.11.0';
 export {

src/schemas/nostr.ts

@@ -9,27 +9,12 @@ const signedEventSchema = n.event()
   .refine((event) => event.id === getEventHash(event), 'Event ID does not match hash')
   .refine(verifyEvent, 'Event signature is invalid');
 
-/** Media data schema from `"media"` tags. */
-const mediaDataSchema = z.object({
-  blurhash: z.string().optional().catch(undefined),
-  cid: z.string().optional().catch(undefined),
-  description: z.string().max(200).optional().catch(undefined),
-  height: z.number().int().positive().optional().catch(undefined),
-  mime: z.string().optional().catch(undefined),
-  name: z.string().optional().catch(undefined),
-  size: z.number().int().positive().optional().catch(undefined),
-  width: z.number().int().positive().optional().catch(undefined),
-});
-
 /** Kind 0 content schema for the Ditto server admin user. */
 const serverMetaSchema = n.metadata().and(z.object({
   tagline: z.string().optional().catch(undefined),
   email: z.string().optional().catch(undefined),
 }));
 
-/** Media data from `"media"` tags. */
-type MediaData = z.infer<typeof mediaDataSchema>;
-
 /** NIP-11 Relay Information Document. */
 const relayInfoDocSchema = z.object({
   name: z.string().transform((val) => val.slice(0, 30)).optional().catch(undefined),
@@ -47,12 +32,4 @@ const emojiTagSchema = z.tuple([z.literal('emoji'), z.string(), z.string().url()
 /** NIP-30 custom emoji tag. */
 type EmojiTag = z.infer<typeof emojiTagSchema>;
 
-export {
+export { type EmojiTag, emojiTagSchema, relayInfoDocSchema, serverMetaSchema, signedEventSchema };
-  type EmojiTag,
-  emojiTagSchema,
-  type MediaData,
-  mediaDataSchema,
-  relayInfoDocSchema,
-  serverMetaSchema,
-  signedEventSchema,
-};

src/upload.ts

@@ -1,5 +1,5 @@
 import { Conf } from '@/config.ts';
-import { insertUnattachedMedia } from '@/db/unattached-media.ts';
+import { insertUnattachedMedia, UnattachedMedia } from '@/db/unattached-media.ts';
 import { configUploader as uploader } from '@/uploaders/config.ts';
 
 interface FileMeta {
@@ -8,25 +8,40 @@ interface FileMeta {
 }
 
 /** Upload a file, track it in the database, and return the resulting media object. */
-async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal) {
+async function uploadFile(file: File, meta: FileMeta, signal?: AbortSignal): Promise<UnattachedMedia> {
-  const { name, type, size } = file;
+  const { type, size } = file;
   const { pubkey, description } = meta;
 
   if (file.size > Conf.maxUploadSize) {
     throw new Error('File size is too large.');
   }
 
-  const { url } = await uploader.upload(file, { signal });
+  const { url, sha256, cid } = await uploader.upload(file, { signal });
+
+  const data: string[][] = [
+    ['url', url],
+    ['m', type],
+    ['size', size.toString()],
+  ];
+
+  if (sha256) {
+    data.push(['x', sha256]);
+  }
+
+  if (cid) {
+    data.push(['cid', cid]);
+  }
+
+  if (description) {
+    data.push(['alt', description]);
+  }
 
   return insertUnattachedMedia({
+    id: crypto.randomUUID(),
     pubkey,
     url,
-    data: {
+    data,
-      name,
+    uploaded_at: Date.now(),
-      size,
-      description,
-      mime: type,
-    },
   });
 }
 

src/utils/media.test.ts

@@ -0,0 +1,17 @@
+import { assertEquals } from '@std/assert';
+
+import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts';
+
+Deno.test('getUrlMediaType', () => {
+  assertEquals(getUrlMediaType('https://example.com/image.png'), 'image/png');
+  assertEquals(getUrlMediaType('https://example.com/index.html'), 'text/html');
+  assertEquals(getUrlMediaType('https://example.com/yolo'), undefined);
+  assertEquals(getUrlMediaType('https://example.com/'), undefined);
+});
+
+Deno.test('isPermittedMediaType', () => {
+  assertEquals(isPermittedMediaType('image/png', ['image', 'video']), true);
+  assertEquals(isPermittedMediaType('video/webm', ['image', 'video']), true);
+  assertEquals(isPermittedMediaType('audio/ogg', ['image', 'video']), false);
+  assertEquals(isPermittedMediaType('application/json', ['image', 'video']), false);
+});

src/utils/media.ts

@@ -0,0 +1,24 @@
+import { typeByExtension } from '@std/media-types';
+
+/** Get media type of the filename in the URL by its extension, if any. */
+export function getUrlMediaType(url: string): string | undefined {
+  try {
+    const { pathname } = new URL(url);
+    const ext = pathname.split('.').pop() ?? '';
+    return typeByExtension(ext);
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Check if the base type matches any of the permitted types.
+ *
+ * ```ts
+ * isPermittedMediaType('image/png', ['image', 'video']); // true
+ * ```
+ */
+export function isPermittedMediaType(mediaType: string, permitted: string[]): boolean {
+  const [baseType, _subType] = mediaType.split('/');
+  return permitted.includes(baseType);
+}

src/utils/note.test.ts

@@ -0,0 +1,24 @@
+import { assertEquals } from '@std/assert';
+
+import { getMediaLinks, parseNoteContent } from '@/utils/note.ts';
+
+Deno.test('parseNoteContent', () => {
+  const { html, links, firstUrl } = parseNoteContent('Hello, world!');
+  assertEquals(html, 'Hello, world!');
+  assertEquals(links, []);
+  assertEquals(firstUrl, undefined);
+});
+
+Deno.test('getMediaLinks', () => {
+  const links = [
+    { href: 'https://example.com/image.png' },
+    { href: 'https://example.com/index.html' },
+    { href: 'https://example.com/yolo' },
+    { href: 'https://example.com/' },
+  ];
+  const mediaLinks = getMediaLinks(links);
+  assertEquals(mediaLinks, [[
+    ['url', 'https://example.com/image.png'],
+    ['m', 'image/png'],
+  ]]);
+});

src/utils/note.ts

@@ -4,8 +4,7 @@ import linkify from 'linkifyjs';
 import { nip19, nip21 } from 'nostr-tools';
 
 import { Conf } from '@/config.ts';
-import { mime } from '@/deps.ts';
+import { getUrlMediaType, isPermittedMediaType } from '@/utils/media.ts';
-import { type DittoAttachment } from '@/views/mastodon/attachments.ts';
 
 linkify.registerCustomProtocol('nostr', true);
 linkify.registerCustomProtocol('wss');
@@ -58,20 +57,17 @@ function parseNoteContent(content: string): ParsedNoteContent {
   };
 }
 
-function getMediaLinks(links: Link[]): DittoAttachment[] {
+/** Returns a matrix of tags. Each item is a list of NIP-94 tags representing a file. */
-  return links.reduce<DittoAttachment[]>((acc, link) => {
+function getMediaLinks(links: Pick<Link, 'href'>[]): string[][][] {
-    const mimeType = getUrlMimeType(link.href);
+  return links.reduce<string[][][]>((acc, link) => {
-    if (!mimeType) return acc;
+    const mediaType = getUrlMediaType(link.href);
+    if (!mediaType) return acc;
 
-    const [baseType, _subType] = mimeType.split('/');
+    if (isPermittedMediaType(mediaType, ['audio', 'image', 'video'])) {
-
+      acc.push([
-    if (['audio', 'image', 'video'].includes(baseType)) {
+        ['url', link.href],
-      acc.push({
+        ['m', mediaType],
-        url: link.href,
+      ]);
-        data: {
-          mime: mimeType,
-        },
-      });
     }
 
     return acc;
@@ -79,7 +75,7 @@ function getMediaLinks(links: Link[]): DittoAttachment[] {
 }
 
 function isNonMediaLink({ href }: Link): boolean {
-  return /^https?:\/\//.test(href) && !getUrlMimeType(href);
+  return /^https?:\/\//.test(href) && !getUrlMediaType(href);
 }
 
 /** Ensures the Link is a URL so it can be parsed. */
@@ -87,16 +83,6 @@ function isLinkURL(link: Link): boolean {
   return link.type === 'url';
 }
 
-/** `npm:mime` treats `.com` as a file extension, so parse the full URL to get its path first. */
-function getUrlMimeType(url: string): string | undefined {
-  try {
-    const { pathname } = new URL(url);
-    return mime.getType(pathname) || undefined;
-  } catch (_e) {
-    return undefined;
-  }
-}
-
 /** Get pubkey from decoded bech32 entity, or undefined if not applicable. */
 function getDecodedPubkey(decoded: nip19.DecodeResult): string | undefined {
   switch (decoded.type) {

src/views/mastodon/attachments.ts

@@ -1,20 +1,24 @@
-import * as TypeFest from 'type-fest';
+/** Render Mastodon media attachment. */
+function renderAttachment(media: { id?: string; data: string[][] }) {
+  const { id, data: tags } = media;
 
-import { UnattachedMedia } from '@/db/unattached-media.ts';
+  const m = tags.find(([name]) => name === 'm')?.[1];
+  const url = tags.find(([name]) => name === 'url')?.[1];
+  const alt = tags.find(([name]) => name === 'alt')?.[1];
+  const cid = tags.find(([name]) => name === 'cid')?.[1];
+  const blurhash = tags.find(([name]) => name === 'blurhash')?.[1];
 
-type DittoAttachment = TypeFest.SetOptional<UnattachedMedia, 'id' | 'pubkey' | 'uploaded_at'>;
+  if (!url) return;
 
-function renderAttachment(media: DittoAttachment) {
-  const { id, data, url } = media;
   return {
-    id: id ?? url ?? data.cid,
+    id: id ?? url,
-    type: getAttachmentType(data.mime ?? ''),
+    type: getAttachmentType(m ?? ''),
     url,
     preview_url: url,
     remote_url: null,
-    description: data.description ?? '',
+    description: alt ?? '',
-    blurhash: data.blurhash || null,
+    blurhash: blurhash || null,
-    cid: data.cid,
+    cid: cid,
   };
 }
 
@@ -32,4 +36,4 @@ function getAttachmentType(mime: string): string {
   }
 }
 
-export { type DittoAttachment, renderAttachment };
+export { renderAttachment };

src/views/mastodon/statuses.ts

@@ -1,18 +1,17 @@
-import { NostrEvent, NSchema as n } from '@nostrify/nostrify';
+import { NostrEvent } from '@nostrify/nostrify';
 import { isCWTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts';
 import { nip19 } from 'nostr-tools';
 
 import { Conf } from '@/config.ts';
 import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
-import { getMediaLinks, parseNoteContent } from '@/note.ts';
 import { Storages } from '@/storages.ts';
 import { findReplyTag } from '@/tags.ts';
 import { nostrDate } from '@/utils.ts';
+import { getMediaLinks, parseNoteContent } from '@/utils/note.ts';
 import { unfurlCardCached } from '@/utils/unfurl.ts';
 import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
-import { DittoAttachment, renderAttachment } from '@/views/mastodon/attachments.ts';
+import { renderAttachment } from '@/views/mastodon/attachments.ts';
 import { renderEmojis } from '@/views/mastodon/emojis.ts';
-import { mediaDataSchema } from '@/schemas/nostr.ts';
 
 interface RenderStatusOpts {
   viewerPubkey?: string;
@@ -79,11 +78,11 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
 
   const mediaLinks = getMediaLinks(links);
 
-  const mediaTags: DittoAttachment[] = event.tags
+  const imeta: string[][][] = event.tags
-    .filter((tag) => tag[0] === 'media')
+    .filter(([name]) => name === 'imeta')
-    .map(([_, url, json]) => ({ url, data: n.json().pipe(mediaDataSchema).parse(json) }));
+    .map(([_, ...entries]) => entries.map((entry) => entry.split(' ')));
 
-  const media = [...mediaLinks, ...mediaTags];
+  const media = [...mediaLinks, ...imeta];
 
   return {
     id: event.id,
@@ -107,7 +106,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
     pinned: Boolean(pinEvent),
     reblog: null,
     application: null,
-    media_attachments: media.map(renderAttachment),
+    media_attachments: media.map((m) => renderAttachment({ data: m })).filter(Boolean),
     mentions,
     tags: [],
     emojis: renderEmojis(event),