Merge branch 'media' into 'develop'

Media uploads

Closes #50

See merge request soapbox-pub/ditto!37
This commit is contained in:
Alex Gleason 2023-09-10 15:26:33 +00:00
commit 35b91812fc
No known key found for this signature in database
29 changed files with 788 additions and 162 deletions

View File

@ -29,6 +29,7 @@ import {
import { appCredentialsController, createAppController } from './controllers/api/apps.ts';
import { emptyArrayController, emptyObjectController } from './controllers/api/fallback.ts';
import { instanceController } from './controllers/api/instance.ts';
import { mediaController } from './controllers/api/media.ts';
import { notificationsController } from './controllers/api/notifications.ts';
import { createTokenController, oauthAuthorizeController, oauthController } from './controllers/api/oauth.ts';
import { frontendConfigController, updateConfigController } from './controllers/api/pleroma.ts';
@ -56,7 +57,7 @@ import { nodeInfoController, nodeInfoSchemaController } from './controllers/well
import { nostrController } from './controllers/well-known/nostr.ts';
import { webfingerController } from './controllers/well-known/webfinger.ts';
import { auth19, requirePubkey } from './middleware/auth19.ts';
import { auth98, requireAdmin } from './middleware/auth98.ts';
import { auth98, requireRole } from './middleware/auth98.ts';
interface AppEnv extends HonoEnv {
Variables: {
@ -121,6 +122,9 @@ app.get('/api/v1/statuses/:id{[0-9a-f]{64}}', statusController);
app.post('/api/v1/statuses/:id{[0-9a-f]{64}}/favourite', favouriteController);
app.post('/api/v1/statuses', requirePubkey, createStatusController);
app.post('/api/v1/media', requireRole('user', { validatePayload: false }), mediaController);
app.post('/api/v2/media', requireRole('user', { validatePayload: false }), mediaController);
app.get('/api/v1/timelines/home', requirePubkey, homeTimelineController);
app.get('/api/v1/timelines/public', publicTimelineController);
app.get('/api/v1/timelines/tag/:hashtag', hashtagTimelineController);
@ -137,7 +141,7 @@ app.get('/api/v1/trends', trendingTagsController);
app.get('/api/v1/notifications', requirePubkey, notificationsController);
app.get('/api/v1/favourites', requirePubkey, favouritesController);
app.post('/api/v1/pleroma/admin/config', requireAdmin, updateConfigController);
app.post('/api/v1/pleroma/admin/config', requireRole('admin'), updateConfigController);
// Not (yet) implemented.
app.get('/api/v1/bookmarks', emptyArrayController);

View File

@ -1,4 +1,4 @@
import { dotenv, getPublicKey, nip19, secp } from '@/deps.ts';
import { dotenv, getPublicKey, nip19, secp, z } from '@/deps.ts';
/** Load environment config from `.env` */
await dotenv.load({
@ -42,7 +42,7 @@ const Conf = {
const { protocol, host } = Conf.url;
return `${protocol === 'https:' ? 'wss:' : 'ws:'}//${host}/relay`;
},
/** Domain of the Ditto server, including the protocol. */
/** Origin of the Ditto server, including the protocol and port. */
get localDomain() {
return Deno.env.get('LOCAL_DOMAIN') || 'http://localhost:8000';
},
@ -58,13 +58,88 @@ const Conf = {
get adminEmail() {
return Deno.env.get('ADMIN_EMAIL') || 'webmaster@localhost';
},
/** S3 media storage configuration. */
s3: {
get endPoint() {
return Deno.env.get('S3_ENDPOINT')!;
},
get region() {
return Deno.env.get('S3_REGION')!;
},
get accessKey() {
return Deno.env.get('S3_ACCESS_KEY');
},
get secretKey() {
return Deno.env.get('S3_SECRET_KEY');
},
get bucket() {
return Deno.env.get('S3_BUCKET');
},
get pathStyle() {
return optionalBooleanSchema.parse(Deno.env.get('S3_PATH_STYLE'));
},
get port() {
return optionalNumberSchema.parse(Deno.env.get('S3_PORT'));
},
get sessionToken() {
return Deno.env.get('S3_SESSION_TOKEN');
},
get useSSL() {
return optionalBooleanSchema.parse(Deno.env.get('S3_USE_SSL'));
},
},
/** IPFS uploader configuration. */
ipfs: {
/** Base URL for private IPFS API calls. */
get apiUrl() {
return Deno.env.get('IPFS_API_URL') || 'http://localhost:5001';
},
},
/** Module to upload files with. */
get uploader() {
return Deno.env.get('DITTO_UPLOADER');
},
/** Media base URL for uploads. */
get mediaDomain() {
const value = Deno.env.get('MEDIA_DOMAIN');
if (!value) {
const url = Conf.url;
url.host = `media.${url.host}`;
return url.toString();
}
return value;
},
/** Max upload size for files in number of bytes. Default 100MiB. */
get maxUploadSize() {
return Number(Deno.env.get('MAX_UPLOAD_SIZE') || 100 * 1024 * 1024);
},
/** Domain of the Ditto server as a `URL` object, for easily grabbing the `hostname`, etc. */
get url() {
return new URL(Conf.localDomain);
},
/** Merges the path with the localDomain. */
local(path: string): string {
const url = new URL(path.startsWith('/') ? path : new URL(path).pathname, Conf.localDomain);
return mergePaths(Conf.localDomain, path);
},
};
const optionalBooleanSchema = z
.enum(['true', 'false'])
.optional()
.transform((value) => value !== undefined ? value === 'true' : undefined);
const optionalNumberSchema = z
.string()
.optional()
.transform((value) => value !== undefined ? Number(value) : undefined);
function mergePaths(base: string, path: string) {
const url = new URL(
path.startsWith('/') ? path : new URL(path).pathname,
base,
);
if (!path.startsWith('/')) {
// Copy query parameters from the original URL to the new URL
@ -73,7 +148,6 @@ const Conf = {
}
return url.toString();
},
};
}
export { Conf };

View File

@ -2,7 +2,7 @@ import { type AppController } from '@/app.ts';
import { type Filter, findReplyTag, z } from '@/deps.ts';
import * as mixer from '@/mixer.ts';
import { getAuthor, getFollowedPubkeys, getFollows, syncUser } from '@/queries.ts';
import { booleanParamSchema } from '@/schema.ts';
import { booleanParamSchema, fileSchema } from '@/schema.ts';
import { jsonMetaContentSchema } from '@/schemas/nostr.ts';
import { toAccount, toRelationship, toStatus } from '@/transformers/nostr-to-mastoapi.ts';
import { isFollowing, lookupAccount, Time } from '@/utils.ts';
@ -113,8 +113,6 @@ const accountStatusesController: AppController = async (c) => {
return paginated(c, events, statuses);
};
const fileSchema = z.custom<File>((value) => value instanceof File);
const updateCredentialsSchema = z.object({
display_name: z.string().optional(),
note: z.string().optional(),

View File

@ -0,0 +1,52 @@
import { AppController } from '@/app.ts';
import { Conf } from '@/config.ts';
import { insertUnattachedMedia } from '@/db/unattached-media.ts';
import { z } from '@/deps.ts';
import { fileSchema } from '@/schema.ts';
import { configUploader as uploader } from '@/uploaders/config.ts';
import { parseBody } from '@/utils/web.ts';
import { renderAttachment } from '@/views/attachment.ts';
const uploadSchema = fileSchema
.refine((file) => !!file.type, 'File type is required.')
.refine((file) => file.size <= Conf.maxUploadSize, 'File size is too large.');
const mediaBodySchema = z.object({
file: uploadSchema,
thumbnail: uploadSchema.optional(),
description: z.string().optional(),
focus: z.string().optional(),
});
const mediaController: AppController = async (c) => {
const result = mediaBodySchema.safeParse(await parseBody(c.req.raw));
if (!result.success) {
return c.json({ error: 'Bad request.', schema: result.error }, 422);
}
try {
const { file, description } = result.data;
const { cid } = await uploader.upload(file);
const url = new URL(`/ipfs/${cid}`, Conf.mediaDomain).toString();
const media = await insertUnattachedMedia({
pubkey: c.get('pubkey')!,
url,
data: {
name: file.name,
mime: file.type,
size: file.size,
description,
},
});
return c.json(renderAttachment(media));
} catch (e) {
console.error(e);
return c.json({ error: 'Failed to upload file.' }, 500);
}
};
export { mediaController };

View File

@ -4,6 +4,7 @@ import { getAncestors, getDescendants, getEvent } from '@/queries.ts';
import { toStatus } from '@/transformers/nostr-to-mastoapi.ts';
import { createEvent, paginationSchema, parseBody } from '@/utils/web.ts';
import { renderEventAccounts } from '@/views.ts';
import { getUnattachedMediaByIds } from '@/db/unattached-media.ts';
const createStatusSchema = z.object({
in_reply_to_id: z.string().regex(/[0-9a-f]{64}/).nullish(),
@ -40,7 +41,10 @@ const createStatusController: AppController = async (c) => {
const body = await parseBody(c.req.raw);
const result = createStatusSchema.safeParse(body);
if (result.success) {
if (!result.success) {
return c.json({ error: 'Bad request', schema: result.error }, 400);
}
const { data } = result;
if (data.visibility !== 'public') {
@ -51,10 +55,6 @@ const createStatusController: AppController = async (c) => {
return c.json({ error: 'Polls are not yet supported.' }, 422);
}
if (data.media_ids?.length) {
return c.json({ error: 'Media uploads are not yet supported.' }, 422);
}
const tags: string[][] = [];
if (data.in_reply_to_id) {
@ -69,6 +69,14 @@ const createStatusController: AppController = async (c) => {
tags.push(['subject', data.spoiler_text]);
}
if (data.media_ids?.length) {
const media = await getUnattachedMediaByIds(data.media_ids)
.then((media) => media.filter(({ pubkey }) => pubkey === c.get('pubkey')))
.then((media) => media.map(({ url, data }) => ['media', url, data]));
tags.push(...media);
}
const event = await createEvent({
kind: 1,
content: data.status ?? '',
@ -76,9 +84,6 @@ const createStatusController: AppController = async (c) => {
}, c);
return c.json(await toStatus(event, c.get('pubkey')));
} else {
return c.json({ error: 'Bad request', schema: result.error }, 400);
}
};
const contextController: AppController = async (c) => {

View File

@ -1,6 +1,9 @@
import * as eventsDB from '@/db/events.ts';
import { deleteUnattachedMediaByUrl, getUnattachedMedia } from '@/db/unattached-media.ts';
import { cron } from '@/deps.ts';
import { Time } from '@/utils/time.ts';
import { configUploader as uploader } from '@/uploaders/config.ts';
import { cidFromUrl } from '@/utils/ipfs.ts';
/** Clean up old remote events. */
async function cleanupEvents() {
@ -14,6 +17,29 @@ async function cleanupEvents() {
console.log(`Cleaned up ${result?.numDeletedRows ?? 0} old remote events.`);
}
/** Delete files that aren't attached to any events. */
async function cleanupMedia() {
console.log('Deleting orphaned media files...');
const until = new Date(Date.now() - Time.minutes(15));
const media = await getUnattachedMedia(until);
for (const { url } of media) {
const cid = cidFromUrl(new URL(url))!;
try {
await uploader.delete(cid);
await deleteUnattachedMediaByUrl(url);
} catch (e) {
console.error(`Failed to delete file ${url}`);
console.error(e);
}
}
console.log(`Removed ${media?.length ?? 0} orphaned media files.`);
}
await cleanupEvents();
await cleanupMedia();
cron.every15Minute(cleanupEvents);
cron.every15Minute(cleanupMedia);

View File

@ -10,6 +10,7 @@ interface DittoDB {
tags: TagRow;
users: UserRow;
relays: RelayRow;
unattached_media: UnattachedMediaRow;
}
interface EventRow {
@ -46,6 +47,14 @@ interface RelayRow {
active: boolean;
}
interface UnattachedMediaRow {
id: string;
pubkey: string;
url: string;
data: string;
uploaded_at: Date;
}
const db = new Kysely<DittoDB>({
dialect: new DenoSqliteDialect({
database: new Sqlite(Conf.dbPath),

View File

@ -6,12 +6,12 @@ import { insertUser } from '@/db/users.ts';
Deno.test('count filters', async () => {
assertEquals(await countFilters([{ kinds: [1] }]), 0);
await insertEvent(event55920b75);
await insertEvent(event55920b75, { user: undefined });
assertEquals(await countFilters([{ kinds: [1] }]), 1);
});
Deno.test('insert and filter events', async () => {
await insertEvent(event55920b75);
await insertEvent(event55920b75, { user: undefined });
assertEquals(await getFilters([{ kinds: [1] }]), [event55920b75]);
assertEquals(await getFilters([{ kinds: [3] }]), []);
@ -24,14 +24,14 @@ Deno.test('insert and filter events', async () => {
});
Deno.test('delete events', async () => {
await insertEvent(event55920b75);
await insertEvent(event55920b75, { user: undefined });
assertEquals(await getFilters([{ kinds: [1] }]), [event55920b75]);
await deleteFilters([{ kinds: [1] }]);
assertEquals(await getFilters([{ kinds: [1] }]), []);
});
Deno.test('query events with local filter', async () => {
await insertEvent(event55920b75);
await insertEvent(event55920b75, { user: undefined });
assertEquals(await getFilters([{}]), [event55920b75]);
assertEquals(await getFilters([{ local: true }]), []);

View File

@ -1,58 +1,67 @@
import { db, type TagRow } from '@/db.ts';
import { type Event, type Insertable, SqliteError } from '@/deps.ts';
import { db } from '@/db.ts';
import { type Event, SqliteError } from '@/deps.ts';
import { isParameterizedReplaceableKind } from '@/kinds.ts';
import { jsonMetaContentSchema } from '@/schemas/nostr.ts';
import { EventData } from '@/types.ts';
import { isNostrId, isURL } from '@/utils.ts';
import type { DittoFilter, GetFiltersOpts } from '@/filter.ts';
import { jsonMetaContentSchema } from '@/schemas/nostr.ts';
type TagCondition = ({ event, count }: { event: Event; count: number }) => boolean;
/** Function to decide whether or not to index a tag. */
type TagCondition = ({ event, count, value }: {
event: Event;
data: EventData;
count: number;
value: string;
}) => boolean;
/** Conditions for when to index certain tags. */
const tagConditions: Record<string, TagCondition> = {
'd': ({ event, count }) => 30000 <= event.kind && event.kind < 40000 && count === 0,
'e': ({ count }) => count < 15,
'p': ({ event, count }) => event.kind === 3 || count < 15,
'proxy': ({ count }) => count === 0,
'q': ({ event, count }) => event.kind === 1 && count === 0,
't': ({ count }) => count < 5,
'd': ({ event, count }) => count === 0 && isParameterizedReplaceableKind(event.kind),
'e': ({ count, value }) => count < 15 && isNostrId(value),
'media': ({ count, value, data }) => (data.user || count < 4) && isURL(value),
'p': ({ event, count, value }) => (count < 15 || event.kind === 3) && isNostrId(value),
'proxy': ({ count, value }) => count === 0 && isURL(value),
'q': ({ event, count, value }) => count === 0 && event.kind === 1 && isNostrId(value),
't': ({ count, value }) => count < 5 && value.length < 50,
};
/** Insert an event (and its tags) into the database. */
function insertEvent(event: Event): Promise<void> {
function insertEvent(event: Event, data: EventData): Promise<void> {
return db.transaction().execute(async (trx) => {
/** Insert the event into the database. */
async function addEvent() {
await trx.insertInto('events')
.values({
...event,
tags: JSON.stringify(event.tags),
})
.values({ ...event, tags: JSON.stringify(event.tags) })
.execute();
}
/** Add search data to the FTS table. */
async function indexSearch() {
const searchContent = buildSearchContent(event);
if (searchContent) {
if (!searchContent) return;
await trx.insertInto('events_fts')
.values({ id: event.id, content: searchContent.substring(0, 1000) })
.execute();
}
const tagCounts: Record<string, number> = {};
const tags = event.tags.reduce<Insertable<TagRow>[]>((results, [name, value]) => {
tagCounts[name] = (tagCounts[name] || 0) + 1;
/** Index event tags depending on the conditions defined above. */
async function indexTags() {
const tags = filterIndexableTags(event, data);
const rows = tags.map(([tag, value]) => ({ event_id: event.id, tag, value }));
if (value && tagConditions[name]?.({ event, count: tagCounts[name] - 1 })) {
results.push({
event_id: event.id,
tag: name,
value,
});
}
return results;
}, []);
if (tags.length) {
if (!tags.length) return;
await trx.insertInto('tags')
.values(tags)
.values(rows)
.execute();
}
// Run the queries.
await Promise.all([
addEvent(),
indexTags(),
indexSearch(),
]);
}).catch((error) => {
// Don't throw for duplicate events.
if (error instanceof SqliteError && error.code === 19) {
@ -181,6 +190,40 @@ async function countFilters<K extends number>(filters: DittoFilter<K>[]): Promis
return Number(count);
}
/** Return only the tags that should be indexed. */
function filterIndexableTags(event: Event, data: EventData): string[][] {
const tagCounts: Record<string, number> = {};
function getCount(name: string) {
return tagCounts[name] || 0;
}
function incrementCount(name: string) {
tagCounts[name] = getCount(name) + 1;
}
function checkCondition(name: string, value: string, condition: TagCondition) {
return condition({
event,
data,
count: getCount(name),
value,
});
}
return event.tags.reduce<string[][]>((results, tag) => {
const [name, value] = tag;
const condition = tagConditions[name] as TagCondition | undefined;
if (value && condition && value.length < 200 && checkCondition(name, value, condition)) {
results.push(tag);
}
incrementCount(name);
return results;
}, []);
}
/** Build a search index from the event. */
function buildSearchContent(event: Event): string {
switch (event.kind) {

View File

@ -0,0 +1,34 @@
import { Kysely, sql } from '@/deps.ts';
export async function up(db: Kysely<any>): Promise<void> {
await db.schema
.createTable('unattached_media')
.addColumn('id', 'text', (c) => c.primaryKey())
.addColumn('pubkey', 'text', (c) => c.notNull())
.addColumn('url', 'text', (c) => c.notNull())
.addColumn('data', 'text', (c) => c.notNull())
.addColumn('uploaded_at', 'datetime', (c) => c.notNull().defaultTo(sql`CURRENT_TIMESTAMP`))
.execute();
await db.schema
.createIndex('unattached_media_id')
.on('unattached_media')
.column('id')
.execute();
await db.schema
.createIndex('unattached_media_pubkey')
.on('unattached_media')
.column('pubkey')
.execute();
await db.schema
.createIndex('unattached_media_url')
.on('unattached_media')
.column('url')
.execute();
}
export async function down(db: Kysely<any>): Promise<void> {
await db.schema.dropTable('unattached_media').execute();
}

View File

@ -0,0 +1,77 @@
import { db } from '@/db.ts';
import { uuid62 } from '@/deps.ts';
import { type MediaData } from '@/schemas/nostr.ts';
interface UnattachedMedia {
id: string;
pubkey: string;
url: string;
data: MediaData;
uploaded_at: Date;
}
/** Add unattached media into the database. */
async function insertUnattachedMedia(media: Omit<UnattachedMedia, 'id' | 'uploaded_at'>) {
const result = {
id: uuid62.v4(),
uploaded_at: new Date(),
...media,
};
await db.insertInto('unattached_media')
.values({ ...result, data: JSON.stringify(media.data) })
.execute();
return result;
}
/** Select query for unattached media. */
function selectUnattachedMediaQuery() {
return db.selectFrom('unattached_media')
.select([
'unattached_media.id',
'unattached_media.pubkey',
'unattached_media.url',
'unattached_media.data',
'unattached_media.uploaded_at',
]);
}
/** Find attachments that exist but aren't attached to any events. */
function getUnattachedMedia(until: Date) {
return selectUnattachedMediaQuery()
.leftJoin('tags', 'unattached_media.url', 'tags.value')
.where('uploaded_at', '<', until)
.execute();
}
/** Delete unattached media by URL. */
function deleteUnattachedMediaByUrl(url: string) {
return db.deleteFrom('unattached_media')
.where('url', '=', url)
.execute();
}
/** Get unattached media by IDs. */
function getUnattachedMediaByIds(ids: string[]) {
return selectUnattachedMediaQuery()
.where('id', 'in', ids)
.execute();
}
/** Delete rows as an event with media is being created. */
function deleteAttachedMedia(pubkey: string, urls: string[]) {
return db.deleteFrom('unattached_media')
.where('pubkey', '=', pubkey)
.where('url', 'in', urls)
.execute();
}
export {
deleteAttachedMedia,
deleteUnattachedMediaByUrl,
getUnattachedMedia,
getUnattachedMediaByIds,
insertUnattachedMedia,
type UnattachedMedia,
};

View File

@ -66,5 +66,8 @@ export {
export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/v1.0.1/mod.ts';
export { default as tldts } from 'npm:tldts@^6.0.14';
export * as cron from 'https://deno.land/x/deno_cron@v1.0.0/cron.ts';
export { S3Client } from 'https://deno.land/x/s3_lite_client@0.6.1/mod.ts';
export { default as IpfsHash } from 'npm:ipfs-only-hash@^4.0.0';
export { default as uuid62 } from 'npm:uuid62@^1.0.2';
export type * as TypeFest from 'npm:type-fest@^4.3.0';

View File

@ -1,9 +1,14 @@
import { type AppContext, type AppMiddleware } from '@/app.ts';
import { HTTPException } from '@/deps.ts';
import { buildAuthEventTemplate, parseAuthRequest, type ParseAuthRequestOpts } from '@/utils/nip98.ts';
import {
buildAuthEventTemplate,
parseAuthRequest,
type ParseAuthRequestOpts,
validateAuthEvent,
} from '@/utils/nip98.ts';
import { localRequest } from '@/utils/web.ts';
import { signNostrConnect } from '@/sign.ts';
import { findUser } from '@/db/users.ts';
import { signEvent } from '@/sign.ts';
import { findUser, User } from '@/db/users.ts';
/**
* NIP-98 auth.
@ -23,13 +28,16 @@ function auth98(opts: ParseAuthRequestOpts = {}): AppMiddleware {
};
}
/** Require the user to prove they're an admin before invoking the controller. */
const requireAdmin: AppMiddleware = async (c, next) => {
type UserRole = 'user' | 'admin';
/** Require the user to prove their role before invoking the controller. */
function requireRole(role: UserRole, opts?: ParseAuthRequestOpts): AppMiddleware {
return async (c, next) => {
const header = c.req.headers.get('x-nostr-sign');
const proof = c.get('proof') || header ? await obtainProof(c) : undefined;
const proof = c.get('proof') || header ? await obtainProof(c, opts) : undefined;
const user = proof ? await findUser({ pubkey: proof.pubkey }) : undefined;
if (proof && user?.admin) {
if (proof && user && matchesRole(user, role)) {
c.set('pubkey', proof.pubkey);
c.set('proof', proof);
await next();
@ -37,12 +45,30 @@ const requireAdmin: AppMiddleware = async (c, next) => {
throw new HTTPException(401);
}
};
/** Get the proof over Nostr Connect. */
async function obtainProof(c: AppContext) {
const req = localRequest(c);
const event = await buildAuthEventTemplate(req);
return signNostrConnect(event, c);
}
export { auth98, requireAdmin };
/** Check whether the user fulfills the role. */
function matchesRole(user: User, role: UserRole): boolean {
switch (role) {
case 'user':
return true;
case 'admin':
return user.admin;
default:
return false;
}
}
/** Get the proof over Nostr Connect. */
async function obtainProof(c: AppContext, opts?: ParseAuthRequestOpts) {
const req = localRequest(c);
const reqEvent = await buildAuthEventTemplate(req, opts);
const resEvent = await signEvent(reqEvent, c);
const result = await validateAuthEvent(req, resEvent, opts);
if (result.success) {
return result.data;
}
}
export { auth98, requireRole };

View File

@ -1,5 +1,6 @@
import { Conf } from '@/config.ts';
import { linkify, linkifyStr, mime, nip19, nip21 } from '@/deps.ts';
import { type DittoAttachment } from '@/views/attachment.ts';
linkify.registerCustomProtocol('nostr', true);
linkify.registerCustomProtocol('wss');
@ -52,13 +53,8 @@ function parseNoteContent(content: string): ParsedNoteContent {
};
}
interface MediaLink {
url: string;
mimeType: string;
}
function getMediaLinks(links: Link[]): MediaLink[] {
return links.reduce<MediaLink[]>((acc, link) => {
function getMediaLinks(links: Link[]): DittoAttachment[] {
return links.reduce<DittoAttachment[]>((acc, link) => {
const mimeType = getUrlMimeType(link.href);
if (!mimeType) return acc;
@ -67,7 +63,9 @@ function getMediaLinks(links: Link[]): MediaLink[] {
if (['audio', 'image', 'video'].includes(baseType)) {
acc.push({
url: link.href,
mimeType,
data: {
mime: mimeType,
},
});
}
@ -110,4 +108,4 @@ function getDecodedPubkey(decoded: nip19.DecodeResult): string | undefined {
}
}
export { getMediaLinks, type MediaLink, parseNoteContent };
export { getMediaLinks, parseNoteContent };

View File

@ -1,6 +1,7 @@
import { Conf } from '@/config.ts';
import * as eventsDB from '@/db/events.ts';
import { addRelays } from '@/db/relays.ts';
import { deleteAttachedMedia } from '@/db/unattached-media.ts';
import { findUser } from '@/db/users.ts';
import { type Event, LRUCache } from '@/deps.ts';
import { isEphemeralKind } from '@/kinds.ts';
@ -27,6 +28,7 @@ async function handleEvent(event: Event): Promise<void> {
processDeletions(event),
trackRelays(event),
trackHashtags(event),
processMedia(event, data),
streamOut(event, data),
broadcast(event, data),
]);
@ -64,7 +66,7 @@ async function storeEvent(event: Event, data: EventData): Promise<void> {
if (deletion) {
return Promise.reject(new RelayError('blocked', 'event was deleted'));
} else {
await eventsDB.insertEvent(event).catch(console.warn);
await eventsDB.insertEvent(event, data).catch(console.warn);
}
} else {
return Promise.reject(new RelayError('blocked', 'only registered users can post'));
@ -120,6 +122,14 @@ function trackRelays(event: Event) {
return addRelays([...relays]);
}
/** Delete unattached media entries that are attached to the event. */
function processMedia({ tags, pubkey }: Event, { user }: EventData) {
if (user) {
const urls = getTagSet(tags, 'media');
return deleteAttachedMedia(pubkey, [...urls]);
}
}
/** Determine if the event is being received in a timely manner. */
const isFresh = (event: Event): boolean => eventAge(event) < Time.seconds(10);

22
src/precheck.ts Normal file
View File

@ -0,0 +1,22 @@
import { Conf } from '@/config.ts';
/** Ensure the media URL is not on the same host as the local domain. */
function checkMediaHost() {
const { url, mediaDomain } = Conf;
const mediaUrl = new URL(mediaDomain);
if (url.host === mediaUrl.host) {
throw new PrecheckError('For security reasons, MEDIA_DOMAIN cannot be on the same host as LOCAL_DOMAIN.');
}
}
/** Error class for precheck errors. */
class PrecheckError extends Error {
constructor(message: string) {
super(`${message}\nTo disable this check, set DITTO_PRECHECK="false"`);
}
}
if (Deno.env.get('DITTO_PRECHECK') !== 'false') {
checkMediaHost();
}

View File

@ -48,4 +48,16 @@ const safeUrlSchema = z.string().max(2048).url();
/** https://github.com/colinhacks/zod/issues/1630#issuecomment-1365983831 */
const booleanParamSchema = z.enum(['true', 'false']).transform((value) => value === 'true');
export { booleanParamSchema, decode64Schema, emojiTagSchema, filteredArray, hashtagSchema, jsonSchema, safeUrlSchema };
/** Schema for `File` objects. */
const fileSchema = z.custom<File>((value) => value instanceof File);
export {
booleanParamSchema,
decode64Schema,
emojiTagSchema,
fileSchema,
filteredArray,
hashtagSchema,
jsonSchema,
safeUrlSchema,
};

View File

@ -73,9 +73,27 @@ const metaContentSchema = z.object({
lud16: z.string().optional().catch(undefined),
}).partial().passthrough();
/** Media data schema from `"media"` tags. */
const mediaDataSchema = z.object({
blurhash: z.string().optional().catch(undefined),
cid: z.string().optional().catch(undefined),
description: z.string().max(200).optional().catch(undefined),
height: z.number().int().positive().optional().catch(undefined),
mime: z.string().optional().catch(undefined),
name: z.string().optional().catch(undefined),
size: z.number().int().positive().optional().catch(undefined),
width: z.number().int().positive().optional().catch(undefined),
});
/** Media data from `"media"` tags. */
type MediaData = z.infer<typeof mediaDataSchema>;
/** Parses kind 0 content from a JSON string. */
const jsonMetaContentSchema = jsonSchema.pipe(metaContentSchema).catch({});
/** Parses media data from a JSON string. */
const jsonMediaDataSchema = jsonSchema.pipe(mediaDataSchema).catch({});
/** NIP-11 Relay Information Document. */
const relayInfoDocSchema = z.object({
name: z.string().transform((val) => val.slice(0, 30)).optional().catch(undefined),
@ -102,7 +120,10 @@ export {
type ClientREQ,
connectResponseSchema,
filterSchema,
jsonMediaDataSchema,
jsonMetaContentSchema,
type MediaData,
mediaDataSchema,
metaContentSchema,
nostrIdSchema,
relayInfoDocSchema,

View File

@ -1,3 +1,4 @@
import './precheck.ts';
import app from './app.ts';
Deno.serve(app.fetch);

View File

@ -99,4 +99,4 @@ async function signAdminEvent<K extends number = number>(event: EventTemplate<K>
return finishEvent(event, Conf.seckey);
}
export { signAdminEvent, signEvent, signNostrConnect };
export { signAdminEvent, signEvent };

View File

@ -2,14 +2,15 @@ import { isCWTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ad
import { Conf } from '@/config.ts';
import * as eventsDB from '@/db/events.ts';
import { type Event, findReplyTag, lodash, nip19, sanitizeHtml, TTLCache, unfurl, z } from '@/deps.ts';
import { getMediaLinks, type MediaLink, parseNoteContent } from '@/note.ts';
import { type Event, findReplyTag, lodash, nip19, sanitizeHtml, TTLCache, unfurl } from '@/deps.ts';
import { getMediaLinks, parseNoteContent } from '@/note.ts';
import { getAuthor, getFollowedPubkeys, getFollows } from '@/queries.ts';
import { emojiTagSchema, filteredArray } from '@/schema.ts';
import { jsonMetaContentSchema } from '@/schemas/nostr.ts';
import { jsonMediaDataSchema, jsonMetaContentSchema } from '@/schemas/nostr.ts';
import { isFollowing, type Nip05, nostrDate, parseNip05, Time } from '@/utils.ts';
import { verifyNip05Cached } from '@/utils/nip05.ts';
import { findUser } from '@/db/users.ts';
import { DittoAttachment, renderAttachment } from '@/views/attachment.ts';
const DEFAULT_AVATAR = 'https://gleasonator.com/images/avi.png';
const DEFAULT_BANNER = 'https://gleasonator.com/images/banner.png';
@ -118,7 +119,6 @@ async function toStatus(event: Event<1>, viewerPubkey?: string) {
];
const { html, links, firstUrl } = parseNoteContent(event.content);
const mediaLinks = getMediaLinks(links);
const [mentions, card, repliesCount, reblogsCount, favouritesCount, [repostEvent], [reactionEvent]] = await Promise
.all([
@ -140,6 +140,14 @@ async function toStatus(event: Event<1>, viewerPubkey?: string) {
const cw = event.tags.find(isCWTag);
const subject = event.tags.find((tag) => tag[0] === 'subject');
const mediaLinks = getMediaLinks(links);
const mediaTags: DittoAttachment[] = event.tags
.filter((tag) => tag[0] === 'media')
.map(([_, url, json]) => ({ url, data: jsonMediaDataSchema.parse(json) }));
const media = [...mediaLinks, ...mediaTags];
return {
id: event.id,
account,
@ -161,7 +169,7 @@ async function toStatus(event: Event<1>, viewerPubkey?: string) {
bookmarked: false,
reblog: null,
application: null,
media_attachments: mediaLinks.map(renderAttachment),
media_attachments: media.map(renderAttachment),
mentions,
tags: [],
emojis: toEmojis(event),
@ -185,24 +193,6 @@ function buildInlineRecipients(mentions: Mention[]): string {
return `<span class="recipients-inline">${elements.join(' ')} </span>`;
}
const attachmentTypeSchema = z.enum(['image', 'video', 'gifv', 'audio', 'unknown']).catch('unknown');
function renderAttachment({ url, mimeType }: MediaLink) {
const [baseType, _subType] = mimeType.split('/');
const type = attachmentTypeSchema.parse(baseType);
return {
id: url,
type,
url,
preview_url: url,
remote_url: null,
meta: {},
description: '',
blurhash: null,
};
}
interface PreviewCard {
url: string;
title: string;

30
src/uploaders/config.ts Normal file
View File

@ -0,0 +1,30 @@
import { Conf } from '@/config.ts';
import { ipfsUploader } from './ipfs.ts';
import { s3Uploader } from './s3.ts';
import type { Uploader } from './types.ts';
/** Meta-uploader determined from configuration. */
const configUploader: Uploader = {
upload(file) {
return uploader().upload(file);
},
delete(cid) {
return uploader().delete(cid);
},
};
/** Get the uploader module based on configuration. */
function uploader() {
switch (Conf.uploader) {
case 's3':
return s3Uploader;
case 'ipfs':
return ipfsUploader;
default:
return ipfsUploader;
}
}
export { configUploader };

50
src/uploaders/ipfs.ts Normal file
View File

@ -0,0 +1,50 @@
import { Conf } from '@/config.ts';
import { z } from '@/deps.ts';
import type { Uploader } from './types.ts';
/** Response schema for POST `/api/v0/add`. */
const ipfsAddResponseSchema = z.object({
Name: z.string(),
Hash: z.string(),
Size: z.string(),
});
/**
* IPFS uploader. It expects an IPFS node up and running.
* It will try to connect to `http://localhost:5001` by default,
* and upload the file using the REST API.
*/
const ipfsUploader: Uploader = {
async upload(file) {
const url = new URL('/api/v0/add', Conf.ipfs.apiUrl);
const formData = new FormData();
formData.append('file', file);
const response = await fetch(url, {
method: 'POST',
body: formData,
});
const { Hash } = ipfsAddResponseSchema.parse(await response.json());
return {
cid: Hash,
};
},
async delete(cid) {
const url = new URL('/api/v0/pin/rm', Conf.ipfs.apiUrl);
const query = new URLSearchParams();
query.set('arg', cid);
url.search = query.toString();
await fetch(url, {
method: 'POST',
});
},
};
export { ipfsUploader };

33
src/uploaders/s3.ts Normal file
View File

@ -0,0 +1,33 @@
import { Conf } from '@/config.ts';
import { IpfsHash, S3Client } from '@/deps.ts';
import type { Uploader } from './types.ts';
const s3 = new S3Client({ ...Conf.s3 });
/**
* S3-compatible uploader for AWS, Wasabi, DigitalOcean Spaces, and more.
* Files are named by their IPFS CID and exposed at `/ipfs/<cid>`, letting it
* take advantage of IPFS features while not really using IPFS.
*/
const s3Uploader: Uploader = {
async upload(file) {
const cid = await IpfsHash.of(file.stream()) as string;
await s3.putObject(`ipfs/${cid}`, file.stream(), {
metadata: {
'Content-Type': file.type,
'x-amz-acl': 'public-read',
},
});
return {
cid,
};
},
async delete(cid) {
await s3.deleteObject(`ipfs/${cid}`);
},
};
export { s3Uploader };

15
src/uploaders/types.ts Normal file
View File

@ -0,0 +1,15 @@
/** Modular uploader interface, to support uploading to different backends. */
interface Uploader {
/** Upload the file to the backend. */
upload(file: File): Promise<UploadResult>;
/** Delete the file from the backend. */
delete(cid: string): Promise<void>;
}
/** Return value from the uploader after uploading a file. */
interface UploadResult {
/** IPFS CID for the file. */
cid: string;
}
export type { Uploader };

View File

@ -1,6 +1,7 @@
import { type Event, type EventTemplate, getEventHash, nip19, z } from '@/deps.ts';
import { getAuthor } from '@/queries.ts';
import { lookupNip05Cached } from '@/utils/nip05.ts';
import { nostrIdSchema } from '@/schemas/nostr.ts';
/** Get the current time in Nostr format. */
const nostrNow = (): number => Math.floor(Date.now() / 1000);
@ -111,6 +112,21 @@ function eventMatchesTemplate(event: Event, template: EventTemplate): boolean {
return getEventHash(event) === getEventHash({ pubkey: event.pubkey, ...template });
}
/** Test whether the value is a Nostr ID. */
function isNostrId(value: unknown): boolean {
return nostrIdSchema.safeParse(value).success;
}
/** Test whether the value is a URL. */
function isURL(value: unknown): boolean {
try {
new URL(value as string);
return true;
} catch (_) {
return false;
}
}
export {
bech32ToPubkey,
dedupeEvents,
@ -119,7 +135,9 @@ export {
eventMatchesTemplate,
findTag,
isFollowing,
isNostrId,
isRelay,
isURL,
lookupAccount,
type Nip05,
nostrDate,

27
src/utils/ipfs.ts Normal file
View File

@ -0,0 +1,27 @@
/** https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway */
const IPFS_PATH_REGEX = /^\/ipfs\/([^/]+)/;
/** https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway */
const IPFS_HOST_REGEX = /^([^/]+)\.ipfs\./;
/** Get IPFS CID out of a path. */
function cidFromPath(path: string) {
return path.match(IPFS_PATH_REGEX)?.[1];
}
/** Get IPFS CID out of a host. */
function cidFromHost(host: string) {
return host.match(IPFS_HOST_REGEX)?.[1];
}
/** Get IPFS CID out of a URL. */
function cidFromUrl({ protocol, hostname, pathname }: URL) {
switch (protocol) {
case 'ipfs:':
return hostname;
case 'http:':
case 'https:':
return cidFromPath(pathname) || cidFromHost(hostname);
}
}
export { cidFromUrl };

View File

@ -15,13 +15,21 @@ interface ParseAuthRequestOpts {
}
/** Parse the auth event from a Request, returning a zod SafeParse type. */
function parseAuthRequest(req: Request, opts: ParseAuthRequestOpts = {}) {
const { maxAge = Time.minutes(1), validatePayload = true } = opts;
// deno-lint-ignore require-await
async function parseAuthRequest(req: Request, opts: ParseAuthRequestOpts = {}) {
const header = req.headers.get('authorization');
const base64 = header?.match(/^Nostr (.+)$/)?.[1];
const result = decode64EventSchema.safeParse(base64);
const schema = decode64EventSchema
if (!result.success) return result;
return validateAuthEvent(req, result.data, opts);
}
/** Compare the auth event with the request, returning a zod SafeParse type. */
function validateAuthEvent(req: Request, event: Event, opts: ParseAuthRequestOpts = {}) {
const { maxAge = Time.minutes(1), validatePayload = true } = opts;
const schema = signedEventSchema
.refine((event): event is Event<27235> => event.kind === 27235, 'Event must be kind 27235')
.refine((event) => eventAge(event) < maxAge, 'Event expired')
.refine((event) => tagValue(event, 'method') === req.method, 'Event method does not match HTTP request method')
@ -35,22 +43,28 @@ function parseAuthRequest(req: Request, opts: ParseAuthRequestOpts = {}) {
.then((hash) => hash === tagValue(event, 'payload'));
}
return schema.safeParseAsync(base64);
return schema.safeParseAsync(event);
}
/** Create an auth EventTemplate from a Request. */
async function buildAuthEventTemplate(req: Request): Promise<EventTemplate<27235>> {
async function buildAuthEventTemplate(req: Request, opts: ParseAuthRequestOpts = {}): Promise<EventTemplate<27235>> {
const { validatePayload = true } = opts;
const { method, url } = req;
const tags = [
['method', method],
['u', url],
];
if (validatePayload) {
const payload = await req.clone().text().then(sha256);
tags.push(['payload', payload]);
}
return {
kind: 27235,
content: '',
tags: [
['method', method],
['u', url],
['payload', payload],
],
tags,
created_at: nostrNow(),
};
}
@ -60,4 +74,4 @@ function tagValue(event: Event, tagName: string): string | undefined {
return findTag(event.tags, tagName)?.[1];
}
export { buildAuthEventTemplate, parseAuthRequest, type ParseAuthRequestOpts };
export { buildAuthEventTemplate, parseAuthRequest, type ParseAuthRequestOpts, validateAuthEvent };

34
src/views/attachment.ts Normal file
View File

@ -0,0 +1,34 @@
import { UnattachedMedia } from '@/db/unattached-media.ts';
import { type TypeFest } from '@/deps.ts';
type DittoAttachment = TypeFest.SetOptional<UnattachedMedia, 'id' | 'pubkey' | 'uploaded_at'>;
function renderAttachment(media: DittoAttachment) {
const { id, data, url } = media;
return {
id: id ?? url ?? data.cid,
type: getAttachmentType(data.mime ?? ''),
url,
preview_url: url,
remote_url: null,
description: data.description ?? '',
blurhash: data.blurhash || null,
cid: data.cid,
};
}
/** MIME to Mastodon API `Attachment` type. */
function getAttachmentType(mime: string): string {
const [type] = mime.split('/');
switch (type) {
case 'image':
case 'video':
case 'audio':
return type;
default:
return 'unknown';
}
}
export { type DittoAttachment, renderAttachment };