Merge branch 'firehose' into 'develop'
Firehose See merge request soapbox-pub/ditto!9
This commit is contained in:
commit
4580b921c4
|
@ -6,21 +6,22 @@
|
||||||
"dev": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --watch src/server.ts",
|
"dev": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --watch src/server.ts",
|
||||||
"debug": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --inspect src/server.ts",
|
"debug": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --inspect src/server.ts",
|
||||||
"test": "DB_PATH=\":memory:\" deno test --allow-read --allow-write=data --allow-env --unstable src",
|
"test": "DB_PATH=\":memory:\" deno test --allow-read --allow-write=data --allow-env --unstable src",
|
||||||
"check": "deno check --unstable src/server.ts"
|
"check": "deno check --unstable src/server.ts",
|
||||||
|
"relays:sync": "deno run -A --unstable scripts/relays.ts sync",
|
||||||
},
|
},
|
||||||
"imports": {
|
"imports": {
|
||||||
"@/": "./src/",
|
"@/": "./src/",
|
||||||
"~/": "./"
|
"~/": "./"
|
||||||
},
|
},
|
||||||
"lint": {
|
"lint": {
|
||||||
"include": ["src/"],
|
"include": ["src/", "scripts/"],
|
||||||
"rules": {
|
"rules": {
|
||||||
"tags": ["recommended"],
|
"tags": ["recommended"],
|
||||||
"exclude": ["no-explicit-any"]
|
"exclude": ["no-explicit-any"]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"fmt": {
|
"fmt": {
|
||||||
"include": ["src/"],
|
"include": ["src/", "scripts/"],
|
||||||
"useTabs": false,
|
"useTabs": false,
|
||||||
"lineWidth": 120,
|
"lineWidth": 120,
|
||||||
"indentWidth": 2,
|
"indentWidth": 2,
|
||||||
|
|
|
@ -0,0 +1,23 @@
|
||||||
|
import { addRelays } from '@/db/relays.ts';
|
||||||
|
import { filteredArray } from '@/schema.ts';
|
||||||
|
import { relaySchema } from '~/src/utils.ts';
|
||||||
|
|
||||||
|
switch (Deno.args[0]) {
|
||||||
|
case 'sync':
|
||||||
|
await sync(Deno.args.slice(1));
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
console.log('Usage: deno run -A scripts/relays.ts sync <url>');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sync([url]: string[]) {
|
||||||
|
if (!url) {
|
||||||
|
console.error('Error: please provide a URL');
|
||||||
|
Deno.exit(1);
|
||||||
|
}
|
||||||
|
const response = await fetch(url);
|
||||||
|
const data = await response.json();
|
||||||
|
const values = filteredArray(relaySchema).parse(data) as `wss://${string}`[];
|
||||||
|
await addRelays(values);
|
||||||
|
console.log(`Done: added ${values.length} relays.`);
|
||||||
|
}
|
|
@ -1,6 +1,6 @@
|
||||||
import { type Context, cors, type Handler, Hono, type HonoEnv, logger, type MiddlewareHandler } from '@/deps.ts';
|
import { type Context, cors, type Handler, Hono, type HonoEnv, logger, type MiddlewareHandler } from '@/deps.ts';
|
||||||
import { type Event } from '@/event.ts';
|
import { type Event } from '@/event.ts';
|
||||||
import '@/loopback.ts';
|
import '@/firehose.ts';
|
||||||
|
|
||||||
import { actorController } from './controllers/activitypub/actor.ts';
|
import { actorController } from './controllers/activitypub/actor.ts';
|
||||||
import {
|
import {
|
||||||
|
|
|
@ -35,12 +35,9 @@ const Conf = {
|
||||||
['sign', 'verify'],
|
['sign', 'verify'],
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
get relay() {
|
get relay(): `wss://${string}` | `ws://${string}` {
|
||||||
const value = Deno.env.get('DITTO_RELAY');
|
const { protocol, host } = Conf.url;
|
||||||
if (!value) {
|
return `${protocol === 'https:' ? 'wss:' : 'ws:'}//${host}/relay`;
|
||||||
throw new Error('Missing DITTO_RELAY');
|
|
||||||
}
|
|
||||||
return value;
|
|
||||||
},
|
},
|
||||||
get localDomain() {
|
get localDomain() {
|
||||||
return Deno.env.get('LOCAL_DOMAIN') || 'http://localhost:8000';
|
return Deno.env.get('LOCAL_DOMAIN') || 'http://localhost:8000';
|
||||||
|
|
|
@ -8,6 +8,7 @@ interface DittoDB {
|
||||||
events: EventRow;
|
events: EventRow;
|
||||||
tags: TagRow;
|
tags: TagRow;
|
||||||
users: UserRow;
|
users: UserRow;
|
||||||
|
relays: RelayRow;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface EventRow {
|
interface EventRow {
|
||||||
|
@ -34,6 +35,12 @@ interface UserRow {
|
||||||
inserted_at: Date;
|
inserted_at: Date;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface RelayRow {
|
||||||
|
url: string;
|
||||||
|
domain: string;
|
||||||
|
active: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
const db = new Kysely<DittoDB>({
|
const db = new Kysely<DittoDB>({
|
||||||
dialect: new DenoSqliteDialect({
|
dialect: new DenoSqliteDialect({
|
||||||
database: new Sqlite(Conf.dbPath),
|
database: new Sqlite(Conf.dbPath),
|
||||||
|
|
|
@ -0,0 +1,14 @@
|
||||||
|
import { Kysely } from '@/deps.ts';
|
||||||
|
|
||||||
|
export async function up(db: Kysely<any>): Promise<void> {
|
||||||
|
await db.schema
|
||||||
|
.createTable('relays')
|
||||||
|
.addColumn('url', 'text', (col) => col.primaryKey())
|
||||||
|
.addColumn('domain', 'text', (col) => col.notNull())
|
||||||
|
.addColumn('active', 'boolean', (col) => col.notNull())
|
||||||
|
.execute();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function down(db: Kysely<any>): Promise<void> {
|
||||||
|
await db.schema.dropTable('relays').execute();
|
||||||
|
}
|
|
@ -0,0 +1,31 @@
|
||||||
|
import { tldts } from '@/deps.ts';
|
||||||
|
import { db } from '@/db.ts';
|
||||||
|
|
||||||
|
/** Inserts relays into the database, skipping duplicates. */
|
||||||
|
function addRelays(relays: `wss://${string}`[]) {
|
||||||
|
if (!relays.length) return Promise.resolve();
|
||||||
|
|
||||||
|
const values = relays.map((url) => ({
|
||||||
|
url,
|
||||||
|
domain: tldts.getDomain(url)!,
|
||||||
|
active: true,
|
||||||
|
}));
|
||||||
|
|
||||||
|
return db.insertInto('relays')
|
||||||
|
.values(values)
|
||||||
|
.onConflict((oc) => oc.column('url').doNothing())
|
||||||
|
.execute();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Get a list of all known active relay URLs. */
|
||||||
|
async function getActiveRelays(): Promise<string[]> {
|
||||||
|
const rows = await db
|
||||||
|
.selectFrom('relays')
|
||||||
|
.select('relays.url')
|
||||||
|
.where('relays.active', '=', true)
|
||||||
|
.execute();
|
||||||
|
|
||||||
|
return rows.map((row) => row.url);
|
||||||
|
}
|
||||||
|
|
||||||
|
export { addRelays, getActiveRelays };
|
|
@ -59,4 +59,5 @@ export {
|
||||||
type NullableInsertKeys,
|
type NullableInsertKeys,
|
||||||
sql,
|
sql,
|
||||||
} from 'npm:kysely@^0.25.0';
|
} from 'npm:kysely@^0.25.0';
|
||||||
export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/76748303a45fac64a889cd2b9265c6c9b8ef2e8b/mod.ts';
|
export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/v1.0.0/mod.ts';
|
||||||
|
export { default as tldts } from 'npm:tldts@^6.0.14';
|
||||||
|
|
|
@ -0,0 +1,69 @@
|
||||||
|
import { insertEvent, isLocallyFollowed } from '@/db/events.ts';
|
||||||
|
import { addRelays, getActiveRelays } from '@/db/relays.ts';
|
||||||
|
import { findUser } from '@/db/users.ts';
|
||||||
|
import { RelayPool } from '@/deps.ts';
|
||||||
|
import { trends } from '@/trends.ts';
|
||||||
|
import { isRelay, nostrDate, nostrNow } from '@/utils.ts';
|
||||||
|
|
||||||
|
import type { SignedEvent } from '@/event.ts';
|
||||||
|
|
||||||
|
const relays = await getActiveRelays();
|
||||||
|
const pool = new RelayPool(relays);
|
||||||
|
|
||||||
|
// This file watches events on all known relays and performs
|
||||||
|
// side-effects based on them, such as trending hashtag tracking
|
||||||
|
// and storing events for notifications and the home feed.
|
||||||
|
pool.subscribe(
|
||||||
|
[{ kinds: [0, 1, 3, 5, 6, 7, 10002], since: nostrNow() }],
|
||||||
|
relays,
|
||||||
|
handleEvent,
|
||||||
|
undefined,
|
||||||
|
undefined,
|
||||||
|
);
|
||||||
|
|
||||||
|
/** Handle events through the firehose pipeline. */
|
||||||
|
async function handleEvent(event: SignedEvent): Promise<void> {
|
||||||
|
console.info(`firehose: Event<${event.kind}> ${event.id}`);
|
||||||
|
|
||||||
|
trackHashtags(event);
|
||||||
|
trackRelays(event);
|
||||||
|
|
||||||
|
if (await findUser({ pubkey: event.pubkey }) || await isLocallyFollowed(event.pubkey)) {
|
||||||
|
insertEvent(event).catch(console.warn);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Track whenever a hashtag is used, for processing trending tags. */
|
||||||
|
function trackHashtags(event: SignedEvent): void {
|
||||||
|
const date = nostrDate(event.created_at);
|
||||||
|
|
||||||
|
const tags = event.tags
|
||||||
|
.filter((tag) => tag[0] === 't')
|
||||||
|
.map((tag) => tag[1])
|
||||||
|
.slice(0, 5);
|
||||||
|
|
||||||
|
if (!tags.length) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
console.info('tracking tags:', tags);
|
||||||
|
trends.addTagUsages(event.pubkey, tags, date);
|
||||||
|
} catch (_e) {
|
||||||
|
// do nothing
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Tracks known relays in the database. */
|
||||||
|
function trackRelays(event: SignedEvent) {
|
||||||
|
const relays = new Set<`wss://${string}`>();
|
||||||
|
|
||||||
|
event.tags.forEach((tag) => {
|
||||||
|
if (['p', 'e', 'a'].includes(tag[0]) && isRelay(tag[2])) {
|
||||||
|
relays.add(tag[2]);
|
||||||
|
}
|
||||||
|
if (event.kind === 10002 && tag[0] === 'r' && isRelay(tag[1])) {
|
||||||
|
relays.add(tag[1]);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return addRelays([...relays]);
|
||||||
|
}
|
|
@ -1,51 +0,0 @@
|
||||||
import { Conf } from '@/config.ts';
|
|
||||||
import { insertEvent, isLocallyFollowed } from '@/db/events.ts';
|
|
||||||
import { findUser } from '@/db/users.ts';
|
|
||||||
import { RelayPool } from '@/deps.ts';
|
|
||||||
import { trends } from '@/trends.ts';
|
|
||||||
import { nostrDate, nostrNow } from '@/utils.ts';
|
|
||||||
|
|
||||||
import type { SignedEvent } from '@/event.ts';
|
|
||||||
|
|
||||||
const relay = new RelayPool([Conf.relay]);
|
|
||||||
|
|
||||||
// This file watches all events on your Ditto relay and triggers
|
|
||||||
// side-effects based on them. This can be used for things like
|
|
||||||
// notifications, trending hashtag tracking, etc.
|
|
||||||
relay.subscribe(
|
|
||||||
[{ kinds: [1], since: nostrNow() }],
|
|
||||||
[Conf.relay],
|
|
||||||
handleEvent,
|
|
||||||
undefined,
|
|
||||||
undefined,
|
|
||||||
);
|
|
||||||
|
|
||||||
/** Handle events through the loopback pipeline. */
|
|
||||||
async function handleEvent(event: SignedEvent): Promise<void> {
|
|
||||||
console.info('loopback event:', event.id);
|
|
||||||
|
|
||||||
trackHashtags(event);
|
|
||||||
|
|
||||||
if (await findUser({ pubkey: event.pubkey }) || await isLocallyFollowed(event.pubkey)) {
|
|
||||||
insertEvent(event).catch(console.warn);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Track whenever a hashtag is used, for processing trending tags. */
|
|
||||||
function trackHashtags(event: SignedEvent): void {
|
|
||||||
const date = nostrDate(event.created_at);
|
|
||||||
|
|
||||||
const tags = event.tags
|
|
||||||
.filter((tag) => tag[0] === 't')
|
|
||||||
.map((tag) => tag[1])
|
|
||||||
.slice(0, 5);
|
|
||||||
|
|
||||||
if (!tags.length) return;
|
|
||||||
|
|
||||||
try {
|
|
||||||
console.info('tracking tags:', tags);
|
|
||||||
trends.addTagUsages(event.pubkey, tags, date);
|
|
||||||
} catch (_e) {
|
|
||||||
// do nothing
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -20,24 +20,6 @@ const jsonSchema = z.string().transform((value, ctx) => {
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
/** Alias for `safeParse`, but instead of returning a success object it returns the value (or undefined on fail). */
|
|
||||||
function parseValue<T>(schema: z.ZodType<T>, value: unknown): T | undefined {
|
|
||||||
const result = schema.safeParse(value);
|
|
||||||
return result.success ? result.data : undefined;
|
|
||||||
}
|
|
||||||
|
|
||||||
const parseRelay = (relay: string | URL) => parseValue(relaySchema, relay);
|
|
||||||
|
|
||||||
const relaySchema = z.custom<URL>((relay) => {
|
|
||||||
if (typeof relay !== 'string') return false;
|
|
||||||
try {
|
|
||||||
const { protocol } = new URL(relay);
|
|
||||||
return protocol === 'wss:' || protocol === 'ws:';
|
|
||||||
} catch (_e) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
const emojiTagSchema = z.tuple([z.literal('emoji'), z.string(), z.string().url()]);
|
const emojiTagSchema = z.tuple([z.literal('emoji'), z.string(), z.string().url()]);
|
||||||
|
|
||||||
/** https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem */
|
/** https://developer.mozilla.org/en-US/docs/Glossary/Base64#the_unicode_problem */
|
||||||
|
@ -54,4 +36,10 @@ const decode64Schema = z.string().transform((value, ctx) => {
|
||||||
|
|
||||||
const hashtagSchema = z.string().regex(/^\w{1,30}$/);
|
const hashtagSchema = z.string().regex(/^\w{1,30}$/);
|
||||||
|
|
||||||
export { decode64Schema, emojiTagSchema, filteredArray, hashtagSchema, jsonSchema, parseRelay, relaySchema };
|
/**
|
||||||
|
* Limits the length before trying to parse the URL.
|
||||||
|
* https://stackoverflow.com/a/417184/8811886
|
||||||
|
*/
|
||||||
|
const safeUrlSchema = z.string().max(2048).url();
|
||||||
|
|
||||||
|
export { decode64Schema, emojiTagSchema, filteredArray, hashtagSchema, jsonSchema, safeUrlSchema };
|
||||||
|
|
|
@ -1,14 +1,16 @@
|
||||||
import { verifySignature, z } from '@/deps.ts';
|
import { verifySignature, z } from '@/deps.ts';
|
||||||
|
|
||||||
import { jsonSchema } from '../schema.ts';
|
import { jsonSchema, safeUrlSchema } from '../schema.ts';
|
||||||
|
|
||||||
/** Schema to validate Nostr hex IDs such as event IDs and pubkeys. */
|
/** Schema to validate Nostr hex IDs such as event IDs and pubkeys. */
|
||||||
const nostrIdSchema = z.string().regex(/^[0-9a-f]{64}$/);
|
const nostrIdSchema = z.string().regex(/^[0-9a-f]{64}$/);
|
||||||
|
/** Nostr kinds are positive integers. */
|
||||||
|
const kindSchema = z.number().int().positive();
|
||||||
|
|
||||||
/** Nostr event schema. */
|
/** Nostr event schema. */
|
||||||
const eventSchema = z.object({
|
const eventSchema = z.object({
|
||||||
id: nostrIdSchema,
|
id: nostrIdSchema,
|
||||||
kind: z.number(),
|
kind: kindSchema,
|
||||||
tags: z.array(z.array(z.string())),
|
tags: z.array(z.array(z.string())),
|
||||||
content: z.string(),
|
content: z.string(),
|
||||||
created_at: z.number(),
|
created_at: z.number(),
|
||||||
|
@ -21,7 +23,7 @@ const signedEventSchema = eventSchema.refine(verifySignature);
|
||||||
|
|
||||||
/** Nostr relay filter schema. */
|
/** Nostr relay filter schema. */
|
||||||
const filterSchema = z.object({
|
const filterSchema = z.object({
|
||||||
kinds: z.number().int().positive().array().optional(),
|
kinds: kindSchema.array().optional(),
|
||||||
ids: nostrIdSchema.array().optional(),
|
ids: nostrIdSchema.array().optional(),
|
||||||
authors: nostrIdSchema.array().optional(),
|
authors: nostrIdSchema.array().optional(),
|
||||||
since: z.number().int().positive().optional(),
|
since: z.number().int().positive().optional(),
|
||||||
|
@ -67,6 +69,17 @@ const metaContentSchema = z.object({
|
||||||
/** Parses kind 0 content from a JSON string. */
|
/** Parses kind 0 content from a JSON string. */
|
||||||
const jsonMetaContentSchema = jsonSchema.pipe(metaContentSchema).catch({});
|
const jsonMetaContentSchema = jsonSchema.pipe(metaContentSchema).catch({});
|
||||||
|
|
||||||
|
/** NIP-11 Relay Information Document. */
|
||||||
|
const relayInfoDocSchema = z.object({
|
||||||
|
name: z.string().transform((val) => val.slice(0, 30)).optional().catch(undefined),
|
||||||
|
description: z.string().transform((val) => val.slice(0, 3000)).optional().catch(undefined),
|
||||||
|
pubkey: nostrIdSchema.optional().catch(undefined),
|
||||||
|
contact: safeUrlSchema.optional().catch(undefined),
|
||||||
|
supported_nips: z.number().int().positive().array().optional().catch(undefined),
|
||||||
|
software: safeUrlSchema.optional().catch(undefined),
|
||||||
|
icon: safeUrlSchema.optional().catch(undefined),
|
||||||
|
});
|
||||||
|
|
||||||
export {
|
export {
|
||||||
type ClientCLOSE,
|
type ClientCLOSE,
|
||||||
type ClientEVENT,
|
type ClientEVENT,
|
||||||
|
@ -77,5 +90,6 @@ export {
|
||||||
jsonMetaContentSchema,
|
jsonMetaContentSchema,
|
||||||
metaContentSchema,
|
metaContentSchema,
|
||||||
nostrIdSchema,
|
nostrIdSchema,
|
||||||
|
relayInfoDocSchema,
|
||||||
signedEventSchema,
|
signedEventSchema,
|
||||||
};
|
};
|
||||||
|
|
|
@ -142,6 +142,12 @@ function activityJson<T, P extends string>(c: Context<any, P>, object: T) {
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Schema to parse a relay URL. */
|
||||||
|
const relaySchema = z.string().max(255).startsWith('wss://').url();
|
||||||
|
|
||||||
|
/** Check whether the value is a valid relay URL. */
|
||||||
|
const isRelay = (relay: string): relay is `wss://${string}` => relaySchema.safeParse(relay).success;
|
||||||
|
|
||||||
export {
|
export {
|
||||||
activityJson,
|
activityJson,
|
||||||
bech32ToPubkey,
|
bech32ToPubkey,
|
||||||
|
@ -149,6 +155,7 @@ export {
|
||||||
eventAge,
|
eventAge,
|
||||||
eventDateComparator,
|
eventDateComparator,
|
||||||
findTag,
|
findTag,
|
||||||
|
isRelay,
|
||||||
lookupAccount,
|
lookupAccount,
|
||||||
type Nip05,
|
type Nip05,
|
||||||
nostrDate,
|
nostrDate,
|
||||||
|
@ -157,6 +164,7 @@ export {
|
||||||
paginationSchema,
|
paginationSchema,
|
||||||
parseBody,
|
parseBody,
|
||||||
parseNip05,
|
parseNip05,
|
||||||
|
relaySchema,
|
||||||
sha256,
|
sha256,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue