moon
/
ditto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'queries' into 'develop' 

Add Queries module

See merge request soapbox-pub/ditto!13
This commit is contained in:
Alex Gleason 2023-08-17 20:44:12 +00:00
parent b1c5dd0e34 299a1a0db0
commit 3890df1a2d
12 changed files with 118 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.vscode/settings.json vendored
View File

@ -1,7 +1,6 @@
{
"deno.enable": true,
"deno.lint": true,
"deno.unstable": true,
"editor.defaultFormatter": "denoland.vscode-deno",
"path-intellisense.extensionOnImport": true
}

12
deno.json
View File

@ -2,12 +2,12 @@
"$schema": "https://deno.land/x/deno@v1.32.3/cli/schemas/config-file.v1.json",
"lock": false,
"tasks": {
"start": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable src/server.ts",
"dev": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --watch src/server.ts",
"debug": "deno run --allow-read --allow-write=data --allow-env --allow-net --unstable --inspect src/server.ts",
"test": "DB_PATH=\":memory:\" deno test --allow-read --allow-write=data --allow-env --unstable src",
"check": "deno check --unstable src/server.ts",
"relays:sync": "deno run -A --unstable scripts/relays.ts sync",
"start": "deno run --allow-read --allow-write=data --allow-env --allow-net src/server.ts",
"dev": "deno run --allow-read --allow-write=data --allow-env --allow-net --watch src/server.ts",
"debug": "deno run --allow-read --allow-write=data --allow-env --allow-net --inspect src/server.ts",
"test": "DB_PATH=\":memory:\" deno test --allow-read --allow-write=data --allow-env src",
"check": "deno check src/server.ts",
"relays:sync": "deno run -A scripts/relays.ts sync"
},
"imports": {
"@/": "./src/",

90
src/client.ts
View File

@ -1,11 +1,9 @@
import { Conf } from '@/config.ts';
import { Author, type Event, type Filter, findReplyTag, matchFilters, RelayPool, TTLCache } from '@/deps.ts';
import { eventDateComparator, type PaginationParams, Time } from '@/utils.ts';
import { type Event, type Filter, matchFilters, RelayPool, TTLCache } from '@/deps.ts';
import { Time } from '@/utils.ts';
import type { GetFiltersOpts } from '@/types.ts';
const db = await Deno.openKv();
type Pool = InstanceType<typeof RelayPool>;
/** HACK: Websockets in Deno are finnicky... get a new pool every 30 minutes. */
@ -72,88 +70,6 @@ function getFilters<K extends number>(filters: Filter<K>[], opts: GetFiltersOpts
});
}
/** Get a Nostr event by its ID. */
const getEvent = async <K extends number = number>(id: string, kind?: K): Promise<Event<K> | undefined> => {
const event = await (getPool().getEventById(id, Conf.poolRelays, 0) as Promise<Event>);
if (event) {
if (event.id !== id) return undefined;
if (kind && event.kind !== kind) return undefined;
return event as Event<K>;
}
};
/** Get a Nostr `set_medatadata` event for a user's pubkey. */
const getAuthor = async (pubkey: string, timeout = 1000): Promise<Event<0> | undefined> => {
const author = new Author(getPool(), Conf.poolRelays, pubkey);
const event: Event<0> | null = await new Promise((resolve) => {
setTimeout(resolve, timeout, null);
return author.metaData(resolve, 0);
});
return event?.pubkey === pubkey ? event : undefined;
};
/** Get users the given pubkey follows. */
const getFollows = async (pubkey: string): Promise<Event<3> | undefined> => {
const [event] = await getFilters([{ authors: [pubkey], kinds: [3] }], { timeout: 5000 });
// TODO: figure out a better, more generic & flexible way to handle event cache (and timeouts?)
// Prewarm cache in GET `/api/v1/accounts/verify_credentials`
if (event) {
await db.set(['event3', pubkey], event);
return event;
} else {
return (await db.get<Event<3>>(['event3', pubkey])).value || undefined;
}
};
/** Get events from people the user follows. */
async function getFeed(event3: Event<3>, params: PaginationParams): Promise<Event<1>[]> {
const authors = event3.tags
.filter((tag) => tag[0] === 'p')
.map((tag) => tag[1]);
authors.push(event3.pubkey); // see own events in feed
const filter: Filter = {
authors,
kinds: [1],
...params,
};
const results = await getFilters([filter], { timeout: 5000 }) as Event<1>[];
return results.sort(eventDateComparator);
}
/** Get a feed of all known text notes. */
async function getPublicFeed(params: PaginationParams): Promise<Event<1>[]> {
const results = await getFilters([{ kinds: [1], ...params }], { timeout: 5000 });
return results.sort(eventDateComparator);
}
async function getAncestors(event: Event<1>, result = [] as Event<1>[]): Promise<Event<1>[]> {
if (result.length < 100) {
const replyTag = findReplyTag(event);
const inReplyTo = replyTag ? replyTag[1] : undefined;
if (inReplyTo) {
const parentEvent = await getEvent(inReplyTo, 1);
if (parentEvent) {
result.push(parentEvent);
return getAncestors(parentEvent, result);
}
}
}
return result.reverse();
}
function getDescendants(eventId: string): Promise<Event<1>[]> {
return getFilters([{ kinds: [1], '#e': [eventId] }], { limit: 200, timeout: 2000 }) as Promise<Event<1>[]>;
}
/** Publish an event to the Nostr relay. */
function publish(event: Event, relays = Conf.publishRelays): void {
console.log('Publishing event', event, relays);
@ -164,4 +80,4 @@ function publish(event: Event, relays = Conf.publishRelays): void {
}
}
export { getAncestors, getAuthor, getDescendants, getEvent, getFeed, getFilters, getFollows, getPublicFeed, publish };
export { getFilters, publish };

2
src/controllers/activitypub/actor.ts
View File

@ -1,5 +1,5 @@
import { getAuthor } from '@/client.ts';
import { findUser } from '@/db/users.ts';
import { getAuthor } from '@/queries.ts';
import { toActor } from '@/transformers/nostr-to-activitypub.ts';
import { activityJson } from '@/utils.ts';

7
src/controllers/api/accounts.ts
View File

@ -1,7 +1,8 @@
import { type AppController } from '@/app.ts';
import { type Filter, findReplyTag, z } from '@/deps.ts';
import { getAuthor, getFollows, publish } from '@/client.ts';
import { getFilters } from '@/mixer.ts';
import { publish } from '@/client.ts';
import * as mixer from '@/mixer.ts';
import { getAuthor, getFollows } from '@/queries.ts';
import { jsonMetaContentSchema } from '@/schemas/nostr.ts';
import { signEvent } from '@/sign.ts';
import { toAccount, toStatus } from '@/transformers/nostr-to-mastoapi.ts';
@ -116,7 +117,7 @@ const accountStatusesController: AppController = async (c) => {
filter['#t'] = [tagged];
}
let events = await getFilters([filter]);
let events = await mixer.getFilters([filter]);
events.sort(eventDateComparator);
if (exclude_replies) {

13
src/controllers/api/statuses.ts
View File

@ -1,6 +1,7 @@
import { type AppController } from '@/app.ts';
import { getAncestors, getDescendants, getEvent, publish } from '@/client.ts';
import { type Event, ISO6391, Kind, z } from '@/deps.ts';
import { publish } from '@/client.ts';
import { ISO6391, Kind, z } from '@/deps.ts';
import { getAncestors, getDescendants, getEvent } from '@/queries.ts';
import { signEvent } from '@/sign.ts';
import { toStatus } from '@/transformers/nostr-to-mastoapi.ts';
import { nostrNow, parseBody } from '@/utils.ts';
@ -28,9 +29,9 @@ const createStatusSchema = z.object({
const statusController: AppController = async (c) => {
const id = c.req.param('id');
const event = await getEvent(id, 1);
const event = await getEvent(id, { kind: 1 });
if (event) {
return c.json(await toStatus(event as Event<1>));
return c.json(await toStatus(event));
}
return c.json({ error: 'Event not found.' }, 404);
@ -87,7 +88,7 @@ const createStatusController: AppController = async (c) => {
const contextController: AppController = async (c) => {
const id = c.req.param('id');
const event = await getEvent(id, 1);
const event = await getEvent(id, { kind: 1 });
if (event) {
const ancestorEvents = await getAncestors(event);
@ -104,7 +105,7 @@ const contextController: AppController = async (c) => {
const favouriteController: AppController = async (c) => {
const id = c.req.param('id');
const target = await getEvent(id, 1);
const target = await getEvent(id, { kind: 1 });
if (target) {
const event = await signEvent({

9
src/controllers/api/timelines.ts
View File

@ -1,4 +1,4 @@
import { getFeed, getFollows, getPublicFeed } from '@/client.ts';
import { getFeed, getPublicFeed } from '@/queries.ts';
import { toStatus } from '@/transformers/nostr-to-mastoapi.ts';
import { buildLinkHeader, paginationSchema } from '@/utils.ts';
@ -8,12 +8,7 @@ const homeController: AppController = async (c) => {
const params = paginationSchema.parse(c.req.query());
const pubkey = c.get('pubkey')!;
const follows = await getFollows(pubkey);
if (!follows) {
return c.json([]);
}
const events = await getFeed(follows, params);
const events = await getFeed(pubkey, params);
if (!events.length) {
return c.json([]);
}

6
src/controllers/nostr/relay.ts
View File

@ -1,4 +1,4 @@
import { getFilters, insertEvent } from '@/db/events.ts';
import * as eventsDB from '@/db/events.ts';
import { findUser } from '@/db/users.ts';
import { jsonSchema } from '@/schema.ts';
import {
@ -46,7 +46,7 @@ function connectStream(socket: WebSocket) {
}
async function handleReq([_, sub, ...filters]: ClientREQ) {
for (const event of await getFilters(prepareFilters(filters))) {
for (const event of await eventsDB.getFilters(prepareFilters(filters))) {
send(['EVENT', sub, event]);
}
send(['EOSE', sub]);
@ -54,7 +54,7 @@ function connectStream(socket: WebSocket) {
async function handleEvent([_, event]: ClientEVENT) {
if (await findUser({ pubkey: event.pubkey })) {
insertEvent(event);
eventsDB.insertEvent(event);
send(['OK', event.id, true, '']);
} else {
send(['OK', event.id, false, 'blocked: only registered users can post']);

8
src/mixer.ts
View File

@ -1,7 +1,7 @@
import { type Event, matchFilters } from '@/deps.ts';
import { getFilters as getFiltersClient } from '@/client.ts';
import { getFilters as getFiltersDB } from '@/db/events.ts';
import * as client from '@/client.ts';
import * as eventsDB from '@/db/events.ts';
import { eventDateComparator } from '@/utils.ts';
import type { DittoFilter, GetFiltersOpts } from '@/types.ts';
@ -12,8 +12,8 @@ async function getFilters<K extends number>(
opts?: GetFiltersOpts,
): Promise<Event<K>[]> {
const results = await Promise.allSettled([
getFiltersClient(filters, opts),
getFiltersDB(filters, opts),
client.getFilters(filters, opts),
eventsDB.getFilters(filters, opts),
]);
const events = results

86
src/queries.ts Normal file
View File

@ -0,0 +1,86 @@
import { type Event, type Filter, findReplyTag } from '@/deps.ts';
import { type PaginationParams } from '@/utils.ts';
import * as mixer from './mixer.ts';
interface GetEventOpts<K extends number> {
/** Timeout in milliseconds. */
timeout?: number;
/** Event kind. */
kind?: K;
}
/** Get a Nostr event by its ID. */
const getEvent = async <K extends number = number>(
id: string,
opts: GetEventOpts<K> = {},
): Promise<Event<K> | undefined> => {
const { kind, timeout = 1000 } = opts;
const filter: Filter<K> = { ids: [id], limit: 1 };
if (kind) {
filter.kinds = [kind];
}
const [event] = await mixer.getFilters([filter], { limit: 1, timeout });
return event;
};
/** Get a Nostr `set_medatadata` event for a user's pubkey. */
const getAuthor = async (pubkey: string, timeout = 1000): Promise<Event<0> | undefined> => {
const [event] = await mixer.getFilters([{ authors: [pubkey], kinds: [0] }], { timeout });
return event;
};
/** Get users the given pubkey follows. */
const getFollows = async (pubkey: string, timeout = 1000): Promise<Event<3> | undefined> => {
const [event] = await mixer.getFilters([{ authors: [pubkey], kinds: [3] }], { timeout });
return event;
};
/** Get events from people the user follows. */
async function getFeed(pubkey: string, params: PaginationParams): Promise<Event<1>[]> {
const event3 = await getFollows(pubkey);
if (!event3) return [];
const authors = event3.tags
.filter((tag) => tag[0] === 'p')
.map((tag) => tag[1]);
authors.push(event3.pubkey); // see own events in feed
const filter: Filter<1> = {
authors,
kinds: [1],
...params,
};
return mixer.getFilters([filter], { timeout: 5000 });
}
/** Get a feed of all known text notes. */
function getPublicFeed(params: PaginationParams): Promise<Event<1>[]> {
return mixer.getFilters([{ kinds: [1], ...params }], { timeout: 5000 });
}
async function getAncestors(event: Event<1>, result = [] as Event<1>[]): Promise<Event<1>[]> {
if (result.length < 100) {
const replyTag = findReplyTag(event);
const inReplyTo = replyTag ? replyTag[1] : undefined;
if (inReplyTo) {
const parentEvent = await getEvent(inReplyTo, { kind: 1 });
if (parentEvent) {
result.push(parentEvent);
return getAncestors(parentEvent, result);
}
}
}
return result.reverse();
}
function getDescendants(eventId: string): Promise<Event<1>[]> {
return mixer.getFilters([{ kinds: [1], '#e': [eventId] }], { limit: 200, timeout: 2000 });
}
export { getAncestors, getAuthor, getDescendants, getEvent, getFeed, getFollows, getPublicFeed };

2
src/transformers/nostr-to-mastoapi.ts
View File

@ -1,10 +1,10 @@
import { isCWTag } from 'https://gitlab.com/soapbox-pub/mostr/-/raw/c67064aee5ade5e01597c6d23e22e53c628ef0e2/src/nostr/tags.ts';
import { getAuthor } from '@/client.ts';
import { Conf } from '@/config.ts';
import { type Event, findReplyTag, lodash, nip19, sanitizeHtml, TTLCache, unfurl, z } from '@/deps.ts';
import { verifyNip05Cached } from '@/nip05.ts';
import { getMediaLinks, type MediaLink, parseNoteContent } from '@/note.ts';
import { getAuthor } from '@/queries.ts';
import { emojiTagSchema, filteredArray } from '@/schema.ts';
import { jsonMetaContentSchema } from '@/schemas/nostr.ts';
import { type Nip05, nostrDate, parseNip05, Time } from '@/utils.ts';

2
src/utils.ts
View File

@ -1,7 +1,7 @@
import { getAuthor } from '@/client.ts';
import { Conf } from '@/config.ts';
import { type Context, type Event, nip19, parseFormData, z } from '@/deps.ts';
import { lookupNip05Cached } from '@/nip05.ts';
import { getAuthor } from '@/queries.ts';
/** Get the current time in Nostr format. */
const nostrNow = () => Math.floor(new Date().getTime() / 1000);