moon
/
ditto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf: make up to 5 calls to database in hydrateEvents & remove old hydrate functions

This commit is contained in:
P. Reis 2024-04-22 19:51:29 -03:00
parent ed08ac7c17
commit b77c8a00cd
11 changed files with 71 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/controllers/api/accounts.ts
View File

@ -95,7 +95,6 @@ const accountSearchController: AppController = async (c) => {
const results = await hydrateEvents({
events: event ? [event, ...events] : events,
relations: ['author_stats'],
storage: eventsDB,
signal: c.req.raw.signal,
});
@ -164,9 +163,7 @@ const accountStatusesController: AppController = async (c) => {
}
const events = await eventsDB.query([filter], { signal })
.then((events) =>
hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal })
)
.then((events) => hydrateEvents({ events, storage: eventsDB, signal }))
.then((events) => {
if (exclude_replies) {
return events.filter((event) => !findReplyTag(event.tags));
@ -317,9 +314,7 @@ const favouritesController: AppController = async (c) => {
.filter((id): id is string => !!id);
const events1 = await eventsDB.query([{ kinds: [1], ids }], { signal })
.then((events) =>
hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal })
);
.then((events) => hydrateEvents({ events, storage: eventsDB, signal }));
const statuses = await Promise.all(events1.map((event) => renderStatus(event, { viewerPubkey: c.get('pubkey') })));
return paginated(c, events1, statuses);

8
src/controllers/api/search.ts
View File

@ -90,9 +90,7 @@ function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: Abort
}
return searchStore.query([filter], { signal })
.then((events) =>
hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: searchStore, signal })
);
.then((events) => hydrateEvents({ events, storage: searchStore, signal }));
}
/** Get event kinds to search from `type` query param. */
@ -112,9 +110,7 @@ async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise<Nos
const filters = await getLookupFilters(query, signal);
return searchStore.query(filters, { limit: 1, signal })
.then((events) =>
hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: searchStore, signal })
)
.then((events) => hydrateEvents({ events, storage: searchStore, signal }))
.then(([event]) => event);
}

3
src/controllers/api/statuses.ts
View File

@ -41,7 +41,6 @@ const statusController: AppController = async (c) => {
const event = await getEvent(id, {
kind: 1,
relations: ['author', 'event_stats', 'author_stats', 'quote_repost'],
signal: AbortSignal.timeout(1500),
});
@ -135,7 +134,6 @@ const createStatusController: AppController = async (c) => {
if (data.quote_id) {
await hydrateEvents({
events: [event],
relations: ['quote_repost'],
storage: eventsDB,
signal: c.req.raw.signal,
});
@ -241,7 +239,6 @@ const reblogStatusController: AppController = async (c) => {
await hydrateEvents({
events: [reblogEvent],
relations: ['repost', 'author'],
storage: eventsDB,
signal: signal,
});

1
src/controllers/api/streaming.ts
View File

@ -68,7 +68,6 @@ const streamingController: AppController = (c) => {
if (event.kind === 6) {
await hydrateEvents({
events: [event],
relations: ['repost', 'author'],
storage: eventsDB,
signal: AbortSignal.timeout(1000),
});

1
src/controllers/api/timelines.ts
View File

@ -51,7 +51,6 @@ async function renderStatuses(c: AppContext, filters: NostrFilter[]) {
.then((events) =>
hydrateEvents({
events,
relations: ['author', 'author_stats', 'event_stats', 'repost', 'quote_repost'],
storage: eventsDB,
signal,
})

4
src/interfaces/DittoEvent.ts
View File

@ -22,6 +22,6 @@ export interface DittoEvent extends NostrEvent {
event_stats?: EventStats;
d_author?: DittoEvent;
user?: DittoEvent;
repost?: NostrEvent;
quote_repost?: NostrEvent;
repost?: DittoEvent;
quote_repost?: DittoEvent;
}

2
src/pipeline.ts
View File

@ -56,7 +56,7 @@ async function encounterEvent(event: NostrEvent, signal: AbortSignal): Promise<b
/** Hydrate the event with the user, if applicable. */
async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise<void> {
await hydrateEvents({ events: [event], relations: ['author', 'user'], storage: eventsDB, signal });
await hydrateEvents({ events: [event], storage: eventsDB, signal });
const domain = await db
.selectFrom('pubkey_domains')

14
src/queries.ts
View File

@ -24,7 +24,7 @@ const getEvent = async (
opts: GetEventOpts = {},
): Promise<DittoEvent | undefined> => {
debug(`getEvent: ${id}`);
const { kind, relations = [], signal = AbortSignal.timeout(1000) } = opts;
const { kind, signal = AbortSignal.timeout(1000) } = opts;
const filter: NostrFilter = { ids: [id], limit: 1 };
if (kind) {
@ -32,16 +32,16 @@ const getEvent = async (
}
return await optimizer.query([filter], { limit: 1, signal })
.then((events) => hydrateEvents({ events, relations, storage: optimizer, signal }))
.then((events) => hydrateEvents({ events, storage: optimizer, signal }))
.then(([event]) => event);
};
/** Get a Nostr `set_medatadata` event for a user's pubkey. */
const getAuthor = async (pubkey: string, opts: GetEventOpts = {}): Promise<NostrEvent | undefined> => {
const { relations = [], signal = AbortSignal.timeout(1000) } = opts;
const { signal = AbortSignal.timeout(1000) } = opts;
return await optimizer.query([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, signal })
.then((events) => hydrateEvents({ events, relations, storage: optimizer, signal }))
.then((events) => hydrateEvents({ events, storage: optimizer, signal }))
.then(([event]) => event);
};
@ -70,7 +70,7 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi
const inReplyTo = replyTag ? replyTag[1] : undefined;
if (inReplyTo) {
const parentEvent = await getEvent(inReplyTo, { kind: 1, relations: ['author', 'event_stats', 'author_stats'] });
const parentEvent = await getEvent(inReplyTo, { kind: 1 });
if (parentEvent) {
result.push(parentEvent);
@ -84,9 +84,7 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi
function getDescendants(eventId: string, signal = AbortSignal.timeout(2000)): Promise<NostrEvent[]> {
return eventsDB.query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal })
.then((events) =>
hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal })
);
.then((events) => hydrateEvents({ events, storage: eventsDB, signal }));
}
/** Returns whether the pubkey is followed by a local user. */

176
src/storages/hydrate.ts
View File

@ -1,46 +1,75 @@
import { NostrEvent, NStore } from '@nostrify/nostrify';
import { Conf } from '@/config.ts';
import { db } from '@/db.ts';
import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
import { type DittoRelation } from '@/interfaces/DittoFilter.ts';
interface HydrateEventOpts {
events: DittoEvent[];
relations: DittoRelation[];
storage: NStore;
signal?: AbortSignal;
}
/** Hydrate event relationships using the provided storage. */
/** Hydrate events using the provided storage. */
async function hydrateEvents(opts: HydrateEventOpts): Promise<DittoEvent[]> {
const { events, relations, storage, signal } = opts;
const { events, storage, signal } = opts;
if (!events.length || !relations.length) {
if (!events.length) {
return events;
}
for (const relation of relations) {
switch (relation) {
case 'author':
await hydrateAuthors({ events, storage, signal });
break;
case 'author_stats':
await hydrateAuthorStats(events);
break;
case 'event_stats':
await hydrateEventStats(events);
break;
case 'user':
await hydrateUsers({ events, storage, signal });
break;
case 'repost':
await hydrateRepostEvents({ events, storage, signal });
break;
case 'quote_repost':
await hydrateQuoteRepostEvents({ events, storage, signal });
break;
const allEvents: DittoEvent[] = structuredClone(events);
const childrenEventsIds = (events.map((event) => {
if (event.kind === 1) return event.tags.find(([name]) => name === 'q')?.[1]; // possible quote repost
if (event.kind === 6) return event.tags.find(([name]) => name === 'e')?.[1]; // possible repost
return;
}).filter(Boolean)) as string[];
if (childrenEventsIds.length > 0) {
const childrenEvents = await storage.query([{ ids: childrenEventsIds }], { signal });
allEvents.push(...childrenEvents);
if (childrenEvents.length > 0) {
const grandChildrenEventsIds = (childrenEvents.map((event) => {
if (event.kind === 1) return event.tags.find(([name]) => name === 'q')?.[1]; // possible quote repost
return;
}).filter(Boolean)) as string[];
if (grandChildrenEventsIds.length > 0) {
const grandChildrenEvents = await storage.query([{ ids: grandChildrenEventsIds }], { signal });
allEvents.push(...grandChildrenEvents);
}
}
}
await hydrateAuthors({ events: allEvents, storage, signal });
await hydrateAuthorStats(allEvents);
await hydrateEventStats(allEvents);
events.forEach((event) => {
const correspondingEvent = allEvents.find((element) => element.id === event.id);
if (correspondingEvent?.author) event.author = correspondingEvent.author;
if (correspondingEvent?.author_stats) event.author_stats = correspondingEvent.author_stats;
if (correspondingEvent?.event_stats) event.event_stats = correspondingEvent.event_stats;
if (event.kind === 1) {
const quoteId = event.tags.find(([name]) => name === 'q')?.[1];
if (quoteId) {
event.quote_repost = allEvents.find((element) => element.id === quoteId);
}
} else if (event.kind === 6) {
const repostedId = event.tags.find(([name]) => name === 'e')?.[1];
if (repostedId) {
const repostedEvent = allEvents.find((element) => element.id === repostedId);
if (repostedEvent && repostedEvent.tags.find(([name]) => name === 'q')?.[1]) { // The repost is a repost of a quote repost
const postBeingQuoteRepostedId = repostedEvent.tags.find(([name]) => name === 'q')?.[1];
event.repost = {
quote_repost: allEvents.find((element) => element.id === postBeingQuoteRepostedId),
...allEvents.find((element) => element.id === repostedId) as DittoEvent,
};
} else { // The repost is a repost of a normal post
event.repost = allEvents.find((element) => element.id === repostedId);
}
}
}
});
return events;
}
@ -58,23 +87,6 @@ async function hydrateAuthors(opts: Omit<HydrateEventOpts, 'relations'>): Promis
return events;
}
async function hydrateUsers(opts: Omit<HydrateEventOpts, 'relations'>): Promise<DittoEvent[]> {
const { events, storage, signal } = opts;
const pubkeys = new Set([...events].map((event) => event.pubkey));
const users = await storage.query(
[{ kinds: [30361], authors: [Conf.pubkey], '#d': [...pubkeys], limit: pubkeys.size }],
{ signal },
);
for (const event of events) {
event.user = users.find((user) => user.tags.find(([name]) => name === 'd')?.[1] === event.pubkey);
}
return events;
}
async function hydrateAuthorStats(events: DittoEvent[]): Promise<DittoEvent[]> {
const results = await db
.selectFrom('author_stats')
@ -117,84 +129,6 @@ async function hydrateEventStats(events: DittoEvent[]): Promise<DittoEvent[]> {
return events;
}
async function hydrateRepostEvents(opts: Omit<HydrateEventOpts, 'relations'>): Promise<DittoEvent[]> {
const { events, storage, signal } = opts;
const results = await storage.query([{
kinds: [1],
ids: events.map((event) => {
if (event.kind === 6) {
const originalPostId = event.tags.find(([name]) => name === 'e')?.[1];
if (!originalPostId) return event.id;
else return originalPostId;
}
return event.id;
}),
}], { signal });
for (const event of events) {
if (event.kind === 6) {
const originalPostId = event.tags.find(([name]) => name === 'e')?.[1];
if (!originalPostId) continue;
const originalPostEvent = results.find((event) => event.id === originalPostId);
if (!originalPostEvent) continue;
await hydrateEvents({
events: [originalPostEvent],
storage: storage,
signal: signal,
relations: ['author', 'event_stats'],
});
event.repost = originalPostEvent;
}
}
return events;
}
async function hydrateQuoteRepostEvents(opts: Omit<HydrateEventOpts, 'relations'>): Promise<DittoEvent[]> {
const { events, storage, signal } = opts;
const results = await storage.query([{
kinds: [1],
ids: events.map((event) => {
if (event.kind === 1) {
const originalPostId = event.tags.find(([name]) => name === 'q')?.[1];
if (!originalPostId) return event.id;
else return originalPostId;
}
return event.id;
}),
}], { signal });
for (const event of events) {
if (event.kind === 1) {
const originalPostId = event.tags.find(([name]) => name === 'q')?.[1];
if (!originalPostId) continue;
const originalPostEvent = events.find((event) => event.id === originalPostId);
if (!originalPostEvent) {
const originalPostEvent = results.find((event) => event.id === originalPostId);
if (!originalPostEvent) continue;
await hydrateEvents({ events: [originalPostEvent], storage: storage, signal: signal, relations: ['author'] });
event.quote_repost = originalPostEvent;
continue;
}
if (!originalPostEvent.author) {
await hydrateEvents({ events: [originalPostEvent], storage: storage, signal: signal, relations: ['author'] });
event.quote_repost = originalPostEvent;
continue;
}
event.quote_repost = originalPostEvent;
}
}
return events;
}
/** Return a normalized event without any non-standard keys. */
function purifyEvent(event: NostrEvent): NostrEvent {
return {

1
src/storages/search-store.ts
View File

@ -47,7 +47,6 @@ class SearchStore implements NStore {
return hydrateEvents({
events,
relations: ['author', 'event_stats', 'author_stats'],
storage: this.#hydrator,
signal: opts?.signal,
});

8
src/views.ts
View File

@ -20,7 +20,7 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal
}
const authors = await eventsDB.query([{ kinds: [0], authors: [...pubkeys] }], { signal })
.then((events) => hydrateEvents({ events, relations: ['author_stats'], storage: eventsDB, signal }));
.then((events) => hydrateEvents({ events, storage: eventsDB, signal }));
const accounts = await Promise.all(
authors.map((event) => renderAccount(event)),
@ -33,7 +33,7 @@ async function renderAccounts(c: AppContext, authors: string[], signal = AbortSi
const { since, until, limit } = paginationSchema.parse(c.req.query());
const events = await eventsDB.query([{ kinds: [0], authors, since, until, limit }], { signal })
.then((events) => hydrateEvents({ events, relations: ['author_stats'], storage: eventsDB, signal }));
.then((events) => hydrateEvents({ events, storage: eventsDB, signal }));
const accounts = await Promise.all(
events.map((event) => renderAccount(event)),
@ -51,9 +51,7 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal
const { limit } = paginationSchema.parse(c.req.query());
const events = await eventsDB.query([{ kinds: [1], ids, limit }], { signal })
.then((events) =>
hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal })
);
.then((events) => hydrateEvents({ events, storage: eventsDB, signal }));
if (!events.length) {
return c.json([]);