moon
/
ditto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'timelines-hydrate' into 'main' 

Timelines hydrate

Closes #79

See merge request soapbox-pub/ditto!116
This commit is contained in:
Alex Gleason 2024-03-06 19:02:54 +00:00
parent 07d7b3868d 8b9566d79b
commit e1e71be8ea
10 changed files with 76 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/controllers/api/accounts.ts
View File

@ -15,6 +15,7 @@ import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts';
import { renderRelationship } from '@/views/mastodon/relationships.ts';
import { renderStatus } from '@/views/mastodon/statuses.ts';
import { DittoFilter } from '@/interfaces/DittoFilter.ts';
import { hydrateEvents } from '@/storages/hydrate.ts';
const usernameSchema = z
.string().min(1).max(30)
@ -147,7 +148,6 @@ const accountStatusesController: AppController = async (c) => {
const filter: DittoFilter = {
authors: [pubkey],
kinds: [1],
relations: ['author', 'event_stats', 'author_stats'],
since,
until,
limit,
@ -157,11 +157,16 @@ const accountStatusesController: AppController = async (c) => {
filter['#t'] = [tagged];
}
let events = await eventsDB.query([filter], { signal });
if (exclude_replies) {
events = events.filter((event) => !findReplyTag(event.tags));
}
const events = await eventsDB.query([filter], { signal })
.then((events) =>
hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal })
)
.then((events) => {
if (exclude_replies) {
return events.filter((event) => !findReplyTag(event.tags));
}
return events;
});
const statuses = await Promise.all(events.map((event) => renderStatus(event, c.get('pubkey'))));
return paginated(c, events, statuses);
@ -304,10 +309,10 @@ const favouritesController: AppController = async (c) => {
.map((event) => event.tags.find((tag) => tag[0] === 'e')?.[1])
.filter((id): id is string => !!id);
const events1 = await eventsDB.query(
[{ kinds: [1], ids, relations: ['author', 'event_stats', 'author_stats'] }],
{ signal },
);
const events1 = await eventsDB.query([{ kinds: [1], ids }], { signal })
.then((events) =>
hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal })
);
const statuses = await Promise.all(events1.map((event) => renderStatus(event, c.get('pubkey'))));
return paginated(c, events1, statuses);

25
src/controllers/api/search.ts
View File

@ -8,6 +8,7 @@ import { dedupeEvents } from '@/utils.ts';
import { nip05Cache } from '@/utils/nip05.ts';
import { renderAccount } from '@/views/mastodon/accounts.ts';
import { renderStatus } from '@/views/mastodon/statuses.ts';
import { hydrateEvents } from '@/storages/hydrate.ts';
/** Matches NIP-05 names with or without an @ in front. */
const ACCT_REGEX = /^@?(?:([\w.+-]+)@)?([\w.-]+)$/;
@ -69,7 +70,6 @@ function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: Abort
const filter: DittoFilter = {
kinds: typeToKinds(type),
search: q,
relations: ['author', 'event_stats', 'author_stats'],
limit,
};
@ -77,7 +77,10 @@ function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: Abort
filter.authors = [account_id];
}
return searchStore.query([filter], { signal });
return searchStore.query([filter], { signal })
.then((events) =>
hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: searchStore, signal })
);
}
/** Get event kinds to search from `type` query param. */
@ -95,8 +98,12 @@ function typeToKinds(type: SearchQuery['type']): number[] {
/** Resolve a searched value into an event, if applicable. */
async function lookupEvent(query: SearchQuery, signal: AbortSignal): Promise<NostrEvent | undefined> {
const filters = await getLookupFilters(query, signal);
const [event] = await searchStore.query(filters, { limit: 1, signal });
return event;
return searchStore.query(filters, { limit: 1, signal })
.then((events) =>
hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: searchStore, signal })
)
.then(([event]) => event);
}
/** Get filters to lookup the input value. */
@ -115,19 +122,19 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
const result = nip19.decode(q);
switch (result.type) {
case 'npub':
if (accounts) filters.push({ kinds: [0], authors: [result.data], relations: ['author_stats'] });
if (accounts) filters.push({ kinds: [0], authors: [result.data] });
break;
case 'nprofile':
if (accounts) filters.push({ kinds: [0], authors: [result.data.pubkey], relations: ['author_stats'] });
if (accounts) filters.push({ kinds: [0], authors: [result.data.pubkey] });
break;
case 'note':
if (statuses) {
filters.push({ kinds: [1], ids: [result.data], relations: ['author', 'event_stats', 'author_stats'] });
filters.push({ kinds: [1], ids: [result.data] });
}
break;
case 'nevent':
if (statuses) {
filters.push({ kinds: [1], ids: [result.data.id], relations: ['author', 'event_stats', 'author_stats'] });
filters.push({ kinds: [1], ids: [result.data.id] });
}
break;
}
@ -141,7 +148,7 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort
try {
const { pubkey } = await nip05Cache.fetch(q, { signal });
if (pubkey) {
filters.push({ kinds: [0], authors: [pubkey], relations: ['author_stats'] });
filters.push({ kinds: [0], authors: [pubkey] });
}
} catch (_e) {
// do nothing

8
src/controllers/api/timelines.ts
View File

@ -4,6 +4,7 @@ import { type DittoFilter } from '@/interfaces/DittoFilter.ts';
import { getFeedPubkeys } from '@/queries.ts';
import { booleanParamSchema } from '@/schema.ts';
import { eventsDB } from '@/storages.ts';
import { hydrateEvents } from '@/storages/hydrate.ts';
import { paginated, paginationSchema } from '@/utils/api.ts';
import { renderStatus } from '@/views/mastodon/statuses.ts';
@ -34,10 +35,9 @@ const hashtagTimelineController: AppController = (c) => {
async function renderStatuses(c: AppContext, filters: DittoFilter[]) {
const { signal } = c.req.raw;
const events = await eventsDB.query(
filters.map((filter) => ({ ...filter, relations: ['author', 'event_stats', 'author_stats'] })),
{ signal },
);
const events = await eventsDB
.query(filters, { signal })
.then((events) => hydrateEvents({ events, relations: ['author'], storage: eventsDB, signal }));
if (!events.length) {
return c.json([]);

2
src/interfaces/DittoFilter.ts
View File

@ -9,6 +9,4 @@ export type DittoRelation = Exclude<keyof DittoEvent, keyof NostrEvent>;
export interface DittoFilter extends NostrFilter {
/** Whether the event was authored by a local user. */
local?: boolean;
/** Additional fields to add to the returned event. */
relations?: DittoRelation[];
}

83
src/queries.ts
View File

@ -1,9 +1,9 @@
import { cache, eventsDB, reqmeister } from '@/storages.ts';
import { Debug, type NostrEvent } from '@/deps.ts';
import { type AuthorMicrofilter, type IdMicrofilter } from '@/filter.ts';
import { eventsDB, optimizer } from '@/storages.ts';
import { Debug, type NostrEvent, type NostrFilter } from '@/deps.ts';
import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
import { type DittoFilter, type DittoRelation } from '@/interfaces/DittoFilter.ts';
import { type DittoRelation } from '@/interfaces/DittoFilter.ts';
import { findReplyTag, getTagSet } from '@/tags.ts';
import { hydrateEvents } from '@/storages/hydrate.ts';
const debug = Debug('ditto:queries');
@ -22,76 +22,25 @@ const getEvent = async (
opts: GetEventOpts = {},
): Promise<DittoEvent | undefined> => {
debug(`getEvent: ${id}`);
const { kind, relations, signal = AbortSignal.timeout(1000) } = opts;
const microfilter: IdMicrofilter = { ids: [id] };
const { kind, relations = [], signal = AbortSignal.timeout(1000) } = opts;
const [memoryEvent] = await cache.query([microfilter]) as DittoEvent[];
if (memoryEvent && !relations) {
debug(`getEvent: ${id.slice(0, 8)} found in memory`);
return memoryEvent;
}
const filter: DittoFilter = { ids: [id], relations, limit: 1 };
const filter: NostrFilter = { ids: [id], limit: 1 };
if (kind) {
filter.kinds = [kind];
}
const dbEvent = await eventsDB.query([filter], { limit: 1, signal })
return await optimizer.query([filter], { limit: 1, signal })
.then(([event]) => hydrateEvents({ events: [event], relations, storage: optimizer, signal }))
.then(([event]) => event);
// TODO: make this DRY-er.
if (dbEvent && !dbEvent.author) {
const [author] = await cache.query([{ kinds: [0], authors: [dbEvent.pubkey] }]);
dbEvent.author = author;
}
if (dbEvent) {
debug(`getEvent: ${id.slice(0, 8)} found in db`);
return dbEvent;
}
if (memoryEvent && !memoryEvent.author) {
const [author] = await cache.query([{ kinds: [0], authors: [memoryEvent.pubkey] }]);
memoryEvent.author = author;
}
if (memoryEvent) {
debug(`getEvent: ${id.slice(0, 8)} found in memory`);
return memoryEvent;
}
const reqEvent = await reqmeister.req(microfilter, opts).catch(() => undefined);
if (reqEvent) {
debug(`getEvent: ${id.slice(0, 8)} found by reqmeister`);
return reqEvent;
}
debug(`getEvent: ${id.slice(0, 8)} not found`);
};
/** Get a Nostr `set_medatadata` event for a user's pubkey. */
const getAuthor = async (pubkey: string, opts: GetEventOpts = {}): Promise<NostrEvent | undefined> => {
const { relations, signal = AbortSignal.timeout(1000) } = opts;
const microfilter: AuthorMicrofilter = { kinds: [0], authors: [pubkey] };
const { relations = [], signal = AbortSignal.timeout(1000) } = opts;
const [memoryEvent] = await cache.query([microfilter]);
if (memoryEvent && !relations) {
return memoryEvent;
}
const dbEvent = await eventsDB.query(
[{ authors: [pubkey], relations, kinds: [0], limit: 1 }],
{ limit: 1, signal },
).then(([event]) => event);
if (dbEvent) return dbEvent;
if (memoryEvent) return memoryEvent;
return reqmeister.req(microfilter, opts).catch(() => undefined);
return await optimizer.query([{ authors: [pubkey], kinds: [0], limit: 1 }], { limit: 1, signal })
.then(([event]) => hydrateEvents({ events: [event], relations, storage: optimizer, signal }))
.then(([event]) => event);
};
/** Get users the given pubkey follows. */
@ -132,10 +81,10 @@ async function getAncestors(event: NostrEvent, result: NostrEvent[] = []): Promi
}
function getDescendants(eventId: string, signal = AbortSignal.timeout(2000)): Promise<NostrEvent[]> {
return eventsDB.query(
[{ kinds: [1], '#e': [eventId], relations: ['author', 'event_stats', 'author_stats'] }],
{ limit: 200, signal },
);
return eventsDB.query([{ kinds: [1], '#e': [eventId] }], { limit: 200, signal })
.then((events) =>
hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal })
);
}
/** Returns whether the pubkey is followed by a local user. */

43
src/storages/events-db.ts
View File

@ -198,49 +198,6 @@ class EventsDB implements NStore {
.where('users.d_tag', filter.local ? 'is not' : 'is', null);
}
if (filter.relations?.includes('author')) {
query = query
.leftJoin(
(eb) =>
eb
.selectFrom('events')
.selectAll()
.where('kind', '=', 0)
.groupBy('pubkey')
.as('authors'),
(join) => join.onRef('authors.pubkey', '=', 'events.pubkey'),
)
.select([
'authors.id as author_id',
'authors.kind as author_kind',
'authors.pubkey as author_pubkey',
'authors.content as author_content',
'authors.tags as author_tags',
'authors.created_at as author_created_at',
'authors.sig as author_sig',
]);
}
if (filter.relations?.includes('author_stats')) {
query = query
.leftJoin('author_stats', 'author_stats.pubkey', 'events.pubkey')
.select((eb) => [
eb.fn.coalesce('author_stats.followers_count', eb.val(0)).as('author_stats_followers_count'),
eb.fn.coalesce('author_stats.following_count', eb.val(0)).as('author_stats_following_count'),
eb.fn.coalesce('author_stats.notes_count', eb.val(0)).as('author_stats_notes_count'),
]);
}
if (filter.relations?.includes('event_stats')) {
query = query
.leftJoin('event_stats', 'event_stats.event_id', 'events.id')
.select((eb) => [
eb.fn.coalesce('event_stats.replies_count', eb.val(0)).as('stats_replies_count'),
eb.fn.coalesce('event_stats.reposts_count', eb.val(0)).as('stats_reposts_count'),
eb.fn.coalesce('event_stats.reactions_count', eb.val(0)).as('stats_reactions_count'),
]);
}
if (filter.search) {
query = query
.innerJoin('events_fts', 'events_fts.id', 'events.id')

14
src/storages/hydrate.ts
View File

@ -1,21 +1,25 @@
import { type NostrEvent, type NStore } from '@/deps.ts';
import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
import { type DittoFilter } from '@/interfaces/DittoFilter.ts';
import { type DittoRelation } from '@/interfaces/DittoFilter.ts';
interface HydrateEventOpts {
events: DittoEvent[];
filters: DittoFilter[];
relations: DittoRelation[];
storage: NStore;
signal?: AbortSignal;
}
/** Hydrate event relationships using the provided storage. */
async function hydrateEvents(opts: HydrateEventOpts): Promise<DittoEvent[]> {
const { events, filters, storage, signal } = opts;
const { events, relations, storage, signal } = opts;
if (filters.some((filter) => filter.relations?.includes('author'))) {
if (!events.length || !relations.length) {
return events;
}
if (relations.includes('author')) {
const pubkeys = new Set([...events].map((event) => event.pubkey));
const authors = await storage.query([{ kinds: [0], authors: [...pubkeys] }], { signal });
const authors = await storage.query([{ kinds: [0], authors: [...pubkeys], limit: pubkeys.size }], { signal });
for (const event of events) {
event.author = authors.find((author) => author.pubkey === event.pubkey);

2
src/storages/optimizer.ts
View File

@ -23,7 +23,7 @@ class Optimizer implements NStore {
this.#client = opts.client;
}
async event(event: DittoEvent, opts?: NStoreOpts | undefined): Promise<void> {
async event(event: DittoEvent, opts?: NStoreOpts): Promise<void> {
if (opts?.signal?.aborted) return Promise.reject(abortError());
await Promise.all([

7
src/storages/search-store.ts
View File

@ -62,7 +62,12 @@ class SearchStore implements NStore {
events.add(event);
}
return hydrateEvents({ events: [...events], filters, storage: this.#hydrator, signal: opts?.signal });
return hydrateEvents({
events: [...events],
relations: ['author', 'event_stats', 'author_stats'],
storage: this.#hydrator,
signal: opts?.signal,
});
} else {
this.#debug(`Searching for "${query}" locally...`);
return this.#fallback.query(filters, opts);

21
src/views.ts
View File

@ -4,6 +4,7 @@ import { eventsDB } from '@/storages.ts';
import { renderAccount } from '@/views/mastodon/accounts.ts';
import { renderStatus } from '@/views/mastodon/statuses.ts';
import { paginated, paginationSchema } from '@/utils/api.ts';
import { hydrateEvents } from '@/storages/hydrate.ts';
/** Render account objects for the author of each event. */
async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal = AbortSignal.timeout(1000)) {
@ -18,10 +19,8 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal
return c.json([]);
}
const authors = await eventsDB.query(
[{ kinds: [0], authors: [...pubkeys], relations: ['author_stats'] }],
{ signal },
);
const authors = await eventsDB.query([{ kinds: [0], authors: [...pubkeys] }], { signal })
.then((events) => hydrateEvents({ events, relations: ['author_stats'], storage: eventsDB, signal }));
const accounts = await Promise.all(
authors.map((event) => renderAccount(event)),
@ -33,10 +32,8 @@ async function renderEventAccounts(c: AppContext, filters: NostrFilter[], signal
async function renderAccounts(c: AppContext, authors: string[], signal = AbortSignal.timeout(1000)) {
const { since, until, limit } = paginationSchema.parse(c.req.query());
const events = await eventsDB.query(
[{ kinds: [0], authors, relations: ['author_stats'], since, until, limit }],
{ signal },
);
const events = await eventsDB.query([{ kinds: [0], authors, since, until, limit }], { signal })
.then((events) => hydrateEvents({ events, relations: ['author_stats'], storage: eventsDB, signal }));
const accounts = await Promise.all(
events.map((event) => renderAccount(event)),
@ -53,10 +50,10 @@ async function renderStatuses(c: AppContext, ids: string[], signal = AbortSignal
const { limit } = paginationSchema.parse(c.req.query());
const events = await eventsDB.query(
[{ kinds: [1], ids, relations: ['author', 'event_stats', 'author_stats'], limit }],
{ signal },
);
const events = await eventsDB.query([{ kinds: [1], ids, limit }], { signal })
.then((events) =>
hydrateEvents({ events, relations: ['author', 'event_stats', 'author_stats'], storage: eventsDB, signal })
);
if (!events.length) {
return c.json([]);