moon
/
ditto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Nuke the old stats module, support emoji reactions on posts

This commit is contained in:
Alex Gleason 2024-05-24 17:40:51 -05:00
parent ee2065b76b
commit f7c9a96719
No known key found for this signature in database
GPG Key ID: 7211D1F99744FBB7
10 changed files with 134 additions and 299 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scripts/stats-recompute.ts
View File

@ -1,6 +1,8 @@
import { nip19 } from 'nostr-tools';
import { refreshAuthorStats } from '@/stats.ts';
import { DittoDB } from '@/db/DittoDB.ts';
import { Storages } from '@/storages.ts';
import { refreshAuthorStats } from '@/utils/stats.ts';
let pubkey: string;
try {
@ -15,4 +17,7 @@ try {
Deno.exit(1);
}
await refreshAuthorStats(pubkey);
const store = await Storages.db();
const kysely = await DittoDB.getInstance();
await refreshAuthorStats({ pubkey, kysely, store });

2
src/db/DittoTables.ts
View File

@ -19,7 +19,7 @@ interface EventStatsRow {
event_id: string;
replies_count: number;
reposts_count: number;
reactions_count: number;
reactions: string;
}
interface EventRow {

18
src/db/migrations/022_event_stats_reactions.ts Normal file
View File

@ -0,0 +1,18 @@
import { Kysely } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
await db.schema
.alterTable('event_stats')
.addColumn('reactions', 'text', (col) => col.defaultTo('{}'))
.execute();
await db.schema
.alterTable('event_stats')
.dropColumn('reactions_count')
.execute();
}
export async function down(db: Kysely<any>): Promise<void> {
await db.schema.alterTable('event_stats').dropColumn('reactions').execute();
await db.schema.alterTable('event_stats').addColumn('reactions_count', 'integer').execute();
}

2
src/interfaces/DittoEvent.ts
View File

@ -11,7 +11,7 @@ export interface AuthorStats {
export interface EventStats {
replies_count: number;
reposts_count: number;
reactions_count: number;
reactions: Record<string, number>;
}
/** Internal Event representation used by Ditto, including extra keys. */

5
src/pipeline.ts
View File

@ -10,7 +10,6 @@ import { deleteAttachedMedia } from '@/db/unattached-media.ts';
import { DittoEvent } from '@/interfaces/DittoEvent.ts';
import { DVM } from '@/pipeline/DVM.ts';
import { RelayError } from '@/RelayError.ts';
import { updateStats } from '@/stats.ts';
import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts';
import { Storages } from '@/storages.ts';
import { eventAge, nostrDate, nostrNow, parseNip05, Time } from '@/utils.ts';
@ -21,6 +20,7 @@ import { verifyEventWorker } from '@/workers/verify.ts';
import { AdminSigner } from '@/signers/AdminSigner.ts';
import { lnurlCache } from '@/utils/lnurl.ts';
import { nip05Cache } from '@/utils/nip05.ts';
import { updateStats } from '@/utils/stats.ts';
import { getTagSet } from '@/utils/tags.ts';
import { MuteListPolicy } from '@/policies/MuteListPolicy.ts';
@ -121,8 +121,9 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise<voi
async function storeEvent(event: DittoEvent, signal?: AbortSignal): Promise<void> {
if (NKinds.ephemeral(event.kind)) return;
const store = await Storages.db();
const kysely = await DittoDB.getInstance();
await updateStats(event).catch(debug);
await updateStats({ event, store, kysely }).catch(debug);
await store.event(event, { signal });
}

273
src/stats.ts
View File

@ -1,273 +0,0 @@
import { Semaphore } from '@lambdalisue/async';
import { NKinds, NostrEvent, NStore } from '@nostrify/nostrify';
import Debug from '@soapbox/stickynotes/debug';
import { InsertQueryBuilder, Kysely } from 'kysely';
import { LRUCache } from 'lru-cache';
import { SetRequired } from 'type-fest';
import { DittoDB } from '@/db/DittoDB.ts';
import { DittoTables } from '@/db/DittoTables.ts';
import { Storages } from '@/storages.ts';
import { findReplyTag, getTagSet } from '@/utils/tags.ts';
type AuthorStat = keyof Omit<DittoTables['author_stats'], 'pubkey'>;
type EventStat = keyof Omit<DittoTables['event_stats'], 'event_id'>;
type AuthorStatDiff = ['author_stats', pubkey: string, stat: AuthorStat, diff: number];
type EventStatDiff = ['event_stats', eventId: string, stat: EventStat, diff: number];
type StatDiff = AuthorStatDiff | EventStatDiff;
const debug = Debug('ditto:stats');
/** Store stats for the event. */
async function updateStats(event: NostrEvent) {
let prev: NostrEvent | undefined;
const queries: InsertQueryBuilder<DittoTables, any, unknown>[] = [];
// Kind 3 is a special case - replace the count with the new list.
if (event.kind === 3) {
prev = await getPrevEvent(event);
if (!prev || event.created_at >= prev.created_at) {
queries.push(await updateFollowingCountQuery(event));
}
}
const statDiffs = await getStatsDiff(event, prev);
const pubkeyDiffs = statDiffs.filter(([table]) => table === 'author_stats') as AuthorStatDiff[];
const eventDiffs = statDiffs.filter(([table]) => table === 'event_stats') as EventStatDiff[];
if (statDiffs.length) {
debug(JSON.stringify({ id: event.id, pubkey: event.pubkey, kind: event.kind, tags: event.tags, statDiffs }));
}
pubkeyDiffs.forEach(([_, pubkey]) => refreshAuthorStatsDebounced(pubkey));
const kysely = await DittoDB.getInstance();
if (pubkeyDiffs.length) queries.push(authorStatsQuery(kysely, pubkeyDiffs));
if (eventDiffs.length) queries.push(eventStatsQuery(kysely, eventDiffs));
if (queries.length) {
await Promise.all(queries.map((query) => query.execute()));
}
}
/** Calculate stats changes ahead of time so we can build an efficient query. */
async function getStatsDiff(event: NostrEvent, prev: NostrEvent | undefined): Promise<StatDiff[]> {
const store = await Storages.db();
const statDiffs: StatDiff[] = [];
const firstTaggedId = event.tags.find(([name]) => name === 'e')?.[1];
const inReplyToId = findReplyTag(event.tags)?.[1];
switch (event.kind) {
case 1:
statDiffs.push(['author_stats', event.pubkey, 'notes_count', 1]);
if (inReplyToId) {
statDiffs.push(['event_stats', inReplyToId, 'replies_count', 1]);
}
break;
case 3:
statDiffs.push(...getFollowDiff(event, prev));
break;
case 5: {
if (!firstTaggedId) break;
const [repostedEvent] = await store.query(
[{ kinds: [6], ids: [firstTaggedId], authors: [event.pubkey] }],
{ limit: 1 },
);
// Check if the event being deleted is of kind 6,
// if it is then proceed, else just break
if (!repostedEvent) break;
const eventBeingRepostedId = repostedEvent.tags.find(([name]) => name === 'e')?.[1];
const eventBeingRepostedPubkey = repostedEvent.tags.find(([name]) => name === 'p')?.[1];
if (!eventBeingRepostedId || !eventBeingRepostedPubkey) break;
const [eventBeingReposted] = await store.query(
[{ kinds: [1], ids: [eventBeingRepostedId], authors: [eventBeingRepostedPubkey] }],
{ limit: 1 },
);
if (!eventBeingReposted) break;
statDiffs.push(['event_stats', eventBeingRepostedId, 'reposts_count', -1]);
break;
}
case 6:
if (firstTaggedId) {
statDiffs.push(['event_stats', firstTaggedId, 'reposts_count', 1]);
}
break;
case 7:
if (firstTaggedId) {
statDiffs.push(['event_stats', firstTaggedId, 'reactions_count', 1]);
}
}
return statDiffs;
}
/** Create an author stats query from the list of diffs. */
function authorStatsQuery(kysely: Kysely<DittoTables>, diffs: AuthorStatDiff[]) {
const values: DittoTables['author_stats'][] = diffs.map(([_, pubkey, stat, diff]) => {
const row: DittoTables['author_stats'] = {
pubkey,
followers_count: 0,
following_count: 0,
notes_count: 0,
};
row[stat] = diff;
return row;
});
return kysely.insertInto('author_stats')
.values(values)
.onConflict((oc) =>
oc
.column('pubkey')
.doUpdateSet((eb) => ({
followers_count: eb('author_stats.followers_count', '+', eb.ref('excluded.followers_count')),
following_count: eb('author_stats.following_count', '+', eb.ref('excluded.following_count')),
notes_count: eb('author_stats.notes_count', '+', eb.ref('excluded.notes_count')),
}))
);
}
/** Create an event stats query from the list of diffs. */
function eventStatsQuery(kysely: Kysely<DittoTables>, diffs: EventStatDiff[]) {
const values: DittoTables['event_stats'][] = diffs.map(([_, event_id, stat, diff]) => {
const row: DittoTables['event_stats'] = {
event_id,
replies_count: 0,
reposts_count: 0,
reactions_count: 0,
};
row[stat] = diff;
return row;
});
return kysely.insertInto('event_stats')
.values(values)
.onConflict((oc) =>
oc
.column('event_id')
.doUpdateSet((eb) => ({
replies_count: eb('event_stats.replies_count', '+', eb.ref('excluded.replies_count')),
reposts_count: eb('event_stats.reposts_count', '+', eb.ref('excluded.reposts_count')),
reactions_count: eb('event_stats.reactions_count', '+', eb.ref('excluded.reactions_count')),
}))
);
}
/** Get the last version of the event, if any. */
async function getPrevEvent(event: NostrEvent): Promise<NostrEvent | undefined> {
if (NKinds.replaceable(event.kind) || NKinds.parameterizedReplaceable(event.kind)) {
const store = await Storages.db();
const [prev] = await store.query([
{ kinds: [event.kind], authors: [event.pubkey], limit: 1 },
]);
return prev;
}
}
/** Set the following count to the total number of unique "p" tags in the follow list. */
async function updateFollowingCountQuery({ pubkey, tags }: NostrEvent) {
const following_count = new Set(
tags
.filter(([name]) => name === 'p')
.map(([_, value]) => value),
).size;
const kysely = await DittoDB.getInstance();
return kysely.insertInto('author_stats')
.values({
pubkey,
following_count,
followers_count: 0,
notes_count: 0,
})
.onConflict((oc) =>
oc
.column('pubkey')
.doUpdateSet({ following_count })
);
}
/** Compare the old and new follow events (if any), and return a diff array. */
function getFollowDiff(event: NostrEvent, prev?: NostrEvent): AuthorStatDiff[] {
const prevTags = prev?.tags ?? [];
const prevPubkeys = new Set(
prevTags
.filter(([name]) => name === 'p')
.map(([_, value]) => value),
);
const pubkeys = new Set(
event.tags
.filter(([name]) => name === 'p')
.map(([_, value]) => value),
);
const added = [...pubkeys].filter((pubkey) => !prevPubkeys.has(pubkey));
const removed = [...prevPubkeys].filter((pubkey) => !pubkeys.has(pubkey));
return [
...added.map((pubkey): AuthorStatDiff => ['author_stats', pubkey, 'followers_count', 1]),
...removed.map((pubkey): AuthorStatDiff => ['author_stats', pubkey, 'followers_count', -1]),
];
}
/** Refresh the author's stats in the database. */
async function refreshAuthorStats(pubkey: string): Promise<DittoTables['author_stats']> {
const store = await Storages.db();
const stats = await countAuthorStats(store, pubkey);
const kysely = await DittoDB.getInstance();
await kysely.insertInto('author_stats')
.values(stats)
.onConflict((oc) => oc.column('pubkey').doUpdateSet(stats))
.execute();
return stats;
}
/** Calculate author stats from the database. */
async function countAuthorStats(
store: SetRequired<NStore, 'count'>,
pubkey: string,
): Promise<DittoTables['author_stats']> {
const [{ count: followers_count }, { count: notes_count }, [followList]] = await Promise.all([
store.count([{ kinds: [3], '#p': [pubkey] }]),
store.count([{ kinds: [1], authors: [pubkey] }]),
store.query([{ kinds: [3], authors: [pubkey], limit: 1 }]),
]);
return {
pubkey,
followers_count,
following_count: getTagSet(followList?.tags ?? [], 'p').size,
notes_count,
};
}
const authorStatsSemaphore = new Semaphore(10);
const refreshedAuthors = new LRUCache<string, true>({ max: 1000 });
/** Calls `refreshAuthorStats` only once per author. */
function refreshAuthorStatsDebounced(pubkey: string): void {
if (refreshedAuthors.get(pubkey)) {
return;
}
refreshedAuthors.set(pubkey, true);
debug('refreshing author stats:', pubkey);
authorStatsSemaphore
.lock(() => refreshAuthorStats(pubkey).catch(() => {}));
}
export { refreshAuthorStats, refreshAuthorStatsDebounced, updateStats };

21
src/storages/hydrate.ts
View File

@ -2,10 +2,11 @@ import { NostrEvent, NStore } from '@nostrify/nostrify';
import { matchFilter } from 'nostr-tools';
import { DittoDB } from '@/db/DittoDB.ts';
import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
import { DittoTables } from '@/db/DittoTables.ts';
import { Conf } from '@/config.ts';
import { refreshAuthorStatsDebounced } from '@/stats.ts';
import { type DittoEvent } from '@/interfaces/DittoEvent.ts';
import { Storages } from '@/storages.ts';
import { refreshAuthorStatsDebounced } from '@/utils/stats.ts';
import { findQuoteTag } from '@/utils/tags.ts';
interface HydrateOpts {
@ -77,6 +78,11 @@ function assembleEvents(
): DittoEvent[] {
const admin = Conf.pubkey;
const eventStats = stats.events.map((stat) => ({
...stat,
reactions: JSON.parse(stat.reactions),
}));
for (const event of a) {
event.author = b.find((e) => matchFilter({ kinds: [0], authors: [event.pubkey] }, e));
event.user = b.find((e) => matchFilter({ kinds: [30361], authors: [admin], '#d': [event.pubkey] }, e));
@ -120,7 +126,7 @@ function assembleEvents(
}
event.author_stats = stats.authors.find((stats) => stats.pubkey === event.pubkey);
event.event_stats = stats.events.find((stats) => stats.event_id === event.id);
event.event_stats = eventStats.find((stats) => stats.event_id === event.id);
}
return a;
@ -270,7 +276,10 @@ async function gatherAuthorStats(events: DittoEvent[]): Promise<DittoTables['aut
}));
}
function refreshMissingAuthorStats(events: NostrEvent[], stats: DittoTables['author_stats'][]) {
async function refreshMissingAuthorStats(events: NostrEvent[], stats: DittoTables['author_stats'][]) {
const store = await Storages.db();
const kysely = await DittoDB.getInstance();
const pubkeys = new Set<string>(
events
.filter((event) => event.kind === 0)
@ -282,7 +291,7 @@ function refreshMissingAuthorStats(events: NostrEvent[], stats: DittoTables['aut
);
for (const pubkey of missing) {
refreshAuthorStatsDebounced(pubkey);
refreshAuthorStatsDebounced({ pubkey, store, kysely });
}
}
@ -309,8 +318,8 @@ async function gatherEventStats(events: DittoEvent[]): Promise<DittoTables['even
return rows.map((row) => ({
event_id: row.event_id,
reposts_count: Math.max(0, row.reposts_count),
reactions_count: Math.max(0, row.reactions_count),
replies_count: Math.max(0, row.replies_count),
reactions: row.reactions,
}));
}

12
src/utils/stats.test.ts
View File

@ -113,15 +113,13 @@ Deno.test('updateStats with kind 7 increments reactions count', async () => {
const note = genEvent({ kind: 1 });
await updateStats({ ...db, event: note });
await db.store.event(note);
const reaction = genEvent({ kind: 7, tags: [['e', note.id]] });
await updateStats({ ...db, event: reaction });
await db.store.event(reaction);
await updateStats({ ...db, event: genEvent({ kind: 7, content: '+', tags: [['e', note.id]] }) });
await updateStats({ ...db, event: genEvent({ kind: 7, content: '😂', tags: [['e', note.id]] }) });
const stats = await getEventStats(db.kysely, note.id);
assertEquals(stats!.reactions_count, 1);
assertEquals(stats!.reactions, JSON.stringify({ '+': 1, '😂': 1 }));
});
Deno.test('updateStats with kind 5 decrements reactions count', async () => {
@ -132,7 +130,7 @@ Deno.test('updateStats with kind 5 decrements reactions count', async () => {
await db.store.event(note);
const sk = generateSecretKey();
const reaction = genEvent({ kind: 7, tags: [['e', note.id]] }, sk);
const reaction = genEvent({ kind: 7, content: '+', tags: [['e', note.id]] }, sk);
await updateStats({ ...db, event: reaction });
await db.store.event(reaction);
@ -140,7 +138,7 @@ Deno.test('updateStats with kind 5 decrements reactions count', async () => {
const stats = await getEventStats(db.kysely, note.id);
assertEquals(stats!.reactions_count, 0);
assertEquals(stats!.reactions, JSON.stringify({}));
});
Deno.test('countAuthorStats counts author stats from the database', async () => {

76
src/utils/stats.ts
View File

@ -1,5 +1,7 @@
import { Semaphore } from '@lambdalisue/async';
import { NostrEvent, NStore } from '@nostrify/nostrify';
import { Kysely, UpdateObject } from 'kysely';
import { LRUCache } from 'lru-cache';
import { SetRequired } from 'type-fest';
import { DittoTables } from '@/db/DittoTables.ts';
@ -31,7 +33,7 @@ export async function updateStats({ event, kysely, store, x = 1 }: UpdateStatsOp
/** Update stats for kind 1 event. */
async function handleEvent1(kysely: Kysely<DittoTables>, event: NostrEvent, x: number): Promise<void> {
await updateAuthorStats(kysely, event.pubkey, ({ notes_count }) => ({ notes_count: notes_count + x }));
await updateAuthorStats(kysely, event.pubkey, ({ notes_count }) => ({ notes_count: Math.max(0, notes_count + x) }));
}
/** Update stats for kind 3 event. */
@ -47,11 +49,19 @@ async function handleEvent3(kysely: Kysely<DittoTables>, event: NostrEvent, x: n
const { added, removed } = getFollowDiff(event.tags, prev?.tags);
for (const pubkey of added) {
await updateAuthorStats(kysely, pubkey, ({ followers_count }) => ({ followers_count: followers_count + x }));
await updateAuthorStats(
kysely,
pubkey,
({ followers_count }) => ({ followers_count: Math.max(0, followers_count + x) }),
);
}
for (const pubkey of removed) {
await updateAuthorStats(kysely, pubkey, ({ followers_count }) => ({ followers_count: followers_count - x }));
await updateAuthorStats(
kysely,
pubkey,
({ followers_count }) => ({ followers_count: Math.max(0, followers_count - x) }),
);
}
}
@ -70,15 +80,33 @@ async function handleEvent5(kysely: Kysely<DittoTables>, event: NostrEvent, x: -
async function handleEvent6(kysely: Kysely<DittoTables>, event: NostrEvent, x: number): Promise<void> {
const id = event.tags.find(([name]) => name === 'e')?.[1];
if (id) {
await updateEventStats(kysely, id, ({ reposts_count }) => ({ reposts_count: reposts_count + x }));
await updateEventStats(kysely, id, ({ reposts_count }) => ({ reposts_count: Math.max(0, reposts_count + x) }));
}
}
/** Update stats for kind 7 event. */
async function handleEvent7(kysely: Kysely<DittoTables>, event: NostrEvent, x: number): Promise<void> {
const id = event.tags.find(([name]) => name === 'e')?.[1];
if (id) {
await updateEventStats(kysely, id, ({ reactions_count }) => ({ reactions_count: reactions_count + x }));
const emoji = event.content;
if (id && emoji && (['+', '-'].includes(emoji) || /^\p{RGI_Emoji}$/v.test(emoji))) {
await updateEventStats(kysely, id, ({ reactions }) => {
const data: Record<string, number> = JSON.parse(reactions);
// Increment or decrement the emoji count.
data[emoji] = (data[emoji] ?? 0) + x;
// Remove reactions with a count of 0 or less.
for (const key of Object.keys(data)) {
if (data[key] < 1) {
delete data[key];
}
}
return {
reactions: JSON.stringify(data),
};
});
}
}
@ -160,6 +188,7 @@ export async function updateEventStats(
replies_count: 0,
reposts_count: 0,
reactions_count: 0,
reactions: '{}',
};
const prev = await getEventStats(kysely, eventId);
@ -196,3 +225,38 @@ export async function countAuthorStats(
notes_count,
};
}
export interface RefreshAuthorStatsOpts {
pubkey: string;
kysely: Kysely<DittoTables>;
store: SetRequired<NStore, 'count'>;
}
/** Refresh the author's stats in the database. */
export async function refreshAuthorStats(
{ pubkey, kysely, store }: RefreshAuthorStatsOpts,
): Promise<DittoTables['author_stats']> {
const stats = await countAuthorStats(store, pubkey);
await kysely.insertInto('author_stats')
.values(stats)
.onConflict((oc) => oc.column('pubkey').doUpdateSet(stats))
.execute();
return stats;
}
const authorStatsSemaphore = new Semaphore(10);
const refreshedAuthors = new LRUCache<string, true>({ max: 1000 });
/** Calls `refreshAuthorStats` only once per author. */
export function refreshAuthorStatsDebounced(opts: RefreshAuthorStatsOpts): void {
if (refreshedAuthors.get(opts.pubkey)) {
return;
}
refreshedAuthors.set(opts.pubkey, true);
authorStatsSemaphore
.lock(() => refreshAuthorStats(opts).catch(() => {}));
}

15
src/views/mastodon/statuses.ts
View File

@ -82,6 +82,15 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
const media = imeta.length ? imeta : getMediaLinks(links);
/** Pleroma emoji reactions object. */
const reactions = Object.entries(event.event_stats?.reactions ?? {}).reduce((acc, [emoji, count]) => {
if (['+', '-'].includes(emoji)) return acc;
acc.push({ name: emoji, count, me: reactionEvent?.content === emoji });
return acc;
}, [] as { name: string; count: number; me: boolean }[]);
const expiresAt = new Date(Number(event.tags.find(([name]) => name === 'expiration')?.[1]) * 1000);
return {
id: event.id,
account,
@ -96,7 +105,7 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
language: event.tags.find((tag) => tag[0] === 'lang')?.[1] || null,
replies_count: event.event_stats?.replies_count ?? 0,
reblogs_count: event.event_stats?.reposts_count ?? 0,
favourites_count: event.event_stats?.reactions_count ?? 0,
favourites_count: event.event_stats?.reactions['+'] ?? 0,
favourited: reactionEvent?.content === '+',
reblogged: Boolean(repostEvent),
muted: false,
@ -114,6 +123,10 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise<
uri: Conf.external(note),
url: Conf.external(note),
zapped: Boolean(zapEvent),
pleroma: {
emoji_reactions: reactions,
expires_at: !isNaN(expiresAt.getTime()) ? expiresAt.toISOString() : undefined,
},
};
}