From 02e1a4ce589b22ecee3b7aed20e5836e10423379 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 30 Aug 2023 12:04:45 -0500 Subject: [PATCH 01/12] db/events: support "search" filter --- src/db.ts | 8 +++++++- src/db/events.ts | 10 ++++++++++ src/db/migrations/002_events_fts.ts | 9 +++++++++ src/deps.ts | 5 ++++- 4 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 src/db/migrations/002_events_fts.ts diff --git a/src/db.ts b/src/db.ts index 474652c..9f497b2 100644 --- a/src/db.ts +++ b/src/db.ts @@ -6,6 +6,7 @@ import { Conf } from '@/config.ts'; interface DittoDB { events: EventRow; + events_fts: EventFTSRow; tags: TagRow; users: UserRow; relays: RelayRow; @@ -21,6 +22,11 @@ interface EventRow { sig: string; } +interface EventFTSRow { + id: string; + content: string; +} + interface TagRow { tag: string; value_1: string | null; @@ -43,7 +49,7 @@ interface RelayRow { const db = new Kysely({ dialect: new DenoSqliteDialect({ - database: new Sqlite(Conf.dbPath), + database: new Sqlite(Conf.dbPath) as any, }), }); diff --git a/src/db/events.ts b/src/db/events.ts index fdd04c0..0bedccf 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -25,6 +25,10 @@ function insertEvent(event: Event): Promise { }) .execute(); + await trx.insertInto('events_fts') + .values({ id: event.id, content: event.content }) + .execute(); + const tagCounts: Record = {}; const tags = event.tags.reduce[]>((results, tag) => { const tagName = tag[0]; @@ -111,6 +115,12 @@ function getFilterQuery(filter: DittoFilter) { query = query.innerJoin('users', 'users.pubkey', 'events.pubkey'); } + if (filter.search) { + query = query + .innerJoin('events_fts', 'events_fts.id', 'events.id') + .where('events_fts.content', 'match', filter.search); + } + return query; } diff --git a/src/db/migrations/002_events_fts.ts b/src/db/migrations/002_events_fts.ts new file mode 100644 index 0000000..c0341c1 --- /dev/null +++ b/src/db/migrations/002_events_fts.ts @@ -0,0 +1,9 @@ +import { Kysely, sql } from '@/deps.ts'; + +export async function up(db: Kysely): Promise { + await sql`CREATE VIRTUAL TABLE events_fts USING fts5(id, content)`.execute(db); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('events_fts').execute(); +} diff --git a/src/deps.ts b/src/deps.ts index dcef225..31104ac 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -50,7 +50,10 @@ export { export { generateSeededRsa } from 'https://gitlab.com/soapbox-pub/seeded-rsa/-/raw/v1.0.0/mod.ts'; export * as secp from 'npm:@noble/secp256k1@^2.0.0'; export { LRUCache } from 'npm:lru-cache@^10.0.0'; -export { DB as Sqlite, SqliteError } from 'https://deno.land/x/sqlite@v3.7.3/mod.ts'; +export { + DB as Sqlite, + SqliteError, +} from 'https://raw.githubusercontent.com/teleclimber/deno-sqlite/5283320fce74fbfd90b62d379e8703d386ed0b27/mod.ts'; export * as dotenv from 'https://deno.land/std@0.198.0/dotenv/mod.ts'; export { FileMigrationProvider, From 5e9a3dd8d17d6e317a049689ab4aab2121802932 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 30 Aug 2023 12:07:29 -0500 Subject: [PATCH 02/12] db/events: only index kind 1 events in search --- src/db/events.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index 0bedccf..59d831b 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -25,9 +25,11 @@ function insertEvent(event: Event): Promise { }) .execute(); - await trx.insertInto('events_fts') - .values({ id: event.id, content: event.content }) - .execute(); + if (event.kind === 1) { + await trx.insertInto('events_fts') + .values({ id: event.id, content: event.content }) + .execute(); + } const tagCounts: Record = {}; const tags = event.tags.reduce[]>((results, tag) => { From df14ff66bc4642d34f3757766c0c286c53c21650 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 30 Aug 2023 12:16:04 -0500 Subject: [PATCH 03/12] search: search use FTS to search for statuses --- src/controllers/api/search.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 45193f3..c7123bc 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -1,6 +1,7 @@ import { AppController } from '@/app.ts'; +import * as eventsDB from '@/db/events.ts'; import { lookupAccount } from '@/utils.ts'; -import { toAccount } from '@/transformers/nostr-to-mastoapi.ts'; +import { toAccount, toStatus } from '@/transformers/nostr-to-mastoapi.ts'; const searchController: AppController = async (c) => { const q = c.req.query('q'); @@ -13,9 +14,12 @@ const searchController: AppController = async (c) => { // TODO: Support searching statuses and hashtags. const event = await lookupAccount(decodeURIComponent(q)); + const events = await eventsDB.getFilters([{ kinds: [1], search: q }]); + const statuses = await Promise.all(events.map((event) => toStatus(event, c.get('pubkey')))); + return c.json({ accounts: event ? [await toAccount(event)] : [], - statuses: [], + statuses: statuses.filter(Boolean), hashtags: [], }); }; From 675010ddecc2b320784da59ffdf6ca6eff348b39 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 30 Aug 2023 14:03:16 -0500 Subject: [PATCH 04/12] search: fix FTS special characters, optimize search --- src/controllers/api/search.ts | 57 ++++++++++++++++++++++++++++++----- src/db/events.ts | 2 +- src/mixer.ts | 7 +---- src/utils.ts | 6 ++++ 4 files changed, 58 insertions(+), 14 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index c7123bc..674d9f1 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -1,27 +1,70 @@ import { AppController } from '@/app.ts'; import * as eventsDB from '@/db/events.ts'; -import { lookupAccount } from '@/utils.ts'; +import { type Event, nip05, nip19 } from '@/deps.ts'; +import * as mixer from '@/mixer.ts'; +import { lookupNip05Cached } from '@/nip05.ts'; +import { getAuthor } from '@/queries.ts'; import { toAccount, toStatus } from '@/transformers/nostr-to-mastoapi.ts'; +import { bech32ToPubkey, dedupeEvents, Time } from '@/utils.ts'; +import { paginationSchema } from '@/utils/web.ts'; const searchController: AppController = async (c) => { const q = c.req.query('q'); + const params = paginationSchema.parse(c.req.query()); if (!q) { return c.json({ error: 'Missing `q` query parameter.' }, 422); } - // For now, only support looking up accounts. - // TODO: Support searching statuses and hashtags. - const event = await lookupAccount(decodeURIComponent(q)); + const [event, events] = await Promise.all([ + lookupEvent(decodeURIComponent(q)), + eventsDB.getFilters([{ kinds: [1], search: q, ...params }]), + ]); - const events = await eventsDB.getFilters([{ kinds: [1], search: q }]); - const statuses = await Promise.all(events.map((event) => toStatus(event, c.get('pubkey')))); + if (event) { + events.push(event); + } + + const results = dedupeEvents(events); + + const accounts = await Promise.all( + results + .filter((event): event is Event<0> => event.kind === 0) + .map((event) => toAccount(event)), + ); + + const statuses = await Promise.all( + results + .filter((event): event is Event<1> => event.kind === 1) + .map((event) => toStatus(event, c.get('pubkey'))), + ); return c.json({ - accounts: event ? [await toAccount(event)] : [], + accounts: accounts.filter(Boolean), statuses: statuses.filter(Boolean), hashtags: [], }); }; +/** Resolve a searched value into an event, if applicable. */ +async function lookupEvent(value: string): Promise | undefined> { + if (new RegExp(`^${nip19.BECH32_REGEX.source}$`).test(value)) { + const pubkey = bech32ToPubkey(value); + if (pubkey) { + return getAuthor(pubkey); + } + } else if (/^[0-9a-f]{64}$/.test(value)) { + const [event] = await mixer.getFilters( + [{ kinds: [0], authors: [value], limit: 1 }, { kinds: [1], ids: [value], limit: 1 }], + { limit: 1, timeout: Time.seconds(1) }, + ); + return event; + } else if (nip05.NIP05_REGEX.test(value)) { + const pubkey = await lookupNip05Cached(value); + if (pubkey) { + return getAuthor(pubkey); + } + } +} + export { searchController }; diff --git a/src/db/events.ts b/src/db/events.ts index 59d831b..e33298c 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -120,7 +120,7 @@ function getFilterQuery(filter: DittoFilter) { if (filter.search) { query = query .innerJoin('events_fts', 'events_fts.id', 'events.id') - .where('events_fts.content', 'match', filter.search); + .where('events_fts.content', 'match', JSON.stringify(filter.search)); } return query; diff --git a/src/mixer.ts b/src/mixer.ts index 4c160b3..9d18939 100644 --- a/src/mixer.ts +++ b/src/mixer.ts @@ -2,7 +2,7 @@ import { type Event, matchFilters } from '@/deps.ts'; import * as client from '@/client.ts'; import * as eventsDB from '@/db/events.ts'; -import { eventDateComparator } from '@/utils.ts'; +import { dedupeEvents, eventDateComparator } from '@/utils.ts'; import type { DittoFilter, GetFiltersOpts } from '@/filter.ts'; @@ -33,11 +33,6 @@ function unmixEvents(events: Event[], filters: DittoFilter< return events; } -/** Deduplicate events by ID. */ -function dedupeEvents(events: Event[]): Event[] { - return [...new Map(events.map((event) => [event.id, event])).values()]; -} - /** Take the newest events among replaceable ones. */ function takeNewestEvents(events: Event[]): Event[] { const isReplaceable = (kind: number) => diff --git a/src/utils.ts b/src/utils.ts index 7f47f31..d6d24d8 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -101,8 +101,14 @@ function isFollowing(source: Event<3>, targetPubkey: string): boolean { ); } +/** Deduplicate events by ID. */ +function dedupeEvents(events: Event[]): Event[] { + return [...new Map(events.map((event) => [event.id, event])).values()]; +} + export { bech32ToPubkey, + dedupeEvents, eventAge, eventDateComparator, findTag, From 8079679f18506f8a8ed90cdb2a585ca878e7078b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 30 Aug 2023 15:02:28 -0500 Subject: [PATCH 05/12] search: support MastoAPI params, improve performance, improve value lookup --- src/controllers/api/search.ts | 123 ++++++++++++++++++++++++---------- src/mixer.ts | 2 + 2 files changed, 90 insertions(+), 35 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 674d9f1..1adc266 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -1,24 +1,38 @@ import { AppController } from '@/app.ts'; import * as eventsDB from '@/db/events.ts'; -import { type Event, nip05, nip19 } from '@/deps.ts'; +import { type Event, type Filter, nip19, z } from '@/deps.ts'; import * as mixer from '@/mixer.ts'; import { lookupNip05Cached } from '@/nip05.ts'; -import { getAuthor } from '@/queries.ts'; +import { booleanParamSchema } from '@/schema.ts'; import { toAccount, toStatus } from '@/transformers/nostr-to-mastoapi.ts'; -import { bech32ToPubkey, dedupeEvents, Time } from '@/utils.ts'; -import { paginationSchema } from '@/utils/web.ts'; +import { dedupeEvents, Time } from '@/utils.ts'; + +/** Matches NIP-05 names with or without an @ in front. */ +const ACCT_REGEX = /^@?(?:([\w.+-]+)@)?([\w.-]+)$/; + +const searchQuerySchema = z.object({ + q: z.string().transform(decodeURIComponent), + type: z.enum(['accounts', 'statuses', 'hashtags']).optional(), + resolve: booleanParamSchema.optional().transform(Boolean), + following: z.boolean().default(false), + account_id: z.string().optional(), + limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)), +}); + +type SearchQuery = z.infer; const searchController: AppController = async (c) => { - const q = c.req.query('q'); - const params = paginationSchema.parse(c.req.query()); + const result = searchQuerySchema.safeParse(c.req.query()); - if (!q) { - return c.json({ error: 'Missing `q` query parameter.' }, 422); + if (!result.success) { + return c.json({ error: 'Bad request', schema: result.error }, 422); } + const { q, type, limit } = result.data; + const [event, events] = await Promise.all([ - lookupEvent(decodeURIComponent(q)), - eventsDB.getFilters([{ kinds: [1], search: q, ...params }]), + lookupEvent(result.data), + !type || type === 'statuses' ? eventsDB.getFilters([{ kinds: [1], search: q, limit }]) : [] as Event[], ]); if (event) { @@ -27,17 +41,18 @@ const searchController: AppController = async (c) => { const results = dedupeEvents(events); - const accounts = await Promise.all( - results - .filter((event): event is Event<0> => event.kind === 0) - .map((event) => toAccount(event)), - ); - - const statuses = await Promise.all( - results - .filter((event): event is Event<1> => event.kind === 1) - .map((event) => toStatus(event, c.get('pubkey'))), - ); + const [accounts, statuses] = await Promise.all([ + Promise.all( + results + .filter((event): event is Event<0> => event.kind === 0) + .map((event) => toAccount(event)), + ), + Promise.all( + results + .filter((event): event is Event<1> => event.kind === 1) + .map((event) => toStatus(event, c.get('pubkey'))), + ), + ]); return c.json({ accounts: accounts.filter(Boolean), @@ -47,24 +62,62 @@ const searchController: AppController = async (c) => { }; /** Resolve a searched value into an event, if applicable. */ -async function lookupEvent(value: string): Promise | undefined> { - if (new RegExp(`^${nip19.BECH32_REGEX.source}$`).test(value)) { - const pubkey = bech32ToPubkey(value); - if (pubkey) { - return getAuthor(pubkey); +async function lookupEvent(query: SearchQuery): Promise { + const filters = await getLookupFilters(query); + const [event] = await mixer.getFilters(filters, { limit: 1, timeout: Time.seconds(1) }); + return event; +} + +/** Get filters to lookup the input value. */ +async function getLookupFilters({ q, type, resolve }: SearchQuery): Promise { + const filters: Filter[] = []; + + if (!resolve || type === 'hashtags') { + return filters; + } + + if (new RegExp(`^${nip19.BECH32_REGEX.source}$`).test(q)) { + try { + const result = nip19.decode(q); + switch (result.type) { + case 'npub': + filters.push({ kinds: [0], authors: [result.data] }); + break; + case 'nprofile': + filters.push({ kinds: [0], authors: [result.data.pubkey] }); + break; + case 'note': + filters.push({ kinds: [1], ids: [result.data] }); + break; + case 'nevent': + filters.push({ kinds: [1], ids: [result.data.id] }); + break; + } + } catch (_e) { + // do nothing } - } else if (/^[0-9a-f]{64}$/.test(value)) { - const [event] = await mixer.getFilters( - [{ kinds: [0], authors: [value], limit: 1 }, { kinds: [1], ids: [value], limit: 1 }], - { limit: 1, timeout: Time.seconds(1) }, - ); - return event; - } else if (nip05.NIP05_REGEX.test(value)) { - const pubkey = await lookupNip05Cached(value); + } else if (/^[0-9a-f]{64}$/.test(q)) { + filters.push({ kinds: [0], authors: [q] }); + filters.push({ kinds: [1], ids: [q] }); + } else if ((!type || type === 'accounts') && ACCT_REGEX.test(q)) { + const pubkey = await lookupNip05Cached(q); if (pubkey) { - return getAuthor(pubkey); + filters.push({ kinds: [0], authors: [pubkey] }); } } + + if (!type) { + return filters; + } + + return filters.filter(({ kinds }) => { + switch (type) { + case 'accounts': + return kinds?.every((kind) => kind === 0); + case 'statuses': + return kinds?.every((kind) => kind === 1); + } + }); } export { searchController }; diff --git a/src/mixer.ts b/src/mixer.ts index 9d18939..c6788ff 100644 --- a/src/mixer.ts +++ b/src/mixer.ts @@ -11,6 +11,8 @@ async function getFilters( filters: DittoFilter[], opts?: GetFiltersOpts, ): Promise[]> { + if (!filters.length) return Promise.resolve([]); + const results = await Promise.allSettled([ client.getFilters(filters.filter((filter) => !filter.local), opts), eventsDB.getFilters(filters, opts), From 799a9c1e9a6a87c7e7582f7866cf02a5c0b3495b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 30 Aug 2023 15:22:39 -0500 Subject: [PATCH 06/12] search: support searching by account_id --- src/controllers/api/search.ts | 47 +++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 1adc266..a61c0bc 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -28,11 +28,24 @@ const searchController: AppController = async (c) => { return c.json({ error: 'Bad request', schema: result.error }, 422); } - const { q, type, limit } = result.data; + const { q, type, limit, account_id } = result.data; + + const searchAccounts = !type || type === 'accounts'; + const searchStatuses = !type || type === 'statuses'; + + const filter: Filter = { + kinds: searchAccounts ? [0] : [1], + search: q, + limit, + }; + + if (account_id) { + filter.authors = [account_id]; + } const [event, events] = await Promise.all([ lookupEvent(result.data), - !type || type === 'statuses' ? eventsDB.getFilters([{ kinds: [1], search: q, limit }]) : [] as Event[], + searchStatuses ? eventsDB.getFilters([filter]) : [] as Event[], ]); if (event) { @@ -72,6 +85,9 @@ async function lookupEvent(query: SearchQuery): Promise { async function getLookupFilters({ q, type, resolve }: SearchQuery): Promise { const filters: Filter[] = []; + const accounts = !type || type === 'accounts'; + const statuses = !type || type === 'statuses'; + if (!resolve || type === 'hashtags') { return filters; } @@ -81,43 +97,32 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery): Promise { - switch (type) { - case 'accounts': - return kinds?.every((kind) => kind === 0); - case 'statuses': - return kinds?.every((kind) => kind === 1); - } - }); + return filters; } export { searchController }; From 080cfe817b2157044e631df5d2cf29673ea9ec3b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 30 Aug 2023 15:31:07 -0500 Subject: [PATCH 07/12] search: support account_id query param --- src/controllers/api/search.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index a61c0bc..0f926ce 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -4,6 +4,7 @@ import { type Event, type Filter, nip19, z } from '@/deps.ts'; import * as mixer from '@/mixer.ts'; import { lookupNip05Cached } from '@/nip05.ts'; import { booleanParamSchema } from '@/schema.ts'; +import { nostrIdSchema } from '@/schemas/nostr.ts'; import { toAccount, toStatus } from '@/transformers/nostr-to-mastoapi.ts'; import { dedupeEvents, Time } from '@/utils.ts'; @@ -15,7 +16,7 @@ const searchQuerySchema = z.object({ type: z.enum(['accounts', 'statuses', 'hashtags']).optional(), resolve: booleanParamSchema.optional().transform(Boolean), following: z.boolean().default(false), - account_id: z.string().optional(), + account_id: nostrIdSchema.optional(), limit: z.coerce.number().catch(20).transform((value) => Math.min(Math.max(value, 0), 40)), }); @@ -30,11 +31,8 @@ const searchController: AppController = async (c) => { const { q, type, limit, account_id } = result.data; - const searchAccounts = !type || type === 'accounts'; - const searchStatuses = !type || type === 'statuses'; - const filter: Filter = { - kinds: searchAccounts ? [0] : [1], + kinds: typeToKinds(type), search: q, limit, }; @@ -45,7 +43,7 @@ const searchController: AppController = async (c) => { const [event, events] = await Promise.all([ lookupEvent(result.data), - searchStatuses ? eventsDB.getFilters([filter]) : [] as Event[], + (!type || type === 'statuses') ? eventsDB.getFilters([filter]) : [] as Event[], ]); if (event) { @@ -74,6 +72,18 @@ const searchController: AppController = async (c) => { }); }; +/** Get event kinds to search from `type` query param. */ +function typeToKinds(type: SearchQuery['type']): number[] { + switch (type) { + case 'accounts': + return [0]; + case 'statuses': + return [1]; + default: + return [0, 1]; + } +} + /** Resolve a searched value into an event, if applicable. */ async function lookupEvent(query: SearchQuery): Promise { const filters = await getLookupFilters(query); From b7b5e67118ec1e495c685fed4185cdba540c6c6f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 30 Aug 2023 15:40:18 -0500 Subject: [PATCH 08/12] search: refactor/cleanup searchController --- src/controllers/api/search.ts | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 0f926ce..6ba6183 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -29,21 +29,9 @@ const searchController: AppController = async (c) => { return c.json({ error: 'Bad request', schema: result.error }, 422); } - const { q, type, limit, account_id } = result.data; - - const filter: Filter = { - kinds: typeToKinds(type), - search: q, - limit, - }; - - if (account_id) { - filter.authors = [account_id]; - } - const [event, events] = await Promise.all([ lookupEvent(result.data), - (!type || type === 'statuses') ? eventsDB.getFilters([filter]) : [] as Event[], + searchEvents(result.data), ]); if (event) { @@ -72,6 +60,23 @@ const searchController: AppController = async (c) => { }); }; +/** Get events for the search params. */ +function searchEvents({ q, type, limit, account_id }: SearchQuery): Promise { + if (type === 'hashtags') return Promise.resolve([]); + + const filter: Filter = { + kinds: typeToKinds(type), + search: q, + limit, + }; + + if (account_id) { + filter.authors = [account_id]; + } + + return eventsDB.getFilters([filter]); +} + /** Get event kinds to search from `type` query param. */ function typeToKinds(type: SearchQuery['type']): number[] { switch (type) { From af76d8cbaeccf9bba4d1df3597a16495fe4d128a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 30 Aug 2023 16:12:39 -0500 Subject: [PATCH 09/12] Bump kysely-deno-sqlite to v1.0.1, remove `as any` from database type --- src/db.ts | 2 +- src/deps.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/db.ts b/src/db.ts index 9f497b2..d14a9bc 100644 --- a/src/db.ts +++ b/src/db.ts @@ -49,7 +49,7 @@ interface RelayRow { const db = new Kysely({ dialect: new DenoSqliteDialect({ - database: new Sqlite(Conf.dbPath) as any, + database: new Sqlite(Conf.dbPath), }), }); diff --git a/src/deps.ts b/src/deps.ts index 31104ac..a2c640a 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -63,7 +63,7 @@ export { type NullableInsertKeys, sql, } from 'npm:kysely@^0.25.0'; -export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/v1.0.0/mod.ts'; +export { DenoSqliteDialect } from 'https://gitlab.com/soapbox-pub/kysely-deno-sqlite/-/raw/v1.0.1/mod.ts'; export { default as tldts } from 'npm:tldts@^6.0.14'; export type * as TypeFest from 'npm:type-fest@^4.3.0'; From e976f51d550d5d237229ea4ccca47afb6aeede0e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 30 Aug 2023 16:19:21 -0500 Subject: [PATCH 10/12] schemas/nostr: add `search` key to filters --- src/schemas/nostr.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/schemas/nostr.ts b/src/schemas/nostr.ts index 3f9c902..6b5c30c 100644 --- a/src/schemas/nostr.ts +++ b/src/schemas/nostr.ts @@ -29,6 +29,7 @@ const filterSchema = z.object({ since: z.number().int().nonnegative().optional(), until: z.number().int().nonnegative().optional(), limit: z.number().int().nonnegative().optional(), + search: z.string().optional(), }).passthrough().and( z.record( z.custom<`#${string}`>((val) => typeof val === 'string' && val.startsWith('#')), From 3cd6b6f3f39e3a3ef19730d113e063500a3fc57e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 30 Aug 2023 17:09:44 -0500 Subject: [PATCH 11/12] db/events: index profiles in search --- src/db/events.ts | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/db/events.ts b/src/db/events.ts index e33298c..1b30f5b 100644 --- a/src/db/events.ts +++ b/src/db/events.ts @@ -2,6 +2,7 @@ import { db, type TagRow } from '@/db.ts'; import { type Event, type Insertable, SqliteError } from '@/deps.ts'; import type { DittoFilter, GetFiltersOpts } from '@/filter.ts'; +import { jsonMetaContentSchema } from '@/schemas/nostr.ts'; type TagCondition = ({ event, count }: { event: Event; count: number }) => boolean; @@ -25,9 +26,10 @@ function insertEvent(event: Event): Promise { }) .execute(); - if (event.kind === 1) { + const searchContent = buildSearchContent(event); + if (searchContent) { await trx.insertInto('events_fts') - .values({ id: event.id, content: event.content }) + .values({ id: event.id, content: searchContent.substring(0, 1000) }) .execute(); } @@ -143,6 +145,7 @@ async function getFilters( )); } +/** Get number of events that would be returned by filters. */ async function countFilters(filters: DittoFilter[]): Promise { if (!filters.length) return Promise.resolve(0); const query = filters.map(getFilterQuery).reduce((acc, curr) => acc.union(curr)); @@ -155,4 +158,22 @@ async function countFilters(filters: DittoFilter[]): Promis return Number(count); } +/** Build a search index from the event. */ +function buildSearchContent(event: Event): string { + switch (event.kind) { + case 0: + return buildUserSearchContent(event as Event<0>); + case 1: + return event.content; + default: + return ''; + } +} + +/** Build search content for a user. */ +function buildUserSearchContent(event: Event<0>): string { + const { name, nip05, about } = jsonMetaContentSchema.parse(event.content); + return [name, nip05, about].filter(Boolean).join('\n'); +} + export { countFilters, getFilters, insertEvent }; From b9fc663db482e360993b5f9a732432d53c6bf3c2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 30 Aug 2023 17:29:45 -0500 Subject: [PATCH 12/12] Switch to my fork of deno-sqlite with FTS support --- src/deps.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deps.ts b/src/deps.ts index a2c640a..b26bd8f 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -53,7 +53,7 @@ export { LRUCache } from 'npm:lru-cache@^10.0.0'; export { DB as Sqlite, SqliteError, -} from 'https://raw.githubusercontent.com/teleclimber/deno-sqlite/5283320fce74fbfd90b62d379e8703d386ed0b27/mod.ts'; +} from 'https://raw.githubusercontent.com/alexgleason/deno-sqlite/325f66d8c395e7f6f5ee78ebfa42a0eeea4a942b/mod.ts'; export * as dotenv from 'https://deno.land/std@0.198.0/dotenv/mod.ts'; export { FileMigrationProvider,