From 8e68d13ff15ac5744a341a2d022ef8383085d0eb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 15 May 2024 18:49:08 -0500 Subject: [PATCH 1/6] Let custom policy be configured with DITTO_POLICY --- src/config.ts | 4 ++++ src/pipeline.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index 0943736..92bbdc3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -219,6 +219,10 @@ class Conf { static get firehoseEnabled(): boolean { return optionalBooleanSchema.parse(Deno.env.get('FIREHOSE_ENABLED')) ?? true; } + /** Path to the custom policy module. Supports any value Deno's `import()` accepts, including relative path, absolute path, https:, npm:, and jsr:. */ + static get policy(): string { + return Deno.env.get('DITTO_POLICY') || new URL('../data/policy.ts', import.meta.url).toString(); + } } const optionalBooleanSchema = z diff --git a/src/pipeline.ts b/src/pipeline.ts index ec14179..83e3923 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -60,7 +60,7 @@ async function policyFilter(event: NostrEvent): Promise { ]; try { - const CustomPolicy = (await import('../data/policy.ts')).default; + const CustomPolicy = (await import(Conf.policy)).default; policies.push(new CustomPolicy()); } catch (_e) { debug('policy not found - https://docs.soapbox.pub/ditto/policies/'); From 8a672c93ecb94306b941e6776e04fcc8e2d754a2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 15 May 2024 18:53:30 -0500 Subject: [PATCH 2/6] Debug custom policies with ditto:policy --- src/config.ts | 2 +- src/pipeline.ts | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/config.ts b/src/config.ts index 92bbdc3..589386f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -219,7 +219,7 @@ class Conf { static get firehoseEnabled(): boolean { return optionalBooleanSchema.parse(Deno.env.get('FIREHOSE_ENABLED')) ?? true; } - /** Path to the custom policy module. Supports any value Deno's `import()` accepts, including relative path, absolute path, https:, npm:, and jsr:. */ + /** Path to the custom policy module. Must be an absolute path, https:, npm:, or jsr: URI. */ static get policy(): string { return Deno.env.get('DITTO_POLICY') || new URL('../data/policy.ts', import.meta.url).toString(); } diff --git a/src/pipeline.ts b/src/pipeline.ts index 83e3923..6162530 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -2,6 +2,7 @@ import { NKinds, NostrEvent, NPolicy, NSchema as n } from '@nostrify/nostrify'; import { LNURL } from '@nostrify/nostrify/ln'; import { PipePolicy } from '@nostrify/nostrify/policies'; import Debug from '@soapbox/stickynotes/debug'; +import { Stickynotes } from '@soapbox/stickynotes'; import { sql } from 'kysely'; import { Conf } from '@/config.ts'; @@ -55,6 +56,8 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { + const console = new Stickynotes('ditto:policy'); + const policies: NPolicy[] = [ new MuteListPolicy(Conf.pubkey, await Storages.admin()), ]; @@ -62,14 +65,16 @@ async function policyFilter(event: NostrEvent): Promise { try { const CustomPolicy = (await import(Conf.policy)).default; policies.push(new CustomPolicy()); - } catch (_e) { - debug('policy not found - https://docs.soapbox.pub/ditto/policies/'); + console.info(`Using custom policy: ${Conf.policy}`); + } catch { + console.info('Custom policy not found '); } const policy = new PipePolicy(policies.reverse()); const result = await policy.call(event); - debug(JSON.stringify(result)); + console.debug(JSON.stringify(result)); + RelayError.assert(result); } From 6a1b8b0943606f98ecc9b8c84757159d43058841 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 15 May 2024 19:29:58 -0500 Subject: [PATCH 3/6] policy: improve error handling --- src/pipeline.ts | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 6162530..52cb563 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -2,7 +2,6 @@ import { NKinds, NostrEvent, NPolicy, NSchema as n } from '@nostrify/nostrify'; import { LNURL } from '@nostrify/nostrify/ln'; import { PipePolicy } from '@nostrify/nostrify/policies'; import Debug from '@soapbox/stickynotes/debug'; -import { Stickynotes } from '@soapbox/stickynotes'; import { sql } from 'kysely'; import { Conf } from '@/config.ts'; @@ -56,7 +55,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { - const console = new Stickynotes('ditto:policy'); + const debug = Debug('ditto:policy'); const policies: NPolicy[] = [ new MuteListPolicy(Conf.pubkey, await Storages.admin()), @@ -65,17 +64,30 @@ async function policyFilter(event: NostrEvent): Promise { try { const CustomPolicy = (await import(Conf.policy)).default; policies.push(new CustomPolicy()); - console.info(`Using custom policy: ${Conf.policy}`); - } catch { - console.info('Custom policy not found '); + debug(`Using custom policy: ${Conf.policy}`); + } catch (e) { + if (e.code === 'ERR_MODULE_NOT_FOUND') { + debug('Custom policy not found '); + } else { + console.error(`DITTO_POLICY (error importing policy): ${Conf.policy}`, e); + throw new RelayError('blocked', 'policy could not be loaded'); + } } const policy = new PipePolicy(policies.reverse()); - const result = await policy.call(event); - console.debug(JSON.stringify(result)); - - RelayError.assert(result); + try { + const result = await policy.call(event); + debug(JSON.stringify(result)); + RelayError.assert(result); + } catch (e) { + if (e instanceof RelayError) { + throw e; + } else { + console.error('POLICY ERROR:', e); + throw new RelayError('blocked', 'policy error'); + } + } } /** Encounter the event, and return whether it has already been encountered. */ From 9e9ab4088609f08703711d8443b51c34aabb2e14 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 15 May 2024 20:19:49 -0500 Subject: [PATCH 4/6] Run the custom policy in a worker for security --- Dockerfile | 2 +- data/.gitignore | 3 ++- data/policy/.gitignore | 2 ++ deno.json | 2 +- src/config.ts | 2 +- src/pipeline.ts | 7 ++++--- src/workers/policy.ts | 23 +++++++++++++++++++++++ src/workers/policy.worker.ts | 19 +++++++++++++++++++ 8 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 data/policy/.gitignore create mode 100644 src/workers/policy.ts create mode 100644 src/workers/policy.worker.ts diff --git a/Dockerfile b/Dockerfile index f8df815..a0e2194 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM denoland/deno:1.43.3 EXPOSE 4036 WORKDIR /app -RUN mkdir -p data && chown -R deno data +RUN mkdir -p data/policy && chown -R deno data USER deno COPY . . RUN deno cache src/server.ts diff --git a/data/.gitignore b/data/.gitignore index c96a04f..3c46d84 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -1,2 +1,3 @@ * -!.gitignore \ No newline at end of file +!.gitignore +!/policy \ No newline at end of file diff --git a/data/policy/.gitignore b/data/policy/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/data/policy/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/deno.json b/deno.json index 1ead2b9..399f474 100644 --- a/deno.json +++ b/deno.json @@ -13,7 +13,7 @@ "admin:role": "deno run -A scripts/admin-role.ts", "stats:recompute": "deno run -A scripts/stats-recompute.ts" }, - "unstable": ["ffi", "kv"], + "unstable": ["ffi", "kv", "worker-options"], "exclude": ["./public"], "imports": { "@/": "./src/", diff --git a/src/config.ts b/src/config.ts index 589386f..5a30c0e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -221,7 +221,7 @@ class Conf { } /** Path to the custom policy module. Must be an absolute path, https:, npm:, or jsr: URI. */ static get policy(): string { - return Deno.env.get('DITTO_POLICY') || new URL('../data/policy.ts', import.meta.url).toString(); + return Deno.env.get('DITTO_POLICY') || new URL('../data/policy.ts', import.meta.url).pathname; } } diff --git a/src/pipeline.ts b/src/pipeline.ts index 52cb563..6f487ed 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -16,6 +16,7 @@ import { Storages } from '@/storages.ts'; import { getTagSet } from '@/tags.ts'; import { eventAge, nostrDate, nostrNow, parseNip05, Time } from '@/utils.ts'; import { fetchWorker } from '@/workers/fetch.ts'; +import { policyWorker } from '@/workers/policy.ts'; import { TrendsWorker } from '@/workers/trends.ts'; import { verifyEventWorker } from '@/workers/verify.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; @@ -62,11 +63,11 @@ async function policyFilter(event: NostrEvent): Promise { ]; try { - const CustomPolicy = (await import(Conf.policy)).default; - policies.push(new CustomPolicy()); + await policyWorker.import(Conf.policy); + policies.push(policyWorker); debug(`Using custom policy: ${Conf.policy}`); } catch (e) { - if (e.code === 'ERR_MODULE_NOT_FOUND') { + if (e.message.includes('Module not found')) { debug('Custom policy not found '); } else { console.error(`DITTO_POLICY (error importing policy): ${Conf.policy}`, e); diff --git a/src/workers/policy.ts b/src/workers/policy.ts new file mode 100644 index 0000000..3cc03c9 --- /dev/null +++ b/src/workers/policy.ts @@ -0,0 +1,23 @@ +import * as Comlink from 'comlink'; + +import { Conf } from '@/config.ts'; +import type { CustomPolicy } from '@/workers/policy.worker.ts'; + +const policyDir = new URL('../../data/policy', import.meta.url).pathname; + +export const policyWorker = Comlink.wrap( + new Worker( + new URL('./policy.worker.ts', import.meta.url), + { + type: 'module', + deno: { + permissions: { + read: [Conf.policy, policyDir], + write: [policyDir], + net: 'inherit', + env: false, + }, + }, + }, + ), +); diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts new file mode 100644 index 0000000..146a116 --- /dev/null +++ b/src/workers/policy.worker.ts @@ -0,0 +1,19 @@ +import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; +import { ReadOnlyPolicy } from '@nostrify/nostrify/policies'; +import * as Comlink from 'comlink'; + +export class CustomPolicy implements NPolicy { + private policy: NPolicy = new ReadOnlyPolicy(); + + // deno-lint-ignore require-await + async call(event: NostrEvent): Promise { + return this.policy.call(event); + } + + async import(path: string): Promise { + const Policy = (await import(path)).default; + this.policy = new Policy(); + } +} + +Comlink.expose(new CustomPolicy()); From 0b6b62f3b38ef5b1065d1b5e6a3ca9644a8bb72f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 15 May 2024 20:27:54 -0500 Subject: [PATCH 5/6] policyWorker: import deno-safe-fetch --- deno.json | 1 + src/deps.ts | 2 +- src/workers/policy.worker.ts | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 399f474..c8eba2a 100644 --- a/deno.json +++ b/deno.json @@ -34,6 +34,7 @@ "@std/media-types": "jsr:@std/media-types@^0.224.0", "@std/streams": "jsr:@std/streams@^0.223.0", "comlink": "npm:comlink@^4.4.1", + "deno-safe-fetch": "https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts", "fast-stable-stringify": "npm:fast-stable-stringify@^1.0.0", "formdata-helper": "npm:formdata-helper@^0.3.0", "hono": "https://deno.land/x/hono@v3.10.1/mod.ts", diff --git a/src/deps.ts b/src/deps.ts index 7a8fa9a..46d8fec 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -1,4 +1,4 @@ -import 'https://gitlab.com/soapbox-pub/deno-safe-fetch/-/raw/v1.0.0/load.ts'; +import 'deno-safe-fetch'; // @deno-types="npm:@types/lodash@4.14.194" export { default as lodash } from 'https://esm.sh/lodash@4.17.21'; // @deno-types="npm:@types/mime@3.0.0" diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 146a116..4e4bcae 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -1,3 +1,4 @@ +import 'deno-safe-fetch'; import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { ReadOnlyPolicy } from '@nostrify/nostrify/policies'; import * as Comlink from 'comlink'; From f14b64b003fd2849a1b59373a7d7837e1488fe2f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 15 May 2024 20:35:00 -0500 Subject: [PATCH 6/6] Remove useless policy dir --- Dockerfile | 2 +- data/.gitignore | 3 +-- data/policy/.gitignore | 2 -- src/workers/policy.ts | 6 ++---- 4 files changed, 4 insertions(+), 9 deletions(-) delete mode 100644 data/policy/.gitignore diff --git a/Dockerfile b/Dockerfile index a0e2194..f8df815 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM denoland/deno:1.43.3 EXPOSE 4036 WORKDIR /app -RUN mkdir -p data/policy && chown -R deno data +RUN mkdir -p data && chown -R deno data USER deno COPY . . RUN deno cache src/server.ts diff --git a/data/.gitignore b/data/.gitignore index 3c46d84..c96a04f 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -1,3 +1,2 @@ * -!.gitignore -!/policy \ No newline at end of file +!.gitignore \ No newline at end of file diff --git a/data/policy/.gitignore b/data/policy/.gitignore deleted file mode 100644 index c96a04f..0000000 --- a/data/policy/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore \ No newline at end of file diff --git a/src/workers/policy.ts b/src/workers/policy.ts index 3cc03c9..e392667 100644 --- a/src/workers/policy.ts +++ b/src/workers/policy.ts @@ -3,8 +3,6 @@ import * as Comlink from 'comlink'; import { Conf } from '@/config.ts'; import type { CustomPolicy } from '@/workers/policy.worker.ts'; -const policyDir = new URL('../../data/policy', import.meta.url).pathname; - export const policyWorker = Comlink.wrap( new Worker( new URL('./policy.worker.ts', import.meta.url), @@ -12,8 +10,8 @@ export const policyWorker = Comlink.wrap( type: 'module', deno: { permissions: { - read: [Conf.policy, policyDir], - write: [policyDir], + read: [Conf.policy], + write: false, net: 'inherit', env: false, },