Compare commits

..

No commits in common. "spc2" and "streams-fixes" have entirely different histories.

150 changed files with 543 additions and 3923 deletions

View File

@ -1,8 +1,8 @@
image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 image: git.pleroma.social:5050/pleroma/pleroma/ci-base
variables: &global_variables variables: &global_variables
# Only used for the release # Only used for the release
ELIXIR_VER: 1.13.4 ELIXIR_VER: 1.12.3
POSTGRES_DB: pleroma_test POSTGRES_DB: pleroma_test
POSTGRES_USER: postgres POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres POSTGRES_PASSWORD: postgres
@ -72,7 +72,7 @@ check-changelog:
tags: tags:
- amd64 - amd64
build-1.13.4: build-1.12.3:
extends: extends:
- .build_changes_policy - .build_changes_policy
- .using-ci-base - .using-ci-base
@ -85,7 +85,7 @@ build-1.15.7-otp-25:
- .build_changes_policy - .build_changes_policy
- .using-ci-base - .using-ci-base
stage: build stage: build
image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15
allow_failure: true allow_failure: true
script: script:
- mix compile --force - mix compile --force

View File

@ -1,7 +1,7 @@
ARG ELIXIR_IMG=hexpm/elixir ARG ELIXIR_IMG=hexpm/elixir
ARG ELIXIR_VER=1.13.4 ARG ELIXIR_VER=1.12.3
ARG ERLANG_VER=24.3.4.15 ARG ERLANG_VER=24.2.1
ARG ALPINE_VER=3.17.5 ARG ALPINE_VER=3.17.0
FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build

View File

@ -1 +0,0 @@
HTTP Security: By default, don't allow unsafe-eval. The setting needs to be changed to allow Flash emulation.

View File

View File

@ -1 +0,0 @@
Uploader: Add support for uploading attachments using IPFS

View File

@ -1 +0,0 @@
Add NSFW-detecting MRF

View File

@ -1 +0,0 @@
Add DNSRBL MRF

View File

@ -1 +0,0 @@
Add options to the mix prune_objects task

View File

@ -1 +0,0 @@
Add Anti-mention Spam MRF backported from Rebased

View File

@ -1 +0,0 @@
HTTPSignaturePlug: Add :authorized_fetch_mode_exceptions configuration

View File

@ -1 +0,0 @@
Add an option to reject certain domains when authorized fetch is enabled.

View File

@ -1 +0,0 @@
Update Bandit to 1.5.2

View File

@ -1 +0,0 @@
Elixir 1.13 is the minimum required version.

View File

@ -1 +0,0 @@
Fix webfinger spoofing.

View File

@ -1 +0,0 @@
Includes scrobble when available in the account field with statuses

View File

@ -1 +0,0 @@
Logger metadata is now attached to some logs to help with troubleshooting and analysis

View File

@ -1 +0,0 @@
Ensure MediaProxy HTTP requests obey all the defined connection settings

View File

@ -1 +0,0 @@
Add missing indexes on foreign key relationships

View File

@ -1 +0,0 @@
Permit passing --chunk and --step values to the Pleroma.Search.Indexer Mix task

View File

@ -1 +0,0 @@
noop

View File

@ -1 +0,0 @@
Oban queues have refactored to simplify the queue design

View File

@ -1 +0,0 @@
HTTP connection pool adjustments

View File

@ -1 +0,0 @@
Update the documentation for configuring Prometheus metrics.

View File

@ -1 +0,0 @@
PromEx documentation

View File

@ -1 +0,0 @@
Add Qdrant/OpenAI embedding search

View File

@ -1 +0,0 @@
pleroma_ctl: Use realpath(1) instead of readlink(1)

View File

@ -1 +0,0 @@
A 422 error is returned when attempting to reply to a deleted status

View File

@ -1 +0,0 @@
Parsing of RichMedia TTLs for Amazon URLs when query parameters are nil

View File

@ -1 +0,0 @@
Monitoring of search backend health to control the processing of jobs in the search indexing Oban queue

View File

@ -1 +0,0 @@
Display reposted replies with exclude_replies: true

View File

@ -1 +0,0 @@
Support honk-style attachment summaries as alt-text.

View File

@ -1 +0,0 @@
Video thumbnails were not being generated due to a negative cache lookup logic error

View File

@ -1 +0,0 @@
Fix validate_webfinger when running a different domain for Webfinger

View File

@ -1,8 +0,0 @@
FROM elixir:1.13.4-otp-24
# Single RUN statement, otherwise intermediate images are created
# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
RUN apt-get update &&\
apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\
mix local.hex --force &&\
mix local.rebar --force

View File

@ -1 +0,0 @@
docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 --push .

View File

@ -1 +1 @@
docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push . docker buildx build --platform linux/amd64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push .

View File

@ -82,10 +82,6 @@
# region: "us-east-1", # may be required for Amazon AWS # region: "us-east-1", # may be required for Amazon AWS
scheme: "https://" scheme: "https://"
config :pleroma, Pleroma.Uploaders.IPFS,
post_gateway_url: nil,
get_gateway_url: nil
config :pleroma, :emoji, config :pleroma, :emoji,
shortcode_globs: ["/emoji/custom/**/*.png"], shortcode_globs: ["/emoji/custom/**/*.png"],
pack_extensions: [".png", ".gif"], pack_extensions: [".png", ".gif"],
@ -135,13 +131,13 @@
config :logger, :console, config :logger, :console,
level: :debug, level: :debug,
format: "\n$time $metadata[$level] $message\n", format: "\n$time $metadata[$level] $message\n",
metadata: [:actor, :path, :request_id, :type, :user] metadata: [:request_id]
config :logger, :ex_syslogger, config :logger, :ex_syslogger,
level: :debug, level: :debug,
ident: "pleroma", ident: "pleroma",
format: "$metadata[$level] $message", format: "$metadata[$level] $message",
metadata: [:actor, :path, :request_id, :type, :user] metadata: [:request_id]
config :mime, :types, %{ config :mime, :types, %{
"application/xml" => ["xml"], "application/xml" => ["xml"],
@ -192,7 +188,6 @@
allow_relay: true, allow_relay: true,
public: true, public: true,
quarantined_instances: [], quarantined_instances: [],
rejected_instances: [],
static_dir: "instance/static/", static_dir: "instance/static/",
allowed_post_formats: [ allowed_post_formats: [
"text/plain", "text/plain",
@ -411,23 +406,11 @@
accept: [], accept: [],
reject: [] reject: []
config :pleroma, :mrf_dnsrbl,
nameserver: "127.0.0.1",
port: 53,
zone: "bl.pleroma.com"
# threshold of 7 days # threshold of 7 days
config :pleroma, :mrf_object_age, config :pleroma, :mrf_object_age,
threshold: 604_800, threshold: 604_800,
actions: [:delist, :strip_followers] actions: [:delist, :strip_followers]
config :pleroma, :mrf_nsfw_api,
url: "http://127.0.0.1:5000/",
threshold: 0.7,
mark_sensitive: true,
unlist: false,
reject: false
config :pleroma, :mrf_follow_bot, follower_nickname: nil config :pleroma, :mrf_follow_bot, follower_nickname: nil
config :pleroma, :mrf_inline_quote, template: "<bdi>RT:</bdi> {url}" config :pleroma, :mrf_inline_quote, template: "<bdi>RT:</bdi> {url}"
@ -436,8 +419,6 @@
mention_parent: true, mention_parent: true,
mention_quoted: true mention_quoted: true
config :pleroma, :mrf_antimentionspam, user_age_limit: 30_000
config :pleroma, :rich_media, config :pleroma, :rich_media,
enabled: true, enabled: true,
ignore_hosts: [], ignore_hosts: [],
@ -520,8 +501,7 @@
sts: false, sts: false,
sts_max_age: 31_536_000, sts_max_age: 31_536_000,
ct_max_age: 2_592_000, ct_max_age: 2_592_000,
referrer_policy: "same-origin", referrer_policy: "same-origin"
allow_unsafe_eval: false
config :cors_plug, config :cors_plug,
max_age: 86_400, max_age: 86_400,
@ -583,14 +563,24 @@
log: false, log: false,
queues: [ queues: [
activity_expiration: 10, activity_expiration: 10,
token_expiration: 5,
filter_expiration: 1,
backup: 1,
federator_incoming: 5, federator_incoming: 5,
federator_outgoing: 5, federator_outgoing: 5,
ingestion_queue: 50, ingestion_queue: 50,
web_push: 50, web_push: 50,
mailer: 10,
transmogrifier: 20, transmogrifier: 20,
scheduled_activities: 10,
poll_notifications: 10,
background: 5, background: 5,
search_indexing: [limit: 10, paused: true], remote_fetcher: 2,
slow: 1 attachments_cleanup: 1,
new_users_digest: 1,
mute_expire: 5,
search_indexing: 10,
rich_media_expiration: 2
], ],
plugins: [Oban.Plugins.Pruner], plugins: [Oban.Plugins.Pruner],
crontab: [ crontab: [
@ -814,7 +804,7 @@
config :pleroma, configurable_from_database: false config :pleroma, configurable_from_database: false
config :pleroma, Pleroma.Repo, config :pleroma, Pleroma.Repo,
parameters: [gin_fuzzy_search_limit: "500"], parameters: [gin_fuzzy_search_limit: "500", jit: "off"],
prepare: :unnamed prepare: :unnamed
config :pleroma, :connections_pool, config :pleroma, :connections_pool,
@ -828,27 +818,22 @@
config :pleroma, :pools, config :pleroma, :pools,
federation: [ federation: [
size: 75, size: 50,
max_waiting: 20, max_waiting: 10,
recv_timeout: 10_000 recv_timeout: 10_000
], ],
media: [ media: [
size: 75, size: 50,
max_waiting: 20,
recv_timeout: 15_000
],
rich_media: [
size: 25,
max_waiting: 20, max_waiting: 20,
recv_timeout: 15_000 recv_timeout: 15_000
], ],
upload: [ upload: [
size: 25, size: 25,
max_waiting: 20, max_waiting: 5,
recv_timeout: 15_000 recv_timeout: 15_000
], ],
default: [ default: [
size: 50, size: 10,
max_waiting: 2, max_waiting: 2,
recv_timeout: 5_000 recv_timeout: 5_000
] ]
@ -862,10 +847,6 @@
max_connections: 50, max_connections: 50,
timeout: 150_000 timeout: 150_000
], ],
rich_media: [
max_connections: 50,
timeout: 150_000
],
upload: [ upload: [
max_connections: 25, max_connections: 25,
timeout: 300_000 timeout: 300_000
@ -911,6 +892,8 @@
process_chunk_size: 100 process_chunk_size: 100
config :pleroma, ConcurrentLimiter, [ config :pleroma, ConcurrentLimiter, [
{Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
{Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]},
{Pleroma.Search, [max_running: 30, max_waiting: 50]} {Pleroma.Search, [max_running: 30, max_waiting: 50]}
] ]
@ -932,19 +915,6 @@
config :pleroma, Pleroma.Uploaders.Uploader, timeout: 30_000 config :pleroma, Pleroma.Uploaders.Uploader, timeout: 30_000
config :pleroma, Pleroma.Search.QdrantSearch,
qdrant_url: "http://127.0.0.1:6333/",
qdrant_api_key: "",
openai_url: "http://127.0.0.1:11345",
# The healthcheck url has to be set to nil when used with the real openai
# API, as it doesn't have a healthcheck endpoint.
openai_healthcheck_url: "http://127.0.0.1:11345/health",
openai_model: "snowflake/snowflake-arctic-embed-xs",
openai_api_key: "",
qdrant_index_configuration: %{
vectors: %{size: 384, distance: "Cosine"}
}
# Import environment specific config. This must remain at the bottom # Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above. # of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs" import_config "#{Mix.env()}.exs"

View File

@ -136,31 +136,6 @@
} }
] ]
}, },
%{
group: :pleroma,
key: Pleroma.Uploaders.IPFS,
type: :group,
description: "IPFS uploader-related settings",
children: [
%{
key: :get_gateway_url,
type: :string,
description: "GET Gateway URL",
suggestions: [
"https://ipfs.mydomain.com/{CID}",
"https://{CID}.ipfs.mydomain.com/"
]
},
%{
key: :post_gateway_url,
type: :string,
description: "POST Gateway URL",
suggestions: [
"http://localhost:5001/"
]
}
]
},
%{ %{
group: :pleroma, group: :pleroma,
key: Pleroma.Uploaders.S3, key: Pleroma.Uploaders.S3,
@ -774,18 +749,6 @@
{"*.quarantined.com", "Reason"} {"*.quarantined.com", "Reason"}
] ]
}, },
%{
key: :rejected_instances,
type: {:list, :tuple},
key_placeholder: "instance",
value_placeholder: "reason",
description:
"List of ActivityPub instances to reject requests from if authorized_fetch_mode is enabled",
suggestions: [
{"rejected.com", "Reason"},
{"*.rejected.com", "Reason"}
]
},
%{ %{
key: :static_dir, key: :static_dir,
type: :string, type: :string,
@ -1828,12 +1791,6 @@
type: :boolean, type: :boolean,
description: "Require HTTP signatures for AP fetches" description: "Require HTTP signatures for AP fetches"
}, },
%{
key: :authorized_fetch_mode_exceptions,
type: {:list, :string},
description:
"List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)"
},
%{ %{
key: :note_replies_output_limit, key: :note_replies_output_limit,
type: :integer, type: :integer,

View File

@ -35,8 +35,8 @@
# configured to run both http and https servers on # configured to run both http and https servers on
# different ports. # different ports.
# Do not include timestamps in development logs # Do not include metadata nor timestamps in development logs
config :logger, :console, format: "$metadata[$level] $message\n" config :logger, :console, format: "[$level] $message\n"
# Set a higher stacktrace during development. Avoid configuring such # Set a higher stacktrace during development. Avoid configuring such
# in production as building large stacktraces may be expensive. # in production as building large stacktraces may be expensive.

View File

@ -153,12 +153,6 @@
config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock
config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock
config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock
config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock
config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.StaticStubbedConfigMock
config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug, config_impl: Pleroma.StaticStubbedConfigMock
config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug,
http_signatures_impl: Pleroma.StubbedHTTPSignaturesMock
peer_module = peer_module =
if String.to_integer(System.otp_release()) >= 25 do if String.to_integer(System.otp_release()) >= 25 do

View File

@ -21,18 +21,16 @@ Replaces embedded objects with references to them in the `objects` table. Only n
mix pleroma.database remove_embedded_objects [option ...] mix pleroma.database remove_embedded_objects [option ...]
``` ```
### Options ### Options
- `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
## Prune old remote posts from the database ## Prune old remote posts from the database
This will prune remote posts older than 90 days (configurable with [`config :pleroma, :instance, remote_post_retention_days`](../../configuration/cheatsheet.md#instance)) from the database. Pruned posts may be refetched in some cases. This will prune remote posts older than 90 days (configurable with [`config :pleroma, :instance, remote_post_retention_days`](../../configuration/cheatsheet.md#instance)) from the database, they will be refetched from source when accessed.
!!! note
The disk space will only be reclaimed after a proper vacuum. By default Postgresql does this for you on a regular basis, but if your instance has been running for a long time and there are many rows deleted, it may be advantageous to use `VACUUM FULL` (e.g. by using the `--vacuum` option).
!!! danger !!! danger
You may run out of disk space during the execution of the task or vacuuming if you don't have about 1/3rds of the database size free. Vacuum causes a substantial increase in I/O traffic, and may lead to a degraded experience while it is running. The disk space will only be reclaimed after `VACUUM FULL`. You may run out of disk space during the execution of the task or vacuuming if you don't have about 1/3rds of the database size free.
=== "OTP" === "OTP"
@ -47,11 +45,7 @@ This will prune remote posts older than 90 days (configurable with [`config :ple
``` ```
### Options ### Options
- `--vacuum` - run `VACUUM FULL` after the objects are pruned
- `--keep-threads` - Don't prune posts when they are part of a thread where at least one post has seen local interaction (e.g. one of the posts is a local post, or is favourited by a local user, or has been repeated by a local user...). It also won't delete posts when at least one of the posts in that thread is kept (e.g. because one of the posts has seen recent activity).
- `--keep-non-public` - Keep non-public posts like DM's and followers-only, even if they are remote.
- `--prune-orphaned-activities` - Also prune orphaned activities afterwards. Activities are things like Like, Create, Announce, Flag (aka reports). They can significantly help reduce the database size. Note: this can take a very long time.
- `--vacuum` - Run `VACUUM FULL` after the objects are pruned. This should not be used on a regular basis, but is useful if your instance has been running for a long time before pruning.
## Create a conversation for all existing DMs ## Create a conversation for all existing DMs
@ -99,9 +93,6 @@ Can be safely re-run
## Vacuum the database ## Vacuum the database
!!! note
By default Postgresql has an autovacuum deamon running. While the tasks described here can help in some cases, they shouldn't be needed on a regular basis. See [the Postgresql docs on vacuuming](https://www.postgresql.org/docs/current/sql-vacuum.html) for more information on this.
### Analyze ### Analyze
Running an `analyze` vacuum job can improve performance by updating statistics used by the query planner. **It is safe to cancel this.** Running an `analyze` vacuum job can improve performance by updating statistics used by the query planner. **It is safe to cancel this.**

View File

@ -41,7 +41,6 @@ To add configuration to your config file, you can copy it from the base config.
* `allow_relay`: Permits remote instances to subscribe to all public posts of your instance. This may increase the visibility of your instance. * `allow_relay`: Permits remote instances to subscribe to all public posts of your instance. This may increase the visibility of your instance.
* `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details. * `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details.
* `quarantined_instances`: ActivityPub instances where private (DMs, followers-only) activities will not be send. * `quarantined_instances`: ActivityPub instances where private (DMs, followers-only) activities will not be send.
* `rejected_instances`: ActivityPub instances to reject requests from if authorized_fetch_mode is enabled.
* `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML). * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML).
* `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with
older software for theses nicknames. older software for theses nicknames.
@ -285,7 +284,6 @@ Notes:
* `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question * `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question
* `sign_object_fetches`: Sign object fetches with HTTP signatures * `sign_object_fetches`: Sign object fetches with HTTP signatures
* `authorized_fetch_mode`: Require HTTP signatures for AP fetches * `authorized_fetch_mode`: Require HTTP signatures for AP fetches
* `authorized_fetch_mode_exceptions`: List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)
## Pleroma.User ## Pleroma.User
@ -474,7 +472,6 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
* ``ct_max_age``: The maximum age for the `Expect-CT` header if sent. * ``ct_max_age``: The maximum age for the `Expect-CT` header if sent.
* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`. * ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`.
* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header. * ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header.
* `allow_unsafe_eval`: Adds `wasm-unsafe-eval` to the CSP header. Needed for some non-essential frontend features like Flash emulation.
### Pleroma.Web.Plugs.RemoteIp ### Pleroma.Web.Plugs.RemoteIp
@ -664,19 +661,6 @@ config :ex_aws, :s3,
host: "s3.eu-central-1.amazonaws.com" host: "s3.eu-central-1.amazonaws.com"
``` ```
#### Pleroma.Uploaders.IPFS
* `post_gateway_url`: URL with port of POST Gateway (unauthenticated)
* `get_gateway_url`: URL of public GET Gateway
Example:
```elixir
config :pleroma, Pleroma.Uploaders.IPFS,
post_gateway_url: "http://localhost:5001",
get_gateway_url: "http://{CID}.ipfs.mydomain.com"
```
### Upload filters ### Upload filters
#### Pleroma.Upload.Filter.AnonymizeFilename #### Pleroma.Upload.Filter.AnonymizeFilename

View File

@ -10,30 +10,6 @@ To use built-in search that has no external dependencies, set the search module
While it has no external dependencies, it has problems with performance and relevancy. While it has no external dependencies, it has problems with performance and relevancy.
## QdrantSearch
This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`.
The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. To use it, set the search provider and run the fastembed server, see the README in `supplemental/search/fastembed-api`:
> config :pleroma, Pleroma.Search, module: Pleroma.Search.QdrantSearch
Then, start the Qdrant server, see [here](https://qdrant.tech/documentation/quick-start/) for instructions.
You will also need to create the Qdrant index once by running `mix pleroma.search.indexer create_index`. Running `mix pleroma.search.indexer index` will retroactively index the last 100_000 activities.
### Indexing and model options
To see the available configuration options, check out the QdrantSearch section in `config/config.exs`.
The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/). See also [this blog post](https://qdrant.tech/articles/memory-consumption/) that goes into detail.
Different embedding models will need different vector size settings. You can see a list of the models supported by the fastembed server [here](https://qdrant.github.io/fastembed/examples/Supported_Models), including their vector dimensions. These vector dimensions need to be set in the `qdrant_index_configuration`.
E.g, If you want to use `sentence-transformers/all-MiniLM-L6-v2` as a model, you will not need to adjust things, because it and `snowflake-arctic-embed-xs` are both 384 dimensional models. If you want to use `snowflake/snowflake-arctic-embed-l`, you will need to adjust the `size` parameter in the `qdrant_index_configuration` to 1024, as it has a dimension of 1024.
When using a different model, you will need do drop the index and recreate it (`mix pleroma.search.indexer drop_index` and `mix pleroma.search.indexer create_index`), as the different embeddings are not compatible with each other.
## Meilisearch ## Meilisearch
Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 million Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 million

View File

@ -295,7 +295,9 @@ See [Admin-API](admin_api.md)
"id": "9umDrYheeY451cQnEe", "id": "9umDrYheeY451cQnEe",
"name": "Read later", "name": "Read later",
"emoji": "🕓", "emoji": "🕓",
"emoji_url": null "source": {
"emoji": "🕓"
}
} }
] ]
``` ```

View File

@ -1,47 +1,44 @@
# Prometheus / OpenTelemetry Metrics # Prometheus Metrics
Pleroma includes support for exporting metrics via the [prom_ex](https://github.com/akoutmos/prom_ex) library. Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library.
The metrics are exposed by a dedicated webserver/port to improve privacy and security.
Config example: Config example:
``` ```
config :pleroma, Pleroma.PromEx, config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
disabled: false, enabled: true,
manual_metrics_start_delay: :no_delay, auth: {:basic, "myusername", "mypassword"},
drop_metrics_groups: [], ip_whitelist: ["127.0.0.1"],
grafana: [ path: "/api/pleroma/app_metrics",
host: System.get_env("GRAFANA_HOST", "http://localhost:3000"), format: :text
auth_token: System.get_env("GRAFANA_TOKEN"),
upload_dashboards_on_start: false,
folder_name: "BEAM",
annotate_app_lifecycle: true
],
metrics_server: [
port: 4021,
path: "/metrics",
protocol: :http,
pool_size: 5,
cowboy_opts: [],
auth_strategy: :none
],
datasource: "Prometheus"
``` ```
PromEx supports the ability to automatically publish dashboards to your Grafana server as well as register Annotations. If you do not wish to configure this capability you must generate the dashboard JSON files and import them directly. You can find the mix commands in the upstream [documentation](https://hexdocs.pm/prom_ex/Mix.Tasks.PromEx.Dashboard.Export.html). You can find the list of modules enabled in Pleroma for which you should generate dashboards for by examining the contents of the `lib/pleroma/prom_ex.ex` module. * `enabled` (Pleroma extension) enables the endpoint
* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs
* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation)
* `format` sets the output format (`:text` or `:protobuf`)
* `path` sets the path to app metrics page
## prometheus.yml
The following is a bare minimum config example to use with [Prometheus](https://prometheus.io) or Prometheus-compatible software like [VictoriaMetrics](https://victoriametrics.com). ## `/api/pleroma/app_metrics`
### Exports Prometheus application metrics
* Method: `GET`
* Authentication: not required by default (see configuration options above)
* Params: none
* Response: text
## Grafana
### Config example
The following is a config example to use with [Grafana](https://grafana.com)
``` ```
global: - job_name: 'beam'
scrape_interval: 15s metrics_path: /api/pleroma/app_metrics
scheme: https
scrape_configs:
- job_name: 'pleroma'
scheme: http
static_configs: static_configs:
- targets: ['pleroma.soykaf.com:4021'] - targets: ['pleroma.soykaf.com']
``` ```

View File

@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have
- PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので,[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください) - PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので,[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください)
- `postgresql-contrib` 11.0以上 (同上) - `postgresql-contrib` 11.0以上 (同上)
- Elixir 1.13 以上 ([Debianのリポジトリからインストールしないこと ここからインストールすること!](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) - Elixir 1.8 以上 ([Debianのリポジトリからインストールしないこと ここからインストールすること!](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
- `erlang-dev` - `erlang-dev`
- `erlang-nox` - `erlang-nox`
- `git` - `git`

View File

@ -1,7 +1,7 @@
## Required dependencies ## Required dependencies
* PostgreSQL >=11.0 * PostgreSQL >=11.0
* Elixir >=1.13.0 <1.15 * Elixir >=1.11.0 <1.15
* Erlang OTP >=22.2.0 (supported: <27) * Erlang OTP >=22.2.0 (supported: <27)
* git * git
* file / libmagic * file / libmagic

View File

@ -1,15 +0,0 @@
[Unit]
Description=NSFW API
After=docker.service
Requires=docker.service
[Service]
TimeoutStartSec=0
Restart=always
ExecStartPre=-/usr/bin/docker stop %n
ExecStartPre=-/usr/bin/docker rm %n
ExecStartPre=/usr/bin/docker pull eugencepoi/nsfw_api:latest
ExecStart=/usr/bin/docker run --rm -p 127.0.0.1:5000:5000/tcp --env PORT=5000 --name %n eugencepoi/nsfw_api:latest
[Install]
WantedBy=multi-user.target

View File

@ -67,168 +67,43 @@ def run(["prune_objects" | args]) do
OptionParser.parse( OptionParser.parse(
args, args,
strict: [ strict: [
vacuum: :boolean, vacuum: :boolean
keep_threads: :boolean,
keep_non_public: :boolean,
prune_orphaned_activities: :boolean
] ]
) )
start_pleroma() start_pleroma()
deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
time_deadline = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400))
log_message = "Pruning objects older than #{deadline} days" Logger.info("Pruning objects older than #{deadline} days")
log_message = time_deadline =
if Keyword.get(options, :keep_non_public) do NaiveDateTime.utc_now()
log_message <> ", keeping non public posts" |> NaiveDateTime.add(-(deadline * 86_400))
else
log_message
end
log_message = from(o in Object,
if Keyword.get(options, :keep_threads) do where:
log_message <> ", keeping threads intact"
else
log_message
end
log_message =
if Keyword.get(options, :prune_orphaned_activities) do
log_message <> ", pruning orphaned activities"
else
log_message
end
log_message =
if Keyword.get(options, :vacuum) do
log_message <>
", doing a full vacuum (you shouldn't do this as a recurring maintanance task)"
else
log_message
end
Logger.info(log_message)
if Keyword.get(options, :keep_threads) do
# We want to delete objects from threads where
# 1. the newest post is still old
# 2. none of the activities is local
# 3. none of the activities is bookmarked
# 4. optionally none of the posts is non-public
deletable_context =
if Keyword.get(options, :keep_non_public) do
Pleroma.Activity
|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
|> group_by([a], fragment("? ->> 'context'::text", a.data))
|> having(
[a],
not fragment(
# Posts (checked on Create Activity) is non-public
"bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')",
a.data,
^Pleroma.Constants.as_public(),
a.data,
^Pleroma.Constants.as_public(),
a.data
)
)
else
Pleroma.Activity
|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
|> group_by([a], fragment("? ->> 'context'::text", a.data))
end
|> having([a], max(a.updated_at) < ^time_deadline)
|> having([a], not fragment("bool_or(?)", a.local))
|> having([_, b], fragment("max(?::text) is null", b.id))
|> select([a], fragment("? ->> 'context'::text", a.data))
Pleroma.Object
|> where([o], fragment("? ->> 'context'::text", o.data) in subquery(deletable_context))
else
if Keyword.get(options, :keep_non_public) do
Pleroma.Object
|> where(
[o],
fragment( fragment(
"?->'to' \\? ? OR ?->'cc' \\? ?", "?->'to' \\? ? OR ?->'cc' \\? ?",
o.data, o.data,
^Pleroma.Constants.as_public(), ^Pleroma.Constants.as_public(),
o.data, o.data,
^Pleroma.Constants.as_public() ^Pleroma.Constants.as_public()
) ),
) where: o.inserted_at < ^time_deadline,
else where:
Pleroma.Object
end
|> where([o], o.updated_at < ^time_deadline)
|> where(
[o],
fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host()) fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
) )
end
|> Repo.delete_all(timeout: :infinity) |> Repo.delete_all(timeout: :infinity)
if !Keyword.get(options, :keep_threads) do prune_hashtags_query = """
# Without the --keep-threads option, it's possible that bookmarked
# objects have been deleted. We remove the corresponding bookmarks.
"""
delete from public.bookmarks
where id in (
select b.id from public.bookmarks b
left join public.activities a on b.activity_id = a.id
left join public.objects o on a."data" ->> 'object' = o.data ->> 'id'
where o.id is null
)
"""
|> Repo.query([], timeout: :infinity)
end
if Keyword.get(options, :prune_orphaned_activities) do
# Prune activities who link to a single object
"""
delete from public.activities
where id in (
select a.id from public.activities a
left join public.objects o on a.data ->> 'object' = o.data ->> 'id'
left join public.activities a2 on a.data ->> 'object' = a2.data ->> 'id'
left join public.users u on a.data ->> 'object' = u.ap_id
where not a.local
and jsonb_typeof(a."data" -> 'object') = 'string'
and o.id is null
and a2.id is null
and u.id is null
)
"""
|> Repo.query([], timeout: :infinity)
# Prune activities who link to an array of objects
"""
delete from public.activities
where id in (
select a.id from public.activities a
join json_array_elements_text((a."data" -> 'object')::json) as j on jsonb_typeof(a."data" -> 'object') = 'array'
left join public.objects o on j.value = o.data ->> 'id'
left join public.activities a2 on j.value = a2.data ->> 'id'
left join public.users u on j.value = u.ap_id
group by a.id
having max(o.data ->> 'id') is null
and max(a2.data ->> 'id') is null
and max(u.ap_id) is null
)
"""
|> Repo.query([], timeout: :infinity)
end
"""
DELETE FROM hashtags AS ht DELETE FROM hashtags AS ht
WHERE NOT EXISTS ( WHERE NOT EXISTS (
SELECT 1 FROM hashtags_objects hto SELECT 1 FROM hashtags_objects hto
WHERE ht.id = hto.hashtag_id) WHERE ht.id = hto.hashtag_id)
""" """
|> Repo.query()
Repo.query(prune_hashtags_query)
if Keyword.get(options, :vacuum) do if Keyword.get(options, :vacuum) do
Maintenance.vacuum("full") Maintenance.vacuum("full")

View File

@ -1,83 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.Search.Indexer do
import Mix.Pleroma
import Ecto.Query
alias Pleroma.Workers.SearchIndexingWorker
def run(["create_index"]) do
start_pleroma()
with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do
IO.puts("Index created")
else
e -> IO.puts("Could not create index: #{inspect(e)}")
end
end
def run(["drop_index"]) do
start_pleroma()
with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do
IO.puts("Index dropped")
else
e -> IO.puts("Could not drop index: #{inspect(e)}")
end
end
def run(["index" | options]) do
{options, [], []} =
OptionParser.parse(
options,
strict: [
chunk: :integer,
limit: :integer,
step: :integer
]
)
start_pleroma()
chunk_size = Keyword.get(options, :chunk, 100)
limit = Keyword.get(options, :limit, 100_000)
per_step = Keyword.get(options, :step, 1000)
chunks = max(div(limit, per_step), 1)
1..chunks
|> Enum.each(fn step ->
q =
from(a in Pleroma.Activity,
limit: ^per_step,
offset: ^per_step * (^step - 1),
select: [:id],
order_by: [desc: :id]
)
{:ok, ids} =
Pleroma.Repo.transaction(fn ->
Pleroma.Repo.stream(q, timeout: :infinity)
|> Enum.map(fn a ->
a.id
end)
end)
IO.puts("Got #{length(ids)} activities, adding to indexer")
ids
|> Enum.chunk_every(chunk_size)
|> Enum.each(fn chunk ->
IO.puts("Adding #{length(chunk)} activities to indexing queue")
chunk
|> Enum.map(fn id ->
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
end)
|> Oban.insert_all()
end)
end)
end
end

View File

@ -14,7 +14,6 @@ defmodule Pleroma.Application do
@name Mix.Project.config()[:name] @name Mix.Project.config()[:name]
@version Mix.Project.config()[:version] @version Mix.Project.config()[:version]
@repository Mix.Project.config()[:source_url] @repository Mix.Project.config()[:source_url]
@compile_env Mix.env()
def name, do: @name def name, do: @name
def version, do: @version def version, do: @version
@ -52,11 +51,7 @@ def start(_type, _args) do
Pleroma.HTML.compile_scrubbers() Pleroma.HTML.compile_scrubbers()
Pleroma.Config.Oban.warn() Pleroma.Config.Oban.warn()
Config.DeprecationWarnings.warn() Config.DeprecationWarnings.warn()
if @compile_env != :test do
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
end
Pleroma.ApplicationRequirements.verify!() Pleroma.ApplicationRequirements.verify!()
load_custom_modules() load_custom_modules()
Pleroma.Docs.JSON.compile() Pleroma.Docs.JSON.compile()
@ -114,8 +109,7 @@ def start(_type, _args) do
streamer_registry() ++ streamer_registry() ++
background_migrators() ++ background_migrators() ++
shout_child(shout_enabled?()) ++ shout_child(shout_enabled?()) ++
[Pleroma.Gopher.Server] ++ [Pleroma.Gopher.Server]
[Pleroma.Search.Healthcheck]
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options # for other strategies and supported options
@ -168,8 +162,7 @@ defp cachex_children do
expiration: chat_message_id_idempotency_key_expiration(), expiration: chat_message_id_idempotency_key_expiration(),
limit: 500_000 limit: 500_000
), ),
build_cachex("rel_me", limit: 2500), build_cachex("rel_me", limit: 2500)
build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000)
] ]
end end

View File

@ -16,15 +16,4 @@ def parse_address(ip) when is_binary(ip) do
def parse_address(ip) do def parse_address(ip) do
:inet.parse_address(ip) :inet.parse_address(ip)
end end
def parse_cidr(proxy) when is_binary(proxy) do
proxy =
cond do
"/" in String.codepoints(proxy) -> proxy
InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
end
InetCidr.parse_cidr!(proxy, true)
end
end end

View File

@ -25,7 +25,7 @@ def missing_dependencies do
end end
def image_resize(url, options) do def image_resize(url, options) do
with {:ok, env} <- HTTP.get(url, [], http_client_opts()), with {:ok, env} <- HTTP.get(url, [], pool: :media),
{:ok, resized} <- {:ok, resized} <-
Operation.thumbnail_buffer(env.body, options.max_width, Operation.thumbnail_buffer(env.body, options.max_width,
height: options.max_height, height: options.max_height,
@ -45,8 +45,8 @@ def image_resize(url, options) do
@spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()} @spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()}
def video_framegrab(url) do def video_framegrab(url) do
with executable when is_binary(executable) <- System.find_executable("ffmpeg"), with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
{:ok, false} <- @cachex.exists?(:failed_media_helper_cache, url), false <- @cachex.exists?(:failed_media_helper_cache, url),
{:ok, env} <- HTTP.get(url, [], http_client_opts()), {:ok, env} <- HTTP.get(url, [], pool: :media),
{:ok, pid} <- StringIO.open(env.body) do {:ok, pid} <- StringIO.open(env.body) do
body_stream = IO.binstream(pid, 1) body_stream = IO.binstream(pid, 1)
@ -71,19 +71,17 @@ def video_framegrab(url) do
end) end)
case Task.yield(task, 5_000) do case Task.yield(task, 5_000) do
{:ok, result} -> nil ->
{:ok, result}
_ ->
Task.shutdown(task) Task.shutdown(task)
@cachex.put(:failed_media_helper_cache, url, nil) @cachex.put(:failed_media_helper_cache, url, nil)
{:error, {:ffmpeg, :timeout}} {:error, {:ffmpeg, :timeout}}
result ->
{:ok, result}
end end
else else
nil -> {:error, {:ffmpeg, :command_not_found}} nil -> {:error, {:ffmpeg, :command_not_found}}
{:error, _} = error -> error {:error, _} = error -> error
end end
end end
defp http_client_opts, do: Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
end end

View File

@ -1,4 +0,0 @@
defmodule Pleroma.HTTPSignaturesAPI do
@callback validate_conn(conn :: Plug.Conn.t()) :: boolean
@callback signature_for_conn(conn :: Plug.Conn.t()) :: map
end

View File

@ -489,7 +489,7 @@ def create_poll_notifications(%Activity{} = activity) do
NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1 NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1
""" """
@spec get_notified_from_activity(Activity.t(), boolean()) :: list(User.t()) @spec get_notified_from_activity(Activity.t(), boolean()) :: {list(User.t()), list(User.t())}
def get_notified_from_activity(activity, local_only \\ true) def get_notified_from_activity(activity, local_only \\ true)
def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only) def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)

View File

@ -204,7 +204,7 @@ def due_activities(offset \\ 0) do
def job_query(scheduled_activity_id) do def job_query(scheduled_activity_id) do
from(j in Oban.Job, from(j in Oban.Job,
where: j.queue == "federator_outgoing", where: j.queue == "scheduled_activities",
where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id)) where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id))
) )
end end

View File

@ -10,12 +10,8 @@ def remove_from_index(%Pleroma.Object{id: object_id}) do
end end
def search(query, options) do def search(query, options) do
search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
search_module.search(options[:for_user], query, options) search_module.search(options[:for_user], query, options)
end end
def healthcheck_endpoints do
search_module = Pleroma.Config.get([Pleroma.Search, :module])
search_module.healthcheck_endpoints
end
end end

View File

@ -28,7 +28,7 @@ def search(user, search_query, options \\ []) do
|> Activity.with_preloaded_object() |> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users() |> Activity.restrict_deactivated_users()
|> restrict_public(user) |> restrict_public(user)
|> query_with(index_type, search_query) |> query_with(index_type, search_query, :websearch)
|> maybe_restrict_local(user) |> maybe_restrict_local(user)
|> maybe_restrict_author(author) |> maybe_restrict_author(author)
|> maybe_restrict_blocked(user) |> maybe_restrict_blocked(user)
@ -48,15 +48,6 @@ def add_to_index(_activity), do: :ok
@impl true @impl true
def remove_from_index(_object), do: :ok def remove_from_index(_object), do: :ok
@impl true
def create_index, do: :ok
@impl true
def drop_index, do: :ok
@impl true
def healthcheck_endpoints, do: nil
def maybe_restrict_author(query, %User{} = author) do def maybe_restrict_author(query, %User{} = author) do
Activity.Queries.by_author(query, author) Activity.Queries.by_author(query, author)
end end
@ -88,7 +79,25 @@ defp restrict_public(q, _user) do
) )
end end
defp query_with(q, :gin, search_query) do defp query_with(q, :gin, search_query, :plain) do
%{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!(
Pleroma.Repo,
"select current_setting('default_text_search_config')::regconfig::oid;"
)
from([a, o] in q,
where:
fragment(
"to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)",
^tsc,
o.data,
^search_query
)
)
end
defp query_with(q, :gin, search_query, :websearch) do
%{rows: [[tsc]]} = %{rows: [[tsc]]} =
Ecto.Adapters.SQL.query!( Ecto.Adapters.SQL.query!(
Pleroma.Repo, Pleroma.Repo,
@ -106,7 +115,19 @@ defp query_with(q, :gin, search_query) do
) )
end end
defp query_with(q, :rum, search_query) do defp query_with(q, :rum, search_query, :plain) do
from([a, o] in q,
where:
fragment(
"? @@ plainto_tsquery(?)",
o.fts_content,
^search_query
),
order_by: [fragment("? <=> now()::date", o.inserted_at)]
)
end
defp query_with(q, :rum, search_query, :websearch) do
from([a, o] in q, from([a, o] in q,
where: where:
fragment( fragment(

View File

@ -1,86 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Search.Healthcheck do
@doc """
Monitors health of search backend to control processing of events based on health and availability.
"""
use GenServer
require Logger
@queue :search_indexing
@tick :timer.seconds(5)
@timeout :timer.seconds(2)
def start_link(_) do
GenServer.start_link(__MODULE__, [], name: __MODULE__)
end
@impl true
def init(_) do
state = %{healthy: false}
{:ok, state, {:continue, :start}}
end
@impl true
def handle_continue(:start, state) do
tick()
{:noreply, state}
end
@impl true
def handle_info(:check, state) do
urls = Pleroma.Search.healthcheck_endpoints()
new_state =
if check(urls) do
Oban.resume_queue(queue: @queue)
Map.put(state, :healthy, true)
else
Oban.pause_queue(queue: @queue)
Map.put(state, :healthy, false)
end
maybe_log_state_change(state, new_state)
tick()
{:noreply, new_state}
end
@impl true
def handle_call(:state, _from, state) do
{:reply, state, state, :hibernate}
end
def state, do: GenServer.call(__MODULE__, :state)
def check([]), do: true
def check(urls) when is_list(urls) do
Enum.all?(
urls,
fn url ->
case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do
{:ok, %{status: 200}} -> true
_ -> false
end
end
)
end
def check(_), do: true
defp tick do
Process.send_after(self(), :check, @tick)
end
defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do
Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure")
end
defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do
Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass")
end
defp maybe_log_state_change(_, _), do: :ok
end

View File

@ -10,12 +10,6 @@ defmodule Pleroma.Search.Meilisearch do
@behaviour Pleroma.Search.SearchBackend @behaviour Pleroma.Search.SearchBackend
@impl true
def create_index, do: :ok
@impl true
def drop_index, do: :ok
defp meili_headers do defp meili_headers do
private_key = Config.get([Pleroma.Search.Meilisearch, :private_key]) private_key = Config.get([Pleroma.Search.Meilisearch, :private_key])
@ -184,15 +178,4 @@ def add_to_index(activity) do
def remove_from_index(object) do def remove_from_index(object) do
meili_delete("/indexes/objects/documents/#{object.id}") meili_delete("/indexes/objects/documents/#{object.id}")
end end
@impl true
def healthcheck_endpoints do
endpoint =
Config.get([Pleroma.Search.Meilisearch, :url])
|> URI.parse()
|> Map.put(:path, "/health")
|> URI.to_string()
[endpoint]
end
end end

View File

@ -1,182 +0,0 @@
defmodule Pleroma.Search.QdrantSearch do
@behaviour Pleroma.Search.SearchBackend
import Ecto.Query
alias Pleroma.Activity
alias Pleroma.Config.Getting, as: Config
alias __MODULE__.OpenAIClient
alias __MODULE__.QdrantClient
import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
@impl true
def create_index do
payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do
:ok
else
e -> {:error, e}
end
end
@impl true
def drop_index do
with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do
:ok
else
e -> {:error, e}
end
end
def get_embedding(text) do
with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <-
OpenAIClient.post("/v1/embeddings", %{
input: text,
model: Config.get([Pleroma.Search.QdrantSearch, :openai_model])
}) do
{:ok, embedding}
else
_ ->
{:error, "Failed to get embedding"}
end
end
defp actor_from_activity(%{data: %{"actor" => actor}}) do
actor
end
defp actor_from_activity(_), do: nil
defp build_index_payload(activity, embedding) do
actor = actor_from_activity(activity)
published_at = activity.data["published"]
%{
points: [
%{
id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(),
vector: embedding,
payload: %{actor: actor, published_at: published_at}
}
]
}
end
defp build_search_payload(embedding, options) do
base = %{
vector: embedding,
limit: options[:limit] || 20,
offset: options[:offset] || 0
}
if author = options[:author] do
Map.put(base, :filter, %{
must: [%{key: "actor", match: %{value: author.ap_id}}]
})
else
base
end
end
@impl true
def add_to_index(activity) do
# This will only index public or unlisted notes
maybe_search_data = object_to_search_data(activity.object)
if activity.data["type"] == "Create" and maybe_search_data do
with {:ok, embedding} <- get_embedding(maybe_search_data.content),
{:ok, %{status: 200}} <-
QdrantClient.put(
"/collections/posts/points",
build_index_payload(activity, embedding)
) do
:ok
else
e -> {:error, e}
end
else
:ok
end
end
@impl true
def remove_from_index(object) do
activity = Activity.get_by_object_ap_id_with_object(object.data["id"])
id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()
with {:ok, %{status: 200}} <-
QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do
:ok
else
e -> {:error, e}
end
end
@impl true
def search(user, original_query, options) do
query = "Represent this sentence for searching relevant passages: #{original_query}"
with {:ok, embedding} <- get_embedding(query),
{:ok, %{body: %{"result" => result}}} <-
QdrantClient.post(
"/collections/posts/points/search",
build_search_payload(embedding, options)
) do
ids =
Enum.map(result, fn %{"id" => id} ->
Ecto.UUID.dump!(id)
end)
from(a in Activity, where: a.id in ^ids)
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
|> Pleroma.Repo.all()
|> maybe_fetch(user, original_query)
else
_ ->
[]
end
end
@impl true
def healthcheck_endpoints do
qdrant_health =
Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])
|> URI.parse()
|> Map.put(:path, "/healthz")
|> URI.to_string()
openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url])
[qdrant_health, openai_health] |> Enum.filter(& &1)
end
end
defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url]))
plug(Tesla.Middleware.JSON)
plug(Tesla.Middleware.Headers, [
{"Authorization",
"Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
])
end
defmodule Pleroma.Search.QdrantSearch.QdrantClient do
use Tesla
alias Pleroma.Config.Getting, as: Config
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
plug(Tesla.Middleware.JSON)
plug(Tesla.Middleware.Headers, [
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
])
end

View File

@ -21,22 +21,4 @@ defmodule Pleroma.Search.SearchBackend do
from index. from index.
""" """
@callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()} @callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()}
@doc """
Create the index
"""
@callback create_index() :: :ok | {:error, any()}
@doc """
Drop the index
"""
@callback drop_index() :: :ok | {:error, any()}
@doc """
Healthcheck endpoints of search backend infrastructure to monitor for controlling
processing of jobs in the Oban queue.
It is expected a 200 response is healthy and other responses are unhealthy.
"""
@callback healthcheck_endpoints :: list() | nil
end end

View File

@ -62,7 +62,8 @@ defp remove_suffix(uri, [test | rest]) do
defp remove_suffix(uri, []), do: uri defp remove_suffix(uri, []), do: uri
def fetch_public_key(conn) do def fetch_public_key(conn) do
with {:ok, actor_id} <- get_actor_id(conn), with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
{:ok, actor_id} <- key_id_to_actor_id(kid),
{:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
{:ok, public_key} {:ok, public_key}
else else
@ -72,7 +73,8 @@ def fetch_public_key(conn) do
end end
def refetch_public_key(conn) do def refetch_public_key(conn) do
with {:ok, actor_id} <- get_actor_id(conn), with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
{:ok, actor_id} <- key_id_to_actor_id(kid),
{:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id), {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
{:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
{:ok, public_key} {:ok, public_key}
@ -82,16 +84,6 @@ def refetch_public_key(conn) do
end end
end end
def get_actor_id(conn) do
with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
{:ok, actor_id} <- key_id_to_actor_id(kid) do
{:ok, actor_id}
else
e ->
{:error, e}
end
end
def sign(%User{keys: keys} = user, headers) do def sign(%User{keys: keys} = user, headers) do
with {:ok, private_key, _} <- Keys.keys_from_pem(keys) do with {:ok, private_key, _} <- Keys.keys_from_pem(keys) do
HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers) HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers)

View File

@ -239,13 +239,9 @@ defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do
"" ""
end end
if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do
String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path)
else
[base_url, path] [base_url, path]
|> Path.join() |> Path.join()
end end
end
defp url_from_spec(_upload, _base_url, {:url, url}), do: url defp url_from_spec(_upload, _base_url, {:url, url}), do: url
@ -281,9 +277,6 @@ def base_url do
Path.join([upload_base_url, bucket_with_namespace]) Path.join([upload_base_url, bucket_with_namespace])
end end
Pleroma.Uploaders.IPFS ->
@config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
_ -> _ ->
public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
end end

View File

@ -9,6 +9,8 @@ defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do
""" """
@behaviour Pleroma.Upload.Filter @behaviour Pleroma.Upload.Filter
@spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
# Formats not compatible with exiftool at this time # Formats not compatible with exiftool at this time
def filter(%Pleroma.Upload{content_type: "image/heic"}), do: {:ok, :noop} def filter(%Pleroma.Upload{content_type: "image/heic"}), do: {:ok, :noop}
def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop} def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}

View File

@ -38,6 +38,7 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
[{"fill", "yellow"}, {"tint", "40"}] [{"fill", "yellow"}, {"tint", "40"}]
] ]
@spec filter(Pleroma.Upload.t()) :: {:ok, atom()} | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do try do
Filter.Mogrify.do_filter(file, [Enum.random(@filters)]) Filter.Mogrify.do_filter(file, [Enum.random(@filters)])

View File

@ -8,6 +8,7 @@ defmodule Pleroma.Upload.Filter.Mogrify do
@type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
@type conversions :: conversion() | [conversion()] @type conversions :: conversion() | [conversion()]
@spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
try do try do
do_filter(file, Pleroma.Config.get!([__MODULE__, :args])) do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))

View File

@ -1,77 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Uploaders.IPFS do
@behaviour Pleroma.Uploaders.Uploader
require Logger
alias Tesla.Multipart
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
defp get_final_url(method) do
config = @config_impl.get([__MODULE__])
post_base_url = Keyword.get(config, :post_gateway_url)
Path.join([post_base_url, method])
end
def put_file_endpoint do
get_final_url("/api/v0/add")
end
def delete_file_endpoint do
get_final_url("/api/v0/files/rm")
end
@placeholder "{CID}"
def placeholder, do: @placeholder
@impl true
def get_file(file) do
b_url = Pleroma.Upload.base_url()
if String.contains?(b_url, @placeholder) do
{:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}}
else
{:error, "IPFS Get URL doesn't contain 'cid' placeholder"}
end
end
@impl true
def put_file(%Pleroma.Upload{} = upload) do
mp =
Multipart.new()
|> Multipart.add_content_type_param("charset=utf-8")
|> Multipart.add_file(upload.tempfile)
case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do
{:ok, ret} ->
case Jason.decode(ret.body) do
{:ok, ret} ->
if Map.has_key?(ret, "Hash") do
{:ok, {:file, ret["Hash"]}}
else
{:error, "JSON doesn't contain Hash key"}
end
error ->
Logger.error("#{__MODULE__}: #{inspect(error)}")
{:error, "JSON decode failed"}
end
error ->
Logger.error("#{__MODULE__}: #{inspect(error)}")
{:error, "IPFS Gateway upload failed"}
end
end
@impl true
def delete_file(file) do
case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do
{:ok, %{status: 204}} -> :ok
error -> {:error, inspect(error)}
end
end
end

View File

@ -2053,8 +2053,7 @@ defp verify_field_link(field, profile_urls) do
%{scheme: scheme, userinfo: nil, host: host} %{scheme: scheme, userinfo: nil, host: host}
when not_empty_string(host) and scheme in ["http", "https"] <- when not_empty_string(host) and scheme in ["http", "https"] <-
URI.parse(value), URI.parse(value),
{:not_idn, true} <- {:not_idn, true} <- {:not_idn, to_string(:idna.encode(host)) == host},
{:not_idn, match?(^host, to_string(:idna.encode(to_charlist(host))))},
"me" <- Pleroma.Web.RelMe.maybe_put_rel_me(value, profile_urls) do "me" <- Pleroma.Web.RelMe.maybe_put_rel_me(value, profile_urls) do
CommonUtils.to_masto_date(NaiveDateTime.utc_now()) CommonUtils.to_masto_date(NaiveDateTime.utc_now())
else else
@ -2728,7 +2727,7 @@ defp add_to_block(%User{} = user, %User{} = blocked) do
end end
end end
@spec remove_from_block(User.t(), User.t()) :: @spec add_to_block(User.t(), User.t()) ::
{:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()} {:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()}
defp remove_from_block(%User{} = user, %User{} = blocked) do defp remove_from_block(%User{} = user, %User{} = blocked) do
with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do

View File

@ -979,9 +979,8 @@ defp restrict_media(query, _), do: query
defp restrict_replies(query, %{exclude_replies: true}) do defp restrict_replies(query, %{exclude_replies: true}) do
from( from(
[activity, object] in query, [_activity, object] in query,
where: where: fragment("?->>'inReplyTo' is null", object.data)
fragment("?->>'inReplyTo' is null or ?->>'type' = 'Announce'", object.data, activity.data)
) )
end end

View File

@ -52,7 +52,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
when action in [:activity, :object] when action in [:activity, :object]
) )
plug(:log_inbox_metadata when action in [:inbox])
plug(:set_requester_reachable when action in [:inbox]) plug(:set_requester_reachable when action in [:inbox])
plug(:relay_active? when action in [:relay]) plug(:relay_active? when action in [:relay])
@ -522,13 +521,6 @@ defp set_requester_reachable(%Plug.Conn{} = conn, _) do
conn conn
end end
defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do
Logger.metadata(actor: actor, type: type)
conn
end
defp log_inbox_metadata(conn, _), do: conn
def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
with {:ok, object} <- with {:ok, object} <-
ActivityPub.upload( ActivityPub.upload(

View File

@ -1,87 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do
alias Pleroma.Config
alias Pleroma.User
require Pleroma.Constants
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
defp user_has_posted?(%User{} = u), do: u.note_count > 0
defp user_has_age?(%User{} = u) do
user_age_limit = Config.get([:mrf_antimentionspam, :user_age_limit], 30_000)
diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :millisecond)
diff >= user_age_limit
end
defp good_reputation?(%User{} = u) do
user_has_age?(u) and user_has_posted?(u)
end
# copied from HellthreadPolicy
defp get_recipient_count(message) do
recipients = (message["to"] || []) ++ (message["cc"] || [])
follower_collection =
User.get_cached_by_ap_id(message["actor"] || message["attributedTo"]).follower_address
if Enum.member?(recipients, Pleroma.Constants.as_public()) do
recipients =
recipients
|> List.delete(Pleroma.Constants.as_public())
|> List.delete(follower_collection)
{:public, length(recipients)}
else
recipients =
recipients
|> List.delete(follower_collection)
{:not_public, length(recipients)}
end
end
defp object_has_recipients?(%{"object" => object} = activity) do
{_, object_count} = get_recipient_count(object)
{_, activity_count} = get_recipient_count(activity)
object_count + activity_count > 0
end
defp object_has_recipients?(object) do
{_, count} = get_recipient_count(object)
count > 0
end
@impl true
def filter(%{"type" => "Create", "actor" => actor} = activity) do
with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor),
{:has_mentions, true} <- {:has_mentions, object_has_recipients?(activity)},
{:good_reputation, true} <- {:good_reputation, good_reputation?(u)} do
{:ok, activity}
else
{:ok, %User{local: true}} ->
{:ok, activity}
{:has_mentions, false} ->
{:ok, activity}
{:good_reputation, false} ->
{:reject, "[AntiMentionSpamPolicy] User rejected"}
{:error, _} ->
{:reject, "[AntiMentionSpamPolicy] Failed to get or fetch user by ap_id"}
e ->
{:reject, "[AntiMentionSpamPolicy] Unhandled error #{inspect(e)}"}
end
end
# in all other cases, pass through
def filter(message), do: {:ok, message}
@impl true
def describe, do: {:ok, %{}}
end

View File

@ -1,146 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do
@moduledoc """
Dynamic activity filtering based on an RBL database
This MRF makes queries to a custom DNS server which will
respond with values indicating the classification of the domain
the activity originated from. This method has been widely used
in the email anti-spam industry for very fast reputation checks.
e.g., if the DNS response is 127.0.0.1 or empty, the domain is OK
Other values such as 127.0.0.2 may be used for specific classifications.
Information for why the host is blocked can be stored in a corresponding TXT record.
This method is fail-open so if the queries fail the activites are accepted.
An example of software meant for this purpsoe is rbldnsd which can be found
at http://www.corpit.ru/mjt/rbldnsd.html or mirrored at
https://git.pleroma.social/feld/rbldnsd
It is highly recommended that you run your own copy of rbldnsd and use an
external mechanism to sync/share the contents of the zone file. This is
important to keep the latency on the queries as low as possible and prevent
your DNS server from being attacked so it fails and content is permitted.
"""
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
alias Pleroma.Config
require Logger
@query_retries 1
@query_timeout 500
@impl true
def filter(%{"actor" => actor} = object) do
actor_info = URI.parse(actor)
with {:ok, object} <- check_rbl(actor_info, object) do
{:ok, object}
else
_ -> {:reject, "[DNSRBLPolicy]"}
end
end
@impl true
def filter(object), do: {:ok, object}
@impl true
def describe do
mrf_dnsrbl =
Config.get(:mrf_dnsrbl)
|> Enum.into(%{})
{:ok, %{mrf_dnsrbl: mrf_dnsrbl}}
end
@impl true
def config_description do
%{
key: :mrf_dnsrbl,
related_policy: "Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy",
label: "MRF DNSRBL",
description: "DNS RealTime Blackhole Policy",
children: [
%{
key: :nameserver,
type: {:string},
description: "DNSRBL Nameserver to Query (IP or hostame)",
suggestions: ["127.0.0.1"]
},
%{
key: :port,
type: {:string},
description: "Nameserver port",
suggestions: ["53"]
},
%{
key: :zone,
type: {:string},
description: "Root zone for querying",
suggestions: ["bl.pleroma.com"]
}
]
}
end
defp check_rbl(%{host: actor_host}, object) do
with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()),
zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do
query =
Enum.join([actor_host, zone], ".")
|> String.to_charlist()
rbl_response = rblquery(query)
if Enum.empty?(rbl_response) do
{:ok, object}
else
Task.start(fn ->
reason =
case rblquery(query, :txt) do
[[result]] -> result
_ -> "undefined"
end
Logger.warning(
"DNSRBL Rejected activity from #{actor_host} for reason: #{inspect(reason)}"
)
end)
:error
end
else
_ -> {:ok, object}
end
end
defp get_rblhost_ip(rblhost) do
case rblhost |> String.to_charlist() |> :inet_parse.address() do
{:ok, _} -> rblhost |> String.to_charlist() |> :inet_parse.address()
_ -> {:ok, rblhost |> String.to_charlist() |> :inet_res.lookup(:in, :a) |> Enum.random()}
end
end
defp rblquery(query, type \\ :a) do
config = Config.get([:mrf_dnsrbl])
case get_rblhost_ip(config[:nameserver]) do
{:ok, rblnsip} ->
:inet_res.lookup(query, :in, type,
nameservers: [{rblnsip, config[:port]}],
timeout: @query_timeout,
retry: @query_retries
)
_ ->
[]
end
end
end

View File

@ -11,6 +11,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger require Logger
@adapter_options [
pool: :media,
recv_timeout: 10_000
]
@impl true @impl true
def history_awareness, do: :auto def history_awareness, do: :auto
@ -22,14 +27,17 @@ defp prefetch(url) do
Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}") Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
if Pleroma.Config.get(:env) == :test do
fetch(prefetch_url) fetch(prefetch_url)
else
ConcurrentLimiter.limit(__MODULE__, fn ->
Task.start(fn -> fetch(prefetch_url) end)
end)
end
end end
end end
defp fetch(url) do defp fetch(url), do: HTTP.get(url, [], @adapter_options)
http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
HTTP.get(url, [], http_client_opts)
end
defp preload(%{"object" => %{"attachment" => attachments}} = _message) do defp preload(%{"object" => %{"attachment" => attachments}} = _message) do
Enum.each(attachments, fn Enum.each(attachments, fn

View File

@ -1,265 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
@moduledoc """
Hide, delete, or mark sensitive NSFW content with artificial intelligence.
Requires a NSFW API server, configured like so:
config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF,
url: "http://127.0.0.1:5000/",
threshold: 0.7,
mark_sensitive: true,
unlist: false,
reject: false
The NSFW API server must implement an HTTP endpoint like this:
curl http://localhost:5000/?url=https://fedi.com/images/001.jpg
Returning a response like this:
{"score", 0.314}
Where a score is 0-1, with `1` being definitely NSFW.
A good API server is here: https://github.com/EugenCepoi/nsfw_api
You can run it with Docker with a one-liner:
docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest
Options:
- `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/"
- `threshold`: Lowest score to take action on. Default: `0.7`
- `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true`
- `unlist`: Unlist all detected NSFW content? Default: `false`
- `reject`: Reject all detected NSFW content (takes precedence)? Default: `false`
"""
alias Pleroma.Config
alias Pleroma.Constants
alias Pleroma.HTTP
alias Pleroma.User
require Logger
require Pleroma.Constants
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
@policy :mrf_nsfw_api
def build_request_url(url) do
Config.get([@policy, :url])
|> URI.parse()
|> fix_path()
|> Map.put(:query, "url=#{url}")
|> URI.to_string()
end
def parse_url(url) do
request = build_request_url(url)
with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do
Jason.decode(body)
else
error ->
Logger.warning("""
[NsfwApiPolicy]: The API server failed. Skipping.
#{inspect(error)}
""")
error
end
end
def check_url_nsfw(url) when is_binary(url) do
threshold = Config.get([@policy, :threshold])
case parse_url(url) do
{:ok, %{"score" => score}} when score >= threshold ->
{:nsfw, %{url: url, score: score, threshold: threshold}}
{:ok, %{"score" => score}} ->
{:sfw, %{url: url, score: score, threshold: threshold}}
_ ->
{:sfw, %{url: url, score: nil, threshold: threshold}}
end
end
def check_url_nsfw(%{"href" => url}) when is_binary(url) do
check_url_nsfw(url)
end
def check_url_nsfw(url) do
threshold = Config.get([@policy, :threshold])
{:sfw, %{url: url, score: nil, threshold: threshold}}
end
def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do
{:sfw, attachment}
else
{:nsfw, attachment}
end
end
def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do
case check_url_nsfw(url) do
{:sfw, _} -> {:sfw, attachment}
{:nsfw, _} -> {:nsfw, attachment}
end
end
def check_attachment_nsfw(attachment), do: {:sfw, attachment}
def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do
{:sfw, object}
else
{:nsfw, object}
end
end
def check_object_nsfw(%{"object" => %{} = child_object} = object) do
case check_object_nsfw(child_object) do
{:sfw, _} -> {:sfw, object}
{:nsfw, _} -> {:nsfw, object}
end
end
def check_object_nsfw(object), do: {:sfw, object}
@impl true
def filter(object) do
with {:sfw, object} <- check_object_nsfw(object) do
{:ok, object}
else
{:nsfw, _data} -> handle_nsfw(object)
_ -> {:reject, "NSFW: Attachment rejected"}
end
end
defp handle_nsfw(object) do
if Config.get([@policy, :reject]) do
{:reject, object}
else
{:ok,
object
|> maybe_unlist()
|> maybe_mark_sensitive()}
end
end
defp maybe_unlist(object) do
if Config.get([@policy, :unlist]) do
unlist(object)
else
object
end
end
defp maybe_mark_sensitive(object) do
if Config.get([@policy, :mark_sensitive]) do
mark_sensitive(object)
else
object
end
end
def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do
with %User{} = user <- User.get_cached_by_ap_id(actor) do
to =
[user.follower_address | to]
|> List.delete(Constants.as_public())
|> Enum.uniq()
cc =
[Constants.as_public() | cc]
|> List.delete(user.follower_address)
|> Enum.uniq()
object
|> Map.put("to", to)
|> Map.put("cc", cc)
else
_ -> raise "[NsfwApiPolicy]: Could not find user #{actor}"
end
end
def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do
Map.put(object, "object", mark_sensitive(child_object))
end
def mark_sensitive(object) when is_map(object) do
tags = (object["tag"] || []) ++ ["nsfw"]
object
|> Map.put("tag", tags)
|> Map.put("sensitive", true)
end
# Hackney needs a trailing slash
defp fix_path(%URI{path: path} = uri) when is_binary(path) do
path = String.trim_trailing(path, "/") <> "/"
Map.put(uri, :path, path)
end
defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/")
@impl true
def describe do
options = %{
threshold: Config.get([@policy, :threshold]),
mark_sensitive: Config.get([@policy, :mark_sensitive]),
unlist: Config.get([@policy, :unlist]),
reject: Config.get([@policy, :reject])
}
{:ok, %{@policy => options}}
end
@impl true
def config_description do
%{
key: @policy,
related_policy: to_string(__MODULE__),
label: "NSFW API Policy",
description:
"Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.",
children: [
%{
key: :url,
type: :string,
description: "Base URL of the API server.",
suggestions: ["http://127.0.0.1:5000/"]
},
%{
key: :threshold,
type: :float,
description: "Lowest score to take action on. Between 0 and 1.",
suggestions: [0.7]
},
%{
key: :mark_sensitive,
type: :boolean,
description: "Mark sensitive all detected NSFW content?",
suggestions: [true]
},
%{
key: :unlist,
type: :boolean,
description: "Unlist sensitive all detected NSFW content?",
suggestions: [false]
},
%{
key: :reject,
type: :boolean,
description: "Reject sensitive all detected NSFW content (takes precedence)?",
suggestions: [false]
}
]
}
end
end

View File

@ -15,7 +15,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
field(:type, :string, default: "Link") field(:type, :string, default: "Link")
field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream") field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream")
field(:name, :string) field(:name, :string)
field(:summary, :string)
field(:blurhash, :string) field(:blurhash, :string)
embeds_many :url, UrlObjectValidator, primary_key: false do embeds_many :url, UrlObjectValidator, primary_key: false do
@ -45,7 +44,7 @@ def changeset(struct, data) do
|> fix_url() |> fix_url()
struct struct
|> cast(data, [:id, :type, :mediaType, :name, :summary, :blurhash]) |> cast(data, [:id, :type, :mediaType, :name, :blurhash])
|> cast_embed(:url, with: &url_changeset/2, required: true) |> cast_embed(:url, with: &url_changeset/2, required: true)
|> validate_inclusion(:type, ~w[Link Document Audio Image Video]) |> validate_inclusion(:type, ~w[Link Document Audio Image Video])
|> validate_required([:type, :mediaType]) |> validate_required([:type, :mediaType])

View File

@ -50,11 +50,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do
pleroma: %Schema{ pleroma: %Schema{
type: :object, type: :object,
properties: %{ properties: %{
mime_type: %Schema{type: :string, description: "mime type of the attachment"}, mime_type: %Schema{type: :string, description: "mime type of the attachment"}
name: %Schema{
type: :string,
description: "Name of the attachment, typically the filename"
}
} }
} }
}, },

View File

@ -129,22 +129,8 @@ defp attachments(%{params: params} = draft) do
defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft
defp in_reply_to(%{params: %{in_reply_to_status_id: :deleted}} = draft) do defp in_reply_to(%{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do
add_error(draft, dgettext("errors", "Cannot reply to a deleted status")) %__MODULE__{draft | in_reply_to: Activity.get_by_id(id)}
end
defp in_reply_to(%{params: %{in_reply_to_status_id: id} = params} = draft) when is_binary(id) do
activity = Activity.get_by_id(id)
params =
if is_nil(activity) do
# Deleted activities are returned as nil
Map.put(params, :in_reply_to_status_id, :deleted)
else
Map.put(params, :in_reply_to_status_id, activity)
end
in_reply_to(%{draft | params: params})
end end
defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do

View File

@ -38,8 +38,6 @@ defmodule Pleroma.Web.Endpoint do
plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint]) plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint])
plug(Pleroma.Web.Plugs.LoggerMetadataPath)
plug(Pleroma.Web.Plugs.SetLocalePlug) plug(Pleroma.Web.Plugs.SetLocalePlug)
plug(CORSPlug) plug(CORSPlug)
plug(Pleroma.Web.Plugs.HTTPSecurityPlug) plug(Pleroma.Web.Plugs.HTTPSecurityPlug)

View File

@ -44,7 +44,7 @@ def incoming_ap_doc(%{params: params, req_headers: req_headers}) do
end end
def incoming_ap_doc(%{"type" => "Delete"} = params) do def incoming_ap_doc(%{"type" => "Delete"} = params) do
ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow) ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3)
end end
def incoming_ap_doc(params) do def incoming_ap_doc(params) do

View File

@ -9,11 +9,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
alias Pleroma.User alias Pleroma.User
alias Pleroma.UserNote alias Pleroma.UserNote
alias Pleroma.UserRelationship alias Pleroma.UserRelationship
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.AccountView
alias Pleroma.Web.MediaProxy alias Pleroma.Web.MediaProxy
alias Pleroma.Web.PleromaAPI.ScrobbleView
def render("index.json", %{users: users} = opts) do def render("index.json", %{users: users} = opts) do
reading_user = opts[:for] reading_user = opts[:for]
@ -276,13 +274,6 @@ defp do_render("show.json", %{user: user} = opts) do
user.last_status_at && user.last_status_at &&
user.last_status_at |> NaiveDateTime.to_date() |> Date.to_iso8601() user.last_status_at |> NaiveDateTime.to_date() |> Date.to_iso8601()
scrobbles =
ActivityPub.fetch_user_abstract_activities(user, opts[:for], %{
type: "Listen",
limit: 1,
id: user.id
})
%{ %{
id: to_string(user.id), id: to_string(user.id),
username: username_from_nickname(user.nickname), username: username_from_nickname(user.nickname),
@ -331,8 +322,7 @@ defp do_render("show.json", %{user: user} = opts) do
skip_thread_containment: user.skip_thread_containment, skip_thread_containment: user.skip_thread_containment,
background_image: image_url(user.background) |> MediaProxy.url(), background_image: image_url(user.background) |> MediaProxy.url(),
accepts_chat_messages: user.accepts_chat_messages, accepts_chat_messages: user.accepts_chat_messages,
favicon: favicon, favicon: favicon
scrobbles: safe_render_many(scrobbles, ScrobbleView, "show.json")
} }
} }
|> maybe_put_role(user, opts[:for]) |> maybe_put_role(user, opts[:for])

View File

@ -152,7 +152,6 @@ def features do
def federation do def federation do
quarantined = Config.get([:instance, :quarantined_instances], []) quarantined = Config.get([:instance, :quarantined_instances], [])
rejected = Config.get([:instance, :rejected_instances], [])
if Config.get([:mrf, :transparency]) do if Config.get([:mrf, :transparency]) do
{:ok, data} = MRF.describe() {:ok, data} = MRF.describe()
@ -172,12 +171,6 @@ def federation do
|> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end)
|> Map.new() |> Map.new()
}) })
|> Map.put(
:rejected_instances,
rejected
|> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end)
|> Map.new()
)
else else
%{} %{}
end end

View File

@ -624,19 +624,6 @@ def render("attachment.json", %{attachment: attachment}) do
to_string(attachment["id"] || hash_id) to_string(attachment["id"] || hash_id)
end end
description =
if attachment["summary"] do
HTML.strip_tags(attachment["summary"])
else
attachment["name"]
end
name = if attachment["summary"], do: attachment["name"]
pleroma =
%{mime_type: media_type}
|> Maps.put_if_present(:name, name)
%{ %{
id: attachment_id, id: attachment_id,
url: href, url: href,
@ -644,8 +631,8 @@ def render("attachment.json", %{attachment: attachment}) do
preview_url: href_preview, preview_url: href_preview,
text_url: href, text_url: href,
type: type, type: type,
description: description, description: attachment["name"],
pleroma: pleroma, pleroma: %{mime_type: media_type},
blurhash: attachment["blurhash"] blurhash: attachment["blurhash"]
} }
|> Maps.put_if_present(:meta, meta) |> Maps.put_if_present(:meta, meta)

View File

@ -54,10 +54,9 @@ def preview(%Conn{} = conn, %{"sig" => sig64, "url" => url64}) do
defp handle_preview(conn, url) do defp handle_preview(conn, url) do
media_proxy_url = MediaProxy.url(url) media_proxy_url = MediaProxy.url(url)
http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
with {:ok, %{status: status} = head_response} when status in 200..299 <- with {:ok, %{status: status} = head_response} when status in 200..299 <-
Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do
content_type = Tesla.get_header(head_response, "content-type") content_type = Tesla.get_header(head_response, "content-type")
content_length = Tesla.get_header(head_response, "content-length") content_length = Tesla.get_header(head_response, "content-length")
content_length = content_length && String.to_integer(content_length) content_length = content_length && String.to_integer(content_length)

View File

@ -96,7 +96,7 @@ defp put_valid_until(changeset, attrs) do
|> validate_required([:valid_until]) |> validate_required([:valid_until])
end end
@spec create(App.t(), User.t(), map()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()} @spec create(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Ecto.Changeset.t()}
def create(%App{} = app, %User{} = user, attrs \\ %{}) do def create(%App{} = app, %User{} = user, attrs \\ %{}) do
with {:ok, token} <- do_create(app, user, attrs) do with {:ok, token} <- do_create(app, user, attrs) do
if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do

View File

@ -14,27 +14,15 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do
alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.AccountView
def render("show.json", %{scrobble: %Activity{data: %{"type" => "Listen"}} = scrobble} = _opts) do
scrobble_schema_skeleton(scrobble)
end
def render("show.json", %{activity: %Activity{data: %{"type" => "Listen"}} = activity} = opts) do def render("show.json", %{activity: %Activity{data: %{"type" => "Listen"}} = activity} = opts) do
object = Object.normalize(activity, fetch: false)
user = CommonAPI.get_user(activity.data["actor"]) user = CommonAPI.get_user(activity.data["actor"])
created_at = Utils.to_masto_date(activity.data["published"])
scrobble_schema_skeleton(activity)
|> Map.put(:account, AccountView.render("show.json", %{user: user, for: opts[:for]}))
end
def render("index.json", opts) do
safe_render_many(opts.activities, __MODULE__, "show.json", opts)
end
defp scrobble_schema_skeleton(%Activity{data: %{"type" => "Listen"}} = scrobble) do
object = Object.normalize(scrobble, fetch: false)
created_at = Utils.to_masto_date(scrobble.data["published"])
%{ %{
id: scrobble.id, id: activity.id,
account: AccountView.render("show.json", %{user: user, for: opts[:for]}),
created_at: created_at, created_at: created_at,
title: object.data["title"] |> HTML.strip_tags(), title: object.data["title"] |> HTML.strip_tags(),
artist: object.data["artist"] |> HTML.strip_tags(), artist: object.data["artist"] |> HTML.strip_tags(),
@ -43,4 +31,8 @@ defp scrobble_schema_skeleton(%Activity{data: %{"type" => "Listen"}} = scrobble)
length: object.data["length"] length: object.data["length"]
} }
end end
def render("index.json", opts) do
safe_render_many(opts.activities, __MODULE__, "show.json", opts)
end
end end

View File

@ -3,27 +3,26 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
alias Pleroma.Config
import Plug.Conn import Plug.Conn
require Logger require Logger
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
def init(opts), do: opts def init(opts), do: opts
def call(conn, _options) do def call(conn, _options) do
if @config_impl.get([:http_security, :enabled]) do if Config.get([:http_security, :enabled]) do
conn conn
|> merge_resp_headers(headers()) |> merge_resp_headers(headers())
|> maybe_send_sts_header(@config_impl.get([:http_security, :sts])) |> maybe_send_sts_header(Config.get([:http_security, :sts]))
else else
conn conn
end end
end end
def primary_frontend do def primary_frontend do
with %{"name" => frontend} <- @config_impl.get([:frontends, :primary]), with %{"name" => frontend} <- Config.get([:frontends, :primary]),
available <- @config_impl.get([:frontends, :available]), available <- Config.get([:frontends, :available]),
%{} = primary_frontend <- Map.get(available, frontend) do %{} = primary_frontend <- Map.get(available, frontend) do
{:ok, primary_frontend} {:ok, primary_frontend}
end end
@ -38,8 +37,8 @@ def custom_http_frontend_headers do
end end
def headers do def headers do
referrer_policy = @config_impl.get([:http_security, :referrer_policy]) referrer_policy = Config.get([:http_security, :referrer_policy])
report_uri = @config_impl.get([:http_security, :report_uri]) report_uri = Config.get([:http_security, :report_uri])
custom_http_frontend_headers = custom_http_frontend_headers() custom_http_frontend_headers = custom_http_frontend_headers()
headers = [ headers = [
@ -87,10 +86,10 @@ def headers do
@csp_start [Enum.join(static_csp_rules, ";") <> ";"] @csp_start [Enum.join(static_csp_rules, ";") <> ";"]
defp csp_string do defp csp_string do
scheme = @config_impl.get([Pleroma.Web.Endpoint, :url])[:scheme] scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
static_url = Pleroma.Web.Endpoint.static_url() static_url = Pleroma.Web.Endpoint.static_url()
websocket_url = Pleroma.Web.Endpoint.websocket_url() websocket_url = Pleroma.Web.Endpoint.websocket_url()
report_uri = @config_impl.get([:http_security, :report_uri]) report_uri = Config.get([:http_security, :report_uri])
img_src = "img-src 'self' data: blob:" img_src = "img-src 'self' data: blob:"
media_src = "media-src 'self'" media_src = "media-src 'self'"
@ -98,8 +97,8 @@ defp csp_string do
# Strict multimedia CSP enforcement only when MediaProxy is enabled # Strict multimedia CSP enforcement only when MediaProxy is enabled
{img_src, media_src, connect_src} = {img_src, media_src, connect_src} =
if @config_impl.get([:media_proxy, :enabled]) && if Config.get([:media_proxy, :enabled]) &&
!@config_impl.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
sources = build_csp_multimedia_source_list() sources = build_csp_multimedia_source_list()
{ {
@ -116,22 +115,18 @@ defp csp_string do
end end
connect_src = connect_src =
if @config_impl.get([:env]) == :dev do if Config.get(:env) == :dev do
[connect_src, " http://localhost:3035/"] [connect_src, " http://localhost:3035/"]
else else
connect_src connect_src
end end
script_src = script_src =
if @config_impl.get([:http_security, :allow_unsafe_eval]) do if Config.get(:env) == :dev do
if @config_impl.get([:env]) == :dev do
"script-src 'self' 'unsafe-eval'" "script-src 'self' 'unsafe-eval'"
else else
"script-src 'self' 'wasm-unsafe-eval'" "script-src 'self' 'wasm-unsafe-eval'"
end end
else
"script-src 'self'"
end
report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"] report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"]
insecure = if scheme == "https", do: "upgrade-insecure-requests" insecure = if scheme == "https", do: "upgrade-insecure-requests"
@ -166,11 +161,11 @@ defp build_csp_param_from_whitelist(url), do: url
defp build_csp_multimedia_source_list do defp build_csp_multimedia_source_list do
media_proxy_whitelist = media_proxy_whitelist =
[:media_proxy, :whitelist] [:media_proxy, :whitelist]
|> @config_impl.get() |> Config.get()
|> build_csp_from_whitelist([]) |> build_csp_from_whitelist([])
captcha_method = @config_impl.get([Pleroma.Captcha, :method]) captcha_method = Config.get([Pleroma.Captcha, :method])
captcha_endpoint = @config_impl.get([captcha_method, :endpoint]) captcha_endpoint = Config.get([captcha_method, :endpoint])
base_endpoints = base_endpoints =
[ [
@ -178,7 +173,7 @@ defp build_csp_multimedia_source_list do
[Pleroma.Upload, :base_url], [Pleroma.Upload, :base_url],
[Pleroma.Uploaders.S3, :public_endpoint] [Pleroma.Uploaders.S3, :public_endpoint]
] ]
|> Enum.map(&@config_impl.get/1) |> Enum.map(&Config.get/1)
[captcha_endpoint | base_endpoints] [captcha_endpoint | base_endpoints]
|> Enum.map(&build_csp_param/1) |> Enum.map(&build_csp_param/1)
@ -205,7 +200,7 @@ defp build_csp_param(url) when is_binary(url) do
end end
def warn_if_disabled do def warn_if_disabled do
unless Pleroma.Config.get([:http_security, :enabled]) do unless Config.get([:http_security, :enabled]) do
Logger.warning(" Logger.warning("
.i;;;;i. .i;;;;i.
iYcviii;vXY: iYcviii;vXY:
@ -250,8 +245,8 @@ def warn_if_disabled do
end end
defp maybe_send_sts_header(conn, true) do defp maybe_send_sts_header(conn, true) do
max_age_sts = @config_impl.get([:http_security, :sts_max_age]) max_age_sts = Config.get([:http_security, :sts_max_age])
max_age_ct = @config_impl.get([:http_security, :ct_max_age]) max_age_ct = Config.get([:http_security, :ct_max_age])
merge_resp_headers(conn, [ merge_resp_headers(conn, [
{"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"}, {"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"},

View File

@ -3,22 +3,10 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
alias Pleroma.Helpers.InetHelper
import Plug.Conn import Plug.Conn
import Phoenix.Controller, only: [get_format: 1, text: 2] import Phoenix.Controller, only: [get_format: 1, text: 2]
alias Pleroma.Web.ActivityPub.MRF
require Logger require Logger
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
@http_signatures_impl Application.compile_env(
:pleroma,
[__MODULE__, :http_signatures_impl],
HTTPSignatures
)
def init(options) do def init(options) do
options options
end end
@ -31,9 +19,7 @@ def call(conn, _opts) do
if get_format(conn) in ["json", "activity+json"] do if get_format(conn) in ["json", "activity+json"] do
conn conn
|> maybe_assign_valid_signature() |> maybe_assign_valid_signature()
|> maybe_assign_actor_id()
|> maybe_require_signature() |> maybe_require_signature()
|> maybe_filter_requests()
else else
conn conn
end end
@ -47,7 +33,7 @@ defp validate_signature(conn, request_target) do
|> put_req_header("(request-target)", request_target) |> put_req_header("(request-target)", request_target)
|> put_req_header("@request-target", request_target) |> put_req_header("@request-target", request_target)
@http_signatures_impl.validate_conn(conn) HTTPSignatures.validate_conn(conn)
end end
defp validate_signature(conn) do defp validate_signature(conn) do
@ -97,63 +83,20 @@ defp maybe_assign_valid_signature(conn) do
end end
end end
defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do
adapter = Application.get_env(:http_signatures, :adapter)
{:ok, actor_id} = adapter.get_actor_id(conn)
assign(conn, :actor_id, actor_id)
end
defp maybe_assign_actor_id(conn), do: conn
defp has_signature_header?(conn) do defp has_signature_header?(conn) do
conn |> get_req_header("signature") |> Enum.at(0, false) conn |> get_req_header("signature") |> Enum.at(0, false)
end end
defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn
defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do defp maybe_require_signature(conn) do
if @config_impl.get([:activitypub, :authorized_fetch_mode], false) do if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do
exceptions =
@config_impl.get([:activitypub, :authorized_fetch_mode_exceptions], [])
|> Enum.map(&InetHelper.parse_cidr/1)
if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do
conn
else
conn conn
|> put_status(:unauthorized) |> put_status(:unauthorized)
|> text("Request not signed") |> text("Request not signed")
|> halt() |> halt()
end
else else
conn conn
end end
end end
defp maybe_filter_requests(%{halted: true} = conn), do: conn
defp maybe_filter_requests(conn) do
if @config_impl.get([:activitypub, :authorized_fetch_mode], false) and
conn.assigns[:actor_id] do
%{host: host} = URI.parse(conn.assigns.actor_id)
if MRF.subdomain_match?(rejected_domains(), host) do
conn
|> put_status(:unauthorized)
|> halt()
else
conn
end
else
conn
end
end
defp rejected_domains do
@config_impl.get([:instance, :rejected_instances])
|> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples()
|> Pleroma.Web.ActivityPub.MRF.subdomains_regex()
end
end end

View File

@ -1,12 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.LoggerMetadataPath do
def init(opts), do: opts
def call(conn, _) do
Logger.metadata(path: conn.request_path)
conn
end
end

View File

@ -1,18 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Plugs.LoggerMetadataUser do
alias Pleroma.User
def init(opts), do: opts
def call(%{assigns: %{user: user = %User{}}} = conn, _) do
Logger.metadata(user: user.nickname)
conn
end
def call(conn, _) do
conn
end
end

View File

@ -8,7 +8,6 @@ defmodule Pleroma.Web.Plugs.RemoteIp do
""" """
alias Pleroma.Config alias Pleroma.Config
alias Pleroma.Helpers.InetHelper
import Plug.Conn import Plug.Conn
@behaviour Plug @behaviour Plug
@ -31,8 +30,19 @@ defp remote_ip_opts do
proxies = proxies =
Config.get([__MODULE__, :proxies], []) Config.get([__MODULE__, :proxies], [])
|> Enum.concat(reserved) |> Enum.concat(reserved)
|> Enum.map(&InetHelper.parse_cidr/1) |> Enum.map(&maybe_add_cidr/1)
{headers, proxies} {headers, proxies}
end end
defp maybe_add_cidr(proxy) when is_binary(proxy) do
proxy =
cond do
"/" in String.codepoints(proxy) -> proxy
InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
end
InetCidr.parse_cidr!(proxy, true)
end
end end

View File

@ -63,7 +63,6 @@ def perform(_) do
@doc "Push message to web" @doc "Push message to web"
def push_message(body, sub, api_key, subscription) do def push_message(body, sub, api_key, subscription) do
try do
case WebPushEncryption.send_web_push(body, sub, api_key) do case WebPushEncryption.send_web_push(body, sub, api_key) do
{:ok, %{status: code}} when code in 400..499 -> {:ok, %{status: code}} when code in 400..499 ->
Logger.debug("Removing subscription record") Logger.debug("Removing subscription record")
@ -77,11 +76,6 @@ def push_message(body, sub, api_key, subscription) do
Logger.error("Web Push Notification failed with code: #{code}") Logger.error("Web Push Notification failed with code: #{code}")
:error :error
error ->
Logger.error("Web Push Notification failed with #{inspect(error)}")
:error
end
rescue
error -> error ->
Logger.error("Web Push Notification failed with #{inspect(error)}") Logger.error("Web Push Notification failed with #{inspect(error)}")
:error :error

View File

@ -58,7 +58,7 @@ defp check_content_length(headers) do
defp http_options do defp http_options do
[ [
pool: :rich_media, pool: :media,
max_body: Config.get([:rich_media, :max_body], 5_000_000) max_body: Config.get([:rich_media, :max_body], 5_000_000)
] ]
end end

View File

@ -23,7 +23,7 @@ defp aws_signed_url?(image) when is_binary(image) and image != "" do
%URI{host: host, query: query} = URI.parse(image) %URI{host: host, query: query} = URI.parse(image)
is_binary(host) and String.contains?(host, "amazonaws.com") and is_binary(host) and String.contains?(host, "amazonaws.com") and
is_binary(query) and String.contains?(query, "X-Amz-Expires") String.contains?(query, "X-Amz-Expires")
end end
defp aws_signed_url?(_), do: nil defp aws_signed_url?(_), do: nil

View File

@ -29,7 +29,6 @@ defmodule Pleroma.Web.Router do
pipeline :browser do pipeline :browser do
plug(:accepts, ["html"]) plug(:accepts, ["html"])
plug(:fetch_session) plug(:fetch_session)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end end
pipeline :oauth do pipeline :oauth do
@ -68,14 +67,12 @@ defmodule Pleroma.Web.Router do
plug(:fetch_session) plug(:fetch_session)
plug(:authenticate) plug(:authenticate)
plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end end
pipeline :no_auth_or_privacy_expectations_api do pipeline :no_auth_or_privacy_expectations_api do
plug(:base_api) plug(:base_api)
plug(:after_auth) plug(:after_auth)
plug(Pleroma.Web.Plugs.IdempotencyPlug) plug(Pleroma.Web.Plugs.IdempotencyPlug)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end end
# Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported) # Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported)
@ -86,14 +83,12 @@ defmodule Pleroma.Web.Router do
pipeline :api do pipeline :api do
plug(:expect_public_instance_or_user_authentication) plug(:expect_public_instance_or_user_authentication)
plug(:no_auth_or_privacy_expectations_api) plug(:no_auth_or_privacy_expectations_api)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end end
pipeline :authenticated_api do pipeline :authenticated_api do
plug(:expect_user_authentication) plug(:expect_user_authentication)
plug(:no_auth_or_privacy_expectations_api) plug(:no_auth_or_privacy_expectations_api)
plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end end
pipeline :admin_api do pipeline :admin_api do
@ -104,7 +99,6 @@ defmodule Pleroma.Web.Router do
plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug)
plug(Pleroma.Web.Plugs.UserIsStaffPlug) plug(Pleroma.Web.Plugs.UserIsStaffPlug)
plug(Pleroma.Web.Plugs.IdempotencyPlug) plug(Pleroma.Web.Plugs.IdempotencyPlug)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end end
pipeline :require_admin do pipeline :require_admin do
@ -185,7 +179,6 @@ defmodule Pleroma.Web.Router do
plug(:browser) plug(:browser)
plug(:authenticate) plug(:authenticate)
plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug) plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end end
pipeline :well_known do pipeline :well_known do
@ -200,7 +193,6 @@ defmodule Pleroma.Web.Router do
pipeline :pleroma_api do pipeline :pleroma_api do
plug(:accepts, ["html", "json"]) plug(:accepts, ["html", "json"])
plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec)
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
end end
pipeline :mailbox_preview do pipeline :mailbox_preview do

View File

@ -155,16 +155,7 @@ def get_template_from_xml(body) do
end end
end end
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def find_lrdd_template(domain) do def find_lrdd_template(domain) do
@cachex.fetch!(:host_meta_cache, domain, fn _ ->
{:commit, fetch_lrdd_template(domain)}
end)
rescue
e -> {:error, "Cachex error: #{inspect(e)}"}
end
defp fetch_lrdd_template(domain) do
# WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1 # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1
meta_url = "https://#{domain}/.well-known/host-meta" meta_url = "https://#{domain}/.well-known/host-meta"
@ -177,7 +168,7 @@ defp fetch_lrdd_template(domain) do
end end
end end
defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do
case find_lrdd_template(domain) do case find_lrdd_template(domain) do
{:ok, template} -> {:ok, template} ->
String.replace(template, "{uri}", encoded_account) String.replace(template, "{uri}", encoded_account)
@ -187,11 +178,6 @@ defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_bin
end end
end end
defp get_address_from_domain(domain, account) when is_binary(domain) do
encoded_account = URI.encode("acct:#{account}")
get_address_from_domain(domain, encoded_account)
end
defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain} defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain}
@spec finger(String.t()) :: {:ok, map()} | {:error, any()} @spec finger(String.t()) :: {:ok, map()} | {:error, any()}
@ -206,7 +192,9 @@ def finger(account) do
URI.parse(account).host URI.parse(account).host
end end
with address when is_binary(address) <- get_address_from_domain(domain, account), encoded_account = URI.encode("acct:#{account}")
with address when is_binary(address) <- get_address_from_domain(domain, encoded_account),
{:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <- {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <-
HTTP.get( HTTP.get(
address, address,
@ -228,28 +216,10 @@ def finger(account) do
_ -> _ ->
{:error, {:content_type, nil}} {:error, {:content_type, nil}}
end end
|> case do
{:ok, data} -> validate_webfinger(address, data)
error -> error
end
else else
error -> error ->
Logger.debug("Couldn't finger #{account}: #{inspect(error)}") Logger.debug("Couldn't finger #{account}: #{inspect(error)}")
error error
end end
end end
defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do
with [_name, acct_host] <- String.split(acct, "@"),
{_, url} <- {:address, get_address_from_domain(acct_host, subject)},
%URI{host: request_host} <- URI.parse(request_url),
%URI{host: acct_host} <- URI.parse(url),
{_, true} <- {:hosts_match, acct_host == request_host} do
{:ok, data}
else
_ -> {:error, {:webfinger_invalid, request_url, data}}
end
end
defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}}
end end

View File

@ -8,7 +8,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Repo alias Pleroma.Repo
use Pleroma.Workers.WorkerHelper, queue: "slow" use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup"
@impl Oban.Worker @impl Oban.Worker
def perform(%Job{ def perform(%Job{

View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.BackupWorker do defmodule Pleroma.Workers.BackupWorker do
use Oban.Worker, queue: :slow, max_attempts: 1 use Oban.Worker, queue: :backup, max_attempts: 1
alias Oban.Job alias Oban.Job
alias Pleroma.User.Backup alias Pleroma.User.Backup

View File

@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do
import Ecto.Query import Ecto.Query
use Pleroma.Workers.WorkerHelper, queue: "background" use Pleroma.Workers.WorkerHelper, queue: "mailer"
@impl Oban.Worker @impl Oban.Worker
def perform(_job) do def perform(_job) do

Some files were not shown because too many files have changed in this diff Show More