Compare commits
No commits in common. "spc2" and "streams-fixes" have entirely different histories.
spc2
...
streams-fi
|
@ -1,8 +1,8 @@
|
|||
image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24
|
||||
image: git.pleroma.social:5050/pleroma/pleroma/ci-base
|
||||
|
||||
variables: &global_variables
|
||||
# Only used for the release
|
||||
ELIXIR_VER: 1.13.4
|
||||
ELIXIR_VER: 1.12.3
|
||||
POSTGRES_DB: pleroma_test
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
|
@ -72,7 +72,7 @@ check-changelog:
|
|||
tags:
|
||||
- amd64
|
||||
|
||||
build-1.13.4:
|
||||
build-1.12.3:
|
||||
extends:
|
||||
- .build_changes_policy
|
||||
- .using-ci-base
|
||||
|
@ -85,7 +85,7 @@ build-1.15.7-otp-25:
|
|||
- .build_changes_policy
|
||||
- .using-ci-base
|
||||
stage: build
|
||||
image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25
|
||||
image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15
|
||||
allow_failure: true
|
||||
script:
|
||||
- mix compile --force
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
ARG ELIXIR_IMG=hexpm/elixir
|
||||
ARG ELIXIR_VER=1.13.4
|
||||
ARG ERLANG_VER=24.3.4.15
|
||||
ARG ALPINE_VER=3.17.5
|
||||
ARG ELIXIR_VER=1.12.3
|
||||
ARG ERLANG_VER=24.2.1
|
||||
ARG ALPINE_VER=3.17.0
|
||||
|
||||
FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build
|
||||
|
||||
|
|
|
@ -1 +0,0 @@
|
|||
HTTP Security: By default, don't allow unsafe-eval. The setting needs to be changed to allow Flash emulation.
|
|
@ -1 +0,0 @@
|
|||
Uploader: Add support for uploading attachments using IPFS
|
|
@ -1 +0,0 @@
|
|||
Add NSFW-detecting MRF
|
|
@ -1 +0,0 @@
|
|||
Add DNSRBL MRF
|
|
@ -1 +0,0 @@
|
|||
Add options to the mix prune_objects task
|
|
@ -1 +0,0 @@
|
|||
Add Anti-mention Spam MRF backported from Rebased
|
|
@ -1 +0,0 @@
|
|||
HTTPSignaturePlug: Add :authorized_fetch_mode_exceptions configuration
|
|
@ -1 +0,0 @@
|
|||
Add an option to reject certain domains when authorized fetch is enabled.
|
|
@ -1 +0,0 @@
|
|||
Update Bandit to 1.5.2
|
|
@ -1 +0,0 @@
|
|||
Elixir 1.13 is the minimum required version.
|
|
@ -1 +0,0 @@
|
|||
Fix webfinger spoofing.
|
|
@ -1 +0,0 @@
|
|||
Includes scrobble when available in the account field with statuses
|
|
@ -1 +0,0 @@
|
|||
Logger metadata is now attached to some logs to help with troubleshooting and analysis
|
|
@ -1 +0,0 @@
|
|||
Ensure MediaProxy HTTP requests obey all the defined connection settings
|
|
@ -1 +0,0 @@
|
|||
Add missing indexes on foreign key relationships
|
|
@ -1 +0,0 @@
|
|||
Permit passing --chunk and --step values to the Pleroma.Search.Indexer Mix task
|
|
@ -1 +0,0 @@
|
|||
noop
|
|
@ -1 +0,0 @@
|
|||
Oban queues have refactored to simplify the queue design
|
|
@ -1 +0,0 @@
|
|||
HTTP connection pool adjustments
|
|
@ -1 +0,0 @@
|
|||
Update the documentation for configuring Prometheus metrics.
|
|
@ -1 +0,0 @@
|
|||
PromEx documentation
|
|
@ -1 +0,0 @@
|
|||
Add Qdrant/OpenAI embedding search
|
|
@ -1 +0,0 @@
|
|||
pleroma_ctl: Use realpath(1) instead of readlink(1)
|
|
@ -1 +0,0 @@
|
|||
A 422 error is returned when attempting to reply to a deleted status
|
|
@ -1 +0,0 @@
|
|||
Parsing of RichMedia TTLs for Amazon URLs when query parameters are nil
|
|
@ -1 +0,0 @@
|
|||
Monitoring of search backend health to control the processing of jobs in the search indexing Oban queue
|
|
@ -1 +0,0 @@
|
|||
Display reposted replies with exclude_replies: true
|
|
@ -1 +0,0 @@
|
|||
Support honk-style attachment summaries as alt-text.
|
|
@ -1 +0,0 @@
|
|||
Video thumbnails were not being generated due to a negative cache lookup logic error
|
|
@ -1 +0,0 @@
|
|||
Fix validate_webfinger when running a different domain for Webfinger
|
|
@ -1,8 +0,0 @@
|
|||
FROM elixir:1.13.4-otp-24
|
||||
|
||||
# Single RUN statement, otherwise intermediate images are created
|
||||
# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
|
||||
RUN apt-get update &&\
|
||||
apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\
|
||||
mix local.hex --force &&\
|
||||
mix local.rebar --force
|
|
@ -1 +0,0 @@
|
|||
docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 --push .
|
|
@ -1 +1 @@
|
|||
docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push .
|
||||
docker buildx build --platform linux/amd64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push .
|
||||
|
|
|
@ -82,10 +82,6 @@
|
|||
# region: "us-east-1", # may be required for Amazon AWS
|
||||
scheme: "https://"
|
||||
|
||||
config :pleroma, Pleroma.Uploaders.IPFS,
|
||||
post_gateway_url: nil,
|
||||
get_gateway_url: nil
|
||||
|
||||
config :pleroma, :emoji,
|
||||
shortcode_globs: ["/emoji/custom/**/*.png"],
|
||||
pack_extensions: [".png", ".gif"],
|
||||
|
@ -135,13 +131,13 @@
|
|||
config :logger, :console,
|
||||
level: :debug,
|
||||
format: "\n$time $metadata[$level] $message\n",
|
||||
metadata: [:actor, :path, :request_id, :type, :user]
|
||||
metadata: [:request_id]
|
||||
|
||||
config :logger, :ex_syslogger,
|
||||
level: :debug,
|
||||
ident: "pleroma",
|
||||
format: "$metadata[$level] $message",
|
||||
metadata: [:actor, :path, :request_id, :type, :user]
|
||||
metadata: [:request_id]
|
||||
|
||||
config :mime, :types, %{
|
||||
"application/xml" => ["xml"],
|
||||
|
@ -192,7 +188,6 @@
|
|||
allow_relay: true,
|
||||
public: true,
|
||||
quarantined_instances: [],
|
||||
rejected_instances: [],
|
||||
static_dir: "instance/static/",
|
||||
allowed_post_formats: [
|
||||
"text/plain",
|
||||
|
@ -411,23 +406,11 @@
|
|||
accept: [],
|
||||
reject: []
|
||||
|
||||
config :pleroma, :mrf_dnsrbl,
|
||||
nameserver: "127.0.0.1",
|
||||
port: 53,
|
||||
zone: "bl.pleroma.com"
|
||||
|
||||
# threshold of 7 days
|
||||
config :pleroma, :mrf_object_age,
|
||||
threshold: 604_800,
|
||||
actions: [:delist, :strip_followers]
|
||||
|
||||
config :pleroma, :mrf_nsfw_api,
|
||||
url: "http://127.0.0.1:5000/",
|
||||
threshold: 0.7,
|
||||
mark_sensitive: true,
|
||||
unlist: false,
|
||||
reject: false
|
||||
|
||||
config :pleroma, :mrf_follow_bot, follower_nickname: nil
|
||||
|
||||
config :pleroma, :mrf_inline_quote, template: "<bdi>RT:</bdi> {url}"
|
||||
|
@ -436,8 +419,6 @@
|
|||
mention_parent: true,
|
||||
mention_quoted: true
|
||||
|
||||
config :pleroma, :mrf_antimentionspam, user_age_limit: 30_000
|
||||
|
||||
config :pleroma, :rich_media,
|
||||
enabled: true,
|
||||
ignore_hosts: [],
|
||||
|
@ -520,8 +501,7 @@
|
|||
sts: false,
|
||||
sts_max_age: 31_536_000,
|
||||
ct_max_age: 2_592_000,
|
||||
referrer_policy: "same-origin",
|
||||
allow_unsafe_eval: false
|
||||
referrer_policy: "same-origin"
|
||||
|
||||
config :cors_plug,
|
||||
max_age: 86_400,
|
||||
|
@ -583,14 +563,24 @@
|
|||
log: false,
|
||||
queues: [
|
||||
activity_expiration: 10,
|
||||
token_expiration: 5,
|
||||
filter_expiration: 1,
|
||||
backup: 1,
|
||||
federator_incoming: 5,
|
||||
federator_outgoing: 5,
|
||||
ingestion_queue: 50,
|
||||
web_push: 50,
|
||||
mailer: 10,
|
||||
transmogrifier: 20,
|
||||
scheduled_activities: 10,
|
||||
poll_notifications: 10,
|
||||
background: 5,
|
||||
search_indexing: [limit: 10, paused: true],
|
||||
slow: 1
|
||||
remote_fetcher: 2,
|
||||
attachments_cleanup: 1,
|
||||
new_users_digest: 1,
|
||||
mute_expire: 5,
|
||||
search_indexing: 10,
|
||||
rich_media_expiration: 2
|
||||
],
|
||||
plugins: [Oban.Plugins.Pruner],
|
||||
crontab: [
|
||||
|
@ -814,7 +804,7 @@
|
|||
config :pleroma, configurable_from_database: false
|
||||
|
||||
config :pleroma, Pleroma.Repo,
|
||||
parameters: [gin_fuzzy_search_limit: "500"],
|
||||
parameters: [gin_fuzzy_search_limit: "500", jit: "off"],
|
||||
prepare: :unnamed
|
||||
|
||||
config :pleroma, :connections_pool,
|
||||
|
@ -828,27 +818,22 @@
|
|||
|
||||
config :pleroma, :pools,
|
||||
federation: [
|
||||
size: 75,
|
||||
max_waiting: 20,
|
||||
size: 50,
|
||||
max_waiting: 10,
|
||||
recv_timeout: 10_000
|
||||
],
|
||||
media: [
|
||||
size: 75,
|
||||
max_waiting: 20,
|
||||
recv_timeout: 15_000
|
||||
],
|
||||
rich_media: [
|
||||
size: 25,
|
||||
size: 50,
|
||||
max_waiting: 20,
|
||||
recv_timeout: 15_000
|
||||
],
|
||||
upload: [
|
||||
size: 25,
|
||||
max_waiting: 20,
|
||||
max_waiting: 5,
|
||||
recv_timeout: 15_000
|
||||
],
|
||||
default: [
|
||||
size: 50,
|
||||
size: 10,
|
||||
max_waiting: 2,
|
||||
recv_timeout: 5_000
|
||||
]
|
||||
|
@ -862,10 +847,6 @@
|
|||
max_connections: 50,
|
||||
timeout: 150_000
|
||||
],
|
||||
rich_media: [
|
||||
max_connections: 50,
|
||||
timeout: 150_000
|
||||
],
|
||||
upload: [
|
||||
max_connections: 25,
|
||||
timeout: 300_000
|
||||
|
@ -911,6 +892,8 @@
|
|||
process_chunk_size: 100
|
||||
|
||||
config :pleroma, ConcurrentLimiter, [
|
||||
{Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
|
||||
{Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]},
|
||||
{Pleroma.Search, [max_running: 30, max_waiting: 50]}
|
||||
]
|
||||
|
||||
|
@ -932,19 +915,6 @@
|
|||
|
||||
config :pleroma, Pleroma.Uploaders.Uploader, timeout: 30_000
|
||||
|
||||
config :pleroma, Pleroma.Search.QdrantSearch,
|
||||
qdrant_url: "http://127.0.0.1:6333/",
|
||||
qdrant_api_key: "",
|
||||
openai_url: "http://127.0.0.1:11345",
|
||||
# The healthcheck url has to be set to nil when used with the real openai
|
||||
# API, as it doesn't have a healthcheck endpoint.
|
||||
openai_healthcheck_url: "http://127.0.0.1:11345/health",
|
||||
openai_model: "snowflake/snowflake-arctic-embed-xs",
|
||||
openai_api_key: "",
|
||||
qdrant_index_configuration: %{
|
||||
vectors: %{size: 384, distance: "Cosine"}
|
||||
}
|
||||
|
||||
# Import environment specific config. This must remain at the bottom
|
||||
# of this file so it overrides the configuration defined above.
|
||||
import_config "#{Mix.env()}.exs"
|
||||
|
|
|
@ -136,31 +136,6 @@
|
|||
}
|
||||
]
|
||||
},
|
||||
%{
|
||||
group: :pleroma,
|
||||
key: Pleroma.Uploaders.IPFS,
|
||||
type: :group,
|
||||
description: "IPFS uploader-related settings",
|
||||
children: [
|
||||
%{
|
||||
key: :get_gateway_url,
|
||||
type: :string,
|
||||
description: "GET Gateway URL",
|
||||
suggestions: [
|
||||
"https://ipfs.mydomain.com/{CID}",
|
||||
"https://{CID}.ipfs.mydomain.com/"
|
||||
]
|
||||
},
|
||||
%{
|
||||
key: :post_gateway_url,
|
||||
type: :string,
|
||||
description: "POST Gateway URL",
|
||||
suggestions: [
|
||||
"http://localhost:5001/"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
%{
|
||||
group: :pleroma,
|
||||
key: Pleroma.Uploaders.S3,
|
||||
|
@ -774,18 +749,6 @@
|
|||
{"*.quarantined.com", "Reason"}
|
||||
]
|
||||
},
|
||||
%{
|
||||
key: :rejected_instances,
|
||||
type: {:list, :tuple},
|
||||
key_placeholder: "instance",
|
||||
value_placeholder: "reason",
|
||||
description:
|
||||
"List of ActivityPub instances to reject requests from if authorized_fetch_mode is enabled",
|
||||
suggestions: [
|
||||
{"rejected.com", "Reason"},
|
||||
{"*.rejected.com", "Reason"}
|
||||
]
|
||||
},
|
||||
%{
|
||||
key: :static_dir,
|
||||
type: :string,
|
||||
|
@ -1828,12 +1791,6 @@
|
|||
type: :boolean,
|
||||
description: "Require HTTP signatures for AP fetches"
|
||||
},
|
||||
%{
|
||||
key: :authorized_fetch_mode_exceptions,
|
||||
type: {:list, :string},
|
||||
description:
|
||||
"List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)"
|
||||
},
|
||||
%{
|
||||
key: :note_replies_output_limit,
|
||||
type: :integer,
|
||||
|
|
|
@ -35,8 +35,8 @@
|
|||
# configured to run both http and https servers on
|
||||
# different ports.
|
||||
|
||||
# Do not include timestamps in development logs
|
||||
config :logger, :console, format: "$metadata[$level] $message\n"
|
||||
# Do not include metadata nor timestamps in development logs
|
||||
config :logger, :console, format: "[$level] $message\n"
|
||||
|
||||
# Set a higher stacktrace during development. Avoid configuring such
|
||||
# in production as building large stacktraces may be expensive.
|
||||
|
|
|
@ -153,12 +153,6 @@
|
|||
config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock
|
||||
config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock
|
||||
config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock
|
||||
config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock
|
||||
config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.StaticStubbedConfigMock
|
||||
config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug, config_impl: Pleroma.StaticStubbedConfigMock
|
||||
|
||||
config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug,
|
||||
http_signatures_impl: Pleroma.StubbedHTTPSignaturesMock
|
||||
|
||||
peer_module =
|
||||
if String.to_integer(System.otp_release()) >= 25 do
|
||||
|
|
|
@ -21,18 +21,16 @@ Replaces embedded objects with references to them in the `objects` table. Only n
|
|||
mix pleroma.database remove_embedded_objects [option ...]
|
||||
```
|
||||
|
||||
|
||||
### Options
|
||||
- `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
|
||||
|
||||
## Prune old remote posts from the database
|
||||
|
||||
This will prune remote posts older than 90 days (configurable with [`config :pleroma, :instance, remote_post_retention_days`](../../configuration/cheatsheet.md#instance)) from the database. Pruned posts may be refetched in some cases.
|
||||
|
||||
!!! note
|
||||
The disk space will only be reclaimed after a proper vacuum. By default Postgresql does this for you on a regular basis, but if your instance has been running for a long time and there are many rows deleted, it may be advantageous to use `VACUUM FULL` (e.g. by using the `--vacuum` option).
|
||||
This will prune remote posts older than 90 days (configurable with [`config :pleroma, :instance, remote_post_retention_days`](../../configuration/cheatsheet.md#instance)) from the database, they will be refetched from source when accessed.
|
||||
|
||||
!!! danger
|
||||
You may run out of disk space during the execution of the task or vacuuming if you don't have about 1/3rds of the database size free. Vacuum causes a substantial increase in I/O traffic, and may lead to a degraded experience while it is running.
|
||||
The disk space will only be reclaimed after `VACUUM FULL`. You may run out of disk space during the execution of the task or vacuuming if you don't have about 1/3rds of the database size free.
|
||||
|
||||
=== "OTP"
|
||||
|
||||
|
@ -47,11 +45,7 @@ This will prune remote posts older than 90 days (configurable with [`config :ple
|
|||
```
|
||||
|
||||
### Options
|
||||
|
||||
- `--keep-threads` - Don't prune posts when they are part of a thread where at least one post has seen local interaction (e.g. one of the posts is a local post, or is favourited by a local user, or has been repeated by a local user...). It also won't delete posts when at least one of the posts in that thread is kept (e.g. because one of the posts has seen recent activity).
|
||||
- `--keep-non-public` - Keep non-public posts like DM's and followers-only, even if they are remote.
|
||||
- `--prune-orphaned-activities` - Also prune orphaned activities afterwards. Activities are things like Like, Create, Announce, Flag (aka reports). They can significantly help reduce the database size. Note: this can take a very long time.
|
||||
- `--vacuum` - Run `VACUUM FULL` after the objects are pruned. This should not be used on a regular basis, but is useful if your instance has been running for a long time before pruning.
|
||||
- `--vacuum` - run `VACUUM FULL` after the objects are pruned
|
||||
|
||||
## Create a conversation for all existing DMs
|
||||
|
||||
|
@ -99,9 +93,6 @@ Can be safely re-run
|
|||
|
||||
## Vacuum the database
|
||||
|
||||
!!! note
|
||||
By default Postgresql has an autovacuum deamon running. While the tasks described here can help in some cases, they shouldn't be needed on a regular basis. See [the Postgresql docs on vacuuming](https://www.postgresql.org/docs/current/sql-vacuum.html) for more information on this.
|
||||
|
||||
### Analyze
|
||||
|
||||
Running an `analyze` vacuum job can improve performance by updating statistics used by the query planner. **It is safe to cancel this.**
|
||||
|
|
|
@ -41,7 +41,6 @@ To add configuration to your config file, you can copy it from the base config.
|
|||
* `allow_relay`: Permits remote instances to subscribe to all public posts of your instance. This may increase the visibility of your instance.
|
||||
* `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details.
|
||||
* `quarantined_instances`: ActivityPub instances where private (DMs, followers-only) activities will not be send.
|
||||
* `rejected_instances`: ActivityPub instances to reject requests from if authorized_fetch_mode is enabled.
|
||||
* `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML).
|
||||
* `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with
|
||||
older software for theses nicknames.
|
||||
|
@ -285,7 +284,6 @@ Notes:
|
|||
* `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question
|
||||
* `sign_object_fetches`: Sign object fetches with HTTP signatures
|
||||
* `authorized_fetch_mode`: Require HTTP signatures for AP fetches
|
||||
* `authorized_fetch_mode_exceptions`: List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)
|
||||
|
||||
## Pleroma.User
|
||||
|
||||
|
@ -474,7 +472,6 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
|
|||
* ``ct_max_age``: The maximum age for the `Expect-CT` header if sent.
|
||||
* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`.
|
||||
* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header.
|
||||
* `allow_unsafe_eval`: Adds `wasm-unsafe-eval` to the CSP header. Needed for some non-essential frontend features like Flash emulation.
|
||||
|
||||
### Pleroma.Web.Plugs.RemoteIp
|
||||
|
||||
|
@ -664,19 +661,6 @@ config :ex_aws, :s3,
|
|||
host: "s3.eu-central-1.amazonaws.com"
|
||||
```
|
||||
|
||||
#### Pleroma.Uploaders.IPFS
|
||||
|
||||
* `post_gateway_url`: URL with port of POST Gateway (unauthenticated)
|
||||
* `get_gateway_url`: URL of public GET Gateway
|
||||
|
||||
Example:
|
||||
|
||||
```elixir
|
||||
config :pleroma, Pleroma.Uploaders.IPFS,
|
||||
post_gateway_url: "http://localhost:5001",
|
||||
get_gateway_url: "http://{CID}.ipfs.mydomain.com"
|
||||
```
|
||||
|
||||
### Upload filters
|
||||
|
||||
#### Pleroma.Upload.Filter.AnonymizeFilename
|
||||
|
|
|
@ -10,30 +10,6 @@ To use built-in search that has no external dependencies, set the search module
|
|||
|
||||
While it has no external dependencies, it has problems with performance and relevancy.
|
||||
|
||||
## QdrantSearch
|
||||
|
||||
This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`.
|
||||
|
||||
The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. To use it, set the search provider and run the fastembed server, see the README in `supplemental/search/fastembed-api`:
|
||||
|
||||
> config :pleroma, Pleroma.Search, module: Pleroma.Search.QdrantSearch
|
||||
|
||||
Then, start the Qdrant server, see [here](https://qdrant.tech/documentation/quick-start/) for instructions.
|
||||
|
||||
You will also need to create the Qdrant index once by running `mix pleroma.search.indexer create_index`. Running `mix pleroma.search.indexer index` will retroactively index the last 100_000 activities.
|
||||
|
||||
### Indexing and model options
|
||||
|
||||
To see the available configuration options, check out the QdrantSearch section in `config/config.exs`.
|
||||
|
||||
The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/). See also [this blog post](https://qdrant.tech/articles/memory-consumption/) that goes into detail.
|
||||
|
||||
Different embedding models will need different vector size settings. You can see a list of the models supported by the fastembed server [here](https://qdrant.github.io/fastembed/examples/Supported_Models), including their vector dimensions. These vector dimensions need to be set in the `qdrant_index_configuration`.
|
||||
|
||||
E.g, If you want to use `sentence-transformers/all-MiniLM-L6-v2` as a model, you will not need to adjust things, because it and `snowflake-arctic-embed-xs` are both 384 dimensional models. If you want to use `snowflake/snowflake-arctic-embed-l`, you will need to adjust the `size` parameter in the `qdrant_index_configuration` to 1024, as it has a dimension of 1024.
|
||||
|
||||
When using a different model, you will need do drop the index and recreate it (`mix pleroma.search.indexer drop_index` and `mix pleroma.search.indexer create_index`), as the different embeddings are not compatible with each other.
|
||||
|
||||
## Meilisearch
|
||||
|
||||
Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 million
|
||||
|
|
|
@ -295,7 +295,9 @@ See [Admin-API](admin_api.md)
|
|||
"id": "9umDrYheeY451cQnEe",
|
||||
"name": "Read later",
|
||||
"emoji": "🕓",
|
||||
"emoji_url": null
|
||||
"source": {
|
||||
"emoji": "🕓"
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
|
|
|
@ -1,47 +1,44 @@
|
|||
# Prometheus / OpenTelemetry Metrics
|
||||
# Prometheus Metrics
|
||||
|
||||
Pleroma includes support for exporting metrics via the [prom_ex](https://github.com/akoutmos/prom_ex) library.
|
||||
The metrics are exposed by a dedicated webserver/port to improve privacy and security.
|
||||
Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library.
|
||||
|
||||
Config example:
|
||||
|
||||
```
|
||||
config :pleroma, Pleroma.PromEx,
|
||||
disabled: false,
|
||||
manual_metrics_start_delay: :no_delay,
|
||||
drop_metrics_groups: [],
|
||||
grafana: [
|
||||
host: System.get_env("GRAFANA_HOST", "http://localhost:3000"),
|
||||
auth_token: System.get_env("GRAFANA_TOKEN"),
|
||||
upload_dashboards_on_start: false,
|
||||
folder_name: "BEAM",
|
||||
annotate_app_lifecycle: true
|
||||
],
|
||||
metrics_server: [
|
||||
port: 4021,
|
||||
path: "/metrics",
|
||||
protocol: :http,
|
||||
pool_size: 5,
|
||||
cowboy_opts: [],
|
||||
auth_strategy: :none
|
||||
],
|
||||
datasource: "Prometheus"
|
||||
|
||||
config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
|
||||
enabled: true,
|
||||
auth: {:basic, "myusername", "mypassword"},
|
||||
ip_whitelist: ["127.0.0.1"],
|
||||
path: "/api/pleroma/app_metrics",
|
||||
format: :text
|
||||
```
|
||||
|
||||
PromEx supports the ability to automatically publish dashboards to your Grafana server as well as register Annotations. If you do not wish to configure this capability you must generate the dashboard JSON files and import them directly. You can find the mix commands in the upstream [documentation](https://hexdocs.pm/prom_ex/Mix.Tasks.PromEx.Dashboard.Export.html). You can find the list of modules enabled in Pleroma for which you should generate dashboards for by examining the contents of the `lib/pleroma/prom_ex.ex` module.
|
||||
* `enabled` (Pleroma extension) enables the endpoint
|
||||
* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs
|
||||
* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation)
|
||||
* `format` sets the output format (`:text` or `:protobuf`)
|
||||
* `path` sets the path to app metrics page
|
||||
|
||||
## prometheus.yml
|
||||
|
||||
The following is a bare minimum config example to use with [Prometheus](https://prometheus.io) or Prometheus-compatible software like [VictoriaMetrics](https://victoriametrics.com).
|
||||
## `/api/pleroma/app_metrics`
|
||||
|
||||
### Exports Prometheus application metrics
|
||||
|
||||
* Method: `GET`
|
||||
* Authentication: not required by default (see configuration options above)
|
||||
* Params: none
|
||||
* Response: text
|
||||
|
||||
## Grafana
|
||||
|
||||
### Config example
|
||||
|
||||
The following is a config example to use with [Grafana](https://grafana.com)
|
||||
|
||||
```
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'pleroma'
|
||||
scheme: http
|
||||
- job_name: 'beam'
|
||||
metrics_path: /api/pleroma/app_metrics
|
||||
scheme: https
|
||||
static_configs:
|
||||
- targets: ['pleroma.soykaf.com:4021']
|
||||
- targets: ['pleroma.soykaf.com']
|
||||
```
|
||||
|
|
|
@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have
|
|||
|
||||
- PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので,[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください)
|
||||
- `postgresql-contrib` 11.0以上 (同上)
|
||||
- Elixir 1.13 以上 ([Debianのリポジトリからインストールしないこと!!! ここからインストールすること!](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
|
||||
- Elixir 1.8 以上 ([Debianのリポジトリからインストールしないこと!!! ここからインストールすること!](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
|
||||
- `erlang-dev`
|
||||
- `erlang-nox`
|
||||
- `git`
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
## Required dependencies
|
||||
|
||||
* PostgreSQL >=11.0
|
||||
* Elixir >=1.13.0 <1.15
|
||||
* Elixir >=1.11.0 <1.15
|
||||
* Erlang OTP >=22.2.0 (supported: <27)
|
||||
* git
|
||||
* file / libmagic
|
||||
|
|
|
@ -1,15 +0,0 @@
|
|||
[Unit]
|
||||
Description=NSFW API
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
TimeoutStartSec=0
|
||||
Restart=always
|
||||
ExecStartPre=-/usr/bin/docker stop %n
|
||||
ExecStartPre=-/usr/bin/docker rm %n
|
||||
ExecStartPre=/usr/bin/docker pull eugencepoi/nsfw_api:latest
|
||||
ExecStart=/usr/bin/docker run --rm -p 127.0.0.1:5000:5000/tcp --env PORT=5000 --name %n eugencepoi/nsfw_api:latest
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -67,168 +67,43 @@ def run(["prune_objects" | args]) do
|
|||
OptionParser.parse(
|
||||
args,
|
||||
strict: [
|
||||
vacuum: :boolean,
|
||||
keep_threads: :boolean,
|
||||
keep_non_public: :boolean,
|
||||
prune_orphaned_activities: :boolean
|
||||
vacuum: :boolean
|
||||
]
|
||||
)
|
||||
|
||||
start_pleroma()
|
||||
|
||||
deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
|
||||
time_deadline = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400))
|
||||
|
||||
log_message = "Pruning objects older than #{deadline} days"
|
||||
Logger.info("Pruning objects older than #{deadline} days")
|
||||
|
||||
log_message =
|
||||
if Keyword.get(options, :keep_non_public) do
|
||||
log_message <> ", keeping non public posts"
|
||||
else
|
||||
log_message
|
||||
end
|
||||
time_deadline =
|
||||
NaiveDateTime.utc_now()
|
||||
|> NaiveDateTime.add(-(deadline * 86_400))
|
||||
|
||||
log_message =
|
||||
if Keyword.get(options, :keep_threads) do
|
||||
log_message <> ", keeping threads intact"
|
||||
else
|
||||
log_message
|
||||
end
|
||||
|
||||
log_message =
|
||||
if Keyword.get(options, :prune_orphaned_activities) do
|
||||
log_message <> ", pruning orphaned activities"
|
||||
else
|
||||
log_message
|
||||
end
|
||||
|
||||
log_message =
|
||||
if Keyword.get(options, :vacuum) do
|
||||
log_message <>
|
||||
", doing a full vacuum (you shouldn't do this as a recurring maintanance task)"
|
||||
else
|
||||
log_message
|
||||
end
|
||||
|
||||
Logger.info(log_message)
|
||||
|
||||
if Keyword.get(options, :keep_threads) do
|
||||
# We want to delete objects from threads where
|
||||
# 1. the newest post is still old
|
||||
# 2. none of the activities is local
|
||||
# 3. none of the activities is bookmarked
|
||||
# 4. optionally none of the posts is non-public
|
||||
deletable_context =
|
||||
if Keyword.get(options, :keep_non_public) do
|
||||
Pleroma.Activity
|
||||
|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
|
||||
|> group_by([a], fragment("? ->> 'context'::text", a.data))
|
||||
|> having(
|
||||
[a],
|
||||
not fragment(
|
||||
# Posts (checked on Create Activity) is non-public
|
||||
"bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')",
|
||||
a.data,
|
||||
^Pleroma.Constants.as_public(),
|
||||
a.data,
|
||||
^Pleroma.Constants.as_public(),
|
||||
a.data
|
||||
)
|
||||
)
|
||||
else
|
||||
Pleroma.Activity
|
||||
|> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
|
||||
|> group_by([a], fragment("? ->> 'context'::text", a.data))
|
||||
end
|
||||
|> having([a], max(a.updated_at) < ^time_deadline)
|
||||
|> having([a], not fragment("bool_or(?)", a.local))
|
||||
|> having([_, b], fragment("max(?::text) is null", b.id))
|
||||
|> select([a], fragment("? ->> 'context'::text", a.data))
|
||||
|
||||
Pleroma.Object
|
||||
|> where([o], fragment("? ->> 'context'::text", o.data) in subquery(deletable_context))
|
||||
else
|
||||
if Keyword.get(options, :keep_non_public) do
|
||||
Pleroma.Object
|
||||
|> where(
|
||||
[o],
|
||||
fragment(
|
||||
"?->'to' \\? ? OR ?->'cc' \\? ?",
|
||||
o.data,
|
||||
^Pleroma.Constants.as_public(),
|
||||
o.data,
|
||||
^Pleroma.Constants.as_public()
|
||||
)
|
||||
)
|
||||
else
|
||||
Pleroma.Object
|
||||
end
|
||||
|> where([o], o.updated_at < ^time_deadline)
|
||||
|> where(
|
||||
[o],
|
||||
from(o in Object,
|
||||
where:
|
||||
fragment(
|
||||
"?->'to' \\? ? OR ?->'cc' \\? ?",
|
||||
o.data,
|
||||
^Pleroma.Constants.as_public(),
|
||||
o.data,
|
||||
^Pleroma.Constants.as_public()
|
||||
),
|
||||
where: o.inserted_at < ^time_deadline,
|
||||
where:
|
||||
fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
|
||||
)
|
||||
end
|
||||
)
|
||||
|> Repo.delete_all(timeout: :infinity)
|
||||
|
||||
if !Keyword.get(options, :keep_threads) do
|
||||
# Without the --keep-threads option, it's possible that bookmarked
|
||||
# objects have been deleted. We remove the corresponding bookmarks.
|
||||
"""
|
||||
delete from public.bookmarks
|
||||
where id in (
|
||||
select b.id from public.bookmarks b
|
||||
left join public.activities a on b.activity_id = a.id
|
||||
left join public.objects o on a."data" ->> 'object' = o.data ->> 'id'
|
||||
where o.id is null
|
||||
)
|
||||
"""
|
||||
|> Repo.query([], timeout: :infinity)
|
||||
end
|
||||
|
||||
if Keyword.get(options, :prune_orphaned_activities) do
|
||||
# Prune activities who link to a single object
|
||||
"""
|
||||
delete from public.activities
|
||||
where id in (
|
||||
select a.id from public.activities a
|
||||
left join public.objects o on a.data ->> 'object' = o.data ->> 'id'
|
||||
left join public.activities a2 on a.data ->> 'object' = a2.data ->> 'id'
|
||||
left join public.users u on a.data ->> 'object' = u.ap_id
|
||||
where not a.local
|
||||
and jsonb_typeof(a."data" -> 'object') = 'string'
|
||||
and o.id is null
|
||||
and a2.id is null
|
||||
and u.id is null
|
||||
)
|
||||
"""
|
||||
|> Repo.query([], timeout: :infinity)
|
||||
|
||||
# Prune activities who link to an array of objects
|
||||
"""
|
||||
delete from public.activities
|
||||
where id in (
|
||||
select a.id from public.activities a
|
||||
join json_array_elements_text((a."data" -> 'object')::json) as j on jsonb_typeof(a."data" -> 'object') = 'array'
|
||||
left join public.objects o on j.value = o.data ->> 'id'
|
||||
left join public.activities a2 on j.value = a2.data ->> 'id'
|
||||
left join public.users u on j.value = u.ap_id
|
||||
group by a.id
|
||||
having max(o.data ->> 'id') is null
|
||||
and max(a2.data ->> 'id') is null
|
||||
and max(u.ap_id) is null
|
||||
)
|
||||
"""
|
||||
|> Repo.query([], timeout: :infinity)
|
||||
end
|
||||
|
||||
"""
|
||||
prune_hashtags_query = """
|
||||
DELETE FROM hashtags AS ht
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM hashtags_objects hto
|
||||
WHERE ht.id = hto.hashtag_id)
|
||||
"""
|
||||
|> Repo.query()
|
||||
|
||||
Repo.query(prune_hashtags_query)
|
||||
|
||||
if Keyword.get(options, :vacuum) do
|
||||
Maintenance.vacuum("full")
|
||||
|
|
|
@ -1,83 +0,0 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Mix.Tasks.Pleroma.Search.Indexer do
|
||||
import Mix.Pleroma
|
||||
import Ecto.Query
|
||||
|
||||
alias Pleroma.Workers.SearchIndexingWorker
|
||||
|
||||
def run(["create_index"]) do
|
||||
start_pleroma()
|
||||
|
||||
with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do
|
||||
IO.puts("Index created")
|
||||
else
|
||||
e -> IO.puts("Could not create index: #{inspect(e)}")
|
||||
end
|
||||
end
|
||||
|
||||
def run(["drop_index"]) do
|
||||
start_pleroma()
|
||||
|
||||
with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do
|
||||
IO.puts("Index dropped")
|
||||
else
|
||||
e -> IO.puts("Could not drop index: #{inspect(e)}")
|
||||
end
|
||||
end
|
||||
|
||||
def run(["index" | options]) do
|
||||
{options, [], []} =
|
||||
OptionParser.parse(
|
||||
options,
|
||||
strict: [
|
||||
chunk: :integer,
|
||||
limit: :integer,
|
||||
step: :integer
|
||||
]
|
||||
)
|
||||
|
||||
start_pleroma()
|
||||
|
||||
chunk_size = Keyword.get(options, :chunk, 100)
|
||||
limit = Keyword.get(options, :limit, 100_000)
|
||||
per_step = Keyword.get(options, :step, 1000)
|
||||
|
||||
chunks = max(div(limit, per_step), 1)
|
||||
|
||||
1..chunks
|
||||
|> Enum.each(fn step ->
|
||||
q =
|
||||
from(a in Pleroma.Activity,
|
||||
limit: ^per_step,
|
||||
offset: ^per_step * (^step - 1),
|
||||
select: [:id],
|
||||
order_by: [desc: :id]
|
||||
)
|
||||
|
||||
{:ok, ids} =
|
||||
Pleroma.Repo.transaction(fn ->
|
||||
Pleroma.Repo.stream(q, timeout: :infinity)
|
||||
|> Enum.map(fn a ->
|
||||
a.id
|
||||
end)
|
||||
end)
|
||||
|
||||
IO.puts("Got #{length(ids)} activities, adding to indexer")
|
||||
|
||||
ids
|
||||
|> Enum.chunk_every(chunk_size)
|
||||
|> Enum.each(fn chunk ->
|
||||
IO.puts("Adding #{length(chunk)} activities to indexing queue")
|
||||
|
||||
chunk
|
||||
|> Enum.map(fn id ->
|
||||
SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
|
||||
end)
|
||||
|> Oban.insert_all()
|
||||
end)
|
||||
end)
|
||||
end
|
||||
end
|
|
@ -14,7 +14,6 @@ defmodule Pleroma.Application do
|
|||
@name Mix.Project.config()[:name]
|
||||
@version Mix.Project.config()[:version]
|
||||
@repository Mix.Project.config()[:source_url]
|
||||
@compile_env Mix.env()
|
||||
|
||||
def name, do: @name
|
||||
def version, do: @version
|
||||
|
@ -52,11 +51,7 @@ def start(_type, _args) do
|
|||
Pleroma.HTML.compile_scrubbers()
|
||||
Pleroma.Config.Oban.warn()
|
||||
Config.DeprecationWarnings.warn()
|
||||
|
||||
if @compile_env != :test do
|
||||
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
|
||||
end
|
||||
|
||||
Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
|
||||
Pleroma.ApplicationRequirements.verify!()
|
||||
load_custom_modules()
|
||||
Pleroma.Docs.JSON.compile()
|
||||
|
@ -114,8 +109,7 @@ def start(_type, _args) do
|
|||
streamer_registry() ++
|
||||
background_migrators() ++
|
||||
shout_child(shout_enabled?()) ++
|
||||
[Pleroma.Gopher.Server] ++
|
||||
[Pleroma.Search.Healthcheck]
|
||||
[Pleroma.Gopher.Server]
|
||||
|
||||
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
|
||||
# for other strategies and supported options
|
||||
|
@ -168,8 +162,7 @@ defp cachex_children do
|
|||
expiration: chat_message_id_idempotency_key_expiration(),
|
||||
limit: 500_000
|
||||
),
|
||||
build_cachex("rel_me", limit: 2500),
|
||||
build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000)
|
||||
build_cachex("rel_me", limit: 2500)
|
||||
]
|
||||
end
|
||||
|
||||
|
|
|
@ -16,15 +16,4 @@ def parse_address(ip) when is_binary(ip) do
|
|||
def parse_address(ip) do
|
||||
:inet.parse_address(ip)
|
||||
end
|
||||
|
||||
def parse_cidr(proxy) when is_binary(proxy) do
|
||||
proxy =
|
||||
cond do
|
||||
"/" in String.codepoints(proxy) -> proxy
|
||||
InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
|
||||
InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
|
||||
end
|
||||
|
||||
InetCidr.parse_cidr!(proxy, true)
|
||||
end
|
||||
end
|
||||
|
|
|
@ -25,7 +25,7 @@ def missing_dependencies do
|
|||
end
|
||||
|
||||
def image_resize(url, options) do
|
||||
with {:ok, env} <- HTTP.get(url, [], http_client_opts()),
|
||||
with {:ok, env} <- HTTP.get(url, [], pool: :media),
|
||||
{:ok, resized} <-
|
||||
Operation.thumbnail_buffer(env.body, options.max_width,
|
||||
height: options.max_height,
|
||||
|
@ -45,8 +45,8 @@ def image_resize(url, options) do
|
|||
@spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()}
|
||||
def video_framegrab(url) do
|
||||
with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
|
||||
{:ok, false} <- @cachex.exists?(:failed_media_helper_cache, url),
|
||||
{:ok, env} <- HTTP.get(url, [], http_client_opts()),
|
||||
false <- @cachex.exists?(:failed_media_helper_cache, url),
|
||||
{:ok, env} <- HTTP.get(url, [], pool: :media),
|
||||
{:ok, pid} <- StringIO.open(env.body) do
|
||||
body_stream = IO.binstream(pid, 1)
|
||||
|
||||
|
@ -71,19 +71,17 @@ def video_framegrab(url) do
|
|||
end)
|
||||
|
||||
case Task.yield(task, 5_000) do
|
||||
{:ok, result} ->
|
||||
{:ok, result}
|
||||
|
||||
_ ->
|
||||
nil ->
|
||||
Task.shutdown(task)
|
||||
@cachex.put(:failed_media_helper_cache, url, nil)
|
||||
{:error, {:ffmpeg, :timeout}}
|
||||
|
||||
result ->
|
||||
{:ok, result}
|
||||
end
|
||||
else
|
||||
nil -> {:error, {:ffmpeg, :command_not_found}}
|
||||
{:error, _} = error -> error
|
||||
end
|
||||
end
|
||||
|
||||
defp http_client_opts, do: Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
|
||||
end
|
||||
|
|
|
@ -1,4 +0,0 @@
|
|||
defmodule Pleroma.HTTPSignaturesAPI do
|
||||
@callback validate_conn(conn :: Plug.Conn.t()) :: boolean
|
||||
@callback signature_for_conn(conn :: Plug.Conn.t()) :: map
|
||||
end
|
|
@ -489,7 +489,7 @@ def create_poll_notifications(%Activity{} = activity) do
|
|||
|
||||
NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1
|
||||
"""
|
||||
@spec get_notified_from_activity(Activity.t(), boolean()) :: list(User.t())
|
||||
@spec get_notified_from_activity(Activity.t(), boolean()) :: {list(User.t()), list(User.t())}
|
||||
def get_notified_from_activity(activity, local_only \\ true)
|
||||
|
||||
def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
|
||||
|
|
|
@ -204,7 +204,7 @@ def due_activities(offset \\ 0) do
|
|||
|
||||
def job_query(scheduled_activity_id) do
|
||||
from(j in Oban.Job,
|
||||
where: j.queue == "federator_outgoing",
|
||||
where: j.queue == "scheduled_activities",
|
||||
where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id))
|
||||
)
|
||||
end
|
||||
|
|
|
@ -10,12 +10,8 @@ def remove_from_index(%Pleroma.Object{id: object_id}) do
|
|||
end
|
||||
|
||||
def search(query, options) do
|
||||
search_module = Pleroma.Config.get([Pleroma.Search, :module])
|
||||
search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
|
||||
|
||||
search_module.search(options[:for_user], query, options)
|
||||
end
|
||||
|
||||
def healthcheck_endpoints do
|
||||
search_module = Pleroma.Config.get([Pleroma.Search, :module])
|
||||
search_module.healthcheck_endpoints
|
||||
end
|
||||
end
|
||||
|
|
|
@ -28,7 +28,7 @@ def search(user, search_query, options \\ []) do
|
|||
|> Activity.with_preloaded_object()
|
||||
|> Activity.restrict_deactivated_users()
|
||||
|> restrict_public(user)
|
||||
|> query_with(index_type, search_query)
|
||||
|> query_with(index_type, search_query, :websearch)
|
||||
|> maybe_restrict_local(user)
|
||||
|> maybe_restrict_author(author)
|
||||
|> maybe_restrict_blocked(user)
|
||||
|
@ -48,15 +48,6 @@ def add_to_index(_activity), do: :ok
|
|||
@impl true
|
||||
def remove_from_index(_object), do: :ok
|
||||
|
||||
@impl true
|
||||
def create_index, do: :ok
|
||||
|
||||
@impl true
|
||||
def drop_index, do: :ok
|
||||
|
||||
@impl true
|
||||
def healthcheck_endpoints, do: nil
|
||||
|
||||
def maybe_restrict_author(query, %User{} = author) do
|
||||
Activity.Queries.by_author(query, author)
|
||||
end
|
||||
|
@ -88,7 +79,25 @@ defp restrict_public(q, _user) do
|
|||
)
|
||||
end
|
||||
|
||||
defp query_with(q, :gin, search_query) do
|
||||
defp query_with(q, :gin, search_query, :plain) do
|
||||
%{rows: [[tsc]]} =
|
||||
Ecto.Adapters.SQL.query!(
|
||||
Pleroma.Repo,
|
||||
"select current_setting('default_text_search_config')::regconfig::oid;"
|
||||
)
|
||||
|
||||
from([a, o] in q,
|
||||
where:
|
||||
fragment(
|
||||
"to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)",
|
||||
^tsc,
|
||||
o.data,
|
||||
^search_query
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
defp query_with(q, :gin, search_query, :websearch) do
|
||||
%{rows: [[tsc]]} =
|
||||
Ecto.Adapters.SQL.query!(
|
||||
Pleroma.Repo,
|
||||
|
@ -106,7 +115,19 @@ defp query_with(q, :gin, search_query) do
|
|||
)
|
||||
end
|
||||
|
||||
defp query_with(q, :rum, search_query) do
|
||||
defp query_with(q, :rum, search_query, :plain) do
|
||||
from([a, o] in q,
|
||||
where:
|
||||
fragment(
|
||||
"? @@ plainto_tsquery(?)",
|
||||
o.fts_content,
|
||||
^search_query
|
||||
),
|
||||
order_by: [fragment("? <=> now()::date", o.inserted_at)]
|
||||
)
|
||||
end
|
||||
|
||||
defp query_with(q, :rum, search_query, :websearch) do
|
||||
from([a, o] in q,
|
||||
where:
|
||||
fragment(
|
||||
|
|
|
@ -1,86 +0,0 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
defmodule Pleroma.Search.Healthcheck do
|
||||
@doc """
|
||||
Monitors health of search backend to control processing of events based on health and availability.
|
||||
"""
|
||||
use GenServer
|
||||
require Logger
|
||||
|
||||
@queue :search_indexing
|
||||
@tick :timer.seconds(5)
|
||||
@timeout :timer.seconds(2)
|
||||
|
||||
def start_link(_) do
|
||||
GenServer.start_link(__MODULE__, [], name: __MODULE__)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def init(_) do
|
||||
state = %{healthy: false}
|
||||
{:ok, state, {:continue, :start}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_continue(:start, state) do
|
||||
tick()
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(:check, state) do
|
||||
urls = Pleroma.Search.healthcheck_endpoints()
|
||||
|
||||
new_state =
|
||||
if check(urls) do
|
||||
Oban.resume_queue(queue: @queue)
|
||||
Map.put(state, :healthy, true)
|
||||
else
|
||||
Oban.pause_queue(queue: @queue)
|
||||
Map.put(state, :healthy, false)
|
||||
end
|
||||
|
||||
maybe_log_state_change(state, new_state)
|
||||
|
||||
tick()
|
||||
{:noreply, new_state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call(:state, _from, state) do
|
||||
{:reply, state, state, :hibernate}
|
||||
end
|
||||
|
||||
def state, do: GenServer.call(__MODULE__, :state)
|
||||
|
||||
def check([]), do: true
|
||||
|
||||
def check(urls) when is_list(urls) do
|
||||
Enum.all?(
|
||||
urls,
|
||||
fn url ->
|
||||
case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do
|
||||
{:ok, %{status: 200}} -> true
|
||||
_ -> false
|
||||
end
|
||||
end
|
||||
)
|
||||
end
|
||||
|
||||
def check(_), do: true
|
||||
|
||||
defp tick do
|
||||
Process.send_after(self(), :check, @tick)
|
||||
end
|
||||
|
||||
defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do
|
||||
Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure")
|
||||
end
|
||||
|
||||
defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do
|
||||
Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass")
|
||||
end
|
||||
|
||||
defp maybe_log_state_change(_, _), do: :ok
|
||||
end
|
|
@ -10,12 +10,6 @@ defmodule Pleroma.Search.Meilisearch do
|
|||
|
||||
@behaviour Pleroma.Search.SearchBackend
|
||||
|
||||
@impl true
|
||||
def create_index, do: :ok
|
||||
|
||||
@impl true
|
||||
def drop_index, do: :ok
|
||||
|
||||
defp meili_headers do
|
||||
private_key = Config.get([Pleroma.Search.Meilisearch, :private_key])
|
||||
|
||||
|
@ -184,15 +178,4 @@ def add_to_index(activity) do
|
|||
def remove_from_index(object) do
|
||||
meili_delete("/indexes/objects/documents/#{object.id}")
|
||||
end
|
||||
|
||||
@impl true
|
||||
def healthcheck_endpoints do
|
||||
endpoint =
|
||||
Config.get([Pleroma.Search.Meilisearch, :url])
|
||||
|> URI.parse()
|
||||
|> Map.put(:path, "/health")
|
||||
|> URI.to_string()
|
||||
|
||||
[endpoint]
|
||||
end
|
||||
end
|
||||
|
|
|
@ -1,182 +0,0 @@
|
|||
defmodule Pleroma.Search.QdrantSearch do
|
||||
@behaviour Pleroma.Search.SearchBackend
|
||||
import Ecto.Query
|
||||
|
||||
alias Pleroma.Activity
|
||||
alias Pleroma.Config.Getting, as: Config
|
||||
|
||||
alias __MODULE__.OpenAIClient
|
||||
alias __MODULE__.QdrantClient
|
||||
|
||||
import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1]
|
||||
import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3]
|
||||
|
||||
@impl true
|
||||
def create_index do
|
||||
payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration])
|
||||
|
||||
with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do
|
||||
:ok
|
||||
else
|
||||
e -> {:error, e}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def drop_index do
|
||||
with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do
|
||||
:ok
|
||||
else
|
||||
e -> {:error, e}
|
||||
end
|
||||
end
|
||||
|
||||
def get_embedding(text) do
|
||||
with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <-
|
||||
OpenAIClient.post("/v1/embeddings", %{
|
||||
input: text,
|
||||
model: Config.get([Pleroma.Search.QdrantSearch, :openai_model])
|
||||
}) do
|
||||
{:ok, embedding}
|
||||
else
|
||||
_ ->
|
||||
{:error, "Failed to get embedding"}
|
||||
end
|
||||
end
|
||||
|
||||
defp actor_from_activity(%{data: %{"actor" => actor}}) do
|
||||
actor
|
||||
end
|
||||
|
||||
defp actor_from_activity(_), do: nil
|
||||
|
||||
defp build_index_payload(activity, embedding) do
|
||||
actor = actor_from_activity(activity)
|
||||
published_at = activity.data["published"]
|
||||
|
||||
%{
|
||||
points: [
|
||||
%{
|
||||
id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(),
|
||||
vector: embedding,
|
||||
payload: %{actor: actor, published_at: published_at}
|
||||
}
|
||||
]
|
||||
}
|
||||
end
|
||||
|
||||
defp build_search_payload(embedding, options) do
|
||||
base = %{
|
||||
vector: embedding,
|
||||
limit: options[:limit] || 20,
|
||||
offset: options[:offset] || 0
|
||||
}
|
||||
|
||||
if author = options[:author] do
|
||||
Map.put(base, :filter, %{
|
||||
must: [%{key: "actor", match: %{value: author.ap_id}}]
|
||||
})
|
||||
else
|
||||
base
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def add_to_index(activity) do
|
||||
# This will only index public or unlisted notes
|
||||
maybe_search_data = object_to_search_data(activity.object)
|
||||
|
||||
if activity.data["type"] == "Create" and maybe_search_data do
|
||||
with {:ok, embedding} <- get_embedding(maybe_search_data.content),
|
||||
{:ok, %{status: 200}} <-
|
||||
QdrantClient.put(
|
||||
"/collections/posts/points",
|
||||
build_index_payload(activity, embedding)
|
||||
) do
|
||||
:ok
|
||||
else
|
||||
e -> {:error, e}
|
||||
end
|
||||
else
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def remove_from_index(object) do
|
||||
activity = Activity.get_by_object_ap_id_with_object(object.data["id"])
|
||||
id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()
|
||||
|
||||
with {:ok, %{status: 200}} <-
|
||||
QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do
|
||||
:ok
|
||||
else
|
||||
e -> {:error, e}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def search(user, original_query, options) do
|
||||
query = "Represent this sentence for searching relevant passages: #{original_query}"
|
||||
|
||||
with {:ok, embedding} <- get_embedding(query),
|
||||
{:ok, %{body: %{"result" => result}}} <-
|
||||
QdrantClient.post(
|
||||
"/collections/posts/points/search",
|
||||
build_search_payload(embedding, options)
|
||||
) do
|
||||
ids =
|
||||
Enum.map(result, fn %{"id" => id} ->
|
||||
Ecto.UUID.dump!(id)
|
||||
end)
|
||||
|
||||
from(a in Activity, where: a.id in ^ids)
|
||||
|> Activity.with_preloaded_object()
|
||||
|> Activity.restrict_deactivated_users()
|
||||
|> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id))
|
||||
|> Pleroma.Repo.all()
|
||||
|> maybe_fetch(user, original_query)
|
||||
else
|
||||
_ ->
|
||||
[]
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def healthcheck_endpoints do
|
||||
qdrant_health =
|
||||
Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])
|
||||
|> URI.parse()
|
||||
|> Map.put(:path, "/healthz")
|
||||
|> URI.to_string()
|
||||
|
||||
openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url])
|
||||
|
||||
[qdrant_health, openai_health] |> Enum.filter(& &1)
|
||||
end
|
||||
end
|
||||
|
||||
defmodule Pleroma.Search.QdrantSearch.OpenAIClient do
|
||||
use Tesla
|
||||
alias Pleroma.Config.Getting, as: Config
|
||||
|
||||
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url]))
|
||||
plug(Tesla.Middleware.JSON)
|
||||
|
||||
plug(Tesla.Middleware.Headers, [
|
||||
{"Authorization",
|
||||
"Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"}
|
||||
])
|
||||
end
|
||||
|
||||
defmodule Pleroma.Search.QdrantSearch.QdrantClient do
|
||||
use Tesla
|
||||
alias Pleroma.Config.Getting, as: Config
|
||||
|
||||
plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]))
|
||||
plug(Tesla.Middleware.JSON)
|
||||
|
||||
plug(Tesla.Middleware.Headers, [
|
||||
{"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])}
|
||||
])
|
||||
end
|
|
@ -21,22 +21,4 @@ defmodule Pleroma.Search.SearchBackend do
|
|||
from index.
|
||||
"""
|
||||
@callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()}
|
||||
|
||||
@doc """
|
||||
Create the index
|
||||
"""
|
||||
@callback create_index() :: :ok | {:error, any()}
|
||||
|
||||
@doc """
|
||||
Drop the index
|
||||
"""
|
||||
@callback drop_index() :: :ok | {:error, any()}
|
||||
|
||||
@doc """
|
||||
Healthcheck endpoints of search backend infrastructure to monitor for controlling
|
||||
processing of jobs in the Oban queue.
|
||||
|
||||
It is expected a 200 response is healthy and other responses are unhealthy.
|
||||
"""
|
||||
@callback healthcheck_endpoints :: list() | nil
|
||||
end
|
||||
|
|
|
@ -62,7 +62,8 @@ defp remove_suffix(uri, [test | rest]) do
|
|||
defp remove_suffix(uri, []), do: uri
|
||||
|
||||
def fetch_public_key(conn) do
|
||||
with {:ok, actor_id} <- get_actor_id(conn),
|
||||
with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
|
||||
{:ok, actor_id} <- key_id_to_actor_id(kid),
|
||||
{:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
|
||||
{:ok, public_key}
|
||||
else
|
||||
|
@ -72,7 +73,8 @@ def fetch_public_key(conn) do
|
|||
end
|
||||
|
||||
def refetch_public_key(conn) do
|
||||
with {:ok, actor_id} <- get_actor_id(conn),
|
||||
with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
|
||||
{:ok, actor_id} <- key_id_to_actor_id(kid),
|
||||
{:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
|
||||
{:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
|
||||
{:ok, public_key}
|
||||
|
@ -82,16 +84,6 @@ def refetch_public_key(conn) do
|
|||
end
|
||||
end
|
||||
|
||||
def get_actor_id(conn) do
|
||||
with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
|
||||
{:ok, actor_id} <- key_id_to_actor_id(kid) do
|
||||
{:ok, actor_id}
|
||||
else
|
||||
e ->
|
||||
{:error, e}
|
||||
end
|
||||
end
|
||||
|
||||
def sign(%User{keys: keys} = user, headers) do
|
||||
with {:ok, private_key, _} <- Keys.keys_from_pem(keys) do
|
||||
HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers)
|
||||
|
|
|
@ -239,12 +239,8 @@ defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do
|
|||
""
|
||||
end
|
||||
|
||||
if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do
|
||||
String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path)
|
||||
else
|
||||
[base_url, path]
|
||||
|> Path.join()
|
||||
end
|
||||
[base_url, path]
|
||||
|> Path.join()
|
||||
end
|
||||
|
||||
defp url_from_spec(_upload, _base_url, {:url, url}), do: url
|
||||
|
@ -281,9 +277,6 @@ def base_url do
|
|||
Path.join([upload_base_url, bucket_with_namespace])
|
||||
end
|
||||
|
||||
Pleroma.Uploaders.IPFS ->
|
||||
@config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
|
||||
|
||||
_ ->
|
||||
public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
|
||||
end
|
||||
|
|
|
@ -9,6 +9,8 @@ defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do
|
|||
"""
|
||||
@behaviour Pleroma.Upload.Filter
|
||||
|
||||
@spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
|
||||
|
||||
# Formats not compatible with exiftool at this time
|
||||
def filter(%Pleroma.Upload{content_type: "image/heic"}), do: {:ok, :noop}
|
||||
def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}
|
||||
|
|
|
@ -38,6 +38,7 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
|
|||
[{"fill", "yellow"}, {"tint", "40"}]
|
||||
]
|
||||
|
||||
@spec filter(Pleroma.Upload.t()) :: {:ok, atom()} | {:error, String.t()}
|
||||
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
|
||||
try do
|
||||
Filter.Mogrify.do_filter(file, [Enum.random(@filters)])
|
||||
|
|
|
@ -8,6 +8,7 @@ defmodule Pleroma.Upload.Filter.Mogrify do
|
|||
@type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
|
||||
@type conversions :: conversion() | [conversion()]
|
||||
|
||||
@spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
|
||||
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
|
||||
try do
|
||||
do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))
|
||||
|
|
|
@ -1,77 +0,0 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Uploaders.IPFS do
|
||||
@behaviour Pleroma.Uploaders.Uploader
|
||||
require Logger
|
||||
|
||||
alias Tesla.Multipart
|
||||
|
||||
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
|
||||
|
||||
defp get_final_url(method) do
|
||||
config = @config_impl.get([__MODULE__])
|
||||
post_base_url = Keyword.get(config, :post_gateway_url)
|
||||
|
||||
Path.join([post_base_url, method])
|
||||
end
|
||||
|
||||
def put_file_endpoint do
|
||||
get_final_url("/api/v0/add")
|
||||
end
|
||||
|
||||
def delete_file_endpoint do
|
||||
get_final_url("/api/v0/files/rm")
|
||||
end
|
||||
|
||||
@placeholder "{CID}"
|
||||
def placeholder, do: @placeholder
|
||||
|
||||
@impl true
|
||||
def get_file(file) do
|
||||
b_url = Pleroma.Upload.base_url()
|
||||
|
||||
if String.contains?(b_url, @placeholder) do
|
||||
{:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}}
|
||||
else
|
||||
{:error, "IPFS Get URL doesn't contain 'cid' placeholder"}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def put_file(%Pleroma.Upload{} = upload) do
|
||||
mp =
|
||||
Multipart.new()
|
||||
|> Multipart.add_content_type_param("charset=utf-8")
|
||||
|> Multipart.add_file(upload.tempfile)
|
||||
|
||||
case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do
|
||||
{:ok, ret} ->
|
||||
case Jason.decode(ret.body) do
|
||||
{:ok, ret} ->
|
||||
if Map.has_key?(ret, "Hash") do
|
||||
{:ok, {:file, ret["Hash"]}}
|
||||
else
|
||||
{:error, "JSON doesn't contain Hash key"}
|
||||
end
|
||||
|
||||
error ->
|
||||
Logger.error("#{__MODULE__}: #{inspect(error)}")
|
||||
{:error, "JSON decode failed"}
|
||||
end
|
||||
|
||||
error ->
|
||||
Logger.error("#{__MODULE__}: #{inspect(error)}")
|
||||
{:error, "IPFS Gateway upload failed"}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def delete_file(file) do
|
||||
case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do
|
||||
{:ok, %{status: 204}} -> :ok
|
||||
error -> {:error, inspect(error)}
|
||||
end
|
||||
end
|
||||
end
|
|
@ -2053,8 +2053,7 @@ defp verify_field_link(field, profile_urls) do
|
|||
%{scheme: scheme, userinfo: nil, host: host}
|
||||
when not_empty_string(host) and scheme in ["http", "https"] <-
|
||||
URI.parse(value),
|
||||
{:not_idn, true} <-
|
||||
{:not_idn, match?(^host, to_string(:idna.encode(to_charlist(host))))},
|
||||
{:not_idn, true} <- {:not_idn, to_string(:idna.encode(host)) == host},
|
||||
"me" <- Pleroma.Web.RelMe.maybe_put_rel_me(value, profile_urls) do
|
||||
CommonUtils.to_masto_date(NaiveDateTime.utc_now())
|
||||
else
|
||||
|
@ -2728,7 +2727,7 @@ defp add_to_block(%User{} = user, %User{} = blocked) do
|
|||
end
|
||||
end
|
||||
|
||||
@spec remove_from_block(User.t(), User.t()) ::
|
||||
@spec add_to_block(User.t(), User.t()) ::
|
||||
{:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()}
|
||||
defp remove_from_block(%User{} = user, %User{} = blocked) do
|
||||
with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do
|
||||
|
|
|
@ -979,9 +979,8 @@ defp restrict_media(query, _), do: query
|
|||
|
||||
defp restrict_replies(query, %{exclude_replies: true}) do
|
||||
from(
|
||||
[activity, object] in query,
|
||||
where:
|
||||
fragment("?->>'inReplyTo' is null or ?->>'type' = 'Announce'", object.data, activity.data)
|
||||
[_activity, object] in query,
|
||||
where: fragment("?->>'inReplyTo' is null", object.data)
|
||||
)
|
||||
end
|
||||
|
||||
|
|
|
@ -52,7 +52,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
|
|||
when action in [:activity, :object]
|
||||
)
|
||||
|
||||
plug(:log_inbox_metadata when action in [:inbox])
|
||||
plug(:set_requester_reachable when action in [:inbox])
|
||||
plug(:relay_active? when action in [:relay])
|
||||
|
||||
|
@ -522,13 +521,6 @@ defp set_requester_reachable(%Plug.Conn{} = conn, _) do
|
|||
conn
|
||||
end
|
||||
|
||||
defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do
|
||||
Logger.metadata(actor: actor, type: type)
|
||||
conn
|
||||
end
|
||||
|
||||
defp log_inbox_metadata(conn, _), do: conn
|
||||
|
||||
def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
|
||||
with {:ok, object} <-
|
||||
ActivityPub.upload(
|
||||
|
|
|
@ -1,87 +0,0 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.User
|
||||
require Pleroma.Constants
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
defp user_has_posted?(%User{} = u), do: u.note_count > 0
|
||||
|
||||
defp user_has_age?(%User{} = u) do
|
||||
user_age_limit = Config.get([:mrf_antimentionspam, :user_age_limit], 30_000)
|
||||
diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :millisecond)
|
||||
diff >= user_age_limit
|
||||
end
|
||||
|
||||
defp good_reputation?(%User{} = u) do
|
||||
user_has_age?(u) and user_has_posted?(u)
|
||||
end
|
||||
|
||||
# copied from HellthreadPolicy
|
||||
defp get_recipient_count(message) do
|
||||
recipients = (message["to"] || []) ++ (message["cc"] || [])
|
||||
|
||||
follower_collection =
|
||||
User.get_cached_by_ap_id(message["actor"] || message["attributedTo"]).follower_address
|
||||
|
||||
if Enum.member?(recipients, Pleroma.Constants.as_public()) do
|
||||
recipients =
|
||||
recipients
|
||||
|> List.delete(Pleroma.Constants.as_public())
|
||||
|> List.delete(follower_collection)
|
||||
|
||||
{:public, length(recipients)}
|
||||
else
|
||||
recipients =
|
||||
recipients
|
||||
|> List.delete(follower_collection)
|
||||
|
||||
{:not_public, length(recipients)}
|
||||
end
|
||||
end
|
||||
|
||||
defp object_has_recipients?(%{"object" => object} = activity) do
|
||||
{_, object_count} = get_recipient_count(object)
|
||||
{_, activity_count} = get_recipient_count(activity)
|
||||
object_count + activity_count > 0
|
||||
end
|
||||
|
||||
defp object_has_recipients?(object) do
|
||||
{_, count} = get_recipient_count(object)
|
||||
count > 0
|
||||
end
|
||||
|
||||
@impl true
|
||||
def filter(%{"type" => "Create", "actor" => actor} = activity) do
|
||||
with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor),
|
||||
{:has_mentions, true} <- {:has_mentions, object_has_recipients?(activity)},
|
||||
{:good_reputation, true} <- {:good_reputation, good_reputation?(u)} do
|
||||
{:ok, activity}
|
||||
else
|
||||
{:ok, %User{local: true}} ->
|
||||
{:ok, activity}
|
||||
|
||||
{:has_mentions, false} ->
|
||||
{:ok, activity}
|
||||
|
||||
{:good_reputation, false} ->
|
||||
{:reject, "[AntiMentionSpamPolicy] User rejected"}
|
||||
|
||||
{:error, _} ->
|
||||
{:reject, "[AntiMentionSpamPolicy] Failed to get or fetch user by ap_id"}
|
||||
|
||||
e ->
|
||||
{:reject, "[AntiMentionSpamPolicy] Unhandled error #{inspect(e)}"}
|
||||
end
|
||||
end
|
||||
|
||||
# in all other cases, pass through
|
||||
def filter(message), do: {:ok, message}
|
||||
|
||||
@impl true
|
||||
def describe, do: {:ok, %{}}
|
||||
end
|
|
@ -1,146 +0,0 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do
|
||||
@moduledoc """
|
||||
Dynamic activity filtering based on an RBL database
|
||||
|
||||
This MRF makes queries to a custom DNS server which will
|
||||
respond with values indicating the classification of the domain
|
||||
the activity originated from. This method has been widely used
|
||||
in the email anti-spam industry for very fast reputation checks.
|
||||
|
||||
e.g., if the DNS response is 127.0.0.1 or empty, the domain is OK
|
||||
Other values such as 127.0.0.2 may be used for specific classifications.
|
||||
|
||||
Information for why the host is blocked can be stored in a corresponding TXT record.
|
||||
|
||||
This method is fail-open so if the queries fail the activites are accepted.
|
||||
|
||||
An example of software meant for this purpsoe is rbldnsd which can be found
|
||||
at http://www.corpit.ru/mjt/rbldnsd.html or mirrored at
|
||||
https://git.pleroma.social/feld/rbldnsd
|
||||
|
||||
It is highly recommended that you run your own copy of rbldnsd and use an
|
||||
external mechanism to sync/share the contents of the zone file. This is
|
||||
important to keep the latency on the queries as low as possible and prevent
|
||||
your DNS server from being attacked so it fails and content is permitted.
|
||||
"""
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
|
||||
alias Pleroma.Config
|
||||
|
||||
require Logger
|
||||
|
||||
@query_retries 1
|
||||
@query_timeout 500
|
||||
|
||||
@impl true
|
||||
def filter(%{"actor" => actor} = object) do
|
||||
actor_info = URI.parse(actor)
|
||||
|
||||
with {:ok, object} <- check_rbl(actor_info, object) do
|
||||
{:ok, object}
|
||||
else
|
||||
_ -> {:reject, "[DNSRBLPolicy]"}
|
||||
end
|
||||
end
|
||||
|
||||
@impl true
|
||||
def filter(object), do: {:ok, object}
|
||||
|
||||
@impl true
|
||||
def describe do
|
||||
mrf_dnsrbl =
|
||||
Config.get(:mrf_dnsrbl)
|
||||
|> Enum.into(%{})
|
||||
|
||||
{:ok, %{mrf_dnsrbl: mrf_dnsrbl}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def config_description do
|
||||
%{
|
||||
key: :mrf_dnsrbl,
|
||||
related_policy: "Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy",
|
||||
label: "MRF DNSRBL",
|
||||
description: "DNS RealTime Blackhole Policy",
|
||||
children: [
|
||||
%{
|
||||
key: :nameserver,
|
||||
type: {:string},
|
||||
description: "DNSRBL Nameserver to Query (IP or hostame)",
|
||||
suggestions: ["127.0.0.1"]
|
||||
},
|
||||
%{
|
||||
key: :port,
|
||||
type: {:string},
|
||||
description: "Nameserver port",
|
||||
suggestions: ["53"]
|
||||
},
|
||||
%{
|
||||
key: :zone,
|
||||
type: {:string},
|
||||
description: "Root zone for querying",
|
||||
suggestions: ["bl.pleroma.com"]
|
||||
}
|
||||
]
|
||||
}
|
||||
end
|
||||
|
||||
defp check_rbl(%{host: actor_host}, object) do
|
||||
with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()),
|
||||
zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do
|
||||
query =
|
||||
Enum.join([actor_host, zone], ".")
|
||||
|> String.to_charlist()
|
||||
|
||||
rbl_response = rblquery(query)
|
||||
|
||||
if Enum.empty?(rbl_response) do
|
||||
{:ok, object}
|
||||
else
|
||||
Task.start(fn ->
|
||||
reason =
|
||||
case rblquery(query, :txt) do
|
||||
[[result]] -> result
|
||||
_ -> "undefined"
|
||||
end
|
||||
|
||||
Logger.warning(
|
||||
"DNSRBL Rejected activity from #{actor_host} for reason: #{inspect(reason)}"
|
||||
)
|
||||
end)
|
||||
|
||||
:error
|
||||
end
|
||||
else
|
||||
_ -> {:ok, object}
|
||||
end
|
||||
end
|
||||
|
||||
defp get_rblhost_ip(rblhost) do
|
||||
case rblhost |> String.to_charlist() |> :inet_parse.address() do
|
||||
{:ok, _} -> rblhost |> String.to_charlist() |> :inet_parse.address()
|
||||
_ -> {:ok, rblhost |> String.to_charlist() |> :inet_res.lookup(:in, :a) |> Enum.random()}
|
||||
end
|
||||
end
|
||||
|
||||
defp rblquery(query, type \\ :a) do
|
||||
config = Config.get([:mrf_dnsrbl])
|
||||
|
||||
case get_rblhost_ip(config[:nameserver]) do
|
||||
{:ok, rblnsip} ->
|
||||
:inet_res.lookup(query, :in, type,
|
||||
nameservers: [{rblnsip, config[:port]}],
|
||||
timeout: @query_timeout,
|
||||
retry: @query_retries
|
||||
)
|
||||
|
||||
_ ->
|
||||
[]
|
||||
end
|
||||
end
|
||||
end
|
|
@ -11,6 +11,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
|
|||
|
||||
require Logger
|
||||
|
||||
@adapter_options [
|
||||
pool: :media,
|
||||
recv_timeout: 10_000
|
||||
]
|
||||
|
||||
@impl true
|
||||
def history_awareness, do: :auto
|
||||
|
||||
|
@ -22,14 +27,17 @@ defp prefetch(url) do
|
|||
|
||||
Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
|
||||
|
||||
fetch(prefetch_url)
|
||||
if Pleroma.Config.get(:env) == :test do
|
||||
fetch(prefetch_url)
|
||||
else
|
||||
ConcurrentLimiter.limit(__MODULE__, fn ->
|
||||
Task.start(fn -> fetch(prefetch_url) end)
|
||||
end)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp fetch(url) do
|
||||
http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
|
||||
HTTP.get(url, [], http_client_opts)
|
||||
end
|
||||
defp fetch(url), do: HTTP.get(url, [], @adapter_options)
|
||||
|
||||
defp preload(%{"object" => %{"attachment" => attachments}} = _message) do
|
||||
Enum.each(attachments, fn
|
||||
|
|
|
@ -1,265 +0,0 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do
|
||||
@moduledoc """
|
||||
Hide, delete, or mark sensitive NSFW content with artificial intelligence.
|
||||
|
||||
Requires a NSFW API server, configured like so:
|
||||
|
||||
config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF,
|
||||
url: "http://127.0.0.1:5000/",
|
||||
threshold: 0.7,
|
||||
mark_sensitive: true,
|
||||
unlist: false,
|
||||
reject: false
|
||||
|
||||
The NSFW API server must implement an HTTP endpoint like this:
|
||||
|
||||
curl http://localhost:5000/?url=https://fedi.com/images/001.jpg
|
||||
|
||||
Returning a response like this:
|
||||
|
||||
{"score", 0.314}
|
||||
|
||||
Where a score is 0-1, with `1` being definitely NSFW.
|
||||
|
||||
A good API server is here: https://github.com/EugenCepoi/nsfw_api
|
||||
You can run it with Docker with a one-liner:
|
||||
|
||||
docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest
|
||||
|
||||
Options:
|
||||
|
||||
- `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/"
|
||||
- `threshold`: Lowest score to take action on. Default: `0.7`
|
||||
- `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true`
|
||||
- `unlist`: Unlist all detected NSFW content? Default: `false`
|
||||
- `reject`: Reject all detected NSFW content (takes precedence)? Default: `false`
|
||||
"""
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.Constants
|
||||
alias Pleroma.HTTP
|
||||
alias Pleroma.User
|
||||
|
||||
require Logger
|
||||
require Pleroma.Constants
|
||||
|
||||
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
|
||||
@policy :mrf_nsfw_api
|
||||
|
||||
def build_request_url(url) do
|
||||
Config.get([@policy, :url])
|
||||
|> URI.parse()
|
||||
|> fix_path()
|
||||
|> Map.put(:query, "url=#{url}")
|
||||
|> URI.to_string()
|
||||
end
|
||||
|
||||
def parse_url(url) do
|
||||
request = build_request_url(url)
|
||||
|
||||
with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do
|
||||
Jason.decode(body)
|
||||
else
|
||||
error ->
|
||||
Logger.warning("""
|
||||
[NsfwApiPolicy]: The API server failed. Skipping.
|
||||
#{inspect(error)}
|
||||
""")
|
||||
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
def check_url_nsfw(url) when is_binary(url) do
|
||||
threshold = Config.get([@policy, :threshold])
|
||||
|
||||
case parse_url(url) do
|
||||
{:ok, %{"score" => score}} when score >= threshold ->
|
||||
{:nsfw, %{url: url, score: score, threshold: threshold}}
|
||||
|
||||
{:ok, %{"score" => score}} ->
|
||||
{:sfw, %{url: url, score: score, threshold: threshold}}
|
||||
|
||||
_ ->
|
||||
{:sfw, %{url: url, score: nil, threshold: threshold}}
|
||||
end
|
||||
end
|
||||
|
||||
def check_url_nsfw(%{"href" => url}) when is_binary(url) do
|
||||
check_url_nsfw(url)
|
||||
end
|
||||
|
||||
def check_url_nsfw(url) do
|
||||
threshold = Config.get([@policy, :threshold])
|
||||
{:sfw, %{url: url, score: nil, threshold: threshold}}
|
||||
end
|
||||
|
||||
def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do
|
||||
if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do
|
||||
{:sfw, attachment}
|
||||
else
|
||||
{:nsfw, attachment}
|
||||
end
|
||||
end
|
||||
|
||||
def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do
|
||||
case check_url_nsfw(url) do
|
||||
{:sfw, _} -> {:sfw, attachment}
|
||||
{:nsfw, _} -> {:nsfw, attachment}
|
||||
end
|
||||
end
|
||||
|
||||
def check_attachment_nsfw(attachment), do: {:sfw, attachment}
|
||||
|
||||
def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do
|
||||
if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do
|
||||
{:sfw, object}
|
||||
else
|
||||
{:nsfw, object}
|
||||
end
|
||||
end
|
||||
|
||||
def check_object_nsfw(%{"object" => %{} = child_object} = object) do
|
||||
case check_object_nsfw(child_object) do
|
||||
{:sfw, _} -> {:sfw, object}
|
||||
{:nsfw, _} -> {:nsfw, object}
|
||||
end
|
||||
end
|
||||
|
||||
def check_object_nsfw(object), do: {:sfw, object}
|
||||
|
||||
@impl true
|
||||
def filter(object) do
|
||||
with {:sfw, object} <- check_object_nsfw(object) do
|
||||
{:ok, object}
|
||||
else
|
||||
{:nsfw, _data} -> handle_nsfw(object)
|
||||
_ -> {:reject, "NSFW: Attachment rejected"}
|
||||
end
|
||||
end
|
||||
|
||||
defp handle_nsfw(object) do
|
||||
if Config.get([@policy, :reject]) do
|
||||
{:reject, object}
|
||||
else
|
||||
{:ok,
|
||||
object
|
||||
|> maybe_unlist()
|
||||
|> maybe_mark_sensitive()}
|
||||
end
|
||||
end
|
||||
|
||||
defp maybe_unlist(object) do
|
||||
if Config.get([@policy, :unlist]) do
|
||||
unlist(object)
|
||||
else
|
||||
object
|
||||
end
|
||||
end
|
||||
|
||||
defp maybe_mark_sensitive(object) do
|
||||
if Config.get([@policy, :mark_sensitive]) do
|
||||
mark_sensitive(object)
|
||||
else
|
||||
object
|
||||
end
|
||||
end
|
||||
|
||||
def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do
|
||||
with %User{} = user <- User.get_cached_by_ap_id(actor) do
|
||||
to =
|
||||
[user.follower_address | to]
|
||||
|> List.delete(Constants.as_public())
|
||||
|> Enum.uniq()
|
||||
|
||||
cc =
|
||||
[Constants.as_public() | cc]
|
||||
|> List.delete(user.follower_address)
|
||||
|> Enum.uniq()
|
||||
|
||||
object
|
||||
|> Map.put("to", to)
|
||||
|> Map.put("cc", cc)
|
||||
else
|
||||
_ -> raise "[NsfwApiPolicy]: Could not find user #{actor}"
|
||||
end
|
||||
end
|
||||
|
||||
def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do
|
||||
Map.put(object, "object", mark_sensitive(child_object))
|
||||
end
|
||||
|
||||
def mark_sensitive(object) when is_map(object) do
|
||||
tags = (object["tag"] || []) ++ ["nsfw"]
|
||||
|
||||
object
|
||||
|> Map.put("tag", tags)
|
||||
|> Map.put("sensitive", true)
|
||||
end
|
||||
|
||||
# Hackney needs a trailing slash
|
||||
defp fix_path(%URI{path: path} = uri) when is_binary(path) do
|
||||
path = String.trim_trailing(path, "/") <> "/"
|
||||
Map.put(uri, :path, path)
|
||||
end
|
||||
|
||||
defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/")
|
||||
|
||||
@impl true
|
||||
def describe do
|
||||
options = %{
|
||||
threshold: Config.get([@policy, :threshold]),
|
||||
mark_sensitive: Config.get([@policy, :mark_sensitive]),
|
||||
unlist: Config.get([@policy, :unlist]),
|
||||
reject: Config.get([@policy, :reject])
|
||||
}
|
||||
|
||||
{:ok, %{@policy => options}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def config_description do
|
||||
%{
|
||||
key: @policy,
|
||||
related_policy: to_string(__MODULE__),
|
||||
label: "NSFW API Policy",
|
||||
description:
|
||||
"Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.",
|
||||
children: [
|
||||
%{
|
||||
key: :url,
|
||||
type: :string,
|
||||
description: "Base URL of the API server.",
|
||||
suggestions: ["http://127.0.0.1:5000/"]
|
||||
},
|
||||
%{
|
||||
key: :threshold,
|
||||
type: :float,
|
||||
description: "Lowest score to take action on. Between 0 and 1.",
|
||||
suggestions: [0.7]
|
||||
},
|
||||
%{
|
||||
key: :mark_sensitive,
|
||||
type: :boolean,
|
||||
description: "Mark sensitive all detected NSFW content?",
|
||||
suggestions: [true]
|
||||
},
|
||||
%{
|
||||
key: :unlist,
|
||||
type: :boolean,
|
||||
description: "Unlist sensitive all detected NSFW content?",
|
||||
suggestions: [false]
|
||||
},
|
||||
%{
|
||||
key: :reject,
|
||||
type: :boolean,
|
||||
description: "Reject sensitive all detected NSFW content (takes precedence)?",
|
||||
suggestions: [false]
|
||||
}
|
||||
]
|
||||
}
|
||||
end
|
||||
end
|
|
@ -15,7 +15,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
|
|||
field(:type, :string, default: "Link")
|
||||
field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream")
|
||||
field(:name, :string)
|
||||
field(:summary, :string)
|
||||
field(:blurhash, :string)
|
||||
|
||||
embeds_many :url, UrlObjectValidator, primary_key: false do
|
||||
|
@ -45,7 +44,7 @@ def changeset(struct, data) do
|
|||
|> fix_url()
|
||||
|
||||
struct
|
||||
|> cast(data, [:id, :type, :mediaType, :name, :summary, :blurhash])
|
||||
|> cast(data, [:id, :type, :mediaType, :name, :blurhash])
|
||||
|> cast_embed(:url, with: &url_changeset/2, required: true)
|
||||
|> validate_inclusion(:type, ~w[Link Document Audio Image Video])
|
||||
|> validate_required([:type, :mediaType])
|
||||
|
|
|
@ -50,11 +50,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do
|
|||
pleroma: %Schema{
|
||||
type: :object,
|
||||
properties: %{
|
||||
mime_type: %Schema{type: :string, description: "mime type of the attachment"},
|
||||
name: %Schema{
|
||||
type: :string,
|
||||
description: "Name of the attachment, typically the filename"
|
||||
}
|
||||
mime_type: %Schema{type: :string, description: "mime type of the attachment"}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
@ -129,22 +129,8 @@ defp attachments(%{params: params} = draft) do
|
|||
|
||||
defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft
|
||||
|
||||
defp in_reply_to(%{params: %{in_reply_to_status_id: :deleted}} = draft) do
|
||||
add_error(draft, dgettext("errors", "Cannot reply to a deleted status"))
|
||||
end
|
||||
|
||||
defp in_reply_to(%{params: %{in_reply_to_status_id: id} = params} = draft) when is_binary(id) do
|
||||
activity = Activity.get_by_id(id)
|
||||
|
||||
params =
|
||||
if is_nil(activity) do
|
||||
# Deleted activities are returned as nil
|
||||
Map.put(params, :in_reply_to_status_id, :deleted)
|
||||
else
|
||||
Map.put(params, :in_reply_to_status_id, activity)
|
||||
end
|
||||
|
||||
in_reply_to(%{draft | params: params})
|
||||
defp in_reply_to(%{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do
|
||||
%__MODULE__{draft | in_reply_to: Activity.get_by_id(id)}
|
||||
end
|
||||
|
||||
defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do
|
||||
|
|
|
@ -38,8 +38,6 @@ defmodule Pleroma.Web.Endpoint do
|
|||
|
||||
plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint])
|
||||
|
||||
plug(Pleroma.Web.Plugs.LoggerMetadataPath)
|
||||
|
||||
plug(Pleroma.Web.Plugs.SetLocalePlug)
|
||||
plug(CORSPlug)
|
||||
plug(Pleroma.Web.Plugs.HTTPSecurityPlug)
|
||||
|
|
|
@ -44,7 +44,7 @@ def incoming_ap_doc(%{params: params, req_headers: req_headers}) do
|
|||
end
|
||||
|
||||
def incoming_ap_doc(%{"type" => "Delete"} = params) do
|
||||
ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow)
|
||||
ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3)
|
||||
end
|
||||
|
||||
def incoming_ap_doc(params) do
|
||||
|
|
|
@ -9,11 +9,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
|
|||
alias Pleroma.User
|
||||
alias Pleroma.UserNote
|
||||
alias Pleroma.UserRelationship
|
||||
alias Pleroma.Web.ActivityPub.ActivityPub
|
||||
alias Pleroma.Web.CommonAPI.Utils
|
||||
alias Pleroma.Web.MastodonAPI.AccountView
|
||||
alias Pleroma.Web.MediaProxy
|
||||
alias Pleroma.Web.PleromaAPI.ScrobbleView
|
||||
|
||||
def render("index.json", %{users: users} = opts) do
|
||||
reading_user = opts[:for]
|
||||
|
@ -276,13 +274,6 @@ defp do_render("show.json", %{user: user} = opts) do
|
|||
user.last_status_at &&
|
||||
user.last_status_at |> NaiveDateTime.to_date() |> Date.to_iso8601()
|
||||
|
||||
scrobbles =
|
||||
ActivityPub.fetch_user_abstract_activities(user, opts[:for], %{
|
||||
type: "Listen",
|
||||
limit: 1,
|
||||
id: user.id
|
||||
})
|
||||
|
||||
%{
|
||||
id: to_string(user.id),
|
||||
username: username_from_nickname(user.nickname),
|
||||
|
@ -331,8 +322,7 @@ defp do_render("show.json", %{user: user} = opts) do
|
|||
skip_thread_containment: user.skip_thread_containment,
|
||||
background_image: image_url(user.background) |> MediaProxy.url(),
|
||||
accepts_chat_messages: user.accepts_chat_messages,
|
||||
favicon: favicon,
|
||||
scrobbles: safe_render_many(scrobbles, ScrobbleView, "show.json")
|
||||
favicon: favicon
|
||||
}
|
||||
}
|
||||
|> maybe_put_role(user, opts[:for])
|
||||
|
|
|
@ -152,7 +152,6 @@ def features do
|
|||
|
||||
def federation do
|
||||
quarantined = Config.get([:instance, :quarantined_instances], [])
|
||||
rejected = Config.get([:instance, :rejected_instances], [])
|
||||
|
||||
if Config.get([:mrf, :transparency]) do
|
||||
{:ok, data} = MRF.describe()
|
||||
|
@ -172,12 +171,6 @@ def federation do
|
|||
|> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end)
|
||||
|> Map.new()
|
||||
})
|
||||
|> Map.put(
|
||||
:rejected_instances,
|
||||
rejected
|
||||
|> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end)
|
||||
|> Map.new()
|
||||
)
|
||||
else
|
||||
%{}
|
||||
end
|
||||
|
|
|
@ -624,19 +624,6 @@ def render("attachment.json", %{attachment: attachment}) do
|
|||
to_string(attachment["id"] || hash_id)
|
||||
end
|
||||
|
||||
description =
|
||||
if attachment["summary"] do
|
||||
HTML.strip_tags(attachment["summary"])
|
||||
else
|
||||
attachment["name"]
|
||||
end
|
||||
|
||||
name = if attachment["summary"], do: attachment["name"]
|
||||
|
||||
pleroma =
|
||||
%{mime_type: media_type}
|
||||
|> Maps.put_if_present(:name, name)
|
||||
|
||||
%{
|
||||
id: attachment_id,
|
||||
url: href,
|
||||
|
@ -644,8 +631,8 @@ def render("attachment.json", %{attachment: attachment}) do
|
|||
preview_url: href_preview,
|
||||
text_url: href,
|
||||
type: type,
|
||||
description: description,
|
||||
pleroma: pleroma,
|
||||
description: attachment["name"],
|
||||
pleroma: %{mime_type: media_type},
|
||||
blurhash: attachment["blurhash"]
|
||||
}
|
||||
|> Maps.put_if_present(:meta, meta)
|
||||
|
|
|
@ -54,10 +54,9 @@ def preview(%Conn{} = conn, %{"sig" => sig64, "url" => url64}) do
|
|||
|
||||
defp handle_preview(conn, url) do
|
||||
media_proxy_url = MediaProxy.url(url)
|
||||
http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
|
||||
|
||||
with {:ok, %{status: status} = head_response} when status in 200..299 <-
|
||||
Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do
|
||||
Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do
|
||||
content_type = Tesla.get_header(head_response, "content-type")
|
||||
content_length = Tesla.get_header(head_response, "content-length")
|
||||
content_length = content_length && String.to_integer(content_length)
|
||||
|
|
|
@ -96,7 +96,7 @@ defp put_valid_until(changeset, attrs) do
|
|||
|> validate_required([:valid_until])
|
||||
end
|
||||
|
||||
@spec create(App.t(), User.t(), map()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()}
|
||||
@spec create(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Ecto.Changeset.t()}
|
||||
def create(%App{} = app, %User{} = user, attrs \\ %{}) do
|
||||
with {:ok, token} <- do_create(app, user, attrs) do
|
||||
if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do
|
||||
|
|
|
@ -14,27 +14,15 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do
|
|||
alias Pleroma.Web.CommonAPI.Utils
|
||||
alias Pleroma.Web.MastodonAPI.AccountView
|
||||
|
||||
def render("show.json", %{scrobble: %Activity{data: %{"type" => "Listen"}} = scrobble} = _opts) do
|
||||
scrobble_schema_skeleton(scrobble)
|
||||
end
|
||||
|
||||
def render("show.json", %{activity: %Activity{data: %{"type" => "Listen"}} = activity} = opts) do
|
||||
object = Object.normalize(activity, fetch: false)
|
||||
|
||||
user = CommonAPI.get_user(activity.data["actor"])
|
||||
|
||||
scrobble_schema_skeleton(activity)
|
||||
|> Map.put(:account, AccountView.render("show.json", %{user: user, for: opts[:for]}))
|
||||
end
|
||||
|
||||
def render("index.json", opts) do
|
||||
safe_render_many(opts.activities, __MODULE__, "show.json", opts)
|
||||
end
|
||||
|
||||
defp scrobble_schema_skeleton(%Activity{data: %{"type" => "Listen"}} = scrobble) do
|
||||
object = Object.normalize(scrobble, fetch: false)
|
||||
created_at = Utils.to_masto_date(scrobble.data["published"])
|
||||
created_at = Utils.to_masto_date(activity.data["published"])
|
||||
|
||||
%{
|
||||
id: scrobble.id,
|
||||
id: activity.id,
|
||||
account: AccountView.render("show.json", %{user: user, for: opts[:for]}),
|
||||
created_at: created_at,
|
||||
title: object.data["title"] |> HTML.strip_tags(),
|
||||
artist: object.data["artist"] |> HTML.strip_tags(),
|
||||
|
@ -43,4 +31,8 @@ defp scrobble_schema_skeleton(%Activity{data: %{"type" => "Listen"}} = scrobble)
|
|||
length: object.data["length"]
|
||||
}
|
||||
end
|
||||
|
||||
def render("index.json", opts) do
|
||||
safe_render_many(opts.activities, __MODULE__, "show.json", opts)
|
||||
end
|
||||
end
|
||||
|
|
|
@ -3,27 +3,26 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do
|
||||
alias Pleroma.Config
|
||||
import Plug.Conn
|
||||
|
||||
require Logger
|
||||
|
||||
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
|
||||
|
||||
def init(opts), do: opts
|
||||
|
||||
def call(conn, _options) do
|
||||
if @config_impl.get([:http_security, :enabled]) do
|
||||
if Config.get([:http_security, :enabled]) do
|
||||
conn
|
||||
|> merge_resp_headers(headers())
|
||||
|> maybe_send_sts_header(@config_impl.get([:http_security, :sts]))
|
||||
|> maybe_send_sts_header(Config.get([:http_security, :sts]))
|
||||
else
|
||||
conn
|
||||
end
|
||||
end
|
||||
|
||||
def primary_frontend do
|
||||
with %{"name" => frontend} <- @config_impl.get([:frontends, :primary]),
|
||||
available <- @config_impl.get([:frontends, :available]),
|
||||
with %{"name" => frontend} <- Config.get([:frontends, :primary]),
|
||||
available <- Config.get([:frontends, :available]),
|
||||
%{} = primary_frontend <- Map.get(available, frontend) do
|
||||
{:ok, primary_frontend}
|
||||
end
|
||||
|
@ -38,8 +37,8 @@ def custom_http_frontend_headers do
|
|||
end
|
||||
|
||||
def headers do
|
||||
referrer_policy = @config_impl.get([:http_security, :referrer_policy])
|
||||
report_uri = @config_impl.get([:http_security, :report_uri])
|
||||
referrer_policy = Config.get([:http_security, :referrer_policy])
|
||||
report_uri = Config.get([:http_security, :report_uri])
|
||||
custom_http_frontend_headers = custom_http_frontend_headers()
|
||||
|
||||
headers = [
|
||||
|
@ -87,10 +86,10 @@ def headers do
|
|||
@csp_start [Enum.join(static_csp_rules, ";") <> ";"]
|
||||
|
||||
defp csp_string do
|
||||
scheme = @config_impl.get([Pleroma.Web.Endpoint, :url])[:scheme]
|
||||
scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
|
||||
static_url = Pleroma.Web.Endpoint.static_url()
|
||||
websocket_url = Pleroma.Web.Endpoint.websocket_url()
|
||||
report_uri = @config_impl.get([:http_security, :report_uri])
|
||||
report_uri = Config.get([:http_security, :report_uri])
|
||||
|
||||
img_src = "img-src 'self' data: blob:"
|
||||
media_src = "media-src 'self'"
|
||||
|
@ -98,8 +97,8 @@ defp csp_string do
|
|||
|
||||
# Strict multimedia CSP enforcement only when MediaProxy is enabled
|
||||
{img_src, media_src, connect_src} =
|
||||
if @config_impl.get([:media_proxy, :enabled]) &&
|
||||
!@config_impl.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
|
||||
if Config.get([:media_proxy, :enabled]) &&
|
||||
!Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
|
||||
sources = build_csp_multimedia_source_list()
|
||||
|
||||
{
|
||||
|
@ -116,21 +115,17 @@ defp csp_string do
|
|||
end
|
||||
|
||||
connect_src =
|
||||
if @config_impl.get([:env]) == :dev do
|
||||
if Config.get(:env) == :dev do
|
||||
[connect_src, " http://localhost:3035/"]
|
||||
else
|
||||
connect_src
|
||||
end
|
||||
|
||||
script_src =
|
||||
if @config_impl.get([:http_security, :allow_unsafe_eval]) do
|
||||
if @config_impl.get([:env]) == :dev do
|
||||
"script-src 'self' 'unsafe-eval'"
|
||||
else
|
||||
"script-src 'self' 'wasm-unsafe-eval'"
|
||||
end
|
||||
if Config.get(:env) == :dev do
|
||||
"script-src 'self' 'unsafe-eval'"
|
||||
else
|
||||
"script-src 'self'"
|
||||
"script-src 'self' 'wasm-unsafe-eval'"
|
||||
end
|
||||
|
||||
report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"]
|
||||
|
@ -166,11 +161,11 @@ defp build_csp_param_from_whitelist(url), do: url
|
|||
defp build_csp_multimedia_source_list do
|
||||
media_proxy_whitelist =
|
||||
[:media_proxy, :whitelist]
|
||||
|> @config_impl.get()
|
||||
|> Config.get()
|
||||
|> build_csp_from_whitelist([])
|
||||
|
||||
captcha_method = @config_impl.get([Pleroma.Captcha, :method])
|
||||
captcha_endpoint = @config_impl.get([captcha_method, :endpoint])
|
||||
captcha_method = Config.get([Pleroma.Captcha, :method])
|
||||
captcha_endpoint = Config.get([captcha_method, :endpoint])
|
||||
|
||||
base_endpoints =
|
||||
[
|
||||
|
@ -178,7 +173,7 @@ defp build_csp_multimedia_source_list do
|
|||
[Pleroma.Upload, :base_url],
|
||||
[Pleroma.Uploaders.S3, :public_endpoint]
|
||||
]
|
||||
|> Enum.map(&@config_impl.get/1)
|
||||
|> Enum.map(&Config.get/1)
|
||||
|
||||
[captcha_endpoint | base_endpoints]
|
||||
|> Enum.map(&build_csp_param/1)
|
||||
|
@ -205,7 +200,7 @@ defp build_csp_param(url) when is_binary(url) do
|
|||
end
|
||||
|
||||
def warn_if_disabled do
|
||||
unless Pleroma.Config.get([:http_security, :enabled]) do
|
||||
unless Config.get([:http_security, :enabled]) do
|
||||
Logger.warning("
|
||||
.i;;;;i.
|
||||
iYcviii;vXY:
|
||||
|
@ -250,8 +245,8 @@ def warn_if_disabled do
|
|||
end
|
||||
|
||||
defp maybe_send_sts_header(conn, true) do
|
||||
max_age_sts = @config_impl.get([:http_security, :sts_max_age])
|
||||
max_age_ct = @config_impl.get([:http_security, :ct_max_age])
|
||||
max_age_sts = Config.get([:http_security, :sts_max_age])
|
||||
max_age_ct = Config.get([:http_security, :ct_max_age])
|
||||
|
||||
merge_resp_headers(conn, [
|
||||
{"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"},
|
||||
|
|
|
@ -3,22 +3,10 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
|
||||
alias Pleroma.Helpers.InetHelper
|
||||
|
||||
import Plug.Conn
|
||||
import Phoenix.Controller, only: [get_format: 1, text: 2]
|
||||
|
||||
alias Pleroma.Web.ActivityPub.MRF
|
||||
|
||||
require Logger
|
||||
|
||||
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
|
||||
@http_signatures_impl Application.compile_env(
|
||||
:pleroma,
|
||||
[__MODULE__, :http_signatures_impl],
|
||||
HTTPSignatures
|
||||
)
|
||||
|
||||
def init(options) do
|
||||
options
|
||||
end
|
||||
|
@ -31,9 +19,7 @@ def call(conn, _opts) do
|
|||
if get_format(conn) in ["json", "activity+json"] do
|
||||
conn
|
||||
|> maybe_assign_valid_signature()
|
||||
|> maybe_assign_actor_id()
|
||||
|> maybe_require_signature()
|
||||
|> maybe_filter_requests()
|
||||
else
|
||||
conn
|
||||
end
|
||||
|
@ -47,7 +33,7 @@ defp validate_signature(conn, request_target) do
|
|||
|> put_req_header("(request-target)", request_target)
|
||||
|> put_req_header("@request-target", request_target)
|
||||
|
||||
@http_signatures_impl.validate_conn(conn)
|
||||
HTTPSignatures.validate_conn(conn)
|
||||
end
|
||||
|
||||
defp validate_signature(conn) do
|
||||
|
@ -97,63 +83,20 @@ defp maybe_assign_valid_signature(conn) do
|
|||
end
|
||||
end
|
||||
|
||||
defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do
|
||||
adapter = Application.get_env(:http_signatures, :adapter)
|
||||
|
||||
{:ok, actor_id} = adapter.get_actor_id(conn)
|
||||
|
||||
assign(conn, :actor_id, actor_id)
|
||||
end
|
||||
|
||||
defp maybe_assign_actor_id(conn), do: conn
|
||||
|
||||
defp has_signature_header?(conn) do
|
||||
conn |> get_req_header("signature") |> Enum.at(0, false)
|
||||
end
|
||||
|
||||
defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn
|
||||
|
||||
defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do
|
||||
if @config_impl.get([:activitypub, :authorized_fetch_mode], false) do
|
||||
exceptions =
|
||||
@config_impl.get([:activitypub, :authorized_fetch_mode_exceptions], [])
|
||||
|> Enum.map(&InetHelper.parse_cidr/1)
|
||||
|
||||
if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do
|
||||
conn
|
||||
else
|
||||
conn
|
||||
|> put_status(:unauthorized)
|
||||
|> text("Request not signed")
|
||||
|> halt()
|
||||
end
|
||||
defp maybe_require_signature(conn) do
|
||||
if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do
|
||||
conn
|
||||
|> put_status(:unauthorized)
|
||||
|> text("Request not signed")
|
||||
|> halt()
|
||||
else
|
||||
conn
|
||||
end
|
||||
end
|
||||
|
||||
defp maybe_filter_requests(%{halted: true} = conn), do: conn
|
||||
|
||||
defp maybe_filter_requests(conn) do
|
||||
if @config_impl.get([:activitypub, :authorized_fetch_mode], false) and
|
||||
conn.assigns[:actor_id] do
|
||||
%{host: host} = URI.parse(conn.assigns.actor_id)
|
||||
|
||||
if MRF.subdomain_match?(rejected_domains(), host) do
|
||||
conn
|
||||
|> put_status(:unauthorized)
|
||||
|> halt()
|
||||
else
|
||||
conn
|
||||
end
|
||||
else
|
||||
conn
|
||||
end
|
||||
end
|
||||
|
||||
defp rejected_domains do
|
||||
@config_impl.get([:instance, :rejected_instances])
|
||||
|> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples()
|
||||
|> Pleroma.Web.ActivityPub.MRF.subdomains_regex()
|
||||
end
|
||||
end
|
||||
|
|
|
@ -1,12 +0,0 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.Plugs.LoggerMetadataPath do
|
||||
def init(opts), do: opts
|
||||
|
||||
def call(conn, _) do
|
||||
Logger.metadata(path: conn.request_path)
|
||||
conn
|
||||
end
|
||||
end
|
|
@ -1,18 +0,0 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Web.Plugs.LoggerMetadataUser do
|
||||
alias Pleroma.User
|
||||
|
||||
def init(opts), do: opts
|
||||
|
||||
def call(%{assigns: %{user: user = %User{}}} = conn, _) do
|
||||
Logger.metadata(user: user.nickname)
|
||||
conn
|
||||
end
|
||||
|
||||
def call(conn, _) do
|
||||
conn
|
||||
end
|
||||
end
|
|
@ -8,7 +8,6 @@ defmodule Pleroma.Web.Plugs.RemoteIp do
|
|||
"""
|
||||
|
||||
alias Pleroma.Config
|
||||
alias Pleroma.Helpers.InetHelper
|
||||
import Plug.Conn
|
||||
|
||||
@behaviour Plug
|
||||
|
@ -31,8 +30,19 @@ defp remote_ip_opts do
|
|||
proxies =
|
||||
Config.get([__MODULE__, :proxies], [])
|
||||
|> Enum.concat(reserved)
|
||||
|> Enum.map(&InetHelper.parse_cidr/1)
|
||||
|> Enum.map(&maybe_add_cidr/1)
|
||||
|
||||
{headers, proxies}
|
||||
end
|
||||
|
||||
defp maybe_add_cidr(proxy) when is_binary(proxy) do
|
||||
proxy =
|
||||
cond do
|
||||
"/" in String.codepoints(proxy) -> proxy
|
||||
InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
|
||||
InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
|
||||
end
|
||||
|
||||
InetCidr.parse_cidr!(proxy, true)
|
||||
end
|
||||
end
|
||||
|
|
|
@ -63,25 +63,19 @@ def perform(_) do
|
|||
|
||||
@doc "Push message to web"
|
||||
def push_message(body, sub, api_key, subscription) do
|
||||
try do
|
||||
case WebPushEncryption.send_web_push(body, sub, api_key) do
|
||||
{:ok, %{status: code}} when code in 400..499 ->
|
||||
Logger.debug("Removing subscription record")
|
||||
Repo.delete!(subscription)
|
||||
:ok
|
||||
case WebPushEncryption.send_web_push(body, sub, api_key) do
|
||||
{:ok, %{status: code}} when code in 400..499 ->
|
||||
Logger.debug("Removing subscription record")
|
||||
Repo.delete!(subscription)
|
||||
:ok
|
||||
|
||||
{:ok, %{status: code}} when code in 200..299 ->
|
||||
:ok
|
||||
{:ok, %{status: code}} when code in 200..299 ->
|
||||
:ok
|
||||
|
||||
{:ok, %{status: code}} ->
|
||||
Logger.error("Web Push Notification failed with code: #{code}")
|
||||
:error
|
||||
{:ok, %{status: code}} ->
|
||||
Logger.error("Web Push Notification failed with code: #{code}")
|
||||
:error
|
||||
|
||||
error ->
|
||||
Logger.error("Web Push Notification failed with #{inspect(error)}")
|
||||
:error
|
||||
end
|
||||
rescue
|
||||
error ->
|
||||
Logger.error("Web Push Notification failed with #{inspect(error)}")
|
||||
:error
|
||||
|
|
|
@ -58,7 +58,7 @@ defp check_content_length(headers) do
|
|||
|
||||
defp http_options do
|
||||
[
|
||||
pool: :rich_media,
|
||||
pool: :media,
|
||||
max_body: Config.get([:rich_media, :max_body], 5_000_000)
|
||||
]
|
||||
end
|
||||
|
|
|
@ -23,7 +23,7 @@ defp aws_signed_url?(image) when is_binary(image) and image != "" do
|
|||
%URI{host: host, query: query} = URI.parse(image)
|
||||
|
||||
is_binary(host) and String.contains?(host, "amazonaws.com") and
|
||||
is_binary(query) and String.contains?(query, "X-Amz-Expires")
|
||||
String.contains?(query, "X-Amz-Expires")
|
||||
end
|
||||
|
||||
defp aws_signed_url?(_), do: nil
|
||||
|
|
|
@ -29,7 +29,6 @@ defmodule Pleroma.Web.Router do
|
|||
pipeline :browser do
|
||||
plug(:accepts, ["html"])
|
||||
plug(:fetch_session)
|
||||
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
|
||||
end
|
||||
|
||||
pipeline :oauth do
|
||||
|
@ -68,14 +67,12 @@ defmodule Pleroma.Web.Router do
|
|||
plug(:fetch_session)
|
||||
plug(:authenticate)
|
||||
plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec)
|
||||
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
|
||||
end
|
||||
|
||||
pipeline :no_auth_or_privacy_expectations_api do
|
||||
plug(:base_api)
|
||||
plug(:after_auth)
|
||||
plug(Pleroma.Web.Plugs.IdempotencyPlug)
|
||||
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
|
||||
end
|
||||
|
||||
# Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported)
|
||||
|
@ -86,14 +83,12 @@ defmodule Pleroma.Web.Router do
|
|||
pipeline :api do
|
||||
plug(:expect_public_instance_or_user_authentication)
|
||||
plug(:no_auth_or_privacy_expectations_api)
|
||||
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
|
||||
end
|
||||
|
||||
pipeline :authenticated_api do
|
||||
plug(:expect_user_authentication)
|
||||
plug(:no_auth_or_privacy_expectations_api)
|
||||
plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug)
|
||||
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
|
||||
end
|
||||
|
||||
pipeline :admin_api do
|
||||
|
@ -104,7 +99,6 @@ defmodule Pleroma.Web.Router do
|
|||
plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug)
|
||||
plug(Pleroma.Web.Plugs.UserIsStaffPlug)
|
||||
plug(Pleroma.Web.Plugs.IdempotencyPlug)
|
||||
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
|
||||
end
|
||||
|
||||
pipeline :require_admin do
|
||||
|
@ -185,7 +179,6 @@ defmodule Pleroma.Web.Router do
|
|||
plug(:browser)
|
||||
plug(:authenticate)
|
||||
plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug)
|
||||
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
|
||||
end
|
||||
|
||||
pipeline :well_known do
|
||||
|
@ -200,7 +193,6 @@ defmodule Pleroma.Web.Router do
|
|||
pipeline :pleroma_api do
|
||||
plug(:accepts, ["html", "json"])
|
||||
plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec)
|
||||
plug(Pleroma.Web.Plugs.LoggerMetadataUser)
|
||||
end
|
||||
|
||||
pipeline :mailbox_preview do
|
||||
|
|
|
@ -155,16 +155,7 @@ def get_template_from_xml(body) do
|
|||
end
|
||||
end
|
||||
|
||||
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
|
||||
def find_lrdd_template(domain) do
|
||||
@cachex.fetch!(:host_meta_cache, domain, fn _ ->
|
||||
{:commit, fetch_lrdd_template(domain)}
|
||||
end)
|
||||
rescue
|
||||
e -> {:error, "Cachex error: #{inspect(e)}"}
|
||||
end
|
||||
|
||||
defp fetch_lrdd_template(domain) do
|
||||
# WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1
|
||||
meta_url = "https://#{domain}/.well-known/host-meta"
|
||||
|
||||
|
@ -177,7 +168,7 @@ defp fetch_lrdd_template(domain) do
|
|||
end
|
||||
end
|
||||
|
||||
defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do
|
||||
defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do
|
||||
case find_lrdd_template(domain) do
|
||||
{:ok, template} ->
|
||||
String.replace(template, "{uri}", encoded_account)
|
||||
|
@ -187,11 +178,6 @@ defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_bin
|
|||
end
|
||||
end
|
||||
|
||||
defp get_address_from_domain(domain, account) when is_binary(domain) do
|
||||
encoded_account = URI.encode("acct:#{account}")
|
||||
get_address_from_domain(domain, encoded_account)
|
||||
end
|
||||
|
||||
defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain}
|
||||
|
||||
@spec finger(String.t()) :: {:ok, map()} | {:error, any()}
|
||||
|
@ -206,7 +192,9 @@ def finger(account) do
|
|||
URI.parse(account).host
|
||||
end
|
||||
|
||||
with address when is_binary(address) <- get_address_from_domain(domain, account),
|
||||
encoded_account = URI.encode("acct:#{account}")
|
||||
|
||||
with address when is_binary(address) <- get_address_from_domain(domain, encoded_account),
|
||||
{:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <-
|
||||
HTTP.get(
|
||||
address,
|
||||
|
@ -228,28 +216,10 @@ def finger(account) do
|
|||
_ ->
|
||||
{:error, {:content_type, nil}}
|
||||
end
|
||||
|> case do
|
||||
{:ok, data} -> validate_webfinger(address, data)
|
||||
error -> error
|
||||
end
|
||||
else
|
||||
error ->
|
||||
Logger.debug("Couldn't finger #{account}: #{inspect(error)}")
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do
|
||||
with [_name, acct_host] <- String.split(acct, "@"),
|
||||
{_, url} <- {:address, get_address_from_domain(acct_host, subject)},
|
||||
%URI{host: request_host} <- URI.parse(request_url),
|
||||
%URI{host: acct_host} <- URI.parse(url),
|
||||
{_, true} <- {:hosts_match, acct_host == request_host} do
|
||||
{:ok, data}
|
||||
else
|
||||
_ -> {:error, {:webfinger_invalid, request_url, data}}
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}}
|
||||
end
|
||||
|
|
|
@ -8,7 +8,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
|
|||
alias Pleroma.Object
|
||||
alias Pleroma.Repo
|
||||
|
||||
use Pleroma.Workers.WorkerHelper, queue: "slow"
|
||||
use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup"
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(%Job{
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Workers.BackupWorker do
|
||||
use Oban.Worker, queue: :slow, max_attempts: 1
|
||||
use Oban.Worker, queue: :backup, max_attempts: 1
|
||||
|
||||
alias Oban.Job
|
||||
alias Pleroma.User.Backup
|
||||
|
|
|
@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do
|
|||
|
||||
import Ecto.Query
|
||||
|
||||
use Pleroma.Workers.WorkerHelper, queue: "background"
|
||||
use Pleroma.Workers.WorkerHelper, queue: "mailer"
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(_job) do
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue