From 718e8e1edb537aca984216be39b3be5c8af4e6da Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 May 2021 21:39:58 -0500 Subject: [PATCH 001/161] Create NsfwApiPolicy --- config/config.exs | 7 + .../web/activity_pub/mrf/nsfw_api_policy.ex | 185 ++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex diff --git a/config/config.exs b/config/config.exs index 66aee3264..d96dc1646 100644 --- a/config/config.exs +++ b/config/config.exs @@ -404,6 +404,13 @@ threshold: 604_800, actions: [:delist, :strip_followers] +config :pleroma, :mrf_nsfw_api, + url: "http://127.0.0.1:5000/", + threshold: 0.7, + mark_sensitive: true, + unlist: false, + reject: false + config :pleroma, :rich_media, enabled: true, ignore_hosts: [], diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex new file mode 100644 index 000000000..9ad175b1b --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -0,0 +1,185 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do + @moduledoc """ + Hide, delete, or mark sensitive NSFW content with artificial intelligence. + + Requires a NSFW API server, configured like so: + + config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF, + url: "http://127.0.0.1:5000/", + threshold: 0.8, + mark_sensitive: true, + unlist: false, + reject: false + + The NSFW API server must implement an HTTP endpoint like this: + + curl http://localhost:5000/?url=https://fedi.com/images/001.jpg + + Returning a response like this: + + {"score", 0.314} + + Where a score is 0-1, with `1` being definitely NSFW. + + A good API server is here: https://github.com/EugenCepoi/nsfw_api + You can run it with Docker with a one-liner: + + docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest + + Options: + + - `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/" + - `threshold`: Lowest score to take action on. Default: `0.7` + - `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true` + - `unlist`: Unlist all detected NSFW content? Default: `false` + - `reject`: Reject all detected NSFW content (takes precedence)? Default: `false` + """ + alias Pleroma.Config + alias Pleroma.Constants + alias Pleroma.HTTP + alias Pleroma.User + + require Logger + require Pleroma.Constants + + @behaviour Pleroma.Web.ActivityPub.MRF + @policy :mrf_nsfw_api + + defp build_request_url(url) do + Config.get([@policy, :url]) + |> URI.parse() + |> Map.put(:query, "url=#{url}") + |> URI.to_string() + end + + defp parse_url(url) do + request = build_request_url(url) + + with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do + Jason.decode(body) + else + error -> + Logger.warn(""" + [NsfwApiPolicy]: The API server failed. Skipping. + #{inspect(error)} + """) + + error + end + end + + defp check_url_nsfw(url) when is_binary(url) do + threshold = Config.get([@policy, :threshold]) + + case parse_url(url) do + {:ok, %{"score" => score}} when score >= threshold -> + {:nsfw, %{url: url, score: score, threshold: threshold}} + + _ -> + {:sfw, url} + end + end + + defp check_url_nsfw(%{"href" => url}) when is_binary(url) do + check_url_nsfw(url) + end + + defp check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do + if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do + {:sfw, attachment} + else + {:nsfw, attachment} + end + end + + defp check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do + if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do + {:sfw, object} + else + {:nsfw, object} + end + end + + defp check_object_nsfw(%{"object" => %{} = child_object} = object) do + case check_object_nsfw(child_object) do + {:sfw, _} -> {:sfw, object} + {:nsfw, _} -> {:nsfw, object} + end + end + + defp check_object_nsfw(object), do: {:sfw, object} + + @impl true + def filter(object) do + with {:sfw, object} <- check_object_nsfw(object) do + {:ok, object} + else + {:nsfw, _data} -> handle_nsfw(object) + _ -> {:reject, "NSFW: Attachment rejected"} + end + end + + defp handle_nsfw(object) do + if Config.get([@policy, :reject]) do + {:reject, object} + else + {:ok, + object + |> maybe_unlist() + |> maybe_mark_sensitive()} + end + end + + defp maybe_unlist(object) do + if Config.get([@policy, :unlist]) do + unlist(object) + else + object + end + end + + defp maybe_mark_sensitive(object) do + if Config.get([@policy, :mark_sensitive]) do + mark_sensitive(object) + else + object + end + end + + defp unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do + with %User{} = user <- User.get_cached_by_ap_id(actor) do + to = + [user.follower_address | to] + |> List.delete(Constants.as_public()) + |> Enum.uniq() + + cc = + [Constants.as_public() | cc] + |> List.delete(user.follower_address) + |> Enum.uniq() + + object + |> Map.put("to", to) + |> Map.put("cc", cc) + end + end + + defp mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do + Map.put(object, "object", mark_sensitive(child_object)) + end + + defp mark_sensitive(object) when is_map(object) do + tags = (object["tag"] || []) ++ ["nsfw"] + + object + |> Map.put("tag", tags) + |> Map.put("sensitive", true) + end + + @impl true + def describe, do: {:ok, %{}} +end From f15d419062b5f9aba2a2e84257dc2379b44f92e8 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 16 Jun 2021 22:30:18 -0500 Subject: [PATCH 002/161] NsfwApiPolicy: raise if can't fetch user --- lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index 9ad175b1b..63e6af0a0 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -165,6 +165,8 @@ defp unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do object |> Map.put("to", to) |> Map.put("cc", cc) + else + _ -> raise "[NsfwApiPolicy]: Could not fetch user #{actor}" end end From 2b3dfbb42f7ec0c5604876276a81d55a05955416 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 17 Jun 2021 14:36:51 -0500 Subject: [PATCH 003/161] NsfwApiPolicy: add tests --- .../web/activity_pub/mrf/nsfw_api_policy.ex | 45 ++- .../activity_pub/mrf/nsfw_api_policy_test.exs | 267 ++++++++++++++++++ 2 files changed, 299 insertions(+), 13 deletions(-) create mode 100644 test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index 63e6af0a0..9dcdf560e 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -49,14 +49,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do @behaviour Pleroma.Web.ActivityPub.MRF @policy :mrf_nsfw_api - defp build_request_url(url) do + def build_request_url(url) do Config.get([@policy, :url]) |> URI.parse() + |> fix_path() |> Map.put(:query, "url=#{url}") |> URI.to_string() end - defp parse_url(url) do + def parse_url(url) do request = build_request_url(url) with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do @@ -72,23 +73,26 @@ defp parse_url(url) do end end - defp check_url_nsfw(url) when is_binary(url) do + def check_url_nsfw(url) when is_binary(url) do threshold = Config.get([@policy, :threshold]) case parse_url(url) do {:ok, %{"score" => score}} when score >= threshold -> {:nsfw, %{url: url, score: score, threshold: threshold}} + {:ok, %{"score" => score}} -> + {:sfw, %{url: url, score: score, threshold: threshold}} + _ -> - {:sfw, url} + {:sfw, %{url: url, score: nil, threshold: threshold}} end end - defp check_url_nsfw(%{"href" => url}) when is_binary(url) do + def check_url_nsfw(%{"href" => url}) when is_binary(url) do check_url_nsfw(url) end - defp check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do + def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do {:sfw, attachment} else @@ -96,7 +100,14 @@ defp check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do end end - defp check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do + def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do + case check_url_nsfw(url) do + {:sfw, _} -> {:sfw, attachment} + {:nsfw, _} -> {:nsfw, attachment} + end + end + + def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do {:sfw, object} else @@ -104,14 +115,14 @@ defp check_object_nsfw(%{"attachment" => attachments} = object) when is_list(att end end - defp check_object_nsfw(%{"object" => %{} = child_object} = object) do + def check_object_nsfw(%{"object" => %{} = child_object} = object) do case check_object_nsfw(child_object) do {:sfw, _} -> {:sfw, object} {:nsfw, _} -> {:nsfw, object} end end - defp check_object_nsfw(object), do: {:sfw, object} + def check_object_nsfw(object), do: {:sfw, object} @impl true def filter(object) do @@ -150,7 +161,7 @@ defp maybe_mark_sensitive(object) do end end - defp unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do + def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do with %User{} = user <- User.get_cached_by_ap_id(actor) do to = [user.follower_address | to] @@ -166,15 +177,15 @@ defp unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do |> Map.put("to", to) |> Map.put("cc", cc) else - _ -> raise "[NsfwApiPolicy]: Could not fetch user #{actor}" + _ -> raise "[NsfwApiPolicy]: Could not find user #{actor}" end end - defp mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do + def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do Map.put(object, "object", mark_sensitive(child_object)) end - defp mark_sensitive(object) when is_map(object) do + def mark_sensitive(object) when is_map(object) do tags = (object["tag"] || []) ++ ["nsfw"] object @@ -182,6 +193,14 @@ defp mark_sensitive(object) when is_map(object) do |> Map.put("sensitive", true) end + # Hackney needs a trailing slash + defp fix_path(%URI{path: path} = uri) when is_binary(path) do + path = String.trim_trailing(path, "/") <> "/" + Map.put(uri, :path, path) + end + + defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/") + @impl true def describe, do: {:ok, %{}} end diff --git a/test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs b/test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs new file mode 100644 index 000000000..0beb9c2cb --- /dev/null +++ b/test/pleroma/web/activity_pub/mrf/nsfw_api_policy_test.exs @@ -0,0 +1,267 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicyTest do + use Pleroma.DataCase + + import ExUnit.CaptureLog + import Pleroma.Factory + + alias Pleroma.Constants + alias Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy + + require Pleroma.Constants + + @policy :mrf_nsfw_api + + @sfw_url "https://kittens.co/kitty.gif" + @nsfw_url "https://b00bies.com/nsfw.jpg" + @timeout_url "http://time.out/i.jpg" + + setup_all do + clear_config(@policy, + url: "http://127.0.0.1:5000/", + threshold: 0.7, + mark_sensitive: true, + unlist: false, + reject: false + ) + end + + setup do + Tesla.Mock.mock(fn + # NSFW URL + %{method: :get, url: "http://127.0.0.1:5000/?url=#{@nsfw_url}"} -> + %Tesla.Env{status: 200, body: ~s({"score":0.99772077798843384,"url":"#{@nsfw_url}"})} + + # SFW URL + %{method: :get, url: "http://127.0.0.1:5000/?url=#{@sfw_url}"} -> + %Tesla.Env{status: 200, body: ~s({"score":0.00011714912398019806,"url":"#{@sfw_url}"})} + + # Timeout URL + %{method: :get, url: "http://127.0.0.1:5000/?url=#{@timeout_url}"} -> + {:error, :timeout} + + # Fallback URL + %{method: :get, url: "http://127.0.0.1:5000/?url=" <> url} -> + body = + ~s({"error_code":500,"error_reason":"[Errno -2] Name or service not known","url":"#{url}"}) + + %Tesla.Env{status: 500, body: body} + end) + + :ok + end + + describe "build_request_url/1" do + test "it works" do + expected = "http://127.0.0.1:5000/?url=https://b00bies.com/nsfw.jpg" + assert NsfwApiPolicy.build_request_url(@nsfw_url) == expected + end + + test "it adds a trailing slash" do + clear_config([@policy, :url], "http://localhost:5000") + + expected = "http://localhost:5000/?url=https://b00bies.com/nsfw.jpg" + assert NsfwApiPolicy.build_request_url(@nsfw_url) == expected + end + + test "it adds a trailing slash preserving the path" do + clear_config([@policy, :url], "http://localhost:5000/nsfw_api") + + expected = "http://localhost:5000/nsfw_api/?url=https://b00bies.com/nsfw.jpg" + assert NsfwApiPolicy.build_request_url(@nsfw_url) == expected + end + end + + describe "parse_url/1" do + test "returns decoded JSON from the API server" do + expected = %{"score" => 0.99772077798843384, "url" => @nsfw_url} + assert NsfwApiPolicy.parse_url(@nsfw_url) == {:ok, expected} + end + + test "warns when the API server fails" do + expected = "[NsfwApiPolicy]: The API server failed. Skipping." + assert capture_log(fn -> NsfwApiPolicy.parse_url(@timeout_url) end) =~ expected + end + + test "returns {:error, _} tuple when the API server fails" do + capture_log(fn -> + assert {:error, _} = NsfwApiPolicy.parse_url(@timeout_url) + end) + end + end + + describe "check_url_nsfw/1" do + test "returns {:nsfw, _} tuple" do + expected = {:nsfw, %{url: @nsfw_url, score: 0.99772077798843384, threshold: 0.7}} + assert NsfwApiPolicy.check_url_nsfw(@nsfw_url) == expected + end + + test "returns {:sfw, _} tuple" do + expected = {:sfw, %{url: @sfw_url, score: 0.00011714912398019806, threshold: 0.7}} + assert NsfwApiPolicy.check_url_nsfw(@sfw_url) == expected + end + + test "returns {:sfw, _} on failure" do + expected = {:sfw, %{url: @timeout_url, score: nil, threshold: 0.7}} + + capture_log(fn -> + assert NsfwApiPolicy.check_url_nsfw(@timeout_url) == expected + end) + end + + test "works with map URL" do + expected = {:nsfw, %{url: @nsfw_url, score: 0.99772077798843384, threshold: 0.7}} + assert NsfwApiPolicy.check_url_nsfw(%{"href" => @nsfw_url}) == expected + end + end + + describe "check_attachment_nsfw/1" do + test "returns {:nsfw, _} if any items are NSFW" do + attachment = %{"url" => [%{"href" => @nsfw_url}, @nsfw_url, @sfw_url]} + assert NsfwApiPolicy.check_attachment_nsfw(attachment) == {:nsfw, attachment} + end + + test "returns {:sfw, _} if all items are SFW" do + attachment = %{"url" => [%{"href" => @sfw_url}, @sfw_url, @sfw_url]} + assert NsfwApiPolicy.check_attachment_nsfw(attachment) == {:sfw, attachment} + end + + test "works with binary URL" do + attachment = %{"url" => @nsfw_url} + assert NsfwApiPolicy.check_attachment_nsfw(attachment) == {:nsfw, attachment} + end + end + + describe "check_object_nsfw/1" do + test "returns {:nsfw, _} if any items are NSFW" do + object = %{"attachment" => [%{"url" => [%{"href" => @nsfw_url}, @sfw_url]}]} + assert NsfwApiPolicy.check_object_nsfw(object) == {:nsfw, object} + end + + test "returns {:sfw, _} if all items are SFW" do + object = %{"attachment" => [%{"url" => [%{"href" => @sfw_url}, @sfw_url]}]} + assert NsfwApiPolicy.check_object_nsfw(object) == {:sfw, object} + end + + test "works with embedded object" do + object = %{"object" => %{"attachment" => [%{"url" => [%{"href" => @nsfw_url}, @sfw_url]}]}} + assert NsfwApiPolicy.check_object_nsfw(object) == {:nsfw, object} + end + end + + describe "unlist/1" do + test "unlist addressing" do + user = insert(:user) + + object = %{ + "to" => [Constants.as_public()], + "cc" => [user.follower_address, "https://hello.world/users/alex"], + "actor" => user.ap_id + } + + expected = %{ + "to" => [user.follower_address], + "cc" => [Constants.as_public(), "https://hello.world/users/alex"], + "actor" => user.ap_id + } + + assert NsfwApiPolicy.unlist(object) == expected + end + + test "raise if user isn't found" do + object = %{ + "to" => [Constants.as_public()], + "cc" => [], + "actor" => "https://hello.world/users/alex" + } + + assert_raise(RuntimeError, fn -> + NsfwApiPolicy.unlist(object) + end) + end + end + + describe "mark_sensitive/1" do + test "adds nsfw tag and marks sensitive" do + object = %{"tag" => ["yolo"]} + expected = %{"tag" => ["yolo", "nsfw"], "sensitive" => true} + assert NsfwApiPolicy.mark_sensitive(object) == expected + end + + test "works with embedded object" do + object = %{"object" => %{"tag" => ["yolo"]}} + expected = %{"object" => %{"tag" => ["yolo", "nsfw"], "sensitive" => true}} + assert NsfwApiPolicy.mark_sensitive(object) == expected + end + end + + describe "filter/1" do + setup do + user = insert(:user) + + nsfw_object = %{ + "to" => [Constants.as_public()], + "cc" => [user.follower_address], + "actor" => user.ap_id, + "attachment" => [%{"url" => @nsfw_url}] + } + + sfw_object = %{ + "to" => [Constants.as_public()], + "cc" => [user.follower_address], + "actor" => user.ap_id, + "attachment" => [%{"url" => @sfw_url}] + } + + %{user: user, nsfw_object: nsfw_object, sfw_object: sfw_object} + end + + test "passes SFW object through", %{sfw_object: object} do + {:ok, _} = NsfwApiPolicy.filter(object) + end + + test "passes NSFW object through when actions are disabled", %{nsfw_object: object} do + clear_config([@policy, :mark_sensitive], false) + clear_config([@policy, :unlist], false) + clear_config([@policy, :reject], false) + {:ok, _} = NsfwApiPolicy.filter(object) + end + + test "passes NSFW object through when :threshold is 1", %{nsfw_object: object} do + clear_config([@policy, :reject], true) + clear_config([@policy, :threshold], 1) + {:ok, _} = NsfwApiPolicy.filter(object) + end + + test "rejects SFW object through when :threshold is 0", %{sfw_object: object} do + clear_config([@policy, :reject], true) + clear_config([@policy, :threshold], 0) + {:reject, _} = NsfwApiPolicy.filter(object) + end + + test "rejects NSFW when :reject is enabled", %{nsfw_object: object} do + clear_config([@policy, :reject], true) + {:reject, _} = NsfwApiPolicy.filter(object) + end + + test "passes NSFW through when :reject is disabled", %{nsfw_object: object} do + clear_config([@policy, :reject], false) + {:ok, _} = NsfwApiPolicy.filter(object) + end + + test "unlists NSFW when :unlist is enabled", %{user: user, nsfw_object: object} do + clear_config([@policy, :unlist], true) + {:ok, object} = NsfwApiPolicy.filter(object) + assert object["to"] == [user.follower_address] + end + + test "passes NSFW through when :unlist is disabled", %{nsfw_object: object} do + clear_config([@policy, :unlist], false) + {:ok, object} = NsfwApiPolicy.filter(object) + assert object["to"] == [Constants.as_public()] + end + end +end From b293c14a1b01398029dfa80aea306946efc2f284 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 17 Jun 2021 14:52:07 -0500 Subject: [PATCH 004/161] NsfwApiPolicy: add describe/0 and config_description/0 --- .../web/activity_pub/mrf/nsfw_api_policy.ex | 56 ++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index 9dcdf560e..a1560c584 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF, url: "http://127.0.0.1:5000/", - threshold: 0.8, + threshold: 0.7, mark_sensitive: true, unlist: false, reject: false @@ -202,5 +202,57 @@ defp fix_path(%URI{path: path} = uri) when is_binary(path) do defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/") @impl true - def describe, do: {:ok, %{}} + def describe do + options = %{ + threshold: Config.get([@policy, :threshold]), + mark_sensitive: Config.get([@policy, :mark_sensitive]), + unlist: Config.get([@policy, :unlist]), + reject: Config.get([@policy, :reject]) + } + + {:ok, %{@policy => options}} + end + + @impl true + def config_description do + %{ + key: @policy, + related_policy: to_string(__MODULE__), + label: "NSFW API Policy", + description: + "Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.", + children: [ + %{ + key: :url, + type: :string, + description: "Base URL of the API server.", + suggestions: ["http://127.0.0.1:5000/"] + }, + %{ + key: :threshold, + type: :float, + description: "Lowest score to take action on. Between 0 and 1.", + suggestions: [0.7] + }, + %{ + key: :mark_sensitive, + type: :boolean, + description: "Mark sensitive all detected NSFW content?", + suggestions: [true] + }, + %{ + key: :unlist, + type: :boolean, + description: "Unlist sensitive all detected NSFW content?", + suggestions: [false] + }, + %{ + key: :reject, + type: :boolean, + description: "Reject sensitive all detected NSFW content (takes precedence)?", + suggestions: [false] + } + ] + } + end end From c802c3055ef6c1f763d5df68f9e5308093f7d565 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 17 Jun 2021 15:04:40 -0500 Subject: [PATCH 005/161] NsfwApiPolicy: add systemd example file --- installation/nsfw-api.service | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 installation/nsfw-api.service diff --git a/installation/nsfw-api.service b/installation/nsfw-api.service new file mode 100644 index 000000000..ec629df67 --- /dev/null +++ b/installation/nsfw-api.service @@ -0,0 +1,15 @@ +[Unit] +Description=NSFW API +After=docker.service +Requires=docker.service + +[Service] +TimeoutStartSec=0 +Restart=always +ExecStartPre=-/usr/bin/docker stop %n +ExecStartPre=-/usr/bin/docker rm %n +ExecStartPre=/usr/bin/docker pull eugencepoi/nsfw_api:latest +ExecStart=/usr/bin/docker run --rm -p 127.0.0.1:5000:5000/tcp --env PORT=5000 --name %n eugencepoi/nsfw_api:latest + +[Install] +WantedBy=multi-user.target From a704d5499c03cb5609ea38a5f2ef06095ced3ef3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 17 Jun 2021 15:32:42 -0500 Subject: [PATCH 006/161] NsfwApiPolicy: Fall back more generously when functions don't match --- lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index a1560c584..920821f38 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -92,6 +92,11 @@ def check_url_nsfw(%{"href" => url}) when is_binary(url) do check_url_nsfw(url) end + def check_url_nsfw(url) do + threshold = Config.get([@policy, :threshold]) + {:sfw, %{url: url, score: nil, threshold: threshold}} + end + def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do {:sfw, attachment} @@ -107,6 +112,8 @@ def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do end end + def check_attachment_nsfw(attachment), do: {:sfw, attachment} + def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do {:sfw, object} From 9423052e9217aa1358950d37c5c96b11d554b37a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 25 Apr 2022 12:39:36 +0200 Subject: [PATCH 007/161] Add "status" notification type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/notification.ex | 38 ++++++++++++-- .../operations/notification_operation.ex | 4 +- .../controllers/notification_controller.ex | 1 + .../mastodon_api/views/notification_view.ex | 3 ++ lib/pleroma/web/push/impl.ex | 1 + ...00000_add_status_to_notifications_enum.exs | 50 +++++++++++++++++++ test/pleroma/notification_test.exs | 1 + 7 files changed, 92 insertions(+), 6 deletions(-) create mode 100644 priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 52fd2656b..d142baa8b 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -73,6 +73,7 @@ def unread_notifications_count(%User{id: user_id}) do pleroma:report reblog poll + status } def changeset(%Notification{} = notification, attrs) do @@ -397,11 +398,18 @@ defp do_create_notifications(%Activity{} = activity, options) do {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity) potential_receivers = enabled_receivers ++ disabled_receivers + {enabled_subscribers, disabled_subscribers} = get_notified_subscribers_from_activity(activity) + potential_subscribers = (enabled_subscribers ++ disabled_subscribers) -- potential_receivers + notifications = - Enum.map(potential_receivers, fn user -> - do_send = do_send && user in enabled_receivers - create_notification(activity, user, do_send: do_send) - end) + (Enum.map(potential_receivers, fn user -> + do_send = do_send && user in enabled_receivers + create_notification(activity, user, do_send: do_send) + end) ++ + Enum.map(potential_subscribers, fn user -> + do_send = do_send && user in enabled_subscribers + create_notification(activity, user, do_send: do_send, type: "status") + end)) |> Enum.reject(&is_nil/1) {:ok, notifications} @@ -533,6 +541,27 @@ def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, lo def get_notified_from_activity(_, _local_only), do: {[], []} + def get_notified_subscribers_from_activity(activity, local_only \\ true) + + def get_notified_subscribers_from_activity( + %Activity{data: %{"type" => "Create"}} = activity, + local_only + ) do + notification_enabled_ap_ids = + [] + |> Utils.maybe_notify_subscribers(activity) + + potential_receivers = + User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only) + + notification_enabled_users = + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) + + {notification_enabled_users, potential_receivers -- notification_enabled_users} + end + + def get_notified_subscribers_from_activity(_, _), do: {[], []} + # For some activities, only notify the author of the object def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}}) when type in ~w{Like Announce EmojiReact} do @@ -557,7 +586,6 @@ def get_potential_receiver_ap_ids(activity) do [] |> Utils.maybe_notify_to_recipients(activity) |> Utils.maybe_notify_mentioned_recipients(activity) - |> Utils.maybe_notify_subscribers(activity) |> Utils.maybe_notify_followers(activity) |> Enum.uniq() end diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 7f2336ff6..aa965fabb 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -196,7 +196,8 @@ defp notification_type do "pleroma:report", "move", "follow_request", - "poll" + "poll", + "status" ], description: """ The type of event that resulted in the notification. @@ -210,6 +211,7 @@ defp notification_type do - `pleroma:emoji_reaction` - Someone reacted with emoji to your status - `pleroma:chat_mention` - Someone mentioned you in a chat message - `pleroma:report` - Someone was reported + - `status` - Someone you are subscribed to created a status """ } end diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index 932bc6423..9209e8ebd 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -51,6 +51,7 @@ def index(conn, %{account_id: account_id} = params) do move pleroma:emoji_reaction poll + status } def index(%{assigns: %{user: user}} = conn, params) do params = diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 0dc7f3beb..b10b0893c 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -103,6 +103,9 @@ def render( "mention" -> put_status(response, activity, reading_user, status_render_opts) + "status" -> + put_status(response, activity, reading_user, status_render_opts) + "favourite" -> put_status(response, parent_activity_fn.(), reading_user, status_render_opts) diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index daf3eeb9e..77bc2941d 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -183,6 +183,7 @@ def format_title(%{activity: %{data: %{"directMessage" => true}}}, _mastodon_typ def format_title(%{type: type}, mastodon_type) do case mastodon_type || type do "mention" -> "New Mention" + "status" -> "New Status" "follow" -> "New Follower" "follow_request" -> "New Follow Request" "reblog" -> "New Repeat" diff --git a/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs new file mode 100644 index 000000000..62c0afb63 --- /dev/null +++ b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs @@ -0,0 +1,50 @@ +defmodule Pleroma.Repo.Migrations.AddStatusToNotificationsEnum do + use Ecto.Migration + + @disable_ddl_transaction true + + def up do + """ + alter type notification_type add value 'status' + """ + |> execute() + end + + def down do + alter table(:notifications) do + modify(:type, :string) + end + + """ + delete from notifications where type = 'status' + """ + |> execute() + + """ + drop type if exists notification_type + """ + |> execute() + + """ + create type notification_type as enum ( + 'follow', + 'follow_request', + 'mention', + 'move', + 'pleroma:emoji_reaction', + 'pleroma:chat_mention', + 'reblog', + 'favourite', + 'pleroma:report', + 'poll + ) + """ + |> execute() + + """ + alter table notifications + alter column type type notification_type using (type::notification_type) + """ + |> execute() + end +end diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index 805764ea4..eea2fcb67 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -104,6 +104,7 @@ test "it creates a notification for subscribed users" do {:ok, [notification]} = Notification.create_notifications(status) assert notification.user_id == subscriber.id + assert notification.type == "status" end test "does not create a notification for subscribed users if status is a reply" do From bd52e2aec7f07da3bc3609f72f7f1bf5969c9baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 5 Feb 2022 18:03:53 +0100 Subject: [PATCH 008/161] Instance rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/rule.ex | 56 +++++++++ .../admin_api/controllers/rule_controller.ex | 64 ++++++++++ lib/pleroma/web/admin_api/views/rule_view.ex | 21 ++++ .../operations/admin/rule_operation.ex | 113 ++++++++++++++++++ .../web/mastodon_api/views/instance_view.ex | 7 ++ lib/pleroma/web/router.ex | 5 + .../20220203224011_create_rules.exs | 12 ++ .../controllers/rule_controller_test.exs | 78 ++++++++++++ .../controllers/instance_controller_test.exs | 18 ++- 9 files changed, 373 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/rule.ex create mode 100644 lib/pleroma/web/admin_api/controllers/rule_controller.ex create mode 100644 lib/pleroma/web/admin_api/views/rule_view.ex create mode 100644 lib/pleroma/web/api_spec/operations/admin/rule_operation.ex create mode 100644 priv/repo/migrations/20220203224011_create_rules.exs create mode 100644 test/pleroma/web/admin_api/controllers/rule_controller_test.exs diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex new file mode 100644 index 000000000..d772a32bd --- /dev/null +++ b/lib/pleroma/rule.ex @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Rule do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + + alias Pleroma.Repo + alias Pleroma.Rule + + schema "rules" do + field(:priority, :integer, default: 0) + field(:text, :string) + + timestamps() + end + + def changeset(%Rule{} = rule, params \\ %{}) do + rule + |> cast(params, [:priority, :text]) + |> validate_required([:text]) + end + + def query do + Rule + |> order_by(asc: :priority) + end + + def get(id), do: Repo.get(__MODULE__, id) + + def create(params) do + {:ok, rule} = + %Rule{} + |> changeset(params) + |> Repo.insert() + + rule + end + + def update(params, id) do + {:ok, rule} = + get(id) + |> changeset(params) + |> Repo.update() + + rule + end + + def delete(id) do + get(id) + |> Repo.delete() + end +end diff --git a/lib/pleroma/web/admin_api/controllers/rule_controller.ex b/lib/pleroma/web/admin_api/controllers/rule_controller.ex new file mode 100644 index 000000000..2db88b6ba --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/rule_controller.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.RuleController do + use Pleroma.Web, :controller + + alias Pleroma.Repo + alias Pleroma.Rule + alias Pleroma.Web.Plugs.OAuthScopesPlug + + import Pleroma.Web.ControllerHelper, + only: [ + json_response: 3 + ] + + require Logger + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:write"]} + when action in [:create, :update, :delete] + ) + + plug(OAuthScopesPlug, %{scopes: ["admin:read"]} when action == :index) + + action_fallback(AdminAPI.FallbackController) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.RuleOperation + + def index(conn, _) do + rules = + Rule.query() + |> Repo.all() + + render(conn, "index.json", rules: rules) + end + + def create(%{body_params: params} = conn, _) do + rule = + params + |> Rule.create() + + render(conn, "show.json", rule: rule) + end + + def update(%{body_params: params} = conn, %{id: id}) do + rule = + params + |> Rule.update(id) + + render(conn, "show.json", rule: rule) + end + + def delete(conn, %{id: id}) do + with {:ok, _} <- Rule.delete(id) do + json(conn, %{}) + else + _ -> json_response(conn, :bad_request, "") + end + end +end diff --git a/lib/pleroma/web/admin_api/views/rule_view.ex b/lib/pleroma/web/admin_api/views/rule_view.ex new file mode 100644 index 000000000..f29145248 --- /dev/null +++ b/lib/pleroma/web/admin_api/views/rule_view.ex @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.RuleView do + use Pleroma.Web, :view + + require Pleroma.Constants + + def render("index.json", %{rules: rules} = _opts) do + render_many(rules, __MODULE__, "show.json") + end + + def render("show.json", %{rule: rule} = _opts) do + %{ + id: rule.id, + priority: rule.priority, + text: rule.text + } + end +end diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex new file mode 100644 index 000000000..28f2be5e7 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex @@ -0,0 +1,113 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Retrieve a list of instance rules", + operationId: "AdminAPI.RuleController.index", + security: [%{"oAuth" => ["admin:read"]}], + responses: %{ + 200 => + Operation.response("Response", "application/json", %Schema{ + type: :array, + items: rule() + }), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def create_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Create new rule", + operationId: "AdminAPI.RuleController.create", + security: [%{"oAuth" => ["admin:write"]}], + parameters: admin_api_params(), + requestBody: request_body("Parameters", create_request(), required: true), + responses: %{ + 200 => Operation.response("Response", "application/json", rule()), + 400 => Operation.response("Bad Request", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def update_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Modify existing rule", + operationId: "AdminAPI.RuleController.update", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [Operation.parameter(:id, :path, :string, "Rule ID")], + requestBody: request_body("Parameters", update_request(), required: true), + responses: %{ + 200 => Operation.response("Response", "application/json", rule()), + 400 => Operation.response("Bad Request", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def delete_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Delete rule", + operationId: "AdminAPI.RuleController.delete", + parameters: [Operation.parameter(:id, :path, :string, "Rule ID")], + security: [%{"oAuth" => ["admin:write"]}], + responses: %{ + 200 => empty_object_response(), + 404 => Operation.response("Not Found", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + defp create_request do + %Schema{ + type: :object, + required: [:text], + properties: %{ + priority: %Schema{type: :integer}, + text: %Schema{type: :string} + } + } + end + + defp update_request do + %Schema{ + type: :object, + properties: %{ + priority: %Schema{type: :integer}, + text: %Schema{type: :string} + } + } + end + + defp rule do + %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :integer}, + priority: %Schema{type: :integer}, + text: %Schema{type: :string}, + created_at: %Schema{type: :string, format: :"date-time"} + } + } + end +end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index ee52475d5..9652da53a 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -40,6 +40,7 @@ def render("show.json", _) do background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit), + rules: rules(), pleroma: %{ metadata: %{ account_activation_required: Keyword.get(instance, :account_activation_required), @@ -137,4 +138,10 @@ def fields_limits do value_length: Config.get([:instance, :account_field_value_length]) } end + + def rules do + Pleroma.Rule.query() + |> Pleroma.Repo.all() + |> Enum.map(&%{id: &1.id, text: &1.text}) + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ceb6c3cfd..e8b7b17aa 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -229,6 +229,11 @@ defmodule Pleroma.Web.Router do post("/frontends/install", FrontendController, :install) post("/backups", AdminAPIController, :create_backup) + + get("/rules", RuleController, :index) + post("/rules", RuleController, :create) + patch("/rules/:id", RuleController, :update) + delete("/rules/:id", RuleController, :delete) end # AdminAPI: admins and mods (staff) can perform these actions (if enabled by config) diff --git a/priv/repo/migrations/20220203224011_create_rules.exs b/priv/repo/migrations/20220203224011_create_rules.exs new file mode 100644 index 000000000..16f29ca53 --- /dev/null +++ b/priv/repo/migrations/20220203224011_create_rules.exs @@ -0,0 +1,12 @@ +defmodule Pleroma.Repo.Migrations.CreateRules do + use Ecto.Migration + + def change do + create_if_not_exists table(:rules) do + add(:priority, :integer, default: 0, null: false) + add(:text, :text, null: false) + + timestamps() + end + end +end diff --git a/test/pleroma/web/admin_api/controllers/rule_controller_test.exs b/test/pleroma/web/admin_api/controllers/rule_controller_test.exs new file mode 100644 index 000000000..c5c72d293 --- /dev/null +++ b/test/pleroma/web/admin_api/controllers/rule_controller_test.exs @@ -0,0 +1,78 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.RuleControllerTest do + use Pleroma.Web.ConnCase, async: true + + import Pleroma.Factory + + alias Pleroma.Rule + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + {:ok, %{admin: admin, token: token, conn: conn}} + end + + describe "GET /api/pleroma/admin/rules" do + test "sorts rules by priority", %{conn: conn} do + %{id: id1} = Rule.create(%{text: "Example rule"}) + %{id: id2} = Rule.create(%{text: "Second rule", priority: 2}) + %{id: id3} = Rule.create(%{text: "Third rule", priority: 1}) + + response = + conn + |> get("/api/pleroma/admin/rules") + |> json_response_and_validate_schema(:ok) + + assert [%{"id" => ^id1}, %{"id" => ^id3}, %{"id" => ^id2}] = response + end + end + + describe "POST /api/pleroma/admin/rules" do + test "creates a rule", %{conn: conn} do + %{"id" => id} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/rules", %{text: "Example rule"}) + |> json_response_and_validate_schema(:ok) + + assert %{text: "Example rule"} = Rule.get(id) + end + end + + describe "PATCH /api/pleroma/admin/rules" do + test "edits a rule", %{conn: conn} do + %{id: id} = Rule.create(%{text: "Example rule"}) + + conn + |> put_req_header("content-type", "application/json") + |> patch("/api/pleroma/admin/rules/#{id}", %{text: "There are no rules", priority: 2}) + |> json_response_and_validate_schema(:ok) + + assert %{text: "There are no rules", priority: 2} = Rule.get(id) + end + end + + describe "DELETE /api/pleroma/admin/rules" do + test "deletes a rule", %{conn: conn} do + %{id: id} = Rule.create(%{text: "Example rule"}) + + conn + |> put_req_header("content-type", "application/json") + |> delete("/api/pleroma/admin/rules/#{id}") + |> json_response_and_validate_schema(:ok) + + assert [] = + Rule.query() + |> Pleroma.Repo.all() + end + end +end diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index 9845408d6..87beacd64 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do # TODO: Should not need Cachex use Pleroma.Web.ConnCase + alias Pleroma.Rule alias Pleroma.User import Pleroma.Factory @@ -39,7 +40,8 @@ test "get instance information", %{conn: conn} do "banner_upload_limit" => _, "background_image" => from_config_background, "shout_limit" => _, - "description_limit" => _ + "description_limit" => _, + "rules" => _ } = result assert result["pleroma"]["metadata"]["account_activation_required"] != nil @@ -91,4 +93,18 @@ test "get peers", %{conn: conn} do assert ["peer1.com", "peer2.com"] == Enum.sort(result) end + + test "get instance rules", %{conn: conn} do + Rule.create(%{text: "Example rule"}) + Rule.create(%{text: "Second rule"}) + Rule.create(%{text: "Third rule"}) + + conn = get(conn, "/api/v1/instance") + + assert result = json_response_and_validate_schema(conn, 200) + + rules = result["rules"] + + assert length(rules) == 3 + end end From 432599311d3aab8829ed2cc7f795a1662e7a8f82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 5 Feb 2022 19:13:30 +0100 Subject: [PATCH 009/161] Add GET /api/v1/instance/rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../operations/admin/rule_operation.ex | 2 +- .../api_spec/operations/instance_operation.ex | 27 ++++++++++++++++++- .../controllers/instance_controller.ex | 5 ++++ .../web/mastodon_api/views/instance_view.ex | 4 +++ lib/pleroma/web/router.ex | 1 + 5 files changed, 37 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex index 28f2be5e7..ed0d9eaf6 100644 --- a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex @@ -17,7 +17,7 @@ def open_api_operation(action) do def index_operation do %Operation{ tags: ["Instance rule managment"], - summary: "Retrieve a list of instance rules", + summary: "Retrieve list of instance rules", operationId: "AdminAPI.RuleController.index", security: [%{"oAuth" => ["admin:read"]}], responses: %{ diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index 3c4b504fe..e66e5b7a3 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -34,6 +34,17 @@ def peers_operation do } end + def rules_operation do + %Operation{ + tags: ["Instance"], + summary: "Retrieve list of instance rules", + operationId: "InstanceController.rules", + responses: %{ + 200 => Operation.response("Array of domains", "application/json", array_of_rules()) + } + } + end + defp instance do %Schema{ type: :object, @@ -160,7 +171,8 @@ defp instance do "urls" => %{ "streaming_api" => "wss://lain.com" }, - "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)" + "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)", + "rules" => array_of_rules() } } end @@ -172,4 +184,17 @@ defp array_of_domains do example: ["pleroma.site", "lain.com", "bikeshed.party"] } end + + defp array_of_rules do + %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :integer}, + text: %Schema{type: :string} + } + } + } + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index 6410e872c..d6aa89432 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -20,4 +20,9 @@ def show(conn, _params) do def peers(conn, _params) do json(conn, Pleroma.Stats.get_peers()) end + + @doc "GET /api/v1/instance/rules" + def rules(conn, _params) do + render(conn, "rules.json") + end end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 9652da53a..feb94b40b 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -58,6 +58,10 @@ def render("show.json", _) do } end + def render("rules.json", _) do + rules() + end + def features do [ "pleroma_api", diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e8b7b17aa..1246989b0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -606,6 +606,7 @@ defmodule Pleroma.Web.Router do get("/instance", InstanceController, :show) get("/instance/peers", InstanceController, :peers) + get("/instance/rules", InstanceController, :rules) get("/statuses", StatusController, :index) get("/statuses/:id", StatusController, :show) From 384f8bfa786f51f3abec101e2ab78917f324a4fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 7 Feb 2022 23:41:41 +0100 Subject: [PATCH 010/161] Instance rules: Use render_many MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../admin_api/controllers/rule_controller.ex | 2 -- .../web/mastodon_api/views/instance_view.ex | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/rule_controller.ex b/lib/pleroma/web/admin_api/controllers/rule_controller.ex index 2db88b6ba..43b2f209a 100644 --- a/lib/pleroma/web/admin_api/controllers/rule_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/rule_controller.ex @@ -14,8 +14,6 @@ defmodule Pleroma.Web.AdminAPI.RuleController do json_response: 3 ] - require Logger - plug(Pleroma.Web.ApiSpec.CastAndValidate) plug( diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index feb94b40b..8379731e4 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -40,7 +40,7 @@ def render("show.json", _) do background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit), - rules: rules(), + rules: render(__MODULE__, "rules.json"), pleroma: %{ metadata: %{ account_activation_required: Keyword.get(instance, :account_activation_required), @@ -59,7 +59,16 @@ def render("show.json", _) do end def render("rules.json", _) do - rules() + Pleroma.Rule.query() + |> Pleroma.Repo.all() + |> render_many(__MODULE__, "rule.json", as: :rule) + end + + def render("rule.json", %{rule: rule}) do + %{ + id: rule.id, + text: rule.text + } end def features do @@ -142,10 +151,4 @@ def fields_limits do value_length: Config.get([:instance, :account_field_value_length]) } end - - def rules do - Pleroma.Rule.query() - |> Pleroma.Repo.all() - |> Enum.map(&%{id: &1.id, text: &1.text}) - end end From bbf3bc2228ca4bc5c209e418538665240e2aa9ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 7 Feb 2022 23:55:20 +0100 Subject: [PATCH 011/161] Add RuleTest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- test/pleroma/rule_test.exs | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/pleroma/rule_test.exs diff --git a/test/pleroma/rule_test.exs b/test/pleroma/rule_test.exs new file mode 100644 index 000000000..012ac902c --- /dev/null +++ b/test/pleroma/rule_test.exs @@ -0,0 +1,46 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.RuleTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Repo + alias Pleroma.Rule + + test "getting a list of rules sorted by priority" do + %{id: id1} = Rule.create(%{text: "Example rule"}) + %{id: id2} = Rule.create(%{text: "Second rule", priority: 2}) + %{id: id3} = Rule.create(%{text: "Third rule", priority: 1}) + + rules = + Rule.query() + |> Repo.all() + + assert [%{id: ^id1}, %{id: ^id3}, %{id: ^id2}] = rules + end + + test "creating rules" do + %{id: id} = Rule.create(%{text: "Example rule"}) + + assert %{text: "Example rule"} = Rule.get(id) + end + + test "editing rules" do + %{id: id} = Rule.create(%{text: "Example rule"}) + + Rule.update(%{text: "There are no rules", priority: 2}, id) + + assert %{text: "There are no rules", priority: 2} = Rule.get(id) + end + + test "deleting rules" do + %{id: id} = Rule.create(%{text: "Example rule"}) + + Rule.delete(id) + + assert [] = + Rule.query() + |> Pleroma.Repo.all() + end +end From 574db5b988e3caca14d1729a729af83d2f23e214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 20 Feb 2022 21:44:42 +0100 Subject: [PATCH 012/161] Allow submitting an array of rule_ids to /api/v1/reports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/constants.ex | 3 ++- lib/pleroma/rule.ex | 5 ++++ lib/pleroma/web/activity_pub/utils.ex | 8 +++++-- .../web/admin_api/views/report_view.ex | 15 +++++++++++- .../operations/admin/report_operation.ex | 10 ++++++++ .../api_spec/operations/report_operation.ex | 9 +++++++- lib/pleroma/web/common_api.ex | 17 ++++++++++++-- .../web/admin_api/views/report_view_test.exs | 23 +++++++++++++++++-- .../controllers/report_controller_test.exs | 21 +++++++++++++++++ 9 files changed, 102 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index a42c71d23..bf43becb3 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -19,7 +19,8 @@ defmodule Pleroma.Constants do "context_id", "deleted_activity_id", "pleroma_internal", - "generator" + "generator", + "rules" ] ) diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex index d772a32bd..486cff8cc 100644 --- a/lib/pleroma/rule.ex +++ b/lib/pleroma/rule.ex @@ -29,6 +29,11 @@ def query do |> order_by(asc: :priority) end + def get(ids) when is_list(ids) do + from(r in __MODULE__, where: r.id in ^ids) + |> Repo.all() + end + def get(id), do: Repo.get(__MODULE__, id) def create(params) do diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 9cde7805c..72d17e2aa 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -692,14 +692,18 @@ def make_listen_data(params, additional) do #### Flag-related helpers @spec make_flag_data(map(), map()) :: map() - def make_flag_data(%{actor: actor, context: context, content: content} = params, additional) do + def make_flag_data( + %{actor: actor, context: context, content: content} = params, + additional + ) do %{ "type" => "Flag", "actor" => actor.ap_id, "content" => content, "object" => build_flag_object(params), "context" => context, - "state" => "open" + "state" => "open", + "rules" => Map.get(params, :rules, nil) } |> Map.merge(additional) end diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index b761dbb22..ca70f4359 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -6,10 +6,12 @@ defmodule Pleroma.Web.AdminAPI.ReportView do use Pleroma.Web, :view alias Pleroma.HTML + alias Pleroma.Rule alias Pleroma.User alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.Report alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI.InstanceView alias Pleroma.Web.MastodonAPI.StatusView defdelegate merge_account_views(user), to: AdminAPI.AccountView @@ -46,7 +48,8 @@ def render("show.json", %{report: report, user: user, account: account, statuses as: :activity }), state: report.data["state"], - notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}) + notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}), + rules: rules(Map.get(report.data, "rules", nil)) } end @@ -71,4 +74,14 @@ def render("show_note.json", %{ created_at: Utils.to_masto_date(inserted_at) } end + + defp rules(nil) do + [] + end + + defp rules(rule_ids) do + rule_ids + |> Rule.get() + |> render_many(InstanceView, "rule.json", as: :rule) + end end diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index 312e091a5..bb71abbd1 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -169,6 +169,16 @@ defp report do inserted_at: %Schema{type: :string, format: :"date-time"} } } + }, + rules: %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :integer}, + text: %Schema{type: :string} + } + } } } } diff --git a/lib/pleroma/web/api_spec/operations/report_operation.ex b/lib/pleroma/web/api_spec/operations/report_operation.ex index c74ac7d5f..fd68f67a2 100644 --- a/lib/pleroma/web/api_spec/operations/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/report_operation.ex @@ -53,6 +53,12 @@ defp create_request do default: false, description: "If the account is remote, should the report be forwarded to the remote admin?" + }, + rule_ids: %Schema{ + type: :array, + nullable: true, + items: %Schema{type: :number}, + description: "Array of rules" } }, required: [:account_id], @@ -60,7 +66,8 @@ defp create_request do "account_id" => "123", "status_ids" => ["1337"], "comment" => "bad status!", - "forward" => "false" + "forward" => "false", + "rule_ids" => [3] } } end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 1b95ee89c..9f8d4def4 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Conversation.Participation alias Pleroma.Formatter alias Pleroma.Object + alias Pleroma.Rule alias Pleroma.ThreadMute alias Pleroma.User alias Pleroma.UserRelationship @@ -505,14 +506,16 @@ def thread_muted?(_, _), do: false def report(user, data) do with {:ok, account} <- get_reported_account(data.account_id), {:ok, {content_html, _, _}} <- make_report_content_html(data[:comment]), - {:ok, statuses} <- get_report_statuses(account, data) do + {:ok, statuses} <- get_report_statuses(account, data), + rules <- get_report_rules(Map.get(data, :rule_ids, nil)) do ActivityPub.flag(%{ context: Utils.generate_context_id(), actor: user, account: account, statuses: statuses, content: content_html, - forward: Map.get(data, :forward, false) + forward: Map.get(data, :forward, false), + rules: rules }) end end @@ -524,6 +527,16 @@ defp get_reported_account(account_id) do end end + defp get_report_rules(nil) do + nil + end + + defp get_report_rules(rule_ids) do + rule_ids + |> Rule.get() + |> Enum.map(& &1.id) + end + def update_report_state(activity_ids, state) when is_list(activity_ids) do case Utils.update_report_state(activity_ids, state) do :ok -> {:ok, activity_ids} diff --git a/test/pleroma/web/admin_api/views/report_view_test.exs b/test/pleroma/web/admin_api/views/report_view_test.exs index 9637c2b90..519208b45 100644 --- a/test/pleroma/web/admin_api/views/report_view_test.exs +++ b/test/pleroma/web/admin_api/views/report_view_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do import Pleroma.Factory + alias Pleroma.Rule alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.Report alias Pleroma.Web.AdminAPI.ReportView @@ -38,7 +39,8 @@ test "renders a report" do statuses: [], notes: [], state: "open", - id: activity.id + id: activity.id, + rules: [] } result = @@ -76,7 +78,8 @@ test "includes reported statuses" do statuses: [StatusView.render("show.json", %{activity: activity})], state: "open", notes: [], - id: report_activity.id + id: report_activity.id, + rules: [] } result = @@ -168,4 +171,20 @@ test "reports are ordered newest first" do assert report2.id == rendered |> Enum.at(0) |> Map.get(:id) assert report1.id == rendered |> Enum.at(1) |> Map.get(:id) end + + test "renders included rules" do + user = insert(:user) + other_user = insert(:user) + + %{id: id, text: text} = Rule.create(%{text: "Example rule"}) + + {:ok, activity} = + CommonAPI.report(user, %{ + account_id: other_user.id, + rule_ids: [id] + }) + + assert %{rules: [%{id: ^id, text: ^text}]} = + ReportView.render("show.json", Report.extract_report_info(activity)) + end end diff --git a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs index 6d1a63334..77626b821 100644 --- a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs @@ -5,6 +5,8 @@ defmodule Pleroma.Web.MastodonAPI.ReportControllerTest do use Pleroma.Web.ConnCase, async: true + alias Pleroma.Activity + alias Pleroma.Rule alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -44,6 +46,25 @@ test "submit a report with statuses and comment", %{ |> json_response_and_validate_schema(200) end + test "submit a report with rule_ids", %{ + conn: conn, + target_user: target_user + } do + %{id: rule_id} = Rule.create(%{text: "There are no rules"}) + + assert %{"action_taken" => false, "id" => id} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/reports", %{ + "account_id" => target_user.id, + "forward" => "false", + "rule_ids" => [rule_id] + }) + |> json_response_and_validate_schema(200) + + assert %Activity{data: %{"rules" => [^rule_id]}} = Activity.get_report(id) + end + test "account_id is required", %{ conn: conn, activity: activity From d26aadb743c0de6fee7653ac2b90f989862d3c02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 1 Mar 2022 23:50:09 +0100 Subject: [PATCH 013/161] Add tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- test/pleroma/rule_test.exs | 11 +++++++++++ test/pleroma/web/common_api_test.exs | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/test/pleroma/rule_test.exs b/test/pleroma/rule_test.exs index 012ac902c..d710a6312 100644 --- a/test/pleroma/rule_test.exs +++ b/test/pleroma/rule_test.exs @@ -43,4 +43,15 @@ test "deleting rules" do Rule.query() |> Pleroma.Repo.all() end + + test "getting rules by ids" do + %{id: id1} = Rule.create(%{text: "Example rule"}) + %{id: id2} = Rule.create(%{text: "Second rule"}) + %{id: _id3} = Rule.create(%{text: "Third rule"}) + + rules = Rule.get([id1, id2]) + + assert Enum.all?(rules, &(&1.id in [id1, id2])) + assert length(rules) == 2 + end end diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index b502aaa03..103aab397 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -12,6 +12,7 @@ defmodule Pleroma.Web.CommonAPITest do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Rule alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier @@ -1186,6 +1187,33 @@ test "updates state of multiple reports" do assert first_report.data["state"] == "resolved" assert second_report.data["state"] == "resolved" end + + test "creates a report with provided rules" do + reporter = insert(:user) + target_user = insert(:user) + + %{id: rule_id} = Rule.create(%{text: "There are no rules"}) + + reporter_ap_id = reporter.ap_id + target_ap_id = target_user.ap_id + + report_data = %{ + account_id: target_user.id, + rule_ids: [rule_id] + } + + assert {:ok, flag_activity} = CommonAPI.report(reporter, report_data) + + assert %Activity{ + actor: ^reporter_ap_id, + data: %{ + "type" => "Flag", + "object" => [^target_ap_id], + "state" => "open", + "rules" => [^rule_id] + } + } = flag_activity + end end describe "reblog muting" do From 5c383ada8ad7491125a8264a8a03d85fe822f61e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 5 Mar 2022 21:45:34 +0100 Subject: [PATCH 014/161] Correctly order rules by id/creation date MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/rule.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex index 486cff8cc..b1db1dc0c 100644 --- a/lib/pleroma/rule.ex +++ b/lib/pleroma/rule.ex @@ -27,6 +27,7 @@ def changeset(%Rule{} = rule, params \\ %{}) do def query do Rule |> order_by(asc: :priority) + |> order_by(asc: :id) end def get(ids) when is_list(ids) do From b354d70e85bbf0f685f3d56f7377fde2efce4187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 30 May 2022 12:30:03 +0200 Subject: [PATCH 015/161] Apply, suggestions, use strings for actual Mastodon API compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/rule.ex | 2 ++ lib/pleroma/web/admin_api/views/report_view.ex | 10 ++++++---- lib/pleroma/web/admin_api/views/rule_view.ex | 2 +- .../operations/admin/report_operation.ex | 2 +- .../api_spec/operations/admin/rule_operation.ex | 5 ++--- .../api_spec/operations/instance_operation.ex | 2 +- .../web/api_spec/operations/report_operation.ex | 4 ++-- lib/pleroma/web/common_api.ex | 3 +-- .../web/mastodon_api/views/instance_view.ex | 2 +- .../controllers/rule_controller_test.exs | 4 ++++ .../controllers/report_controller_test.exs | 17 +++++++++++++++++ 11 files changed, 38 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex index b1db1dc0c..611e945b3 100644 --- a/lib/pleroma/rule.ex +++ b/lib/pleroma/rule.ex @@ -37,6 +37,8 @@ def get(ids) when is_list(ids) do def get(id), do: Repo.get(__MODULE__, id) + def exists?(id), do: not is_nil(get(id)) + def create(params) do {:ok, rule} = %Rule{} diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index ca70f4359..b4b0be267 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -10,8 +10,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do alias Pleroma.User alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.AdminAPI.RuleView alias Pleroma.Web.CommonAPI.Utils - alias Pleroma.Web.MastodonAPI.InstanceView alias Pleroma.Web.MastodonAPI.StatusView defdelegate merge_account_views(user), to: AdminAPI.AccountView @@ -80,8 +80,10 @@ defp rules(nil) do end defp rules(rule_ids) do - rule_ids - |> Rule.get() - |> render_many(InstanceView, "rule.json", as: :rule) + rules = + rule_ids + |> Rule.get() + + render(RuleView, "index.json", rules: rules) end end diff --git a/lib/pleroma/web/admin_api/views/rule_view.ex b/lib/pleroma/web/admin_api/views/rule_view.ex index f29145248..abfdd593f 100644 --- a/lib/pleroma/web/admin_api/views/rule_view.ex +++ b/lib/pleroma/web/admin_api/views/rule_view.ex @@ -13,7 +13,7 @@ def render("index.json", %{rules: rules} = _opts) do def render("show.json", %{rule: rule} = _opts) do %{ - id: rule.id, + id: to_string(rule.id), priority: rule.priority, text: rule.text } diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index bb71abbd1..b90bbd592 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -175,7 +175,7 @@ defp report do items: %Schema{ type: :object, properties: %{ - id: %Schema{type: :integer}, + id: %Schema{type: :string}, text: %Schema{type: :string} } } diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex index ed0d9eaf6..2360880e4 100644 --- a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex @@ -103,10 +103,9 @@ defp rule do %Schema{ type: :object, properties: %{ - id: %Schema{type: :integer}, + id: %Schema{type: :string}, priority: %Schema{type: :integer}, - text: %Schema{type: :string}, - created_at: %Schema{type: :string, format: :"date-time"} + text: %Schema{type: :string} } } end diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index e66e5b7a3..f3dba108e 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -191,7 +191,7 @@ defp array_of_rules do items: %Schema{ type: :object, properties: %{ - id: %Schema{type: :integer}, + id: %Schema{type: :string}, text: %Schema{type: :string} } } diff --git a/lib/pleroma/web/api_spec/operations/report_operation.ex b/lib/pleroma/web/api_spec/operations/report_operation.ex index fd68f67a2..f5f88974c 100644 --- a/lib/pleroma/web/api_spec/operations/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/report_operation.ex @@ -57,7 +57,7 @@ defp create_request do rule_ids: %Schema{ type: :array, nullable: true, - items: %Schema{type: :number}, + items: %Schema{type: :string}, description: "Array of rules" } }, @@ -67,7 +67,7 @@ defp create_request do "status_ids" => ["1337"], "comment" => "bad status!", "forward" => "false", - "rule_ids" => [3] + "rule_ids" => ["3"] } } end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 9f8d4def4..6fd744ddc 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -533,8 +533,7 @@ defp get_report_rules(nil) do defp get_report_rules(rule_ids) do rule_ids - |> Rule.get() - |> Enum.map(& &1.id) + |> Enum.filter(&Rule.exists?/1) end def update_report_state(activity_ids, state) when is_list(activity_ids) do diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 8379731e4..c7f5ff554 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -66,7 +66,7 @@ def render("rules.json", _) do def render("rule.json", %{rule: rule}) do %{ - id: rule.id, + id: to_string(rule.id), text: rule.text } end diff --git a/test/pleroma/web/admin_api/controllers/rule_controller_test.exs b/test/pleroma/web/admin_api/controllers/rule_controller_test.exs index c5c72d293..96b52b272 100644 --- a/test/pleroma/web/admin_api/controllers/rule_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/rule_controller_test.exs @@ -27,6 +27,10 @@ test "sorts rules by priority", %{conn: conn} do %{id: id2} = Rule.create(%{text: "Second rule", priority: 2}) %{id: id3} = Rule.create(%{text: "Third rule", priority: 1}) + id1 = to_string(id1) + id2 = to_string(id2) + id3 = to_string(id3) + response = conn |> get("/api/pleroma/admin/rules") diff --git a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs index 77626b821..689a7f375 100644 --- a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs @@ -65,6 +65,23 @@ test "submit a report with rule_ids", %{ assert %Activity{data: %{"rules" => [^rule_id]}} = Activity.get_report(id) end + test "rules field is empty if provided wrong rule id", %{ + conn: conn, + target_user: target_user + } do + assert %{"id" => id} = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/reports", %{ + "account_id" => target_user.id, + "forward" => "false", + "rule_ids" => ["-1"] + }) + |> json_response_and_validate_schema(200) + + assert %Activity{data: %{"rules" => []}} = Activity.get_report(id) + end + test "account_id is required", %{ conn: conn, activity: activity From 0ecd6ba35e868eac296b013f743d82a120dd68db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 30 May 2022 12:50:44 +0200 Subject: [PATCH 016/161] AdminAPI: Allow filtering reports by rule_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/activity_pub/activity_pub.ex | 10 +++++++ .../operations/admin/report_operation.ex | 6 ++++ .../controllers/report_controller_test.exs | 29 +++++++++++++++++++ .../web/admin_api/views/report_view_test.exs | 8 +++-- .../controllers/report_controller_test.exs | 2 ++ 5 files changed, 52 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 064f93b22..e54adf611 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1191,6 +1191,15 @@ defp restrict_filtered(query, %{blocking_user: %User{} = user}) do defp restrict_filtered(query, _), do: query + defp restrict_rule(query, %{rule_id: rule_id}) do + from( + activity in query, + where: fragment("(?)->'rules' \\? (?)", activity.data, ^rule_id) + ) + end + + defp restrict_rule(query, _), do: query + defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query defp exclude_poll_votes(query, _) do @@ -1353,6 +1362,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do |> restrict_instance(opts) |> restrict_announce_object_actor(opts) |> restrict_filtered(opts) + |> restrict_rule(opts) |> Activity.restrict_deactivated_users() |> exclude_poll_votes(opts) |> exclude_chat_messages(opts) diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index b90bbd592..18386296f 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -30,6 +30,12 @@ def index_operation do report_state(), "Filter by report state" ), + Operation.parameter( + :rule_id, + :query, + %Schema{type: :string}, + "Filter by selected rule id" + ), Operation.parameter( :limit, :query, diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs index 30dcb87e2..5dc3e7491 100644 --- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do alias Pleroma.ModerationLog alias Pleroma.Repo alias Pleroma.ReportNote + alias Pleroma.Rule alias Pleroma.Web.CommonAPI setup do @@ -313,6 +314,34 @@ test "returns 403 when requested by anonymous" do "error" => "Invalid credentials." } end + + test "returns reports with specified role_id", %{conn: conn} do + [reporter, target_user] = insert_pair(:user) + + %{id: rule_id} = Rule.create(%{text: "Example rule"}) + + rule_id = to_string(rule_id) + + {:ok, %{id: report_id}} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + comment: "", + rule_ids: [rule_id] + }) + + {:ok, _report} = + CommonAPI.report(reporter, %{ + account_id: target_user.id, + comment: "" + }) + + response = + conn + |> get("/api/pleroma/admin/reports?rule_id=#{rule_id}") + |> json_response_and_validate_schema(:ok) + + assert %{"reports" => [%{"id" => ^report_id}]} = response + end end describe "POST /api/pleroma/admin/reports/:id/notes" do diff --git a/test/pleroma/web/admin_api/views/report_view_test.exs b/test/pleroma/web/admin_api/views/report_view_test.exs index 519208b45..1b16aca6a 100644 --- a/test/pleroma/web/admin_api/views/report_view_test.exs +++ b/test/pleroma/web/admin_api/views/report_view_test.exs @@ -176,15 +176,17 @@ test "renders included rules" do user = insert(:user) other_user = insert(:user) - %{id: id, text: text} = Rule.create(%{text: "Example rule"}) + %{id: rule_id, text: text} = Rule.create(%{text: "Example rule"}) + + rule_id = to_string(rule_id) {:ok, activity} = CommonAPI.report(user, %{ account_id: other_user.id, - rule_ids: [id] + rule_ids: [rule_id] }) - assert %{rules: [%{id: ^id, text: ^text}]} = + assert %{rules: [%{id: ^rule_id, text: ^text}]} = ReportView.render("show.json", Report.extract_report_info(activity)) end end diff --git a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs index 689a7f375..509de6899 100644 --- a/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/report_controller_test.exs @@ -52,6 +52,8 @@ test "submit a report with rule_ids", %{ } do %{id: rule_id} = Rule.create(%{text: "There are no rules"}) + rule_id = to_string(rule_id) + assert %{"action_taken" => false, "id" => id} = conn |> put_req_header("content-type", "application/json") From 5846e7d5f6b91ab63270f2104543d874589d39ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 1 Jun 2022 16:03:22 +0200 Subject: [PATCH 017/161] Use Repo.exists? MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/rule.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex index 611e945b3..c8e3470c7 100644 --- a/lib/pleroma/rule.ex +++ b/lib/pleroma/rule.ex @@ -37,7 +37,10 @@ def get(ids) when is_list(ids) do def get(id), do: Repo.get(__MODULE__, id) - def exists?(id), do: not is_nil(get(id)) + def exists?(id) do + from(r in __MODULE__, where: r.id == ^id) + |> Repo.exists?() + end def create(params) do {:ok, rule} = From fa2a6d5d6b24657ddbda4ef11d2e6dbcb59545d3 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Thu, 7 Apr 2022 18:25:02 +0200 Subject: [PATCH 018/161] feat: simple, but not stupid, uploader for IPFS fix: format fix with credo --- config/config.exs | 4 +++ config/description.exs | 24 +++++++++++++ config/dev.exs | 4 +++ lib/pleroma/upload.ex | 13 +++++-- lib/pleroma/uploaders/ipfs.ex | 64 +++++++++++++++++++++++++++++++++++ 5 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/uploaders/ipfs.ex diff --git a/config/config.exs b/config/config.exs index 6a5acda09..7efad0061 100644 --- a/config/config.exs +++ b/config/config.exs @@ -82,6 +82,10 @@ # region: "us-east-1", # may be required for Amazon AWS scheme: "https://" +config :pleroma, Pleroma.Uploaders.IPFS, + post_gateway_url: nil, + get_gateway_url: nil + config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"], pack_extensions: [".png", ".gif"], diff --git a/config/description.exs b/config/description.exs index 7caad18b4..d87bbb9b8 100644 --- a/config/description.exs +++ b/config/description.exs @@ -136,6 +136,30 @@ } ] }, + %{ + group: :pleroma, + key: Pleroma.Uploaders.IPFS, + type: :group, + description: "IPFS uploader-related settings", + children: [ + %{ + key: :get_gateway_url, + type: :string, + description: "GET Gateway URL", + suggestions: [ + "get_gateway_url" + ] + }, + %{ + key: :post_gateway_url, + type: :string, + description: "POST Gateway URL", + suggestions: [ + "post_gateway_url" + ] + } + ] + }, %{ group: :pleroma, key: Pleroma.Uploaders.S3, diff --git a/config/dev.exs b/config/dev.exs index ab3e83c12..89a84bf05 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -58,6 +58,10 @@ # https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects config :phoenix, :plug_init_mode, :runtime +config :pleroma, Pleroma.Uploaders.IPFS, + post_gateway_url: nil, + get_gateway_url: nil + if File.exists?("./config/dev.secret.exs") do import_config "dev.secret.exs" else diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index db2909276..de39bcd6c 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -235,8 +235,14 @@ defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do "" end - [base_url, path] - |> Path.join() + uploader = Config.get([Pleroma.Upload, :uploader]) + + if uploader == Pleroma.Uploaders.IPFS && String.contains?(base_url, "{CID}") do + String.replace(base_url, "{CID}", path) + else + [base_url, path] + |> Path.join() + end end defp url_from_spec(_upload, _base_url, {:url, url}), do: url @@ -273,6 +279,9 @@ def base_url do Path.join([upload_base_url, bucket_with_namespace]) end + Pleroma.Uploaders.IPFS -> + Config.get([Pleroma.Uploaders.IPFS, :get_gateway_url]) + _ -> public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" end diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex new file mode 100644 index 000000000..b46e9322e --- /dev/null +++ b/lib/pleroma/uploaders/ipfs.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Uploaders.IPFS do + @behaviour Pleroma.Uploaders.Uploader + require Logger + + alias Pleroma.Config + alias Tesla.Multipart + + @impl true + def get_file(file) do + b_url = Pleroma.Upload.base_url() + + if String.contains?(b_url, "{CID}") do + {:ok, {:url, String.replace(b_url, "{CID}", URI.decode(file))}} + else + {:error, "IPFS Get URL doesn't contain '{CID}' placeholder"} + end + end + + @impl true + def put_file(%Pleroma.Upload{} = upload) do + config = Config.get([__MODULE__]) + post_base_url = Keyword.get(config, :post_gateway_url) + + mp = + Multipart.new() + |> Multipart.add_content_type_param("charset=utf-8") + |> Multipart.add_file(upload.tempfile) + + final_url = Path.join([post_base_url, "/api/v0/add"]) + + case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do + {:ok, ret} -> + case Jason.decode(ret.body) do + {:ok, ret} -> + {:ok, {:file, ret["Hash"]}} + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "JSON decode failed"} + end + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "IPFS Gateway Upload failed"} + end + end + + @impl true + def delete_file(file) do + config = Config.get([__MODULE__]) + post_base_url = Keyword.get(config, :post_gateway_url) + + final_url = Path.join([post_base_url, "/api/v0/files/rm"]) + + case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do + {:ok, %{status_code: 204}} -> :ok + error -> {:error, inspect(error)} + end + end +end From 43dfa58ebda407a0813d398bee8d0ae3e5c9fd5b Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Mon, 11 Apr 2022 15:10:01 +0200 Subject: [PATCH 019/161] added tests for ipfs uploader. adapted changelog.md accordingly. improved ipfs uploader with external suggestions fix lint description.exs --- CHANGELOG.md | 1 + config/description.exs | 5 +- config/dev.exs | 4 -- docs/configuration/cheatsheet.md | 13 ++++ lib/pleroma/upload.ex | 6 +- lib/pleroma/uploaders/ipfs.ex | 14 +++-- test/pleroma/uploaders/ipfs_test.exs | 88 ++++++++++++++++++++++++++++ 7 files changed, 116 insertions(+), 15 deletions(-) create mode 100644 test/pleroma/uploaders/ipfs_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index f1beb0cd0..b2185b1ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - PleromaAPI: Add `GET /api/v1/pleroma/birthdays` API endpoint - Make backend-rendered pages translatable. This includes emails. Pages returned as a HTTP response are translated using the language specified in the `userLanguage` cookie, or the `Accept-Language` header. Emails are translated using the `language` field when registering. This language can be changed by `PATCH /api/v1/accounts/update_credentials` with the `language` field. - Uploadfilter `Pleroma.Upload.Filter.Exiftool.ReadDescription` returns description values to the FE so they can pre fill the image description field +- Uploader: Add support for uploading attachments using IPFS ### Fixed - Subscription(Bell) Notifications: Don't create from Pipeline Ingested replies diff --git a/config/description.exs b/config/description.exs index d87bbb9b8..1fe5f01f0 100644 --- a/config/description.exs +++ b/config/description.exs @@ -147,7 +147,8 @@ type: :string, description: "GET Gateway URL", suggestions: [ - "get_gateway_url" + "https://ipfs.mydomain.com/<%= cid %>", + "https://<%= cid %>.ipfs.mydomain.com/" ] }, %{ @@ -155,7 +156,7 @@ type: :string, description: "POST Gateway URL", suggestions: [ - "post_gateway_url" + "http://localhost:5001/" ] } ] diff --git a/config/dev.exs b/config/dev.exs index 89a84bf05..ab3e83c12 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -58,10 +58,6 @@ # https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects config :phoenix, :plug_init_mode, :runtime -config :pleroma, Pleroma.Uploaders.IPFS, - post_gateway_url: nil, - get_gateway_url: nil - if File.exists?("./config/dev.secret.exs") do import_config "dev.secret.exs" else diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 74642397b..7e1f9c934 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -614,6 +614,19 @@ config :ex_aws, :s3, host: "s3.eu-central-1.amazonaws.com" ``` +#### Pleroma.Uploaders.IPFS + +* `post_gateway_url`: URL with port of POST Gateway (unauthenticated) +* `get_gateway_url`: URL of public GET Gateway + +Example: + +```elixir +config :pleroma, Pleroma.Uploaders.IPFS, + post_gateway_url: "http://localhost:5001", + get_gateway_url: "http://<%= cid %>.ipfs.mydomain.com" +``` + ### Upload filters #### Pleroma.Upload.Filter.AnonymizeFilename diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index de39bcd6c..b51d23f9e 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -235,10 +235,8 @@ defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do "" end - uploader = Config.get([Pleroma.Upload, :uploader]) - - if uploader == Pleroma.Uploaders.IPFS && String.contains?(base_url, "{CID}") do - String.replace(base_url, "{CID}", path) + if String.contains?(base_url, "<%= cid %>") do + EEx.eval_string(base_url, cid: path) else [base_url, path] |> Path.join() diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index b46e9322e..722c68fa1 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -13,10 +13,10 @@ defmodule Pleroma.Uploaders.IPFS do def get_file(file) do b_url = Pleroma.Upload.base_url() - if String.contains?(b_url, "{CID}") do - {:ok, {:url, String.replace(b_url, "{CID}", URI.decode(file))}} + if String.contains?(b_url, "<%= cid %>") do + {:ok, {:url, EEx.eval_string(b_url, cid: URI.decode(file))}} else - {:error, "IPFS Get URL doesn't contain '{CID}' placeholder"} + {:error, "IPFS Get URL doesn't contain 'cid' placeholder"} end end @@ -36,7 +36,11 @@ def put_file(%Pleroma.Upload{} = upload) do {:ok, ret} -> case Jason.decode(ret.body) do {:ok, ret} -> - {:ok, {:file, ret["Hash"]}} + if Map.has_key?(ret, "Hash") do + {:ok, {:file, ret["Hash"]}} + else + {:error, "JSON doesn't contain Hash value"} + end error -> Logger.error("#{__MODULE__}: #{inspect(error)}") @@ -45,7 +49,7 @@ def put_file(%Pleroma.Upload{} = upload) do error -> Logger.error("#{__MODULE__}: #{inspect(error)}") - {:error, "IPFS Gateway Upload failed"} + {:error, "IPFS Gateway upload failed"} end end diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs new file mode 100644 index 000000000..f9ae046cf --- /dev/null +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -0,0 +1,88 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Uploaders.IPFSTest do + use Pleroma.DataCase + + alias Pleroma.Uploaders.IPFS + alias Tesla.Multipart + + import Mock + import ExUnit.CaptureLog + + setup do + clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.IPFS) + clear_config([Pleroma.Uploaders.IPFS]) + + clear_config( + [Pleroma.Uploaders.IPFS, :get_gateway_url], + "https://<%= cid %>.ipfs.mydomain.com" + ) + + clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001") + end + + describe "get_file/1" do + test "it returns path to ipfs file with cid as subdomain" do + assert IPFS.get_file("testcid") == { + :ok, + {:url, "https://testcid.ipfs.mydomain.com"} + } + end + + test "it returns path to ipfs file with cid as path" do + clear_config( + [Pleroma.Uploaders.IPFS, :get_gateway_url], + "https://ipfs.mydomain.com/ipfs/<%= cid %>" + ) + + assert IPFS.get_file("testcid") == { + :ok, + {:url, "https://ipfs.mydomain.com/ipfs/testcid"} + } + end + end + + describe "put_file/1" do + setup do + file_upload = %Pleroma.Upload{ + name: "image-tet.jpg", + content_type: "image/jpeg", + path: "test_folder/image-tet.jpg", + tempfile: Path.absname("test/instance_static/add/shortcode.png") + } + + [file_upload: file_upload] + end + + test "save file", %{file_upload: file_upload} do + with_mock Pleroma.HTTP, + post: fn _, _, _, _ -> + {:ok, + %Tesla.Env{ + status: 200, + body: "{\"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}" + }} + end do + assert IPFS.put_file(file_upload) == + {:ok, {:file, "bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi"}} + end + end + + test "returns error", %{file_upload: file_upload} do + with_mock Pleroma.HTTP, post: fn _, _, _, _ -> {:error, "IPFS Gateway upload failed"} end do + assert capture_log(fn -> + assert IPFS.put_file(file_upload) == {:error, "IPFS Gateway upload failed"} + end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}" + end + end + end + + describe "delete_file/1" do + test_with_mock "deletes file", Pleroma.HTTP, + post: fn _, _, _, _ -> {:ok, %{status_code: 204}} end do + assert :ok = IPFS.delete_file("image.jpg") + end + end +end From 44659ecd65fb2251f9130fcecf1732b8931104c1 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Sat, 16 Apr 2022 09:38:49 +0200 Subject: [PATCH 020/161] ipfs: revert to String.replace for cid placeholder ipfs: fix lint --- config/description.exs | 4 ++-- docs/configuration/cheatsheet.md | 2 +- lib/pleroma/upload.ex | 4 ++-- lib/pleroma/uploaders/ipfs.ex | 7 +++++-- test/pleroma/uploaders/ipfs_test.exs | 4 ++-- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/config/description.exs b/config/description.exs index 1fe5f01f0..b180e7308 100644 --- a/config/description.exs +++ b/config/description.exs @@ -147,8 +147,8 @@ type: :string, description: "GET Gateway URL", suggestions: [ - "https://ipfs.mydomain.com/<%= cid %>", - "https://<%= cid %>.ipfs.mydomain.com/" + "https://ipfs.mydomain.com/{CID}", + "https://{CID}.ipfs.mydomain.com/" ] }, %{ diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 7e1f9c934..d35b33574 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -624,7 +624,7 @@ Example: ```elixir config :pleroma, Pleroma.Uploaders.IPFS, post_gateway_url: "http://localhost:5001", - get_gateway_url: "http://<%= cid %>.ipfs.mydomain.com" + get_gateway_url: "http://{CID}.ipfs.mydomain.com" ``` ### Upload filters diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index b51d23f9e..8a01cf613 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -235,8 +235,8 @@ defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do "" end - if String.contains?(base_url, "<%= cid %>") do - EEx.eval_string(base_url, cid: path) + if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do + String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path) else [base_url, path] |> Path.join() diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index 722c68fa1..dde520d8e 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -9,12 +9,15 @@ defmodule Pleroma.Uploaders.IPFS do alias Pleroma.Config alias Tesla.Multipart + @placeholder "{CID}" + def placeholder, do: @placeholder + @impl true def get_file(file) do b_url = Pleroma.Upload.base_url() - if String.contains?(b_url, "<%= cid %>") do - {:ok, {:url, EEx.eval_string(b_url, cid: URI.decode(file))}} + if String.contains?(b_url, @placeholder) do + {:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}} else {:error, "IPFS Get URL doesn't contain 'cid' placeholder"} end diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index f9ae046cf..fc87fa378 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -17,7 +17,7 @@ defmodule Pleroma.Uploaders.IPFSTest do clear_config( [Pleroma.Uploaders.IPFS, :get_gateway_url], - "https://<%= cid %>.ipfs.mydomain.com" + "https://{CID}.ipfs.mydomain.com" ) clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001") @@ -34,7 +34,7 @@ test "it returns path to ipfs file with cid as subdomain" do test "it returns path to ipfs file with cid as path" do clear_config( [Pleroma.Uploaders.IPFS, :get_gateway_url], - "https://ipfs.mydomain.com/ipfs/<%= cid %>" + "https://ipfs.mydomain.com/ipfs/{CID}" ) assert IPFS.get_file("testcid") == { From 7c1af86f979ecebcd38995e5278fe2d59a36eda5 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Mon, 9 May 2022 12:15:40 +0200 Subject: [PATCH 021/161] ipfs: refactor final_url generation. add tests for final_url fix lint --- lib/pleroma/uploaders/ipfs.ex | 19 ++++++++++--------- test/pleroma/uploaders/ipfs_test.exs | 13 ++++++++++++- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index dde520d8e..7a7481d81 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -12,6 +12,13 @@ defmodule Pleroma.Uploaders.IPFS do @placeholder "{CID}" def placeholder, do: @placeholder + def get_final_url(method) do + config = Config.get([__MODULE__]) + post_base_url = Keyword.get(config, :post_gateway_url) + + Path.join([post_base_url, method]) + end + @impl true def get_file(file) do b_url = Pleroma.Upload.base_url() @@ -25,15 +32,12 @@ def get_file(file) do @impl true def put_file(%Pleroma.Upload{} = upload) do - config = Config.get([__MODULE__]) - post_base_url = Keyword.get(config, :post_gateway_url) - mp = Multipart.new() |> Multipart.add_content_type_param("charset=utf-8") |> Multipart.add_file(upload.tempfile) - final_url = Path.join([post_base_url, "/api/v0/add"]) + final_url = get_final_url("/api/v0/add") case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do {:ok, ret} -> @@ -42,7 +46,7 @@ def put_file(%Pleroma.Upload{} = upload) do if Map.has_key?(ret, "Hash") do {:ok, {:file, ret["Hash"]}} else - {:error, "JSON doesn't contain Hash value"} + {:error, "JSON doesn't contain Hash key"} end error -> @@ -58,10 +62,7 @@ def put_file(%Pleroma.Upload{} = upload) do @impl true def delete_file(file) do - config = Config.get([__MODULE__]) - post_base_url = Keyword.get(config, :post_gateway_url) - - final_url = Path.join([post_base_url, "/api/v0/files/rm"]) + final_url = get_final_url("/api/v0/files/rm") case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do {:ok, %{status_code: 204}} -> :ok diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index fc87fa378..d567272d2 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -23,6 +23,16 @@ defmodule Pleroma.Uploaders.IPFSTest do clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001") end + describe "get_final_url" do + test "it returns the final url for put_file" do + assert IPFS.get_final_url("/api/v0/add") == "http://localhost:5001/api/v0/add" + end + + test "it returns the final url for delete_file" do + assert IPFS.get_final_url("/api/v0/files/rm") == "http://localhost:5001/api/v0/files/rm" + end + end + describe "get_file/1" do test "it returns path to ipfs file with cid as subdomain" do assert IPFS.get_file("testcid") == { @@ -62,7 +72,8 @@ test "save file", %{file_upload: file_upload} do {:ok, %Tesla.Env{ status: 200, - body: "{\"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}" + body: + "{\"Name\":\"image-tet.jpg\",\"Size\":\"5000\", \"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}" }} end do assert IPFS.put_file(file_upload) == From 98f268e5ecc5bab98c98270a582f8b3f0e3be4e8 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Thu, 9 Jun 2022 19:24:13 +0200 Subject: [PATCH 022/161] ipfs: small refactor and more tests --- lib/pleroma/uploaders/ipfs.ex | 24 ++++++++++++++---------- test/pleroma/uploaders/ipfs_test.exs | 24 ++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index 7a7481d81..9f6f26e2e 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -9,16 +9,24 @@ defmodule Pleroma.Uploaders.IPFS do alias Pleroma.Config alias Tesla.Multipart - @placeholder "{CID}" - def placeholder, do: @placeholder - - def get_final_url(method) do + defp get_final_url(method) do config = Config.get([__MODULE__]) post_base_url = Keyword.get(config, :post_gateway_url) Path.join([post_base_url, method]) end + def put_file_endpoint() do + get_final_url("/api/v0/add") + end + + def delete_file_endpoint() do + get_final_url("/api/v0/files/rm") + end + + @placeholder "{CID}" + def placeholder, do: @placeholder + @impl true def get_file(file) do b_url = Pleroma.Upload.base_url() @@ -37,9 +45,7 @@ def put_file(%Pleroma.Upload{} = upload) do |> Multipart.add_content_type_param("charset=utf-8") |> Multipart.add_file(upload.tempfile) - final_url = get_final_url("/api/v0/add") - - case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do + case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do {:ok, ret} -> case Jason.decode(ret.body) do {:ok, ret} -> @@ -62,9 +68,7 @@ def put_file(%Pleroma.Upload{} = upload) do @impl true def delete_file(file) do - final_url = get_final_url("/api/v0/files/rm") - - case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do + case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do {:ok, %{status_code: 204}} -> :ok error -> {:error, inspect(error)} end diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index d567272d2..f2b880d9f 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -25,11 +25,11 @@ defmodule Pleroma.Uploaders.IPFSTest do describe "get_final_url" do test "it returns the final url for put_file" do - assert IPFS.get_final_url("/api/v0/add") == "http://localhost:5001/api/v0/add" + assert IPFS.put_file_endpoint() == "http://localhost:5001/api/v0/add" end test "it returns the final url for delete_file" do - assert IPFS.get_final_url("/api/v0/files/rm") == "http://localhost:5001/api/v0/files/rm" + assert IPFS.delete_file_endpoint() == "http://localhost:5001/api/v0/files/rm" end end @@ -88,6 +88,26 @@ test "returns error", %{file_upload: file_upload} do end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}" end end + + test "returns error if JSON decode fails", %{file_upload: file_upload} do + with_mocks([ + {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]}, + {Jason, [], [decode: fn _ -> {:error, "JSON decode failed"} end]} + ]) do + assert capture_log(fn -> + assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"} + end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"JSON decode failed\"}" + end + end + + test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do + with_mocks([ + {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]}, + {Jason, [], [decode: fn _ -> {:ok, %{}} end]} + ]) do + assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"} + end + end end describe "delete_file/1" do From 254f2ea85400ebd692fc4a45f5ac22fedd49ec09 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Thu, 9 Jun 2022 23:38:50 +0200 Subject: [PATCH 023/161] ipfs: remove unused alias fix analysis job --- lib/pleroma/uploaders/ipfs.ex | 4 ++-- test/pleroma/uploaders/ipfs_test.exs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index 9f6f26e2e..32e06c5cf 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -16,11 +16,11 @@ defp get_final_url(method) do Path.join([post_base_url, method]) end - def put_file_endpoint() do + def put_file_endpoint do get_final_url("/api/v0/add") end - def delete_file_endpoint() do + def delete_file_endpoint do get_final_url("/api/v0/files/rm") end diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index f2b880d9f..5edb6266b 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -6,7 +6,6 @@ defmodule Pleroma.Uploaders.IPFSTest do use Pleroma.DataCase alias Pleroma.Uploaders.IPFS - alias Tesla.Multipart import Mock import ExUnit.CaptureLog From 5e097eb91def0efd3cd0008309fd524fcfd88e15 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Tue, 28 Jun 2022 17:53:44 +0200 Subject: [PATCH 024/161] ipfs: better tests with @ilja suggestions --- test/pleroma/uploaders/ipfs_test.exs | 38 ++++++++++++++++++---------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index 5edb6266b..9df3f239c 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Uploaders.IPFSTest do use Pleroma.DataCase alias Pleroma.Uploaders.IPFS + alias Tesla.Multipart import Mock import ExUnit.CaptureLog @@ -62,12 +63,17 @@ test "it returns path to ipfs file with cid as path" do tempfile: Path.absname("test/instance_static/add/shortcode.png") } - [file_upload: file_upload] + mp = + Multipart.new() + |> Multipart.add_content_type_param("charset=utf-8") + |> Multipart.add_file(file_upload.tempfile) + + [file_upload: file_upload, mp: mp] end test "save file", %{file_upload: file_upload} do with_mock Pleroma.HTTP, - post: fn _, _, _, _ -> + post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> {:ok, %Tesla.Env{ status: 200, @@ -81,7 +87,10 @@ test "save file", %{file_upload: file_upload} do end test "returns error", %{file_upload: file_upload} do - with_mock Pleroma.HTTP, post: fn _, _, _, _ -> {:error, "IPFS Gateway upload failed"} end do + with_mock Pleroma.HTTP, + post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + {:error, "IPFS Gateway upload failed"} + end do assert capture_log(fn -> assert IPFS.put_file(file_upload) == {:error, "IPFS Gateway upload failed"} end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}" @@ -89,21 +98,22 @@ test "returns error", %{file_upload: file_upload} do end test "returns error if JSON decode fails", %{file_upload: file_upload} do - with_mocks([ - {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]}, - {Jason, [], [decode: fn _ -> {:error, "JSON decode failed"} end]} - ]) do + with_mock Pleroma.HTTP, [], + post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + {:ok, %Tesla.Env{status: 200, body: 'invalid'}} + end do assert capture_log(fn -> assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"} - end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"JSON decode failed\"}" + end) =~ + "Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError{data: \"invalid\", position: 0, token: nil}}" end end test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do - with_mocks([ - {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]}, - {Jason, [], [decode: fn _ -> {:ok, %{}} end]} - ]) do + with_mock Pleroma.HTTP, [], + post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + {:ok, %Tesla.Env{status: 200, body: '{"key": "value"}'}} + end do assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"} end end @@ -111,7 +121,9 @@ test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_ describe "delete_file/1" do test_with_mock "deletes file", Pleroma.HTTP, - post: fn _, _, _, _ -> {:ok, %{status_code: 204}} end do + post: fn "http://localhost:5001/api/v0/files/rm", "", [], params: [arg: "image.jpg"] -> + {:ok, %{status_code: 204}} + end do assert :ok = IPFS.delete_file("image.jpg") end end From 21d9091f5e422493ff69fe59db9c965e0d511369 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Fri, 8 Jul 2022 10:06:46 +0200 Subject: [PATCH 025/161] ipfs: replacing single quotes with double quotes --- test/pleroma/uploaders/ipfs_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index 9df3f239c..853d185e5 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -100,7 +100,7 @@ test "returns error", %{file_upload: file_upload} do test "returns error if JSON decode fails", %{file_upload: file_upload} do with_mock Pleroma.HTTP, [], post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> - {:ok, %Tesla.Env{status: 200, body: 'invalid'}} + {:ok, %Tesla.Env{status: 200, body: "invalid"}} end do assert capture_log(fn -> assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"} @@ -112,7 +112,7 @@ test "returns error if JSON decode fails", %{file_upload: file_upload} do test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do with_mock Pleroma.HTTP, [], post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> - {:ok, %Tesla.Env{status: 200, body: '{"key": "value"}'}} + {:ok, %Tesla.Env{status: 200, body: "{\"key\": \"value\"}"}} end do assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"} end From 3ed39e310939d90ddbad7bd7ffa1ebd8aca6e74c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Fri, 8 Jul 2022 21:28:23 +0200 Subject: [PATCH 026/161] Add test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- test/pleroma/notification_test.exs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index eea2fcb67..c43502eb5 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -128,6 +128,21 @@ test "does not create a notification for subscribed users if status is a reply" subscriber_notifications = Notification.for_user(subscriber) assert Enum.empty?(subscriber_notifications) end + + test "does not create subscriber notification if mentioned" do + user = insert(:user) + subscriber = insert(:user) + + User.subscribe(subscriber, user) + + {:ok, status} = CommonAPI.post(user, %{status: "mentioning @#{subscriber.nickname}"}) + {:ok, [notification] = notifications} = Notification.create_notifications(status) + + assert length(notifications) == 1 + + assert notification.user_id == subscriber.id + assert notification.type == "mention" + end end test "create_poll_notifications/1" do From c899af1d6acad1895240a0247e9b91eca5db08df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 14 Apr 2022 20:09:43 +0200 Subject: [PATCH 027/161] Reject requests from specified instances if `authorized_fetch_mode` is enabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- config/config.exs | 1 + config/description.exs | 12 ++++ docs/configuration/cheatsheet.md | 1 + lib/pleroma/signature.ex | 16 +++-- .../web/mastodon_api/views/instance_view.ex | 7 +++ lib/pleroma/web/plugs/http_signature_plug.ex | 40 ++++++++++++ test/pleroma/signature_test.exs | 8 +++ .../web/plugs/http_signature_plug_test.exs | 63 +++++++++++++++++-- 8 files changed, 140 insertions(+), 8 deletions(-) diff --git a/config/config.exs b/config/config.exs index 0fc959807..cfa16f766 100644 --- a/config/config.exs +++ b/config/config.exs @@ -216,6 +216,7 @@ allow_relay: true, public: true, quarantined_instances: [], + rejected_instances: [], static_dir: "instance/static/", allowed_post_formats: [ "text/plain", diff --git a/config/description.exs b/config/description.exs index b29348edf..a75f40929 100644 --- a/config/description.exs +++ b/config/description.exs @@ -714,6 +714,18 @@ {"*.quarantined.com", "Reason"} ] }, + %{ + key: :rejected_instances, + type: {:list, :tuple}, + key_placeholder: "instance", + value_placeholder: "reason", + description: + "List of ActivityPub instances to reject requests from if authorized_fetch_mode is enabled", + suggestions: [ + {"rejected.com", "Reason"}, + {"*.rejected.com", "Reason"} + ] + }, %{ key: :static_dir, type: :string, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 6e13b9622..84a5bdb98 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -41,6 +41,7 @@ To add configuration to your config file, you can copy it from the base config. * `allow_relay`: Permits remote instances to subscribe to all public posts of your instance. This may increase the visibility of your instance. * `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details. * `quarantined_instances`: ActivityPub instances where private (DMs, followers-only) activities will not be send. +* `rejected_instances`: ActivityPub instances to reject requests from if authorized_fetch_mode is enabled. * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML). * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with older software for theses nicknames. diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index dbe6fd209..d5ba5c4fb 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -37,8 +37,7 @@ def key_id_to_actor_id(key_id) do end def fetch_public_key(conn) do - with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), - {:ok, actor_id} <- key_id_to_actor_id(kid), + with {:ok, actor_id} <- get_actor_id(conn), {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} else @@ -48,8 +47,7 @@ def fetch_public_key(conn) do end def refetch_public_key(conn) do - with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), - {:ok, actor_id} <- key_id_to_actor_id(kid), + with {:ok, actor_id} <- get_actor_id(conn), {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id), {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} @@ -59,6 +57,16 @@ def refetch_public_key(conn) do end end + def get_actor_id(conn) do + with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), + {:ok, actor_id} <- key_id_to_actor_id(kid) do + {:ok, actor_id} + else + e -> + {:error, e} + end + end + def sign(%User{} = user, headers) do with {:ok, %{keys: keys}} <- User.ensure_keys_present(user), {:ok, private_key, _} <- Keys.keys_from_pem(keys) do diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 62931bd41..017bd62e2 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -105,6 +105,7 @@ def features do def federation do quarantined = Config.get([:instance, :quarantined_instances], []) + rejected = Config.get([:instance, :rejected_instances], []) if Config.get([:mrf, :transparency]) do {:ok, data} = MRF.describe() @@ -124,6 +125,12 @@ def federation do |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) |> Map.new() }) + |> Map.put( + :rejected_instances, + rejected + |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) + |> Map.new() + ) else %{} end diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex index d023754a6..cf80b9b14 100644 --- a/lib/pleroma/web/plugs/http_signature_plug.ex +++ b/lib/pleroma/web/plugs/http_signature_plug.ex @@ -5,6 +5,10 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do import Plug.Conn import Phoenix.Controller, only: [get_format: 1, text: 2] + + alias Pleroma.Config + alias Pleroma.Web.ActivityPub.MRF + require Logger def init(options) do @@ -19,7 +23,9 @@ def call(conn, _opts) do if get_format(conn) == "activity+json" do conn |> maybe_assign_valid_signature() + |> maybe_assign_actor_id() |> maybe_require_signature() + |> maybe_filter_requests() else conn end @@ -46,6 +52,16 @@ defp maybe_assign_valid_signature(conn) do end end + defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do + adapter = Application.get_env(:http_signatures, :adapter) + + {:ok, actor_id} = adapter.get_actor_id(conn) + + assign(conn, :actor_id, actor_id) + end + + defp maybe_assign_actor_id(conn), do: conn + defp has_signature_header?(conn) do conn |> get_req_header("signature") |> Enum.at(0, false) end @@ -62,4 +78,28 @@ defp maybe_require_signature(conn) do conn end end + + defp maybe_filter_requests(%{halted: true} = conn), do: conn + + defp maybe_filter_requests(conn) do + if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do + %{host: host} = URI.parse(conn.assigns.actor_id) + + if MRF.subdomain_match?(rejected_domains(), host) do + conn + |> put_status(:unauthorized) + |> halt() + else + conn + end + else + conn + end + end + + defp rejected_domains do + Config.get([:instance, :rejected_instances]) + |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples() + |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() + end end diff --git a/test/pleroma/signature_test.exs b/test/pleroma/signature_test.exs index 92d05f26c..8f94efdc3 100644 --- a/test/pleroma/signature_test.exs +++ b/test/pleroma/signature_test.exs @@ -70,6 +70,14 @@ test "it returns error when not found user" do end end + describe "get_actor_id/1" do + test "it returns actor id" do + ap_id = "https://mastodon.social/users/lambadalambda" + + assert Signature.get_actor_id(make_fake_conn(ap_id)) == {:ok, ap_id} + end + end + describe "sign/2" do test "it returns signature headers" do user = diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs index 2d8fba3cd..de68e8823 100644 --- a/test/pleroma/web/plugs/http_signature_plug_test.exs +++ b/test/pleroma/web/plugs/http_signature_plug_test.exs @@ -10,11 +10,15 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do import Phoenix.Controller, only: [put_format: 2] import Mock - test "it call HTTPSignatures to check validity if the actor sighed it" do + test "it call HTTPSignatures to check validity if the actor signed it" do params = %{"actor" => "http://mastodon.example.org/users/admin"} conn = build_conn(:get, "/doesntmattter", params) - with_mock HTTPSignatures, validate_conn: fn _ -> true end do + with_mock HTTPSignatures, + validate_conn: fn _ -> true end, + signature_for_conn: fn _ -> + %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} + end do conn = conn |> put_req_header( @@ -41,7 +45,11 @@ test "it call HTTPSignatures to check validity if the actor sighed it" do end test "when signature header is present", %{conn: conn} do - with_mock HTTPSignatures, validate_conn: fn _ -> false end do + with_mock HTTPSignatures, + validate_conn: fn _ -> false end, + signature_for_conn: fn _ -> + %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} + end do conn = conn |> put_req_header( @@ -58,7 +66,11 @@ test "when signature header is present", %{conn: conn} do assert called(HTTPSignatures.validate_conn(:_)) end - with_mock HTTPSignatures, validate_conn: fn _ -> true end do + with_mock HTTPSignatures, + validate_conn: fn _ -> true end, + signature_for_conn: fn _ -> + %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} + end do conn = conn |> put_req_header( @@ -82,4 +94,47 @@ test "halts the connection when `signature` header is not present", %{conn: conn assert conn.resp_body == "Request not signed" end end + + test "rejects requests from `rejected_instances` when `authorized_fetch_mode` is enabled" do + clear_config([:activitypub, :authorized_fetch_mode], true) + clear_config([:instance, :rejected_instances], [{"mastodon.example.org", "no reason"}]) + + with_mock HTTPSignatures, + validate_conn: fn _ -> true end, + signature_for_conn: fn _ -> + %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} + end do + conn = + build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == true + assert called(HTTPSignatures.validate_conn(:_)) + end + + with_mock HTTPSignatures, + validate_conn: fn _ -> true end, + signature_for_conn: fn _ -> + %{"keyId" => "http://allowed.example.org/users/admin#main-key"} + end do + conn = + build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"}) + |> put_req_header( + "signature", + "keyId=\"http://allowed.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == false + assert called(HTTPSignatures.validate_conn(:_)) + end + end end From 78d1105bffee7ece8a2b972d3cb58a6e41d86828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 19 Feb 2023 22:02:38 +0100 Subject: [PATCH 028/161] Fix down migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../20220319000000_add_status_to_notifications_enum.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs index 62c0afb63..c3bc85894 100644 --- a/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs +++ b/priv/repo/migrations/20220319000000_add_status_to_notifications_enum.exs @@ -36,7 +36,8 @@ def down do 'reblog', 'favourite', 'pleroma:report', - 'poll + 'poll', + 'update' ) """ |> execute() From 2ae1b802f260e9ad8eaa585907d9505545ceb872 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 9 Mar 2023 10:21:11 +0100 Subject: [PATCH 029/161] AttachmentValidator: Add support for Honk "summary" + "name" As used by Honk and supported by Mastodon --- .../object_validators/attachment_validator.ex | 3 ++- .../object_validators/attachment_validator_test.exs | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index 398020bff..766421e60 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do field(:type, :string) field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream") field(:name, :string) + field(:summary, :string) field(:blurhash, :string) embeds_many :url, UrlObjectValidator, primary_key: false do @@ -44,7 +45,7 @@ def changeset(struct, data) do |> fix_url() struct - |> cast(data, [:id, :type, :mediaType, :name, :blurhash]) + |> cast(data, [:id, :type, :mediaType, :name, :summary, :blurhash]) |> cast_embed(:url, with: &url_changeset/2, required: true) |> validate_inclusion(:type, ~w[Link Document Audio Image Video]) |> validate_required([:type, :mediaType]) diff --git a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs index 77f2044e9..8d561603c 100644 --- a/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/attachment_validator_test.exs @@ -25,19 +25,22 @@ test "fails without url" do end test "works with honkerific attachments" do - attachment = %{ + honk = %{ "mediaType" => "", - "name" => "", - "summary" => "298p3RG7j27tfsZ9RQ.jpg", + "summary" => "Select your spirit chonk", + "name" => "298p3RG7j27tfsZ9RQ.jpg", "type" => "Document", "url" => "https://honk.tedunangst.com/d/298p3RG7j27tfsZ9RQ.jpg" } assert {:ok, attachment} = - AttachmentValidator.cast_and_validate(attachment) + honk + |> AttachmentValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) assert attachment.mediaType == "application/octet-stream" + assert attachment.summary == "Select your spirit chonk" + assert attachment.name == "298p3RG7j27tfsZ9RQ.jpg" end test "works with an unknown but valid mime type" do From 197647a04e66c1af3ae691a4507612fdbee9c48c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 9 Mar 2023 10:35:57 +0100 Subject: [PATCH 030/161] MastoAPI Attachment: Use "summary" for descriptions if present --- .../web/api_spec/schemas/attachment.ex | 6 +- .../web/mastodon_api/views/status_view.ex | 17 ++- .../mastodon_api/views/status_view_test.exs | 101 ++++++++++++------ 3 files changed, 87 insertions(+), 37 deletions(-) diff --git a/lib/pleroma/web/api_spec/schemas/attachment.ex b/lib/pleroma/web/api_spec/schemas/attachment.ex index 48634a14f..e89f2ddd0 100644 --- a/lib/pleroma/web/api_spec/schemas/attachment.ex +++ b/lib/pleroma/web/api_spec/schemas/attachment.ex @@ -50,7 +50,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do pleroma: %Schema{ type: :object, properties: %{ - mime_type: %Schema{type: :string, description: "mime type of the attachment"} + mime_type: %Schema{type: :string, description: "mime type of the attachment"}, + name: %Schema{ + type: :string, + description: "Name of the attachment, typically the filename" + } } } }, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 0a8c98b44..7a3af8acb 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -571,6 +571,19 @@ def render("attachment.json", %{attachment: attachment}) do to_string(attachment["id"] || hash_id) end + description = + if attachment["summary"] do + HTML.strip_tags(attachment["summary"]) + else + attachment["name"] + end + + name = if attachment["summary"], do: attachment["name"] + + pleroma = + %{mime_type: media_type} + |> Maps.put_if_present(:name, name) + %{ id: attachment_id, url: href, @@ -578,8 +591,8 @@ def render("attachment.json", %{attachment: attachment}) do preview_url: href_preview, text_url: href, type: type, - description: attachment["name"], - pleroma: %{mime_type: media_type}, + description: description, + pleroma: pleroma, blurhash: attachment["blurhash"] } |> Maps.put_if_present(:meta, meta) diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index f76b115b7..4580da75b 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -456,45 +456,78 @@ test "create mentions from the 'tag' field" do assert mention.url == recipient.ap_id end - test "attachments" do - object = %{ - "type" => "Image", - "url" => [ - %{ - "mediaType" => "image/png", - "href" => "someurl", - "width" => 200, - "height" => 100 - } - ], - "blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn", - "uuid" => 6 - } + describe "attachments" do + test "Complete Mastodon style" do + object = %{ + "type" => "Image", + "url" => [ + %{ + "mediaType" => "image/png", + "href" => "someurl", + "width" => 200, + "height" => 100 + } + ], + "blurhash" => "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn", + "uuid" => 6 + } - expected = %{ - id: "1638338801", - type: "image", - url: "someurl", - remote_url: "someurl", - preview_url: "someurl", - text_url: "someurl", - description: nil, - pleroma: %{mime_type: "image/png"}, - meta: %{original: %{width: 200, height: 100, aspect: 2}}, - blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn" - } + expected = %{ + id: "1638338801", + type: "image", + url: "someurl", + remote_url: "someurl", + preview_url: "someurl", + text_url: "someurl", + description: nil, + pleroma: %{mime_type: "image/png"}, + meta: %{original: %{width: 200, height: 100, aspect: 2}}, + blurhash: "UJJ8X[xYW,%Jtq%NNFbXB5j]IVM|9GV=WHRn" + } - api_spec = Pleroma.Web.ApiSpec.spec() + api_spec = Pleroma.Web.ApiSpec.spec() - assert expected == StatusView.render("attachment.json", %{attachment: object}) - assert_schema(expected, "Attachment", api_spec) + assert expected == StatusView.render("attachment.json", %{attachment: object}) + assert_schema(expected, "Attachment", api_spec) - # If theres a "id", use that instead of the generated one - object = Map.put(object, "id", 2) - result = StatusView.render("attachment.json", %{attachment: object}) + # If theres a "id", use that instead of the generated one + object = Map.put(object, "id", 2) + result = StatusView.render("attachment.json", %{attachment: object}) - assert %{id: "2"} = result - assert_schema(result, "Attachment", api_spec) + assert %{id: "2"} = result + assert_schema(result, "Attachment", api_spec) + end + + test "Honkerific" do + object = %{ + "type" => "Image", + "url" => [ + %{ + "mediaType" => "image/png", + "href" => "someurl" + } + ], + "name" => "fool.jpeg", + "summary" => "they have played us for absolute fools." + } + + expected = %{ + blurhash: nil, + description: "they have played us for absolute fools.", + id: "1638338801", + pleroma: %{mime_type: "image/png", name: "fool.jpeg"}, + preview_url: "someurl", + remote_url: "someurl", + text_url: "someurl", + type: "image", + url: "someurl" + } + + api_spec = Pleroma.Web.ApiSpec.spec() + + assert expected == StatusView.render("attachment.json", %{attachment: object}) + assert_schema(expected, "Attachment", api_spec) + end end test "put the url advertised in the Activity in to the url attribute" do From 9363ef53a34c9d96191bccaece76dd4e01f493b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 14 May 2023 15:02:58 +0200 Subject: [PATCH 031/161] Add test for 'status' notification type for NotificationView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../views/notification_view_test.exs | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index 6ea894691..92de6c6a7 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -286,4 +286,31 @@ test "muted notification" do test_notifications_rendering([notification], user, [expected]) end + + test "Subscribed status notification" do + user = insert(:user) + subscriber = insert(:user) + + User.subscribe(subscriber, user) + + {:ok, activity} = CommonAPI.post(user, %{status: "hi"}) + {:ok, [notification]} = Notification.create_notifications(activity) + + user = User.get_cached_by_id(user.id) + + expected = %{ + id: to_string(notification.id), + pleroma: %{is_seen: false, is_muted: false}, + type: "status", + account: + AccountView.render("show.json", %{ + user: user, + for: subscriber + }), + status: StatusView.render("show.json", %{activity: activity, for: subscriber}), + created_at: Utils.to_masto_date(notification.inserted_at) + } + + test_notifications_rendering([notification], subscriber, [expected]) + end end From 1ab4ab8d38687634735e1415f395b072718ab1ab Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 18 Jul 2023 18:24:30 -0400 Subject: [PATCH 032/161] Extract translatable strings --- changelog.d/3907.skip | 0 priv/gettext/config_descriptions.pot | 84 ++++++++++++++++++++++++++++ priv/gettext/errors.pot | 36 +++++++++--- priv/gettext/oauth_scopes.pot | 40 +++++++++++++ 4 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 changelog.d/3907.skip diff --git a/changelog.d/3907.skip b/changelog.d/3907.skip new file mode 100644 index 000000000..e69de29bb diff --git a/priv/gettext/config_descriptions.pot b/priv/gettext/config_descriptions.pot index 4f60e1c85..b4792868b 100644 --- a/priv/gettext/config_descriptions.pot +++ b/priv/gettext/config_descriptions.pot @@ -5973,3 +5973,87 @@ msgstr "" msgctxt "config label at :pleroma-:instance > :languages" msgid "Languages" msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji" +msgid "Reject or force-unlisted emojis whose URLs or names match a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html)." +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji > :federated_timeline_removal_shortcode" +msgid " A list of patterns which result in message with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji > :federated_timeline_removal_url" +msgid " A list of patterns which result in message with emojis whose URLs match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji > :remove_shortcode" +msgid " A list of patterns which result in emoji whose shortcode matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-:mrf_emoji > :remove_url" +msgid " A list of patterns which result in emoji whose URL matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles.\n\n Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.\n" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-Pleroma.User.Backup > :process_chunk_size" +msgid "The number of activities to fetch in the backup job for each chunk." +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config description at :pleroma-Pleroma.User.Backup > :process_wait_time" +msgid "The amount of time to wait for backup to report progress, in milliseconds. If no progress is received from the backup job for that much time, terminate it and deem it failed." +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji" +msgid "MRF Emoji" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji > :federated_timeline_removal_shortcode" +msgid "Federated timeline removal shortcode" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji > :federated_timeline_removal_url" +msgid "Federated timeline removal url" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji > :remove_shortcode" +msgid "Remove shortcode" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-:mrf_emoji > :remove_url" +msgid "Remove url" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-Pleroma.User.Backup > :process_chunk_size" +msgid "Process Chunk Size" +msgstr "" + +#: lib/pleroma/docs/translator.ex:5 +#, elixir-autogen, elixir-format +msgctxt "config label at :pleroma-Pleroma.User.Backup > :process_wait_time" +msgid "Process Wait Time" +msgstr "" diff --git a/priv/gettext/errors.pot b/priv/gettext/errors.pot index d320ee1bd..aca77f8fa 100644 --- a/priv/gettext/errors.pot +++ b/priv/gettext/errors.pot @@ -110,7 +110,7 @@ msgstr "" msgid "Can't display this activity" msgstr "" -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:334 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:346 #, elixir-autogen, elixir-format msgid "Can't find user" msgstr "" @@ -198,7 +198,7 @@ msgstr "" msgid "Invalid password." msgstr "" -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:267 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:279 #, elixir-autogen, elixir-format msgid "Invalid request" msgstr "" @@ -225,7 +225,7 @@ msgstr "" #: lib/pleroma/web/feed/tag_controller.ex:16 #: lib/pleroma/web/feed/user_controller.ex:69 #: lib/pleroma/web/o_status/o_status_controller.ex:132 -#: lib/pleroma/web/plugs/uploaded_media.ex:104 +#: lib/pleroma/web/plugs/uploaded_media.ex:84 #, elixir-autogen, elixir-format msgid "Not found" msgstr "" @@ -235,7 +235,7 @@ msgstr "" msgid "Poll's author can't vote" msgstr "" -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:499 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:511 #: lib/pleroma/web/mastodon_api/controllers/fallback_controller.ex:20 #: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:39 #: lib/pleroma/web/mastodon_api/controllers/poll_controller.ex:51 @@ -341,7 +341,7 @@ msgstr "" msgid "CAPTCHA expired" msgstr "" -#: lib/pleroma/web/plugs/uploaded_media.ex:77 +#: lib/pleroma/web/plugs/uploaded_media.ex:57 #, elixir-autogen, elixir-format msgid "Failed" msgstr "" @@ -361,7 +361,7 @@ msgstr "" msgid "Insufficient permissions: %{permissions}." msgstr "" -#: lib/pleroma/web/plugs/uploaded_media.ex:131 +#: lib/pleroma/web/plugs/uploaded_media.ex:111 #, elixir-autogen, elixir-format msgid "Internal Error" msgstr "" @@ -557,7 +557,7 @@ msgstr "" msgid "Access denied" msgstr "" -#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:331 +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:343 #, elixir-autogen, elixir-format msgid "This API requires an authenticated user" msgstr "" @@ -567,7 +567,7 @@ msgstr "" msgid "User is not an admin." msgstr "" -#: lib/pleroma/user/backup.ex:73 +#: lib/pleroma/user/backup.ex:78 #, elixir-format msgid "Last export was less than a day ago" msgid_plural "Last export was less than %{days} days ago" @@ -607,3 +607,23 @@ msgstr "" #, elixir-autogen, elixir-format msgid "User isn't privileged." msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:267 +#, elixir-autogen, elixir-format +msgid "Bio is too long" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:270 +#, elixir-autogen, elixir-format +msgid "Name is too long" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:273 +#, elixir-autogen, elixir-format +msgid "One or more field entries are too long" +msgstr "" + +#: lib/pleroma/web/mastodon_api/controllers/account_controller.ex:276 +#, elixir-autogen, elixir-format +msgid "Too many field entries" +msgstr "" diff --git a/priv/gettext/oauth_scopes.pot b/priv/gettext/oauth_scopes.pot index 50ad0dd9e..83328770e 100644 --- a/priv/gettext/oauth_scopes.pot +++ b/priv/gettext/oauth_scopes.pot @@ -219,3 +219,43 @@ msgstr "" #, elixir-autogen, elixir-format msgid "read:mutes" msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "push" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "read:backups" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "read:chats" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "read:media" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "read:reports" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "write:chats" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "write:follow" +msgstr "" + +#: lib/pleroma/web/api_spec/scopes/translator.ex:5 +#, elixir-autogen, elixir-format +msgid "write:reports" +msgstr "" From b6a9d87f16a4806eab7a6da874d6f75b65d4f214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 15 Mar 2023 19:44:42 +0100 Subject: [PATCH 033/161] Display reposted replies with exclude_replies: true MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/show-reposter-replies.add | 1 + lib/pleroma/web/activity_pub/activity_pub.ex | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changelog.d/show-reposter-replies.add diff --git a/changelog.d/show-reposter-replies.add b/changelog.d/show-reposter-replies.add new file mode 100644 index 000000000..3b852ec3b --- /dev/null +++ b/changelog.d/show-reposter-replies.add @@ -0,0 +1 @@ +Display reposted replies with exclude_replies: true \ No newline at end of file diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 3979d418e..4b956c680 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -964,8 +964,9 @@ defp restrict_media(query, _), do: query defp restrict_replies(query, %{exclude_replies: true}) do from( - [_activity, object] in query, - where: fragment("?->>'inReplyTo' is null", object.data) + [activity, object] in query, + where: + fragment("?->>'inReplyTo' is null or ?->>'type' = 'Announce'", object.data, activity.data) ) end From f271ea6e432d685c113582e5944d79e12c153016 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 16 Dec 2023 18:22:32 +0100 Subject: [PATCH 034/161] Move Plugs.RemoteIP.maybe_add_cidr/1 to InetHelper.parse_cidr/1 --- lib/pleroma/helpers/inet_helper.ex | 11 +++++++++++ lib/pleroma/web/plugs/remote_ip.ex | 14 ++------------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex index 704d37f8a..3500fc679 100644 --- a/lib/pleroma/helpers/inet_helper.ex +++ b/lib/pleroma/helpers/inet_helper.ex @@ -16,4 +16,15 @@ def parse_address(ip) when is_binary(ip) do def parse_address(ip) do :inet.parse_address(ip) end + + def parse_cidr(proxy) when is_binary(proxy) do + proxy = + cond do + "/" in String.codepoints(proxy) -> proxy + InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32" + InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" + end + + InetCidr.parse(proxy, true) + end end diff --git a/lib/pleroma/web/plugs/remote_ip.ex b/lib/pleroma/web/plugs/remote_ip.ex index f207d9fef..3a4bffb50 100644 --- a/lib/pleroma/web/plugs/remote_ip.ex +++ b/lib/pleroma/web/plugs/remote_ip.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.Plugs.RemoteIp do """ alias Pleroma.Config + alias Pleroma.Helpers.InetHelper import Plug.Conn @behaviour Plug @@ -30,19 +31,8 @@ defp remote_ip_opts do proxies = Config.get([__MODULE__, :proxies], []) |> Enum.concat(reserved) - |> Enum.map(&maybe_add_cidr/1) + |> Enum.map(&InetHelper.parse_cidr/1) {headers, proxies} end - - defp maybe_add_cidr(proxy) when is_binary(proxy) do - proxy = - cond do - "/" in String.codepoints(proxy) -> proxy - InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32" - InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" - end - - InetCidr.parse(proxy, true) - end end From 086ba59d0346be870dc7df2660fbb55666bf0af7 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 16 Dec 2023 18:56:46 +0100 Subject: [PATCH 035/161] HTTPSignaturePlug: Add :authorized_fetch_mode_exceptions --- changelog.d/auth-fetch-exception.add | 1 + config/description.exs | 6 ++++++ docs/configuration/cheatsheet.md | 1 + lib/pleroma/web/plugs/http_signature_plug.ex | 20 ++++++++++++++----- .../web/plugs/http_signature_plug_test.exs | 19 ++++++++++++++++++ 5 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 changelog.d/auth-fetch-exception.add diff --git a/changelog.d/auth-fetch-exception.add b/changelog.d/auth-fetch-exception.add new file mode 100644 index 000000000..98efb903e --- /dev/null +++ b/changelog.d/auth-fetch-exception.add @@ -0,0 +1 @@ +HTTPSignaturePlug: Add :authorized_fetch_mode_exceptions configuration \ No newline at end of file diff --git a/config/description.exs b/config/description.exs index b152981c4..2fed1a152 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1771,6 +1771,12 @@ type: :boolean, description: "Require HTTP signatures for AP fetches" }, + %{ + key: :authorized_fetch_mode_exceptions, + type: {:list, :string}, + description: + "List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this)" + }, %{ key: :note_replies_output_limit, type: :integer, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index a4cae4dbb..06933ba76 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -279,6 +279,7 @@ Notes: * `deny_follow_blocked`: Whether to disallow following an account that has blocked the user in question * `sign_object_fetches`: Sign object fetches with HTTP signatures * `authorized_fetch_mode`: Require HTTP signatures for AP fetches +* `authorized_fetch_mode_exceptions`: List of IPs (CIDR format accepted) to exempt from HTTP Signatures requirement (for example to allow debugging, you shouldn't otherwise need this) ## Pleroma.User diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex index e814efc2c..7ec202662 100644 --- a/lib/pleroma/web/plugs/http_signature_plug.ex +++ b/lib/pleroma/web/plugs/http_signature_plug.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do + alias Pleroma.Helpers.InetHelper + import Plug.Conn import Phoenix.Controller, only: [get_format: 1, text: 2] require Logger @@ -89,12 +91,20 @@ defp has_signature_header?(conn) do defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn - defp maybe_require_signature(conn) do + defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do - conn - |> put_status(:unauthorized) - |> text("Request not signed") - |> halt() + exceptions = + Pleroma.Config.get([:activitypub, :authorized_fetch_mode_exceptions], []) + |> Enum.map(&InetHelper.parse_cidr/1) + + if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do + conn + else + conn + |> put_status(:unauthorized) + |> text("Request not signed") + |> halt() + end else conn end diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs index 2d8fba3cd..deb7e4a23 100644 --- a/test/pleroma/web/plugs/http_signature_plug_test.exs +++ b/test/pleroma/web/plugs/http_signature_plug_test.exs @@ -81,5 +81,24 @@ test "halts the connection when `signature` header is not present", %{conn: conn assert conn.state == :sent assert conn.resp_body == "Request not signed" end + + test "exempts specific IPs from `authorized_fetch_mode_exceptions`", %{conn: conn} do + clear_config([:activitypub, :authorized_fetch_mode_exceptions], ["192.168.0.0/24"]) + + with_mock HTTPSignatures, validate_conn: fn _ -> false end do + conn = + conn + |> Map.put(:remote_ip, {192, 168, 0, 1}) + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> HTTPSignaturePlug.call(%{}) + + assert conn.remote_ip == {192, 168, 0, 1} + assert conn.halted == false + assert called(HTTPSignatures.validate_conn(:_)) + end + end end end From 7e3bbdded5dea73a0bad3a8905839e42d476e506 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 20 Dec 2023 23:39:12 +0000 Subject: [PATCH 036/161] Elixir 1.13 is the minimum required version --- .gitlab-ci.yml | 2 +- Dockerfile | 6 +++--- changelog.d/bump-elixir.change | 1 + mix.exs | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changelog.d/bump-elixir.change diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eb31a8086..8f10790da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ image: git.pleroma.social:5050/pleroma/pleroma/ci-base variables: &global_variables # Only used for the release - ELIXIR_VER: 1.12.3 + ELIXIR_VER: 1.13.4 POSTGRES_DB: pleroma_test POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres diff --git a/Dockerfile b/Dockerfile index 69c3509de..72461305c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ARG ELIXIR_IMG=hexpm/elixir -ARG ELIXIR_VER=1.12.3 -ARG ERLANG_VER=24.2.1 -ARG ALPINE_VER=3.17.0 +ARG ELIXIR_VER=1.13.4 +ARG ERLANG_VER=24.3.4.15 +ARG ALPINE_VER=3.17.5 FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build diff --git a/changelog.d/bump-elixir.change b/changelog.d/bump-elixir.change new file mode 100644 index 000000000..afb25d4e7 --- /dev/null +++ b/changelog.d/bump-elixir.change @@ -0,0 +1 @@ +Elixir 1.13 is the minimum required version. diff --git a/mix.exs b/mix.exs index b4b77b161..2056e591d 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ def project do [ app: :pleroma, version: version("2.6.51"), - elixir: "~> 1.11", + elixir: "~> 1.13", elixirc_paths: elixirc_paths(Mix.env()), compilers: Mix.compilers(), elixirc_options: [warnings_as_errors: warnings_as_errors()], From f6fee39e42e633ff4298291dca0db656c92dad81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 23 Dec 2023 15:51:20 +0100 Subject: [PATCH 037/161] Add changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/instance-rules.add | 1 + lib/pleroma/web/api_spec.ex | 1 + .../api_spec/operations/instance_operation.ex | 2 +- .../web/mastodon_api/views/instance_view.ex | 18 +++++++++--------- 4 files changed, 12 insertions(+), 10 deletions(-) create mode 100644 changelog.d/instance-rules.add diff --git a/changelog.d/instance-rules.add b/changelog.d/instance-rules.add new file mode 100644 index 000000000..42f3cbfa1 --- /dev/null +++ b/changelog.d/instance-rules.add @@ -0,0 +1 @@ +Add instance rules \ No newline at end of file diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 163226ce5..0eedf4aea 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -97,6 +97,7 @@ def spec(opts \\ []) do "Frontend managment", "Instance configuration", "Instance documents", + "Instance rule managment", "Invites", "MediaProxy cache", "OAuth application managment", diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index a22eb5bc9..452e56e45 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -48,7 +48,7 @@ def peers_operation do def rules_operation do %Operation{ - tags: ["Instance"], + tags: ["Instance misc"], summary: "Retrieve list of instance rules", operationId: "InstanceController.rules", responses: %{ diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index fa3726d4a..62c6b9e2e 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -73,15 +73,6 @@ def render("show2.json", _) do }) end - defp common_information(instance) do - %{ - languages: Keyword.get(instance, :languages, ["en"]), - rules: render(__MODULE__, "rules.json"), - title: Keyword.get(instance, :name), - version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})" - } - end - def render("rules.json", _) do Pleroma.Rule.query() |> Pleroma.Repo.all() @@ -95,6 +86,15 @@ def render("rule.json", %{rule: rule}) do } end + defp common_information(instance) do + %{ + languages: Keyword.get(instance, :languages, ["en"]), + rules: render(__MODULE__, "rules.json"), + title: Keyword.get(instance, :name), + version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})" + } + end + def features do [ "pleroma_api", From ddb9e90c405369496fdf9e6dfed593eff8d5dc5c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 15:59:25 -0500 Subject: [PATCH 038/161] Update minimum elixir version found in various docs --- docs/installation/debian_based_jp.md | 2 +- docs/installation/generic_dependencies.include | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index 1424ad7f4..502eefaf8 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have - PostgreSQL 9.6以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください) - `postgresql-contrib` 9.6以上 (同上) -- Elixir 1.8 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) +- Elixir 1.13 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) - `erlang-dev` - `erlang-nox` - `git` diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include index 3365a36a8..e0cfd3264 100644 --- a/docs/installation/generic_dependencies.include +++ b/docs/installation/generic_dependencies.include @@ -1,7 +1,7 @@ ## Required dependencies * PostgreSQL >=9.6 -* Elixir >=1.11.0 <1.15 +* Elixir >=1.13.0 <1.15 * Erlang OTP >=22.2.0 <26 * git * file / libmagic From 90b442727e4e2e56b4b68a15172a5ef7516531df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Fri, 19 Jan 2024 17:53:37 +0100 Subject: [PATCH 039/161] Update Admin API docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- docs/development/API/admin_api.md | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/docs/development/API/admin_api.md b/docs/development/API/admin_api.md index 7d31ee262..b8dc419e2 100644 --- a/docs/development/API/admin_api.md +++ b/docs/development/API/admin_api.md @@ -1751,3 +1751,50 @@ Note that this differs from the Mastodon API variant: Mastodon API only returns ```json {} ``` + + +## `GET /api/v1/pleroma/admin/rules` + +### List rules + +- Response: JSON, list of rules + +```json +[ + { + "id": "1", + "priority": 1, + "text": "There are no rules" + } +] +``` + +## `POST /api/v1/pleroma/admin/rules` + +### Create a rule + +- Params: + - `text`: string, required, rule content + - `priority`: integer, optional, rule ordering priority + +- Response: JSON, a single rule + +## `PATCH /api/v1/pleroma/admin/rules/:id` + +### Update a rule + +- Params: + - `text`: string, optional, rule content + - `priority`: integer, optional, rule ordering priority + +- Response: JSON, a single rule + +## `DELETE /api/v1/pleroma/admin/rules/:id` + +### Delete a rule + +- Response: JSON, empty object + +```json +{} +``` From 1ed8ae2d8e86ed26d4e21f59e95995795bcb282b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 31 Jan 2024 22:55:58 +0100 Subject: [PATCH 040/161] Add changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/status-notification-type.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/status-notification-type.add diff --git a/changelog.d/status-notification-type.add b/changelog.d/status-notification-type.add new file mode 100644 index 000000000..a6e94fa87 --- /dev/null +++ b/changelog.d/status-notification-type.add @@ -0,0 +1 @@ +Add "status" notification type \ No newline at end of file From ac977bdb1c58fac826f6325a3b1550ff389439ca Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 20 Feb 2024 08:45:48 +0100 Subject: [PATCH 041/161] StealEmojiPolicy: Sanitize shortcodes Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/3245 --- .../activity_pub/mrf/steal_emoji_policy.ex | 2 ++ .../mrf/steal_emoji_policy_test.exs | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index f66c379b5..12accfadd 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -34,6 +34,7 @@ defp steal_emoji({shortcode, url}, emoji_dir_path) do |> Path.basename() |> Path.extname() + shortcode = Path.basename(shortcode) file_path = Path.join(emoji_dir_path, shortcode <> (extension || ".png")) case File.write(file_path, response.body) do @@ -76,6 +77,7 @@ def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = messa new_emojis = foreign_emojis |> Enum.reject(fn {shortcode, _url} -> shortcode in installed_emoji end) + |> Enum.reject(fn {shortcode, _url} -> String.contains?(shortcode, ["/", "\\"]) end) |> Enum.filter(fn {shortcode, _url} -> reject_emoji? = [:mrf_steal_emoji, :rejected_shortcodes] diff --git a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs index 89d32352f..e7fb337ec 100644 --- a/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs @@ -60,6 +60,32 @@ test "Steals emoji on unknown shortcode from allowed remote host", %{ |> File.exists?() end + test "rejects invalid shortcodes", %{path: path} do + message = %{ + "type" => "Create", + "object" => %{ + "emoji" => [{"fired/fox", "https://example.org/emoji/firedfox"}], + "actor" => "https://example.org/users/admin" + } + } + + fullpath = Path.join(path, "fired/fox.png") + + Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox"} -> + %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")} + end) + + clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468) + + refute "firedfox" in installed() + refute File.exists?(path) + + assert {:ok, _message} = StealEmojiPolicy.filter(message) + + refute "fired/fox" in installed() + refute File.exists?(fullpath) + end + test "reject regex shortcode", %{message: message} do refute "firedfox" in installed() From be075a43363519505dcfe2dba1fbb19e0326b668 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 20 Feb 2024 09:16:36 +0100 Subject: [PATCH 042/161] Security release 2.6.2 --- CHANGELOG.md | 5 +++++ mix.exs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83b3065ce..92e5e6134 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## 2.6.2 + +### Security +- MRF StealEmojiPolicy: Sanitize shortcodes (thanks to Hazel K for the report + ## 2.6.1 ### Changed - - Document maximum supported version of Erlang & Elixir diff --git a/mix.exs b/mix.exs index d420c11e4..c95c2a82f 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("2.6.1"), + version: version("2.6.2"), elixir: "~> 1.11", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix] ++ Mix.compilers(), From 291d531e4cbdb5b63edb5b43914d82dafe356907 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 29 Jan 2024 10:18:11 -0500 Subject: [PATCH 043/161] Unify notification push and streaming events for both local and federated activities This also removes generation of notifications for blocked/filtered/muted users and threads. --- changelog.d/web_push_filtered.fix | 1 + lib/pleroma/notification.ex | 45 ++-- lib/pleroma/web/activity_pub/activity_pub.ex | 3 +- lib/pleroma/web/activity_pub/side_effects.ex | 39 ++-- test/pleroma/notification_test.exs | 201 ++---------------- .../web/activity_pub/side_effects_test.exs | 25 --- 6 files changed, 68 insertions(+), 246 deletions(-) create mode 100644 changelog.d/web_push_filtered.fix diff --git a/changelog.d/web_push_filtered.fix b/changelog.d/web_push_filtered.fix new file mode 100644 index 000000000..b9159362a --- /dev/null +++ b/changelog.d/web_push_filtered.fix @@ -0,0 +1 @@ +Web Push notifications are no longer generated for muted/blocked threads and users. diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 710b19866..654c9c98d 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -361,36 +361,32 @@ def dismiss(%{id: user_id} = _user, id) do end end - @spec create_notifications(Activity.t(), keyword()) :: {:ok, [Notification.t()] | []} - def create_notifications(activity, options \\ []) + @spec create_notifications(Activity.t()) :: {:ok, [Notification.t()] | []} + def create_notifications(activity) - def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity, options) do + def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do object = Object.normalize(activity, fetch: false) if object && object.data["type"] == "Answer" do {:ok, []} else - do_create_notifications(activity, options) + do_create_notifications(activity) end end - def create_notifications(%Activity{data: %{"type" => type}} = activity, options) + def create_notifications(%Activity{data: %{"type" => type}} = activity) when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do - do_create_notifications(activity, options) + do_create_notifications(activity) end - def create_notifications(_, _), do: {:ok, []} + def create_notifications(_), do: {:ok, []} - defp do_create_notifications(%Activity{} = activity, options) do - do_send = Keyword.get(options, :do_send, true) - - {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity) - potential_receivers = enabled_receivers ++ disabled_receivers + defp do_create_notifications(%Activity{} = activity) do + enabled_receivers = get_notified_from_activity(activity) notifications = - Enum.map(potential_receivers, fn user -> - do_send = do_send && user in enabled_receivers - create_notification(activity, user, do_send: do_send) + Enum.map(enabled_receivers, fn user -> + create_notification(activity, user) end) |> Enum.reject(&is_nil/1) @@ -450,7 +446,6 @@ defp type_from_activity_object(%{data: %{"type" => "Create"}} = activity) do # TODO move to sql, too. def create_notification(%Activity{} = activity, %User{} = user, opts \\ []) do - do_send = Keyword.get(opts, :do_send, true) type = Keyword.get(opts, :type, type_from_activity(activity)) unless skip?(activity, user, opts) do @@ -465,11 +460,6 @@ def create_notification(%Activity{} = activity, %User{} = user, opts \\ []) do |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() - if do_send do - Streamer.stream(["user", "user:notification"], notification) - Push.send(notification) - end - notification end end @@ -527,10 +517,7 @@ def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, lo |> exclude_relationship_restricted_ap_ids(activity) |> exclude_thread_muter_ap_ids(activity) - notification_enabled_users = - Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) - - {notification_enabled_users, potential_receivers -- notification_enabled_users} + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) end def get_notified_from_activity(_, _local_only), do: {[], []} @@ -748,4 +735,12 @@ def mark_context_as_read(%User{id: id}, context) do ) |> Repo.update_all(set: [seen: true]) end + + @spec send(list(Notification.t())) :: :ok + def send(notifications) do + Enum.each(notifications, fn notification -> + Streamer.stream(["user", "user:notification"], notification) + Push.send(notification) + end) + end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 2017c696d..a6ec67025 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -202,7 +202,8 @@ defp insert_activity_with_expiration(data, local, recipients) do end def notify_and_stream(activity) do - Notification.create_notifications(activity) + {:ok, notifications} = Notification.create_notifications(activity) + Notification.send(notifications) original_activity = case activity do diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 5cb8a9700..982927e16 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -21,7 +21,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.Push alias Pleroma.Web.Streamer alias Pleroma.Workers.PollWorker @@ -125,7 +124,7 @@ def handle( nil end - {:ok, notifications} = Notification.create_notifications(object, do_send: false) + {:ok, notifications} = Notification.create_notifications(object) meta = meta @@ -184,7 +183,11 @@ def handle(%{data: %{"type" => "Like"}} = object, meta) do liked_object = Object.get_by_ap_id(object.data["object"]) Utils.add_like_to_object(object, liked_object) - Notification.create_notifications(object) + {:ok, notifications} = Notification.create_notifications(object) + + meta = + meta + |> add_notifications(notifications) {:ok, object, meta} end @@ -202,7 +205,7 @@ def handle(%{data: %{"type" => "Like"}} = object, meta) do def handle(%{data: %{"type" => "Create"}} = activity, meta) do with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta), %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do - {:ok, notifications} = Notification.create_notifications(activity, do_send: false) + {:ok, notifications} = Notification.create_notifications(activity) {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object) {:ok, _user} = ActivityPub.update_last_status_at_if_public(user, object) @@ -258,11 +261,20 @@ def handle(%{data: %{"type" => "Announce"}} = object, meta) do Utils.add_announce_to_object(object, announced_object) - if !User.internal?(user) do - Notification.create_notifications(object) + notifications = + if !User.is_internal_user?(user) do + {:ok, notifications} = Notification.create_notifications(object) - ap_streamer().stream_out(object) - end + ap_streamer().stream_out(object) + + notifications + else + [] + end + + meta = + meta + |> add_notifications(notifications) {:ok, object, meta} end @@ -283,7 +295,11 @@ def handle(%{data: %{"type" => "EmojiReact"}} = object, meta) do reacted_object = Object.get_by_ap_id(object.data["object"]) Utils.add_emoji_reaction_to_object(object, reacted_object) - Notification.create_notifications(object) + {:ok, notifications} = Notification.create_notifications(object) + + meta = + meta + |> add_notifications(notifications) {:ok, object, meta} end @@ -587,10 +603,7 @@ defp delete_object(object) do defp send_notifications(meta) do Keyword.get(meta, :notifications, []) - |> Enum.each(fn notification -> - Streamer.stream(["user", "user:notification"], notification) - Push.send(notification) - end) + |> Notification.send() meta end diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index 4cf14e65b..392fd53c2 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -6,7 +6,6 @@ defmodule Pleroma.NotificationTest do use Pleroma.DataCase, async: false import Pleroma.Factory - import Mock alias Pleroma.FollowingRelationship alias Pleroma.Notification @@ -18,8 +17,6 @@ defmodule Pleroma.NotificationTest do alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.NotificationView - alias Pleroma.Web.Push - alias Pleroma.Web.Streamer setup do Mox.stub_with(Pleroma.UnstubbedConfigMock, Pleroma.Config) @@ -175,158 +172,7 @@ test "create_poll_notifications/1" do assert [user2.id, user3.id, user1.id] == Enum.map(notifications, & &1.user_id) end - describe "CommonApi.post/2 notification-related functionality" do - test_with_mock "creates but does NOT send notification to blocker user", - Push, - [:passthrough], - [] do - user = insert(:user) - blocker = insert(:user) - {:ok, _user_relationship} = User.block(blocker, user) - - {:ok, _activity} = CommonAPI.post(user, %{status: "hey @#{blocker.nickname}!"}) - - blocker_id = blocker.id - assert [%Notification{user_id: ^blocker_id}] = Repo.all(Notification) - refute called(Push.send(:_)) - end - - test_with_mock "creates but does NOT send notification to notification-muter user", - Push, - [:passthrough], - [] do - user = insert(:user) - muter = insert(:user) - {:ok, _user_relationships} = User.mute(muter, user) - - {:ok, _activity} = CommonAPI.post(user, %{status: "hey @#{muter.nickname}!"}) - - muter_id = muter.id - assert [%Notification{user_id: ^muter_id}] = Repo.all(Notification) - refute called(Push.send(:_)) - end - - test_with_mock "creates but does NOT send notification to thread-muter user", - Push, - [:passthrough], - [] do - user = insert(:user) - thread_muter = insert(:user) - - {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{thread_muter.nickname}!"}) - - {:ok, _} = CommonAPI.add_mute(thread_muter, activity) - - {:ok, _same_context_activity} = - CommonAPI.post(user, %{ - status: "hey-hey-hey @#{thread_muter.nickname}!", - in_reply_to_status_id: activity.id - }) - - [pre_mute_notification, post_mute_notification] = - Repo.all(from(n in Notification, where: n.user_id == ^thread_muter.id, order_by: n.id)) - - pre_mute_notification_id = pre_mute_notification.id - post_mute_notification_id = post_mute_notification.id - - assert called( - Push.send( - :meck.is(fn - %Notification{id: ^pre_mute_notification_id} -> true - _ -> false - end) - ) - ) - - refute called( - Push.send( - :meck.is(fn - %Notification{id: ^post_mute_notification_id} -> true - _ -> false - end) - ) - ) - end - end - describe "create_notification" do - @tag needs_streamer: true - test "it creates a notification for user and send to the 'user' and the 'user:notification' stream" do - %{user: user, token: oauth_token} = oauth_access(["read"]) - - task = - Task.async(fn -> - {:ok, _topic} = Streamer.get_topic_and_add_socket("user", user, oauth_token) - assert_receive {:render_with_user, _, _, _, _}, 4_000 - end) - - task_user_notification = - Task.async(fn -> - {:ok, _topic} = - Streamer.get_topic_and_add_socket("user:notification", user, oauth_token) - - assert_receive {:render_with_user, _, _, _, _}, 4_000 - end) - - activity = insert(:note_activity) - - notify = Notification.create_notification(activity, user) - assert notify.user_id == user.id - Task.await(task) - Task.await(task_user_notification) - end - - test "it creates a notification for user if the user blocks the activity author" do - activity = insert(:note_activity) - author = User.get_cached_by_ap_id(activity.data["actor"]) - user = insert(:user) - {:ok, _user_relationship} = User.block(user, author) - - assert Notification.create_notification(activity, user) - end - - test "it creates a notification for the user if the user mutes the activity author" do - muter = insert(:user) - muted = insert(:user) - {:ok, _} = User.mute(muter, muted) - muter = Repo.get(User, muter.id) - {:ok, activity} = CommonAPI.post(muted, %{status: "Hi @#{muter.nickname}"}) - - notification = Notification.create_notification(activity, muter) - - assert notification.id - assert notification.seen - end - - test "notification created if user is muted without notifications" do - muter = insert(:user) - muted = insert(:user) - - {:ok, _user_relationships} = User.mute(muter, muted, %{notifications: false}) - - {:ok, activity} = CommonAPI.post(muted, %{status: "Hi @#{muter.nickname}"}) - - assert Notification.create_notification(activity, muter) - end - - test "it creates a notification for an activity from a muted thread" do - muter = insert(:user) - other_user = insert(:user) - {:ok, activity} = CommonAPI.post(muter, %{status: "hey"}) - CommonAPI.add_mute(muter, activity) - - {:ok, activity} = - CommonAPI.post(other_user, %{ - status: "Hi @#{muter.nickname}", - in_reply_to_status_id: activity.id - }) - - notification = Notification.create_notification(activity, muter) - - assert notification.id - assert notification.seen - end - test "it disables notifications from strangers" do follower = insert(:user) @@ -680,7 +526,7 @@ test "it sends notifications to addressed users in new messages" do status: "hey @#{other_user.nickname}!" }) - {enabled_receivers, _disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert other_user in enabled_receivers end @@ -712,7 +558,7 @@ test "it sends notifications to mentioned users in new messages" do {:ok, activity} = Transmogrifier.handle_incoming(create_activity) - {enabled_receivers, _disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert other_user in enabled_receivers end @@ -739,7 +585,7 @@ test "it does not send notifications to users who are only cc in new messages" d {:ok, activity} = Transmogrifier.handle_incoming(create_activity) - {enabled_receivers, _disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert other_user not in enabled_receivers end @@ -756,8 +602,7 @@ test "it does not send notification to mentioned users in likes" do {:ok, activity_two} = CommonAPI.favorite(third_user, activity_one.id) - {enabled_receivers, _disabled_receivers} = - Notification.get_notified_from_activity(activity_two) + enabled_receivers = Notification.get_notified_from_activity(activity_two) assert other_user not in enabled_receivers end @@ -779,7 +624,7 @@ test "it only notifies the post's author in likes" do |> Map.put("to", [other_user.ap_id | like_data["to"]]) |> ActivityPub.persist(local: true) - {enabled_receivers, _disabled_receivers} = Notification.get_notified_from_activity(like) + enabled_receivers = Notification.get_notified_from_activity(like) assert other_user not in enabled_receivers end @@ -796,39 +641,36 @@ test "it does not send notification to mentioned users in announces" do {:ok, activity_two} = CommonAPI.repeat(activity_one.id, third_user) - {enabled_receivers, _disabled_receivers} = - Notification.get_notified_from_activity(activity_two) + enabled_receivers = Notification.get_notified_from_activity(activity_two) assert other_user not in enabled_receivers end - test "it returns blocking recipient in disabled recipients list" do + test "it does not return blocking recipient in recipients list" do user = insert(:user) other_user = insert(:user) {:ok, _user_relationship} = User.block(other_user, user) {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{other_user.nickname}!"}) - {enabled_receivers, disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert [] == enabled_receivers - assert [other_user] == disabled_receivers end - test "it returns notification-muting recipient in disabled recipients list" do + test "it does not return notification-muting recipient in recipients list" do user = insert(:user) other_user = insert(:user) {:ok, _user_relationships} = User.mute(other_user, user) {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{other_user.nickname}!"}) - {enabled_receivers, disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert [] == enabled_receivers - assert [other_user] == disabled_receivers end - test "it returns thread-muting recipient in disabled recipients list" do + test "it does not return thread-muting recipient in recipients list" do user = insert(:user) other_user = insert(:user) @@ -842,14 +684,12 @@ test "it returns thread-muting recipient in disabled recipients list" do in_reply_to_status_id: activity.id }) - {enabled_receivers, disabled_receivers} = - Notification.get_notified_from_activity(same_context_activity) + enabled_receivers = Notification.get_notified_from_activity(same_context_activity) - assert [other_user] == disabled_receivers refute other_user in enabled_receivers end - test "it returns non-following domain-blocking recipient in disabled recipients list" do + test "it does not return non-following domain-blocking recipient in recipients list" do blocked_domain = "blocked.domain" user = insert(:user, %{ap_id: "https://#{blocked_domain}/@actor"}) other_user = insert(:user) @@ -858,10 +698,9 @@ test "it returns non-following domain-blocking recipient in disabled recipients {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{other_user.nickname}!"}) - {enabled_receivers, disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert [] == enabled_receivers - assert [other_user] == disabled_receivers end test "it returns following domain-blocking recipient in enabled recipients list" do @@ -874,10 +713,9 @@ test "it returns following domain-blocking recipient in enabled recipients list" {:ok, activity} = CommonAPI.post(user, %{status: "hey @#{other_user.nickname}!"}) - {enabled_receivers, disabled_receivers} = Notification.get_notified_from_activity(activity) + enabled_receivers = Notification.get_notified_from_activity(activity) assert [other_user] == enabled_receivers - assert [] == disabled_receivers end test "it sends edited notifications to those who repeated a status" do @@ -897,11 +735,10 @@ test "it sends edited notifications to those who repeated a status" do status: "hey @#{other_user.nickname}! mew mew" }) - {enabled_receivers, _disabled_receivers} = - Notification.get_notified_from_activity(edit_activity) + enabled_receivers = Notification.get_notified_from_activity(edit_activity) assert repeated_user in enabled_receivers - assert other_user not in enabled_receivers + refute other_user in enabled_receivers end end @@ -1189,13 +1026,13 @@ test "it doesn't return notifications for muted thread", %{user: user} do assert Notification.for_user(user) == [] end - test "it returns notifications from a muted user when with_muted is set", %{user: user} do + test "it doesn't return notifications from a muted user when with_muted is set", %{user: user} do muted = insert(:user) {:ok, _user_relationships} = User.mute(user, muted) {:ok, _activity} = CommonAPI.post(muted, %{status: "hey @#{user.nickname}"}) - assert length(Notification.for_user(user, %{with_muted: true})) == 1 + assert Enum.empty?(Notification.for_user(user, %{with_muted: true})) end test "it doesn't return notifications from a blocked user when with_muted is set", %{ diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 94cc80b76..7af50e12c 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -827,31 +827,6 @@ test "creates a notification", %{announce: announce, poster: poster} do {:ok, announce, _} = SideEffects.handle(announce) assert Repo.get_by(Notification, user_id: poster.id, activity_id: announce.id) end - - test "it streams out the announce", %{announce: announce} do - with_mocks([ - { - Pleroma.Web.Streamer, - [], - [ - stream: fn _, _ -> nil end - ] - }, - { - Pleroma.Web.Push, - [], - [ - send: fn _ -> nil end - ] - } - ]) do - {:ok, announce, _} = SideEffects.handle(announce) - - assert called(Pleroma.Web.Streamer.stream(["user", "list"], announce)) - - assert called(Pleroma.Web.Push.send(:_)) - end - end end describe "removing a follower" do From c25fda34e7560d0d8eb5fdbfb444b641c4c4bf53 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 30 Jan 2024 10:41:52 -0500 Subject: [PATCH 044/161] Skip generating notifications for internal users --- lib/pleroma/notification.ex | 7 +++++++ lib/pleroma/web/activity_pub/side_effects.ex | 11 ++--------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 654c9c98d..a80279fa6 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -630,6 +630,7 @@ def skip?(activity, user, opts \\ []) def skip?(%Activity{} = activity, %User{} = user, opts) do [ :self, + :internal, :invisible, :block_from_strangers, :recently_followed, @@ -649,6 +650,12 @@ def skip?(:self, %Activity{} = activity, %User{} = user, opts) do end end + def skip?(:internal, %Activity{} = activity, _user, _opts) do + actor = activity.data["actor"] + user = User.get_cached_by_ap_id(actor) + User.internal?(user) + end + def skip?(:invisible, %Activity{} = activity, _user, _opts) do actor = activity.data["actor"] user = User.get_cached_by_ap_id(actor) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 982927e16..7ae16eb57 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -261,16 +261,9 @@ def handle(%{data: %{"type" => "Announce"}} = object, meta) do Utils.add_announce_to_object(object, announced_object) - notifications = - if !User.is_internal_user?(user) do - {:ok, notifications} = Notification.create_notifications(object) + {:ok, notifications} = Notification.create_notifications(object) - ap_streamer().stream_out(object) - - notifications - else - [] - end + if !User.internal?(user), do: ap_streamer().stream_out(object) meta = meta From 7dfd148ff8a4f2d349d6d6f92d788effdaab36f3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 9 Dec 2023 18:32:26 -0500 Subject: [PATCH 045/161] Logger metadata for inbound federation requests --- config/config.exs | 4 ++-- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 32c8509be..537517688 100644 --- a/config/config.exs +++ b/config/config.exs @@ -131,13 +131,13 @@ config :logger, :console, level: :debug, format: "\n$time $metadata[$level] $message\n", - metadata: [:request_id] + metadata: [:actor, :request_id, :type] config :logger, :ex_syslogger, level: :debug, ident: "pleroma", format: "$metadata[$level] $message", - metadata: [:request_id] + metadata: [:actor, :request_id, :type] config :mime, :types, %{ "application/xml" => ["xml"], diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index e38a94966..d2b2cae0b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -52,6 +52,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do when action in [:activity, :object] ) + plug(:log_inbox_metadata when action in [:inbox]) plug(:set_requester_reachable when action in [:inbox]) plug(:relay_active? when action in [:relay]) @@ -521,6 +522,13 @@ defp set_requester_reachable(%Plug.Conn{} = conn, _) do conn end + defp log_inbox_metadata(conn = %{params: %{"actor" => actor, "type" => type}}, _) do + Logger.metadata(actor: actor, type: type) + conn + end + + defp log_inbox_metadata(conn, _), do: conn + def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do with {:ok, object} <- ActivityPub.upload( From 40823462e7779fb79a4fcd458daa5e7095a6030b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 17 Dec 2023 18:20:22 -0500 Subject: [PATCH 046/161] Logger metadata for request path and authenticated user --- config/config.exs | 4 ++-- lib/pleroma/web/endpoint.ex | 2 ++ lib/pleroma/web/plugs/logger_metadata_path.ex | 12 ++++++++++++ lib/pleroma/web/plugs/logger_metadata_user.ex | 18 ++++++++++++++++++ lib/pleroma/web/router.ex | 8 ++++++++ 5 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/web/plugs/logger_metadata_path.ex create mode 100644 lib/pleroma/web/plugs/logger_metadata_user.ex diff --git a/config/config.exs b/config/config.exs index 537517688..83e7a33e3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -131,13 +131,13 @@ config :logger, :console, level: :debug, format: "\n$time $metadata[$level] $message\n", - metadata: [:actor, :request_id, :type] + metadata: [:actor, :path, :request_id, :type, :user] config :logger, :ex_syslogger, level: :debug, ident: "pleroma", format: "$metadata[$level] $message", - metadata: [:actor, :request_id, :type] + metadata: [:actor, :path, :request_id, :type, :user] config :mime, :types, %{ "application/xml" => ["xml"], diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 2e2104904..fef907ace 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -38,6 +38,8 @@ defmodule Pleroma.Web.Endpoint do plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint]) + plug(Pleroma.Web.Plugs.LoggerMetadataPath) + plug(Pleroma.Web.Plugs.SetLocalePlug) plug(CORSPlug) plug(Pleroma.Web.Plugs.HTTPSecurityPlug) diff --git a/lib/pleroma/web/plugs/logger_metadata_path.ex b/lib/pleroma/web/plugs/logger_metadata_path.ex new file mode 100644 index 000000000..a5553cfc8 --- /dev/null +++ b/lib/pleroma/web/plugs/logger_metadata_path.ex @@ -0,0 +1,12 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.LoggerMetadataPath do + def init(opts), do: opts + + def call(conn, _) do + Logger.metadata(path: conn.request_path) + conn + end +end diff --git a/lib/pleroma/web/plugs/logger_metadata_user.ex b/lib/pleroma/web/plugs/logger_metadata_user.ex new file mode 100644 index 000000000..6a5c0041d --- /dev/null +++ b/lib/pleroma/web/plugs/logger_metadata_user.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.LoggerMetadataUser do + alias Pleroma.User + + def init(opts), do: opts + + def call(%{assigns: %{user: user = %User{}}} = conn, _) do + Logger.metadata(user: user.nickname) + conn + end + + def call(conn, _) do + conn + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 4fe0cb02f..f0414cc35 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -29,6 +29,7 @@ defmodule Pleroma.Web.Router do pipeline :browser do plug(:accepts, ["html"]) plug(:fetch_session) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :oauth do @@ -67,12 +68,14 @@ defmodule Pleroma.Web.Router do plug(:fetch_session) plug(:authenticate) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :no_auth_or_privacy_expectations_api do plug(:base_api) plug(:after_auth) plug(Pleroma.Web.Plugs.IdempotencyPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end # Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported) @@ -83,12 +86,14 @@ defmodule Pleroma.Web.Router do pipeline :api do plug(:expect_public_instance_or_user_authentication) plug(:no_auth_or_privacy_expectations_api) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :authenticated_api do plug(:expect_user_authentication) plug(:no_auth_or_privacy_expectations_api) plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :admin_api do @@ -99,6 +104,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Web.Plugs.UserIsStaffPlug) plug(Pleroma.Web.Plugs.IdempotencyPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :require_admin do @@ -179,6 +185,7 @@ defmodule Pleroma.Web.Router do plug(:browser) plug(:authenticate) plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :well_known do @@ -193,6 +200,7 @@ defmodule Pleroma.Web.Router do pipeline :pleroma_api do plug(:accepts, ["html", "json"]) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :mailbox_preview do From 99cee755d8798c0743b96fb11e55f283f0195b85 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 17 Dec 2023 18:20:34 -0500 Subject: [PATCH 047/161] Show Logger metadata in dev --- config/dev.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index fe8de5045..f23719fe3 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -35,8 +35,8 @@ # configured to run both http and https servers on # different ports. -# Do not include metadata nor timestamps in development logs -config :logger, :console, format: "[$level] $message\n" +# Do not include timestamps in development logs +config :logger, :console, format: "$metadata[$level] $message\n" # Set a higher stacktrace during development. Avoid configuring such # in production as building large stacktraces may be expensive. From 462d5aa5cbe3194bad56a7c973e9b200742ef6ca Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 19 Mar 2024 20:53:40 -0400 Subject: [PATCH 048/161] logger: remove request_id metadata which is not useful --- config/config.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 83e7a33e3..383eb3768 100644 --- a/config/config.exs +++ b/config/config.exs @@ -131,13 +131,13 @@ config :logger, :console, level: :debug, format: "\n$time $metadata[$level] $message\n", - metadata: [:actor, :path, :request_id, :type, :user] + metadata: [:actor, :path, :type, :user] config :logger, :ex_syslogger, level: :debug, ident: "pleroma", format: "$metadata[$level] $message", - metadata: [:actor, :path, :request_id, :type, :user] + metadata: [:actor, :path, :type, :user] config :mime, :types, %{ "application/xml" => ["xml"], From 9e6cf45906b9d56834d032d4e0fa436cc5e17031 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 6 Apr 2024 11:43:07 +0200 Subject: [PATCH 049/161] /api/v1/accounts/familiar_followers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/familiar-followers.add | 1 + lib/pleroma/user.ex | 34 +++++++++++++ .../api_spec/operations/account_operation.ex | 42 ++++++++++++++++ .../controllers/account_controller.ex | 34 ++++++++++++- .../web/mastodon_api/views/account_view.ex | 19 +++++++ lib/pleroma/web/router.ex | 1 + test/pleroma/user_test.exs | 14 ++++++ .../controllers/account_controller_test.exs | 49 +++++++++++++++++++ 8 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 changelog.d/familiar-followers.add diff --git a/changelog.d/familiar-followers.add b/changelog.d/familiar-followers.add new file mode 100644 index 000000000..6e7ec9d25 --- /dev/null +++ b/changelog.d/familiar-followers.add @@ -0,0 +1 @@ +Implement `/api/v1/accounts/familiar_followers` \ No newline at end of file diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 778e20526..6d6aa98b5 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1404,6 +1404,40 @@ def get_friends_ids(%User{} = user, page \\ nil) do |> Repo.all() end + @spec get_familiar_followers_query(User.t(), User.t(), pos_integer() | nil) :: Ecto.Query.t() + def get_familiar_followers_query(%User{} = user, %User{} = current_user, nil) do + friends = + get_friends_query(current_user) + |> where([u], not u.hide_follows) + |> select([u], u.id) + + User.Query.build(%{is_active: true}) + |> where([u], u.id not in ^[user.id, current_user.id]) + |> join(:inner, [u], r in FollowingRelationship, + as: :followers_relationships, + on: r.following_id == ^user.id and r.follower_id == u.id + ) + |> where([followers_relationships: r], r.state == ^:follow_accept) + |> where([followers_relationships: r], r.follower_id in subquery(friends)) + end + + def get_familiar_followers_query(%User{} = user, %User{} = current_user, page) do + user + |> get_familiar_followers_query(current_user, nil) + |> User.Query.paginate(page, 20) + end + + @spec get_familiar_followers_query(User.t(), User.t()) :: Ecto.Query.t() + def get_familiar_followers_query(%User{} = user, %User{} = current_user), + do: get_familiar_followers_query(user, current_user, nil) + + @spec get_familiar_followers(User.t(), User.t(), pos_integer() | nil) :: {:ok, list(User.t())} + def get_familiar_followers(%User{} = user, %User{} = current_user, page \\ nil) do + user + |> get_familiar_followers_query(current_user, page) + |> Repo.all() + end + def increase_note_count(%User{} = user) do User |> where(id: ^user.id) diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 36025e47a..244f18dc7 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do alias Pleroma.Web.ApiSpec.Schemas.ActorType alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.BooleanLike + alias Pleroma.Web.ApiSpec.Schemas.FlakeID alias Pleroma.Web.ApiSpec.Schemas.List alias Pleroma.Web.ApiSpec.Schemas.Status alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -513,6 +514,47 @@ def identity_proofs_operation do } end + def familiar_followers_operation do + %Operation{ + tags: ["Retrieve account information"], + summary: "Followers you know", + operationId: "AccountController.relationships", + description: "Returns followers of given account you know.", + security: [%{"oAuth" => ["read:follows"]}], + parameters: [ + Operation.parameter( + :id, + :query, + %Schema{ + oneOf: [%Schema{type: :array, items: %Schema{type: :string}}, %Schema{type: :string}] + }, + "Account IDs", + example: "123" + ) + ], + responses: %{ + 200 => + Operation.response("Accounts", "application/json", %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: %Schema{ + title: "Account", + type: :object, + properties: %{ + id: FlakeID, + accounts: %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: Account, + example: [Account.schema().example] + } + } + } + }) + } + } + end + defp create_request do %Schema{ title: "AccountCreateRequest", diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 9226a2deb..47e6f0a64 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -72,7 +72,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do %{scopes: ["follow", "write:blocks"]} when action in [:block, :unblock] ) - plug(OAuthScopesPlug, %{scopes: ["read:follows"]} when action == :relationships) + plug( + OAuthScopesPlug, + %{scopes: ["read:follows"]} when action in [:relationships, :familiar_followers] + ) plug( OAuthScopesPlug, @@ -629,6 +632,35 @@ def endorsements(%{assigns: %{user: user}} = conn, params) do ) end + @doc "GET /api/v1/accounts/familiar_followers" + def familiar_followers( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, + _id + ) do + users = + User.get_all_by_ids(List.wrap(id)) + |> Enum.map(&%{id: &1.id, accounts: get_familiar_followers(&1, user)}) + + conn + |> render("familiar_followers.json", + for: user, + users: users, + as: :user + ) + end + + defp get_familiar_followers(%{id: id} = user, %{id: id}) do + User.get_familiar_followers(user, user) + end + + defp get_familiar_followers(%{hide_followers: true}, _current_user) do + [] + end + + defp get_familiar_followers(user, current_user) do + User.get_familiar_followers(user, current_user) + end + @doc "GET /api/v1/identity_proofs" def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params) end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 267c3e3ed..6976ca6e5 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -193,6 +193,25 @@ def render("relationships.json", %{user: user, targets: targets} = opts) do render_many(targets, AccountView, "relationship.json", render_opts) end + def render("familiar_followers.json", %{users: users} = opts) do + opts = + opts + |> Map.merge(%{as: :user}) + |> Map.delete(:users) + + users + |> render_many(AccountView, "familiar_followers.json", opts) + end + + def render("familiar_followers.json", %{user: %{id: id, accounts: accounts}} = opts) do + accounts = + accounts + |> render_many(AccountView, "show.json", opts) + |> Enum.filter(&Enum.any?/1) + + %{id: id, accounts: accounts} + end + defp do_render("show.json", %{user: user} = opts) do self = opts[:for] == user diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 4fe0cb02f..644f6cc81 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -633,6 +633,7 @@ defmodule Pleroma.Web.Router do patch("/accounts/update_credentials", AccountController, :update_credentials) get("/accounts/relationships", AccountController, :relationships) + get("/accounts/familiar_followers", AccountController, :familiar_followers) get("/accounts/:id/lists", AccountController, :lists) get("/accounts/:id/identity_proofs", AccountController, :identity_proofs) get("/endorsements", AccountController, :endorsements) diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index a93f81659..48391d871 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -2894,6 +2894,20 @@ test "should report error on non-existing alias" do end end + describe "get_familiar_followers/3" do + test "returns familiar followers for a pair of users" do + user1 = insert(:user) + %{id: id2} = user2 = insert(:user) + user3 = insert(:user) + _user4 = insert(:user) + + User.follow(user1, user2) + User.follow(user2, user3) + + assert [%{id: ^id2}] = User.get_familiar_followers(user3, user1) + end + end + describe "account endorsements" do test "it pins people" do user = insert(:user) diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs index aa7726a9c..e87b33960 100644 --- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs @@ -2172,6 +2172,55 @@ test "max pinned accounts", %{user: user, conn: conn} do end end + describe "familiar followers" do + setup do: oauth_access(["read:follows"]) + + test "fetch user familiar followers", %{user: user, conn: conn} do + %{id: id1} = other_user1 = insert(:user) + %{id: id2} = other_user2 = insert(:user) + _ = insert(:user) + + User.follow(user, other_user1) + User.follow(other_user1, other_user2) + + assert [%{"accounts" => [%{"id" => ^id1}], "id" => ^id2}] = + conn + |> put_req_header("content-type", "application/json") + |> get("/api/v1/accounts/familiar_followers?id[]=#{id2}") + |> json_response_and_validate_schema(200) + end + + test "returns empty array if followers are hidden", %{user: user, conn: conn} do + other_user1 = insert(:user, hide_follows: true) + %{id: id2} = other_user2 = insert(:user) + _ = insert(:user) + + User.follow(user, other_user1) + User.follow(other_user1, other_user2) + + assert [%{"accounts" => [], "id" => ^id2}] = + conn + |> put_req_header("content-type", "application/json") + |> get("/api/v1/accounts/familiar_followers?id[]=#{id2}") + |> json_response_and_validate_schema(200) + end + + test "it respects hide_followers", %{user: user, conn: conn} do + other_user1 = insert(:user) + %{id: id2} = other_user2 = insert(:user, hide_followers: true) + _ = insert(:user) + + User.follow(user, other_user1) + User.follow(other_user1, other_user2) + + assert [%{"accounts" => [], "id" => ^id2}] = + conn + |> put_req_header("content-type", "application/json") + |> get("/api/v1/accounts/familiar_followers?id[]=#{id2}") + |> json_response_and_validate_schema(200) + end + end + describe "remove from followers" do setup do: oauth_access(["follow"]) From ccc3ac241f5b7c88b36efe60a4f9e5d791d2d49a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 6 Apr 2024 10:54:59 +0200 Subject: [PATCH 050/161] Add hint to rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- docs/development/API/admin_api.md | 5 +++- lib/pleroma/rule.ex | 3 ++- lib/pleroma/web/admin_api/views/rule_view.ex | 3 ++- .../operations/admin/report_operation.ex | 3 ++- .../operations/admin/rule_operation.ex | 9 +++++--- .../api_spec/operations/instance_operation.ex | 3 ++- .../web/mastodon_api/views/instance_view.ex | 3 ++- .../20240406000000_add_hint_to_rules.exs | 13 +++++++++++ .../controllers/instance_controller_test.exs | 23 ++++++++++++++----- 9 files changed, 50 insertions(+), 15 deletions(-) create mode 100644 priv/repo/migrations/20240406000000_add_hint_to_rules.exs diff --git a/docs/development/API/admin_api.md b/docs/development/API/admin_api.md index 7bbed16d6..5b373b8e1 100644 --- a/docs/development/API/admin_api.md +++ b/docs/development/API/admin_api.md @@ -1764,7 +1764,8 @@ Note that this differs from the Mastodon API variant: Mastodon API only returns { "id": "1", "priority": 1, - "text": "There are no rules" + "text": "There are no rules", + "hint": null } ] ``` @@ -1775,6 +1776,7 @@ Note that this differs from the Mastodon API variant: Mastodon API only returns - Params: - `text`: string, required, rule content + - `hint`: string, optional, rule description - `priority`: integer, optional, rule ordering priority - Response: JSON, a single rule @@ -1785,6 +1787,7 @@ Note that this differs from the Mastodon API variant: Mastodon API only returns - Params: - `text`: string, optional, rule content + - `hint`: string, optional, rule description - `priority`: integer, optional, rule ordering priority - Response: JSON, a single rule diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex index c8e3470c7..3ba413214 100644 --- a/lib/pleroma/rule.ex +++ b/lib/pleroma/rule.ex @@ -14,13 +14,14 @@ defmodule Pleroma.Rule do schema "rules" do field(:priority, :integer, default: 0) field(:text, :string) + field(:hint, :string) timestamps() end def changeset(%Rule{} = rule, params \\ %{}) do rule - |> cast(params, [:priority, :text]) + |> cast(params, [:priority, :text, :hint]) |> validate_required([:text]) end diff --git a/lib/pleroma/web/admin_api/views/rule_view.ex b/lib/pleroma/web/admin_api/views/rule_view.ex index abfdd593f..606443f05 100644 --- a/lib/pleroma/web/admin_api/views/rule_view.ex +++ b/lib/pleroma/web/admin_api/views/rule_view.ex @@ -15,7 +15,8 @@ def render("show.json", %{rule: rule} = _opts) do %{ id: to_string(rule.id), priority: rule.priority, - text: rule.text + text: rule.text, + hint: rule.hint } end end diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index 971231f4d..25a604beb 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -182,7 +182,8 @@ defp report do type: :object, properties: %{ id: %Schema{type: :string}, - text: %Schema{type: :string} + text: %Schema{type: :string}, + hint: %Schema{type: :string, nullable: true} } } } diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex index 2360880e4..c3a3ecc7c 100644 --- a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex @@ -84,7 +84,8 @@ defp create_request do required: [:text], properties: %{ priority: %Schema{type: :integer}, - text: %Schema{type: :string} + text: %Schema{type: :string}, + hint: %Schema{type: :string} } } end @@ -94,7 +95,8 @@ defp update_request do type: :object, properties: %{ priority: %Schema{type: :integer}, - text: %Schema{type: :string} + text: %Schema{type: :string}, + hint: %Schema{type: :string} } } end @@ -105,7 +107,8 @@ defp rule do properties: %{ id: %Schema{type: :string}, priority: %Schema{type: :integer}, - text: %Schema{type: :string} + text: %Schema{type: :string}, + hint: %Schema{type: :string, nullable: true} } } end diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index faf79c2ea..4cd457ae7 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -364,7 +364,8 @@ defp array_of_rules do type: :object, properties: %{ id: %Schema{type: :string}, - text: %Schema{type: :string} + text: %Schema{type: :string}, + hint: %Schema{type: :string} } } } diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 6a3d445c5..d00f1bd9e 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -85,7 +85,8 @@ def render("rules.json", _) do def render("rule.json", %{rule: rule}) do %{ id: to_string(rule.id), - text: rule.text + text: rule.text, + hint: rule.hint || "" } end diff --git a/priv/repo/migrations/20240406000000_add_hint_to_rules.exs b/priv/repo/migrations/20240406000000_add_hint_to_rules.exs new file mode 100644 index 000000000..273290560 --- /dev/null +++ b/priv/repo/migrations/20240406000000_add_hint_to_rules.exs @@ -0,0 +1,13 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.AddHintToRules do + use Ecto.Migration + + def change do + alter table(:rules) do + add_if_not_exists(:hint, :text) + end + end +end diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index bfc672ff7..373a84303 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -129,16 +129,27 @@ test "get instance information v2", %{conn: conn} do end test "get instance rules", %{conn: conn} do - Rule.create(%{text: "Example rule"}) - Rule.create(%{text: "Second rule"}) - Rule.create(%{text: "Third rule"}) + Rule.create(%{text: "Example rule", hint: "Rule description", priority: 1}) + Rule.create(%{text: "Third rule", priority: 2}) + Rule.create(%{text: "Second rule", priority: 1}) conn = get(conn, "/api/v1/instance") assert result = json_response_and_validate_schema(conn, 200) - rules = result["rules"] - - assert length(rules) == 3 + assert [ + %{ + "text" => "Example rule", + "hint" => "Rule description" + }, + %{ + "text" => "Second rule", + "hint" => "" + }, + %{ + "text" => "Third rule", + "hint" => "" + } + ] = result["rules"] end end From 88412daf118f15a40119f0b47a740a442ec5040c Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Fri, 12 Apr 2024 09:15:06 +0000 Subject: [PATCH 051/161] Apply @lanodan's suggestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/api_spec/operations/account_operation.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 244f18dc7..85f02166f 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -517,9 +517,10 @@ def identity_proofs_operation do def familiar_followers_operation do %Operation{ tags: ["Retrieve account information"], - summary: "Followers you know", - operationId: "AccountController.relationships", - description: "Returns followers of given account you know.", + summary: "Followers that you follow", + operationId: "AccountController.familiar_followers", + description: + "Obtain a list of all accounts that follow a given account, filtered for accounts you follow.", security: [%{"oAuth" => ["read:follows"]}], parameters: [ Operation.parameter( From dd031848112ef812bdb7af9d485360cc4f0ba13a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 7 May 2024 11:54:45 -0400 Subject: [PATCH 052/161] Strip actor from objects before federating --- lib/pleroma/constants.ex | 1 + test/fixtures/create-chat-message.json | 8 ++++---- test/pleroma/user/backup_test.exs | 2 -- .../web/activity_pub/transmogrifier/chat_message_test.exs | 2 -- test/pleroma/web/activity_pub/transmogrifier_test.exs | 5 +++-- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index d814b4931..4d2cd0b62 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Constants do const(object_internal_fields, do: [ + "actor", "reactions", "reaction_count", "likes", diff --git a/test/fixtures/create-chat-message.json b/test/fixtures/create-chat-message.json index 9c23a1c9b..a5e5f559b 100644 --- a/test/fixtures/create-chat-message.json +++ b/test/fixtures/create-chat-message.json @@ -1,10 +1,10 @@ { - "actor": "http://2hu.gensokyo/users/raymoo", - "id": "http://2hu.gensokyo/objects/1", + "actor": "http://mastodon.example.org/users/admin", + "id": "http://mastodon.example.org/objects/1", "object": { - "attributedTo": "http://2hu.gensokyo/users/raymoo", + "attributedTo": "http://mastodon.example.org/users/admin", "content": "You expected a cute girl? Too bad. ", - "id": "http://2hu.gensokyo/objects/2", + "id": "http://mastodon.example.org/objects/2", "published": "2020-02-12T14:08:20Z", "to": [ "http://2hu.gensokyo/users/marisa" diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs index 5503d15bc..e7187df35 100644 --- a/test/pleroma/user/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -221,7 +221,6 @@ test "it creates a zip archive with user data" do "orderedItems" => [ %{ "object" => %{ - "actor" => "http://cofe.io/users/cofe", "content" => "status1", "type" => "Note" }, @@ -229,7 +228,6 @@ test "it creates a zip archive with user data" do }, %{ "object" => %{ - "actor" => "http://cofe.io/users/cofe", "content" => "status2" } }, diff --git a/test/pleroma/web/activity_pub/transmogrifier/chat_message_test.exs b/test/pleroma/web/activity_pub/transmogrifier/chat_message_test.exs index c798a0fc9..086641750 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/chat_message_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/chat_message_test.exs @@ -116,8 +116,6 @@ test "it fetches the actor if they aren't in our system" do data = File.read!("test/fixtures/create-chat-message.json") |> Jason.decode!() - |> Map.put("actor", "http://mastodon.example.org/users/admin") - |> put_in(["object", "actor"], "http://mastodon.example.org/users/admin") _recipient = insert(:user, ap_id: List.first(data["to"]), local: true) diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index a49e459a6..5d84b8403 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -169,7 +169,7 @@ test "it inlines private announced objects" do {:ok, modified} = Transmogrifier.prepare_outgoing(announce_activity.data) assert modified["object"]["content"] == "hey" - assert modified["object"]["actor"] == modified["object"]["attributedTo"] + assert activity.actor == modified["object"]["attributedTo"] end test "it turns mentions into tags" do @@ -220,7 +220,7 @@ test "it sets the 'attributedTo' property to the actor of the object if it doesn {:ok, activity} = CommonAPI.post(user, %{status: "hey"}) {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) - assert modified["object"]["actor"] == modified["object"]["attributedTo"] + assert activity.actor == modified["object"]["attributedTo"] end test "it strips internal hashtag data" do @@ -266,6 +266,7 @@ test "it strips internal fields" do assert is_nil(modified["object"]["announcements"]) assert is_nil(modified["object"]["announcement_count"]) assert is_nil(modified["object"]["generator"]) + assert is_nil(modified["object"]["actor"]) end test "it strips internal fields of article" do From 3cad57bf48180ee5f308ad491c21bcf231a7ba69 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 7 May 2024 17:23:41 -0400 Subject: [PATCH 053/161] Add configuration[statuses][characters_reserved_per_url] to /api/v2/instance Fixes #3250 --- changelog.d/characters_reserved_per_url.add | 1 + lib/pleroma/web/api_spec/operations/instance_operation.ex | 5 +++++ lib/pleroma/web/mastodon_api/views/instance_view.ex | 1 + 3 files changed, 7 insertions(+) create mode 100644 changelog.d/characters_reserved_per_url.add diff --git a/changelog.d/characters_reserved_per_url.add b/changelog.d/characters_reserved_per_url.add new file mode 100644 index 000000000..0ca1a2801 --- /dev/null +++ b/changelog.d/characters_reserved_per_url.add @@ -0,0 +1 @@ +Add configuration[statuses][characters_reserved_per_url] to /api/v2/instance (https://docs.joinmastodon.org/entities/Instance/#characters_reserved_per_url) diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index 708b74b12..57aec83a2 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -285,6 +285,11 @@ defp instance2 do type: :object, description: "A map with poll limits for local statuses", properties: %{ + characters_reserved_per_url: %Schema{ + type: :integer, + description: + "Each URL in a status will be assumed to be exactly this many characters." + }, max_characters: %Schema{ type: :integer, description: "Posts character limit (CW/Subject included in the counter)" diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 210b46d2c..337b2cc83 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -213,6 +213,7 @@ defp configuration do defp configuration2 do configuration() + |> put_in([:statuses, :characters_reserved_per_url], 0) |> Map.merge(%{ urls: %{ streaming: Pleroma.Web.Endpoint.websocket_url(), From b979389958e2d96212cf54ad917d55da86524e30 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 7 May 2024 17:45:02 -0400 Subject: [PATCH 054/161] Add configuration[accounts][max_pinned_statuses] to /api/v2/instance Also add the absent max_featured_tags to the api spec for /api/v2/instance --- .../web/api_spec/operations/instance_operation.ex | 13 +++++++++++++ lib/pleroma/web/mastodon_api/views/instance_view.ex | 1 + 2 files changed, 14 insertions(+) diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index 57aec83a2..bc37cae75 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -272,6 +272,19 @@ defp instance2 do type: :object, description: "Instance configuration", properties: %{ + accounts: %Schema{ + type: :object, + properties: %{ + max_featured_tags: %Schema{ + type: :integer, + description: "The maximum number of featured tags allowed for each account." + }, + max_pinned_statuses: %Schema{ + type: :integer, + description: "The maximum number of pinned statuses for each account." + } + } + }, urls: %Schema{ type: :object, properties: %{ diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 337b2cc83..890dd3977 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -213,6 +213,7 @@ defp configuration do defp configuration2 do configuration() + |> put_in([:accounts, :max_pinned_statuses], Config.get([:instance, :max_pinned_statuses], 0)) |> put_in([:statuses, :characters_reserved_per_url], 0) |> Map.merge(%{ urls: %{ From 06c26bf9c964986b9018ca843be94767f41636a3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 7 May 2024 17:46:05 -0400 Subject: [PATCH 055/161] Add the absent max_featured_tags to the api spec for /api/v1/instance --- .../web/api_spec/operations/instance_operation.ex | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index bc37cae75..b6c411c07 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -50,6 +50,15 @@ defp instance do %Schema{ type: :object, properties: %{ + accounts: %Schema{ + type: :object, + properties: %{ + max_featured_tags: %Schema{ + type: :integer, + description: "The maximum number of featured tags allowed for each account." + } + } + }, uri: %Schema{type: :string, description: "The domain name of the instance"}, title: %Schema{type: :string, description: "The title of the website"}, description: %Schema{ From acf73f7e13515c64a9cfa935f6102f7ffa32585b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 7 May 2024 17:48:40 -0400 Subject: [PATCH 056/161] Update changelog entry --- changelog.d/characters_reserved_per_url.add | 1 - changelog.d/mastodon_api_v2.add | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changelog.d/characters_reserved_per_url.add create mode 100644 changelog.d/mastodon_api_v2.add diff --git a/changelog.d/characters_reserved_per_url.add b/changelog.d/characters_reserved_per_url.add deleted file mode 100644 index 0ca1a2801..000000000 --- a/changelog.d/characters_reserved_per_url.add +++ /dev/null @@ -1 +0,0 @@ -Add configuration[statuses][characters_reserved_per_url] to /api/v2/instance (https://docs.joinmastodon.org/entities/Instance/#characters_reserved_per_url) diff --git a/changelog.d/mastodon_api_v2.add b/changelog.d/mastodon_api_v2.add new file mode 100644 index 000000000..d53aa35c4 --- /dev/null +++ b/changelog.d/mastodon_api_v2.add @@ -0,0 +1 @@ +Add new parameters to /api/v2/instance: configuration[accounts][max_pinned_statuses] and configuration[statuses][characters_reserved_per_url] From cd7e2138d11901fc7a0c8c2f22b7a5d57383a555 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 14 May 2024 14:13:37 +0400 Subject: [PATCH 057/161] Search: Basic Qdrant/Ollama search --- config/config.exs | 9 ++ lib/mix/tasks/pleroma/search/indexer.ex | 60 ++++++++++++ lib/pleroma/search/qdrant_search.ex | 117 ++++++++++++++++++++++++ 3 files changed, 186 insertions(+) create mode 100644 lib/mix/tasks/pleroma/search/indexer.ex create mode 100644 lib/pleroma/search/qdrant_search.ex diff --git a/config/config.exs b/config/config.exs index b69044a2b..f74eda6b2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -915,6 +915,15 @@ config :pleroma, Pleroma.Uploaders.Uploader, timeout: 30_000 +config :pleroma, Pleroma.Search.QdrantSearch, + qdrant_url: "http://127.0.0.1:6333/", + qdrant_api_key: nil, + ollama_url: "http://127.0.0.1:11434", + ollama_model: "snowflake-arctic-embed:xs", + qdrant_index_configuration: %{ + vectors: %{size: 384, distance: "Cosine"} + } + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex new file mode 100644 index 000000000..ffa2f3c94 --- /dev/null +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -0,0 +1,60 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search.Indexer do + import Mix.Pleroma + import Ecto.Query + + alias Pleroma.Workers.SearchIndexingWorker + + def run(["index" | options]) do + {options, [], []} = + OptionParser.parse( + options, + strict: [ + limit: :integer + ] + ) + + start_pleroma() + + limit = Keyword.get(options, :limit, 100_000) + + per_step = 1000 + chunks = max(div(limit, per_step), 1) + + 1..chunks + |> Enum.each(fn step -> + q = + from(a in Pleroma.Activity, + limit: ^per_step, + offset: ^per_step * (^step - 1), + select: [:id], + order_by: [desc: :id] + ) + + {:ok, ids} = + Pleroma.Repo.transaction(fn -> + Pleroma.Repo.stream(q, timeout: :infinity) + |> Enum.map(fn a -> + a.id + end) + end) + + IO.puts("Got #{length(ids)} activities, adding to indexer") + + ids + |> Enum.chunk_every(100) + |> Enum.each(fn chunk -> + IO.puts("Adding #{length(chunk)} activities to indexing queue") + + chunk + |> Enum.map(fn id -> + SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id}) + end) + |> Oban.insert_all() + end) + end) + end +end diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex new file mode 100644 index 000000000..fcaa9e686 --- /dev/null +++ b/lib/pleroma/search/qdrant_search.ex @@ -0,0 +1,117 @@ +defmodule Pleroma.Search.QdrantSearch do + @behaviour Pleroma.Search.SearchBackend + import Ecto.Query + alias Pleroma.Activity + + alias __MODULE__.QdrantClient + alias __MODULE__.OllamaClient + + import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] + + def initialize_index() do + payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) + QdrantClient.put("/collections/posts", payload) + end + + def drop_index() do + QdrantClient.delete("/collections/posts") + end + + def get_embedding(text) do + with {:ok, %{body: %{"embedding" => embedding}}} <- + OllamaClient.post("/api/embeddings", %{ + prompt: text, + model: Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_model]) + }) + |> IO.inspect() do + {:ok, embedding} + else + _ -> + {:error, "Failed to get embedding"} + end + end + + defp build_index_payload(activity, embedding) do + %{ + points: [ + %{ + id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(), + vector: embedding + } + ] + } + end + + defp build_search_payload(embedding) do + %{ + vector: embedding, + limit: 20 + } + end + + @impl true + def add_to_index(activity) do + # This will only index public or unlisted notes + maybe_search_data = object_to_search_data(activity.object) + IO.puts("TRYING TO INDEX\n\n") + + if activity.data["type"] == "Create" and maybe_search_data do + with {:ok, embedding} <- get_embedding(maybe_search_data.content), + {:ok, %{status: 200}} <- + QdrantClient.put( + "/collections/posts/points", + build_index_payload(activity, embedding) + ) do + :ok + else + e -> {:error, e} + end + else + :ok + end + end + + @impl true + def search(_user, query, _options) do + with {:ok, embedding} <- get_embedding(query), + {:ok, %{body: %{"result" => result}}} <- + QdrantClient.post("/collections/posts/points/search", build_search_payload(embedding)) do + ids = + Enum.map(result, fn %{"id" => id} -> + Ecto.UUID.dump!(id) + end) + + from(a in Activity, where: a.id in ^ids) + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id)) + |> Pleroma.Repo.all() + else + _ -> + [] + end + end + + @impl true + def remove_from_index(_object) do + :ok + end +end + +defmodule Pleroma.Search.QdrantSearch.OllamaClient do + use Tesla + + plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_url])) + plug(Tesla.Middleware.JSON) +end + +defmodule Pleroma.Search.QdrantSearch.QdrantClient do + use Tesla + + plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])) + plug(Tesla.Middleware.JSON) + + plug(Tesla.Middleware.Headers, [ + {"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])} + ]) +end From bb08a766f4c1bd84c98e245c1871c46fcc7c7a8d Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 14 May 2024 14:26:41 +0400 Subject: [PATCH 058/161] QdrantSearch: Remove debugging stuff --- lib/pleroma/search/qdrant_search.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index fcaa9e686..726a30b3b 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -22,8 +22,7 @@ def get_embedding(text) do OllamaClient.post("/api/embeddings", %{ prompt: text, model: Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_model]) - }) - |> IO.inspect() do + }) do {:ok, embedding} else _ -> @@ -53,7 +52,6 @@ defp build_search_payload(embedding) do def add_to_index(activity) do # This will only index public or unlisted notes maybe_search_data = object_to_search_data(activity.object) - IO.puts("TRYING TO INDEX\n\n") if activity.data["type"] == "Create" and maybe_search_data do with {:ok, embedding} <- get_embedding(maybe_search_data.content), From 1490ff30af7001adc386b4fec54c62e1a524d7d6 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 14 May 2024 15:09:38 +0400 Subject: [PATCH 059/161] QdrantSearch: Add query prefix. --- lib/pleroma/search/qdrant_search.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 726a30b3b..31e7754ae 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -71,6 +71,8 @@ def add_to_index(activity) do @impl true def search(_user, query, _options) do + query = "Represent this sentence for searching relevant passages: #{query}" + with {:ok, embedding} <- get_embedding(query), {:ok, %{body: %{"result" => result}}} <- QdrantClient.post("/collections/posts/points/search", build_search_payload(embedding)) do From c50f0f31f418037063bd97efcdc0f60b89594212 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 14 May 2024 16:56:58 +0400 Subject: [PATCH 060/161] Docs/Search: Add basic documentation of the qdrant search --- docs/configuration/search.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 0316c9bf4..682d1e52a 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -10,6 +10,12 @@ To use built-in search that has no external dependencies, set the search module While it has no external dependencies, it has problems with performance and relevancy. +## QdrantSearch + +This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings, for now only the [Ollama](Ollama) api is supported. + +The default settings will support a setup where both Ollama and Qdrant run on the same system as pleroma. The embedding model used by Ollama will need to be pulled first (e.g. `ollama pull snowflake-arctic-embed:xs`) for the embedding to work. + ## Meilisearch Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 million From 1261c43a7af48ed6e6753461944659391c4c58cc Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 14 May 2024 17:19:36 +0400 Subject: [PATCH 061/161] SearchBackend: Add create_index --- lib/mix/tasks/pleroma/search/indexer.ex | 6 ++++++ lib/pleroma/search/qdrant_search.ex | 3 ++- lib/pleroma/search/search_backend.ex | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex index ffa2f3c94..326646b69 100644 --- a/lib/mix/tasks/pleroma/search/indexer.ex +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -8,6 +8,12 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do alias Pleroma.Workers.SearchIndexingWorker + def run(["create_index"]) do + Application.ensure_all_started(:pleroma) + + Pleroma.Config.get([Pleroma.Search, :module]).create_index() + end + def run(["index" | options]) do {options, [], []} = OptionParser.parse( diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 31e7754ae..315262cb3 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -8,7 +8,8 @@ defmodule Pleroma.Search.QdrantSearch do import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] - def initialize_index() do + @impl true + def create_index() do payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) QdrantClient.put("/collections/posts", payload) end diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index 68bc48cec..5be0169d0 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -21,4 +21,9 @@ defmodule Pleroma.Search.SearchBackend do from index. """ @callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()} + + @doc """ + Create the index + """ + @callback create_index() :: :ok | {:error, any()} end From 2965ed47bdae43fcddb7258aa2e667aec5be018b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 15 May 2024 16:40:31 -0400 Subject: [PATCH 062/161] Changelog for stripping actor from objects --- changelog.d/strip-object-actor.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/strip-object-actor.fix diff --git a/changelog.d/strip-object-actor.fix b/changelog.d/strip-object-actor.fix new file mode 100644 index 000000000..71cf7ee65 --- /dev/null +++ b/changelog.d/strip-object-actor.fix @@ -0,0 +1 @@ +Strip actor property from objects before federating From a9be4907c0d7b34e5564584d2d040632c32f2aa3 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 16 May 2024 10:47:24 +0400 Subject: [PATCH 063/161] SearchBackend: Add drop_index --- lib/mix/tasks/pleroma/search/indexer.ex | 18 ++++++++++++++++-- lib/pleroma/search/database_search.ex | 6 ++++++ lib/pleroma/search/meilisearch.ex | 6 ++++++ lib/pleroma/search/qdrant_search.ex | 14 ++++++++++++-- lib/pleroma/search/search_backend.ex | 5 +++++ 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex index 326646b69..81a9fced6 100644 --- a/lib/mix/tasks/pleroma/search/indexer.ex +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -9,9 +9,23 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do alias Pleroma.Workers.SearchIndexingWorker def run(["create_index"]) do - Application.ensure_all_started(:pleroma) + start_pleroma() - Pleroma.Config.get([Pleroma.Search, :module]).create_index() + with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do + IO.puts("Index created") + else + e -> IO.puts("Could not create index: #{inspect(e)}") + end + end + + def run(["drop_index"]) do + start_pleroma() + + with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do + IO.puts("Index dropped") + else + e -> IO.puts("Could not drop index: #{inspect(e)}") + end end def run(["index" | options]) do diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 31bfc7e33..24a1ff431 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -48,6 +48,12 @@ def add_to_index(_activity), do: :ok @impl true def remove_from_index(_object), do: :ok + @impl true + def create_index, do: :ok + + @impl true + def drop_index, do: :ok + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 2bff663e8..50f5984d6 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -10,6 +10,12 @@ defmodule Pleroma.Search.Meilisearch do @behaviour Pleroma.Search.SearchBackend + @impl true + def create_index, do: :ok + + @impl true + def drop_index, do: :ok + defp meili_headers do private_key = Config.get([Pleroma.Search.Meilisearch, :private_key]) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 315262cb3..4bd35c17c 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -11,11 +11,21 @@ defmodule Pleroma.Search.QdrantSearch do @impl true def create_index() do payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) - QdrantClient.put("/collections/posts", payload) + + with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do + :ok + else + e -> {:error, e} + end end + @impl true def drop_index() do - QdrantClient.delete("/collections/posts") + with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do + :ok + else + e -> {:error, e} + end end def get_embedding(text) do diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index 5be0169d0..9735ab3f4 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -26,4 +26,9 @@ defmodule Pleroma.Search.SearchBackend do Create the index """ @callback create_index() :: :ok | {:error, any()} + + @doc """ + Drop the index + """ + @callback drop_index() :: :ok | {:error, any()} end From 7f8a9329e566140f9f36cecac58b13097b7e0519 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 May 2024 16:03:05 -0400 Subject: [PATCH 064/161] Startup detection for configured MRF modules that are missing or incorrectly defined --- changelog.d/missing-mrfs.add | 1 + lib/pleroma/application_requirements.ex | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 changelog.d/missing-mrfs.add diff --git a/changelog.d/missing-mrfs.add b/changelog.d/missing-mrfs.add new file mode 100644 index 000000000..6a17f9e1a --- /dev/null +++ b/changelog.d/missing-mrfs.add @@ -0,0 +1 @@ +Startup detection for configured MRF modules that are missing or incorrectly defined diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 819245481..8c0df64fc 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -28,6 +28,7 @@ def verify! do |> check_welcome_message_config!() |> check_rum!() |> check_repo_pool_size!() + |> check_mrfs() |> handle_result() end @@ -234,4 +235,25 @@ defp check_filter(filter, command_required) do true end end + + defp check_mrfs(:ok) do + mrfs = Config.get!([:mrf, :policies]) + + missing_mrfs = + Enum.reduce(mrfs, [], fn x, acc -> + if Code.ensure_compiled(x) do + acc + else + acc ++ [x] + end + end) + + if Enum.empty?(missing_mrfs) do + :ok + else + {:error, "The following MRF modules are configured but missing: #{inspect(missing_mrfs)}"} + end + end + + defp check_mrfs(result), do: result end From 9988dc22273a22cd262c84adde184fcab4a4e8ae Mon Sep 17 00:00:00 2001 From: feld Date: Thu, 16 May 2024 23:33:48 +0000 Subject: [PATCH 065/161] Revert "Merge branch 'strip-object-actor' into 'develop'" This reverts merge request !4105 --- changelog.d/strip-object-actor.fix | 1 - lib/pleroma/constants.ex | 1 - test/fixtures/create-chat-message.json | 8 ++++---- test/pleroma/user/backup_test.exs | 2 ++ .../web/activity_pub/transmogrifier/chat_message_test.exs | 2 ++ test/pleroma/web/activity_pub/transmogrifier_test.exs | 5 ++--- 6 files changed, 10 insertions(+), 9 deletions(-) delete mode 100644 changelog.d/strip-object-actor.fix diff --git a/changelog.d/strip-object-actor.fix b/changelog.d/strip-object-actor.fix deleted file mode 100644 index 71cf7ee65..000000000 --- a/changelog.d/strip-object-actor.fix +++ /dev/null @@ -1 +0,0 @@ -Strip actor property from objects before federating diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index ac4bf2ffb..3a5e35301 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -9,7 +9,6 @@ defmodule Pleroma.Constants do const(object_internal_fields, do: [ - "actor", "reactions", "reaction_count", "likes", diff --git a/test/fixtures/create-chat-message.json b/test/fixtures/create-chat-message.json index a5e5f559b..9c23a1c9b 100644 --- a/test/fixtures/create-chat-message.json +++ b/test/fixtures/create-chat-message.json @@ -1,10 +1,10 @@ { - "actor": "http://mastodon.example.org/users/admin", - "id": "http://mastodon.example.org/objects/1", + "actor": "http://2hu.gensokyo/users/raymoo", + "id": "http://2hu.gensokyo/objects/1", "object": { - "attributedTo": "http://mastodon.example.org/users/admin", + "attributedTo": "http://2hu.gensokyo/users/raymoo", "content": "You expected a cute girl? Too bad. ", - "id": "http://mastodon.example.org/objects/2", + "id": "http://2hu.gensokyo/objects/2", "published": "2020-02-12T14:08:20Z", "to": [ "http://2hu.gensokyo/users/marisa" diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs index e7187df35..5503d15bc 100644 --- a/test/pleroma/user/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -221,6 +221,7 @@ test "it creates a zip archive with user data" do "orderedItems" => [ %{ "object" => %{ + "actor" => "http://cofe.io/users/cofe", "content" => "status1", "type" => "Note" }, @@ -228,6 +229,7 @@ test "it creates a zip archive with user data" do }, %{ "object" => %{ + "actor" => "http://cofe.io/users/cofe", "content" => "status2" } }, diff --git a/test/pleroma/web/activity_pub/transmogrifier/chat_message_test.exs b/test/pleroma/web/activity_pub/transmogrifier/chat_message_test.exs index 086641750..c798a0fc9 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/chat_message_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/chat_message_test.exs @@ -116,6 +116,8 @@ test "it fetches the actor if they aren't in our system" do data = File.read!("test/fixtures/create-chat-message.json") |> Jason.decode!() + |> Map.put("actor", "http://mastodon.example.org/users/admin") + |> put_in(["object", "actor"], "http://mastodon.example.org/users/admin") _recipient = insert(:user, ap_id: List.first(data["to"]), local: true) diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 5d84b8403..a49e459a6 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -169,7 +169,7 @@ test "it inlines private announced objects" do {:ok, modified} = Transmogrifier.prepare_outgoing(announce_activity.data) assert modified["object"]["content"] == "hey" - assert activity.actor == modified["object"]["attributedTo"] + assert modified["object"]["actor"] == modified["object"]["attributedTo"] end test "it turns mentions into tags" do @@ -220,7 +220,7 @@ test "it sets the 'attributedTo' property to the actor of the object if it doesn {:ok, activity} = CommonAPI.post(user, %{status: "hey"}) {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) - assert activity.actor == modified["object"]["attributedTo"] + assert modified["object"]["actor"] == modified["object"]["attributedTo"] end test "it strips internal hashtag data" do @@ -266,7 +266,6 @@ test "it strips internal fields" do assert is_nil(modified["object"]["announcements"]) assert is_nil(modified["object"]["announcement_count"]) assert is_nil(modified["object"]["generator"]) - assert is_nil(modified["object"]["actor"]) end test "it strips internal fields of article" do From 069ce4448c556af90293cde9b9872c3d53eb894b Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 11:55:17 +0400 Subject: [PATCH 066/161] Add basic fastembed server --- python/fastembed-server.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 python/fastembed-server.py diff --git a/python/fastembed-server.py b/python/fastembed-server.py new file mode 100644 index 000000000..fa3f7c82b --- /dev/null +++ b/python/fastembed-server.py @@ -0,0 +1,21 @@ +from fastembed import TextEmbedding +from fastapi import FastAPI +from pydantic import BaseModel + +model = TextEmbedding("snowflake/snowflake-arctic-embed-xs") + +app = FastAPI() + +class EmbeddingRequest(BaseModel): + model: str + prompt: str + +@app.post("/api/embeddings") +def embeddings(request: EmbeddingRequest): + embeddings = next(model.embed(request.prompt)).tolist() + return {"embedding": embeddings} + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=11345) From 769773a500d4c6ec021776b493f7d98c9f87e81e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 12:08:42 +0400 Subject: [PATCH 067/161] Add dockerfile --- python/Dockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 python/Dockerfile diff --git a/python/Dockerfile b/python/Dockerfile new file mode 100644 index 000000000..f83c1c1b3 --- /dev/null +++ b/python/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.9 + +WORKDIR /code +COPY fastembed-server.py /workdir/fastembed-server.py + +RUN pip install --no-cache-dir --upgrade fastembed fastapi uvicorn + +CMD ["python", "/workdir/fastembed-server.py"] From 61e9027131843858b017d3b7c18c3a396d5656a9 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 12:19:42 +0400 Subject: [PATCH 068/161] Add docker compose file for fastembed server --- python/compose.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 python/compose.yml diff --git a/python/compose.yml b/python/compose.yml new file mode 100644 index 000000000..d4cb31722 --- /dev/null +++ b/python/compose.yml @@ -0,0 +1,5 @@ +services: + web: + build: . + ports: + - "11345:11345" From 933117785fb1b5b671c61d09671cf6418b105187 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 13:43:47 +0400 Subject: [PATCH 069/161] QdrantSearch: Add basic test --- lib/pleroma/search/qdrant_search.ex | 11 ++-- test/pleroma/search/qdrant_search_test.exs | 65 ++++++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 test/pleroma/search/qdrant_search_test.exs diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 4bd35c17c..2d9315a2f 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -5,12 +5,13 @@ defmodule Pleroma.Search.QdrantSearch do alias __MODULE__.QdrantClient alias __MODULE__.OllamaClient + alias Pleroma.Config.Getting, as: Config import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] @impl true def create_index() do - payload = Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) + payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do :ok @@ -32,7 +33,7 @@ def get_embedding(text) do with {:ok, %{body: %{"embedding" => embedding}}} <- OllamaClient.post("/api/embeddings", %{ prompt: text, - model: Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_model]) + model: Config.get([Pleroma.Search.QdrantSearch, :ollama_model]) }) do {:ok, embedding} else @@ -111,15 +112,17 @@ def remove_from_index(_object) do defmodule Pleroma.Search.QdrantSearch.OllamaClient do use Tesla + alias Pleroma.Config.Getting, as: Config - plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :ollama_url])) + plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :ollama_url])) plug(Tesla.Middleware.JSON) end defmodule Pleroma.Search.QdrantSearch.QdrantClient do use Tesla + alias Pleroma.Config.Getting, as: Config - plug(Tesla.Middleware.BaseUrl, Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])) + plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])) plug(Tesla.Middleware.JSON) plug(Tesla.Middleware.Headers, [ diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs new file mode 100644 index 000000000..9be246a9a --- /dev/null +++ b/test/pleroma/search/qdrant_search_test.exs @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.QdrantSearchTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + import Mox + + alias Pleroma.Web.CommonAPI + alias Pleroma.UnstubbedConfigMock, as: Config + alias Pleroma.Search.QdrantSearch + alias Pleroma.Workers.SearchIndexingWorker + + describe "Qdrant search" do + test "indexes a public post on creation" do + user = insert(:user) + + Tesla.Mock.mock(fn + %{method: :post, url: "https://ollama.url/api/embeddings"} -> + send(self(), "posted_to_ollama") + Tesla.Mock.json(%{embedding: [1, 2, 3]}) + + %{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} -> + send(self(), "posted_to_qdrant") + + assert match?(%{"points" => [%{"vector" => [1, 2, 3]}]}, Jason.decode!(body)) + + Tesla.Mock.json("ok") + end) + + Config + |> expect(:get, 4, fn + [Pleroma.Search, :module], nil -> + QdrantSearch + + [Pleroma.Search.QdrantSearch, key], nil -> + %{ + ollama_model: "a_model", + ollama_url: "https://ollama.url", + qdrant_url: "https://qdrant.url" + }[key] + end) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + args = %{"op" => "add_to_index", "activity" => activity.id} + + assert_enqueued( + worker: SearchIndexingWorker, + args: args + ) + + assert :ok = perform_job(SearchIndexingWorker, args) + assert_received("posted_to_ollama") + assert_received("posted_to_qdrant") + end + end +end From e3933a067feae1f087616f675657d6ff99b2782b Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 14:04:32 +0400 Subject: [PATCH 070/161] QdrantSearch: Implement post deletion --- lib/pleroma/search/qdrant_search.ex | 18 +++++++++++++----- test/pleroma/search/qdrant_search_test.exs | 16 ++++++++++++++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 2d9315a2f..acfaaff52 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -81,6 +81,19 @@ def add_to_index(activity) do end end + @impl true + def remove_from_index(object) do + activity = Activity.get_by_object_ap_id_with_object(object.data["id"]) + id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!() + + with {:ok, %{status: 200}} <- + QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do + :ok + else + e -> {:error, e} + end + end + @impl true def search(_user, query, _options) do query = "Represent this sentence for searching relevant passages: #{query}" @@ -103,11 +116,6 @@ def search(_user, query, _options) do [] end end - - @impl true - def remove_from_index(_object) do - :ok - end end defmodule Pleroma.Search.QdrantSearch.OllamaClient do diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index 9be246a9a..e816311aa 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -15,7 +15,7 @@ defmodule Pleroma.Search.QdrantSearchTest do alias Pleroma.Workers.SearchIndexingWorker describe "Qdrant search" do - test "indexes a public post on creation" do + test "indexes a public post on creation, deletes from the index on deletion" do user = insert(:user) Tesla.Mock.mock(fn @@ -29,10 +29,14 @@ test "indexes a public post on creation" do assert match?(%{"points" => [%{"vector" => [1, 2, 3]}]}, Jason.decode!(body)) Tesla.Mock.json("ok") + + %{method: :post, url: "https://qdrant.url/collections/posts/points/delete"} -> + send(self(), "deleted_from_qdrant") + Tesla.Mock.json("ok") end) Config - |> expect(:get, 4, fn + |> expect(:get, 6, fn [Pleroma.Search, :module], nil -> QdrantSearch @@ -60,6 +64,14 @@ test "indexes a public post on creation" do assert :ok = perform_job(SearchIndexingWorker, args) assert_received("posted_to_ollama") assert_received("posted_to_qdrant") + + {:ok, _} = CommonAPI.delete(activity.id, user) + + delete_args = %{"op" => "remove_from_index", "object" => activity.object.id} + assert_enqueued(worker: SearchIndexingWorker, args: delete_args) + assert :ok = perform_job(SearchIndexingWorker, delete_args) + + assert_received("deleted_from_qdrant") end end end From 39525bcec7c685cb28ca4702b6e145a78e733fee Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 14:07:47 +0400 Subject: [PATCH 071/161] Add qdrant changelog --- changelog.d/qdrant_search.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/qdrant_search.add diff --git a/changelog.d/qdrant_search.add b/changelog.d/qdrant_search.add new file mode 100644 index 000000000..6f9e39e23 --- /dev/null +++ b/changelog.d/qdrant_search.add @@ -0,0 +1 @@ +Add Qdrant/Ollama search From 3345ddd2d4ef380929cc231118a5fb6486c0bd5c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 18 May 2024 15:02:22 +0400 Subject: [PATCH 072/161] Linting --- lib/pleroma/search/qdrant_search.ex | 11 ++++++----- test/pleroma/search/qdrant_search_test.exs | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index acfaaff52..a6c6c6a0d 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -1,16 +1,17 @@ defmodule Pleroma.Search.QdrantSearch do @behaviour Pleroma.Search.SearchBackend import Ecto.Query - alias Pleroma.Activity - alias __MODULE__.QdrantClient - alias __MODULE__.OllamaClient + alias Pleroma.Activity alias Pleroma.Config.Getting, as: Config + alias __MODULE__.OllamaClient + alias __MODULE__.QdrantClient + import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] @impl true - def create_index() do + def create_index do payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do @@ -21,7 +22,7 @@ def create_index() do end @impl true - def drop_index() do + def drop_index do with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do :ok else diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index e816311aa..698894cdb 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -9,9 +9,9 @@ defmodule Pleroma.Search.QdrantSearchTest do import Pleroma.Factory import Mox - alias Pleroma.Web.CommonAPI - alias Pleroma.UnstubbedConfigMock, as: Config alias Pleroma.Search.QdrantSearch + alias Pleroma.UnstubbedConfigMock, as: Config + alias Pleroma.Web.CommonAPI alias Pleroma.Workers.SearchIndexingWorker describe "Qdrant search" do From d07d49227fd3bf716fa22e402685f27e31a0f6d3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 May 2024 16:26:21 -0400 Subject: [PATCH 073/161] PleromaAPI: marking notifications as read no longer returns notifications --- changelog.d/mark-read.fix | 1 + lib/pleroma/notification.ex | 17 +++--------- .../pleroma_notification_operation.ex | 8 +----- .../controllers/notification_controller.ex | 20 ++++++++------ test/pleroma/notification_test.exs | 4 +-- .../notification_controller_test.exs | 27 ++++++++++--------- 6 files changed, 33 insertions(+), 44 deletions(-) create mode 100644 changelog.d/mark-read.fix diff --git a/changelog.d/mark-read.fix b/changelog.d/mark-read.fix new file mode 100644 index 000000000..346eb19e2 --- /dev/null +++ b/changelog.d/mark-read.fix @@ -0,0 +1 @@ +The query for marking notifications as read has been simplified diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index a80279fa6..cb9bd92b8 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -280,15 +280,10 @@ def set_read_up_to(%{id: user_id} = user, id) do select: n.id ) - {:ok, %{ids: {_, notification_ids}}} = - Multi.new() - |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) - |> Marker.multi_set_last_read_id(user, "notifications") - |> Repo.transaction() - - for_user_query(user) - |> where([n], n.id in ^notification_ids) - |> Repo.all() + Multi.new() + |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) + |> Marker.multi_set_last_read_id(user, "notifications") + |> Repo.transaction() end @spec read_one(User.t(), String.t()) :: @@ -299,10 +294,6 @@ def read_one(%User{} = user, notification_id) do |> Multi.update(:update, changeset(notification, %{seen: true})) |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() - |> case do - {:ok, %{update: notification}} -> {:ok, notification} - {:error, :update, changeset, _} -> {:error, changeset} - end end end diff --git a/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex index a994345db..0e2865191 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema - alias Pleroma.Web.ApiSpec.NotificationOperation alias Pleroma.Web.ApiSpec.Schemas.ApiError import Pleroma.Web.ApiSpec.Helpers @@ -35,12 +34,7 @@ def mark_as_read_operation do Operation.response( "A Notification or array of Notifications", "application/json", - %Schema{ - anyOf: [ - %Schema{type: :array, items: NotificationOperation.notification()}, - NotificationOperation.notification() - ] - } + %Schema{type: :string} ), 400 => Operation.response("Bad Request", "application/json", ApiError) } diff --git a/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex b/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex index f860eaf7e..435ccfabe 100644 --- a/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex @@ -23,8 +23,9 @@ def mark_as_read( } = conn, _ ) do - with {:ok, notification} <- Notification.read_one(user, notification_id) do - render(conn, "show.json", notification: notification, for: user) + with {:ok, _} <- Notification.read_one(user, notification_id) do + conn + |> json("ok") else {:error, message} -> conn @@ -38,11 +39,14 @@ def mark_as_read( conn, _ ) do - notifications = - user - |> Notification.set_read_up_to(max_id) - |> Enum.take(80) - - render(conn, "index.json", notifications: notifications, for: user) + with {:ok, _} <- Notification.set_read_up_to(user, max_id) do + conn + |> json("ok") + else + {:error, message} -> + conn + |> put_status(:bad_request) + |> json(%{"error" => message}) + end end end diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index 392fd53c2..1dd0c26f7 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -449,9 +449,7 @@ test "it sets all notifications as read up to a specified notification ID" do status: "hey yet again @#{other_user.nickname}!" }) - [_, read_notification] = Notification.set_read_up_to(other_user, n2.id) - - assert read_notification.activity.object + Notification.set_read_up_to(other_user, n2.id) [n3, n2, n1] = Notification.for_user(other_user) diff --git a/test/pleroma/web/pleroma_api/controllers/notification_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/notification_controller_test.exs index b8c7964f9..036cbf176 100644 --- a/test/pleroma/web/pleroma_api/controllers/notification_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/notification_controller_test.exs @@ -21,13 +21,11 @@ test "it marks a single notification as read", %{user: user1, conn: conn} do {:ok, [notification1]} = Notification.create_notifications(activity1) {:ok, [notification2]} = Notification.create_notifications(activity2) - response = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/pleroma/notifications/read", %{id: notification1.id}) - |> json_response_and_validate_schema(:ok) + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/pleroma/notifications/read", %{id: notification1.id}) + |> json_response_and_validate_schema(:ok) - assert %{"pleroma" => %{"is_seen" => true}} = response assert Repo.get(Notification, notification1.id).seen refute Repo.get(Notification, notification2.id).seen end @@ -40,14 +38,17 @@ test "it marks multiple notifications as read", %{user: user1, conn: conn} do [notification3, notification2, notification1] = Notification.for_user(user1, %{limit: 3}) - [response1, response2] = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/pleroma/notifications/read", %{max_id: notification2.id}) - |> json_response_and_validate_schema(:ok) + refute Repo.get(Notification, notification1.id).seen + refute Repo.get(Notification, notification2.id).seen + refute Repo.get(Notification, notification3.id).seen + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/pleroma/notifications/read", %{max_id: notification2.id}) + |> json_response_and_validate_schema(:ok) + + [notification3, notification2, notification1] = Notification.for_user(user1, %{limit: 3}) - assert %{"pleroma" => %{"is_seen" => true}} = response1 - assert %{"pleroma" => %{"is_seen" => true}} = response2 assert Repo.get(Notification, notification1.id).seen assert Repo.get(Notification, notification2.id).seen refute Repo.get(Notification, notification3.id).seen From 72ec261a69a7dda7ab95667e425824ab7758b636 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 12:17:46 +0400 Subject: [PATCH 074/161] B QdrantSearch: Switch to OpenAI api --- changelog.d/qdrant_search.add | 2 +- config/config.exs | 7 ++++--- lib/pleroma/search/qdrant_search.ex | 19 ++++++++++++------- test/pleroma/search/qdrant_search_test.exs | 15 +++++++++------ 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/changelog.d/qdrant_search.add b/changelog.d/qdrant_search.add index 6f9e39e23..9801131d1 100644 --- a/changelog.d/qdrant_search.add +++ b/changelog.d/qdrant_search.add @@ -1 +1 @@ -Add Qdrant/Ollama search +Add Qdrant/OpenAI embedding search diff --git a/config/config.exs b/config/config.exs index f74eda6b2..dd0150c66 100644 --- a/config/config.exs +++ b/config/config.exs @@ -917,9 +917,10 @@ config :pleroma, Pleroma.Search.QdrantSearch, qdrant_url: "http://127.0.0.1:6333/", - qdrant_api_key: nil, - ollama_url: "http://127.0.0.1:11434", - ollama_model: "snowflake-arctic-embed:xs", + qdrant_api_key: "", + openai_url: "http://127.0.0.1:11345", + openai_model: "snowflake", + openai_api_key: "", qdrant_index_configuration: %{ vectors: %{size: 384, distance: "Cosine"} } diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index a6c6c6a0d..5ae04be78 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Search.QdrantSearch do alias Pleroma.Activity alias Pleroma.Config.Getting, as: Config - alias __MODULE__.OllamaClient + alias __MODULE__.OpenAIClient alias __MODULE__.QdrantClient import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] @@ -31,10 +31,10 @@ def drop_index do end def get_embedding(text) do - with {:ok, %{body: %{"embedding" => embedding}}} <- - OllamaClient.post("/api/embeddings", %{ - prompt: text, - model: Config.get([Pleroma.Search.QdrantSearch, :ollama_model]) + with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <- + OpenAIClient.post("/v1/embeddings", %{ + input: text, + model: Config.get([Pleroma.Search.QdrantSearch, :openai_model]) }) do {:ok, embedding} else @@ -119,12 +119,17 @@ def search(_user, query, _options) do end end -defmodule Pleroma.Search.QdrantSearch.OllamaClient do +defmodule Pleroma.Search.QdrantSearch.OpenAIClient do use Tesla alias Pleroma.Config.Getting, as: Config - plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :ollama_url])) + plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url])) plug(Tesla.Middleware.JSON) + + plug(Tesla.Middleware.Headers, [ + {"Authorization", + "Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"} + ]) end defmodule Pleroma.Search.QdrantSearch.QdrantClient do diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index 698894cdb..a2f9cc7ec 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -19,9 +19,12 @@ test "indexes a public post on creation, deletes from the index on deletion" do user = insert(:user) Tesla.Mock.mock(fn - %{method: :post, url: "https://ollama.url/api/embeddings"} -> - send(self(), "posted_to_ollama") - Tesla.Mock.json(%{embedding: [1, 2, 3]}) + %{method: :post, url: "https://openai.url/v1/embeddings"} -> + send(self(), "posted_to_openai") + + Tesla.Mock.json(%{ + data: [%{embedding: [1, 2, 3]}] + }) %{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} -> send(self(), "posted_to_qdrant") @@ -42,8 +45,8 @@ test "indexes a public post on creation, deletes from the index on deletion" do [Pleroma.Search.QdrantSearch, key], nil -> %{ - ollama_model: "a_model", - ollama_url: "https://ollama.url", + openai_model: "a_model", + openai_url: "https://openai.url", qdrant_url: "https://qdrant.url" }[key] end) @@ -62,7 +65,7 @@ test "indexes a public post on creation, deletes from the index on deletion" do ) assert :ok = perform_job(SearchIndexingWorker, args) - assert_received("posted_to_ollama") + assert_received("posted_to_openai") assert_received("posted_to_qdrant") {:ok, _} = CommonAPI.delete(activity.id, user) From b9af017a4cf1025c7d8245fa4f1dbcb678ddd4b9 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 12:33:49 +0400 Subject: [PATCH 075/161] B FastembedServer: Switch to OpenAI api, support changing models --- python/fastembed-server.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/fastembed-server.py b/python/fastembed-server.py index fa3f7c82b..dd4a7a9c8 100644 --- a/python/fastembed-server.py +++ b/python/fastembed-server.py @@ -2,18 +2,20 @@ from fastembed import TextEmbedding from fastapi import FastAPI from pydantic import BaseModel -model = TextEmbedding("snowflake/snowflake-arctic-embed-xs") +models = {} app = FastAPI() class EmbeddingRequest(BaseModel): model: str - prompt: str + input: str -@app.post("/api/embeddings") +@app.post("/v1/embeddings") def embeddings(request: EmbeddingRequest): - embeddings = next(model.embed(request.prompt)).tolist() - return {"embedding": embeddings} + model = models.get(request.model) or TextEmbedding(request.model) + models[request.model] = model + embeddings = next(model.embed(request.input)).tolist() + return {"data": [{"embedding": embeddings}]} if __name__ == "__main__": import uvicorn From c139a9f38c06ab4485b98b56b9ad4cce4d57be12 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 12:39:54 +0400 Subject: [PATCH 076/161] B Config: Set default Qdrant embedder to our fastembed-api server --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index dd0150c66..c3b20947d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -919,7 +919,7 @@ qdrant_url: "http://127.0.0.1:6333/", qdrant_api_key: "", openai_url: "http://127.0.0.1:11345", - openai_model: "snowflake", + openai_model: "snowflake/snowflake-arctic-embed-xs", openai_api_key: "", qdrant_index_configuration: %{ vectors: %{size: 384, distance: "Cosine"} From e142ea400a9ed3595f8d432edd90ea26fc7d2eb5 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 12:42:08 +0400 Subject: [PATCH 077/161] Docs: Switch docs from Ollama to OpenAI. --- docs/configuration/search.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 682d1e52a..388f5acd1 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -12,9 +12,9 @@ While it has no external dependencies, it has problems with performance and rele ## QdrantSearch -This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings, for now only the [Ollama](Ollama) api is supported. +This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`. -The default settings will support a setup where both Ollama and Qdrant run on the same system as pleroma. The embedding model used by Ollama will need to be pulled first (e.g. `ollama pull snowflake-arctic-embed:xs`) for the embedding to work. +The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. ## Meilisearch From dd48810186e3b4ee14e1d3727f37bd470d0711a4 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 12:47:08 +0400 Subject: [PATCH 078/161] B FastembedAPI: Move to more appropriate folder --- {python => supplemental/search/fastembed-api}/Dockerfile | 0 {python => supplemental/search/fastembed-api}/compose.yml | 0 {python => supplemental/search/fastembed-api}/fastembed-server.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {python => supplemental/search/fastembed-api}/Dockerfile (100%) rename {python => supplemental/search/fastembed-api}/compose.yml (100%) rename {python => supplemental/search/fastembed-api}/fastembed-server.py (100%) diff --git a/python/Dockerfile b/supplemental/search/fastembed-api/Dockerfile similarity index 100% rename from python/Dockerfile rename to supplemental/search/fastembed-api/Dockerfile diff --git a/python/compose.yml b/supplemental/search/fastembed-api/compose.yml similarity index 100% rename from python/compose.yml rename to supplemental/search/fastembed-api/compose.yml diff --git a/python/fastembed-server.py b/supplemental/search/fastembed-api/fastembed-server.py similarity index 100% rename from python/fastembed-server.py rename to supplemental/search/fastembed-api/fastembed-server.py From 8329ad521419119f89e3e2577269475190cfe921 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 12:59:03 +0400 Subject: [PATCH 079/161] B FastembedAPI: Add requirements.txt --- supplemental/search/fastembed-api/Dockerfile | 3 ++- supplemental/search/fastembed-api/requirements.txt | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 supplemental/search/fastembed-api/requirements.txt diff --git a/supplemental/search/fastembed-api/Dockerfile b/supplemental/search/fastembed-api/Dockerfile index f83c1c1b3..c1e0ef51f 100644 --- a/supplemental/search/fastembed-api/Dockerfile +++ b/supplemental/search/fastembed-api/Dockerfile @@ -2,7 +2,8 @@ FROM python:3.9 WORKDIR /code COPY fastembed-server.py /workdir/fastembed-server.py +COPY requirements.txt /workdir/requirements.txt -RUN pip install --no-cache-dir --upgrade fastembed fastapi uvicorn +RUN pip install -r /workdir/requirements.txt CMD ["python", "/workdir/fastembed-server.py"] diff --git a/supplemental/search/fastembed-api/requirements.txt b/supplemental/search/fastembed-api/requirements.txt new file mode 100644 index 000000000..db67a8402 --- /dev/null +++ b/supplemental/search/fastembed-api/requirements.txt @@ -0,0 +1,4 @@ +fastapi==0.111.0 +fastembed==0.2.7 +pydantic==1.10.15 +uvicorn==0.29.0 From 23881842ae33a294e344cef0cc2f1385ea6819f9 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 13:04:27 +0400 Subject: [PATCH 080/161] B FastembedAPI: Add readme --- supplemental/search/fastembed-api/README.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 supplemental/search/fastembed-api/README.md diff --git a/supplemental/search/fastembed-api/README.md b/supplemental/search/fastembed-api/README.md new file mode 100644 index 000000000..63a037207 --- /dev/null +++ b/supplemental/search/fastembed-api/README.md @@ -0,0 +1,6 @@ +# About +This is a minimal implementation of the [OpenAI Embeddings API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings) meant to be used with the QdrantSearch backend. + +# Usage + +The easiest way to run it is to just use docker compose with `docker compose up`. This starts the server on the default configured port. Different models can be used, for a full list of supported models, check the [fastembed documentation](https://qdrant.github.io/fastembed/examples/Supported_Models/). The first time a model is requested it will be downloaded, which can take a few seconds. From 6a3a0cc0f5995185428c92f3c53e9c8524ea6856 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 13:20:37 +0400 Subject: [PATCH 081/161] Docs: Write docs for the QdrantSearch --- docs/configuration/search.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 388f5acd1..6598e533f 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -14,7 +14,25 @@ While it has no external dependencies, it has problems with performance and rele This uses the vector search engine [Qdrant](https://qdrant.tech) to search the posts in a vector space. This needs a way to generate embeddings and uses the [OpenAI API](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is implemented by several project besides OpenAI itself, including the python-based fastembed-server found in `supplemental/search/fastembed-api`. -The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. +The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. To use it, set the search provider and run the fastembed server, see the README in `supplemental/search/fastembed-api`: + +https://qdrant.github.io/fastembed/examples/Supported_Models/ + +> config :pleroma, Pleroma.Search, module: Pleroma.Search.QdrantSearch + +You will also need to create the Qdrant index once by running `mix pleroma.search.indexer create_index`. Running `mix pleroma.search.indexer index` will retroactively index the last 100_000 activities. + +### Indexing and model options + +To see the available configuration options, check out the QdrantSearch section in `config/config.exs`. + +The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/). + +Different embedding models will need different vector size settings. You can see a list of the models supported by the fastembed server [here](https://qdrant.github.io/fastembed/examples/Supported_Models), including their vector dimensions. These vector dimensions need to be set in the `qdrant_index_configuration`. + +E.g, If you want to use `sentence-transformers/all-MiniLM-L6-v2` as a model, you will not need to adjust things, because it and `snowflake-arctic-embed-xs` are both 384 dimensional models. If you want to use `snowflake/snowflake-arctic-embed-l`, you will need to adjust the `size` parameter in the `qdrant_index_configuration` to 1024, as it has a dimension of 1024. + +When using a different model, you will need do drop the index and recreate it (`mix pleroma.search.indexer drop_index` and `mix pleroma.search.indexer create_index`), as the different embeddings are not compatible with each other. ## Meilisearch From 6ec306d0684f3c5c05d768a3c431008925f21f15 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 13:24:24 +0400 Subject: [PATCH 082/161] Docs: Add more information about index memory consumption. --- docs/configuration/search.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index 6598e533f..ed85acd2a 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -26,7 +26,7 @@ You will also need to create the Qdrant index once by running `mix pleroma.searc To see the available configuration options, check out the QdrantSearch section in `config/config.exs`. -The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/). +The default indexing option work for the default model (`snowflake-arctic-embed-xs`). To optimize for a low memory footprint, adjust the index configuration as described in the [Qdrant docs](https://qdrant.tech/documentation/guides/optimize/). See also [this blog post](https://qdrant.tech/articles/memory-consumption/) that goes into detail. Different embedding models will need different vector size settings. You can see a list of the models supported by the fastembed server [here](https://qdrant.github.io/fastembed/examples/Supported_Models), including their vector dimensions. These vector dimensions need to be set in the `qdrant_index_configuration`. From dbaab6f54e306e5fb930ce1ed0699631c8aeaae1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 13:38:31 +0400 Subject: [PATCH 083/161] Docs: Mention running the Qdrant server --- docs/configuration/search.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/configuration/search.md b/docs/configuration/search.md index ed85acd2a..d34f84d4f 100644 --- a/docs/configuration/search.md +++ b/docs/configuration/search.md @@ -16,10 +16,10 @@ This uses the vector search engine [Qdrant](https://qdrant.tech) to search the p The default settings will support a setup where both the fastembed server and Qdrant run on the same system as pleroma. To use it, set the search provider and run the fastembed server, see the README in `supplemental/search/fastembed-api`: -https://qdrant.github.io/fastembed/examples/Supported_Models/ - > config :pleroma, Pleroma.Search, module: Pleroma.Search.QdrantSearch +Then, start the Qdrant server, see [here](https://qdrant.tech/documentation/quick-start/) for instructions. + You will also need to create the Qdrant index once by running `mix pleroma.search.indexer create_index`. Running `mix pleroma.search.indexer index` will retroactively index the last 100_000 activities. ### Indexing and model options From 1b4f1db9b2990f725a06f0dff41980c51853c5e9 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 19 May 2024 14:41:05 +0400 Subject: [PATCH 084/161] QdrantSearch: Support pagination. --- lib/pleroma/search/qdrant_search.ex | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 5ae04be78..283c43075 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -54,10 +54,11 @@ defp build_index_payload(activity, embedding) do } end - defp build_search_payload(embedding) do + defp build_search_payload(embedding, options) do %{ vector: embedding, - limit: 20 + limit: options[:limit] || 20, + offset: options[:offset] || 0 } end @@ -96,12 +97,15 @@ def remove_from_index(object) do end @impl true - def search(_user, query, _options) do + def search(_user, query, options) do query = "Represent this sentence for searching relevant passages: #{query}" with {:ok, embedding} <- get_embedding(query), {:ok, %{body: %{"result" => result}}} <- - QdrantClient.post("/collections/posts/points/search", build_search_payload(embedding)) do + QdrantClient.post( + "/collections/posts/points/search", + build_search_payload(embedding, options) + ) do ids = Enum.map(result, fn %{"id" => id} -> Ecto.UUID.dump!(id) From 226874c9d603be72699d5aa5434616efffe3f239 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 20 May 2024 13:12:12 +0400 Subject: [PATCH 085/161] CI: Add new builders for base images --- .gitlab-ci.yml | 6 +++--- ci/elixir-1.13/Dockerfile | 8 ++++++++ ci/elixir-1.13/build_and_push.sh | 1 + ci/elixir-1.15-otp25/build_and_push.sh | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 ci/elixir-1.13/Dockerfile create mode 100755 ci/elixir-1.13/build_and_push.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2e321c978..eba769af8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: git.pleroma.social:5050/pleroma/pleroma/ci-base +image: git.pleroma.social:5050/pleroma/pleroma/ci-base:1.13.4-otp24 variables: &global_variables # Only used for the release @@ -72,7 +72,7 @@ check-changelog: tags: - amd64 -build-1.12.3: +build-1.13.4: extends: - .build_changes_policy - .using-ci-base @@ -85,7 +85,7 @@ build-1.15.7-otp-25: - .build_changes_policy - .using-ci-base stage: build - image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15 + image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 allow_failure: true script: - mix compile --force diff --git a/ci/elixir-1.13/Dockerfile b/ci/elixir-1.13/Dockerfile new file mode 100644 index 000000000..b8bceb3d9 --- /dev/null +++ b/ci/elixir-1.13/Dockerfile @@ -0,0 +1,8 @@ +FROM elixir:1.13.4-otp-24 + +# Single RUN statement, otherwise intermediate images are created +# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run +RUN apt-get update &&\ + apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\ + mix local.hex --force &&\ + mix local.rebar --force diff --git a/ci/elixir-1.13/build_and_push.sh b/ci/elixir-1.13/build_and_push.sh new file mode 100755 index 000000000..53af4245f --- /dev/null +++ b/ci/elixir-1.13/build_and_push.sh @@ -0,0 +1 @@ +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13-otp24 --push . diff --git a/ci/elixir-1.15-otp25/build_and_push.sh b/ci/elixir-1.15-otp25/build_and_push.sh index 06fe74f34..a28e0d33c 100755 --- a/ci/elixir-1.15-otp25/build_and_push.sh +++ b/ci/elixir-1.15-otp25/build_and_push.sh @@ -1 +1 @@ -docker buildx build --platform linux/amd64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push . +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push . From f8411a351de07f14fdc9c9eca30109feaadf6f93 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 20 May 2024 13:30:31 +0400 Subject: [PATCH 086/161] CI: Specify version fully in base image tag --- ci/elixir-1.13/build_and_push.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/elixir-1.13/build_and_push.sh b/ci/elixir-1.13/build_and_push.sh index 53af4245f..d848344a3 100755 --- a/ci/elixir-1.13/build_and_push.sh +++ b/ci/elixir-1.13/build_and_push.sh @@ -1 +1 @@ -docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13-otp24 --push . +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp24 --push . From f5c029524752e1820ea29f6557647823ae89ecf1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 20 May 2024 13:32:25 +0400 Subject: [PATCH 087/161] CI: Specify correct image name. --- .gitlab-ci.yml | 2 +- ci/elixir-1.13/build_and_push.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eba769af8..21d7b2242 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: git.pleroma.social:5050/pleroma/pleroma/ci-base:1.13.4-otp24 +image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 variables: &global_variables # Only used for the release diff --git a/ci/elixir-1.13/build_and_push.sh b/ci/elixir-1.13/build_and_push.sh index d848344a3..64e1856db 100755 --- a/ci/elixir-1.13/build_and_push.sh +++ b/ci/elixir-1.13/build_and_push.sh @@ -1 +1 @@ -docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp24 --push . +docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 --push . From 36fa0debfe66d3b706eeaa09227edd8b82c70aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 20 May 2024 23:25:50 +0200 Subject: [PATCH 088/161] Fix `get_notified_from` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/notification.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 55c47e966..942aa7198 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -526,7 +526,7 @@ def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, lo Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) end - def get_notified_from_activity(_, _local_only), do: {[], []} + def get_notified_from_activity(_, _local_only), do: [] def get_notified_subscribers_from_activity(activity, local_only \\ true) @@ -544,7 +544,7 @@ def get_notified_subscribers_from_activity( Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) end - def get_notified_subscribers_from_activity(_, _), do: {[], []} + def get_notified_subscribers_from_activity(_, _), do: [] # For some activities, only notify the author of the object def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}}) From d1b053f3ba4170021c511b0d06a41405d3ab07d3 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 12:57:30 +0400 Subject: [PATCH 089/161] Webfinger: Add test showing wrong webfinger behavior --- .../webfinger/graf-imposter-webfinger.json | 41 +++++++++++++++++++ test/pleroma/web/web_finger_test.exs | 15 +++++++ 2 files changed, 56 insertions(+) create mode 100644 test/fixtures/webfinger/graf-imposter-webfinger.json diff --git a/test/fixtures/webfinger/graf-imposter-webfinger.json b/test/fixtures/webfinger/graf-imposter-webfinger.json new file mode 100644 index 000000000..e7010f606 --- /dev/null +++ b/test/fixtures/webfinger/graf-imposter-webfinger.json @@ -0,0 +1,41 @@ +{ + "subject": "acct:graf@poa.st", + "aliases": [ + "https://fba.ryona.agenc/webfingertest" + ], + "links": [ + { + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html", + "href": "https://fba.ryona.agenc/webfingertest" + }, + { + "rel": "self", + "type": "application/activity+json", + "href": "https://fba.ryona.agenc/webfingertest" + }, + { + "rel": "http://ostatus.org/schema/1.0/subscribe", + "template": "https://fba.ryona.agenc/contact/follow?url={uri}" + }, + { + "rel": "http://schemas.google.com/g/2010#updates-from", + "type": "application/atom+xml", + "href": "" + }, + { + "rel": "salmon", + "href": "https://fba.ryona.agenc/salmon/friendica" + }, + { + "rel": "http://microformats.org/profile/hcard", + "type": "text/html", + "href": "https://fba.ryona.agenc/hcard/friendica" + }, + { + "rel": "http://joindiaspora.com/seed_location", + "type": "text/html", + "href": "https://fba.ryona.agenc" + } + ] +} diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs index be5e08776..6530fbc56 100644 --- a/test/pleroma/web/web_finger_test.exs +++ b/test/pleroma/web/web_finger_test.exs @@ -204,4 +204,19 @@ test "refuses to process XML remote entities" do assert :error = WebFinger.finger("pekorino@pawoo.net") end end + + test "prevents forgeries" do + Tesla.Mock.mock(fn + %{url: "https://fba.ryona.agency/.well-known/webfinger?resource=acct:graf@fba.ryona.agency"} -> + fake_webfinger = + File.read!("test/fixtures/webfinger/graf-imposter-webfinger.json") |> Jason.decode!() + + Tesla.Mock.json(fake_webfinger) + + %{url: "https://fba.ryona.agency/.well-known/host-meta"} -> + {:ok, %Tesla.Env{status: 404}} + end) + + refute {:ok, _} = WebFinger.finger("graf@fba.ryona.agency") + end end From b15f8b06425edbfc3a7cef2a55c609b12ee14377 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 23 Aug 2023 13:10:19 -0500 Subject: [PATCH 090/161] Prevent webfinger spoofing --- lib/pleroma/web/web_finger.ex | 16 ++++++++ .../tesla_mock/gleasonator.com_host_meta | 4 ++ test/fixtures/tesla_mock/webfinger_spoof.json | 28 ++++++++++++++ test/pleroma/web/web_finger_test.exs | 38 +++++++++++-------- 4 files changed, 71 insertions(+), 15 deletions(-) create mode 100644 test/fixtures/tesla_mock/gleasonator.com_host_meta create mode 100644 test/fixtures/tesla_mock/webfinger_spoof.json diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 26fb8af84..a84a4351b 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -216,10 +216,26 @@ def finger(account) do _ -> {:error, {:content_type, nil}} end + |> case do + {:ok, data} -> validate_webfinger(address, data) + error -> error + end else error -> Logger.debug("Couldn't finger #{account}: #{inspect(error)}") error end end + + defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do + with %URI{host: request_host} <- URI.parse(url), + [_name, acct_host] <- String.split(acct, "@"), + {_, true} <- {:hosts_match, acct_host == request_host} do + {:ok, data} + else + _ -> {:error, {:webfinger_invalid, url, data}} + end + end + + defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}} end diff --git a/test/fixtures/tesla_mock/gleasonator.com_host_meta b/test/fixtures/tesla_mock/gleasonator.com_host_meta new file mode 100644 index 000000000..c1a432519 --- /dev/null +++ b/test/fixtures/tesla_mock/gleasonator.com_host_meta @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/test/fixtures/tesla_mock/webfinger_spoof.json b/test/fixtures/tesla_mock/webfinger_spoof.json new file mode 100644 index 000000000..7c2a11f69 --- /dev/null +++ b/test/fixtures/tesla_mock/webfinger_spoof.json @@ -0,0 +1,28 @@ +{ + "aliases": [ + "https://gleasonator.com/users/alex", + "https://mitra.social/users/alex" + ], + "links": [ + { + "href": "https://gleasonator.com/users/alex", + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html" + }, + { + "href": "https://gleasonator.com/users/alex", + "rel": "self", + "type": "application/activity+json" + }, + { + "href": "https://gleasonator.com/users/alex", + "rel": "self", + "type": "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"" + }, + { + "rel": "http://ostatus.org/schema/1.0/subscribe", + "template": "https://gleasonator.com/ostatus_subscribe?acct={uri}" + } + ], + "subject": "acct:trump@whitehouse.gov" +} diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs index 6530fbc56..84a8e19d5 100644 --- a/test/pleroma/web/web_finger_test.exs +++ b/test/pleroma/web/web_finger_test.exs @@ -76,15 +76,6 @@ test "returns the ActivityPub actor URI for an ActivityPub user" do {:ok, _data} = WebFinger.finger(user) end - test "returns the ActivityPub actor URI and subscribe address for an ActivityPub user with the ld+json mimetype" do - user = "kaniini@gerzilla.de" - - {:ok, data} = WebFinger.finger(user) - - assert data["ap_id"] == "https://gerzilla.de/channel/kaniini" - assert data["subscribe_address"] == "https://gerzilla.de/follow?f=&url={uri}" - end - test "it work for AP-only user" do user = "kpherox@mstdn.jp" @@ -99,12 +90,6 @@ test "it work for AP-only user" do assert data["subscribe_address"] == "https://mstdn.jp/authorize_interaction?acct={uri}" end - test "it works for friendica" do - user = "lain@squeet.me" - - {:ok, _data} = WebFinger.finger(user) - end - test "it gets the xrd endpoint" do {:ok, template} = WebFinger.find_lrdd_template("social.heldscal.la") @@ -203,6 +188,29 @@ test "refuses to process XML remote entities" do assert :error = WebFinger.finger("pekorino@pawoo.net") end + + test "prevents spoofing" do + Tesla.Mock.mock(fn + %{ + url: "https://gleasonator.com/.well-known/webfinger?resource=acct:alex@gleasonator.com" + } -> + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/webfinger_spoof.json"), + headers: [{"content-type", "application/jrd+json"}] + }} + + %{url: "https://gleasonator.com/.well-known/host-meta"} -> + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/gleasonator.com_host_meta") + }} + end) + + {:error, _data} = WebFinger.finger("alex@gleasonator.com") + end end test "prevents forgeries" do From 206ea92837f8016d66a2b87f7f7338d814735a92 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 12:59:10 +0400 Subject: [PATCH 091/161] Webfinger: Fix test --- test/pleroma/web/web_finger_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs index 84a8e19d5..8a550a6ba 100644 --- a/test/pleroma/web/web_finger_test.exs +++ b/test/pleroma/web/web_finger_test.exs @@ -213,6 +213,7 @@ test "prevents spoofing" do end end + @tag capture_log: true test "prevents forgeries" do Tesla.Mock.mock(fn %{url: "https://fba.ryona.agency/.well-known/webfinger?resource=acct:graf@fba.ryona.agency"} -> @@ -225,6 +226,6 @@ test "prevents forgeries" do {:ok, %Tesla.Env{status: 404}} end) - refute {:ok, _} = WebFinger.finger("graf@fba.ryona.agency") + assert {:error, _} = WebFinger.finger("graf@fba.ryona.agency") end end From 4491e8c9a3e2cdeb1b8e9cb98015dc1d0435c65c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 13:01:23 +0400 Subject: [PATCH 092/161] Add changelog --- changelog.d/fix-webfinger-spoofing.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/fix-webfinger-spoofing.fix diff --git a/changelog.d/fix-webfinger-spoofing.fix b/changelog.d/fix-webfinger-spoofing.fix new file mode 100644 index 000000000..7b3c9490a --- /dev/null +++ b/changelog.d/fix-webfinger-spoofing.fix @@ -0,0 +1 @@ +Fix webfinger spoofing. From 91c93ce3cd62a916c7d367979473f94e36cf1873 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 13:14:59 +0400 Subject: [PATCH 093/161] Changelog: Adjust changelog type --- ...fix-webfinger-spoofing.fix => fix-webfinger-spoofing.security} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{fix-webfinger-spoofing.fix => fix-webfinger-spoofing.security} (100%) diff --git a/changelog.d/fix-webfinger-spoofing.fix b/changelog.d/fix-webfinger-spoofing.security similarity index 100% rename from changelog.d/fix-webfinger-spoofing.fix rename to changelog.d/fix-webfinger-spoofing.security From 84bb854056e406d5235dd442c28127891a8a8a86 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 15:12:29 +0400 Subject: [PATCH 094/161] Webfinger: Allow managing account for subdomain --- lib/pleroma/web/web_finger.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index a84a4351b..e149d9247 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -230,7 +230,7 @@ def finger(account) do defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do with %URI{host: request_host} <- URI.parse(url), [_name, acct_host] <- String.split(acct, "@"), - {_, true} <- {:hosts_match, acct_host == request_host} do + {_, true} <- {:hosts_match_or_subdomain, String.ends_with?(request_host, acct_host)} do {:ok, data} else _ -> {:error, {:webfinger_invalid, url, data}} From 29b968ce2006de47d8f1dbc161756e35ba5944a1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 12:57:30 +0400 Subject: [PATCH 095/161] Webfinger: Add test showing wrong webfinger behavior --- .../webfinger/graf-imposter-webfinger.json | 41 +++++++++++++++++++ test/pleroma/web/web_finger_test.exs | 15 +++++++ 2 files changed, 56 insertions(+) create mode 100644 test/fixtures/webfinger/graf-imposter-webfinger.json diff --git a/test/fixtures/webfinger/graf-imposter-webfinger.json b/test/fixtures/webfinger/graf-imposter-webfinger.json new file mode 100644 index 000000000..e7010f606 --- /dev/null +++ b/test/fixtures/webfinger/graf-imposter-webfinger.json @@ -0,0 +1,41 @@ +{ + "subject": "acct:graf@poa.st", + "aliases": [ + "https://fba.ryona.agenc/webfingertest" + ], + "links": [ + { + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html", + "href": "https://fba.ryona.agenc/webfingertest" + }, + { + "rel": "self", + "type": "application/activity+json", + "href": "https://fba.ryona.agenc/webfingertest" + }, + { + "rel": "http://ostatus.org/schema/1.0/subscribe", + "template": "https://fba.ryona.agenc/contact/follow?url={uri}" + }, + { + "rel": "http://schemas.google.com/g/2010#updates-from", + "type": "application/atom+xml", + "href": "" + }, + { + "rel": "salmon", + "href": "https://fba.ryona.agenc/salmon/friendica" + }, + { + "rel": "http://microformats.org/profile/hcard", + "type": "text/html", + "href": "https://fba.ryona.agenc/hcard/friendica" + }, + { + "rel": "http://joindiaspora.com/seed_location", + "type": "text/html", + "href": "https://fba.ryona.agenc" + } + ] +} diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs index be5e08776..6530fbc56 100644 --- a/test/pleroma/web/web_finger_test.exs +++ b/test/pleroma/web/web_finger_test.exs @@ -204,4 +204,19 @@ test "refuses to process XML remote entities" do assert :error = WebFinger.finger("pekorino@pawoo.net") end end + + test "prevents forgeries" do + Tesla.Mock.mock(fn + %{url: "https://fba.ryona.agency/.well-known/webfinger?resource=acct:graf@fba.ryona.agency"} -> + fake_webfinger = + File.read!("test/fixtures/webfinger/graf-imposter-webfinger.json") |> Jason.decode!() + + Tesla.Mock.json(fake_webfinger) + + %{url: "https://fba.ryona.agency/.well-known/host-meta"} -> + {:ok, %Tesla.Env{status: 404}} + end) + + refute {:ok, _} = WebFinger.finger("graf@fba.ryona.agency") + end end From 364f6e1620876dcfc1d228e2db17190d74b6f0ce Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 23 Aug 2023 13:10:19 -0500 Subject: [PATCH 096/161] Prevent webfinger spoofing --- lib/pleroma/web/web_finger.ex | 16 ++++++++ .../tesla_mock/gleasonator.com_host_meta | 4 ++ test/fixtures/tesla_mock/webfinger_spoof.json | 28 ++++++++++++++ test/pleroma/web/web_finger_test.exs | 38 +++++++++++-------- 4 files changed, 71 insertions(+), 15 deletions(-) create mode 100644 test/fixtures/tesla_mock/gleasonator.com_host_meta create mode 100644 test/fixtures/tesla_mock/webfinger_spoof.json diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index f95dc2458..0d6a686c3 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -216,10 +216,26 @@ def finger(account) do _ -> {:error, {:content_type, nil}} end + |> case do + {:ok, data} -> validate_webfinger(address, data) + error -> error + end else error -> Logger.debug("Couldn't finger #{account}: #{inspect(error)}") error end end + + defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do + with %URI{host: request_host} <- URI.parse(url), + [_name, acct_host] <- String.split(acct, "@"), + {_, true} <- {:hosts_match, acct_host == request_host} do + {:ok, data} + else + _ -> {:error, {:webfinger_invalid, url, data}} + end + end + + defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}} end diff --git a/test/fixtures/tesla_mock/gleasonator.com_host_meta b/test/fixtures/tesla_mock/gleasonator.com_host_meta new file mode 100644 index 000000000..c1a432519 --- /dev/null +++ b/test/fixtures/tesla_mock/gleasonator.com_host_meta @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/test/fixtures/tesla_mock/webfinger_spoof.json b/test/fixtures/tesla_mock/webfinger_spoof.json new file mode 100644 index 000000000..7c2a11f69 --- /dev/null +++ b/test/fixtures/tesla_mock/webfinger_spoof.json @@ -0,0 +1,28 @@ +{ + "aliases": [ + "https://gleasonator.com/users/alex", + "https://mitra.social/users/alex" + ], + "links": [ + { + "href": "https://gleasonator.com/users/alex", + "rel": "http://webfinger.net/rel/profile-page", + "type": "text/html" + }, + { + "href": "https://gleasonator.com/users/alex", + "rel": "self", + "type": "application/activity+json" + }, + { + "href": "https://gleasonator.com/users/alex", + "rel": "self", + "type": "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"" + }, + { + "rel": "http://ostatus.org/schema/1.0/subscribe", + "template": "https://gleasonator.com/ostatus_subscribe?acct={uri}" + } + ], + "subject": "acct:trump@whitehouse.gov" +} diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs index 6530fbc56..84a8e19d5 100644 --- a/test/pleroma/web/web_finger_test.exs +++ b/test/pleroma/web/web_finger_test.exs @@ -76,15 +76,6 @@ test "returns the ActivityPub actor URI for an ActivityPub user" do {:ok, _data} = WebFinger.finger(user) end - test "returns the ActivityPub actor URI and subscribe address for an ActivityPub user with the ld+json mimetype" do - user = "kaniini@gerzilla.de" - - {:ok, data} = WebFinger.finger(user) - - assert data["ap_id"] == "https://gerzilla.de/channel/kaniini" - assert data["subscribe_address"] == "https://gerzilla.de/follow?f=&url={uri}" - end - test "it work for AP-only user" do user = "kpherox@mstdn.jp" @@ -99,12 +90,6 @@ test "it work for AP-only user" do assert data["subscribe_address"] == "https://mstdn.jp/authorize_interaction?acct={uri}" end - test "it works for friendica" do - user = "lain@squeet.me" - - {:ok, _data} = WebFinger.finger(user) - end - test "it gets the xrd endpoint" do {:ok, template} = WebFinger.find_lrdd_template("social.heldscal.la") @@ -203,6 +188,29 @@ test "refuses to process XML remote entities" do assert :error = WebFinger.finger("pekorino@pawoo.net") end + + test "prevents spoofing" do + Tesla.Mock.mock(fn + %{ + url: "https://gleasonator.com/.well-known/webfinger?resource=acct:alex@gleasonator.com" + } -> + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/webfinger_spoof.json"), + headers: [{"content-type", "application/jrd+json"}] + }} + + %{url: "https://gleasonator.com/.well-known/host-meta"} -> + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/gleasonator.com_host_meta") + }} + end) + + {:error, _data} = WebFinger.finger("alex@gleasonator.com") + end end test "prevents forgeries" do From eafcb7b4ec368038aafa440ea32abe417a805f41 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 12:59:10 +0400 Subject: [PATCH 097/161] Webfinger: Fix test --- test/pleroma/web/web_finger_test.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/web_finger_test.exs b/test/pleroma/web/web_finger_test.exs index 84a8e19d5..8a550a6ba 100644 --- a/test/pleroma/web/web_finger_test.exs +++ b/test/pleroma/web/web_finger_test.exs @@ -213,6 +213,7 @@ test "prevents spoofing" do end end + @tag capture_log: true test "prevents forgeries" do Tesla.Mock.mock(fn %{url: "https://fba.ryona.agency/.well-known/webfinger?resource=acct:graf@fba.ryona.agency"} -> @@ -225,6 +226,6 @@ test "prevents forgeries" do {:ok, %Tesla.Env{status: 404}} end) - refute {:ok, _} = WebFinger.finger("graf@fba.ryona.agency") + assert {:error, _} = WebFinger.finger("graf@fba.ryona.agency") end end From 275fdb26c1472d3109721590080dea863c769794 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 13:01:23 +0400 Subject: [PATCH 098/161] Add changelog --- changelog.d/fix-webfinger-spoofing.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/fix-webfinger-spoofing.fix diff --git a/changelog.d/fix-webfinger-spoofing.fix b/changelog.d/fix-webfinger-spoofing.fix new file mode 100644 index 000000000..7b3c9490a --- /dev/null +++ b/changelog.d/fix-webfinger-spoofing.fix @@ -0,0 +1 @@ +Fix webfinger spoofing. From 2212287b0047d356592da82b02170b25fa1a4011 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 13:14:59 +0400 Subject: [PATCH 099/161] Changelog: Adjust changelog type --- ...fix-webfinger-spoofing.fix => fix-webfinger-spoofing.security} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{fix-webfinger-spoofing.fix => fix-webfinger-spoofing.security} (100%) diff --git a/changelog.d/fix-webfinger-spoofing.fix b/changelog.d/fix-webfinger-spoofing.security similarity index 100% rename from changelog.d/fix-webfinger-spoofing.fix rename to changelog.d/fix-webfinger-spoofing.security From 20fa400082df4c504768190f1ecbd407c9a6376f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 15:12:29 +0400 Subject: [PATCH 100/161] Webfinger: Allow managing account for subdomain --- lib/pleroma/web/web_finger.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 0d6a686c3..668d7d576 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -230,7 +230,7 @@ def finger(account) do defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do with %URI{host: request_host} <- URI.parse(url), [_name, acct_host] <- String.split(acct, "@"), - {_, true} <- {:hosts_match, acct_host == request_host} do + {_, true} <- {:hosts_match_or_subdomain, String.ends_with?(request_host, acct_host)} do {:ok, data} else _ -> {:error, {:webfinger_invalid, url, data}} From 239c9c3f1ce60a95b389c2f4ee1e717f4907c381 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 17:40:20 +0400 Subject: [PATCH 101/161] Mix: Update version --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index c95c2a82f..d0ee061c8 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("2.6.2"), + version: version("2.6.3"), elixir: "~> 1.11", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix] ++ Mix.compilers(), From 7b4e6d4c16a246ef4ae958a1536b00320441b63e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 17:44:10 +0400 Subject: [PATCH 102/161] Collect changelog --- CHANGELOG.md | 5 +++++ changelog.d/fix-webfinger-spoofing.security | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/fix-webfinger-spoofing.security diff --git a/CHANGELOG.md b/CHANGELOG.md index 92e5e6134..75d2aa415 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## 2.6.3 + +### Security +- Fix webfinger spoofing. + ## 2.6.2 ### Security diff --git a/changelog.d/fix-webfinger-spoofing.security b/changelog.d/fix-webfinger-spoofing.security deleted file mode 100644 index 7b3c9490a..000000000 --- a/changelog.d/fix-webfinger-spoofing.security +++ /dev/null @@ -1 +0,0 @@ -Fix webfinger spoofing. From 1f2f7e044d1be1e56789ce01ce4e54dd86a74f36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 22 May 2024 15:52:10 +0200 Subject: [PATCH 103/161] Revert "Webfinger: Allow managing account for subdomain" This reverts commit 84bb854056e406d5235dd442c28127891a8a8a86. --- lib/pleroma/web/web_finger.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index e149d9247..a84a4351b 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -230,7 +230,7 @@ def finger(account) do defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do with %URI{host: request_host} <- URI.parse(url), [_name, acct_host] <- String.split(acct, "@"), - {_, true} <- {:hosts_match_or_subdomain, String.ends_with?(request_host, acct_host)} do + {_, true} <- {:hosts_match, acct_host == request_host} do {:ok, data} else _ -> {:error, {:webfinger_invalid, url, data}} From d0b18e338bfed05c6b2c4a8f5c63d865d9eb669c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 24 Aug 2023 00:37:39 +0200 Subject: [PATCH 104/161] Fix validate_webfinger when running a different domain for Webfinger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/application.ex | 3 ++- lib/pleroma/web/web_finger.ex | 30 ++++++++++++++++++++++-------- test/pleroma/user_test.exs | 4 ++-- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 75154f94c..649bb11c8 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -162,7 +162,8 @@ defp cachex_children do expiration: chat_message_id_idempotency_key_expiration(), limit: 500_000 ), - build_cachex("rel_me", limit: 2500) + build_cachex("rel_me", limit: 2500), + build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000) ] end diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index a84a4351b..e653b3338 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -155,7 +155,16 @@ def get_template_from_xml(body) do end end + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def find_lrdd_template(domain) do + @cachex.fetch!(:host_meta_cache, domain, fn _ -> + {:commit, fetch_lrdd_template(domain)} + end) + rescue + e -> {:error, "Cachex error: #{inspect(e)}"} + end + + defp fetch_lrdd_template(domain) do # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1 meta_url = "https://#{domain}/.well-known/host-meta" @@ -168,7 +177,7 @@ def find_lrdd_template(domain) do end end - defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do + defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do case find_lrdd_template(domain) do {:ok, template} -> String.replace(template, "{uri}", encoded_account) @@ -178,6 +187,11 @@ defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do end end + defp get_address_from_domain(domain, account) when is_binary(domain) do + encoded_account = URI.encode("acct:#{account}") + get_address_from_domain(domain, encoded_account) + end + defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain} @spec finger(String.t()) :: {:ok, map()} | {:error, any()} @@ -192,9 +206,7 @@ def finger(account) do URI.parse(account).host end - encoded_account = URI.encode("acct:#{account}") - - with address when is_binary(address) <- get_address_from_domain(domain, encoded_account), + with address when is_binary(address) <- get_address_from_domain(domain, account), {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <- HTTP.get( address, @@ -227,13 +239,15 @@ def finger(account) do end end - defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do - with %URI{host: request_host} <- URI.parse(url), - [_name, acct_host] <- String.split(acct, "@"), + defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do + with [_name, acct_host] <- String.split(acct, "@"), + {_, url} <- {:address, get_address_from_domain(acct_host, subject)}, + %URI{host: request_host} <- URI.parse(request_url), + %URI{host: acct_host} <- URI.parse(url), {_, true} <- {:hosts_match, acct_host == request_host} do {:ok, data} else - _ -> {:error, {:webfinger_invalid, url, data}} + _ -> {:error, {:webfinger_invalid, request_url, data}} end end diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 48391d871..7f1a8d893 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -877,7 +877,7 @@ test "gets an existing user by nickname starting with http" do setup do: clear_config([Pleroma.Web.WebFinger, :update_nickname_on_user_fetch], true) test "for mastodon" do - Tesla.Mock.mock(fn + Tesla.Mock.mock_global(fn %{url: "https://example.com/.well-known/host-meta"} -> %Tesla.Env{ status: 302, @@ -935,7 +935,7 @@ test "for mastodon" do end test "for pleroma" do - Tesla.Mock.mock(fn + Tesla.Mock.mock_global(fn %{url: "https://example.com/.well-known/host-meta"} -> %Tesla.Env{ status: 302, From 70cabbf6dc2f8440484f1e56d3aa2d27f65ee88f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 24 Aug 2023 01:09:00 +0200 Subject: [PATCH 105/161] Fix tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- test/pleroma/user_test.exs | 102 +--------------- .../web_finger/web_finger_controller_test.exs | 5 + test/support/http_request_mock.ex | 114 ++++++++++++++++++ 3 files changed, 125 insertions(+), 96 deletions(-) diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 7f1a8d893..5b7a65658 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -877,109 +877,19 @@ test "gets an existing user by nickname starting with http" do setup do: clear_config([Pleroma.Web.WebFinger, :update_nickname_on_user_fetch], true) test "for mastodon" do - Tesla.Mock.mock_global(fn - %{url: "https://example.com/.well-known/host-meta"} -> - %Tesla.Env{ - status: 302, - headers: [{"location", "https://sub.example.com/.well-known/host-meta"}] - } - - %{url: "https://sub.example.com/.well-known/host-meta"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/masto-host-meta.xml" - |> File.read!() - |> String.replace("{{domain}}", "sub.example.com") - } - - %{url: "https://sub.example.com/.well-known/webfinger?resource=acct:a@example.com"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/masto-webfinger.json" - |> File.read!() - |> String.replace("{{nickname}}", "a") - |> String.replace("{{domain}}", "example.com") - |> String.replace("{{subdomain}}", "sub.example.com"), - headers: [{"content-type", "application/jrd+json"}] - } - - %{url: "https://sub.example.com/users/a"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/masto-user.json" - |> File.read!() - |> String.replace("{{nickname}}", "a") - |> String.replace("{{domain}}", "sub.example.com"), - headers: [{"content-type", "application/activity+json"}] - } - - %{url: "https://sub.example.com/users/a/collections/featured"} -> - %Tesla.Env{ - status: 200, - body: - File.read!("test/fixtures/users_mock/masto_featured.json") - |> String.replace("{{domain}}", "sub.example.com") - |> String.replace("{{nickname}}", "a"), - headers: [{"content-type", "application/activity+json"}] - } - end) - - ap_id = "a@example.com" + ap_id = "a@mastodon.example" {:ok, fetched_user} = User.get_or_fetch(ap_id) - assert fetched_user.ap_id == "https://sub.example.com/users/a" - assert fetched_user.nickname == "a@example.com" + assert fetched_user.ap_id == "https://sub.mastodon.example/users/a" + assert fetched_user.nickname == "a@mastodon.example" end test "for pleroma" do - Tesla.Mock.mock_global(fn - %{url: "https://example.com/.well-known/host-meta"} -> - %Tesla.Env{ - status: 302, - headers: [{"location", "https://sub.example.com/.well-known/host-meta"}] - } - - %{url: "https://sub.example.com/.well-known/host-meta"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/pleroma-host-meta.xml" - |> File.read!() - |> String.replace("{{domain}}", "sub.example.com") - } - - %{url: "https://sub.example.com/.well-known/webfinger?resource=acct:a@example.com"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/pleroma-webfinger.json" - |> File.read!() - |> String.replace("{{nickname}}", "a") - |> String.replace("{{domain}}", "example.com") - |> String.replace("{{subdomain}}", "sub.example.com"), - headers: [{"content-type", "application/jrd+json"}] - } - - %{url: "https://sub.example.com/users/a"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/pleroma-user.json" - |> File.read!() - |> String.replace("{{nickname}}", "a") - |> String.replace("{{domain}}", "sub.example.com"), - headers: [{"content-type", "application/activity+json"}] - } - end) - - ap_id = "a@example.com" + ap_id = "a@pleroma.example" {:ok, fetched_user} = User.get_or_fetch(ap_id) - assert fetched_user.ap_id == "https://sub.example.com/users/a" - assert fetched_user.nickname == "a@example.com" + assert fetched_user.ap_id == "https://sub.pleroma.example/users/a" + assert fetched_user.nickname == "a@pleroma.example" end end diff --git a/test/pleroma/web/web_finger/web_finger_controller_test.exs b/test/pleroma/web/web_finger/web_finger_controller_test.exs index 80e072163..f501c6e44 100644 --- a/test/pleroma/web/web_finger/web_finger_controller_test.exs +++ b/test/pleroma/web/web_finger/web_finger_controller_test.exs @@ -56,6 +56,11 @@ test "Webfinger JRD" do end test "reach user on tld, while pleroma is running on subdomain" do + Pleroma.Web.Endpoint.config_change( + [{Pleroma.Web.Endpoint, url: [host: "sub.example.com"]}], + [] + ) + clear_config([Pleroma.Web.Endpoint, :url, :host], "sub.example.com") clear_config([Pleroma.Web.WebFinger, :domain], "example.com") diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index f656c9412..20e410424 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1521,6 +1521,120 @@ def get("https://friends.grishka.me/users/1", _, _, _) do }} end + def get("https://mastodon.example/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 302, + headers: [{"location", "https://sub.mastodon.example/.well-known/host-meta"}] + }} + end + + def get("https://sub.mastodon.example/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/masto-host-meta.xml" + |> File.read!() + |> String.replace("{{domain}}", "sub.mastodon.example") + }} + end + + def get( + "https://sub.mastodon.example/.well-known/webfinger?resource=acct:a@mastodon.example", + _, + _, + _ + ) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/masto-webfinger.json" + |> File.read!() + |> String.replace("{{nickname}}", "a") + |> String.replace("{{domain}}", "mastodon.example") + |> String.replace("{{subdomain}}", "sub.mastodon.example"), + headers: [{"content-type", "application/jrd+json"}] + }} + end + + def get("https://sub.mastodon.example/users/a", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/masto-user.json" + |> File.read!() + |> String.replace("{{nickname}}", "a") + |> String.replace("{{domain}}", "sub.mastodon.example"), + headers: [{"content-type", "application/activity+json"}] + }} + end + + def get("https://sub.mastodon.example/users/a/collections/featured", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + File.read!("test/fixtures/users_mock/masto_featured.json") + |> String.replace("{{domain}}", "sub.mastodon.example") + |> String.replace("{{nickname}}", "a"), + headers: [{"content-type", "application/activity+json"}] + }} + end + + def get("https://pleroma.example/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 302, + headers: [{"location", "https://sub.pleroma.example/.well-known/host-meta"}] + }} + end + + def get("https://sub.pleroma.example/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/pleroma-host-meta.xml" + |> File.read!() + |> String.replace("{{domain}}", "sub.pleroma.example") + }} + end + + def get( + "https://sub.pleroma.example/.well-known/webfinger?resource=acct:a@pleroma.example", + _, + _, + _ + ) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/pleroma-webfinger.json" + |> File.read!() + |> String.replace("{{nickname}}", "a") + |> String.replace("{{domain}}", "pleroma.example") + |> String.replace("{{subdomain}}", "sub.pleroma.example"), + headers: [{"content-type", "application/jrd+json"}] + }} + end + + def get("https://sub.pleroma.example/users/a", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/pleroma-user.json" + |> File.read!() + |> String.replace("{{nickname}}", "a") + |> String.replace("{{domain}}", "sub.pleroma.example"), + headers: [{"content-type", "application/activity+json"}] + }} + end + def get(url, query, body, headers) do {:error, "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"} From d536d58080d68598ca282263159f9d565a048642 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 22 May 2024 15:53:32 +0200 Subject: [PATCH 106/161] changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/webfinger-validation.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/webfinger-validation.fix diff --git a/changelog.d/webfinger-validation.fix b/changelog.d/webfinger-validation.fix new file mode 100644 index 000000000..e64312666 --- /dev/null +++ b/changelog.d/webfinger-validation.fix @@ -0,0 +1 @@ +Fix validate_webfinger when running a different domain for Webfinger \ No newline at end of file From 5f1f574f01ea18170a228a8cb273e143d2f05ab4 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 18:45:34 +0400 Subject: [PATCH 107/161] WebFingerControllerTest: Restore host after test. --- test/pleroma/web/web_finger/web_finger_controller_test.exs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/pleroma/web/web_finger/web_finger_controller_test.exs b/test/pleroma/web/web_finger/web_finger_controller_test.exs index f501c6e44..80e072163 100644 --- a/test/pleroma/web/web_finger/web_finger_controller_test.exs +++ b/test/pleroma/web/web_finger/web_finger_controller_test.exs @@ -56,11 +56,6 @@ test "Webfinger JRD" do end test "reach user on tld, while pleroma is running on subdomain" do - Pleroma.Web.Endpoint.config_change( - [{Pleroma.Web.Endpoint, url: [host: "sub.example.com"]}], - [] - ) - clear_config([Pleroma.Web.Endpoint, :url, :host], "sub.example.com") clear_config([Pleroma.Web.WebFinger, :domain], "example.com") From 50ffbd980e8f9aee48788cea90b723c2dcca017d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 22 May 2024 15:52:10 +0200 Subject: [PATCH 108/161] Revert "Webfinger: Allow managing account for subdomain" This reverts commit 84bb854056e406d5235dd442c28127891a8a8a86. --- lib/pleroma/web/web_finger.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 668d7d576..0d6a686c3 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -230,7 +230,7 @@ def finger(account) do defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do with %URI{host: request_host} <- URI.parse(url), [_name, acct_host] <- String.split(acct, "@"), - {_, true} <- {:hosts_match_or_subdomain, String.ends_with?(request_host, acct_host)} do + {_, true} <- {:hosts_match, acct_host == request_host} do {:ok, data} else _ -> {:error, {:webfinger_invalid, url, data}} From b245a5c8c2a554b18f9e22c050abf59e41eda5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 24 Aug 2023 00:37:39 +0200 Subject: [PATCH 109/161] Fix validate_webfinger when running a different domain for Webfinger MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/application.ex | 3 ++- lib/pleroma/web/web_finger.ex | 30 ++++++++++++++++++++++-------- test/pleroma/user_test.exs | 4 ++-- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index e68a3c57e..385e3872d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -210,7 +210,8 @@ defp cachex_children do expiration: chat_message_id_idempotency_key_expiration(), limit: 500_000 ), - build_cachex("rel_me", limit: 2500) + build_cachex("rel_me", limit: 2500), + build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000) ] end diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 0d6a686c3..398742200 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -155,7 +155,16 @@ def get_template_from_xml(body) do end end + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def find_lrdd_template(domain) do + @cachex.fetch!(:host_meta_cache, domain, fn _ -> + {:commit, fetch_lrdd_template(domain)} + end) + rescue + e -> {:error, "Cachex error: #{inspect(e)}"} + end + + defp fetch_lrdd_template(domain) do # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1 meta_url = "https://#{domain}/.well-known/host-meta" @@ -168,7 +177,7 @@ def find_lrdd_template(domain) do end end - defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do + defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do case find_lrdd_template(domain) do {:ok, template} -> String.replace(template, "{uri}", encoded_account) @@ -178,6 +187,11 @@ defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do end end + defp get_address_from_domain(domain, account) when is_binary(domain) do + encoded_account = URI.encode("acct:#{account}") + get_address_from_domain(domain, encoded_account) + end + defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain} @spec finger(String.t()) :: {:ok, map()} | {:error, any()} @@ -192,9 +206,7 @@ def finger(account) do URI.parse(account).host end - encoded_account = URI.encode("acct:#{account}") - - with address when is_binary(address) <- get_address_from_domain(domain, encoded_account), + with address when is_binary(address) <- get_address_from_domain(domain, account), {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <- HTTP.get( address, @@ -227,13 +239,15 @@ def finger(account) do end end - defp validate_webfinger(url, %{"subject" => "acct:" <> acct} = data) do - with %URI{host: request_host} <- URI.parse(url), - [_name, acct_host] <- String.split(acct, "@"), + defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do + with [_name, acct_host] <- String.split(acct, "@"), + {_, url} <- {:address, get_address_from_domain(acct_host, subject)}, + %URI{host: request_host} <- URI.parse(request_url), + %URI{host: acct_host} <- URI.parse(url), {_, true} <- {:hosts_match, acct_host == request_host} do {:ok, data} else - _ -> {:error, {:webfinger_invalid, url, data}} + _ -> {:error, {:webfinger_invalid, request_url, data}} end end diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 7f60b959a..f64299370 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -872,7 +872,7 @@ test "gets an existing user by nickname starting with http" do setup do: clear_config([Pleroma.Web.WebFinger, :update_nickname_on_user_fetch], true) test "for mastodon" do - Tesla.Mock.mock(fn + Tesla.Mock.mock_global(fn %{url: "https://example.com/.well-known/host-meta"} -> %Tesla.Env{ status: 302, @@ -930,7 +930,7 @@ test "for mastodon" do end test "for pleroma" do - Tesla.Mock.mock(fn + Tesla.Mock.mock_global(fn %{url: "https://example.com/.well-known/host-meta"} -> %Tesla.Env{ status: 302, From 45b5e6ecd8e647026bbdcdb454d75e5e586f5bb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 24 Aug 2023 01:09:00 +0200 Subject: [PATCH 110/161] Fix tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- test/pleroma/user_test.exs | 102 +---------- .../web_finger/web_finger_controller_test.exs | 2 +- test/support/http_request_mock.ex | 171 ++++++++++++++++++ 3 files changed, 178 insertions(+), 97 deletions(-) diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index f64299370..b1ff52768 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -872,109 +872,19 @@ test "gets an existing user by nickname starting with http" do setup do: clear_config([Pleroma.Web.WebFinger, :update_nickname_on_user_fetch], true) test "for mastodon" do - Tesla.Mock.mock_global(fn - %{url: "https://example.com/.well-known/host-meta"} -> - %Tesla.Env{ - status: 302, - headers: [{"location", "https://sub.example.com/.well-known/host-meta"}] - } - - %{url: "https://sub.example.com/.well-known/host-meta"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/masto-host-meta.xml" - |> File.read!() - |> String.replace("{{domain}}", "sub.example.com") - } - - %{url: "https://sub.example.com/.well-known/webfinger?resource=acct:a@example.com"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/masto-webfinger.json" - |> File.read!() - |> String.replace("{{nickname}}", "a") - |> String.replace("{{domain}}", "example.com") - |> String.replace("{{subdomain}}", "sub.example.com"), - headers: [{"content-type", "application/jrd+json"}] - } - - %{url: "https://sub.example.com/users/a"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/masto-user.json" - |> File.read!() - |> String.replace("{{nickname}}", "a") - |> String.replace("{{domain}}", "sub.example.com"), - headers: [{"content-type", "application/activity+json"}] - } - - %{url: "https://sub.example.com/users/a/collections/featured"} -> - %Tesla.Env{ - status: 200, - body: - File.read!("test/fixtures/users_mock/masto_featured.json") - |> String.replace("{{domain}}", "sub.example.com") - |> String.replace("{{nickname}}", "a"), - headers: [{"content-type", "application/activity+json"}] - } - end) - - ap_id = "a@example.com" + ap_id = "a@mastodon.example" {:ok, fetched_user} = User.get_or_fetch(ap_id) - assert fetched_user.ap_id == "https://sub.example.com/users/a" - assert fetched_user.nickname == "a@example.com" + assert fetched_user.ap_id == "https://sub.mastodon.example/users/a" + assert fetched_user.nickname == "a@mastodon.example" end test "for pleroma" do - Tesla.Mock.mock_global(fn - %{url: "https://example.com/.well-known/host-meta"} -> - %Tesla.Env{ - status: 302, - headers: [{"location", "https://sub.example.com/.well-known/host-meta"}] - } - - %{url: "https://sub.example.com/.well-known/host-meta"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/pleroma-host-meta.xml" - |> File.read!() - |> String.replace("{{domain}}", "sub.example.com") - } - - %{url: "https://sub.example.com/.well-known/webfinger?resource=acct:a@example.com"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/pleroma-webfinger.json" - |> File.read!() - |> String.replace("{{nickname}}", "a") - |> String.replace("{{domain}}", "example.com") - |> String.replace("{{subdomain}}", "sub.example.com"), - headers: [{"content-type", "application/jrd+json"}] - } - - %{url: "https://sub.example.com/users/a"} -> - %Tesla.Env{ - status: 200, - body: - "test/fixtures/webfinger/pleroma-user.json" - |> File.read!() - |> String.replace("{{nickname}}", "a") - |> String.replace("{{domain}}", "sub.example.com"), - headers: [{"content-type", "application/activity+json"}] - } - end) - - ap_id = "a@example.com" + ap_id = "a@pleroma.example" {:ok, fetched_user} = User.get_or_fetch(ap_id) - assert fetched_user.ap_id == "https://sub.example.com/users/a" - assert fetched_user.nickname == "a@example.com" + assert fetched_user.ap_id == "https://sub.pleroma.example/users/a" + assert fetched_user.nickname == "a@pleroma.example" end end diff --git a/test/pleroma/web/web_finger/web_finger_controller_test.exs b/test/pleroma/web/web_finger/web_finger_controller_test.exs index 5e3ac26f9..e01cec5e4 100644 --- a/test/pleroma/web/web_finger/web_finger_controller_test.exs +++ b/test/pleroma/web/web_finger/web_finger_controller_test.exs @@ -48,7 +48,7 @@ test "Webfinger JRD" do ] end - test "reach user on tld, while pleroma is runned on subdomain" do + test "reach user on tld, while pleroma is running on subdomain" do Pleroma.Web.Endpoint.config_change( [{Pleroma.Web.Endpoint, url: [host: "sub.example.com"]}], [] diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 78a367024..82d8c38d7 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1464,6 +1464,177 @@ def get("https://misskey.io/notes/8vs6wxufd0", _, _, _) do }} end + def get("https://google.com/", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/google.html")}} + end + + def get("https://yahoo.com/", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/yahoo.html")}} + end + + def get("https://example.com/error", _, _, _), do: {:error, :overload} + + def get("https://example.com/ogp-missing-title", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/rich_media/ogp-missing-title.html") + }} + end + + def get("https://example.com/oembed", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.html")}} + end + + def get("https://example.com/oembed.json", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.json")}} + end + + def get("https://example.com/twitter-card", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/twitter_card.html")}} + end + + def get("https://example.com/non-ogp", _, _, _) do + {:ok, + %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/non_ogp_embed.html")}} + end + + def get("https://example.com/empty", _, _, _) do + {:ok, %Tesla.Env{status: 200, body: "hello"}} + end + + def get("https://friends.grishka.me/posts/54642", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/smithereen_non_anonymous_poll.json"), + headers: activitypub_object_headers() + }} + end + + def get("https://friends.grishka.me/users/1", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/smithereen_user.json"), + headers: activitypub_object_headers() + }} + end + + def get("https://mastodon.example/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 302, + headers: [{"location", "https://sub.mastodon.example/.well-known/host-meta"}] + }} + end + + def get("https://sub.mastodon.example/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/masto-host-meta.xml" + |> File.read!() + |> String.replace("{{domain}}", "sub.mastodon.example") + }} + end + + def get( + "https://sub.mastodon.example/.well-known/webfinger?resource=acct:a@mastodon.example", + _, + _, + _ + ) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/masto-webfinger.json" + |> File.read!() + |> String.replace("{{nickname}}", "a") + |> String.replace("{{domain}}", "mastodon.example") + |> String.replace("{{subdomain}}", "sub.mastodon.example"), + headers: [{"content-type", "application/jrd+json"}] + }} + end + + def get("https://sub.mastodon.example/users/a", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/masto-user.json" + |> File.read!() + |> String.replace("{{nickname}}", "a") + |> String.replace("{{domain}}", "sub.mastodon.example"), + headers: [{"content-type", "application/activity+json"}] + }} + end + + def get("https://sub.mastodon.example/users/a/collections/featured", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + File.read!("test/fixtures/users_mock/masto_featured.json") + |> String.replace("{{domain}}", "sub.mastodon.example") + |> String.replace("{{nickname}}", "a"), + headers: [{"content-type", "application/activity+json"}] + }} + end + + def get("https://pleroma.example/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 302, + headers: [{"location", "https://sub.pleroma.example/.well-known/host-meta"}] + }} + end + + def get("https://sub.pleroma.example/.well-known/host-meta", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/pleroma-host-meta.xml" + |> File.read!() + |> String.replace("{{domain}}", "sub.pleroma.example") + }} + end + + def get( + "https://sub.pleroma.example/.well-known/webfinger?resource=acct:a@pleroma.example", + _, + _, + _ + ) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/pleroma-webfinger.json" + |> File.read!() + |> String.replace("{{nickname}}", "a") + |> String.replace("{{domain}}", "pleroma.example") + |> String.replace("{{subdomain}}", "sub.pleroma.example"), + headers: [{"content-type", "application/jrd+json"}] + }} + end + + def get("https://sub.pleroma.example/users/a", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: + "test/fixtures/webfinger/pleroma-user.json" + |> File.read!() + |> String.replace("{{nickname}}", "a") + |> String.replace("{{domain}}", "sub.pleroma.example"), + headers: [{"content-type", "application/activity+json"}] + }} + end + def get(url, query, body, headers) do {:error, "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"} From c42527dc2efe6d25310e44cfec7396c51ced5cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 22 May 2024 15:53:32 +0200 Subject: [PATCH 111/161] changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/webfinger-validation.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/webfinger-validation.fix diff --git a/changelog.d/webfinger-validation.fix b/changelog.d/webfinger-validation.fix new file mode 100644 index 000000000..e64312666 --- /dev/null +++ b/changelog.d/webfinger-validation.fix @@ -0,0 +1 @@ +Fix validate_webfinger when running a different domain for Webfinger \ No newline at end of file From 53a3176d2414bf4af523f1d9d13fc082fd23ea43 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Wed, 22 May 2024 18:45:34 +0400 Subject: [PATCH 112/161] WebFingerControllerTest: Restore host after test. --- test/pleroma/web/web_finger/web_finger_controller_test.exs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/pleroma/web/web_finger/web_finger_controller_test.exs b/test/pleroma/web/web_finger/web_finger_controller_test.exs index e01cec5e4..cc7125ce4 100644 --- a/test/pleroma/web/web_finger/web_finger_controller_test.exs +++ b/test/pleroma/web/web_finger/web_finger_controller_test.exs @@ -49,11 +49,6 @@ test "Webfinger JRD" do end test "reach user on tld, while pleroma is running on subdomain" do - Pleroma.Web.Endpoint.config_change( - [{Pleroma.Web.Endpoint, url: [host: "sub.example.com"]}], - [] - ) - clear_config([Pleroma.Web.Endpoint, :url, :host], "sub.example.com") clear_config([Pleroma.Web.WebFinger, :domain], "example.com") From 818712f99f165011aaaad5fd82c40304004ace23 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 23 May 2024 00:35:38 +0200 Subject: [PATCH 113/161] pleroma_ctl: Use realpath(1) instead of readlink(1) From realpath(1) in POSIX 202x Draft 4.1: > If file does not name a symbolic link, readlink shall write a diagnostic > message to standard error and exit with non-zero status. Which also doesn't includes `-f`, in preference of `realpath`. --- changelog.d/realpath-over-readlink.fix | 1 + rel/files/bin/pleroma_ctl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/realpath-over-readlink.fix diff --git a/changelog.d/realpath-over-readlink.fix b/changelog.d/realpath-over-readlink.fix new file mode 100644 index 000000000..479561b95 --- /dev/null +++ b/changelog.d/realpath-over-readlink.fix @@ -0,0 +1 @@ +pleroma_ctl: Use realpath(1) instead of readlink(1) diff --git a/rel/files/bin/pleroma_ctl b/rel/files/bin/pleroma_ctl index 87c486514..6f0dba3a8 100755 --- a/rel/files/bin/pleroma_ctl +++ b/rel/files/bin/pleroma_ctl @@ -134,7 +134,7 @@ if [ -z "$1" ] || [ "$1" = "help" ]; then " else - SCRIPT=$(readlink -f "$0") + SCRIPT=$(realpath "$0") SCRIPTPATH=$(dirname "$SCRIPT") FULL_ARGS="$*" From 94e4f215896dc7976a54fd146daf3e286602925a Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 23 May 2024 14:38:30 +0400 Subject: [PATCH 114/161] QdrantSearch: Deal with actor restrictions --- lib/pleroma/search/qdrant_search.ex | 22 ++++- test/pleroma/search/qdrant_search_test.exs | 95 +++++++++++++++++++++- 2 files changed, 114 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 283c43075..9cb34ef71 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -43,23 +43,41 @@ def get_embedding(text) do end end + defp actor_from_activity(%{data: %{"actor" => actor}}) do + actor + end + + defp actor_from_activity(_), do: nil + defp build_index_payload(activity, embedding) do + actor = actor_from_activity(activity) + published_at = activity.data["published"] + %{ points: [ %{ id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(), - vector: embedding + vector: embedding, + payload: %{actor: actor, published_at: published_at} } ] } end defp build_search_payload(embedding, options) do - %{ + base = %{ vector: embedding, limit: options[:limit] || 20, offset: options[:offset] || 0 } + + if options[:actor] do + Map.put(base, :filter, %{ + must: [%{key: "actor", match: %{value: options[:actor].ap_id}}] + }) + else + base + end end @impl true diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index a2f9cc7ec..371074dcf 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -15,6 +15,94 @@ defmodule Pleroma.Search.QdrantSearchTest do alias Pleroma.Workers.SearchIndexingWorker describe "Qdrant search" do + test "searches for a term by encoding it and sending it to qdrant" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + Config + |> expect(:get, 3, fn + [Pleroma.Search, :module], nil -> + QdrantSearch + + [Pleroma.Search.QdrantSearch, key], nil -> + %{ + openai_model: "a_model", + openai_url: "https://openai.url", + qdrant_url: "https://qdrant.url" + }[key] + end) + + Tesla.Mock.mock(fn + %{url: "https://openai.url/v1/embeddings", method: :post} -> + Tesla.Mock.json(%{ + data: [%{embedding: [1, 2, 3]}] + }) + + %{url: "https://qdrant.url/collections/posts/points/search", method: :post, body: body} -> + data = Jason.decode!(body) + refute data["filter"] + + Tesla.Mock.json(%{ + result: [%{"id" => activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()}] + }) + end) + + results = QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{}) + + assert results == [activity] + end + + test "for a given actor, ask for only relevant matches" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "guys i just don't wanna leave the swamp", + visibility: "public" + }) + + Config + |> expect(:get, 3, fn + [Pleroma.Search, :module], nil -> + QdrantSearch + + [Pleroma.Search.QdrantSearch, key], nil -> + %{ + openai_model: "a_model", + openai_url: "https://openai.url", + qdrant_url: "https://qdrant.url" + }[key] + end) + + Tesla.Mock.mock(fn + %{url: "https://openai.url/v1/embeddings", method: :post} -> + Tesla.Mock.json(%{ + data: [%{embedding: [1, 2, 3]}] + }) + + %{url: "https://qdrant.url/collections/posts/points/search", method: :post, body: body} -> + data = Jason.decode!(body) + + assert data["filter"] == %{ + "must" => [%{"key" => "actor", "match" => %{"value" => user.ap_id}}] + } + + Tesla.Mock.json(%{ + result: [%{"id" => activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!()}] + }) + end) + + results = + QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{actor: user}) + + assert results == [activity] + end + test "indexes a public post on creation, deletes from the index on deletion" do user = insert(:user) @@ -29,7 +117,12 @@ test "indexes a public post on creation, deletes from the index on deletion" do %{method: :put, url: "https://qdrant.url/collections/posts/points", body: body} -> send(self(), "posted_to_qdrant") - assert match?(%{"points" => [%{"vector" => [1, 2, 3]}]}, Jason.decode!(body)) + data = Jason.decode!(body) + %{"points" => [%{"vector" => vector, "payload" => payload}]} = data + + assert vector == [1, 2, 3] + assert payload["actor"] + assert payload["published_at"] Tesla.Mock.json("ok") From a566ad56e1434715d00067b1e49be66b6787f5ba Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 23 May 2024 18:55:16 +0400 Subject: [PATCH 115/161] QdrantSearch: Fix actor / author restriction --- lib/pleroma/search/qdrant_search.ex | 4 ++-- test/pleroma/search/qdrant_search_test.exs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 9cb34ef71..19e8cd4bf 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -71,9 +71,9 @@ defp build_search_payload(embedding, options) do offset: options[:offset] || 0 } - if options[:actor] do + if author = options[:author] do Map.put(base, :filter, %{ - must: [%{key: "actor", match: %{value: options[:actor].ap_id}}] + must: [%{key: "actor", match: %{value: author.ap_id}}] }) else base diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index 371074dcf..46485392e 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -98,7 +98,7 @@ test "for a given actor, ask for only relevant matches" do end) results = - QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{actor: user}) + QdrantSearch.search(nil, "guys i just don't wanna leave the swamp", %{author: user}) assert results == [activity] end From 618b77071afb480b763a493bfcd9b376effedaaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 25 May 2024 09:10:11 +0200 Subject: [PATCH 116/161] Update pleroma_api.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/api-docs-2.skip | 0 docs/development/API/pleroma_api.md | 4 +--- 2 files changed, 1 insertion(+), 3 deletions(-) create mode 100644 changelog.d/api-docs-2.skip diff --git a/changelog.d/api-docs-2.skip b/changelog.d/api-docs-2.skip new file mode 100644 index 000000000..e69de29bb diff --git a/docs/development/API/pleroma_api.md b/docs/development/API/pleroma_api.md index 267dfc1ec..57d333ffe 100644 --- a/docs/development/API/pleroma_api.md +++ b/docs/development/API/pleroma_api.md @@ -295,9 +295,7 @@ See [Admin-API](admin_api.md) "id": "9umDrYheeY451cQnEe", "name": "Read later", "emoji": "🕓", - "source": { - "emoji": "🕓" - } + "emoji_url": null } ] ``` From 61a3b793165e92d6f23a2e59fb9b95e06737cf25 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 25 May 2024 14:20:47 -0400 Subject: [PATCH 117/161] Search backend healthcheck process --- changelog.d/search-healthcheck.add | 1 + config/config.exs | 2 +- lib/pleroma/application.ex | 3 +- lib/pleroma/search.ex | 5 ++ lib/pleroma/search/database_search.ex | 3 + lib/pleroma/search/healthcheck.ex | 85 +++++++++++++++++++++++++++ lib/pleroma/search/meilisearch.ex | 11 ++++ lib/pleroma/search/search_backend.ex | 8 +++ 8 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 changelog.d/search-healthcheck.add create mode 100644 lib/pleroma/search/healthcheck.ex diff --git a/changelog.d/search-healthcheck.add b/changelog.d/search-healthcheck.add new file mode 100644 index 000000000..4974925e7 --- /dev/null +++ b/changelog.d/search-healthcheck.add @@ -0,0 +1 @@ +Monitoring of search backend health to control the processing of jobs in the search indexing Oban queue diff --git a/config/config.exs b/config/config.exs index b69044a2b..8b9a588b7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -579,7 +579,7 @@ attachments_cleanup: 1, new_users_digest: 1, mute_expire: 5, - search_indexing: 10, + search_indexing: [limit: 10, paused: true], rich_media_expiration: 2 ], plugins: [Oban.Plugins.Pruner], diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 649bb11c8..d266d1836 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -109,7 +109,8 @@ def start(_type, _args) do streamer_registry() ++ background_migrators() ++ shout_child(shout_enabled?()) ++ - [Pleroma.Gopher.Server] + [Pleroma.Gopher.Server] ++ + [Pleroma.Search.Healthcheck] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index 3b266e59b..e8dbcca1f 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -14,4 +14,9 @@ def search(query, options) do search_module.search(options[:for_user], query, options) end + + def healthcheck_endpoints do + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + search_module.healthcheck_endpoints + end end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 31bfc7e33..11e99e7f1 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -48,6 +48,9 @@ def add_to_index(_activity), do: :ok @impl true def remove_from_index(_object), do: :ok + @impl true + def healthcheck_endpoints, do: nil + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) end diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex new file mode 100644 index 000000000..495aee930 --- /dev/null +++ b/lib/pleroma/search/healthcheck.ex @@ -0,0 +1,85 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only +defmodule Pleroma.Search.Healthcheck do + @doc """ + Monitors health of search backend to control processing of events based on health and availability. + """ + use GenServer + require Logger + + @tick :timer.seconds(60) + @queue :search_indexing + + def start_link(_) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(_) do + state = %{healthy: false} + {:ok, state, {:continue, :start}} + end + + @impl true + def handle_continue(:start, state) do + tick() + {:noreply, state} + end + + @impl true + def handle_info(:check, state) do + urls = Pleroma.Search.healthcheck_endpoints() + + new_state = + if healthy?(urls) do + Oban.resume_queue(queue: @queue) + Map.put(state, :healthy, true) + else + Oban.pause_queue(queue: @queue) + Map.put(state, :healthy, false) + end + + maybe_log_state_change(state, new_state) + + tick() + {:noreply, new_state} + end + + @impl true + def handle_call(:check, _from, state) do + status = Map.get(state, :healthy) + + {:reply, status, state, :hibernate} + end + + defp healthy?([]), do: true + + defp healthy?(urls) when is_list(urls) do + Enum.all?( + urls, + fn url -> + case Pleroma.HTTP.get(url) do + {:ok, %{status: 200}} -> true + _ -> false + end + end + ) + end + + defp healthy?(_), do: true + + defp tick do + Process.send_after(self(), :check, @tick) + end + + defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do + Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure") + end + + defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do + Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass") + end + + defp maybe_log_state_change(_, _), do: :ok +end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 2bff663e8..08c2f3d86 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -178,4 +178,15 @@ def add_to_index(activity) do def remove_from_index(object) do meili_delete("/indexes/objects/documents/#{object.id}") end + + @impl true + def healthcheck_endpoints do + endpoint = + Config.get([Pleroma.Search.Meilisearch, :url]) + |> URI.parse() + |> Map.put(:path, "/health") + |> URI.to_string() + + [endpoint] + end end diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index 68bc48cec..13c887bc2 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -21,4 +21,12 @@ defmodule Pleroma.Search.SearchBackend do from index. """ @callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()} + + @doc """ + Healthcheck endpoints of search backend infrastructure to monitor for controlling + processing of jobs in the Oban queue. + + It is expected a 200 response is healthy and other responses are unhealthy. + """ + @callback healthcheck_endpoints :: list() | nil end From 3474b42ce396150b21f26ed35bea46ad61f57d5f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 25 May 2024 16:55:29 -0400 Subject: [PATCH 118/161] Drop TTL to 5 seconds --- lib/pleroma/search/healthcheck.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex index 495aee930..9a2d9fdd6 100644 --- a/lib/pleroma/search/healthcheck.ex +++ b/lib/pleroma/search/healthcheck.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Search.Healthcheck do use GenServer require Logger - @tick :timer.seconds(60) + @tick :timer.seconds(5) @queue :search_indexing def start_link(_) do From 354b700bedf8ad6e9187245977165ebd7bc2fa1c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 26 May 2024 14:01:00 -0400 Subject: [PATCH 119/161] Assert that AWS URLs without query parameters do not crash --- .../web/rich_media/parser/ttl/aws_signed_url_test.exs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs index cd8be8675..cc28aa7f3 100644 --- a/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs +++ b/test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs @@ -10,6 +10,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do alias Pleroma.UnstubbedConfigMock, as: ConfigMock alias Pleroma.Web.RichMedia.Card + alias Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl setup do ConfigMock @@ -82,6 +83,12 @@ test "s3 signed url is parsed and correct ttl is set for rich media" do assert DateTime.diff(scheduled_at, timestamp_dt) == valid_till end + test "AWS URL for an image without expiration works" do + og_data = %{"image" => "https://amazonaws.com/image.png"} + + assert is_nil(AwsSignedUrl.ttl(og_data, "")) + end + defp construct_s3_url(timestamp, valid_till) do "https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{timestamp}&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host" end From 807782b7f96ee0e053ad59b464766d750f8a8800 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 25 May 2024 16:27:59 -0400 Subject: [PATCH 120/161] Fix rich media parsing some Amazon URLs --- changelog.d/richmediattl.fix | 1 + lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/richmediattl.fix diff --git a/changelog.d/richmediattl.fix b/changelog.d/richmediattl.fix new file mode 100644 index 000000000..98de63015 --- /dev/null +++ b/changelog.d/richmediattl.fix @@ -0,0 +1 @@ +Parsing of RichMedia TTLs for Amazon URLs when query parameters are nil diff --git a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex index 948c727e1..1172a120a 100644 --- a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex +++ b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex @@ -23,7 +23,7 @@ defp aws_signed_url?(image) when is_binary(image) and image != "" do %URI{host: host, query: query} = URI.parse(image) is_binary(host) and String.contains?(host, "amazonaws.com") and - String.contains?(query, "X-Amz-Expires") + is_binary(query) and String.contains?(query, "X-Amz-Expires") end defp aws_signed_url?(_), do: nil From f2b0d5f1d02e243a7a1a6f339b59e5abcb8e1bd8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 26 May 2024 14:11:41 -0400 Subject: [PATCH 121/161] Make it easier to read the state for debugging purposes and expose functions for testing --- lib/pleroma/search/healthcheck.ex | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex index 9a2d9fdd6..170f29344 100644 --- a/lib/pleroma/search/healthcheck.ex +++ b/lib/pleroma/search/healthcheck.ex @@ -32,7 +32,7 @@ def handle_info(:check, state) do urls = Pleroma.Search.healthcheck_endpoints() new_state = - if healthy?(urls) do + if check(urls) do Oban.resume_queue(queue: @queue) Map.put(state, :healthy, true) else @@ -47,15 +47,15 @@ def handle_info(:check, state) do end @impl true - def handle_call(:check, _from, state) do - status = Map.get(state, :healthy) - - {:reply, status, state, :hibernate} + def handle_call(:state, _from, state) do + {:reply, state, state, :hibernate} end - defp healthy?([]), do: true + def state, do: GenServer.call(__MODULE__, :state) - defp healthy?(urls) when is_list(urls) do + def check([]), do: true + + def check(urls) when is_list(urls) do Enum.all?( urls, fn url -> @@ -67,7 +67,7 @@ defp healthy?(urls) when is_list(urls) do ) end - defp healthy?(_), do: true + def check(_), do: true defp tick do Process.send_after(self(), :check, @tick) From 03f4b461895802259c895c81462a3e9d0d31c1e5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 26 May 2024 14:21:24 -0400 Subject: [PATCH 122/161] Test that healthchecks behave correctly for the expected HTTP responses --- test/pleroma/search/healthcheck_test.exs | 49 ++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 test/pleroma/search/healthcheck_test.exs diff --git a/test/pleroma/search/healthcheck_test.exs b/test/pleroma/search/healthcheck_test.exs new file mode 100644 index 000000000..e7649d949 --- /dev/null +++ b/test/pleroma/search/healthcheck_test.exs @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Search.HealthcheckTest do + use Pleroma.DataCase + + import Tesla.Mock + + alias Pleroma.Search.Healthcheck + + @good1 "http://good1.example.com/healthz" + @good2 "http://good2.example.com/health" + @bad "http://bad.example.com/healthy" + + setup do + mock(fn + %{method: :get, url: @good1} -> + %Tesla.Env{ + status: 200, + body: "" + } + + %{method: :get, url: @good2} -> + %Tesla.Env{ + status: 200, + body: "" + } + + %{method: :get, url: @bad} -> + %Tesla.Env{ + status: 503, + body: "" + } + end) + + :ok + end + + test "true for 200 responses" do + assert Healthcheck.check([@good1]) + assert Healthcheck.check([@good1, @good2]) + end + + test "false if any response is not a 200" do + refute Healthcheck.check([@bad]) + refute Healthcheck.check([@good1, @bad]) + end +end From d4769b076a95ce2281dba5673c410eb098445bba Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 26 May 2024 15:13:59 -0400 Subject: [PATCH 123/161] Return a 422 when trying to reply to a deleted status --- changelog.d/reply-to-deleted.change | 1 + lib/pleroma/web/common_api/activity_draft.ex | 18 ++++++++++++++++-- .../controllers/status_controller_test.exs | 10 ++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 changelog.d/reply-to-deleted.change diff --git a/changelog.d/reply-to-deleted.change b/changelog.d/reply-to-deleted.change new file mode 100644 index 000000000..8b952ee7a --- /dev/null +++ b/changelog.d/reply-to-deleted.change @@ -0,0 +1 @@ +A 422 error is returned when attempting to reply to a deleted status diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index bc46a8a36..8aa1e258d 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -129,8 +129,22 @@ defp attachments(%{params: params} = draft) do defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft - defp in_reply_to(%{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do - %__MODULE__{draft | in_reply_to: Activity.get_by_id(id)} + defp in_reply_to(%{params: %{in_reply_to_status_id: :deleted}} = draft) do + add_error(draft, dgettext("errors", "Cannot reply to a deleted status")) + end + + defp in_reply_to(%{params: %{in_reply_to_status_id: id} = params} = draft) when is_binary(id) do + activity = Activity.get_by_id(id) + + params = + if is_nil(activity) do + # Deleted activities are returned as nil + Map.put(params, :in_reply_to_status_id, :deleted) + else + Map.put(params, :in_reply_to_status_id, activity) + end + + in_reply_to(%{draft | params: params}) end defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 80c1ed099..f34911e5b 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -235,6 +235,16 @@ test "replying to a status", %{user: user, conn: conn} do assert Activity.get_in_reply_to_activity(activity).id == replied_to.id end + test "replying to a deleted status", %{user: user, conn: conn} do + {:ok, status} = CommonAPI.post(user, %{status: "cofe"}) + {:ok, _deleted_status} = CommonAPI.delete(status.id, user) + + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses", %{"status" => "xD", "in_reply_to_id" => status.id}) + |> json_response_and_validate_schema(422) + end + test "replying to a direct message with visibility other than direct", %{ user: user, conn: conn From d9b82255b9cf49176f8ef1d5a87abf7d80769a47 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 26 May 2024 15:23:12 -0400 Subject: [PATCH 124/161] Add an HTTP timeout for the healthcheck --- lib/pleroma/search/healthcheck.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex index 170f29344..e562c8478 100644 --- a/lib/pleroma/search/healthcheck.ex +++ b/lib/pleroma/search/healthcheck.ex @@ -8,8 +8,9 @@ defmodule Pleroma.Search.Healthcheck do use GenServer require Logger - @tick :timer.seconds(5) @queue :search_indexing + @tick :timer.seconds(5) + @timeout :timer.seconds(2) def start_link(_) do GenServer.start_link(__MODULE__, [], name: __MODULE__) @@ -59,7 +60,7 @@ def check(urls) when is_list(urls) do Enum.all?( urls, fn url -> - case Pleroma.HTTP.get(url) do + case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do {:ok, %{status: 200}} -> true _ -> false end From d35b69d2686e62cc5076bd7a33449f98f8a11a85 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 13:18:02 +0400 Subject: [PATCH 125/161] Pleroma.Search: Remove wrong (but irrelevant) results --- lib/pleroma/search.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index e8dbcca1f..fd0218cb8 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -10,13 +10,12 @@ def remove_from_index(%Pleroma.Object{id: object_id}) do end def search(query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) - + search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.search(options[:for_user], query, options) end def healthcheck_endpoints do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.healthcheck_endpoints end end From 8b76f56050a609bf562053cb7201a9204901490e Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 13:57:42 +0400 Subject: [PATCH 126/161] QdrantSearch: Add healthcheck for qdrant --- lib/pleroma/search/qdrant_search.ex | 11 +++++++++++ test/pleroma/search/qdrant_search_test.exs | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 19e8cd4bf..3c3ffce16 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -139,6 +139,17 @@ def search(_user, query, options) do [] end end + + @impl true + def healthcheck_endpoints do + qdrant_health = + Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]) + |> URI.parse() + |> Map.put(:path, "/healthz") + |> URI.to_string() + + [qdrant_health] + end end defmodule Pleroma.Search.QdrantSearch.OpenAIClient do diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index 46485392e..b389aa816 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -15,6 +15,18 @@ defmodule Pleroma.Search.QdrantSearchTest do alias Pleroma.Workers.SearchIndexingWorker describe "Qdrant search" do + test "returns the correct healthcheck endpoints" do + Config + |> expect(:get, 1, fn + [Pleroma.Search.QdrantSearch, key], nil -> + %{qdrant_url: "https://qdrant.url"}[key] + end) + + health_endpoints = QdrantSearch.healthcheck_endpoints() + + assert "https://qdrant.url/healthz" in health_endpoints + end + test "searches for a term by encoding it and sending it to qdrant" do user = insert(:user) From ec3f3fef7798111641f08020d5fd7ae16e407b89 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 14:15:04 +0400 Subject: [PATCH 127/161] Fastembed Server: Add health check endpoint --- supplemental/search/fastembed-api/fastembed-server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/supplemental/search/fastembed-api/fastembed-server.py b/supplemental/search/fastembed-api/fastembed-server.py index dd4a7a9c8..02da69db2 100644 --- a/supplemental/search/fastembed-api/fastembed-server.py +++ b/supplemental/search/fastembed-api/fastembed-server.py @@ -17,6 +17,10 @@ def embeddings(request: EmbeddingRequest): embeddings = next(model.embed(request.input)).tolist() return {"data": [{"embedding": embeddings}]} +@app.get("/health") +def health(): + return {"status": "ok"} + if __name__ == "__main__": import uvicorn From f4c04e6b2dce6d75d148ca520aaef27005ecaa82 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 14:21:55 +0400 Subject: [PATCH 128/161] QdrantSearch: Add health checks. --- config/config.exs | 3 +++ lib/pleroma/search/qdrant_search.ex | 4 +++- test/pleroma/search/qdrant_search_test.exs | 20 +++++++++++++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index d891a5218..f388dfe52 100644 --- a/config/config.exs +++ b/config/config.exs @@ -919,6 +919,9 @@ qdrant_url: "http://127.0.0.1:6333/", qdrant_api_key: "", openai_url: "http://127.0.0.1:11345", + # The healthcheck url has to be set to nil when used with the real openai + # API, as it doesn't have a healthcheck endpoint. + openai_healthcheck_url: "http://127.0.0.1:11345/health", openai_model: "snowflake/snowflake-arctic-embed-xs", openai_api_key: "", qdrant_index_configuration: %{ diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 3c3ffce16..429ae05b8 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -148,7 +148,9 @@ def healthcheck_endpoints do |> Map.put(:path, "/healthz") |> URI.to_string() - [qdrant_health] + openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url]) + + [qdrant_health, openai_health] |> Enum.filter(& &1) end end diff --git a/test/pleroma/search/qdrant_search_test.exs b/test/pleroma/search/qdrant_search_test.exs index b389aa816..47a77a391 100644 --- a/test/pleroma/search/qdrant_search_test.exs +++ b/test/pleroma/search/qdrant_search_test.exs @@ -16,15 +16,29 @@ defmodule Pleroma.Search.QdrantSearchTest do describe "Qdrant search" do test "returns the correct healthcheck endpoints" do + # No openai healthcheck URL Config - |> expect(:get, 1, fn + |> expect(:get, 2, fn [Pleroma.Search.QdrantSearch, key], nil -> %{qdrant_url: "https://qdrant.url"}[key] end) - health_endpoints = QdrantSearch.healthcheck_endpoints() + [health_endpoint] = QdrantSearch.healthcheck_endpoints() - assert "https://qdrant.url/healthz" in health_endpoints + assert "https://qdrant.url/healthz" == health_endpoint + + # Set openai healthcheck URL + Config + |> expect(:get, 2, fn + [Pleroma.Search.QdrantSearch, key], nil -> + %{qdrant_url: "https://qdrant.url", openai_healthcheck_url: "https://openai.url/health"}[ + key + ] + end) + + [_, health_endpoint] = QdrantSearch.healthcheck_endpoints() + + assert "https://openai.url/health" == health_endpoint end test "searches for a term by encoding it and sending it to qdrant" do From ddf103eca04c9571ba8310915556cc51cd4a9af8 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 14:35:08 +0400 Subject: [PATCH 129/161] QdrantSearch: Fetch a post in search if possible. --- lib/pleroma/search/qdrant_search.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex index 429ae05b8..b659bb682 100644 --- a/lib/pleroma/search/qdrant_search.ex +++ b/lib/pleroma/search/qdrant_search.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Search.QdrantSearch do alias __MODULE__.QdrantClient import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] + import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3] @impl true def create_index do @@ -115,8 +116,8 @@ def remove_from_index(object) do end @impl true - def search(_user, query, options) do - query = "Represent this sentence for searching relevant passages: #{query}" + def search(user, original_query, options) do + query = "Represent this sentence for searching relevant passages: #{original_query}" with {:ok, embedding} <- get_embedding(query), {:ok, %{body: %{"result" => result}}} <- @@ -134,6 +135,7 @@ def search(_user, query, options) do |> Activity.restrict_deactivated_users() |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id)) |> Pleroma.Repo.all() + |> maybe_fetch(user, original_query) else _ -> [] From f214c2cdac4a94fae51e7679223df9557c6a1827 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 15:23:33 +0400 Subject: [PATCH 130/161] NotificationTest: Remove impossible case. --- test/pleroma/notification_test.exs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index ecdb32e32..2c582c708 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -859,22 +859,6 @@ test "repeating an activity which is already deleted does not generate a notific assert Enum.empty?(Notification.for_user(user)) end - test "replying to a deleted post without tagging does not generate a notification" do - user = insert(:user) - other_user = insert(:user) - - {:ok, activity} = CommonAPI.post(user, %{status: "test post"}) - {:ok, _deletion_activity} = CommonAPI.delete(activity.id, user) - - {:ok, _reply_activity} = - CommonAPI.post(other_user, %{ - status: "test reply", - in_reply_to_status_id: activity.id - }) - - assert Enum.empty?(Notification.for_user(user)) - end - test "notifications are deleted if a local user is deleted" do user = insert(:user) other_user = insert(:user) From 3055c1598b43ee9460b88880e2752c68e9cf6edb Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 17:22:18 +0400 Subject: [PATCH 131/161] IPFSTest: Fix configuration mocking --- config/test.exs | 1 + lib/pleroma/upload.ex | 2 +- lib/pleroma/uploaders/ipfs.ex | 7 +-- test/pleroma/uploaders/ipfs_test.exs | 70 +++++++++++++++++++--------- 4 files changed, 55 insertions(+), 25 deletions(-) diff --git a/config/test.exs b/config/test.exs index 9b4113dd5..3345bb3a9 100644 --- a/config/test.exs +++ b/config/test.exs @@ -153,6 +153,7 @@ config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock +config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock peer_module = if String.to_integer(System.otp_release()) >= 25 do diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 2c6b23c39..35c7c02a5 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -282,7 +282,7 @@ def base_url do end Pleroma.Uploaders.IPFS -> - Config.get([Pleroma.Uploaders.IPFS, :get_gateway_url]) + @config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url]) _ -> public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index 32e06c5cf..d171e4652 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -6,11 +6,12 @@ defmodule Pleroma.Uploaders.IPFS do @behaviour Pleroma.Uploaders.Uploader require Logger - alias Pleroma.Config alias Tesla.Multipart + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + defp get_final_url(method) do - config = Config.get([__MODULE__]) + config = @config_impl.get([__MODULE__]) post_base_url = Keyword.get(config, :post_gateway_url) Path.join([post_base_url, method]) @@ -69,7 +70,7 @@ def put_file(%Pleroma.Upload{} = upload) do @impl true def delete_file(file) do case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do - {:ok, %{status_code: 204}} -> :ok + {:ok, %{status: 204}} -> :ok error -> {:error, inspect(error)} end end diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index 853d185e5..cf325b54f 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -8,22 +8,22 @@ defmodule Pleroma.Uploaders.IPFSTest do alias Pleroma.Uploaders.IPFS alias Tesla.Multipart - import Mock import ExUnit.CaptureLog + import Mock + import Mox - setup do - clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.IPFS) - clear_config([Pleroma.Uploaders.IPFS]) - - clear_config( - [Pleroma.Uploaders.IPFS, :get_gateway_url], - "https://{CID}.ipfs.mydomain.com" - ) - - clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001") - end + alias Pleroma.UnstubbedConfigMock, as: Config describe "get_final_url" do + setup do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS] -> + [post_gateway_url: "http://localhost:5001"] + end) + + :ok + end + test "it returns the final url for put_file" do assert IPFS.put_file_endpoint() == "http://localhost:5001/api/v0/add" end @@ -34,7 +34,21 @@ test "it returns the final url for delete_file" do end describe "get_file/1" do + setup do + Config + |> expect(:get, fn [Pleroma.Upload, :uploader] -> Pleroma.Uploaders.IPFS end) + |> expect(:get, fn [Pleroma.Upload, :base_url] -> nil end) + |> expect(:get, fn [Pleroma.Uploaders.IPFS, :public_endpoint] -> nil end) + + :ok + end + test "it returns path to ipfs file with cid as subdomain" do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] -> + "https://{CID}.ipfs.mydomain.com" + end) + assert IPFS.get_file("testcid") == { :ok, {:url, "https://testcid.ipfs.mydomain.com"} @@ -42,10 +56,10 @@ test "it returns path to ipfs file with cid as subdomain" do end test "it returns path to ipfs file with cid as path" do - clear_config( - [Pleroma.Uploaders.IPFS, :get_gateway_url], + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] -> "https://ipfs.mydomain.com/ipfs/{CID}" - ) + end) assert IPFS.get_file("testcid") == { :ok, @@ -56,6 +70,11 @@ test "it returns path to ipfs file with cid as path" do describe "put_file/1" do setup do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS] -> + [post_gateway_url: "http://localhost:5001"] + end) + file_upload = %Pleroma.Upload{ name: "image-tet.jpg", content_type: "image/jpeg", @@ -73,7 +92,7 @@ test "it returns path to ipfs file with cid as path" do test "save file", %{file_upload: file_upload} do with_mock Pleroma.HTTP, - post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] -> {:ok, %Tesla.Env{ status: 200, @@ -88,7 +107,7 @@ test "save file", %{file_upload: file_upload} do test "returns error", %{file_upload: file_upload} do with_mock Pleroma.HTTP, - post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] -> {:error, "IPFS Gateway upload failed"} end do assert capture_log(fn -> @@ -99,19 +118,19 @@ test "returns error", %{file_upload: file_upload} do test "returns error if JSON decode fails", %{file_upload: file_upload} do with_mock Pleroma.HTTP, [], - post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] -> {:ok, %Tesla.Env{status: 200, body: "invalid"}} end do assert capture_log(fn -> assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"} end) =~ - "Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError{data: \"invalid\", position: 0, token: nil}}" + "Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError" end end test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do with_mock Pleroma.HTTP, [], - post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] -> {:ok, %Tesla.Env{status: 200, body: "{\"key\": \"value\"}"}} end do assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"} @@ -120,9 +139,18 @@ test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_ end describe "delete_file/1" do + setup do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS] -> + [post_gateway_url: "http://localhost:5001"] + end) + + :ok + end + test_with_mock "deletes file", Pleroma.HTTP, post: fn "http://localhost:5001/api/v0/files/rm", "", [], params: [arg: "image.jpg"] -> - {:ok, %{status_code: 204}} + {:ok, %{status: 204}} end do assert :ok = IPFS.delete_file("image.jpg") end From ed93af64e14e1e82cf4840b1a160df8eddecc55c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 17:50:34 +0400 Subject: [PATCH 132/161] Add changelog --- changelog.d/add-nsfw-mrf.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/add-nsfw-mrf.add diff --git a/changelog.d/add-nsfw-mrf.add b/changelog.d/add-nsfw-mrf.add new file mode 100644 index 000000000..ce62c7ed0 --- /dev/null +++ b/changelog.d/add-nsfw-mrf.add @@ -0,0 +1 @@ +Add NSFW-detecting MRF From a50c657427a2dfe9d48c25529a179fe634d30e48 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:17:02 -0400 Subject: [PATCH 133/161] Add a dedicated connection pool for Rich Media Sharing this pool with regular Media is problematic as Rich Media will connect to many different domains and thrash the pool, but regular Media will have predictable connections to the webservers hosting media for the fediverse servers you peer with. --- config/config.exs | 9 +++++++++ lib/pleroma/web/rich_media/helpers.ex | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 8b9a588b7..b8030651f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -827,6 +827,11 @@ max_waiting: 20, recv_timeout: 15_000 ], + rich_media: [ + size: 25, + max_waiting: 20, + recv_timeout: 15_000 + ], upload: [ size: 25, max_waiting: 5, @@ -847,6 +852,10 @@ max_connections: 50, timeout: 150_000 ], + rich_media: [ + max_connections: 50, + timeout: 150_000 + ], upload: [ max_connections: 25, timeout: 300_000 diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 119994458..ea41bd285 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -58,7 +58,7 @@ defp check_content_length(headers) do defp http_options do [ - pool: :media, + pool: :rich_media, max_body: Config.get([:rich_media, :max_body], 5_000_000) ] end From 6708f154a4f7ad46b4637d4d566b8cf81e3ebb7b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:18:58 -0400 Subject: [PATCH 134/161] Rework Gun connection pool sizes to make better use of the default 250 connections --- config/config.exs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index b8030651f..a025defeb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -818,27 +818,27 @@ config :pleroma, :pools, federation: [ - size: 50, - max_waiting: 10, + size: 75, + max_waiting: 20, recv_timeout: 10_000 ], media: [ - size: 50, + size: 75, max_waiting: 20, recv_timeout: 15_000 ], rich_media: [ size: 25, max_waiting: 20, - recv_timeout: 15_000 - ], + recv_timeout: 15_000 + ], upload: [ size: 25, - max_waiting: 5, + max_waiting: 20, recv_timeout: 15_000 ], default: [ - size: 10, + size: 50, max_waiting: 2, recv_timeout: 5_000 ] From d272eb62cd4da45e5ef82fcc0126d2cf799d292a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:22:45 -0400 Subject: [PATCH 135/161] Trust the connection pools to enforce the concurrency limitations --- .../web/activity_pub/mrf/media_proxy_warming_policy.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index c95d35bb9..f10dc3ce5 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -30,9 +30,7 @@ defp prefetch(url) do if Pleroma.Config.get(:env) == :test do fetch(prefetch_url) else - ConcurrentLimiter.limit(__MODULE__, fn -> - Task.start(fn -> fetch(prefetch_url) end) - end) + Task.start(fn -> fetch(prefetch_url) end) end end end From 37d79b76bb770cb294c0a54777b435dc49c042ab Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:24:54 -0400 Subject: [PATCH 136/161] Use the configured http client options for mediaproxy --- .../web/activity_pub/mrf/media_proxy_warming_policy.ex | 10 ++++------ lib/pleroma/web/media_proxy/media_proxy_controller.ex | 3 ++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index f10dc3ce5..e5eb6896a 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -11,11 +11,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @adapter_options [ - pool: :media, - recv_timeout: 10_000 - ] - @impl true def history_awareness, do: :auto @@ -35,7 +30,10 @@ defp prefetch(url) do end end - defp fetch(url), do: HTTP.get(url, [], @adapter_options) + defp fetch(url) do + http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) + HTTP.get(url, [], http_client_opts) + end defp preload(%{"object" => %{"attachment" => attachments}} = _message) do Enum.each(attachments, fn diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index c11484ecb..0b446e0a6 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -54,9 +54,10 @@ def preview(%Conn{} = conn, %{"sig" => sig64, "url" => url64}) do defp handle_preview(conn, url) do media_proxy_url = MediaProxy.url(url) + http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) with {:ok, %{status: status} = head_response} when status in 200..299 <- - Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do + Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do content_type = Tesla.get_header(head_response, "content-type") content_length = Tesla.get_header(head_response, "content-length") content_length = content_length && String.to_integer(content_length) From 8b61d4e3e124c4fafeaabe58a8c7673f767b2871 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:28:31 -0400 Subject: [PATCH 137/161] Changelogs --- changelog.d/mediaproxy-http.fix | 1 + changelog.d/pools.change | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/mediaproxy-http.fix create mode 100644 changelog.d/pools.change diff --git a/changelog.d/mediaproxy-http.fix b/changelog.d/mediaproxy-http.fix new file mode 100644 index 000000000..4ff6430e0 --- /dev/null +++ b/changelog.d/mediaproxy-http.fix @@ -0,0 +1 @@ +Ensure MediaProxy HTTP requests obey all the defined connection settings diff --git a/changelog.d/pools.change b/changelog.d/pools.change new file mode 100644 index 000000000..3c689195a --- /dev/null +++ b/changelog.d/pools.change @@ -0,0 +1 @@ +HTTP connection pool adjustments From e4f1325f78e9be9fb200358d73794f15794c39bd Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 19:44:41 +0400 Subject: [PATCH 138/161] InetHelper: Don't use deprecated function. --- lib/pleroma/helpers/inet_helper.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex index 3500fc679..00e18649e 100644 --- a/lib/pleroma/helpers/inet_helper.ex +++ b/lib/pleroma/helpers/inet_helper.ex @@ -25,6 +25,6 @@ def parse_cidr(proxy) when is_binary(proxy) do InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" end - InetCidr.parse(proxy, true) + InetCidr.parse_cidr!(proxy, true) end end From 284cd0abe5fd34d0bb31281614a7dc9249731b40 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 20:04:12 +0400 Subject: [PATCH 139/161] Add changelog --- changelog.d/support-honk-image-summaries.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/support-honk-image-summaries.add diff --git a/changelog.d/support-honk-image-summaries.add b/changelog.d/support-honk-image-summaries.add new file mode 100644 index 000000000..052c03f95 --- /dev/null +++ b/changelog.d/support-honk-image-summaries.add @@ -0,0 +1 @@ +Support honk-style attachment summaries as alt-text. From f4693dc6710c8c8ac878c2845793c7d138f90c04 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 22:32:42 -0500 Subject: [PATCH 140/161] Update Prometheus/Grafana docs for PromEx --- changelog.d/prometheus-docs.change | 1 + docs/development/API/prometheus.md | 73 ++++++++++++++++-------------- 2 files changed, 39 insertions(+), 35 deletions(-) create mode 100644 changelog.d/prometheus-docs.change diff --git a/changelog.d/prometheus-docs.change b/changelog.d/prometheus-docs.change new file mode 100644 index 000000000..a9bd1e2e9 --- /dev/null +++ b/changelog.d/prometheus-docs.change @@ -0,0 +1 @@ +Update the documentation for configuring Prometheus metrics. diff --git a/docs/development/API/prometheus.md b/docs/development/API/prometheus.md index a5158d905..140291fe0 100644 --- a/docs/development/API/prometheus.md +++ b/docs/development/API/prometheus.md @@ -1,44 +1,47 @@ -# Prometheus Metrics +# Prometheus / OpenTelemetry Metrics -Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library. +Pleroma includes support for exporting metrics via the [prom_ex](https://github.com/akoutmos/prom_ex) library. +The metrics are exposed by a dedicated webserver/port to improve privacy and security. Config example: ``` -config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, - enabled: true, - auth: {:basic, "myusername", "mypassword"}, - ip_whitelist: ["127.0.0.1"], - path: "/api/pleroma/app_metrics", - format: :text -``` - -* `enabled` (Pleroma extension) enables the endpoint -* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs -* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation) -* `format` sets the output format (`:text` or `:protobuf`) -* `path` sets the path to app metrics page - - -## `/api/pleroma/app_metrics` - -### Exports Prometheus application metrics - -* Method: `GET` -* Authentication: not required by default (see configuration options above) -* Params: none -* Response: text - -## Grafana - -### Config example - -The following is a config example to use with [Grafana](https://grafana.com) +config :pleroma, Pleroma.PromEx, + disabled: false, + manual_metrics_start_delay: :no_delay, + drop_metrics_groups: [], + grafana: [ + host: System.get_env("GRAFANA_HOST", "http://localhost:3000"), + auth_token: System.get_env("GRAFANA_TOKEN"), + upload_dashboards_on_start: false, + folder_name: "BEAM", + annotate_app_lifecycle: true + ], + metrics_server: [ + port: 4021, + path: "/metrics", + protocol: :http, + pool_size: 5, + cowboy_opts: [], + auth_strategy: :none + ], + datasource: "Prometheus" ``` - - job_name: 'beam' - metrics_path: /api/pleroma/app_metrics - scheme: https + +PromEx supports the ability to automatically publish dashboards to your Grafana server as well as register Annotations. If you do not wish to configure this capability you must generate the dashboard JSON files and import them directly. You can find the mix commands in the upstream [documentation](https://hexdocs.pm/prom_ex/Mix.Tasks.PromEx.Dashboard.Export.html). You can find the list of modules enabled in Pleroma for which you should generate dashboards for by examining the contents of the `lib/pleroma/prom_ex.ex` module. + +## prometheus.yml + +The following is a bare minimum config example to use with [Prometheus](https://prometheus.io) or Prometheus-compatible software like [VictoriaMetrics](https://victoriametrics.com). + +``` +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'pleroma' + scheme: http static_configs: - - targets: ['pleroma.soykaf.com'] + - targets: ['pleroma.soykaf.com:4021'] ``` From 7258ab1aed53da796e24bdab81f39ea1d358a549 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 12:20:00 -0400 Subject: [PATCH 141/161] Changelog --- changelog.d/promexdocs.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/promexdocs.add diff --git a/changelog.d/promexdocs.add b/changelog.d/promexdocs.add new file mode 100644 index 000000000..dda972994 --- /dev/null +++ b/changelog.d/promexdocs.add @@ -0,0 +1 @@ +PromEx documentation From 0bddca361d12f347ca9907c5ddb5d1464a17b32a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 24 Jan 2021 14:56:45 -0600 Subject: [PATCH 142/161] DNSRBL in an MRF --- changelog.d/add-rbl-mrf.add | 1 + config/config.exs | 5 + .../web/activity_pub/mrf/dnsrbl_policy.ex | 142 ++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 changelog.d/add-rbl-mrf.add create mode 100644 lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex diff --git a/changelog.d/add-rbl-mrf.add b/changelog.d/add-rbl-mrf.add new file mode 100644 index 000000000..363270fb9 --- /dev/null +++ b/changelog.d/add-rbl-mrf.add @@ -0,0 +1 @@ +Add DNSRBL MRF diff --git a/config/config.exs b/config/config.exs index b93de52e1..1fb0f3911 100644 --- a/config/config.exs +++ b/config/config.exs @@ -410,6 +410,11 @@ accept: [], reject: [] +config :pleroma, :mrf_dnsrbl, + nameserver: "127.0.0.1", + port: 53, + zone: "bl.pleroma.com" + # threshold of 7 days config :pleroma, :mrf_object_age, threshold: 604_800, diff --git a/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex new file mode 100644 index 000000000..9543cc545 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex @@ -0,0 +1,142 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do + @moduledoc """ + Dynamic activity filtering based on an RBL database + + This MRF makes queries to a custom DNS server which will + respond with values indicating the classification of the domain + the activity originated from. This method has been widely used + in the email anti-spam industry for very fast reputation checks. + + e.g., if the DNS response is 127.0.0.1 or empty, the domain is OK + Other values such as 127.0.0.2 may be used for specific classifications. + + Information for why the host is blocked can be stored in a corresponding TXT record. + + This method is fail-open so if the queries fail the activites are accepted. + + An example of software meant for this purpsoe is rbldnsd which can be found + at http://www.corpit.ru/mjt/rbldnsd.html or mirrored at + https://git.pleroma.social/feld/rbldnsd + + It is highly recommended that you run your own copy of rbldnsd and use an + external mechanism to sync/share the contents of the zone file. This is + important to keep the latency on the queries as low as possible and prevent + your DNS server from being attacked so it fails and content is permitted. + """ + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.Config + + require Logger + + @query_retries 1 + @query_timeout 500 + + @impl true + def filter(%{"actor" => actor} = object) do + actor_info = URI.parse(actor) + + with {:ok, object} <- check_rbl(actor_info, object) do + {:ok, object} + else + _ -> {:reject, "[DNSRBLPolicy]"} + end + end + + @impl true + def filter(object), do: {:ok, object} + + @impl true + def describe do + mrf_dnsrbl = + Config.get(:mrf_dnsrbl) + |> Enum.into(%{}) + + {:ok, %{mrf_dnsrbl: mrf_dnsrbl}} + end + + @impl true + def config_description do + %{ + key: :mrf_dnsrbl, + related_policy: "Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy", + label: "MRF DNSRBL", + description: "DNS RealTime Blackhole Policy", + children: [ + %{ + key: :nameserver, + type: {:string}, + description: "DNSRBL Nameserver to Query (IP or hostame)", + suggestions: ["127.0.0.1"] + }, + %{ + key: :port, + type: {:string}, + description: "Nameserver port", + suggestions: ["53"] + }, + %{ + key: :zone, + type: {:string}, + description: "Root zone for querying", + suggestions: ["bl.pleroma.com"] + } + ] + } + end + + defp check_rbl(%{host: actor_host}, object) do + with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()), + zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do + query = + Enum.join([actor_host, zone], ".") + |> String.to_charlist() + + rbl_response = rblquery(query) + + if Enum.empty?(rbl_response) do + {:ok, object} + else + Task.start(fn -> + reason = rblquery(query, :txt) || "undefined" + + Logger.warning( + "DNSRBL Rejected activity from #{actor_host} for reason: #{inspect(reason)}" + ) + end) + + :error + end + else + _ -> {:ok, object} + end + end + + defp get_rblhost_ip(rblhost) do + case rblhost |> String.to_charlist() |> :inet_parse.address() do + {:ok, _} -> rblhost |> String.to_charlist() |> :inet_parse.address() + _ -> {:ok, rblhost |> String.to_charlist() |> :inet_res.lookup(:in, :a) |> Enum.random()} + end + end + + defp rblquery(query, type \\ :a) do + config = Config.get([:mrf_dnsrbl]) + + case get_rblhost_ip(config[:nameserver]) do + {:ok, rblnsip} -> + :inet_res.lookup(query, :in, type, + nameservers: [{rblnsip, config[:port]}], + timeout: @query_timeout, + retry: @query_retries + ) + + _ -> + [] + end + end +end From 5e963736cee55aa8f4bb9d9fba451ff3864ddaa8 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 21 May 2023 15:26:02 -0500 Subject: [PATCH 143/161] Add AntiMentionSpamPolicy --- .../mrf/anti_mention_spam_policy.ex | 87 +++++++++++++++++++ .../mrf/anti_mention_spam_policy_test.exs | 65 ++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex create mode 100644 test/pleroma/web/activity_pub/mrf/anti_mention_spam_policy_test.exs diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex new file mode 100644 index 000000000..ad97a1552 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do + alias Pleroma.User + require Pleroma.Constants + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + defp user_has_followers?(%User{} = u), do: u.follower_count > 0 + defp user_has_posted?(%User{} = u), do: u.note_count > 0 + + defp user_has_age?(%User{} = u) do + now = NaiveDateTime.utc_now() + diff = u.inserted_at |> NaiveDateTime.diff(now, :second) + diff > :timer.seconds(30) + end + + defp good_reputation?(%User{} = u) do + user_has_age?(u) and user_has_followers?(u) and user_has_posted?(u) + end + + # copied from HellthreadPolicy + defp get_recipient_count(message) do + recipients = (message["to"] || []) ++ (message["cc"] || []) + + follower_collection = + User.get_cached_by_ap_id(message["actor"] || message["attributedTo"]).follower_address + + if Enum.member?(recipients, Pleroma.Constants.as_public()) do + recipients = + recipients + |> List.delete(Pleroma.Constants.as_public()) + |> List.delete(follower_collection) + + {:public, length(recipients)} + else + recipients = + recipients + |> List.delete(follower_collection) + + {:not_public, length(recipients)} + end + end + + defp object_has_recipients?(%{"object" => object} = activity) do + {_, object_count} = get_recipient_count(object) + {_, activity_count} = get_recipient_count(activity) + object_count + activity_count > 0 + end + + defp object_has_recipients?(object) do + {_, count} = get_recipient_count(object) + count > 0 + end + + @impl true + def filter(%{"type" => "Create", "actor" => actor} = activity) do + with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor), + {:has_mentions, true} <- {:has_mentions, object_has_recipients?(activity)}, + {:good_reputation, true} <- {:good_reputation, good_reputation?(u)} do + {:ok, activity} + else + {:ok, %User{local: true}} -> + {:ok, activity} + + {:has_mentions, false} -> + {:ok, activity} + + {:good_reputation, false} -> + {:reject, "[AntiMentionSpamPolicy] User rejected"} + + {:error, _} -> + {:reject, "[AntiMentionSpamPolicy] Failed to get or fetch user by ap_id"} + + e -> + {:reject, "[AntiMentionSpamPolicy] Unhandled error #{inspect(e)}"} + end + end + + # in all other cases, pass through + def filter(message), do: {:ok, message} + + @impl true + def describe, do: {:ok, %{}} +end diff --git a/test/pleroma/web/activity_pub/mrf/anti_mention_spam_policy_test.exs b/test/pleroma/web/activity_pub/mrf/anti_mention_spam_policy_test.exs new file mode 100644 index 000000000..63947858c --- /dev/null +++ b/test/pleroma/web/activity_pub/mrf/anti_mention_spam_policy_test.exs @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicyTest do + use Pleroma.DataCase + import Pleroma.Factory + alias Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy + + test "it allows posts without mentions" do + user = insert(:user, local: false) + assert user.note_count == 0 + + message = %{ + "type" => "Create", + "actor" => user.ap_id + } + + {:ok, _message} = AntiMentionSpamPolicy.filter(message) + end + + test "it allows posts from users with followers, posts, and age" do + user = + insert(:user, + local: false, + follower_count: 1, + note_count: 1, + inserted_at: ~N[1970-01-01 00:00:00] + ) + + message = %{ + "type" => "Create", + "actor" => user.ap_id + } + + {:ok, _message} = AntiMentionSpamPolicy.filter(message) + end + + test "it allows posts from local users" do + user = insert(:user, local: true) + + message = %{ + "type" => "Create", + "actor" => user.ap_id + } + + {:ok, _message} = AntiMentionSpamPolicy.filter(message) + end + + test "it rejects posts with mentions from users without followers" do + user = insert(:user, local: false, follower_count: 0) + + message = %{ + "type" => "Create", + "actor" => user.ap_id, + "object" => %{ + "to" => ["https://pleroma.soykaf.com/users/1"], + "cc" => ["https://pleroma.soykaf.com/users/1"], + "actor" => user.ap_id + } + } + + {:reject, _message} = AntiMentionSpamPolicy.filter(message) + end +end From 64cacc3694c0441d3f3f5886b301bbf93f590cb6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 21 May 2023 19:31:56 -0500 Subject: [PATCH 144/161] AntiMentionSpamPolicy: fix user age check --- lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex index ad97a1552..0cb3313b2 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex @@ -12,9 +12,8 @@ defp user_has_followers?(%User{} = u), do: u.follower_count > 0 defp user_has_posted?(%User{} = u), do: u.note_count > 0 defp user_has_age?(%User{} = u) do - now = NaiveDateTime.utc_now() - diff = u.inserted_at |> NaiveDateTime.diff(now, :second) - diff > :timer.seconds(30) + diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :second) + diff >= :timer.seconds(30) end defp good_reputation?(%User{} = u) do From 02d8ce8f0ba615fa0946064052113fb05dd0b6a2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 22 May 2023 15:50:34 -0500 Subject: [PATCH 145/161] AntiMentionSpamPolicy: remove followers check --- lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex index 0cb3313b2..9cdb2077f 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do @behaviour Pleroma.Web.ActivityPub.MRF.Policy - defp user_has_followers?(%User{} = u), do: u.follower_count > 0 defp user_has_posted?(%User{} = u), do: u.note_count > 0 defp user_has_age?(%User{} = u) do @@ -17,7 +16,7 @@ defp user_has_age?(%User{} = u) do end defp good_reputation?(%User{} = u) do - user_has_age?(u) and user_has_followers?(u) and user_has_posted?(u) + user_has_age?(u) and user_has_posted?(u) end # copied from HellthreadPolicy From 0d092a3d4fd89a7f8df30f080087bd24ce53c597 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 12:26:55 -0400 Subject: [PATCH 146/161] Changelog --- changelog.d/anti-mentionspam-mrf.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/anti-mentionspam-mrf.add diff --git a/changelog.d/anti-mentionspam-mrf.add b/changelog.d/anti-mentionspam-mrf.add new file mode 100644 index 000000000..9466f85f4 --- /dev/null +++ b/changelog.d/anti-mentionspam-mrf.add @@ -0,0 +1 @@ +Add Anti-mention Spam MRF backported from Rebased From cab6372d7a1bdf50436eff1b4023fd6e05586dbc Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 12:31:29 -0400 Subject: [PATCH 147/161] Make user age limit configurable Switch to milliseconds for consistency with other configuration options in codebase --- config/config.exs | 2 ++ .../web/activity_pub/mrf/anti_mention_spam_policy.ex | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index b93de52e1..51773d830 100644 --- a/config/config.exs +++ b/config/config.exs @@ -430,6 +430,8 @@ mention_parent: true, mention_quoted: true +config :pleroma, :mrf_antimentionspam, user_age_limit: 30_000 + config :pleroma, :rich_media, enabled: true, ignore_hosts: [], diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex index 9cdb2077f..531e75ce8 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do + alias Pleroma.Config alias Pleroma.User require Pleroma.Constants @@ -11,8 +12,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do defp user_has_posted?(%User{} = u), do: u.note_count > 0 defp user_has_age?(%User{} = u) do - diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :second) - diff >= :timer.seconds(30) + user_age_limit = Config.get([:mrf_antimentionspam, :user_age_limit], 30_000) + diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :millisecond) + diff >= user_age_limit end defp good_reputation?(%User{} = u) do From 1c699144d23aa4a86ff8b6ebef7d760ce9e3a4e2 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 21:26:40 +0400 Subject: [PATCH 148/161] HttpSecurityPlug: Don't allow unsafe-eval by default --- config/config.exs | 3 +- config/test.exs | 1 + lib/pleroma/application.ex | 7 +- lib/pleroma/web/plugs/http_security_plug.ex | 49 +++-- .../web/plugs/http_security_plug_test.exs | 208 ++++++++++++++---- 5 files changed, 204 insertions(+), 64 deletions(-) diff --git a/config/config.exs b/config/config.exs index 4752bbbde..f861daf04 100644 --- a/config/config.exs +++ b/config/config.exs @@ -519,7 +519,8 @@ sts: false, sts_max_age: 31_536_000, ct_max_age: 2_592_000, - referrer_policy: "same-origin" + referrer_policy: "same-origin", + allow_unsafe_eval: false config :cors_plug, max_age: 86_400, diff --git a/config/test.exs b/config/test.exs index 3345bb3a9..b5c9c6e4a 100644 --- a/config/test.exs +++ b/config/test.exs @@ -154,6 +154,7 @@ config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock +config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.UnstubbedConfigMock peer_module = if String.to_integer(System.otp_release()) >= 25 do diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d266d1836..0d9757b44 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Application do @name Mix.Project.config()[:name] @version Mix.Project.config()[:version] @repository Mix.Project.config()[:source_url] + @compile_env Mix.env() def name, do: @name def version, do: @version @@ -51,7 +52,11 @@ def start(_type, _args) do Pleroma.HTML.compile_scrubbers() Pleroma.Config.Oban.warn() Config.DeprecationWarnings.warn() - Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() + + if @compile_env != :test do + Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() + end + Pleroma.ApplicationRequirements.verify!() load_custom_modules() Pleroma.Docs.JSON.compile() diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index a27dcd0ab..a1dc6c02a 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -3,26 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do - alias Pleroma.Config import Plug.Conn require Logger + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + def init(opts), do: opts def call(conn, _options) do - if Config.get([:http_security, :enabled]) do + if @config_impl.get([:http_security, :enabled]) do conn |> merge_resp_headers(headers()) - |> maybe_send_sts_header(Config.get([:http_security, :sts])) + |> maybe_send_sts_header(@config_impl.get([:http_security, :sts])) else conn end end def primary_frontend do - with %{"name" => frontend} <- Config.get([:frontends, :primary]), - available <- Config.get([:frontends, :available]), + with %{"name" => frontend} <- @config_impl.get([:frontends, :primary]), + available <- @config_impl.get([:frontends, :available]), %{} = primary_frontend <- Map.get(available, frontend) do {:ok, primary_frontend} end @@ -37,8 +38,8 @@ def custom_http_frontend_headers do end def headers do - referrer_policy = Config.get([:http_security, :referrer_policy]) - report_uri = Config.get([:http_security, :report_uri]) + referrer_policy = @config_impl.get([:http_security, :referrer_policy]) + report_uri = @config_impl.get([:http_security, :report_uri]) custom_http_frontend_headers = custom_http_frontend_headers() headers = [ @@ -86,10 +87,10 @@ def headers do @csp_start [Enum.join(static_csp_rules, ";") <> ";"] defp csp_string do - scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] + scheme = @config_impl.get([Pleroma.Web.Endpoint, :url])[:scheme] static_url = Pleroma.Web.Endpoint.static_url() websocket_url = Pleroma.Web.Endpoint.websocket_url() - report_uri = Config.get([:http_security, :report_uri]) + report_uri = @config_impl.get([:http_security, :report_uri]) img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" @@ -97,8 +98,8 @@ defp csp_string do # Strict multimedia CSP enforcement only when MediaProxy is enabled {img_src, media_src, connect_src} = - if Config.get([:media_proxy, :enabled]) && - !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do + if @config_impl.get([:media_proxy, :enabled]) && + !@config_impl.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do sources = build_csp_multimedia_source_list() { @@ -115,17 +116,21 @@ defp csp_string do end connect_src = - if Config.get(:env) == :dev do + if @config_impl.get(:env) == :dev do [connect_src, " http://localhost:3035/"] else connect_src end script_src = - if Config.get(:env) == :dev do - "script-src 'self' 'unsafe-eval'" + if @config_impl.get([:http_security, :allow_unsafe_eval]) do + if @config_impl.get(:env) == :dev do + "script-src 'self' 'unsafe-eval'" + else + "script-src 'self' 'wasm-unsafe-eval'" + end else - "script-src 'self' 'wasm-unsafe-eval'" + "script-src 'self'" end report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"] @@ -161,11 +166,11 @@ defp build_csp_param_from_whitelist(url), do: url defp build_csp_multimedia_source_list do media_proxy_whitelist = [:media_proxy, :whitelist] - |> Config.get() + |> @config_impl.get() |> build_csp_from_whitelist([]) - captcha_method = Config.get([Pleroma.Captcha, :method]) - captcha_endpoint = Config.get([captcha_method, :endpoint]) + captcha_method = @config_impl.get([Pleroma.Captcha, :method]) + captcha_endpoint = @config_impl.get([captcha_method, :endpoint]) base_endpoints = [ @@ -173,7 +178,7 @@ defp build_csp_multimedia_source_list do [Pleroma.Upload, :base_url], [Pleroma.Uploaders.S3, :public_endpoint] ] - |> Enum.map(&Config.get/1) + |> Enum.map(&@config_impl.get/1) [captcha_endpoint | base_endpoints] |> Enum.map(&build_csp_param/1) @@ -200,7 +205,7 @@ defp build_csp_param(url) when is_binary(url) do end def warn_if_disabled do - unless Config.get([:http_security, :enabled]) do + unless Pleroma.Config.get([:http_security, :enabled]) do Logger.warning(" .i;;;;i. iYcviii;vXY: @@ -245,8 +250,8 @@ def warn_if_disabled do end defp maybe_send_sts_header(conn, true) do - max_age_sts = Config.get([:http_security, :sts_max_age]) - max_age_ct = Config.get([:http_security, :ct_max_age]) + max_age_sts = @config_impl.get([:http_security, :sts_max_age]) + max_age_ct = @config_impl.get([:http_security, :ct_max_age]) merge_resp_headers(conn, [ {"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"}, diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs index c79170382..80ad1fa7d 100644 --- a/test/pleroma/web/plugs/http_security_plug_test.exs +++ b/test/pleroma/web/plugs/http_security_plug_test.exs @@ -3,14 +3,52 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do - use Pleroma.Web.ConnCase + use Pleroma.Web.ConnCase, async: true alias Plug.Conn - describe "http security enabled" do - setup do: clear_config([:http_security, :enabled], true) + import Mox - test "it sends CSP headers when enabled", %{conn: conn} do + setup do + base_config = Pleroma.Config.get([:http_security]) + %{base_config: base_config} + end + + defp mock_config(config, additional \\ %{}) do + Pleroma.UnstubbedConfigMock + |> stub(:get, fn + [:http_security, key] -> config[key] + key -> additional[key] + end) + end + + describe "http security enabled" do + setup %{base_config: base_config} do + %{base_config: Keyword.put(base_config, :enabled, true)} + end + + test "it does not contain unsafe-eval", %{conn: conn, base_config: base_config} do + mock_config(base_config) + + conn = get(conn, "/api/v1/instance") + [header] = Conn.get_resp_header(conn, "content-security-policy") + refute header =~ ~r/unsafe-eval/ + end + + test "with allow_unsafe_eval set, it does contain it", %{conn: conn, base_config: base_config} do + base_config = + base_config + |> Keyword.put(:allow_unsafe_eval, true) + + mock_config(base_config) + + conn = get(conn, "/api/v1/instance") + [header] = Conn.get_resp_header(conn, "content-security-policy") + assert header =~ ~r/unsafe-eval/ + end + + test "it sends CSP headers when enabled", %{conn: conn, base_config: base_config} do + mock_config(base_config) conn = get(conn, "/api/v1/instance") refute Conn.get_resp_header(conn, "x-xss-protection") == [] @@ -22,8 +60,10 @@ test "it sends CSP headers when enabled", %{conn: conn} do refute Conn.get_resp_header(conn, "content-security-policy") == [] end - test "it sends STS headers when enabled", %{conn: conn} do - clear_config([:http_security, :sts], true) + test "it sends STS headers when enabled", %{conn: conn, base_config: base_config} do + base_config + |> Keyword.put(:sts, true) + |> mock_config() conn = get(conn, "/api/v1/instance") @@ -31,8 +71,10 @@ test "it sends STS headers when enabled", %{conn: conn} do refute Conn.get_resp_header(conn, "expect-ct") == [] end - test "it does not send STS headers when disabled", %{conn: conn} do - clear_config([:http_security, :sts], false) + test "it does not send STS headers when disabled", %{conn: conn, base_config: base_config} do + base_config + |> Keyword.put(:sts, false) + |> mock_config() conn = get(conn, "/api/v1/instance") @@ -40,19 +82,30 @@ test "it does not send STS headers when disabled", %{conn: conn} do assert Conn.get_resp_header(conn, "expect-ct") == [] end - test "referrer-policy header reflects configured value", %{conn: conn} do - resp = get(conn, "/api/v1/instance") + test "referrer-policy header reflects configured value", %{ + conn: conn, + base_config: base_config + } do + mock_config(base_config) + resp = get(conn, "/api/v1/instance") assert Conn.get_resp_header(resp, "referrer-policy") == ["same-origin"] - clear_config([:http_security, :referrer_policy], "no-referrer") + base_config + |> Keyword.put(:referrer_policy, "no-referrer") + |> mock_config resp = get(conn, "/api/v1/instance") assert Conn.get_resp_header(resp, "referrer-policy") == ["no-referrer"] end - test "it sends `report-to` & `report-uri` CSP response headers", %{conn: conn} do + test "it sends `report-to` & `report-uri` CSP response headers", %{ + conn: conn, + base_config: base_config + } do + mock_config(base_config) + conn = get(conn, "/api/v1/instance") [csp] = Conn.get_resp_header(conn, "content-security-policy") @@ -65,7 +118,11 @@ test "it sends `report-to` & `report-uri` CSP response headers", %{conn: conn} d "{\"endpoints\":[{\"url\":\"https://endpoint.com\"}],\"group\":\"csp-endpoint\",\"max-age\":10886400}" end - test "default values for img-src and media-src with disabled media proxy", %{conn: conn} do + test "default values for img-src and media-src with disabled media proxy", %{ + conn: conn, + base_config: base_config + } do + mock_config(base_config) conn = get(conn, "/api/v1/instance") [csp] = Conn.get_resp_header(conn, "content-security-policy") @@ -73,60 +130,129 @@ test "default values for img-src and media-src with disabled media proxy", %{con assert csp =~ "img-src 'self' data: blob: https:;" end - test "it sets the Service-Worker-Allowed header", %{conn: conn} do - clear_config([:http_security, :enabled], true) - clear_config([:frontends, :primary], %{"name" => "fedi-fe", "ref" => "develop"}) + test "it sets the Service-Worker-Allowed header", %{conn: conn, base_config: base_config} do + base_config + |> Keyword.put(:enabled, true) - clear_config([:frontends, :available], %{ - "fedi-fe" => %{ - "name" => "fedi-fe", - "custom-http-headers" => [{"service-worker-allowed", "/"}] - } - }) + additional_config = + %{} + |> Map.put([:frontends, :primary], %{"name" => "fedi-fe", "ref" => "develop"}) + |> Map.put( + [:frontends, :available], + %{ + "fedi-fe" => %{ + "name" => "fedi-fe", + "custom-http-headers" => [{"service-worker-allowed", "/"}] + } + } + ) + mock_config(base_config, additional_config) conn = get(conn, "/api/v1/instance") assert Conn.get_resp_header(conn, "service-worker-allowed") == ["/"] end end describe "img-src and media-src" do - setup do - clear_config([:http_security, :enabled], true) - clear_config([:media_proxy, :enabled], true) - clear_config([:media_proxy, :proxy_opts, :redirect_on_failure], false) + setup %{base_config: base_config} do + base_config = + base_config + |> Keyword.put(:enabled, true) + + additional_config = + %{} + |> Map.put([:media_proxy, :enabled], true) + |> Map.put([:media_proxy, :proxy_opts, :redirect_on_failure], false) + |> Map.put([:media_proxy, :whitelist], []) + + %{base_config: base_config, additional_config: additional_config} end - test "media_proxy with base_url", %{conn: conn} do + test "media_proxy with base_url", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do url = "https://example.com" - clear_config([:media_proxy, :base_url], url) + + additional_config = + additional_config + |> Map.put([:media_proxy, :base_url], url) + + mock_config(base_config, additional_config) + assert_media_img_src(conn, url) end - test "upload with base url", %{conn: conn} do + test "upload with base url", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do url = "https://example2.com" - clear_config([Pleroma.Upload, :base_url], url) + + additional_config = + additional_config + |> Map.put([Pleroma.Upload, :base_url], url) + + mock_config(base_config, additional_config) + assert_media_img_src(conn, url) end - test "with S3 public endpoint", %{conn: conn} do + test "with S3 public endpoint", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do url = "https://example3.com" - clear_config([Pleroma.Uploaders.S3, :public_endpoint], url) + + additional_config = + additional_config + |> Map.put([Pleroma.Uploaders.S3, :public_endpoint], url) + + mock_config(base_config, additional_config) assert_media_img_src(conn, url) end - test "with captcha endpoint", %{conn: conn} do - clear_config([Pleroma.Captcha.Mock, :endpoint], "https://captcha.com") + test "with captcha endpoint", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do + additional_config = + additional_config + |> Map.put([Pleroma.Captcha.Mock, :endpoint], "https://captcha.com") + |> Map.put([Pleroma.Captcha, :method], Pleroma.Captcha.Mock) + + mock_config(base_config, additional_config) assert_media_img_src(conn, "https://captcha.com") end - test "with media_proxy whitelist", %{conn: conn} do - clear_config([:media_proxy, :whitelist], ["https://example6.com", "https://example7.com"]) + test "with media_proxy whitelist", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do + additional_config = + additional_config + |> Map.put([:media_proxy, :whitelist], ["https://example6.com", "https://example7.com"]) + + mock_config(base_config, additional_config) assert_media_img_src(conn, "https://example7.com https://example6.com") end # TODO: delete after removing support bare domains for media proxy whitelist - test "with media_proxy bare domains whitelist (deprecated)", %{conn: conn} do - clear_config([:media_proxy, :whitelist], ["example4.com", "example5.com"]) + test "with media_proxy bare domains whitelist (deprecated)", %{ + conn: conn, + base_config: base_config, + additional_config: additional_config + } do + additional_config = + additional_config + |> Map.put([:media_proxy, :whitelist], ["example4.com", "example5.com"]) + + mock_config(base_config, additional_config) assert_media_img_src(conn, "example5.com example4.com") end end @@ -138,8 +264,10 @@ defp assert_media_img_src(conn, url) do assert csp =~ "img-src 'self' data: blob: #{url};" end - test "it does not send CSP headers when disabled", %{conn: conn} do - clear_config([:http_security, :enabled], false) + test "it does not send CSP headers when disabled", %{conn: conn, base_config: base_config} do + base_config + |> Keyword.put(:enabled, false) + |> mock_config conn = get(conn, "/api/v1/instance") From fc7ce339edc40cb791d321a20f01f2568337b845 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 21:28:20 +0400 Subject: [PATCH 149/161] Cheatsheet: Add allow_unsafe_eval --- docs/configuration/cheatsheet.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index ca2ce6369..78997c4db 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -472,6 +472,7 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start * ``ct_max_age``: The maximum age for the `Expect-CT` header if sent. * ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`. * ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header. +* `allow_unsafe_eval`: Adds `wasm-unsafe-eval` to the CSP header. Needed for some non-essential frontend features like Flash emulation. ### Pleroma.Web.Plugs.RemoteIp From c67b41415b369d67c25356205bf69de2d99a291c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 11 Jun 2023 20:24:18 +0400 Subject: [PATCH 150/161] Changelog: Add changelog entry. --- changelog.d/3904.security | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3904.security diff --git a/changelog.d/3904.security b/changelog.d/3904.security new file mode 100644 index 000000000..04836d4e8 --- /dev/null +++ b/changelog.d/3904.security @@ -0,0 +1 @@ +HTTP Security: By default, don't allow unsafe-eval. The setting needs to be changed to allow Flash emulation. From 0847d9ebafa38007aeef0a6677588211994ab546 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 14:35:30 +0000 Subject: [PATCH 151/161] Oban queue simplification --- changelog.d/oban-queues.change | 1 + config/config.exs | 12 +------ lib/pleroma/scheduled_activity.ex | 2 +- lib/pleroma/web/federator.ex | 2 +- .../workers/attachments_cleanup_worker.ex | 2 +- lib/pleroma/workers/backup_worker.ex | 2 +- .../workers/cron/new_users_digest_worker.ex | 2 +- lib/pleroma/workers/mailer_worker.ex | 2 +- lib/pleroma/workers/mute_expire_worker.ex | 2 +- lib/pleroma/workers/poll_worker.ex | 2 +- lib/pleroma/workers/purge_expired_activity.ex | 4 +-- lib/pleroma/workers/purge_expired_filter.ex | 4 +-- lib/pleroma/workers/purge_expired_token.ex | 2 +- lib/pleroma/workers/remote_fetcher_worker.ex | 2 +- .../workers/rich_media_expiration_worker.ex | 2 +- .../workers/scheduled_activity_worker.ex | 2 +- .../20240527144418_oban_queues_refactor.exs | 32 +++++++++++++++++++ 17 files changed, 50 insertions(+), 27 deletions(-) create mode 100644 changelog.d/oban-queues.change create mode 100644 priv/repo/migrations/20240527144418_oban_queues_refactor.exs diff --git a/changelog.d/oban-queues.change b/changelog.d/oban-queues.change new file mode 100644 index 000000000..16df6409a --- /dev/null +++ b/changelog.d/oban-queues.change @@ -0,0 +1 @@ +Oban queues have refactored to simplify the queue design diff --git a/config/config.exs b/config/config.exs index b93de52e1..b52021373 100644 --- a/config/config.exs +++ b/config/config.exs @@ -574,24 +574,14 @@ log: false, queues: [ activity_expiration: 10, - token_expiration: 5, - filter_expiration: 1, - backup: 1, federator_incoming: 5, federator_outgoing: 5, ingestion_queue: 50, web_push: 50, - mailer: 10, transmogrifier: 20, - scheduled_activities: 10, - poll_notifications: 10, background: 5, - remote_fetcher: 2, - attachments_cleanup: 1, - new_users_digest: 1, - mute_expire: 5, search_indexing: [limit: 10, paused: true], - rich_media_expiration: 2 + slow: 1 ], plugins: [Oban.Plugins.Pruner], crontab: [ diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 63c6cb45b..c361d7d89 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -204,7 +204,7 @@ def due_activities(offset \\ 0) do def job_query(scheduled_activity_id) do from(j in Oban.Job, - where: j.queue == "scheduled_activities", + where: j.queue == "federator_outgoing", where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id)) ) end diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 1f2c3835a..4b30fd21d 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -44,7 +44,7 @@ def incoming_ap_doc(%{params: params, req_headers: req_headers}) do end def incoming_ap_doc(%{"type" => "Delete"} = params) do - ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3) + ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow) end def incoming_ap_doc(params) do diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 4c1764053..0b570b70b 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do alias Pleroma.Object alias Pleroma.Repo - use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup" + use Pleroma.Workers.WorkerHelper, queue: "slow" @impl Oban.Worker def perform(%Job{ diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index a485ddb4b..54ac31a3c 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.BackupWorker do - use Oban.Worker, queue: :backup, max_attempts: 1 + use Oban.Worker, queue: :slow, max_attempts: 1 alias Oban.Job alias Pleroma.User.Backup diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 1c3e445aa..d2abb2d3b 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do import Ecto.Query - use Pleroma.Workers.WorkerHelper, queue: "mailer" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(_job) do diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index 940716558..652bf77e0 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MailerWorker do - use Pleroma.Workers.WorkerHelper, queue: "mailer" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex index 8ce458d48..8ad287a7f 100644 --- a/lib/pleroma/workers/mute_expire_worker.ex +++ b/lib/pleroma/workers/mute_expire_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MuteExpireWorker do - use Pleroma.Workers.WorkerHelper, queue: "mute_expire" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index 022d026f8..70df54193 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.PollWorker do @moduledoc """ Generates notifications when a poll ends. """ - use Pleroma.Workers.WorkerHelper, queue: "poll_notifications" + use Pleroma.Workers.WorkerHelper, queue: "background" alias Pleroma.Activity alias Pleroma.Notification diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index e554684fe..a65593b6e 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do Worker which purges expired activity. """ - use Oban.Worker, queue: :activity_expiration, max_attempts: 1, unique: [period: :infinity] + use Oban.Worker, queue: :slow, max_attempts: 1, unique: [period: :infinity] import Ecto.Query @@ -59,7 +59,7 @@ defp find_user(ap_id) do def get_expiration(id) do from(j in Oban.Job, where: j.state == "scheduled", - where: j.queue == "activity_expiration", + where: j.queue == "slow", where: fragment("?->>'activity_id' = ?", j.args, ^id) ) |> Pleroma.Repo.one() diff --git a/lib/pleroma/workers/purge_expired_filter.ex b/lib/pleroma/workers/purge_expired_filter.ex index 9114aeb7f..1f6931e4c 100644 --- a/lib/pleroma/workers/purge_expired_filter.ex +++ b/lib/pleroma/workers/purge_expired_filter.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do Worker which purges expired filters """ - use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [period: :infinity] + use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: :infinity] import Ecto.Query @@ -38,7 +38,7 @@ def timeout(_job), do: :timer.seconds(5) def get_expiration(id) do from(j in Job, where: j.state == "scheduled", - where: j.queue == "filter_expiration", + where: j.queue == "background", where: fragment("?->'filter_id' = ?", j.args, ^id) ) |> Repo.one() diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex index 2ccd9e80b..1854bf561 100644 --- a/lib/pleroma/workers/purge_expired_token.ex +++ b/lib/pleroma/workers/purge_expired_token.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredToken do Worker which purges expired OAuth tokens """ - use Oban.Worker, queue: :token_expiration, max_attempts: 1 + use Oban.Worker, queue: :background, max_attempts: 1 @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) :: {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()} diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index c26418483..ed04c54b2 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do alias Pleroma.Object.Fetcher - use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do diff --git a/lib/pleroma/workers/rich_media_expiration_worker.ex b/lib/pleroma/workers/rich_media_expiration_worker.ex index d7ae497a7..0b74687cf 100644 --- a/lib/pleroma/workers/rich_media_expiration_worker.ex +++ b/lib/pleroma/workers/rich_media_expiration_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.RichMediaExpirationWorker do alias Pleroma.Web.RichMedia.Card use Oban.Worker, - queue: :rich_media_expiration + queue: :background @impl Oban.Worker def perform(%Job{args: %{"url" => url} = _args}) do diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index 4df84d00f..ab62686f4 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do The worker to post scheduled activity. """ - use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" + use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" alias Pleroma.Repo alias Pleroma.ScheduledActivity diff --git a/priv/repo/migrations/20240527144418_oban_queues_refactor.exs b/priv/repo/migrations/20240527144418_oban_queues_refactor.exs new file mode 100644 index 000000000..64ee28dfd --- /dev/null +++ b/priv/repo/migrations/20240527144418_oban_queues_refactor.exs @@ -0,0 +1,32 @@ +defmodule Pleroma.Repo.Migrations.ObanQueuesRefactor do + use Ecto.Migration + + @changed_queues [ + {"attachments_cleanup", "slow"}, + {"mailer", "background"}, + {"mute_expire", "background"}, + {"poll_notifications", "background"}, + {"activity_expiration", "slow"}, + {"filter_expiration", "background"}, + {"token_expiration", "background"}, + {"remote_fetcher", "background"}, + {"rich_media_expiration", "background"} + ] + + def up do + Enum.each(@changed_queues, fn {old, new} -> + execute("UPDATE oban_jobs SET queue = '#{new}' WHERE queue = '#{old}';") + end) + + # Handled special as reverting this would not be ideal and leaving it is harmless + execute( + "UPDATE oban_jobs SET queue = 'federator_outgoing' WHERE queue = 'scheduled_activities';" + ) + end + + def down do + # Just move all slow queue jobs to background queue if we are reverting + # as the slow queue will not be processing jobs + execute("UPDATE oban_jobs SET queue = 'background' WHERE queue = 'slow';") + end +end From f63e44b8bc8e4e2f21fe21f1407a85d072dcab6d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 13:46:15 -0400 Subject: [PATCH 152/161] Fix Oban related tests --- test/pleroma/scheduled_activity_test.exs | 3 +-- .../scheduled_activity_controller_test.exs | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/test/pleroma/scheduled_activity_test.exs b/test/pleroma/scheduled_activity_test.exs index 4818e8bcf..aaf643cfc 100644 --- a/test/pleroma/scheduled_activity_test.exs +++ b/test/pleroma/scheduled_activity_test.exs @@ -31,8 +31,7 @@ test "scheduled activities with jobs when ScheduledActivity enabled" do {:ok, sa1} = ScheduledActivity.create(user, attrs) {:ok, sa2} = ScheduledActivity.create(user, attrs) - jobs = - Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args)) + jobs = Repo.all(from(j in Oban.Job, where: j.queue == "federator_outgoing", select: j.args)) assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}] end diff --git a/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs index 632242221..2d6b2aee2 100644 --- a/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/scheduled_activity_controller_test.exs @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do + use Oban.Testing, repo: Pleroma.Repo use Pleroma.Web.ConnCase, async: true alias Pleroma.Repo @@ -78,7 +79,7 @@ test "updates a scheduled activity" do } ) - job = Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) + job = Repo.one(from(j in Oban.Job, where: j.queue == "federator_outgoing")) assert job.args == %{"activity_id" => scheduled_activity.id} assert DateTime.truncate(job.scheduled_at, :second) == to_datetime(scheduled_at) @@ -124,9 +125,11 @@ test "deletes a scheduled activity" do } ) - job = Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) - - assert job.args == %{"activity_id" => scheduled_activity.id} + assert_enqueued( + worker: Pleroma.Workers.ScheduledActivityWorker, + args: %{"activity_id" => scheduled_activity.id}, + queue: :federator_outgoing + ) res_conn = conn @@ -135,7 +138,11 @@ test "deletes a scheduled activity" do assert %{} = json_response_and_validate_schema(res_conn, 200) refute Repo.get(ScheduledActivity, scheduled_activity.id) - refute Repo.get(Oban.Job, job.id) + + refute_enqueued( + worker: Pleroma.Workers.ScheduledActivityWorker, + args: %{"activity_id" => scheduled_activity.id} + ) res_conn = conn From 29eac86dc0bb246e983afe4209332194bf11bed0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 13:53:22 -0400 Subject: [PATCH 153/161] Logger metadata changelog --- changelog.d/logger-metadata.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/logger-metadata.add diff --git a/changelog.d/logger-metadata.add b/changelog.d/logger-metadata.add new file mode 100644 index 000000000..6c627a972 --- /dev/null +++ b/changelog.d/logger-metadata.add @@ -0,0 +1 @@ +Logger metadata is now attached to some logs to help with troubleshooting and analysis From 6b8c15a4a1fdbc2a2a4ef194d9519e717470c632 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 14:11:42 -0400 Subject: [PATCH 154/161] Remove MediaProxyWarmingPolicy config for ConcurrentLimiter as we are not using it --- config/config.exs | 1 - .../web/activity_pub/mrf/media_proxy_warming_policy.ex | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/config/config.exs b/config/config.exs index a025defeb..3c35f439a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -902,7 +902,6 @@ config :pleroma, ConcurrentLimiter, [ {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, - {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}, {Pleroma.Search, [max_running: 30, max_waiting: 50]} ] diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index e5eb6896a..0c5b53def 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -22,11 +22,7 @@ defp prefetch(url) do Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}") - if Pleroma.Config.get(:env) == :test do - fetch(prefetch_url) - else - Task.start(fn -> fetch(prefetch_url) end) - end + fetch(prefetch_url) end end From ba511a30b923cc9248686702f75a4041d69c7bee Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 14:12:38 -0400 Subject: [PATCH 155/161] RichMedia use of ConcurrentLimiter was removed in the refactor --- config/config.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 3c35f439a..e7e751d8a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -901,7 +901,6 @@ process_chunk_size: 100 config :pleroma, ConcurrentLimiter, [ - {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, {Pleroma.Search, [max_running: 30, max_waiting: 50]} ] From 81e44ced0c7251b5a6b585f297e1e00fad08c6d1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 22:13:20 +0400 Subject: [PATCH 156/161] HTTPSecurityPlug: Fix tests --- config/test.exs | 2 +- lib/pleroma/web/plugs/http_security_plug.ex | 4 ++-- test/pleroma/web/plugs/http_security_plug_test.exs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/test.exs b/config/test.exs index b5c9c6e4a..6c88ad3c6 100644 --- a/config/test.exs +++ b/config/test.exs @@ -154,7 +154,7 @@ config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock -config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.UnstubbedConfigMock +config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.StaticStubbedConfigMock peer_module = if String.to_integer(System.otp_release()) >= 25 do diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index a1dc6c02a..38f6c511e 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -116,7 +116,7 @@ defp csp_string do end connect_src = - if @config_impl.get(:env) == :dev do + if @config_impl.get([:env]) == :dev do [connect_src, " http://localhost:3035/"] else connect_src @@ -124,7 +124,7 @@ defp csp_string do script_src = if @config_impl.get([:http_security, :allow_unsafe_eval]) do - if @config_impl.get(:env) == :dev do + if @config_impl.get([:env]) == :dev do "script-src 'self' 'unsafe-eval'" else "script-src 'self' 'wasm-unsafe-eval'" diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs index 80ad1fa7d..11a351a41 100644 --- a/test/pleroma/web/plugs/http_security_plug_test.exs +++ b/test/pleroma/web/plugs/http_security_plug_test.exs @@ -15,7 +15,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do end defp mock_config(config, additional \\ %{}) do - Pleroma.UnstubbedConfigMock + Pleroma.StaticStubbedConfigMock |> stub(:get, fn [:http_security, key] -> config[key] key -> additional[key] From bb86a01b9b5889155f60f08143755dd101d925f0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 15:20:47 -0400 Subject: [PATCH 157/161] Credo --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index d2b2cae0b..e6161455d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -522,7 +522,7 @@ defp set_requester_reachable(%Plug.Conn{} = conn, _) do conn end - defp log_inbox_metadata(conn = %{params: %{"actor" => actor, "type" => type}}, _) do + defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do Logger.metadata(actor: actor, type: type) conn end From 73d58c22d4a9a87539cf1c3a33083464fc4b8540 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 May 2024 08:09:19 +0400 Subject: [PATCH 158/161] Linting --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index d2b2cae0b..e6161455d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -522,7 +522,7 @@ defp set_requester_reachable(%Plug.Conn{} = conn, _) do conn end - defp log_inbox_metadata(conn = %{params: %{"actor" => actor, "type" => type}}, _) do + defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do Logger.metadata(actor: actor, type: type) conn end From f5978da67633110acbc6494ff6f07b3f07424779 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 May 2024 14:00:25 +0400 Subject: [PATCH 159/161] HTTPSignaturePlugTest: Rewrite to use mox. --- config/test.exs | 4 + lib/pleroma/http_signatures_api.ex | 4 + lib/pleroma/web/plugs/http_signature_plug.ex | 19 +- .../web/plugs/http_signature_plug_test.exs | 219 +++++++++--------- test/support/data_case.ex | 1 + test/support/http_signatures_proxy.ex | 9 + test/support/mocks.ex | 1 + 7 files changed, 144 insertions(+), 113 deletions(-) create mode 100644 lib/pleroma/http_signatures_api.ex create mode 100644 test/support/http_signatures_proxy.ex diff --git a/config/test.exs b/config/test.exs index 6c88ad3c6..0d4c82e0e 100644 --- a/config/test.exs +++ b/config/test.exs @@ -155,6 +155,10 @@ config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Web.Plugs.HTTPSecurityPlug, config_impl: Pleroma.StaticStubbedConfigMock +config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug, config_impl: Pleroma.StaticStubbedConfigMock + +config :pleroma, Pleroma.Web.Plugs.HTTPSignaturePlug, + http_signatures_impl: Pleroma.StubbedHTTPSignaturesMock peer_module = if String.to_integer(System.otp_release()) >= 25 do diff --git a/lib/pleroma/http_signatures_api.ex b/lib/pleroma/http_signatures_api.ex new file mode 100644 index 000000000..8e73dc98e --- /dev/null +++ b/lib/pleroma/http_signatures_api.ex @@ -0,0 +1,4 @@ +defmodule Pleroma.HTTPSignaturesAPI do + @callback validate_conn(conn :: Plug.Conn.t()) :: boolean + @callback signature_for_conn(conn :: Plug.Conn.t()) :: map +end diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex index 71b2a5f51..6bf2dd432 100644 --- a/lib/pleroma/web/plugs/http_signature_plug.ex +++ b/lib/pleroma/web/plugs/http_signature_plug.ex @@ -8,11 +8,17 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do import Plug.Conn import Phoenix.Controller, only: [get_format: 1, text: 2] - alias Pleroma.Config alias Pleroma.Web.ActivityPub.MRF require Logger + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + @http_signatures_impl Application.compile_env( + :pleroma, + [__MODULE__, :http_signatures_impl], + HTTPSignatures + ) + def init(options) do options end @@ -41,7 +47,7 @@ defp validate_signature(conn, request_target) do |> put_req_header("(request-target)", request_target) |> put_req_header("@request-target", request_target) - HTTPSignatures.validate_conn(conn) + @http_signatures_impl.validate_conn(conn) end defp validate_signature(conn) do @@ -108,9 +114,9 @@ defp has_signature_header?(conn) do defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do - if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do + if @config_impl.get([:activitypub, :authorized_fetch_mode], false) do exceptions = - Pleroma.Config.get([:activitypub, :authorized_fetch_mode_exceptions], []) + @config_impl.get([:activitypub, :authorized_fetch_mode_exceptions], []) |> Enum.map(&InetHelper.parse_cidr/1) if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do @@ -129,7 +135,8 @@ defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do defp maybe_filter_requests(%{halted: true} = conn), do: conn defp maybe_filter_requests(conn) do - if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do + if @config_impl.get([:activitypub, :authorized_fetch_mode], false) and + conn.assigns[:actor_id] do %{host: host} = URI.parse(conn.assigns.actor_id) if MRF.subdomain_match?(rejected_domains(), host) do @@ -145,7 +152,7 @@ defp maybe_filter_requests(conn) do end defp rejected_domains do - Config.get([:instance, :rejected_instances]) + @config_impl.get([:instance, :rejected_instances]) |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples() |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() end diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs index b871d956e..5f049dc45 100644 --- a/test/pleroma/web/plugs/http_signature_plug_test.exs +++ b/test/pleroma/web/plugs/http_signature_plug_test.exs @@ -3,89 +3,88 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do - use Pleroma.Web.ConnCase + use Pleroma.Web.ConnCase, async: true alias Pleroma.Web.Plugs.HTTPSignaturePlug + alias Pleroma.StubbedHTTPSignaturesMock, as: HTTPSignaturesMock + alias Pleroma.StaticStubbedConfigMock, as: ConfigMock import Plug.Conn import Phoenix.Controller, only: [put_format: 2] - import Mock + import Mox - test "it call HTTPSignatures to check validity if the actor signed it" do + test "it calls HTTPSignatures to check validity if the actor signed it" do params = %{"actor" => "http://mastodon.example.org/users/admin"} conn = build_conn(:get, "/doesntmattter", params) - with_mock HTTPSignatures, - validate_conn: fn _ -> true end, - signature_for_conn: fn _ -> - %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} - end do - conn = - conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> put_format("activity+json") - |> HTTPSignaturePlug.call(%{}) + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) - assert conn.assigns.valid_signature == true - assert conn.halted == false - assert called(HTTPSignatures.validate_conn(:_)) - end + conn = + conn + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == false end describe "requires a signature when `authorized_fetch_mode` is enabled" do setup do - clear_config([:activitypub, :authorized_fetch_mode], true) - params = %{"actor" => "http://mastodon.example.org/users/admin"} conn = build_conn(:get, "/doesntmattter", params) |> put_format("activity+json") [conn: conn] end - test "when signature header is present", %{conn: conn} do - with_mock HTTPSignatures, - validate_conn: fn _ -> false end, - signature_for_conn: fn _ -> - %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} - end do - conn = - conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> HTTPSignaturePlug.call(%{}) + test "when signature header is present", %{conn: orig_conn} do + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> [] end) - assert conn.assigns.valid_signature == false - assert conn.halted == true - assert conn.status == 401 - assert conn.state == :sent - assert conn.resp_body == "Request not signed" - assert called(HTTPSignatures.validate_conn(:_)) - end + HTTPSignaturesMock + |> expect(:validate_conn, 2, fn _ -> false end) - with_mock HTTPSignatures, - validate_conn: fn _ -> true end, - signature_for_conn: fn _ -> - %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} - end do - conn = - conn - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> HTTPSignaturePlug.call(%{}) + conn = + orig_conn + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> HTTPSignaturePlug.call(%{}) - assert conn.assigns.valid_signature == true - assert conn.halted == false - assert called(HTTPSignatures.validate_conn(:_)) - end + assert conn.assigns.valid_signature == false + assert conn.halted == true + assert conn.status == 401 + assert conn.state == :sent + assert conn.resp_body == "Request not signed" + + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) + + conn = + orig_conn + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == false end test "halts the connection when `signature` header is not present", %{conn: conn} do + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> [] end) + conn = HTTPSignaturePlug.call(conn, %{}) assert conn.assigns[:valid_signature] == nil assert conn.halted == true @@ -95,65 +94,71 @@ test "halts the connection when `signature` header is not present", %{conn: conn end test "exempts specific IPs from `authorized_fetch_mode_exceptions`", %{conn: conn} do - clear_config([:activitypub, :authorized_fetch_mode_exceptions], ["192.168.0.0/24"]) + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:activitypub, :authorized_fetch_mode_exceptions], [] -> + ["192.168.0.0/24"] + end) + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) - with_mock HTTPSignatures, validate_conn: fn _ -> false end do - conn = - conn - |> Map.put(:remote_ip, {192, 168, 0, 1}) - |> put_req_header( - "signature", - "keyId=\"http://mastodon.example.org/users/admin#main-key" - ) - |> HTTPSignaturePlug.call(%{}) + HTTPSignaturesMock + |> expect(:validate_conn, 2, fn _ -> false end) - assert conn.remote_ip == {192, 168, 0, 1} - assert conn.halted == false - assert called(HTTPSignatures.validate_conn(:_)) - end - end - end - - test "rejects requests from `rejected_instances` when `authorized_fetch_mode` is enabled" do - clear_config([:activitypub, :authorized_fetch_mode], true) - clear_config([:instance, :rejected_instances], [{"mastodon.example.org", "no reason"}]) - - with_mock HTTPSignatures, - validate_conn: fn _ -> true end, - signature_for_conn: fn _ -> - %{"keyId" => "http://mastodon.example.org/users/admin#main-key"} - end do conn = - build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + conn + |> Map.put(:remote_ip, {192, 168, 0, 1}) |> put_req_header( "signature", "keyId=\"http://mastodon.example.org/users/admin#main-key" ) - |> put_format("activity+json") |> HTTPSignaturePlug.call(%{}) - assert conn.assigns.valid_signature == true - assert conn.halted == true - assert called(HTTPSignatures.validate_conn(:_)) - end - - with_mock HTTPSignatures, - validate_conn: fn _ -> true end, - signature_for_conn: fn _ -> - %{"keyId" => "http://allowed.example.org/users/admin#main-key"} - end do - conn = - build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"}) - |> put_req_header( - "signature", - "keyId=\"http://allowed.example.org/users/admin#main-key" - ) - |> put_format("activity+json") - |> HTTPSignaturePlug.call(%{}) - - assert conn.assigns.valid_signature == true + assert conn.remote_ip == {192, 168, 0, 1} assert conn.halted == false - assert called(HTTPSignatures.validate_conn(:_)) end end + + test "rejects requests from `rejected_instances` when `authorized_fetch_mode` is enabled" do + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:instance, :rejected_instances] -> + [{"mastodon.example.org", "no reason"}] + end) + + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) + + conn = + build_conn(:get, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == true + + ConfigMock + |> expect(:get, fn [:activitypub, :authorized_fetch_mode], false -> true end) + |> expect(:get, fn [:instance, :rejected_instances] -> + [{"mastodon.example.org", "no reason"}] + end) + + HTTPSignaturesMock + |> expect(:validate_conn, fn _ -> true end) + + conn = + build_conn(:get, "/doesntmattter", %{"actor" => "http://allowed.example.org/users/admin"}) + |> put_req_header( + "signature", + "keyId=\"http://allowed.example.org/users/admin#main-key" + ) + |> put_format("activity+json") + |> HTTPSignaturePlug.call(%{}) + + assert conn.assigns.valid_signature == true + assert conn.halted == false + end end diff --git a/test/support/data_case.ex b/test/support/data_case.ex index 14403f0b8..52d4bef1a 100644 --- a/test/support/data_case.ex +++ b/test/support/data_case.ex @@ -116,6 +116,7 @@ def stub_pipeline do Mox.stub_with(Pleroma.Web.FederatorMock, Pleroma.Web.Federator) Mox.stub_with(Pleroma.ConfigMock, Pleroma.Config) Mox.stub_with(Pleroma.StaticStubbedConfigMock, Pleroma.Test.StaticConfig) + Mox.stub_with(Pleroma.StubbedHTTPSignaturesMock, Pleroma.Test.HTTPSignaturesProxy) end def ensure_local_uploader(context) do diff --git a/test/support/http_signatures_proxy.ex b/test/support/http_signatures_proxy.ex new file mode 100644 index 000000000..4c6b39d19 --- /dev/null +++ b/test/support/http_signatures_proxy.ex @@ -0,0 +1,9 @@ +defmodule Pleroma.Test.HTTPSignaturesProxy do + @behaviour Pleroma.HTTPSignaturesAPI + + @impl true + defdelegate validate_conn(conn), to: HTTPSignatures + + @impl true + defdelegate signature_for_conn(conn), to: HTTPSignatures +end diff --git a/test/support/mocks.ex b/test/support/mocks.ex index d906f0e1d..63cbc49ab 100644 --- a/test/support/mocks.ex +++ b/test/support/mocks.ex @@ -28,6 +28,7 @@ Mox.defmock(Pleroma.ConfigMock, for: Pleroma.Config.Getting) Mox.defmock(Pleroma.UnstubbedConfigMock, for: Pleroma.Config.Getting) Mox.defmock(Pleroma.StaticStubbedConfigMock, for: Pleroma.Config.Getting) +Mox.defmock(Pleroma.StubbedHTTPSignaturesMock, for: Pleroma.HTTPSignaturesAPI) Mox.defmock(Pleroma.LoggerMock, for: Pleroma.Logging) From 8066645f711f38986b3d0f9c0b34d6956563da6a Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 May 2024 14:20:48 +0400 Subject: [PATCH 160/161] Linting --- test/pleroma/web/plugs/http_signature_plug_test.exs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs index 5f049dc45..9d07270bb 100644 --- a/test/pleroma/web/plugs/http_signature_plug_test.exs +++ b/test/pleroma/web/plugs/http_signature_plug_test.exs @@ -4,13 +4,14 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlugTest do use Pleroma.Web.ConnCase, async: true - alias Pleroma.Web.Plugs.HTTPSignaturePlug - alias Pleroma.StubbedHTTPSignaturesMock, as: HTTPSignaturesMock - alias Pleroma.StaticStubbedConfigMock, as: ConfigMock - import Plug.Conn - import Phoenix.Controller, only: [put_format: 2] + alias Pleroma.StaticStubbedConfigMock, as: ConfigMock + alias Pleroma.StubbedHTTPSignaturesMock, as: HTTPSignaturesMock + alias Pleroma.Web.Plugs.HTTPSignaturePlug + import Mox + import Phoenix.Controller, only: [put_format: 2] + import Plug.Conn test "it calls HTTPSignatures to check validity if the actor signed it" do params = %{"actor" => "http://mastodon.example.org/users/admin"} From 335691bae1a002c1a3bec956884fe665114285ec Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 May 2024 14:38:44 +0400 Subject: [PATCH 161/161] Add changelog --- changelog.d/authorized-fetch-rejections.add | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/authorized-fetch-rejections.add diff --git a/changelog.d/authorized-fetch-rejections.add b/changelog.d/authorized-fetch-rejections.add new file mode 100644 index 000000000..66e15a979 --- /dev/null +++ b/changelog.d/authorized-fetch-rejections.add @@ -0,0 +1 @@ +Add an option to reject certain domains when authorized fetch is enabled.