moon
/
spc-pleroma
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'bugfix/mrf-ingestion' into 'develop' 

Bugfix: MRF and Pipeline Ingestion

See merge request pleroma/secteam/pleroma!15
This commit is contained in:
rinpatch 2020-09-17 12:13:36 +00:00
parent eff7f9892d
commit 22d49993d9
12 changed files with 121 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
CHANGELOG.md
View File

@ -3,6 +3,17 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## unreleased-patch - ???
### Security
- Fix most MRF rules either crashing or not being applied to objects passed into the Common Pipeline (ChatMessage, Question, Answer, Audio, Event)
### Fixed
- Welcome Chat messages preventing user registration with MRF Simple Policy applied to the local instance
- Mastodon API: the public timeline returning an error when the `reply_visibility` parameter is set to `self` for an unauthenticated user
## [2.1.1] - 2020-09-08
### Security

24
lib/pleroma/web/activity_pub/mrf.ex
View File

@ -5,16 +5,34 @@
defmodule Pleroma.Web.ActivityPub.MRF do
@callback filter(Map.t()) :: {:ok | :reject, Map.t()}
def filter(policies, %{} = object) do
def filter(policies, %{} = message) do
policies
|> Enum.reduce({:ok, object}, fn
policy, {:ok, object} -> policy.filter(object)
|> Enum.reduce({:ok, message}, fn
policy, {:ok, message} -> policy.filter(message)
_, error -> error
end)
end
def filter(%{} = object), do: get_policies() |> filter(object)
def pipeline_filter(%{} = message, meta) do
object = meta[:object_data]
ap_id = message["object"]
if object && ap_id do
with {:ok, message} <- filter(Map.put(message, "object", object)) do
meta = Keyword.put(meta, :object_data, message["object"])
{:ok, Map.put(message, "object", ap_id), meta}
else
{err, message} -> {err, message, meta}
end
else
{err, message} = filter(message)
{err, message, meta}
end
end
def get_policies do
Pleroma.Config.get([:mrf, :policies], []) |> get_policies()
end

59
lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
View File

@ -20,9 +20,17 @@ defp string_matches?(string, pattern) do
String.match?(string, pattern)
end
defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} = message) do
defp object_payload(%{} = object) do
[object["content"], object["summary"], object["name"]]
|> Enum.filter(& &1)
|> Enum.join("\n")
end
defp check_reject(%{"object" => %{} = object} = message) do
payload = object_payload(object)
if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
string_matches?(content, pattern) or string_matches?(summary, pattern)
string_matches?(payload, pattern)
end) do
{:reject, "[KeywordPolicy] Matches with rejected keyword"}
else
@ -30,12 +38,12 @@ defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} =
end
end
defp check_ftl_removal(
%{"to" => to, "object" => %{"content" => content, "summary" => summary}} = message
) do
defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do
payload = object_payload(object)
if Pleroma.Constants.as_public() in to and
Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
string_matches?(content, pattern) or string_matches?(summary, pattern)
string_matches?(payload, pattern)
end) do
to = List.delete(to, Pleroma.Constants.as_public())
cc = [Pleroma.Constants.as_public() | message["cc"] || []]
@ -51,35 +59,24 @@ defp check_ftl_removal(
end
end
defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do
content =
if is_binary(content) do
content
else
""
end
summary =
if is_binary(summary) do
summary
else
""
end
{content, summary} =
defp check_replace(%{"object" => %{} = object} = message) do
object =
["content", "name", "summary"]
|> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end)
|> Enum.reduce(object, fn field, object ->
data =
Enum.reduce(
Pleroma.Config.get([:mrf_keyword, :replace]),
{content, summary},
fn {pattern, replacement}, {content_acc, summary_acc} ->
{String.replace(content_acc, pattern, replacement),
String.replace(summary_acc, pattern, replacement)}
end
object[field],
fn {pat, repl}, acc -> String.replace(acc, pat, repl) end
)
{:ok,
message
|> put_in(["object", "content"], content)
|> put_in(["object", "summary"], summary)}
Map.put(object, field, data)
end)
message = Map.put(message, "object", object)
{:ok, message}
end
@impl true

3
lib/pleroma/web/activity_pub/mrf/subchain_policy.ex
View File

@ -28,8 +28,7 @@ def filter(%{"actor" => actor} = message) do
}"
)
subchain
|> MRF.filter(message)
MRF.filter(subchain, message)
else
_e -> {:ok, message}
end

8
lib/pleroma/web/activity_pub/pipeline.ex
View File

@ -26,13 +26,17 @@ def common_pipeline(object, meta) do
{:error, e} ->
{:error, e}
{:reject, e} ->
{:reject, e}
end
end
def do_common_pipeline(object, meta) do
with {_, {:ok, validated_object, meta}} <-
{:validate_object, ObjectValidator.validate(object, meta)},
{_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)},
{_, {:ok, mrfd_object, meta}} <-
{:mrf_object, MRF.pipeline_filter(validated_object, meta)},
{_, {:ok, activity, meta}} <-
{:persist_object, ActivityPub.persist(mrfd_object, meta)},
{_, {:ok, activity, meta}} <-
@ -40,7 +44,7 @@ def do_common_pipeline(object, meta) do
{_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do
{:ok, activity, meta}
else
{:mrf_object, {:reject, _}} -> {:ok, nil, meta}
{:mrf_object, {:reject, message, _}} -> {:reject, message}
e -> {:error, e}
end
end

3
lib/pleroma/web/api_spec/operations/chat_operation.ex
View File

@ -184,7 +184,8 @@ def post_chat_message_operation do
"application/json",
ChatMessage
),
400 => Operation.response("Bad Request", "application/json", ApiError)
400 => Operation.response("Bad Request", "application/json", ApiError),
422 => Operation.response("MRF Rejection", "application/json", ApiError)
},
security: [
%{

2
lib/pleroma/web/api_spec/operations/status_operation.ex
View File

@ -55,7 +55,7 @@ def create_operation do
"application/json",
%Schema{oneOf: [Status, ScheduledStatus]}
),
422 => Operation.response("Bad Request", "application/json", ApiError)
422 => Operation.response("Bad Request / MRF Rejection", "application/json", ApiError)
}
}
end

3
lib/pleroma/web/common_api/common_api.ex
View File

@ -49,6 +49,9 @@ def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ [])
local: true
)} do
{:ok, activity}
else
{:common_pipeline, {:reject, _} = e} -> e
e -> e
end
end

10
lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
View File

@ -90,6 +90,16 @@ def post_chat_message(
conn
|> put_view(MessageReferenceView)
|> render("show.json", chat_message_reference: cm_ref)
else
{:reject, message} ->
conn
|> put_status(:unprocessable_entity)
|> json(%{error: message})
{:error, message} ->
conn
|> put_status(:bad_request)
|> json(%{error: message})
end
end

16
test/web/activity_pub/pipeline_test.exs
View File

@ -26,7 +26,7 @@ test "when given an `object_data` in meta, Federation will receive a the origina
{
Pleroma.Web.ActivityPub.MRF,
[],
[filter: fn o -> {:ok, o} end]
[pipeline_filter: fn o, m -> {:ok, o, m} end]
},
{
Pleroma.Web.ActivityPub.ActivityPub,
@ -51,7 +51,7 @@ test "when given an `object_data` in meta, Federation will receive a the origina
Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity))
assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
refute called(Pleroma.Web.Federator.publish(activity))
@ -68,7 +68,7 @@ test "it goes through validation, filtering, persisting, side effects and federa
{
Pleroma.Web.ActivityPub.MRF,
[],
[filter: fn o -> {:ok, o} end]
[pipeline_filter: fn o, m -> {:ok, o, m} end]
},
{
Pleroma.Web.ActivityPub.ActivityPub,
@ -93,7 +93,7 @@ test "it goes through validation, filtering, persisting, side effects and federa
Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity))
assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
assert_called(Pleroma.Web.Federator.publish(activity))
@ -109,7 +109,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
{
Pleroma.Web.ActivityPub.MRF,
[],
[filter: fn o -> {:ok, o} end]
[pipeline_filter: fn o, m -> {:ok, o, m} end]
},
{
Pleroma.Web.ActivityPub.ActivityPub,
@ -131,7 +131,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity))
assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
end
@ -148,7 +148,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
{
Pleroma.Web.ActivityPub.MRF,
[],
[filter: fn o -> {:ok, o} end]
[pipeline_filter: fn o, m -> {:ok, o, m} end]
},
{
Pleroma.Web.ActivityPub.ActivityPub,
@ -170,7 +170,7 @@ test "it goes through validation, filtering, persisting, side effects without fe
Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
assert_called(Pleroma.Web.ActivityPub.ObjectValidator.validate(activity, meta))
assert_called(Pleroma.Web.ActivityPub.MRF.filter(activity))
assert_called(Pleroma.Web.ActivityPub.MRF.pipeline_filter(activity, meta))
assert_called(Pleroma.Web.ActivityPub.ActivityPub.persist(activity, meta))
assert_called(Pleroma.Web.ActivityPub.SideEffects.handle(activity, meta))
end

11
test/web/common_api/common_api_test.exs
View File

@ -213,6 +213,17 @@ test "it reject messages over the local limit" do
assert message == :content_too_long
end
test "it reject messages via MRF" do
clear_config([:mrf_keyword, :reject], ["GNO"])
clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy])
author = insert(:user)
recipient = insert(:user)
assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} ==
CommonAPI.post_chat_message(author, recipient, "GNO/Linux")
end
end
describe "unblocking" do

19
test/web/pleroma_api/controllers/chat_controller_test.exs
View File

@ -100,7 +100,7 @@ test "it fails if there is no content", %{conn: conn, user: user} do
|> post("/api/v1/pleroma/chats/#{chat.id}/messages")
|> json_response_and_validate_schema(400)
assert result
assert %{"error" => "no_content"} == result
end
test "it works with an attachment", %{conn: conn, user: user} do
@ -126,6 +126,23 @@ test "it works with an attachment", %{conn: conn, user: user} do
assert result["attachment"]
end
test "gets MRF reason when rejected", %{conn: conn, user: user} do
clear_config([:mrf_keyword, :reject], ["GNO"])
clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy])
other_user = insert(:user)
{:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
result =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "GNO/Linux"})
|> json_response_and_validate_schema(422)
assert %{"error" => "[KeywordPolicy] Matches with rejected keyword"} == result
end
end
describe "DELETE /api/v1/pleroma/chats/:id/messages/:message_id" do