From aedbbec88aa0a9a38e588eabfbecb8058652002b Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 11 Jul 2020 15:48:45 +0300 Subject: [PATCH 01/50] Add Pleroma.Utils.command_available?/1 and use where appropriate --- lib/pleroma/upload/filter/exiftool.ex | 9 ++++++++- lib/pleroma/utils.ex | 15 +++++++++++++++ mix.exs | 6 +++--- test/upload/filter/exiftool_test.exs | 2 ++ 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index c7fb6aefa..94622acd0 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -9,8 +9,15 @@ defmodule Pleroma.Upload.Filter.Exiftool do """ @behaviour Pleroma.Upload.Filter + require Logger + def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) + if Pleroma.Utils.command_available?("exiftool") do + System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) + else + Logger.warn("exiftool is not available, filter #{__MODULE__} skipped") + end + :ok end diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex index 6b8e3accf..21d1159be 100644 --- a/lib/pleroma/utils.ex +++ b/lib/pleroma/utils.ex @@ -9,4 +9,19 @@ def compile_dir(dir) when is_binary(dir) do |> Enum.map(&Path.join(dir, &1)) |> Kernel.ParallelCompiler.compile() end + + @doc """ + POSIX-compliant check if command is available in the system + + ## Examples + iex> command_available?("git") + true + iex> command_available?("wrongcmd") + false + + """ + @spec command_available?(String.t()) :: boolean() + def command_available?(command) do + match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"])) + end end diff --git a/mix.exs b/mix.exs index d7992ee37..5cd06e8fd 100644 --- a/mix.exs +++ b/mix.exs @@ -234,10 +234,10 @@ defp aliases do defp version(version) do identifier_filter = ~r/[^0-9a-z\-]+/i - {_cmdgit, cmdgit_err} = System.cmd("sh", ["-c", "command -v git"]) + git_available? = Pleroma.Utils.command_available?("git") git_pre_release = - if cmdgit_err == 0 do + if git_available? do {tag, tag_err} = System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true) @@ -263,7 +263,7 @@ defp version(version) do # Branch name as pre-release version component, denoted with a dot branch_name = - with 0 <- cmdgit_err, + with true <- git_available?, {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), branch_name <- String.trim(branch_name), branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, diff --git a/test/upload/filter/exiftool_test.exs b/test/upload/filter/exiftool_test.exs index a1b7e46cd..8ed7d650b 100644 --- a/test/upload/filter/exiftool_test.exs +++ b/test/upload/filter/exiftool_test.exs @@ -7,6 +7,8 @@ defmodule Pleroma.Upload.Filter.ExiftoolTest do alias Pleroma.Upload.Filter test "apply exiftool filter" do + assert Pleroma.Utils.command_available?("exiftool") + File.cp!( "test/fixtures/DSCN0010.jpg", "test/fixtures/DSCN0010_tmp.jpg" From 05187d497da1844005eaf96fbcab65840a578bb1 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 11 Jul 2020 16:09:46 +0300 Subject: [PATCH 02/50] One can not simply call application modules from mix.exs --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 5cd06e8fd..a775e54a4 100644 --- a/mix.exs +++ b/mix.exs @@ -234,7 +234,7 @@ defp aliases do defp version(version) do identifier_filter = ~r/[^0-9a-z\-]+/i - git_available? = Pleroma.Utils.command_available?("git") + git_available? = match?({_output, 0}, System.cmd("sh", ["-c", "command -v git"])) git_pre_release = if git_available? do From 45bd64e2a7a08377e260e93c8e1744166bfc133a Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 11 Jul 2020 18:11:23 +0300 Subject: [PATCH 03/50] Error in Filter.Exiftool if exiftool not found --- lib/pleroma/upload/filter/exiftool.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index 94622acd0..6a40e152f 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -14,11 +14,10 @@ defmodule Pleroma.Upload.Filter.Exiftool do def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do if Pleroma.Utils.command_available?("exiftool") do System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) + :ok else - Logger.warn("exiftool is not available, filter #{__MODULE__} skipped") + {:error, "exiftool command not found"} end - - :ok end def filter(_), do: :ok From 523f1b93a48d88ef8aa04ca17d51d1d0916b6093 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 11 Jul 2020 18:15:51 +0300 Subject: [PATCH 04/50] Remove Logger requirement --- lib/pleroma/upload/filter/exiftool.ex | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index 6a40e152f..e1b976c98 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -9,8 +9,6 @@ defmodule Pleroma.Upload.Filter.Exiftool do """ @behaviour Pleroma.Upload.Filter - require Logger - def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do if Pleroma.Utils.command_available?("exiftool") do System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) From 3116a75e80144dff79232c8676bd28ed285a14d9 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sat, 11 Jul 2020 18:22:03 +0300 Subject: [PATCH 05/50] Check if mogrify available before calling it --- lib/pleroma/upload/filter/mogrifun.ex | 9 ++++++--- lib/pleroma/upload/filter/mogrify.ex | 10 +++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex index 7d95577a4..8f362333d 100644 --- a/lib/pleroma/upload/filter/mogrifun.ex +++ b/lib/pleroma/upload/filter/mogrifun.ex @@ -35,9 +35,12 @@ defmodule Pleroma.Upload.Filter.Mogrifun do ] def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - Filter.Mogrify.do_filter(file, [Enum.random(@filters)]) - - :ok + if Pleroma.Utils.command_available?("mogrify") do + Filter.Mogrify.do_filter(file, [Enum.random(@filters)]) + :ok + else + {:error, "mogrify command not found"} + end end def filter(_), do: :ok diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex index 2eb758006..4bd0c2eb4 100644 --- a/lib/pleroma/upload/filter/mogrify.ex +++ b/lib/pleroma/upload/filter/mogrify.ex @@ -9,10 +9,14 @@ defmodule Pleroma.Upload.Filter.Mogrify do @type conversions :: conversion() | [conversion()] def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - filters = Pleroma.Config.get!([__MODULE__, :args]) + if Pleroma.Utils.command_available?("mogrify") do + filters = Pleroma.Config.get!([__MODULE__, :args]) - do_filter(file, filters) - :ok + do_filter(file, filters) + :ok + else + {:error, "mogrify command not found"} + end end def filter(_), do: :ok From 7bcd7a959519ea023b075142aa36005f80503ba4 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 11 Jun 2020 20:23:10 +0200 Subject: [PATCH 06/50] QuestionValidator: Create --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- .../web/activity_pub/object_validator.ex | 41 +++++++- .../object_validators/common_validations.ex | 4 +- .../create_question_validator.ex | 94 +++++++++++++++++++ .../object_validators/note_validator.ex | 2 +- .../question_options_validator.ex | 47 ++++++++++ .../object_validators/question_validator.ex | 89 ++++++++++++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 10 +- .../web/activity_pub/transmogrifier.ex | 17 +++- test/web/activity_pub/transmogrifier_test.exs | 2 +- 10 files changed, 297 insertions(+), 11 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex create mode 100644 lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex create mode 100644 lib/pleroma/web/activity_pub/object_validators/question_validator.ex diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index bc7b5d95a..462aa57a6 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -95,7 +95,7 @@ defp increase_poll_votes_if_vote(%{ defp increase_poll_votes_if_vote(_create_data), do: :noop - @object_types ["ChatMessage"] + @object_types ["ChatMessage", "Question"] @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} def persist(%{"type" => type} = object, meta) when type in @object_types do with {:ok, object} <- Object.create(object) do diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index df926829c..5cc66d7bd 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -16,10 +16,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.CreateQuestionValidator alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator @@ -105,17 +107,30 @@ def validate(%{"type" => "ChatMessage"} = object, meta) do end end + def validate(%{"type" => "Question"} = object, meta) do + with {:ok, object} <- + object + |> QuestionValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + def validate(%{"type" => "EmojiReact"} = object, meta) do with {:ok, object} <- object |> EmojiReactValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object |> Map.from_struct()) + object = stringify_keys(object) {:ok, object, meta} end end - def validate(%{"type" => "Create", "object" => object} = create_activity, meta) do + def validate( + %{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object} = create_activity, + meta + ) do with {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, create_activity} <- @@ -127,12 +142,27 @@ def validate(%{"type" => "Create", "object" => object} = create_activity, meta) end end + def validate( + %{"type" => "Create", "object" => %{"type" => "Question"} = object} = create_activity, + meta + ) do + with {:ok, object_data} <- cast_and_apply(object), + meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), + {:ok, create_activity} <- + create_activity + |> CreateQuestionValidator.cast_and_validate(meta) + |> Ecto.Changeset.apply_action(:insert) do + create_activity = stringify_keys(create_activity) + {:ok, create_activity, meta} + end + end + def validate(%{"type" => "Announce"} = object, meta) do with {:ok, object} <- object |> AnnounceValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object |> Map.from_struct()) + object = stringify_keys(object) {:ok, object, meta} end end @@ -141,8 +171,13 @@ def cast_and_apply(%{"type" => "ChatMessage"} = object) do ChatMessageValidator.cast_and_apply(object) end + def cast_and_apply(%{"type" => "Question"} = object) do + QuestionValidator.cast_and_apply(object) + end + def cast_and_apply(o), do: {:error, {:validator_not_set, o}} + # is_struct/1 isn't present in Elixir 1.8.x def stringify_keys(%{__struct__: _} = object) do object |> Map.from_struct() diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index aeef31945..e746b9360 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do alias Pleroma.Object alias Pleroma.User - def validate_recipients_presence(cng, fields \\ [:to, :cc]) do + def validate_any_presence(cng, fields) do non_empty = fields |> Enum.map(fn field -> get_field(cng, field) end) @@ -24,7 +24,7 @@ def validate_recipients_presence(cng, fields \\ [:to, :cc]) do fields |> Enum.reduce(cng, fn field, cng -> cng - |> add_error(field, "no recipients in any field") + |> add_error(field, "none of #{inspect(fields)} present") end) end end diff --git a/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex new file mode 100644 index 000000000..f09207418 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex @@ -0,0 +1,94 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +# Code based on CreateChatMessageValidator +# NOTES +# - Can probably be a generic create validator +# - doesn't embed, will only get the object id +defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateQuestionValidator do + use Ecto.Schema + + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.ObjectValidators.Types + + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + @primary_key false + + embedded_schema do + field(:id, Types.ObjectID, primary_key: true) + field(:actor, Types.ObjectID) + field(:type, :string) + field(:to, Types.Recipients, default: []) + field(:cc, Types.Recipients, default: []) + field(:object, Types.ObjectID) + end + + def cast_and_apply(data) do + data + |> cast_data + |> apply_action(:insert) + end + + def cast_data(data) do + cast(%__MODULE__{}, data, __schema__(:fields)) + end + + def cast_and_validate(data, meta \\ []) do + cast_data(data) + |> validate_data(meta) + end + + def validate_data(cng, meta \\ []) do + cng + |> validate_required([:actor, :type, :object]) + |> validate_inclusion(:type, ["Create"]) + |> validate_actor_presence() + |> validate_any_presence([:to, :cc]) + |> validate_recipients_match(meta) + |> validate_actors_match(meta) + |> validate_object_nonexistence() + end + + def validate_object_nonexistence(cng) do + cng + |> validate_change(:object, fn :object, object_id -> + if Object.get_cached_by_ap_id(object_id) do + [{:object, "The object to create already exists"}] + else + [] + end + end) + end + + def validate_actors_match(cng, meta) do + object_actor = meta[:object_data]["actor"] + + cng + |> validate_change(:actor, fn :actor, actor -> + if actor == object_actor do + [] + else + [{:actor, "Actor doesn't match with object actor"}] + end + end) + end + + def validate_recipients_match(cng, meta) do + object_recipients = meta[:object_data]["to"] || [] + + cng + |> validate_change(:to, fn :to, recipients -> + activity_set = MapSet.new(recipients) + object_set = MapSet.new(object_recipients) + + if MapSet.equal?(activity_set, object_set) do + [] + else + [{:to, "Recipients don't match with object recipients"}] + end + end) + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex index 56b93dde8..a65fe2354 100644 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -34,7 +34,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do field(:replies_count, :integer, default: 0) field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) - field(:inRepyTo, :string) + field(:inReplyTo, :string) field(:uri, ObjectValidators.Uri) field(:likes, {:array, :string}, default: []) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex new file mode 100644 index 000000000..9bc7e0cc0 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex @@ -0,0 +1,47 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do + use Ecto.Schema + + alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsRepliesValidator + + import Ecto.Changeset + + @primary_key false + + embedded_schema do + field(:name, :string) + embeds_one(:replies, QuestionOptionsRepliesValidator) + field(:type, :string) + end + + def changeset(struct, data) do + struct + |> cast(data, [:name, :type]) + |> cast_embed(:replies) + |> validate_inclusion(:type, ["Note"]) + |> validate_required([:name, :type]) + end +end + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsRepliesValidator do + use Ecto.Schema + + import Ecto.Changeset + + @primary_key false + + embedded_schema do + field(:totalItems, :integer) + field(:type, :string) + end + + def changeset(struct, data) do + struct + |> cast(data, __schema__(:fields)) + |> validate_inclusion(:type, ["Collection"]) + |> validate_required([:type]) + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex new file mode 100644 index 000000000..f94d79352 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -0,0 +1,89 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do + use Ecto.Schema + + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.Types + + import Ecto.Changeset + + @primary_key false + @derive Jason.Encoder + + # Extends from NoteValidator + embedded_schema do + field(:id, Types.ObjectID, primary_key: true) + field(:to, {:array, :string}, default: []) + field(:cc, {:array, :string}, default: []) + field(:bto, {:array, :string}, default: []) + field(:bcc, {:array, :string}, default: []) + # TODO: Write type + field(:tag, {:array, :map}, default: []) + field(:type, :string) + field(:content, :string) + field(:context, :string) + field(:actor, Types.ObjectID) + field(:attributedTo, Types.ObjectID) + field(:summary, :string) + field(:published, Types.DateTime) + # TODO: Write type + field(:emoji, :map, default: %{}) + field(:sensitive, :boolean, default: false) + # TODO: Write type + field(:attachment, {:array, :map}, default: []) + field(:replies_count, :integer, default: 0) + field(:like_count, :integer, default: 0) + field(:announcement_count, :integer, default: 0) + field(:inReplyTo, :string) + field(:uri, Types.Uri) + + field(:likes, {:array, :string}, default: []) + field(:announcements, {:array, :string}, default: []) + + # see if needed + field(:conversation, :string) + field(:context_id, :string) + + field(:closed, Types.DateTime) + field(:voters, {:array, Types.ObjectID}, default: []) + embeds_many(:anyOf, QuestionOptionsValidator) + embeds_many(:oneOf, QuestionOptionsValidator) + end + + def cast_and_apply(data) do + data + |> cast_data + |> apply_action(:insert) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> changeset(data) + end + + def changeset(struct, data) do + struct + |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf]) + |> cast_embed(:anyOf) + |> cast_embed(:oneOf) + end + + def validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Question"]) + |> validate_required([:id, :actor, :type, :content, :context]) + |> CommonValidations.validate_any_presence([:cc, :to]) + |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_any_presence([:oneOf, :anyOf]) + end +end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 1d2c296a5..a78ec411f 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -268,9 +268,15 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do end end + def handle_object_creation(%{"type" => "Question"} = object, meta) do + with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do + {:ok, object, meta} + end + end + # Nothing to do - def handle_object_creation(object) do - {:ok, object} + def handle_object_creation(object, meta) do + {:ok, object, meta} end defp undo_like(nil, object), do: delete_object(object) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index f37bcab3e..da5dc23bc 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -457,7 +457,7 @@ def handle_incoming( %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, options ) - when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer", "Audio"] do + when objtype in ["Article", "Event", "Note", "Video", "Page", "Answer", "Audio"] do actor = Containment.get_actor(data) with nil <- Activity.get_create_by_object_ap_id(object["id"]), @@ -613,6 +613,21 @@ def handle_incoming( |> handle_incoming(options) end + def handle_incoming( + %{"type" => "Create", "object" => %{"type" => "Question"} = object} = data, + _options + ) do + data = + data + |> Map.put("object", fix_object(object)) + |> fix_addressing() + + with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), + {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do + {:ok, activity} + end + end + def handle_incoming( %{"type" => "Create", "object" => %{"type" => "ChatMessage"}} = data, _options diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 248b410c6..73949b558 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -222,7 +222,7 @@ test "it works for incoming questions" do {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) - object = Object.normalize(activity) + object = Object.normalize(activity, false) assert Enum.all?(object.data["oneOf"], fn choice -> choice["name"] in [ From c5efaf6b00bf582a6eef7b5728fe5042f5fcc702 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 11 Jun 2020 20:43:01 +0200 Subject: [PATCH 07/50] AnswerValidator: Create --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- .../web/activity_pub/object_validator.ex | 20 ++++++- .../object_validators/answer_validator.ex | 58 +++++++++++++++++++ .../web/activity_pub/transmogrifier.ex | 7 ++- 4 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/answer_validator.ex diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 462aa57a6..d8cc8d24f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -95,7 +95,7 @@ defp increase_poll_votes_if_vote(%{ defp increase_poll_votes_if_vote(_create_data), do: :noop - @object_types ["ChatMessage", "Question"] + @object_types ["ChatMessage", "Question", "Answer"] @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} def persist(%{"type" => type} = object, meta) when type in @object_types do with {:ok, object} <- Object.create(object) do diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 5cc66d7bd..c89311187 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator @@ -117,6 +118,16 @@ def validate(%{"type" => "Question"} = object, meta) do end end + def validate(%{"type" => "Answer"} = object, meta) do + with {:ok, object} <- + object + |> AnswerValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + def validate(%{"type" => "EmojiReact"} = object, meta) do with {:ok, object} <- object @@ -143,9 +154,10 @@ def validate( end def validate( - %{"type" => "Create", "object" => %{"type" => "Question"} = object} = create_activity, + %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity, meta - ) do + ) + when objtype in ["Question", "Answer"] do with {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, create_activity} <- @@ -175,6 +187,10 @@ def cast_and_apply(%{"type" => "Question"} = object) do QuestionValidator.cast_and_apply(object) end + def cast_and_apply(%{"type" => "Answer"} = object) do + QuestionValidator.cast_and_apply(object) + end + def cast_and_apply(o), do: {:error, {:validator_not_set, o}} # is_struct/1 isn't present in Elixir 1.8.x diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex new file mode 100644 index 000000000..0b51eccfa --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex @@ -0,0 +1,58 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do + use Ecto.Schema + + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.ObjectValidators.Types + + import Ecto.Changeset + + @primary_key false + @derive Jason.Encoder + + # Extends from NoteValidator + embedded_schema do + field(:id, Types.ObjectID, primary_key: true) + field(:to, {:array, :string}, default: []) + field(:cc, {:array, :string}, default: []) + field(:bto, {:array, :string}, default: []) + field(:bcc, {:array, :string}, default: []) + field(:type, :string) + field(:name, :string) + field(:inReplyTo, :string) + field(:attributedTo, Types.ObjectID) + end + + def cast_and_apply(data) do + data + |> cast_data + |> apply_action(:insert) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> changeset(data) + end + + def changeset(struct, data) do + struct + |> cast(data, __schema__(:fields)) + end + + def validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Answer"]) + |> validate_required([:id, :inReplyTo, :name]) + |> CommonValidations.validate_any_presence([:cc, :to]) + |> CommonValidations.validate_actor_presence() + end +end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index da5dc23bc..9900602e4 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -457,7 +457,7 @@ def handle_incoming( %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, options ) - when objtype in ["Article", "Event", "Note", "Video", "Page", "Answer", "Audio"] do + when objtype in ["Article", "Event", "Note", "Video", "Page", "Audio"] do actor = Containment.get_actor(data) with nil <- Activity.get_create_by_object_ap_id(object["id"]), @@ -614,9 +614,10 @@ def handle_incoming( end def handle_incoming( - %{"type" => "Create", "object" => %{"type" => "Question"} = object} = data, + %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, _options - ) do + ) + when objtype in ["Question", "Answer"] do data = data |> Map.put("object", fix_object(object)) From 89a2433154b05c378f9c5b3224375049f3dbc8af Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 12 Jun 2020 21:22:05 +0200 Subject: [PATCH 08/50] QuestionOptionsValidator: inline schema for replies --- .../question_options_validator.ex | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex index 9bc7e0cc0..8291d7b9f 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex @@ -5,42 +5,32 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do use Ecto.Schema - alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsRepliesValidator - import Ecto.Changeset @primary_key false embedded_schema do field(:name, :string) - embeds_one(:replies, QuestionOptionsRepliesValidator) + + embeds_one :replies, Replies do + field(:totalItems, :integer) + field(:type, :string) + end + field(:type, :string) end def changeset(struct, data) do struct |> cast(data, [:name, :type]) - |> cast_embed(:replies) + |> cast_embed(:replies, with: &replies_changeset/2) |> validate_inclusion(:type, ["Note"]) |> validate_required([:name, :type]) end -end -defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsRepliesValidator do - use Ecto.Schema - - import Ecto.Changeset - - @primary_key false - - embedded_schema do - field(:totalItems, :integer) - field(:type, :string) - end - - def changeset(struct, data) do + def replies_changeset(struct, data) do struct - |> cast(data, __schema__(:fields)) + |> cast(data, [:totalItems, :type]) |> validate_inclusion(:type, ["Collection"]) |> validate_required([:type]) end From 10bd08ef07bd91995589ad37cb25e6889dac59b3 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 14 Jun 2020 22:25:04 +0200 Subject: [PATCH 09/50] transmogrifier_test: test date, anyOf and oneOf completely --- .../question_options_validator.ex | 2 +- test/web/activity_pub/transmogrifier_test.exs | 35 ++++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex index 8291d7b9f..478b3b5cf 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do embedded_schema do field(:name, :string) - embeds_one :replies, Replies do + embeds_one :replies, Replies, primary_key: false do field(:totalItems, :integer) field(:type, :string) end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 73949b558..4184b93ce 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -224,14 +224,33 @@ test "it works for incoming questions" do object = Object.normalize(activity, false) - assert Enum.all?(object.data["oneOf"], fn choice -> - choice["name"] in [ - "Dunno", - "Everyone knows that!", - "25 char limit is dumb", - "I can't even fit a funny" - ] - end) + assert object.data["closed"] == "2019-05-11T09:03:36Z" + + assert object.data["anyOf"] == [] + + assert Enum.sort(object.data["oneOf"]) == + Enum.sort([ + %{ + "name" => "25 char limit is dumb", + "replies" => %{"totalItems" => 0, "type" => "Collection"}, + "type" => "Note" + }, + %{ + "name" => "Dunno", + "replies" => %{"totalItems" => 0, "type" => "Collection"}, + "type" => "Note" + }, + %{ + "name" => "Everyone knows that!", + "replies" => %{"totalItems" => 1, "type" => "Collection"}, + "type" => "Note" + }, + %{ + "name" => "I can't even fit a funny", + "replies" => %{"totalItems" => 1, "type" => "Collection"}, + "type" => "Note" + } + ]) end test "it works for incoming listens" do From 4644a8bd109faf6c684767fc60c37e298b8c5c07 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 15 Jun 2020 00:30:45 +0200 Subject: [PATCH 10/50] Fix multiple-choice poll detection --- lib/pleroma/object.ex | 17 ++++++++++------- lib/pleroma/web/common_api/common_api.ex | 9 +++++++-- lib/pleroma/web/mastodon_api/views/poll_view.ex | 4 ++-- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 546c4ea01..4dd929cfd 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -255,6 +255,12 @@ def increase_replies_count(ap_id) do end end + defp poll_is_multiple?(%Object{data: %{"anyOf" => anyOf}}) do + !Enum.empty?(anyOf) + end + + defp poll_is_multiple?(_), do: false + def decrease_replies_count(ap_id) do Object |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id))) @@ -281,10 +287,10 @@ def decrease_replies_count(ap_id) do def increase_vote_count(ap_id, name, actor) do with %Object{} = object <- Object.normalize(ap_id), "Question" <- object.data["type"] do - multiple = Map.has_key?(object.data, "anyOf") + key = if poll_is_multiple?(object), do: "anyOf", else: "oneOf" options = - (object.data["anyOf"] || object.data["oneOf"] || []) + object.data[key] |> Enum.map(fn %{"name" => ^name} = option -> Kernel.update_in(option["replies"]["totalItems"], &(&1 + 1)) @@ -296,11 +302,8 @@ def increase_vote_count(ap_id, name, actor) do voters = [actor | object.data["voters"] || []] |> Enum.uniq() data = - if multiple do - Map.put(object.data, "anyOf", options) - else - Map.put(object.data, "oneOf", options) - end + object.data + |> Map.put(key, options) |> Map.put("voters", voters) object diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 4d5b0decf..692ceab1e 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -340,8 +340,13 @@ defp validate_existing_votes(%{ap_id: ap_id}, object) do end end - defp get_options_and_max_count(%{data: %{"anyOf" => any_of}}), do: {any_of, Enum.count(any_of)} - defp get_options_and_max_count(%{data: %{"oneOf" => one_of}}), do: {one_of, 1} + defp get_options_and_max_count(%{data: %{"anyOf" => any_of}}) + when is_list(any_of) and any_of != [], + do: {any_of, Enum.count(any_of)} + + defp get_options_and_max_count(%{data: %{"oneOf" => one_of}}) + when is_list(one_of) and one_of != [], + do: {one_of, 1} defp normalize_and_validate_choices(choices, object) do choices = Enum.map(choices, fn i -> if is_binary(i), do: String.to_integer(i), else: i end) diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex index 59a5deb28..73c990e2e 100644 --- a/lib/pleroma/web/mastodon_api/views/poll_view.ex +++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex @@ -28,10 +28,10 @@ def render("show.json", %{object: object, multiple: multiple, options: options} def render("show.json", %{object: object} = params) do case object.data do - %{"anyOf" => options} when is_list(options) -> + %{"anyOf" => options} when is_list(options) and options != [] -> render(__MODULE__, "show.json", Map.merge(params, %{multiple: true, options: options})) - %{"oneOf" => options} when is_list(options) -> + %{"oneOf" => options} when is_list(options) and options != [] -> render(__MODULE__, "show.json", Map.merge(params, %{multiple: false, options: options})) _ -> From 6b9c4bc1f1e5013cdfc084603cdf6cfa83ac2778 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 14 Jun 2020 22:24:00 +0200 Subject: [PATCH 11/50] fetcher: more descriptive variable names --- lib/pleroma/object/fetcher.ex | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 3e2949ee2..e1ab4ef8b 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -23,21 +23,21 @@ defp touch_changeset(changeset) do Ecto.Changeset.put_change(changeset, :updated_at, updated_at) end - defp maybe_reinject_internal_fields(data, %{data: %{} = old_data}) do + defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields()) - Map.merge(data, internal_fields) + Map.merge(new_data, internal_fields) end - defp maybe_reinject_internal_fields(data, _), do: data + defp maybe_reinject_internal_fields(_, new_data), do: new_data @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()} - defp reinject_object(struct, data) do - Logger.debug("Reinjecting object #{data["id"]}") + defp reinject_object(%Object{} = object, new_data) do + Logger.debug("Reinjecting object #{new_data["id"]}") - with data <- Transmogrifier.fix_object(data), - data <- maybe_reinject_internal_fields(data, struct), - changeset <- Object.change(struct, %{data: data}), + with new_data <- Transmogrifier.fix_object(new_data), + data <- maybe_reinject_internal_fields(object, new_data), + changeset <- Object.change(object, %{data: data}), changeset <- touch_changeset(changeset), {:ok, object} <- Repo.insert_or_update(changeset), {:ok, object} <- Object.set_cache(object) do @@ -51,8 +51,8 @@ defp reinject_object(struct, data) do def refetch_object(%Object{data: %{"id" => id}} = object) do with {:local, false} <- {:local, Object.local?(object)}, - {:ok, data} <- fetch_and_contain_remote_object_from_id(id), - {:ok, object} <- reinject_object(object, data) do + {:ok, new_data} <- fetch_and_contain_remote_object_from_id(id), + {:ok, object} <- reinject_object(object, new_data) do {:ok, object} else {:local, true} -> {:ok, object} From ad867ccfa18afe2a6104cad4d3035834c5a264c8 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 14 Jun 2020 22:01:14 +0200 Subject: [PATCH 12/50] fetcher: Reinject Question through validator --- lib/pleroma/object/fetcher.ex | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index e1ab4ef8b..3956bb727 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.Repo alias Pleroma.Signature alias Pleroma.Web.ActivityPub.InternalFetchActor + alias Pleroma.Web.ActivityPub.ObjectValidator alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.Federator @@ -32,6 +33,24 @@ defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do defp maybe_reinject_internal_fields(_, new_data), do: new_data @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()} + defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do + Logger.debug("Reinjecting object #{new_data["id"]}") + + with new_data <- Transmogrifier.fix_object(new_data), + data <- maybe_reinject_internal_fields(object, new_data), + {:ok, data, _} <- ObjectValidator.validate(data, %{}), + changeset <- Object.change(object, %{data: data}), + changeset <- touch_changeset(changeset), + {:ok, object} <- Repo.insert_or_update(changeset), + {:ok, object} <- Object.set_cache(object) do + {:ok, object} + else + e -> + Logger.error("Error while processing object: #{inspect(e)}") + {:error, e} + end + end + defp reinject_object(%Object{} = object, new_data) do Logger.debug("Reinjecting object #{new_data["id"]}") From 47ba796f415740c443cd8477c121280656b13032 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 15 Jun 2020 02:20:18 +0200 Subject: [PATCH 13/50] create_question_validator: remove validate_recipients_match --- .../create_question_validator.ex | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex index f09207418..6d3f71566 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex @@ -47,7 +47,6 @@ def validate_data(cng, meta \\ []) do |> validate_inclusion(:type, ["Create"]) |> validate_actor_presence() |> validate_any_presence([:to, :cc]) - |> validate_recipients_match(meta) |> validate_actors_match(meta) |> validate_object_nonexistence() end @@ -75,20 +74,4 @@ def validate_actors_match(cng, meta) do end end) end - - def validate_recipients_match(cng, meta) do - object_recipients = meta[:object_data]["to"] || [] - - cng - |> validate_change(:to, fn :to, recipients -> - activity_set = MapSet.new(recipients) - object_set = MapSet.new(object_recipients) - - if MapSet.equal?(activity_set, object_set) do - [] - else - [{:to, "Recipients don't match with object recipients"}] - end - end) - end end From 173f69c854adf966d52b3767c4de43a0b1ce5b00 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 15 Jun 2020 05:18:30 +0200 Subject: [PATCH 14/50] question_validator: fix for mastodon poll expiration Mastodon activities do not have a "closed" field, this could be seen on https://pouet.it/users/lanodan_tmp/statuses/104345126997708380 which runs Mastodon 3.1.4 (SDF runs 3.1.2) --- .../object_validators/question_validator.ex | 10 ++++++++++ lib/pleroma/web/mastodon_api/views/poll_view.ex | 14 ++++++-------- test/fixtures/mastodon-question-activity.json | 1 - 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index f94d79352..605cb56f8 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -71,7 +71,17 @@ def cast_data(data) do |> changeset(data) end + def fix(data) do + cond do + is_binary(data["closed"]) -> data + is_binary(data["endTime"]) -> Map.put(data, "closed", data["endTime"]) + true -> Map.drop(data, ["closed"]) + end + end + def changeset(struct, data) do + data = fix(data) + struct |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf]) |> cast_embed(:anyOf) diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex index 73c990e2e..ce595ae8a 100644 --- a/lib/pleroma/web/mastodon_api/views/poll_view.ex +++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex @@ -40,15 +40,13 @@ def render("show.json", %{object: object} = params) do end defp end_time_and_expired(object) do - case object.data["closed"] || object.data["endTime"] do - end_time when is_binary(end_time) -> - end_time = NaiveDateTime.from_iso8601!(end_time) - expired = NaiveDateTime.compare(end_time, NaiveDateTime.utc_now()) == :lt + if object.data["closed"] do + end_time = NaiveDateTime.from_iso8601!(object.data["closed"]) + expired = NaiveDateTime.compare(end_time, NaiveDateTime.utc_now()) == :lt - {Utils.to_masto_date(end_time), expired} - - _ -> - {nil, false} + {Utils.to_masto_date(end_time), expired} + else + {nil, false} end end diff --git a/test/fixtures/mastodon-question-activity.json b/test/fixtures/mastodon-question-activity.json index ac329c7d5..3648b9f90 100644 --- a/test/fixtures/mastodon-question-activity.json +++ b/test/fixtures/mastodon-question-activity.json @@ -49,7 +49,6 @@ "en": "

Why is Tenshi eating a corndog so cute?

" }, "endTime": "2019-05-11T09:03:36Z", - "closed": "2019-05-11T09:03:36Z", "attachment": [], "tag": [], "replies": { From 4f70fd4105e90c8fc06a1eb6bd70084874bae3a5 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 18 Jun 2020 19:54:56 +0200 Subject: [PATCH 15/50] question_validator: remove conversation field --- .../web/activity_pub/object_validators/question_validator.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 605cb56f8..211f520c4 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -45,7 +45,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do field(:announcements, {:array, :string}, default: []) # see if needed - field(:conversation, :string) field(:context_id, :string) field(:closed, Types.DateTime) From 82895a40123ca24258a95b9ac7e117dd8b1e698e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 18 Jun 2020 04:05:42 +0200 Subject: [PATCH 16/50] SideEffects: port ones from ActivityPub.do_create and ActivityPub.insert --- lib/pleroma/object/containment.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 13 +---- lib/pleroma/web/activity_pub/builder.ex | 15 ++++++ .../web/activity_pub/object_validator.ex | 6 +-- .../object_validators/answer_validator.ex | 7 ++- .../object_validators/common_validations.ex | 38 ++++++++++++++ ...lidator.ex => create_generic_validator.ex} | 39 ++++++++++++--- .../object_validators/question_validator.ex | 28 +++++++++-- lib/pleroma/web/activity_pub/side_effects.ex | 30 +++++++++++- .../web/activity_pub/transmogrifier.ex | 49 ++++++++++++------- lib/pleroma/web/common_api/common_api.ex | 21 ++++---- lib/pleroma/web/common_api/utils.ex | 11 ----- test/web/activity_pub/transmogrifier_test.exs | 2 +- 13 files changed, 188 insertions(+), 73 deletions(-) rename lib/pleroma/web/activity_pub/object_validators/{create_question_validator.ex => create_generic_validator.ex} (70%) diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex index 99608b8a5..bc88e8a0c 100644 --- a/lib/pleroma/object/containment.ex +++ b/lib/pleroma/object/containment.ex @@ -55,7 +55,7 @@ defp compare_uris(%URI{host: host} = _id_uri, %URI{host: host} = _other_uri), do defp compare_uris(_id_uri, _other_uri), do: :error @doc """ - Checks that an imported AP object's actor matches the domain it came from. + Checks that an imported AP object's actor matches the host it came from. """ def contain_origin(_id, %{"actor" => nil}), do: :error diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d8cc8d24f..9d13a06c4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -66,7 +66,7 @@ defp check_remote_limit(%{"object" => %{"content" => content}}) when not is_nil( defp check_remote_limit(_), do: true - defp increase_note_count_if_public(actor, object) do + def increase_note_count_if_public(actor, object) do if is_public?(object), do: User.increase_note_count(actor), else: {:ok, actor} end @@ -85,16 +85,6 @@ defp increase_replies_count_if_reply(%{ defp increase_replies_count_if_reply(_create_data), do: :noop - defp increase_poll_votes_if_vote(%{ - "object" => %{"inReplyTo" => reply_ap_id, "name" => name}, - "type" => "Create", - "actor" => actor - }) do - Object.increase_vote_count(reply_ap_id, name, actor) - end - - defp increase_poll_votes_if_vote(_create_data), do: :noop - @object_types ["ChatMessage", "Question", "Answer"] @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} def persist(%{"type" => type} = object, meta) when type in @object_types do @@ -258,7 +248,6 @@ defp do_create(%{to: to, actor: actor, context: context, object: object} = param with {:ok, activity} <- insert(create_data, local, fake), {:fake, false, activity} <- {:fake, fake, activity}, _ <- increase_replies_count_if_reply(create_data), - _ <- increase_poll_votes_if_vote(create_data), {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity}, {:ok, _actor} <- increase_note_count_if_public(actor, activity), _ <- notify_and_stream(activity), diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index d5f3610ed..e97381954 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -115,6 +115,21 @@ def chat_message(actor, recipient, content, opts \\ []) do end end + def answer(user, object, name) do + {:ok, + %{ + "type" => "Answer", + "actor" => user.ap_id, + "cc" => [object.data["actor"]], + "to" => [], + "name" => name, + "inReplyTo" => object.data["id"], + "context" => object.data["context"], + "published" => DateTime.utc_now() |> DateTime.to_iso8601(), + "id" => Utils.generate_object_id() + }, []} + end + @spec tombstone(String.t(), String.t()) :: {:ok, map(), keyword()} def tombstone(actor, id) do {:ok, diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index c89311187..a24aaf00c 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.CreateQuestionValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator @@ -162,7 +162,7 @@ def validate( meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, create_activity} <- create_activity - |> CreateQuestionValidator.cast_and_validate(meta) + |> CreateGenericValidator.cast_and_validate(meta) |> Ecto.Changeset.apply_action(:insert) do create_activity = stringify_keys(create_activity) {:ok, create_activity, meta} @@ -188,7 +188,7 @@ def cast_and_apply(%{"type" => "Question"} = object) do end def cast_and_apply(%{"type" => "Answer"} = object) do - QuestionValidator.cast_and_apply(object) + AnswerValidator.cast_and_apply(object) end def cast_and_apply(o), do: {:error, {:validator_not_set, o}} diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex index 0b51eccfa..8d4c92520 100644 --- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex @@ -13,22 +13,25 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do @primary_key false @derive Jason.Encoder - # Extends from NoteValidator embedded_schema do field(:id, Types.ObjectID, primary_key: true) field(:to, {:array, :string}, default: []) field(:cc, {:array, :string}, default: []) + + # is this actually needed? field(:bto, {:array, :string}, default: []) field(:bcc, {:array, :string}, default: []) + field(:type, :string) field(:name, :string) field(:inReplyTo, :string) field(:attributedTo, Types.ObjectID) + field(:actor, Types.ObjectID) end def cast_and_apply(data) do data - |> cast_data + |> cast_data() |> apply_action(:insert) end diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index e746b9360..140555a45 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -42,6 +42,19 @@ def validate_actor_presence(cng, options \\ []) do end) end + def validate_actor_is_active(cng, options \\ []) do + field_name = Keyword.get(options, :field_name, :actor) + + cng + |> validate_change(field_name, fn field_name, actor -> + if %User{deactivated: false} = User.get_cached_by_ap_id(actor) do + [] + else + [{field_name, "can't find user (or deactivated)"}] + end + end) + end + def validate_object_presence(cng, options \\ []) do field_name = Keyword.get(options, :field_name, :object) allowed_types = Keyword.get(options, :allowed_types, false) @@ -77,4 +90,29 @@ def validate_object_or_user_presence(cng, options \\ []) do if actor_cng.valid?, do: actor_cng, else: object_cng end + + def validate_host_match(cng, fields \\ [:id, :actor]) do + unique_hosts = + fields + |> Enum.map(fn field -> + %URI{host: host} = + cng + |> get_field(field) + |> URI.parse() + + host + end) + |> Enum.uniq() + |> Enum.count() + + if unique_hosts == 1 do + cng + else + fields + |> Enum.reduce(cng, fn field, cng -> + cng + |> add_error(field, "hosts of #{inspect(fields)} aren't matching") + end) + end + end end diff --git a/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex similarity index 70% rename from lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex rename to lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index 6d3f71566..4ad4ca0de 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -4,9 +4,8 @@ # Code based on CreateChatMessageValidator # NOTES -# - Can probably be a generic create validator # - doesn't embed, will only get the object id -defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateQuestionValidator do +defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do use Ecto.Schema alias Pleroma.Object @@ -26,29 +25,53 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateQuestionValidator do field(:object, Types.ObjectID) end + def cast_data(data) do + %__MODULE__{} + |> changeset(data) + end + def cast_and_apply(data) do data |> cast_data |> apply_action(:insert) end - def cast_data(data) do - cast(%__MODULE__{}, data, __schema__(:fields)) + def cast_and_validate(data, meta \\ []) do + data + |> cast_data + |> validate_data(meta) end - def cast_and_validate(data, meta \\ []) do - cast_data(data) - |> validate_data(meta) + def changeset(struct, data) do + struct + |> cast(data, __schema__(:fields)) end def validate_data(cng, meta \\ []) do cng |> validate_required([:actor, :type, :object]) |> validate_inclusion(:type, ["Create"]) - |> validate_actor_presence() + |> validate_actor_is_active() |> validate_any_presence([:to, :cc]) |> validate_actors_match(meta) |> validate_object_nonexistence() + |> validate_object_containment() + end + + def validate_object_containment(cng) do + actor = get_field(cng, :actor) + + cng + |> validate_change(:object, fn :object, object_id -> + %URI{host: object_id_host} = URI.parse(object_id) + %URI{host: actor_host} = URI.parse(actor) + + if object_id_host == actor_host do + [] + else + [{:object, "The host of the object id doesn't match with the host of the actor"}] + end + end) end def validate_object_nonexistence(cng) do diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 211f520c4..f7f3b1354 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do use Ecto.Schema + alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator alias Pleroma.Web.ActivityPub.ObjectValidators.Types @@ -40,13 +41,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do field(:announcement_count, :integer, default: 0) field(:inReplyTo, :string) field(:uri, Types.Uri) + # short identifier for PleromaFE to group statuses by context + field(:context_id, :integer) field(:likes, {:array, :string}, default: []) field(:announcements, {:array, :string}, default: []) - # see if needed - field(:context_id, :string) - field(:closed, Types.DateTime) field(:voters, {:array, Types.ObjectID}, default: []) embeds_many(:anyOf, QuestionOptionsValidator) @@ -70,7 +70,7 @@ def cast_data(data) do |> changeset(data) end - def fix(data) do + defp fix_closed(data) do cond do is_binary(data["closed"]) -> data is_binary(data["endTime"]) -> Map.put(data, "closed", data["endTime"]) @@ -78,6 +78,23 @@ def fix(data) do end end + # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults + defp fix_defaults(data) do + %{data: %{"id" => context}, id: context_id} = Utils.create_context(data["context"]) + + data + |> Map.put_new_lazy("id", &Utils.generate_object_id/0) + |> Map.put_new_lazy("published", &Utils.make_date/0) + |> Map.put_new("context", context) + |> Map.put_new("context_id", context_id) + end + + defp fix(data) do + data + |> fix_closed() + |> fix_defaults() + end + def changeset(struct, data) do data = fix(data) @@ -92,7 +109,8 @@ def validate_data(data_cng) do |> validate_inclusion(:type, ["Question"]) |> validate_required([:id, :actor, :type, :content, :context]) |> CommonValidations.validate_any_presence([:cc, :to]) - |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_actor_is_active() |> CommonValidations.validate_any_presence([:oneOf, :anyOf]) + |> CommonValidations.validate_host_match() end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index a78ec411f..c17197bec 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do """ alias Pleroma.Activity alias Pleroma.Activity.Ir.Topics + alias Pleroma.ActivityExpiration alias Pleroma.Chat alias Pleroma.Chat.MessageReference alias Pleroma.FollowingRelationship @@ -19,6 +20,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Push alias Pleroma.Web.Streamer + alias Pleroma.Workers.BackgroundWorker def handle(object, meta \\ []) @@ -135,10 +137,24 @@ def handle(%{data: %{"type" => "Like"}} = object, meta) do # Tasks this handles # - Actually create object # - Rollback if we couldn't create it + # - Increase the user note count + # - Increase the reply count # - Set up notifications def handle(%{data: %{"type" => "Create"}} = activity, meta) do - with {:ok, _object, meta} <- handle_object_creation(meta[:object_data], meta) do + with {:ok, object, meta} <- handle_object_creation(meta[:object_data], meta), + %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do {:ok, notifications} = Notification.create_notifications(activity, do_send: false) + {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object) + + if in_reply_to = object.data["inReplyTo"] do + Object.increase_replies_count(in_reply_to) + end + + if expires_at = activity.data["expires_at"] do + ActivityExpiration.create(activity, expires_at) + end + + BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id}) meta = meta @@ -268,6 +284,18 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do end end + def handle_object_creation(%{"type" => "Answer"} = object_map, meta) do + with {:ok, object, meta} <- Pipeline.common_pipeline(object_map, meta) do + Object.increase_vote_count( + object.data["inReplyTo"], + object.data["name"], + object.data["actor"] + ) + + {:ok, object, meta} + end + end + def handle_object_creation(%{"type" => "Question"} = object, meta) do with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do {:ok, object, meta} diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9900602e4..26325d5de 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -419,6 +419,29 @@ defp get_reported(objects) do end) end + # Compatibility wrapper for Mastodon votes + defp handle_create(%{"object" => %{"type" => "Answer"}} = data, _user) do + handle_incoming(data) + end + + defp handle_create(%{"object" => object} = data, user) do + %{ + to: data["to"], + object: object, + actor: user, + context: object["context"], + local: false, + published: data["published"], + additional: + Map.take(data, [ + "cc", + "directMessage", + "id" + ]) + } + |> ActivityPub.create() + end + def handle_incoming(data, options \\ []) # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them @@ -461,26 +484,14 @@ def handle_incoming( actor = Containment.get_actor(data) with nil <- Activity.get_create_by_object_ap_id(object["id"]), - {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor), - data <- Map.put(data, "actor", actor) |> fix_addressing() do - object = fix_object(object, options) + {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor) do + data = + data + |> Map.put("object", fix_object(object, options)) + |> Map.put("actor", actor) + |> fix_addressing() - params = %{ - to: data["to"], - object: object, - actor: user, - context: object["context"], - local: false, - published: data["published"], - additional: - Map.take(data, [ - "cc", - "directMessage", - "id" - ]) - } - - with {:ok, created_activity} <- ActivityPub.create(params) do + with {:ok, created_activity} <- handle_create(data, user) do reply_depth = (options[:depth] || 0) + 1 if Federator.allowed_thread_distance?(reply_depth) do diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 692ceab1e..c08e0ffeb 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -308,18 +308,19 @@ def vote(user, %{data: %{"type" => "Question"}} = object, choices) do {:ok, options, choices} <- normalize_and_validate_choices(choices, object) do answer_activities = Enum.map(choices, fn index -> - answer_data = make_answer_data(user, object, Enum.at(options, index)["name"]) + {:ok, answer_object, _meta} = + Builder.answer(user, object, Enum.at(options, index)["name"]) - {:ok, activity} = - ActivityPub.create(%{ - to: answer_data["to"], - actor: user, - context: object.data["context"], - object: answer_data, - additional: %{"cc" => answer_data["cc"]} - }) + {:ok, activity_data, _meta} = Builder.create(user, answer_object, []) - activity + {:ok, activity, _meta} = + activity_data + |> Map.put("cc", answer_object["cc"]) + |> Map.put("context", answer_object["context"]) + |> Pipeline.common_pipeline(local: true) + + # TODO: Do preload of Pleroma.Object in Pipeline + Activity.normalize(activity.data) end) object = Object.get_cached_by_ap_id(object.data["id"]) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 9c38b73eb..9d7b24eb2 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -548,17 +548,6 @@ def conversation_id_to_context(id) do end end - def make_answer_data(%User{ap_id: ap_id}, object, name) do - %{ - "type" => "Answer", - "actor" => ap_id, - "cc" => [object.data["actor"]], - "to" => [], - "name" => name, - "inReplyTo" => object.data["id"] - } - end - def validate_character_limit("" = _full_payload, [] = _attachments) do {:error, dgettext("errors", "Cannot post an empty status without attachments")} end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 4184b93ce..62b5b06aa 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -282,7 +282,7 @@ test "it works for incoming listens" do assert object.data["length"] == 180_000 end - test "it rewrites Note votes to Answers and increments vote counters on question activities" do + test "it rewrites Note votes to Answer and increments vote counters on Question activities" do user = insert(:user) {:ok, activity} = From 39870d99b810940ccce430411c9fc6939f760663 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 19 Jun 2020 23:31:19 +0200 Subject: [PATCH 17/50] transmogrifier tests: Move & enhance in specialised modules --- .../transmogrifier/answer_handling_test.exs | 78 ++++++++++++++++ .../transmogrifier/question_handling_test.exs | 65 +++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 92 ------------------- 3 files changed, 143 insertions(+), 92 deletions(-) create mode 100644 test/web/activity_pub/transmogrifier/answer_handling_test.exs create mode 100644 test/web/activity_pub/transmogrifier/question_handling_test.exs diff --git a/test/web/activity_pub/transmogrifier/answer_handling_test.exs b/test/web/activity_pub/transmogrifier/answer_handling_test.exs new file mode 100644 index 000000000..0f6605c3f --- /dev/null +++ b/test/web/activity_pub/transmogrifier/answer_handling_test.exs @@ -0,0 +1,78 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.AnswerHandlingTest do + use Pleroma.DataCase + + alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.CommonAPI + + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + test "incoming, rewrites Note to Answer and increments vote counters" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "suya...", + poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10} + }) + + object = Object.normalize(activity) + + data = + File.read!("test/fixtures/mastodon-vote.json") + |> Poison.decode!() + |> Kernel.put_in(["to"], user.ap_id) + |> Kernel.put_in(["object", "inReplyTo"], object.data["id"]) + |> Kernel.put_in(["object", "to"], user.ap_id) + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + answer_object = Object.normalize(activity) + assert answer_object.data["type"] == "Answer" + assert answer_object.data["inReplyTo"] == object.data["id"] + + new_object = Object.get_by_ap_id(object.data["id"]) + assert new_object.data["replies_count"] == object.data["replies_count"] + + assert Enum.any?( + new_object.data["oneOf"], + fn + %{"name" => "suya..", "replies" => %{"totalItems" => 1}} -> true + _ -> false + end + ) + end + + test "outgoing, rewrites Answer to Note" do + user = insert(:user) + + {:ok, poll_activity} = + CommonAPI.post(user, %{ + status: "suya...", + poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10} + }) + + poll_object = Object.normalize(poll_activity) + # TODO: Replace with CommonAPI vote creation when implemented + data = + File.read!("test/fixtures/mastodon-vote.json") + |> Poison.decode!() + |> Kernel.put_in(["to"], user.ap_id) + |> Kernel.put_in(["object", "inReplyTo"], poll_object.data["id"]) + |> Kernel.put_in(["object", "to"], user.ap_id) + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) + + assert data["object"]["type"] == "Note" + end +end diff --git a/test/web/activity_pub/transmogrifier/question_handling_test.exs b/test/web/activity_pub/transmogrifier/question_handling_test.exs new file mode 100644 index 000000000..b7b9a1a7b --- /dev/null +++ b/test/web/activity_pub/transmogrifier/question_handling_test.exs @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do + use Pleroma.DataCase + + alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.Transmogrifier + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + test "Mastodon Question activity" do + data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!() + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + + object = Object.normalize(activity, false) + + assert object.data["closed"] == "2019-05-11T09:03:36Z" + + assert object.data["context"] == + "tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation" + + assert object.data["context_id"] + + assert object.data["anyOf"] == [] + + assert Enum.sort(object.data["oneOf"]) == + Enum.sort([ + %{ + "name" => "25 char limit is dumb", + "replies" => %{"totalItems" => 0, "type" => "Collection"}, + "type" => "Note" + }, + %{ + "name" => "Dunno", + "replies" => %{"totalItems" => 0, "type" => "Collection"}, + "type" => "Note" + }, + %{ + "name" => "Everyone knows that!", + "replies" => %{"totalItems" => 1, "type" => "Collection"}, + "type" => "Note" + }, + %{ + "name" => "I can't even fit a funny", + "replies" => %{"totalItems" => 1, "type" => "Collection"}, + "type" => "Note" + } + ]) + end + + test "returns an error if received a second time" do + data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!() + + assert {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + + assert {:error, {:validate_object, {:error, _}}} = Transmogrifier.handle_incoming(data) + end +end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 62b5b06aa..272431135 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -217,42 +217,6 @@ test "it works for incoming notices with hashtags" do assert Enum.at(object.data["tag"], 2) == "moo" end - test "it works for incoming questions" do - data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!() - - {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) - - object = Object.normalize(activity, false) - - assert object.data["closed"] == "2019-05-11T09:03:36Z" - - assert object.data["anyOf"] == [] - - assert Enum.sort(object.data["oneOf"]) == - Enum.sort([ - %{ - "name" => "25 char limit is dumb", - "replies" => %{"totalItems" => 0, "type" => "Collection"}, - "type" => "Note" - }, - %{ - "name" => "Dunno", - "replies" => %{"totalItems" => 0, "type" => "Collection"}, - "type" => "Note" - }, - %{ - "name" => "Everyone knows that!", - "replies" => %{"totalItems" => 1, "type" => "Collection"}, - "type" => "Note" - }, - %{ - "name" => "I can't even fit a funny", - "replies" => %{"totalItems" => 1, "type" => "Collection"}, - "type" => "Note" - } - ]) - end - test "it works for incoming listens" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams", @@ -282,38 +246,6 @@ test "it works for incoming listens" do assert object.data["length"] == 180_000 end - test "it rewrites Note votes to Answer and increments vote counters on Question activities" do - user = insert(:user) - - {:ok, activity} = - CommonAPI.post(user, %{ - status: "suya...", - poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10} - }) - - object = Object.normalize(activity) - - data = - File.read!("test/fixtures/mastodon-vote.json") - |> Poison.decode!() - |> Kernel.put_in(["to"], user.ap_id) - |> Kernel.put_in(["object", "inReplyTo"], object.data["id"]) - |> Kernel.put_in(["object", "to"], user.ap_id) - - {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) - answer_object = Object.normalize(activity) - assert answer_object.data["type"] == "Answer" - object = Object.get_by_ap_id(object.data["id"]) - - assert Enum.any?( - object.data["oneOf"], - fn - %{"name" => "suya..", "replies" => %{"totalItems" => 1}} -> true - _ -> false - end - ) - end - test "it works for incoming notices with contentMap" do data = File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!() @@ -1280,30 +1212,6 @@ test "successfully reserializes a message with AS2 objects in IR" do end end - test "Rewrites Answers to Notes" do - user = insert(:user) - - {:ok, poll_activity} = - CommonAPI.post(user, %{ - status: "suya...", - poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10} - }) - - poll_object = Object.normalize(poll_activity) - # TODO: Replace with CommonAPI vote creation when implemented - data = - File.read!("test/fixtures/mastodon-vote.json") - |> Poison.decode!() - |> Kernel.put_in(["to"], user.ap_id) - |> Kernel.put_in(["object", "inReplyTo"], poll_object.data["id"]) - |> Kernel.put_in(["object", "to"], user.ap_id) - - {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) - {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) - - assert data["object"]["type"] == "Note" - end - describe "fix_explicit_addressing" do setup do user = insert(:user) From fe6924d00d710e7e568d0ed40c02d3c516d91460 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 19 Jun 2020 23:43:36 +0200 Subject: [PATCH 18/50] CreateGenericValidator: add expires_at --- .../activity_pub/object_validators/create_generic_validator.ex | 1 + lib/pleroma/web/activity_pub/side_effects.ex | 2 ++ 2 files changed, 3 insertions(+) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index 4ad4ca0de..f467ccc7c 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -23,6 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do field(:to, Types.Recipients, default: []) field(:cc, Types.Recipients, default: []) field(:object, Types.ObjectID) + field(:expires_at, Types.DateTime) end def cast_data(data) do diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index c17197bec..5104d38ee 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -139,6 +139,8 @@ def handle(%{data: %{"type" => "Like"}} = object, meta) do # - Rollback if we couldn't create it # - Increase the user note count # - Increase the reply count + # - Increase replies count + # - Set up ActivityExpiration # - Set up notifications def handle(%{data: %{"type" => "Create"}} = activity, meta) do with {:ok, object, meta} <- handle_object_creation(meta[:object_data], meta), From 435a65b9768b9f2c614ba5e00d83753b5834a695 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 20 Jun 2020 00:07:39 +0200 Subject: [PATCH 19/50] QuestionValidator: Use AttachmentValidator --- .../activity_pub/object_validators/question_validator.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index f7f3b1354..56f02777d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do use Ecto.Schema alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator alias Pleroma.Web.ActivityPub.ObjectValidators.Types @@ -34,8 +35,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do # TODO: Write type field(:emoji, :map, default: %{}) field(:sensitive, :boolean, default: false) - # TODO: Write type - field(:attachment, {:array, :map}, default: []) + embeds_many(:attachment, AttachmentValidator) field(:replies_count, :integer, default: 0) field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) @@ -99,7 +99,8 @@ def changeset(struct, data) do data = fix(data) struct - |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf]) + |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment]) + |> cast_embed(:attachment) |> cast_embed(:anyOf) |> cast_embed(:oneOf) end From d713930ea7fe78f705f534febe4ceaf0a0216d24 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sat, 20 Jun 2020 00:23:04 +0200 Subject: [PATCH 20/50] Fixup for EctoType module move --- .../object_validators/answer_validator.ex | 8 ++++---- .../create_generic_validator.ex | 14 +++++++------- .../object_validators/question_validator.ex | 18 +++++++++--------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex index 8d4c92520..9861eec7f 100644 --- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations - alias Pleroma.Web.ActivityPub.ObjectValidators.Types import Ecto.Changeset @@ -14,7 +14,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do @derive Jason.Encoder embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:to, {:array, :string}, default: []) field(:cc, {:array, :string}, default: []) @@ -25,8 +25,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do field(:type, :string) field(:name, :string) field(:inReplyTo, :string) - field(:attributedTo, Types.ObjectID) - field(:actor, Types.ObjectID) + field(:attributedTo, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) end def cast_and_apply(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index f467ccc7c..97e2def10 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -8,8 +8,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object - alias Pleroma.Web.ActivityPub.ObjectValidators.Types import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -17,13 +17,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) - field(:actor, Types.ObjectID) + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:actor, ObjectValidators.ObjectID) field(:type, :string) - field(:to, Types.Recipients, default: []) - field(:cc, Types.Recipients, default: []) - field(:object, Types.ObjectID) - field(:expires_at, Types.DateTime) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:object, ObjectValidators.ObjectID) + field(:expires_at, ObjectValidators.DateTime) end def cast_data(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 56f02777d..53cf35d40 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -5,11 +5,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do use Ecto.Schema - alias Pleroma.Web.ActivityPub.Utils + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.Types + alias Pleroma.Web.ActivityPub.Utils import Ecto.Changeset @@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do # Extends from NoteValidator embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:to, {:array, :string}, default: []) field(:cc, {:array, :string}, default: []) field(:bto, {:array, :string}, default: []) @@ -28,10 +28,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do field(:type, :string) field(:content, :string) field(:context, :string) - field(:actor, Types.ObjectID) - field(:attributedTo, Types.ObjectID) + field(:actor, ObjectValidators.ObjectID) + field(:attributedTo, ObjectValidators.ObjectID) field(:summary, :string) - field(:published, Types.DateTime) + field(:published, ObjectValidators.DateTime) # TODO: Write type field(:emoji, :map, default: %{}) field(:sensitive, :boolean, default: false) @@ -40,15 +40,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) field(:inReplyTo, :string) - field(:uri, Types.Uri) + field(:uri, ObjectValidators.Uri) # short identifier for PleromaFE to group statuses by context field(:context_id, :integer) field(:likes, {:array, :string}, default: []) field(:announcements, {:array, :string}, default: []) - field(:closed, Types.DateTime) - field(:voters, {:array, Types.ObjectID}, default: []) + field(:closed, ObjectValidators.DateTime) + field(:voters, {:array, ObjectValidators.ObjectID}, default: []) embeds_many(:anyOf, QuestionOptionsValidator) embeds_many(:oneOf, QuestionOptionsValidator) end From c19bdc811e526f83a2120c58f858044f4ff96e5f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 23 Jun 2020 05:30:34 +0200 Subject: [PATCH 21/50] Fix attachments in polls --- .../object_validators/url_object_validator.ex | 2 +- .../web/activity_pub/transmogrifier.ex | 12 ++- test/fixtures/tesla_mock/poll_attachment.json | 99 +++++++++++++++++++ test/object/fetcher_test.exs | 7 ++ test/support/http_request_mock.ex | 8 ++ test/web/activity_pub/transmogrifier_test.exs | 27 ++++- 6 files changed, 145 insertions(+), 10 deletions(-) create mode 100644 test/fixtures/tesla_mock/poll_attachment.json diff --git a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex index f64fac46d..881030f38 100644 --- a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do embedded_schema do field(:type, :string) field(:href, ObjectValidators.Uri) - field(:mediaType, :string) + field(:mediaType, :string, default: "application/octet-stream") end def changeset(struct, data) do diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 26325d5de..0ad982720 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -240,13 +240,17 @@ def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachm if href do attachment_url = - %{"href" => href} + %{ + "href" => href, + "type" => Map.get(url || %{}, "type", "Link") + } |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("type", Map.get(url || %{}, "type")) - %{"url" => [attachment_url]} + %{ + "url" => [attachment_url], + "type" => data["type"] || "Document" + } |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("type", data["type"]) |> Maps.put_if_present("name", data["name"]) else nil diff --git a/test/fixtures/tesla_mock/poll_attachment.json b/test/fixtures/tesla_mock/poll_attachment.json new file mode 100644 index 000000000..92e822dc8 --- /dev/null +++ b/test/fixtures/tesla_mock/poll_attachment.json @@ -0,0 +1,99 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + "https://patch.cx/schemas/litepub-0.1.jsonld", + { + "@language": "und" + } + ], + "actor": "https://patch.cx/users/rin", + "anyOf": [], + "attachment": [ + { + "mediaType": "image/jpeg", + "name": "screenshot_mpv:Totoro@01:18:44.345.jpg", + "type": "Document", + "url": "https://shitposter.club/media/3bb4c4d402f8fdcc7f80963c3d7cf6f10f936897fd374922ade33199d2f86d87.jpg?name=screenshot_mpv%3ATotoro%4001%3A18%3A44.345.jpg" + } + ], + "attributedTo": "https://patch.cx/users/rin", + "cc": [ + "https://patch.cx/users/rin/followers" + ], + "closed": "2020-06-19T23:22:02.754678Z", + "content": "@rinpatch", + "closed": "2019-09-19T00:32:36.785333", + "content": "can you vote on this poll?", + "id": "https://patch.cx/objects/tesla_mock/poll_attachment", + "oneOf": [ + { + "name": "a", + "replies": { + "totalItems": 0, + "type": "Collection" + }, + "type": "Note" + }, + { + "name": "A", + "replies": { + "totalItems": 0, + "type": "Collection" + }, + "type": "Note" + }, + { + "name": "Aa", + "replies": { + "totalItems": 0, + "type": "Collection" + }, + "type": "Note" + }, + { + "name": "AA", + "replies": { + "totalItems": 0, + "type": "Collection" + }, + "type": "Note" + }, + { + "name": "AAa", + "replies": { + "totalItems": 1, + "type": "Collection" + }, + "type": "Note" + }, + { + "name": "AAA", + "replies": { + "totalItems": 3, + "type": "Collection" + }, + "type": "Note" + } + ], + "published": "2020-06-19T23:12:02.786113Z", + "sensitive": false, + "summary": "", + "tag": [ + { + "href": "https://mastodon.sdf.org/users/rinpatch", + "name": "@rinpatch@mastodon.sdf.org", + "type": "Mention" + } + ], + "to": [ + "https://www.w3.org/ns/activitystreams#Public", + "https://mastodon.sdf.org/users/rinpatch" + ], + "type": "Question", + "voters": [ + "https://shitposter.club/users/moonman", + "https://skippers-bin.com/users/7v1w1r8ce6", + "https://mastodon.sdf.org/users/rinpatch", + "https://mastodon.social/users/emelie" + ] +} diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index d9098ea1b..16cfa7f5c 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -177,6 +177,13 @@ test "handle HTTP 404 response" do "https://mastodon.example.org/users/userisgone404" ) end + + test "it can fetch pleroma polls with attachments" do + {:ok, object} = + Fetcher.fetch_object_from_id("https://patch.cx/objects/tesla_mock/poll_attachment") + + assert object + end end describe "pruning" do diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 19a202654..eeeba7880 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -82,6 +82,14 @@ def get("https://mastodon.sdf.org/users/rinpatch", _, _, _) do }} end + def get("https://patch.cx/objects/tesla_mock/poll_attachment", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/poll_attachment.json") + }} + end + def get( "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/emelie", _, diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 272431135..7269e81bb 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -620,7 +620,8 @@ test "it remaps video URLs as attachments if necessary" do %{ "href" => "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4", - "mediaType" => "video/mp4" + "mediaType" => "video/mp4", + "type" => "Link" } ] } @@ -639,7 +640,8 @@ test "it remaps video URLs as attachments if necessary" do %{ "href" => "https://framatube.org/static/webseed/6050732a-8a7a-43d4-a6cd-809525a1d206-1080.mp4", - "mediaType" => "video/mp4" + "mediaType" => "video/mp4", + "type" => "Link" } ] } @@ -1459,8 +1461,13 @@ test "returns modified object when attachment is map" do "attachment" => [ %{ "mediaType" => "video/mp4", + "type" => "Document", "url" => [ - %{"href" => "https://peertube.moe/stat-480.mp4", "mediaType" => "video/mp4"} + %{ + "href" => "https://peertube.moe/stat-480.mp4", + "mediaType" => "video/mp4", + "type" => "Link" + } ] } ] @@ -1477,14 +1484,24 @@ test "returns modified object when attachment is list" do "attachment" => [ %{ "mediaType" => "video/mp4", + "type" => "Document", "url" => [ - %{"href" => "https://pe.er/stat-480.mp4", "mediaType" => "video/mp4"} + %{ + "href" => "https://pe.er/stat-480.mp4", + "mediaType" => "video/mp4", + "type" => "Link" + } ] }, %{ "mediaType" => "video/mp4", + "type" => "Document", "url" => [ - %{"href" => "https://pe.er/stat-480.mp4", "mediaType" => "video/mp4"} + %{ + "href" => "https://pe.er/stat-480.mp4", + "mediaType" => "video/mp4", + "type" => "Link" + } ] } ] From bfe2dafd398114240fcc2d3472799d6770904f6a Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 26 Jun 2020 00:07:43 +0200 Subject: [PATCH 22/50] {Answer,Question}Validator: Keep both actor and attributedTo for now but sync them --- lib/pleroma/web/activity_pub/builder.ex | 1 + .../object_validators/answer_validator.ex | 8 ++++++-- .../object_validators/common_validations.ex | 18 ++++++++++++++++++ .../create_generic_validator.ex | 6 +++--- .../object_validators/question_validator.ex | 6 +++++- lib/pleroma/web/activity_pub/transmogrifier.ex | 7 ++++++- 6 files changed, 39 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index e97381954..49ce5a938 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -120,6 +120,7 @@ def answer(user, object, name) do %{ "type" => "Answer", "actor" => user.ap_id, + "attributedTo" => user.ap_id, "cc" => [object.data["actor"]], "to" => [], "name" => name, diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex index 9861eec7f..ebddd5038 100644 --- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex @@ -26,6 +26,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do field(:name, :string) field(:inReplyTo, :string) field(:attributedTo, ObjectValidators.ObjectID) + + # TODO: Remove actor on objects field(:actor, ObjectValidators.ObjectID) end @@ -54,8 +56,10 @@ def changeset(struct, data) do def validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Answer"]) - |> validate_required([:id, :inReplyTo, :name]) + |> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor]) |> CommonValidations.validate_any_presence([:cc, :to]) - |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_fields_match([:actor, :attributedTo]) + |> CommonValidations.validate_actor_is_active() + |> CommonValidations.validate_host_match() end end diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index 140555a45..e981dacaa 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -115,4 +115,22 @@ def validate_host_match(cng, fields \\ [:id, :actor]) do end) end end + + def validate_fields_match(cng, fields) do + unique_fields = + fields + |> Enum.map(fn field -> get_field(cng, field) end) + |> Enum.uniq() + |> Enum.count() + + if unique_fields == 1 do + cng + else + fields + |> Enum.reduce(cng, fn field, cng -> + cng + |> add_error(field, "Fields #{inspect(fields)} aren't matching") + end) + end + end end diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index 97e2def10..54ea14f89 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -87,14 +87,14 @@ def validate_object_nonexistence(cng) do end def validate_actors_match(cng, meta) do - object_actor = meta[:object_data]["actor"] + attributed_to = meta[:object_data]["attributedTo"] || meta[:object_data]["actor"] cng |> validate_change(:actor, fn :actor, actor -> - if actor == object_actor do + if actor == attributed_to do [] else - [{:actor, "Actor doesn't match with object actor"}] + [{:actor, "Actor doesn't match with object attributedTo"}] end end) end diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 53cf35d40..466b3e6c2 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -28,7 +28,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do field(:type, :string) field(:content, :string) field(:context, :string) + + # TODO: Remove actor on objects field(:actor, ObjectValidators.ObjectID) + field(:attributedTo, ObjectValidators.ObjectID) field(:summary, :string) field(:published, ObjectValidators.DateTime) @@ -108,8 +111,9 @@ def changeset(struct, data) do def validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Question"]) - |> validate_required([:id, :actor, :type, :content, :context]) + |> validate_required([:id, :actor, :attributedTo, :type, :content, :context]) |> CommonValidations.validate_any_presence([:cc, :to]) + |> CommonValidations.validate_fields_match([:actor, :attributedTo]) |> CommonValidations.validate_actor_is_active() |> CommonValidations.validate_any_presence([:oneOf, :anyOf]) |> CommonValidations.validate_host_match() diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 0ad982720..6ab8a52c1 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -157,7 +157,12 @@ def fix_addressing(object) do end def fix_actor(%{"attributedTo" => actor} = object) do - Map.put(object, "actor", Containment.get_actor(%{"actor" => actor})) + actor = Containment.get_actor(%{"actor" => actor}) + + # TODO: Remove actor field for Objects + object + |> Map.put("actor", actor) + |> Map.put("attributedTo", actor) end def fix_in_reply_to(object, options \\ []) From 922ca232988b90b7a4fb5918bb76c383c90fd770 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 2 Jul 2020 05:47:18 +0200 Subject: [PATCH 23/50] Question: Add tests on HTML tags in options Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/1362 --- .../transmogrifier/question_handling_test.exs | 35 +++++++++++++++++++ .../web/mastodon_api/views/poll_view_test.exs | 29 +++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/test/web/activity_pub/transmogrifier/question_handling_test.exs b/test/web/activity_pub/transmogrifier/question_handling_test.exs index b7b9a1a7b..fba8106b5 100644 --- a/test/web/activity_pub/transmogrifier/question_handling_test.exs +++ b/test/web/activity_pub/transmogrifier/question_handling_test.exs @@ -55,6 +55,41 @@ test "Mastodon Question activity" do ]) end + test "Mastodon Question activity with HTML tags in plaintext" do + options = [ + %{ + "type" => "Note", + "name" => "", + "replies" => %{"totalItems" => 0, "type" => "Collection"} + }, + %{ + "type" => "Note", + "name" => "", + "replies" => %{"totalItems" => 0, "type" => "Collection"} + }, + %{ + "type" => "Note", + "name" => "", + "replies" => %{"totalItems" => 1, "type" => "Collection"} + }, + %{ + "type" => "Note", + "name" => "", + "replies" => %{"totalItems" => 1, "type" => "Collection"} + } + ] + + data = + File.read!("test/fixtures/mastodon-question-activity.json") + |> Poison.decode!() + |> Kernel.put_in(["object", "oneOf"], options) + + {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data) + object = Object.normalize(activity, false) + + assert Enum.sort(object.data["oneOf"]) == Enum.sort(options) + end + test "returns an error if received a second time" do data = File.read!("test/fixtures/mastodon-question-activity.json") |> Poison.decode!() diff --git a/test/web/mastodon_api/views/poll_view_test.exs b/test/web/mastodon_api/views/poll_view_test.exs index 76672f36c..b7e2f17ef 100644 --- a/test/web/mastodon_api/views/poll_view_test.exs +++ b/test/web/mastodon_api/views/poll_view_test.exs @@ -135,4 +135,33 @@ test "does not crash on polls with no end date" do assert result[:expires_at] == nil assert result[:expired] == false end + + test "doesn't strips HTML tags" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + status: "What's with the smug face?", + poll: %{ + options: [ + "", + "", + "", + "" + ], + expires_in: 20 + } + }) + + object = Object.normalize(activity) + + assert %{ + options: [ + %{title: "", votes_count: 0}, + %{title: "", votes_count: 0}, + %{title: "", votes_count: 0}, + %{title: "", votes_count: 0} + ] + } = PollView.render("show.json", %{object: object}) + end end From e4beff90f5670876184b2593c1b4a49f2339d048 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 2 Jul 2020 05:45:19 +0200 Subject: [PATCH 24/50] Create Question: Add context field to create --- lib/pleroma/web/activity_pub/builder.ex | 10 +++++++++- .../create_generic_validator.ex | 17 +++++++++++++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 2 ++ .../transmogrifier/question_handling_test.exs | 14 ++++++++++++++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index 49ce5a938..1b4c421b8 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -80,6 +80,13 @@ def delete(actor, object_id) do end def create(actor, object, recipients) do + context = + if is_map(object) do + object["context"] + else + nil + end + {:ok, %{ "id" => Utils.generate_activity_id(), @@ -88,7 +95,8 @@ def create(actor, object, recipients) do "object" => object, "type" => "Create", "published" => DateTime.utc_now() |> DateTime.to_iso8601() - }, []} + } + |> Pleroma.Maps.put_if_present("context", context), []} end def chat_message(actor, recipient, content, opts \\ []) do diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index 54ea14f89..ff889330e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -24,6 +24,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do field(:cc, ObjectValidators.Recipients, default: []) field(:object, ObjectValidators.ObjectID) field(:expires_at, ObjectValidators.DateTime) + + # Should be moved to object, done for CommonAPI.Utils.make_context + field(:context, :string) end def cast_data(data) do @@ -55,6 +58,7 @@ def validate_data(cng, meta \\ []) do |> validate_actor_is_active() |> validate_any_presence([:to, :cc]) |> validate_actors_match(meta) + |> validate_context_match(meta) |> validate_object_nonexistence() |> validate_object_containment() end @@ -98,4 +102,17 @@ def validate_actors_match(cng, meta) do end end) end + + def validate_context_match(cng, %{object_data: %{"context" => object_context}}) do + cng + |> validate_change(:context, fn :context, context -> + if context == object_context do + [] + else + [{:context, "context field not matching between Create and object (#{object_context})"}] + end + end) + end + + def validate_context_match(cng, _), do: cng end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 6ab8a52c1..edabe1130 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -643,6 +643,8 @@ def handle_incoming( |> Map.put("object", fix_object(object)) |> fix_addressing() + data = Map.put_new(data, "context", data["object"]["context"]) + with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do {:ok, activity} diff --git a/test/web/activity_pub/transmogrifier/question_handling_test.exs b/test/web/activity_pub/transmogrifier/question_handling_test.exs index fba8106b5..12516c4ab 100644 --- a/test/web/activity_pub/transmogrifier/question_handling_test.exs +++ b/test/web/activity_pub/transmogrifier/question_handling_test.exs @@ -8,6 +8,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.QuestionHandlingTest do alias Pleroma.Activity alias Pleroma.Object alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.CommonAPI + + import Pleroma.Factory setup_all do Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -23,6 +26,8 @@ test "Mastodon Question activity" do assert object.data["closed"] == "2019-05-11T09:03:36Z" + assert object.data["context"] == activity.data["context"] + assert object.data["context"] == "tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation" @@ -53,6 +58,15 @@ test "Mastodon Question activity" do "type" => "Note" } ]) + + user = insert(:user) + + {:ok, reply_activity} = CommonAPI.post(user, %{status: "hewwo", in_reply_to_id: activity.id}) + + reply_object = Object.normalize(reply_activity, false) + + assert reply_object.data["context"] == object.data["context"] + assert reply_object.data["context_id"] == object.data["context_id"] end test "Mastodon Question activity with HTML tags in plaintext" do From c2c3dd46133499db4102a946f07be87efdf82f1a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 2 Aug 2020 13:42:23 -0500 Subject: [PATCH 25/50] Migrate legacy tags set by AdminFE to match TagPolicy, #2010 --- .../20200802170532_fix_legacy_tags.exs | 37 +++++++++++++++++++ .../20200802170532_fix_legacy_tags_test.exs | 24 ++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 priv/repo/migrations/20200802170532_fix_legacy_tags.exs create mode 100644 test/migrations/20200802170532_fix_legacy_tags_test.exs diff --git a/priv/repo/migrations/20200802170532_fix_legacy_tags.exs b/priv/repo/migrations/20200802170532_fix_legacy_tags.exs new file mode 100644 index 000000000..f7274b44e --- /dev/null +++ b/priv/repo/migrations/20200802170532_fix_legacy_tags.exs @@ -0,0 +1,37 @@ +# Fix legacy tags set by AdminFE that don't align with TagPolicy MRF + +defmodule Pleroma.Repo.Migrations.FixLegacyTags do + use Ecto.Migration + alias Pleroma.Repo + alias Pleroma.User + import Ecto.Query + + @old_new_map %{ + "force_nsfw" => "mrf_tag:media-force-nsfw", + "strip_media" => "mrf_tag:media-strip", + "force_unlisted" => "mrf_tag:force-unlisted", + "sandbox" => "mrf_tag:sandbox", + "disable_remote_subscription" => "mrf_tag:disable-remote-subscription", + "disable_any_subscription" => "mrf_tag:disable-any-subscription" + } + + def change do + legacy_tags = Map.keys(@old_new_map) + + from(u in User, where: fragment("? && ?", u.tags, ^legacy_tags)) + |> Repo.all() + |> Enum.each(fn user -> + fix_tags_changeset(user) + |> Repo.update() + end) + end + + defp fix_tags_changeset(%User{tags: tags} = user) do + new_tags = + Enum.map(tags, fn tag -> + Map.get(@old_new_map, tag, tag) + end) + + Ecto.Changeset.change(user, tags: new_tags) + end +end diff --git a/test/migrations/20200802170532_fix_legacy_tags_test.exs b/test/migrations/20200802170532_fix_legacy_tags_test.exs new file mode 100644 index 000000000..3b4dee407 --- /dev/null +++ b/test/migrations/20200802170532_fix_legacy_tags_test.exs @@ -0,0 +1,24 @@ +defmodule Pleroma.Repo.Migrations.FixLegacyTagsTest do + alias Pleroma.User + use Pleroma.DataCase + import Pleroma.Factory + import Pleroma.Tests.Helpers + + setup_all do: require_migration("20200802170532_fix_legacy_tags") + + test "change/0 converts legacy user tags into correct values", %{migration: migration} do + user = insert(:user, tags: ["force_nsfw", "force_unlisted", "verified"]) + user2 = insert(:user) + + assert :ok == migration.change() + + fixed_user = User.get_by_id(user.id) + fixed_user2 = User.get_by_id(user2.id) + + assert fixed_user.tags == ["mrf_tag:media-force-nsfw", "mrf_tag:force-unlisted", "verified"] + assert fixed_user2.tags == [] + + # user2 should not have been updated + assert fixed_user2.updated_at == fixed_user2.inserted_at + end +end From 56e9bf33932bacfdffd700b97e3117fc593cac11 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Tue, 4 Aug 2020 14:35:47 +0300 Subject: [PATCH 26/50] Unify Config.get behaviour for atom/list key param --- lib/pleroma/config.ex | 34 +++++++++++++++++++++++++++------- test/config_test.exs | 28 ++++++++++++++++++++++++++++ test/support/helpers.ex | 14 ++++++++++++-- 3 files changed, 67 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index cc80deff5..88d1972ba 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -11,13 +11,33 @@ def get(key), do: get(key, nil) def get([key], default), do: get(key, default) - def get([parent_key | keys], default) do - case :pleroma - |> Application.get_env(parent_key) - |> get_in(keys) do - nil -> default - any -> any - end + def get([root_key | keys], default) do + # This is to mimic Application.get_env/3 behaviour that returns `nil` if the + # actual value is `nil`. + Enum.reduce_while(keys, Application.get_env(:pleroma, root_key), fn key, config -> + case key do + [last_key] when is_map(config) -> + {:halt, Map.get(config, last_key, default)} + + [last_key] when is_list(config) -> + {:halt, Keyword.get(config, last_key, default)} + + _ -> + case config do + %{^key => value} -> + {:cont, value} + + [_ | _] -> + case :lists.keyfind(key, 1, config) do + {_, value} -> {:cont, value} + _ -> {:halt, default} + end + + _ -> + {:halt, default} + end + end + end) end def get(key, default) do diff --git a/test/config_test.exs b/test/config_test.exs index a46ab4302..3f3da06d0 100644 --- a/test/config_test.exs +++ b/test/config_test.exs @@ -28,6 +28,34 @@ test "get/1 with a list of keys" do assert Pleroma.Config.get([:azerty, :uiop], true) == true end + describe "nil values" do + setup do + Pleroma.Config.put(:lorem, nil) + Pleroma.Config.put(:ipsum, %{dolor: [sit: nil]}) + Pleroma.Config.put(:dolor, sit: %{amet: nil}) + + on_exit(fn -> Enum.each(~w(lorem ipsum dolor)a, &Pleroma.Config.delete/1) end) + end + + test "get/1 with an atom for nil value" do + assert Pleroma.Config.get(:lorem) == nil + end + + test "get/2 with an atom for nil value" do + assert Pleroma.Config.get(:lorem, true) == nil + end + + test "get/1 with a list of keys for nil value" do + assert Pleroma.Config.get([:ipsum, :dolor, :sit]) == nil + assert Pleroma.Config.get([:dolor, :sit, :amet]) == nil + end + + test "get/2 with a list of keys for nil value" do + assert Pleroma.Config.get([:ipsum, :dolor, :sit], true) == nil + assert Pleroma.Config.get([:dolor, :sit, :amet], true) == nil + end + end + test "get/1 when value is false" do Pleroma.Config.put([:instance, :false_test], false) Pleroma.Config.put([:instance, :nested], []) diff --git a/test/support/helpers.ex b/test/support/helpers.ex index 5cbf2e291..7d729541d 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -17,9 +17,19 @@ defmacro clear_config(config_path) do defmacro clear_config(config_path, do: yield) do quote do - initial_setting = Config.get(unquote(config_path)) + initial_setting = Config.get(unquote(config_path), :__clear_config_absent__) unquote(yield) - on_exit(fn -> Config.put(unquote(config_path), initial_setting) end) + + on_exit(fn -> + case initial_setting do + :__clear_config_absent__ -> + Config.delete(unquote(config_path)) + + _ -> + Config.put(unquote(config_path), initial_setting) + end + end) + :ok end end From 0f088d8ce35150d7baa0591a25c831fce0181239 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 4 Aug 2020 14:23:35 +0200 Subject: [PATCH 27/50] question_validator: Allow content to be an empty-string (blank) --- .../activity_pub/object_validators/question_validator.ex | 2 +- .../transmogrifier/question_handling_test.exs | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 466b3e6c2..d248c6aec 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -111,7 +111,7 @@ def changeset(struct, data) do def validate_data(data_cng) do data_cng |> validate_inclusion(:type, ["Question"]) - |> validate_required([:id, :actor, :attributedTo, :type, :content, :context]) + |> validate_required([:id, :actor, :attributedTo, :type, :context]) |> CommonValidations.validate_any_presence([:cc, :to]) |> CommonValidations.validate_fields_match([:actor, :attributedTo]) |> CommonValidations.validate_actor_is_active() diff --git a/test/web/activity_pub/transmogrifier/question_handling_test.exs b/test/web/activity_pub/transmogrifier/question_handling_test.exs index 12516c4ab..9fb965d7f 100644 --- a/test/web/activity_pub/transmogrifier/question_handling_test.exs +++ b/test/web/activity_pub/transmogrifier/question_handling_test.exs @@ -111,4 +111,13 @@ test "returns an error if received a second time" do assert {:error, {:validate_object, {:error, _}}} = Transmogrifier.handle_incoming(data) end + + test "accepts a Question with no content" do + data = + File.read!("test/fixtures/mastodon-question-activity.json") + |> Poison.decode!() + |> Kernel.put_in(["object", "content"], "") + + assert {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(data) + end end From 184742af5eed2c48ba8518f1e114cbe0655ad467 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 3 Aug 2020 22:32:51 -0500 Subject: [PATCH 28/50] Unique apps.client_id for new installations, fixes #2022 --- ...200804183107_add_unique_index_to_app_client_id.exs | 7 +++++++ test/web/oauth/app_test.exs | 11 +++++++++++ 2 files changed, 18 insertions(+) create mode 100644 priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs diff --git a/priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs b/priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs new file mode 100644 index 000000000..83de18096 --- /dev/null +++ b/priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddUniqueIndexToAppClientId do + use Ecto.Migration + + def change do + create(unique_index(:apps, [:client_id])) + end +end diff --git a/test/web/oauth/app_test.exs b/test/web/oauth/app_test.exs index 899af648e..993a490e0 100644 --- a/test/web/oauth/app_test.exs +++ b/test/web/oauth/app_test.exs @@ -29,5 +29,16 @@ test "gets exist app and updates scopes" do assert exist_app.id == app.id assert exist_app.scopes == ["read", "write", "follow", "push"] end + + test "has unique client_id" do + insert(:oauth_app, client_name: "", redirect_uris: "", client_id: "boop") + + error = + catch_error(insert(:oauth_app, client_name: "", redirect_uris: "", client_id: "boop")) + + assert %Ecto.ConstraintError{} = error + assert error.constraint == "apps_client_id_index" + assert error.type == :unique + end end end From 079e410d6efcb39e72a238c13e52bd1898b442a2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 4 Aug 2020 13:12:23 -0500 Subject: [PATCH 29/50] Add a migration to clean up activity_expirations table --- ...0804180322_remove_nonlocal_expirations.exs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs diff --git a/priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs b/priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs new file mode 100644 index 000000000..389935f0d --- /dev/null +++ b/priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs @@ -0,0 +1,19 @@ +defmodule Pleroma.Repo.Migrations.RemoveNonlocalExpirations do + use Ecto.Migration + + def up do + statement = """ + DELETE FROM + activity_expirations A USING activities B + WHERE + A.activity_id = B.id + AND B.local = false; + """ + + execute(statement) + end + + def down do + :ok + end +end From 577b11167cb55203d30c43773f40108a87b2be6d Mon Sep 17 00:00:00 2001 From: Karol Kosek Date: Wed, 5 Aug 2020 00:01:30 +0200 Subject: [PATCH 30/50] templates/layout/app.html.eex: fix link color --- lib/pleroma/web/templates/layout/app.html.eex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index 5836ec1e0..51603fe0c 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -37,7 +37,7 @@ } a { - color: color: #d8a070; + color: #d8a070; text-decoration: none; } From f341a8e142ad9d4c92afc4a97ef387df068e38e0 Mon Sep 17 00:00:00 2001 From: MK Fain Date: Wed, 5 Aug 2020 02:01:27 +0000 Subject: [PATCH 31/50] Update filter_view.ex to return whole_word actual value --- lib/pleroma/web/mastodon_api/views/filter_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/filter_view.ex b/lib/pleroma/web/mastodon_api/views/filter_view.ex index aeff646f5..c37f624e0 100644 --- a/lib/pleroma/web/mastodon_api/views/filter_view.ex +++ b/lib/pleroma/web/mastodon_api/views/filter_view.ex @@ -25,7 +25,7 @@ def render("show.json", %{filter: filter}) do context: filter.context, expires_at: expires_at, irreversible: filter.hide, - whole_word: false + whole_word: filter.whole_word } end end From 6f60ac9f41d9511afa71986f000a2fc6c637b0c5 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 5 Aug 2020 13:00:49 +0300 Subject: [PATCH 32/50] Refactor config --- lib/pleroma/config.ex | 61 ++++++++++++-------------- test/application_requirements_test.exs | 5 ++- test/config_test.exs | 16 +++++++ 3 files changed, 49 insertions(+), 33 deletions(-) diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index 88d1972ba..98099ca58 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -11,33 +11,11 @@ def get(key), do: get(key, nil) def get([key], default), do: get(key, default) - def get([root_key | keys], default) do - # This is to mimic Application.get_env/3 behaviour that returns `nil` if the - # actual value is `nil`. - Enum.reduce_while(keys, Application.get_env(:pleroma, root_key), fn key, config -> - case key do - [last_key] when is_map(config) -> - {:halt, Map.get(config, last_key, default)} - - [last_key] when is_list(config) -> - {:halt, Keyword.get(config, last_key, default)} - - _ -> - case config do - %{^key => value} -> - {:cont, value} - - [_ | _] -> - case :lists.keyfind(key, 1, config) do - {_, value} -> {:cont, value} - _ -> {:halt, default} - end - - _ -> - {:halt, default} - end - end - end) + def get([_ | _] = path, default) do + case fetch(path) do + {:ok, value} -> value + :error -> default + end end def get(key, default) do @@ -54,6 +32,22 @@ def get!(key) do end end + def fetch([root_key | keys]) do + Enum.reduce_while(keys, Application.fetch_env(:pleroma, root_key), fn + key, {:ok, config} when is_map(config) or is_list(config) -> + case Access.fetch(config, key) do + :error -> + {:halt, :error} + + value -> + {:cont, value} + end + + _key, _config -> + {:halt, :error} + end) + end + def put([key], value), do: put(key, value) def put([parent_key | keys], value) do @@ -70,12 +64,15 @@ def put(key, value) do def delete([key]), do: delete(key) - def delete([parent_key | keys]) do - {_, parent} = - Application.get_env(:pleroma, parent_key) - |> get_and_update_in(keys, fn _ -> :pop end) + def delete([parent_key | keys] = path) do + with {:ok, _} <- fetch(path) do + {_, parent} = + parent_key + |> get() + |> get_and_update_in(keys, fn _ -> :pop end) - Application.put_env(:pleroma, parent_key, parent) + Application.put_env(:pleroma, parent_key, parent) + end end def delete(key) do diff --git a/test/application_requirements_test.exs b/test/application_requirements_test.exs index 21d24ddd0..e96295955 100644 --- a/test/application_requirements_test.exs +++ b/test/application_requirements_test.exs @@ -127,7 +127,10 @@ test "doesn't do anything if rum disabled" do :ok end - setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check]) + setup do + Pleroma.Config.get(:i_am_aware_this_may_cause_data_loss, 42) |> IO.inspect() + clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check]) + end test "raises if it detects unapplied migrations" do assert_raise Pleroma.ApplicationRequirements.VerifyError, diff --git a/test/config_test.exs b/test/config_test.exs index 3f3da06d0..e2c18304e 100644 --- a/test/config_test.exs +++ b/test/config_test.exs @@ -117,5 +117,21 @@ test "delete/2 with a list of keys" do Pleroma.Config.put([:delete_me, :delete_me], hello: "world", world: "Hello") Pleroma.Config.delete([:delete_me, :delete_me, :world]) assert Pleroma.Config.get([:delete_me, :delete_me]) == [hello: "world"] + + assert Pleroma.Config.delete([:this_key_does_not_exist]) + assert Pleroma.Config.delete([:non, :existing, :key]) + end + + test "fetch/1" do + Pleroma.Config.put([:lorem], :ipsum) + Pleroma.Config.put([:ipsum], dolor: :sit) + + assert Pleroma.Config.fetch([:lorem]) == {:ok, :ipsum} + assert Pleroma.Config.fetch([:ipsum, :dolor]) == {:ok, :sit} + assert Pleroma.Config.fetch([:lorem, :ipsum]) == :error + assert Pleroma.Config.fetch([:loremipsum]) == :error + + Pleroma.Config.delete([:lorem]) + Pleroma.Config.delete([:ipsum]) end end From 00c4c6a382d9965ea42236232094c4352c9ebae1 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 5 Aug 2020 12:24:34 +0200 Subject: [PATCH 33/50] CommonValidations: Remove superfluous function The `is_active` functionality was integrated into the presence checker. --- .../object_validators/answer_validator.ex | 2 +- .../object_validators/common_validations.ex | 13 ------------- .../object_validators/create_generic_validator.ex | 2 +- .../object_validators/question_validator.ex | 2 +- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex index ebddd5038..323367642 100644 --- a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex @@ -59,7 +59,7 @@ def validate_data(data_cng) do |> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor]) |> CommonValidations.validate_any_presence([:cc, :to]) |> CommonValidations.validate_fields_match([:actor, :attributedTo]) - |> CommonValidations.validate_actor_is_active() + |> CommonValidations.validate_actor_presence() |> CommonValidations.validate_host_match() end end diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index 57d4456aa..67352f801 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -47,19 +47,6 @@ def validate_actor_presence(cng, options \\ []) do end) end - def validate_actor_is_active(cng, options \\ []) do - field_name = Keyword.get(options, :field_name, :actor) - - cng - |> validate_change(field_name, fn field_name, actor -> - if %User{deactivated: false} = User.get_cached_by_ap_id(actor) do - [] - else - [{field_name, "can't find user (or deactivated)"}] - end - end) - end - def validate_object_presence(cng, options \\ []) do field_name = Keyword.get(options, :field_name, :object) allowed_types = Keyword.get(options, :allowed_types, false) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index ff889330e..2569df7f6 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -55,7 +55,7 @@ def validate_data(cng, meta \\ []) do cng |> validate_required([:actor, :type, :object]) |> validate_inclusion(:type, ["Create"]) - |> validate_actor_is_active() + |> validate_actor_presence() |> validate_any_presence([:to, :cc]) |> validate_actors_match(meta) |> validate_context_match(meta) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index d248c6aec..694cb6730 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -114,7 +114,7 @@ def validate_data(data_cng) do |> validate_required([:id, :actor, :attributedTo, :type, :context]) |> CommonValidations.validate_any_presence([:cc, :to]) |> CommonValidations.validate_fields_match([:actor, :attributedTo]) - |> CommonValidations.validate_actor_is_active() + |> CommonValidations.validate_actor_presence() |> CommonValidations.validate_any_presence([:oneOf, :anyOf]) |> CommonValidations.validate_host_match() end From 70522989d9d1119e5b3d86151f633f849d92f307 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 5 Aug 2020 11:14:58 +0000 Subject: [PATCH 34/50] Apply 1 suggestion(s) to 1 file(s) --- lib/pleroma/web/mastodon_api/views/poll_view.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex index ce595ae8a..1bfc99259 100644 --- a/lib/pleroma/web/mastodon_api/views/poll_view.ex +++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex @@ -28,10 +28,10 @@ def render("show.json", %{object: object, multiple: multiple, options: options} def render("show.json", %{object: object} = params) do case object.data do - %{"anyOf" => options} when is_list(options) and options != [] -> + %{"anyOf" => [ _ | _] = options} -> render(__MODULE__, "show.json", Map.merge(params, %{multiple: true, options: options})) - %{"oneOf" => options} when is_list(options) and options != [] -> + %{"oneOf" => [ _ | _] = options} -> render(__MODULE__, "show.json", Map.merge(params, %{multiple: false, options: options})) _ -> From b5f0cef156c1d1dd0376a791d8b4be48591f2c27 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 5 Aug 2020 11:33:21 +0000 Subject: [PATCH 35/50] Apply 1 suggestion(s) to 1 file(s) --- lib/pleroma/object.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 4dd929cfd..b3e654857 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -255,9 +255,7 @@ def increase_replies_count(ap_id) do end end - defp poll_is_multiple?(%Object{data: %{"anyOf" => anyOf}}) do - !Enum.empty?(anyOf) - end + defp poll_is_multiple?(%Object{data: %{"anyOf" => [_ | _]}}), do: true defp poll_is_multiple?(_), do: false From f889400d05e86d8d9509577946a0ab3a55b3eabb Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 5 Aug 2020 14:51:33 +0200 Subject: [PATCH 36/50] Questions: Move fixes to validators. --- .../create_generic_validator.ex | 19 +++++++++++++++++-- .../object_validators/question_validator.ex | 10 ++++++++-- .../web/activity_pub/transmogrifier.ex | 11 ++--------- .../web/mastodon_api/views/poll_view.ex | 4 ++-- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index 2569df7f6..60868eae0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -29,7 +29,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do field(:context, :string) end - def cast_data(data) do + def cast_data(data, meta \\ []) do + data = fix(data, meta) + %__MODULE__{} |> changeset(data) end @@ -42,7 +44,7 @@ def cast_and_apply(data) do def cast_and_validate(data, meta \\ []) do data - |> cast_data + |> cast_data(meta) |> validate_data(meta) end @@ -51,6 +53,19 @@ def changeset(struct, data) do |> cast(data, __schema__(:fields)) end + defp fix_context(data, meta) do + if object = meta[:object_data] do + Map.put_new(data, "context", object["context"]) + else + data + end + end + + defp fix(data, meta) do + data + |> fix_context(meta) + end + def validate_data(cng, meta \\ []) do cng |> validate_required([:actor, :type, :object]) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 694cb6730..f47acf606 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -83,17 +83,23 @@ defp fix_closed(data) do # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults defp fix_defaults(data) do - %{data: %{"id" => context}, id: context_id} = Utils.create_context(data["context"]) + %{data: %{"id" => context}, id: context_id} = + Utils.create_context(data["context"] || data["conversation"]) data - |> Map.put_new_lazy("id", &Utils.generate_object_id/0) |> Map.put_new_lazy("published", &Utils.make_date/0) |> Map.put_new("context", context) |> Map.put_new("context_id", context_id) end + defp fix_attribution(data) do + data + |> Map.put_new("actor", data["attributedTo"]) + end + defp fix(data) do data + |> fix_attribution() |> fix_closed() |> fix_defaults() end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index f85a26679..7381d4476 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -634,17 +634,10 @@ def handle_incoming( end def handle_incoming( - %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, + %{"type" => "Create", "object" => %{"type" => objtype}} = data, _options ) - when objtype in ["Question", "Answer"] do - data = - data - |> Map.put("object", fix_object(object)) - |> fix_addressing() - - data = Map.put_new(data, "context", data["object"]["context"]) - + when objtype in ["Question", "Answer", "ChatMessage"] do with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do {:ok, activity} diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex index 1bfc99259..1208dc9a0 100644 --- a/lib/pleroma/web/mastodon_api/views/poll_view.ex +++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex @@ -28,10 +28,10 @@ def render("show.json", %{object: object, multiple: multiple, options: options} def render("show.json", %{object: object} = params) do case object.data do - %{"anyOf" => [ _ | _] = options} -> + %{"anyOf" => [_ | _] = options} -> render(__MODULE__, "show.json", Map.merge(params, %{multiple: true, options: options})) - %{"oneOf" => [ _ | _] = options} -> + %{"oneOf" => [_ | _] = options} -> render(__MODULE__, "show.json", Map.merge(params, %{multiple: false, options: options})) _ -> From 5221879c358a7859d54013597c9ed9ccbb494155 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 5 Aug 2020 15:40:32 +0200 Subject: [PATCH 37/50] Fix linting. --- lib/pleroma/object.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index b3e654857..052ad413b 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -255,7 +255,7 @@ def increase_replies_count(ap_id) do end end - defp poll_is_multiple?(%Object{data: %{"anyOf" => [_ | _]}}), do: true + defp poll_is_multiple?(%Object{data: %{"anyOf" => [_ | _]}}), do: true defp poll_is_multiple?(_), do: false From 97b57014496003cabb416766457552ef854fa658 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 5 Aug 2020 17:46:14 +0300 Subject: [PATCH 38/50] Update clear_config macro --- test/application_requirements_test.exs | 5 +---- test/support/helpers.ex | 8 ++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/test/application_requirements_test.exs b/test/application_requirements_test.exs index e96295955..21d24ddd0 100644 --- a/test/application_requirements_test.exs +++ b/test/application_requirements_test.exs @@ -127,10 +127,7 @@ test "doesn't do anything if rum disabled" do :ok end - setup do - Pleroma.Config.get(:i_am_aware_this_may_cause_data_loss, 42) |> IO.inspect() - clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check]) - end + setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check]) test "raises if it detects unapplied migrations" do assert_raise Pleroma.ApplicationRequirements.VerifyError, diff --git a/test/support/helpers.ex b/test/support/helpers.ex index 7d729541d..ecd4b1e18 100644 --- a/test/support/helpers.ex +++ b/test/support/helpers.ex @@ -17,16 +17,16 @@ defmacro clear_config(config_path) do defmacro clear_config(config_path, do: yield) do quote do - initial_setting = Config.get(unquote(config_path), :__clear_config_absent__) + initial_setting = Config.fetch(unquote(config_path)) unquote(yield) on_exit(fn -> case initial_setting do - :__clear_config_absent__ -> + :error -> Config.delete(unquote(config_path)) - _ -> - Config.put(unquote(config_path), initial_setting) + {:ok, value} -> + Config.put(unquote(config_path), value) end end) From 8c57a299b463b7e5916addbbd3571b35e1742ebd Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 5 Aug 2020 18:23:12 +0300 Subject: [PATCH 39/50] Handle non-list keys in Config.fetch/1 --- lib/pleroma/config.ex | 2 ++ test/config_test.exs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index 98099ca58..a8329cc1e 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -32,6 +32,8 @@ def get!(key) do end end + def fetch(key) when is_atom(key), do: fetch([key]) + def fetch([root_key | keys]) do Enum.reduce_while(keys, Application.fetch_env(:pleroma, root_key), fn key, {:ok, config} when is_map(config) or is_list(config) -> diff --git a/test/config_test.exs b/test/config_test.exs index e2c18304e..1556e4237 100644 --- a/test/config_test.exs +++ b/test/config_test.exs @@ -127,9 +127,11 @@ test "fetch/1" do Pleroma.Config.put([:ipsum], dolor: :sit) assert Pleroma.Config.fetch([:lorem]) == {:ok, :ipsum} + assert Pleroma.Config.fetch(:lorem) == {:ok, :ipsum} assert Pleroma.Config.fetch([:ipsum, :dolor]) == {:ok, :sit} assert Pleroma.Config.fetch([:lorem, :ipsum]) == :error assert Pleroma.Config.fetch([:loremipsum]) == :error + assert Pleroma.Config.fetch(:loremipsum) == :error Pleroma.Config.delete([:lorem]) Pleroma.Config.delete([:ipsum]) From 2173945f9012ec0db82a73fc7ed9423899dfd28f Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 5 Aug 2020 17:26:03 +0200 Subject: [PATCH 40/50] MailerTest: Give it some time. --- test/emails/mailer_test.exs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/emails/mailer_test.exs b/test/emails/mailer_test.exs index e6e34cba8..3da45056b 100644 --- a/test/emails/mailer_test.exs +++ b/test/emails/mailer_test.exs @@ -19,6 +19,7 @@ defmodule Pleroma.Emails.MailerTest do test "not send email when mailer is disabled" do Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], false) Mailer.deliver(@email) + :timer.sleep(100) refute_email_sent( from: {"Pleroma", "noreply@example.com"}, @@ -30,6 +31,7 @@ test "not send email when mailer is disabled" do test "send email" do Mailer.deliver(@email) + :timer.sleep(100) assert_email_sent( from: {"Pleroma", "noreply@example.com"}, @@ -41,6 +43,7 @@ test "send email" do test "perform" do Mailer.perform(:deliver_async, @email, []) + :timer.sleep(100) assert_email_sent( from: {"Pleroma", "noreply@example.com"}, From 9c96fc052a89789b398794761741783eaa86d6a1 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 5 Aug 2020 17:26:53 +0200 Subject: [PATCH 41/50] CommonValidations: Extract modification right checker --- .../object_validators/common_validations.ex | 27 ++++++++++++++++++ .../object_validators/delete_validator.ex | 28 +------------------ .../delete_validation_test.exs | 2 +- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index 67352f801..e4c5d9619 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -125,4 +125,31 @@ def validate_fields_match(cng, fields) do end) end end + + def same_domain?(cng, field_one \\ :actor, field_two \\ :object) do + actor_uri = + cng + |> get_field(field_one) + |> URI.parse() + + object_uri = + cng + |> get_field(field_two) + |> URI.parse() + + object_uri.host == actor_uri.host + end + + # This figures out if a user is able to create, delete or modify something + # based on the domain and superuser status + def validate_modification_rights(cng) do + actor = User.get_cached_by_ap_id(get_field(cng, :actor)) + + if User.superuser?(actor) || same_domain?(cng) do + cng + else + cng + |> add_error(:actor, "is not allowed to modify object") + end + end end diff --git a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex index 93a7b0e0b..2634e8d4d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do alias Pleroma.Activity alias Pleroma.EctoType.ActivityPub.ObjectValidators - alias Pleroma.User import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -59,7 +58,7 @@ def validate_data(cng) do |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Delete"]) |> validate_actor_presence() - |> validate_deletion_rights() + |> validate_modification_rights() |> validate_object_or_user_presence(allowed_types: @deletable_types) |> add_deleted_activity_id() end @@ -68,31 +67,6 @@ def do_not_federate?(cng) do !same_domain?(cng) end - defp same_domain?(cng) do - actor_uri = - cng - |> get_field(:actor) - |> URI.parse() - - object_uri = - cng - |> get_field(:object) - |> URI.parse() - - object_uri.host == actor_uri.host - end - - def validate_deletion_rights(cng) do - actor = User.get_cached_by_ap_id(get_field(cng, :actor)) - - if User.superuser?(actor) || same_domain?(cng) do - cng - else - cng - |> add_error(:actor, "is not allowed to delete object") - end - end - def cast_and_validate(data) do data |> cast_data diff --git a/test/web/activity_pub/object_validators/delete_validation_test.exs b/test/web/activity_pub/object_validators/delete_validation_test.exs index 42cd18298..02683b899 100644 --- a/test/web/activity_pub/object_validators/delete_validation_test.exs +++ b/test/web/activity_pub/object_validators/delete_validation_test.exs @@ -87,7 +87,7 @@ test "it's invalid if the actor of the object and the actor of delete are from d {:error, cng} = ObjectValidator.validate(invalid_other_actor, []) - assert {:actor, {"is not allowed to delete object", []}} in cng.errors + assert {:actor, {"is not allowed to modify object", []}} in cng.errors end test "it's valid if the actor of the object is a local superuser", From 3655175639a004976ef8296a0838e72642ba0b11 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 5 Aug 2020 17:36:27 +0200 Subject: [PATCH 42/50] CommonValidations: Refactor `same_domain?` --- .../object_validators/common_validations.ex | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index e4c5d9619..82a9d39b5 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -126,18 +126,21 @@ def validate_fields_match(cng, fields) do end end - def same_domain?(cng, field_one \\ :actor, field_two \\ :object) do - actor_uri = - cng - |> get_field(field_one) - |> URI.parse() + def same_domain?(cng, fields \\ [:actor, :object]) do + unique_domains = + fields + |> Enum.map(fn field -> + %URI{host: host} = + cng + |> get_field(field) + |> URI.parse() - object_uri = - cng - |> get_field(field_two) - |> URI.parse() + host + end) + |> Enum.uniq() + |> Enum.count() - object_uri.host == actor_uri.host + unique_domains == 1 end # This figures out if a user is able to create, delete or modify something From 9d7ce1a6d014499eb4d55190b81e55da849b5ad0 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 5 Aug 2020 17:56:12 +0200 Subject: [PATCH 43/50] CommonValidations: More refactors. --- .../object_validators/common_validations.ex | 53 +++++++------------ 1 file changed, 18 insertions(+), 35 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex index 82a9d39b5..603d87b8e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -84,20 +84,7 @@ def validate_object_or_user_presence(cng, options \\ []) do end def validate_host_match(cng, fields \\ [:id, :actor]) do - unique_hosts = - fields - |> Enum.map(fn field -> - %URI{host: host} = - cng - |> get_field(field) - |> URI.parse() - - host - end) - |> Enum.uniq() - |> Enum.count() - - if unique_hosts == 1 do + if same_domain?(cng, fields) do cng else fields @@ -109,13 +96,7 @@ def validate_host_match(cng, fields \\ [:id, :actor]) do end def validate_fields_match(cng, fields) do - unique_fields = - fields - |> Enum.map(fn field -> get_field(cng, field) end) - |> Enum.uniq() - |> Enum.count() - - if unique_fields == 1 do + if map_unique?(cng, fields) do cng else fields @@ -126,21 +107,23 @@ def validate_fields_match(cng, fields) do end end + defp map_unique?(cng, fields, func \\ & &1) do + Enum.reduce_while(fields, nil, fn field, acc -> + value = + cng + |> get_field(field) + |> func.() + + case {value, acc} do + {value, nil} -> {:cont, value} + {value, value} -> {:cont, value} + _ -> {:halt, false} + end + end) + end + def same_domain?(cng, fields \\ [:actor, :object]) do - unique_domains = - fields - |> Enum.map(fn field -> - %URI{host: host} = - cng - |> get_field(field) - |> URI.parse() - - host - end) - |> Enum.uniq() - |> Enum.count() - - unique_domains == 1 + map_unique?(cng, fields, fn value -> URI.parse(value).host end) end # This figures out if a user is able to create, delete or modify something From 7569f225f1d43c6435eda6b62fd5eff3cd3408e0 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Wed, 5 Aug 2020 19:38:55 +0300 Subject: [PATCH 44/50] Move checks to application startup --- lib/pleroma/application.ex | 18 ++++++++++++++++++ lib/pleroma/upload/filter/exiftool.ex | 14 +++++++++----- lib/pleroma/upload/filter/mogrifun.ex | 8 +++++--- lib/pleroma/upload/filter/mogrify.ex | 12 ++++++------ 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 0ffb55358..c0b5db9f1 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -47,6 +47,7 @@ def start(_type, _args) do Pleroma.ApplicationRequirements.verify!() setup_instrumenters() load_custom_modules() + check_system_commands() Pleroma.Docs.JSON.compile() adapter = Application.get_env(:tesla, :adapter) @@ -249,4 +250,21 @@ defp http_children(Tesla.Adapter.Gun, _) do end defp http_children(_, _), do: [] + + defp check_system_commands do + filters = Config.get([Pleroma.Upload, :filters]) + + check_filter = fn filter, command_required -> + with true <- filter in filters, + false <- Pleroma.Utils.command_available?(command_required) do + Logger.error( + "#{filter} is specified in list of Pleroma.Upload filters, but the #{command_required} command is not found" + ) + end + end + + check_filter.(Pleroma.Upload.Filters.Exiftool, "exiftool") + check_filter.(Pleroma.Upload.Filters.Mogrify, "mogrify") + check_filter.(Pleroma.Upload.Filters.Mogrifun, "mogrify") + end end diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index e1b976c98..ea8798fe3 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -9,12 +9,16 @@ defmodule Pleroma.Upload.Filter.Exiftool do """ @behaviour Pleroma.Upload.Filter + @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - if Pleroma.Utils.command_available?("exiftool") do - System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) - :ok - else - {:error, "exiftool command not found"} + try do + case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do + {_response, 0} -> :ok + {error, 1} -> {:error, error} + end + rescue + _e in ErlangError -> + {:error, "exiftool command not found"} end end diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex index 8f362333d..a8503ac24 100644 --- a/lib/pleroma/upload/filter/mogrifun.ex +++ b/lib/pleroma/upload/filter/mogrifun.ex @@ -34,12 +34,14 @@ defmodule Pleroma.Upload.Filter.Mogrifun do [{"fill", "yellow"}, {"tint", "40"}] ] + @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - if Pleroma.Utils.command_available?("mogrify") do + try do Filter.Mogrify.do_filter(file, [Enum.random(@filters)]) :ok - else - {:error, "mogrify command not found"} + rescue + _e in ErlangError -> + {:error, "mogrify command not found"} end end diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex index 4bd0c2eb4..7a45add5a 100644 --- a/lib/pleroma/upload/filter/mogrify.ex +++ b/lib/pleroma/upload/filter/mogrify.ex @@ -8,14 +8,14 @@ defmodule Pleroma.Upload.Filter.Mogrify do @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} @type conversions :: conversion() | [conversion()] + @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do - if Pleroma.Utils.command_available?("mogrify") do - filters = Pleroma.Config.get!([__MODULE__, :args]) - - do_filter(file, filters) + try do + do_filter(file, Pleroma.Config.get!([__MODULE__, :args])) :ok - else - {:error, "mogrify command not found"} + rescue + _e in ErlangError -> + {:error, "mogrify command not found"} end end From d6ab9f2132cdcbed303c9ef0941bf7210e49c5d6 Mon Sep 17 00:00:00 2001 From: Mary Kate Date: Wed, 5 Aug 2020 15:36:25 -0500 Subject: [PATCH 45/50] update test for whole_word in filter --- .../controllers/filter_controller_test.exs | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/test/web/mastodon_api/controllers/filter_controller_test.exs b/test/web/mastodon_api/controllers/filter_controller_test.exs index f29547d13..6e94150b4 100644 --- a/test/web/mastodon_api/controllers/filter_controller_test.exs +++ b/test/web/mastodon_api/controllers/filter_controller_test.exs @@ -64,11 +64,13 @@ test "fetching a list of filters" do test "get a filter" do %{user: user, conn: conn} = oauth_access(["read:filters"]) + # check whole_word true query = %Pleroma.Filter{ user_id: user.id, filter_id: 2, phrase: "knight", - context: ["home"] + context: ["home"], + whole_word: false } {:ok, filter} = Pleroma.Filter.create(query) @@ -76,6 +78,25 @@ test "get a filter" do conn = get(conn, "/api/v1/filters/#{filter.filter_id}") assert response = json_response_and_validate_schema(conn, 200) + assert response["whole_word"] == false + + # check whole_word false + %{user: user, conn: conn} = oauth_access(["read:filters"]) + + query = %Pleroma.Filter{ + user_id: user.id, + filter_id: 3, + phrase: "knight", + context: ["home"], + whole_word: true + } + + {:ok, filter} = Pleroma.Filter.create(query) + + conn = get(conn, "/api/v1/filters/#{filter.filter_id}") + + assert response = json_response_and_validate_schema(conn, 200) + assert response["whole_word"] == true end test "update a filter" do @@ -86,7 +107,8 @@ test "update a filter" do filter_id: 2, phrase: "knight", context: ["home"], - hide: true + hide: true, + whole_word: true } {:ok, _filter} = Pleroma.Filter.create(query) @@ -108,6 +130,7 @@ test "update a filter" do assert response["phrase"] == new.phrase assert response["context"] == new.context assert response["irreversible"] == true + assert response["whole_word"] == true end test "delete a filter" do From f785dba09bc6c6624c17350356632d008f701183 Mon Sep 17 00:00:00 2001 From: Mary Kate Date: Wed, 5 Aug 2020 15:39:11 -0500 Subject: [PATCH 46/50] changelog for filter whole_word fix --- CHANGELOG.md | 1 + test/web/mastodon_api/controllers/filter_controller_test.exs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de017e30a..572f9e84b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Fix CSP policy generation to include remote Captcha services - Fix edge case where MediaProxy truncates media, usually caused when Caddy is serving content for the other Federated instance. - Emoji Packs could not be listed when instance was set to `public: false` +- Fix whole_word always returning false on filter get requests ## [Unreleased (patch)] diff --git a/test/web/mastodon_api/controllers/filter_controller_test.exs b/test/web/mastodon_api/controllers/filter_controller_test.exs index 6e94150b4..0d426ec34 100644 --- a/test/web/mastodon_api/controllers/filter_controller_test.exs +++ b/test/web/mastodon_api/controllers/filter_controller_test.exs @@ -64,7 +64,7 @@ test "fetching a list of filters" do test "get a filter" do %{user: user, conn: conn} = oauth_access(["read:filters"]) - # check whole_word true + # check whole_word false query = %Pleroma.Filter{ user_id: user.id, filter_id: 2, @@ -80,7 +80,7 @@ test "get a filter" do assert response = json_response_and_validate_schema(conn, 200) assert response["whole_word"] == false - # check whole_word false + # check whole_word true %{user: user, conn: conn} = oauth_access(["read:filters"]) query = %Pleroma.Filter{ From 135ae4e35a3e6a084eb611ce3a21c7a6c6bba9fc Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 6 Aug 2020 16:00:00 +0300 Subject: [PATCH 47/50] [#2025] Defaulted OAuth login scopes choice to all scopes when user selects no scopes. --- lib/pleroma/web/oauth/oauth_controller.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index f29b3cb57..dd00600ea 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -76,6 +76,13 @@ defp do_authorize(%Plug.Conn{} = conn, params) do available_scopes = (app && app.scopes) || [] scopes = Scopes.fetch_scopes(params, available_scopes) + scopes = + if scopes == [] do + available_scopes + else + scopes + end + # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template render(conn, Authenticator.auth_template(), %{ response_type: params["response_type"], From e639eee82e1e0136bf6e64e571f2b05b5b7b948c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 6 Aug 2020 18:01:29 -0500 Subject: [PATCH 48/50] restricted_nicknames: Add names from MastoAPI endpoints --- config/config.exs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 933a899ab..393f74372 100644 --- a/config/config.exs +++ b/config/config.exs @@ -515,7 +515,13 @@ "user-search", "user_exists", "users", - "web" + "web", + "verify_credentials", + "update_credentials", + "relationships", + "search", + "confirmation_resend", + "mfa" ], email_blacklist: [] From ebb30128af653d146091fa2317418103fd1e46a3 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 7 Aug 2020 16:20:13 +0200 Subject: [PATCH 49/50] Changelog: Add information about the object age policy --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 572f9e84b..e2a855bef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Changed +- **Breaking:** Added the ObjectAgePolicy to the default set of MRFs. This will delist and strip the follower collection of any message received that is older than 7 days. This will stop users from seeing very old messages in the timelines. The messages can still be viewed on the user's page and in conversations. They also still trigger notifications. - **Breaking:** Elixir >=1.9 is now required (was >= 1.8) - **Breaking:** Configuration: `:auto_linker, :opts` moved to `:pleroma, Pleroma.Formatter`. Old config namespace is deprecated. - In Conversations, return only direct messages as `last_status` From 60fe0a08f0ed4b83847995f4e0a5ff10dcf9d336 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 7 Aug 2020 17:59:55 +0200 Subject: [PATCH 50/50] Docs: Remove wrong / confusing auth docs. --- docs/configuration/cheatsheet.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index f23cf4fe4..89036ded0 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -858,9 +858,6 @@ Warning: it's discouraged to use this feature because of the associated security ### :auth -* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator. -* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication. - Authentication / authorization settings. * `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`.