From 547def67a76854aa4c9c8438eb1ee4dfa36fd8ac Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 29 May 2022 11:36:00 -0400 Subject: [PATCH 01/51] Allow Updates by every actor on the same origin --- .../object_validators/update_validator.ex | 4 +++- .../update_handling_test.exs | 24 ++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex index a5def312e..1e940a400 100644 --- a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex @@ -51,7 +51,9 @@ def validate_updating_rights(cng) do with actor = get_field(cng, :actor), object = get_field(cng, :object), {:ok, object_id} <- ObjectValidators.ObjectID.cast(object), - true <- actor == object_id do + actor_uri <- URI.parse(actor), + object_uri <- URI.parse(object_id), + true <- actor_uri.host == object_uri.host do cng else _e -> diff --git a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs index 94bc5a89b..f2a22d370 100644 --- a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs @@ -32,7 +32,7 @@ test "validates a basic object", %{valid_update: valid_update} do test "returns an error if the object can't be updated by the actor", %{ valid_update: valid_update } do - other_user = insert(:user) + other_user = insert(:user, local: false) update = valid_update @@ -40,5 +40,27 @@ test "returns an error if the object can't be updated by the actor", %{ assert {:error, _cng} = ObjectValidator.validate(update, []) end + + test "validates as long as the object is same-origin with the actor", %{ + valid_update: valid_update + } do + other_user = insert(:user) + + update = + valid_update + |> Map.put("actor", other_user.ap_id) + + assert {:ok, _update, []} = ObjectValidator.validate(update, []) + end + + test "validates if the object is not of an Actor type" do + note = insert(:note) + updated_note = note.data |> Map.put("content", "edited content") + other_user = insert(:user) + + {:ok, update, _} = Builder.update(other_user, updated_note) + + assert {:ok, _update, []} = ObjectValidator.validate(update, []) + end end end From 0f6a5eb9a299629f295372f4d5ecdd9083a19717 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 29 May 2022 12:54:57 -0400 Subject: [PATCH 02/51] Handle Note and Question Updates --- lib/pleroma/web/activity_pub/side_effects.ex | 82 ++++++++++++++++--- .../web/activity_pub/side_effects_test.exs | 23 ++++++ 2 files changed, 95 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index b997c15db..aeddf3ed8 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -153,23 +153,25 @@ def handle( # Tasks this handles: # - Update the user + # - Update a non-user object (Note, Question, etc.) # # For a local user, we also get a changeset with the full information, so we # can update non-federating, non-activitypub settings as well. @impl true def handle(%{data: %{"type" => "Update", "object" => updated_object}} = object, meta) do - if changeset = Keyword.get(meta, :user_update_changeset) do - changeset - |> User.update_and_set_cache() + updated_object_id = updated_object["id"] + + with {_, true} <- {:has_id, is_binary(updated_object_id)}, + {_, user} <- {:user, Pleroma.User.get_by_ap_id(updated_object_id)} do + if user do + handle_update_user(object, meta) + else + handle_update_object(object, meta) + end else - {:ok, new_user_data} = ActivityPub.user_data_from_user_object(updated_object) - - User.get_by_ap_id(updated_object["id"]) - |> User.remote_user_changeset(new_user_data) - |> User.update_and_set_cache() + _ -> + {:ok, object, meta} end - - {:ok, object, meta} end # Tasks this handles: @@ -390,6 +392,66 @@ def handle(object, meta) do {:ok, object, meta} end + defp handle_update_user( + %{data: %{"type" => "Update", "object" => updated_object}} = object, + meta + ) do + if changeset = Keyword.get(meta, :user_update_changeset) do + changeset + |> User.update_and_set_cache() + else + {:ok, new_user_data} = ActivityPub.user_data_from_user_object(updated_object) + + User.get_by_ap_id(updated_object["id"]) + |> User.remote_user_changeset(new_user_data) + |> User.update_and_set_cache() + end + + {:ok, object, meta} + end + + @updatable_object_types ["Note", "Question"] + # We do not allow poll options to be changed, but the poll description can be. + @updatable_fields [ + "source", + "tag", + "updated", + "emoji", + "content", + "summary", + "sensitive", + "attachment", + "generator" + ] + defp handle_update_object( + %{data: %{"type" => "Update", "object" => updated_object}} = object, + meta + ) do + orig_object = Object.get_by_ap_id(updated_object["id"]) + orig_object_data = orig_object.data + + if orig_object_data["type"] in @updatable_object_types do + updated_object_data = + @updatable_fields + |> Enum.reduce( + orig_object_data, + fn field, acc -> + if Map.has_key?(updated_object, field) do + Map.put(acc, field, updated_object[field]) + else + Map.drop(acc, [field]) + end + end + ) + + orig_object + |> Object.change(%{data: updated_object_data}) + |> Object.update_and_set_cache() + end + + {:ok, object, meta} + end + def handle_object_creation(%{"type" => "ChatMessage"} = object, _activity, meta) do with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do actor = User.get_cached_by_ap_id(object.data["actor"]) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 64c4a8c14..f72753116 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -140,6 +140,29 @@ test "it uses a given changeset to update", %{user: user, update: update} do end end + describe "update notes" do + setup do + user = insert(:user) + note = insert(:note, user: user) + + updated_note = + note.data + |> Map.put("summary", "edited summary") + |> Map.put("content", "edited content") + + {:ok, update_data, []} = Builder.update(user, updated_note) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + %{user: user, object_id: note.id, update_data: update_data, update: update} + end + + test "it updates the note", %{object_id: object_id, update: update} do + {:ok, _, _} = SideEffects.handle(update) + new_note = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "edited summary", "content" => "edited content"} = new_note.data + end + end + describe "EmojiReact objects" do setup do poster = insert(:user) From 5e8aac0e07cf54d527643e9793b92f3c0b3826e2 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 29 May 2022 13:54:16 -0400 Subject: [PATCH 03/51] Record edit history for Note and Question Updates --- lib/pleroma/web/activity_pub/side_effects.ex | 34 +++++++++++++++++ priv/static/schemas/litepub-0.1.jsonld | 3 +- .../web/activity_pub/side_effects_test.exs | 37 ++++++++++++++++++- 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index aeddf3ed8..c4d56fa20 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -410,6 +410,26 @@ defp handle_update_user( {:ok, object, meta} end + defp history_for_object(object) do + with history <- Map.get(object, "formerRepresentations"), + true <- is_map(history), + "OrderedCollection" <- Map.get(history, "type"), + true <- is_list(Map.get(history, "orderedItems")), + true <- is_integer(Map.get(history, "totalItems")) do + history + else + _ -> history_skeleton() + end + end + + defp history_skeleton do + %{ + "type" => "OrderedCollection", + "totalItems" => 0, + "orderedItems" => [] + } + end + @updatable_object_types ["Note", "Question"] # We do not allow poll options to be changed, but the poll description can be. @updatable_fields [ @@ -431,6 +451,19 @@ defp handle_update_object( orig_object_data = orig_object.data if orig_object_data["type"] in @updatable_object_types do + # Put edit history + # Note that we may have got the edit history by first fetching the object + history = history_for_object(orig_object_data) + + latest_history_item = + orig_object_data + |> Map.drop(["id", "formerRepresentations"]) + + new_history = + history + |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) + |> Map.put("totalItems", history["totalItems"] + 1) + updated_object_data = @updatable_fields |> Enum.reduce( @@ -443,6 +476,7 @@ defp handle_update_object( end end ) + |> Map.put("formerRepresentations", new_history) orig_object |> Object.change(%{data: updated_object_data}) diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld index 946099a6e..650118475 100644 --- a/priv/static/schemas/litepub-0.1.jsonld +++ b/priv/static/schemas/litepub-0.1.jsonld @@ -36,7 +36,8 @@ "@id": "as:alsoKnownAs", "@type": "@id" }, - "vcard": "http://www.w3.org/2006/vcard/ns#" + "vcard": "http://www.w3.org/2006/vcard/ns#", + "formerRepresentations": "litepub:formerRepresentations" } ] } diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index f72753116..5c60504d4 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -153,7 +153,7 @@ test "it uses a given changeset to update", %{user: user, update: update} do {:ok, update_data, []} = Builder.update(user, updated_note) {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) - %{user: user, object_id: note.id, update_data: update_data, update: update} + %{user: user, note: note, object_id: note.id, update_data: update_data, update: update} end test "it updates the note", %{object_id: object_id, update: update} do @@ -161,6 +161,41 @@ test "it updates the note", %{object_id: object_id, update: update} do new_note = Pleroma.Object.get_by_id(object_id) assert %{"summary" => "edited summary", "content" => "edited content"} = new_note.data end + + test "it records the original note in formerRepresentations", %{ + note: note, + object_id: object_id, + update: update + } do + {:ok, _, _} = SideEffects.handle(update) + %{data: new_note} = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "edited summary", "content" => "edited content"} = new_note + + assert [Map.drop(note.data, ["id", "formerRepresentations"])] == + new_note["formerRepresentations"]["orderedItems"] + + assert new_note["formerRepresentations"]["totalItems"] == 1 + end + + test "it puts the original note at the front of formerRepresentations", %{ + note: note, + object_id: object_id, + update: update + } do + {:ok, _, _} = SideEffects.handle(update) + %{data: first_edit} = Pleroma.Object.get_by_id(object_id) + {:ok, _, _} = SideEffects.handle(update) + %{data: new_note} = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "edited summary", "content" => "edited content"} = new_note + + original_version = Map.drop(note.data, ["id", "formerRepresentations"]) + first_edit = Map.drop(first_edit, ["id", "formerRepresentations"]) + + assert [first_edit, original_version] == + new_note["formerRepresentations"]["orderedItems"] + + assert new_note["formerRepresentations"]["totalItems"] == 2 + end end describe "EmojiReact objects" do From 8acfe95f3e9d4183fd513cfe828500c852db4d5f Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 29 May 2022 22:16:03 -0400 Subject: [PATCH 04/51] Allow updating polls --- lib/pleroma/web/activity_pub/side_effects.ex | 81 ++++++++++++++----- .../web/activity_pub/side_effects_test.exs | 64 ++++++++++++++- 2 files changed, 125 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index c4d56fa20..ac327280c 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -443,14 +443,29 @@ defp history_skeleton do "attachment", "generator" ] - defp handle_update_object( - %{data: %{"type" => "Update", "object" => updated_object}} = object, - meta - ) do - orig_object = Object.get_by_ap_id(updated_object["id"]) - orig_object_data = orig_object.data + defp update_content_fields(orig_object_data, updated_object) do + @updatable_fields + |> Enum.reduce( + %{data: orig_object_data, updated: false}, + fn field, %{data: data, updated: updated} -> + updated = updated or Map.get(updated_object, field) != Map.get(orig_object_data, field) - if orig_object_data["type"] in @updatable_object_types do + data = + if Map.has_key?(updated_object, field) do + Map.put(data, field, updated_object[field]) + else + Map.drop(data, [field]) + end + + %{data: data, updated: updated} + end + ) + end + + defp maybe_update_history(updated_object, orig_object_data, updated) do + if not updated do + updated_object + else # Put edit history # Note that we may have got the edit history by first fetching the object history = history_for_object(orig_object_data) @@ -464,19 +479,47 @@ defp handle_update_object( |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) |> Map.put("totalItems", history["totalItems"] + 1) + updated_object + |> Map.put("formerRepresentations", new_history) + end + end + + defp maybe_update_poll(to_be_updated, updated_object) do + choice_key = fn data -> + if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" + end + + with true <- to_be_updated["type"] == "Question", + key <- choice_key.(updated_object), + true <- key == choice_key.(to_be_updated), + orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])), + new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])), + true <- orig_choices == new_choices do + # Choices are the same, but counts are different + to_be_updated + |> Map.put(key, updated_object[key]) + else + # Choices (or vote type) have changed, do not allow this + _ -> to_be_updated + end + end + + defp handle_update_object( + %{data: %{"type" => "Update", "object" => updated_object}} = object, + meta + ) do + orig_object = Object.get_by_ap_id(updated_object["id"]) + orig_object_data = orig_object.data + + if orig_object_data["type"] in @updatable_object_types do + %{data: updated_object_data, updated: updated} = + orig_object_data + |> update_content_fields(updated_object) + updated_object_data = - @updatable_fields - |> Enum.reduce( - orig_object_data, - fn field, acc -> - if Map.has_key?(updated_object, field) do - Map.put(acc, field, updated_object[field]) - else - Map.drop(acc, [field]) - end - end - ) - |> Map.put("formerRepresentations", new_history) + updated_object_data + |> maybe_update_history(orig_object_data, updated) + |> maybe_update_poll(updated_object) orig_object |> Object.change(%{data: updated_object_data}) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 5c60504d4..62394b058 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -178,15 +178,24 @@ test "it records the original note in formerRepresentations", %{ end test "it puts the original note at the front of formerRepresentations", %{ + user: user, note: note, object_id: object_id, update: update } do {:ok, _, _} = SideEffects.handle(update) %{data: first_edit} = Pleroma.Object.get_by_id(object_id) + + second_updated_note = + note.data + |> Map.put("summary", "edited summary 2") + |> Map.put("content", "edited content 2") + + {:ok, second_update_data, []} = Builder.update(user, second_updated_note) + {:ok, update, _meta} = ActivityPub.persist(second_update_data, local: true) {:ok, _, _} = SideEffects.handle(update) %{data: new_note} = Pleroma.Object.get_by_id(object_id) - assert %{"summary" => "edited summary", "content" => "edited content"} = new_note + assert %{"summary" => "edited summary 2", "content" => "edited content 2"} = new_note original_version = Map.drop(note.data, ["id", "formerRepresentations"]) first_edit = Map.drop(first_edit, ["id", "formerRepresentations"]) @@ -196,6 +205,59 @@ test "it puts the original note at the front of formerRepresentations", %{ assert new_note["formerRepresentations"]["totalItems"] == 2 end + + test "it does not prepend to formerRepresentations if no actual changes are made", %{ + note: note, + object_id: object_id, + update: update + } do + {:ok, _, _} = SideEffects.handle(update) + %{data: _first_edit} = Pleroma.Object.get_by_id(object_id) + + {:ok, _, _} = SideEffects.handle(update) + %{data: new_note} = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "edited summary", "content" => "edited content"} = new_note + + original_version = Map.drop(note.data, ["id", "formerRepresentations"]) + + assert [original_version] == + new_note["formerRepresentations"]["orderedItems"] + + assert new_note["formerRepresentations"]["totalItems"] == 1 + end + end + + describe "update questions" do + setup do + user = insert(:user) + question = insert(:question, user: user) + + %{user: user, data: question.data, id: question.id} + end + + test "allows updating choice count without generating edit history", %{ + user: user, + data: data, + id: id + } do + new_choices = + data["oneOf"] + |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) + + updated_question = data |> Map.put("oneOf", new_choices) + + {:ok, update_data, []} = Builder.update(user, updated_question) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + {:ok, _, _} = SideEffects.handle(update) + + %{data: new_question} = Pleroma.Object.get_by_id(id) + + assert [%{"replies" => %{"totalItems" => 5}}, %{"replies" => %{"totalItems" => 5}}] = + new_question["oneOf"] + + refute Map.has_key?(new_question, "formerRepresentations") + end end describe "EmojiReact objects" do From c004eb0fa2c0a754a0fb839a961e35f406c57445 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 29 May 2022 23:50:31 -0400 Subject: [PATCH 05/51] Implement mastodon api for showing edit history --- lib/pleroma/object.ex | 20 +++++ lib/pleroma/web/activity_pub/side_effects.ex | 22 +---- .../api_spec/operations/status_operation.ex | 82 +++++++++++++++++++ .../controllers/status_controller.ex | 30 ++++++- .../web/mastodon_api/views/status_view.ex | 65 +++++++++++++++ lib/pleroma/web/router.ex | 3 + .../controllers/status_controller_test.exs | 46 +++++++++++ 7 files changed, 245 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index fe264b5e0..a893f2c1a 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -425,4 +425,24 @@ def object_data_hashtags(%{"tag" => tags}) when is_list(tags) do end def object_data_hashtags(_), do: [] + + def history_for(object) do + with history <- Map.get(object, "formerRepresentations"), + true <- is_map(history), + "OrderedCollection" <- Map.get(history, "type"), + true <- is_list(Map.get(history, "orderedItems")), + true <- is_integer(Map.get(history, "totalItems")) do + history + else + _ -> history_skeleton() + end + end + + defp history_skeleton do + %{ + "type" => "OrderedCollection", + "totalItems" => 0, + "orderedItems" => [] + } + end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index ac327280c..894c0ceef 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -410,26 +410,6 @@ defp handle_update_user( {:ok, object, meta} end - defp history_for_object(object) do - with history <- Map.get(object, "formerRepresentations"), - true <- is_map(history), - "OrderedCollection" <- Map.get(history, "type"), - true <- is_list(Map.get(history, "orderedItems")), - true <- is_integer(Map.get(history, "totalItems")) do - history - else - _ -> history_skeleton() - end - end - - defp history_skeleton do - %{ - "type" => "OrderedCollection", - "totalItems" => 0, - "orderedItems" => [] - } - end - @updatable_object_types ["Note", "Question"] # We do not allow poll options to be changed, but the poll description can be. @updatable_fields [ @@ -468,7 +448,7 @@ defp maybe_update_history(updated_object, orig_object_data, updated) do else # Put edit history # Note that we may have got the edit history by first fetching the object - history = history_for_object(orig_object_data) + history = Object.history_for(orig_object_data) latest_history_item = orig_object_data diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 639f24d49..e5322707f 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -6,9 +6,13 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.AccountOperation + alias Pleroma.Web.ApiSpec.Schemas.Account alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.Attachment alias Pleroma.Web.ApiSpec.Schemas.BooleanLike + alias Pleroma.Web.ApiSpec.Schemas.Emoji alias Pleroma.Web.ApiSpec.Schemas.FlakeID + alias Pleroma.Web.ApiSpec.Schemas.Poll alias Pleroma.Web.ApiSpec.Schemas.ScheduledStatus alias Pleroma.Web.ApiSpec.Schemas.Status alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -434,6 +438,29 @@ def bookmarks_operation do } end + def show_history_operation do + %Operation{ + tags: ["Retrieve status history"], + summary: "Status history", + description: "View history of a status", + operationId: "StatusController.show_history", + security: [%{"oAuth" => ["read:statuses"]}], + parameters: [ + id_param() + ], + responses: %{ + 200 => status_history_response(), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + + def show_source_operation do + end + + def update_operation do + end + def array_of_statuses do %Schema{type: :array, items: Status, example: [Status.schema().example]} end @@ -579,6 +606,61 @@ defp status_response do Operation.response("Status", "application/json", Status) end + defp status_history_response do + Operation.response( + "Status History", + "application/json", + %Schema{ + title: "Status history", + description: "Response schema for history of a status", + type: :array, + items: %Schema{ + type: :object, + properties: %{ + account: %Schema{ + allOf: [Account], + description: "The account that authored this status" + }, + content: %Schema{ + type: :string, + format: :html, + description: "HTML-encoded status content" + }, + sensitive: %Schema{ + type: :boolean, + description: "Is this status marked as sensitive content?" + }, + spoiler_text: %Schema{ + type: :string, + description: + "Subject or summary line, below which status content is collapsed until expanded" + }, + created_at: %Schema{ + type: :string, + format: "date-time", + description: "The date when this status was created" + }, + media_attachments: %Schema{ + type: :array, + items: Attachment, + description: "Media that is attached to this status" + }, + emojis: %Schema{ + type: :array, + items: Emoji, + description: "Custom emoji to be used when rendering status content" + }, + poll: %Schema{ + allOf: [Poll], + nullable: true, + description: "The poll attached to the status" + } + } + } + } + ) + end + defp context do %Schema{ title: "StatusContext", diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 42a95bdc5..72d85f1ec 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -38,7 +38,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do :index, :show, :card, - :context + :context, + :show_history, + :show_source ] ) @@ -49,7 +51,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do :create, :delete, :reblog, - :unreblog + :unreblog, + :update ] ) @@ -191,6 +194,29 @@ def create(%{assigns: %{user: _user}, body_params: %{media_ids: _} = params} = c create(%Plug.Conn{conn | body_params: params}, %{}) end + @doc "GET /api/v1/statuses/:id/history" + def show_history(%{assigns: %{user: user}} = conn, %{id: id} = params) do + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + true <- Visibility.visible_for_user?(activity, user) do + try_render(conn, "history.json", + activity: activity, + for: user, + with_direct_conversation_id: true, + with_muted: Map.get(params, :with_muted, false) + ) + else + _ -> {:error, :not_found} + end + end + + @doc "GET /api/v1/statuses/:id/source" + def show_source(%{assigns: %{user: _user}} = _conn, %{id: _id} = _params) do + end + + @doc "PUT /api/v1/statuses/:id" + def update(%{assigns: %{user: _user}} = _conn, %{id: _id} = _params) do + end + @doc "GET /api/v1/statuses/:id" def show(%{assigns: %{user: user}} = conn, %{id: id} = params) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 1ebfd6740..c50e0d3da 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -384,6 +384,71 @@ def render("show.json", _) do nil end + def render("history.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do + object = Object.normalize(activity, fetch: false) + + hashtags = Object.hashtags(object) + + user = CommonAPI.get_user(activity.data["actor"]) + + past_history = + Object.history_for(object.data) + |> Map.get("orderedItems") + |> Enum.map(&Map.put(&1, "id", object.data["id"])) + |> Enum.map(&%Object{data: &1, id: object.id}) + + history = [object | past_history] + + individual_opts = + opts + |> Map.put(:as, :object) + |> Map.put(:user, user) + |> Map.put(:hashtags, hashtags) + + render_many(history, StatusView, "history_item.json", individual_opts) + end + + def render( + "history_item.json", + %{activity: activity, user: user, object: object, hashtags: hashtags} = opts + ) do + sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw") + + attachment_data = object.data["attachment"] || [] + attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) + + created_at = Utils.to_masto_date(object.data["updated"] || object.data["published"]) + + content = + object + |> render_content() + + content_html = + content + |> Activity.HTML.get_cached_scrubbed_html_for_activity( + User.html_filter_policy(opts[:for]), + activity, + "mastoapi:content" + ) + + summary = object.data["summary"] || "" + + %{ + account: + AccountView.render("show.json", %{ + user: user, + for: opts[:for] + }), + content: content_html, + sensitive: sensitive, + spoiler_text: summary, + created_at: created_at, + media_attachments: attachments, + emojis: build_emojis(object.data["emoji"]), + poll: render(PollView, "show.json", object: object, for: opts[:for]) + } + end + def render("card.json", %{rich_media: rich_media, page_url: page_url}) do page_url_data = URI.parse(page_url) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ceb6c3cfd..2d2e5365e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -552,6 +552,9 @@ defmodule Pleroma.Web.Router do get("/bookmarks", StatusController, :bookmarks) post("/statuses", StatusController, :create) + get("/statuses/:id/history", StatusController, :show_history) + get("/statuses/:id/source", StatusController, :show_source) + put("/statuses/:id", StatusController, :update) delete("/statuses/:id", StatusController, :delete) post("/statuses/:id/reblog", StatusController, :reblog) post("/statuses/:id/unreblog", StatusController, :unreblog) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index dc6912b7b..d98dc0a92 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1990,4 +1990,50 @@ test "show" do } = result end end + + describe "get status history" do + setup do + oauth_access(["read:statuses"]) + end + + test "unedited post", %{conn: conn} do + activity = insert(:note_activity) + + conn = get(conn, "/api/v1/statuses/#{activity.id}/history") + + assert [_] = json_response_and_validate_schema(conn, 200) + end + + test "edited post", %{conn: conn} do + note = + insert( + :note, + data: %{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "mew mew 2", + "summary" => "title 2" + }, + %{ + "type" => "Note", + "content" => "mew mew 1", + "summary" => "title 1" + } + ], + "totalItems" => 2 + } + } + ) + + activity = insert(:note_activity, note: note) + + conn = get(conn, "/api/v1/statuses/#{activity.id}/history") + + assert [_, %{"spoiler_text" => "title 2"}, %{"spoiler_text" => "title 1"}] = + json_response_and_validate_schema(conn, 200) + end + end end From 393b50884607f9aca4d6e08bf429c8fe8f426f96 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 30 May 2022 00:59:23 -0400 Subject: [PATCH 06/51] Implement viewing source --- .../api_spec/operations/status_operation.ex | 36 +++++++++++++++++++ .../controllers/status_controller.ex | 11 +++++- .../web/mastodon_api/views/status_view.ex | 10 ++++++ .../controllers/status_controller_test.exs | 18 ++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index e5322707f..617aba460 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -456,6 +456,20 @@ def show_history_operation do end def show_source_operation do + %Operation{ + tags: ["Retrieve status source"], + summary: "Status source", + description: "View source of a status", + operationId: "StatusController.show_source", + security: [%{"oAuth" => ["read:statuses"]}], + parameters: [ + id_param() + ], + responses: %{ + 200 => status_source_response(), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } end def update_operation do @@ -661,6 +675,28 @@ defp status_history_response do ) end + defp status_source_response do + Operation.response( + "Status Source", + "application/json", + %Schema{ + type: :object, + properties: %{ + id: FlakeID, + text: %Schema{ + type: :string, + description: "Raw source of status content" + }, + spoiler_text: %Schema{ + type: :string, + description: + "Subject or summary line, below which status content is collapsed until expanded" + } + } + } + ) + end + defp context do %Schema{ title: "StatusContext", diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 72d85f1ec..ea9e08aa8 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -210,7 +210,16 @@ def show_history(%{assigns: %{user: user}} = conn, %{id: id} = params) do end @doc "GET /api/v1/statuses/:id/source" - def show_source(%{assigns: %{user: _user}} = _conn, %{id: _id} = _params) do + def show_source(%{assigns: %{user: user}} = conn, %{id: id} = _params) do + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + true <- Visibility.visible_for_user?(activity, user) do + try_render(conn, "source.json", + activity: activity, + for: user + ) + else + _ -> {:error, :not_found} + end end @doc "PUT /api/v1/statuses/:id" diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c50e0d3da..8d4685ffa 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -449,6 +449,16 @@ def render( } end + def render("source.json", %{activity: %{data: %{"object" => _object}} = activity} = _opts) do + object = Object.normalize(activity, fetch: false) + + %{ + id: activity.id, + text: Map.get(object.data, "source", ""), + spoiler_text: Map.get(object.data, "summary", "") + } + end + def render("card.json", %{rich_media: rich_media, page_url: page_url}) do page_url_data = URI.parse(page_url) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index d98dc0a92..f27cf5048 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2036,4 +2036,22 @@ test "edited post", %{conn: conn} do json_response_and_validate_schema(conn, 200) end end + + describe "get status source" do + setup do + oauth_access(["read:statuses"]) + end + + test "it returns the source", %{conn: conn} do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "mew mew #abc", spoiler_text: "#def"}) + + conn = get(conn, "/api/v1/statuses/#{activity.id}/source") + + id = activity.id + + assert %{"id" => ^id, "text" => "mew mew #abc", "spoiler_text" => "#def"} = json_response_and_validate_schema(conn, 200) + end + end end From b613a9ec6b68972c81dfe2f0175572bc7bd547f9 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 31 May 2022 14:29:12 -0400 Subject: [PATCH 07/51] Implement mastodon api for editing status --- lib/pleroma/constants.ex | 24 ++++++ lib/pleroma/web/activity_pub/builder.ex | 13 ++- lib/pleroma/web/activity_pub/side_effects.ex | 16 +--- .../api_spec/operations/status_operation.ex | 72 ++++++++++++++++- lib/pleroma/web/common_api.ex | 36 +++++++++ .../controllers/status_controller.ex | 21 ++++- test/pleroma/web/common_api_test.exs | 29 +++++++ .../controllers/status_controller_test.exs | 79 ++++++++++++++++++- 8 files changed, 271 insertions(+), 19 deletions(-) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index a42c71d23..bbb95104f 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -27,4 +27,28 @@ defmodule Pleroma.Constants do do: ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css) ) + + const(status_updatable_fields, + do: [ + "source", + "tag", + "updated", + "emoji", + "content", + "summary", + "sensitive", + "attachment", + "generator" + ] + ) + + const(actor_types, + do: [ + "Application", + "Group", + "Organization", + "Person", + "Service" + ] + ) end diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index 5b25138a4..532047599 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -218,10 +218,16 @@ def like(actor, object) do end end - # Retricted to user updates for now, always public @spec update(User.t(), Object.t()) :: {:ok, map(), keyword()} def update(actor, object) do - to = [Pleroma.Constants.as_public(), actor.follower_address] + {to, cc} = + if object["type"] in Pleroma.Constants.actor_types() do + # User updates, always public + {[Pleroma.Constants.as_public(), actor.follower_address], []} + else + # Status updates, follow the recipients in the object + {object["to"] || [], object["cc"] || []} + end {:ok, %{ @@ -229,7 +235,8 @@ def update(actor, object) do "type" => "Update", "actor" => actor.ap_id, "object" => object, - "to" => to + "to" => to, + "cc" => cc }, []} end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 894c0ceef..49054c320 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -25,6 +25,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do alias Pleroma.Web.Streamer alias Pleroma.Workers.PollWorker + require Pleroma.Constants require Logger @cachex Pleroma.Config.get([:cachex, :provider], Cachex) @@ -411,20 +412,8 @@ defp handle_update_user( end @updatable_object_types ["Note", "Question"] - # We do not allow poll options to be changed, but the poll description can be. - @updatable_fields [ - "source", - "tag", - "updated", - "emoji", - "content", - "summary", - "sensitive", - "attachment", - "generator" - ] defp update_content_fields(orig_object_data, updated_object) do - @updatable_fields + Pleroma.Constants.status_updatable_fields() |> Enum.reduce( %{data: orig_object_data, updated: false}, fn field, %{data: data, updated: updated} -> @@ -502,6 +491,7 @@ defp handle_update_object( |> maybe_update_poll(updated_object) orig_object + |> Repo.preload(:hashtags) |> Object.change(%{data: updated_object_data}) |> Object.update_and_set_cache() end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 617aba460..c69307a4d 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -473,6 +473,22 @@ def show_source_operation do end def update_operation do + %Operation{ + tags: ["Update status"], + summary: "Update status", + description: "Change the content of a status", + operationId: "StatusController.update", + security: [%{"oAuth" => ["write:statuses"]}], + parameters: [ + id_param() + ], + requestBody: request_body("Parameters", update_request(), required: true), + responses: %{ + 200 => status_response(), + 403 => Operation.response("Forbidden", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } end def array_of_statuses do @@ -578,6 +594,60 @@ defp create_request do } end + defp update_request do + %Schema{ + title: "StatusUpdateRequest", + type: :object, + properties: %{ + status: %Schema{ + type: :string, + nullable: true, + description: + "Text content of the status. If `media_ids` is provided, this becomes optional. Attaching a `poll` is optional while `status` is provided." + }, + media_ids: %Schema{ + nullable: true, + type: :array, + items: %Schema{type: :string}, + description: "Array of Attachment ids to be attached as media." + }, + poll: poll_params(), + sensitive: %Schema{ + allOf: [BooleanLike], + nullable: true, + description: "Mark status and attached media as sensitive?" + }, + spoiler_text: %Schema{ + type: :string, + nullable: true, + description: + "Text to be shown as a warning or subject before the actual content. Statuses are generally collapsed behind this field." + }, + content_type: %Schema{ + type: :string, + nullable: true, + description: + "The MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint." + }, + to: %Schema{ + type: :array, + nullable: true, + items: %Schema{type: :string}, + description: + "A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply" + } + }, + example: %{ + "status" => "What time is it?", + "sensitive" => "false", + "poll" => %{ + "options" => ["Cofe", "Adventure"], + "expires_in" => 420 + } + } + } + end + def poll_params do %Schema{ nullable: true, @@ -690,7 +760,7 @@ defp status_source_response do spoiler_text: %Schema{ type: :string, description: - "Subject or summary line, below which status content is collapsed until expanded" + "Subject or summary line, below which status content is collapsed until expanded" } } } diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 1b95ee89c..e60c26053 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -402,6 +402,42 @@ def post(user, %{status: _} = data) do end end + def update(user, orig_activity, changes) do + with orig_object <- Object.normalize(orig_activity), + {:ok, new_object} <- make_update_data(user, orig_object, changes), + {:ok, update_data, _} <- Builder.update(user, new_object), + {:ok, update, _} <- Pipeline.common_pipeline(update_data, local: true) do + {:ok, update} + else + _ -> {:error, nil} + end + end + + defp make_update_data(user, orig_object, changes) do + kept_params = %{ + visibility: Visibility.get_visibility(orig_object) + } + + params = Map.merge(changes, kept_params) + + with {:ok, draft} <- ActivityDraft.create(user, params) do + change = + Pleroma.Constants.status_updatable_fields() + |> Enum.reduce(orig_object.data, fn key, acc -> + if Map.has_key?(draft.object, key) do + acc |> Map.put(key, Map.get(draft.object, key)) + else + acc |> Map.drop([key]) + end + end) + |> Map.put("updated", Utils.make_date()) + + {:ok, change} + else + _ -> {:error, nil} + end + end + @spec pin(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, term()} def pin(id, %User{} = user) do with %Activity{} = activity <- create_activity_by_id(id), diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index ea9e08aa8..fa86e9dc0 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -223,7 +223,26 @@ def show_source(%{assigns: %{user: user}} = conn, %{id: id} = _params) do end @doc "PUT /api/v1/statuses/:id" - def update(%{assigns: %{user: _user}} = _conn, %{id: _id} = _params) do + def update(%{assigns: %{user: user}, body_params: body_params} = conn, %{id: id} = params) do + with {_, %Activity{}} = {_, activity} <- {:activity, Activity.get_by_id_with_object(id)}, + {_, true} <- {:visible, Visibility.visible_for_user?(activity, user)}, + {_, true} <- {:is_create, activity.data["type"] == "Create"}, + actor <- Activity.user_actor(activity), + {_, true} <- {:own_status, actor.id == user.id}, + changes <- body_params |> put_application(conn), + {_, {:ok, _update_activity}} <- {:pipeline, CommonAPI.update(user, activity, changes)}, + {_, %Activity{}} = {_, activity} <- {:refetched, Activity.get_by_id_with_object(id)} do + try_render(conn, "show.json", + activity: activity, + for: user, + with_direct_conversation_id: true, + with_muted: Map.get(params, :with_muted, false) + ) + else + {:own_status, _} -> {:error, :forbidden} + {:pipeline, _} -> {:error, :internal_server_error} + _ -> {:error, :not_found} + end end @doc "GET /api/v1/statuses/:id" diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index b502aaa03..af91fdf74 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1541,4 +1541,33 @@ test "unreact_with_emoji" do end end end + + describe "update/3" do + test "updates a post" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{status: "foo1", spoiler_text: "title 1"}) + + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2"}) + + updated_object = Object.normalize(updated) + assert updated_object.data["content"] == "updated 2" + assert Map.get(updated_object.data, "summary", "") == "" + assert Map.has_key?(updated_object.data, "updated") + end + + test "does not change visibility" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{status: "foo1", spoiler_text: "title 1", visibility: "private"}) + + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2"}) + + updated_object = Object.normalize(updated) + assert updated_object.data["content"] == "updated 2" + assert Map.get(updated_object.data, "summary", "") == "" + assert Visibility.get_visibility(updated_object) == "private" + assert Visibility.get_visibility(updated) == "private" + end + end end diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index f27cf5048..d4ca2d618 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2051,7 +2051,84 @@ test "it returns the source", %{conn: conn} do id = activity.id - assert %{"id" => ^id, "text" => "mew mew #abc", "spoiler_text" => "#def"} = json_response_and_validate_schema(conn, 200) + assert %{"id" => ^id, "text" => "mew mew #abc", "spoiler_text" => "#def"} = + json_response_and_validate_schema(conn, 200) + end + end + + describe "update status" do + setup do + oauth_access(["write:statuses"]) + end + + test "it updates the status", %{conn: conn, user: user} do + {:ok, activity} = CommonAPI.post(user, %{status: "mew mew #abc", spoiler_text: "#def"}) + + response = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "edited", + "spoiler_text" => "lol" + }) + |> json_response_and_validate_schema(200) + + assert response["content"] == "edited" + assert response["spoiler_text"] == "lol" + end + + test "it does not update visibility", %{conn: conn, user: user} do + {:ok, activity} = + CommonAPI.post(user, %{ + status: "mew mew #abc", + spoiler_text: "#def", + visibility: "private" + }) + + response = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "edited", + "spoiler_text" => "lol" + }) + |> json_response_and_validate_schema(200) + + assert response["visibility"] == "private" + end + + test "it refuses to update when original post is not by the user", %{conn: conn} do + another_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(another_user, %{status: "mew mew #abc", spoiler_text: "#def"}) + + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "edited", + "spoiler_text" => "lol" + }) + |> json_response_and_validate_schema(:forbidden) + end + + test "it returns 404 if the user cannot see the post", %{conn: conn} do + another_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(another_user, %{ + status: "mew mew #abc", + spoiler_text: "#def", + visibility: "private" + }) + + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "edited", + "spoiler_text" => "lol" + }) + |> json_response_and_validate_schema(:not_found) end end end From 410e177b2ac3177f0645d7728b2ea922ba3c24d3 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 1 Jun 2022 12:02:03 -0400 Subject: [PATCH 08/51] Strip internal fields in formerRepresentation --- .../web/activity_pub/transmogrifier.ex | 19 ++++++- .../web/activity_pub/transmogrifier_test.exs | 54 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index a70330f0e..5750396a4 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -902,7 +902,24 @@ def prepare_attachments(object) do end def strip_internal_fields(object) do - Map.drop(object, Pleroma.Constants.object_internal_fields()) + outer = Map.drop(object, Pleroma.Constants.object_internal_fields()) + + case outer do + %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> + update_in( + outer["formerRepresentations"]["orderedItems"], + &Enum.map( + &1, + fn + item when is_map(item) -> Map.drop(item, Pleroma.Constants.object_internal_fields()) + item -> item + end + ) + ) + + _ -> + outer + end end defp strip_internal_tags(%{"tag" => tags} = object) do diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index 335fe1a30..dae07cf21 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -575,4 +575,58 @@ test "puts dimensions into attachment url field" do assert Transmogrifier.fix_attachments(object) == expected end end + + describe "strip_internal_fields/1" do + test "it strips internal fields in formerRepresentations" do + original = %{ + "formerRepresentations" => %{ + "orderedItems" => [ + %{"generator" => %{}} + ] + } + } + + stripped = Transmogrifier.strip_internal_fields(original) + + refute Map.has_key?( + Enum.at(stripped["formerRepresentations"]["orderedItems"], 0), + "generator" + ) + end + + test "it strips internal fields in maybe badly-formed formerRepresentations" do + original = %{ + "formerRepresentations" => %{ + "orderedItems" => [ + %{"generator" => %{}}, + "https://example.com/1" + ] + } + } + + stripped = Transmogrifier.strip_internal_fields(original) + + refute Map.has_key?( + Enum.at(stripped["formerRepresentations"]["orderedItems"], 0), + "generator" + ) + + assert Enum.at(stripped["formerRepresentations"]["orderedItems"], 1) == + "https://example.com/1" + end + + test "it ignores if formerRepresentations does not look like an OrderedCollection" do + original = %{ + "formerRepresentations" => %{ + "items" => [ + %{"generator" => %{}} + ] + } + } + + stripped = Transmogrifier.strip_internal_fields(original) + + assert Map.has_key?(Enum.at(stripped["formerRepresentations"]["items"], 0), "generator") + end + end end From fa31ae50e6ec44a3921a60d2a6c19e864f0511e7 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 1 Jun 2022 19:30:50 -0400 Subject: [PATCH 09/51] Inject history when object is refetched --- lib/pleroma/object.ex | 22 +++ lib/pleroma/object/fetcher.ex | 27 +++ lib/pleroma/web/activity_pub/side_effects.ex | 24 +-- test/pleroma/object/fetcher_test.exs | 187 +++++++++++++++++++ 4 files changed, 237 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index a893f2c1a..670ab8743 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -445,4 +445,26 @@ defp history_skeleton do "orderedItems" => [] } end + + def maybe_update_history(updated_object, orig_object_data, updated) do + if not updated do + updated_object + else + # Put edit history + # Note that we may have got the edit history by first fetching the object + history = Object.history_for(orig_object_data) + + latest_history_item = + orig_object_data + |> Map.drop(["id", "formerRepresentations"]) + + new_history = + history + |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) + |> Map.put("totalItems", history["totalItems"] + 1) + + updated_object + |> Map.put("formerRepresentations", new_history) + end + end end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index deb3dc711..ce816c1fc 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -26,8 +26,35 @@ defp touch_changeset(changeset) do end defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do + has_history? = fn + %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> true + _ -> false + end + internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields()) + remote_history_exists? = has_history?.(new_data) + + # If the remote history exists, we treat that as the only source of truth. + new_data = + if has_history?.(old_data) and not remote_history_exists? do + Map.put(new_data, "formerRepresentations", old_data["formerRepresentations"]) + else + new_data + end + + # If the remote does not have history information, we need to manage it ourselves + new_data = + if not remote_history_exists? do + changed? = + Pleroma.Constants.status_updatable_fields() + |> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end) + + new_data |> Object.maybe_update_history(old_data, changed?) + else + new_data + end + Map.merge(new_data, internal_fields) end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 49054c320..52a343de7 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -431,28 +431,6 @@ defp update_content_fields(orig_object_data, updated_object) do ) end - defp maybe_update_history(updated_object, orig_object_data, updated) do - if not updated do - updated_object - else - # Put edit history - # Note that we may have got the edit history by first fetching the object - history = Object.history_for(orig_object_data) - - latest_history_item = - orig_object_data - |> Map.drop(["id", "formerRepresentations"]) - - new_history = - history - |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) - |> Map.put("totalItems", history["totalItems"] + 1) - - updated_object - |> Map.put("formerRepresentations", new_history) - end - end - defp maybe_update_poll(to_be_updated, updated_object) do choice_key = fn data -> if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" @@ -487,7 +465,7 @@ defp handle_update_object( updated_object_data = updated_object_data - |> maybe_update_history(orig_object_data, updated) + |> Object.maybe_update_history(orig_object_data, updated) |> maybe_update_poll(updated_object) orig_object diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index 98130f434..5a79e064f 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -269,4 +269,191 @@ test "it can refetch pruned objects" do refute called(Pleroma.Signature.sign(:_, :_)) end end + + describe "refetching" do + setup do + object1 = %{ + "id" => "https://mastodon.social/1", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "type" => "Note", + "content" => "test 1", + "bcc" => [], + "bto" => [], + "cc" => [], + "to" => [], + "summary" => "" + } + + object2 = %{ + "id" => "https://mastodon.social/2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "type" => "Note", + "content" => "test 2", + "bcc" => [], + "bto" => [], + "cc" => [], + "to" => [], + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "orig 2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "bcc" => [], + "bto" => [], + "cc" => [], + "to" => [], + "summary" => "" + } + ], + "totalItems" => 1 + } + } + + mock(fn + %{ + method: :get, + url: "https://mastodon.social/1" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: Jason.encode!(object1) + } + + %{ + method: :get, + url: "https://mastodon.social/2" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: Jason.encode!(object2) + } + + %{ + method: :get, + url: "https://mastodon.social/users/emelie/collections/featured" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: + Jason.encode!(%{ + "id" => "https://mastodon.social/users/emelie/collections/featured", + "type" => "OrderedCollection", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "orderedItems" => [], + "totalItems" => 0 + }) + } + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + %{object1: object1, object2: object2} + end + + test "it keeps formerRepresentations if remote does not have this attr", %{object1: object1} do + full_object1 = + object1 + |> Map.merge(%{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "orig 2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "bcc" => [], + "bto" => [], + "cc" => [], + "to" => [], + "summary" => "" + } + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object1) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{"formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]}} = + refetched.data + end + + test "it uses formerRepresentations from remote if possible", %{object2: object2} do + {:ok, o} = Object.create(object2) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{"formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]}} = + refetched.data + end + + test "it replaces formerRepresentations with the one from remote", %{object2: object2} do + full_object2 = + object2 + |> Map.merge(%{ + "content" => "mew mew #def", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"type" => "Note", "content" => "mew mew 2"} + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object2) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{ + "content" => "test 2", + "formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]} + } = refetched.data + end + + test "it adds to formerRepresentations if the remote does not have one and the object has changed", + %{object1: object1} do + full_object1 = + object1 + |> Map.merge(%{ + "content" => "mew mew #def", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"type" => "Note", "content" => "mew mew 1"} + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object1) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{ + "content" => "test 1", + "formerRepresentations" => %{ + "orderedItems" => [ + %{"content" => "mew mew #def"}, + %{"content" => "mew mew 1"} + ], + "totalItems" => 2 + } + } = refetched.data + end + end end From 8bac8147d4079c0ba0a54753bbab904e46dadbfc Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 3 Jun 2022 21:15:17 -0400 Subject: [PATCH 10/51] Stream out edits --- lib/pleroma/web/activity_pub/activity_pub.ex | 13 ++++++-- lib/pleroma/web/activity_pub/side_effects.ex | 6 ++++ lib/pleroma/web/streamer.ex | 14 +++++++++ lib/pleroma/web/views/streamer_view.ex | 31 ++++++++++++++++++ test/pleroma/web/streamer_test.exs | 33 ++++++++++++++++++++ 5 files changed, 95 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 064f93b22..179e6763b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -190,7 +190,16 @@ defp insert_activity_with_expiration(data, local, recipients) do def notify_and_stream(activity) do Notification.create_notifications(activity) - conversation = create_or_bump_conversation(activity, activity.actor) + original_activity = + case activity do + %{data: %{"type" => "Update"}, object: %{data: %{"id" => id}}} -> + Activity.get_create_by_object_ap_id_with_object(id) + + _ -> + activity + end + + conversation = create_or_bump_conversation(original_activity, original_activity.actor) participations = get_participations(conversation) stream_out(activity) stream_out_participations(participations) @@ -256,7 +265,7 @@ def stream_out_participations(_, _), do: :noop @impl true def stream_out(%Activity{data: %{"type" => data_type}} = activity) - when data_type in ["Create", "Announce", "Delete"] do + when data_type in ["Create", "Announce", "Delete", "Update"] do activity |> Topics.get_activity_topics() |> Streamer.stream(activity) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 52a343de7..05f9b9bd9 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -472,6 +472,12 @@ defp handle_update_object( |> Repo.preload(:hashtags) |> Object.change(%{data: updated_object_data}) |> Object.update_and_set_cache() + + if updated do + object + |> Activity.normalize() + |> ActivityPub.notify_and_stream() + end end {:ok, object, meta} diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index ff7f62a1e..8b7fb985b 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -296,6 +296,20 @@ defp push_to_socket(topic, %Activity{ defp push_to_socket(_topic, %Activity{data: %{"type" => "Delete"}}), do: :noop + defp push_to_socket(topic, %Activity{data: %{"type" => "Update"}} = item) do + anon_render = StreamerView.render("status_update.json", item) + + Registry.dispatch(@registry, topic, fn list -> + Enum.each(list, fn {pid, auth?} -> + if auth? do + send(pid, {:render_with_user, StreamerView, "status_update.json", item}) + else + send(pid, {:text, anon_render}) + end + end) + end) + end + defp push_to_socket(topic, item) do anon_render = StreamerView.render("update.json", item) diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index 16c2b7d61..797762d90 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -25,6 +25,22 @@ def render("update.json", %Activity{} = activity, %User{} = user) do |> Jason.encode!() end + def render("status_update.json", %Activity{} = activity, %User{} = user) do + activity = Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) + + %{ + event: "status.update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "show.json", + activity: activity, + for: user + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + def render("notification.json", %Notification{} = notify, %User{} = user) do %{ event: "notification", @@ -51,6 +67,21 @@ def render("update.json", %Activity{} = activity) do |> Jason.encode!() end + def render("status_update.json", %Activity{} = activity) do + activity = Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) + + %{ + event: "status.update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "show.json", + activity: activity + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + def render("chat_update.json", %{chat_message_reference: cm_ref}) do # Explicitly giving the cmr for the object here, so we don't accidentally # send a later 'last_message' that was inserted between inserting this and diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 4d4fed070..5ceaf763f 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -442,6 +442,20 @@ test "it sends follow relationships updates to the 'user' stream", %{ "state" => "follow_accept" } = Jason.decode!(payload) end + + test "it streams edits in the 'user' stream", %{user: user, token: oauth_token} do + sender = insert(:user) + {:ok, _, _, _} = CommonAPI.follow(user, sender) + + {:ok, activity} = CommonAPI.post(sender, %{status: "hey"}) + + Streamer.get_topic_and_add_socket("user", user, oauth_token) + {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"}) + edited = Pleroma.Activity.normalize(edited) + + assert_receive {:render_with_user, _, "status_update.json", ^edited} + refute Streamer.filtered_by_user?(user, edited) + end end describe "public streams" do @@ -484,6 +498,25 @@ test "handles deletions" do assert_receive {:text, event} assert %{"event" => "delete", "payload" => ^activity_id} = Jason.decode!(event) end + + test "it streams edits in the 'public' stream" do + sender = insert(:user) + + Streamer.get_topic_and_add_socket("public", nil, nil) + {:ok, activity} = CommonAPI.post(sender, %{status: "hey"}) + assert_receive {:text, _} + + {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"}) + + edited = Pleroma.Activity.normalize(edited) + + %{id: activity_id} = Pleroma.Activity.get_create_by_object_ap_id(edited.object.data["id"]) + + assert_receive {:text, event} + assert %{"event" => "status.update", "payload" => payload} = Jason.decode!(event) + assert %{"id" => ^activity_id} = Jason.decode!(payload) + refute Streamer.filtered_by_user?(sender, edited) + end end describe "thread_containment/2" do From fdaa8640838c741500839f66fc7902c88d449afd Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 3 Jun 2022 21:17:16 -0400 Subject: [PATCH 11/51] Test that own edits are streamed --- test/pleroma/web/streamer_test.exs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 5ceaf763f..56cbf1b7b 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -456,6 +456,17 @@ test "it streams edits in the 'user' stream", %{user: user, token: oauth_token} assert_receive {:render_with_user, _, "status_update.json", ^edited} refute Streamer.filtered_by_user?(user, edited) end + + test "it streams own edits in the 'user' stream", %{user: user, token: oauth_token} do + {:ok, activity} = CommonAPI.post(user, %{status: "hey"}) + + Streamer.get_topic_and_add_socket("user", user, oauth_token) + {:ok, edited} = CommonAPI.update(user, activity, %{status: "mew mew"}) + edited = Pleroma.Activity.normalize(edited) + + assert_receive {:render_with_user, _, "status_update.json", ^edited} + refute Streamer.filtered_by_user?(user, edited) + end end describe "public streams" do From 3249ac1f12b69718cacc193c020e8bdccf167a9e Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 3 Jun 2022 21:47:40 -0400 Subject: [PATCH 12/51] Show edited_at in MastodonAPI/show --- lib/pleroma/web/api_spec/schemas/status.ex | 6 ++++++ .../web/mastodon_api/views/status_view.ex | 11 +++++++++++ .../web/mastodon_api/views/status_view_test.exs | 16 ++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 6e6e30315..f803caec2 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -73,6 +73,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do format: "date-time", description: "The date when this status was created" }, + edited_at: %Schema{ + type: :string, + format: "date-time", + nullable: true, + description: "The date when this status was last edited" + }, emojis: %Schema{ type: :array, items: Emoji, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 8d4685ffa..4afba4b33 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -258,6 +258,16 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} created_at = Utils.to_masto_date(object.data["published"]) + edited_at = + with %{"updated" => updated} <- object.data, + date <- Utils.to_masto_date(updated), + true <- date != "" do + date + else + _ -> + nil + end + reply_to = get_reply_to(activity, opts) reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"]) @@ -346,6 +356,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} content: content_html, text: opts[:with_source] && object.data["source"], created_at: created_at, + edited_at: edited_at, reblogs_count: announcement_count, replies_count: object.data["repliesCount"] || 0, favourites_count: like_count, diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 5d81c92b9..44cc003f3 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -246,6 +246,7 @@ test "a note activity" do content: HTML.filter_tags(object_data["content"]), text: nil, created_at: created_at, + edited_at: nil, reblogs_count: 0, replies_count: 0, favourites_count: 0, @@ -708,4 +709,19 @@ test "has a field for parent visibility" do status = StatusView.render("show.json", activity: visible, for: poster) assert status.pleroma.parent_visible end + + test "it shows edited_at" do + poster = insert(:user) + + {:ok, post} = CommonAPI.post(poster, %{status: "hey"}) + + status = StatusView.render("show.json", activity: post) + refute status.edited_at + + {:ok, _} = CommonAPI.update(poster, post, %{status: "mew mew"}) + edited = Pleroma.Activity.normalize(post) + + status = StatusView.render("show.json", activity: edited) + assert status.edited_at + end end From 72ac940618efe56e14f02e23e5af75ae275a5c26 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 3 Jun 2022 21:50:49 -0400 Subject: [PATCH 13/51] Fix SideEffectsTest --- test/pleroma/web/activity_pub/side_effects_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 62394b058..c709ac75a 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -144,6 +144,7 @@ test "it uses a given changeset to update", %{user: user, update: update} do setup do user = insert(:user) note = insert(:note, user: user) + _note_activity = insert(:note_activity, note: note) updated_note = note.data From fe2d4778eee5e8b4fe24f8e1d16d1065e9430027 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 4 Jun 2022 12:56:56 -0400 Subject: [PATCH 14/51] Expose content type of status sources --- .../API/differences_in_mastoapi_responses.md | 4 +++ .../api_spec/operations/status_operation.ex | 4 +++ lib/pleroma/web/common_api/activity_draft.ex | 5 ++- lib/pleroma/web/common_api/utils.ex | 2 +- .../web/mastodon_api/views/status_view.ex | 27 ++++++++++++-- .../mastodon_api/views/status_view_test.exs | 36 +++++++++++++++++++ 6 files changed, 73 insertions(+), 5 deletions(-) diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index 73c46fff8..4007c63c8 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -40,6 +40,10 @@ Has these additional fields under the `pleroma` object: - `parent_visible`: If the parent of this post is visible to the user or not. - `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise. +The `GET /api/v1/statuses/:id/source` endpoint additionally has the following attributes: + +- `content_type`: The content type of the status source. + ## Scheduled statuses Has these additional fields in `params`: diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index c69307a4d..e921128c7 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -761,6 +761,10 @@ defp status_source_response do type: :string, description: "Subject or summary line, below which status content is collapsed until expanded" + }, + content_type: %Schema{ + type: :string, + description: "The content type of the source" } } } diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 7c21c8c3a..9af635da8 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -224,7 +224,10 @@ defp object(draft) do object = note_data |> Map.put("emoji", emoji) - |> Map.put("source", draft.status) + |> Map.put("source", %{ + "content" => draft.status, + "mediaType" => Utils.get_content_type(draft.params[:content_type]) + }) |> Map.put("generator", draft.params[:generator]) %__MODULE__{draft | object: object} diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index ce850b038..4c6a26384 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -219,7 +219,7 @@ def make_content_html(%ActivityDraft{} = draft) do |> maybe_add_attachments(draft.attachments, attachment_links) end - defp get_content_type(content_type) do + def get_content_type(content_type) do if Enum.member?(Config.get([:instance, :allowed_post_formats]), content_type) do content_type else diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 4afba4b33..f798b2624 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -354,7 +354,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} reblog: nil, card: card, content: content_html, - text: opts[:with_source] && object.data["source"], + text: opts[:with_source] && get_source_text(object.data["source"]), created_at: created_at, edited_at: edited_at, reblogs_count: announcement_count, @@ -465,8 +465,9 @@ def render("source.json", %{activity: %{data: %{"object" => _object}} = activity %{ id: activity.id, - text: Map.get(object.data, "source", ""), - spoiler_text: Map.get(object.data, "summary", "") + text: get_source_text(Map.get(object.data, "source", "")), + spoiler_text: Map.get(object.data, "summary", ""), + content_type: get_source_content_type(object.data["source"]) } end @@ -687,4 +688,24 @@ defp build_image_url(%URI{} = image_url_data, %URI{} = page_url_data) do end defp build_image_url(_, _), do: nil + + defp get_source_text(%{"content" => content} = _source) do + content + end + + defp get_source_text(source) when is_binary(source) do + source + end + + defp get_source_text(_) do + "" + end + + defp get_source_content_type(%{"mediaType" => type} = _source) do + type + end + + defp get_source_content_type(_source) do + Utils.get_content_type(nil) + end end diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index 44cc003f3..297889449 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -724,4 +724,40 @@ test "it shows edited_at" do status = StatusView.render("show.json", activity: edited) assert status.edited_at end + + test "with a source object" do + note = + insert(:note, + data: %{"source" => %{"content" => "object source", "mediaType" => "text/markdown"}} + ) + + activity = insert(:note_activity, note: note) + + status = StatusView.render("show.json", activity: activity, with_source: true) + assert status.text == "object source" + end + + describe "source.json" do + test "with a source object, renders both source and content type" do + note = + insert(:note, + data: %{"source" => %{"content" => "object source", "mediaType" => "text/markdown"}} + ) + + activity = insert(:note_activity, note: note) + + status = StatusView.render("source.json", activity: activity) + assert status.text == "object source" + assert status.content_type == "text/markdown" + end + + test "with a source string, renders source and put text/plain as the content type" do + note = insert(:note, data: %{"source" => "string source"}) + activity = insert(:note_activity, note: note) + + status = StatusView.render("source.json", activity: activity) + assert status.text == "string source" + assert status.content_type == "text/plain" + end + end end From 97eabb20473d962065cb32782737ee10c794617b Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 4 Jun 2022 13:15:07 -0400 Subject: [PATCH 15/51] Fix CommonAPITest --- test/pleroma/web/common_api_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index af91fdf74..2f1197c37 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -586,7 +586,7 @@ test "it filters out obviously bad tags when accepting a post as HTML" do object = Object.normalize(activity, fetch: false) assert object.data["content"] == "

2hu

alert('xss')" - assert object.data["source"] == post + assert object.data["source"]["content"] == post end test "it filters out obviously bad tags when accepting a post as Markdown" do @@ -603,7 +603,7 @@ test "it filters out obviously bad tags when accepting a post as Markdown" do object = Object.normalize(activity, fetch: false) assert object.data["content"] == "

2hu

" - assert object.data["source"] == post + assert object.data["source"]["content"] == post end test "it does not allow replies to direct messages that are not direct messages themselves" do From 06a3998013aca1f74c563d261d050543056c1255 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 5 Jun 2022 15:02:25 -0400 Subject: [PATCH 16/51] Create Update notifications --- lib/pleroma/notification.ex | 22 +++++++- ...85734_add_update_to_notifications_enum.exs | 51 +++++++++++++++++++ test/pleroma/notification_test.exs | 46 +++++++++++++++++ 3 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 52fd2656b..82aeb1802 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -385,7 +385,7 @@ def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = act end def create_notifications(%Activity{data: %{"type" => type}} = activity, options) - when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag"] do + when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do do_create_notifications(activity, options) end @@ -439,6 +439,9 @@ defp type_from_activity(%{data: %{"type" => type}} = activity) do activity |> type_from_activity_object() + "Update" -> + "update" + t -> raise "No notification type for activity type #{t}" end @@ -513,7 +516,7 @@ def create_poll_notifications(%Activity{} = activity) do def get_notified_from_activity(activity, local_only \\ true) def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only) - when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag"] do + when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag", "Update"] do potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity) potential_receivers = @@ -553,6 +556,21 @@ def get_potential_receiver_ap_ids(%{data: %{"type" => "Flag", "actor" => actor}} (User.all_superusers() |> Enum.map(fn user -> user.ap_id end)) -- [actor] end + # Update activity: notify all who repeated this + def get_potential_receiver_ap_ids(%{data: %{"type" => "Update", "actor" => actor}} = activity) do + with %Object{data: %{"id" => object_id}} <- Object.normalize(activity, fetch: false) do + repeaters = + Activity.Queries.by_type("Announce") + |> Activity.Queries.by_object_id(object_id) + |> Activity.with_joined_user_actor() + |> where([a, u], u.local) + |> select([a, u], u.ap_id) + |> Repo.all() + + repeaters -- [actor] + end + end + def get_potential_receiver_ap_ids(activity) do [] |> Utils.maybe_notify_to_recipients(activity) diff --git a/priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs b/priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs new file mode 100644 index 000000000..0656c885f --- /dev/null +++ b/priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs @@ -0,0 +1,51 @@ +defmodule Pleroma.Repo.Migrations.AddUpdateToNotificationsEnum do + use Ecto.Migration + + @disable_ddl_transaction true + + def up do + """ + alter type notification_type add value 'update' + """ + |> execute() + end + + # 20210717000000_add_poll_to_notifications_enum.exs + def down do + alter table(:notifications) do + modify(:type, :string) + end + + """ + delete from notifications where type = 'update' + """ + |> execute() + + """ + drop type if exists notification_type + """ + |> execute() + + """ + create type notification_type as enum ( + 'follow', + 'follow_request', + 'mention', + 'move', + 'pleroma:emoji_reaction', + 'pleroma:chat_mention', + 'reblog', + 'favourite', + 'pleroma:report', + 'poll' + ) + """ + |> execute() + + """ + alter table notifications + alter column type type notification_type using (type::notification_type) + """ + |> execute() + end +end diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index 805764ea4..a000c0efd 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -127,6 +127,28 @@ test "does not create a notification for subscribed users if status is a reply" subscriber_notifications = Notification.for_user(subscriber) assert Enum.empty?(subscriber_notifications) end + + test "it sends edited notifications to those who repeated a status" do + user = insert(:user) + repeated_user = insert(:user) + other_user = insert(:user) + + {:ok, activity_one} = + CommonAPI.post(user, %{ + status: "hey @#{other_user.nickname}!" + }) + + {:ok, _activity_two} = CommonAPI.repeat(activity_one.id, repeated_user) + + {:ok, _edit_activity} = + CommonAPI.update(user, activity_one, %{ + status: "hey @#{other_user.nickname}! mew mew" + }) + + assert [%{type: "reblog"}] = Notification.for_user(user) + assert [%{type: "update"}] = Notification.for_user(repeated_user) + assert [%{type: "mention"}] = Notification.for_user(other_user) + end end test "create_poll_notifications/1" do @@ -839,6 +861,30 @@ test "it returns following domain-blocking recipient in enabled recipients list" assert [other_user] == enabled_receivers assert [] == disabled_receivers end + + test "it sends edited notifications to those who repeated a status" do + user = insert(:user) + repeated_user = insert(:user) + other_user = insert(:user) + + {:ok, activity_one} = + CommonAPI.post(user, %{ + status: "hey @#{other_user.nickname}!" + }) + + {:ok, _activity_two} = CommonAPI.repeat(activity_one.id, repeated_user) + + {:ok, edit_activity} = + CommonAPI.update(user, activity_one, %{ + status: "hey @#{other_user.nickname}! mew mew" + }) + + {enabled_receivers, _disabled_receivers} = + Notification.get_notified_from_activity(edit_activity) + + assert repeated_user in enabled_receivers + assert other_user not in enabled_receivers + end end describe "notification lifecycle" do From 532f6ae3ede9b0795a164ca170314b95d5113fc8 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 5 Jun 2022 16:34:42 -0400 Subject: [PATCH 17/51] Return update notification in mastodon api --- .../controllers/notification_controller.ex | 1 + .../mastodon_api/views/notification_view.ex | 15 ++++++++--- .../views/notification_view_test.exs | 26 +++++++++++++++++++ 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index 932bc6423..e93930771 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -51,6 +51,7 @@ def index(conn, %{account_id: account_id} = params) do move pleroma:emoji_reaction poll + update } def index(%{assigns: %{user: user}} = conn, params) do params = diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 0dc7f3beb..b5b5b2376 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -19,7 +19,11 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView - @parent_types ~w{Like Announce EmojiReact} + defp object_id_for(%{data: %{"object" => %{"id" => id}}}) when is_binary(id), do: id + + defp object_id_for(%{data: %{"object" => id}}) when is_binary(id), do: id + + @parent_types ~w{Like Announce EmojiReact Update} def render("index.json", %{notifications: notifications, for: reading_user} = opts) do activities = Enum.map(notifications, & &1.activity) @@ -30,7 +34,7 @@ def render("index.json", %{notifications: notifications, for: reading_user} = op %{data: %{"type" => type}} -> type in @parent_types end) - |> Enum.map(& &1.data["object"]) + |> Enum.map(&object_id_for/1) |> Activity.create_by_object_ap_id() |> Activity.with_preloaded_object(:left) |> Pleroma.Repo.all() @@ -78,9 +82,9 @@ def render( parent_activity_fn = fn -> if opts[:parent_activities] do - Activity.Queries.find_by_object_ap_id(opts[:parent_activities], activity.data["object"]) + Activity.Queries.find_by_object_ap_id(opts[:parent_activities], object_id_for(activity)) else - Activity.get_create_by_object_ap_id(activity.data["object"]) + Activity.get_create_by_object_ap_id(object_id_for(activity)) end end @@ -109,6 +113,9 @@ def render( "reblog" -> put_status(response, parent_activity_fn.(), reading_user, status_render_opts) + "update" -> + put_status(response, parent_activity_fn.(), reading_user, status_render_opts) + "move" -> put_target(response, activity, reading_user, %{}) diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index 8e4c9136a..d3d74f5cd 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -237,6 +237,32 @@ test "Report notification" do test_notifications_rendering([notification], moderator_user, [expected]) end + test "Edit notification" do + user = insert(:user) + repeat_user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "mew"}) + {:ok, _} = CommonAPI.repeat(activity.id, repeat_user) + {:ok, update} = CommonAPI.update(user, activity, %{status: "mew mew"}) + + user = Pleroma.User.get_by_ap_id(user.ap_id) + activity = Pleroma.Activity.normalize(activity) + update = Pleroma.Activity.normalize(update) + + {:ok, [notification]} = Notification.create_notifications(update) + + expected = %{ + id: to_string(notification.id), + pleroma: %{is_seen: false, is_muted: false}, + type: "update", + account: AccountView.render("show.json", %{user: user, for: repeat_user}), + created_at: Utils.to_masto_date(notification.inserted_at), + status: StatusView.render("show.json", %{activity: activity, for: repeat_user}) + } + + test_notifications_rendering([notification], repeat_user, [expected]) + end + test "muted notification" do user = insert(:user) another_user = insert(:user) From d2d3532e5f3e5bcedc91fd0f5ac4ca69043348db Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 5 Jun 2022 16:35:01 -0400 Subject: [PATCH 18/51] Lint --- lib/pleroma/notification.ex | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 82aeb1802..2906c599d 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -516,7 +516,16 @@ def create_poll_notifications(%Activity{} = activity) do def get_notified_from_activity(activity, local_only \\ true) def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only) - when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag", "Update"] do + when type in [ + "Create", + "Like", + "Announce", + "Follow", + "Move", + "EmojiReact", + "Flag", + "Update" + ] do potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity) potential_receivers = From 237b220d71bfe7db66db12549851fb93900a060a Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 8 Jun 2022 11:05:48 -0400 Subject: [PATCH 19/51] Add object id to uploaded attachments --- lib/pleroma/upload.ex | 2 ++ test/pleroma/upload_test.exs | 30 ++++++++++++++++-------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 242813dcd..7480c57a6 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -36,6 +36,7 @@ defmodule Pleroma.Upload do alias Ecto.UUID alias Pleroma.Config alias Pleroma.Maps + alias Pleroma.Web.ActivityPub.Utils require Logger @type source :: @@ -88,6 +89,7 @@ def store(upload, opts \\ []) do {:ok, url_spec} <- Pleroma.Uploaders.Uploader.put_file(opts.uploader, upload) do {:ok, %{ + "id" => Utils.generate_object_id(), "type" => opts.activity_type, "mediaType" => upload.content_type, "url" => [ diff --git a/test/pleroma/upload_test.exs b/test/pleroma/upload_test.exs index f2795f985..6584c2def 100644 --- a/test/pleroma/upload_test.exs +++ b/test/pleroma/upload_test.exs @@ -49,20 +49,22 @@ def put_file(upload), do: TestUploaderBase.put_file(upload, __MODULE__) test "it returns file" do File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") - assert Upload.store(@upload_file) == - {:ok, - %{ - "name" => "image.jpg", - "type" => "Document", - "mediaType" => "image/jpeg", - "url" => [ - %{ - "href" => "http://localhost:4001/media/post-process-file.jpg", - "mediaType" => "image/jpeg", - "type" => "Link" - } - ] - }} + assert {:ok, result} = Upload.store(@upload_file) + + assert result == + %{ + "id" => result["id"], + "name" => "image.jpg", + "type" => "Document", + "mediaType" => "image/jpeg", + "url" => [ + %{ + "href" => "http://localhost:4001/media/post-process-file.jpg", + "mediaType" => "image/jpeg", + "type" => "Link" + } + ] + } Task.await(Agent.get(TestUploaderSuccess, fn task_pid -> task_pid end)) end From aafd7a687dea7595ee9431451d8e170fc3ff909e Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 8 Jun 2022 11:45:24 -0400 Subject: [PATCH 20/51] Return the corresponding object id in attachment view --- lib/pleroma/web/common_api/utils.ex | 8 ++++++-- .../web/mastodon_api/views/status_view.ex | 13 +++++++++++-- .../controllers/status_controller_test.exs | 19 +++++++++++++++++++ test/support/factory.ex | 12 ++++++++++++ 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 4c6a26384..5fc8c3220 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -37,7 +37,7 @@ def attachments_from_ids_no_descs([]), do: [] def attachments_from_ids_no_descs(ids) do Enum.map(ids, fn media_id -> - case Repo.get(Object, media_id) do + case get_attachment(media_id) do %Object{data: data} -> data _ -> nil end @@ -51,13 +51,17 @@ def attachments_from_ids_descs(ids, descs_str) do {_, descs} = Jason.decode(descs_str) Enum.map(ids, fn media_id -> - with %Object{data: data} <- Repo.get(Object, media_id) do + with %Object{data: data} <- get_attachment(media_id) do Map.put(data, "name", descs[media_id]) end end) |> Enum.reject(&is_nil/1) end + defp get_attachment(media_id) do + Repo.get(Object, media_id) + end + @spec get_to_and_cc(ActivityDraft.t()) :: {list(String.t()), list(String.t())} def get_to_and_cc(%{in_reply_to_conversation: %Participation{} = participation}) do diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index f798b2624..43f5fa02e 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -523,10 +523,19 @@ def render("attachment.json", %{attachment: attachment}) do true -> "unknown" end - <> = :crypto.hash(:md5, href) + attachment_id = + with {_, ap_id} when is_binary(ap_id) <- {:ap_id, attachment["id"]}, + {_, %Object{data: _object_data, id: object_id}} <- + {:object, Object.get_by_ap_id(ap_id)} do + to_string(object_id) + else + _ -> + <> = :crypto.hash(:md5, href) + to_string(attachment["id"] || hash_id) + end %{ - id: to_string(attachment["id"] || hash_id), + id: attachment_id, url: href, remote_url: href, preview_url: href_preview, diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index d4ca2d618..3bfe5ea79 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2077,6 +2077,25 @@ test "it updates the status", %{conn: conn, user: user} do assert response["spoiler_text"] == "lol" end + test "it updates the attachments", %{conn: conn, user: user} do + attachment = insert(:attachment, user: user) + attachment_id = to_string(attachment.id) + + {:ok, activity} = CommonAPI.post(user, %{status: "mew mew #abc", spoiler_text: "#def"}) + + response = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "mew mew #abc", + "spoiler_text" => "#def", + "media_ids" => [attachment_id] + }) + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^attachment_id}] = response["media_attachments"] + end + test "it does not update visibility", %{conn: conn, user: user} do {:ok, activity} = CommonAPI.post(user, %{ diff --git a/test/support/factory.ex b/test/support/factory.ex index 09456debf..aaadae9bd 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -111,6 +111,18 @@ def note_factory(attrs \\ %{}) do } end + def attachment_factory(attrs \\ %{}) do + user = attrs[:user] || insert(:user) + + data = + attachment_data(user.ap_id, nil) + |> Map.put("id", Pleroma.Web.ActivityPub.Utils.generate_object_id()) + + %Pleroma.Object{ + data: merge_attributes(data, Map.get(attrs, :data, %{})) + } + end + def attachment_note_factory(attrs \\ %{}) do user = attrs[:user] || insert(:user) {length, attrs} = Map.pop(attrs, :length, 1) From c3593639adfdd6f9e086aaab18bda5c83bcfcc8b Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 9 Jun 2022 11:39:51 -0400 Subject: [PATCH 21/51] Fix incorrectly cached content after editing --- lib/pleroma/activity/html.ex | 36 ++++++++++++++++ lib/pleroma/application.ex | 1 + lib/pleroma/web/activity_pub/side_effects.ex | 25 +++++++---- .../web/mastodon_api/views/status_view.ex | 41 ++++++++++++++++--- .../controllers/status_controller_test.exs | 16 +++++++- test/pleroma/web/metadata/utils_test.exs | 16 +++++++- 6 files changed, 119 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/activity/html.ex b/lib/pleroma/activity/html.ex index 071a89c8d..706b2d36c 100644 --- a/lib/pleroma/activity/html.ex +++ b/lib/pleroma/activity/html.ex @@ -8,6 +8,40 @@ defmodule Pleroma.Activity.HTML do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + # We store a list of cache keys related to an activity in a + # separate cache, scrubber_management_cache. It has the same + # size as scrubber_cache (see application.ex). Every time we add + # a cache to scrubber_cache, we update scrubber_management_cache. + # + # The most recent write of a certain key in the management cache + # is the same as the most recent write of any record related to that + # key in the main cache. + # Assuming LRW ( https://hexdocs.pm/cachex/Cachex.Policy.LRW.html ), + # this means when the management cache is evicted by cachex, all + # related records in the main cache will also have been evicted. + + defp get_cache_keys_for(activity_id) do + with {:ok, list} when is_list(list) <- @cachex.get(:scrubber_management_cache, activity_id) do + list + else + _ -> [] + end + end + + defp add_cache_key_for(activity_id, additional_key) do + current = get_cache_keys_for(activity_id) + + unless additional_key in current do + @cachex.put(:scrubber_management_cache, activity_id, [additional_key | current]) + end + end + + def invalidate_cache_for(activity_id) do + keys = get_cache_keys_for(activity_id) + Enum.map(keys, &@cachex.del(:scrubber_cache, &1)) + @cachex.del(:scrubber_management_cache, activity_id) + end + def get_cached_scrubbed_html_for_activity( content, scrubbers, @@ -19,6 +53,8 @@ def get_cached_scrubbed_html_for_activity( @cachex.fetch!(:scrubber_cache, key, fn _key -> object = Object.normalize(activity, fetch: false) + + add_cache_key_for(activity.id, key) HTML.ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback) end) end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d808bc732..e6b733f9b 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -189,6 +189,7 @@ defp cachex_children do build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500), build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), build_cachex("scrubber", limit: 2500), + build_cachex("scrubber_management", limit: 2500), build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 05f9b9bd9..d387d9362 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -455,7 +455,8 @@ defp handle_update_object( %{data: %{"type" => "Update", "object" => updated_object}} = object, meta ) do - orig_object = Object.get_by_ap_id(updated_object["id"]) + orig_object_ap_id = updated_object["id"] + orig_object = Object.get_by_ap_id(orig_object_ap_id) orig_object_data = orig_object.data if orig_object_data["type"] in @updatable_object_types do @@ -468,15 +469,21 @@ defp handle_update_object( |> Object.maybe_update_history(orig_object_data, updated) |> maybe_update_poll(updated_object) - orig_object - |> Repo.preload(:hashtags) - |> Object.change(%{data: updated_object_data}) - |> Object.update_and_set_cache() + changeset = + orig_object + |> Repo.preload(:hashtags) + |> Object.change(%{data: updated_object_data}) - if updated do - object - |> Activity.normalize() - |> ActivityPub.notify_and_stream() + with {:ok, new_object} <- Repo.update(changeset), + {:ok, _} <- Object.invalid_object_cache(new_object), + {:ok, _} <- Object.set_cache(new_object), + # The metadata/utils.ex uses the object id for the cache. + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do + if updated do + object + |> Activity.normalize() + |> ActivityPub.notify_and_stream() + end end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 43f5fa02e..9cb2adcf9 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -272,6 +272,16 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"]) + history_len = + 1 + + (Object.history_for(object.data) + |> Map.get("orderedItems") + |> length()) + + # See render("history.json", ...) for more details + # Here the implicit index of the current content is 0 + chrono_order = history_len - 1 + content = object |> render_content() @@ -281,14 +291,14 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} |> Activity.HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, - "mastoapi:content" + "mastoapi:content:#{chrono_order}" ) content_plaintext = content |> Activity.HTML.get_cached_stripped_html_for_activity( activity, - "mastoapi:content" + "mastoapi:content:#{chrono_order}" ) summary = object.data["summary"] || "" @@ -410,9 +420,25 @@ def render("history.json", %{activity: %{data: %{"object" => _object}} = activit history = [object | past_history] + history_len = length(history) + + history = + Enum.with_index( + history, + fn object, index -> + %{ + # The history is prepended every time there is a new edit. + # In chrono_order, the oldest item is always at 0, and so on. + # The chrono_order is an invariant kept between edits. + chrono_order: history_len - 1 - index, + object: object + } + end + ) + individual_opts = opts - |> Map.put(:as, :object) + |> Map.put(:as, :item) |> Map.put(:user, user) |> Map.put(:hashtags, hashtags) @@ -421,7 +447,12 @@ def render("history.json", %{activity: %{data: %{"object" => _object}} = activit def render( "history_item.json", - %{activity: activity, user: user, object: object, hashtags: hashtags} = opts + %{ + activity: activity, + user: user, + item: %{object: object, chrono_order: chrono_order}, + hashtags: hashtags + } = opts ) do sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw") @@ -439,7 +470,7 @@ def render( |> Activity.HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, - "mastoapi:content" + "mastoapi:content:#{chrono_order}" ) summary = object.data["summary"] || "" diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 3bfe5ea79..c077670ed 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2061,9 +2061,15 @@ test "it returns the source", %{conn: conn} do oauth_access(["write:statuses"]) end - test "it updates the status", %{conn: conn, user: user} do + test "it updates the status" do + %{conn: conn, user: user} = oauth_access(["write:statuses", "read:statuses"]) + {:ok, activity} = CommonAPI.post(user, %{status: "mew mew #abc", spoiler_text: "#def"}) + conn + |> get("/api/v1/statuses/#{activity.id}") + |> json_response_and_validate_schema(200) + response = conn |> put_req_header("content-type", "application/json") @@ -2075,6 +2081,14 @@ test "it updates the status", %{conn: conn, user: user} do assert response["content"] == "edited" assert response["spoiler_text"] == "lol" + + response = + conn + |> get("/api/v1/statuses/#{activity.id}") + |> json_response_and_validate_schema(200) + + assert response["content"] == "edited" + assert response["spoiler_text"] == "lol" end test "it updates the attachments", %{conn: conn, user: user} do diff --git a/test/pleroma/web/metadata/utils_test.exs b/test/pleroma/web/metadata/utils_test.exs index ce8ed5683..5f2f4a056 100644 --- a/test/pleroma/web/metadata/utils_test.exs +++ b/test/pleroma/web/metadata/utils_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.UtilsTest do - use Pleroma.DataCase, async: true + use Pleroma.DataCase, async: false import Pleroma.Factory alias Pleroma.Web.Metadata.Utils @@ -22,6 +22,20 @@ test "it returns text without encode HTML" do assert Utils.scrub_html_and_truncate(note) == "Pleroma's really cool!" end + + test "it does not return old content after editing" do + user = insert(:user) + + {:ok, activity} = Pleroma.Web.CommonAPI.post(user, %{status: "mew mew #def"}) + + object = Pleroma.Object.normalize(activity) + assert Utils.scrub_html_and_truncate(object) == "mew mew #def" + + {:ok, update} = Pleroma.Web.CommonAPI.update(user, activity, %{status: "mew mew #abc"}) + update = Pleroma.Activity.normalize(update) + object = Pleroma.Object.normalize(update) + assert Utils.scrub_html_and_truncate(object) == "mew mew #abc" + end end describe "scrub_html_and_truncate/2" do From 27f3d802f2fd6e9d002654993d8eedb92d120055 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 11 Jun 2022 10:35:36 -0400 Subject: [PATCH 22/51] Expose history and source apis to anon users --- .../web/mastodon_api/controllers/status_controller.ex | 10 ++++++---- lib/pleroma/web/router.ex | 4 ++-- .../controllers/status_controller_test.exs | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index fa86e9dc0..e594ea491 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -195,8 +195,9 @@ def create(%{assigns: %{user: _user}, body_params: %{media_ids: _} = params} = c end @doc "GET /api/v1/statuses/:id/history" - def show_history(%{assigns: %{user: user}} = conn, %{id: id} = params) do - with %Activity{} = activity <- Activity.get_by_id_with_object(id), + def show_history(%{assigns: assigns} = conn, %{id: id} = params) do + with user = assigns[:user], + %Activity{} = activity <- Activity.get_by_id_with_object(id), true <- Visibility.visible_for_user?(activity, user) do try_render(conn, "history.json", activity: activity, @@ -210,8 +211,9 @@ def show_history(%{assigns: %{user: user}} = conn, %{id: id} = params) do end @doc "GET /api/v1/statuses/:id/source" - def show_source(%{assigns: %{user: user}} = conn, %{id: id} = _params) do - with %Activity{} = activity <- Activity.get_by_id_with_object(id), + def show_source(%{assigns: assigns} = conn, %{id: id} = _params) do + with user = assigns[:user], + %Activity{} = activity <- Activity.get_by_id_with_object(id), true <- Visibility.visible_for_user?(activity, user) do try_render(conn, "source.json", activity: activity, diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 2d2e5365e..4a999f0c2 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -552,8 +552,6 @@ defmodule Pleroma.Web.Router do get("/bookmarks", StatusController, :bookmarks) post("/statuses", StatusController, :create) - get("/statuses/:id/history", StatusController, :show_history) - get("/statuses/:id/source", StatusController, :show_source) put("/statuses/:id", StatusController, :update) delete("/statuses/:id", StatusController, :delete) post("/statuses/:id/reblog", StatusController, :reblog) @@ -611,6 +609,8 @@ defmodule Pleroma.Web.Router do get("/statuses/:id/card", StatusController, :card) get("/statuses/:id/favourited_by", StatusController, :favourited_by) get("/statuses/:id/reblogged_by", StatusController, :reblogged_by) + get("/statuses/:id/history", StatusController, :show_history) + get("/statuses/:id/source", StatusController, :show_source) get("/custom_emojis", CustomEmojiController, :index) diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index c077670ed..04f1c17db 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1993,7 +1993,7 @@ test "show" do describe "get status history" do setup do - oauth_access(["read:statuses"]) + %{conn: build_conn()} end test "unedited post", %{conn: conn} do @@ -2039,7 +2039,7 @@ test "edited post", %{conn: conn} do describe "get status source" do setup do - oauth_access(["read:statuses"]) + %{conn: build_conn()} end test "it returns the source", %{conn: conn} do From 7451f0e81f1fd378a3ff23d437e3cc6780d62fb4 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 11 Jun 2022 12:02:16 -0400 Subject: [PATCH 23/51] Send the correct update in streamer get_create_by_ap_id_with_object() seems to fetch the old object. Why this happens needs further investigation. --- lib/pleroma/web/streamer.ex | 8 ++++-- lib/pleroma/web/views/streamer_view.ex | 4 --- test/pleroma/web/streamer_test.exs | 37 +++++++++++++++++++++++--- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 8b7fb985b..fe909df0a 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -297,12 +297,16 @@ defp push_to_socket(topic, %Activity{ defp push_to_socket(_topic, %Activity{data: %{"type" => "Delete"}}), do: :noop defp push_to_socket(topic, %Activity{data: %{"type" => "Update"}} = item) do - anon_render = StreamerView.render("status_update.json", item) + create_activity = + Pleroma.Activity.get_create_by_object_ap_id(item.object.data["id"]) + |> Map.put(:object, item.object) + + anon_render = StreamerView.render("status_update.json", create_activity) Registry.dispatch(@registry, topic, fn list -> Enum.each(list, fn {pid, auth?} -> if auth? do - send(pid, {:render_with_user, StreamerView, "status_update.json", item}) + send(pid, {:render_with_user, StreamerView, "status_update.json", create_activity}) else send(pid, {:text, anon_render}) end diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index 797762d90..6a55242b0 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -26,8 +26,6 @@ def render("update.json", %Activity{} = activity, %User{} = user) do end def render("status_update.json", %Activity{} = activity, %User{} = user) do - activity = Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) - %{ event: "status.update", payload: @@ -68,8 +66,6 @@ def render("update.json", %Activity{} = activity) do end def render("status_update.json", %Activity{} = activity) do - activity = Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) - %{ event: "status.update", payload: diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 56cbf1b7b..4891bf499 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -451,9 +451,9 @@ test "it streams edits in the 'user' stream", %{user: user, token: oauth_token} Streamer.get_topic_and_add_socket("user", user, oauth_token) {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"}) - edited = Pleroma.Activity.normalize(edited) + create = Pleroma.Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) - assert_receive {:render_with_user, _, "status_update.json", ^edited} + assert_receive {:render_with_user, _, "status_update.json", ^create} refute Streamer.filtered_by_user?(user, edited) end @@ -462,9 +462,9 @@ test "it streams own edits in the 'user' stream", %{user: user, token: oauth_tok Streamer.get_topic_and_add_socket("user", user, oauth_token) {:ok, edited} = CommonAPI.update(user, activity, %{status: "mew mew"}) - edited = Pleroma.Activity.normalize(edited) + create = Pleroma.Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) - assert_receive {:render_with_user, _, "status_update.json", ^edited} + assert_receive {:render_with_user, _, "status_update.json", ^create} refute Streamer.filtered_by_user?(user, edited) end end @@ -528,6 +528,35 @@ test "it streams edits in the 'public' stream" do assert %{"id" => ^activity_id} = Jason.decode!(payload) refute Streamer.filtered_by_user?(sender, edited) end + + test "it streams multiple edits in the 'public' stream correctly" do + sender = insert(:user) + + Streamer.get_topic_and_add_socket("public", nil, nil) + {:ok, activity} = CommonAPI.post(sender, %{status: "hey"}) + assert_receive {:text, _} + + {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"}) + + edited = Pleroma.Activity.normalize(edited) + + %{id: activity_id} = Pleroma.Activity.get_create_by_object_ap_id(edited.object.data["id"]) + + assert_receive {:text, event} + assert %{"event" => "status.update", "payload" => payload} = Jason.decode!(event) + assert %{"id" => ^activity_id} = Jason.decode!(payload) + refute Streamer.filtered_by_user?(sender, edited) + + {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew 2"}) + + edited = Pleroma.Activity.normalize(edited) + + %{id: activity_id} = Pleroma.Activity.get_create_by_object_ap_id(edited.object.data["id"]) + assert_receive {:text, event} + assert %{"event" => "status.update", "payload" => payload} = Jason.decode!(event) + assert %{"id" => ^activity_id, "content" => "mew mew 2"} = Jason.decode!(payload) + refute Streamer.filtered_by_user?(sender, edited) + end end describe "thread_containment/2" do From 95b39223281a61f3ee7d52776df2713952de3be0 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 11 Jun 2022 16:28:59 -0400 Subject: [PATCH 24/51] Workaround with_index does not support function in Elixir 1.9 --- .../web/mastodon_api/views/status_view.ex | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 9cb2adcf9..6ede89803 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -423,18 +423,16 @@ def render("history.json", %{activity: %{data: %{"object" => _object}} = activit history_len = length(history) history = - Enum.with_index( - history, - fn object, index -> - %{ - # The history is prepended every time there is a new edit. - # In chrono_order, the oldest item is always at 0, and so on. - # The chrono_order is an invariant kept between edits. - chrono_order: history_len - 1 - index, - object: object - } - end - ) + Enum.zip(history_len..0, history) + |> Enum.map(fn {chrono_order, object} -> + %{ + # The history is prepended every time there is a new edit. + # In chrono_order, the oldest item is always at 0, and so on. + # The chrono_order is an invariant kept between edits. + chrono_order: chrono_order, + object: object + } + end) individual_opts = opts From 44613db853226015207977ee958ebbf4d26f7c00 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 11 Jun 2022 19:52:07 -0400 Subject: [PATCH 25/51] Show original status at the first of history --- lib/pleroma/web/mastodon_api/views/status_view.ex | 11 +++++------ .../controllers/status_controller_test.exs | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6ede89803..8439431eb 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -418,13 +418,12 @@ def render("history.json", %{activity: %{data: %{"object" => _object}} = activit |> Enum.map(&Map.put(&1, "id", object.data["id"])) |> Enum.map(&%Object{data: &1, id: object.id}) - history = [object | past_history] - - history_len = length(history) - history = - Enum.zip(history_len..0, history) - |> Enum.map(fn {chrono_order, object} -> + [object | past_history] + # Mastodon expects the original to be at the first + |> Enum.reverse() + |> Enum.with_index() + |> Enum.map(fn {object, chrono_order} -> %{ # The history is prepended every time there is a new edit. # In chrono_order, the oldest item is always at 0, and so on. diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index 04f1c17db..05c5d9ed5 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2032,7 +2032,7 @@ test "edited post", %{conn: conn} do conn = get(conn, "/api/v1/statuses/#{activity.id}/history") - assert [_, %{"spoiler_text" => "title 2"}, %{"spoiler_text" => "title 1"}] = + assert [%{"spoiler_text" => "title 1"}, %{"spoiler_text" => "title 2"}, _] = json_response_and_validate_schema(conn, 200) end end From 06da000c5d4fc8d71bd36bfb4cec9cbf4399dfe8 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 21 Jun 2022 12:32:44 -0400 Subject: [PATCH 26/51] Add editing to features --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index ee52475d5..4f613416b 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -68,6 +68,7 @@ def features do "shareable_emoji_packs", "multifetch", "pleroma:api/v1/notifications:include_types_filter", + "editing", if Config.get([:activitypub, :blockers_visible]) do "blockers_visible" end, From 01321c88b5ef0d6c236b968cc47eaa8873054b1e Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 24 Jun 2022 10:10:22 -0400 Subject: [PATCH 27/51] Convert incoming Updated object into Pleroma format --- .../web/activity_pub/object_validator.ex | 16 +++++++++ .../article_note_page_validator.ex | 5 ++- .../article_note_page_validator_test.exs | 5 +++ .../update_handling_test.exs | 35 ++++++++++++++++++- 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index f3e31c931..b3105c46e 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -141,6 +141,22 @@ def validate(%{"type" => type} = object, meta) end end + def validate( + %{"type" => "Update", "object" => %{"type" => objtype} = object} = update_activity, + meta + ) + when objtype in ~w[Question Answer Audio Video Event Article Note Page] do + with {:ok, object_data} <- cast_and_apply(object), + meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), + {:ok, update_activity} <- + update_activity + |> UpdateValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + update_activity = stringify_keys(update_activity) + {:ok, update_activity, meta} + end + end + def validate(%{"type" => type} = object, meta) when type in ~w[Accept Reject Follow Update Like EmojiReact Announce ChatMessage Answer] do diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index ca335bc8a..0a0d30e45 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -49,7 +49,10 @@ defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"]) defp fix_url(data), do: data - defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data + defp fix_tag(%{"tag" => tag} = data) when is_list(tag) do + Map.put(data, "tag", Enum.filter(tag, &is_map/1)) + end + defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag]) defp fix_tag(data), do: Map.drop(data, ["tag"]) diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index f93537ed8..c87b07547 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -31,6 +31,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest test "a basic note validates", %{note: note} do %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) end + + test "a note from factory validates" do + note = insert(:note) + %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note.data) + end end test "a Note from Roadhouse validates" do diff --git a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs index f2a22d370..b812b0b0e 100644 --- a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs @@ -60,7 +60,40 @@ test "validates if the object is not of an Actor type" do {:ok, update, _} = Builder.update(other_user, updated_note) - assert {:ok, _update, []} = ObjectValidator.validate(update, []) + assert {:ok, _update, _} = ObjectValidator.validate(update, []) + end + end + + describe "update note" do + test "converts object into Pleroma's format" do + mastodon_tags = [ + %{ + "icon" => %{ + "mediaType" => "image/png", + "type" => "Image", + "url" => "https://somewhere.org/emoji/url/1.png" + }, + "id" => "https://somewhere.org/emoji/1", + "name" => ":some_emoji:", + "type" => "Emoji", + "updated" => "2021-04-07T11:00:00Z" + } + ] + + user = insert(:user) + note = insert(:note, user: user) + + updated_note = + note.data + |> Map.put("content", "edited content") + |> Map.put("tag", mastodon_tags) + + {:ok, update, _} = Builder.update(user, updated_note) + + assert {:ok, _update, meta} = ObjectValidator.validate(update, []) + + assert %{"emoji" => %{"some_emoji" => "https://somewhere.org/emoji/url/1.png"}} = + meta[:object_data] end end end From ee0738319169cf5c7d31dcaf641d9b744c7a72dc Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 24 Jun 2022 10:26:01 -0400 Subject: [PATCH 28/51] Use meta[:object_data] in SideEffects for Update --- lib/pleroma/web/activity_pub/side_effects.ex | 2 + .../web/activity_pub/side_effects_test.exs | 49 ++++++++++++++----- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index d387d9362..aa4183bf6 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -459,6 +459,8 @@ defp handle_update_object( orig_object = Object.get_by_ap_id(orig_object_ap_id) orig_object_data = orig_object.data + updated_object = meta[:object_data] + if orig_object_data["type"] in @updatable_object_types do %{data: updated_object_data, updated: updated} = orig_object_data diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index c709ac75a..8e84db774 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -154,21 +154,44 @@ test "it uses a given changeset to update", %{user: user, update: update} do {:ok, update_data, []} = Builder.update(user, updated_note) {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) - %{user: user, note: note, object_id: note.id, update_data: update_data, update: update} + %{ + user: user, + note: note, + object_id: note.id, + update_data: update_data, + update: update, + updated_note: updated_note + } end - test "it updates the note", %{object_id: object_id, update: update} do - {:ok, _, _} = SideEffects.handle(update) + test "it updates the note", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) new_note = Pleroma.Object.get_by_id(object_id) assert %{"summary" => "edited summary", "content" => "edited content"} = new_note.data end + test "it updates using object_data", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + updated_note = Map.put(updated_note, "summary", "mew mew") + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + new_note = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "mew mew", "content" => "edited content"} = new_note.data + end + test "it records the original note in formerRepresentations", %{ note: note, object_id: object_id, - update: update + update: update, + updated_note: updated_note } do - {:ok, _, _} = SideEffects.handle(update) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) %{data: new_note} = Pleroma.Object.get_by_id(object_id) assert %{"summary" => "edited summary", "content" => "edited content"} = new_note @@ -182,9 +205,10 @@ test "it puts the original note at the front of formerRepresentations", %{ user: user, note: note, object_id: object_id, - update: update + update: update, + updated_note: updated_note } do - {:ok, _, _} = SideEffects.handle(update) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) %{data: first_edit} = Pleroma.Object.get_by_id(object_id) second_updated_note = @@ -194,7 +218,7 @@ test "it puts the original note at the front of formerRepresentations", %{ {:ok, second_update_data, []} = Builder.update(user, second_updated_note) {:ok, update, _meta} = ActivityPub.persist(second_update_data, local: true) - {:ok, _, _} = SideEffects.handle(update) + {:ok, _, _} = SideEffects.handle(update, object_data: second_updated_note) %{data: new_note} = Pleroma.Object.get_by_id(object_id) assert %{"summary" => "edited summary 2", "content" => "edited content 2"} = new_note @@ -210,12 +234,13 @@ test "it puts the original note at the front of formerRepresentations", %{ test "it does not prepend to formerRepresentations if no actual changes are made", %{ note: note, object_id: object_id, - update: update + update: update, + updated_note: updated_note } do - {:ok, _, _} = SideEffects.handle(update) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) %{data: _first_edit} = Pleroma.Object.get_by_id(object_id) - {:ok, _, _} = SideEffects.handle(update) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) %{data: new_note} = Pleroma.Object.get_by_id(object_id) assert %{"summary" => "edited summary", "content" => "edited content"} = new_note @@ -250,7 +275,7 @@ test "allows updating choice count without generating edit history", %{ {:ok, update_data, []} = Builder.update(user, updated_question) {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) - {:ok, _, _} = SideEffects.handle(update) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_question) %{data: new_question} = Pleroma.Object.get_by_id(id) From e0d6da4e7d52e2cdd0fc5290e4ff3a23da7398f6 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 24 Jun 2022 10:54:11 -0400 Subject: [PATCH 29/51] Fix CommonAPITest --- .../web/activity_pub/object_validators/attachment_validator.ex | 3 ++- .../web/activity_pub/object_validators/common_fields.ex | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index d1c61ac82..ca6e39612 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do @primary_key false embedded_schema do + field(:id, :string) field(:type, :string) field(:mediaType, :string, default: "application/octet-stream") field(:name, :string) @@ -43,7 +44,7 @@ def changeset(struct, data) do |> fix_url() struct - |> cast(data, [:type, :mediaType, :name, :blurhash]) + |> cast(data, [:id, :type, :mediaType, :name, :blurhash]) |> cast_embed(:url, with: &url_changeset/2) |> validate_inclusion(:type, ~w[Link Document Audio Image Video]) |> validate_required([:type, :mediaType, :url]) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index 8e768ffbf..a59a6e545 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -33,6 +33,7 @@ defmacro object_fields do field(:content, :string) field(:published, ObjectValidators.DateTime) + field(:updated, ObjectValidators.DateTime) field(:emoji, ObjectValidators.Emoji, default: %{}) embeds_many(:attachment, AttachmentValidator) end From 99a6f5031638da2eed237f91c6dded9e25717599 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 25 Jun 2022 00:32:22 -0400 Subject: [PATCH 30/51] Unify the logic of updating objects --- lib/pleroma/object.ex | 42 ----- lib/pleroma/object/fetcher.ex | 9 +- lib/pleroma/object/updater.ex | 157 ++++++++++++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 62 ++----- lib/pleroma/web/common_api.ex | 10 +- .../web/mastodon_api/views/status_view.ex | 4 +- 6 files changed, 183 insertions(+), 101 deletions(-) create mode 100644 lib/pleroma/object/updater.ex diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 670ab8743..fe264b5e0 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -425,46 +425,4 @@ def object_data_hashtags(%{"tag" => tags}) when is_list(tags) do end def object_data_hashtags(_), do: [] - - def history_for(object) do - with history <- Map.get(object, "formerRepresentations"), - true <- is_map(history), - "OrderedCollection" <- Map.get(history, "type"), - true <- is_list(Map.get(history, "orderedItems")), - true <- is_integer(Map.get(history, "totalItems")) do - history - else - _ -> history_skeleton() - end - end - - defp history_skeleton do - %{ - "type" => "OrderedCollection", - "totalItems" => 0, - "orderedItems" => [] - } - end - - def maybe_update_history(updated_object, orig_object_data, updated) do - if not updated do - updated_object - else - # Put edit history - # Note that we may have got the edit history by first fetching the object - history = Object.history_for(orig_object_data) - - latest_history_item = - orig_object_data - |> Map.drop(["id", "formerRepresentations"]) - - new_history = - history - |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) - |> Map.put("totalItems", history["totalItems"] + 1) - - updated_object - |> Map.put("formerRepresentations", new_history) - end - end end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index ce816c1fc..d81fdcf24 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -50,7 +50,14 @@ defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do Pleroma.Constants.status_updatable_fields() |> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end) - new_data |> Object.maybe_update_history(old_data, changed?) + %{updated_object: updated_object} = + new_data + |> Object.Updater.maybe_update_history(old_data, + updated: changed?, + use_history_in_new_object?: false + ) + + updated_object else new_data end diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex new file mode 100644 index 000000000..03136c38e --- /dev/null +++ b/lib/pleroma/object/updater.ex @@ -0,0 +1,157 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Object.Updater do + require Pleroma.Constants + + def update_content_fields(orig_object_data, updated_object) do + Pleroma.Constants.status_updatable_fields() + |> Enum.reduce( + %{data: orig_object_data, updated: false}, + fn field, %{data: data, updated: updated} -> + updated = updated or Map.get(updated_object, field) != Map.get(orig_object_data, field) + + data = + if Map.has_key?(updated_object, field) do + Map.put(data, field, updated_object[field]) + else + Map.drop(data, [field]) + end + + %{data: data, updated: updated} + end + ) + end + + def maybe_history(object) do + with history <- Map.get(object, "formerRepresentations"), + true <- is_map(history), + "OrderedCollection" <- Map.get(history, "type"), + true <- is_list(Map.get(history, "orderedItems")), + true <- is_integer(Map.get(history, "totalItems")) do + history + else + _ -> nil + end + end + + def history_for(object) do + with history when not is_nil(history) <- maybe_history(object) do + history + else + _ -> history_skeleton() + end + end + + defp history_skeleton do + %{ + "type" => "OrderedCollection", + "totalItems" => 0, + "orderedItems" => [] + } + end + + def maybe_update_history( + updated_object, + orig_object_data, + opts + ) do + updated = opts[:updated] + use_history_in_new_object? = opts[:use_history_in_new_object?] + + if not updated do + %{updated_object: updated_object, used_history_in_new_object?: false} + else + # Put edit history + # Note that we may have got the edit history by first fetching the object + {new_history, used_history_in_new_object?} = + with true <- use_history_in_new_object?, + updated_history when not is_nil(updated_history) <- maybe_history(updated_object) do + {updated_history, true} + else + _ -> + history = history_for(orig_object_data) + + latest_history_item = + orig_object_data + |> Map.drop(["id", "formerRepresentations"]) + + updated_history = + history + |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) + |> Map.put("totalItems", history["totalItems"] + 1) + + {updated_history, false} + end + + updated_object = + updated_object + |> Map.put("formerRepresentations", new_history) + + %{updated_object: updated_object, used_history_in_new_object?: used_history_in_new_object?} + end + end + + defp maybe_update_poll(to_be_updated, updated_object) do + choice_key = fn data -> + if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" + end + + with true <- to_be_updated["type"] == "Question", + key <- choice_key.(updated_object), + true <- key == choice_key.(to_be_updated), + orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])), + new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])), + true <- orig_choices == new_choices do + # Choices are the same, but counts are different + to_be_updated + |> Map.put(key, updated_object[key]) + else + # Choices (or vote type) have changed, do not allow this + _ -> to_be_updated + end + end + + # This calculates the data to be sent as the object of an Update. + # new_data's formerRepresentations is not considered. + # formerRepresentations is added to the returned data. + def make_update_object_data(original_data, new_data, date) do + %{data: updated_data, updated: updated} = + original_data + |> update_content_fields(new_data) + + if not updated do + updated_data + else + %{updated_object: updated_data} = + updated_data + |> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false) + + updated_data + |> Map.put("updated", date) + end + end + + # This calculates the data of the new Object from an Update. + # new_data's formerRepresentations is considered. + def make_new_object_data_from_update_object(original_data, new_data) do + %{data: updated_data, updated: updated} = + original_data + |> update_content_fields(new_data) + + %{updated_object: updated_data, used_history_in_new_object?: used_history_in_new_object?} = + updated_data + |> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false) + + updated_data = + updated_data + |> maybe_update_poll(new_data) + + %{ + updated_data: updated_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } + end +end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index aa4183bf6..7345a6904 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -412,45 +412,6 @@ defp handle_update_user( end @updatable_object_types ["Note", "Question"] - defp update_content_fields(orig_object_data, updated_object) do - Pleroma.Constants.status_updatable_fields() - |> Enum.reduce( - %{data: orig_object_data, updated: false}, - fn field, %{data: data, updated: updated} -> - updated = updated or Map.get(updated_object, field) != Map.get(orig_object_data, field) - - data = - if Map.has_key?(updated_object, field) do - Map.put(data, field, updated_object[field]) - else - Map.drop(data, [field]) - end - - %{data: data, updated: updated} - end - ) - end - - defp maybe_update_poll(to_be_updated, updated_object) do - choice_key = fn data -> - if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" - end - - with true <- to_be_updated["type"] == "Question", - key <- choice_key.(updated_object), - true <- key == choice_key.(to_be_updated), - orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])), - new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])), - true <- orig_choices == new_choices do - # Choices are the same, but counts are different - to_be_updated - |> Map.put(key, updated_object[key]) - else - # Choices (or vote type) have changed, do not allow this - _ -> to_be_updated - end - end - defp handle_update_object( %{data: %{"type" => "Update", "object" => updated_object}} = object, meta @@ -462,14 +423,11 @@ defp handle_update_object( updated_object = meta[:object_data] if orig_object_data["type"] in @updatable_object_types do - %{data: updated_object_data, updated: updated} = - orig_object_data - |> update_content_fields(updated_object) - - updated_object_data = - updated_object_data - |> Object.maybe_update_history(orig_object_data, updated) - |> maybe_update_poll(updated_object) + %{ + updated_data: updated_object_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } = Object.Updater.make_new_object_data_from_update_object(orig_object_data, updated_object) changeset = orig_object @@ -481,6 +439,16 @@ defp handle_update_object( {:ok, _} <- Object.set_cache(new_object), # The metadata/utils.ex uses the object id for the cache. {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do + if used_history_in_new_object? do + with create_activity when not is_nil(create_activity) <- + Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id), + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do + nil + else + _ -> nil + end + end + if updated do object |> Activity.normalize() diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index e60c26053..e5a78c102 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -422,15 +422,7 @@ defp make_update_data(user, orig_object, changes) do with {:ok, draft} <- ActivityDraft.create(user, params) do change = - Pleroma.Constants.status_updatable_fields() - |> Enum.reduce(orig_object.data, fn key, acc -> - if Map.has_key?(draft.object, key) do - acc |> Map.put(key, Map.get(draft.object, key)) - else - acc |> Map.drop([key]) - end - end) - |> Map.put("updated", Utils.make_date()) + Object.Updater.make_update_object_data(orig_object.data, draft.object, Utils.make_date()) {:ok, change} else diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 8439431eb..54e025aae 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -274,7 +274,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} history_len = 1 + - (Object.history_for(object.data) + (Object.Updater.history_for(object.data) |> Map.get("orderedItems") |> length()) @@ -413,7 +413,7 @@ def render("history.json", %{activity: %{data: %{"object" => _object}} = activit user = CommonAPI.get_user(activity.data["actor"]) past_history = - Object.history_for(object.data) + Object.Updater.history_for(object.data) |> Map.get("orderedItems") |> Enum.map(&Map.put(&1, "id", object.data["id"])) |> Enum.map(&%Object{data: &1, id: object.id}) From 40953a8f5c299e55b3f186bd6fdebe1bbf6e7401 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 25 Jun 2022 01:03:46 -0400 Subject: [PATCH 31/51] Reuse formerRepresentations from remote if possible --- lib/pleroma/object/updater.ex | 8 +++- test/pleroma/object/updater_test.exs | 65 ++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 test/pleroma/object/updater_test.exs diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 03136c38e..0b21f6c99 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -67,7 +67,7 @@ def maybe_update_history( # Note that we may have got the edit history by first fetching the object {new_history, used_history_in_new_object?} = with true <- use_history_in_new_object?, - updated_history when not is_nil(updated_history) <- maybe_history(updated_object) do + updated_history when not is_nil(updated_history) <- maybe_history(opts[:new_data]) do {updated_history, true} else _ -> @@ -142,7 +142,11 @@ def make_new_object_data_from_update_object(original_data, new_data) do %{updated_object: updated_data, used_history_in_new_object?: used_history_in_new_object?} = updated_data - |> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false) + |> maybe_update_history(original_data, + updated: updated, + use_history_in_new_object?: true, + new_data: new_data + ) updated_data = updated_data diff --git a/test/pleroma/object/updater_test.exs b/test/pleroma/object/updater_test.exs new file mode 100644 index 000000000..ef13439b9 --- /dev/null +++ b/test/pleroma/object/updater_test.exs @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Object.UpdaterTest do + use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + alias Pleroma.Object.Updater + + describe "make_update_object_data/3" do + setup do + note = insert(:note) + %{original_data: note.data} + end + + test "it makes an updated field", %{original_data: original_data} do + new_data = Map.put(original_data, "content", "new content") + + date = Pleroma.Web.ActivityPub.Utils.make_date() + update_object_data = Updater.make_update_object_data(original_data, new_data, date) + assert %{"updated" => ^date} = update_object_data + end + + test "it creates formerRepresentations", %{original_data: original_data} do + new_data = Map.put(original_data, "content", "new content") + + date = Pleroma.Web.ActivityPub.Utils.make_date() + update_object_data = Updater.make_update_object_data(original_data, new_data, date) + + history_item = original_data |> Map.drop(["id", "formerRepresentations"]) + + assert %{ + "formerRepresentations" => %{ + "totalItems" => 1, + "orderedItems" => [^history_item] + } + } = update_object_data + end + end + + describe "make_new_object_data_from_update_object/2" do + test "it reuses formerRepresentations if it exists" do + %{data: original_data} = insert(:note) + + new_data = + original_data + |> Map.put("content", "edited") + + date = Pleroma.Web.ActivityPub.Utils.make_date() + update_object_data = Updater.make_update_object_data(original_data, new_data, date) + + %{ + updated_data: _updated_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } = Updater.make_new_object_data_from_update_object(original_data, update_object_data) + + assert updated + assert used_history_in_new_object? + end + end +end From e98579b1da78b47d6848ec55042640e539e44f6c Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 25 Jun 2022 01:17:09 -0400 Subject: [PATCH 32/51] Verify that formerRepresentation provided in Update is used --- test/pleroma/object/updater_test.exs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/test/pleroma/object/updater_test.exs b/test/pleroma/object/updater_test.exs index ef13439b9..7e9b44823 100644 --- a/test/pleroma/object/updater_test.exs +++ b/test/pleroma/object/updater_test.exs @@ -52,14 +52,25 @@ test "it reuses formerRepresentations if it exists" do date = Pleroma.Web.ActivityPub.Utils.make_date() update_object_data = Updater.make_update_object_data(original_data, new_data, date) + history = update_object_data["formerRepresentations"]["orderedItems"] + + update_object_data = + update_object_data + |> put_in( + ["formerRepresentations", "orderedItems"], + history ++ [Map.put(original_data, "summary", "additional summary")] + ) + |> put_in(["formerRepresentations", "totalItems"], length(history) + 1) + %{ - updated_data: _updated_data, + updated_data: updated_data, updated: updated, used_history_in_new_object?: used_history_in_new_object? } = Updater.make_new_object_data_from_update_object(original_data, update_object_data) assert updated assert used_history_in_new_object? + assert updated_data["formerRepresentations"] == update_object_data["formerRepresentations"] end end end From 9c6dae942d2ec5e2314af1d345cf2aeed504aae8 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 25 Jun 2022 09:23:09 -0400 Subject: [PATCH 33/51] Fix local updates causing emojis to be lost --- lib/pleroma/web/activity_pub/side_effects.ex | 9 ++++++++- test/pleroma/web/common_api_test.exs | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 7345a6904..747f467db 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -420,7 +420,14 @@ defp handle_update_object( orig_object = Object.get_by_ap_id(orig_object_ap_id) orig_object_data = orig_object.data - updated_object = meta[:object_data] + updated_object = + if meta[:local] do + # If this is a local Update, we don't process it by transmogrifier, + # so we use the embedded object as-is. + updated_object + else + meta[:object_data] + end if orig_object_data["type"] in @updatable_object_types do %{ diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 2f1197c37..32d6562d7 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1569,5 +1569,20 @@ test "does not change visibility" do assert Visibility.get_visibility(updated_object) == "private" assert Visibility.get_visibility(updated) == "private" end + + test "updates a post with emoji" do + [{emoji1, _}, {emoji2, _} | _] = Pleroma.Emoji.get_all() + + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{status: "foo1", spoiler_text: "title 1 :#{emoji1}:"}) + + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2 :#{emoji2}:"}) + + updated_object = Object.normalize(updated) + assert updated_object.data["content"] == "updated 2 :#{emoji2}:" + assert %{^emoji2 => _} = updated_object.data["emoji"] + end end end From 5321fd001233076e7f2b6ea00adeb41ecf7e180a Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 25 Jun 2022 10:03:19 -0400 Subject: [PATCH 34/51] Do not put meta[:object_data] for local Updates --- .../web/activity_pub/object_validator.ex | 12 ++++++- .../update_handling_test.exs | 31 +++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index b3105c46e..d4bf9c31e 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -146,7 +146,8 @@ def validate( meta ) when objtype in ~w[Question Answer Audio Video Event Article Note Page] do - with {:ok, object_data} <- cast_and_apply(object), + with {_, false} <- {:local, Access.get(meta, :local, false)}, + {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, update_activity} <- update_activity @@ -154,6 +155,15 @@ def validate( |> Ecto.Changeset.apply_action(:insert) do update_activity = stringify_keys(update_activity) {:ok, update_activity, meta} + else + {:local, _} -> + with {:ok, object} <- + update_activity + |> UpdateValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end end end diff --git a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs index b812b0b0e..198c35cd3 100644 --- a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs @@ -95,5 +95,36 @@ test "converts object into Pleroma's format" do assert %{"emoji" => %{"some_emoji" => "https://somewhere.org/emoji/url/1.png"}} = meta[:object_data] end + + test "returns no object_data in meta for a local Update" do + user = insert(:user) + note = insert(:note, user: user) + + updated_note = + note.data + |> Map.put("content", "edited content") + + {:ok, update, _} = Builder.update(user, updated_note) + + assert {:ok, _update, meta} = ObjectValidator.validate(update, local: true) + assert is_nil(meta[:object_data]) + end + + test "returns object_data in meta for a remote Update" do + user = insert(:user) + note = insert(:note, user: user) + + updated_note = + note.data + |> Map.put("content", "edited content") + + {:ok, update, _} = Builder.update(user, updated_note) + + assert {:ok, _update, meta} = ObjectValidator.validate(update, local: false) + assert meta[:object_data] + + assert {:ok, _update, meta} = ObjectValidator.validate(update, []) + assert meta[:object_data] + end end end From 014096aeefe88348323db74e2ab7f81e0184bfee Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 25 Jun 2022 11:20:46 -0400 Subject: [PATCH 35/51] Make outbound transmogrifier aware of edit history --- lib/pleroma/constants.ex | 12 +++ lib/pleroma/web/activity_pub/side_effects.ex | 3 +- .../web/activity_pub/transmogrifier.ex | 52 ++++++++----- .../web/activity_pub/transmogrifier_test.exs | 75 ++++++++++--------- test/pleroma/web/common_api_test.exs | 21 ++++++ 5 files changed, 109 insertions(+), 54 deletions(-) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index bbb95104f..1b3d09d73 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -42,6 +42,18 @@ defmodule Pleroma.Constants do ] ) + const(updatable_object_types, + do: [ + "Note", + "Question", + "Audio", + "Video", + "Event", + "Article", + "Page" + ] + ) + const(actor_types, do: [ "Application", diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 747f467db..f56e357bf 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -411,7 +411,6 @@ defp handle_update_user( {:ok, object, meta} end - @updatable_object_types ["Note", "Question"] defp handle_update_object( %{data: %{"type" => "Update", "object" => updated_object}} = object, meta @@ -429,7 +428,7 @@ defp handle_update_object( meta[:object_data] end - if orig_object_data["type"] in @updatable_object_types do + if orig_object_data["type"] in Pleroma.Constants.updatable_object_types() do %{ updated_data: updated_object_data, updated: updated, diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 5750396a4..cccee342f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -687,6 +687,24 @@ def prepare_object(object) do |> strip_internal_fields |> strip_internal_tags |> set_type + |> maybe_process_history + end + + defp maybe_process_history(%{"formerRepresentations" => %{"orderedItems" => history}} = object) do + processed_history = + Enum.map( + history, + fn + item when is_map(item) -> prepare_object(item) + item -> item + end + ) + + put_in(object, ["formerRepresentations", "orderedItems"], processed_history) + end + + defp maybe_process_history(object) do + object end # @doc @@ -711,6 +729,21 @@ def prepare_outgoing(%{"type" => activity_type, "object" => object_id} = data) {:ok, data} end + def prepare_outgoing(%{"type" => "Update", "object" => %{"type" => objtype} = object} = data) + when objtype in Pleroma.Constants.updatable_object_types() do + object = + object + |> prepare_object + + data = + data + |> Map.put("object", object) + |> Map.merge(Utils.make_json_ld_header()) + |> Map.delete("bcc") + + {:ok, data} + end + def prepare_outgoing(%{"type" => "Announce", "actor" => ap_id, "object" => object_id} = data) do object = object_id @@ -902,24 +935,7 @@ def prepare_attachments(object) do end def strip_internal_fields(object) do - outer = Map.drop(object, Pleroma.Constants.object_internal_fields()) - - case outer do - %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> - update_in( - outer["formerRepresentations"]["orderedItems"], - &Enum.map( - &1, - fn - item when is_map(item) -> Map.drop(item, Pleroma.Constants.object_internal_fields()) - item -> item - end - ) - ) - - _ -> - outer - end + Map.drop(object, Pleroma.Constants.object_internal_fields()) end defp strip_internal_tags(%{"tag" => tags} = object) do diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index dae07cf21..6520eabc9 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -312,6 +312,28 @@ test "custom emoji urls are URI encoded" do assert url == "http://localhost:4001/emoji/dino%20walking.gif" end + + test "Updates of Notes are handled" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "everybody do the dinosaur :dinosaur:"}) + {:ok, update} = CommonAPI.update(user, activity, %{status: "mew mew :blank:"}) + + {:ok, prepared} = Transmogrifier.prepare_outgoing(update.data) + + assert %{ + "content" => "mew mew :blank:", + "tag" => [%{"name" => ":blank:", "type" => "Emoji"}], + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "content" => "everybody do the dinosaur :dinosaur:", + "tag" => [%{"name" => ":dinosaur:", "type" => "Emoji"}] + } + ] + } + } = prepared["object"] + end end describe "user upgrade" do @@ -576,57 +598,42 @@ test "puts dimensions into attachment url field" do end end - describe "strip_internal_fields/1" do - test "it strips internal fields in formerRepresentations" do + describe "prepare_object/1" do + test "it processes history" do original = %{ "formerRepresentations" => %{ "orderedItems" => [ - %{"generator" => %{}} + %{ + "generator" => %{}, + "emoji" => %{"blobcat" => "http://localhost:4001/emoji/blobcat.png"} + } ] } } - stripped = Transmogrifier.strip_internal_fields(original) + processed = Transmogrifier.prepare_object(original) - refute Map.has_key?( - Enum.at(stripped["formerRepresentations"]["orderedItems"], 0), - "generator" - ) + history_item = Enum.at(processed["formerRepresentations"]["orderedItems"], 0) + + refute Map.has_key?(history_item, "generator") + + assert [%{"name" => ":blobcat:"}] = history_item["tag"] end - test "it strips internal fields in maybe badly-formed formerRepresentations" do - original = %{ - "formerRepresentations" => %{ - "orderedItems" => [ - %{"generator" => %{}}, - "https://example.com/1" - ] - } - } - - stripped = Transmogrifier.strip_internal_fields(original) - - refute Map.has_key?( - Enum.at(stripped["formerRepresentations"]["orderedItems"], 0), - "generator" - ) - - assert Enum.at(stripped["formerRepresentations"]["orderedItems"], 1) == - "https://example.com/1" - end - - test "it ignores if formerRepresentations does not look like an OrderedCollection" do + test "it works when there is no or bad history" do original = %{ "formerRepresentations" => %{ "items" => [ - %{"generator" => %{}} + %{ + "generator" => %{}, + "emoji" => %{"blobcat" => "http://localhost:4001/emoji/blobcat.png"} + } ] } } - stripped = Transmogrifier.strip_internal_fields(original) - - assert Map.has_key?(Enum.at(stripped["formerRepresentations"]["items"], 0), "generator") + processed = Transmogrifier.prepare_object(original) + assert processed["formerRepresentations"] == original["formerRepresentations"] end end end diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 32d6562d7..842c75e21 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1584,5 +1584,26 @@ test "updates a post with emoji" do assert updated_object.data["content"] == "updated 2 :#{emoji2}:" assert %{^emoji2 => _} = updated_object.data["emoji"] end + + test "updates a post with emoji and federate properly" do + [{emoji1, _}, {emoji2, _} | _] = Pleroma.Emoji.get_all() + + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{status: "foo1", spoiler_text: "title 1 :#{emoji1}:"}) + + clear_config([:instance, :federating], true) + + with_mock Pleroma.Web.Federator, + publish: fn p -> nil end do + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2 :#{emoji2}:"}) + + assert updated.data["object"]["content"] == "updated 2 :#{emoji2}:" + assert %{^emoji2 => _} = updated.data["object"]["emoji"] + + assert called(Pleroma.Web.Federator.publish(updated)) + end + end end end From 4367489a3e8eb8682d717014eea9092d7679c070 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 3 Jul 2022 20:02:52 -0400 Subject: [PATCH 36/51] Pass history items through ObjectValidator for updatable object types --- .../web/activity_pub/object_validator.ex | 69 +++++++++++++++++-- .../article_note_page_validator_test.exs | 44 ++++++++++++ 2 files changed, 106 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index d4bf9c31e..12278a46b 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -128,15 +128,20 @@ def validate(%{"type" => type} = object, meta) end with {:ok, object} <- - object - |> validator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) + do_separate_with_history(object, fn object -> + with {:ok, object} <- + object + |> validator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) - # Insert copy of hashtags as strings for the non-hashtag table indexing - tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object}) - object = Map.put(object, "tag", tag) + # Insert copy of hashtags as strings for the non-hashtag table indexing + tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object}) + object = Map.put(object, "tag", tag) + {:ok, object} + end + end) do {:ok, object, meta} end end @@ -262,4 +267,54 @@ def fetch_actor_and_object(object) do Object.normalize(object["object"], fetch: true) :ok end + + defp for_each_history_item( + %{"type" => "OrderedCollection", "orderedItems" => items} = history, + object, + fun + ) do + processed_items = + Enum.map(items, fn item -> + with item <- Map.put(item, "id", object["id"]), + {:ok, item} <- fun.(item) do + item + else + _ -> nil + end + end) + + if Enum.all?(processed_items, &(not is_nil(&1))) do + {:ok, Map.put(history, "orderedItems", processed_items)} + else + {:error, :invalid_history} + end + end + + defp for_each_history_item(nil, _object, _fun) do + {:ok, nil} + end + + defp for_each_history_item(_, _object, _fun) do + {:error, :invalid_history} + end + + # fun is (object -> {:ok, validated_object_with_string_keys}) + defp do_separate_with_history(object, fun) do + with history <- object["formerRepresentations"], + object <- Map.drop(object, ["formerRepresentations"]), + {_, {:ok, object}} <- {:main_body, fun.(object)}, + {_, {:ok, history}} <- {:history_items, for_each_history_item(history, object, fun)} do + object = + if history do + Map.put(object, "formerRepresentations", history) + else + object + end + + {:ok, object} + else + {:main_body, e} -> e + {:history_items, e} -> e + end + end end diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index c87b07547..11195c40d 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest do use Pleroma.DataCase, async: true + alias Pleroma.Web.ActivityPub.ObjectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator alias Pleroma.Web.ActivityPub.Utils @@ -38,6 +39,49 @@ test "a note from factory validates" do end end + describe "Note with history" do + setup do + user = insert(:user) + {:ok, activity} = Pleroma.Web.CommonAPI.post(user, %{status: "mew mew :dinosaur:"}) + {:ok, edit} = Pleroma.Web.CommonAPI.update(user, activity, %{status: "edited :blank:"}) + + {:ok, %{"object" => external_rep}} = + Pleroma.Web.ActivityPub.Transmogrifier.prepare_outgoing(edit.data) + + %{external_rep: external_rep} + end + + test "edited note", %{external_rep: external_rep} do + assert %{"formerRepresentations" => %{"orderedItems" => [%{"tag" => [_]}]}} = external_rep + + {:ok, validate_res, []} = ObjectValidator.validate(external_rep, []) + + assert %{"formerRepresentations" => %{"orderedItems" => [%{"emoji" => %{"dinosaur" => _}}]}} = + validate_res + end + + test "edited note, badly-formed formerRepresentations", %{external_rep: external_rep} do + external_rep = Map.put(external_rep, "formerRepresentations", %{}) + + assert {:error, _} = ObjectValidator.validate(external_rep, []) + end + + test "edited note, badly-formed history item", %{external_rep: external_rep} do + history_item = + Enum.at(external_rep["formerRepresentations"]["orderedItems"], 0) + |> Map.put("type", "Foo") + + external_rep = + put_in( + external_rep, + ["formerRepresentations", "orderedItems"], + [history_item] + ) + + assert {:error, _} = ObjectValidator.validate(external_rep, []) + end + end + test "a Note from Roadhouse validates" do insert(:user, ap_id: "https://macgirvin.com/channel/mike") From 5ce118d970d3d7a2a5dd0a3719feb1d53be6b5ae Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 3 Jul 2022 20:19:50 -0400 Subject: [PATCH 37/51] Validate object data for incoming Update activities In Create validator we do not validate the object data, but that is because the object itself will go through the pipeline again, which is not the case for Update. Thus, we added validation for objects in Update activities. --- .../web/activity_pub/object_validator.ex | 7 +++- .../update_handling_test.exs | 38 +++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 12278a46b..3ccb4a3d6 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -152,8 +152,8 @@ def validate( ) when objtype in ~w[Question Answer Audio Video Event Article Note Page] do with {_, false} <- {:local, Access.get(meta, :local, false)}, - {:ok, object_data} <- cast_and_apply(object), - meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), + {_, {:ok, object_data, _}} <- {:object_validation, validate(object, meta)}, + meta = Keyword.put(meta, :object_data, object_data), {:ok, update_activity} <- update_activity |> UpdateValidator.cast_and_validate() @@ -169,6 +169,9 @@ def validate( object = stringify_keys(object) {:ok, object, meta} end + + {:object_validation, e} -> + e end end diff --git a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs index 198c35cd3..a09dbf5c6 100644 --- a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs @@ -127,4 +127,42 @@ test "returns object_data in meta for a remote Update" do assert meta[:object_data] end end + + describe "update with history" do + setup do + user = insert(:user) + {:ok, activity} = Pleroma.Web.CommonAPI.post(user, %{status: "mew mew :dinosaur:"}) + {:ok, edit} = Pleroma.Web.CommonAPI.update(user, activity, %{status: "edited :blank:"}) + {:ok, external_rep} = Pleroma.Web.ActivityPub.Transmogrifier.prepare_outgoing(edit.data) + %{external_rep: external_rep} + end + + test "edited note", %{external_rep: external_rep} do + {:ok, _validate_res, meta} = ObjectValidator.validate(external_rep, []) + + assert %{"formerRepresentations" => %{"orderedItems" => [%{"emoji" => %{"dinosaur" => _}}]}} = + meta[:object_data] + end + + test "edited note, badly-formed formerRepresentations", %{external_rep: external_rep} do + external_rep = put_in(external_rep, ["object", "formerRepresentations"], %{}) + + assert {:error, _} = ObjectValidator.validate(external_rep, []) + end + + test "edited note, badly-formed history item", %{external_rep: external_rep} do + history_item = + Enum.at(external_rep["object"]["formerRepresentations"]["orderedItems"], 0) + |> Map.put("type", "Foo") + + external_rep = + put_in( + external_rep, + ["object", "formerRepresentations", "orderedItems"], + [history_item] + ) + + assert {:error, _} = ObjectValidator.validate(external_rep, []) + end + end end From f84ed44cea1e5793dd899c74c38336a1721889e6 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Wed, 6 Jul 2022 01:19:53 -0400 Subject: [PATCH 38/51] Fix cannot get full history on object fetch --- .../web/activity_pub/object_validator.ex | 13 ++- test/pleroma/object/fetcher_test.exs | 80 +++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 3ccb4a3d6..9f446100d 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -103,8 +103,8 @@ def validate( meta ) when objtype in ~w[Question Answer Audio Video Event Article Note Page] do - with {:ok, object_data} <- cast_and_apply(object), - meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), + with {:ok, object_data} <- cast_and_apply_and_stringify_with_history(object), + meta = Keyword.put(meta, :object_data, object_data), {:ok, create_activity} <- create_activity |> CreateGenericValidator.cast_and_validate(meta) @@ -212,6 +212,15 @@ def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do def validate(o, m), do: {:error, {:validator_not_set, {o, m}}} + def cast_and_apply_and_stringify_with_history(object) do + do_separate_with_history(object, fn object -> + with {:ok, object_data} <- cast_and_apply(object), + object_data <- object_data |> stringify_keys() do + {:ok, object_data} + end + end) + end + def cast_and_apply(%{"type" => "ChatMessage"} = object) do ChatMessageValidator.cast_and_apply(object) end diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index 5a79e064f..1f97f36f7 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -456,4 +456,84 @@ test "it adds to formerRepresentations if the remote does not have one and the o } = refetched.data end end + + describe "fetch with history" do + setup do + object2 = %{ + "id" => "https://mastodon.social/2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "type" => "Note", + "content" => "test 2", + "bcc" => [], + "bto" => [], + "cc" => ["https://mastodon.social/users/emelie/followers"], + "to" => [], + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "orig 2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "bcc" => [], + "bto" => [], + "cc" => ["https://mastodon.social/users/emelie/followers"], + "to" => [], + "summary" => "" + } + ], + "totalItems" => 1 + } + } + + mock(fn + %{ + method: :get, + url: "https://mastodon.social/2" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: Jason.encode!(object2) + } + + %{ + method: :get, + url: "https://mastodon.social/users/emelie/collections/featured" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: + Jason.encode!(%{ + "id" => "https://mastodon.social/users/emelie/collections/featured", + "type" => "OrderedCollection", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "orderedItems" => [], + "totalItems" => 0 + }) + } + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + %{object2: object2} + end + + test "it gets history", %{object2: object2} do + {:ok, object} = Fetcher.fetch_object_from_id(object2["id"]) + + assert %{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [%{}] + } + } = object.data + end + end end From 069554e9253a47f99225e12cc0ee99700fb89c6e Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 7 Jul 2022 15:11:29 -0400 Subject: [PATCH 39/51] Guard against outdated Updates It is possible for an earlier Update to be received by us later. For this, we now (1) only allows Updates to poll counts if there is no updated field, or the updated field is the same as the last updated date or creation date; (2) does not allow updating anything if the updated field is older than the last updated date or creation date; (3) allows updating updatable fields otherwise (normal updates); (4) if only the updated field is changed, it does not create a new history item on its own. --- lib/pleroma/object/updater.ex | 65 ++++++++++--- .../web/activity_pub/side_effects_test.exs | 96 ++++++++++++++++++- test/pleroma/web/common_api_test.exs | 2 +- 3 files changed, 145 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 0b21f6c99..3d34c3f27 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -10,7 +10,10 @@ def update_content_fields(orig_object_data, updated_object) do |> Enum.reduce( %{data: orig_object_data, updated: false}, fn field, %{data: data, updated: updated} -> - updated = updated or Map.get(updated_object, field) != Map.get(orig_object_data, field) + updated = + updated or + (field != "updated" and + Map.get(updated_object, field) != Map.get(orig_object_data, field)) data = if Map.has_key?(updated_object, field) do @@ -136,21 +139,57 @@ def make_update_object_data(original_data, new_data, date) do # This calculates the data of the new Object from an Update. # new_data's formerRepresentations is considered. def make_new_object_data_from_update_object(original_data, new_data) do - %{data: updated_data, updated: updated} = - original_data - |> update_content_fields(new_data) + update_is_reasonable = + with {_, updated} when not is_nil(updated) <- {:cur_updated, new_data["updated"]}, + {_, {:ok, updated_time, _}} <- {:cur_updated, DateTime.from_iso8601(updated)}, + {_, last_updated} when not is_nil(last_updated) <- + {:last_updated, original_data["updated"] || original_data["published"]}, + {_, {:ok, last_updated_time, _}} <- + {:last_updated, DateTime.from_iso8601(last_updated)}, + :gt <- DateTime.compare(updated_time, last_updated_time) do + :update_everything + else + # only allow poll updates + {:cur_updated, _} -> :no_content_update + :eq -> :no_content_update + # allow all updates + {:last_updated, _} -> :update_everything + # allow no updates + _ -> false + end - %{updated_object: updated_data, used_history_in_new_object?: used_history_in_new_object?} = - updated_data - |> maybe_update_history(original_data, - updated: updated, - use_history_in_new_object?: true, - new_data: new_data - ) + %{ + updated_object: updated_data, + used_history_in_new_object?: used_history_in_new_object?, + updated: updated + } = + if update_is_reasonable == :update_everything do + %{data: updated_data, updated: updated} = + original_data + |> update_content_fields(new_data) + + updated_data + |> maybe_update_history(original_data, + updated: updated, + use_history_in_new_object?: true, + new_data: new_data + ) + |> Map.put(:updated, updated) + else + %{ + updated_object: original_data, + used_history_in_new_object?: false, + updated: false + } + end updated_data = - updated_data - |> maybe_update_poll(new_data) + if update_is_reasonable != false do + updated_data + |> maybe_update_poll(new_data) + else + updated_data + end %{ updated_data: updated_data, diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 8e84db774..3b313b769 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -142,14 +142,19 @@ test "it uses a given changeset to update", %{user: user, update: update} do describe "update notes" do setup do + make_time = fn -> + Pleroma.Web.ActivityPub.Utils.make_date() + end + user = insert(:user) - note = insert(:note, user: user) + note = insert(:note, user: user, data: %{"published" => make_time.()}) _note_activity = insert(:note_activity, note: note) updated_note = note.data |> Map.put("summary", "edited summary") |> Map.put("content", "edited content") + |> Map.put("updated", make_time.()) {:ok, update_data, []} = Builder.update(user, updated_note) {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) @@ -170,8 +175,69 @@ test "it updates the note", %{ updated_note: updated_note } do {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + updated_time = updated_note["updated"] + new_note = Pleroma.Object.get_by_id(object_id) - assert %{"summary" => "edited summary", "content" => "edited content"} = new_note.data + + assert %{ + "summary" => "edited summary", + "content" => "edited content", + "updated" => ^updated_time + } = new_note.data + end + + test "it rejects updates with no updated attribute in object", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + old_note = Pleroma.Object.get_by_id(object_id) + updated_note = Map.drop(updated_note, ["updated"]) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + new_note = Pleroma.Object.get_by_id(object_id) + assert old_note.data == new_note.data + end + + test "it rejects updates with updated attribute older than what we have in the original object", + %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + old_note = Pleroma.Object.get_by_id(object_id) + {:ok, creation_time, _} = DateTime.from_iso8601(old_note.data["published"]) + + updated_note = + Map.put(updated_note, "updated", DateTime.to_iso8601(DateTime.add(creation_time, -10))) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + new_note = Pleroma.Object.get_by_id(object_id) + assert old_note.data == new_note.data + end + + test "it rejects updates with updated attribute older than the last Update", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + old_note = Pleroma.Object.get_by_id(object_id) + {:ok, creation_time, _} = DateTime.from_iso8601(old_note.data["published"]) + + updated_note = + Map.put(updated_note, "updated", DateTime.to_iso8601(DateTime.add(creation_time, +10))) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + + old_note = Pleroma.Object.get_by_id(object_id) + {:ok, update_time, _} = DateTime.from_iso8601(old_note.data["updated"]) + + updated_note = + Map.put(updated_note, "updated", DateTime.to_iso8601(DateTime.add(update_time, -5))) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + + new_note = Pleroma.Object.get_by_id(object_id) + assert old_note.data == new_note.data end test "it updates using object_data", %{ @@ -215,6 +281,14 @@ test "it puts the original note at the front of formerRepresentations", %{ note.data |> Map.put("summary", "edited summary 2") |> Map.put("content", "edited content 2") + |> Map.put( + "updated", + first_edit["updated"] + |> DateTime.from_iso8601() + |> elem(1) + |> DateTime.add(10) + |> DateTime.to_iso8601() + ) {:ok, second_update_data, []} = Builder.update(user, second_updated_note) {:ok, update, _meta} = ActivityPub.persist(second_update_data, local: true) @@ -238,7 +312,18 @@ test "it does not prepend to formerRepresentations if no actual changes are made updated_note: updated_note } do {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) - %{data: _first_edit} = Pleroma.Object.get_by_id(object_id) + %{data: first_edit} = Pleroma.Object.get_by_id(object_id) + + updated_note = + updated_note + |> Map.put( + "updated", + first_edit["updated"] + |> DateTime.from_iso8601() + |> elem(1) + |> DateTime.add(10) + |> DateTime.to_iso8601() + ) {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) %{data: new_note} = Pleroma.Object.get_by_id(object_id) @@ -270,7 +355,10 @@ test "allows updating choice count without generating edit history", %{ data["oneOf"] |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) - updated_question = data |> Map.put("oneOf", new_choices) + updated_question = + data + |> Map.put("oneOf", new_choices) + |> Map.put("updated", Pleroma.Web.ActivityPub.Utils.make_date()) {:ok, update_data, []} = Builder.update(user, updated_question) {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 842c75e21..24f3a1a93 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1596,7 +1596,7 @@ test "updates a post with emoji and federate properly" do clear_config([:instance, :federating], true) with_mock Pleroma.Web.Federator, - publish: fn p -> nil end do + publish: fn _p -> nil end do {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2 :#{emoji2}:"}) assert updated.data["object"]["content"] == "updated 2 :#{emoji2}:" From 11a6e8842079d8f29417e0de31fda85c7b7cb1be Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Thu, 7 Jul 2022 15:22:04 -0400 Subject: [PATCH 40/51] Test that Question updates are viable --- .../web/activity_pub/side_effects_test.exs | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 3b313b769..0db3a8ea6 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -341,7 +341,12 @@ test "it does not prepend to formerRepresentations if no actual changes are made describe "update questions" do setup do user = insert(:user) - question = insert(:question, user: user) + + question = + insert(:question, + user: user, + data: %{"published" => Pleroma.Web.ActivityPub.Utils.make_date()} + ) %{user: user, data: question.data, id: question.id} end @@ -372,6 +377,59 @@ test "allows updating choice count without generating edit history", %{ refute Map.has_key?(new_question, "formerRepresentations") end + + test "allows updating choice count without updated field", %{ + user: user, + data: data, + id: id + } do + new_choices = + data["oneOf"] + |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) + + updated_question = + data + |> Map.put("oneOf", new_choices) + + {:ok, update_data, []} = Builder.update(user, updated_question) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_question) + + %{data: new_question} = Pleroma.Object.get_by_id(id) + + assert [%{"replies" => %{"totalItems" => 5}}, %{"replies" => %{"totalItems" => 5}}] = + new_question["oneOf"] + + refute Map.has_key?(new_question, "formerRepresentations") + end + + test "allows updating choice count with updated field same as the creation date", %{ + user: user, + data: data, + id: id + } do + new_choices = + data["oneOf"] + |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) + + updated_question = + data + |> Map.put("oneOf", new_choices) + |> Map.put("updated", data["published"]) + + {:ok, update_data, []} = Builder.update(user, updated_question) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_question) + + %{data: new_question} = Pleroma.Object.get_by_id(id) + + assert [%{"replies" => %{"totalItems" => 5}}, %{"replies" => %{"totalItems" => 5}}] = + new_question["oneOf"] + + refute Map.has_key?(new_question, "formerRepresentations") + end end describe "EmojiReact objects" do From 04ded94a50fbabb194ab9e9c5cf8f08937f85d64 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 9 Jul 2022 18:00:42 -0400 Subject: [PATCH 41/51] Fix remote emoji in subject disappearing after edits --- lib/pleroma/web/common_api.ex | 9 +++++- test/pleroma/web/common_api_test.exs | 42 ++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index e5a78c102..89f5dd606 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -415,7 +415,14 @@ def update(user, orig_activity, changes) do defp make_update_data(user, orig_object, changes) do kept_params = %{ - visibility: Visibility.get_visibility(orig_object) + visibility: Visibility.get_visibility(orig_object), + in_reply_to_id: + with replied_id when is_binary(replied_id) <- orig_object.data["inReplyTo"], + %Activity{id: activity_id} <- Activity.get_create_by_object_ap_id(replied_id) do + activity_id + else + _ -> nil + end } params = Map.merge(changes, kept_params) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 24f3a1a93..fcfcb9a2e 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1605,5 +1605,47 @@ test "updates a post with emoji and federate properly" do assert called(Pleroma.Web.Federator.publish(updated)) end end + + test "editing a post that copied a remote title with remote emoji should keep that emoji" do + remote_emoji_uri = "https://remote.org/emoji.png" + + note = + insert( + :note, + data: %{ + "summary" => ":remoteemoji:", + "emoji" => %{ + "remoteemoji" => remote_emoji_uri + }, + "tag" => [ + %{ + "type" => "Emoji", + "name" => "remoteemoji", + "icon" => %{"url" => remote_emoji_uri} + } + ] + } + ) + + note_activity = insert(:note_activity, note: note) + + user = insert(:user) + + {:ok, reply} = + CommonAPI.post(user, %{ + status: "reply", + spoiler_text: ":remoteemoji:", + in_reply_to_id: note_activity.id + }) + + assert reply.object.data["emoji"]["remoteemoji"] == remote_emoji_uri + + {:ok, edit} = + CommonAPI.update(user, reply, %{status: "reply mew mew", spoiler_text: ":remoteemoji:"}) + + edited_note = Pleroma.Object.normalize(edit) + + assert edited_note.data["emoji"]["remoteemoji"] == remote_emoji_uri + end end end From eba9b0760f294482823b9bd55a430979fc2d21af Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 15:56:36 -0400 Subject: [PATCH 42/51] Make MRF Keyword history-aware --- lib/pleroma/object/updater.ex | 40 ++++++ .../web/activity_pub/mrf/keyword_policy.ex | 55 ++++++-- .../activity_pub/mrf/keyword_policy_test.exs | 131 ++++++++++++++++++ test/pleroma/web/common_api_test.exs | 17 +++ 4 files changed, 231 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 3d34c3f27..6381320bd 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -197,4 +197,44 @@ def make_new_object_data_from_update_object(original_data, new_data) do used_history_in_new_object?: used_history_in_new_object? } end + + defp for_each_history_item(%{"orderedItems" => items} = history, _object, fun) do + new_items = + Enum.map(items, fun) + |> Enum.reduce_while( + {:ok, []}, + fn + {:ok, item}, {:ok, acc} -> {:cont, {:ok, acc ++ [item]}} + e, _acc -> {:halt, e} + end + ) + + case new_items do + {:ok, items} -> {:ok, Map.put(history, "orderedItems", items)} + e -> e + end + end + + defp for_each_history_item(history, _, _) do + {:ok, history} + end + + def do_with_history(object, fun) do + with history <- object["formerRepresentations"], + object <- Map.drop(object, ["formerRepresentations"]), + {_, {:ok, object}} <- {:main_body, fun.(object)}, + {_, {:ok, history}} <- {:history_items, for_each_history_item(history, object, fun)} do + object = + if history do + Map.put(object, "formerRepresentations", history) + else + object + end + + {:ok, object} + else + {:main_body, e} -> e + {:history_items, e} -> e + end + end end diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 00b64744f..687ec6c2f 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -27,24 +27,46 @@ defp object_payload(%{} = object) do end defp check_reject(%{"object" => %{} = object} = message) do - payload = object_payload(object) + with {:ok, _new_object} <- + Pleroma.Object.Updater.do_with_history(object, fn object -> + payload = object_payload(object) - if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern -> - string_matches?(payload, pattern) - end) do - {:reject, "[KeywordPolicy] Matches with rejected keyword"} - else + if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern -> + string_matches?(payload, pattern) + end) do + {:reject, "[KeywordPolicy] Matches with rejected keyword"} + else + {:ok, message} + end + end) do {:ok, message} + else + e -> e end end - defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do - payload = object_payload(object) + defp check_ftl_removal(%{"type" => "Create", "to" => to, "object" => %{} = object} = message) do + check_keyword = fn object -> + payload = object_payload(object) - if Pleroma.Constants.as_public() in to and - Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern -> + if Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern -> string_matches?(payload, pattern) end) do + {:should_delist, nil} + else + {:ok, %{}} + end + end + + should_delist? = fn object -> + with {:ok, _} <- Pleroma.Object.Updater.do_with_history(object, check_keyword) do + false + else + _ -> true + end + end + + if Pleroma.Constants.as_public() in to and should_delist?.(object) do to = List.delete(to, Pleroma.Constants.as_public()) cc = [Pleroma.Constants.as_public() | message["cc"] || []] @@ -59,8 +81,12 @@ defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do end end + defp check_ftl_removal(message) do + {:ok, message} + end + defp check_replace(%{"object" => %{} = object} = message) do - object = + replace_kw = fn object -> ["content", "name", "summary"] |> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end) |> Enum.reduce(object, fn field, object -> @@ -73,6 +99,10 @@ defp check_replace(%{"object" => %{} = object} = message) do Map.put(object, field, data) end) + |> (fn object -> {:ok, object} end).() + end + + {:ok, object} = Pleroma.Object.Updater.do_with_history(object, replace_kw) message = Map.put(message, "object", object) @@ -80,7 +110,8 @@ defp check_replace(%{"object" => %{} = object} = message) do end @impl true - def filter(%{"type" => "Create", "object" => %{"content" => _content}} = message) do + def filter(%{"type" => type, "object" => %{"content" => _content}} = message) + when type in ["Create", "Update"] do with {:ok, message} <- check_reject(message), {:ok, message} <- check_ftl_removal(message), {:ok, message} <- check_replace(message) do diff --git a/test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs b/test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs index bfa8e8f59..a0e77d7b9 100644 --- a/test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs @@ -79,6 +79,54 @@ test "rejects if regex matches in summary" do KeywordPolicy.filter(message) end) end + + test "rejects if string matches in history" do + clear_config([:mrf_keyword, :reject], ["pun"]) + + message = %{ + "type" => "Create", + "object" => %{ + "content" => "just a daily reminder that compLAINer is a good", + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "content" => "just a daily reminder that compLAINer is a good pun", + "summary" => "" + } + ] + } + } + } + + assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} = + KeywordPolicy.filter(message) + end + + test "rejects Updates" do + clear_config([:mrf_keyword, :reject], ["pun"]) + + message = %{ + "type" => "Update", + "object" => %{ + "content" => "just a daily reminder that compLAINer is a good", + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "content" => "just a daily reminder that compLAINer is a good pun", + "summary" => "" + } + ] + } + } + } + + assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} = + KeywordPolicy.filter(message) + end end describe "delisting from ftl based on keywords" do @@ -157,6 +205,31 @@ test "delists if regex matches in summary" do not (["https://www.w3.org/ns/activitystreams#Public"] == result["to"]) end) end + + test "delists if string matches in history" do + clear_config([:mrf_keyword, :federated_timeline_removal], ["pun"]) + + message = %{ + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Create", + "object" => %{ + "content" => "just a daily reminder that compLAINer is a good", + "summary" => "", + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "content" => "just a daily reminder that compLAINer is a good pun", + "summary" => "" + } + ] + } + } + } + + {:ok, result} = KeywordPolicy.filter(message) + assert ["https://www.w3.org/ns/activitystreams#Public"] == result["cc"] + refute ["https://www.w3.org/ns/activitystreams#Public"] == result["to"] + end end describe "replacing keywords" do @@ -221,5 +294,63 @@ test "replaces keyword if regex matches in summary" do result == "ZFS is free software" end) end + + test "replaces keyword if string matches in history" do + clear_config([:mrf_keyword, :replace], [{"opensource", "free software"}]) + + message = %{ + "type" => "Create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "object" => %{ + "content" => "ZFS is opensource", + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"content" => "ZFS is opensource mew mew", "summary" => ""} + ] + } + } + } + + {:ok, + %{ + "object" => %{ + "content" => "ZFS is free software", + "formerRepresentations" => %{ + "orderedItems" => [%{"content" => "ZFS is free software mew mew"}] + } + } + }} = KeywordPolicy.filter(message) + end + + test "replaces keyword in Updates" do + clear_config([:mrf_keyword, :replace], [{"opensource", "free software"}]) + + message = %{ + "type" => "Update", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "object" => %{ + "content" => "ZFS is opensource", + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"content" => "ZFS is opensource mew mew", "summary" => ""} + ] + } + } + } + + {:ok, + %{ + "object" => %{ + "content" => "ZFS is free software", + "formerRepresentations" => %{ + "orderedItems" => [%{"content" => "ZFS is free software mew mew"}] + } + } + }} = KeywordPolicy.filter(message) + end end end diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index fcfcb9a2e..c87e64d9e 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1647,5 +1647,22 @@ test "editing a post that copied a remote title with remote emoji should keep th assert edited_note.data["emoji"]["remoteemoji"] == remote_emoji_uri end + + test "respects MRF" do + user = insert(:user) + + clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy]) + clear_config([:mrf_keyword, :replace], [{"updated", "mewmew"}]) + + {:ok, activity} = CommonAPI.post(user, %{status: "foo1", spoiler_text: "updated 1"}) + assert Object.normalize(activity).data["summary"] == "mewmew 1" + + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2"}) + + updated_object = Object.normalize(updated) + assert updated_object.data["content"] == "mewmew 2" + assert Map.get(updated_object.data, "summary", "") == "" + assert Map.has_key?(updated_object.data, "updated") + end end end From cd19537f391b792ee67c728320801d5a247ceb2c Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 17:48:39 -0400 Subject: [PATCH 43/51] Make EnsureRePrepended history-aware --- lib/pleroma/object/updater.ex | 4 +- lib/pleroma/web/activity_pub/mrf.ex | 45 +++++++++++++++- .../activity_pub/mrf/ensure_re_prepended.ex | 6 ++- lib/pleroma/web/activity_pub/mrf/policy.ex | 3 +- .../mrf/ensure_re_prepended_test.exs | 51 ++++++++++++++++++- 5 files changed, 102 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index 6381320bd..ab38d3ed2 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -198,7 +198,7 @@ def make_new_object_data_from_update_object(original_data, new_data) do } end - defp for_each_history_item(%{"orderedItems" => items} = history, _object, fun) do + def for_each_history_item(%{"orderedItems" => items} = history, _object, fun) do new_items = Enum.map(items, fun) |> Enum.reduce_while( @@ -215,7 +215,7 @@ defp for_each_history_item(%{"orderedItems" => items} = history, _object, fun) d end end - defp for_each_history_item(history, _, _) do + def for_each_history_item(history, _, _) do {:ok, history} end diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 323ecdbf1..ff9f84497 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -53,10 +53,53 @@ defmodule Pleroma.Web.ActivityPub.MRF do @required_description_keys [:key, :related_policy] + def filter_one(policy, message) do + should_plug_history? = + if function_exported?(policy, :history_awareness, 0) do + policy.history_awareness() + else + :manual + end + |> Kernel.==(:auto) + + if not should_plug_history? do + policy.filter(message) + else + main_result = policy.filter(message) + + with {_, {:ok, main_message}} <- {:main, main_result}, + {_, + %{ + "formerRepresentations" => %{ + "orderedItems" => [_ | _] + } + }} = {_, object} <- {:object, message["object"]}, + {_, {:ok, new_history}} <- + {:history, + Pleroma.Object.Updater.for_each_history_item( + object["formerRepresentations"], + object, + fn item -> + with {:ok, filtered} <- policy.filter(Map.put(message, "object", item)) do + {:ok, filtered["object"]} + else + e -> e + end + end + )} do + {:ok, put_in(main_message, ["object", "formerRepresentations"], new_history)} + else + {:main, _} -> main_result + {:object, _} -> main_result + {:history, e} -> e + end + end + end + def filter(policies, %{} = message) do policies |> Enum.reduce({:ok, message}, fn - policy, {:ok, message} -> policy.filter(message) + policy, {:ok, message} -> filter_one(policy, message) _, error -> error end) end diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index 51596c09f..a148cc1e7 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless]) + def history_awareness, do: :auto + def filter_by_summary( %{data: %{"summary" => parent_summary}} = _in_reply_to, %{"summary" => child_summary} = child @@ -27,8 +29,8 @@ def filter_by_summary( def filter_by_summary(_in_reply_to, child), do: child - def filter(%{"type" => "Create", "object" => child_object} = object) - when is_map(child_object) do + def filter(%{"type" => type, "object" => child_object} = object) + when type in ["Create", "Update"] and is_map(child_object) do child = child_object["inReplyTo"] |> Object.normalize(fetch: false) diff --git a/lib/pleroma/web/activity_pub/mrf/policy.ex b/lib/pleroma/web/activity_pub/mrf/policy.ex index 0ac250c3d..0234de4d5 100644 --- a/lib/pleroma/web/activity_pub/mrf/policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/policy.ex @@ -12,5 +12,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do label: String.t(), description: String.t() } - @optional_callbacks config_description: 0 + @callback history_awareness() :: :auto | :manual + @optional_callbacks config_description: 0, history_awareness: 0 end diff --git a/test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs b/test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs index bc2f09e29..859e6f1e9 100644 --- a/test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs +++ b/test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrependedTest do alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.EnsureRePrepended describe "rewrites summary" do @@ -35,10 +36,58 @@ test "it adds `re:` to summary object when child summary containts re-subject of assert {:ok, res} = EnsureRePrepended.filter(message) assert res["object"]["summary"] == "re: object-summary" end + + test "it adds `re:` to history" do + message = %{ + "type" => "Create", + "object" => %{ + "summary" => "object-summary", + "inReplyTo" => %Activity{object: %Object{data: %{"summary" => "object-summary"}}}, + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "summary" => "object-summary", + "inReplyTo" => %Activity{object: %Object{data: %{"summary" => "object-summary"}}} + } + ] + } + } + } + + assert {:ok, res} = MRF.filter_one(EnsureRePrepended, message) + assert res["object"]["summary"] == "re: object-summary" + + assert Enum.at(res["object"]["formerRepresentations"]["orderedItems"], 0)["summary"] == + "re: object-summary" + end + + test "it accepts Updates" do + message = %{ + "type" => "Update", + "object" => %{ + "summary" => "object-summary", + "inReplyTo" => %Activity{object: %Object{data: %{"summary" => "object-summary"}}}, + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "summary" => "object-summary", + "inReplyTo" => %Activity{object: %Object{data: %{"summary" => "object-summary"}}} + } + ] + } + } + } + + assert {:ok, res} = MRF.filter_one(EnsureRePrepended, message) + assert res["object"]["summary"] == "re: object-summary" + + assert Enum.at(res["object"]["formerRepresentations"]["orderedItems"], 0)["summary"] == + "re: object-summary" + end end describe "skip filter" do - test "it skip if type isn't 'Create'" do + test "it skip if type isn't 'Create' or 'Update'" do message = %{ "type" => "Annotation", "object" => %{"summary" => "object-summary"} From 0a337063e14a63b3ed80776b493e3c9c56dd95d1 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 22:23:57 -0400 Subject: [PATCH 44/51] Make ForceMentionsInContent history-aware --- .../mrf/force_mentions_in_content.ex | 7 +- .../mrf/force_mentions_in_content_test.exs | 95 +++++++++++++++++++ 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex index 255910b2f..70224561c 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex @@ -11,6 +11,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @impl true + def history_awareness, do: :auto + defp do_extract({:a, attrs, _}, acc) do if Enum.find(attrs, fn {name, value} -> name == "class" && value in ["mention", "u-url mention", "mention u-url"] @@ -74,11 +77,11 @@ defp clean_recipients(recipients, object) do @impl true def filter( %{ - "type" => "Create", + "type" => type, "object" => %{"type" => "Note", "to" => to, "inReplyTo" => in_reply_to} } = object ) - when is_list(to) and is_binary(in_reply_to) do + when type in ["Create", "Update"] and is_list(to) and is_binary(in_reply_to) do # image-only posts from pleroma apparently reach this MRF without the content field content = object["object"]["content"] || "" diff --git a/test/pleroma/web/activity_pub/mrf/force_mentions_in_content_test.exs b/test/pleroma/web/activity_pub/mrf/force_mentions_in_content_test.exs index 125b14a59..b349a4bb7 100644 --- a/test/pleroma/web/activity_pub/mrf/force_mentions_in_content_test.exs +++ b/test/pleroma/web/activity_pub/mrf/force_mentions_in_content_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContentTest do alias Pleroma.Constants alias Pleroma.Object + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent alias Pleroma.Web.CommonAPI @@ -161,4 +162,98 @@ test "with markdown formatting" do assert filtered == "

@luigi I'ma tired...

" end + + test "aware of history" do + mario = insert(:user, nickname: "mario") + wario = insert(:user, nickname: "wario") + + {:ok, post1} = CommonAPI.post(mario, %{status: "Letsa go!"}) + + activity = %{ + "type" => "Create", + "actor" => wario.ap_id, + "object" => %{ + "type" => "Note", + "actor" => wario.ap_id, + "content" => "WHA-HA!", + "to" => [ + mario.ap_id, + Constants.as_public() + ], + "inReplyTo" => post1.object.data["id"], + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "type" => "Note", + "actor" => wario.ap_id, + "content" => "WHA-HA!", + "to" => [ + mario.ap_id, + Constants.as_public() + ], + "inReplyTo" => post1.object.data["id"] + } + ] + } + } + } + + expected = + "@mario WHA-HA!" + + assert {:ok, + %{ + "object" => %{ + "content" => ^expected, + "formerRepresentations" => %{"orderedItems" => [%{"content" => ^expected}]} + } + }} = MRF.filter_one(ForceMentionsInContent, activity) + end + + test "works with Updates" do + mario = insert(:user, nickname: "mario") + wario = insert(:user, nickname: "wario") + + {:ok, post1} = CommonAPI.post(mario, %{status: "Letsa go!"}) + + activity = %{ + "type" => "Update", + "actor" => wario.ap_id, + "object" => %{ + "type" => "Note", + "actor" => wario.ap_id, + "content" => "WHA-HA!", + "to" => [ + mario.ap_id, + Constants.as_public() + ], + "inReplyTo" => post1.object.data["id"], + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "type" => "Note", + "actor" => wario.ap_id, + "content" => "WHA-HA!", + "to" => [ + mario.ap_id, + Constants.as_public() + ], + "inReplyTo" => post1.object.data["id"] + } + ] + } + } + } + + expected = + "@mario WHA-HA!" + + assert {:ok, + %{ + "object" => %{ + "content" => ^expected, + "formerRepresentations" => %{"orderedItems" => [%{"content" => ^expected}]} + } + }} = MRF.filter_one(ForceMentionsInContent, activity) + end end From dce7e429286dfe8cb44a27c50713a03f0e696357 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 22:34:03 -0400 Subject: [PATCH 45/51] Make MediaProxyWarmingPolicy history-aware --- .../mrf/media_proxy_warming_policy.ex | 9 ++-- .../mrf/media_proxy_warming_policy_test.exs | 44 +++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index 0eac8f021..c95d35bb9 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -16,6 +16,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do recv_timeout: 10_000 ] + @impl true + def history_awareness, do: :auto + defp prefetch(url) do # Fetching only proxiable resources if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do @@ -54,10 +57,8 @@ defp preload(%{"object" => %{"attachment" => attachments}} = _message) do end @impl true - def filter( - %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message - ) - when is_list(attachments) and length(attachments) > 0 do + def filter(%{"type" => type, "object" => %{"attachment" => attachments} = _object} = message) + when type in ["Create", "Update"] and is_list(attachments) and length(attachments) > 0 do preload(message) {:ok, message} diff --git a/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs b/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs index 09301c6ca..6557c3a98 100644 --- a/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do use Pleroma.Tests.Helpers alias Pleroma.HTTP + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy import Mock @@ -22,6 +23,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do } } + @message_with_history %{ + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "content", + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "content", + "attachment" => [ + %{"url" => [%{"href" => "http://example.com/image.jpg"}]} + ] + } + ] + } + } + } + setup do: clear_config([:media_proxy, :enabled], true) test "it prefetches media proxy URIs" do @@ -50,4 +70,28 @@ test "it does nothing when no attachments are present" do refute called(HTTP.get(:_, :_, :_)) end end + + test "history-aware" do + Tesla.Mock.mock(fn %{method: :get, url: "http://example.com/image.jpg"} -> + {:ok, %Tesla.Env{status: 200, body: ""}} + end) + + with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do + MRF.filter_one(MediaProxyWarmingPolicy, @message_with_history) + + assert called(HTTP.get(:_, :_, :_)) + end + end + + test "works with Updates" do + Tesla.Mock.mock(fn %{method: :get, url: "http://example.com/image.jpg"} -> + {:ok, %Tesla.Env{status: 200, body: ""}} + end) + + with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do + MRF.filter_one(MediaProxyWarmingPolicy, @message_with_history |> Map.put("type", "Update")) + + assert called(HTTP.get(:_, :_, :_)) + end + end end From fc7ce5f93c4031863cbaf62b72dce55b5b6b0390 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 22:41:04 -0400 Subject: [PATCH 46/51] Make NoPlaceholderTextPolicy history-aware --- .../mrf/no_placeholder_text_policy.ex | 7 +++- .../mrf/no_placeholder_text_policy_test.exs | 41 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index aab647d8e..f81e9e52a 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -6,14 +6,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do @moduledoc "Ensure no content placeholder is present (such as the dot from mastodon)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @impl true + def history_awareness, do: :auto + @impl true def filter( %{ - "type" => "Create", + "type" => type, "object" => %{"content" => content, "attachment" => _} = _child_object } = object ) - when content in [".", "

.

"] do + when type in ["Create", "Update"] and content in [".", "

.

"] do {:ok, put_in(object, ["object", "content"], "")} end diff --git a/test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs b/test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs index acc7c3566..3533c2bc8 100644 --- a/test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicyTest do use Pleroma.DataCase, async: true + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy test "it clears content object" do @@ -20,6 +21,46 @@ test "it clears content object" do assert res["object"]["content"] == "" end + test "history-aware" do + message = %{ + "type" => "Create", + "object" => %{ + "content" => ".", + "attachment" => "image", + "formerRepresentations" => %{ + "orderedItems" => [%{"content" => ".", "attachment" => "image"}] + } + } + } + + assert {:ok, res} = MRF.filter_one(NoPlaceholderTextPolicy, message) + + assert %{ + "content" => "", + "formerRepresentations" => %{"orderedItems" => [%{"content" => ""}]} + } = res["object"] + end + + test "works with Updates" do + message = %{ + "type" => "Update", + "object" => %{ + "content" => ".", + "attachment" => "image", + "formerRepresentations" => %{ + "orderedItems" => [%{"content" => ".", "attachment" => "image"}] + } + } + } + + assert {:ok, res} = MRF.filter_one(NoPlaceholderTextPolicy, message) + + assert %{ + "content" => "", + "formerRepresentations" => %{"orderedItems" => [%{"content" => ""}]} + } = res["object"] + end + @messages [ %{ "type" => "Create", From 46a5c06853c21e720b41a4b38a4d88a38a218ad4 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 22:50:38 -0400 Subject: [PATCH 47/51] Make NormalizeMarkup history-aware --- .../web/activity_pub/mrf/normalize_markup.ex | 6 +- .../mrf/normalize_markup_test.exs | 59 +++++++++++++++---- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index dc2c19d49..2dfc9a901 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -9,7 +9,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do @behaviour Pleroma.Web.ActivityPub.MRF.Policy @impl true - def filter(%{"type" => "Create", "object" => child_object} = object) do + def history_awareness, do: :auto + + @impl true + def filter(%{"type" => type, "object" => child_object} = object) + when type in ["Create", "Update"] do scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy]) content = diff --git a/test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs b/test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs index 20176b63b..66a8f4e44 100644 --- a/test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs +++ b/test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkupTest do use Pleroma.DataCase, async: true + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.NormalizeMarkup @html_sample """ @@ -16,24 +17,58 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkupTest do """ - test "it filter html tags" do - expected = """ - this is in bold -

this is a paragraph

- this is a linebreak
- this is a link with allowed "rel" attribute: - this is a link with not allowed "rel" attribute: example.com - this is an image:
- alert('hacked') - """ + @expected """ + this is in bold +

this is a paragraph

+ this is a linebreak
+ this is a link with allowed "rel" attribute: + this is a link with not allowed "rel" attribute: example.com + this is an image:
+ alert('hacked') + """ + test "it filter html tags" do message = %{"type" => "Create", "object" => %{"content" => @html_sample}} assert {:ok, res} = NormalizeMarkup.filter(message) - assert res["object"]["content"] == expected + assert res["object"]["content"] == @expected end - test "it skips filter if type isn't `Create`" do + test "history-aware" do + message = %{ + "type" => "Create", + "object" => %{ + "content" => @html_sample, + "formerRepresentations" => %{"orderedItems" => [%{"content" => @html_sample}]} + } + } + + assert {:ok, res} = MRF.filter_one(NormalizeMarkup, message) + + assert %{ + "content" => @expected, + "formerRepresentations" => %{"orderedItems" => [%{"content" => @expected}]} + } = res["object"] + end + + test "works with Updates" do + message = %{ + "type" => "Update", + "object" => %{ + "content" => @html_sample, + "formerRepresentations" => %{"orderedItems" => [%{"content" => @html_sample}]} + } + } + + assert {:ok, res} = MRF.filter_one(NormalizeMarkup, message) + + assert %{ + "content" => @expected, + "formerRepresentations" => %{"orderedItems" => [%{"content" => @expected}]} + } = res["object"] + end + + test "it skips filter if type isn't `Create` or `Update`" do message = %{"type" => "Note", "object" => %{}} assert {:ok, res} = NormalizeMarkup.filter(message) From 82c8fc1ede26837d024ecc2fd1231c6d2a3c2c3e Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 23 Jul 2022 23:24:25 -0400 Subject: [PATCH 48/51] Make NoEmptyPolicy work with Update --- .../web/activity_pub/mrf/no_empty_policy.ex | 15 +++++++++--- .../activity_pub/mrf/no_empty_policy_test.exs | 23 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex index 4dc96e068..855cda3b9 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do @impl true def filter(%{"actor" => actor} = object) do with true <- is_local?(actor), + true <- is_eligible_type?(object), true <- is_note?(object), false <- has_attachment?(object), true <- only_mentions?(object) do @@ -32,7 +33,6 @@ defp is_local?(actor) do end defp has_attachment?(%{ - "type" => "Create", "object" => %{"type" => "Note", "attachment" => attachments} }) when length(attachments) > 0, @@ -40,7 +40,13 @@ defp has_attachment?(%{ defp has_attachment?(_), do: false - defp only_mentions?(%{"type" => "Create", "object" => %{"type" => "Note", "source" => source}}) do + defp only_mentions?(%{"object" => %{"type" => "Note", "source" => source}}) do + source = + case source do + %{"content" => text} -> text + _ -> source + end + non_mentions = source |> String.split() |> Enum.filter(&(not String.starts_with?(&1, "@"))) |> length @@ -53,9 +59,12 @@ defp only_mentions?(%{"type" => "Create", "object" => %{"type" => "Note", "sourc defp only_mentions?(_), do: false - defp is_note?(%{"type" => "Create", "object" => %{"type" => "Note"}}), do: true + defp is_note?(%{"object" => %{"type" => "Note"}}), do: true defp is_note?(_), do: false + defp is_eligible_type?(%{"type" => type}) when type in ["Create", "Update"], do: true + defp is_eligible_type?(_), do: false + @impl true def describe, do: {:ok, %{}} end diff --git a/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs b/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs index fe4bb8f0a..386ed395f 100644 --- a/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs @@ -151,4 +151,27 @@ test "Notes with no content are denied" do assert NoEmptyPolicy.filter(message) == {:reject, "[NoEmptyPolicy]"} end + + test "works with Update" do + message = %{ + "actor" => "http://localhost:4001/users/testuser", + "cc" => ["http://localhost:4001/users/testuser/followers"], + "object" => %{ + "actor" => "http://localhost:4001/users/testuser", + "attachment" => [], + "cc" => ["http://localhost:4001/users/testuser/followers"], + "source" => "", + "to" => [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "type" => "Note" + }, + "to" => [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "type" => "Update" + } + + assert NoEmptyPolicy.filter(message) == {:reject, "[NoEmptyPolicy]"} + end end From d877d2a4e7449e942b4d192f283824eebcade563 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 24 Jul 2022 00:02:39 -0400 Subject: [PATCH 49/51] Make HashtagPolicy history-aware --- .../web/activity_pub/mrf/hashtag_policy.ex | 47 ++++++++++--- .../activity_pub/mrf/hashtag_policy_test.exs | 70 +++++++++++++++++++ 2 files changed, 107 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex index 2142b7add..b73fd974c 100644 --- a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -16,6 +16,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @impl true + def history_awareness, do: :manual + defp check_reject(message, hashtags) do if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do {:reject, "[HashtagPolicy] Matches with rejected keyword"} @@ -47,22 +50,46 @@ defp check_ftl_removal(%{"to" => to} = message, hashtags) do defp check_ftl_removal(message, _hashtags), do: {:ok, message} - defp check_sensitive(message, hashtags) do - if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do - {:ok, Kernel.put_in(message, ["object", "sensitive"], true)} - else - {:ok, message} - end + defp check_sensitive(message) do + {:ok, new_object} = + Object.Updater.do_with_history(message["object"], fn object -> + hashtags = Object.hashtags(%Object{data: object}) + + if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do + {:ok, Map.put(object, "sensitive", true)} + else + {:ok, object} + end + end) + + {:ok, Map.put(message, "object", new_object)} end @impl true - def filter(%{"type" => "Create", "object" => object} = message) do - hashtags = Object.hashtags(%Object{data: object}) + def filter(%{"type" => type, "object" => object} = message) when type in ["Create", "Update"] do + history_items = + with %{"formerRepresentations" => %{"orderedItems" => items}} <- object do + items + else + _ -> [] + end + + historical_hashtags = + Enum.reduce(history_items, [], fn item, acc -> + acc ++ Object.hashtags(%Object{data: item}) + end) + + hashtags = Object.hashtags(%Object{data: object}) ++ historical_hashtags if hashtags != [] do with {:ok, message} <- check_reject(message, hashtags), - {:ok, message} <- check_ftl_removal(message, hashtags), - {:ok, message} <- check_sensitive(message, hashtags) do + {:ok, message} <- + (if "type" == "Create" do + check_ftl_removal(message, hashtags) + else + {:ok, message} + end), + {:ok, message} <- check_sensitive(message) do {:ok, message} end else diff --git a/test/pleroma/web/activity_pub/mrf/hashtag_policy_test.exs b/test/pleroma/web/activity_pub/mrf/hashtag_policy_test.exs index 7f2d78a4c..32991c966 100644 --- a/test/pleroma/web/activity_pub/mrf/hashtag_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/hashtag_policy_test.exs @@ -20,6 +20,76 @@ test "it sets the sensitive property with relevant hashtags" do assert modified["object"]["sensitive"] end + test "it is history-aware" do + activity = %{ + "type" => "Create", + "object" => %{ + "content" => "hey", + "tag" => [] + } + } + + activity_data = + activity + |> put_in( + ["object", "formerRepresentations"], + %{ + "type" => "OrderedCollection", + "orderedItems" => [ + Map.put( + activity["object"], + "tag", + [%{"type" => "Hashtag", "name" => "#nsfw"}] + ) + ] + } + ) + + {:ok, modified} = + Pleroma.Web.ActivityPub.MRF.filter_one( + Pleroma.Web.ActivityPub.MRF.HashtagPolicy, + activity_data + ) + + refute modified["object"]["sensitive"] + assert Enum.at(modified["object"]["formerRepresentations"]["orderedItems"], 0)["sensitive"] + end + + test "it works with Update" do + activity = %{ + "type" => "Update", + "object" => %{ + "content" => "hey", + "tag" => [] + } + } + + activity_data = + activity + |> put_in( + ["object", "formerRepresentations"], + %{ + "type" => "OrderedCollection", + "orderedItems" => [ + Map.put( + activity["object"], + "tag", + [%{"type" => "Hashtag", "name" => "#nsfw"}] + ) + ] + } + ) + + {:ok, modified} = + Pleroma.Web.ActivityPub.MRF.filter_one( + Pleroma.Web.ActivityPub.MRF.HashtagPolicy, + activity_data + ) + + refute modified["object"]["sensitive"] + assert Enum.at(modified["object"]["formerRepresentations"]["orderedItems"], 0)["sensitive"] + end + test "it doesn't sets the sensitive property with irrelevant hashtags" do user = insert(:user) From 997f08b3500a983e8b27db9a6e4745582bb4763c Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 24 Jul 2022 00:18:09 -0400 Subject: [PATCH 50/51] Make AntiLinkSpamPolicy history-aware --- .../activity_pub/mrf/anti_link_spam_policy.ex | 3 ++ .../mrf/anti_link_spam_policy_test.exs | 33 ++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index f0504ead4..3ec9c52ee 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -9,6 +9,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do require Logger + @impl true + def history_awareness, do: :auto + # has the user successfully posted before? defp old_user?(%User{} = u) do u.note_count > 0 || u.follower_count > 0 diff --git a/test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs b/test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs index 8c7d4de5c..303d7ca1e 100644 --- a/test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do import Pleroma.Factory import ExUnit.CaptureLog + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy @linkless_message %{ @@ -49,11 +50,23 @@ test "it disallows posts with links" do assert user.note_count == 0 - message = - @linkful_message - |> Map.put("actor", user.ap_id) + message = %{ + "type" => "Create", + "actor" => user.ap_id, + "object" => %{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "content" => "hi world!" + } + ] + }, + "content" => "mew" + } + } - {:reject, _} = AntiLinkSpamPolicy.filter(message) + {:reject, _} = MRF.filter_one(AntiLinkSpamPolicy, message) end test "it allows posts with links for local users" do @@ -67,6 +80,18 @@ test "it allows posts with links for local users" do {:ok, _message} = AntiLinkSpamPolicy.filter(message) end + + test "it disallows posts with links in history" do + user = insert(:user, local: false) + + assert user.note_count == 0 + + message = + @linkful_message + |> Map.put("actor", user.ap_id) + + {:reject, _} = AntiLinkSpamPolicy.filter(message) + end end describe "with old user" do From a4fa286d200b4f0c0ac9f453eb3e0a0526560a20 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Tue, 2 Aug 2022 10:15:56 -0400 Subject: [PATCH 51/51] Use actor_types() to determine whether the Update is for user --- lib/pleroma/web/activity_pub/side_effects.ex | 5 +++-- test/pleroma/web/activity_pub/side_effects_test.exs | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index f56e357bf..5eefd2824 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -163,8 +163,9 @@ def handle(%{data: %{"type" => "Update", "object" => updated_object}} = object, updated_object_id = updated_object["id"] with {_, true} <- {:has_id, is_binary(updated_object_id)}, - {_, user} <- {:user, Pleroma.User.get_by_ap_id(updated_object_id)} do - if user do + %{"type" => type} <- updated_object, + {_, is_user} <- {:is_user, type in Pleroma.Constants.actor_types()} do + if is_user do handle_update_user(object, meta) else handle_update_object(object, meta) diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 0db3a8ea6..9d3490ecd 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -118,7 +118,10 @@ test "it blocks but does not unfollow if the relevant setting is set", %{ describe "update users" do setup do user = insert(:user, local: false) - {:ok, update_data, []} = Builder.update(user, %{"id" => user.ap_id, "name" => "new name!"}) + + {:ok, update_data, []} = + Builder.update(user, %{"id" => user.ap_id, "type" => "Person", "name" => "new name!"}) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) %{user: user, update_data: update_data, update: update}