Merge branch 'feature/database-compaction' into 'develop'

database compaction

See merge request pleroma/pleroma!473

CHANGELOG.md

@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc.
 - [Prometheus](https://prometheus.io/) metrics
 - Support for Mastodon's remote interaction
+- Mix Tasks: `mix pleroma.database remove_embedded_objects`
 - Federation: Support for reports
 - Configuration: `safe_dm_mentions` option
 - Configuration: `link_name` option

lib/mix/tasks/pleroma/database.ex

@@ -0,0 +1,51 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Database do
+  alias Mix.Tasks.Pleroma.Common
+  require Logger
+  use Mix.Task
+
+  @shortdoc "A collection of database related tasks"
+  @moduledoc """
+   A collection of database related tasks
+
+   ## Replace embedded objects with their references
+
+   Replaces embedded objects with references to them in the `objects` table. Only needs to be ran once. The reason why this is not a migration is because it could significantly increase the database size after being ran, however after this `VACUUM FULL` will be able to reclaim about 20% (really depends on what is in the database, your mileage may vary) of the db size before the migration.
+
+       mix pleroma.database remove_embedded_objects
+
+    Options:
+    - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
+  """
+  def run(["remove_embedded_objects" | args]) do
+    {options, [], []} =
+      OptionParser.parse(
+        args,
+        strict: [
+          vacuum: :boolean
+        ]
+      )
+
+    Common.start_pleroma()
+    Logger.info("Removing embedded objects")
+
+    Pleroma.Repo.query!(
+      "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
+      [],
+      timeout: :infinity
+    )
+
+    if Keyword.get(options, :vacuum) do
+      Logger.info("Runnning VACUUM FULL")
+
+      Pleroma.Repo.query!(
+        "vacuum full;",
+        [],
+        timeout: :infinity
+      )
+    end
+  end
+end

lib/pleroma/activity.ex

@@ -10,6 +10,7 @@ defmodule Pleroma.Activity do
   alias Pleroma.Object
   alias Pleroma.Repo
 
+  import Ecto.Changeset
   import Ecto.Query
 
   @type t :: %__MODULE__{}
@@ -79,6 +80,13 @@ def get_by_ap_id(ap_id) do
     )
   end
 
+  def change(struct, params \\ %{}) do
+    struct
+    |> cast(params, [:data])
+    |> validate_required([:data])
+    |> unique_constraint(:ap_id, name: :activities_unique_apid_index)
+  end
+
   def get_by_ap_id_with_object(ap_id) do
     Repo.one(
       from(
@@ -196,28 +204,35 @@ def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
 
   def create_by_object_ap_id_with_object(_), do: nil
 
-  def get_create_by_object_ap_id_with_object(ap_id) do
+  def get_create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
     ap_id
     |> create_by_object_ap_id_with_object()
     |> Repo.one()
   end
 
+  def get_create_by_object_ap_id_with_object(_), do: nil
+
+  defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do
+    get_create_by_object_ap_id_with_object(ap_id)
+  end
+
+  defp get_in_reply_to_activity_from_object(_), do: nil
+
+  def get_in_reply_to_activity(%Activity{data: %{"object" => object}}) do
+    get_in_reply_to_activity_from_object(Object.normalize(object))
+  end
+
   def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"])
   def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id)
   def normalize(_), do: nil
 
-  def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do
-    get_create_by_object_ap_id(ap_id)
-  end
-
-  def get_in_reply_to_activity(_), do: nil
-
   def delete_by_ap_id(id) when is_binary(id) do
     by_object_ap_id(id)
     |> select([u], u)
     |> Repo.delete_all()
     |> elem(1)
     |> Enum.find(fn
+      %{data: %{"type" => "Create", "object" => ap_id}} when is_binary(ap_id) -> ap_id == id
       %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id
       _ -> nil
     end)
@@ -245,54 +260,4 @@ def all_by_actor_and_id(actor, status_ids) do
     |> where([s], s.actor == ^actor)
     |> Repo.all()
   end
-
-  def increase_replies_count(nil), do: nil
-
-  def increase_replies_count(object_ap_id) do
-    from(a in create_by_object_ap_id(object_ap_id),
-      update: [
-        set: [
-          data:
-            fragment(
-              """
-              jsonb_set(?, '{object, repliesCount}',
-                (coalesce((?->'object'->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true)
-              """,
-              a.data,
-              a.data
-            )
-        ]
-      ]
-    )
-    |> Repo.update_all([])
-    |> case do
-      {1, [activity]} -> activity
-      _ -> {:error, "Not found"}
-    end
-  end
-
-  def decrease_replies_count(nil), do: nil
-
-  def decrease_replies_count(object_ap_id) do
-    from(a in create_by_object_ap_id(object_ap_id),
-      update: [
-        set: [
-          data:
-            fragment(
-              """
-              jsonb_set(?, '{object, repliesCount}',
-                (greatest(0, (?->'object'->>'repliesCount')::int - 1))::varchar::jsonb, true)
-              """,
-              a.data,
-              a.data
-            )
-        ]
-      ]
-    )
-    |> Repo.update_all([])
-    |> case do
-      {1, [activity]} -> activity
-      _ -> {:error, "Not found"}
-    end
-  end
 end

lib/pleroma/gopher/server.ex

@@ -38,6 +38,7 @@ def init([ip, port]) do
 defmodule Pleroma.Gopher.Server.ProtocolHandler do
   alias Pleroma.Activity
   alias Pleroma.HTML
+  alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Visibility
@@ -75,14 +76,14 @@ def render_activities(activities) do
     |> Enum.map(fn activity ->
       user = User.get_cached_by_ap_id(activity.data["actor"])
 
-      object = activity.data["object"]
+      object = Object.normalize(activity.data["object"])
       like_count = object["like_count"] || 0
       announcement_count = object["announcement_count"] || 0
 
       link("Post ##{activity.id} by #{user.nickname}", "/notices/#{activity.id}") <>
         info("#{like_count} likes, #{announcement_count} repeats") <>
         "i\tfake\t(NULL)\t0\r\n" <>
-        info(HTML.strip_tags(String.replace(activity.data["object"]["content"], "<br>", "\r")))
+        info(HTML.strip_tags(String.replace(object["content"], "<br>", "\r")))
     end)
     |> Enum.join("i\tfake\t(NULL)\t0\r\n")
   end

lib/pleroma/html.ex

@@ -32,7 +32,8 @@ def get_cached_scrubbed_html_for_activity(content, scrubbers, activity, key \\ "
     key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
 
     Cachex.fetch!(:scrubber_cache, key, fn _key ->
-      ensure_scrubbed_html(content, scrubbers, activity.data["object"]["fake"] || false)
+      object = Pleroma.Object.normalize(activity)
+      ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false)
     end)
   end
 

lib/pleroma/object.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Object do
 
   alias Pleroma.Activity
   alias Pleroma.Object
+  alias Pleroma.Object.Fetcher
   alias Pleroma.ObjectTombstone
   alias Pleroma.Repo
   alias Pleroma.User
@@ -40,41 +41,44 @@ def get_by_ap_id(ap_id) do
     Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
   end
 
+  def normalize(_, fetch_remote \\ true)
   # If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
   # Use this whenever possible, especially when walking graphs in an O(N) loop!
-  def normalize(%Activity{object: %Object{} = object}), do: object
+  def normalize(%Object{} = object, _), do: object
+  def normalize(%Activity{object: %Object{} = object}, _), do: object
 
   # A hack for fake activities
-  def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}) do
+  def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _) do
     %Object{id: "pleroma:fake_object_id", data: data}
   end
 
   # Catch and log Object.normalize() calls where the Activity's child object is not
   # preloaded.
-  def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}) do
+  def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote) do
     Logger.debug(
       "Object.normalize() called without preloaded object (#{ap_id}).  Consider preloading the object!"
     )
 
     Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
 
-    normalize(ap_id)
+    normalize(ap_id, fetch_remote)
   end
 
-  def normalize(%Activity{data: %{"object" => ap_id}}) do
+  def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote) do
     Logger.debug(
       "Object.normalize() called without preloaded object (#{ap_id}).  Consider preloading the object!"
     )
 
     Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
 
-    normalize(ap_id)
+    normalize(ap_id, fetch_remote)
   end
 
   # Old way, try fetching the object through cache.
-  def normalize(%{"id" => ap_id}), do: normalize(ap_id)
-  def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
-  def normalize(_), do: nil
+  def normalize(%{"id" => ap_id}, fetch_remote), do: normalize(ap_id, fetch_remote)
+  def normalize(ap_id, false) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
+  def normalize(ap_id, true) when is_binary(ap_id), do: Fetcher.fetch_object_from_id!(ap_id)
+  def normalize(_, _), do: nil
 
   # Owned objects can only be mutated by their owner
   def authorize_mutation(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}),

lib/pleroma/object/containment.ex

@@ -0,0 +1,61 @@
+defmodule Pleroma.Object.Containment do
+  @moduledoc """
+  # Object Containment
+
+  This module contains some useful functions for containing objects to specific
+  origins and determining those origins.  They previously lived in the
+  ActivityPub `Transmogrifier` module.
+
+  Object containment is an important step in validating remote objects to prevent
+  spoofing, therefore removal of object containment functions is NOT recommended.
+  """
+  def get_actor(%{"actor" => actor}) when is_binary(actor) do
+    actor
+  end
+
+  def get_actor(%{"actor" => actor}) when is_list(actor) do
+    if is_binary(Enum.at(actor, 0)) do
+      Enum.at(actor, 0)
+    else
+      Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end)
+      |> Map.get("id")
+    end
+  end
+
+  def get_actor(%{"actor" => %{"id" => id}}) when is_bitstring(id) do
+    id
+  end
+
+  def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
+    get_actor(%{"actor" => actor})
+  end
+
+  @doc """
+  Checks that an imported AP object's actor matches the domain it came from.
+  """
+  def contain_origin(_id, %{"actor" => nil}), do: :error
+
+  def contain_origin(id, %{"actor" => _actor} = params) do
+    id_uri = URI.parse(id)
+    actor_uri = URI.parse(get_actor(params))
+
+    if id_uri.host == actor_uri.host do
+      :ok
+    else
+      :error
+    end
+  end
+
+  def contain_origin_from_id(_id, %{"id" => nil}), do: :error
+
+  def contain_origin_from_id(id, %{"id" => other_id} = _params) do
+    id_uri = URI.parse(id)
+    other_uri = URI.parse(other_id)
+
+    if id_uri.host == other_uri.host do
+      :ok
+    else
+      :error
+    end
+  end
+end

lib/pleroma/object/fetcher.ex

@@ -0,0 +1,75 @@
+defmodule Pleroma.Object.Fetcher do
+  alias Pleroma.Object
+  alias Pleroma.Object.Containment
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.OStatus
+
+  require Logger
+
+  @httpoison Application.get_env(:pleroma, :httpoison)
+
+  # TODO:
+  # This will create a Create activity, which we need internally at the moment.
+  def fetch_object_from_id(id) do
+    if object = Object.get_cached_by_ap_id(id) do
+      {:ok, object}
+    else
+      Logger.info("Fetching #{id} via AP")
+
+      with {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
+           nil <- Object.normalize(data, false),
+           params <- %{
+             "type" => "Create",
+             "to" => data["to"],
+             "cc" => data["cc"],
+             "actor" => data["actor"] || data["attributedTo"],
+             "object" => data
+           },
+           :ok <- Containment.contain_origin(id, params),
+           {:ok, activity} <- Transmogrifier.handle_incoming(params) do
+        {:ok, Object.normalize(activity, false)}
+      else
+        {:error, {:reject, nil}} ->
+          {:reject, nil}
+
+        object = %Object{} ->
+          {:ok, object}
+
+        _e ->
+          Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
+
+          case OStatus.fetch_activity_from_url(id) do
+            {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"], false)}
+            e -> e
+          end
+      end
+    end
+  end
+
+  def fetch_object_from_id!(id) do
+    with {:ok, object} <- fetch_object_from_id(id) do
+      object
+    else
+      _e ->
+        nil
+    end
+  end
+
+  def fetch_and_contain_remote_object_from_id(id) do
+    Logger.info("Fetching object #{id} via AP")
+
+    with true <- String.starts_with?(id, "http"),
+         {:ok, %{body: body, status: code}} when code in 200..299 <-
+           @httpoison.get(
+             id,
+             [{:Accept, "application/activity+json"}]
+           ),
+         {:ok, data} <- Jason.decode(body),
+         :ok <- Containment.contain_origin_from_id(id, data) do
+      {:ok, data}
+    else
+      e ->
+        {:error, e}
+    end
+  end
+end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Instances
   alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.Object.Fetcher
   alias Pleroma.Pagination
   alias Pleroma.Repo
   alias Pleroma.Upload
@@ -14,7 +15,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
-  alias Pleroma.Web.OStatus
   alias Pleroma.Web.WebFinger
 
   import Ecto.Query
@@ -95,7 +95,6 @@ def increase_replies_count_if_reply(%{
         "type" => "Create"
       }) do
     if is_public?(object) do
-      Activity.increase_replies_count(reply_ap_id)
       Object.increase_replies_count(reply_ap_id)
     end
   end
@@ -106,7 +105,6 @@ def decrease_replies_count_if_reply(%Object{
         data: %{"inReplyTo" => reply_ap_id} = object
       }) do
     if is_public?(object) do
-      Activity.decrease_replies_count(reply_ap_id)
       Object.decrease_replies_count(reply_ap_id)
     end
   end
@@ -121,7 +119,7 @@ def insert(map, local \\ true, fake \\ false) when is_map(map) do
          {:ok, map} <- MRF.filter(map),
          {recipients, _, _} = get_recipients(map),
          {:fake, false, map, recipients} <- {:fake, fake, map, recipients},
-         {:ok, object} <- insert_full_object(map) do
+         {:ok, map, object} <- insert_full_object(map) do
       {:ok, activity} =
         Repo.insert(%Activity{
           data: map,
@@ -170,6 +168,7 @@ def stream_out(activity) do
     public = "https://www.w3.org/ns/activitystreams#Public"
 
     if activity.data["type"] in ["Create", "Announce", "Delete"] do
+      object = Object.normalize(activity.data["object"])
       Pleroma.Web.Streamer.stream("user", activity)
       Pleroma.Web.Streamer.stream("list", activity)
 
@@ -181,12 +180,12 @@ def stream_out(activity) do
         end
 
         if activity.data["type"] in ["Create"] do
-          activity.data["object"]
+          object.data
           |> Map.get("tag", [])
           |> Enum.filter(fn tag -> is_bitstring(tag) end)
           |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end)
 
-          if activity.data["object"]["attachment"] != [] do
+          if object.data["attachment"] != [] do
             Pleroma.Web.Streamer.stream("public:media", activity)
 
             if activity.local do
@@ -572,37 +571,49 @@ defp restrict_since(query, %{"since_id" => since_id}) do
 
   defp restrict_since(query, _), do: query
 
+  defp restrict_tag_reject(_query, %{"tag_reject" => _tag_reject, "skip_preload" => true}) do
+    raise "Can't use the child object without preloading!"
+  end
+
   defp restrict_tag_reject(query, %{"tag_reject" => tag_reject})
        when is_list(tag_reject) and tag_reject != [] do
     from(
-      activity in query,
-      where: fragment(~s(\(not \(? #> '{"object","tag"}'\) \\?| ?\)), activity.data, ^tag_reject)
+      [_activity, object] in query,
+      where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
     )
   end
 
   defp restrict_tag_reject(query, _), do: query
 
+  defp restrict_tag_all(_query, %{"tag_all" => _tag_all, "skip_preload" => true}) do
+    raise "Can't use the child object without preloading!"
+  end
+
   defp restrict_tag_all(query, %{"tag_all" => tag_all})
        when is_list(tag_all) and tag_all != [] do
     from(
-      activity in query,
-      where: fragment(~s(\(? #> '{"object","tag"}'\) \\?& ?), activity.data, ^tag_all)
+      [_activity, object] in query,
+      where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
     )
   end
 
   defp restrict_tag_all(query, _), do: query
 
+  defp restrict_tag(_query, %{"tag" => _tag, "skip_preload" => true}) do
+    raise "Can't use the child object without preloading!"
+  end
+
   defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do
     from(
-      activity in query,
-      where: fragment(~s(\(? #> '{"object","tag"}'\) \\?| ?), activity.data, ^tag)
+      [_activity, object] in query,
+      where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
     )
   end
 
   defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do
     from(
-      activity in query,
-      where: fragment(~s(? <@ (? #> '{"object","tag"}'\)), ^tag, activity.data)
+      [_activity, object] in query,
+      where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
     )
   end
 
@@ -667,10 +678,14 @@ defp restrict_favorited_by(query, %{"favorited_by" => ap_id}) do
 
   defp restrict_favorited_by(query, _), do: query
 
+  defp restrict_media(_query, %{"only_media" => _val, "skip_preload" => true}) do
+    raise "Can't use the child object without preloading!"
+  end
+
   defp restrict_media(query, %{"only_media" => val}) when val == "true" or val == "1" do
     from(
-      activity in query,
-      where: fragment(~s(not (? #> '{"object","attachment"}' = ?\)), activity.data, ^[])
+      [_activity, object] in query,
+      where: fragment("not (?)->'attachment' = (?)", object.data, ^[])
     )
   end
 
@@ -866,7 +881,7 @@ def user_data_from_user_object(data) do
   end
 
   def fetch_and_prepare_user_from_ap_id(ap_id) do
-    with {:ok, data} <- fetch_and_contain_remote_object_from_id(ap_id) do
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
       user_data_from_user_object(data)
     else
       e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
@@ -976,60 +991,6 @@ def publish_one(%{inbox: inbox, json: json, actor: actor, id: id} = params) do
     end
   end
 
-  # TODO:
-  # This will create a Create activity, which we need internally at the moment.
-  def fetch_object_from_id(id) do
-    if object = Object.get_cached_by_ap_id(id) do
-      {:ok, object}
-    else
-      with {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
-           nil <- Object.normalize(data),
-           params <- %{
-             "type" => "Create",
-             "to" => data["to"],
-             "cc" => data["cc"],
-             "actor" => data["actor"] || data["attributedTo"],
-             "object" => data
-           },
-           :ok <- Transmogrifier.contain_origin(id, params),
-           {:ok, activity} <- Transmogrifier.handle_incoming(params) do
-        {:ok, Object.normalize(activity)}
-      else
-        {:error, {:reject, nil}} ->
-          {:reject, nil}
-
-        object = %Object{} ->
-          {:ok, object}
-
-        _e ->
-          Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
-
-          case OStatus.fetch_activity_from_url(id) do
-            {:ok, [activity | _]} -> {:ok, Object.normalize(activity)}
-            e -> e
-          end
-      end
-    end
-  end
-
-  def fetch_and_contain_remote_object_from_id(id) do
-    Logger.info("Fetching object #{id} via AP")
-
-    with true <- String.starts_with?(id, "http"),
-         {:ok, %{body: body, status: code}} when code in 200..299 <-
-           @httpoison.get(
-             id,
-             [{:Accept, "application/activity+json"}]
-           ),
-         {:ok, data} <- Jason.decode(body),
-         :ok <- Transmogrifier.contain_origin_from_id(id, data) do
-      {:ok, data}
-    else
-      e ->
-        {:error, e}
-    end
-  end
-
   # filter out broken threads
   def contain_broken_threads(%Activity{} = activity, %User{} = user) do
     entire_thread_visible_for_user?(activity, user)

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   alias Pleroma.Activity
   alias Pleroma.Object
+  alias Pleroma.Object.Fetcher
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.ObjectView
@@ -173,7 +174,7 @@ def inbox(conn, %{"type" => "Create"} = params) do
       "Signature missing or not from author, relayed Create message, fetching object from source"
     )
 
-    ActivityPub.fetch_object_from_id(params["object"]["id"])
+    Fetcher.fetch_object_from_id(params["object"]["id"])
 
     json(conn, "ok")
   end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -8,8 +8,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   """
   alias Pleroma.Activity
   alias Pleroma.Object
+  alias Pleroma.Object.Containment
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
@@ -18,56 +20,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   require Logger
 
-  def get_actor(%{"actor" => actor}) when is_binary(actor) do
-    actor
-  end
-
-  def get_actor(%{"actor" => actor}) when is_list(actor) do
-    if is_binary(Enum.at(actor, 0)) do
-      Enum.at(actor, 0)
-    else
-      Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end)
-      |> Map.get("id")
-    end
-  end
-
-  def get_actor(%{"actor" => %{"id" => id}}) when is_bitstring(id) do
-    id
-  end
-
-  def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
-    get_actor(%{"actor" => actor})
-  end
-
-  @doc """
-  Checks that an imported AP object's actor matches the domain it came from.
-  """
-  def contain_origin(_id, %{"actor" => nil}), do: :error
-
-  def contain_origin(id, %{"actor" => _actor} = params) do
-    id_uri = URI.parse(id)
-    actor_uri = URI.parse(get_actor(params))
-
-    if id_uri.host == actor_uri.host do
-      :ok
-    else
-      :error
-    end
-  end
-
-  def contain_origin_from_id(_id, %{"id" => nil}), do: :error
-
-  def contain_origin_from_id(id, %{"id" => other_id} = _params) do
-    id_uri = URI.parse(id)
-    other_uri = URI.parse(other_id)
-
-    if id_uri.host == other_uri.host do
-      :ok
-    else
-      :error
-    end
-  end
-
   @doc """
   Modifies an incoming AP object (mastodon format) to our internal format.
   """
@@ -188,7 +140,7 @@ def fix_addressing(object) do
 
   def fix_actor(%{"attributedTo" => actor} = object) do
     object
-    |> Map.put("actor", get_actor(%{"actor" => actor}))
+    |> Map.put("actor", Containment.get_actor(%{"actor" => actor}))
   end
 
   # Check for standardisation
@@ -223,7 +175,7 @@ def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object)
           ""
       end
 
-    case fetch_obj_helper(in_reply_to_id) do
+    case get_obj_helper(in_reply_to_id) do
       {:ok, replied_object} ->
         with %Activity{} = _activity <-
                Activity.get_create_by_object_ap_id(replied_object.data["id"]) do
@@ -448,7 +400,7 @@ def handle_incoming(%{"id" => id}) when not (is_binary(id) and length(id) > 8),
   # - emoji
   def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data)
       when objtype in ["Article", "Note", "Video", "Page"] do
-    actor = get_actor(data)
+    actor = Containment.get_actor(data)
 
     data =
       Map.put(data, "actor", actor)
@@ -506,7 +458,7 @@ def handle_incoming(
   def handle_incoming(
         %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data
       ) do
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"),
@@ -532,7 +484,7 @@ def handle_incoming(
   def handle_incoming(
         %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data
       ) do
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
@@ -556,9 +508,9 @@ def handle_incoming(
   def handle_incoming(
         %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data
       ) do
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
+         {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do
       {:ok, activity}
     else
@@ -569,9 +521,9 @@ def handle_incoming(
   def handle_incoming(
         %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data
       ) do
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
+         {:ok, object} <- get_obj_helper(object_id),
          public <- Visibility.is_public?(data),
          {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do
       {:ok, activity}
@@ -624,10 +576,10 @@ def handle_incoming(
       ) do
     object_id = Utils.get_ap_id(object_id)
 
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
-         :ok <- contain_origin(actor.ap_id, object.data),
+         {:ok, object} <- get_obj_helper(object_id),
+         :ok <- Containment.contain_origin(actor.ap_id, object.data),
          {:ok, activity} <- ActivityPub.delete(object, false) do
       {:ok, activity}
     else
@@ -643,9 +595,9 @@ def handle_incoming(
           "id" => id
         } = data
       ) do
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
+         {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do
       {:ok, activity}
     else
@@ -713,9 +665,9 @@ def handle_incoming(
           "id" => id
         } = data
       ) do
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
+         {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do
       {:ok, activity}
     else
@@ -725,9 +677,6 @@ def handle_incoming(
 
   def handle_incoming(_), do: :error
 
-  def fetch_obj_helper(id) when is_bitstring(id), do: ActivityPub.fetch_object_from_id(id)
-  def fetch_obj_helper(obj) when is_map(obj), do: ActivityPub.fetch_object_from_id(obj["id"])
-
   def get_obj_helper(id) do
     if object = Object.normalize(id), do: {:ok, object}, else: nil
   end
@@ -764,9 +713,9 @@ def prepare_object(object) do
   #  internal -> Mastodon
   #  """
 
-  def prepare_outgoing(%{"type" => "Create", "object" => object} = data) do
+  def prepare_outgoing(%{"type" => "Create", "object" => object_id} = data) do
     object =
-      object
+      Object.normalize(object_id).data
       |> prepare_object
 
     data =
@@ -827,7 +776,7 @@ def prepare_outgoing(%{"type" => _type} = data) do
 
   def maybe_fix_object_url(data) do
     if is_binary(data["object"]) and not String.starts_with?(data["object"], "http") do
-      case fetch_obj_helper(data["object"]) do
+      case get_obj_helper(data["object"]) do
         {:ok, relative_object} ->
           if relative_object.data["external_url"] do
             _data =

lib/pleroma/web/activity_pub/utils.ex

@@ -234,14 +234,18 @@ def lazy_put_object_defaults(map, activity, _fake) do
   @doc """
   Inserts a full object if it is contained in an activity.
   """
-  def insert_full_object(%{"object" => %{"type" => type} = object_data})
+  def insert_full_object(%{"object" => %{"type" => type} = object_data} = map)
       when is_map(object_data) and type in @supported_object_types do
     with {:ok, object} <- Object.create(object_data) do
-      {:ok, object}
+      map =
+        map
+        |> Map.put("object", object.data["id"])
+
+      {:ok, map, object}
     end
   end
 
-  def insert_full_object(_), do: {:ok, nil}
+  def insert_full_object(map), do: {:ok, map, nil}
 
   def update_object_in_activities(%{data: %{"id" => id}} = object) do
     # TODO

lib/pleroma/web/activity_pub/visibility.ex

@@ -41,16 +41,21 @@ def visible_for_user?(activity, user) do
   # guard
   def entire_thread_visible_for_user?(nil, _user), do: false
 
-  # child
+  # XXX: Probably even more inefficient than the previous implementation intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop
+  # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
+
   def entire_thread_visible_for_user?(
-        %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail,
+        %Activity{} = tail,
+        # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail,
         user
-      )
-      when is_binary(parent_id) do
+      ) do
+    case Object.normalize(tail) do
+      %{data: %{"inReplyTo" => parent_id}} when is_binary(parent_id) ->
         parent = Activity.get_in_reply_to_activity(tail)
         visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user)
-  end
 
-  # root
-  def entire_thread_visible_for_user?(tail, user), do: visible_for_user?(tail, user)
+      _ ->
+        visible_for_user?(tail, user)
+    end
+  end
 end

lib/pleroma/web/common_api/common_api.ex

@@ -125,7 +125,10 @@ def get_visibility(%{"in_reply_to_status_id" => status_id}) when not is_nil(stat
         "public"
 
       in_reply_to ->
-        Pleroma.Web.MastodonAPI.StatusView.get_visibility(in_reply_to.data["object"])
+        # XXX: these heuristics should be moved out of MastodonAPI.
+        with %Object{} = object <- Object.normalize(in_reply_to) do
+          Pleroma.Web.MastodonAPI.StatusView.get_visibility(object)
+        end
     end
   end
 
@@ -214,8 +217,10 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do
     with %Activity{
            actor: ^user_ap_id,
            data: %{
-             "type" => "Create",
-             "object" => %{
+             "type" => "Create"
+           },
+           object: %Object{
+             data: %{
                "to" => object_to,
                "type" => "Note"
              }

lib/pleroma/web/common_api/utils.ex

@@ -208,7 +208,7 @@ def make_note_data(
         context,
         content_html,
         attachments,
-        inReplyTo,
+        in_reply_to,
         tags,
         cw \\ nil,
         cc \\ []
@@ -225,9 +225,11 @@ def make_note_data(
       "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq()
     }
 
-    if inReplyTo do
+    if in_reply_to do
+      in_reply_to_object = Object.normalize(in_reply_to.data["object"])
+
       object
-      |> Map.put("inReplyTo", inReplyTo.data["object"]["id"])
+      |> Map.put("inReplyTo", in_reply_to_object.data["id"])
     else
       object
     end

lib/pleroma/web/federator/federator.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.Federator do
   alias Pleroma.Activity
+  alias Pleroma.Object.Containment
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
@@ -136,7 +137,7 @@ def perform(:incoming_ap_doc, params) do
     # actor shouldn't be acting on objects outside their own AP server.
     with {:ok, _user} <- ap_enabled_actor(params["actor"]),
          nil <- Activity.normalize(params["id"]),
-         :ok <- Transmogrifier.contain_origin_from_id(params["actor"], params),
+         :ok <- Containment.contain_origin_from_id(params["actor"], params),
          {:ok, activity} <- Transmogrifier.handle_incoming(params) do
       {:ok, activity}
     else

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -4,13 +4,13 @@
 
 defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   use Pleroma.Web, :controller
-
   alias Ecto.Changeset
   alias Pleroma.Activity
   alias Pleroma.Config
   alias Pleroma.Filter
   alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.Object.Fetcher
   alias Pleroma.Pagination
   alias Pleroma.Repo
   alias Pleroma.ScheduledActivity
@@ -543,10 +543,11 @@ def unpin_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
   end
 
   def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with %Activity{} = activity <- Activity.get_by_id(id),
+    with %Activity{} = activity <- Activity.get_by_id_with_object(id),
+         %Object{} = object <- Object.normalize(activity),
          %User{} = user <- User.get_by_nickname(user.nickname),
          true <- Visibility.visible_for_user?(activity, user),
-         {:ok, user} <- User.bookmark(user, activity.data["object"]["id"]) do
+         {:ok, user} <- User.bookmark(user, object.data["id"]) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -554,10 +555,11 @@ def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
   end
 
   def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with %Activity{} = activity <- Activity.get_by_id(id),
+    with %Activity{} = activity <- Activity.get_by_id_with_object(id),
+         %Object{} = object <- Object.normalize(activity),
          %User{} = user <- User.get_by_nickname(user.nickname),
          true <- Visibility.visible_for_user?(activity, user),
-         {:ok, user} <- User.unbookmark(user, activity.data["object"]["id"]) do
+         {:ok, user} <- User.unbookmark(user, object.data["id"]) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -681,7 +683,8 @@ def upload(%{assigns: %{user: user}} = conn, %{"file" => file} = data) do
   end
 
   def favourited_by(conn, %{"id" => id}) do
-    with %Activity{data: %{"object" => %{"likes" => likes}}} <- Activity.get_by_id(id) do
+    with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
+         %Object{data: %{"likes" => likes}} <- Object.normalize(object) do
       q = from(u in User, where: u.ap_id in ^likes)
       users = Repo.all(q)
 
@@ -694,7 +697,8 @@ def favourited_by(conn, %{"id" => id}) do
   end
 
   def reblogged_by(conn, %{"id" => id}) do
-    with %Activity{data: %{"object" => %{"announcements" => announces}}} <- Activity.get_by_id(id) do
+    with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
+         %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do
       q = from(u in User, where: u.ap_id in ^announces)
       users = Repo.all(q)
 
@@ -997,7 +1001,7 @@ def unsubscribe(%{assigns: %{user: user}} = conn, %{"id" => id}) do
   def status_search(user, query) do
     fetched =
       if Regex.match?(~r/https?:/, query) do
-        with {:ok, object} <- ActivityPub.fetch_object_from_id(query),
+        with {:ok, object} <- Fetcher.fetch_object_from_id(query),
              %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
              true <- Visibility.visible_for_user?(activity, user) do
           [activity]
@@ -1008,13 +1012,13 @@ def status_search(user, query) do
 
     q =
       from(
-        a in Activity,
+        [a, o] in Activity.with_preloaded_object(Activity),
         where: fragment("?->>'type' = 'Create'", a.data),
         where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
         where:
           fragment(
-            "to_tsvector('english', ?->'object'->>'content') @@ plainto_tsquery('english', ?)",
-            a.data,
+            "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
+            o.data,
             ^query
           ),
         limit: 20,

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
   alias Pleroma.Activity
   alias Pleroma.HTML
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
@@ -19,8 +20,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   defp get_replied_to_activities(activities) do
     activities
     |> Enum.map(fn
-      %{data: %{"type" => "Create", "object" => %{"inReplyTo" => in_reply_to}}} ->
-        in_reply_to != "" && in_reply_to
+      %{data: %{"type" => "Create", "object" => object}} ->
+        object = Object.normalize(object)
+        object.data["inReplyTo"] != "" && object.data["inReplyTo"]
 
       _ ->
         nil
@@ -29,7 +31,8 @@ defp get_replied_to_activities(activities) do
     |> Activity.create_by_object_ap_id()
     |> Repo.all()
     |> Enum.reduce(%{}, fn activity, acc ->
-      Map.put(acc, activity.data["object"]["id"], activity)
+      object = Object.normalize(activity.data["object"])
+      Map.put(acc, object.data["id"], activity)
     end)
   end
 
@@ -55,8 +58,8 @@ defp get_context_id(%{data: %{"context" => context}}) when is_binary(context),
   defp get_context_id(_), do: nil
 
   defp reblogged?(activity, user) do
-    object = activity.data["object"] || %{}
-    present?(user && user.ap_id in (object["announcements"] || []))
+    object = Object.normalize(activity) || %{}
+    present?(user && user.ap_id in (object.data["announcements"] || []))
   end
 
   def render("index.json", opts) do
@@ -122,14 +125,16 @@ def render(
     }
   end
 
-  def render("status.json", %{activity: %{data: %{"object" => object}} = activity} = opts) do
+  def render("status.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do
+    object = Object.normalize(activity)
+
     user = get_user(activity.data["actor"])
 
-    like_count = object["like_count"] || 0
-    announcement_count = object["announcement_count"] || 0
+    like_count = object.data["like_count"] || 0
+    announcement_count = object.data["announcement_count"] || 0
 
-    tags = object["tag"] || []
-    sensitive = object["sensitive"] || Enum.member?(tags, "nsfw")
+    tags = object.data["tag"] || []
+    sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw")
 
     mentions =
       activity.recipients
@@ -137,15 +142,17 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
       |> Enum.filter(& &1)
       |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end)
 
-    favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
-    bookmarked = opts[:for] && object["id"] in opts[:for].bookmarks
+    favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || [])
 
-    attachment_data = object["attachment"] || []
+    bookmarked = opts[:for] && object.data["id"] in opts[:for].bookmarks
+
+    attachment_data = object.data["attachment"] || []
     attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)
 
-    created_at = Utils.to_masto_date(object["published"])
+    created_at = Utils.to_masto_date(object.data["published"])
 
     reply_to = get_reply_to(activity, opts)
+
     reply_to_user = reply_to && get_user(reply_to.data["actor"])
 
     content =
@@ -167,7 +174,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
         "mastoapi:content"
       )
 
-    summary = object["summary"] || ""
+    summary = object.data["summary"] || ""
 
     summary_html =
       summary
@@ -190,12 +197,12 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
       if user.local do
         Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity)
       else
-        object["external_url"] || object["id"]
+        object.data["external_url"] || object.data["id"]
       end
 
     %{
       id: to_string(activity.id),
-      uri: object["id"],
+      uri: object.data["id"],
       url: url,
       account: AccountView.render("account.json", %{user: user}),
       in_reply_to_id: reply_to && to_string(reply_to.id),
@@ -205,7 +212,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
       content: content_html,
       created_at: created_at,
       reblogs_count: announcement_count,
-      replies_count: object["repliesCount"] || 0,
+      replies_count: object.data["repliesCount"] || 0,
       favourites_count: like_count,
       reblogged: reblogged?(activity, opts[:for]),
       favourited: present?(favorited),
@@ -223,7 +230,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
         website: nil
       },
       language: nil,
-      emojis: build_emojis(activity.data["object"]["emoji"]),
+      emojis: build_emojis(object.data["emoji"]),
       pleroma: %{
         local: activity.local,
         conversation_id: get_context_id(activity),
@@ -305,15 +312,19 @@ def render("attachment.json", %{attachment: attachment}) do
   end
 
   def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do
-    with nil <- replied_to_activities[activity.data["object"]["inReplyTo"]] do
+    object = Object.normalize(activity.data["object"])
+
+    with nil <- replied_to_activities[object.data["inReplyTo"]] do
       # If user didn't participate in the thread
       Activity.get_in_reply_to_activity(activity)
     end
   end
 
-  def get_reply_to(%{data: %{"object" => object}}, _) do
-    if object["inReplyTo"] && object["inReplyTo"] != "" do
-      Activity.get_create_by_object_ap_id(object["inReplyTo"])
+  def get_reply_to(%{data: %{"object" => _object}} = activity, _) do
+    object = Object.normalize(activity)
+
+    if object.data["inReplyTo"] && object.data["inReplyTo"] != "" do
+      Activity.get_create_by_object_ap_id(object.data["inReplyTo"])
     else
       nil
     end
@@ -321,8 +332,8 @@ def get_reply_to(%{data: %{"object" => object}}, _) do
 
   def get_visibility(object) do
     public = "https://www.w3.org/ns/activitystreams#Public"
-    to = object["to"] || []
-    cc = object["cc"] || []
+    to = object.data["to"] || []
+    cc = object.data["cc"] || []
 
     cond do
       public in to ->
@@ -343,25 +354,25 @@ def get_visibility(object) do
     end
   end
 
-  def render_content(%{"type" => "Video"} = object) do
-    with name when not is_nil(name) and name != "" <- object["name"] do
-      "<p><a href=\"#{object["id"]}\">#{name}</a></p>#{object["content"]}"
+  def render_content(%{data: %{"type" => "Video"}} = object) do
+    with name when not is_nil(name) and name != "" <- object.data["name"] do
+      "<p><a href=\"#{object.data["id"]}\">#{name}</a></p>#{object.data["content"]}"
     else
-      _ -> object["content"] || ""
+      _ -> object.data["content"] || ""
     end
   end
 
-  def render_content(%{"type" => object_type} = object)
+  def render_content(%{data: %{"type" => object_type}} = object)
       when object_type in ["Article", "Page"] do
-    with summary when not is_nil(summary) and summary != "" <- object["name"],
-         url when is_bitstring(url) <- object["url"] do
-      "<p><a href=\"#{url}\">#{summary}</a></p>#{object["content"]}"
+    with summary when not is_nil(summary) and summary != "" <- object.data["name"],
+         url when is_bitstring(url) <- object.data["url"] do
+      "<p><a href=\"#{url}\">#{summary}</a></p>#{object.data["content"]}"
     else
-      _ -> object["content"] || ""
+      _ -> object.data["content"] || ""
     end
   end
 
-  def render_content(object), do: object["content"] || ""
+  def render_content(object), do: object.data["content"] || ""
 
   @doc """
   Builds a dictionary tags.

lib/pleroma/web/ostatus/activity_representer.ex

@@ -54,23 +54,16 @@ defp get_mentions(to) do
     end)
   end
 
-  defp get_links(%{local: true, data: data}) do
+  defp get_links(%{local: true}, %{"id" => object_id}) do
     h = fn str -> [to_charlist(str)] end
 
     [
-      {:link, [type: ['application/atom+xml'], href: h.(data["object"]["id"]), rel: 'self'], []},
-      {:link, [type: ['text/html'], href: h.(data["object"]["id"]), rel: 'alternate'], []}
+      {:link, [type: ['application/atom+xml'], href: h.(object_id), rel: 'self'], []},
+      {:link, [type: ['text/html'], href: h.(object_id), rel: 'alternate'], []}
     ]
   end
 
-  defp get_links(%{
-         local: false,
-         data: %{
-           "object" => %{
-             "external_url" => external_url
-           }
-         }
-       }) do
+  defp get_links(%{local: false}, %{"external_url" => external_url}) do
     h = fn str -> [to_charlist(str)] end
 
     [
@@ -78,7 +71,7 @@ defp get_links(%{
     ]
   end
 
-  defp get_links(_activity), do: []
+  defp get_links(_activity, _object_data), do: []
 
   defp get_emoji_links(emojis) do
     Enum.map(emojis, fn {emoji, file} ->
@@ -88,14 +81,16 @@ defp get_emoji_links(emojis) do
 
   def to_simple_form(activity, user, with_author \\ false)
 
-  def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do
+  def to_simple_form(%{data: %{"type" => "Create"}} = activity, user, with_author) do
     h = fn str -> [to_charlist(str)] end
 
-    updated_at = activity.data["object"]["published"]
-    inserted_at = activity.data["object"]["published"]
+    object = Object.normalize(activity.data["object"])
+
+    updated_at = object.data["published"]
+    inserted_at = object.data["published"]
 
     attachments =
-      Enum.map(activity.data["object"]["attachment"] || [], fn attachment ->
+      Enum.map(object.data["attachment"] || [], fn attachment ->
         url = hd(attachment["url"])
 
         {:link,
@@ -108,7 +103,7 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user,
     mentions = activity.recipients |> get_mentions
 
     categories =
-      (activity.data["object"]["tag"] || [])
+      (object.data["tag"] || [])
       |> Enum.map(fn tag ->
         if is_binary(tag) do
           {:category, [term: to_charlist(tag)], []}
@@ -118,11 +113,11 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user,
       end)
       |> Enum.filter(& &1)
 
-    emoji_links = get_emoji_links(activity.data["object"]["emoji"] || %{})
+    emoji_links = get_emoji_links(object.data["emoji"] || %{})
 
     summary =
-      if activity.data["object"]["summary"] do
-        [{:summary, [], h.(activity.data["object"]["summary"])}]
+      if object.data["summary"] do
+        [{:summary, [], h.(object.data["summary"])}]
       else
         []
       end
@@ -131,10 +126,9 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user,
       {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']},
       {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']},
       # For notes, federate the object id.
-      {:id, h.(activity.data["object"]["id"])},
+      {:id, h.(object.data["id"])},
       {:title, ['New note by #{user.nickname}']},
-      {:content, [type: 'html'],
-       h.(activity.data["object"]["content"] |> String.replace(~r/[\n\r]/, ""))},
+      {:content, [type: 'html'], h.(object.data["content"] |> String.replace(~r/[\n\r]/, ""))},
       {:published, h.(inserted_at)},
       {:updated, h.(updated_at)},
       {:"ostatus:conversation", [ref: h.(activity.data["context"])],
@@ -142,7 +136,7 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user,
       {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}
     ] ++
       summary ++
-      get_links(activity) ++
+      get_links(activity, object.data) ++
       categories ++ attachments ++ in_reply_to ++ author ++ mentions ++ emoji_links
   end
 

lib/pleroma/web/ostatus/handlers/note_handler.ex

@@ -113,8 +113,9 @@ def handle_note(entry, doc \\ nil) do
          cw <- OStatus.get_cw(entry),
          in_reply_to <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry),
          in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to),
-         in_reply_to <-
-           (in_reply_to_activity && in_reply_to_activity.data["object"]["id"]) || in_reply_to,
+         in_reply_to_object <-
+           (in_reply_to_activity && Object.normalize(in_reply_to_activity)) || nil,
+         in_reply_to <- (in_reply_to_object && in_reply_to_object.data["id"]) || in_reply_to,
          attachments <- OStatus.get_attachments(entry),
          context <- get_context(entry, in_reply_to),
          tags <- OStatus.get_tags(entry),

lib/pleroma/web/twitter_api/controllers/util_controller.ex

@@ -75,7 +75,7 @@ def remote_subscribe(conn, %{"user" => %{"nickname" => nick, "profile" => profil
 
   def remote_follow(%{assigns: %{user: user}} = conn, %{"acct" => acct}) do
     if is_status?(acct) do
-      {:ok, object} = ActivityPub.fetch_object_from_id(acct)
+      {:ok, object} = Pleroma.Object.Fetcher.fetch_object_from_id(acct)
       %Activity{id: activity_id} = Activity.get_create_by_object_ap_id(object.data["id"])
       redirect(conn, to: "/notice/#{activity_id}")
     else
@@ -101,7 +101,7 @@ def remote_follow(%{assigns: %{user: user}} = conn, %{"acct" => acct}) do
   end
 
   defp is_status?(acct) do
-    case ActivityPub.fetch_and_contain_remote_object_from_id(acct) do
+    case Pleroma.Object.Fetcher.fetch_and_contain_remote_object_from_id(acct) do
       {:ok, %{"type" => type}} when type in ["Article", "Note", "Video", "Page", "Question"] ->
         true
 

lib/pleroma/web/twitter_api/twitter_api.ex

@@ -266,6 +266,7 @@ defp parse_int(string, default) when is_binary(string) do
 
   defp parse_int(_, default), do: default
 
+  # TODO: unify the search query with MastoAPI one and do only pagination here
   def search(_user, %{"q" => query} = params) do
     limit = parse_int(params["rpp"], 20)
     page = parse_int(params["page"], 1)
@@ -273,13 +274,13 @@ def search(_user, %{"q" => query} = params) do
 
     q =
       from(
-        a in Activity,
+        [a, o] in Activity.with_preloaded_object(Activity),
         where: fragment("?->>'type' = 'Create'", a.data),
         where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
         where:
           fragment(
-            "to_tsvector('english', ?->'object'->>'content') @@ plainto_tsquery('english', ?)",
-            a.data,
+            "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
+            o.data,
             ^query
           ),
         limit: ^limit,

lib/pleroma/web/twitter_api/views/activity_view.ex

@@ -224,15 +224,17 @@ def render("activity.json", %{activity: %{data: %{"type" => "Like"}} = activity}
 
   def render(
         "activity.json",
-        %{activity: %{data: %{"type" => "Create", "object" => object}} = activity} = opts
+        %{activity: %{data: %{"type" => "Create", "object" => object_id}} = activity} = opts
       ) do
     user = get_user(activity.data["actor"], opts)
 
-    created_at = object["published"] |> Utils.date_to_asctime()
-    like_count = object["like_count"] || 0
-    announcement_count = object["announcement_count"] || 0
-    favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
-    repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || [])
+    object = Object.normalize(object_id)
+
+    created_at = object.data["published"] |> Utils.date_to_asctime()
+    like_count = object.data["like_count"] || 0
+    announcement_count = object.data["announcement_count"] || 0
+    favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || [])
+    repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || [])
     pinned = activity.id in user.info.pinned_activities
 
     attentions =
@@ -245,12 +247,12 @@ def render(
 
     conversation_id = get_context_id(activity, opts)
 
-    tags = activity.data["object"]["tag"] || []
-    possibly_sensitive = activity.data["object"]["sensitive"] || Enum.member?(tags, "nsfw")
+    tags = object.data["tag"] || []
+    possibly_sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw")
 
     tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags
 
-    {summary, content} = render_content(object)
+    {summary, content} = render_content(object.data)
 
     html =
       content
@@ -259,7 +261,7 @@ def render(
         activity,
         "twitterapi:content"
       )
-      |> Formatter.emojify(object["emoji"])
+      |> Formatter.emojify(object.data["emoji"])
 
     text =
       if content do
@@ -284,7 +286,7 @@ def render(
 
     %{
       "id" => activity.id,
-      "uri" => activity.data["object"]["id"],
+      "uri" => object.data["id"],
       "user" => UserView.render("show.json", %{user: user, for: opts[:for]}),
       "statusnet_html" => html,
       "text" => text,
@@ -297,20 +299,20 @@ def render(
       "in_reply_to_ostatus_uri" => reply_user && reply_user.ap_id,
       "in_reply_to_user_id" => reply_user && reply_user.id,
       "statusnet_conversation_id" => conversation_id,
-      "attachments" => (object["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts),
+      "attachments" => (object.data["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts),
       "attentions" => attentions,
       "fave_num" => like_count,
       "repeat_num" => announcement_count,
       "favorited" => !!favorited,
       "repeated" => !!repeated,
       "pinned" => pinned,
-      "external_url" => object["external_url"] || object["id"],
+      "external_url" => object.data["external_url"] || object.data["id"],
       "tags" => tags,
       "activity_type" => "post",
       "possibly_sensitive" => possibly_sensitive,
       "visibility" => StatusView.get_visibility(object),
       "summary" => summary,
-      "summary_html" => summary |> Formatter.emojify(object["emoji"]),
+      "summary_html" => summary |> Formatter.emojify(object.data["emoji"]),
       "card" => card,
       "muted" => CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user)
     }

test/activity_test.exs

@@ -28,18 +28,4 @@ test "returns the activity that created an object" do
 
     assert activity == found_activity
   end
-
-  test "reply count" do
-    %{id: id, data: %{"object" => %{"id" => object_ap_id}}} = activity = insert(:note_activity)
-
-    replies_count = activity.data["object"]["repliesCount"] || 0
-    expected_increase = replies_count + 1
-    Activity.increase_replies_count(object_ap_id)
-    %{data: %{"object" => %{"repliesCount" => actual_increase}}} = Activity.get_by_id(id)
-    assert expected_increase == actual_increase
-    expected_decrease = expected_increase - 1
-    Activity.decrease_replies_count(object_ap_id)
-    %{data: %{"object" => %{"repliesCount" => actual_decrease}}} = Activity.get_by_id(id)
-    assert expected_decrease == actual_decrease
-  end
 end

test/object/containment_test.exs

@@ -0,0 +1,58 @@
+defmodule Pleroma.Object.ContainmentTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Object.Containment
+  alias Pleroma.User
+
+  import Pleroma.Factory
+
+  describe "general origin containment" do
+    test "contain_origin_from_id() catches obvious spoofing attempts" do
+      data = %{
+        "id" => "http://example.com/~alyssa/activities/1234.json"
+      }
+
+      :error =
+        Containment.contain_origin_from_id(
+          "http://example.org/~alyssa/activities/1234.json",
+          data
+        )
+    end
+
+    test "contain_origin_from_id() allows alternate IDs within the same origin domain" do
+      data = %{
+        "id" => "http://example.com/~alyssa/activities/1234.json"
+      }
+
+      :ok =
+        Containment.contain_origin_from_id(
+          "http://example.com/~alyssa/activities/1234",
+          data
+        )
+    end
+
+    test "contain_origin_from_id() allows matching IDs" do
+      data = %{
+        "id" => "http://example.com/~alyssa/activities/1234.json"
+      }
+
+      :ok =
+        Containment.contain_origin_from_id(
+          "http://example.com/~alyssa/activities/1234.json",
+          data
+        )
+    end
+
+    test "users cannot be collided through fake direction spoofing attempts" do
+      _user =
+        insert(:user, %{
+          nickname: "rye@niu.moe",
+          local: false,
+          ap_id: "https://niu.moe/users/rye",
+          follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"})
+        })
+
+      {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye")
+    end
+  end
+end

test/object/fetcher_test.exs

@@ -0,0 +1,90 @@
+defmodule Pleroma.Object.FetcherTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Object.Fetcher
+  import Tesla.Mock
+
+  setup do
+    mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  describe "actor origin containment" do
+    test "it rejects objects with a bogus origin" do
+      {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity.json")
+    end
+
+    test "it rejects objects when attributedTo is wrong (variant 1)" do
+      {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity2.json")
+    end
+
+    test "it rejects objects when attributedTo is wrong (variant 2)" do
+      {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity3.json")
+    end
+  end
+
+  describe "fetching an object" do
+    test "it fetches an object" do
+      {:ok, object} =
+        Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
+
+      assert activity = Activity.get_create_by_object_ap_id(object.data["id"])
+      assert activity.data["id"]
+
+      {:ok, object_again} =
+        Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
+
+      assert [attachment] = object.data["attachment"]
+      assert is_list(attachment["url"])
+
+      assert object == object_again
+    end
+
+    test "it works with objects only available via Ostatus" do
+      {:ok, object} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873")
+      assert activity = Activity.get_create_by_object_ap_id(object.data["id"])
+      assert activity.data["id"]
+
+      {:ok, object_again} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873")
+
+      assert object == object_again
+    end
+
+    test "it correctly stitches up conversations between ostatus and ap" do
+      last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394"
+      {:ok, object} = Fetcher.fetch_object_from_id(last)
+
+      object = Object.get_by_ap_id(object.data["inReplyTo"])
+      assert object
+    end
+  end
+
+  describe "implementation quirks" do
+    test "it can fetch plume articles" do
+      {:ok, object} =
+        Fetcher.fetch_object_from_id(
+          "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
+        )
+
+      assert object
+    end
+
+    test "it can fetch peertube videos" do
+      {:ok, object} =
+        Fetcher.fetch_object_from_id(
+          "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
+        )
+
+      assert object
+    end
+
+    test "all objects with fake directions are rejected by the object fetcher" do
+      {:error, _} =
+        Fetcher.fetch_and_contain_remote_object_from_id(
+          "https://info.pleroma.site/activity4.json"
+        )
+    end
+  end
+end

test/object_test.exs

@@ -5,9 +5,15 @@
 defmodule Pleroma.ObjectTest do
   use Pleroma.DataCase
   import Pleroma.Factory
+  import Tesla.Mock
   alias Pleroma.Object
   alias Pleroma.Repo
 
+  setup do
+    mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
   test "returns an object by it's AP id" do
     object = insert(:note)
     found_object = Object.get_by_ap_id(object.data["id"])
@@ -58,4 +64,26 @@ test "ensures cache is cleared for the object" do
       assert cached_object.data["type"] == "Tombstone"
     end
   end
+
+  describe "normalizer" do
+    test "fetches unknown objects by default" do
+      %Object{} =
+        object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367")
+
+      assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367"
+    end
+
+    test "fetches unknown objects when fetch_remote is explicitly true" do
+      %Object{} =
+        object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367", true)
+
+      assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367"
+    end
+
+    test "does not fetch unknown objects when fetch_remote is false" do
+      assert is_nil(
+               Object.normalize("http://mastodon.example.org/@admin/99541947525187367", false)
+             )
+    end
+  end
 end

test/scheduled_activity_worker_test.exs

@@ -14,6 +14,6 @@ test "creates a status from the scheduled activity" do
 
     refute Repo.get(ScheduledActivity, scheduled_activity.id)
     activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id))
-    assert activity.data["object"]["content"] == "hi"
+    assert Pleroma.Object.normalize(activity).data["content"] == "hi"
   end
 end

test/user_test.exs

@@ -5,6 +5,7 @@
 defmodule Pleroma.UserTest do
   alias Pleroma.Activity
   alias Pleroma.Builders.UserBuilder
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
@@ -256,7 +257,7 @@ test "it sends a welcome message if it is set" do
 
       activity = Repo.one(Pleroma.Activity)
       assert registered_user.ap_id in activity.recipients
-      assert activity.data["object"]["content"] =~ "cool site"
+      assert Object.normalize(activity).data["content"] =~ "cool site"
       assert activity.actor == welcome_user.ap_id
 
       Pleroma.Config.put([:instance, :welcome_user_nickname], nil)
@@ -1132,14 +1133,14 @@ test "bookmarks" do
         "status" => "heweoo!"
       })
 
-    id1 = activity1.data["object"]["id"]
+    id1 = Object.normalize(activity1).data["id"]
 
     {:ok, activity2} =
       CommonAPI.post(user, %{
         "status" => "heweoo!"
       })
 
-    id2 = activity2.data["object"]["id"]
+    id2 = Object.normalize(activity2).data["id"]
 
     assert {:ok, user_state1} = User.bookmark(user, id1)
     assert user_state1.bookmarks == [id1]

test/web/activity_pub/activity_pub_test.exs

@@ -84,17 +84,21 @@ test "it fetches the appropriate tag-restricted posts" do
       {:ok, status_two} = CommonAPI.post(user, %{"status" => ". #essais"})
       {:ok, status_three} = CommonAPI.post(user, %{"status" => ". #test #reject"})
 
-      fetch_one = ActivityPub.fetch_activities([], %{"tag" => "test"})
-      fetch_two = ActivityPub.fetch_activities([], %{"tag" => ["test", "essais"]})
+      fetch_one = ActivityPub.fetch_activities([], %{"type" => "Create", "tag" => "test"})
+
+      fetch_two =
+        ActivityPub.fetch_activities([], %{"type" => "Create", "tag" => ["test", "essais"]})
 
       fetch_three =
         ActivityPub.fetch_activities([], %{
+          "type" => "Create",
           "tag" => ["test", "essais"],
           "tag_reject" => ["reject"]
         })
 
       fetch_four =
         ActivityPub.fetch_activities([], %{
+          "type" => "Create",
           "tag" => ["test"],
           "tag_all" => ["test", "reject"]
         })
@@ -192,8 +196,10 @@ test "adds an id to a given object if it lacks one and is a note and inserts it
       }
 
       {:ok, %Activity{} = activity} = ActivityPub.insert(data)
-      assert is_binary(activity.data["object"]["id"])
-      assert %Object{} = Object.get_by_ap_id(activity.data["object"]["id"])
+      object = Object.normalize(activity.data["object"])
+
+      assert is_binary(object.data["id"])
+      assert %Object{} = Object.get_by_ap_id(activity.data["object"])
     end
   end
 
@@ -206,7 +212,11 @@ test "removes doubled 'to' recipients" do
           to: ["user1", "user1", "user2"],
           actor: user,
           context: "",
-          object: %{}
+          object: %{
+            "to" => ["user1", "user1", "user2"],
+            "type" => "Note",
+            "content" => "testing"
+          }
         })
 
       assert activity.data["to"] == ["user1", "user2"]
@@ -244,25 +254,21 @@ test "increases replies count" do
       # public
       {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "public"))
       assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
-      assert data["object"]["repliesCount"] == 1
       assert object.data["repliesCount"] == 1
 
       # unlisted
       {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "unlisted"))
       assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
-      assert data["object"]["repliesCount"] == 2
       assert object.data["repliesCount"] == 2
 
       # private
       {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "private"))
       assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
-      assert data["object"]["repliesCount"] == 2
       assert object.data["repliesCount"] == 2
 
       # direct
       {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "direct"))
       assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
-      assert data["object"]["repliesCount"] == 2
       assert object.data["repliesCount"] == 2
     end
   end
@@ -680,40 +686,13 @@ test "works with base64 encoded images" do
     end
   end
 
-  describe "fetching an object" do
-    test "it fetches an object" do
-      {:ok, object} =
-        ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
+  describe "fetch the latest Follow" do
+    test "fetches the latest Follow activity" do
+      %Activity{data: %{"type" => "Follow"}} = activity = insert(:follow_activity)
+      follower = Repo.get_by(User, ap_id: activity.data["actor"])
+      followed = Repo.get_by(User, ap_id: activity.data["object"])
 
-      assert activity = Activity.get_create_by_object_ap_id(object.data["id"])
-      assert activity.data["id"]
-
-      {:ok, object_again} =
-        ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
-
-      assert [attachment] = object.data["attachment"]
-      assert is_list(attachment["url"])
-
-      assert object == object_again
-    end
-
-    test "it works with objects only available via Ostatus" do
-      {:ok, object} = ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873")
-      assert activity = Activity.get_create_by_object_ap_id(object.data["id"])
-      assert activity.data["id"]
-
-      {:ok, object_again} =
-        ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873")
-
-      assert object == object_again
-    end
-
-    test "it correctly stitches up conversations between ostatus and ap" do
-      last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394"
-      {:ok, object} = ActivityPub.fetch_object_from_id(last)
-
-      object = Object.get_by_ap_id(object.data["inReplyTo"])
-      assert object
+      assert activity == Utils.fetch_latest_follow(follower, followed)
     end
   end
 
@@ -804,10 +783,10 @@ test "decrements user note count only for public activities" do
       {:ok, a4} =
         CommonAPI.post(User.get_by_id(user.id), %{"status" => "yeah", "visibility" => "direct"})
 
-      {:ok, _} = a1.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
-      {:ok, _} = a2.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
-      {:ok, _} = a3.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
-      {:ok, _} = a4.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete()
+      {:ok, _} = Object.normalize(a1) |> ActivityPub.delete()
+      {:ok, _} = Object.normalize(a2) |> ActivityPub.delete()
+      {:ok, _} = Object.normalize(a3) |> ActivityPub.delete()
+      {:ok, _} = Object.normalize(a4) |> ActivityPub.delete()
 
       user = User.get_by_id(user.id)
       assert user.info.note_count == 10
@@ -849,22 +828,18 @@ test "decreases reply count" do
 
       _ = CommonAPI.delete(direct_reply.id, user2)
       assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
-      assert data["object"]["repliesCount"] == 2
       assert object.data["repliesCount"] == 2
 
       _ = CommonAPI.delete(private_reply.id, user2)
       assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
-      assert data["object"]["repliesCount"] == 2
       assert object.data["repliesCount"] == 2
 
       _ = CommonAPI.delete(public_reply.id, user2)
       assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
-      assert data["object"]["repliesCount"] == 1
       assert object.data["repliesCount"] == 1
 
       _ = CommonAPI.delete(unlisted_reply.id, user2)
       assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id)
-      assert data["object"]["repliesCount"] == 0
       assert object.data["repliesCount"] == 0
     end
   end
@@ -906,7 +881,10 @@ test "it filters broken threads" do
       activities = ActivityPub.fetch_activities([user1.ap_id | user1.following])
 
       private_activity_1 = Activity.get_by_ap_id_with_object(private_activity_1.data["id"])
-      assert [public_activity, private_activity_1, private_activity_3] == activities
+
+      assert [public_activity, private_activity_1, private_activity_3] ==
+               activities
+
       assert length(activities) == 3
 
       activities = ActivityPub.contain_timeline(activities, user1)
@@ -916,15 +894,6 @@ test "it filters broken threads" do
     end
   end
 
-  test "it can fetch plume articles" do
-    {:ok, object} =
-      ActivityPub.fetch_object_from_id(
-        "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/"
-      )
-
-    assert object
-  end
-
   describe "update" do
     test "it creates an update activity with the new user data" do
       user = insert(:user)
@@ -946,15 +915,6 @@ test "it creates an update activity with the new user data" do
     end
   end
 
-  test "it can fetch peertube videos" do
-    {:ok, object} =
-      ActivityPub.fetch_object_from_id(
-        "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
-      )
-
-    assert object
-  end
-
   test "returned pinned statuses" do
     Pleroma.Config.put([:instance, :max_pinned_statuses], 3)
     user = insert(:user)

test/web/activity_pub/transmogrifier_test.exs

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
   use Pleroma.DataCase
   alias Pleroma.Activity
   alias Pleroma.Object
+  alias Pleroma.Object.Fetcher
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -50,14 +51,14 @@ test "it fetches replied-to activities if we don't have them" do
         |> Map.put("object", object)
 
       {:ok, returned_activity} = Transmogrifier.handle_incoming(data)
+      returned_object = Object.normalize(returned_activity.data["object"])
 
       assert activity =
                Activity.get_create_by_object_ap_id(
                  "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment"
                )
 
-      assert returned_activity.data["object"]["inReplyToAtomUri"] ==
-               "https://shitposter.club/notice/2827873"
+      assert returned_object.data["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873"
     end
 
     test "it works for incoming notices" do
@@ -80,7 +81,7 @@ test "it works for incoming notices" do
 
       assert data["actor"] == "http://mastodon.example.org/users/admin"
 
-      object = data["object"]
+      object = Object.normalize(data["object"]).data
       assert object["id"] == "http://mastodon.example.org/users/admin/statuses/99512778738411822"
 
       assert object["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
@@ -107,7 +108,9 @@ test "it works for incoming notices with hashtags" do
       data = File.read!("test/fixtures/mastodon-post-activity-hashtag.json") |> Poison.decode!()
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
-      assert Enum.at(data["object"]["tag"], 2) == "moo"
+      object = Object.normalize(data["object"])
+
+      assert Enum.at(object.data["tag"], 2) == "moo"
     end
 
     test "it works for incoming notices with contentMap" do
@@ -115,8 +118,9 @@ test "it works for incoming notices with contentMap" do
         File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!()
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+      object = Object.normalize(data["object"])
 
-      assert data["object"]["content"] ==
+      assert object.data["content"] ==
                "<p><span class=\"h-card\"><a href=\"http://localtesting.pleroma.lol/users/lain\" class=\"u-url mention\">@<span>lain</span></a></span></p>"
     end
 
@@ -124,8 +128,9 @@ test "it works for incoming notices with to/cc not being an array (kroeg)" do
       data = File.read!("test/fixtures/kroeg-post-activity.json") |> Poison.decode!()
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+      object = Object.normalize(data["object"])
 
-      assert data["object"]["content"] ==
+      assert object.data["content"] ==
                "<p>henlo from my Psion netBook</p><p>message sent from my Psion netBook</p>"
     end
 
@@ -141,24 +146,27 @@ test "it works for incoming notices with tag not being an array (kroeg)" do
       data = File.read!("test/fixtures/kroeg-array-less-emoji.json") |> Poison.decode!()
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+      object = Object.normalize(data["object"])
 
-      assert data["object"]["emoji"] == %{
+      assert object.data["emoji"] == %{
                "icon_e_smile" => "https://puckipedia.com/forum/images/smilies/icon_e_smile.png"
              }
 
       data = File.read!("test/fixtures/kroeg-array-less-hashtag.json") |> Poison.decode!()
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+      object = Object.normalize(data["object"])
 
-      assert "test" in data["object"]["tag"]
+      assert "test" in object.data["tag"]
     end
 
     test "it works for incoming notices with url not being a string (prismo)" do
       data = File.read!("test/fixtures/prismo-url-map.json") |> Poison.decode!()
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+      object = Object.normalize(data["object"])
 
-      assert data["object"]["url"] == "https://prismo.news/posts/83"
+      assert object.data["url"] == "https://prismo.news/posts/83"
     end
 
     test "it cleans up incoming notices which are not really DMs" do
@@ -180,15 +188,15 @@ test "it cleans up incoming notices which are not really DMs" do
 
       data = Map.put(data, "object", object)
 
-      {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
+      {:ok, %Activity{data: data, local: false} = activity} = Transmogrifier.handle_incoming(data)
 
       assert data["to"] == []
       assert data["cc"] == to
 
-      object = data["object"]
+      object_data = Object.normalize(activity).data
 
-      assert object["to"] == []
-      assert object["cc"] == to
+      assert object_data["to"] == []
+      assert object_data["cc"] == to
     end
 
     test "it works for incoming follow requests" do
@@ -231,14 +239,14 @@ test "it works for incoming likes" do
       data =
         File.read!("test/fixtures/mastodon-like.json")
         |> Poison.decode!()
-        |> Map.put("object", activity.data["object"]["id"])
+        |> Map.put("object", activity.data["object"])
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
 
       assert data["actor"] == "http://mastodon.example.org/users/admin"
       assert data["type"] == "Like"
       assert data["id"] == "http://mastodon.example.org/users/admin#likes/2"
-      assert data["object"] == activity.data["object"]["id"]
+      assert data["object"] == activity.data["object"]
     end
 
     test "it returns an error for incoming unlikes wihout a like activity" do
@@ -248,7 +256,7 @@ test "it returns an error for incoming unlikes wihout a like activity" do
       data =
         File.read!("test/fixtures/mastodon-undo-like.json")
         |> Poison.decode!()
-        |> Map.put("object", activity.data["object"]["id"])
+        |> Map.put("object", activity.data["object"])
 
       assert Transmogrifier.handle_incoming(data) == :error
     end
@@ -260,7 +268,7 @@ test "it works for incoming unlikes with an existing like activity" do
       like_data =
         File.read!("test/fixtures/mastodon-like.json")
         |> Poison.decode!()
-        |> Map.put("object", activity.data["object"]["id"])
+        |> Map.put("object", activity.data["object"])
 
       {:ok, %Activity{data: like_data, local: false}} = Transmogrifier.handle_incoming(like_data)
 
@@ -302,7 +310,7 @@ test "it works for incoming announces with an existing activity" do
       data =
         File.read!("test/fixtures/mastodon-announce.json")
         |> Poison.decode!()
-        |> Map.put("object", activity.data["object"]["id"])
+        |> Map.put("object", activity.data["object"])
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
 
@@ -312,7 +320,7 @@ test "it works for incoming announces with an existing activity" do
       assert data["id"] ==
                "http://mastodon.example.org/users/admin/statuses/99542391527669785/activity"
 
-      assert data["object"] == activity.data["object"]["id"]
+      assert data["object"] == activity.data["object"]
 
       assert Activity.get_create_by_object_ap_id(data["object"]).id == activity.id
     end
@@ -324,7 +332,7 @@ test "it does not clobber the addressing on announce activities" do
       data =
         File.read!("test/fixtures/mastodon-announce.json")
         |> Poison.decode!()
-        |> Map.put("object", activity.data["object"]["id"])
+        |> Map.put("object", Object.normalize(activity).data["id"])
         |> Map.put("to", ["http://mastodon.example.org/users/admin/followers"])
         |> Map.put("cc", [])
 
@@ -450,7 +458,7 @@ test "it works for incoming deletes" do
 
       object =
         data["object"]
-        |> Map.put("id", activity.data["object"]["id"])
+        |> Map.put("id", activity.data["object"])
 
       data =
         data
@@ -471,7 +479,7 @@ test "it fails for incoming deletes with spoofed origin" do
 
       object =
         data["object"]
-        |> Map.put("id", activity.data["object"]["id"])
+        |> Map.put("id", activity.data["object"])
 
       data =
         data
@@ -489,7 +497,7 @@ test "it works for incoming unannounces with an existing notice" do
       announce_data =
         File.read!("test/fixtures/mastodon-announce.json")
         |> Poison.decode!()
-        |> Map.put("object", activity.data["object"]["id"])
+        |> Map.put("object", activity.data["object"])
 
       {:ok, %Activity{data: announce_data, local: false}} =
         Transmogrifier.handle_incoming(announce_data)
@@ -504,7 +512,7 @@ test "it works for incoming unannounces with an existing notice" do
 
       assert data["type"] == "Undo"
       assert data["object"]["type"] == "Announce"
-      assert data["object"]["object"] == activity.data["object"]["id"]
+      assert data["object"]["object"] == activity.data["object"]
 
       assert data["object"]["id"] ==
                "http://mastodon.example.org/users/admin/statuses/99542391527669785/activity"
@@ -783,7 +791,7 @@ test "it rejects activities without a valid ID" do
 
     test "it remaps video URLs as attachments if necessary" do
       {:ok, object} =
-        ActivityPub.fetch_object_from_id(
+        Fetcher.fetch_object_from_id(
           "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
         )
 
@@ -1088,10 +1096,6 @@ test "it fixes the actor URL property to be a proper URI" do
   end
 
   describe "actor origin containment" do
-    test "it rejects objects with a bogus origin" do
-      {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity.json")
-    end
-
     test "it rejects activities which reference objects with bogus origins" do
       data = %{
         "@context" => "https://www.w3.org/ns/activitystreams",
@@ -1105,10 +1109,6 @@ test "it rejects activities which reference objects with bogus origins" do
       :error = Transmogrifier.handle_incoming(data)
     end
 
-    test "it rejects objects when attributedTo is wrong (variant 1)" do
-      {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity2.json")
-    end
-
     test "it rejects activities which reference objects that have an incorrect attribution (variant 1)" do
       data = %{
         "@context" => "https://www.w3.org/ns/activitystreams",
@@ -1122,10 +1122,6 @@ test "it rejects activities which reference objects that have an incorrect attri
       :error = Transmogrifier.handle_incoming(data)
     end
 
-    test "it rejects objects when attributedTo is wrong (variant 2)" do
-      {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity3.json")
-    end
-
     test "it rejects activities which reference objects that have an incorrect attribution (variant 2)" do
       data = %{
         "@context" => "https://www.w3.org/ns/activitystreams",
@@ -1140,62 +1136,6 @@ test "it rejects activities which reference objects that have an incorrect attri
     end
   end
 
-  describe "general origin containment" do
-    test "contain_origin_from_id() catches obvious spoofing attempts" do
-      data = %{
-        "id" => "http://example.com/~alyssa/activities/1234.json"
-      }
-
-      :error =
-        Transmogrifier.contain_origin_from_id(
-          "http://example.org/~alyssa/activities/1234.json",
-          data
-        )
-    end
-
-    test "contain_origin_from_id() allows alternate IDs within the same origin domain" do
-      data = %{
-        "id" => "http://example.com/~alyssa/activities/1234.json"
-      }
-
-      :ok =
-        Transmogrifier.contain_origin_from_id(
-          "http://example.com/~alyssa/activities/1234",
-          data
-        )
-    end
-
-    test "contain_origin_from_id() allows matching IDs" do
-      data = %{
-        "id" => "http://example.com/~alyssa/activities/1234.json"
-      }
-
-      :ok =
-        Transmogrifier.contain_origin_from_id(
-          "http://example.com/~alyssa/activities/1234.json",
-          data
-        )
-    end
-
-    test "users cannot be collided through fake direction spoofing attempts" do
-      insert(:user, %{
-        nickname: "rye@niu.moe",
-        local: false,
-        ap_id: "https://niu.moe/users/rye",
-        follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"})
-      })
-
-      {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye")
-    end
-
-    test "all objects with fake directions are rejected by the object fetcher" do
-      {:error, _} =
-        ActivityPub.fetch_and_contain_remote_object_from_id(
-          "https://info.pleroma.site/activity4.json"
-        )
-    end
-  end
-
   describe "reserialization" do
     test "successfully reserializes a message with inReplyTo == nil" do
       user = insert(:user)

test/web/common_api/common_api_test.exs

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.CommonAPITest do
   use Pleroma.DataCase
   alias Pleroma.Activity
+  alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
 
@@ -32,14 +33,16 @@ test "it de-duplicates tags" do
     user = insert(:user)
     {:ok, activity} = CommonAPI.post(user, %{"status" => "#2hu #2HU"})
 
-    assert activity.data["object"]["tag"] == ["2hu"]
+    object = Object.normalize(activity.data["object"])
+
+    assert object.data["tag"] == ["2hu"]
   end
 
   test "it adds emoji in the object" do
     user = insert(:user)
     {:ok, activity} = CommonAPI.post(user, %{"status" => ":moominmamma:"})
 
-    assert activity.data["object"]["emoji"]["moominmamma"]
+    assert Object.normalize(activity).data["emoji"]["moominmamma"]
   end
 
   test "it adds emoji when updating profiles" do
@@ -64,8 +67,9 @@ test "it filters out obviously bad tags when accepting a post as HTML" do
           "content_type" => "text/html"
         })
 
-      content = activity.data["object"]["content"]
-      assert content == "<p><b>2hu</b></p>alert('xss')"
+      object = Object.normalize(activity.data["object"])
+
+      assert object.data["content"] == "<p><b>2hu</b></p>alert('xss')"
     end
 
     test "it filters out obviously bad tags when accepting a post as Markdown" do
@@ -79,8 +83,9 @@ test "it filters out obviously bad tags when accepting a post as Markdown" do
           "content_type" => "text/markdown"
         })
 
-      content = activity.data["object"]["content"]
-      assert content == "<p><b>2hu</b></p>alert('xss')"
+      object = Object.normalize(activity.data["object"])
+
+      assert object.data["content"] == "<p><b>2hu</b></p>alert('xss')"
     end
   end
 

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -1879,7 +1879,7 @@ test "search doesn't show statuses that it shouldn't", %{conn: conn} do
     capture_log(fn ->
       conn =
         conn
-        |> get("/api/v1/search", %{"q" => activity.data["object"]["id"]})
+        |> get("/api/v1/search", %{"q" => Object.normalize(activity).data["id"]})
 
       assert results = json_response(conn, 200)
 
@@ -2791,9 +2791,9 @@ test "Repeated posts that are replies incorrectly have in_reply_to_id null", %{c
 
     assert %{"content" => "xD", "id" => id} = json_response(conn1, 200)
 
-    activity = Activity.get_by_id(id)
+    activity = Activity.get_by_id_with_object(id)
 
-    assert activity.data["object"]["inReplyTo"] == replied_to.data["object"]["id"]
+    assert Object.normalize(activity).data["inReplyTo"] == Object.normalize(replied_to).data["id"]
     assert Activity.get_in_reply_to_activity(activity).id == replied_to.id
 
     # Reblog from the third user

test/web/mastodon_api/status_view_test.exs

@@ -6,8 +6,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
   use Pleroma.DataCase
 
   alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
@@ -53,14 +54,14 @@ test "tries to get a user by nickname if fetching by ap_id doesn't work" do
 
   test "a note with null content" do
     note = insert(:note_activity)
+    note_object = Object.normalize(note.data["object"])
 
     data =
-      note.data
-      |> put_in(["object", "content"], nil)
+      note_object.data
+      |> Map.put("content", nil)
 
-    note =
-      note
-      |> Map.put(:data, data)
+    Object.change(note_object, %{data: data})
+    |> Object.update_and_set_cache()
 
     User.get_cached_by_ap_id(note.data["actor"])
 
@@ -230,7 +231,7 @@ test "a peertube video" do
     user = insert(:user)
 
     {:ok, object} =
-      ActivityPub.fetch_object_from_id(
+      Pleroma.Object.Fetcher.fetch_object_from_id(
         "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
       )
 

test/web/ostatus/ostatus_test.exs

@@ -28,34 +28,35 @@ test "don't insert create notes twice" do
   test "handle incoming note - GS, Salmon" do
     incoming = File.read!("test/fixtures/incoming_note_activity.xml")
     {:ok, [activity]} = OStatus.handle_incoming(incoming)
+    object = Object.normalize(activity.data["object"])
 
     user = User.get_by_ap_id(activity.data["actor"])
     assert user.info.note_count == 1
     assert activity.data["type"] == "Create"
-    assert activity.data["object"]["type"] == "Note"
+    assert object.data["type"] == "Note"
 
-    assert activity.data["object"]["id"] ==
-             "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note"
+    assert object.data["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note"
 
     assert activity.data["published"] == "2017-04-23T14:51:03+00:00"
-    assert activity.data["object"]["published"] == "2017-04-23T14:51:03+00:00"
+    assert object.data["published"] == "2017-04-23T14:51:03+00:00"
 
     assert activity.data["context"] ==
              "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b"
 
     assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"]
-    assert activity.data["object"]["emoji"] == %{"marko" => "marko.png", "reimu" => "reimu.png"}
+    assert object.data["emoji"] == %{"marko" => "marko.png", "reimu" => "reimu.png"}
     assert activity.local == false
   end
 
   test "handle incoming notes - GS, subscription" do
     incoming = File.read!("test/fixtures/ostatus_incoming_post.xml")
     {:ok, [activity]} = OStatus.handle_incoming(incoming)
+    object = Object.normalize(activity.data["object"])
 
     assert activity.data["type"] == "Create"
-    assert activity.data["object"]["type"] == "Note"
-    assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211"
-    assert activity.data["object"]["content"] == "Will it blend?"
+    assert object.data["type"] == "Note"
+    assert object.data["actor"] == "https://social.heldscal.la/user/23211"
+    assert object.data["content"] == "Will it blend?"
     user = User.get_cached_by_ap_id(activity.data["actor"])
     assert User.ap_followers(user) in activity.data["to"]
     assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"]
@@ -64,20 +65,22 @@ test "handle incoming notes - GS, subscription" do
   test "handle incoming notes with attachments - GS, subscription" do
     incoming = File.read!("test/fixtures/incoming_websub_gnusocial_attachments.xml")
     {:ok, [activity]} = OStatus.handle_incoming(incoming)
+    object = Object.normalize(activity.data["object"])
 
     assert activity.data["type"] == "Create"
-    assert activity.data["object"]["type"] == "Note"
-    assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211"
-    assert activity.data["object"]["attachment"] |> length == 2
-    assert activity.data["object"]["external_url"] == "https://social.heldscal.la/notice/2020923"
+    assert object.data["type"] == "Note"
+    assert object.data["actor"] == "https://social.heldscal.la/user/23211"
+    assert object.data["attachment"] |> length == 2
+    assert object.data["external_url"] == "https://social.heldscal.la/notice/2020923"
     assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"]
   end
 
   test "handle incoming notes with tags" do
     incoming = File.read!("test/fixtures/ostatus_incoming_post_tag.xml")
     {:ok, [activity]} = OStatus.handle_incoming(incoming)
+    object = Object.normalize(activity.data["object"])
 
-    assert activity.data["object"]["tag"] == ["nsfw"]
+    assert object.data["tag"] == ["nsfw"]
     assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"]
   end
 
@@ -92,10 +95,11 @@ test "handle incoming notes - Mastodon, salmon, reply" do
 
     incoming = File.read!("test/fixtures/incoming_reply_mastodon.xml")
     {:ok, [activity]} = OStatus.handle_incoming(incoming)
+    object = Object.normalize(activity.data["object"])
 
     assert activity.data["type"] == "Create"
-    assert activity.data["object"]["type"] == "Note"
-    assert activity.data["object"]["actor"] == "https://mastodon.social/users/lambadalambda"
+    assert object.data["type"] == "Note"
+    assert object.data["actor"] == "https://mastodon.social/users/lambadalambda"
     assert activity.data["context"] == "2hu"
     assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"]
   end
@@ -103,42 +107,47 @@ test "handle incoming notes - Mastodon, salmon, reply" do
   test "handle incoming notes - Mastodon, with CW" do
     incoming = File.read!("test/fixtures/mastodon-note-cw.xml")
     {:ok, [activity]} = OStatus.handle_incoming(incoming)
+    object = Object.normalize(activity.data["object"])
 
     assert activity.data["type"] == "Create"
-    assert activity.data["object"]["type"] == "Note"
-    assert activity.data["object"]["actor"] == "https://mastodon.social/users/lambadalambda"
-    assert activity.data["object"]["summary"] == "technologic"
+    assert object.data["type"] == "Note"
+    assert object.data["actor"] == "https://mastodon.social/users/lambadalambda"
+    assert object.data["summary"] == "technologic"
     assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"]
   end
 
   test "handle incoming unlisted messages, put public into cc" do
     incoming = File.read!("test/fixtures/mastodon-note-unlisted.xml")
     {:ok, [activity]} = OStatus.handle_incoming(incoming)
+    object = Object.normalize(activity.data["object"])
+
     refute "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"]
     assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["cc"]
-    refute "https://www.w3.org/ns/activitystreams#Public" in activity.data["object"]["to"]
-    assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["object"]["cc"]
+    refute "https://www.w3.org/ns/activitystreams#Public" in object.data["to"]
+    assert "https://www.w3.org/ns/activitystreams#Public" in object.data["cc"]
   end
 
   test "handle incoming retweets - Mastodon, with CW" do
     incoming = File.read!("test/fixtures/cw_retweet.xml")
     {:ok, [[_activity, retweeted_activity]]} = OStatus.handle_incoming(incoming)
+    retweeted_object = Object.normalize(retweeted_activity.data["object"])
 
-    assert retweeted_activity.data["object"]["summary"] == "Hey."
+    assert retweeted_object.data["summary"] == "Hey."
   end
 
   test "handle incoming notes - GS, subscription, reply" do
     incoming = File.read!("test/fixtures/ostatus_incoming_reply.xml")
     {:ok, [activity]} = OStatus.handle_incoming(incoming)
+    object = Object.normalize(activity.data["object"])
 
     assert activity.data["type"] == "Create"
-    assert activity.data["object"]["type"] == "Note"
-    assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211"
+    assert object.data["type"] == "Note"
+    assert object.data["actor"] == "https://social.heldscal.la/user/23211"
 
-    assert activity.data["object"]["content"] ==
+    assert object.data["content"] ==
              "@<a href=\"https://gs.archae.me/user/4687\" class=\"h-card u-url p-nickname mention\" title=\"shpbot\">shpbot</a> why not indeed."
 
-    assert activity.data["object"]["inReplyTo"] ==
+    assert object.data["inReplyTo"] ==
              "tag:gs.archae.me,2017-04-30:noticeId=778260:objectType=note"
 
     assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"]
@@ -150,17 +159,18 @@ test "handle incoming retweets - GS, subscription" do
 
     assert activity.data["type"] == "Announce"
     assert activity.data["actor"] == "https://social.heldscal.la/user/23211"
-    assert activity.data["object"] == retweeted_activity.data["object"]["id"]
+    assert activity.data["object"] == retweeted_activity.data["object"]
     assert "https://pleroma.soykaf.com/users/lain" in activity.data["to"]
     refute activity.local
 
     retweeted_activity = Activity.get_by_id(retweeted_activity.id)
+    retweeted_object = Object.normalize(retweeted_activity.data["object"])
     assert retweeted_activity.data["type"] == "Create"
     assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain"
     refute retweeted_activity.local
-    assert retweeted_activity.data["object"]["announcement_count"] == 1
-    assert String.contains?(retweeted_activity.data["object"]["content"], "mastodon")
-    refute String.contains?(retweeted_activity.data["object"]["content"], "Test account")
+    assert retweeted_object.data["announcement_count"] == 1
+    assert String.contains?(retweeted_object.data["content"], "mastodon")
+    refute String.contains?(retweeted_object.data["content"], "Test account")
   end
 
   test "handle incoming retweets - GS, subscription - local message" do
@@ -192,10 +202,11 @@ test "handle incoming retweets - GS, subscription - local message" do
   test "handle incoming retweets - Mastodon, salmon" do
     incoming = File.read!("test/fixtures/share.xml")
     {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming)
+    retweeted_object = Object.normalize(retweeted_activity.data["object"])
 
     assert activity.data["type"] == "Announce"
     assert activity.data["actor"] == "https://mastodon.social/users/lambadalambda"
-    assert activity.data["object"] == retweeted_activity.data["object"]["id"]
+    assert activity.data["object"] == retweeted_activity.data["object"]
 
     assert activity.data["id"] ==
              "tag:mastodon.social,2017-05-03:objectId=4934452:objectType=Status"
@@ -204,7 +215,7 @@ test "handle incoming retweets - Mastodon, salmon" do
     assert retweeted_activity.data["type"] == "Create"
     assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain"
     refute retweeted_activity.local
-    refute String.contains?(retweeted_activity.data["object"]["content"], "Test account")
+    refute String.contains?(retweeted_object.data["content"], "Test account")
   end
 
   test "handle incoming favorites - GS, websub" do
@@ -214,7 +225,7 @@ test "handle incoming favorites - GS, websub" do
 
       assert activity.data["type"] == "Like"
       assert activity.data["actor"] == "https://social.heldscal.la/user/23211"
-      assert activity.data["object"] == favorited_activity.data["object"]["id"]
+      assert activity.data["object"] == favorited_activity.data["object"]
 
       assert activity.data["id"] ==
                "tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061643:2017-05-05T09:12:50+00:00"
@@ -223,7 +234,7 @@ test "handle incoming favorites - GS, websub" do
       assert favorited_activity.data["type"] == "Create"
       assert favorited_activity.data["actor"] == "https://shitposter.club/user/1"
 
-      assert favorited_activity.data["object"]["id"] ==
+      assert favorited_activity.data["object"] ==
                "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment"
 
       refute favorited_activity.local
@@ -258,17 +269,17 @@ test "handle incoming favorites with locally available object - GS, websub" do
   test "handle incoming replies" do
     incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml")
     {:ok, [activity]} = OStatus.handle_incoming(incoming)
+    object = Object.normalize(activity.data["object"])
 
     assert activity.data["type"] == "Create"
-    assert activity.data["object"]["type"] == "Note"
+    assert object.data["type"] == "Note"
 
-    assert activity.data["object"]["inReplyTo"] ==
+    assert object.data["inReplyTo"] ==
              "http://pleroma.example.org:4000/objects/55bce8fc-b423-46b1-af71-3759ab4670bc"
 
     assert "http://pleroma.example.org:4000/users/lain5" in activity.data["to"]
 
-    assert activity.data["object"]["id"] ==
-             "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note"
+    assert object.data["id"] == "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note"
 
     assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"]
   end
@@ -495,7 +506,7 @@ test "it builds a missing status from an html url" do
 
         assert activity.data["actor"] == "https://shitposter.club/user/1"
 
-        assert activity.data["object"]["id"] ==
+        assert activity.data["object"] ==
                  "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment"
       end)
     end
@@ -504,7 +515,7 @@ test "it works for atom notes, too" do
       url = "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056"
       {:ok, [activity]} = OStatus.fetch_activity_from_url(url)
       assert activity.data["actor"] == "https://social.sakamoto.gq/users/eal"
-      assert activity.data["object"]["id"] == url
+      assert activity.data["object"] == url
     end
   end
 

test/web/twitter_api/twitter_api_test.exs

@@ -46,13 +46,14 @@ test "create a status" do
     }
 
     {:ok, activity = %Activity{}} = TwitterAPI.create_status(user, input)
+    object = Object.normalize(activity.data["object"])
 
     expected_text =
       "Hello again, <span class='h-card'><a data-user='#{mentioned_user.id}' class='u-url mention' href='shp'>@<span>shp</span></a></span>.&lt;script&gt;&lt;/script&gt;<br>This is on another :moominmamma: line. <a class='hashtag' data-tag='2hu' href='http://localhost:4001/tag/2hu' rel='tag'>#2hu</a> <a class='hashtag' data-tag='epic' href='http://localhost:4001/tag/epic' rel='tag'>#epic</a> <a class='hashtag' data-tag='phantasmagoric' href='http://localhost:4001/tag/phantasmagoric' rel='tag'>#phantasmagoric</a><br><a href=\"http://example.org/image.jpg\" class='attachment'>image.jpg</a>"
 
-    assert get_in(activity.data, ["object", "content"]) == expected_text
-    assert get_in(activity.data, ["object", "type"]) == "Note"
-    assert get_in(activity.data, ["object", "actor"]) == user.ap_id
+    assert get_in(object.data, ["content"]) == expected_text
+    assert get_in(object.data, ["type"]) == "Note"
+    assert get_in(object.data, ["actor"]) == user.ap_id
     assert get_in(activity.data, ["actor"]) == user.ap_id
     assert Enum.member?(get_in(activity.data, ["cc"]), User.ap_followers(user))
 
@@ -65,18 +66,18 @@ test "create a status" do
     assert activity.local == true
 
     assert %{"moominmamma" => "http://localhost:4001/finmoji/128px/moominmamma-128.png"} =
-             activity.data["object"]["emoji"]
+             object.data["emoji"]
 
     # hashtags
-    assert activity.data["object"]["tag"] == ["2hu", "epic", "phantasmagoric"]
+    assert object.data["tag"] == ["2hu", "epic", "phantasmagoric"]
 
     # Add a context
     assert is_binary(get_in(activity.data, ["context"]))
-    assert is_binary(get_in(activity.data, ["object", "context"]))
+    assert is_binary(get_in(object.data, ["context"]))
 
-    assert is_list(activity.data["object"]["attachment"])
+    assert is_list(object.data["attachment"])
 
-    assert activity.data["object"] == Object.get_by_ap_id(activity.data["object"]["id"]).data
+    assert activity.data["object"] == object.data["id"]
 
     user = User.get_by_ap_id(user.ap_id)
 
@@ -91,6 +92,7 @@ test "create a status that is a reply" do
     }
 
     {:ok, activity = %Activity{}} = TwitterAPI.create_status(user, input)
+    object = Object.normalize(activity.data["object"])
 
     input = %{
       "status" => "Here's your (you).",
@@ -98,13 +100,13 @@ test "create a status that is a reply" do
     }
 
     {:ok, reply = %Activity{}} = TwitterAPI.create_status(user, input)
+    reply_object = Object.normalize(reply.data["object"])
 
     assert get_in(reply.data, ["context"]) == get_in(activity.data, ["context"])
 
-    assert get_in(reply.data, ["object", "context"]) ==
-             get_in(activity.data, ["object", "context"])
+    assert get_in(reply_object.data, ["context"]) == get_in(object.data, ["context"])
 
-    assert get_in(reply.data, ["object", "inReplyTo"]) == get_in(activity.data, ["object", "id"])
+    assert get_in(reply_object.data, ["inReplyTo"]) == get_in(activity.data, ["object"])
     assert Activity.get_in_reply_to_activity(reply).id == activity.id
   end
 

test/web/twitter_api/views/activity_view_test.exs

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do
   use Pleroma.DataCase
 
   alias Pleroma.Activity
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -125,10 +126,11 @@ test "a create activity with a note" do
     other_user = insert(:user, %{nickname: "shp"})
 
     {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!", "visibility" => "direct"})
+    object = Object.normalize(activity.data["object"])
 
     result = ActivityView.render("activity.json", activity: activity)
 
-    convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"])
+    convo_id = Utils.context_to_conversation_id(object.data["context"])
 
     expected = %{
       "activity_type" => "post",
@@ -136,8 +138,8 @@ test "a create activity with a note" do
       "attentions" => [
         UserView.render("show.json", %{user: other_user})
       ],
-      "created_at" => activity.data["object"]["published"] |> Utils.date_to_asctime(),
-      "external_url" => activity.data["object"]["id"],
+      "created_at" => object.data["published"] |> Utils.date_to_asctime(),
+      "external_url" => object.data["id"],
       "fave_num" => 0,
       "favorited" => false,
       "id" => activity.id,
@@ -161,7 +163,7 @@ test "a create activity with a note" do
         }\">@<span>shp</span></a></span>!",
       "tags" => [],
       "text" => "Hey @shp!",
-      "uri" => activity.data["object"]["id"],
+      "uri" => object.data["id"],
       "user" => UserView.render("show.json", %{user: user}),
       "visibility" => "direct",
       "card" => nil,
@@ -175,8 +177,9 @@ test "a list of activities" do
     user = insert(:user)
     other_user = insert(:user, %{nickname: "shp"})
     {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"})
+    object = Object.normalize(activity.data["object"])
 
-    convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"])
+    convo_id = Utils.context_to_conversation_id(object.data["context"])
 
     mocks = [
       {
@@ -277,9 +280,9 @@ test "an announce activity" do
     other_user = insert(:user, %{nickname: "shp"})
 
     {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"})
-    {:ok, announce, _object} = CommonAPI.repeat(activity.id, other_user)
+    {:ok, announce, object} = CommonAPI.repeat(activity.id, other_user)
 
-    convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"])
+    convo_id = Utils.context_to_conversation_id(object.data["context"])
 
     activity = Activity.get_by_id(activity.id)
 
@@ -357,7 +360,7 @@ test "a delete activity" do
 
   test "a peertube video" do
     {:ok, object} =
-      ActivityPub.fetch_object_from_id(
+      Pleroma.Object.Fetcher.fetch_object_from_id(
         "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
       )