Merge remote-tracking branch 'upstream/develop' into confirm-users

CHANGELOG.md

@@ -60,6 +60,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 - Fix ability to update Pleroma Chat push notifications with PUT /api/v1/push/subscription and alert type pleroma:chat_mention
 - Emoji Reaction activity filtering from blocked and muted accounts.
+- StealEmojiPolicy creates dir for emojis, if it doesn't exist.
 
 ## [2.2.1] - 2020-12-22
 

config/config.exs

@@ -63,14 +63,6 @@
   filters: [Pleroma.Upload.Filter.Dedupe],
   link_name: false,
   proxy_remote: false,
-  proxy_opts: [
-    redirect_on_failure: false,
-    max_body_length: 25 * 1_048_576,
-    http: [
-      follow_redirect: true,
-      pool: :upload
-    ]
-  ],
   filename_display_max_length: 30,
   default_description: nil
 

config/description.exs

@@ -101,74 +101,10 @@
       %{
         key: :proxy_remote,
         type: :boolean,
-        description:
+        description: """
-          "If enabled, requests to media stored using a remote uploader will be proxied instead of being redirected"
+        Proxy requests to the remote uploader.\n
-      },
+        Useful if media upload endpoint is not internet accessible.
-      %{
+        """
-        key: :proxy_opts,
-        label: "Proxy Options",
-        type: :keyword,
-        description: "Options for Pleroma.ReverseProxy",
-        suggestions: [
-          redirect_on_failure: false,
-          max_body_length: 25 * 1_048_576,
-          http: [
-            follow_redirect: true,
-            pool: :media
-          ]
-        ],
-        children: [
-          %{
-            key: :redirect_on_failure,
-            type: :boolean,
-            description:
-              "Redirects the client to the real remote URL if there's any HTTP errors. " <>
-                "Any error during body processing will not be redirected as the response is chunked."
-          },
-          %{
-            key: :max_body_length,
-            type: :integer,
-            description:
-              "Limits the content length to be approximately the " <>
-                "specified length. It is validated with the `content-length` header and also verified when proxying."
-          },
-          %{
-            key: :http,
-            label: "HTTP",
-            type: :keyword,
-            description: "HTTP options",
-            children: [
-              %{
-                key: :adapter,
-                type: :keyword,
-                description: "Adapter specific options",
-                children: [
-                  %{
-                    key: :ssl_options,
-                    type: :keyword,
-                    label: "SSL Options",
-                    description: "SSL options for HTTP adapter",
-                    children: [
-                      %{
-                        key: :versions,
-                        type: {:list, :atom},
-                        description: "List of TLS versions to use",
-                        suggestions: [:tlsv1, ":tlsv1.1", ":tlsv1.2"]
-                      }
-                    ]
-                  }
-                ]
-              },
-              %{
-                key: :proxy_url,
-                label: "Proxy URL",
-                type: [:string, :tuple],
-                description: "Proxy URL",
-                suggestions: ["127.0.0.1:8123", {:socks5, :localhost, 9050}]
-              }
-            ]
-          }
-        ]
       },
       %{
         key: :filename_display_max_length,
@@ -1550,7 +1486,7 @@
       %{
         key: :enabled,
         type: :boolean,
-        description: "Enables proxying of remote media to the instance's proxy"
+        description: "Enables proxying of remote media via the instance's proxy"
       },
       %{
         key: :base_url,
@@ -1587,80 +1523,41 @@
       },
       %{
         key: :proxy_opts,
-        label: "Proxy Options",
+        label: "Advanced MediaProxy Options",
         type: :keyword,
-        description: "Options for Pleroma.ReverseProxy",
+        description: "Internal Pleroma.ReverseProxy settings",
         suggestions: [
           redirect_on_failure: false,
           max_body_length: 25 * 1_048_576,
-          max_read_duration: 30_000,
+          max_read_duration: 30_000
-          http: [
-            follow_redirect: true,
-            pool: :media
-          ]
         ],
         children: [
           %{
             key: :redirect_on_failure,
             type: :boolean,
-            description:
+            description: """
-              "Redirects the client to the real remote URL if there's any HTTP errors. " <>
+            Redirects the client to the origin server upon encountering HTTP errors.\n
-                "Any error during body processing will not be redirected as the response is chunked."
+            Note that files larger than Max Body Length will trigger an error. (e.g., Peertube videos)\n\n
+            **WARNING:** This setting will allow larger files to be accessed, but exposes the\n
+            IP addresses of your users to the other servers, bypassing the MediaProxy.
+            """
           },
           %{
             key: :max_body_length,
             type: :integer,
-            description:
+            description: "Maximum file size allowed through the Pleroma MediaProxy cache."
-              "Limits the content length to be approximately the " <>
-                "specified length. It is validated with the `content-length` header and also verified when proxying."
           },
           %{
             key: :max_read_duration,
             type: :integer,
-            description: "Timeout (in milliseconds) of GET request to remote URI."
+            description: "Timeout (in milliseconds) of GET request to the remote URI."
-          },
-          %{
-            key: :http,
-            label: "HTTP",
-            type: :keyword,
-            description: "HTTP options",
-            children: [
-              %{
-                key: :adapter,
-                type: :keyword,
-                description: "Adapter specific options",
-                children: [
-                  %{
-                    key: :ssl_options,
-                    type: :keyword,
-                    label: "SSL Options",
-                    description: "SSL options for HTTP adapter",
-                    children: [
-                      %{
-                        key: :versions,
-                        type: {:list, :atom},
-                        description: "List of TLS version to use",
-                        suggestions: [:tlsv1, ":tlsv1.1", ":tlsv1.2"]
-                      }
-                    ]
-                  }
-                ]
-              },
-              %{
-                key: :proxy_url,
-                label: "Proxy URL",
-                type: [:string, :tuple],
-                description: "Proxy URL",
-                suggestions: ["127.0.0.1:8123", {:socks5, :localhost, 9050}]
-              }
-            ]
           }
         ]
       },
       %{
         key: :whitelist,
         type: {:list, :string},
-        description: "List of hosts with scheme to bypass the mediaproxy",
+        description: "List of hosts with scheme to bypass the MediaProxy",
         suggestions: ["http://example.com"]
       }
     ]

config/emoji.txt

@@ -1,2 +1,4 @@
 firefox, /emoji/Firefox.gif, Gif,Fun
 blank, /emoji/blank.png, Fun
+dinosaur, /emoji/dino walking.gif, Gif
+external_emoji, https://example.com/emoji.png

lib/pleroma/activity.ex

@@ -274,7 +274,7 @@ defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}
   defp get_in_reply_to_activity_from_object(_), do: nil
 
   def get_in_reply_to_activity(%Activity{} = activity) do
-    get_in_reply_to_activity_from_object(Object.normalize(activity))
+    get_in_reply_to_activity_from_object(Object.normalize(activity, fetch: false))
   end
 
   def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"])

lib/pleroma/activity/ir/topics.ex

@@ -8,7 +8,7 @@ defmodule Pleroma.Activity.Ir.Topics do
 
   def get_activity_topics(activity) do
     activity
-    |> Object.normalize()
+    |> Object.normalize(fetch: false)
     |> generate_topics(activity)
     |> List.flatten()
   end

lib/pleroma/conversation.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Conversation do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Conversation.Participation.RecipientShip
+  alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
   use Ecto.Schema
@@ -58,7 +59,7 @@ def maybe_create_recipientships(participation, activity) do
   def create_or_bump_for(activity, opts \\ []) do
     with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity),
          "Create" <- activity.data["type"],
-         object <- Pleroma.Object.normalize(activity),
+         %Object{} = object <- Object.normalize(activity, fetch: false),
          true <- object.data["type"] in ["Note", "Question"],
          ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"] do
       {:ok, conversation} = create_for_ap_id(ap_id)

lib/pleroma/emails/user_email.ex

@@ -119,7 +119,7 @@ def digest_email(user) do
       notifications
       |> Enum.filter(&(&1.activity.data["type"] == "Create"))
       |> Enum.map(fn notification ->
-        object = Pleroma.Object.normalize(notification.activity)
+        object = Pleroma.Object.normalize(notification.activity, fetch: false)
 
         if not is_nil(object) do
           object = update_in(object.data["content"], &format_links/1)
@@ -142,7 +142,7 @@ def digest_email(user) do
         if not is_nil(from) do
           %{
             data: notification,
-            object: Pleroma.Object.normalize(notification.activity),
+            object: Pleroma.Object.normalize(notification.activity, fetch: false),
             from: User.get_by_ap_id(notification.activity.actor)
           }
         end

lib/pleroma/emoji/formatter.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Emoji.Formatter do
   alias Pleroma.Emoji
   alias Pleroma.HTML
+  alias Pleroma.Web
   alias Pleroma.Web.MediaProxy
 
   def emojify(text) do
@@ -43,7 +44,7 @@ def get_emoji_map(text) when is_binary(text) do
     Emoji.get_all()
     |> Enum.filter(fn {emoji, %Emoji{}} -> String.contains?(text, ":#{emoji}:") end)
     |> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc ->
-      Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
+      Map.put(acc, name, to_string(URI.merge(Web.base_url(), file)))
     end)
   end
 

lib/pleroma/gopher/server.ex

@@ -76,7 +76,7 @@ def render_activities(activities) do
     |> Enum.map(fn activity ->
       user = User.get_cached_by_ap_id(activity.data["actor"])
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
       like_count = object.data["like_count"] || 0
       announcement_count = object.data["announcement_count"] || 0
 

lib/pleroma/html.ex

@@ -59,7 +59,7 @@ def get_cached_scrubbed_html_for_activity(
     key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
 
     @cachex.fetch!(:scrubber_cache, key, fn _key ->
-      object = Pleroma.Object.normalize(activity)
+      object = Pleroma.Object.normalize(activity, fetch: false)
       ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
     end)
   end

lib/pleroma/notification.ex

@@ -358,7 +358,7 @@ def dismiss(%{id: user_id} = _user, id) do
   def create_notifications(activity, options \\ [])
 
   def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity, options) do
-    object = Object.normalize(activity, false)
+    object = Object.normalize(activity, fetch: false)
 
     if object && object.data["type"] == "Answer" do
       {:ok, []}
@@ -625,7 +625,7 @@ def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity,
   def skip?(:filtered, %{data: %{"type" => type}}, _) when type in ["Follow", "Move"], do: false
 
   def skip?(:filtered, activity, user) do
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     cond do
       is_nil(object) ->

lib/pleroma/object.ex

@@ -108,39 +108,42 @@ defp warn_on_no_object_preloaded(ap_id) do
     Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
   end
 
-  def normalize(_, fetch_remote \\ true, options \\ [])
+  def normalize(_, options \\ [fetch: false])
 
   # If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
   # Use this whenever possible, especially when walking graphs in an O(N) loop!
-  def normalize(%Object{} = object, _, _), do: object
+  def normalize(%Object{} = object, _), do: object
-  def normalize(%Activity{object: %Object{} = object}, _, _), do: object
+  def normalize(%Activity{object: %Object{} = object}, _), do: object
 
   # A hack for fake activities
-  def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _, _) do
+  def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _) do
     %Object{id: "pleroma:fake_object_id", data: data}
   end
 
   # No preloaded object
-  def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote, _) do
+  def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, options) do
     warn_on_no_object_preloaded(ap_id)
-    normalize(ap_id, fetch_remote)
+    normalize(ap_id, options)
   end
 
   # No preloaded object
-  def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote, _) do
+  def normalize(%Activity{data: %{"object" => ap_id}}, options) do
     warn_on_no_object_preloaded(ap_id)
-    normalize(ap_id, fetch_remote)
+    normalize(ap_id, options)
   end
 
   # Old way, try fetching the object through cache.
-  def normalize(%{"id" => ap_id}, fetch_remote, _), do: normalize(ap_id, fetch_remote)
+  def normalize(%{"id" => ap_id}, options), do: normalize(ap_id, options)
-  def normalize(ap_id, false, _) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
 
-  def normalize(ap_id, true, options) when is_binary(ap_id) do
+  def normalize(ap_id, options) when is_binary(ap_id) do
-    Fetcher.fetch_object_from_id!(ap_id, options)
+    if Keyword.get(options, :fetch) do
+      Fetcher.fetch_object_from_id!(ap_id, options)
+    else
+      get_cached_by_ap_id(ap_id)
+    end
   end
 
-  def normalize(_, _, _), do: nil
+  def normalize(_, _), do: nil
 
   # Owned objects can only be accessed by their owner
   def authorize_access(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}) do
@@ -285,7 +288,7 @@ def decrease_replies_count(ap_id) do
   end
 
   def increase_vote_count(ap_id, name, actor) do
-    with %Object{} = object <- Object.normalize(ap_id),
+    with %Object{} = object <- Object.normalize(ap_id, fetch: false),
          "Question" <- object.data["type"] do
       key = if poll_is_multiple?(object), do: "anyOf", else: "oneOf"
 
@@ -326,7 +329,7 @@ def local?(%Object{data: %{"id" => id}}) do
   end
 
   def replies(object, opts \\ []) do
-    object = Object.normalize(object)
+    object = Object.normalize(object, fetch: false)
 
     query =
       Object

lib/pleroma/object/fetcher.ex

@@ -83,13 +83,13 @@ def fetch_object_from_id(id, options \\ []) do
     with {_, nil} <- {:fetch_object, Object.get_cached_by_ap_id(id)},
          {_, true} <- {:allowed_depth, Federator.allowed_thread_distance?(options[:depth])},
          {_, {:ok, data}} <- {:fetch, fetch_and_contain_remote_object_from_id(id)},
-         {_, nil} <- {:normalize, Object.normalize(data, false)},
+         {_, nil} <- {:normalize, Object.normalize(data, fetch: false)},
          params <- prepare_activity_params(data),
          {_, :ok} <- {:containment, Containment.contain_origin(id, params)},
          {_, {:ok, activity}} <-
            {:transmogrifier, Transmogrifier.handle_incoming(params, options)},
          {_, _data, %Object{} = object} <-
-           {:object, data, Object.normalize(activity, false)} do
+           {:object, data, Object.normalize(activity, fetch: false)} do
       {:ok, object}
     else
       {:allowed_depth, false} ->

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -608,11 +608,7 @@ def fetch_user_activities(user, reading_user, params \\ %{}) do
         |> Map.put(:muting_user, reading_user)
       end
 
-    pagination_type =
+    pagination_type = Map.get(params, :pagination_type) || :keyset
-      cond do
-        !Map.has_key?(params, :offset) -> :keyset
-        true -> :offset
-      end
 
     %{
       godmode: params[:godmode],

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -128,7 +128,7 @@ def activity(conn, _params) do
   end
 
   defp maybe_set_tracking_data(conn, %Activity{data: %{"type" => "Create"}} = activity) do
-    object_id = Object.normalize(activity).id
+    object_id = Object.normalize(activity, fetch: false).id
     assign(conn, :tracking_fun_data, object_id)
   end
 
@@ -434,7 +434,7 @@ defp handle_user_activity(
   end
 
   defp handle_user_activity(%User{} = user, %{"type" => "Delete"} = params) do
-    with %Object{} = object <- Object.normalize(params["object"]),
+    with %Object{} = object <- Object.normalize(params["object"], fetch: false),
          true <- user.is_moderator || user.ap_id == object.data["actor"],
          {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]),
          {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do
@@ -445,7 +445,7 @@ defp handle_user_activity(%User{} = user, %{"type" => "Delete"} = params) do
   end
 
   defp handle_user_activity(%User{} = user, %{"type" => "Like"} = params) do
-    with %Object{} = object <- Object.normalize(params["object"]),
+    with %Object{} = object <- Object.normalize(params["object"], fetch: false),
          {_, {:ok, like_object, meta}} <- {:build_object, Builder.like(user, object)},
          {_, {:ok, %Activity{} = activity, _meta}} <-
            {:common_pipeline,

lib/pleroma/web/activity_pub/builder.ex

@@ -80,7 +80,7 @@ def undo(actor, object) do
 
   @spec delete(User.t(), String.t()) :: {:ok, map(), keyword()}
   def delete(actor, object_id) do
-    object = Object.normalize(object_id, false)
+    object = Object.normalize(object_id, fetch: false)
 
     user = !object && User.get_cached_by_ap_id(object_id)
 

lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex

@@ -31,7 +31,7 @@ def filter(%{"type" => "Create", "object" => child_object} = object)
       when is_map(child_object) do
     child =
       child_object["inReplyTo"]
-      |> Object.normalize(child_object["inReplyTo"])
+      |> Object.normalize(fetch: false)
       |> filter_by_summary(child_object)
 
     object = Map.put(object, "object", child)

lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex

@@ -10,73 +10,75 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
   @moduledoc "Detect new emojis by their shortcode and steals them"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
-  defp remote_host?(host), do: host != Config.get([Pleroma.Web.Endpoint, :url, :host])
-
   defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], [])
 
-  defp steal_emoji({shortcode, url}) do
+  defp steal_emoji({shortcode, url}, emoji_dir_path) do
     url = Pleroma.Web.MediaProxy.url(url)
-    {:ok, response} = Pleroma.HTTP.get(url)
-    size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000)
 
-    if byte_size(response.body) <= size_limit do
+    with {:ok, %{status: status} = response} when status in 200..299 <- Pleroma.HTTP.get(url) do
-      emoji_dir_path =
+      size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000)
-        Config.get(
+
-          [:mrf_steal_emoji, :path],
+      if byte_size(response.body) <= size_limit do
-          Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
+        extension =
+          url
+          |> URI.parse()
+          |> Map.get(:path)
+          |> Path.basename()
+          |> Path.extname()
+
+        file_path = Path.join(emoji_dir_path, shortcode <> (extension || ".png"))
+
+        case File.write(file_path, response.body) do
+          :ok ->
+            shortcode
+
+          e ->
+            Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}")
+            nil
+        end
+      else
+        Logger.debug(
+          "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{
+            size_limit
+          } B)"
         )
 
-      extension =
+        nil
-        url
-        |> URI.parse()
-        |> Map.get(:path)
-        |> Path.basename()
-        |> Path.extname()
-
-      file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")])
-
-      try do
-        :ok = File.write(file_path, response.body)
-
-        shortcode
-      rescue
-        e ->
-          Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}")
-          nil
       end
     else
-      Logger.debug(
+      e ->
-        "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{
+        Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}")
-          size_limit
+        nil
-        } B)"
-      )
-
-      nil
     end
-  rescue
-    e ->
-      Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}")
-      nil
   end
 
   @impl true
   def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do
     host = URI.parse(actor).host
 
-    if remote_host?(host) and accept_host?(host) do
+    if host != Pleroma.Web.Endpoint.host() and accept_host?(host) do
       installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
 
+      emoji_dir_path =
+        Config.get(
+          [:mrf_steal_emoji, :path],
+          Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
+        )
+
+      File.mkdir_p(emoji_dir_path)
+
       new_emojis =
         foreign_emojis
-        |> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end)
+        |> Enum.reject(fn {shortcode, _url} -> shortcode in installed_emoji end)
         |> Enum.filter(fn {shortcode, _url} ->
           reject_emoji? =
-            Config.get([:mrf_steal_emoji, :rejected_shortcodes], [])
+            [:mrf_steal_emoji, :rejected_shortcodes]
+            |> Config.get([])
             |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end)
 
           !reject_emoji?
         end)
-        |> Enum.map(&steal_emoji(&1))
+        |> Enum.map(&steal_emoji(&1, emoji_dir_path))
         |> Enum.filter(& &1)
 
       if !Enum.empty?(new_emojis) do

lib/pleroma/web/activity_pub/object_validator.ex

@@ -288,7 +288,7 @@ def fetch_actor(object) do
 
   def fetch_actor_and_object(object) do
     fetch_actor(object)
-    Object.normalize(object["object"], true)
+    Object.normalize(object["object"], fetch: true)
     :ok
   end
 end

lib/pleroma/web/activity_pub/publisher.ex

@@ -129,7 +129,7 @@ defp recipients(actor, activity) do
 
     fetchers =
       with %Activity{data: %{"type" => "Delete"}} <- activity,
-           %Object{id: object_id} <- Object.normalize(activity),
+           %Object{id: object_id} <- Object.normalize(activity, fetch: false),
            fetchers <- User.get_delivered_users_by_object_id(object_id),
            _ <- Delivery.delete_all_by_object_id(object_id) do
         fetchers

lib/pleroma/web/activity_pub/side_effects.ex

@@ -270,7 +270,7 @@ def handle(%{data: %{"type" => "EmojiReact"}} = object, meta) do
   @impl true
   def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, meta) do
     deleted_object =
-      Object.normalize(deleted_object, false) ||
+      Object.normalize(deleted_object, fetch: false) ||
         User.get_cached_by_ap_id(deleted_object)
 
     result =

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -653,7 +653,9 @@ def handle_incoming(_, _), do: :error
 
   @spec get_obj_helper(String.t(), Keyword.t()) :: {:ok, Object.t()} | nil
   def get_obj_helper(id, options \\ []) do
-    case Object.normalize(id, true, options) do
+    options = Keyword.put(options, :fetch, true)
+
+    case Object.normalize(id, options) do
       %Object{} = object -> {:ok, object}
       _ -> nil
     end
@@ -672,7 +674,7 @@ def get_embedded_obj_helper(%{"attributedTo" => attributed_to, "id" => object_id
              "actor" => attributed_to,
              "object" => data
            }) do
-      {:ok, Object.normalize(activity)}
+      {:ok, Object.normalize(activity, fetch: false)}
     else
       _ -> get_obj_helper(object_id)
     end
@@ -763,7 +765,7 @@ def prepare_outgoing(%{"type" => activity_type, "object" => object_id} = data)
       when activity_type in ["Create", "Listen"] do
     object =
       object_id
-      |> Object.normalize()
+      |> Object.normalize(fetch: false)
       |> Map.get(:data)
       |> prepare_object
 
@@ -779,7 +781,7 @@ def prepare_outgoing(%{"type" => activity_type, "object" => object_id} = data)
   def prepare_outgoing(%{"type" => "Announce", "actor" => ap_id, "object" => object_id} = data) do
     object =
       object_id
-      |> Object.normalize()
+      |> Object.normalize(fetch: false)
 
     data =
       if Visibility.is_private?(object) && object.data["actor"] == ap_id do
@@ -919,7 +921,7 @@ def add_emoji_tags(object), do: object
 
   defp build_emoji_tag({name, url}) do
     %{
-      "icon" => %{"url" => url, "type" => "Image"},
+      "icon" => %{"url" => "#{URI.encode(url)}", "type" => "Image"},
       "name" => ":" <> name <> ":",
       "type" => "Emoji",
       "updated" => "1970-01-01T00:00:00Z",

lib/pleroma/web/activity_pub/views/object_view.ex

@@ -18,7 +18,7 @@ def render("object.json", %{object: %Object{} = object}) do
   def render("object.json", %{object: %Activity{data: %{"type" => activity_type}} = activity})
       when activity_type in ["Create", "Listen"] do
     base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     additional =
       Transmogrifier.prepare_object(activity.data)
@@ -29,7 +29,7 @@ def render("object.json", %{object: %Activity{data: %{"type" => activity_type}}
 
   def render("object.json", %{object: %Activity{} = activity}) do
     base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     additional =
       Transmogrifier.prepare_object(activity.data)

lib/pleroma/web/admin_api/controllers/admin_api_controller.ex

@@ -110,7 +110,8 @@ def list_user_statuses(%{assigns: %{user: admin}} = conn, %{"nickname" => nickna
           limit: page_size,
           offset: (page - 1) * page_size,
           godmode: godmode,
-          exclude_reblogs: not with_reblogs
+          exclude_reblogs: not with_reblogs,
+          pagination_type: :offset
         })
 
       conn

lib/pleroma/web/admin_api/views/account_view.ex

@@ -69,6 +69,7 @@ def render("show.json", %{user: user}) do
 
     %{
       "id" => user.id,
+      "email" => user.email,
       "avatar" => avatar,
       "nickname" => user.nickname,
       "display_name" => display_name,

lib/pleroma/web/common_api.ex

@@ -142,7 +142,7 @@ def delete(activity_id, user) do
     with {_, %Activity{data: %{"object" => _, "type" => "Create"}} = activity} <-
            {:find_activity, Activity.get_by_id(activity_id)},
          {_, %Object{} = object, _} <-
-           {:find_object, Object.normalize(activity, false), activity},
+           {:find_object, Object.normalize(activity, fetch: false), activity},
          true <- User.superuser?(user) || user.ap_id == object.data["actor"],
          {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]),
          {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do
@@ -173,7 +173,7 @@ def delete(activity_id, user) do
 
   def repeat(id, user, params \\ %{}) do
     with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id),
-         object = %Object{} <- Object.normalize(activity, false),
+         object = %Object{} <- Object.normalize(activity, fetch: false),
          {_, nil} <- {:existing_announce, Utils.get_existing_announce(user.ap_id, object)},
          public = public_announce?(object, params),
          {:ok, announce, _} <- Builder.announce(user, object, public: public),
@@ -191,7 +191,7 @@ def repeat(id, user, params \\ %{}) do
   def unrepeat(id, user) do
     with {_, %Activity{data: %{"type" => "Create"}} = activity} <-
            {:find_activity, Activity.get_by_id(id)},
-         %Object{} = note <- Object.normalize(activity, false),
+         %Object{} = note <- Object.normalize(activity, fetch: false),
          %Activity{} = announce <- Utils.get_existing_announce(user.ap_id, note),
          {:ok, undo, _} <- Builder.undo(user, announce),
          {:ok, activity, _} <- Pipeline.common_pipeline(undo, local: true) do
@@ -253,7 +253,7 @@ def favorite_helper(user, id) do
   def unfavorite(id, user) do
     with {_, %Activity{data: %{"type" => "Create"}} = activity} <-
            {:find_activity, Activity.get_by_id(id)},
-         %Object{} = note <- Object.normalize(activity, false),
+         %Object{} = note <- Object.normalize(activity, fetch: false),
          %Activity{} = like <- Utils.get_existing_like(user.ap_id, note),
          {:ok, undo, _} <- Builder.undo(user, like),
          {:ok, activity, _} <- Pipeline.common_pipeline(undo, local: true) do
@@ -266,7 +266,7 @@ def unfavorite(id, user) do
 
   def react_with_emoji(id, user, emoji) do
     with %Activity{} = activity <- Activity.get_by_id(id),
-         object <- Object.normalize(activity),
+         object <- Object.normalize(activity, fetch: false),
          {:ok, emoji_react, _} <- Builder.emoji_react(user, object, emoji),
          {:ok, activity, _} <- Pipeline.common_pipeline(emoji_react, local: true) do
       {:ok, activity}
@@ -377,7 +377,7 @@ def get_visibility(_, in_reply_to, _), do: {"public", get_replied_to_visibility(
   def get_replied_to_visibility(nil), do: nil
 
   def get_replied_to_visibility(activity) do
-    with %Object{} = object <- Object.normalize(activity) do
+    with %Object{} = object <- Object.normalize(activity, fetch: false) do
       Visibility.get_visibility(object)
     end
   end

lib/pleroma/web/common_api/utils.ex

@@ -319,7 +319,7 @@ def make_note_data(%ActivityDraft{} = draft) do
   defp add_in_reply_to(object, nil), do: object
 
   defp add_in_reply_to(object, in_reply_to) do
-    with %Object{} = in_reply_to_object <- Object.normalize(in_reply_to) do
+    with %Object{} = in_reply_to_object <- Object.normalize(in_reply_to, fetch: false) do
       Map.put(object, "inReplyTo", in_reply_to_object.data["id"])
     else
       _ -> object
@@ -399,7 +399,7 @@ def maybe_notify_mentioned_recipients(
         %Activity{data: %{"to" => _to, "type" => type} = data} = activity
       )
       when type == "Create" do
-    object = Object.normalize(activity, false)
+    object = Object.normalize(activity, fetch: false)
 
     object_data =
       cond do

lib/pleroma/web/embed_controller.ex

@@ -31,7 +31,7 @@ def show(conn, %{"id" => id}) do
   end
 
   defp get_counts(%Activity{} = activity) do
-    %Object{data: data} = Object.normalize(activity)
+    %Object{data: data} = Object.normalize(activity, fetch: false)
 
     %{
       likes: Map.get(data, "like_count", 0),

lib/pleroma/web/feed/feed_view.ex

@@ -23,7 +23,7 @@ def pub_date(date) when is_binary(date) do
   def pub_date(%DateTime{} = date), do: Timex.format!(date, "{RFC822}")
 
   def prepare_activity(activity, opts \\ []) do
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     actor =
       if opts[:actor] do

lib/pleroma/web/mastodon_api/controllers/status_controller.ex

@@ -318,7 +318,7 @@ def favourited_by(%{assigns: %{user: user}} = conn, %{id: id}) do
     with true <- Pleroma.Config.get([:instance, :show_reactions]),
          %Activity{} = activity <- Activity.get_by_id_with_object(id),
          {:visible, true} <- {:visible, Visibility.visible_for_user?(activity, user)},
-         %Object{data: %{"likes" => likes}} <- Object.normalize(activity) do
+         %Object{data: %{"likes" => likes}} <- Object.normalize(activity, fetch: false) do
       users =
         User
         |> Ecto.Query.where([u], u.ap_id in ^likes)
@@ -339,7 +339,7 @@ def reblogged_by(%{assigns: %{user: user}} = conn, %{id: id}) do
     with %Activity{} = activity <- Activity.get_by_id_with_object(id),
          {:visible, true} <- {:visible, Visibility.visible_for_user?(activity, user)},
          %Object{data: %{"announcements" => announces, "id" => ap_id}} <-
-           Object.normalize(activity) do
+           Object.normalize(activity, fetch: false) do
       announces =
         "Announce"
         |> Activity.Queries.by_type()

lib/pleroma/web/mastodon_api/views/notification_view.ex

@@ -139,7 +139,7 @@ defp put_emoji(response, activity) do
   end
 
   defp put_chat_message(response, activity, reading_user, opts) do
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
     author = User.get_cached_by_ap_id(object.data["actor"])
     chat = Pleroma.Chat.get(reading_user.id, author.ap_id)
     cm_ref = MessageReference.for_chat_and_object(chat, object)

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -41,7 +41,7 @@ defp get_replied_to_activities(activities) do
     activities
     |> Enum.map(fn
       %{data: %{"type" => "Create"}} = activity ->
-        object = Object.normalize(activity)
+        object = Object.normalize(activity, fetch: false)
         object && object.data["inReplyTo"] != "" && object.data["inReplyTo"]
 
       _ ->
@@ -51,7 +51,7 @@ defp get_replied_to_activities(activities) do
     |> Activity.create_by_object_ap_id_with_object()
     |> Repo.all()
     |> Enum.reduce(%{}, fn activity, acc ->
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
       if object, do: Map.put(acc, object.data["id"], activity), else: acc
     end)
   end
@@ -65,7 +65,7 @@ defp get_context_id(%{data: %{"context" => context}}) when is_binary(context),
   defp get_context_id(_), do: nil
 
   defp reblogged?(activity, user) do
-    object = Object.normalize(activity) || %{}
+    object = Object.normalize(activity, fetch: false) || %{}
     present?(user && user.ap_id in (object.data["announcements"] || []))
   end
 
@@ -84,7 +84,7 @@ def render("index.json", opts) do
     parent_activities =
       activities
       |> Enum.filter(&(&1.data["type"] == "Announce" && &1.data["object"]))
-      |> Enum.map(&Object.normalize(&1).data["id"])
+      |> Enum.map(&Object.normalize(&1, fetch: false).data["id"])
       |> Activity.create_by_object_ap_id()
       |> Activity.with_preloaded_object(:left)
       |> Activity.with_preloaded_bookmark(reading_user)
@@ -124,7 +124,7 @@ def render(
       ) do
     user = CommonAPI.get_user(activity.data["actor"])
     created_at = Utils.to_masto_date(activity.data["published"])
-    activity_object = Object.normalize(activity)
+    activity_object = Object.normalize(activity, fetch: false)
 
     reblogged_parent_activity =
       if opts[:parent_activities] do
@@ -193,7 +193,7 @@ def render(
   end
 
   def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     user = CommonAPI.get_user(activity.data["actor"])
     user_follower_address = user.follower_address
@@ -451,7 +451,7 @@ def render("context.json", %{activity: activity, activities: activities, user: u
   end
 
   def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     with nil <- replied_to_activities[object.data["inReplyTo"]] do
       # If user didn't participate in the thread
@@ -460,7 +460,7 @@ def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do
   end
 
   def get_reply_to(%{data: %{"object" => _object}} = activity, _) do
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     if object.data["inReplyTo"] && object.data["inReplyTo"] != "" do
       Activity.get_create_by_object_ap_id(object.data["inReplyTo"])

lib/pleroma/web/o_status/o_status_controller.ex

@@ -74,14 +74,14 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do
       cond do
         format in ["json", "activity+json"] ->
           if activity.local do
-            %{data: %{"id" => redirect_url}} = Object.normalize(activity)
+            %{data: %{"id" => redirect_url}} = Object.normalize(activity, fetch: false)
             redirect(conn, external: redirect_url)
           else
             {:error, :not_found}
           end
 
         activity.data["type"] == "Create" ->
-          %Object{} = object = Object.normalize(activity)
+          %Object{} = object = Object.normalize(activity, fetch: false)
 
           RedirectController.redirector_with_meta(
             conn,
@@ -112,7 +112,7 @@ def notice_player(conn, %{"id" => id}) do
     with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id_with_object(id),
          true <- Visibility.is_public?(activity),
          {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)},
-         %Object{} = object <- Object.normalize(activity),
+         %Object{} = object <- Object.normalize(activity, fetch: false),
          %{data: %{"attachment" => [%{"url" => [url | _]} | _]}} <- object,
          true <- String.starts_with?(url["mediaType"], ["audio", "video"]) do
       conn

lib/pleroma/web/pleroma_api/controllers/chat_controller.ex

@@ -82,7 +82,7 @@ def post_chat_message(
              media_id: params[:media_id],
              idempotency_key: idempotency_key(conn)
            ),
-         message <- Object.normalize(activity, false),
+         message <- Object.normalize(activity, fetch: false),
          cm_ref <- MessageReference.for_chat_and_object(chat, message) do
       conn
       |> put_view(MessageReferenceView)

lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex

@@ -29,7 +29,7 @@ def index(%{assigns: %{user: user}} = conn, %{id: activity_id} = params) do
     with true <- Pleroma.Config.get([:instance, :show_reactions]),
          %Activity{} = activity <- Activity.get_by_id_with_object(activity_id),
          %Object{data: %{"reactions" => reactions}} when is_list(reactions) <-
-           Object.normalize(activity) do
+           Object.normalize(activity, fetch: false) do
       reactions =
         reactions
         |> filter(params)

lib/pleroma/web/pleroma_api/views/backup_view.ex

@@ -10,6 +10,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do
 
   def render("show.json", %{backup: %Backup{} = backup}) do
     %{
+      id: backup.id,
       content_type: backup.content_type,
       url: download_url(backup),
       file_size: backup.file_size,

lib/pleroma/web/pleroma_api/views/scrobble_view.ex

@@ -15,7 +15,7 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do
   alias Pleroma.Web.MastodonAPI.AccountView
 
   def render("show.json", %{activity: %Activity{data: %{"type" => "Listen"}} = activity} = opts) do
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     user = CommonAPI.get_user(activity.data["actor"])
     created_at = Utils.to_masto_date(activity.data["published"])

lib/pleroma/web/plugs/uploaded_media.ex

@@ -87,8 +87,15 @@ defp get_media(conn, {:static_dir, directory}, _, opts) do
   end
 
   defp get_media(conn, {:url, url}, true, _) do
+    proxy_opts = [
+      http: [
+        follow_redirect: true,
+        pool: :upload
+      ]
+    ]
+
     conn
-    |> Pleroma.ReverseProxy.call(url, Pleroma.Config.get([Pleroma.Upload, :proxy_opts], []))
+    |> Pleroma.ReverseProxy.call(url, proxy_opts)
   end
 
   defp get_media(conn, {:url, url}, _, _) do

lib/pleroma/web/push/impl.ex

@@ -32,7 +32,7 @@ def perform(
     mastodon_type = notification.type
     gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
     avatar_url = User.avatar_url(actor)
-    object = Object.normalize(activity, false)
+    object = Object.normalize(activity, fetch: false)
     user = User.get_cached_by_id(user_id)
     direct_conversation_id = Activity.direct_conversation_id(activity, user)
 

lib/pleroma/web/rich_media/helpers.ex

@@ -69,7 +69,7 @@ def fetch_data_for_object(object) do
 
   def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
     with true <- Config.get([:rich_media, :enabled]),
-         %Object{} = object <- Object.normalize(activity) do
+         %Object{} = object <- Object.normalize(activity, fetch: false) do
       fetch_data_for_object(object)
     else
       _ -> %{}

lib/pleroma/web/static_fe/static_fe_controller.ex

@@ -122,7 +122,7 @@ defp not_found(conn, message) do
   end
 
   defp get_counts(%Activity{} = activity) do
-    %Object{data: data} = Object.normalize(activity)
+    %Object{data: data} = Object.normalize(activity, fetch: false)
 
     %{
       likes: data["like_count"] || 0,

lib/pleroma/web/streamer.ex

@@ -151,7 +151,7 @@ def filtered_by_user?(%User{} = user, %Activity{} = item, streamed_type) do
     recipients = MapSet.new(item.recipients)
     domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.domain_blocks)
 
-    with parent <- Object.normalize(item) || item,
+    with parent <- Object.normalize(item, fetch: false) || item,
          true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(item.actor not in &1)),
          true <- item.data["type"] != "Announce" || item.actor not in reblog_muted_ap_ids,
          true <-

test/mix/tasks/pleroma/user_test.exs

@@ -114,7 +114,7 @@ test "a remote user's create activity is deleted when the object has been pruned
 
       {:ok, post} = CommonAPI.post(user, %{status: "uguu"})
       {:ok, post2} = CommonAPI.post(user2, %{status: "test"})
-      obj = Object.normalize(post2)
+      obj = Object.normalize(post2, fetch: false)
 
       {:ok, like_object, meta} = Pleroma.Web.ActivityPub.Builder.like(user, obj)
 
@@ -130,7 +130,7 @@ test "a remote user's create activity is deleted when the object has been pruned
 
       clear_config([:instance, :federating], true)
 
-      object = Object.normalize(post)
+      object = Object.normalize(post, fetch: false)
       Object.prune(object)
 
       with_mock Pleroma.Web.Federator,

test/pleroma/activity_test.exs

@@ -25,7 +25,7 @@ test "returns an activity by it's AP id" do
 
   test "returns activities by it's objects AP ids" do
     activity = insert(:note_activity)
-    object_data = Object.normalize(activity).data
+    object_data = Object.normalize(activity, fetch: false).data
 
     [found_activity] = Activity.get_all_create_by_object_ap_id(object_data["id"])
 
@@ -34,7 +34,7 @@ test "returns activities by it's objects AP ids" do
 
   test "returns the activity that created an object" do
     activity = insert(:note_activity)
-    object_data = Object.normalize(activity).data
+    object_data = Object.normalize(activity, fetch: false).data
 
     found_activity = Activity.get_create_by_object_ap_id(object_data["id"])
 

test/pleroma/bbs/handler_test.exs

@@ -54,7 +54,7 @@ test "posting" do
       )
 
     assert activity.actor == user.ap_id
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
     assert object.data["content"] == "this is a test post"
   end
 
@@ -63,7 +63,7 @@ test "replying" do
     another_user = insert(:user)
 
     {:ok, activity} = CommonAPI.post(another_user, %{status: "this is a test post"})
-    activity_object = Object.normalize(activity)
+    activity_object = Object.normalize(activity, fetch: false)
 
     output =
       capture_io(fn ->
@@ -82,7 +82,7 @@ test "replying" do
 
     assert reply.actor == user.ap_id
 
-    reply_object_data = Object.normalize(reply).data
+    reply_object_data = Object.normalize(reply, fetch: false).data
     assert reply_object_data["content"] == "this is a reply"
     assert reply_object_data["inReplyTo"] == activity_object.data["id"]
   end

test/pleroma/conversation/participation_test.exs

@@ -175,8 +175,8 @@ test "gets all the participations for a user, ordered by updated at descending"
 
     assert [participation_one, participation_two] = Participation.for_user(user)
 
-    object2 = Pleroma.Object.normalize(activity_two)
+    object2 = Pleroma.Object.normalize(activity_two, fetch: false)
-    object3 = Pleroma.Object.normalize(activity_three)
+    object3 = Pleroma.Object.normalize(activity_three, fetch: false)
 
     user = Repo.get(Pleroma.User, user.id)
 

test/pleroma/conversation_test.exs

@@ -48,7 +48,7 @@ test "public posts don't create conversations" do
     user = insert(:user)
     {:ok, activity} = CommonAPI.post(user, %{status: "Hey"})
 
-    object = Pleroma.Object.normalize(activity)
+    object = Pleroma.Object.normalize(activity, fetch: false)
     context = object.data["context"]
 
     conversation = Conversation.get_for_ap_id(context)
@@ -64,7 +64,7 @@ test "it creates or updates a conversation and participations for a given DM" do
     {:ok, activity} =
       CommonAPI.post(har, %{status: "Hey @#{jafnhar.nickname}", visibility: "direct"})
 
-    object = Pleroma.Object.normalize(activity)
+    object = Pleroma.Object.normalize(activity, fetch: false)
     context = object.data["context"]
 
     conversation =
@@ -86,7 +86,7 @@ test "it creates or updates a conversation and participations for a given DM" do
         in_reply_to_status_id: activity.id
       })
 
-    object = Pleroma.Object.normalize(activity)
+    object = Pleroma.Object.normalize(activity, fetch: false)
     context = object.data["context"]
 
     conversation_two =
@@ -110,7 +110,7 @@ test "it creates or updates a conversation and participations for a given DM" do
         in_reply_to_status_id: activity.id
       })
 
-    object = Pleroma.Object.normalize(activity)
+    object = Pleroma.Object.normalize(activity, fetch: false)
     context = object.data["context"]
 
     conversation_three =

test/pleroma/html_test.exs

@@ -175,7 +175,7 @@ test "extracts the url" do
             "I think I just found the best github repo https://github.com/komeiji-satori/Dress"
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
       {:ok, url} = HTML.extract_first_external_url_from_object(object)
       assert url == "https://github.com/komeiji-satori/Dress"
     end
@@ -190,7 +190,7 @@ test "skips mentions" do
             "@#{other_user.nickname} install misskey! https://github.com/syuilo/misskey/blob/develop/docs/setup.en.md"
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
       {:ok, url} = HTML.extract_first_external_url_from_object(object)
 
       assert url == "https://github.com/syuilo/misskey/blob/develop/docs/setup.en.md"
@@ -206,7 +206,7 @@ test "skips hashtags" do
           status: "#cofe https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72255140"
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
       {:ok, url} = HTML.extract_first_external_url_from_object(object)
 
       assert url == "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72255140"
@@ -222,7 +222,7 @@ test "skips microformats hashtags" do
           content_type: "text/html"
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
       {:ok, url} = HTML.extract_first_external_url_from_object(object)
 
       assert url == "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72255140"
@@ -233,7 +233,7 @@ test "does not crash when there is an HTML entity in a link" do
 
       {:ok, activity} = CommonAPI.post(user, %{status: "\"http://cofe.com/?boomer=ok&foo=bar\""})
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert {:ok, nil} = HTML.extract_first_external_url_from_object(object)
     end
@@ -247,7 +247,7 @@ test "skips attachment links" do
             "<a href=\"https://pleroma.gov/media/d24caa3a498e21e0298377a9ca0149a4f4f8b767178aacf837542282e2d94fb1.png?name=image.png\" class=\"attachment\">image.png</a>"
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert {:ok, nil} = HTML.extract_first_external_url_from_object(object)
     end

test/pleroma/object_test.exs

@@ -256,23 +256,22 @@ test "With custom base_url" do
   end
 
   describe "normalizer" do
-    test "fetches unknown objects by default" do
+    @url "http://mastodon.example.org/@admin/99541947525187367"
-      %Object{} =
+    test "does not fetch unknown objects by default" do
-        object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367")
+      assert nil == Object.normalize(@url)
-
-      assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367"
     end
 
-    test "fetches unknown objects when fetch_remote is explicitly true" do
+    test "fetches unknown objects when fetch is explicitly true" do
-      %Object{} =
+      %Object{} = object = Object.normalize(@url, fetch: true)
-        object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367", true)
 
-      assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367"
+      assert object.data["url"] == @url
     end
 
-    test "does not fetch unknown objects when fetch_remote is false" do
+    test "does not fetch unknown objects when fetch is false" do
       assert is_nil(
-               Object.normalize("http://mastodon.example.org/@admin/99541947525187367", false)
+               Object.normalize(@url,
+                 fetch: false
+               )
              )
     end
   end
@@ -310,7 +309,10 @@ test "refetches if the time since the last refetch is greater than the interval"
       mock_modified: mock_modified
     } do
       %Object{} =
-        object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
+        object =
+        Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d",
+          fetch: true
+        )
 
       Object.set_cache(object)
 
@@ -332,7 +334,10 @@ test "refetches if the time since the last refetch is greater than the interval"
 
     test "returns the old object if refetch fails", %{mock_modified: mock_modified} do
       %Object{} =
-        object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
+        object =
+        Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d",
+          fetch: true
+        )
 
       Object.set_cache(object)
 
@@ -355,7 +360,10 @@ test "does not refetch if the time since the last refetch is greater than the in
       mock_modified: mock_modified
     } do
       %Object{} =
-        object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
+        object =
+        Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d",
+          fetch: true
+        )
 
       Object.set_cache(object)
 
@@ -377,7 +385,10 @@ test "does not refetch if the time since the last refetch is greater than the in
 
     test "preserves internal fields on refetch", %{mock_modified: mock_modified} do
       %Object{} =
-        object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
+        object =
+        Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d",
+          fetch: true
+        )
 
       Object.set_cache(object)
 

test/pleroma/user/welcome_chat_message_test.exs

@@ -28,8 +28,10 @@ test "send a chat welcome message" do
       {:ok, %Pleroma.Activity{} = activity} = WelcomeChatMessage.post_message(user)
 
       assert user.ap_id in activity.recipients
-      assert Pleroma.Object.normalize(activity).data["type"] == "ChatMessage"
+      assert Pleroma.Object.normalize(activity, fetch: false).data["type"] == "ChatMessage"
-      assert Pleroma.Object.normalize(activity).data["content"] == "Hello, welcome to Blob/Cat!"
+
+      assert Pleroma.Object.normalize(activity, fetch: false).data["content"] ==
+               "Hello, welcome to Blob/Cat!"
     end
   end
 end

test/pleroma/user/welcome_message_test.exs

@@ -28,7 +28,9 @@ test "send a direct welcome message" do
       {:ok, %Pleroma.Activity{} = activity} = WelcomeMessage.post_message(user)
       assert user.ap_id in activity.recipients
       assert activity.data["directMessage"] == true
-      assert Pleroma.Object.normalize(activity).data["content"] =~ "Hello. Welcome to Pleroma"
+
+      assert Pleroma.Object.normalize(activity, fetch: false).data["content"] =~
+               "Hello. Welcome to Pleroma"
     end
   end
 end

test/pleroma/user_test.exs

@@ -438,7 +438,7 @@ test "it sends a welcome message if it is set" do
 
       activity = Repo.one(Pleroma.Activity)
       assert registered_user.ap_id in activity.recipients
-      assert Object.normalize(activity).data["content"] =~ "direct message"
+      assert Object.normalize(activity, fetch: false).data["content"] =~ "direct message"
       assert activity.actor == welcome_user.ap_id
     end
 
@@ -454,7 +454,7 @@ test "it sends a welcome chat message if it is set" do
 
       activity = Repo.one(Pleroma.Activity)
       assert registered_user.ap_id in activity.recipients
-      assert Object.normalize(activity).data["content"] =~ "chat message"
+      assert Object.normalize(activity, fetch: false).data["content"] =~ "chat message"
       assert activity.actor == welcome_user.ap_id
     end
 
@@ -493,7 +493,7 @@ test "it sends a welcome chat message when Simple policy applied to local instan
 
       activity = Repo.one(Pleroma.Activity)
       assert registered_user.ap_id in activity.recipients
-      assert Object.normalize(activity).data["content"] =~ "chat message"
+      assert Object.normalize(activity, fetch: false).data["content"] =~ "chat message"
       assert activity.actor == welcome_user.ap_id
     end
 

test/pleroma/web/activity_pub/activity_pub_controller_test.exs

@@ -219,7 +219,7 @@ test "it doesn't return a local-only object", %{conn: conn} do
 
       assert Pleroma.Web.ActivityPub.Visibility.is_local_public?(post)
 
-      object = Object.normalize(post, false)
+      object = Object.normalize(post, fetch: false)
       uuid = String.split(object.data["id"], "/") |> List.last()
 
       conn =
@@ -712,7 +712,7 @@ test "it rejects reads from other users", %{conn: conn} do
 
     test "it returns a note activity in a collection", %{conn: conn} do
       note_activity = insert(:direct_note_activity)
-      note_object = Object.normalize(note_activity)
+      note_object = Object.normalize(note_activity, fetch: false)
       user = User.get_cached_by_ap_id(hd(note_activity.data["to"]))
 
       conn =
@@ -999,7 +999,7 @@ test "it returns 200 even if there're no activities", %{conn: conn} do
 
     test "it returns a note activity in a collection", %{conn: conn} do
       note_activity = insert(:note_activity)
-      note_object = Object.normalize(note_activity)
+      note_object = Object.normalize(note_activity, fetch: false)
       user = User.get_cached_by_ap_id(note_activity.data["actor"])
 
       conn =
@@ -1073,7 +1073,7 @@ test "it inserts an incoming create activity into the database", %{
 
       assert Activity.get_by_ap_id(result["id"])
       assert result["object"]
-      assert %Object{data: object} = Object.normalize(result["object"])
+      assert %Object{data: object} = Object.normalize(result["object"], fetch: false)
       assert object["content"] == activity["object"]["content"]
     end
 
@@ -1109,7 +1109,7 @@ test "it inserts an incoming sensitive activity into the database", %{
 
       assert Activity.get_by_ap_id(response["id"])
       assert response["object"]
-      assert %Object{data: response_object} = Object.normalize(response["object"])
+      assert %Object{data: response_object} = Object.normalize(response["object"], fetch: false)
       assert response_object["sensitive"] == true
       assert response_object["content"] == activity["object"]["content"]
 
@@ -1137,7 +1137,7 @@ test "it rejects an incoming activity with bogus type", %{conn: conn, activity:
 
     test "it erects a tombstone when receiving a delete activity", %{conn: conn} do
       note_activity = insert(:note_activity)
-      note_object = Object.normalize(note_activity)
+      note_object = Object.normalize(note_activity, fetch: false)
       user = User.get_cached_by_ap_id(note_activity.data["actor"])
 
       data = %{
@@ -1162,7 +1162,7 @@ test "it erects a tombstone when receiving a delete activity", %{conn: conn} do
 
     test "it rejects delete activity of object from other actor", %{conn: conn} do
       note_activity = insert(:note_activity)
-      note_object = Object.normalize(note_activity)
+      note_object = Object.normalize(note_activity, fetch: false)
       user = insert(:user)
 
       data = %{
@@ -1183,7 +1183,7 @@ test "it rejects delete activity of object from other actor", %{conn: conn} do
 
     test "it increases like count when receiving a like action", %{conn: conn} do
       note_activity = insert(:note_activity)
-      note_object = Object.normalize(note_activity)
+      note_object = Object.normalize(note_activity, fetch: false)
       user = User.get_cached_by_ap_id(note_activity.data["actor"])
 
       data = %{
@@ -1240,7 +1240,7 @@ test "it doesn't spreads faulty attributedTo or actor fields", %{
       assert cirno_outbox["attributedTo"] == nil
       assert cirno_outbox["actor"] == cirno.ap_id
 
-      assert cirno_object = Object.normalize(cirno_outbox["object"])
+      assert cirno_object = Object.normalize(cirno_outbox["object"], fetch: false)
       assert cirno_object.data["actor"] == cirno.ap_id
       assert cirno_object.data["attributedTo"] == cirno.ap_id
     end
@@ -1503,7 +1503,7 @@ test "does not require authentication", %{conn: conn} do
     test "it tracks a signed object fetch", %{conn: conn} do
       user = insert(:user, local: false)
       activity = insert(:note_activity)
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       object_path = String.trim_leading(object.data["id"], Pleroma.Web.Endpoint.url())
 
@@ -1519,7 +1519,7 @@ test "it tracks a signed object fetch", %{conn: conn} do
     test "it tracks a signed activity fetch", %{conn: conn} do
       user = insert(:user, local: false)
       activity = insert(:note_activity)
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       activity_path = String.trim_leading(activity.data["id"], Pleroma.Web.Endpoint.url())
 
@@ -1536,7 +1536,7 @@ test "it tracks a signed object fetch when the json is cached", %{conn: conn} do
       user = insert(:user, local: false)
       other_user = insert(:user, local: false)
       activity = insert(:note_activity)
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       object_path = String.trim_leading(object.data["id"], Pleroma.Web.Endpoint.url())
 
@@ -1560,7 +1560,7 @@ test "it tracks a signed activity fetch when the json is cached", %{conn: conn}
       user = insert(:user, local: false)
       other_user = insert(:user, local: false)
       activity = insert(:note_activity)
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       activity_path = String.trim_leading(activity.data["id"], Pleroma.Web.Endpoint.url())
 
@@ -1650,7 +1650,7 @@ test "POST /api/ap/upload_media", %{conn: conn} do
       assert activity_response["actor"] == user.ap_id
 
       assert %Object{data: %{"attachment" => [attachment]}} =
-               Object.normalize(activity_response["object"])
+               Object.normalize(activity_response["object"], fetch: false)
 
       assert attachment["type"] == "Document"
       assert attachment["name"] == desc

test/pleroma/web/activity_pub/activity_pub_test.exs

@@ -321,7 +321,7 @@ test "adds a context when none is there" do
       }
 
       {:ok, %Activity{} = activity} = ActivityPub.insert(data)
-      object = Pleroma.Object.normalize(activity)
+      object = Pleroma.Object.normalize(activity, fetch: false)
 
       assert is_binary(activity.data["context"])
       assert is_binary(object.data["context"])
@@ -344,7 +344,7 @@ test "adds an id to a given object if it lacks one and is a note and inserts it
       }
 
       {:ok, %Activity{} = activity} = ActivityPub.insert(data)
-      assert object = Object.normalize(activity)
+      assert object = Object.normalize(activity, fetch: false)
       assert is_binary(object.data["id"])
     end
   end
@@ -678,7 +678,7 @@ test "doesn't return announce activities with blocked users in 'cc'" do
 
     {:ok, activity_two} = CommonAPI.post(blockee, %{status: "hey! @#{friend.nickname}"})
 
-    assert object = Pleroma.Object.normalize(activity_two)
+    assert object = Pleroma.Object.normalize(activity_two, fetch: false)
 
     data = %{
       "actor" => friend.ap_id,

test/pleroma/web/activity_pub/mrf/steal_emoji_policy_test.exs

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do
   use Pleroma.DataCase
 
   alias Pleroma.Config
+  alias Pleroma.Emoji
   alias Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy
 
   setup_all do
@@ -14,55 +15,91 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do
   end
 
   setup do
-    emoji_path = Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
+    emoji_path = [:instance, :static_dir] |> Config.get() |> Path.join("emoji/stolen")
-    File.rm_rf!(emoji_path)
-    File.mkdir!(emoji_path)
 
-    Pleroma.Emoji.reload()
+    Emoji.reload()
+
+    message = %{
+      "type" => "Create",
+      "object" => %{
+        "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}],
+        "actor" => "https://example.org/users/admin"
+      }
+    }
 
     on_exit(fn ->
       File.rm_rf!(emoji_path)
     end)
 
-    :ok
+    [message: message, path: emoji_path]
   end
 
-  test "does nothing by default" do
+  test "does nothing by default", %{message: message} do
-    installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+    refute "firedfox" in installed()
-    refute "firedfox" in installed_emoji
 
-    message = %{
+    assert {:ok, _message} = StealEmojiPolicy.filter(message)
-      "type" => "Create",
-      "object" => %{
-        "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}],
-        "actor" => "https://example.org/users/admin"
-      }
-    }
 
-    assert {:ok, message} == StealEmojiPolicy.filter(message)
+    refute "firedfox" in installed()
-
-    installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
-    refute "firedfox" in installed_emoji
   end
 
-  test "Steals emoji on unknown shortcode from allowed remote host" do
+  test "Steals emoji on unknown shortcode from allowed remote host", %{
-    installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+    message: message,
-    refute "firedfox" in installed_emoji
+    path: path
+  } do
+    refute "firedfox" in installed()
+    refute File.exists?(path)
 
-    message = %{
+    clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468)
-      "type" => "Create",
-      "object" => %{
-        "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}],
-        "actor" => "https://example.org/users/admin"
-      }
-    }
 
-    clear_config([:mrf_steal_emoji, :hosts], ["example.org"])
+    assert {:ok, _message} = StealEmojiPolicy.filter(message)
-    clear_config([:mrf_steal_emoji, :size_limit], 284_468)
 
-    assert {:ok, message} == StealEmojiPolicy.filter(message)
+    assert "firedfox" in installed()
+    assert File.exists?(path)
 
-    installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+    assert path
-    assert "firedfox" in installed_emoji
+           |> Path.join("firedfox.png")
+           |> File.exists?()
   end
+
+  test "reject shortcode", %{message: message} do
+    refute "firedfox" in installed()
+
+    clear_config(:mrf_steal_emoji,
+      hosts: ["example.org"],
+      size_limit: 284_468,
+      rejected_shortcodes: [~r/firedfox/]
+    )
+
+    assert {:ok, _message} = StealEmojiPolicy.filter(message)
+
+    refute "firedfox" in installed()
+  end
+
+  test "reject if size is above the limit", %{message: message} do
+    refute "firedfox" in installed()
+
+    clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 50_000)
+
+    assert {:ok, _message} = StealEmojiPolicy.filter(message)
+
+    refute "firedfox" in installed()
+  end
+
+  test "reject if host returns error", %{message: message} do
+    refute "firedfox" in installed()
+
+    Tesla.Mock.mock(fn %{method: :get, url: "https://example.org/emoji/firedfox.png"} ->
+      {:ok, %Tesla.Env{status: 404, body: "Not found"}}
+    end)
+
+    clear_config(:mrf_steal_emoji, hosts: ["example.org"], size_limit: 284_468)
+
+    ExUnit.CaptureLog.capture_log(fn ->
+      assert {:ok, _message} = StealEmojiPolicy.filter(message)
+    end) =~ "MRF.StealEmojiPolicy: Failed to fetch https://example.org/emoji/firedfox.png"
+
+    refute "firedfox" in installed()
+  end
+
+  defp installed, do: Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
 end

test/pleroma/web/activity_pub/object_validators/announce_validation_test.exs

@@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidationTest do
       announcer = insert(:user)
       {:ok, post_activity} = CommonAPI.post(user, %{status: "uguu"})
 
-      object = Object.normalize(post_activity, false)
+      object = Object.normalize(post_activity, fetch: false)
       {:ok, valid_announce, []} = Builder.announce(announcer, object)
 
       %{
@@ -81,7 +81,7 @@ test "returns an error if the actor can't announce the object", %{
       {:ok, post_activity} =
         CommonAPI.post(user, %{status: "a secret post", visibility: "private"})
 
-      object = Object.normalize(post_activity, false)
+      object = Object.normalize(post_activity, fetch: false)
 
       # Another user can't announce it
       {:ok, announce, []} = Builder.announce(announcer, object, public: false)

test/pleroma/web/activity_pub/object_validators/chat_validation_test.exs

@@ -17,7 +17,7 @@ test "it is invalid if the object already exists" do
       user = insert(:user)
       recipient = insert(:user)
       {:ok, activity} = CommonAPI.post_chat_message(user, recipient, "hey")
-      object = Object.normalize(activity, false)
+      object = Object.normalize(activity, fetch: false)
 
       {:ok, create_data, _} = Builder.create(user, object.data, [recipient.ap_id])
 

test/pleroma/web/activity_pub/publisher_test.exs

@@ -322,7 +322,7 @@ test "publish to url with with different ports" do
       actor = insert(:user)
 
       note_activity = insert(:note_activity, user: actor)
-      object = Object.normalize(note_activity)
+      object = Object.normalize(note_activity, fetch: false)
 
       activity_path = String.trim_leading(note_activity.data["id"], Pleroma.Web.Endpoint.url())
       object_path = String.trim_leading(object.data["id"], Pleroma.Web.Endpoint.url())

test/pleroma/web/activity_pub/side_effects/delete_test.exs

@@ -51,7 +51,7 @@ test "it handles user deletions", %{delete_user: delete, user: user} do
       {:ok, op} = CommonAPI.post(other_user, %{status: "big oof"})
       {:ok, post} = CommonAPI.post(user, %{status: "hey", in_reply_to_id: op})
       {:ok, favorite} = CommonAPI.favorite(user, post.id)
-      object = Object.normalize(post)
+      object = Object.normalize(post, fetch: false)
       {:ok, delete_data, _meta} = Builder.delete(user, object.data["id"])
       {:ok, delete, _meta} = ActivityPub.persist(delete_data, local: true)
 
@@ -93,7 +93,7 @@ test "it handles object deletions", %{
       user = User.get_by_id(user.id)
       assert user.note_count == 0
 
-      object = Object.normalize(op.data["object"], false)
+      object = Object.normalize(op.data["object"], fetch: false)
 
       assert object.data["repliesCount"] == 0
     end
@@ -124,7 +124,7 @@ test "it handles object deletions when the object itself has been pruned", %{
       user = User.get_by_id(user.id)
       assert user.note_count == 0
 
-      object = Object.normalize(op.data["object"], false)
+      object = Object.normalize(op.data["object"], fetch: false)
 
       assert object.data["repliesCount"] == 0
     end

test/pleroma/web/activity_pub/transmogrifier/announce_handling_test.exs

@@ -130,7 +130,7 @@ test "it works for incoming announces with an inlined activity" do
     assert data["id"] ==
              "http://mastodon.example.org/users/admin/statuses/99542391527669785/activity"
 
-    object = Object.normalize(data["object"])
+    object = Object.normalize(data["object"], fetch: false)
 
     assert object.data["id"] == "http://mastodon.example.org/@admin/99541947525187368"
     assert object.data["content"] == "this is a private toot"
@@ -158,7 +158,7 @@ test "it does not clobber the addressing on announce activities" do
     data =
       File.read!("test/fixtures/mastodon-announce.json")
       |> Jason.decode!()
-      |> Map.put("object", Object.normalize(activity).data["id"])
+      |> Map.put("object", Object.normalize(activity, fetch: false).data["id"])
       |> Map.put("to", ["http://mastodon.example.org/users/admin/followers"])
       |> Map.put("cc", [])
 

test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs

@@ -26,7 +26,7 @@ test "incoming, rewrites Note to Answer and increments vote counters" do
         poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10}
       })
 
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
     assert object.data["repliesCount"] == nil
 
     data =
@@ -37,7 +37,7 @@ test "incoming, rewrites Note to Answer and increments vote counters" do
       |> Kernel.put_in(["object", "to"], user.ap_id)
 
     {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
-    answer_object = Object.normalize(activity)
+    answer_object = Object.normalize(activity, fetch: false)
     assert answer_object.data["type"] == "Answer"
     assert answer_object.data["inReplyTo"] == object.data["id"]
 
@@ -62,7 +62,7 @@ test "outgoing, rewrites Answer to Note" do
         poll: %{options: ["suya", "suya.", "suya.."], expires_in: 10}
       })
 
-    poll_object = Object.normalize(poll_activity)
+    poll_object = Object.normalize(poll_activity, fetch: false)
     # TODO: Replace with CommonAPI vote creation when implemented
     data =
       File.read!("test/fixtures/mastodon-vote.json")

test/pleroma/web/activity_pub/transmogrifier/article_handling_test.exs

@@ -25,7 +25,7 @@ test "Pterotype (Wordpress Plugin) Article" do
 
     {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
 
-    object = Object.normalize(data["object"])
+    object = Object.normalize(data["object"], fetch: false)
 
     assert object.data["name"] == "The end is near: Mastodon plans to drop OStatus support"
 
@@ -75,7 +75,7 @@ test "Prismo Article" do
     data = File.read!("test/fixtures/prismo-url-map.json") |> Jason.decode!()
 
     {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
-    object = Object.normalize(data["object"])
+    object = Object.normalize(data["object"], fetch: false)
 
     assert object.data["url"] == "https://prismo.news/posts/83"
   end

test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs

@@ -35,7 +35,7 @@ test "it works for incoming listens" do
 
     {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
 
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     assert object.data["title"] == "lain radio episode 1"
     assert object.data["artist"] == "lain"
@@ -57,7 +57,7 @@ test "Funkwhale Audio object" do
 
     {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
 
-    assert object = Object.normalize(activity, false)
+    assert object = Object.normalize(activity, fetch: false)
 
     assert object.data["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
 

test/pleroma/web/activity_pub/transmogrifier/delete_handling_test.exs

@@ -40,7 +40,7 @@ test "it works for incoming deletes" do
     assert actor == deleting_user.ap_id
 
     # Objects are replaced by a tombstone object.
-    object = Object.normalize(activity.data["object"])
+    object = Object.normalize(activity.data["object"], fetch: false)
     assert object.data["type"] == "Tombstone"
   end
 
@@ -48,7 +48,7 @@ test "it works for incoming when the object has been pruned" do
     activity = insert(:note_activity)
 
     {:ok, object} =
-      Object.normalize(activity.data["object"])
+      Object.normalize(activity.data["object"], fetch: false)
       |> Repo.delete()
 
     # TODO: mock cachex

test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs

@@ -28,7 +28,7 @@ test "it works for incoming notices with tag not being an array (kroeg)" do
       data = File.read!("test/fixtures/kroeg-array-less-emoji.json") |> Jason.decode!()
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
-      object = Object.normalize(data["object"])
+      object = Object.normalize(data["object"], fetch: false)
 
       assert object.data["emoji"] == %{
                "icon_e_smile" => "https://puckipedia.com/forum/images/smilies/icon_e_smile.png"
@@ -37,7 +37,7 @@ test "it works for incoming notices with tag not being an array (kroeg)" do
       data = File.read!("test/fixtures/kroeg-array-less-hashtag.json") |> Jason.decode!()
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
-      object = Object.normalize(data["object"])
+      object = Object.normalize(data["object"], fetch: false)
 
       assert "test" in object.data["tag"]
     end
@@ -66,7 +66,7 @@ test "it cleans up incoming notices which are not really DMs" do
       assert data["to"] == []
       assert data["cc"] == to
 
-      object_data = Object.normalize(activity).data
+      object_data = Object.normalize(activity, fetch: false).data
 
       assert object_data["to"] == []
       assert object_data["cc"] == to
@@ -78,7 +78,7 @@ test "it ignores an incoming notice if we already have it" do
       data =
         File.read!("test/fixtures/mastodon-post-activity.json")
         |> Jason.decode!()
-        |> Map.put("object", Object.normalize(activity).data)
+        |> Map.put("object", Object.normalize(activity, fetch: false).data)
 
       {:ok, returned_activity} = Transmogrifier.handle_incoming(data)
 
@@ -97,7 +97,7 @@ test "it fetches reply-to activities if we don't have them" do
 
       data = Map.put(data, "object", object)
       {:ok, returned_activity} = Transmogrifier.handle_incoming(data)
-      returned_object = Object.normalize(returned_activity, false)
+      returned_object = Object.normalize(returned_activity, fetch: false)
 
       assert %Activity{} =
                Activity.get_create_by_object_ap_id(
@@ -123,7 +123,7 @@ test "it does not fetch reply-to activities beyond max replies depth limit" do
         allowed_thread_distance?: fn _ -> false end do
         {:ok, returned_activity} = Transmogrifier.handle_incoming(data)
 
-        returned_object = Object.normalize(returned_activity, false)
+        returned_object = Object.normalize(returned_activity, fetch: false)
 
         refute Activity.get_create_by_object_ap_id(
                  "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment"
@@ -179,7 +179,7 @@ test "it works for incoming notices" do
 
       assert data["actor"] == "http://mastodon.example.org/users/admin"
 
-      object_data = Object.normalize(data["object"]).data
+      object_data = Object.normalize(data["object"], fetch: false).data
 
       assert object_data["id"] ==
                "http://mastodon.example.org/users/admin/statuses/99512778738411822"
@@ -209,7 +209,7 @@ test "it works for incoming notices without the sensitive property but an nsfw h
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
 
-      object_data = Object.normalize(data["object"], false).data
+      object_data = Object.normalize(data["object"], fetch: false).data
 
       assert object_data["sensitive"] == true
     end
@@ -218,7 +218,7 @@ test "it works for incoming notices with hashtags" do
       data = File.read!("test/fixtures/mastodon-post-activity-hashtag.json") |> Jason.decode!()
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
-      object = Object.normalize(data["object"])
+      object = Object.normalize(data["object"], fetch: false)
 
       assert Enum.at(object.data["tag"], 2) == "moo"
     end
@@ -227,7 +227,7 @@ test "it works for incoming notices with contentMap" do
       data = File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Jason.decode!()
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
-      object = Object.normalize(data["object"])
+      object = Object.normalize(data["object"], fetch: false)
 
       assert object.data["content"] ==
                "<p><span class=\"h-card\"><a href=\"http://localtesting.pleroma.lol/users/lain\" class=\"u-url mention\">@<span>lain</span></a></span></p>"
@@ -237,7 +237,7 @@ test "it works for incoming notices with to/cc not being an array (kroeg)" do
       data = File.read!("test/fixtures/kroeg-post-activity.json") |> Jason.decode!()
 
       {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
-      object = Object.normalize(data["object"])
+      object = Object.normalize(data["object"], fetch: false)
 
       assert object.data["content"] ==
                "<p>henlo from my Psion netBook</p><p>message sent from my Psion netBook</p>"
@@ -725,7 +725,7 @@ test "sets `replies` collection with a limited number of self-replies" do
           in_reply_to_status_id: id1
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
       replies_uris = Enum.map([self_reply1, self_reply2], fn a -> a.object.data["id"] end)
 
       assert %{"type" => "Collection", "items" => ^replies_uris} =

test/pleroma/web/activity_pub/transmogrifier/question_handling_test.exs

@@ -22,7 +22,7 @@ test "Mastodon Question activity" do
 
     {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
 
-    object = Object.normalize(activity, false)
+    object = Object.normalize(activity, fetch: false)
 
     assert object.data["url"] == "https://mastodon.sdf.org/@rinpatch/102070944809637304"
 
@@ -65,7 +65,7 @@ test "Mastodon Question activity" do
 
     {:ok, reply_activity} = CommonAPI.post(user, %{status: "hewwo", in_reply_to_id: activity.id})
 
-    reply_object = Object.normalize(reply_activity, false)
+    reply_object = Object.normalize(reply_activity, fetch: false)
 
     assert reply_object.data["context"] == object.data["context"]
     assert reply_object.data["context_id"] == object.data["context_id"]
@@ -101,7 +101,7 @@ test "Mastodon Question activity with HTML tags in plaintext" do
       |> Kernel.put_in(["object", "oneOf"], options)
 
     {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
-    object = Object.normalize(activity, false)
+    object = Object.normalize(activity, fetch: false)
 
     assert Enum.sort(object.data["oneOf"]) == Enum.sort(options)
   end
@@ -147,7 +147,7 @@ test "Mastodon Question activity with custom emojis" do
       |> Kernel.put_in(["object", "tag"], tag)
 
     {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
-    object = Object.normalize(activity, false)
+    object = Object.normalize(activity, fetch: false)
 
     assert object.data["oneOf"] == options
 

test/pleroma/web/activity_pub/transmogrifier/video_handling_test.exs

@@ -24,7 +24,7 @@ test "skip converting the content when it is nil" do
 
     {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
 
-    assert object = Object.normalize(activity, false)
+    assert object = Object.normalize(activity, fetch: false)
 
     assert object.data["content"] == nil
   end
@@ -34,7 +34,7 @@ test "it converts content of object to html" do
 
     {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
 
-    assert object = Object.normalize(activity, false)
+    assert object = Object.normalize(activity, fetch: false)
 
     assert object.data["content"] ==
              "<p>Après avoir mené avec un certain succès la campagne « Dégooglisons Internet » en 2014, l’association Framasoft annonce fin 2019 arrêter progressivement un certain nombre de ses services alternatifs aux GAFAM. Pourquoi ?</p><p>Transcription par @aprilorg ici : <a href=\"https://www.april.org/deframasoftisons-internet-pierre-yves-gosset-framasoft\">https://www.april.org/deframasoftisons-internet-pierre-yves-gosset-framasoft</a></p>"
@@ -70,7 +70,7 @@ test "it remaps video URLs as attachments if necessary" do
 
     {:ok, %Activity{local: false} = activity} = Transmogrifier.handle_incoming(data)
 
-    assert object = Object.normalize(activity, false)
+    assert object = Object.normalize(activity, fetch: false)
 
     assert object.data["attachment"] == [
              %{

test/pleroma/web/activity_pub/transmogrifier_test.exs

@@ -56,7 +56,7 @@ test "it accepts Flag activities" do
       other_user = insert(:user)
 
       {:ok, activity} = CommonAPI.post(user, %{status: "test post"})
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       note_obj = %{
         "type" => "Note",
@@ -281,6 +281,21 @@ test "it can handle Listen activities" do
 
       {:ok, _modified} = Transmogrifier.prepare_outgoing(activity.data)
     end
+
+    test "custom emoji urls are URI encoded" do
+      # :dinosaur: filename has a space -> dino walking.gif
+      user = insert(:user)
+
+      {:ok, activity} = CommonAPI.post(user, %{status: "everybody do the dinosaur :dinosaur:"})
+
+      {:ok, prepared} = Transmogrifier.prepare_outgoing(activity.data)
+
+      assert length(prepared["object"]["tag"]) == 1
+
+      url = prepared["object"]["tag"] |> List.first() |> Map.get("icon") |> Map.get("url")
+
+      assert url == "http://localhost:4001/emoji/dino%20walking.gif"
+    end
   end
 
   describe "user upgrade" do

test/pleroma/web/activity_pub/utils_test.exs

@@ -165,7 +165,7 @@ test "fetches existing votes" do
           }
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
       {:ok, votes, object} = CommonAPI.vote(other_user, object, [0, 1])
       assert Enum.sort(Utils.get_existing_votes(other_user.ap_id, object)) == Enum.sort(votes)
     end
@@ -183,7 +183,7 @@ test "fetches only Create activities" do
           }
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
       {:ok, [vote], object} = CommonAPI.vote(other_user, object, [0])
       {:ok, _activity} = CommonAPI.favorite(user, activity.id)
       [fetched_vote] = Utils.get_existing_votes(other_user.ap_id, object)
@@ -242,7 +242,7 @@ test "updates the state of the given follow activity" do
     test "updates likes" do
       user = insert(:user)
       activity = insert(:note_activity)
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert {:ok, updated_object} =
                Utils.update_element_in_object(
@@ -302,7 +302,7 @@ test "removes ap_id from likes" do
   describe "get_existing_like/2" do
     test "fetches existing like" do
       note_activity = insert(:note_activity)
-      assert object = Object.normalize(note_activity)
+      assert object = Object.normalize(note_activity, fetch: false)
 
       user = insert(:user)
       refute Utils.get_existing_like(user.ap_id, object)
@@ -320,7 +320,7 @@ test "returns nil if announce not found" do
 
     test "fetches existing announce" do
       note_activity = insert(:note_activity)
-      assert object = Object.normalize(note_activity)
+      assert object = Object.normalize(note_activity, fetch: false)
       actor = insert(:user)
 
       {:ok, announce} = CommonAPI.repeat(note_activity.id, actor)
@@ -412,7 +412,7 @@ test "returns false" do
   describe "lazy_put_activity_defaults/2" do
     test "returns map with id and published data" do
       note_activity = insert(:note_activity)
-      object = Object.normalize(note_activity)
+      object = Object.normalize(note_activity, fetch: false)
       res = Utils.lazy_put_activity_defaults(%{"context" => object.data["id"]})
       assert res["context"] == object.data["id"]
       assert res["context_id"] == object.id
@@ -431,7 +431,7 @@ test "returns map with fake id and published data" do
 
     test "returns activity data with object" do
       note_activity = insert(:note_activity)
-      object = Object.normalize(note_activity)
+      object = Object.normalize(note_activity, fetch: false)
 
       res =
         Utils.lazy_put_activity_defaults(%{

test/pleroma/web/activity_pub/views/object_view_test.exs

@@ -24,7 +24,7 @@ test "renders a note object" do
 
   test "renders a note activity" do
     note = insert(:note_activity)
-    object = Object.normalize(note)
+    object = Object.normalize(note, fetch: false)
 
     result = ObjectView.render("object.json", %{object: note})
 
@@ -56,7 +56,7 @@ test "renders `replies` collection for a note activity" do
 
   test "renders a like activity" do
     note = insert(:note_activity)
-    object = Object.normalize(note)
+    object = Object.normalize(note, fetch: false)
     user = insert(:user)
 
     {:ok, like_activity} = CommonAPI.favorite(user, note.id)
@@ -70,7 +70,7 @@ test "renders a like activity" do
 
   test "renders an announce activity" do
     note = insert(:note_activity)
-    object = Object.normalize(note)
+    object = Object.normalize(note, fetch: false)
     user = insert(:user)
 
     {:ok, announce_activity} = CommonAPI.repeat(note.id, user)

test/pleroma/web/admin_api/controllers/chat_controller_test.exs

@@ -36,7 +36,7 @@ test "it deletes a message from the chat", %{conn: conn, admin: admin} do
       {:ok, message} =
         CommonAPI.post_chat_message(user, recipient, "Hello darkness my old friend")
 
-      object = Object.normalize(message, false)
+      object = Object.normalize(message, fetch: false)
 
       chat = Chat.get(user.id, recipient.ap_id)
       recipient_chat = Chat.get(recipient.id, user.ap_id)
@@ -143,7 +143,7 @@ test "it returns a chat", %{conn: conn} do
       recipient = insert(:user)
 
       {:ok, message} = CommonAPI.post_chat_message(user, recipient, "Yo")
-      object = Object.normalize(message, false)
+      object = Object.normalize(message, fetch: false)
       chat = Chat.get(user.id, recipient.ap_id)
       cm_ref = MessageReference.for_chat_and_object(chat, object)
 
@@ -183,7 +183,7 @@ test "GET /api/pleroma/admin/chats/:id", %{conn: conn, chat: chat} do
       recipient = insert(:user)
 
       {:ok, message} = CommonAPI.post_chat_message(user, recipient, "Yo")
-      object = Object.normalize(message, false)
+      object = Object.normalize(message, fetch: false)
       chat = Chat.get(user.id, recipient.ap_id)
       cm_ref = MessageReference.for_chat_and_object(chat, object)
 

test/pleroma/web/admin_api/controllers/user_controller_test.exs

@@ -953,6 +953,7 @@ defp user_response(user, attrs \\ %{}) do
     %{
       "deactivated" => user.deactivated,
       "id" => user.id,
+      "email" => user.email,
       "nickname" => user.nickname,
       "roles" => %{"admin" => false, "moderator" => false},
       "local" => user.local,

test/pleroma/web/admin_api/views/account_view_test.exs

@@ -0,0 +1,16 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.AccountViewTest do
+  use Pleroma.DataCase, async: true
+  import Pleroma.Factory
+  alias Pleroma.Web.AdminAPI.AccountView
+
+  describe "show.json" do
+    test "renders the user's email" do
+      user = insert(:user, email: "yolo@yolofam.tld")
+      assert %{"email" => "yolo@yolofam.tld"} = AccountView.render("show.json", %{user: user})
+    end
+  end
+end

test/pleroma/web/common_api_test.exs

@@ -39,7 +39,7 @@ test "it posts a poll" do
           poll: %{expires_in: 600, options: ["reimu", "marisa"]}
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert object.data["type"] == "Question"
       assert object.data["oneOf"] |> length() == 2
@@ -174,7 +174,7 @@ test "it adds html newlines" do
 
       assert other_user.ap_id not in activity.recipients
 
-      object = Object.normalize(activity, false)
+      object = Object.normalize(activity, fetch: false)
 
       assert object.data["content"] == "uguu<br/>uguuu"
     end
@@ -194,7 +194,7 @@ test "it linkifies" do
 
       assert other_user.ap_id not in activity.recipients
 
-      object = Object.normalize(activity, false)
+      object = Object.normalize(activity, fetch: false)
 
       assert object.data["content"] ==
                "<a href=\"https://example.org\" rel=\"ugc\">https://example.org</a> is the site of <span class=\"h-card\"><a class=\"u-url mention\" data-user=\"#{
@@ -215,7 +215,7 @@ test "it posts a chat message" do
 
       assert activity.data["type"] == "Create"
       assert activity.local
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert object.data["type"] == "ChatMessage"
       assert object.data["to"] == [recipient.ap_id]
@@ -281,7 +281,7 @@ test "it works with pruned objects" do
 
       clear_config([:instance, :federating], true)
 
-      Object.normalize(post, false)
+      Object.normalize(post, fetch: false)
       |> Object.prune()
 
       with_mock Pleroma.Web.Federator,
@@ -491,7 +491,7 @@ test "it de-duplicates tags" do
     user = insert(:user)
     {:ok, activity} = CommonAPI.post(user, %{status: "#2hu #2HU"})
 
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     assert object.data["tag"] == ["2hu"]
   end
@@ -500,10 +500,23 @@ test "it adds emoji in the object" do
     user = insert(:user)
     {:ok, activity} = CommonAPI.post(user, %{status: ":firefox:"})
 
-    assert Object.normalize(activity).data["emoji"]["firefox"]
+    assert Object.normalize(activity, fetch: false).data["emoji"]["firefox"]
   end
 
   describe "posting" do
+    test "it adds an emoji on an external site" do
+      user = insert(:user)
+      {:ok, activity} = CommonAPI.post(user, %{status: "hey :external_emoji:"})
+
+      assert %{"external_emoji" => url} = Object.normalize(activity).data["emoji"]
+      assert url == "https://example.com/emoji.png"
+
+      {:ok, activity} = CommonAPI.post(user, %{status: "hey :blank:"})
+
+      assert %{"blank" => url} = Object.normalize(activity).data["emoji"]
+      assert url == "#{Pleroma.Web.base_url()}/emoji/blank.png"
+    end
+
     test "deactivated users can't post" do
       user = insert(:user, deactivated: true)
       assert {:error, _} = CommonAPI.post(user, %{status: "ye"})
@@ -539,7 +552,7 @@ test "it filters out obviously bad tags when accepting a post as HTML" do
           content_type: "text/html"
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert object.data["content"] == "<p><b>2hu</b></p>alert(&#39;xss&#39;)"
       assert object.data["source"] == post
@@ -556,7 +569,7 @@ test "it filters out obviously bad tags when accepting a post as Markdown" do
           content_type: "text/markdown"
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert object.data["content"] == "<p><b>2hu</b></p>alert(&#39;xss&#39;)"
       assert object.data["source"] == post
@@ -1211,7 +1224,7 @@ test "does not allow to vote twice" do
           poll: %{options: ["Yes", "No"], expires_in: 20}
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       {:ok, _, object} = CommonAPI.vote(other_user, object, [0])
 
@@ -1231,7 +1244,7 @@ test "returns a valid activity" do
           length: 180_000
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert object.data["title"] == "lain radio episode 1"
 
@@ -1250,7 +1263,7 @@ test "respects visibility=private" do
           visibility: "private"
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert object.data["title"] == "lain radio episode 1"
 

test/pleroma/web/feed/tag_controller_test.exs

@@ -24,7 +24,7 @@ test "gets a feed (ATOM)", %{conn: conn} do
     user = insert(:user)
     {:ok, activity1} = CommonAPI.post(user, %{status: "yeah #PleromaArt"})
 
-    object = Object.normalize(activity1)
+    object = Object.normalize(activity1, fetch: false)
 
     object_data =
       Map.put(object.data, "attachment", [
@@ -91,7 +91,7 @@ test "gets a feed (RSS)", %{conn: conn} do
     user = insert(:user)
     {:ok, activity1} = CommonAPI.post(user, %{status: "yeah #PleromaArt"})
 
-    object = Object.normalize(activity1)
+    object = Object.normalize(activity1, fetch: false)
 
     object_data =
       Map.put(object.data, "attachment", [
@@ -147,8 +147,8 @@ test "gets a feed (RSS)", %{conn: conn} do
              "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4"
            ]
 
-    obj1 = Object.normalize(activity1)
+    obj1 = Object.normalize(activity1, fetch: false)
-    obj2 = Object.normalize(activity2)
+    obj2 = Object.normalize(activity2, fetch: false)
 
     assert xpath(xml, ~x"//channel/item/description/text()"sl) == [
              HtmlEntities.decode(FeedView.activity_content(obj2.data)),

test/pleroma/web/feed/user_controller_test.exs

@@ -58,7 +58,7 @@ defmodule Pleroma.Web.Feed.UserControllerTest do
         )
 
       note_activity2 = insert(:note_activity, note: note2)
-      object = Object.normalize(note_activity)
+      object = Object.normalize(note_activity, fetch: false)
 
       [user: user, object: object, max_id: note_activity2.id]
     end

test/pleroma/web/mastodon_api/controllers/account_controller_test.exs

@@ -469,6 +469,21 @@ test "muted reactions", %{user: user, conn: conn} do
                }
              ] = result
     end
+
+    test "paginates a user's statuses", %{user: user, conn: conn} do
+      {:ok, post_1} = CommonAPI.post(user, %{status: "first post"})
+      {:ok, post_2} = CommonAPI.post(user, %{status: "second post"})
+
+      response_1 = get(conn, "/api/v1/accounts/#{user.id}/statuses?limit=1")
+      assert [res] = json_response(response_1, 200)
+      assert res["id"] == post_2.id
+
+      response_2 = get(conn, "/api/v1/accounts/#{user.id}/statuses?limit=1&max_id=#{res["id"]}")
+      assert [res] = json_response(response_2, 200)
+      assert res["id"] == post_1.id
+
+      refute response_1 == response_2
+    end
   end
 
   defp local_and_remote_activities(%{local: local, remote: remote}) do

test/pleroma/web/mastodon_api/controllers/poll_controller_test.exs

@@ -20,7 +20,7 @@ test "returns poll entity for object id", %{user: user, conn: conn} do
           poll: %{options: ["what Mastodon't", "n't what Mastodoes"], expires_in: 20}
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       conn = get(conn, "/api/v1/polls/#{object.id}")
 
@@ -39,7 +39,7 @@ test "does not expose polls for private statuses", %{conn: conn} do
           visibility: "private"
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       conn = get(conn, "/api/v1/polls/#{object.id}")
 
@@ -63,7 +63,7 @@ test "votes are added to the poll", %{conn: conn} do
           }
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       conn =
         conn
@@ -85,7 +85,7 @@ test "author can't vote", %{user: user, conn: conn} do
           poll: %{options: ["Yes", "No"], expires_in: 20}
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert conn
              |> put_req_header("content-type", "application/json")
@@ -106,7 +106,7 @@ test "does not allow multiple choices on a single-choice question", %{conn: conn
           poll: %{options: ["half empty", "half full"], expires_in: 20}
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert conn
              |> put_req_header("content-type", "application/json")
@@ -129,7 +129,7 @@ test "does not allow choice index to be greater than options count", %{conn: con
           poll: %{options: ["Yes", "No"], expires_in: 20}
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       conn =
         conn
@@ -158,7 +158,7 @@ test "returns 404 when poll is private and not available for user", %{conn: conn
           visibility: "private"
         })
 
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       conn =
         conn

test/pleroma/web/mastodon_api/controllers/search_controller_test.exs

@@ -309,7 +309,7 @@ test "search doesn't show statuses that it shouldn't", %{conn: conn} do
         })
 
       capture_log(fn ->
-        q = Object.normalize(activity).data["id"]
+        q = Object.normalize(activity, fetch: false).data["id"]
 
         results =
           conn

test/pleroma/web/mastodon_api/controllers/status_controller_test.exs

@@ -800,7 +800,7 @@ test "if user is authenticated", %{local: local, remote: remote} do
     test "when you created it" do
       %{user: author, conn: conn} = oauth_access(["write:statuses"])
       activity = insert(:note_activity, user: author)
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       content = object.data["content"]
       source = object.data["source"]
@@ -1374,7 +1374,9 @@ test "Repeated posts that are replies incorrectly have in_reply_to_id null", %{c
 
     activity = Activity.get_by_id_with_object(id)
 
-    assert Object.normalize(activity).data["inReplyTo"] == Object.normalize(replied_to).data["id"]
+    assert Object.normalize(activity, fetch: false).data["inReplyTo"] ==
+             Object.normalize(replied_to, fetch: false).data["id"]
+
     assert Activity.get_in_reply_to_activity(activity).id == replied_to.id
 
     # Reblog from the third user

test/pleroma/web/mastodon_api/views/notification_view_test.exs

@@ -44,7 +44,7 @@ test "ChatMessage notification" do
 
     {:ok, [notification]} = Notification.create_notifications(activity)
 
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
     chat = Chat.get(recipient.id, user.ap_id)
 
     cm_ref = MessageReference.for_chat_and_object(chat, object)

test/pleroma/web/mastodon_api/views/poll_view_test.exs

@@ -29,7 +29,7 @@ test "renders a poll" do
         }
       })
 
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     expected = %{
       emojis: [],
@@ -72,7 +72,7 @@ test "detects if it is multiple choice" do
 
     voter = insert(:user)
 
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     {:ok, _votes, object} = CommonAPI.vote(voter, object, [0, 1])
 
@@ -98,7 +98,7 @@ test "detects emoji" do
         }
       })
 
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     assert %{emojis: [%{shortcode: "blank"}]} = PollView.render("show.json", %{object: object})
   end
@@ -117,7 +117,7 @@ test "detects vote status" do
         }
       })
 
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     {:ok, _, object} = CommonAPI.vote(other_user, object, [1, 2])
 
@@ -129,7 +129,7 @@ test "detects vote status" do
   end
 
   test "does not crash on polls with no end date" do
-    object = Object.normalize("https://skippers-bin.com/notes/7x9tmrp97i")
+    object = Object.normalize("https://skippers-bin.com/notes/7x9tmrp97i", fetch: true)
     result = PollView.render("show.json", %{object: object})
 
     assert result[:expires_at] == nil
@@ -153,7 +153,7 @@ test "doesn't strips HTML tags" do
         }
       })
 
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     assert %{
              options: [

test/pleroma/web/mastodon_api/views/status_view_test.exs

@@ -61,7 +61,7 @@ test "works correctly with badly formatted emojis" do
     {:ok, activity} = CommonAPI.post(user, %{status: "yo"})
 
     activity
-    |> Object.normalize(false)
+    |> Object.normalize(fetch: false)
     |> Object.update_data(%{"reactions" => %{"☕" => [user.ap_id], "x" => 1}})
 
     activity = Activity.get_by_id(activity.id)
@@ -204,7 +204,7 @@ test "tries to get a user by nickname if fetching by ap_id doesn't work" do
 
   test "a note with null content" do
     note = insert(:note_activity)
-    note_object = Object.normalize(note)
+    note_object = Object.normalize(note, fetch: false)
 
     data =
       note_object.data
@@ -223,7 +223,7 @@ test "a note with null content" do
 
   test "a note activity" do
     note = insert(:note_activity)
-    object_data = Object.normalize(note).data
+    object_data = Object.normalize(note, fetch: false).data
     user = User.get_cached_by_ap_id(note.data["actor"])
 
     convo_id = Utils.context_to_conversation_id(object_data["context"])

test/pleroma/web/o_status/o_status_controller_test.exs

@@ -72,7 +72,7 @@ test "redirects to /notice/:id for html format for activity", %{
 
     test "redirects to /notice/id for html format", %{conn: conn} do
       note_activity = insert(:note_activity)
-      object = Object.normalize(note_activity)
+      object = Object.normalize(note_activity, fetch: false)
       [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, object.data["id"]))
       url = "/objects/#{uuid}"
 
@@ -82,7 +82,7 @@ test "redirects to /notice/id for html format", %{conn: conn} do
 
     test "404s on private objects", %{conn: conn} do
       note_activity = insert(:direct_note_activity)
-      object = Object.normalize(note_activity)
+      object = Object.normalize(note_activity, fetch: false)
       [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, object.data["id"]))
 
       conn
@@ -133,7 +133,7 @@ test "redirects to a proper object URL when json requested and the object is loc
       conn: conn
     } do
       note_activity = insert(:note_activity)
-      expected_redirect_url = Object.normalize(note_activity).data["id"]
+      expected_redirect_url = Object.normalize(note_activity, fetch: false).data["id"]
 
       redirect_url =
         conn
@@ -230,7 +230,7 @@ test "does not require authentication on non-federating instances", %{
   describe "GET /notice/:id/embed_player" do
     setup do
       note_activity = insert(:note_activity)
-      object = Pleroma.Object.normalize(note_activity)
+      object = Pleroma.Object.normalize(note_activity, fetch: false)
 
       object_data =
         Map.put(object.data, "attachment", [
@@ -287,7 +287,7 @@ test "404s when activity is direct message", %{conn: conn} do
 
     test "404s when attachment is empty", %{conn: conn} do
       note_activity = insert(:note_activity)
-      object = Pleroma.Object.normalize(note_activity)
+      object = Pleroma.Object.normalize(note_activity, fetch: false)
       object_data = Map.put(object.data, "attachment", [])
 
       object
@@ -301,7 +301,7 @@ test "404s when attachment is empty", %{conn: conn} do
 
     test "404s when attachment isn't audio or video", %{conn: conn} do
       note_activity = insert(:note_activity)
-      object = Pleroma.Object.normalize(note_activity)
+      object = Pleroma.Object.normalize(note_activity, fetch: false)
 
       object_data =
         Map.put(object.data, "attachment", [

test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs

@@ -22,7 +22,7 @@ test "it marks one message as read", %{conn: conn, user: user} do
       {:ok, create} = CommonAPI.post_chat_message(other_user, user, "sup")
       {:ok, _create} = CommonAPI.post_chat_message(other_user, user, "sup part 2")
       {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
-      object = Object.normalize(create, false)
+      object = Object.normalize(create, fetch: false)
       cm_ref = MessageReference.for_chat_and_object(chat, object)
 
       assert cm_ref.unread == true
@@ -52,7 +52,7 @@ test "given a `last_read_id`, it marks everything until then as read", %{
       {:ok, create} = CommonAPI.post_chat_message(other_user, user, "sup")
       {:ok, _create} = CommonAPI.post_chat_message(other_user, user, "sup part 2")
       {:ok, chat} = Chat.get_or_create(user.id, other_user.ap_id)
-      object = Object.normalize(create, false)
+      object = Object.normalize(create, fetch: false)
       cm_ref = MessageReference.for_chat_and_object(chat, object)
 
       assert cm_ref.unread == true
@@ -158,7 +158,7 @@ test "it deletes a message from the chat", %{conn: conn, user: user} do
 
       {:ok, other_message} = CommonAPI.post_chat_message(recipient, user, "nico nico ni")
 
-      object = Object.normalize(message, false)
+      object = Object.normalize(message, fetch: false)
 
       chat = Chat.get(user.id, recipient.ap_id)
 
@@ -176,7 +176,7 @@ test "it deletes a message from the chat", %{conn: conn, user: user} do
       assert %{data: %{"type" => "Tombstone"}} = Object.get_by_id(object.id)
 
       # Deleting other people's messages just removes the reference
-      object = Object.normalize(other_message, false)
+      object = Object.normalize(other_message, fetch: false)
       cm_ref = MessageReference.for_chat_and_object(chat, object)
 
       result =

test/pleroma/web/pleroma_api/views/backup_view_test.exs

@@ -0,0 +1,18 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.BackupViewTest do
+  use Pleroma.DataCase, async: true
+  alias Pleroma.User.Backup
+  alias Pleroma.Web.PleromaAPI.BackupView
+  import Pleroma.Factory
+
+  test "it renders the ID" do
+    user = insert(:user)
+    backup = Backup.new(user)
+
+    result = BackupView.render("show.json", backup: backup)
+    assert result.id == backup.id
+  end
+end

test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs

@@ -31,7 +31,7 @@ test "it displays a chat message" do
 
     chat = Chat.get(user.id, recipient.ap_id)
 
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     cm_ref = MessageReference.for_chat_and_object(chat, object)
 
@@ -58,7 +58,7 @@ test "it displays a chat message" do
         media_id: upload.id
       )
 
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     cm_ref = MessageReference.for_chat_and_object(chat, object)
 

test/pleroma/web/pleroma_api/views/chat_view_test.exs

@@ -35,7 +35,7 @@ test "it represents a chat" do
 
     {:ok, chat_message_creation} = CommonAPI.post_chat_message(user, recipient, "hello")
 
-    chat_message = Object.normalize(chat_message_creation, false)
+    chat_message = Object.normalize(chat_message_creation, fetch: false)
 
     {:ok, chat} = Chat.get_or_create(user.id, recipient.ap_id)
 

test/pleroma/web/push/impl_test.exs

@@ -118,7 +118,7 @@ test "renders title and body for create activity" do
           "<span>Lorem ipsum dolor sit amet</span>, consectetur :firefox: adipiscing elit. Fusce sagittis finibus turpis."
       })
 
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     assert Impl.format_body(
              %{
@@ -137,7 +137,7 @@ test "renders title and body for follow activity" do
     user = insert(:user, nickname: "Bob")
     other_user = insert(:user)
     {:ok, _, _, activity} = CommonAPI.follow(user, other_user)
-    object = Object.normalize(activity, false)
+    object = Object.normalize(activity, fetch: false)
 
     assert Impl.format_body(%{activity: activity, type: "follow"}, user, object) ==
              "@Bob has followed you"
@@ -156,7 +156,7 @@ test "renders title and body for announce activity" do
       })
 
     {:ok, announce_activity} = CommonAPI.repeat(activity.id, user)
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     assert Impl.format_body(%{activity: announce_activity}, user, object) ==
              "@#{user.nickname} repeated: Lorem ipsum dolor sit amet, consectetur  adipiscing elit. Fusce sagittis fini..."
@@ -175,7 +175,7 @@ test "renders title and body for like activity" do
       })
 
     {:ok, activity} = CommonAPI.favorite(user, activity.id)
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     assert Impl.format_body(%{activity: activity, type: "favourite"}, user, object) ==
              "@Bob has favorited your post"
@@ -193,7 +193,7 @@ test "renders title and body for pleroma:emoji_reaction activity" do
       })
 
     {:ok, activity} = CommonAPI.react_with_emoji(activity.id, user, "👍")
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, fetch: false)
 
     assert Impl.format_body(%{activity: activity, type: "pleroma:emoji_reaction"}, user, object) ==
              "@Bob reacted with 👍"
@@ -221,7 +221,7 @@ test "builds content for chat messages" do
       recipient = insert(:user)
 
       {:ok, chat} = CommonAPI.post_chat_message(user, recipient, "hey")
-      object = Object.normalize(chat, false)
+      object = Object.normalize(chat, fetch: false)
       [notification] = Notification.for_user(recipient)
 
       res = Impl.build_content(notification, user, object)
@@ -245,7 +245,7 @@ test "builds content for chat messages with no content" do
       {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id)
 
       {:ok, chat} = CommonAPI.post_chat_message(user, recipient, nil, media_id: upload.id)
-      object = Object.normalize(chat, false)
+      object = Object.normalize(chat, fetch: false)
       [notification] = Notification.for_user(recipient)
 
       res = Impl.build_content(notification, user, object)
@@ -271,7 +271,7 @@ test "hides contents of notifications when option enabled" do
       notif = insert(:notification, user: user2, activity: activity)
 
       actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert Impl.build_content(notif, actor, object) == %{
                body: "New Direct Message"
@@ -286,7 +286,7 @@ test "hides contents of notifications when option enabled" do
       notif = insert(:notification, user: user2, activity: activity, type: "mention")
 
       actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert Impl.build_content(notif, actor, object) == %{
                body: "New Mention"
@@ -297,7 +297,7 @@ test "hides contents of notifications when option enabled" do
       notif = insert(:notification, user: user2, activity: activity, type: "favourite")
 
       actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert Impl.build_content(notif, actor, object) == %{
                body: "New Favorite"
@@ -320,7 +320,7 @@ test "returns regular content when hiding contents option disabled" do
       notif = insert(:notification, user: user2, activity: activity)
 
       actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert Impl.build_content(notif, actor, object) == %{
                body:
@@ -338,7 +338,7 @@ test "returns regular content when hiding contents option disabled" do
       notif = insert(:notification, user: user2, activity: activity, type: "mention")
 
       actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert Impl.build_content(notif, actor, object) == %{
                body:
@@ -351,7 +351,7 @@ test "returns regular content when hiding contents option disabled" do
       notif = insert(:notification, user: user2, activity: activity, type: "favourite")
 
       actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
-      object = Object.normalize(activity)
+      object = Object.normalize(activity, fetch: false)
 
       assert Impl.build_content(notif, actor, object) == %{
                body: "@Bob has favorited your post",

test/pleroma/web/streamer_test.exs

@@ -266,7 +266,7 @@ test "it sends chat messages to the 'user:pleroma_chat' stream", %{
       {:ok, create_activity} =
         CommonAPI.post_chat_message(other_user, user, "hey cirno", idempotency_key: "123")
 
-      object = Object.normalize(create_activity, false)
+      object = Object.normalize(create_activity, fetch: false)
       chat = Chat.get(user.id, other_user.ap_id)
       cm_ref = MessageReference.for_chat_and_object(chat, object)
       cm_ref = %{cm_ref | chat: chat, object: object}
@@ -284,7 +284,7 @@ test "it sends chat messages to the 'user' stream", %{user: user, token: oauth_t
       other_user = insert(:user)
 
       {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno")
-      object = Object.normalize(create_activity, false)
+      object = Object.normalize(create_activity, fetch: false)
       chat = Chat.get(user.id, other_user.ap_id)
       cm_ref = MessageReference.for_chat_and_object(chat, object)
       cm_ref = %{cm_ref | chat: chat, object: object}

test/pleroma/workers/scheduled_activity_worker_test.exs

@@ -36,7 +36,7 @@ test "creates a status from the scheduled activity" do
 
     refute Repo.get(ScheduledActivity, scheduled_activity.id)
     activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id))
-    assert Pleroma.Object.normalize(activity).data["content"] == "hi"
+    assert Pleroma.Object.normalize(activity, fetch: false).data["content"] == "hi"
   end
 
   test "adds log message if ScheduledActivity isn't find" do

test/support/factory.ex

@@ -259,7 +259,7 @@ def announce_activity_factory(attrs \\ %{}) do
 
   def like_activity_factory(attrs \\ %{}) do
     note_activity = attrs[:note_activity] || insert(:note_activity)
-    object = Object.normalize(note_activity)
+    object = Object.normalize(note_activity, fetch: false)
     user = insert(:user)
 
     data =