Merge remote-tracking branch 'remotes/origin/develop' into output-of-relationships-in-statuses

# Conflicts:
#	CHANGELOG.md

CHANGELOG.md

@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Added
 - NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list.
 - Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses.
+- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required.
 - Configuration: `:extensions/:output_relationships_in_statuses_by_default` option (if `false`, disables the output of account/pleroma/relationship for statuses and notifications by default, breaking the compatibility with older PleromaFE versions).
 <details>
   <summary>API Changes</summary>

config/config.exs

@@ -58,20 +58,6 @@
 
 config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
 
-config :pleroma, :hackney_pools,
-  federation: [
-    max_connections: 50,
-    timeout: 150_000
-  ],
-  media: [
-    max_connections: 50,
-    timeout: 150_000
-  ],
-  upload: [
-    max_connections: 25,
-    timeout: 300_000
-  ]
-
 # Upload configuration
 config :pleroma, Pleroma.Upload,
   uploader: Pleroma.Uploaders.Local,
@@ -184,21 +170,13 @@
   "application/ld+json" => ["activity+json"]
 }
 
-config :tesla, adapter: Tesla.Adapter.Hackney
+config :tesla, adapter: Tesla.Adapter.Gun
-
 # Configures http settings, upstream proxy etc.
 config :pleroma, :http,
   proxy_url: nil,
   send_user_agent: true,
   user_agent: :default,
-  adapter: [
+  adapter: []
-    ssl_options: [
-      # Workaround for remote server certificate chain issues
-      partial_chain: &:hackney_connect.partial_chain/1,
-      # We don't support TLS v1.3 yet
-      versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"]
-    ]
-  ]
 
 config :pleroma, :instance,
   name: "Pleroma",
@@ -626,6 +604,49 @@
   parameters: [gin_fuzzy_search_limit: "500"],
   prepare: :unnamed
 
+config :pleroma, :connections_pool,
+  checkin_timeout: 250,
+  max_connections: 250,
+  retry: 1,
+  retry_timeout: 1000,
+  await_up_timeout: 5_000
+
+config :pleroma, :pools,
+  federation: [
+    size: 50,
+    max_overflow: 10,
+    timeout: 150_000
+  ],
+  media: [
+    size: 50,
+    max_overflow: 10,
+    timeout: 150_000
+  ],
+  upload: [
+    size: 25,
+    max_overflow: 5,
+    timeout: 300_000
+  ],
+  default: [
+    size: 10,
+    max_overflow: 2,
+    timeout: 10_000
+  ]
+
+config :pleroma, :hackney_pools,
+  federation: [
+    max_connections: 50,
+    timeout: 150_000
+  ],
+  media: [
+    max_connections: 50,
+    timeout: 150_000
+  ],
+  upload: [
+    max_connections: 25,
+    timeout: 300_000
+  ]
+
 config :pleroma, :restrict_unauthenticated,
   timelines: %{local: false, federated: false},
   profiles: %{local: false, remote: false},

config/description.exs

@@ -2932,6 +2932,219 @@
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :connections_pool,
+    type: :group,
+    description: "Advanced settings for `gun` connections pool",
+    children: [
+      %{
+        key: :checkin_timeout,
+        type: :integer,
+        description: "Timeout to checkin connection from pool. Default: 250ms.",
+        suggestions: [250]
+      },
+      %{
+        key: :max_connections,
+        type: :integer,
+        description: "Maximum number of connections in the pool. Default: 250 connections.",
+        suggestions: [250]
+      },
+      %{
+        key: :retry,
+        type: :integer,
+        description:
+          "Number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.",
+        suggestions: [1]
+      },
+      %{
+        key: :retry_timeout,
+        type: :integer,
+        description:
+          "Time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.",
+        suggestions: [1000]
+      },
+      %{
+        key: :await_up_timeout,
+        type: :integer,
+        description: "Timeout while `gun` will wait until connection is up. Default: 5000ms.",
+        suggestions: [5000]
+      }
+    ]
+  },
+  %{
+    group: :pleroma,
+    key: :pools,
+    type: :group,
+    description: "Advanced settings for `gun` workers pools",
+    children: [
+      %{
+        key: :federation,
+        type: :keyword,
+        description: "Settings for federation pool.",
+        children: [
+          %{
+            key: :size,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [50]
+          },
+          %{
+            key: :max_overflow,
+            type: :integer,
+            description: "Number of additional workers if pool is under load.",
+            suggestions: [10]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `gun` will wait for response.",
+            suggestions: [150_000]
+          }
+        ]
+      },
+      %{
+        key: :media,
+        type: :keyword,
+        description: "Settings for media pool.",
+        children: [
+          %{
+            key: :size,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [50]
+          },
+          %{
+            key: :max_overflow,
+            type: :integer,
+            description: "Number of additional workers if pool is under load.",
+            suggestions: [10]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `gun` will wait for response.",
+            suggestions: [150_000]
+          }
+        ]
+      },
+      %{
+        key: :upload,
+        type: :keyword,
+        description: "Settings for upload pool.",
+        children: [
+          %{
+            key: :size,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [25]
+          },
+          %{
+            key: :max_overflow,
+            type: :integer,
+            description: "Number of additional workers if pool is under load.",
+            suggestions: [5]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `gun` will wait for response.",
+            suggestions: [300_000]
+          }
+        ]
+      },
+      %{
+        key: :default,
+        type: :keyword,
+        description: "Settings for default pool.",
+        children: [
+          %{
+            key: :size,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [10]
+          },
+          %{
+            key: :max_overflow,
+            type: :integer,
+            description: "Number of additional workers if pool is under load.",
+            suggestions: [2]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `gun` will wait for response.",
+            suggestions: [10_000]
+          }
+        ]
+      }
+    ]
+  },
+  %{
+    group: :pleroma,
+    key: :hackney_pools,
+    type: :group,
+    description: "Advanced settings for `hackney` connections pools",
+    children: [
+      %{
+        key: :federation,
+        type: :keyword,
+        description: "Settings for federation pool.",
+        children: [
+          %{
+            key: :max_connections,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [50]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `hackney` will wait for response.",
+            suggestions: [150_000]
+          }
+        ]
+      },
+      %{
+        key: :media,
+        type: :keyword,
+        description: "Settings for media pool.",
+        children: [
+          %{
+            key: :max_connections,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [50]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `hackney` will wait for response.",
+            suggestions: [150_000]
+          }
+        ]
+      },
+      %{
+        key: :upload,
+        type: :keyword,
+        description: "Settings for upload pool.",
+        children: [
+          %{
+            key: :max_connections,
+            type: :integer,
+            description: "Number workers in the pool.",
+            suggestions: [25]
+          },
+          %{
+            key: :timeout,
+            type: :integer,
+            description: "Timeout while `hackney` will wait for response.",
+            suggestions: [300_000]
+          }
+        ]
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: :restrict_unauthenticated,

config/test.exs

@@ -90,6 +90,8 @@
 
 config :pleroma, :modules, runtime_dir: "test/fixtures/modules"
 
+config :pleroma, Pleroma.Gun, Pleroma.GunMock
+
 config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: true
 
 config :pleroma, Pleroma.Plugs.RemoteIp, enabled: false

docs/API/admin_api.md

@@ -841,6 +841,8 @@ Some modifications are necessary to save the config settings correctly:
 Most of the settings will be applied in `runtime`, this means that you don't need to restart the instance. But some settings are applied in `compile time` and require a reboot of the instance, such as:
 - all settings inside these keys:
   - `:hackney_pools`
+  - `:connections_pool`
+  - `:pools`
   - `:chat`
 - partially settings inside these keys:
   - `:seconds_valid` in `Pleroma.Captcha`

docs/configuration/cheatsheet.md

@@ -369,8 +369,7 @@ Available caches:
 * `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`)
 * `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`)
 * `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default`
-* `adapter`: array of hackney options
+* `adapter`: array of adapter options
-
 
 ### :hackney_pools
 
@@ -389,6 +388,42 @@ For each pool, the options are:
 * `timeout` - retention duration for connections
 
 
+### :connections_pool
+
+*For `gun` adapter*
+
+Advanced settings for connections pool. Pool with opened connections. These connections can be reused in worker pools.
+
+For big instances it's recommended to increase `config :pleroma, :connections_pool, max_connections: 500` up to 500-1000.
+It will increase memory usage, but federation would work faster.
+
+* `:checkin_timeout` - timeout to checkin connection from pool. Default: 250ms.
+* `:max_connections` - maximum number of connections in the pool. Default: 250 connections.
+* `:retry` - number of retries, while `gun` will try to reconnect if connection goes down. Default: 1.
+* `:retry_timeout` - time between retries when `gun` will try to reconnect in milliseconds. Default: 1000ms.
+* `:await_up_timeout` - timeout while `gun` will wait until connection is up. Default: 5000ms.
+
+### :pools
+
+*For `gun` adapter*
+
+Advanced settings for workers pools.
+
+There are four pools used:
+
+* `:federation` for the federation jobs.
+  You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs.
+* `:media` for rich media, media proxy
+* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`)
+* `:default` for other requests
+
+For each pool, the options are:
+
+* `:size` - how much workers the pool can hold
+* `:timeout` - timeout while `gun` will wait for response
+* `:max_overflow` - additional workers if pool is under load
+
+
 ## Captcha
 
 ### Pleroma.Captcha

lib/mix/tasks/pleroma/benchmark.ex

@@ -75,4 +75,43 @@ def run(["render_timeline", nickname | _] = args) do
       inputs: inputs
     )
   end
+
+  def run(["adapters"]) do
+    start_pleroma()
+
+    :ok =
+      Pleroma.Gun.Conn.open(
+        "https://httpbin.org/stream-bytes/1500",
+        :gun_connections
+      )
+
+    Process.sleep(1_500)
+
+    Benchee.run(
+      %{
+        "Without conn and without pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [pool: :no_pool, receive_conn: false]
+            )
+        end,
+        "Without conn and with pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [receive_conn: false]
+            )
+        end,
+        "With reused conn and without pool" => fn ->
+          {:ok, %Tesla.Env{}} =
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
+              adapter: [pool: :no_pool]
+            )
+        end,
+        "With reused conn and with pool" => fn ->
+          {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
+        end
+      },
+      parallel: 10
+    )
+  end
 end

lib/mix/tasks/pleroma/emoji.ex

@@ -4,13 +4,13 @@
 
 defmodule Mix.Tasks.Pleroma.Emoji do
   use Mix.Task
+  import Mix.Pleroma
 
   @shortdoc "Manages emoji packs"
   @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")
 
   def run(["ls-packs" | args]) do
-    Mix.Pleroma.start_pleroma()
+    start_pleroma()
-    Application.ensure_all_started(:hackney)
 
     {options, [], []} = parse_global_opts(args)
 
@@ -36,8 +36,7 @@ def run(["ls-packs" | args]) do
   end
 
   def run(["get-packs" | args]) do
-    Mix.Pleroma.start_pleroma()
+    start_pleroma()
-    Application.ensure_all_started(:hackney)
 
     {options, pack_names, []} = parse_global_opts(args)
 
@@ -135,7 +134,7 @@ def run(["get-packs" | args]) do
   end
 
   def run(["gen-pack", src]) do
-    Application.ensure_all_started(:hackney)
+    start_pleroma()
 
     proposed_name = Path.basename(src) |> Path.rootname()
     name = String.trim(IO.gets("Pack name [#{proposed_name}]: "))

lib/pleroma/application.ex

@@ -3,8 +3,12 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Application do
-  import Cachex.Spec
   use Application
+
+  import Cachex.Spec
+
+  alias Pleroma.Config
+
   require Logger
 
   @name Mix.Project.config()[:name]
@@ -18,9 +22,9 @@ def named_version, do: @name <> " " <> @version
   def repository, do: @repository
 
   def user_agent do
-    case Pleroma.Config.get([:http, :user_agent], :default) do
+    case Config.get([:http, :user_agent], :default) do
       :default ->
-        info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
+        info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
         named_version() <> "; " <> info
 
       custom ->
@@ -33,27 +37,51 @@ def user_agent do
   def start(_type, _args) do
     Pleroma.Config.Holder.save_default()
     Pleroma.HTML.compile_scrubbers()
-    Pleroma.Config.DeprecationWarnings.warn()
+    Config.DeprecationWarnings.warn()
     Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
     Pleroma.Repo.check_migrations_applied!()
     setup_instrumenters()
     load_custom_modules()
 
+    adapter = Application.get_env(:tesla, :adapter)
+
+    if adapter == Tesla.Adapter.Gun do
+      if version = Pleroma.OTPVersion.version() do
+        [major, minor] =
+          version
+          |> String.split(".")
+          |> Enum.map(&String.to_integer/1)
+          |> Enum.take(2)
+
+        if (major == 22 and minor < 2) or major < 22 do
+          raise "
+            !!!OTP VERSION WARNING!!!
+            You are using gun adapter with OTP version #{version}, which doesn't support correct handling of unordered certificates chains.
+            "
+        end
+      else
+        raise "
+          !!!OTP VERSION WARNING!!!
+          To support correct handling of unordered certificates chains - OTP version must be > 22.2.
+          "
+      end
+    end
+
     # Define workers and child supervisors to be supervised
     children =
       [
         Pleroma.Repo,
-        Pleroma.Config.TransferTask,
+        Config.TransferTask,
         Pleroma.Emoji,
         Pleroma.Captcha,
         Pleroma.Plugs.RateLimiter.Supervisor
       ] ++
         cachex_children() ++
-        hackney_pool_children() ++
+        http_children(adapter, @env) ++
         [
           Pleroma.Stats,
           Pleroma.JobQueueMonitor,
-          {Oban, Pleroma.Config.get(Oban)}
+          {Oban, Config.get(Oban)}
         ] ++
         task_children(@env) ++
         streamer_child(@env) ++
@@ -70,7 +98,7 @@ def start(_type, _args) do
   end
 
   def load_custom_modules do
-    dir = Pleroma.Config.get([:modules, :runtime_dir])
+    dir = Config.get([:modules, :runtime_dir])
 
     if dir && File.exists?(dir) do
       dir
@@ -111,20 +139,6 @@ defp setup_instrumenters do
     Pleroma.Web.Endpoint.Instrumenter.setup()
   end
 
-  def enabled_hackney_pools do
-    [:media] ++
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        [:federation]
-      else
-        []
-      end ++
-      if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
-        [:upload]
-      else
-        []
-      end
-  end
-
   defp cachex_children do
     [
       build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
@@ -146,7 +160,7 @@ defp idempotency_expiration,
     do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
 
   defp seconds_valid_interval,
-    do: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid]))
+    do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
 
   defp build_cachex(type, opts),
     do: %{
@@ -155,7 +169,7 @@ defp build_cachex(type, opts),
       type: :worker
     }
 
-  defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled])
+  defp chat_enabled?, do: Config.get([:chat, :enabled])
 
   defp streamer_child(env) when env in [:test, :benchmark], do: []
 
@@ -169,13 +183,6 @@ defp chat_child(_env, true) do
 
   defp chat_child(_, _), do: []
 
-  defp hackney_pool_children do
-    for pool <- enabled_hackney_pools() do
-      options = Pleroma.Config.get([:hackney_pools, pool])
-      :hackney_pool.child_spec(pool, options)
-    end
-  end
-
   defp task_children(:test) do
     [
       %{
@@ -200,4 +207,31 @@ defp task_children(_) do
       }
     ]
   end
+
+  # start hackney and gun pools in tests
+  defp http_children(_, :test) do
+    hackney_options = Config.get([:hackney_pools, :federation])
+    hackney_pool = :hackney_pool.child_spec(:federation, hackney_options)
+    [hackney_pool, Pleroma.Pool.Supervisor]
+  end
+
+  defp http_children(Tesla.Adapter.Hackney, _) do
+    pools = [:federation, :media]
+
+    pools =
+      if Config.get([Pleroma.Upload, :proxy_remote]) do
+        [:upload | pools]
+      else
+        pools
+      end
+
+    for pool <- pools do
+      options = Config.get([:hackney_pools, pool])
+      :hackney_pool.child_spec(pool, options)
+    end
+  end
+
+  defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor]
+
+  defp http_children(_, _), do: []
 end

lib/pleroma/config/config_db.ex

@@ -278,8 +278,6 @@ defp do_convert({:proxy_url, {type, host, port}}) do
     }
   end
 
-  defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]}
-
   defp do_convert(entity) when is_tuple(entity) do
     value =
       entity
@@ -323,15 +321,6 @@ defp do_transform(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}
     {:proxy_url, {do_transform_string(type), parse_host(host), port}}
   end
 
-  defp do_transform(%{"tuple" => [":partial_chain", entity]}) do
-    {partial_chain, []} =
-      entity
-      |> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "")
-      |> Code.eval_string()
-
-    {:partial_chain, partial_chain}
-  end
-
   defp do_transform(%{"tuple" => entity}) do
     Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end)
   end

lib/pleroma/config/transfer_task.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Config.TransferTask do
   use Task
 
+  alias Pleroma.Config
   alias Pleroma.ConfigDB
   alias Pleroma.Repo
 
@@ -18,7 +19,9 @@ defmodule Pleroma.Config.TransferTask do
     {:pleroma, Oban},
     {:pleroma, :rate_limit},
     {:pleroma, :markup},
-    {:plerome, :streamer}
+    {:pleroma, :streamer},
+    {:pleroma, :pools},
+    {:pleroma, :connections_pool}
   ]
 
   @reboot_time_subkeys [
@@ -32,45 +35,33 @@ defmodule Pleroma.Config.TransferTask do
     {:pleroma, :gopher, [:enabled]}
   ]
 
-  @reject [nil, :prometheus]
-
   def start_link(_) do
     load_and_update_env()
-    if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
+    if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo)
     :ignore
   end
 
-  @spec load_and_update_env([ConfigDB.t()]) :: :ok | false
+  @spec load_and_update_env([ConfigDB.t()], boolean()) :: :ok
-  def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
+  def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do
-    with {:configurable, true} <-
+    with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do
-           {:configurable, Pleroma.Config.get(:configurable_from_database)},
-         true <- Ecto.Adapters.SQL.table_exists?(Repo, "config"),
-         started_applications <- Application.started_applications() do
       # We need to restart applications for loaded settings take effect
 
-      in_db = Repo.all(ConfigDB)
+      # TODO: some problem with prometheus after restart!
-
+      reject_restart =
-      with_deleted = in_db ++ deleted
+        if restart_pleroma? do
-
+          [nil, :prometheus]
-      reject_for_restart = if restart_pleroma?, do: @reject, else: [:pleroma | @reject]
-
-      applications =
-        with_deleted
-        |> Enum.map(&merge_and_update(&1))
-        |> Enum.uniq()
-        # TODO: some problem with prometheus after restart!
-        |> Enum.reject(&(&1 in reject_for_restart))
-
-      # to be ensured that pleroma will be restarted last
-      applications =
-        if :pleroma in applications do
-          List.delete(applications, :pleroma) ++ [:pleroma]
         else
-          Restarter.Pleroma.rebooted()
+          [:pleroma, nil, :prometheus]
-          applications
         end
 
-      Enum.each(applications, &restart(started_applications, &1, Pleroma.Config.get(:env)))
+      started_applications = Application.started_applications()
+
+      (Repo.all(ConfigDB) ++ deleted_settings)
+      |> Enum.map(&merge_and_update/1)
+      |> Enum.uniq()
+      |> Enum.reject(&(&1 in reject_restart))
+      |> maybe_set_pleroma_last()
+      |> Enum.each(&restart(started_applications, &1, Config.get(:env)))
 
       :ok
     else
@@ -78,42 +69,54 @@ def load_and_update_env(deleted \\ [], restart_pleroma? \\ true) do
     end
   end
 
+  defp maybe_set_pleroma_last(apps) do
+    # to be ensured that pleroma will be restarted last
+    if :pleroma in apps do
+      apps
+      |> List.delete(:pleroma)
+      |> List.insert_at(-1, :pleroma)
+    else
+      Restarter.Pleroma.rebooted()
+      apps
+    end
+  end
+
+  defp group_for_restart(:logger, key, _, merged_value) do
+    # change logger configuration in runtime, without restart
+    if Keyword.keyword?(merged_value) and
+         key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
+      Logger.configure_backend(key, merged_value)
+    else
+      Logger.configure([{key, merged_value}])
+    end
+
+    nil
+  end
+
+  defp group_for_restart(group, _, _, _) when group != :pleroma, do: group
+
+  defp group_for_restart(group, key, value, _) do
+    if pleroma_need_restart?(group, key, value), do: group
+  end
+
   defp merge_and_update(setting) do
     try do
       key = ConfigDB.from_string(setting.key)
       group = ConfigDB.from_string(setting.group)
 
-      default = Pleroma.Config.Holder.default_config(group, key)
+      default = Config.Holder.default_config(group, key)
       value = ConfigDB.from_binary(setting.value)
 
       merged_value =
-        if Ecto.get_meta(setting, :state) == :deleted do
+        cond do
-          default
+          Ecto.get_meta(setting, :state) == :deleted -> default
-        else
+          can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value)
-          if can_be_merged?(default, value) do
+          true -> value
-            ConfigDB.merge_group(group, key, default, value)
-          else
-            value
-          end
         end
 
       :ok = update_env(group, key, merged_value)
 
-      if group != :logger do
+      group_for_restart(group, key, value, merged_value)
-        if group != :pleroma or pleroma_need_restart?(group, key, value) do
-          group
-        end
-      else
-        # change logger configuration in runtime, without restart
-        if Keyword.keyword?(merged_value) and
-             key not in [:compile_time_application, :backends, :compile_time_purge_matching] do
-          Logger.configure_backend(key, merged_value)
-        else
-          Logger.configure([{key, merged_value}])
-        end
-
-        nil
-      end
     rescue
       error ->
         error_msg =

lib/pleroma/gun/api.ex

@@ -0,0 +1,45 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun.API do
+  @behaviour Pleroma.Gun
+
+  alias Pleroma.Gun
+
+  @gun_keys [
+    :connect_timeout,
+    :http_opts,
+    :http2_opts,
+    :protocols,
+    :retry,
+    :retry_timeout,
+    :trace,
+    :transport,
+    :tls_opts,
+    :tcp_opts,
+    :socks_opts,
+    :ws_opts
+  ]
+
+  @impl Gun
+  def open(host, port, opts \\ %{}), do: :gun.open(host, port, Map.take(opts, @gun_keys))
+
+  @impl Gun
+  defdelegate info(pid), to: :gun
+
+  @impl Gun
+  defdelegate close(pid), to: :gun
+
+  @impl Gun
+  defdelegate await_up(pid, timeout \\ 5_000), to: :gun
+
+  @impl Gun
+  defdelegate connect(pid, opts), to: :gun
+
+  @impl Gun
+  defdelegate await(pid, ref), to: :gun
+
+  @impl Gun
+  defdelegate set_owner(pid, owner), to: :gun
+end

lib/pleroma/gun/conn.ex

@@ -0,0 +1,196 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun.Conn do
+  @moduledoc """
+  Struct for gun connection data
+  """
+  alias Pleroma.Gun
+  alias Pleroma.Pool.Connections
+
+  require Logger
+
+  @type gun_state :: :up | :down
+  @type conn_state :: :active | :idle
+
+  @type t :: %__MODULE__{
+          conn: pid(),
+          gun_state: gun_state(),
+          conn_state: conn_state(),
+          used_by: [pid()],
+          last_reference: pos_integer(),
+          crf: float(),
+          retries: pos_integer()
+        }
+
+  defstruct conn: nil,
+            gun_state: :open,
+            conn_state: :init,
+            used_by: [],
+            last_reference: 0,
+            crf: 1,
+            retries: 0
+
+  @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil
+  def open(url, name, opts \\ [])
+  def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts)
+
+  def open(%URI{} = uri, name, opts) do
+    pool_opts = Pleroma.Config.get([:connections_pool], [])
+
+    opts =
+      opts
+      |> Enum.into(%{})
+      |> Map.put_new(:retry, pool_opts[:retry] || 1)
+      |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000)
+      |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
+      |> maybe_add_tls_opts(uri)
+
+    key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
+
+    conn_pid =
+      if Connections.count(name) < opts[:max_connection] do
+        do_open(uri, opts)
+      else
+        close_least_used_and_do_open(name, uri, opts)
+      end
+
+    if is_pid(conn_pid) do
+      conn = %Pleroma.Gun.Conn{
+        conn: conn_pid,
+        gun_state: :up,
+        conn_state: :active,
+        last_reference: :os.system_time(:second)
+      }
+
+      :ok = Gun.set_owner(conn_pid, Process.whereis(name))
+      Connections.add_conn(name, key, conn)
+    end
+  end
+
+  defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts
+
+  defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do
+    tls_opts = [
+      verify: :verify_peer,
+      cacertfile: CAStore.file_path(),
+      depth: 20,
+      reuse_sessions: false,
+      verify_fun:
+        {&:ssl_verify_hostname.verify_fun/3,
+         [check_hostname: Pleroma.HTTP.Connection.format_host(host)]}
+    ]
+
+    tls_opts =
+      if Keyword.keyword?(opts[:tls_opts]) do
+        Keyword.merge(tls_opts, opts[:tls_opts])
+      else
+        tls_opts
+      end
+
+    Map.put(opts, :tls_opts, tls_opts)
+  end
+
+  defp do_open(uri, %{proxy: {proxy_host, proxy_port}} = opts) do
+    connect_opts =
+      uri
+      |> destination_opts()
+      |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
+
+    with open_opts <- Map.delete(opts, :tls_opts),
+         {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
+         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
+         stream <- Gun.connect(conn, connect_opts),
+         {:response, :fin, 200, _} <- Gun.await(conn, stream) do
+      conn
+    else
+      error ->
+        Logger.warn(
+          "Opening proxied connection to #{compose_uri_log(uri)} failed with error #{
+            inspect(error)
+          }"
+        )
+
+        error
+    end
+  end
+
+  defp do_open(uri, %{proxy: {proxy_type, proxy_host, proxy_port}} = opts) do
+    version =
+      proxy_type
+      |> to_string()
+      |> String.last()
+      |> case do
+        "4" -> 4
+        _ -> 5
+      end
+
+    socks_opts =
+      uri
+      |> destination_opts()
+      |> add_http2_opts(uri.scheme, Map.get(opts, :tls_opts, []))
+      |> Map.put(:version, version)
+
+    opts =
+      opts
+      |> Map.put(:protocols, [:socks])
+      |> Map.put(:socks_opts, socks_opts)
+
+    with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
+         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+      conn
+    else
+      error ->
+        Logger.warn(
+          "Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{
+            inspect(error)
+          }"
+        )
+
+        error
+    end
+  end
+
+  defp do_open(%URI{host: host, port: port} = uri, opts) do
+    host = Pleroma.HTTP.Connection.parse_host(host)
+
+    with {:ok, conn} <- Gun.open(host, port, opts),
+         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+      conn
+    else
+      error ->
+        Logger.warn(
+          "Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}"
+        )
+
+        error
+    end
+  end
+
+  defp destination_opts(%URI{host: host, port: port}) do
+    host = Pleroma.HTTP.Connection.parse_host(host)
+    %{host: host, port: port}
+  end
+
+  defp add_http2_opts(opts, "https", tls_opts) do
+    Map.merge(opts, %{protocols: [:http2], transport: :tls, tls_opts: tls_opts})
+  end
+
+  defp add_http2_opts(opts, _, _), do: opts
+
+  defp close_least_used_and_do_open(name, uri, opts) do
+    with [{key, conn} | _conns] <- Connections.get_unused_conns(name),
+         :ok <- Gun.close(conn.conn) do
+      Connections.remove_conn(name, key)
+
+      do_open(uri, opts)
+    else
+      [] -> {:error, :pool_overflowed}
+    end
+  end
+
+  def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
+    "#{scheme}://#{host}#{path}"
+  end
+end

lib/pleroma/gun/gun.ex

@@ -0,0 +1,31 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Gun do
+  @callback open(charlist(), pos_integer(), map()) :: {:ok, pid()}
+  @callback info(pid()) :: map()
+  @callback close(pid()) :: :ok
+  @callback await_up(pid, pos_integer()) :: {:ok, atom()} | {:error, atom()}
+  @callback connect(pid(), map()) :: reference()
+  @callback await(pid(), reference()) :: {:response, :fin, 200, []}
+  @callback set_owner(pid(), pid()) :: :ok
+
+  @api Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
+
+  defp api, do: @api
+
+  def open(host, port, opts), do: api().open(host, port, opts)
+
+  def info(pid), do: api().info(pid)
+
+  def close(pid), do: api().close(pid)
+
+  def await_up(pid, timeout \\ 5_000), do: api().await_up(pid, timeout)
+
+  def connect(pid, opts), do: api().connect(pid, opts)
+
+  def await(pid, ref), do: api().await(pid, ref)
+
+  def set_owner(pid, owner), do: api().set_owner(pid, owner)
+end

lib/pleroma/http/adapter_helper.ex

@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.AdapterHelper do
+  alias Pleroma.HTTP.Connection
+
+  @type proxy ::
+          {Connection.host(), pos_integer()}
+          | {Connection.proxy_type(), Connection.host(), pos_integer()}
+
+  @callback options(keyword(), URI.t()) :: keyword()
+  @callback after_request(keyword()) :: :ok
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(opts, _uri) do
+    proxy = Pleroma.Config.get([:http, :proxy_url], nil)
+    maybe_add_proxy(opts, format_proxy(proxy))
+  end
+
+  @spec maybe_get_conn(URI.t(), keyword()) :: keyword()
+  def maybe_get_conn(_uri, opts), do: opts
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(_opts), do: :ok
+
+  @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
+  def format_proxy(nil), do: nil
+
+  def format_proxy(proxy_url) do
+    case Connection.parse_proxy(proxy_url) do
+      {:ok, host, port} -> {host, port}
+      {:ok, type, host, port} -> {type, host, port}
+      _ -> nil
+    end
+  end
+
+  @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
+  def maybe_add_proxy(opts, nil), do: opts
+  def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
+end

lib/pleroma/http/adapter_helper/gun.ex

@@ -0,0 +1,77 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.AdapterHelper.Gun do
+  @behaviour Pleroma.HTTP.AdapterHelper
+
+  alias Pleroma.HTTP.AdapterHelper
+  alias Pleroma.Pool.Connections
+
+  require Logger
+
+  @defaults [
+    connect_timeout: 5_000,
+    domain_lookup_timeout: 5_000,
+    tls_handshake_timeout: 5_000,
+    retry: 1,
+    retry_timeout: 1000,
+    await_up_timeout: 5_000
+  ]
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(incoming_opts \\ [], %URI{} = uri) do
+    proxy =
+      Pleroma.Config.get([:http, :proxy_url])
+      |> AdapterHelper.format_proxy()
+
+    config_opts = Pleroma.Config.get([:http, :adapter], [])
+
+    @defaults
+    |> Keyword.merge(config_opts)
+    |> add_scheme_opts(uri)
+    |> AdapterHelper.maybe_add_proxy(proxy)
+    |> maybe_get_conn(uri, incoming_opts)
+  end
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(opts) do
+    if opts[:conn] && opts[:body_as] != :chunks do
+      Connections.checkout(opts[:conn], self(), :gun_connections)
+    end
+
+    :ok
+  end
+
+  defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
+
+  defp add_scheme_opts(opts, %{scheme: "https"}) do
+    opts
+    |> Keyword.put(:certificates_verification, true)
+    |> Keyword.put(:tls_opts, log_level: :warning)
+  end
+
+  defp maybe_get_conn(adapter_opts, uri, incoming_opts) do
+    {receive_conn?, opts} =
+      adapter_opts
+      |> Keyword.merge(incoming_opts)
+      |> Keyword.pop(:receive_conn, true)
+
+    if Connections.alive?(:gun_connections) and receive_conn? do
+      checkin_conn(uri, opts)
+    else
+      opts
+    end
+  end
+
+  defp checkin_conn(uri, opts) do
+    case Connections.checkin(uri, :gun_connections) do
+      nil ->
+        Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts])
+        opts
+
+      conn when is_pid(conn) ->
+        Keyword.merge(opts, conn: conn, close_conn: false)
+    end
+  end
+end

lib/pleroma/http/adapter_helper/hackney.ex

@@ -0,0 +1,43 @@
+defmodule Pleroma.HTTP.AdapterHelper.Hackney do
+  @behaviour Pleroma.HTTP.AdapterHelper
+
+  @defaults [
+    connect_timeout: 10_000,
+    recv_timeout: 20_000,
+    follow_redirect: true,
+    force_redirect: true,
+    pool: :federation
+  ]
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(connection_opts \\ [], %URI{} = uri) do
+    proxy = Pleroma.Config.get([:http, :proxy_url])
+
+    config_opts = Pleroma.Config.get([:http, :adapter], [])
+
+    @defaults
+    |> Keyword.merge(config_opts)
+    |> Keyword.merge(connection_opts)
+    |> add_scheme_opts(uri)
+    |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
+  end
+
+  defp add_scheme_opts(opts, %URI{scheme: "http"}), do: opts
+
+  defp add_scheme_opts(opts, %URI{scheme: "https", host: host}) do
+    ssl_opts = [
+      ssl_options: [
+        # Workaround for remote server certificate chain issues
+        partial_chain: &:hackney_connect.partial_chain/1,
+
+        # We don't support TLS v1.3 yet
+        versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"],
+        server_name_indication: to_charlist(host)
+      ]
+    ]
+
+    Keyword.merge(opts, ssl_opts)
+  end
+
+  def after_request(_), do: :ok
+end

lib/pleroma/http/connection.ex

@@ -4,40 +4,121 @@
 
 defmodule Pleroma.HTTP.Connection do
   @moduledoc """
-  Connection for http-requests.
+  Configure Tesla.Client with default and customized adapter options.
   """
 
-  @hackney_options [
+  alias Pleroma.Config
-    connect_timeout: 10_000,
+  alias Pleroma.HTTP.AdapterHelper
-    recv_timeout: 20_000,
+
-    follow_redirect: true,
+  require Logger
-    force_redirect: true,
+
-    pool: :federation
+  @defaults [pool: :federation]
-  ]
+
-  @adapter Application.get_env(:tesla, :adapter)
+  @type ip_address :: ipv4_address() | ipv6_address()
+  @type ipv4_address :: {0..255, 0..255, 0..255, 0..255}
+  @type ipv6_address ::
+          {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535}
+  @type proxy_type() :: :socks4 | :socks5
+  @type host() :: charlist() | ip_address()
 
   @doc """
-  Configure a client connection
+  Merge default connection & adapter options with received ones.
-
-  # Returns
-
-  Tesla.Env.client
   """
-  @spec new(Keyword.t()) :: Tesla.Env.client()
+
-  def new(opts \\ []) do
+  @spec options(URI.t(), keyword()) :: keyword()
-    Tesla.client([], {@adapter, hackney_options(opts)})
+  def options(%URI{} = uri, opts \\ []) do
+    @defaults
+    |> pool_timeout()
+    |> Keyword.merge(opts)
+    |> adapter_helper().options(uri)
   end
 
-  # fetch Hackney options
+  defp pool_timeout(opts) do
-  #
+    {config_key, default} =
-  def hackney_options(opts) do
+      if adapter() == Tesla.Adapter.Gun do
-    options = Keyword.get(opts, :adapter, [])
+        {:pools, Config.get([:pools, :default, :timeout])}
-    adapter_options = Pleroma.Config.get([:http, :adapter], [])
+      else
-    proxy_url = Pleroma.Config.get([:http, :proxy_url], nil)
+        {:hackney_pools, 10_000}
+      end
 
-    @hackney_options
+    timeout = Config.get([config_key, opts[:pool], :timeout], default)
-    |> Keyword.merge(adapter_options)
+
-    |> Keyword.merge(options)
+    Keyword.merge(opts, timeout: timeout)
-    |> Keyword.merge(proxy: proxy_url)
+  end
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(opts), do: adapter_helper().after_request(opts)
+
+  defp adapter, do: Application.get_env(:tesla, :adapter)
+
+  defp adapter_helper do
+    case adapter() do
+      Tesla.Adapter.Gun -> AdapterHelper.Gun
+      Tesla.Adapter.Hackney -> AdapterHelper.Hackney
+      _ -> AdapterHelper
+    end
+  end
+
+  @spec parse_proxy(String.t() | tuple() | nil) ::
+          {:ok, host(), pos_integer()}
+          | {:ok, proxy_type(), host(), pos_integer()}
+          | {:error, atom()}
+          | nil
+
+  def parse_proxy(nil), do: nil
+
+  def parse_proxy(proxy) when is_binary(proxy) do
+    with [host, port] <- String.split(proxy, ":"),
+         {port, ""} <- Integer.parse(port) do
+      {:ok, parse_host(host), port}
+    else
+      {_, _} ->
+        Logger.warn("Parsing port failed #{inspect(proxy)}")
+        {:error, :invalid_proxy_port}
+
+      :error ->
+        Logger.warn("Parsing port failed #{inspect(proxy)}")
+        {:error, :invalid_proxy_port}
+
+      _ ->
+        Logger.warn("Parsing proxy failed #{inspect(proxy)}")
+        {:error, :invalid_proxy}
+    end
+  end
+
+  def parse_proxy(proxy) when is_tuple(proxy) do
+    with {type, host, port} <- proxy do
+      {:ok, type, parse_host(host), port}
+    else
+      _ ->
+        Logger.warn("Parsing proxy failed #{inspect(proxy)}")
+        {:error, :invalid_proxy}
+    end
+  end
+
+  @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address()
+  def parse_host(host) when is_list(host), do: host
+  def parse_host(host) when is_atom(host), do: to_charlist(host)
+
+  def parse_host(host) when is_binary(host) do
+    host = to_charlist(host)
+
+    case :inet.parse_address(host) do
+      {:error, :einval} -> host
+      {:ok, ip} -> ip
+    end
+  end
+
+  @spec format_host(String.t()) :: charlist()
+  def format_host(host) do
+    host_charlist = to_charlist(host)
+
+    case :inet.parse_address(host_charlist) do
+      {:error, :einval} ->
+        :idna.encode(host_charlist)
+
+      {:ok, _ip} ->
+        host_charlist
+    end
   end
 end

lib/pleroma/http/http.ex

@@ -4,21 +4,47 @@
 
 defmodule Pleroma.HTTP do
   @moduledoc """
-
+    Wrapper for `Tesla.request/2`.
   """
 
   alias Pleroma.HTTP.Connection
+  alias Pleroma.HTTP.Request
   alias Pleroma.HTTP.RequestBuilder, as: Builder
+  alias Tesla.Client
+  alias Tesla.Env
+
+  require Logger
 
   @type t :: __MODULE__
 
   @doc """
-  Builds and perform http request.
+  Performs GET request.
+
+  See `Pleroma.HTTP.request/5`
+  """
+  @spec get(Request.url() | nil, Request.headers(), keyword()) ::
+          nil | {:ok, Env.t()} | {:error, any()}
+  def get(url, headers \\ [], options \\ [])
+  def get(nil, _, _), do: nil
+  def get(url, headers, options), do: request(:get, url, "", headers, options)
+
+  @doc """
+  Performs POST request.
+
+  See `Pleroma.HTTP.request/5`
+  """
+  @spec post(Request.url(), String.t(), Request.headers(), keyword()) ::
+          {:ok, Env.t()} | {:error, any()}
+  def post(url, body, headers \\ [], options \\ []),
+    do: request(:post, url, body, headers, options)
+
+  @doc """
+  Builds and performs http request.
 
   # Arguments:
   `method` - :get, :post, :put, :delete
-  `url`
+  `url` - full url
-  `body`
+  `body` - request body
   `headers` - a keyworld list of headers, e.g. `[{"content-type", "text/plain"}]`
   `options` - custom, per-request middleware or adapter options
 
@@ -26,61 +52,66 @@ defmodule Pleroma.HTTP do
   `{:ok, %Tesla.Env{}}` or `{:error, error}`
 
   """
-  def request(method, url, body \\ "", headers \\ [], options \\ []) do
+  @spec request(atom(), Request.url(), String.t(), Request.headers(), keyword()) ::
-    try do
+          {:ok, Env.t()} | {:error, any()}
-      options =
+  def request(method, url, body, headers, options) when is_binary(url) do
-        process_request_options(options)
-        |> process_sni_options(url)
-
-      params = Keyword.get(options, :params, [])
-
-      %{}
-      |> Builder.method(method)
-      |> Builder.headers(headers)
-      |> Builder.opts(options)
-      |> Builder.url(url)
-      |> Builder.add_param(:body, :body, body)
-      |> Builder.add_param(:query, :query, params)
-      |> Enum.into([])
-      |> (&Tesla.request(Connection.new(options), &1)).()
-    rescue
-      e ->
-        {:error, e}
-    catch
-      :exit, e ->
-        {:error, e}
-    end
-  end
-
-  defp process_sni_options(options, nil), do: options
-
-  defp process_sni_options(options, url) do
     uri = URI.parse(url)
-    host = uri.host |> to_charlist()
+    adapter_opts = Connection.options(uri, options[:adapter] || [])
+    options = put_in(options[:adapter], adapter_opts)
+    params = options[:params] || []
+    request = build_request(method, headers, options, url, body, params)
 
-    case uri.scheme do
+    adapter = Application.get_env(:tesla, :adapter)
-      "https" -> options ++ [ssl: [server_name_indication: host]]
+    client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter)
-      _ -> options
+
-    end
+    pid = Process.whereis(adapter_opts[:pool])
+
+    pool_alive? =
+      if adapter == Tesla.Adapter.Gun && pid do
+        Process.alive?(pid)
+      else
+        false
+      end
+
+    request_opts =
+      adapter_opts
+      |> Enum.into(%{})
+      |> Map.put(:env, Pleroma.Config.get([:env]))
+      |> Map.put(:pool_alive?, pool_alive?)
+
+    response = request(client, request, request_opts)
+
+    Connection.after_request(adapter_opts)
+
+    response
   end
 
-  def process_request_options(options) do
+  @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()}
-    Keyword.merge(Pleroma.HTTP.Connection.hackney_options([]), options)
+  def request(%Client{} = client, request, %{env: :test}), do: request(client, request)
+
+  def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request)
+
+  def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request)
+
+  def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do
+    :poolboy.transaction(
+      pool,
+      &Pleroma.Pool.Request.execute(&1, client, request, timeout),
+      timeout
+    )
   end
 
-  @doc """
+  @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
-  Performs GET request.
+  def request(client, request), do: Tesla.request(client, request)
 
-  See `Pleroma.HTTP.request/5`
+  defp build_request(method, headers, options, url, body, params) do
-  """
+    Builder.new()
-  def get(url, headers \\ [], options \\ []),
+    |> Builder.method(method)
-    do: request(:get, url, "", headers, options)
+    |> Builder.headers(headers)
-
+    |> Builder.opts(options)
-  @doc """
+    |> Builder.url(url)
-  Performs POST request.
+    |> Builder.add_param(:body, :body, body)
-
+    |> Builder.add_param(:query, :query, params)
-  See `Pleroma.HTTP.request/5`
+    |> Builder.convert_to_keyword()
-  """
+  end
-  def post(url, body, headers \\ [], options \\ []),
-    do: request(:post, url, body, headers, options)
 end

lib/pleroma/http/request.ex

@@ -0,0 +1,23 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Request do
+  @moduledoc """
+  Request struct.
+  """
+  defstruct method: :get, url: "", query: [], headers: [], body: "", opts: []
+
+  @type method :: :head | :get | :delete | :trace | :options | :post | :put | :patch
+  @type url :: String.t()
+  @type headers :: [{String.t(), String.t()}]
+
+  @type t :: %__MODULE__{
+          method: method(),
+          url: url(),
+          query: keyword(),
+          headers: headers(),
+          body: String.t(),
+          opts: keyword()
+        }
+end

lib/pleroma/http/request_builder.ex

@@ -7,136 +7,87 @@ defmodule Pleroma.HTTP.RequestBuilder do
   Helper functions for building Tesla requests
   """
 
+  alias Pleroma.HTTP.Request
+  alias Tesla.Multipart
+
   @doc """
-  Specify the request method when building a request
+  Creates new request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - m (atom) - Request method
-
-  ## Returns
-
-  Map
   """
-  @spec method(map(), atom) :: map()
+  @spec new(Request.t()) :: Request.t()
-  def method(request, m) do
+  def new(%Request{} = request \\ %Request{}), do: request
-    Map.put_new(request, :method, m)
-  end
 
   @doc """
   Specify the request method when building a request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - u (String) - Request URL
-
-  ## Returns
-
-  Map
   """
-  @spec url(map(), String.t()) :: map()
+  @spec method(Request.t(), Request.method()) :: Request.t()
-  def url(request, u) do
+  def method(request, m), do: %{request | method: m}
-    Map.put_new(request, :url, u)
+
-  end
+  @doc """
+  Specify the request method when building a request
+  """
+  @spec url(Request.t(), Request.url()) :: Request.t()
+  def url(request, u), do: %{request | url: u}
 
   @doc """
   Add headers to the request
   """
-  @spec headers(map(), list(tuple)) :: map()
+  @spec headers(Request.t(), Request.headers()) :: Request.t()
-  def headers(request, header_list) do
+  def headers(request, headers) do
-    header_list =
+    headers_list =
       if Pleroma.Config.get([:http, :send_user_agent]) do
-        header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+        [{"user-agent", Pleroma.Application.user_agent()} | headers]
       else
-        header_list
+        headers
       end
 
-    Map.put_new(request, :headers, header_list)
+    %{request | headers: headers_list}
   end
 
   @doc """
   Add custom, per-request middleware or adapter options to the request
   """
-  @spec opts(map(), Keyword.t()) :: map()
+  @spec opts(Request.t(), keyword()) :: Request.t()
-  def opts(request, options) do
+  def opts(request, options), do: %{request | opts: options}
-    Map.put_new(request, :opts, options)
-  end
 
   @doc """
   Add optional parameters to the request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - definitions (Map) - Map of parameter name to parameter location.
-  - options (KeywordList) - The provided optional parameters
-
-  ## Returns
-
-  Map
   """
-  @spec add_optional_params(map(), %{optional(atom) => atom}, keyword()) :: map()
+  @spec add_param(Request.t(), atom(), atom(), any()) :: Request.t()
-  def add_optional_params(request, _, []), do: request
+  def add_param(request, :query, :query, values), do: %{request | query: values}
 
-  def add_optional_params(request, definitions, [{key, value} | tail]) do
+  def add_param(request, :body, :body, value), do: %{request | body: value}
-    case definitions do
-      %{^key => location} ->
-        request
-        |> add_param(location, key, value)
-        |> add_optional_params(definitions, tail)
-
-      _ ->
-        add_optional_params(request, definitions, tail)
-    end
-  end
-
-  @doc """
-  Add optional parameters to the request
-
-  ## Parameters
-
-  - request (Map) - Collected request options
-  - location (atom) - Where to put the parameter
-  - key (atom) - The name of the parameter
-  - value (any) - The value of the parameter
-
-  ## Returns
-
-  Map
-  """
-  @spec add_param(map(), atom, atom, any()) :: map()
-  def add_param(request, :query, :query, values), do: Map.put(request, :query, values)
-
-  def add_param(request, :body, :body, value), do: Map.put(request, :body, value)
 
   def add_param(request, :body, key, value) do
     request
-    |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
+    |> Map.put(:body, Multipart.new())
     |> Map.update!(
       :body,
-      &Tesla.Multipart.add_field(
+      &Multipart.add_field(
         &1,
         key,
         Jason.encode!(value),
-        headers: [{:"Content-Type", "application/json"}]
+        headers: [{"content-type", "application/json"}]
       )
     )
   end
 
   def add_param(request, :file, name, path) do
     request
-    |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
+    |> Map.put(:body, Multipart.new())
-    |> Map.update!(:body, &Tesla.Multipart.add_file(&1, path, name: name))
+    |> Map.update!(:body, &Multipart.add_file(&1, path, name: name))
   end
 
   def add_param(request, :form, name, value) do
-    request
+    Map.update(request, :body, %{name => value}, &Map.put(&1, name, value))
-    |> Map.update(:body, %{name => value}, &Map.put(&1, name, value))
   end
 
   def add_param(request, location, key, value) do
     Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}]))
   end
+
+  def convert_to_keyword(request) do
+    request
+    |> Map.from_struct()
+    |> Enum.into([])
+  end
 end

lib/pleroma/object/fetcher.ex

@@ -141,7 +141,7 @@ defp make_signature(id, date) do
         date: date
       })
 
-    [{:Signature, signature}]
+    [{"signature", signature}]
   end
 
   defp sign_fetch(headers, id, date) do
@@ -154,7 +154,7 @@ defp sign_fetch(headers, id, date) do
 
   defp maybe_date_fetch(headers, date) do
     if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
-      headers ++ [{:Date, date}]
+      headers ++ [{"date", date}]
     else
       headers
     end
@@ -166,7 +166,7 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
     date = Pleroma.Signature.signed_date()
 
     headers =
-      [{:Accept, "application/activity+json"}]
+      [{"accept", "application/activity+json"}]
       |> maybe_date_fetch(date)
       |> sign_fetch(id, date)
 

lib/pleroma/otp_version.ex

@@ -0,0 +1,28 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.OTPVersion do
+  @spec version() :: String.t() | nil
+  def version do
+    # OTP Version https://erlang.org/doc/system_principles/versions.html#otp-version
+    [
+      Path.join(:code.root_dir(), "OTP_VERSION"),
+      Path.join([:code.root_dir(), "releases", :erlang.system_info(:otp_release), "OTP_VERSION"])
+    ]
+    |> get_version_from_files()
+  end
+
+  @spec get_version_from_files([Path.t()]) :: String.t() | nil
+  def get_version_from_files([]), do: nil
+
+  def get_version_from_files([path | paths]) do
+    if File.exists?(path) do
+      path
+      |> File.read!()
+      |> String.replace(~r/\r|\n|\s/, "")
+    else
+      get_version_from_files(paths)
+    end
+  end
+end

lib/pleroma/pool/connections.ex

@@ -0,0 +1,283 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool.Connections do
+  use GenServer
+
+  alias Pleroma.Config
+  alias Pleroma.Gun
+
+  require Logger
+
+  @type domain :: String.t()
+  @type conn :: Pleroma.Gun.Conn.t()
+
+  @type t :: %__MODULE__{
+          conns: %{domain() => conn()},
+          opts: keyword()
+        }
+
+  defstruct conns: %{}, opts: []
+
+  @spec start_link({atom(), keyword()}) :: {:ok, pid()}
+  def start_link({name, opts}) do
+    GenServer.start_link(__MODULE__, opts, name: name)
+  end
+
+  @impl true
+  def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}}
+
+  @spec checkin(String.t() | URI.t(), atom()) :: pid() | nil
+  def checkin(url, name)
+  def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name)
+
+  def checkin(%URI{} = uri, name) do
+    timeout = Config.get([:connections_pool, :checkin_timeout], 250)
+
+    GenServer.call(name, {:checkin, uri}, timeout)
+  end
+
+  @spec alive?(atom()) :: boolean()
+  def alive?(name) do
+    if pid = Process.whereis(name) do
+      Process.alive?(pid)
+    else
+      false
+    end
+  end
+
+  @spec get_state(atom()) :: t()
+  def get_state(name) do
+    GenServer.call(name, :state)
+  end
+
+  @spec count(atom()) :: pos_integer()
+  def count(name) do
+    GenServer.call(name, :count)
+  end
+
+  @spec get_unused_conns(atom()) :: [{domain(), conn()}]
+  def get_unused_conns(name) do
+    GenServer.call(name, :unused_conns)
+  end
+
+  @spec checkout(pid(), pid(), atom()) :: :ok
+  def checkout(conn, pid, name) do
+    GenServer.cast(name, {:checkout, conn, pid})
+  end
+
+  @spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok
+  def add_conn(name, key, conn) do
+    GenServer.cast(name, {:add_conn, key, conn})
+  end
+
+  @spec remove_conn(atom(), String.t()) :: :ok
+  def remove_conn(name, key) do
+    GenServer.cast(name, {:remove_conn, key})
+  end
+
+  @impl true
+  def handle_cast({:add_conn, key, conn}, state) do
+    state = put_in(state.conns[key], conn)
+
+    Process.monitor(conn.conn)
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_cast({:checkout, conn_pid, pid}, state) do
+    state =
+      with true <- Process.alive?(conn_pid),
+           {key, conn} <- find_conn(state.conns, conn_pid),
+           used_by <- List.keydelete(conn.used_by, pid, 0) do
+        conn_state = if used_by == [], do: :idle, else: conn.conn_state
+
+        put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by})
+      else
+        false ->
+          Logger.debug("checkout for closed conn #{inspect(conn_pid)}")
+          state
+
+        nil ->
+          Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state")
+          state
+      end
+
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_cast({:remove_conn, key}, state) do
+    state = put_in(state.conns, Map.delete(state.conns, key))
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_call({:checkin, uri}, from, state) do
+    key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
+
+    case state.conns[key] do
+      %{conn: pid, gun_state: :up} = conn ->
+        time = :os.system_time(:second)
+        last_reference = time - conn.last_reference
+        crf = crf(last_reference, 100, conn.crf)
+
+        state =
+          put_in(state.conns[key], %{
+            conn
+            | last_reference: time,
+              crf: crf,
+              conn_state: :active,
+              used_by: [from | conn.used_by]
+          })
+
+        {:reply, pid, state}
+
+      %{gun_state: :down} ->
+        {:reply, nil, state}
+
+      nil ->
+        {:reply, nil, state}
+    end
+  end
+
+  @impl true
+  def handle_call(:state, _from, state), do: {:reply, state, state}
+
+  @impl true
+  def handle_call(:count, _from, state) do
+    {:reply, Enum.count(state.conns), state}
+  end
+
+  @impl true
+  def handle_call(:unused_conns, _from, state) do
+    unused_conns =
+      state.conns
+      |> Enum.filter(&filter_conns/1)
+      |> Enum.sort(&sort_conns/2)
+
+    {:reply, unused_conns, state}
+  end
+
+  defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true
+  defp filter_conns(_), do: false
+
+  defp sort_conns({_, c1}, {_, c2}) do
+    c1.crf <= c2.crf and c1.last_reference <= c2.last_reference
+  end
+
+  @impl true
+  def handle_info({:gun_up, conn_pid, _protocol}, state) do
+    %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid)
+
+    host =
+      case :inet.ntoa(host) do
+        {:error, :einval} -> host
+        ip -> ip
+      end
+
+    key = "#{scheme}:#{host}:#{port}"
+
+    state =
+      with {key, conn} <- find_conn(state.conns, conn_pid, key),
+           {true, key} <- {Process.alive?(conn_pid), key} do
+        put_in(state.conns[key], %{
+          conn
+          | gun_state: :up,
+            conn_state: :active,
+            retries: 0
+        })
+      else
+        {false, key} ->
+          put_in(
+            state.conns,
+            Map.delete(state.conns, key)
+          )
+
+        nil ->
+          :ok = Gun.close(conn_pid)
+
+          state
+      end
+
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do
+    retries = Config.get([:connections_pool, :retry], 1)
+    # we can't get info on this pid, because pid is dead
+    state =
+      with {key, conn} <- find_conn(state.conns, conn_pid),
+           {true, key} <- {Process.alive?(conn_pid), key} do
+        if conn.retries == retries do
+          :ok = Gun.close(conn.conn)
+
+          put_in(
+            state.conns,
+            Map.delete(state.conns, key)
+          )
+        else
+          put_in(state.conns[key], %{
+            conn
+            | gun_state: :down,
+              retries: conn.retries + 1
+          })
+        end
+      else
+        {false, key} ->
+          put_in(
+            state.conns,
+            Map.delete(state.conns, key)
+          )
+
+        nil ->
+          Logger.debug(":gun_down for conn which isn't found in state")
+
+          state
+      end
+
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do
+    Logger.debug("received DOWM message for #{inspect(conn_pid)} reason -> #{inspect(reason)}")
+
+    state =
+      with {key, conn} <- find_conn(state.conns, conn_pid) do
+        Enum.each(conn.used_by, fn {pid, _ref} ->
+          Process.exit(pid, reason)
+        end)
+
+        put_in(
+          state.conns,
+          Map.delete(state.conns, key)
+        )
+      else
+        nil ->
+          Logger.debug(":DOWN for conn which isn't found in state")
+
+          state
+      end
+
+    {:noreply, state}
+  end
+
+  defp find_conn(conns, conn_pid) do
+    Enum.find(conns, fn {_key, conn} ->
+      conn.conn == conn_pid
+    end)
+  end
+
+  defp find_conn(conns, conn_pid, conn_key) do
+    Enum.find(conns, fn {key, conn} ->
+      key == conn_key and conn.conn == conn_pid
+    end)
+  end
+
+  def crf(current, steps, crf) do
+    1 + :math.pow(0.5, current / steps) * crf
+  end
+end

lib/pleroma/pool/pool.ex

@@ -0,0 +1,22 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool do
+  def child_spec(opts) do
+    poolboy_opts =
+      opts
+      |> Keyword.put(:worker_module, Pleroma.Pool.Request)
+      |> Keyword.put(:name, {:local, opts[:name]})
+      |> Keyword.put(:size, opts[:size])
+      |> Keyword.put(:max_overflow, opts[:max_overflow])
+
+    %{
+      id: opts[:id] || {__MODULE__, make_ref()},
+      start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]},
+      restart: :permanent,
+      shutdown: 5000,
+      type: :worker
+    }
+  end
+end

lib/pleroma/pool/request.ex

@@ -0,0 +1,65 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool.Request do
+  use GenServer
+
+  require Logger
+
+  def start_link(args) do
+    GenServer.start_link(__MODULE__, args)
+  end
+
+  @impl true
+  def init(_), do: {:ok, []}
+
+  @spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) ::
+          {:ok, Tesla.Env.t()} | {:error, any()}
+  def execute(pid, client, request, timeout) do
+    GenServer.call(pid, {:execute, client, request}, timeout)
+  end
+
+  @impl true
+  def handle_call({:execute, client, request}, _from, state) do
+    response = Pleroma.HTTP.request(client, request)
+
+    {:reply, response, state}
+  end
+
+  @impl true
+  def handle_info({:gun_data, _conn, _stream, _, _}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_up, _conn, _protocol}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_error, _conn, _stream, _error}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do
+    {:noreply, state}
+  end
+
+  @impl true
+  def handle_info(msg, state) do
+    Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}")
+    {:noreply, state}
+  end
+end

lib/pleroma/pool/supervisor.ex

@@ -0,0 +1,42 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool.Supervisor do
+  use Supervisor
+
+  alias Pleroma.Config
+  alias Pleroma.Pool
+
+  def start_link(args) do
+    Supervisor.start_link(__MODULE__, args, name: __MODULE__)
+  end
+
+  def init(_) do
+    conns_child = %{
+      id: Pool.Connections,
+      start:
+        {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]}
+    }
+
+    Supervisor.init([conns_child | pools()], strategy: :one_for_one)
+  end
+
+  defp pools do
+    pools = Config.get(:pools)
+
+    pools =
+      if Config.get([Pleroma.Upload, :proxy_remote]) == false do
+        Keyword.delete(pools, :upload)
+      else
+        pools
+      end
+
+    for {pool_name, pool_opts} <- pools do
+      pool_opts
+      |> Keyword.put(:id, {Pool, pool_name})
+      |> Keyword.put(:name, pool_name)
+      |> Pool.child_spec()
+    end
+  end
+end

lib/pleroma/reverse_proxy/client.ex

@@ -3,19 +3,23 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy.Client do
-  @callback request(atom(), String.t(), [tuple()], String.t(), list()) ::
+  @type status :: pos_integer()
-              {:ok, pos_integer(), [tuple()], reference() | map()}
+  @type header_name :: String.t()
-              | {:ok, pos_integer(), [tuple()]}
+  @type header_value :: String.t()
+  @type headers :: [{header_name(), header_value()}]
+
+  @callback request(atom(), String.t(), headers(), String.t(), list()) ::
+              {:ok, status(), headers(), reference() | map()}
+              | {:ok, status(), headers()}
               | {:ok, reference()}
               | {:error, term()}
 
-  @callback stream_body(reference() | pid() | map()) ::
+  @callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()}
-              {:ok, binary()} | :done | {:error, String.t()}
 
   @callback close(reference() | pid() | map()) :: :ok
 
-  def request(method, url, headers, "", opts \\ []) do
+  def request(method, url, headers, body \\ "", opts \\ []) do
-    client().request(method, url, headers, "", opts)
+    client().request(method, url, headers, body, opts)
   end
 
   def stream_body(ref), do: client().stream_body(ref)
@@ -23,6 +27,12 @@ def stream_body(ref), do: client().stream_body(ref)
   def close(ref), do: client().close(ref)
 
   defp client do
-    Pleroma.Config.get([Pleroma.ReverseProxy.Client], :hackney)
+    :tesla
+    |> Application.get_env(:adapter)
+    |> client()
   end
+
+  defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney
+  defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla
+  defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client)
 end

lib/pleroma/reverse_proxy/client/hackney.ex

@@ -0,0 +1,24 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ReverseProxy.Client.Hackney do
+  @behaviour Pleroma.ReverseProxy.Client
+
+  @impl true
+  def request(method, url, headers, body, opts \\ []) do
+    :hackney.request(method, url, headers, body, opts)
+  end
+
+  @impl true
+  def stream_body(ref) do
+    case :hackney.stream_body(ref) do
+      :done -> :done
+      {:ok, data} -> {:ok, data, ref}
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  @impl true
+  def close(ref), do: :hackney.close(ref)
+end

lib/pleroma/reverse_proxy/client/tesla.ex

@@ -0,0 +1,90 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ReverseProxy.Client.Tesla do
+  @behaviour Pleroma.ReverseProxy.Client
+
+  @type headers() :: [{String.t(), String.t()}]
+  @type status() :: pos_integer()
+
+  @spec request(atom(), String.t(), headers(), String.t(), keyword()) ::
+          {:ok, status(), headers}
+          | {:ok, status(), headers, map()}
+          | {:error, atom() | String.t()}
+          | no_return()
+
+  @impl true
+  def request(method, url, headers, body, opts \\ []) do
+    check_adapter()
+
+    opts = Keyword.put(opts, :body_as, :chunks)
+
+    with {:ok, response} <-
+           Pleroma.HTTP.request(
+             method,
+             url,
+             body,
+             headers,
+             Keyword.put(opts, :adapter, opts)
+           ) do
+      if is_map(response.body) and method != :head do
+        {:ok, response.status, response.headers, response.body}
+      else
+        {:ok, response.status, response.headers}
+      end
+    else
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  @impl true
+  @spec stream_body(map()) ::
+          {:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
+  def stream_body(%{pid: pid, opts: opts, fin: true}) do
+    # if connection was reused, but in tesla were redirects,
+    # tesla returns new opened connection, which must be closed manually
+    if opts[:old_conn], do: Tesla.Adapter.Gun.close(pid)
+    # if there were redirects we need to checkout old conn
+    conn = opts[:old_conn] || opts[:conn]
+
+    if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections)
+
+    :done
+  end
+
+  def stream_body(client) do
+    case read_chunk!(client) do
+      {:fin, body} ->
+        {:ok, body, Map.put(client, :fin, true)}
+
+      {:nofin, part} ->
+        {:ok, part, client}
+
+      {:error, error} ->
+        {:error, error}
+    end
+  end
+
+  defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do
+    adapter = check_adapter()
+    adapter.read_chunk(pid, stream, opts)
+  end
+
+  @impl true
+  @spec close(map) :: :ok | no_return()
+  def close(%{pid: pid}) do
+    adapter = check_adapter()
+    adapter.close(pid)
+  end
+
+  defp check_adapter do
+    adapter = Application.get_env(:tesla, :adapter)
+
+    unless adapter == Tesla.Adapter.Gun do
+      raise "#{adapter} doesn't support reading body in chunks"
+    end
+
+    adapter
+  end
+end

lib/pleroma/reverse_proxy/reverse_proxy.ex

@@ -3,8 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy do
-  alias Pleroma.HTTP
-
   @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
                       ~w(if-unmodified-since if-none-match if-range range)
   @resp_cache_headers ~w(etag date last-modified)
@@ -58,10 +56,10 @@ defmodule Pleroma.ReverseProxy do
 
   * `req_headers`, `resp_headers` additional headers.
 
-  * `http`: options for [hackney](https://github.com/benoitc/hackney).
+  * `http`: options for [hackney](https://github.com/benoitc/hackney) or [gun](https://github.com/ninenines/gun).
 
   """
-  @default_hackney_options [pool: :media]
+  @default_options [pool: :media]
 
   @inline_content_types [
     "image/gif",
@@ -94,11 +92,7 @@ defmodule Pleroma.ReverseProxy do
   def call(_conn, _url, _opts \\ [])
 
   def call(conn = %{method: method}, url, opts) when method in @methods do
-    hackney_opts =
+    client_opts = Keyword.merge(@default_options, Keyword.get(opts, :http, []))
-      Pleroma.HTTP.Connection.hackney_options([])
-      |> Keyword.merge(@default_hackney_options)
-      |> Keyword.merge(Keyword.get(opts, :http, []))
-      |> HTTP.process_request_options()
 
     req_headers = build_req_headers(conn.req_headers, opts)
 
@@ -110,7 +104,7 @@ def call(conn = %{method: method}, url, opts) when method in @methods do
       end
 
     with {:ok, nil} <- Cachex.get(:failed_proxy_url_cache, url),
-         {:ok, code, headers, client} <- request(method, url, req_headers, hackney_opts),
+         {:ok, code, headers, client} <- request(method, url, req_headers, client_opts),
          :ok <-
            header_length_constraint(
              headers,
@@ -156,11 +150,11 @@ def call(conn, _, _) do
     |> halt()
   end
 
-  defp request(method, url, headers, hackney_opts) do
+  defp request(method, url, headers, opts) do
     Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
     method = method |> String.downcase() |> String.to_existing_atom()
 
-    case client().request(method, url, headers, "", hackney_opts) do
+    case client().request(method, url, headers, "", opts) do
       {:ok, code, headers, client} when code in @valid_resp_codes ->
         {:ok, code, downcase_headers(headers), client}
 
@@ -210,7 +204,7 @@ defp chunk_reply(conn, client, opts, sent_so_far, duration) do
              duration,
              Keyword.get(opts, :max_read_duration, @max_read_duration)
            ),
-         {:ok, data} <- client().stream_body(client),
+         {:ok, data, client} <- client().stream_body(client),
          {:ok, duration} <- increase_read_duration(duration),
          sent_so_far = sent_so_far + byte_size(data),
          :ok <-

lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do

lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex

@@ -12,17 +12,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
   require Logger
 
-  @hackney_options [
+  @options [
-    pool: :media,
+    pool: :media
-    recv_timeout: 10_000
   ]
 
   def perform(:prefetch, url) do
     Logger.debug("Prefetching #{inspect(url)}")
 
+    opts =
+      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
+        Keyword.put(@options, :recv_timeout, 10_000)
+      else
+        @options
+      end
+
     url
     |> MediaProxy.url()
-    |> HTTP.get([], adapter: @hackney_options)
+    |> HTTP.get([], adapter: opts)
   end
 
   def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do

lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do

lib/pleroma/web/rel_me.ex

@@ -3,11 +3,9 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RelMe do
-  @hackney_options [
+  @options [
     pool: :media,
-    recv_timeout: 2_000,
+    max_body: 2_000_000
-    max_body: 2_000_000,
-    with_body: true
   ]
 
   if Pleroma.Config.get(:env) == :test do
@@ -25,8 +23,18 @@ def parse(url) when is_binary(url) do
   def parse(_), do: {:error, "No URL provided"}
 
   defp parse_url(url) do
+    opts =
+      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
+        Keyword.merge(@options,
+          recv_timeout: 2_000,
+          with_body: true
+        )
+      else
+        @options
+      end
+
     with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <-
-           Pleroma.HTTP.get(url, [], adapter: @hackney_options),
+           Pleroma.HTTP.get(url, [], adapter: opts),
          {:ok, html_tree} <- Floki.parse_document(html),
          data <-
            Floki.attribute(html_tree, "link[rel~=me]", "href") ++

lib/pleroma/web/rich_media/parser.ex

@@ -3,11 +3,9 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parser do
-  @hackney_options [
+  @options [
     pool: :media,
-    recv_timeout: 2_000,
+    max_body: 2_000_000
-    max_body: 2_000_000,
-    with_body: true
   ]
 
   defp parsers do
@@ -77,8 +75,18 @@ defp get_ttl_from_image(data, url) do
   end
 
   defp parse_url(url) do
+    opts =
+      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
+        Keyword.merge(@options,
+          recv_timeout: 2_000,
+          with_body: true
+        )
+      else
+        @options
+      end
+
     try do
-      {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
+      {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: opts)
 
       html
       |> parse_html()

lib/pleroma/web/web_finger/web_finger.ex

@@ -173,7 +173,8 @@ def find_lrdd_template(domain) do
       get_template_from_xml(body)
     else
       _ ->
-        with {:ok, %{body: body}} <- HTTP.get("https://#{domain}/.well-known/host-meta", []) do
+        with {:ok, %{body: body, status: status}} when status in 200..299 <-
+               HTTP.get("https://#{domain}/.well-known/host-meta", []) do
           get_template_from_xml(body)
         else
           e -> {:error, "Can't find LRDD template: #{inspect(e)}"}
@@ -205,7 +206,7 @@ def finger(account) do
     with response <-
            HTTP.get(
              address,
-             Accept: "application/xrd+xml,application/jrd+json"
+             [{"accept", "application/xrd+xml,application/jrd+json"}]
            ),
          {:ok, %{status: status, body: body}} when status in 200..299 <- response do
       doc = XML.parse_document(body)

mix.exs

@@ -119,7 +119,15 @@ defp deps do
       {:calendar, "~> 0.17.4"},
       {:cachex, "~> 3.2"},
       {:poison, "~> 3.0", override: true},
-      {:tesla, "~> 1.3", override: true},
+      # {:tesla, "~> 1.3", override: true},
+      {:tesla,
+       git: "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git",
+       ref: "61b7503cef33f00834f78ddfafe0d5d9dec2270b",
+       override: true},
+      {:castore, "~> 0.1"},
+      {:cowlib, "~> 2.8", override: true},
+      {:gun,
+       github: "ninenines/gun", ref: "e1a69b36b180a574c0ac314ced9613fdd52312cc", override: true},
       {:jason, "~> 1.0"},
       {:mogrify, "~> 0.6.1"},
       {:ex_aws, "~> 2.1"},

mix.lock

@@ -10,6 +10,7 @@
   "cachex": {:hex, :cachex, "3.2.0", "a596476c781b0646e6cb5cd9751af2e2974c3e0d5498a8cab71807618b74fe2f", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "aef93694067a43697ae0531727e097754a9e992a1e7946296f5969d6dd9ac986"},
   "calendar": {:hex, :calendar, "0.17.6", "ec291cb2e4ba499c2e8c0ef5f4ace974e2f9d02ae9e807e711a9b0c7850b9aee", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "738d0e17a93c2ccfe4ddc707bdc8e672e9074c8569498483feb1c4530fb91b2b"},
   "captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "e0f16822d578866e186a0974d65ad58cddc1e2ab", [ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"]},
+  "castore": {:hex, :castore, "0.1.5", "591c763a637af2cc468a72f006878584bc6c306f8d111ef8ba1d4c10e0684010", [:mix], [], "hexpm", "6db356b2bc6cc22561e051ff545c20ad064af57647e436650aa24d7d06cd941a"},
   "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"},
   "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
   "comeonin": {:hex, :comeonin, "4.1.2", "3eb5620fd8e35508991664b4c2b04dd41e52f1620b36957be837c1d7784b7592", [:mix], [{:argon2_elixir, "~> 1.2", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:bcrypt_elixir, "~> 0.12.1 or ~> 1.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: true]}, {:pbkdf2_elixir, "~> 0.12", [hex: :pbkdf2_elixir, repo: "hexpm", optional: true]}], "hexpm", "d8700a0ca4dbb616c22c9b3f6dd539d88deaafec3efe66869d6370c9a559b3e9"},
@@ -46,6 +47,7 @@
   "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"},
   "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"},
   "gettext": {:hex, :gettext, "0.17.4", "f13088e1ec10ce01665cf25f5ff779e7df3f2dc71b37084976cf89d1aa124d5c", [:mix], [], "hexpm", "3c75b5ea8288e2ee7ea503ff9e30dfe4d07ad3c054576a6e60040e79a801e14d"},
+  "gun": {:git, "https://github.com/ninenines/gun.git", "e1a69b36b180a574c0ac314ced9613fdd52312cc", [ref: "e1a69b36b180a574c0ac314ced9613fdd52312cc"]},
   "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"},
   "html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"},
   "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"},
@@ -101,7 +103,7 @@
   "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "e3928e1d2889a308aaf3e42755809ac21cffd77cb58eef01cbfdab4ce2fd1e21"},
   "syslog": {:hex, :syslog, "1.0.6", "995970c9aa7feb380ac493302138e308d6e04fd57da95b439a6df5bb3bf75076", [:rebar3], [], "hexpm", "769ddfabd0d2a16f3f9c17eb7509951e0ca4f68363fb26f2ee51a8ec4a49881a"},
   "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm", "4738382e36a0a9a2b6e25d67c960e40e1a2c95560b9f936d8e29de8cd858480f"},
-  "tesla": {:hex, :tesla, "1.3.2", "deb92c5c9ce35e747a395ba413ca78593a4f75bf0e1545630ee2e3d34264021e", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "7567704c4790e21bd9a961b56d0b6a988ff68cc4dacfe6b2106e258da1d5cdda"},
+  "tesla": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/tesla.git", "61b7503cef33f00834f78ddfafe0d5d9dec2270b", [ref: "61b7503cef33f00834f78ddfafe0d5d9dec2270b"]},
   "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "f354efb2400dd7a80fd9eb6c8419068c4f632da4ac47f3d8822d6e33f08bc852"},
   "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"},
   "tzdata": {:hex, :tzdata, "0.5.22", "f2ba9105117ee0360eae2eca389783ef7db36d533899b2e84559404dbc77ebb8", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "cd66c8a1e6a9e121d1f538b01bef459334bb4029a1ffb4eeeb5e4eae0337e7b6"},

priv/repo/migrations/20190408123347_create_conversations.exs

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Repo.Migrations.CreateConversations do

restarter/lib/pleroma.ex

@@ -62,7 +62,7 @@ def handle_cast(:refresh, _state) do
   end
 
   def handle_cast({:restart, :test, _}, state) do
-    Logger.warn("pleroma restarted")
+    Logger.debug("pleroma manually restarted")
     {:noreply, Map.put(state, :need_reboot, false)}
   end
 
@@ -75,7 +75,7 @@ def handle_cast({:restart, _, delay}, state) do
   def handle_cast({:after_boot, _}, %{after_boot: true} = state), do: {:noreply, state}
 
   def handle_cast({:after_boot, :test}, state) do
-    Logger.warn("pleroma restarted")
+    Logger.debug("pleroma restarted after boot")
     state = %{state | after_boot: true, rebooted: true}
     {:noreply, state}
   end

test/activity/ir/topics_test.exs

@@ -83,7 +83,7 @@ test "converts tags to hash tags", %{activity: %{object: %{data: data} = object}
       assert Enum.member?(topics, "hashtag:bar")
     end
 
-    test "only converts strinngs to hash tags", %{
+    test "only converts strings to hash tags", %{
       activity: %{object: %{data: data} = object} = activity
     } do
       tagged_data = Map.put(data, "tag", [2])

test/config/config_db_test.exs

@@ -478,14 +478,6 @@ test "simple keyword" do
       assert ConfigDB.from_binary(binary) == [key: "value"]
     end
 
-    test "keyword with partial_chain key" do
-      binary =
-        ConfigDB.transform([%{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]}])
-
-      assert binary == :erlang.term_to_binary(partial_chain: &:hackney_connect.partial_chain/1)
-      assert ConfigDB.from_binary(binary) == [partial_chain: &:hackney_connect.partial_chain/1]
-    end
-
     test "keyword" do
       binary =
         ConfigDB.transform([

test/fixtures/users_mock/localhost.json

@@ -0,0 +1,41 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "http://localhost:4001/schemas/litepub-0.1.jsonld",
+    {
+      "@language": "und"
+    }
+  ],
+  "attachment": [],
+  "endpoints": {
+    "oauthAuthorizationEndpoint": "http://localhost:4001/oauth/authorize",
+    "oauthRegistrationEndpoint": "http://localhost:4001/api/v1/apps",
+    "oauthTokenEndpoint": "http://localhost:4001/oauth/token",
+    "sharedInbox": "http://localhost:4001/inbox"
+  },
+  "followers": "http://localhost:4001/users/{{nickname}}/followers",
+  "following": "http://localhost:4001/users/{{nickname}}/following",
+  "icon": {
+    "type": "Image",
+    "url": "http://localhost:4001/media/4e914f5b84e4a259a3f6c2d2edc9ab642f2ab05f3e3d9c52c81fc2d984b3d51e.jpg"
+  },
+  "id": "http://localhost:4001/users/{{nickname}}",
+  "image": {
+    "type": "Image",
+    "url": "http://localhost:4001/media/f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg?name=f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg"
+  },
+  "inbox": "http://localhost:4001/users/{{nickname}}/inbox",
+  "manuallyApprovesFollowers": false,
+  "name": "{{nickname}}",
+  "outbox": "http://localhost:4001/users/{{nickname}}/outbox",
+  "preferredUsername": "{{nickname}}",
+  "publicKey": {
+    "id": "http://localhost:4001/users/{{nickname}}#main-key",
+    "owner": "http://localhost:4001/users/{{nickname}}",
+    "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5DLtwGXNZElJyxFGfcVc\nXANhaMadj/iYYQwZjOJTV9QsbtiNBeIK54PJrYuU0/0YIdrvS1iqheX5IwXRhcwa\nhm3ZyLz7XeN9st7FBni4BmZMBtMpxAuYuu5p/jbWy13qAiYOhPreCx0wrWgm/lBD\n9mkgaxIxPooBE0S4ZWEJIDIV1Vft3AWcRUyWW1vIBK0uZzs6GYshbQZB952S0yo4\nFzI1hABGHncH8UvuFauh4EZ8tY7/X5I0pGRnDOcRN1dAht5w5yTA+6r5kebiFQjP\nIzN/eCO/a9Flrj9YGW7HDNtjSOH0A31PLRGlJtJO3yK57dnf5ppyCZGfL4emShQo\ncQIDAQAB\n-----END PUBLIC KEY-----\n\n"
+  },
+  "summary": "your friendly neighborhood pleroma developer<br>I like cute things and distributed systems, and really hate delete and redrafts",
+  "tag": [],
+  "type": "Person",
+  "url": "http://localhost:4001/users/{{nickname}}"
+}

test/fixtures/warnings/otp_version/21.1

@@ -0,0 +1 @@
+21.1

test/fixtures/warnings/otp_version/22.1

@@ -0,0 +1 @@
+22.1

test/fixtures/warnings/otp_version/22.4

@@ -0,0 +1 @@
+22.4

test/fixtures/warnings/otp_version/23.0

@@ -0,0 +1 @@
+23.0

test/http/adapter_helper/gun_test.exs

@@ -0,0 +1,258 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.AdapterHelper.GunTest do
+  use ExUnit.Case, async: true
+  use Pleroma.Tests.Helpers
+
+  import Mox
+
+  alias Pleroma.Config
+  alias Pleroma.Gun.Conn
+  alias Pleroma.HTTP.AdapterHelper.Gun
+  alias Pleroma.Pool.Connections
+
+  setup :verify_on_exit!
+
+  defp gun_mock(_) do
+    gun_mock()
+    :ok
+  end
+
+  defp gun_mock do
+    Pleroma.GunMock
+    |> stub(:open, fn _, _, _ -> Task.start_link(fn -> Process.sleep(1000) end) end)
+    |> stub(:await_up, fn _, _ -> {:ok, :http} end)
+    |> stub(:set_owner, fn _, _ -> :ok end)
+  end
+
+  describe "options/1" do
+    setup do: clear_config([:http, :adapter], a: 1, b: 2)
+
+    test "https url with default port" do
+      uri = URI.parse("https://example.com")
+
+      opts = Gun.options([receive_conn: false], uri)
+      assert opts[:certificates_verification]
+      assert opts[:tls_opts][:log_level] == :warning
+    end
+
+    test "https ipv4 with default port" do
+      uri = URI.parse("https://127.0.0.1")
+
+      opts = Gun.options([receive_conn: false], uri)
+      assert opts[:certificates_verification]
+      assert opts[:tls_opts][:log_level] == :warning
+    end
+
+    test "https ipv6 with default port" do
+      uri = URI.parse("https://[2a03:2880:f10c:83:face:b00c:0:25de]")
+
+      opts = Gun.options([receive_conn: false], uri)
+      assert opts[:certificates_verification]
+      assert opts[:tls_opts][:log_level] == :warning
+    end
+
+    test "https url with non standart port" do
+      uri = URI.parse("https://example.com:115")
+
+      opts = Gun.options([receive_conn: false], uri)
+
+      assert opts[:certificates_verification]
+    end
+
+    test "get conn on next request" do
+      gun_mock()
+      level = Application.get_env(:logger, :level)
+      Logger.configure(level: :debug)
+      on_exit(fn -> Logger.configure(level: level) end)
+      uri = URI.parse("http://some-domain2.com")
+
+      opts = Gun.options(uri)
+
+      assert opts[:conn] == nil
+      assert opts[:close_conn] == nil
+
+      Process.sleep(50)
+      opts = Gun.options(uri)
+
+      assert is_pid(opts[:conn])
+      assert opts[:close_conn] == false
+    end
+
+    test "merges with defaul http adapter config" do
+      defaults = Gun.options([receive_conn: false], URI.parse("https://example.com"))
+      assert Keyword.has_key?(defaults, :a)
+      assert Keyword.has_key?(defaults, :b)
+    end
+
+    test "default ssl adapter opts with connection" do
+      gun_mock()
+      uri = URI.parse("https://some-domain.com")
+
+      :ok = Conn.open(uri, :gun_connections)
+
+      opts = Gun.options(uri)
+
+      assert opts[:certificates_verification]
+      refute opts[:tls_opts] == []
+
+      assert opts[:close_conn] == false
+      assert is_pid(opts[:conn])
+    end
+
+    test "parses string proxy host & port" do
+      proxy = Config.get([:http, :proxy_url])
+      Config.put([:http, :proxy_url], "localhost:8123")
+      on_exit(fn -> Config.put([:http, :proxy_url], proxy) end)
+
+      uri = URI.parse("https://some-domain.com")
+      opts = Gun.options([receive_conn: false], uri)
+      assert opts[:proxy] == {'localhost', 8123}
+    end
+
+    test "parses tuple proxy scheme host and port" do
+      proxy = Config.get([:http, :proxy_url])
+      Config.put([:http, :proxy_url], {:socks, 'localhost', 1234})
+      on_exit(fn -> Config.put([:http, :proxy_url], proxy) end)
+
+      uri = URI.parse("https://some-domain.com")
+      opts = Gun.options([receive_conn: false], uri)
+      assert opts[:proxy] == {:socks, 'localhost', 1234}
+    end
+
+    test "passed opts have more weight than defaults" do
+      proxy = Config.get([:http, :proxy_url])
+      Config.put([:http, :proxy_url], {:socks5, 'localhost', 1234})
+      on_exit(fn -> Config.put([:http, :proxy_url], proxy) end)
+      uri = URI.parse("https://some-domain.com")
+      opts = Gun.options([receive_conn: false, proxy: {'example.com', 4321}], uri)
+
+      assert opts[:proxy] == {'example.com', 4321}
+    end
+  end
+
+  describe "options/1 with receive_conn parameter" do
+    setup :gun_mock
+
+    test "receive conn by default" do
+      uri = URI.parse("http://another-domain.com")
+      :ok = Conn.open(uri, :gun_connections)
+
+      received_opts = Gun.options(uri)
+      assert received_opts[:close_conn] == false
+      assert is_pid(received_opts[:conn])
+    end
+
+    test "don't receive conn if receive_conn is false" do
+      uri = URI.parse("http://another-domain.com")
+      :ok = Conn.open(uri, :gun_connections)
+
+      opts = [receive_conn: false]
+      received_opts = Gun.options(opts, uri)
+      assert received_opts[:close_conn] == nil
+      assert received_opts[:conn] == nil
+    end
+  end
+
+  describe "after_request/1" do
+    setup :gun_mock
+
+    test "body_as not chunks" do
+      uri = URI.parse("http://some-domain.com")
+      :ok = Conn.open(uri, :gun_connections)
+      opts = Gun.options(uri)
+      :ok = Gun.after_request(opts)
+      conn = opts[:conn]
+
+      assert %Connections{
+               conns: %{
+                 "http:some-domain.com:80" => %Pleroma.Gun.Conn{
+                   conn: ^conn,
+                   conn_state: :idle,
+                   used_by: []
+                 }
+               }
+             } = Connections.get_state(:gun_connections)
+    end
+
+    test "body_as chunks" do
+      uri = URI.parse("http://some-domain.com")
+      :ok = Conn.open(uri, :gun_connections)
+      opts = Gun.options([body_as: :chunks], uri)
+      :ok = Gun.after_request(opts)
+      conn = opts[:conn]
+      self = self()
+
+      assert %Connections{
+               conns: %{
+                 "http:some-domain.com:80" => %Pleroma.Gun.Conn{
+                   conn: ^conn,
+                   conn_state: :active,
+                   used_by: [{^self, _}]
+                 }
+               }
+             } = Connections.get_state(:gun_connections)
+    end
+
+    test "with no connection" do
+      uri = URI.parse("http://uniq-domain.com")
+
+      :ok = Conn.open(uri, :gun_connections)
+
+      opts = Gun.options([body_as: :chunks], uri)
+      conn = opts[:conn]
+      opts = Keyword.delete(opts, :conn)
+      self = self()
+
+      :ok = Gun.after_request(opts)
+
+      assert %Connections{
+               conns: %{
+                 "http:uniq-domain.com:80" => %Pleroma.Gun.Conn{
+                   conn: ^conn,
+                   conn_state: :active,
+                   used_by: [{^self, _}]
+                 }
+               }
+             } = Connections.get_state(:gun_connections)
+    end
+
+    test "with ipv4" do
+      uri = URI.parse("http://127.0.0.1")
+      :ok = Conn.open(uri, :gun_connections)
+      opts = Gun.options(uri)
+      :ok = Gun.after_request(opts)
+      conn = opts[:conn]
+
+      assert %Connections{
+               conns: %{
+                 "http:127.0.0.1:80" => %Pleroma.Gun.Conn{
+                   conn: ^conn,
+                   conn_state: :idle,
+                   used_by: []
+                 }
+               }
+             } = Connections.get_state(:gun_connections)
+    end
+
+    test "with ipv6" do
+      uri = URI.parse("http://[2a03:2880:f10c:83:face:b00c:0:25de]")
+      :ok = Conn.open(uri, :gun_connections)
+      opts = Gun.options(uri)
+      :ok = Gun.after_request(opts)
+      conn = opts[:conn]
+
+      assert %Connections{
+               conns: %{
+                 "http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Pleroma.Gun.Conn{
+                   conn: ^conn,
+                   conn_state: :idle,
+                   used_by: []
+                 }
+               }
+             } = Connections.get_state(:gun_connections)
+    end
+  end
+end

test/http/adapter_helper/hackney_test.exs

@@ -0,0 +1,47 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do
+  use ExUnit.Case, async: true
+  use Pleroma.Tests.Helpers
+
+  alias Pleroma.HTTP.AdapterHelper.Hackney
+
+  setup_all do
+    uri = URI.parse("http://domain.com")
+    {:ok, uri: uri}
+  end
+
+  describe "options/2" do
+    setup do: clear_config([:http, :adapter], a: 1, b: 2)
+
+    test "add proxy and opts from config", %{uri: uri} do
+      opts = Hackney.options([proxy: "localhost:8123"], uri)
+
+      assert opts[:a] == 1
+      assert opts[:b] == 2
+      assert opts[:proxy] == "localhost:8123"
+    end
+
+    test "respect connection opts and no proxy", %{uri: uri} do
+      opts = Hackney.options([a: 2, b: 1], uri)
+
+      assert opts[:a] == 2
+      assert opts[:b] == 1
+      refute Keyword.has_key?(opts, :proxy)
+    end
+
+    test "add opts for https" do
+      uri = URI.parse("https://domain.com")
+
+      opts = Hackney.options(uri)
+
+      assert opts[:ssl_options] == [
+               partial_chain: &:hackney_connect.partial_chain/1,
+               versions: [:tlsv1, :"tlsv1.1", :"tlsv1.2"],
+               server_name_indication: 'domain.com'
+             ]
+    end
+  end
+end

test/http/adapter_helper_test.exs

@@ -0,0 +1,28 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.AdapterHelperTest do
+  use ExUnit.Case, async: true
+
+  alias Pleroma.HTTP.AdapterHelper
+
+  describe "format_proxy/1" do
+    test "with nil" do
+      assert AdapterHelper.format_proxy(nil) == nil
+    end
+
+    test "with string" do
+      assert AdapterHelper.format_proxy("127.0.0.1:8123") == {{127, 0, 0, 1}, 8123}
+    end
+
+    test "localhost with port" do
+      assert AdapterHelper.format_proxy("localhost:8123") == {'localhost', 8123}
+    end
+
+    test "tuple" do
+      assert AdapterHelper.format_proxy({:socks4, :localhost, 9050}) ==
+               {:socks4, 'localhost', 9050}
+    end
+  end
+end

test/http/connection_test.exs

@@ -0,0 +1,135 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.ConnectionTest do
+  use ExUnit.Case, async: true
+  use Pleroma.Tests.Helpers
+
+  import ExUnit.CaptureLog
+
+  alias Pleroma.Config
+  alias Pleroma.HTTP.Connection
+
+  describe "parse_host/1" do
+    test "as atom to charlist" do
+      assert Connection.parse_host(:localhost) == 'localhost'
+    end
+
+    test "as string to charlist" do
+      assert Connection.parse_host("localhost.com") == 'localhost.com'
+    end
+
+    test "as string ip to tuple" do
+      assert Connection.parse_host("127.0.0.1") == {127, 0, 0, 1}
+    end
+  end
+
+  describe "parse_proxy/1" do
+    test "ip with port" do
+      assert Connection.parse_proxy("127.0.0.1:8123") == {:ok, {127, 0, 0, 1}, 8123}
+    end
+
+    test "host with port" do
+      assert Connection.parse_proxy("localhost:8123") == {:ok, 'localhost', 8123}
+    end
+
+    test "as tuple" do
+      assert Connection.parse_proxy({:socks4, :localhost, 9050}) ==
+               {:ok, :socks4, 'localhost', 9050}
+    end
+
+    test "as tuple with string host" do
+      assert Connection.parse_proxy({:socks5, "localhost", 9050}) ==
+               {:ok, :socks5, 'localhost', 9050}
+    end
+  end
+
+  describe "parse_proxy/1 errors" do
+    test "ip without port" do
+      capture_log(fn ->
+        assert Connection.parse_proxy("127.0.0.1") == {:error, :invalid_proxy}
+      end) =~ "parsing proxy fail \"127.0.0.1\""
+    end
+
+    test "host without port" do
+      capture_log(fn ->
+        assert Connection.parse_proxy("localhost") == {:error, :invalid_proxy}
+      end) =~ "parsing proxy fail \"localhost\""
+    end
+
+    test "host with bad port" do
+      capture_log(fn ->
+        assert Connection.parse_proxy("localhost:port") == {:error, :invalid_proxy_port}
+      end) =~ "parsing port in proxy fail \"localhost:port\""
+    end
+
+    test "ip with bad port" do
+      capture_log(fn ->
+        assert Connection.parse_proxy("127.0.0.1:15.9") == {:error, :invalid_proxy_port}
+      end) =~ "parsing port in proxy fail \"127.0.0.1:15.9\""
+    end
+
+    test "as tuple without port" do
+      capture_log(fn ->
+        assert Connection.parse_proxy({:socks5, :localhost}) == {:error, :invalid_proxy}
+      end) =~ "parsing proxy fail {:socks5, :localhost}"
+    end
+
+    test "with nil" do
+      assert Connection.parse_proxy(nil) == nil
+    end
+  end
+
+  describe "options/3" do
+    setup do: clear_config([:http, :proxy_url])
+
+    test "without proxy_url in config" do
+      Config.delete([:http, :proxy_url])
+
+      opts = Connection.options(%URI{})
+      refute Keyword.has_key?(opts, :proxy)
+    end
+
+    test "parses string proxy host & port" do
+      Config.put([:http, :proxy_url], "localhost:8123")
+
+      opts = Connection.options(%URI{})
+      assert opts[:proxy] == {'localhost', 8123}
+    end
+
+    test "parses tuple proxy scheme host and port" do
+      Config.put([:http, :proxy_url], {:socks, 'localhost', 1234})
+
+      opts = Connection.options(%URI{})
+      assert opts[:proxy] == {:socks, 'localhost', 1234}
+    end
+
+    test "passed opts have more weight than defaults" do
+      Config.put([:http, :proxy_url], {:socks5, 'localhost', 1234})
+
+      opts = Connection.options(%URI{}, proxy: {'example.com', 4321})
+
+      assert opts[:proxy] == {'example.com', 4321}
+    end
+  end
+
+  describe "format_host/1" do
+    test "with domain" do
+      assert Connection.format_host("example.com") == 'example.com'
+    end
+
+    test "with idna domain" do
+      assert Connection.format_host("ですexample.com") == 'xn--example-183fne.com'
+    end
+
+    test "with ipv4" do
+      assert Connection.format_host("127.0.0.1") == '127.0.0.1'
+    end
+
+    test "with ipv6" do
+      assert Connection.format_host("2a03:2880:f10c:83:face:b00c:0:25de") ==
+               '2a03:2880:f10c:83:face:b00c:0:25de'
+    end
+  end
+end

test/http/request_builder_test.exs

@@ -5,6 +5,8 @@
 defmodule Pleroma.HTTP.RequestBuilderTest do
   use ExUnit.Case, async: true
   use Pleroma.Tests.Helpers
+  alias Pleroma.Config
+  alias Pleroma.HTTP.Request
   alias Pleroma.HTTP.RequestBuilder
 
   describe "headers/2" do
@@ -12,48 +14,31 @@ defmodule Pleroma.HTTP.RequestBuilderTest do
     setup do: clear_config([:http, :user_agent])
 
     test "don't send pleroma user agent" do
-      assert RequestBuilder.headers(%{}, []) == %{headers: []}
+      assert RequestBuilder.headers(%Request{}, []) == %Request{headers: []}
     end
 
     test "send pleroma user agent" do
-      Pleroma.Config.put([:http, :send_user_agent], true)
+      Config.put([:http, :send_user_agent], true)
-      Pleroma.Config.put([:http, :user_agent], :default)
+      Config.put([:http, :user_agent], :default)
 
-      assert RequestBuilder.headers(%{}, []) == %{
+      assert RequestBuilder.headers(%Request{}, []) == %Request{
-               headers: [{"User-Agent", Pleroma.Application.user_agent()}]
+               headers: [{"user-agent", Pleroma.Application.user_agent()}]
              }
     end
 
     test "send custom user agent" do
-      Pleroma.Config.put([:http, :send_user_agent], true)
+      Config.put([:http, :send_user_agent], true)
-      Pleroma.Config.put([:http, :user_agent], "totally-not-pleroma")
+      Config.put([:http, :user_agent], "totally-not-pleroma")
 
-      assert RequestBuilder.headers(%{}, []) == %{
+      assert RequestBuilder.headers(%Request{}, []) == %Request{
-               headers: [{"User-Agent", "totally-not-pleroma"}]
+               headers: [{"user-agent", "totally-not-pleroma"}]
              }
     end
   end
 
-  describe "add_optional_params/3" do
-    test "don't add if keyword is empty" do
-      assert RequestBuilder.add_optional_params(%{}, %{}, []) == %{}
-    end
-
-    test "add query parameter" do
-      assert RequestBuilder.add_optional_params(
-               %{},
-               %{query: :query, body: :body, another: :val},
-               [
-                 {:query, "param1=val1&param2=val2"},
-                 {:body, "some body"}
-               ]
-             ) == %{query: "param1=val1&param2=val2", body: "some body"}
-    end
-  end
-
   describe "add_param/4" do
     test "add file parameter" do
-      %{
+      %Request{
         body: %Tesla.Multipart{
           boundary: _,
           content_type_params: [],
@@ -70,7 +55,7 @@ test "add file parameter" do
             }
           ]
         }
-      } = RequestBuilder.add_param(%{}, :file, "filename.png", "some-path/filename.png")
+      } = RequestBuilder.add_param(%Request{}, :file, "filename.png", "some-path/filename.png")
     end
 
     test "add key to body" do
@@ -82,7 +67,7 @@ test "add key to body" do
             %Tesla.Multipart.Part{
               body: "\"someval\"",
               dispositions: [name: "somekey"],
-              headers: ["Content-Type": "application/json"]
+              headers: [{"content-type", "application/json"}]
             }
           ]
         }

test/http_test.exs

@@ -3,8 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.HTTPTest do
-  use Pleroma.DataCase
+  use ExUnit.Case, async: true
+  use Pleroma.Tests.Helpers
   import Tesla.Mock
+  alias Pleroma.HTTP
 
   setup do
     mock(fn
@@ -27,7 +29,7 @@ defmodule Pleroma.HTTPTest do
 
   describe "get/1" do
     test "returns successfully result" do
-      assert Pleroma.HTTP.get("http://example.com/hello") == {
+      assert HTTP.get("http://example.com/hello") == {
                :ok,
                %Tesla.Env{status: 200, body: "hello"}
              }
@@ -36,7 +38,7 @@ test "returns successfully result" do
 
   describe "get/2 (with headers)" do
     test "returns successfully result for json content-type" do
-      assert Pleroma.HTTP.get("http://example.com/hello", [{"content-type", "application/json"}]) ==
+      assert HTTP.get("http://example.com/hello", [{"content-type", "application/json"}]) ==
                {
                  :ok,
                  %Tesla.Env{
@@ -50,7 +52,7 @@ test "returns successfully result for json content-type" do
 
   describe "post/2" do
     test "returns successfully result" do
-      assert Pleroma.HTTP.post("http://example.com/world", "") == {
+      assert HTTP.post("http://example.com/world", "") == {
                :ok,
                %Tesla.Env{status: 200, body: "world"}
              }

test/notification_test.exs

@@ -784,12 +784,20 @@ test "notifications are deleted if a remote user is deleted" do
         "object" => remote_user.ap_id
       }
 
+      remote_user_url = remote_user.ap_id
+
+      Tesla.Mock.mock(fn
+        %{method: :get, url: ^remote_user_url} ->
+          %Tesla.Env{status: 404, body: ""}
+      end)
+
       {:ok, _delete_activity} = Transmogrifier.handle_incoming(delete_user_message)
       ObanHelpers.perform_all()
 
       assert Enum.empty?(Notification.for_user(local_user))
     end
 
+    @tag capture_log: true
     test "move activity generates a notification" do
       %{ap_id: old_ap_id} = old_user = insert(:user)
       %{ap_id: new_ap_id} = new_user = insert(:user, also_known_as: [old_ap_id])
@@ -799,6 +807,18 @@ test "move activity generates a notification" do
       User.follow(follower, old_user)
       User.follow(other_follower, old_user)
 
+      old_user_url = old_user.ap_id
+
+      body =
+        File.read!("test/fixtures/users_mock/localhost.json")
+        |> String.replace("{{nickname}}", old_user.nickname)
+        |> Jason.encode!()
+
+      Tesla.Mock.mock(fn
+        %{method: :get, url: ^old_user_url} ->
+          %Tesla.Env{status: 200, body: body}
+      end)
+
       Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user)
       ObanHelpers.perform_all()
 

test/otp_version_test.exs

@@ -0,0 +1,42 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.OTPVersionTest do
+  use ExUnit.Case, async: true
+
+  alias Pleroma.OTPVersion
+
+  describe "check/1" do
+    test "22.4" do
+      assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/22.4"]) ==
+               "22.4"
+    end
+
+    test "22.1" do
+      assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/22.1"]) ==
+               "22.1"
+    end
+
+    test "21.1" do
+      assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/21.1"]) ==
+               "21.1"
+    end
+
+    test "23.0" do
+      assert OTPVersion.get_version_from_files(["test/fixtures/warnings/otp_version/23.0"]) ==
+               "23.0"
+    end
+
+    test "with non existance file" do
+      assert OTPVersion.get_version_from_files([
+               "test/fixtures/warnings/otp_version/non-exising",
+               "test/fixtures/warnings/otp_version/22.4"
+             ]) == "22.4"
+    end
+
+    test "empty paths" do
+      assert OTPVersion.get_version_from_files([]) == nil
+    end
+  end
+end

test/pool/connections_test.exs

@@ -0,0 +1,760 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Pool.ConnectionsTest do
+  use ExUnit.Case, async: true
+  use Pleroma.Tests.Helpers
+
+  import ExUnit.CaptureLog
+  import Mox
+
+  alias Pleroma.Gun.Conn
+  alias Pleroma.GunMock
+  alias Pleroma.Pool.Connections
+
+  setup :verify_on_exit!
+
+  setup_all do
+    name = :test_connections
+    {:ok, pid} = Connections.start_link({name, [checkin_timeout: 150]})
+    {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.GunMock)
+
+    on_exit(fn ->
+      if Process.alive?(pid), do: GenServer.stop(name)
+    end)
+
+    {:ok, name: name}
+  end
+
+  defp open_mock(num \\ 1) do
+    GunMock
+    |> expect(:open, num, &start_and_register(&1, &2, &3))
+    |> expect(:await_up, num, fn _, _ -> {:ok, :http} end)
+    |> expect(:set_owner, num, fn _, _ -> :ok end)
+  end
+
+  defp connect_mock(mock) do
+    mock
+    |> expect(:connect, &connect(&1, &2))
+    |> expect(:await, &await(&1, &2))
+  end
+
+  defp info_mock(mock), do: expect(mock, :info, &info(&1))
+
+  defp start_and_register('gun-not-up.com', _, _), do: {:error, :timeout}
+
+  defp start_and_register(host, port, _) do
+    {:ok, pid} = Task.start_link(fn -> Process.sleep(1000) end)
+
+    scheme =
+      case port do
+        443 -> "https"
+        _ -> "http"
+      end
+
+    Registry.register(GunMock, pid, %{
+      origin_scheme: scheme,
+      origin_host: host,
+      origin_port: port
+    })
+
+    {:ok, pid}
+  end
+
+  defp info(pid) do
+    [{_, info}] = Registry.lookup(GunMock, pid)
+    info
+  end
+
+  defp connect(pid, _) do
+    ref = make_ref()
+    Registry.register(GunMock, ref, pid)
+    ref
+  end
+
+  defp await(pid, ref) do
+    [{_, ^pid}] = Registry.lookup(GunMock, ref)
+    {:response, :fin, 200, []}
+  end
+
+  defp now, do: :os.system_time(:second)
+
+  describe "alive?/2" do
+    test "is alive", %{name: name} do
+      assert Connections.alive?(name)
+    end
+
+    test "returns false if not started" do
+      refute Connections.alive?(:some_random_name)
+    end
+  end
+
+  test "opens connection and reuse it on next request", %{name: name} do
+    open_mock()
+    url = "http://some-domain.com"
+    key = "http:some-domain.com:80"
+    refute Connections.checkin(url, name)
+    :ok = Conn.open(url, name)
+
+    conn = Connections.checkin(url, name)
+    assert is_pid(conn)
+    assert Process.alive?(conn)
+
+    self = self()
+
+    %Connections{
+      conns: %{
+        ^key => %Conn{
+          conn: ^conn,
+          gun_state: :up,
+          used_by: [{^self, _}],
+          conn_state: :active
+        }
+      }
+    } = Connections.get_state(name)
+
+    reused_conn = Connections.checkin(url, name)
+
+    assert conn == reused_conn
+
+    %Connections{
+      conns: %{
+        ^key => %Conn{
+          conn: ^conn,
+          gun_state: :up,
+          used_by: [{^self, _}, {^self, _}],
+          conn_state: :active
+        }
+      }
+    } = Connections.get_state(name)
+
+    :ok = Connections.checkout(conn, self, name)
+
+    %Connections{
+      conns: %{
+        ^key => %Conn{
+          conn: ^conn,
+          gun_state: :up,
+          used_by: [{^self, _}],
+          conn_state: :active
+        }
+      }
+    } = Connections.get_state(name)
+
+    :ok = Connections.checkout(conn, self, name)
+
+    %Connections{
+      conns: %{
+        ^key => %Conn{
+          conn: ^conn,
+          gun_state: :up,
+          used_by: [],
+          conn_state: :idle
+        }
+      }
+    } = Connections.get_state(name)
+  end
+
+  test "reuse connection for idna domains", %{name: name} do
+    open_mock()
+    url = "http://ですsome-domain.com"
+    refute Connections.checkin(url, name)
+
+    :ok = Conn.open(url, name)
+
+    conn = Connections.checkin(url, name)
+    assert is_pid(conn)
+    assert Process.alive?(conn)
+
+    self = self()
+
+    %Connections{
+      conns: %{
+        "http:ですsome-domain.com:80" => %Conn{
+          conn: ^conn,
+          gun_state: :up,
+          used_by: [{^self, _}],
+          conn_state: :active
+        }
+      }
+    } = Connections.get_state(name)
+
+    reused_conn = Connections.checkin(url, name)
+
+    assert conn == reused_conn
+  end
+
+  test "reuse for ipv4", %{name: name} do
+    open_mock()
+    url = "http://127.0.0.1"
+
+    refute Connections.checkin(url, name)
+
+    :ok = Conn.open(url, name)
+
+    conn = Connections.checkin(url, name)
+    assert is_pid(conn)
+    assert Process.alive?(conn)
+
+    self = self()
+
+    %Connections{
+      conns: %{
+        "http:127.0.0.1:80" => %Conn{
+          conn: ^conn,
+          gun_state: :up,
+          used_by: [{^self, _}],
+          conn_state: :active
+        }
+      }
+    } = Connections.get_state(name)
+
+    reused_conn = Connections.checkin(url, name)
+
+    assert conn == reused_conn
+
+    :ok = Connections.checkout(conn, self, name)
+    :ok = Connections.checkout(reused_conn, self, name)
+
+    %Connections{
+      conns: %{
+        "http:127.0.0.1:80" => %Conn{
+          conn: ^conn,
+          gun_state: :up,
+          used_by: [],
+          conn_state: :idle
+        }
+      }
+    } = Connections.get_state(name)
+  end
+
+  test "reuse for ipv6", %{name: name} do
+    open_mock()
+    url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]"
+
+    refute Connections.checkin(url, name)
+
+    :ok = Conn.open(url, name)
+
+    conn = Connections.checkin(url, name)
+    assert is_pid(conn)
+    assert Process.alive?(conn)
+
+    self = self()
+
+    %Connections{
+      conns: %{
+        "http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Conn{
+          conn: ^conn,
+          gun_state: :up,
+          used_by: [{^self, _}],
+          conn_state: :active
+        }
+      }
+    } = Connections.get_state(name)
+
+    reused_conn = Connections.checkin(url, name)
+
+    assert conn == reused_conn
+  end
+
+  test "up and down ipv4", %{name: name} do
+    open_mock()
+    |> info_mock()
+    |> allow(self(), name)
+
+    self = self()
+    url = "http://127.0.0.1"
+    :ok = Conn.open(url, name)
+    conn = Connections.checkin(url, name)
+    send(name, {:gun_down, conn, nil, nil, nil})
+    send(name, {:gun_up, conn, nil})
+
+    %Connections{
+      conns: %{
+        "http:127.0.0.1:80" => %Conn{
+          conn: ^conn,
+          gun_state: :up,
+          used_by: [{^self, _}],
+          conn_state: :active
+        }
+      }
+    } = Connections.get_state(name)
+  end
+
+  test "up and down ipv6", %{name: name} do
+    self = self()
+
+    open_mock()
+    |> info_mock()
+    |> allow(self, name)
+
+    url = "http://[2a03:2880:f10c:83:face:b00c:0:25de]"
+    :ok = Conn.open(url, name)
+    conn = Connections.checkin(url, name)
+    send(name, {:gun_down, conn, nil, nil, nil})
+    send(name, {:gun_up, conn, nil})
+
+    %Connections{
+      conns: %{
+        "http:2a03:2880:f10c:83:face:b00c:0:25de:80" => %Conn{
+          conn: ^conn,
+          gun_state: :up,
+          used_by: [{^self, _}],
+          conn_state: :active
+        }
+      }
+    } = Connections.get_state(name)
+  end
+
+  test "reuses connection based on protocol", %{name: name} do
+    open_mock(2)
+    http_url = "http://some-domain.com"
+    http_key = "http:some-domain.com:80"
+    https_url = "https://some-domain.com"
+    https_key = "https:some-domain.com:443"
+
+    refute Connections.checkin(http_url, name)
+    :ok = Conn.open(http_url, name)
+    conn = Connections.checkin(http_url, name)
+    assert is_pid(conn)
+    assert Process.alive?(conn)
+
+    refute Connections.checkin(https_url, name)
+    :ok = Conn.open(https_url, name)
+    https_conn = Connections.checkin(https_url, name)
+
+    refute conn == https_conn
+
+    reused_https = Connections.checkin(https_url, name)
+
+    refute conn == reused_https
+
+    assert reused_https == https_conn
+
+    %Connections{
+      conns: %{
+        ^http_key => %Conn{
+          conn: ^conn,
+          gun_state: :up
+        },
+        ^https_key => %Conn{
+          conn: ^https_conn,
+          gun_state: :up
+        }
+      }
+    } = Connections.get_state(name)
+  end
+
+  test "connection can't get up", %{name: name} do
+    expect(GunMock, :open, &start_and_register(&1, &2, &3))
+    url = "http://gun-not-up.com"
+
+    assert capture_log(fn ->
+             refute Conn.open(url, name)
+             refute Connections.checkin(url, name)
+           end) =~
+             "Opening connection to http://gun-not-up.com failed with error {:error, :timeout}"
+  end
+
+  test "process gun_down message and then gun_up", %{name: name} do
+    self = self()
+
+    open_mock()
+    |> info_mock()
+    |> allow(self, name)
+
+    url = "http://gun-down-and-up.com"
+    key = "http:gun-down-and-up.com:80"
+    :ok = Conn.open(url, name)
+    conn = Connections.checkin(url, name)
+
+    assert is_pid(conn)
+    assert Process.alive?(conn)
+
+    %Connections{
+      conns: %{
+        ^key => %Conn{
+          conn: ^conn,
+          gun_state: :up,
+          used_by: [{^self, _}]
+        }
+      }
+    } = Connections.get_state(name)
+
+    send(name, {:gun_down, conn, :http, nil, nil})
+
+    %Connections{
+      conns: %{
+        ^key => %Conn{
+          conn: ^conn,
+          gun_state: :down,
+          used_by: [{^self, _}]
+        }
+      }
+    } = Connections.get_state(name)
+
+    send(name, {:gun_up, conn, :http})
+
+    conn2 = Connections.checkin(url, name)
+    assert conn == conn2
+
+    assert is_pid(conn2)
+    assert Process.alive?(conn2)
+
+    %Connections{
+      conns: %{
+        ^key => %Conn{
+          conn: _,
+          gun_state: :up,
+          used_by: [{^self, _}, {^self, _}]
+        }
+      }
+    } = Connections.get_state(name)
+  end
+
+  test "async processes get same conn for same domain", %{name: name} do
+    open_mock()
+    url = "http://some-domain.com"
+    :ok = Conn.open(url, name)
+
+    tasks =
+      for _ <- 1..5 do
+        Task.async(fn ->
+          Connections.checkin(url, name)
+        end)
+      end
+
+    tasks_with_results = Task.yield_many(tasks)
+
+    results =
+      Enum.map(tasks_with_results, fn {task, res} ->
+        res || Task.shutdown(task, :brutal_kill)
+      end)
+
+    conns = for {:ok, value} <- results, do: value
+
+    %Connections{
+      conns: %{
+        "http:some-domain.com:80" => %Conn{
+          conn: conn,
+          gun_state: :up
+        }
+      }
+    } = Connections.get_state(name)
+
+    assert Enum.all?(conns, fn res -> res == conn end)
+  end
+
+  test "remove frequently used and idle", %{name: name} do
+    open_mock(3)
+    self = self()
+    http_url = "http://some-domain.com"
+    https_url = "https://some-domain.com"
+    :ok = Conn.open(https_url, name)
+    :ok = Conn.open(http_url, name)
+
+    conn1 = Connections.checkin(https_url, name)
+
+    [conn2 | _conns] =
+      for _ <- 1..4 do
+        Connections.checkin(http_url, name)
+      end
+
+    http_key = "http:some-domain.com:80"
+
+    %Connections{
+      conns: %{
+        ^http_key => %Conn{
+          conn: ^conn2,
+          gun_state: :up,
+          conn_state: :active,
+          used_by: [{^self, _}, {^self, _}, {^self, _}, {^self, _}]
+        },
+        "https:some-domain.com:443" => %Conn{
+          conn: ^conn1,
+          gun_state: :up,
+          conn_state: :active,
+          used_by: [{^self, _}]
+        }
+      }
+    } = Connections.get_state(name)
+
+    :ok = Connections.checkout(conn1, self, name)
+
+    another_url = "http://another-domain.com"
+    :ok = Conn.open(another_url, name)
+    conn = Connections.checkin(another_url, name)
+
+    %Connections{
+      conns: %{
+        "http:another-domain.com:80" => %Conn{
+          conn: ^conn,
+          gun_state: :up
+        },
+        ^http_key => %Conn{
+          conn: _,
+          gun_state: :up
+        }
+      }
+    } = Connections.get_state(name)
+  end
+
+  describe "with proxy" do
+    test "as ip", %{name: name} do
+      open_mock()
+      |> connect_mock()
+
+      url = "http://proxy-string.com"
+      key = "http:proxy-string.com:80"
+      :ok = Conn.open(url, name, proxy: {{127, 0, 0, 1}, 8123})
+
+      conn = Connections.checkin(url, name)
+
+      %Connections{
+        conns: %{
+          ^key => %Conn{
+            conn: ^conn,
+            gun_state: :up
+          }
+        }
+      } = Connections.get_state(name)
+
+      reused_conn = Connections.checkin(url, name)
+
+      assert reused_conn == conn
+    end
+
+    test "as host", %{name: name} do
+      open_mock()
+      |> connect_mock()
+
+      url = "http://proxy-tuple-atom.com"
+      :ok = Conn.open(url, name, proxy: {'localhost', 9050})
+      conn = Connections.checkin(url, name)
+
+      %Connections{
+        conns: %{
+          "http:proxy-tuple-atom.com:80" => %Conn{
+            conn: ^conn,
+            gun_state: :up
+          }
+        }
+      } = Connections.get_state(name)
+
+      reused_conn = Connections.checkin(url, name)
+
+      assert reused_conn == conn
+    end
+
+    test "as ip and ssl", %{name: name} do
+      open_mock()
+      |> connect_mock()
+
+      url = "https://proxy-string.com"
+
+      :ok = Conn.open(url, name, proxy: {{127, 0, 0, 1}, 8123})
+      conn = Connections.checkin(url, name)
+
+      %Connections{
+        conns: %{
+          "https:proxy-string.com:443" => %Conn{
+            conn: ^conn,
+            gun_state: :up
+          }
+        }
+      } = Connections.get_state(name)
+
+      reused_conn = Connections.checkin(url, name)
+
+      assert reused_conn == conn
+    end
+
+    test "as host and ssl", %{name: name} do
+      open_mock()
+      |> connect_mock()
+
+      url = "https://proxy-tuple-atom.com"
+      :ok = Conn.open(url, name, proxy: {'localhost', 9050})
+      conn = Connections.checkin(url, name)
+
+      %Connections{
+        conns: %{
+          "https:proxy-tuple-atom.com:443" => %Conn{
+            conn: ^conn,
+            gun_state: :up
+          }
+        }
+      } = Connections.get_state(name)
+
+      reused_conn = Connections.checkin(url, name)
+
+      assert reused_conn == conn
+    end
+
+    test "with socks type", %{name: name} do
+      open_mock()
+
+      url = "http://proxy-socks.com"
+
+      :ok = Conn.open(url, name, proxy: {:socks5, 'localhost', 1234})
+
+      conn = Connections.checkin(url, name)
+
+      %Connections{
+        conns: %{
+          "http:proxy-socks.com:80" => %Conn{
+            conn: ^conn,
+            gun_state: :up
+          }
+        }
+      } = Connections.get_state(name)
+
+      reused_conn = Connections.checkin(url, name)
+
+      assert reused_conn == conn
+    end
+
+    test "with socks4 type and ssl", %{name: name} do
+      open_mock()
+      url = "https://proxy-socks.com"
+
+      :ok = Conn.open(url, name, proxy: {:socks4, 'localhost', 1234})
+
+      conn = Connections.checkin(url, name)
+
+      %Connections{
+        conns: %{
+          "https:proxy-socks.com:443" => %Conn{
+            conn: ^conn,
+            gun_state: :up
+          }
+        }
+      } = Connections.get_state(name)
+
+      reused_conn = Connections.checkin(url, name)
+
+      assert reused_conn == conn
+    end
+  end
+
+  describe "crf/3" do
+    setup do
+      crf = Connections.crf(1, 10, 1)
+      {:ok, crf: crf}
+    end
+
+    test "more used will have crf higher", %{crf: crf} do
+      # used 3 times
+      crf1 = Connections.crf(1, 10, crf)
+      crf1 = Connections.crf(1, 10, crf1)
+
+      # used 2 times
+      crf2 = Connections.crf(1, 10, crf)
+
+      assert crf1 > crf2
+    end
+
+    test "recently used will have crf higher on equal references", %{crf: crf} do
+      # used 3 sec ago
+      crf1 = Connections.crf(3, 10, crf)
+
+      # used 4 sec ago
+      crf2 = Connections.crf(4, 10, crf)
+
+      assert crf1 > crf2
+    end
+
+    test "equal crf on equal reference and time", %{crf: crf} do
+      # used 2 times
+      crf1 = Connections.crf(1, 10, crf)
+
+      # used 2 times
+      crf2 = Connections.crf(1, 10, crf)
+
+      assert crf1 == crf2
+    end
+
+    test "recently used will have higher crf", %{crf: crf} do
+      crf1 = Connections.crf(2, 10, crf)
+      crf1 = Connections.crf(1, 10, crf1)
+
+      crf2 = Connections.crf(3, 10, crf)
+      crf2 = Connections.crf(4, 10, crf2)
+      assert crf1 > crf2
+    end
+  end
+
+  describe "get_unused_conns/1" do
+    test "crf is equalent, sorting by reference", %{name: name} do
+      Connections.add_conn(name, "1", %Conn{
+        conn_state: :idle,
+        last_reference: now() - 1
+      })
+
+      Connections.add_conn(name, "2", %Conn{
+        conn_state: :idle,
+        last_reference: now()
+      })
+
+      assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
+    end
+
+    test "reference is equalent, sorting by crf", %{name: name} do
+      Connections.add_conn(name, "1", %Conn{
+        conn_state: :idle,
+        crf: 1.999
+      })
+
+      Connections.add_conn(name, "2", %Conn{
+        conn_state: :idle,
+        crf: 2
+      })
+
+      assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
+    end
+
+    test "higher crf and lower reference", %{name: name} do
+      Connections.add_conn(name, "1", %Conn{
+        conn_state: :idle,
+        crf: 3,
+        last_reference: now() - 1
+      })
+
+      Connections.add_conn(name, "2", %Conn{
+        conn_state: :idle,
+        crf: 2,
+        last_reference: now()
+      })
+
+      assert [{"2", _unused_conn} | _others] = Connections.get_unused_conns(name)
+    end
+
+    test "lower crf and lower reference", %{name: name} do
+      Connections.add_conn(name, "1", %Conn{
+        conn_state: :idle,
+        crf: 1.99,
+        last_reference: now() - 1
+      })
+
+      Connections.add_conn(name, "2", %Conn{
+        conn_state: :idle,
+        crf: 2,
+        last_reference: now()
+      })
+
+      assert [{"1", _unused_conn} | _others] = Connections.get_unused_conns(name)
+    end
+  end
+
+  test "count/1" do
+    name = :test_count
+    {:ok, _} = Connections.start_link({name, [checkin_timeout: 150]})
+    assert Connections.count(name) == 0
+    Connections.add_conn(name, "1", %Conn{conn: self()})
+    assert Connections.count(name) == 1
+    Connections.remove_conn(name, "1")
+    assert Connections.count(name) == 0
+  end
+end

test/reverse_proxy/reverse_proxy_test.exs

@@ -4,13 +4,16 @@
 
 defmodule Pleroma.ReverseProxyTest do
   use Pleroma.Web.ConnCase, async: true
+
   import ExUnit.CaptureLog
   import Mox
+
   alias Pleroma.ReverseProxy
   alias Pleroma.ReverseProxy.ClientMock
+  alias Plug.Conn
 
   setup_all do
-    {:ok, _} = Registry.start_link(keys: :unique, name: Pleroma.ReverseProxy.ClientMock)
+    {:ok, _} = Registry.start_link(keys: :unique, name: ClientMock)
     :ok
   end
 
@@ -21,7 +24,7 @@ defp user_agent_mock(user_agent, invokes) do
 
     ClientMock
     |> expect(:request, fn :get, url, _, _, _ ->
-      Registry.register(Pleroma.ReverseProxy.ClientMock, url, 0)
+      Registry.register(ClientMock, url, 0)
 
       {:ok, 200,
        [
@@ -29,14 +32,14 @@ defp user_agent_mock(user_agent, invokes) do
          {"content-length", byte_size(json) |> to_string()}
        ], %{url: url}}
     end)
-    |> expect(:stream_body, invokes, fn %{url: url} ->
+    |> expect(:stream_body, invokes, fn %{url: url} = client ->
-      case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do
+      case Registry.lookup(ClientMock, url) do
         [{_, 0}] ->
-          Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1))
+          Registry.update_value(ClientMock, url, &(&1 + 1))
-          {:ok, json}
+          {:ok, json, client}
 
         [{_, 1}] ->
-          Registry.unregister(Pleroma.ReverseProxy.ClientMock, url)
+          Registry.unregister(ClientMock, url)
           :done
       end
     end)
@@ -78,7 +81,39 @@ test "closed connection", %{conn: conn} do
     assert conn.halted
   end
 
-  describe "max_body " do
+  defp stream_mock(invokes, with_close? \\ false) do
+    ClientMock
+    |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ ->
+      Registry.register(ClientMock, "/stream-bytes/" <> length, 0)
+
+      {:ok, 200, [{"content-type", "application/octet-stream"}],
+       %{url: "/stream-bytes/" <> length}}
+    end)
+    |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} = client ->
+      max = String.to_integer(length)
+
+      case Registry.lookup(ClientMock, "/stream-bytes/" <> length) do
+        [{_, current}] when current < max ->
+          Registry.update_value(
+            ClientMock,
+            "/stream-bytes/" <> length,
+            &(&1 + 10)
+          )
+
+          {:ok, "0123456789", client}
+
+        [{_, ^max}] ->
+          Registry.unregister(ClientMock, "/stream-bytes/" <> length)
+          :done
+      end
+    end)
+
+    if with_close? do
+      expect(ClientMock, :close, fn _ -> :ok end)
+    end
+  end
+
+  describe "max_body" do
     test "length returns error if content-length more than option", %{conn: conn} do
       user_agent_mock("hackney/1.15.1", 0)
 
@@ -94,38 +129,6 @@ test "length returns error if content-length more than option", %{conn: conn} do
              end) == ""
     end
 
-    defp stream_mock(invokes, with_close? \\ false) do
-      ClientMock
-      |> expect(:request, fn :get, "/stream-bytes/" <> length, _, _, _ ->
-        Registry.register(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length, 0)
-
-        {:ok, 200, [{"content-type", "application/octet-stream"}],
-         %{url: "/stream-bytes/" <> length}}
-      end)
-      |> expect(:stream_body, invokes, fn %{url: "/stream-bytes/" <> length} ->
-        max = String.to_integer(length)
-
-        case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length) do
-          [{_, current}] when current < max ->
-            Registry.update_value(
-              Pleroma.ReverseProxy.ClientMock,
-              "/stream-bytes/" <> length,
-              &(&1 + 10)
-            )
-
-            {:ok, "0123456789"}
-
-          [{_, ^max}] ->
-            Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/stream-bytes/" <> length)
-            :done
-        end
-      end)
-
-      if with_close? do
-        expect(ClientMock, :close, fn _ -> :ok end)
-      end
-    end
-
     test "max_body_length returns error if streaming body more than that option", %{conn: conn} do
       stream_mock(3, true)
 
@@ -214,24 +217,24 @@ test "streaming", %{conn: conn} do
     conn = ReverseProxy.call(conn, "/stream-bytes/200")
     assert conn.state == :chunked
     assert byte_size(conn.resp_body) == 200
-    assert Plug.Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"]
+    assert Conn.get_resp_header(conn, "content-type") == ["application/octet-stream"]
   end
 
   defp headers_mock(_) do
     ClientMock
     |> expect(:request, fn :get, "/headers", headers, _, _ ->
-      Registry.register(Pleroma.ReverseProxy.ClientMock, "/headers", 0)
+      Registry.register(ClientMock, "/headers", 0)
       {:ok, 200, [{"content-type", "application/json"}], %{url: "/headers", headers: headers}}
     end)
-    |> expect(:stream_body, 2, fn %{url: url, headers: headers} ->
+    |> expect(:stream_body, 2, fn %{url: url, headers: headers} = client ->
-      case Registry.lookup(Pleroma.ReverseProxy.ClientMock, url) do
+      case Registry.lookup(ClientMock, url) do
         [{_, 0}] ->
-          Registry.update_value(Pleroma.ReverseProxy.ClientMock, url, &(&1 + 1))
+          Registry.update_value(ClientMock, url, &(&1 + 1))
           headers = for {k, v} <- headers, into: %{}, do: {String.capitalize(k), v}
-          {:ok, Jason.encode!(%{headers: headers})}
+          {:ok, Jason.encode!(%{headers: headers}), client}
 
         [{_, 1}] ->
-          Registry.unregister(Pleroma.ReverseProxy.ClientMock, url)
+          Registry.unregister(ClientMock, url)
           :done
       end
     end)
@@ -244,7 +247,7 @@ defp headers_mock(_) do
 
     test "header passes", %{conn: conn} do
       conn =
-        Plug.Conn.put_req_header(
+        Conn.put_req_header(
           conn,
           "accept",
           "text/html"
@@ -257,7 +260,7 @@ test "header passes", %{conn: conn} do
 
     test "header is filtered", %{conn: conn} do
       conn =
-        Plug.Conn.put_req_header(
+        Conn.put_req_header(
           conn,
           "accept-language",
           "en-US"
@@ -290,18 +293,18 @@ test "add cache-control", %{conn: conn} do
   defp disposition_headers_mock(headers) do
     ClientMock
     |> expect(:request, fn :get, "/disposition", _, _, _ ->
-      Registry.register(Pleroma.ReverseProxy.ClientMock, "/disposition", 0)
+      Registry.register(ClientMock, "/disposition", 0)
 
       {:ok, 200, headers, %{url: "/disposition"}}
     end)
-    |> expect(:stream_body, 2, fn %{url: "/disposition"} ->
+    |> expect(:stream_body, 2, fn %{url: "/disposition"} = client ->
-      case Registry.lookup(Pleroma.ReverseProxy.ClientMock, "/disposition") do
+      case Registry.lookup(ClientMock, "/disposition") do
         [{_, 0}] ->
-          Registry.update_value(Pleroma.ReverseProxy.ClientMock, "/disposition", &(&1 + 1))
+          Registry.update_value(ClientMock, "/disposition", &(&1 + 1))
-          {:ok, ""}
+          {:ok, "", client}
 
         [{_, 1}] ->
-          Registry.unregister(Pleroma.ReverseProxy.ClientMock, "/disposition")
+          Registry.unregister(ClientMock, "/disposition")
           :done
       end
     end)

test/support/http_request_mock.ex

@@ -107,7 +107,7 @@ def get(
         "https://osada.macgirvin.com/.well-known/webfinger?resource=acct:mike@osada.macgirvin.com",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -120,7 +120,7 @@ def get(
         "https://social.heldscal.la/.well-known/webfinger?resource=https://social.heldscal.la/user/29191",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -141,7 +141,7 @@ def get(
         "https://pawoo.net/.well-known/webfinger?resource=acct:https://pawoo.net/users/pekorino",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -167,7 +167,7 @@ def get(
         "https://social.stopwatchingus-heidelberg.de/.well-known/webfinger?resource=acct:https://social.stopwatchingus-heidelberg.de/user/18330",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -188,7 +188,7 @@ def get(
         "https://mamot.fr/.well-known/webfinger?resource=acct:https://mamot.fr/users/Skruyb",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -201,7 +201,7 @@ def get(
         "https://social.heldscal.la/.well-known/webfinger?resource=nonexistant@social.heldscal.la",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -214,7 +214,7 @@ def get(
         "https://squeet.me/xrd/?uri=lain@squeet.me",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -227,7 +227,7 @@ def get(
         "https://mst3k.interlinked.me/users/luciferMysticus",
         _,
         _,
-        Accept: "application/activity+json"
+        [{"accept", "application/activity+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -248,7 +248,7 @@ def get(
         "https://hubzilla.example.org/channel/kaniini",
         _,
         _,
-        Accept: "application/activity+json"
+        [{"accept", "application/activity+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -257,7 +257,7 @@ def get(
      }}
   end
 
-  def get("https://niu.moe/users/rye", _, _, Accept: "application/activity+json") do
+  def get("https://niu.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -265,7 +265,7 @@ def get("https://niu.moe/users/rye", _, _, Accept: "application/activity+json")
      }}
   end
 
-  def get("https://n1u.moe/users/rye", _, _, Accept: "application/activity+json") do
+  def get("https://n1u.moe/users/rye", _, _, [{"accept", "application/activity+json"}]) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -284,7 +284,7 @@ def get("http://mastodon.example.org/users/admin/statuses/100787282858396771", _
      }}
   end
 
-  def get("https://puckipedia.com/", _, _, Accept: "application/activity+json") do
+  def get("https://puckipedia.com/", _, _, [{"accept", "application/activity+json"}]) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -308,9 +308,9 @@ def get("https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
      }}
   end
 
-  def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _, _,
+  def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _, _, [
-        Accept: "application/activity+json"
+        {"accept", "application/activity+json"}
-      ) do
+      ]) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -318,7 +318,7 @@ def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _,
      }}
   end
 
-  def get("https://mobilizon.org/@tcit", _, _, Accept: "application/activity+json") do
+  def get("https://mobilizon.org/@tcit", _, _, [{"accept", "application/activity+json"}]) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -358,7 +358,7 @@ def get("https://wedistribute.org/wp-json/pterotype/v1/actor/-blog", _, _, _) do
      }}
   end
 
-  def get("http://mastodon.example.org/users/admin", _, _, Accept: "application/activity+json") do
+  def get("http://mastodon.example.org/users/admin", _, _, _) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -366,7 +366,9 @@ def get("http://mastodon.example.org/users/admin", _, _, Accept: "application/ac
      }}
   end
 
-  def get("http://mastodon.example.org/users/relay", _, _, Accept: "application/activity+json") do
+  def get("http://mastodon.example.org/users/relay", _, _, [
+        {"accept", "application/activity+json"}
+      ]) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -374,7 +376,9 @@ def get("http://mastodon.example.org/users/relay", _, _, Accept: "application/ac
      }}
   end
 
-  def get("http://mastodon.example.org/users/gargron", _, _, Accept: "application/activity+json") do
+  def get("http://mastodon.example.org/users/gargron", _, _, [
+        {"accept", "application/activity+json"}
+      ]) do
     {:error, :nxdomain}
   end
 
@@ -557,7 +561,7 @@ def get(
         "http://mastodon.example.org/@admin/99541947525187367",
         _,
         _,
-        Accept: "application/activity+json"
+        _
       ) do
     {:ok,
      %Tesla.Env{
@@ -582,7 +586,7 @@ def get("https://shitposter.club/notice/7369654", _, _, _) do
      }}
   end
 
-  def get("https://mstdn.io/users/mayuutann", _, _, Accept: "application/activity+json") do
+  def get("https://mstdn.io/users/mayuutann", _, _, [{"accept", "application/activity+json"}]) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -594,7 +598,7 @@ def get(
         "https://mstdn.io/users/mayuutann/statuses/99568293732299394",
         _,
         _,
-        Accept: "application/activity+json"
+        [{"accept", "application/activity+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -614,7 +618,7 @@ def get("https://pleroma.soykaf.com/users/lain/feed.atom", _, _, _) do
      }}
   end
 
-  def get(url, _, _, Accept: "application/xrd+xml,application/jrd+json")
+  def get(url, _, _, [{"accept", "application/xrd+xml,application/jrd+json"}])
       when url in [
              "https://pleroma.soykaf.com/.well-known/webfinger?resource=acct:https://pleroma.soykaf.com/users/lain",
              "https://pleroma.soykaf.com/.well-known/webfinger?resource=https://pleroma.soykaf.com/users/lain"
@@ -641,7 +645,7 @@ def get(
         "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/1",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -685,7 +689,7 @@ def get(
         "https://shitposter.club/.well-known/webfinger?resource=https://shitposter.club/user/5381",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -738,7 +742,7 @@ def get(
         "https://social.sakamoto.gq/.well-known/webfinger?resource=https://social.sakamoto.gq/users/eal",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -751,7 +755,7 @@ def get(
         "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056",
         _,
         _,
-        Accept: "application/atom+xml"
+        [{"accept", "application/atom+xml"}]
       ) do
     {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sakamoto.atom")}}
   end
@@ -768,7 +772,7 @@ def get(
         "https://mastodon.social/.well-known/webfinger?resource=https://mastodon.social/users/lambadalambda",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -790,7 +794,7 @@ def get(
         "http://gs.example.org/.well-known/webfinger?resource=http://gs.example.org:4040/index.php/user/1",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -804,7 +808,7 @@ def get(
         "http://gs.example.org:4040/index.php/user/1",
         _,
         _,
-        Accept: "application/activity+json"
+        [{"accept", "application/activity+json"}]
       ) do
     {:ok, %Tesla.Env{status: 406, body: ""}}
   end
@@ -840,7 +844,7 @@ def get(
         "https://squeet.me/xrd?uri=lain@squeet.me",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -853,7 +857,7 @@ def get(
         "https://social.heldscal.la/.well-known/webfinger?resource=shp@social.heldscal.la",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -866,7 +870,7 @@ def get(
         "https://social.heldscal.la/.well-known/webfinger?resource=invalid_content@social.heldscal.la",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok, %Tesla.Env{status: 200, body: ""}}
   end
@@ -883,7 +887,7 @@ def get(
         "http://framatube.org/main/xrd?uri=framasoft@framatube.org",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -905,7 +909,7 @@ def get(
         "http://gnusocial.de/main/xrd?uri=winterdienst@gnusocial.de",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -942,7 +946,7 @@ def get(
         "https://gerzilla.de/xrd/?uri=kaniini@gerzilla.de",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -1005,7 +1009,7 @@ def get("https://apfed.club/channel/indio", _, _, _) do
      %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/osada-user-indio.json")}}
   end
 
-  def get("https://social.heldscal.la/user/23211", _, _, Accept: "application/activity+json") do
+  def get("https://social.heldscal.la/user/23211", _, _, [{"accept", "application/activity+json"}]) do
     {:ok, Tesla.Mock.json(%{"id" => "https://social.heldscal.la/user/23211"}, status: 200)}
   end
 
@@ -1138,7 +1142,7 @@ def get(
         "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=lain@zetsubou.xn--q9jyb4c",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -1151,7 +1155,7 @@ def get(
         "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=https://zetsubou.xn--q9jyb4c/users/lain",
         _,
         _,
-        Accept: "application/xrd+xml,application/jrd+json"
+        [{"accept", "application/xrd+xml,application/jrd+json"}]
       ) do
     {:ok,
      %Tesla.Env{
@@ -1173,7 +1177,9 @@ def get(
      }}
   end
 
-  def get("https://info.pleroma.site/activity.json", _, _, Accept: "application/activity+json") do
+  def get("https://info.pleroma.site/activity.json", _, _, [
+        {"accept", "application/activity+json"}
+      ]) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -1185,7 +1191,9 @@ def get("https://info.pleroma.site/activity.json", _, _, _) do
     {:ok, %Tesla.Env{status: 404, body: ""}}
   end
 
-  def get("https://info.pleroma.site/activity2.json", _, _, Accept: "application/activity+json") do
+  def get("https://info.pleroma.site/activity2.json", _, _, [
+        {"accept", "application/activity+json"}
+      ]) do
     {:ok,
      %Tesla.Env{
        status: 200,
@@ -1197,7 +1205,9 @@ def get("https://info.pleroma.site/activity2.json", _, _, _) do
     {:ok, %Tesla.Env{status: 404, body: ""}}
   end
 
-  def get("https://info.pleroma.site/activity3.json", _, _, Accept: "application/activity+json") do
+  def get("https://info.pleroma.site/activity3.json", _, _, [
+        {"accept", "application/activity+json"}
+      ]) do
     {:ok,
      %Tesla.Env{
        status: 200,

test/test_helper.exs

@@ -6,7 +6,10 @@
 ExUnit.start(exclude: [:federated | os_exclude])
 
 Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual)
+
 Mox.defmock(Pleroma.ReverseProxy.ClientMock, for: Pleroma.ReverseProxy.Client)
+Mox.defmock(Pleroma.GunMock, for: Pleroma.Gun)
+
 {:ok, _} = Application.ensure_all_started(:ex_machina)
 
 ExUnit.after_suite(fn _results ->

test/user_invite_token_test.exs

@@ -4,7 +4,6 @@
 
 defmodule Pleroma.UserInviteTokenTest do
   use ExUnit.Case, async: true
-  use Pleroma.DataCase
   alias Pleroma.UserInviteToken
 
   describe "valid_invite?/1 one time invites" do
@@ -64,7 +63,6 @@ test "expires today returns true", %{invite: invite} do
 
     test "expires yesterday returns false", %{invite: invite} do
       invite = %{invite | expires_at: Date.add(Date.utc_today(), -1)}
-      invite = Repo.insert!(invite)
       refute UserInviteToken.valid_invite?(invite)
     end
   end
@@ -82,7 +80,6 @@ test "not overdue date and less uses returns true", %{invite: invite} do
 
     test "overdue date and less uses returns false", %{invite: invite} do
       invite = %{invite | expires_at: Date.add(Date.utc_today(), -1)}
-      invite = Repo.insert!(invite)
       refute UserInviteToken.valid_invite?(invite)
     end
 
@@ -93,7 +90,6 @@ test "not overdue date with more uses returns false", %{invite: invite} do
 
     test "overdue date with more uses returns false", %{invite: invite} do
       invite = %{invite | expires_at: Date.add(Date.utc_today(), -1), uses: 5}
-      invite = Repo.insert!(invite)
       refute UserInviteToken.valid_invite?(invite)
     end
   end

test/web/activity_pub/mrf/anti_followbot_policy_test.exs

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicyTest do

test/web/activity_pub/mrf/anti_link_spam_policy_test.exs

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do
@@ -110,6 +110,15 @@ test "it allows posts with links" do
   end
 
   describe "with unknown actors" do
+    setup do
+      Tesla.Mock.mock(fn
+        %{method: :get, url: "http://invalid.actor"} ->
+          %Tesla.Env{status: 500, body: ""}
+      end)
+
+      :ok
+    end
+
     test "it rejects posts without links" do
       message =
         @linkless_message

test/web/activity_pub/mrf/ensure_re_prepended_test.exs

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrependedTest do

test/web/activity_pub/mrf/no_placeholder_text_policy_test.exs

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicyTest do

test/web/activity_pub/mrf/normalize_markup_test.exs

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkupTest do

test/web/activity_pub/mrf/object_age_policy_test.exs

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicyTest do

test/web/activity_pub/mrf/reject_non_public_test.exs

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublicTest do

test/web/activity_pub/mrf/simple_policy_test.exs

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do

test/web/activity_pub/relay_test.exs

@@ -89,6 +89,11 @@ test "returns error when object is unknown" do
           }
         )
 
+      Tesla.Mock.mock(fn
+        %{method: :get, url: "http://mastodon.example.org/eee/99541947525187367"} ->
+          %Tesla.Env{status: 500, body: ""}
+      end)
+
       assert capture_log(fn ->
                assert Relay.publish(activity) == {:error, nil}
              end) =~ "[error] error: nil"

test/web/admin_api/admin_api_controller_test.exs

@@ -2369,9 +2369,6 @@ test "update config setting & delete with fallback to default value", %{
     end
 
     test "common config example", %{conn: conn} do
-      adapter = Application.get_env(:tesla, :adapter)
-      on_exit(fn -> Application.put_env(:tesla, :adapter, adapter) end)
-
       conn =
         post(conn, "/api/pleroma/admin/config", %{
           configs: [
@@ -2384,23 +2381,16 @@ test "common config example", %{conn: conn} do
                 %{"tuple" => [":seconds_valid", 60]},
                 %{"tuple" => [":path", ""]},
                 %{"tuple" => [":key1", nil]},
-                %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]},
                 %{"tuple" => [":regex1", "~r/https:\/\/example.com/"]},
                 %{"tuple" => [":regex2", "~r/https:\/\/example.com/u"]},
                 %{"tuple" => [":regex3", "~r/https:\/\/example.com/i"]},
                 %{"tuple" => [":regex4", "~r/https:\/\/example.com/s"]},
                 %{"tuple" => [":name", "Pleroma"]}
               ]
-            },
-            %{
-              "group" => ":tesla",
-              "key" => ":adapter",
-              "value" => "Tesla.Adapter.Httpc"
             }
           ]
         })
 
-      assert Application.get_env(:tesla, :adapter) == Tesla.Adapter.Httpc
       assert Config.get([Pleroma.Captcha.NotReal, :name]) == "Pleroma"
 
       assert json_response(conn, 200) == %{
@@ -2414,7 +2404,6 @@ test "common config example", %{conn: conn} do
                      %{"tuple" => [":seconds_valid", 60]},
                      %{"tuple" => [":path", ""]},
                      %{"tuple" => [":key1", nil]},
-                     %{"tuple" => [":partial_chain", "&:hackney_connect.partial_chain/1"]},
                      %{"tuple" => [":regex1", "~r/https:\\/\\/example.com/"]},
                      %{"tuple" => [":regex2", "~r/https:\\/\\/example.com/u"]},
                      %{"tuple" => [":regex3", "~r/https:\\/\\/example.com/i"]},
@@ -2427,19 +2416,12 @@ test "common config example", %{conn: conn} do
                      ":seconds_valid",
                      ":path",
                      ":key1",
-                     ":partial_chain",
                      ":regex1",
                      ":regex2",
                      ":regex3",
                      ":regex4",
                      ":name"
                    ]
-                 },
-                 %{
-                   "group" => ":tesla",
-                   "key" => ":adapter",
-                   "value" => "Tesla.Adapter.Httpc",
-                   "db" => [":adapter"]
                  }
                ]
              }

test/web/common_api/common_api_utils_test.exs

@@ -472,6 +472,13 @@ test "returns recipients when object not found" do
       activity = insert(:note_activity, user: user, note: object)
       Pleroma.Repo.delete(object)
 
+      obj_url = activity.data["object"]
+
+      Tesla.Mock.mock(fn
+        %{method: :get, url: ^obj_url} ->
+          %Tesla.Env{status: 404, body: ""}
+      end)
+
       assert Utils.maybe_notify_mentioned_recipients(["test-test"], activity) == [
                "test-test"
              ]

test/web/mastodon_api/controllers/notification_controller_test.exs

@@ -472,11 +472,24 @@ test "see notifications after muting user with notifications and with_muted para
     assert length(json_response(conn, 200)) == 1
   end
 
+  @tag capture_log: true
   test "see move notifications" do
     old_user = insert(:user)
     new_user = insert(:user, also_known_as: [old_user.ap_id])
     %{user: follower, conn: conn} = oauth_access(["read:notifications"])
 
+    old_user_url = old_user.ap_id
+
+    body =
+      File.read!("test/fixtures/users_mock/localhost.json")
+      |> String.replace("{{nickname}}", old_user.nickname)
+      |> Jason.encode!()
+
+    Tesla.Mock.mock(fn
+      %{method: :get, url: ^old_user_url} ->
+        %Tesla.Env{status: 200, body: body}
+    end)
+
     User.follow(follower, old_user)
     Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user)
     Pleroma.Tests.ObanHelpers.perform_all()

test/web/mastodon_api/views/notification_view_test.exs

@@ -112,11 +112,24 @@ test "Follow notification" do
     test_notifications_rendering([notification], followed, [])
   end
 
+  @tag capture_log: true
   test "Move notification" do
     old_user = insert(:user)
     new_user = insert(:user, also_known_as: [old_user.ap_id])
     follower = insert(:user)
 
+    old_user_url = old_user.ap_id
+
+    body =
+      File.read!("test/fixtures/users_mock/localhost.json")
+      |> String.replace("{{nickname}}", old_user.nickname)
+      |> Jason.encode!()
+
+    Tesla.Mock.mock(fn
+      %{method: :get, url: ^old_user_url} ->
+        %Tesla.Env{status: 200, body: body}
+    end)
+
     User.follow(follower, old_user)
     Pleroma.Web.ActivityPub.ActivityPub.move(old_user, new_user)
     Pleroma.Tests.ObanHelpers.perform_all()

test/web/mastodon_api/views/status_view_test.exs

@@ -94,6 +94,23 @@ test "returns a temporary ap_id based user for activities missing db users" do
     Repo.delete(user)
     Cachex.clear(:user_cache)
 
+    finger_url =
+      "https://localhost/.well-known/webfinger?resource=acct:#{user.nickname}@localhost"
+
+    Tesla.Mock.mock_global(fn
+      %{method: :get, url: "http://localhost/.well-known/host-meta"} ->
+        %Tesla.Env{status: 404, body: ""}
+
+      %{method: :get, url: "https://localhost/.well-known/host-meta"} ->
+        %Tesla.Env{status: 404, body: ""}
+
+      %{
+        method: :get,
+        url: ^finger_url
+      } ->
+        %Tesla.Env{status: 404, body: ""}
+    end)
+
     %{account: ms_user} = StatusView.render("show.json", activity: activity)
 
     assert ms_user.acct == "erroruser@example.com"

test/web/push/impl_test.exs

@@ -134,7 +134,7 @@ test "renders title and body for follow activity" do
     user = insert(:user, nickname: "Bob")
     other_user = insert(:user)
     {:ok, _, _, activity} = CommonAPI.follow(user, other_user)
-    object = Object.normalize(activity)
+    object = Object.normalize(activity, false)
 
     assert Impl.format_body(%{activity: activity}, user, object) == "@Bob has followed you"
 

test/web/streamer/streamer_test.exs

@@ -121,6 +121,18 @@ test "it doesn't send notify to the 'user:notification' stream' when a domain is
     test "it sends follow activities to the 'user:notification' stream", %{
       user: user
     } do
+      user_url = user.ap_id
+
+      body =
+        File.read!("test/fixtures/users_mock/localhost.json")
+        |> String.replace("{{nickname}}", user.nickname)
+        |> Jason.encode!()
+
+      Tesla.Mock.mock_global(fn
+        %{method: :get, url: ^user_url} ->
+          %Tesla.Env{status: 200, body: body}
+      end)
+
       user2 = insert(:user)
       task = Task.async(fn -> assert_receive {:text, _}, @streamer_timeout end)