From a50c657427a2dfe9d48c25529a179fe634d30e48 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:17:02 -0400 Subject: [PATCH 1/7] Add a dedicated connection pool for Rich Media Sharing this pool with regular Media is problematic as Rich Media will connect to many different domains and thrash the pool, but regular Media will have predictable connections to the webservers hosting media for the fediverse servers you peer with. --- config/config.exs | 9 +++++++++ lib/pleroma/web/rich_media/helpers.ex | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 8b9a588b7..b8030651f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -827,6 +827,11 @@ max_waiting: 20, recv_timeout: 15_000 ], + rich_media: [ + size: 25, + max_waiting: 20, + recv_timeout: 15_000 + ], upload: [ size: 25, max_waiting: 5, @@ -847,6 +852,10 @@ max_connections: 50, timeout: 150_000 ], + rich_media: [ + max_connections: 50, + timeout: 150_000 + ], upload: [ max_connections: 25, timeout: 300_000 diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 119994458..ea41bd285 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -58,7 +58,7 @@ defp check_content_length(headers) do defp http_options do [ - pool: :media, + pool: :rich_media, max_body: Config.get([:rich_media, :max_body], 5_000_000) ] end From 6708f154a4f7ad46b4637d4d566b8cf81e3ebb7b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:18:58 -0400 Subject: [PATCH 2/7] Rework Gun connection pool sizes to make better use of the default 250 connections --- config/config.exs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config/config.exs b/config/config.exs index b8030651f..a025defeb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -818,27 +818,27 @@ config :pleroma, :pools, federation: [ - size: 50, - max_waiting: 10, + size: 75, + max_waiting: 20, recv_timeout: 10_000 ], media: [ - size: 50, + size: 75, max_waiting: 20, recv_timeout: 15_000 ], rich_media: [ size: 25, max_waiting: 20, - recv_timeout: 15_000 - ], + recv_timeout: 15_000 + ], upload: [ size: 25, - max_waiting: 5, + max_waiting: 20, recv_timeout: 15_000 ], default: [ - size: 10, + size: 50, max_waiting: 2, recv_timeout: 5_000 ] From d272eb62cd4da45e5ef82fcc0126d2cf799d292a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:22:45 -0400 Subject: [PATCH 3/7] Trust the connection pools to enforce the concurrency limitations --- .../web/activity_pub/mrf/media_proxy_warming_policy.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index c95d35bb9..f10dc3ce5 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -30,9 +30,7 @@ defp prefetch(url) do if Pleroma.Config.get(:env) == :test do fetch(prefetch_url) else - ConcurrentLimiter.limit(__MODULE__, fn -> - Task.start(fn -> fetch(prefetch_url) end) - end) + Task.start(fn -> fetch(prefetch_url) end) end end end From 37d79b76bb770cb294c0a54777b435dc49c042ab Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:24:54 -0400 Subject: [PATCH 4/7] Use the configured http client options for mediaproxy --- .../web/activity_pub/mrf/media_proxy_warming_policy.ex | 10 ++++------ lib/pleroma/web/media_proxy/media_proxy_controller.ex | 3 ++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index f10dc3ce5..e5eb6896a 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -11,11 +11,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @adapter_options [ - pool: :media, - recv_timeout: 10_000 - ] - @impl true def history_awareness, do: :auto @@ -35,7 +30,10 @@ defp prefetch(url) do end end - defp fetch(url), do: HTTP.get(url, [], @adapter_options) + defp fetch(url) do + http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) + HTTP.get(url, [], http_client_opts) + end defp preload(%{"object" => %{"attachment" => attachments}} = _message) do Enum.each(attachments, fn diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index c11484ecb..0b446e0a6 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -54,9 +54,10 @@ def preview(%Conn{} = conn, %{"sig" => sig64, "url" => url64}) do defp handle_preview(conn, url) do media_proxy_url = MediaProxy.url(url) + http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) with {:ok, %{status: status} = head_response} when status in 200..299 <- - Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do + Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do content_type = Tesla.get_header(head_response, "content-type") content_length = Tesla.get_header(head_response, "content-length") content_length = content_length && String.to_integer(content_length) From 8b61d4e3e124c4fafeaabe58a8c7673f767b2871 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 11:28:31 -0400 Subject: [PATCH 5/7] Changelogs --- changelog.d/mediaproxy-http.fix | 1 + changelog.d/pools.change | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/mediaproxy-http.fix create mode 100644 changelog.d/pools.change diff --git a/changelog.d/mediaproxy-http.fix b/changelog.d/mediaproxy-http.fix new file mode 100644 index 000000000..4ff6430e0 --- /dev/null +++ b/changelog.d/mediaproxy-http.fix @@ -0,0 +1 @@ +Ensure MediaProxy HTTP requests obey all the defined connection settings diff --git a/changelog.d/pools.change b/changelog.d/pools.change new file mode 100644 index 000000000..3c689195a --- /dev/null +++ b/changelog.d/pools.change @@ -0,0 +1 @@ +HTTP connection pool adjustments From 6b8c15a4a1fdbc2a2a4ef194d9519e717470c632 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 14:11:42 -0400 Subject: [PATCH 6/7] Remove MediaProxyWarmingPolicy config for ConcurrentLimiter as we are not using it --- config/config.exs | 1 - .../web/activity_pub/mrf/media_proxy_warming_policy.ex | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/config/config.exs b/config/config.exs index a025defeb..3c35f439a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -902,7 +902,6 @@ config :pleroma, ConcurrentLimiter, [ {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, - {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}, {Pleroma.Search, [max_running: 30, max_waiting: 50]} ] diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index e5eb6896a..0c5b53def 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -22,11 +22,7 @@ defp prefetch(url) do Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}") - if Pleroma.Config.get(:env) == :test do - fetch(prefetch_url) - else - Task.start(fn -> fetch(prefetch_url) end) - end + fetch(prefetch_url) end end From ba511a30b923cc9248686702f75a4041d69c7bee Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 27 May 2024 14:12:38 -0400 Subject: [PATCH 7/7] RichMedia use of ConcurrentLimiter was removed in the refactor --- config/config.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 3c35f439a..e7e751d8a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -901,7 +901,6 @@ process_chunk_size: 100 config :pleroma, ConcurrentLimiter, [ - {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, {Pleroma.Search, [max_running: 30, max_waiting: 50]} ]