Merge branch 'httpfixes' into 'develop'

Some HTTP and connection pool improvements

See merge request pleroma/pleroma!4124
This commit is contained in:
lain 2024-05-28 04:38:01 +00:00
commit 8ff0c32903
6 changed files with 24 additions and 22 deletions

View File

@ -0,0 +1 @@
Ensure MediaProxy HTTP requests obey all the defined connection settings

1
changelog.d/pools.change Normal file
View File

@ -0,0 +1 @@
HTTP connection pool adjustments

View File

@ -827,22 +827,27 @@
config :pleroma, :pools, config :pleroma, :pools,
federation: [ federation: [
size: 50, size: 75,
max_waiting: 10, max_waiting: 20,
recv_timeout: 10_000 recv_timeout: 10_000
], ],
media: [ media: [
size: 50, size: 75,
max_waiting: 20,
recv_timeout: 15_000
],
rich_media: [
size: 25,
max_waiting: 20, max_waiting: 20,
recv_timeout: 15_000 recv_timeout: 15_000
], ],
upload: [ upload: [
size: 25, size: 25,
max_waiting: 5, max_waiting: 20,
recv_timeout: 15_000 recv_timeout: 15_000
], ],
default: [ default: [
size: 10, size: 50,
max_waiting: 2, max_waiting: 2,
recv_timeout: 5_000 recv_timeout: 5_000
] ]
@ -856,6 +861,10 @@
max_connections: 50, max_connections: 50,
timeout: 150_000 timeout: 150_000
], ],
rich_media: [
max_connections: 50,
timeout: 150_000
],
upload: [ upload: [
max_connections: 25, max_connections: 25,
timeout: 300_000 timeout: 300_000
@ -901,8 +910,6 @@
process_chunk_size: 100 process_chunk_size: 100
config :pleroma, ConcurrentLimiter, [ config :pleroma, ConcurrentLimiter, [
{Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
{Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]},
{Pleroma.Search, [max_running: 30, max_waiting: 50]} {Pleroma.Search, [max_running: 30, max_waiting: 50]}
] ]

View File

@ -11,11 +11,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
require Logger require Logger
@adapter_options [
pool: :media,
recv_timeout: 10_000
]
@impl true @impl true
def history_awareness, do: :auto def history_awareness, do: :auto
@ -27,17 +22,14 @@ defp prefetch(url) do
Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}") Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
if Pleroma.Config.get(:env) == :test do fetch(prefetch_url)
fetch(prefetch_url)
else
ConcurrentLimiter.limit(__MODULE__, fn ->
Task.start(fn -> fetch(prefetch_url) end)
end)
end
end end
end end
defp fetch(url), do: HTTP.get(url, [], @adapter_options) defp fetch(url) do
http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
HTTP.get(url, [], http_client_opts)
end
defp preload(%{"object" => %{"attachment" => attachments}} = _message) do defp preload(%{"object" => %{"attachment" => attachments}} = _message) do
Enum.each(attachments, fn Enum.each(attachments, fn

View File

@ -54,9 +54,10 @@ def preview(%Conn{} = conn, %{"sig" => sig64, "url" => url64}) do
defp handle_preview(conn, url) do defp handle_preview(conn, url) do
media_proxy_url = MediaProxy.url(url) media_proxy_url = MediaProxy.url(url)
http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media)
with {:ok, %{status: status} = head_response} when status in 200..299 <- with {:ok, %{status: status} = head_response} when status in 200..299 <-
Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do
content_type = Tesla.get_header(head_response, "content-type") content_type = Tesla.get_header(head_response, "content-type")
content_length = Tesla.get_header(head_response, "content-length") content_length = Tesla.get_header(head_response, "content-length")
content_length = content_length && String.to_integer(content_length) content_length = content_length && String.to_integer(content_length)

View File

@ -58,7 +58,7 @@ defp check_content_length(headers) do
defp http_options do defp http_options do
[ [
pool: :media, pool: :rich_media,
max_body: Config.get([:rich_media, :max_body], 5_000_000) max_body: Config.get([:rich_media, :max_body], 5_000_000)
] ]
end end