Rich media: Add failure tracking
This commit is contained in:
parent
46236d1d87
commit
19691389b9
|
@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
|
||||||
|
|
||||||
## unreleased-patch - ???
|
## unreleased-patch - ???
|
||||||
|
|
||||||
|
### Added
|
||||||
|
- Rich media failure tracking (along with `:failure_backoff` option)
|
||||||
|
|
||||||
### Fixed
|
### Fixed
|
||||||
- Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers
|
- Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers
|
||||||
|
|
||||||
|
|
|
@ -412,6 +412,7 @@
|
||||||
Pleroma.Web.RichMedia.Parsers.TwitterCard,
|
Pleroma.Web.RichMedia.Parsers.TwitterCard,
|
||||||
Pleroma.Web.RichMedia.Parsers.OEmbed
|
Pleroma.Web.RichMedia.Parsers.OEmbed
|
||||||
],
|
],
|
||||||
|
failure_backoff: 60_000,
|
||||||
ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl]
|
ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl]
|
||||||
|
|
||||||
config :pleroma, :media_proxy,
|
config :pleroma, :media_proxy,
|
||||||
|
|
|
@ -2385,6 +2385,13 @@
|
||||||
suggestions: [
|
suggestions: [
|
||||||
Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl
|
Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
%{
|
||||||
|
key: :failure_backoff,
|
||||||
|
type: :integer,
|
||||||
|
description:
|
||||||
|
"Amount of milliseconds after request failure, during which the request will not be retried.",
|
||||||
|
suggestions: [60_000]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
|
|
@ -361,6 +361,7 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
|
||||||
* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`.
|
* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`.
|
||||||
* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"].
|
* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"].
|
||||||
* `parsers`: list of Rich Media parsers.
|
* `parsers`: list of Rich Media parsers.
|
||||||
|
* `failure_backoff`: Amount of milliseconds after request failure, during which the request will not be retried.
|
||||||
|
|
||||||
## HTTP server
|
## HTTP server
|
||||||
|
|
||||||
|
|
|
@ -17,14 +17,25 @@ def parse(url), do: parse_url(url)
|
||||||
else
|
else
|
||||||
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
|
@spec parse(String.t()) :: {:ok, map()} | {:error, any()}
|
||||||
def parse(url) do
|
def parse(url) do
|
||||||
Cachex.fetch!(:rich_media_cache, url, fn _ ->
|
with {:ok, data} <- get_cached_or_parse(url),
|
||||||
with {:ok, data} <- parse_url(url) do
|
{:ok, _} <- set_ttl_based_on_image(data, url) do
|
||||||
{:commit, {:ok, data}}
|
{:ok, data}
|
||||||
else
|
else
|
||||||
error -> {:ignore, error}
|
error ->
|
||||||
end
|
Logger.error(fn -> "Rich media error: #{inspect(error)}" end)
|
||||||
end)
|
end
|
||||||
|> set_ttl_based_on_image(url)
|
end
|
||||||
|
|
||||||
|
defp get_cached_or_parse(url) do
|
||||||
|
case Cachex.fetch!(:rich_media_cache, url, fn _ -> {:commit, parse_url(url)} end) do
|
||||||
|
{:ok, _data} = res ->
|
||||||
|
res
|
||||||
|
|
||||||
|
{:error, _} = e ->
|
||||||
|
ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
|
||||||
|
Cachex.expire(:rich_media_cache, url, ttl)
|
||||||
|
e
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -50,22 +61,21 @@ def ttl(data, url) do
|
||||||
config :pleroma, :rich_media,
|
config :pleroma, :rich_media,
|
||||||
ttl_setters: [MyModule]
|
ttl_setters: [MyModule]
|
||||||
"""
|
"""
|
||||||
@spec set_ttl_based_on_image({:ok, map()} | {:error, any()}, String.t()) ::
|
@spec set_ttl_based_on_image(map(), String.t()) ::
|
||||||
{:ok, map()} | {:error, any()}
|
{:ok, Integer.t() | :noop} | {:error, :no_key}
|
||||||
def set_ttl_based_on_image({:ok, data}, url) do
|
def set_ttl_based_on_image(data, url) do
|
||||||
with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url),
|
case get_ttl_from_image(data, url) do
|
||||||
{:ok, ttl} when is_number(ttl) <- get_ttl_from_image(data, url) do
|
{:ok, ttl} when is_number(ttl) ->
|
||||||
Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
|
ttl = ttl * 1000
|
||||||
{:ok, data}
|
|
||||||
else
|
|
||||||
_ ->
|
|
||||||
{:ok, data}
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def set_ttl_based_on_image({:error, _} = error, _) do
|
case Cachex.expire_at(:rich_media_cache, url, ttl) do
|
||||||
Logger.error("parsing error: #{inspect(error)}")
|
{:ok, true} -> {:ok, ttl}
|
||||||
error
|
{:ok, false} -> {:error, :no_key}
|
||||||
|
end
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
{:ok, :noop}
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
defp get_ttl_from_image(data, url) do
|
defp get_ttl_from_image(data, url) do
|
||||||
|
|
|
@ -55,7 +55,7 @@ test "s3 signed url is parsed and correct ttl is set for rich media" do
|
||||||
|
|
||||||
Cachex.put(:rich_media_cache, url, metadata)
|
Cachex.put(:rich_media_cache, url, metadata)
|
||||||
|
|
||||||
Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image({:ok, metadata}, url)
|
Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image(metadata, url)
|
||||||
|
|
||||||
{:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url)
|
{:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue