Merge remote-tracking branch 'upstream/develop' into spc2

This commit is contained in:
Moon Man 2024-05-14 13:50:29 +00:00
commit 80e0e0c466
27 changed files with 130 additions and 202 deletions

View File

@ -0,0 +1 @@
Mastodon API: Remove deprecated GET /api/v1/statuses/:id/card endpoint https://github.com/mastodon/mastodon/pull/11213

View File

@ -0,0 +1 @@
Include image description in status media cards

1
changelog.d/fep-2c59.add Normal file
View File

@ -0,0 +1 @@
Implement FEP-2c59, add "webfinger" to user actor

View File

@ -0,0 +1 @@
Framegrabs with ffmpeg will execute with a 5 second timeout and cache the URLs of failures with a TTL of 15 minutes to prevent excessive retries.

View File

@ -0,0 +1 @@
Add new parameters to /api/v2/instance: configuration[accounts][max_pinned_statuses] and configuration[statuses][characters_reserved_per_url]

View File

@ -0,0 +1 @@
ReceiverWorker: Make sure non-{:ok, _} is returned as {:error, …}

View File

@ -3522,7 +3522,7 @@
}, },
%{ %{
key: :initial_indexing_chunk_size, key: :initial_indexing_chunk_size,
type: :int, type: :integer,
description: description:
"Amount of posts in a batch when running the initial indexing operation. Should probably not be more than 100000" <> "Amount of posts in a batch when running the initial indexing operation. Should probably not be more than 100000" <>
" since there's a limit on maximum insert size", " since there's a limit on maximum insert size",

View File

@ -156,6 +156,7 @@ defp cachex_children do
build_cachex("web_resp", limit: 2500), build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
build_cachex("failed_proxy_url", limit: 2500), build_cachex("failed_proxy_url", limit: 2500),
build_cachex("failed_media_helper_url", default_ttl: :timer.minutes(15), limit: 2_500),
build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000), build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
build_cachex("chat_message_id_idempotency_key", build_cachex("chat_message_id_idempotency_key",
expiration: chat_message_id_idempotency_key_expiration(), expiration: chat_message_id_idempotency_key_expiration(),

View File

@ -12,6 +12,8 @@ defmodule Pleroma.Helpers.MediaHelper do
require Logger require Logger
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def missing_dependencies do def missing_dependencies do
Enum.reduce([ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc -> Enum.reduce([ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc ->
if Pleroma.Utils.command_available?(executable) do if Pleroma.Utils.command_available?(executable) do
@ -43,11 +45,13 @@ def image_resize(url, options) do
@spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()} @spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()}
def video_framegrab(url) do def video_framegrab(url) do
with executable when is_binary(executable) <- System.find_executable("ffmpeg"), with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
false <- @cachex.exists?(:failed_media_helper_cache, url),
{:ok, env} <- HTTP.get(url, [], pool: :media), {:ok, env} <- HTTP.get(url, [], pool: :media),
{:ok, pid} <- StringIO.open(env.body) do {:ok, pid} <- StringIO.open(env.body) do
body_stream = IO.binstream(pid, 1) body_stream = IO.binstream(pid, 1)
result = task =
Task.async(fn ->
Exile.stream!( Exile.stream!(
[ [
executable, executable,
@ -64,8 +68,17 @@ def video_framegrab(url) do
stderr: :disable stderr: :disable
) )
|> Enum.into(<<>>) |> Enum.into(<<>>)
end)
case Task.yield(task, 5_000) do
nil ->
Task.shutdown(task)
@cachex.put(:failed_media_helper_cache, url, nil)
{:error, {:ffmpeg, :timeout}}
result ->
{:ok, result} {:ok, result}
end
else else
nil -> {:error, {:ffmpeg, :command_not_found}} nil -> {:error, {:ffmpeg, :command_not_found}}
{:error, _} = error -> error {:error, _} = error -> error

View File

@ -67,8 +67,13 @@ def render("service.json", %{user: user}) do
def render("user.json", %{user: %User{nickname: nil} = user}), def render("user.json", %{user: %User{nickname: nil} = user}),
do: render("service.json", %{user: user}) do: render("service.json", %{user: user})
def render("user.json", %{user: %User{nickname: "internal." <> _} = user}), def render("user.json", %{user: %User{nickname: "internal." <> _} = user}) do
do: render("service.json", %{user: user}) |> Map.put("preferredUsername", user.nickname) render("service.json", %{user: user})
|> Map.merge(%{
"preferredUsername" => user.nickname,
"webfinger" => "acct:#{User.full_nickname(user)}"
})
end
def render("user.json", %{user: user}) do def render("user.json", %{user: user}) do
{:ok, _, public_key} = Keys.keys_from_pem(user.keys) {:ok, _, public_key} = Keys.keys_from_pem(user.keys)
@ -121,7 +126,8 @@ def render("user.json", %{user: user}) do
"discoverable" => user.is_discoverable, "discoverable" => user.is_discoverable,
"capabilities" => capabilities, "capabilities" => capabilities,
"alsoKnownAs" => user.also_known_as, "alsoKnownAs" => user.also_known_as,
"vcard:bday" => birthday "vcard:bday" => birthday,
"webfinger" => "acct:#{User.full_nickname(user)}"
} }
|> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
|> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user))

View File

@ -50,6 +50,15 @@ defp instance do
%Schema{ %Schema{
type: :object, type: :object,
properties: %{ properties: %{
accounts: %Schema{
type: :object,
properties: %{
max_featured_tags: %Schema{
type: :integer,
description: "The maximum number of featured tags allowed for each account."
}
}
},
uri: %Schema{type: :string, description: "The domain name of the instance"}, uri: %Schema{type: :string, description: "The domain name of the instance"},
title: %Schema{type: :string, description: "The title of the website"}, title: %Schema{type: :string, description: "The title of the website"},
description: %Schema{ description: %Schema{
@ -272,6 +281,19 @@ defp instance2 do
type: :object, type: :object,
description: "Instance configuration", description: "Instance configuration",
properties: %{ properties: %{
accounts: %Schema{
type: :object,
properties: %{
max_featured_tags: %Schema{
type: :integer,
description: "The maximum number of featured tags allowed for each account."
},
max_pinned_statuses: %Schema{
type: :integer,
description: "The maximum number of pinned statuses for each account."
}
}
},
urls: %Schema{ urls: %Schema{
type: :object, type: :object,
properties: %{ properties: %{
@ -285,6 +307,11 @@ defp instance2 do
type: :object, type: :object,
description: "A map with poll limits for local statuses", description: "A map with poll limits for local statuses",
properties: %{ properties: %{
characters_reserved_per_url: %Schema{
type: :integer,
description:
"Each URL in a status will be assumed to be exactly this many characters."
},
max_characters: %Schema{ max_characters: %Schema{
type: :integer, type: :integer,
description: "Posts character limit (CW/Subject included in the counter)" description: "Posts character limit (CW/Subject included in the counter)"

View File

@ -58,6 +58,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
format: :uri, format: :uri,
description: "Preview thumbnail" description: "Preview thumbnail"
}, },
image_description: %Schema{
type: :string,
description: "Alternate text that describes what is in the thumbnail"
},
title: %Schema{type: :string, description: "Title of linked resource"}, title: %Schema{type: :string, description: "Title of linked resource"},
description: %Schema{type: :string, description: "Description of preview"} description: %Schema{type: :string, description: "Description of preview"}
} }

View File

@ -25,7 +25,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
alias Pleroma.Web.OAuth.Token alias Pleroma.Web.OAuth.Token
alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.Web.Plugs.OAuthScopesPlug
alias Pleroma.Web.Plugs.RateLimiter alias Pleroma.Web.Plugs.RateLimiter
alias Pleroma.Web.RichMedia.Card
plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false) plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false)
@ -39,7 +38,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
when action in [ when action in [
:index, :index,
:show, :show,
:card,
:context, :context,
:show_history, :show_history,
:show_source :show_source
@ -474,21 +472,6 @@ def unmute_conversation(
end end
end end
@doc "GET /api/v1/statuses/:id/card"
@deprecated "https://github.com/tootsuite/mastodon/pull/11213"
def card(
%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: status_id}}}} = conn,
_
) do
with %Activity{} = activity <- Activity.get_by_id(status_id),
true <- Visibility.visible_for_user?(activity, user),
%Card{} = card_data <- Card.get_by_activity(activity) do
render(conn, "card.json", card_data)
else
_ -> render_error(conn, :not_found, "Record not found")
end
end
@doc "GET /api/v1/statuses/:id/favourited_by" @doc "GET /api/v1/statuses/:id/favourited_by"
def favourited_by( def favourited_by(
%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn,

View File

@ -213,6 +213,8 @@ defp configuration do
defp configuration2 do defp configuration2 do
configuration() configuration()
|> put_in([:accounts, :max_pinned_statuses], Config.get([:instance, :max_pinned_statuses], 0))
|> put_in([:statuses, :characters_reserved_per_url], 0)
|> Map.merge(%{ |> Map.merge(%{
urls: %{ urls: %{
streaming: Pleroma.Web.Endpoint.websocket_url(), streaming: Pleroma.Web.Endpoint.websocket_url(),

View File

@ -583,6 +583,7 @@ def render("card.json", %Card{fields: rich_media}) do
provider_url: page_url_data.scheme <> "://" <> page_url_data.host, provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
url: page_url, url: page_url,
image: image_url, image: image_url,
image_description: rich_media["image:alt"] || "",
title: rich_media["title"] || "", title: rich_media["title"] || "",
description: rich_media["description"] || "", description: rich_media["description"] || "",
pleroma: %{ pleroma: %{

View File

@ -2,16 +2,6 @@
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> # Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Backfill.Task do
alias Pleroma.Web.RichMedia.Backfill
def run(args) do
Task.Supervisor.start_child(Pleroma.TaskSupervisor, Backfill, :run, [args],
name: {:global, {:rich_media, args.url_hash}}
)
end
end
defmodule Pleroma.Web.RichMedia.Backfill do defmodule Pleroma.Web.RichMedia.Backfill do
alias Pleroma.Web.RichMedia.Card alias Pleroma.Web.RichMedia.Card
alias Pleroma.Web.RichMedia.Parser alias Pleroma.Web.RichMedia.Parser
@ -99,3 +89,13 @@ defp stream_update(%{activity_id: activity_id}) do
defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val) defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val)
defp negative_cache(key, ttl \\ nil), do: @cachex.put(:rich_media_cache, key, nil, ttl: ttl) defp negative_cache(key, ttl \\ nil), do: @cachex.put(:rich_media_cache, key, nil, ttl: ttl)
end end
defmodule Pleroma.Web.RichMedia.Backfill.Task do
alias Pleroma.Web.RichMedia.Backfill
def run(args) do
Task.Supervisor.start_child(Pleroma.TaskSupervisor, Backfill, :run, [args],
name: {:global, {:rich_media, args.url_hash}}
)
end
end

View File

@ -47,7 +47,7 @@ def delete(url) do
@cachex.del(:rich_media_cache, url_hash) @cachex.del(:rich_media_cache, url_hash)
case get_by_url(url) do case get_by_url(url) do
%__MODULE{} = card -> Repo.delete(card) %__MODULE__{} = card -> Repo.delete(card)
nil -> :ok nil -> :ok
end end
end end

View File

@ -56,7 +56,7 @@ defp check_content_length(headers) do
end end
end end
defp http_options() do defp http_options do
[ [
pool: :media, pool: :media,
max_body: Config.get([:rich_media, :max_body], 5_000_000) max_body: Config.get([:rich_media, :max_body], 5_000_000)

View File

@ -6,11 +6,12 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.Opengraph do
@behaviour Pleroma.Web.RichMedia.Parser.TTL @behaviour Pleroma.Web.RichMedia.Parser.TTL
@impl true @impl true
def ttl(%{"ttl" => ttl_string}, _url) do def ttl(%{"ttl" => ttl_string}, _url) when is_binary(ttl_string) do
with ttl <- String.to_integer(ttl_string) do try do
ttl = String.to_integer(ttl_string)
now = DateTime.utc_now() |> DateTime.to_unix() now = DateTime.utc_now() |> DateTime.to_unix()
now + ttl now + ttl
else rescue
_ -> nil _ -> nil
end end
end end

View File

@ -776,7 +776,6 @@ defmodule Pleroma.Web.Router do
get("/statuses", StatusController, :index) get("/statuses", StatusController, :index)
get("/statuses/:id", StatusController, :show) get("/statuses/:id", StatusController, :show)
get("/statuses/:id/context", StatusController, :context) get("/statuses/:id/context", StatusController, :context)
get("/statuses/:id/card", StatusController, :card)
get("/statuses/:id/favourited_by", StatusController, :favourited_by) get("/statuses/:id/favourited_by", StatusController, :favourited_by)
get("/statuses/:id/reblogged_by", StatusController, :reblogged_by) get("/statuses/:id/reblogged_by", StatusController, :reblogged_by)
get("/statuses/:id/history", StatusController, :show_history) get("/statuses/:id/history", StatusController, :show_history)

View File

@ -52,7 +52,8 @@ defp process_errors(errors) do
{:error, {:reject, reason}} -> {:cancel, reason} {:error, {:reject, reason}} -> {:cancel, reason}
{:signature, false} -> {:cancel, :invalid_signature} {:signature, false} -> {:cancel, :invalid_signature}
{:error, {:error, reason = "Object has been deleted"}} -> {:cancel, reason} {:error, {:error, reason = "Object has been deleted"}} -> {:cancel, reason}
e -> e {:error, _} = e -> e
e -> {:error, e}
end end
end end
end end

View File

@ -2,6 +2,7 @@
"@context": [ "@context": [
"https://www.w3.org/ns/activitystreams", "https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1", "https://w3id.org/security/v1",
"https://purl.archive.org/socialweb/webfinger",
{ {
"Emoji": "toot:Emoji", "Emoji": "toot:Emoji",
"Hashtag": "as:Hashtag", "Hashtag": "as:Hashtag",

View File

@ -91,6 +91,13 @@ test "renders AKAs" do
assert %{"alsoKnownAs" => ^akas} = UserView.render("user.json", %{user: user}) assert %{"alsoKnownAs" => ^akas} = UserView.render("user.json", %{user: user})
end end
test "renders full nickname" do
clear_config([Pleroma.Web.WebFinger, :domain], "plemora.dev")
user = insert(:user, nickname: "user")
assert %{"webfinger" => "acct:user@plemora.dev"} = UserView.render("user.json", %{user: user})
end
describe "endpoints" do describe "endpoints" do
test "local users have a usable endpoints structure" do test "local users have a usable endpoints structure" do
user = insert(:user) user = insert(:user)

View File

@ -329,62 +329,6 @@ test "posting a fake status", %{conn: conn} do
assert real_status == fake_status assert real_status == fake_status
end end
test "fake statuses' preview card is not cached", %{conn: conn} do
Pleroma.StaticStubbedConfigMock
|> stub(:get, fn
[:rich_media, :enabled] -> true
path -> Pleroma.Test.StaticConfig.get(path)
end)
Tesla.Mock.mock_global(fn
env ->
apply(HttpRequestMock, :request, [env])
end)
conn1 =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/v1/statuses", %{
"status" => "https://example.com/ogp",
"preview" => true
})
conn2 =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/v1/statuses", %{
"status" => "https://example.com/twitter-card",
"preview" => true
})
assert %{"card" => %{"title" => "The Rock"}} = json_response_and_validate_schema(conn1, 200)
assert %{"card" => %{"title" => "Small Island Developing States Photo Submission"}} =
json_response_and_validate_schema(conn2, 200)
end
test "posting a status with OGP link preview", %{conn: conn} do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
Pleroma.StaticStubbedConfigMock
|> stub(:get, fn
[:rich_media, :enabled] -> true
path -> Pleroma.Test.StaticConfig.get(path)
end)
conn =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/v1/statuses", %{
"status" => "https://example.com/ogp"
})
assert %{"id" => id, "card" => %{"title" => "The Rock"}} =
json_response_and_validate_schema(conn, 200)
assert Activity.get_by_id(id)
end
test "posting a direct status", %{conn: conn} do test "posting a direct status", %{conn: conn} do
user2 = insert(:user) user2 = insert(:user)
content = "direct cofe @#{user2.nickname}" content = "direct cofe @#{user2.nickname}"
@ -1699,91 +1643,6 @@ test "on pin removes deletion job, on unpin reschedule deletion" do
end end
end end
describe "cards" do
setup do
Pleroma.StaticStubbedConfigMock
|> stub(:get, fn
[:rich_media, :enabled] -> true
path -> Pleroma.Test.StaticConfig.get(path)
end)
oauth_access(["read:statuses"])
end
test "returns rich-media card", %{conn: conn, user: user} do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
{:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp"})
card_data = %{
"image" => "http://ia.media-imdb.com/images/rock.jpg",
"provider_name" => "example.com",
"provider_url" => "https://example.com",
"title" => "The Rock",
"type" => "link",
"url" => "https://example.com/ogp",
"description" =>
"Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
"pleroma" => %{
"opengraph" => %{
"image" => "http://ia.media-imdb.com/images/rock.jpg",
"title" => "The Rock",
"type" => "video.movie",
"url" => "https://example.com/ogp",
"description" =>
"Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer."
}
}
}
response =
conn
|> get("/api/v1/statuses/#{activity.id}/card")
|> json_response_and_validate_schema(200)
assert response == card_data
# works with private posts
{:ok, activity} =
CommonAPI.post(user, %{status: "https://example.com/ogp", visibility: "direct"})
response_two =
conn
|> get("/api/v1/statuses/#{activity.id}/card")
|> json_response_and_validate_schema(200)
assert response_two == card_data
end
test "replaces missing description with an empty string", %{conn: conn, user: user} do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
{:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp-missing-data"})
response =
conn
|> get("/api/v1/statuses/#{activity.id}/card")
|> json_response_and_validate_schema(:ok)
assert response == %{
"type" => "link",
"title" => "Pleroma",
"description" => "",
"image" => nil,
"provider_name" => "example.com",
"provider_url" => "https://example.com",
"url" => "https://example.com/ogp-missing-data",
"pleroma" => %{
"opengraph" => %{
"title" => "Pleroma",
"type" => "website",
"url" => "https://example.com/ogp-missing-data"
}
}
}
end
end
test "bookmarks" do test "bookmarks" do
bookmarks_uri = "/api/v1/bookmarks" bookmarks_uri = "/api/v1/bookmarks"

View File

@ -738,7 +738,7 @@ test "a rich media card without a site name renders correctly" do
{:ok, card} = {:ok, card} =
Card.create(page_url, %{image: page_url <> "/example.jpg", title: "Example website"}) Card.create(page_url, %{image: page_url <> "/example.jpg", title: "Example website"})
%{provider_name: "example.com"} = StatusView.render("card.json", card) assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card))
end end
test "a rich media card without a site name or image renders correctly" do test "a rich media card without a site name or image renders correctly" do
@ -751,7 +751,7 @@ test "a rich media card without a site name or image renders correctly" do
{:ok, card} = Card.create(page_url, fields) {:ok, card} = Card.create(page_url, fields)
%{provider_name: "example.com"} = StatusView.render("card.json", card) assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card))
end end
test "a rich media card without an image renders correctly" do test "a rich media card without an image renders correctly" do
@ -765,7 +765,24 @@ test "a rich media card without an image renders correctly" do
{:ok, card} = Card.create(page_url, fields) {:ok, card} = Card.create(page_url, fields)
%{provider_name: "example.com"} = StatusView.render("card.json", card) assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card))
end
test "a rich media card without descriptions returns the fields with empty strings" do
page_url = "https://example.com"
fields = %{
"url" => page_url,
"site_name" => "Example site name",
"title" => "Example website"
}
{:ok, card} = Card.create(page_url, fields)
assert match?(
%{description: "", image_description: ""},
StatusView.render("card.json", card)
)
end end
test "a rich media card with all relevant data renders correctly" do test "a rich media card with all relevant data renders correctly" do
@ -781,7 +798,7 @@ test "a rich media card with all relevant data renders correctly" do
{:ok, card} = Card.create(page_url, fields) {:ok, card} = Card.create(page_url, fields)
%{provider_name: "example.com"} = StatusView.render("card.json", card) assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card))
end end
test "a rich media card has all media proxied" do test "a rich media card has all media proxied" do

View File

@ -39,7 +39,7 @@ test "crawls URL in activity" do
assert %Card{url_hash: ^url_hash, fields: _} = Card.get_by_activity(activity) assert %Card{url_hash: ^url_hash, fields: _} = Card.get_by_activity(activity)
end end
test "recrawls URLs on updates" do test "recrawls URLs on status edits/updates" do
original_url = "https://google.com/" original_url = "https://google.com/"
original_url_hash = Card.url_to_hash(original_url) original_url_hash = Card.url_to_hash(original_url)
updated_url = "https://yahoo.com/" updated_url = "https://yahoo.com/"