Compare commits

..

No commits in common. "80e0e0c466a2e1c9ec2176bd94ecdc4a9502ba0b" and "5d4913bb93866488740636c118a74c18c1c2510f" have entirely different histories.

27 changed files with 202 additions and 130 deletions

View File

@ -1 +0,0 @@
Mastodon API: Remove deprecated GET /api/v1/statuses/:id/card endpoint https://github.com/mastodon/mastodon/pull/11213

View File

@ -1 +0,0 @@
Include image description in status media cards

View File

@ -1 +0,0 @@
Implement FEP-2c59, add "webfinger" to user actor

View File

@ -1 +0,0 @@
Framegrabs with ffmpeg will execute with a 5 second timeout and cache the URLs of failures with a TTL of 15 minutes to prevent excessive retries.

View File

@ -1 +0,0 @@
Add new parameters to /api/v2/instance: configuration[accounts][max_pinned_statuses] and configuration[statuses][characters_reserved_per_url]

View File

@ -1 +0,0 @@
ReceiverWorker: Make sure non-{:ok, _} is returned as {:error, …}

View File

@ -3522,7 +3522,7 @@
},
%{
key: :initial_indexing_chunk_size,
type: :integer,
type: :int,
description:
"Amount of posts in a batch when running the initial indexing operation. Should probably not be more than 100000" <>
" since there's a limit on maximum insert size",

View File

@ -156,7 +156,6 @@ defp cachex_children do
build_cachex("web_resp", limit: 2500),
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
build_cachex("failed_proxy_url", limit: 2500),
build_cachex("failed_media_helper_url", default_ttl: :timer.minutes(15), limit: 2_500),
build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
build_cachex("chat_message_id_idempotency_key",
expiration: chat_message_id_idempotency_key_expiration(),

View File

@ -12,8 +12,6 @@ defmodule Pleroma.Helpers.MediaHelper do
require Logger
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def missing_dependencies do
Enum.reduce([ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc ->
if Pleroma.Utils.command_available?(executable) do
@ -45,40 +43,29 @@ def image_resize(url, options) do
@spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()}
def video_framegrab(url) do
with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
false <- @cachex.exists?(:failed_media_helper_cache, url),
{:ok, env} <- HTTP.get(url, [], pool: :media),
{:ok, pid} <- StringIO.open(env.body) do
body_stream = IO.binstream(pid, 1)
task =
Task.async(fn ->
Exile.stream!(
[
executable,
"-i",
"pipe:0",
"-vframes",
"1",
"-f",
"mjpeg",
"pipe:1"
],
input: body_stream,
ignore_epipe: true,
stderr: :disable
)
|> Enum.into(<<>>)
end)
result =
Exile.stream!(
[
executable,
"-i",
"pipe:0",
"-vframes",
"1",
"-f",
"mjpeg",
"pipe:1"
],
input: body_stream,
ignore_epipe: true,
stderr: :disable
)
|> Enum.into(<<>>)
case Task.yield(task, 5_000) do
nil ->
Task.shutdown(task)
@cachex.put(:failed_media_helper_cache, url, nil)
{:error, {:ffmpeg, :timeout}}
result ->
{:ok, result}
end
{:ok, result}
else
nil -> {:error, {:ffmpeg, :command_not_found}}
{:error, _} = error -> error

View File

@ -67,13 +67,8 @@ def render("service.json", %{user: user}) do
def render("user.json", %{user: %User{nickname: nil} = user}),
do: render("service.json", %{user: user})
def render("user.json", %{user: %User{nickname: "internal." <> _} = user}) do
render("service.json", %{user: user})
|> Map.merge(%{
"preferredUsername" => user.nickname,
"webfinger" => "acct:#{User.full_nickname(user)}"
})
end
def render("user.json", %{user: %User{nickname: "internal." <> _} = user}),
do: render("service.json", %{user: user}) |> Map.put("preferredUsername", user.nickname)
def render("user.json", %{user: user}) do
{:ok, _, public_key} = Keys.keys_from_pem(user.keys)
@ -126,8 +121,7 @@ def render("user.json", %{user: user}) do
"discoverable" => user.is_discoverable,
"capabilities" => capabilities,
"alsoKnownAs" => user.also_known_as,
"vcard:bday" => birthday,
"webfinger" => "acct:#{User.full_nickname(user)}"
"vcard:bday" => birthday
}
|> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
|> Map.merge(maybe_make_image(&User.banner_url/2, "image", user))

View File

@ -50,15 +50,6 @@ defp instance do
%Schema{
type: :object,
properties: %{
accounts: %Schema{
type: :object,
properties: %{
max_featured_tags: %Schema{
type: :integer,
description: "The maximum number of featured tags allowed for each account."
}
}
},
uri: %Schema{type: :string, description: "The domain name of the instance"},
title: %Schema{type: :string, description: "The title of the website"},
description: %Schema{
@ -281,19 +272,6 @@ defp instance2 do
type: :object,
description: "Instance configuration",
properties: %{
accounts: %Schema{
type: :object,
properties: %{
max_featured_tags: %Schema{
type: :integer,
description: "The maximum number of featured tags allowed for each account."
},
max_pinned_statuses: %Schema{
type: :integer,
description: "The maximum number of pinned statuses for each account."
}
}
},
urls: %Schema{
type: :object,
properties: %{
@ -307,11 +285,6 @@ defp instance2 do
type: :object,
description: "A map with poll limits for local statuses",
properties: %{
characters_reserved_per_url: %Schema{
type: :integer,
description:
"Each URL in a status will be assumed to be exactly this many characters."
},
max_characters: %Schema{
type: :integer,
description: "Posts character limit (CW/Subject included in the counter)"

View File

@ -58,10 +58,6 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
format: :uri,
description: "Preview thumbnail"
},
image_description: %Schema{
type: :string,
description: "Alternate text that describes what is in the thumbnail"
},
title: %Schema{type: :string, description: "Title of linked resource"},
description: %Schema{type: :string, description: "Description of preview"}
}

View File

@ -25,6 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
alias Pleroma.Web.OAuth.Token
alias Pleroma.Web.Plugs.OAuthScopesPlug
alias Pleroma.Web.Plugs.RateLimiter
alias Pleroma.Web.RichMedia.Card
plug(Pleroma.Web.ApiSpec.CastAndValidate, replace_params: false)
@ -38,6 +39,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
when action in [
:index,
:show,
:card,
:context,
:show_history,
:show_source
@ -472,6 +474,21 @@ def unmute_conversation(
end
end
@doc "GET /api/v1/statuses/:id/card"
@deprecated "https://github.com/tootsuite/mastodon/pull/11213"
def card(
%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: status_id}}}} = conn,
_
) do
with %Activity{} = activity <- Activity.get_by_id(status_id),
true <- Visibility.visible_for_user?(activity, user),
%Card{} = card_data <- Card.get_by_activity(activity) do
render(conn, "card.json", card_data)
else
_ -> render_error(conn, :not_found, "Record not found")
end
end
@doc "GET /api/v1/statuses/:id/favourited_by"
def favourited_by(
%{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn,

View File

@ -213,8 +213,6 @@ defp configuration do
defp configuration2 do
configuration()
|> put_in([:accounts, :max_pinned_statuses], Config.get([:instance, :max_pinned_statuses], 0))
|> put_in([:statuses, :characters_reserved_per_url], 0)
|> Map.merge(%{
urls: %{
streaming: Pleroma.Web.Endpoint.websocket_url(),

View File

@ -583,7 +583,6 @@ def render("card.json", %Card{fields: rich_media}) do
provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
url: page_url,
image: image_url,
image_description: rich_media["image:alt"] || "",
title: rich_media["title"] || "",
description: rich_media["description"] || "",
pleroma: %{

View File

@ -2,6 +2,16 @@
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.RichMedia.Backfill.Task do
alias Pleroma.Web.RichMedia.Backfill
def run(args) do
Task.Supervisor.start_child(Pleroma.TaskSupervisor, Backfill, :run, [args],
name: {:global, {:rich_media, args.url_hash}}
)
end
end
defmodule Pleroma.Web.RichMedia.Backfill do
alias Pleroma.Web.RichMedia.Card
alias Pleroma.Web.RichMedia.Parser
@ -89,13 +99,3 @@ defp stream_update(%{activity_id: activity_id}) do
defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val)
defp negative_cache(key, ttl \\ nil), do: @cachex.put(:rich_media_cache, key, nil, ttl: ttl)
end
defmodule Pleroma.Web.RichMedia.Backfill.Task do
alias Pleroma.Web.RichMedia.Backfill
def run(args) do
Task.Supervisor.start_child(Pleroma.TaskSupervisor, Backfill, :run, [args],
name: {:global, {:rich_media, args.url_hash}}
)
end
end

View File

@ -47,7 +47,7 @@ def delete(url) do
@cachex.del(:rich_media_cache, url_hash)
case get_by_url(url) do
%__MODULE__{} = card -> Repo.delete(card)
%__MODULE{} = card -> Repo.delete(card)
nil -> :ok
end
end

View File

@ -56,7 +56,7 @@ defp check_content_length(headers) do
end
end
defp http_options do
defp http_options() do
[
pool: :media,
max_body: Config.get([:rich_media, :max_body], 5_000_000)

View File

@ -6,12 +6,11 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.Opengraph do
@behaviour Pleroma.Web.RichMedia.Parser.TTL
@impl true
def ttl(%{"ttl" => ttl_string}, _url) when is_binary(ttl_string) do
try do
ttl = String.to_integer(ttl_string)
def ttl(%{"ttl" => ttl_string}, _url) do
with ttl <- String.to_integer(ttl_string) do
now = DateTime.utc_now() |> DateTime.to_unix()
now + ttl
rescue
else
_ -> nil
end
end

View File

@ -776,6 +776,7 @@ defmodule Pleroma.Web.Router do
get("/statuses", StatusController, :index)
get("/statuses/:id", StatusController, :show)
get("/statuses/:id/context", StatusController, :context)
get("/statuses/:id/card", StatusController, :card)
get("/statuses/:id/favourited_by", StatusController, :favourited_by)
get("/statuses/:id/reblogged_by", StatusController, :reblogged_by)
get("/statuses/:id/history", StatusController, :show_history)

View File

@ -52,8 +52,7 @@ defp process_errors(errors) do
{:error, {:reject, reason}} -> {:cancel, reason}
{:signature, false} -> {:cancel, :invalid_signature}
{:error, {:error, reason = "Object has been deleted"}} -> {:cancel, reason}
{:error, _} = e -> e
e -> {:error, e}
e -> e
end
end
end

View File

@ -2,7 +2,6 @@
"@context": [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
"https://purl.archive.org/socialweb/webfinger",
{
"Emoji": "toot:Emoji",
"Hashtag": "as:Hashtag",

View File

@ -91,13 +91,6 @@ test "renders AKAs" do
assert %{"alsoKnownAs" => ^akas} = UserView.render("user.json", %{user: user})
end
test "renders full nickname" do
clear_config([Pleroma.Web.WebFinger, :domain], "plemora.dev")
user = insert(:user, nickname: "user")
assert %{"webfinger" => "acct:user@plemora.dev"} = UserView.render("user.json", %{user: user})
end
describe "endpoints" do
test "local users have a usable endpoints structure" do
user = insert(:user)

View File

@ -329,6 +329,62 @@ test "posting a fake status", %{conn: conn} do
assert real_status == fake_status
end
test "fake statuses' preview card is not cached", %{conn: conn} do
Pleroma.StaticStubbedConfigMock
|> stub(:get, fn
[:rich_media, :enabled] -> true
path -> Pleroma.Test.StaticConfig.get(path)
end)
Tesla.Mock.mock_global(fn
env ->
apply(HttpRequestMock, :request, [env])
end)
conn1 =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/v1/statuses", %{
"status" => "https://example.com/ogp",
"preview" => true
})
conn2 =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/v1/statuses", %{
"status" => "https://example.com/twitter-card",
"preview" => true
})
assert %{"card" => %{"title" => "The Rock"}} = json_response_and_validate_schema(conn1, 200)
assert %{"card" => %{"title" => "Small Island Developing States Photo Submission"}} =
json_response_and_validate_schema(conn2, 200)
end
test "posting a status with OGP link preview", %{conn: conn} do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
Pleroma.StaticStubbedConfigMock
|> stub(:get, fn
[:rich_media, :enabled] -> true
path -> Pleroma.Test.StaticConfig.get(path)
end)
conn =
conn
|> put_req_header("content-type", "application/json")
|> post("/api/v1/statuses", %{
"status" => "https://example.com/ogp"
})
assert %{"id" => id, "card" => %{"title" => "The Rock"}} =
json_response_and_validate_schema(conn, 200)
assert Activity.get_by_id(id)
end
test "posting a direct status", %{conn: conn} do
user2 = insert(:user)
content = "direct cofe @#{user2.nickname}"
@ -1643,6 +1699,91 @@ test "on pin removes deletion job, on unpin reschedule deletion" do
end
end
describe "cards" do
setup do
Pleroma.StaticStubbedConfigMock
|> stub(:get, fn
[:rich_media, :enabled] -> true
path -> Pleroma.Test.StaticConfig.get(path)
end)
oauth_access(["read:statuses"])
end
test "returns rich-media card", %{conn: conn, user: user} do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
{:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp"})
card_data = %{
"image" => "http://ia.media-imdb.com/images/rock.jpg",
"provider_name" => "example.com",
"provider_url" => "https://example.com",
"title" => "The Rock",
"type" => "link",
"url" => "https://example.com/ogp",
"description" =>
"Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
"pleroma" => %{
"opengraph" => %{
"image" => "http://ia.media-imdb.com/images/rock.jpg",
"title" => "The Rock",
"type" => "video.movie",
"url" => "https://example.com/ogp",
"description" =>
"Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer."
}
}
}
response =
conn
|> get("/api/v1/statuses/#{activity.id}/card")
|> json_response_and_validate_schema(200)
assert response == card_data
# works with private posts
{:ok, activity} =
CommonAPI.post(user, %{status: "https://example.com/ogp", visibility: "direct"})
response_two =
conn
|> get("/api/v1/statuses/#{activity.id}/card")
|> json_response_and_validate_schema(200)
assert response_two == card_data
end
test "replaces missing description with an empty string", %{conn: conn, user: user} do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
{:ok, activity} = CommonAPI.post(user, %{status: "https://example.com/ogp-missing-data"})
response =
conn
|> get("/api/v1/statuses/#{activity.id}/card")
|> json_response_and_validate_schema(:ok)
assert response == %{
"type" => "link",
"title" => "Pleroma",
"description" => "",
"image" => nil,
"provider_name" => "example.com",
"provider_url" => "https://example.com",
"url" => "https://example.com/ogp-missing-data",
"pleroma" => %{
"opengraph" => %{
"title" => "Pleroma",
"type" => "website",
"url" => "https://example.com/ogp-missing-data"
}
}
}
end
end
test "bookmarks" do
bookmarks_uri = "/api/v1/bookmarks"

View File

@ -738,7 +738,7 @@ test "a rich media card without a site name renders correctly" do
{:ok, card} =
Card.create(page_url, %{image: page_url <> "/example.jpg", title: "Example website"})
assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card))
%{provider_name: "example.com"} = StatusView.render("card.json", card)
end
test "a rich media card without a site name or image renders correctly" do
@ -751,7 +751,7 @@ test "a rich media card without a site name or image renders correctly" do
{:ok, card} = Card.create(page_url, fields)
assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card))
%{provider_name: "example.com"} = StatusView.render("card.json", card)
end
test "a rich media card without an image renders correctly" do
@ -765,24 +765,7 @@ test "a rich media card without an image renders correctly" do
{:ok, card} = Card.create(page_url, fields)
assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card))
end
test "a rich media card without descriptions returns the fields with empty strings" do
page_url = "https://example.com"
fields = %{
"url" => page_url,
"site_name" => "Example site name",
"title" => "Example website"
}
{:ok, card} = Card.create(page_url, fields)
assert match?(
%{description: "", image_description: ""},
StatusView.render("card.json", card)
)
%{provider_name: "example.com"} = StatusView.render("card.json", card)
end
test "a rich media card with all relevant data renders correctly" do
@ -798,7 +781,7 @@ test "a rich media card with all relevant data renders correctly" do
{:ok, card} = Card.create(page_url, fields)
assert match?(%{provider_name: "example.com"}, StatusView.render("card.json", card))
%{provider_name: "example.com"} = StatusView.render("card.json", card)
end
test "a rich media card has all media proxied" do

View File

@ -39,7 +39,7 @@ test "crawls URL in activity" do
assert %Card{url_hash: ^url_hash, fields: _} = Card.get_by_activity(activity)
end
test "recrawls URLs on status edits/updates" do
test "recrawls URLs on updates" do
original_url = "https://google.com/"
original_url_hash = Card.url_to_hash(original_url)
updated_url = "https://yahoo.com/"