From 6b8c5e12dfe759ac1286e81e72ad7f8727e01386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 26 Oct 2023 23:30:38 +0200 Subject: [PATCH 01/42] Add contact account to InstanceView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/instance-contact-account.add | 1 + config/description.exs | 6 ++++++ .../web/mastodon_api/views/instance_view.ex | 17 +++++++++++++++++ .../controllers/instance_controller_test.exs | 12 ++++++++++++ 4 files changed, 36 insertions(+) create mode 100644 changelog.d/instance-contact-account.add diff --git a/changelog.d/instance-contact-account.add b/changelog.d/instance-contact-account.add new file mode 100644 index 000000000..e119446d2 --- /dev/null +++ b/changelog.d/instance-contact-account.add @@ -0,0 +1 @@ +Add contact account to InstanceView \ No newline at end of file diff --git a/config/description.exs b/config/description.exs index d18649ae8..239ba7cb0 100644 --- a/config/description.exs +++ b/config/description.exs @@ -566,6 +566,12 @@ "Cool instance" ] }, + %{ + key: :contact_username, + type: :string, + description: "Instance owner username", + suggestions: ["admin"] + }, %{ key: :limit, type: :integer, diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 1b01d7371..06df63b0f 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -30,6 +30,7 @@ def render("show.json", _) do languages: Keyword.get(instance, :languages, ["en"]), registrations: Keyword.get(instance, :registrations_open), approval_required: Keyword.get(instance, :account_approval_required), + contact_account: contact_account(Keyword.get(instance, :contact_username)), # Extra (not present in Mastodon): max_toot_chars: Keyword.get(instance, :limit), max_media_attachments: Keyword.get(instance, :max_media_attachments), @@ -141,4 +142,20 @@ def fields_limits do value_length: Config.get([:instance, :account_field_value_length]) } end + + defp contact_account(nil), do: nil + + defp contact_account("@" <> username) do + contact_account(username) + end + + defp contact_account(username) do + user = Pleroma.User.get_cached_by_nickname(username) + + if user do + Pleroma.Web.MastodonAPI.AccountView.render("show.json", %{user: user, for: nil}) + else + nil + end + end end diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index a556ef6a8..cfa86468f 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -106,4 +106,16 @@ test "instance languages", %{conn: conn} do |> get("/api/v1/instance") |> json_response_and_validate_schema(200) end + + test "get instance contact information", %{conn: conn} do + user = insert(:user, %{local: true}) + + clear_config([:instance, :contact_username], user.nickname) + + conn = get(conn, "/api/v1/instance") + + assert result = json_response_and_validate_schema(conn, 200) + + assert result["contact_account"]["id"] == user.id + end end From 93370b870a8d16b5dada5ef7c7e3ef2de378395e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 7 Aug 2022 23:56:52 +0200 Subject: [PATCH 02/42] Expose nonAnonymous field from Smithereen polls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../object_validators/question_validator.ex | 1 + lib/pleroma/web/api_spec/schemas/poll.ex | 11 ++++++++++- lib/pleroma/web/mastodon_api/views/poll_view.ex | 5 ++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 621085e6c..7f9d4d648 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -29,6 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do field(:closed, ObjectValidators.DateTime) field(:voters, {:array, ObjectValidators.ObjectID}, default: []) + field(:nonAnonymous, :boolean) embeds_many(:anyOf, QuestionOptionsValidator) embeds_many(:oneOf, QuestionOptionsValidator) end diff --git a/lib/pleroma/web/api_spec/schemas/poll.ex b/lib/pleroma/web/api_spec/schemas/poll.ex index 91570582b..cb2ffdc68 100644 --- a/lib/pleroma/web/api_spec/schemas/poll.ex +++ b/lib/pleroma/web/api_spec/schemas/poll.ex @@ -56,6 +56,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do } }, description: "Possible answers for the poll." + }, + pleroma: %Schema{ + type: :object, + properties: %{ + non_anonymous: %Schema{type: :boolean, description: "Is the voters collection public?"} + } } }, example: %{ @@ -79,7 +85,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do votes_count: 4 } ], - emojis: [] + emojis: [], + pleroma: %{ + non_anonymous: false + } } }) end diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex index 34e23873e..1e3c9f36d 100644 --- a/lib/pleroma/web/mastodon_api/views/poll_view.ex +++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex @@ -21,7 +21,10 @@ def render("show.json", %{object: object, multiple: multiple, options: options} votes_count: votes_count, voters_count: voters_count(object), options: options, - emojis: Pleroma.Web.MastodonAPI.StatusView.build_emojis(object.data["emoji"]) + emojis: Pleroma.Web.MastodonAPI.StatusView.build_emojis(object.data["emoji"]), + pleroma: %{ + non_anonymous: object.data["nonAnonymous"] || false + } } if params[:for] do From e5bd1ee80154b31457567c3d31de6c63e0146352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 8 Aug 2022 10:45:43 +0200 Subject: [PATCH 03/42] Add entry to @context, tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/public-polls.add | 1 + priv/static/schemas/litepub-0.1.jsonld | 4 +++- .../smithereen_non_anonymous_poll.json | 1 + test/fixtures/tesla_mock/smithereen_user.json | 1 + .../web/mastodon_api/views/poll_view_test.exs | 10 +++++++++- test/support/http_request_mock.ex | 18 ++++++++++++++++++ 6 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 changelog.d/public-polls.add create mode 100644 test/fixtures/tesla_mock/smithereen_non_anonymous_poll.json create mode 100644 test/fixtures/tesla_mock/smithereen_user.json diff --git a/changelog.d/public-polls.add b/changelog.d/public-polls.add new file mode 100644 index 000000000..0dae0c38e --- /dev/null +++ b/changelog.d/public-polls.add @@ -0,0 +1 @@ +Expose nonAnonymous field from Smithereen polls \ No newline at end of file diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld index b499a96f5..d156cba55 100644 --- a/priv/static/schemas/litepub-0.1.jsonld +++ b/priv/static/schemas/litepub-0.1.jsonld @@ -40,7 +40,9 @@ "@type": "@id" }, "vcard": "http://www.w3.org/2006/vcard/ns#", - "formerRepresentations": "litepub:formerRepresentations" + "formerRepresentations": "litepub:formerRepresentations", + "sm": "http://smithereen.software/ns#", + "nonAnonymous": "sm:nonAnonymous" } ] } diff --git a/test/fixtures/tesla_mock/smithereen_non_anonymous_poll.json b/test/fixtures/tesla_mock/smithereen_non_anonymous_poll.json new file mode 100644 index 000000000..2b343ea64 --- /dev/null +++ b/test/fixtures/tesla_mock/smithereen_non_anonymous_poll.json @@ -0,0 +1 @@ +{"type":"Question","id":"https://friends.grishka.me/posts/54642","attributedTo":"https://friends.grishka.me/users/1","content":"

здесь тоже можно что-то написать отдельно от опроса

","published":"2021-09-04T00:22:16Z","url":"https://friends.grishka.me/posts/54642","to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://friends.grishka.me/users/1/followers"],"replies":{"type":"Collection","id":"https://friends.grishka.me/posts/54642/replies","first":{"type":"CollectionPage","items":[],"partOf":"https://friends.grishka.me/posts/54642/replies","next":"https://friends.grishka.me/posts/54642/replies?page=1"}},"sensitive":false,"likes":"https://friends.grishka.me/posts/54642/likes","name":"тестовый опрос","oneOf":[{"type":"Note","id":"https://friends.grishka.me/posts/54642#options/76","name":"тестовый ответ 1","replies":{"type":"Collection","id":"https://friends.grishka.me/activitypub/objects/polls/24/options/76/votes","totalItems":4,"items":[]}},{"type":"Note","id":"https://friends.grishka.me/posts/54642#options/77","name":"тестовый ответ 2","replies":{"type":"Collection","id":"https://friends.grishka.me/activitypub/objects/polls/24/options/77/votes","totalItems":4,"items":[]}},{"type":"Note","id":"https://friends.grishka.me/posts/54642#options/78","name":"тестовый ответ 3","replies":{"type":"Collection","id":"https://friends.grishka.me/activitypub/objects/polls/24/options/78/votes","totalItems":6,"items":[]}}],"votersCount":14,"nonAnonymous":true,"@context":["https://www.w3.org/ns/activitystreams",{"sensitive":"as:sensitive","toot":"http://joinmastodon.org/ns#","sm":"http://smithereen.software/ns#","votersCount":"toot:votersCount","nonAnonymous":"sm:nonAnonymous"}]} \ No newline at end of file diff --git a/test/fixtures/tesla_mock/smithereen_user.json b/test/fixtures/tesla_mock/smithereen_user.json new file mode 100644 index 000000000..6468fc519 --- /dev/null +++ b/test/fixtures/tesla_mock/smithereen_user.json @@ -0,0 +1 @@ +{"type":"Person","id":"https://friends.grishka.me/users/1","name":"Григорий Клюшников","icon":{"type":"Image","image":{"type":"Image","url":"https://friends.grishka.me/i/6QLsOws97AWp5N_osd74C1IC1ijnFopyCBD9MSEeXNQ/q:93/bG9jYWw6Ly8vcy91cGxvYWRzL2F2YXRhcnMvNTYzODRhODEwODk5ZTRjMzI4YmY4YmQwM2Q2MWM3NmMud2VicA.jpg","mediaType":"image/jpeg","width":1280,"height":960},"width":573,"height":572,"cropRegion":[0.26422762870788574,0.3766937553882599,0.7113820910453796,0.9728997349739075],"url":"https://friends.grishka.me/i/ql_49PQcETAWgY_nC-Qj63H_Oa6FyOAEoWFkUSSkUvQ/c:573:572:nowe:338:362/q:93/bG9jYWw6Ly8vcy91cGxvYWRzL2F2YXRhcnMvNTYzODRhODEwODk5ZTRjMzI4YmY4YmQwM2Q2MWM3NmMud2VicA.jpg","mediaType":"image/jpeg"},"summary":"

Делаю эту хрень, пытаюсь вырвать социальные сети из жадных лап корпораций

\n

\n

\n

\n

\n

\n

\n

\n

This server does NOT support direct messages. Please write me on Telegram or Matrix.

","url":"https://friends.grishka.me/grishka","preferredUsername":"grishka","inbox":"https://friends.grishka.me/users/1/inbox","outbox":"https://friends.grishka.me/users/1/outbox","followers":"https://friends.grishka.me/users/1/followers","following":"https://friends.grishka.me/users/1/following","endpoints":{"sharedInbox":"https://friends.grishka.me/activitypub/sharedInbox","collectionSimpleQuery":"https://friends.grishka.me/users/1/collectionQuery"},"publicKey":{"id":"https://friends.grishka.me/users/1#main-key","owner":"https://friends.grishka.me/users/1","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAjlakm+i/d9ER/hIeR7KfiFW+SdLZj2SkKIeM8cmR+YFJuh9ghFqXrkFEjcaqUnAFqe5gYDNSQACnDLA8y4DnzjfGNIohKAnRoa9x6GORmfKQvcnjaTZ53S1NvUiPPyc0Pv/vfCtY/Ab0CEXe5BLqL38oZn817Jf7pBrPRTYH7m012kvwAUTT6k0Y8lPITBEG7nzYbbuGcrN9Y/RDdwE08jmBXlZ45bahRH3VNXVpQE17dCzJB+7k+iJ1R7YCoI+DuMlBYGXGE2KVk46NZTuLnOjFV9SyXfWX4/SrJM4oxev+SX2N75tQgmNZmVVHeqg2ZcbC0WCfNjJOi2HHS9MujwIDAQAB\n-----END PUBLIC KEY-----\n"},"wall":"https://friends.grishka.me/users/1/wall","firstName":"Григорий","lastName":"Клюшников","middleName":"Александрович","vcard:bday":"1993-01-22","gender":"http://schema.org#Male","supportsFriendRequests":true,"friends":"https://friends.grishka.me/users/1/friends","groups":"https://friends.grishka.me/users/1/groups","capabilities":{"supportsFriendRequests":true},"@context":["https://www.w3.org/ns/activitystreams",{"sm":"http://smithereen.software/ns#","cropRegion":{"@id":"sm:cropRegion","@container":"@list"},"wall":{"@id":"sm:wall","@type":"@id"},"collectionSimpleQuery":"sm:collectionSimpleQuery","sc":"http://schema.org#","firstName":"sc:givenName","lastName":"sc:familyName","middleName":"sc:additionalName","gender":{"@id":"sc:gender","@type":"sc:GenderType"},"maidenName":"sm:maidenName","friends":{"@id":"sm:friends","@type":"@id"},"groups":{"@id":"sm:groups","@type":"@id"},"vcard":"http://www.w3.org/2006/vcard/ns#","capabilities":"litepub:capabilities","supportsFriendRequests":"sm:supportsFriendRequests","litepub":"http://litepub.social/ns#"},"https://w3id.org/security/v1"]} \ No newline at end of file diff --git a/test/pleroma/web/mastodon_api/views/poll_view_test.exs b/test/pleroma/web/mastodon_api/views/poll_view_test.exs index a73d862fd..3aa73c224 100644 --- a/test/pleroma/web/mastodon_api/views/poll_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/poll_view_test.exs @@ -43,7 +43,8 @@ test "renders a poll" do %{title: "why are you even asking?", votes_count: 0} ], votes_count: 0, - voters_count: 0 + voters_count: 0, + pleroma: %{non_anonymous: false} } result = PollView.render("show.json", %{object: object}) @@ -165,4 +166,11 @@ test "doesn't strips HTML tags" do ] } = PollView.render("show.json", %{object: object}) end + + test "that poll is non anonymous" do + object = Object.normalize("https://friends.grishka.me/posts/54642", fetch: true) + result = PollView.render("show.json", %{object: object}) + + assert result[:pleroma][:non_anonymous] == true + end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 78a367024..2707804bc 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -1464,6 +1464,24 @@ def get("https://misskey.io/notes/8vs6wxufd0", _, _, _) do }} end + def get("https://friends.grishka.me/posts/54642", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/smithereen_non_anonymous_poll.json"), + headers: activitypub_object_headers() + }} + end + + def get("https://friends.grishka.me/users/1", _, _, _) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/smithereen_user.json"), + headers: activitypub_object_headers() + }} + end + def get(url, query, body, headers) do {:error, "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"} From 50e7706b269d6008ae4778db86bc4462ffce5ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 20 Nov 2022 23:19:52 +0100 Subject: [PATCH 04/42] Verify link ownership with rel="me" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/link-verification.add | 1 + lib/pleroma/user.ex | 63 ++++++++++++++++++- lib/pleroma/workers/background_worker.ex | 5 ++ test/pleroma/user_test.exs | 25 ++++++++ .../mastodon_api/update_credentials_test.exs | 20 +++--- 5 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 changelog.d/link-verification.add diff --git a/changelog.d/link-verification.add b/changelog.d/link-verification.add new file mode 100644 index 000000000..d8b11ebbc --- /dev/null +++ b/changelog.d/link-verification.add @@ -0,0 +1 @@ +Verify profile link ownership with rel="me" \ No newline at end of file diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ce125d608..d81aa5252 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -8,6 +8,7 @@ defmodule Pleroma.User do import Ecto.Changeset import Ecto.Query import Ecto, only: [assoc: 2] + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] alias Ecto.Multi alias Pleroma.Activity @@ -595,9 +596,23 @@ def update_changeset(struct, params \\ %{}) do defp put_fields(changeset) do if raw_fields = get_change(changeset, :raw_fields) do + old_fields = changeset.data.raw_fields + raw_fields = raw_fields |> Enum.filter(fn %{"name" => n} -> n != "" end) + |> Enum.map(fn field -> + previous = + old_fields + |> Enum.find(fn %{"value" => value} -> field["value"] == value end) + + if previous && Map.has_key?(previous, "verified_at") do + field + |> Map.put("verified_at", previous["verified_at"]) + else + field + end + end) fields = raw_fields @@ -1198,6 +1213,10 @@ def update_and_set_cache(struct, params) do def update_and_set_cache(changeset) do with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do + if get_change(changeset, :raw_fields) do + BackgroundWorker.enqueue("verify_fields_links", %{"user_id" => user.id}) + end + set_cache(user) end end @@ -1970,8 +1989,47 @@ def perform(:delete, %User{} = user) do maybe_delete_from_db(user) end + def perform(:verify_fields_links, user) do + profile_urls = [user.ap_id] + + fields = + user.raw_fields + |> Enum.map(&verify_field_link(&1, profile_urls)) + + changeset = + user + |> update_changeset(%{raw_fields: fields}) + + with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do + set_cache(user) + end + end + def perform(:set_activation_async, user, status), do: set_activation(user, status) + defp verify_field_link(field, profile_urls) do + verified_at = + with %{"value" => value} <- field, + {:verified_at, nil} <- {:verified_at, Map.get(field, "verified_at")}, + %{scheme: scheme, userinfo: nil, host: host} + when not_empty_string(host) and scheme in ["http", "https"] <- + URI.parse(value), + {:not_idn, true} <- {:not_idn, to_string(:idna.encode(host)) == host}, + attr <- Pleroma.Web.RelMe.maybe_put_rel_me(value, profile_urls) do + if attr == "me" do + CommonUtils.to_masto_date(NaiveDateTime.utc_now()) + end + else + {:verified_at, value} when not_empty_string(value) -> + value + + _ -> + nil + end + + Map.put(field, "verified_at", verified_at) + end + @spec external_users_query() :: Ecto.Query.t() def external_users_query do User.Query.build(%{ @@ -2659,10 +2717,11 @@ def sanitize_html(%User{} = user) do # - display name def sanitize_html(%User{} = user, filter) do fields = - Enum.map(user.fields, fn %{"name" => name, "value" => value} -> + Enum.map(user.fields, fn %{"name" => name, "value" => value} = fields -> %{ "name" => name, - "value" => HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + "value" => HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly), + "verified_at" => Map.get(fields, "verified_at") } end) diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 794417612..eef1c4f15 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -40,6 +40,11 @@ def perform(%Job{ Pleroma.FollowingRelationship.move_following(origin, target) end + def perform(%Job{args: %{"op" => "verify_fields_links", "user_id" => user_id}}) do + user = User.get_by_id(user_id) + User.perform(:verify_fields_links, user) + end + def perform(%Job{args: %{"op" => "delete_instance", "host" => host}}) do Instance.perform(:delete_instance, host) end diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 7f60b959a..93de980c9 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -2913,4 +2913,29 @@ test "it doesn't pin users you do not follow" do refute User.endorses?(user, pinned_user) end end + + test "it checks fields links for a backlink" do + user = insert(:user, ap_id: "https://social.example.org/users/lain") + + fields = [ + %{"name" => "Link", "value" => "http://example.com/rel_me/null"}, + %{"name" => "Verified link", "value" => "http://example.com/rel_me/link"}, + %{"name" => "Not a link", "value" => "i'm not a link"} + ] + + user + |> User.update_and_set_cache(%{raw_fields: fields}) + + ObanHelpers.perform_all() + + user = User.get_cached_by_id(user.id) + + assert [ + %{"verified_at" => nil}, + %{"verified_at" => verified_at}, + %{"verified_at" => nil} + ] = user.fields + + assert is_binary(verified_at) + end end diff --git a/test/pleroma/web/mastodon_api/update_credentials_test.exs b/test/pleroma/web/mastodon_api/update_credentials_test.exs index 45412bb34..c1db21ac2 100644 --- a/test/pleroma/web/mastodon_api/update_credentials_test.exs +++ b/test/pleroma/web/mastodon_api/update_credentials_test.exs @@ -502,10 +502,15 @@ test "update fields", %{conn: conn} do |> json_response_and_validate_schema(200) assert account_data["fields"] == [ - %{"name" => "foo", "value" => "bar"}, + %{ + "name" => "foo", + "value" => "bar", + "verified_at" => nil + }, %{ "name" => "link.io", - "value" => ~S(cofe.io) + "value" => ~S(cofe.io), + "verified_at" => nil } ] @@ -564,8 +569,8 @@ test "emojis in fields labels", %{conn: conn} do |> json_response_and_validate_schema(200) assert account_data["fields"] == [ - %{"name" => ":firefox:", "value" => "is best 2hu"}, - %{"name" => "they wins", "value" => ":blank:"} + %{"name" => ":firefox:", "value" => "is best 2hu", "verified_at" => nil}, + %{"name" => "they wins", "value" => ":blank:", "verified_at" => nil} ] assert account_data["source"]["fields"] == [ @@ -593,10 +598,11 @@ test "update fields via x-www-form-urlencoded", %{conn: conn} do |> json_response_and_validate_schema(200) assert account["fields"] == [ - %{"name" => "foo", "value" => "bar"}, + %{"name" => "foo", "value" => "bar", "verified_at" => nil}, %{ "name" => "link", - "value" => ~S(http://cofe.io) + "value" => ~S(http://cofe.io), + "verified_at" => nil } ] @@ -618,7 +624,7 @@ test "update fields with empty name", %{conn: conn} do |> json_response_and_validate_schema(200) assert account["fields"] == [ - %{"name" => "foo", "value" => ""} + %{"name" => "foo", "value" => "", "verified_at" => nil} ] end From fa02a1e634632bb672b53c244179d3088ffe8423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Fri, 19 Jan 2024 18:05:46 +0100 Subject: [PATCH 05/42] Update MastoAPI responses docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- docs/development/API/differences_in_mastoapi_responses.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index 48a9c104c..70c31272d 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -39,6 +39,7 @@ Has these additional fields under the `pleroma` object: - `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint. - `parent_visible`: If the parent of this post is visible to the user or not. - `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise. +- `non_anonymous`: true if the source post specifies the poll results are not anonymous. Currently only implemented by Smithereen. The `GET /api/v1/statuses/:id/source` endpoint additionally has the following attributes: From a5f64ffd0cba5e183744cf46506656ba61d36eda Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Fri, 19 Jan 2024 21:28:15 +0000 Subject: [PATCH 06/42] =?UTF-8?q?Apply=20lanodan=E2=80=99s=20suggestion=20?= =?UTF-8?q?to=201=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/pleroma/web/api_spec/schemas/poll.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/schemas/poll.ex b/lib/pleroma/web/api_spec/schemas/poll.ex index cb2ffdc68..eeb1a5490 100644 --- a/lib/pleroma/web/api_spec/schemas/poll.ex +++ b/lib/pleroma/web/api_spec/schemas/poll.ex @@ -60,7 +60,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do pleroma: %Schema{ type: :object, properties: %{ - non_anonymous: %Schema{type: :boolean, description: "Is the voters collection public?"} + non_anonymous: %Schema{type: :boolean, description: "Can voters be publicly identified?"} } } }, From def088ce520ffac3e200cbf63b049f1d918d2b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 21 Jan 2024 18:23:24 +0100 Subject: [PATCH 07/42] format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/api_spec/schemas/poll.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/schemas/poll.ex b/lib/pleroma/web/api_spec/schemas/poll.ex index eeb1a5490..20cf5b061 100644 --- a/lib/pleroma/web/api_spec/schemas/poll.ex +++ b/lib/pleroma/web/api_spec/schemas/poll.ex @@ -60,7 +60,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do pleroma: %Schema{ type: :object, properties: %{ - non_anonymous: %Schema{type: :boolean, description: "Can voters be publicly identified?"} + non_anonymous: %Schema{ + type: :boolean, + description: "Can voters be publicly identified?" + } } } }, From acb9e46074346ad28ad6444a170bdd5e00c74910 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 27 Feb 2024 13:25:26 +0100 Subject: [PATCH 08/42] Add some missing fields to instanceV2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/instance-v2.skip | 0 config/description.exs | 8 ++++++++ .../web/mastodon_api/views/instance_view.ex | 20 +++++++++++++++---- 3 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 changelog.d/instance-v2.skip diff --git a/changelog.d/instance-v2.skip b/changelog.d/instance-v2.skip new file mode 100644 index 000000000..e69de29bb diff --git a/config/description.exs b/config/description.exs index 78e7710cb..6c13bde31 100644 --- a/config/description.exs +++ b/config/description.exs @@ -566,6 +566,14 @@ "Cool instance" ] }, + %{ + key: :status_page, + type: :string, + description: "A page where people can see the status of the server during an outage", + suggestions: [ + "https://status.pleroma.example.org" + ] + }, %{ key: :limit, type: :integer, diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 7f73917d6..e4c6c81e1 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -63,7 +63,8 @@ def render("show2.json", _) do registrations: %{ enabled: Keyword.get(instance, :registrations_open), approval_required: Keyword.get(instance, :account_approval_required), - message: nil + message: nil, + url: nil }, contact: %{ email: Keyword.get(instance, :email), @@ -78,7 +79,8 @@ defp common_information(instance) do %{ title: Keyword.get(instance, :name), version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", - languages: Keyword.get(instance, :languages, ["en"]) + languages: Keyword.get(instance, :languages, ["en"]), + rules: [] } end @@ -170,13 +172,17 @@ defp fields_limits do defp configuration do %{ + accounts: %{ + max_featured_tags: 0 + }, statuses: %{ max_characters: Config.get([:instance, :limit]), max_media_attachments: Config.get([:instance, :max_media_attachments]) }, media_attachments: %{ image_size_limit: Config.get([:instance, :upload_limit]), - video_size_limit: Config.get([:instance, :upload_limit]) + video_size_limit: Config.get([:instance, :upload_limit]), + supported_mime_types: ["application/octet-stream"] }, polls: %{ max_options: Config.get([:instance, :poll_limits, :max_options]), @@ -190,7 +196,13 @@ defp configuration do defp configuration2 do configuration() |> Map.merge(%{ - urls: %{streaming: Pleroma.Web.Endpoint.websocket_url()} + urls: %{ + streaming: Pleroma.Web.Endpoint.websocket_url(), + status: Config.get([:instance, :status_page]) + }, + vapid: %{ + public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + } }) end From d415686bb9248e1c5b8913b7c3b96f74e9b8f230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 26 Feb 2024 23:45:02 +0100 Subject: [PATCH 09/42] Allow to group bookmarks in folders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/bookmark-folders.add | 1 + .../API/differences_in_mastoapi_responses.md | 7 + docs/development/API/pleroma_api.md | 46 ++++++ lib/pleroma/bookmark.ex | 31 +++- lib/pleroma/bookmark_folder.ex | 115 ++++++++++++++ lib/pleroma/web/api_spec.ex | 3 +- .../pleroma_bookmark_folder_operation.ex | 125 +++++++++++++++ .../api_spec/operations/status_operation.ex | 22 ++- .../web/api_spec/schemas/bookmark_folder.ex | 26 +++ .../controllers/status_controller.ex | 17 +- .../web/mastodon_api/views/status_view.ex | 28 +++- .../controllers/bookmark_folder_controller.ex | 68 ++++++++ .../pleroma_api/views/bookmark_folder_view.ex | 44 ++++++ lib/pleroma/web/router.ex | 5 + ...20240223165000_create_bookmark_folders.exs | 27 ++++ test/pleroma/bookmark_folder_test.exs | 60 +++++++ test/pleroma/bookmark_test.exs | 17 +- .../controllers/status_controller_test.exs | 54 +++++++ .../mastodon_api/views/status_view_test.exs | 3 +- .../bookmark_folder_controller_test.ex | 148 ++++++++++++++++++ uploads/.gitignore | 3 - 21 files changed, 829 insertions(+), 21 deletions(-) create mode 100644 changelog.d/bookmark-folders.add create mode 100644 lib/pleroma/bookmark_folder.ex create mode 100644 lib/pleroma/web/api_spec/operations/pleroma_bookmark_folder_operation.ex create mode 100644 lib/pleroma/web/api_spec/schemas/bookmark_folder.ex create mode 100644 lib/pleroma/web/pleroma_api/controllers/bookmark_folder_controller.ex create mode 100644 lib/pleroma/web/pleroma_api/views/bookmark_folder_view.ex create mode 100644 priv/repo/migrations/20240223165000_create_bookmark_folders.exs create mode 100644 test/pleroma/bookmark_folder_test.exs create mode 100644 test/pleroma/web/pleroma_api/controllers/bookmark_folder_controller_test.ex delete mode 100644 uploads/.gitignore diff --git a/changelog.d/bookmark-folders.add b/changelog.d/bookmark-folders.add new file mode 100644 index 000000000..d9b03cecc --- /dev/null +++ b/changelog.d/bookmark-folders.add @@ -0,0 +1 @@ +Allow to group bookmarks in folders \ No newline at end of file diff --git a/docs/development/API/differences_in_mastoapi_responses.md b/docs/development/API/differences_in_mastoapi_responses.md index 2937b2301..d72717483 100644 --- a/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/development/API/differences_in_mastoapi_responses.md @@ -40,6 +40,7 @@ Has these additional fields under the `pleroma` object: - `parent_visible`: If the parent of this post is visible to the user or not. - `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise. - `quotes_count`: the count of status quotes. +- `bookmark_folder`: the ID of the folder bookmark is stored within (if any). The `GET /api/v1/statuses/:id/source` endpoint additionally has the following attributes: @@ -65,6 +66,12 @@ Some apps operate under the assumption that no more than 4 attachments can be re Pleroma does not process remote images and therefore cannot include fields such as `meta` and `blurhash`. It does not support focal points or aspect ratios. The frontend is expected to handle it. +## Bookmarks + +The `GET /api/v1/bookmarks` endpoint accepts optional parameter `folder_id` for bookmark folder ID. + +The `POST /api/v1/statuses/:id/bookmark` endpoint accepts optional parameter `folder_id` for bookmark folder ID. + ## Accounts The `id` parameter can also be the `nickname` of the user. This only works in these endpoints, not the deeper nested ones for following etc. diff --git a/docs/development/API/pleroma_api.md b/docs/development/API/pleroma_api.md index 060af5c14..267dfc1ec 100644 --- a/docs/development/API/pleroma_api.md +++ b/docs/development/API/pleroma_api.md @@ -283,6 +283,52 @@ See [Admin-API](admin_api.md) * `id`: the id of the status * Response: JSON, returns a list of Mastodon Status entities +## `GET /api/v1/pleroma/bookmark_folders` +### Gets user bookmark folders +* Authentication: required + +* Response: JSON. Returns a list of bookmark folders. +* Example response: +```json +[ + { + "id": "9umDrYheeY451cQnEe", + "name": "Read later", + "emoji": "🕓", + "source": { + "emoji": "🕓" + } + } +] +``` + +## `POST /api/v1/pleroma/bookmark_folders` +### Creates a bookmark folder +* Authentication: required + +* Params: + * `name`: folder name + * `emoji`: folder emoji (optional) +* Response: JSON. Returns a single bookmark folder. + +## `PATCH /api/v1/pleroma/bookmark_folders/:id` +### Updates a bookmark folder +* Authentication: required + +* Params: + * `id`: folder id + * `name`: folder name (optional) + * `emoji`: folder emoji (optional) +* Response: JSON. Returns a single bookmark folder. + +## `DELETE /api/v1/pleroma/bookmark_folders/:id` +### Deletes a bookmark folder +* Authentication: required + +* Params: + * `id`: folder id +* Response: JSON. Returns a single bookmark folder. + ## `/api/v1/pleroma/mascot` ### Gets user mascot image * Method `GET` diff --git a/lib/pleroma/bookmark.ex b/lib/pleroma/bookmark.ex index b83d72446..1a2a63b82 100644 --- a/lib/pleroma/bookmark.ex +++ b/lib/pleroma/bookmark.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Bookmark do alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.BookmarkFolder alias Pleroma.Repo alias Pleroma.User @@ -18,33 +19,46 @@ defmodule Pleroma.Bookmark do schema "bookmarks" do belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType) + belongs_to(:folder, BookmarkFolder, type: FlakeId.Ecto.CompatType) timestamps() end @spec create(Ecto.UUID.t(), Ecto.UUID.t()) :: {:ok, Bookmark.t()} | {:error, Ecto.Changeset.t()} - def create(user_id, activity_id) do + def create(user_id, activity_id, folder_id \\ nil) do attrs = %{ user_id: user_id, - activity_id: activity_id + activity_id: activity_id, + folder_id: folder_id } %Bookmark{} - |> cast(attrs, [:user_id, :activity_id]) + |> cast(attrs, [:user_id, :activity_id, :folder_id]) |> validate_required([:user_id, :activity_id]) |> unique_constraint(:activity_id, name: :bookmarks_user_id_activity_id_index) - |> Repo.insert() + |> Repo.insert( + on_conflict: [set: [folder_id: folder_id]], + conflict_target: [:user_id, :activity_id] + ) end @spec for_user_query(Ecto.UUID.t()) :: Ecto.Query.t() - def for_user_query(user_id) do + def for_user_query(user_id, folder_id \\ nil) do Bookmark |> where(user_id: ^user_id) + |> maybe_filter_by_folder(folder_id) |> join(:inner, [b], activity in assoc(b, :activity)) |> preload([b, a], activity: a) end + defp maybe_filter_by_folder(query, nil), do: query + + defp maybe_filter_by_folder(query, folder_id) do + query + |> where(folder_id: ^folder_id) + end + def get(user_id, activity_id) do Bookmark |> where(user_id: ^user_id) @@ -62,4 +76,11 @@ def destroy(user_id, activity_id) do |> Repo.one() |> Repo.delete() end + + def set_folder(bookmark, folder_id) do + bookmark + |> cast(%{folder_id: folder_id}, [:folder_id]) + |> validate_required([:folder_id]) + |> Repo.update() + end end diff --git a/lib/pleroma/bookmark_folder.ex b/lib/pleroma/bookmark_folder.ex new file mode 100644 index 000000000..14d37e197 --- /dev/null +++ b/lib/pleroma/bookmark_folder.ex @@ -0,0 +1,115 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.BookmarkFolder do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + + alias Pleroma.BookmarkFolder + alias Pleroma.Emoji + alias Pleroma.Repo + alias Pleroma.User + + @type t :: %__MODULE__{} + @primary_key {:id, FlakeId.Ecto.CompatType, autogenerate: true} + + schema "bookmark_folders" do + field(:name, :string) + field(:emoji, :string) + + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) + + timestamps() + end + + def get_by_id(id), do: Repo.get_by(BookmarkFolder, id: id) + + def create(user_id, name, emoji \\ nil) do + %BookmarkFolder{} + |> cast( + %{ + user_id: user_id, + name: name, + emoji: emoji + }, + [:user_id, :name, :emoji] + ) + |> validate_required([:user_id, :name]) + |> fix_emoji() + |> validate_emoji() + |> unique_constraint([:user_id, :name]) + |> Repo.insert() + end + + def update(folder_id, name, emoji \\ nil) do + get_by_id(folder_id) + |> cast( + %{ + name: name, + emoji: emoji + }, + [:name, :emoji] + ) + |> fix_emoji() + |> validate_emoji() + |> unique_constraint([:user_id, :name]) + |> Repo.update() + end + + defp fix_emoji(changeset) do + with {:emoji_field, emoji} when is_binary(emoji) <- + {:emoji_field, get_field(changeset, :emoji)}, + {:fixed_emoji, emoji} <- + {:fixed_emoji, + emoji + |> Pleroma.Emoji.fully_qualify_emoji() + |> Pleroma.Emoji.maybe_quote()} do + put_change(changeset, :emoji, emoji) + else + {:emoji_field, _} -> changeset + end + end + + defp validate_emoji(changeset) do + validate_change(changeset, :emoji, fn + :emoji, nil -> + [] + + :emoji, emoji -> + if Emoji.unicode?(emoji) or valid_local_custom_emoji?(emoji) do + [] + else + [emoji: "Invalid emoji"] + end + end) + end + + defp valid_local_custom_emoji?(emoji) do + with %{file: _path} <- Emoji.get(emoji) do + true + else + _ -> false + end + end + + def delete(folder_id) do + BookmarkFolder + |> Repo.get_by(id: folder_id) + |> Repo.delete() + end + + def for_user(user_id) do + BookmarkFolder + |> where(user_id: ^user_id) + |> Repo.all() + end + + def belongs_to_user?(folder_id, user_id) do + BookmarkFolder + |> where(id: ^folder_id, user_id: ^user_id) + |> Repo.exists?() + end +end diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 3588608f2..10d221571 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -137,7 +137,8 @@ def spec(opts \\ []) do "Scheduled statuses", "Search", "Status actions", - "Media attachments" + "Media attachments", + "Bookmark folders" ] }, %{ diff --git a/lib/pleroma/web/api_spec/operations/pleroma_bookmark_folder_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_bookmark_folder_operation.ex new file mode 100644 index 000000000..eaa683125 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/pleroma_bookmark_folder_operation.ex @@ -0,0 +1,125 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.PleromaBookmarkFolderOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.BookmarkFolder + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + + import Pleroma.Web.ApiSpec.Helpers + + @spec open_api_operation(any()) :: any() + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Bookmark folders"], + summary: "All bookmark folders", + security: [%{"oAuth" => ["read:bookmarks"]}], + operationId: "PleromaAPI.BookmarkFolderController.index", + responses: %{ + 200 => + Operation.response("Array of Bookmark Folders", "application/json", %Schema{ + type: :array, + items: BookmarkFolder + }) + } + } + end + + def create_operation do + %Operation{ + tags: ["Bookmark folders"], + summary: "Create a bookmark folder", + security: [%{"oAuth" => ["write:bookmarks"]}], + operationId: "PleromaAPI.BookmarkFolderController.create", + requestBody: request_body("Parameters", create_request(), required: true), + responses: %{ + 200 => Operation.response("Bookmark Folder", "application/json", BookmarkFolder), + 422 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def update_operation do + %Operation{ + tags: ["Bookmark folders"], + summary: "Update a bookmark folder", + security: [%{"oAuth" => ["write:bookmarks"]}], + operationId: "PleromaAPI.BookmarkFolderController.update", + parameters: [id_param()], + requestBody: request_body("Parameters", update_request(), required: true), + responses: %{ + 200 => Operation.response("Bookmark Folder", "application/json", BookmarkFolder), + 403 => Operation.response("Forbidden", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError), + 422 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def delete_operation do + %Operation{ + tags: ["Bookmark folders"], + summary: "Delete a bookmark folder", + security: [%{"oAuth" => ["write:bookmarks"]}], + operationId: "PleromaAPI.BookmarkFolderController.delete", + parameters: [id_param()], + responses: %{ + 200 => Operation.response("Bookmark Folder", "application/json", BookmarkFolder), + 403 => Operation.response("Forbidden", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + + defp create_request do + %Schema{ + title: "BookmarkFolderCreateRequest", + type: :object, + properties: %{ + name: %Schema{ + type: :string, + description: "Folder name" + }, + emoji: %Schema{ + type: :string, + nullable: true, + description: "Folder emoji" + } + } + } + end + + defp update_request do + %Schema{ + title: "BookmarkFolderUpdateRequest", + type: :object, + properties: %{ + name: %Schema{ + type: :string, + nullable: true, + description: "Folder name" + }, + emoji: %Schema{ + type: :string, + nullable: true, + description: "Folder emoji" + } + } + } + end + + def id_param do + Operation.parameter(:id, :path, FlakeID.schema(), "Bookmark Folder ID", + example: "9umDrYheeY451cQnEe", + required: true + ) + end +end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index ef4e34044..1717c68c8 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -256,6 +256,18 @@ def bookmark_operation do description: "Privately bookmark a status", operationId: "StatusController.bookmark", parameters: [id_param()], + requestBody: + request_body("Parameters", %Schema{ + title: "StatusUpdateRequest", + type: :object, + properties: %{ + folder_id: %Schema{ + nullable: true, + allOf: [FlakeID], + description: "ID of bookmarks folder, if any" + } + } + }), responses: %{ 200 => status_response() } @@ -430,7 +442,15 @@ def bookmarks_operation do summary: "Bookmarked statuses", description: "Statuses the user has bookmarked", operationId: "StatusController.bookmarks", - parameters: pagination_params(), + parameters: [ + Operation.parameter( + :folder_id, + :query, + FlakeID.schema(), + "If provided, only display bookmarks from given folder" + ) + | pagination_params() + ], security: [%{"oAuth" => ["read:bookmarks"]}], responses: %{ 200 => Operation.response("Array of Statuses", "application/json", array_of_statuses()) diff --git a/lib/pleroma/web/api_spec/schemas/bookmark_folder.ex b/lib/pleroma/web/api_spec/schemas/bookmark_folder.ex new file mode 100644 index 000000000..e8b4f43b7 --- /dev/null +++ b/lib/pleroma/web/api_spec/schemas/bookmark_folder.ex @@ -0,0 +1,26 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Schemas.BookmarkFolder do + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "BookmarkFolder", + description: "Response schema for a bookmark folder", + type: :object, + properties: %{ + id: FlakeID, + name: %Schema{type: :string, description: "Folder name"}, + emoji: %Schema{type: :string, description: "Folder emoji", nullable: true} + }, + example: %{ + "id" => "9toJCu5YZW7O7gfvH6", + "name" => "Read later", + "emoji" => nil + } + }) +end diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 5aa7bddf0..4f6de8a00 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -12,6 +12,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.BookmarkFolder alias Pleroma.Object alias Pleroma.Repo alias Pleroma.ScheduledActivity @@ -411,13 +412,22 @@ def unpin( @doc "POST /api/v1/statuses/:id/bookmark" def bookmark( - %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, + %{ + assigns: %{user: user}, + private: %{open_api_spex: %{body_params: body_params, params: %{id: id}}} + } = conn, _ ) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), %User{} = user <- User.get_cached_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), - {:ok, _bookmark} <- Bookmark.create(user.id, activity.id) do + folder_id <- Map.get(body_params, :folder_id, nil), + folder_id <- + if(folder_id && BookmarkFolder.belongs_to_user?(folder_id, user.id), + do: folder_id, + else: nil + ), + {:ok, _bookmark} <- Bookmark.create(user.id, activity.id, folder_id) do try_render(conn, "show.json", activity: activity, for: user, as: :activity) end end @@ -573,10 +583,11 @@ def favourites( @doc "GET /api/v1/bookmarks" def bookmarks(%{assigns: %{user: user}, private: %{open_api_spex: %{params: params}}} = conn, _) do user = User.get_cached_by_id(user.id) + folder_id = Map.get(params, :folder_id) bookmarks = user.id - |> Bookmark.for_user_query() + |> Bookmark.for_user_query(folder_id) |> Pleroma.Pagination.fetch_paginated(params) activities = diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 6303e72ce..e464f60dc 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -184,7 +184,14 @@ def render( favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) - bookmarked = Activity.get_bookmark(reblogged_parent_activity, opts[:for]) != nil + bookmark = Activity.get_bookmark(reblogged_parent_activity, opts[:for]) + + bookmark_folder = + if bookmark != nil do + bookmark.folder_id + else + nil + end mentions = activity.recipients @@ -213,7 +220,7 @@ def render( favourites_count: 0, reblogged: reblogged?(reblogged_parent_activity, opts[:for]), favourited: present?(favorited), - bookmarked: present?(bookmarked), + bookmarked: present?(bookmark), muted: false, pinned: pinned?, sensitive: false, @@ -227,7 +234,8 @@ def render( emojis: [], pleroma: %{ local: activity.local, - pinned_at: pinned_at + pinned_at: pinned_at, + bookmark_folder: bookmark_folder } } end @@ -264,7 +272,14 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) - bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil + bookmark = Activity.get_bookmark(activity, opts[:for]) + + bookmark_folder = + if bookmark != nil do + bookmark.folder_id + else + nil + end client_posted_this_activity = opts[:for] && user.id == opts[:for].id @@ -418,7 +433,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} favourites_count: like_count, reblogged: reblogged?(activity, opts[:for]), favourited: present?(favorited), - bookmarked: present?(bookmarked), + bookmarked: present?(bookmark), muted: muted, pinned: pinned?, sensitive: sensitive, @@ -448,7 +463,8 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} emoji_reactions: emoji_reactions, parent_visible: visible_for_user?(reply_to, opts[:for]), pinned_at: pinned_at, - quotes_count: object.data["quotesCount"] || 0 + quotes_count: object.data["quotesCount"] || 0, + bookmark_folder: bookmark_folder } } end diff --git a/lib/pleroma/web/pleroma_api/controllers/bookmark_folder_controller.ex b/lib/pleroma/web/pleroma_api/controllers/bookmark_folder_controller.ex new file mode 100644 index 000000000..6d6e2e940 --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/bookmark_folder_controller.ex @@ -0,0 +1,68 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.BookmarkFolderController do + use Pleroma.Web, :controller + + alias Pleroma.BookmarkFolder + alias Pleroma.Web.Plugs.OAuthScopesPlug + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + # Note: scope not present in Mastodon: read:bookmarks + plug(OAuthScopesPlug, %{scopes: ["read:bookmarks"]} when action == :index) + + # Note: scope not present in Mastodon: write:bookmarks + plug( + OAuthScopesPlug, + %{scopes: ["write:bookmarks"]} when action in [:create, :update, :delete] + ) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBookmarkFolderOperation + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + + def index(%{assigns: %{user: user}} = conn, _params) do + with folders <- BookmarkFolder.for_user(user.id) do + conn + |> render("index.json", %{folders: folders, as: :folder}) + end + end + + def create( + %{assigns: %{user: user}, private: %{open_api_spex: %{body_params: params}}} = conn, + _ + ) do + with {:ok, folder} <- BookmarkFolder.create(user.id, params[:name], params[:emoji]) do + render(conn, "show.json", folder: folder) + end + end + + def update( + %{ + assigns: %{user: user}, + private: %{open_api_spex: %{body_params: params, params: %{id: id}}} + } = conn, + _ + ) do + with true <- BookmarkFolder.belongs_to_user?(id, user.id), + {:ok, folder} <- BookmarkFolder.update(id, params[:name], params[:emoji]) do + render(conn, "show.json", folder: folder) + else + false -> {:error, :forbidden} + end + end + + def delete( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, + _ + ) do + with true <- BookmarkFolder.belongs_to_user?(id, user.id), + {:ok, folder} <- BookmarkFolder.delete(id) do + render(conn, "show.json", folder: folder) + else + false -> {:error, :forbidden} + end + end +end diff --git a/lib/pleroma/web/pleroma_api/views/bookmark_folder_view.ex b/lib/pleroma/web/pleroma_api/views/bookmark_folder_view.ex new file mode 100644 index 000000000..fc6ad59d0 --- /dev/null +++ b/lib/pleroma/web/pleroma_api/views/bookmark_folder_view.ex @@ -0,0 +1,44 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.BookmarkFolderView do + use Pleroma.Web, :view + + alias Pleroma.BookmarkFolder + alias Pleroma.Emoji + alias Pleroma.Web.Endpoint + + def render("show.json", %{folder: %BookmarkFolder{} = folder}) do + %{ + id: folder.id |> to_string(), + name: folder.name, + emoji: get_emoji(folder.emoji), + source: %{ + emoji: folder.emoji + } + } + end + + def render("index.json", %{folders: folders} = opts) do + render_many(folders, __MODULE__, "show.json", Map.delete(opts, :folders)) + end + + defp get_emoji(nil) do + nil + end + + defp get_emoji(emoji) do + if Emoji.unicode?(emoji) do + emoji + else + emoji = Emoji.get(emoji) + + if emoji != nil do + Endpoint.url() |> URI.merge(emoji.relative_url) |> to_string() + else + nil + end + end + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 8ba845364..4fe0cb02f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -580,6 +580,11 @@ defmodule Pleroma.Web.Router do get("/backups", BackupController, :index) post("/backups", BackupController, :create) + + get("/bookmark_folders", BookmarkFolderController, :index) + post("/bookmark_folders", BookmarkFolderController, :create) + patch("/bookmark_folders/:id", BookmarkFolderController, :update) + delete("/bookmark_folders/:id", BookmarkFolderController, :delete) end scope [] do diff --git a/priv/repo/migrations/20240223165000_create_bookmark_folders.exs b/priv/repo/migrations/20240223165000_create_bookmark_folders.exs new file mode 100644 index 000000000..016916968 --- /dev/null +++ b/priv/repo/migrations/20240223165000_create_bookmark_folders.exs @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.CreateBookmarkFolders do + use Ecto.Migration + + def change do + create_if_not_exists table(:bookmark_folders, primary_key: false) do + add(:id, :uuid, primary_key: true) + add(:name, :string, null: false) + add(:emoji, :string) + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + + timestamps() + end + + alter table(:bookmarks) do + add_if_not_exists( + :folder_id, + references(:bookmark_folders, type: :uuid, on_delete: :nilify_all) + ) + end + + create_if_not_exists(unique_index(:bookmark_folders, [:user_id, :name])) + end +end diff --git a/test/pleroma/bookmark_folder_test.exs b/test/pleroma/bookmark_folder_test.exs new file mode 100644 index 000000000..c45129b0e --- /dev/null +++ b/test/pleroma/bookmark_folder_test.exs @@ -0,0 +1,60 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.BookmarkFolderTest do + use Pleroma.DataCase, async: true + import Pleroma.Factory + alias Pleroma.BookmarkFolder + + describe "create/3" do + test "with valid params" do + user = insert(:user) + {:ok, folder} = BookmarkFolder.create(user.id, "Read later", "🕓") + assert folder.user_id == user.id + assert folder.name == "Read later" + assert folder.emoji == "🕓" + end + + test "with invalid params" do + {:error, changeset} = BookmarkFolder.create(nil, "", "not an emoji") + refute changeset.valid? + + assert changeset.errors == [ + emoji: {"Invalid emoji", []}, + user_id: {"can't be blank", [validation: :required]}, + name: {"can't be blank", [validation: :required]} + ] + end + end + + test "update/3" do + user = insert(:user) + {:ok, folder} = BookmarkFolder.create(user.id, "Read ltaer") + {:ok, folder} = BookmarkFolder.update(folder.id, "Read later") + assert folder.name == "Read later" + end + + test "for_user/1" do + user = insert(:user) + other_user = insert(:user) + + {:ok, _} = BookmarkFolder.create(user.id, "Folder 1") + {:ok, _} = BookmarkFolder.create(user.id, "Folder 2") + {:ok, _} = BookmarkFolder.create(other_user.id, "Folder 3") + + folders = BookmarkFolder.for_user(user.id) + + assert length(folders) == 2 + end + + test "belongs_to_user?/2" do + user = insert(:user) + other_user = insert(:user) + + {:ok, folder} = BookmarkFolder.create(user.id, "Folder") + + assert true == BookmarkFolder.belongs_to_user?(folder.id, user.id) + assert false == BookmarkFolder.belongs_to_user?(folder.id, other_user.id) + end +end diff --git a/test/pleroma/bookmark_test.exs b/test/pleroma/bookmark_test.exs index 57144ded3..a2ed24c26 100644 --- a/test/pleroma/bookmark_test.exs +++ b/test/pleroma/bookmark_test.exs @@ -6,15 +6,17 @@ defmodule Pleroma.BookmarkTest do use Pleroma.DataCase, async: true import Pleroma.Factory alias Pleroma.Bookmark + alias Pleroma.BookmarkFolder alias Pleroma.Web.CommonAPI - describe "create/2" do + describe "create/3" do test "with valid params" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{status: "Some cool information"}) {:ok, bookmark} = Bookmark.create(user.id, activity.id) assert bookmark.user_id == user.id assert bookmark.activity_id == activity.id + assert bookmark.folder_id == nil end test "with invalid params" do @@ -26,6 +28,19 @@ test "with invalid params" do activity_id: {"can't be blank", [validation: :required]} ] end + + test "update existing bookmark folder" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{status: "Some cool information"}) + + {:ok, bookmark} = Bookmark.create(user.id, activity.id) + assert bookmark.folder_id == nil + + {:ok, bookmark_folder} = BookmarkFolder.create(user.id, "Read later") + + {:ok, bookmark} = Bookmark.create(user.id, activity.id, bookmark_folder.id) + assert bookmark.folder_id == bookmark_folder.id + end end describe "destroy/2" do diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index f95f15ec3..3d8a0fa99 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -1828,6 +1828,60 @@ test "bookmarks" do json_response_and_validate_schema(bookmarks, 200) end + test "bookmark folders" do + %{conn: conn, user: user} = oauth_access(["write:bookmarks", "read:bookmarks"]) + + {:ok, folder} = Pleroma.BookmarkFolder.create(user.id, "folder") + author = insert(:user) + + folder_bookmarks_uri = "/api/v1/bookmarks?folder_id=#{folder.id}" + + {:ok, activity1} = CommonAPI.post(author, %{status: "heweoo?"}) + {:ok, activity2} = CommonAPI.post(author, %{status: "heweoo!"}) + + # Add bookmark with a folder + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses/#{activity1.id}/bookmark", %{folder_id: folder.id}) + + assert json_response_and_validate_schema(response, 200)["bookmarked"] == true + + assert json_response_and_validate_schema(response, 200)["pleroma"]["bookmark_folder"] == + folder.id + + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses/#{activity2.id}/bookmark") + + assert json_response_and_validate_schema(response, 200)["bookmarked"] == true + assert json_response_and_validate_schema(response, 200)["pleroma"]["bookmark_folder"] == nil + + bookmarks = + get(conn, folder_bookmarks_uri) + |> json_response_and_validate_schema(200) + + assert length(bookmarks) == 1 + + # Update folder for existing bookmark + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/statuses/#{activity2.id}/bookmark", %{folder_id: folder.id}) + + assert json_response_and_validate_schema(response, 200)["bookmarked"] == true + + assert json_response_and_validate_schema(response, 200)["pleroma"]["bookmark_folder"] == + folder.id + + bookmarks = + get(conn, folder_bookmarks_uri) + |> json_response_and_validate_schema(200) + + assert length(bookmarks) == 2 + end + describe "conversation muting" do setup do: oauth_access(["write:mutes"]) diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index d82eeb7af..f007bb317 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -341,7 +341,8 @@ test "a note activity" do emoji_reactions: [], parent_visible: false, pinned_at: nil, - quotes_count: 0 + quotes_count: 0, + bookmark_folder: nil } } diff --git a/test/pleroma/web/pleroma_api/controllers/bookmark_folder_controller_test.ex b/test/pleroma/web/pleroma_api/controllers/bookmark_folder_controller_test.ex new file mode 100644 index 000000000..71b850859 --- /dev/null +++ b/test/pleroma/web/pleroma_api/controllers/bookmark_folder_controller_test.ex @@ -0,0 +1,148 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only +defmodule Pleroma.Web.PleromaAPI.BookmarkFolderControllerTest do + use Pleroma.Web.ConnCase + + alias Pleroma.BookmarkFolder + # alias Pleroma.Object + # alias Pleroma.Tests.Helpers + # alias Pleroma.UnstubbedConfigMock, as: ConfigMock + # alias Pleroma.User + # alias Pleroma.Web.ActivityPub.ActivityPub + # alias Pleroma.Web.CommonAPI + + # import Mox + import Pleroma.Factory + + describe "GET /api/v1/pleroma/bookmark_folders" do + setup do: oauth_access(["read:bookmarks"]) + + test "it lists bookmark folders", %{conn: conn, user: user} do + {:ok, folder} = BookmarkFolder.create(user.id, "Bookmark folder") + + folder_id = folder.id + + result = + conn + |> get("/api/v1/pleroma/bookmark_folders") + |> json_response_and_validate_schema(200) + + assert [ + %{ + "id" => ^folder_id, + "name" => "Bookmark folder", + "emoji" => nil, + "source" => %{ + "emoji" => nil + } + } + ] = result + end + end + + describe "POST /api/v1/pleroma/bookmark_folders" do + setup do: oauth_access(["write:bookmarks"]) + + test "it creates a bookmark folder", %{conn: conn} do + result = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/pleroma/bookmark_folders", %{ + name: "Bookmark folder", + emoji: "📁" + }) + |> json_response_and_validate_schema(200) + + assert %{ + "name" => "Bookmark folder", + "emoji" => "📁", + "source" => %{ + "emoji" => "📁" + } + } = result + end + + test "it returns error for invalid emoji", %{conn: conn} do + result = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/pleroma/bookmark_folders", %{ + name: "Bookmark folder", + emoji: "not an emoji" + }) + |> json_response_and_validate_schema(422) + + assert %{"error" => "Invalid emoji"} = result + end + end + + describe "PATCH /api/v1/pleroma/bookmark_folders/:id" do + setup do: oauth_access(["write:bookmarks"]) + + test "it updates a bookmark folder", %{conn: conn, user: user} do + {:ok, folder} = BookmarkFolder.create(user.id, "Bookmark folder") + + result = + conn + |> put_req_header("content-type", "application/json") + |> patch("/api/v1/pleroma/bookmark_folders/#{folder.id}", %{ + name: "bookmark folder" + }) + |> json_response_and_validate_schema(200) + + assert %{ + "name" => "bookmark folder" + } = result + end + + test "it returns error when updating others' folders", %{conn: conn} do + other_user = insert(:user) + + {:ok, folder} = BookmarkFolder.create(other_user.id, "Bookmark folder") + + result = + conn + |> put_req_header("content-type", "application/json") + |> patch("/api/v1/pleroma/bookmark_folders/#{folder.id}", %{ + name: "bookmark folder" + }) + |> json_response_and_validate_schema(403) + + assert %{ + "error" => "Access denied" + } = result + end + end + + describe "DELETE /api/v1/pleroma/bookmark_folders/:id" do + setup do: oauth_access(["write:bookmarks"]) + + test "it deleting a bookmark folder", %{conn: conn, user: user} do + {:ok, folder} = BookmarkFolder.create(user.id, "Bookmark folder") + + assert conn + |> delete("/api/v1/pleroma/bookmark_folders/#{folder.id}") + |> json_response_and_validate_schema(200) + + folders = BookmarkFolder.for_user(user.id) + + assert length(folders) == 0 + end + + test "it returns error when deleting others' folders", %{conn: conn} do + other_user = insert(:user) + + {:ok, folder} = BookmarkFolder.create(other_user.id, "Bookmark folder") + + result = + conn + |> patch("/api/v1/pleroma/bookmark_folders/#{folder.id}") + |> json_response_and_validate_schema(403) + + assert %{ + "error" => "Access denied" + } = result + end + end +end diff --git a/uploads/.gitignore b/uploads/.gitignore deleted file mode 100644 index 523e584a7..000000000 --- a/uploads/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Git will ignore everything in this directory except this file. -* -!.gitignore From 9cfa4e67b11e5a1a7d09330581383dc67fcf6fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Fri, 1 Mar 2024 18:14:31 +0100 Subject: [PATCH 10/42] Add ForceMention mrf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/force-mention-mrf.add | 1 + config/config.exs | 4 + docs/configuration/cheatsheet.md | 7 +- .../web/activity_pub/mrf/force_mention.ex | 59 +++++++++++++++ test/fixtures/minds-invalid-mention-post.json | 1 + .../minds-pleroma-mentioned-post.json | 1 + .../activity_pub/mrf/force_mention_test.exs | 73 +++++++++++++++++++ 7 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 changelog.d/force-mention-mrf.add create mode 100644 lib/pleroma/web/activity_pub/mrf/force_mention.ex create mode 100644 test/fixtures/minds-invalid-mention-post.json create mode 100644 test/fixtures/minds-pleroma-mentioned-post.json create mode 100644 test/pleroma/web/activity_pub/mrf/force_mention_test.exs diff --git a/changelog.d/force-mention-mrf.add b/changelog.d/force-mention-mrf.add new file mode 100644 index 000000000..46ac14244 --- /dev/null +++ b/changelog.d/force-mention-mrf.add @@ -0,0 +1 @@ +Add ForceMention MRF \ No newline at end of file diff --git a/config/config.exs b/config/config.exs index 435387a64..d0496cef8 100644 --- a/config/config.exs +++ b/config/config.exs @@ -415,6 +415,10 @@ config :pleroma, :mrf_inline_quote, template: "RT: {url}" +config :pleroma, :mrf_force_mention, + mention_parent: true, + mention_quoted: true + config :pleroma, :rich_media, enabled: true, ignore_hosts: [], diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 7bba7b26e..89a461b47 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -161,7 +161,8 @@ To add configuration to your config file, you can copy it from the base config. * `Pleroma.Web.ActivityPub.MRF.KeywordPolicy`: Rejects or removes from the federated timeline or replaces keywords. (See [`:mrf_keyword`](#mrf_keyword)). * `Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent`: Forces every mentioned user to be reflected in the post content. * `Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy`: Forces quote post URLs to be reflected in the message content inline. - * `Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy`: Force a Link tag for posts quoting another post. (may break outgoing federation of quote posts with older Pleroma versions) + * `Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy`: Force a Link tag for posts quoting another post. (may break outgoing federation of quote posts with older Pleroma versions). + * `Pleroma.Web.ActivityPub.MRF.ForceMention`: Forces posts to include a mention of the author of parent post or the author of quoted post. * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). * `transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. @@ -272,6 +273,10 @@ Notes: #### :mrf_inline_quote * `template`: The template to append to the post. `{url}` will be replaced with the actual link to the quoted post. Default: `RT: {url}` +#### :mrf_force_mention +* `mention_parent`: Whether to append mention of parent post author +* `mention_quoted`: Whether to append mention of parent quoted author + ### :activitypub * `unfollow_blocked`: Whether blocks result in people getting unfollowed * `outgoing_blocks`: Whether to federate blocks to other instances diff --git a/lib/pleroma/web/activity_pub/mrf/force_mention.ex b/lib/pleroma/web/activity_pub/mrf/force_mention.ex new file mode 100644 index 000000000..3853489fc --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/force_mention.ex @@ -0,0 +1,59 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.ForceMention do + require Pleroma.Constants + + alias Pleroma.Config + alias Pleroma.Object + alias Pleroma.User + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + defp get_author(url) do + with %Object{data: %{"actor" => actor}} <- Object.normalize(url, fetch: false), + %User{ap_id: ap_id, nickname: nickname} <- User.get_cached_by_ap_id(actor) do + %{"type" => "Mention", "href" => ap_id, "name" => "@#{nickname}"} + else + _ -> nil + end + end + + defp prepend_author(tags, _, false), do: tags + + defp prepend_author(tags, nil, _), do: tags + + defp prepend_author(tags, url, _) do + actor = get_author(url) + + if not is_nil(actor) do + [actor | tags] + else + tags + end + end + + @impl true + def filter(%{"type" => "Create", "object" => %{"tag" => tag} = object} = activity) do + tag = + tag + |> prepend_author( + object["inReplyTo"], + Config.get([:mrf_force_mention, :mention_parent, true]) + ) + |> prepend_author( + object["quoteUrl"], + Config.get([:mrf_force_mention, :mention_quoted, true]) + ) + |> Enum.uniq() + + {:ok, put_in(activity["object"]["tag"], tag)} + end + + @impl true + def filter(object), do: {:ok, object} + + @impl true + def describe, do: {:ok, %{}} +end diff --git a/test/fixtures/minds-invalid-mention-post.json b/test/fixtures/minds-invalid-mention-post.json new file mode 100644 index 000000000..ea2cb2739 --- /dev/null +++ b/test/fixtures/minds-invalid-mention-post.json @@ -0,0 +1 @@ +{"@context":"https://www.w3.org/ns/activitystreams","type":"Note","id":"https://www.minds.com/api/activitypub/users/1198929502760083472/entities/urn:comment:1600926863310458883:0:0:0:1600932467852709903","attributedTo":"https://www.minds.com/api/activitypub/users/1198929502760083472","content":"\u003Ca class=\u0022u-url mention\u0022 href=\u0022https://www.minds.com/lain\u0022 target=\u0022_blank\u0022\u003E@lain\u003C/a\u003E corn syrup.","to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://www.minds.com/api/activitypub/users/1198929502760083472/followers","https://lain.com/users/lain"],"tag":[{"type":"Mention","href":"https://www.minds.com/api/activitypub/users/464237775479123984","name":"@lain"}],"url":"https://www.minds.com/newsfeed/1600926863310458883?focusedCommentUrn=urn:comment:1600926863310458883:0:0:0:1600932467852709903","published":"2024-02-04T17:34:03+00:00","inReplyTo":"https://lain.com/objects/36254095-c839-4167-bcc2-b361d5de9198","source":{"content":"@lain corn syrup.","mediaType":"text/plain"}} \ No newline at end of file diff --git a/test/fixtures/minds-pleroma-mentioned-post.json b/test/fixtures/minds-pleroma-mentioned-post.json new file mode 100644 index 000000000..9dfa42c90 --- /dev/null +++ b/test/fixtures/minds-pleroma-mentioned-post.json @@ -0,0 +1 @@ +{"@context":["https://www.w3.org/ns/activitystreams","https://lain.com/schemas/litepub-0.1.jsonld",{"@language":"und"}],"actor":"https://lain.com/users/lain","attachment":[],"attributedTo":"https://lain.com/users/lain","cc":["https://lain.com/users/lain/followers"],"content":"which diet is the best for cognitive dissonance","context":"https://lain.com/contexts/98c8a130-e813-4797-8973-600e80114317","conversation":"https://lain.com/contexts/98c8a130-e813-4797-8973-600e80114317","id":"https://lain.com/objects/36254095-c839-4167-bcc2-b361d5de9198","published":"2024-02-04T17:11:23.931890Z","repliesCount":11,"sensitive":null,"source":{"content":"which diet is the best for cognitive dissonance","mediaType":"text/plain"},"summary":"","tag":[],"to":["https://www.w3.org/ns/activitystreams#Public"],"type":"Note"} \ No newline at end of file diff --git a/test/pleroma/web/activity_pub/mrf/force_mention_test.exs b/test/pleroma/web/activity_pub/mrf/force_mention_test.exs new file mode 100644 index 000000000..b026bab66 --- /dev/null +++ b/test/pleroma/web/activity_pub/mrf/force_mention_test.exs @@ -0,0 +1,73 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionTest do + use Pleroma.DataCase + require Pleroma.Constants + + alias Pleroma.Web.ActivityPub.MRF.ForceMention + + import Pleroma.Factory + + test "adds mention to a reply" do + lain = + insert(:user, ap_id: "https://lain.com/users/lain", nickname: "lain@lain.com", local: false) + + niobleoum = + insert(:user, + ap_id: "https://www.minds.com/api/activitypub/users/1198929502760083472", + nickname: "niobleoum@minds.com", + local: false + ) + + status = File.read!("test/fixtures/minds-pleroma-mentioned-post.json") |> Jason.decode!() + + status_activity = %{ + "type" => "Create", + "actor" => lain.ap_id, + "object" => status + } + + Pleroma.Web.ActivityPub.Transmogrifier.handle_incoming(status_activity) + + reply = File.read!("test/fixtures/minds-invalid-mention-post.json") |> Jason.decode!() + + reply_activity = %{ + "type" => "Create", + "actor" => niobleoum.ap_id, + "object" => reply + } + + {:ok, %{"object" => %{"tag" => tag}}} = ForceMention.filter(reply_activity) + + assert Enum.find(tag, fn %{"href" => href} -> href == lain.ap_id end) + end + + test "adds mention to a quote" do + user1 = insert(:user, ap_id: "https://misskey.io/users/83ssedkv53") + user2 = insert(:user, ap_id: "https://misskey.io/users/7rkrarq81i") + + status = File.read!("test/fixtures/tesla_mock/misskey.io_8vs6wxufd0.json") |> Jason.decode!() + + status_activity = %{ + "type" => "Create", + "actor" => user1.ap_id, + "object" => status + } + + Pleroma.Web.ActivityPub.Transmogrifier.handle_incoming(status_activity) + + quote_post = File.read!("test/fixtures/quote_post/misskey_quote_post.json") |> Jason.decode!() + + quote_activity = %{ + "type" => "Create", + "actor" => user2.ap_id, + "object" => quote_post + } + + {:ok, %{"object" => %{"tag" => tag}}} = ForceMention.filter(quote_activity) + + assert Enum.find(tag, fn %{"href" => href} -> href == user1.ap_id end) + end +end From 6ad4acea50ab0ac682f0f207d60a640a819c7748 Mon Sep 17 00:00:00 2001 From: Kaede Fujisaki Date: Sat, 2 Mar 2024 18:09:08 +0900 Subject: [PATCH 11/42] Consider a case when inbox is nil --- lib/pleroma/web/activity_pub/publisher.ex | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 9e7d00519..6267d0760 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -159,17 +159,20 @@ defp signature_host(%URI{port: port, scheme: scheme, host: host}) do end defp should_federate?(inbox, public) do - if public do - true - else - %{host: host} = URI.parse(inbox) + cond do + inbox == nil -> + false + public -> + true + true -> + %{host: host} = URI.parse(inbox) - quarantined_instances = - Config.get([:instance, :quarantined_instances], []) - |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples() - |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() + quarantined_instances = + Config.get([:instance, :quarantined_instances], []) + |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples() + |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() - !Pleroma.Web.ActivityPub.MRF.subdomain_match?(quarantined_instances, host) + !Pleroma.Web.ActivityPub.MRF.subdomain_match?(quarantined_instances, host) end end From 2e548c31d4756bcd97d52f9b55dd9d700656b832 Mon Sep 17 00:00:00 2001 From: Kaede Fujisaki Date: Sat, 2 Mar 2024 18:13:36 +0900 Subject: [PATCH 12/42] Add changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 063d51d4c..cc994348e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [Unreleased] + +### Fixed +- Federation: Do not tend to send activities when user's inbox is nil. + ## 2.6.2 ### Security From fb1873b6ec05cb7a0f4fb39267d493d2e2d9d0b1 Mon Sep 17 00:00:00 2001 From: Kaede Fujisaki Date: Sat, 2 Mar 2024 18:23:56 +0900 Subject: [PATCH 13/42] add changelog.d --- CHANGELOG.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc994348e..063d51d4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,11 +4,6 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [Unreleased] - -### Fixed -- Federation: Do not tend to send activities when user's inbox is nil. - ## 2.6.2 ### Security From 1311f8314e90753bf8a8f781f81f89317657b727 Mon Sep 17 00:00:00 2001 From: Kaede Fujisaki Date: Sat, 2 Mar 2024 18:24:39 +0900 Subject: [PATCH 14/42] add changelog.d --- changelog.d/issue-3241.fix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/issue-3241.fix diff --git a/changelog.d/issue-3241.fix b/changelog.d/issue-3241.fix new file mode 100644 index 000000000..d46db9805 --- /dev/null +++ b/changelog.d/issue-3241.fix @@ -0,0 +1 @@ +Handle cases when users.inbox is nil. From 0242c1f691fffd6c45a2b33763a4a7c25f02e65f Mon Sep 17 00:00:00 2001 From: Kaede Fujisaki Date: Sat, 2 Mar 2024 18:34:12 +0900 Subject: [PATCH 15/42] fmt --- lib/pleroma/web/activity_pub/publisher.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 6267d0760..f4665275c 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -162,8 +162,10 @@ defp should_federate?(inbox, public) do cond do inbox == nil -> false + public -> true + true -> %{host: host} = URI.parse(inbox) From 1422082bf238b09927e036f4de9deebb164e0653 Mon Sep 17 00:00:00 2001 From: tusooa Date: Thu, 7 Mar 2024 04:43:56 +0000 Subject: [PATCH 16/42] Apply ledyba's suggestion(s) to 1 file(s) --- lib/pleroma/web/activity_pub/publisher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index f4665275c..792c6a1c5 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -160,7 +160,7 @@ defp signature_host(%URI{port: port, scheme: scheme, host: host}) do defp should_federate?(inbox, public) do cond do - inbox == nil -> + is_nil(inbox) -> false public -> From 47ce33a90d09878d5231ce9654853ebcd1b88c08 Mon Sep 17 00:00:00 2001 From: tusooa Date: Thu, 7 Mar 2024 11:55:31 +0000 Subject: [PATCH 17/42] Apply tusooa's suggestion --- lib/pleroma/user.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d81aa5252..ecf537475 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2015,10 +2015,8 @@ defp verify_field_link(field, profile_urls) do when not_empty_string(host) and scheme in ["http", "https"] <- URI.parse(value), {:not_idn, true} <- {:not_idn, to_string(:idna.encode(host)) == host}, - attr <- Pleroma.Web.RelMe.maybe_put_rel_me(value, profile_urls) do - if attr == "me" do - CommonUtils.to_masto_date(NaiveDateTime.utc_now()) - end + "me" <- Pleroma.Web.RelMe.maybe_put_rel_me(value, profile_urls) do + CommonUtils.to_masto_date(NaiveDateTime.utc_now()) else {:verified_at, value} when not_empty_string(value) -> value From 961a5dd4cda7f807455001e1eb936d9c1e14c895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 7 Mar 2024 13:32:50 +0100 Subject: [PATCH 18/42] Add test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- test/pleroma/user_test.exs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index d6ed744f1..a93f81659 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -2953,4 +2953,26 @@ test "it checks fields links for a backlink" do assert is_binary(verified_at) end + + test "updating fields does not invalidate previously validated links" do + user = insert(:user, ap_id: "https://social.example.org/users/lain") + + user + |> User.update_and_set_cache(%{ + raw_fields: [%{"name" => "verified link", "value" => "http://example.com/rel_me/link"}] + }) + + ObanHelpers.perform_all() + + %User{fields: [%{"verified_at" => verified_at}]} = user = User.get_cached_by_id(user.id) + + user + |> User.update_and_set_cache(%{ + raw_fields: [%{"name" => "Verified link", "value" => "http://example.com/rel_me/link"}] + }) + + user = User.get_cached_by_id(user.id) + + assert [%{"verified_at" => ^verified_at}] = user.fields + end end From 54ff7234b9b99ecb5b181d255369f17b5520ace8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 7 Mar 2024 17:38:21 -0500 Subject: [PATCH 19/42] Fix ffmpeg framegrabs with Exile --- changelog.d/framegrabs.fix | 1 + lib/pleroma/helpers/media_helper.ex | 36 ++++++++++++++++------------- 2 files changed, 21 insertions(+), 16 deletions(-) create mode 100644 changelog.d/framegrabs.fix diff --git a/changelog.d/framegrabs.fix b/changelog.d/framegrabs.fix new file mode 100644 index 000000000..dc0466f1b --- /dev/null +++ b/changelog.d/framegrabs.fix @@ -0,0 +1 @@ +Video framegrabs were not working correctly after the change to use Exile to execute ffmpeg diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex index 1a414b37f..7864296fa 100644 --- a/lib/pleroma/helpers/media_helper.ex +++ b/lib/pleroma/helpers/media_helper.ex @@ -40,28 +40,32 @@ def image_resize(url, options) do end # Note: video thumbnail is intentionally not resized (always has original dimensions) + @spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()} def video_framegrab(url) do with executable when is_binary(executable) <- System.find_executable("ffmpeg"), {:ok, env} <- HTTP.get(url, [], pool: :media), {:ok, pid} <- StringIO.open(env.body) do body_stream = IO.binstream(pid, 1) - Exile.stream!( - [ - executable, - "-i", - "pipe:0", - "-vframes", - "1", - "-f", - "mjpeg", - "pipe:1" - ], - input: body_stream, - ignore_epipe: true, - stderr: :disable - ) - |> Enum.into(<<>>) + result = + Exile.stream!( + [ + executable, + "-i", + "pipe:0", + "-vframes", + "1", + "-f", + "mjpeg", + "pipe:1" + ], + input: body_stream, + ignore_epipe: true, + stderr: :disable + ) + |> Enum.into(<<>>) + + {:ok, result} else nil -> {:error, {:ffmpeg, :command_not_found}} {:error, _} = error -> error From bb0b17f4d9a235f79b4b6fabdc5e3dbe472cd259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 7 Mar 2024 20:14:05 +0100 Subject: [PATCH 20/42] Include following/followers in backups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- changelog.d/backups-follows.add | 1 + lib/pleroma/user/backup.ex | 21 ++++++++++++++++++++- test/pleroma/user/backup_test.exs | 13 +++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 changelog.d/backups-follows.add diff --git a/changelog.d/backups-follows.add b/changelog.d/backups-follows.add new file mode 100644 index 000000000..a55c436f6 --- /dev/null +++ b/changelog.d/backups-follows.add @@ -0,0 +1 @@ +Include following/followers in backups \ No newline at end of file diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index b7f00bbf7..65e0baccd 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -196,7 +196,14 @@ defp wait_backup(backup, current_processed, task) do end end - @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] + @files [ + 'actor.json', + 'outbox.json', + 'likes.json', + 'bookmarks.json', + 'followers.json', + 'following.json' + ] @spec export(Pleroma.User.Backup.t(), pid()) :: {:ok, String.t()} | :error def export(%__MODULE__{} = backup, caller_pid) do backup = Repo.preload(backup, :user) @@ -207,6 +214,8 @@ def export(%__MODULE__{} = backup, caller_pid) do :ok <- statuses(dir, backup.user, caller_pid), :ok <- likes(dir, backup.user, caller_pid), :ok <- bookmarks(dir, backup.user, caller_pid), + :ok <- followers(dir, backup.user, caller_pid), + :ok <- following(dir, backup.user, caller_pid), {:ok, zip_path} <- :zip.create(backup.file_name, @files, cwd: dir), {:ok, _} <- File.rm_rf(dir) do {:ok, zip_path} @@ -357,6 +366,16 @@ defp statuses(dir, user, caller_pid) do caller_pid ) end + + defp followers(dir, user, caller_pid) do + User.get_followers_query(user) + |> write(dir, "followers", fn a -> {:ok, a.ap_id} end, caller_pid) + end + + defp following(dir, user, caller_pid) do + User.get_friends_query(user) + |> write(dir, "following", fn a -> {:ok, a.ap_id} end, caller_pid) + end end defmodule Pleroma.User.Backup.ProcessorAPI do diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs index 0ac57e334..5503d15bc 100644 --- a/test/pleroma/user/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -166,6 +166,7 @@ test "it removes outdated backups after creating a fresh one" do test "it creates a zip archive with user data" do user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + %{ap_id: other_ap_id} = other_user = insert(:user) {:ok, %{object: %{data: %{"id" => id1}}} = status1} = CommonAPI.post(user, %{status: "status1"}) @@ -182,6 +183,8 @@ test "it creates a zip archive with user data" do Bookmark.create(user.id, status2.id) Bookmark.create(user.id, status3.id) + CommonAPI.follow(user, other_user) + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() assert {:ok, path} = Backup.export(backup, self()) assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) @@ -261,6 +264,16 @@ test "it creates a zip archive with user data" do "type" => "OrderedCollection" } = Jason.decode!(json) + assert {:ok, {'following.json', json}} = :zip.zip_get('following.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "following.json", + "orderedItems" => [^other_ap_id], + "totalItems" => 1, + "type" => "OrderedCollection" + } = Jason.decode!(json) + :zip.zip_close(zipfile) File.rm!(path) end From 8b651fab1d711a18e8046271a83bcdd756a26670 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 15 Mar 2024 16:20:16 +0100 Subject: [PATCH 21/42] AttachmentValidator: Set "Link" as default type --- .../activity_pub/object_validators/attachment_validator.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index 398020bff..72975f348 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -12,13 +12,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do @primary_key false embedded_schema do field(:id, :string) - field(:type, :string) + field(:type, :string, default: "Link") field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream") field(:name, :string) field(:blurhash, :string) embeds_many :url, UrlObjectValidator, primary_key: false do - field(:type, :string) + field(:type, :string, default: "Link") field(:href, ObjectValidators.Uri) field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream") field(:width, :integer) From 48c22a67dec0b55a1bc48ffc4229352248bceb26 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 15 Mar 2024 16:21:23 +0100 Subject: [PATCH 22/42] QuestionOptionsValidator: set default AS types --- .../object_validators/question_options_validator.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex index 541945fa4..8d7f7b9fa 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex @@ -14,10 +14,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do embeds_one :replies, Replies, primary_key: false do field(:totalItems, :integer) - field(:type, :string) + field(:type, :string, default: "Collection") end - field(:type, :string) + field(:type, :string, default: "Note") end def changeset(struct, data) do From 4ad1d02d7e5f6a3deade296b2baba374898bc033 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 15 Mar 2024 16:25:02 +0100 Subject: [PATCH 23/42] changelog.d/transient-validators-defaults.change: insert --- changelog.d/transient-validators-defaults.change | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/transient-validators-defaults.change diff --git a/changelog.d/transient-validators-defaults.change b/changelog.d/transient-validators-defaults.change new file mode 100644 index 000000000..9da42dca6 --- /dev/null +++ b/changelog.d/transient-validators-defaults.change @@ -0,0 +1 @@ +Set defaults values on transient objects (attachment, poll options) validators \ No newline at end of file From f0cca36e07fd3e0b4677e4a5daf626f3ef74dfec Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 17 Mar 2024 15:39:24 +0400 Subject: [PATCH 24/42] CI: Remove RUM tests. --- .gitlab-ci.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8f1839c42..09ce2efd9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -169,25 +169,6 @@ unit-testing-1.12-erratic: - mix ecto.migrate - mix test --only=erratic -unit-testing-1.12-rum: - extends: - - .build_changes_policy - - .using-ci-base - stage: test - cache: *testing_cache_policy - services: - - name: git.pleroma.social:5050/pleroma/pleroma/postgres-with-rum-13 - alias: postgres - command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] - variables: - <<: *global_variables - RUM_ENABLED: "true" - script: - - mix ecto.create - - mix ecto.migrate - - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" - - mix test --preload-modules - formatting-1.13: extends: .build_changes_policy image: &formatting_elixir elixir:1.13-alpine From caf855cf9cb9a5cadd2519237c9e7916007f4850 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 17 Mar 2024 16:57:45 +0400 Subject: [PATCH 25/42] ActivityPub.Publisher: Don't try federating if a user doesn't have an inbox. --- lib/pleroma/web/activity_pub/publisher.ex | 24 +++++++------------ .../web/activity_pub/publisher_test.exs | 12 ++++++++++ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 792c6a1c5..a42b4844e 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -158,24 +158,18 @@ defp signature_host(%URI{port: port, scheme: scheme, host: host}) do end end - defp should_federate?(inbox, public) do - cond do - is_nil(inbox) -> - false + def should_federate?(nil, _), do: false + def should_federate?(_, true), do: true - public -> - true + def should_federate?(inbox, _) do + %{host: host} = URI.parse(inbox) - true -> - %{host: host} = URI.parse(inbox) + quarantined_instances = + Config.get([:instance, :quarantined_instances], []) + |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples() + |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() - quarantined_instances = - Config.get([:instance, :quarantined_instances], []) - |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples() - |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() - - !Pleroma.Web.ActivityPub.MRF.subdomain_match?(quarantined_instances, host) - end + !Pleroma.Web.ActivityPub.MRF.subdomain_match?(quarantined_instances, host) end @spec recipients(User.t(), Activity.t()) :: [[User.t()]] diff --git a/test/pleroma/web/activity_pub/publisher_test.exs b/test/pleroma/web/activity_pub/publisher_test.exs index 7aa06a5c4..870f1f77a 100644 --- a/test/pleroma/web/activity_pub/publisher_test.exs +++ b/test/pleroma/web/activity_pub/publisher_test.exs @@ -25,6 +25,17 @@ defmodule Pleroma.Web.ActivityPub.PublisherTest do setup_all do: clear_config([:instance, :federating], true) + describe "should_federate?/1" do + test "it returns false when the inbox is nil" do + refute Publisher.should_federate?(nil, false) + refute Publisher.should_federate?(nil, true) + end + + test "it returns true when public is true" do + assert Publisher.should_federate?(false, true) + end + end + describe "gather_webfinger_links/1" do test "it returns links" do user = insert(:user) @@ -205,6 +216,7 @@ test "publish to url with with different ports" do refute called(Instances.set_reachable(inbox)) end + @tag capture_log: true test_with_mock "calls `Instances.set_unreachable` on target inbox on non-2xx HTTP response code", Instances, [:passthrough], From a48f5f860e41ef8a04a0184d2c28cb35a86bbaaa Mon Sep 17 00:00:00 2001 From: Matthieu Rakotojaona Date: Wed, 13 Mar 2024 13:46:37 +0100 Subject: [PATCH 26/42] Notifications: filter on users rather than activities --- changelog.d/notifications.fix | 1 + lib/pleroma/following_relationship.ex | 10 +++++----- lib/pleroma/notification.ex | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) create mode 100644 changelog.d/notifications.fix diff --git a/changelog.d/notifications.fix b/changelog.d/notifications.fix new file mode 100644 index 000000000..a2d2eaea9 --- /dev/null +++ b/changelog.d/notifications.fix @@ -0,0 +1 @@ +Notifications: improve performance by filtering on users table instead of activities table \ No newline at end of file diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index 15664c876..f38c2fce9 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -241,13 +241,13 @@ def find(following_relationships, follower, following) do end @doc """ - For a query with joined activity, - keeps rows where activity's actor is followed by user -or- is NOT domain-blocked by user. + For a query with joined activity's actor, + keeps rows where actor is followed by user -or- is NOT domain-blocked by user. """ def keep_following_or_not_domain_blocked(query, user) do where( query, - [_, activity], + [_, user_actor: user_actor], fragment( # "(actor's domain NOT in domain_blocks) OR (actor IS in followed AP IDs)" """ @@ -255,9 +255,9 @@ def keep_following_or_not_domain_blocked(query, user) do ? = ANY(SELECT ap_id FROM users AS u INNER JOIN following_relationships AS fr ON u.id = fr.following_id WHERE fr.follower_id = ? AND fr.state = ?) """, - activity.actor, + user_actor.ap_id, ^user.domain_blocks, - activity.actor, + user_actor.ap_id, ^User.binary_id(user.id), ^accept_state_code() ) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 368e609d2..710b19866 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -137,7 +137,7 @@ defp exclude_blocked(query, user, opts) do blocked_ap_ids = opts[:blocked_users_ap_ids] || User.blocked_users_ap_ids(user) query - |> where([n, a], a.actor not in ^blocked_ap_ids) + |> where([..., user_actor: user_actor], user_actor.ap_id not in ^blocked_ap_ids) |> FollowingRelationship.keep_following_or_not_domain_blocked(user) end @@ -148,7 +148,7 @@ defp exclude_blockers(query, user) do blocker_ap_ids = User.incoming_relationships_ungrouped_ap_ids(user, [:block]) query - |> where([n, a], a.actor not in ^blocker_ap_ids) + |> where([..., user_actor: user_actor], user_actor.ap_id not in ^blocker_ap_ids) end end @@ -161,7 +161,7 @@ defp exclude_notification_muted(query, user, opts) do opts[:notification_muted_users_ap_ids] || User.notification_muted_users_ap_ids(user) query - |> where([n, a], a.actor not in ^notification_muted_ap_ids) + |> where([..., user_actor: user_actor], user_actor.ap_id not in ^notification_muted_ap_ids) |> join(:left, [n, a], tm in ThreadMute, on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data), as: :thread_mute From 60c4cb21ea329e7256297242b05b7f1e9d53e314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 18 Mar 2024 13:54:15 +0100 Subject: [PATCH 27/42] InstanceView: Update features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 84e9a0d3c..210b46d2c 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -130,7 +130,8 @@ def features do "profile_directory" end, "pleroma:get:main/ostatus", - "pleroma:group_actors" + "pleroma:group_actors", + "pleroma:bookmark_folders" ] |> Enum.filter(& &1) end From 7f97fbc1ae8d38c88d32739e968b4f2572c755a8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 18 Mar 2024 15:36:26 -0400 Subject: [PATCH 28/42] Update minimum Postgres version to 11.0; disable JIT This release is where JIT was introduced and it should be disabled. Pleroma's queries do not benefit from JIT, but it can increase latency of queries. --- .gitlab-ci.yml | 2 +- changelog.d/postgres-jit.change | 1 + config/config.exs | 2 +- docs/installation/debian_based_jp.md | 4 ++-- docs/installation/generic_dependencies.include | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 changelog.d/postgres-jit.change diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 09ce2efd9..79bb2fd7e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -113,7 +113,7 @@ benchmark: variables: MIX_ENV: benchmark services: - - name: postgres:9.6-alpine + - name: postgres:11.22-alpine alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: diff --git a/changelog.d/postgres-jit.change b/changelog.d/postgres-jit.change new file mode 100644 index 000000000..38225b06b --- /dev/null +++ b/changelog.d/postgres-jit.change @@ -0,0 +1 @@ +Disable jit by default for PostgreSQL diff --git a/config/config.exs b/config/config.exs index d0496cef8..32c8509be 100644 --- a/config/config.exs +++ b/config/config.exs @@ -799,7 +799,7 @@ config :pleroma, configurable_from_database: false config :pleroma, Pleroma.Repo, - parameters: [gin_fuzzy_search_limit: "500"], + parameters: [gin_fuzzy_search_limit: "500", jit: "off"], prepare: :unnamed config :pleroma, :connections_pool, diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index 1424ad7f4..b6b5c9c07 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -12,8 +12,8 @@ Note: This article is potentially outdated because at this time we may not have ### 必要なソフトウェア -- PostgreSQL 9.6以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください) -- `postgresql-contrib` 9.6以上 (同上) +- PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください) +- `postgresql-contrib` 11.0以上 (同上) - Elixir 1.8 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください) - `erlang-dev` - `erlang-nox` diff --git a/docs/installation/generic_dependencies.include b/docs/installation/generic_dependencies.include index aebf21e7c..6572716ed 100644 --- a/docs/installation/generic_dependencies.include +++ b/docs/installation/generic_dependencies.include @@ -1,6 +1,6 @@ ## Required dependencies -* PostgreSQL >=9.6 +* PostgreSQL >=11.0 * Elixir >=1.11.0 <1.15 * Erlang OTP >=22.2.0 (supported: <27) * git From 1413d2e517e51a71c71d6365f93ea94fabc61593 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 18 Mar 2024 15:42:15 -0400 Subject: [PATCH 29/42] Remove vestiges of old Postgres support --- lib/pleroma/application.ex | 23 +---------------------- lib/pleroma/search/database_search.ex | 9 +-------- 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index de668052f..f2b234022 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -119,28 +119,7 @@ def start(_type, _args) do max_restarts = Application.get_env(:pleroma, __MODULE__)[:max_restarts] opts = [strategy: :one_for_one, name: Pleroma.Supervisor, max_restarts: max_restarts] - result = Supervisor.start_link(children, opts) - - set_postgres_server_version() - - result - end - - defp set_postgres_server_version do - version = - with %{rows: [[version]]} <- Ecto.Adapters.SQL.query!(Pleroma.Repo, "show server_version"), - {num, _} <- Float.parse(version) do - num - else - e -> - Logger.warning( - "Could not get the postgres version: #{inspect(e)}.\nSetting the default value of 9.6" - ) - - 9.6 - end - - :persistent_term.put({Pleroma.Repo, :postgres_version}, version) + Supervisor.start_link(children, opts) end def load_custom_modules do diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index c6311e0c7..31bfc7e33 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -23,19 +23,12 @@ def search(user, search_query, options \\ []) do offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) - search_function = - if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do - :websearch - else - :plain - end - try do Activity |> Activity.with_preloaded_object() |> Activity.restrict_deactivated_users() |> restrict_public(user) - |> query_with(index_type, search_query, search_function) + |> query_with(index_type, search_query, :websearch) |> maybe_restrict_local(user) |> maybe_restrict_author(author) |> maybe_restrict_blocked(user) From b822a912ad1a597484fab3465e1bd90e756abdc8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 18 Mar 2024 16:15:40 -0400 Subject: [PATCH 30/42] Remove test for postgres < 11 --- test/pleroma/search/database_search_test.exs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/pleroma/search/database_search_test.exs b/test/pleroma/search/database_search_test.exs index 6c47ff425..d8dd09063 100644 --- a/test/pleroma/search/database_search_test.exs +++ b/test/pleroma/search/database_search_test.exs @@ -35,21 +35,6 @@ test "it does not find local-only posts for anonymous users" do assert [] = Search.search(nil, "wednesday") end - test "using plainto_tsquery on postgres < 11" do - old_version = :persistent_term.get({Pleroma.Repo, :postgres_version}) - :persistent_term.put({Pleroma.Repo, :postgres_version}, 10.0) - on_exit(fn -> :persistent_term.put({Pleroma.Repo, :postgres_version}, old_version) end) - - user = insert(:user) - {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) - {:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"}) - - # plainto doesn't understand complex queries - assert [result] = Search.search(nil, "wednesday -dudes") - - assert result.id == post.id - end - test "using websearch_to_tsquery" do user = insert(:user) {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"}) From 357553a64acf8edf180621b3984e596664815248 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 18 Mar 2024 16:24:23 -0400 Subject: [PATCH 31/42] Remove usage of :persistent_term for Postgres version storage, fix test This test should not have been passing. The search result's activity id should not be the same id as the local post. capture_log was not being used. Removed. --- .../controllers/search_controller_test.exs | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs index b05487abe..ad4144da4 100644 --- a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs @@ -322,26 +322,20 @@ test "search", %{conn: conn} do end test "search fetches remote statuses and prefers them over other results", %{conn: conn} do - old_version = :persistent_term.get({Pleroma.Repo, :postgres_version}) - :persistent_term.put({Pleroma.Repo, :postgres_version}, 10.0) - on_exit(fn -> :persistent_term.put({Pleroma.Repo, :postgres_version}, old_version) end) + {:ok, %{id: activity_id}} = + CommonAPI.post(insert(:user), %{ + status: "check out http://mastodon.example.org/@admin/99541947525187367" + }) - capture_log(fn -> - {:ok, %{id: activity_id}} = - CommonAPI.post(insert(:user), %{ - status: "check out http://mastodon.example.org/@admin/99541947525187367" - }) + %{"url" => result_url, "id" => result_id} = + conn + |> get("/api/v1/search?q=http://mastodon.example.org/@admin/99541947525187367") + |> json_response_and_validate_schema(200) + |> Map.get("statuses") + |> List.first() - results = - conn - |> get("/api/v1/search?q=http://mastodon.example.org/@admin/99541947525187367") - |> json_response_and_validate_schema(200) - - assert [ - %{"url" => "http://mastodon.example.org/@admin/99541947525187367"}, - %{"id" => ^activity_id} - ] = results["statuses"] - end) + refute match?(^result_id, activity_id) + assert match?(^result_url, "http://mastodon.example.org/@admin/99541947525187367") end test "search doesn't show statuses that it shouldn't", %{conn: conn} do From 923803a5332f97a7ad4eaa1d77e38b3a7e6e2367 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 19 Mar 2024 10:34:37 +0400 Subject: [PATCH 32/42] Tests: Explicitly set db pool size and max cases to the same value. --- config/test.exs | 2 +- test/test_helper.exs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/test.exs b/config/test.exs index 9d752bdf8..80b01932c 100644 --- a/config/test.exs +++ b/config/test.exs @@ -49,7 +49,7 @@ hostname: System.get_env("DB_HOST") || "localhost", port: System.get_env("DB_PORT") || "5432", pool: Ecto.Adapters.SQL.Sandbox, - pool_size: 50 + pool_size: System.schedulers_online() * 2 config :pleroma, :dangerzone, override_repo_pool_size: true diff --git a/test/test_helper.exs b/test/test_helper.exs index e65f7c1d1..4de50483c 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -4,6 +4,8 @@ Code.put_compiler_option(:warnings_as_errors, true) +ExUnit.configure(max_cases: System.schedulers_online() * 2) + ExUnit.start(exclude: [:federated, :erratic]) if match?({:unix, :darwin}, :os.type()) do From 3cc8414c2ec06e221d8157ecedd55edc6e8def8a Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 19 Mar 2024 10:38:29 +0400 Subject: [PATCH 33/42] Add changelog --- changelog.d/test-improvements.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/test-improvements.skip diff --git a/changelog.d/test-improvements.skip b/changelog.d/test-improvements.skip new file mode 100644 index 000000000..e69de29bb From 665947ab2a55f71d8e0f457c75f48213f868f5df Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 19 Mar 2024 11:03:05 +0400 Subject: [PATCH 34/42] Tests: Reduced the max case number to make tests more stable. --- test/test_helper.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_helper.exs b/test/test_helper.exs index 4de50483c..a117584ae 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -4,7 +4,7 @@ Code.put_compiler_option(:warnings_as_errors, true) -ExUnit.configure(max_cases: System.schedulers_online() * 2) +ExUnit.configure(max_cases: System.schedulers_online()) ExUnit.start(exclude: [:federated, :erratic]) From 9617189e963b89f4353168e20d48ff086a0c53bb Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 19 Mar 2024 13:51:04 +0400 Subject: [PATCH 35/42] Tests: Actually run the bookmark folder tests. --- ...der_controller_test.ex => bookmark_folder_controller_test.exs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/pleroma/web/pleroma_api/controllers/{bookmark_folder_controller_test.ex => bookmark_folder_controller_test.exs} (100%) diff --git a/test/pleroma/web/pleroma_api/controllers/bookmark_folder_controller_test.ex b/test/pleroma/web/pleroma_api/controllers/bookmark_folder_controller_test.exs similarity index 100% rename from test/pleroma/web/pleroma_api/controllers/bookmark_folder_controller_test.ex rename to test/pleroma/web/pleroma_api/controllers/bookmark_folder_controller_test.exs From afae3a94a4d91627ec204f9e1909d0400ec6497f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 19 Mar 2024 13:54:35 +0400 Subject: [PATCH 36/42] CI: Move changelog check to later in the pipeline No reason to not run tests. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79bb2fd7e..dab52e4c6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,10 +26,10 @@ cache: &global_cache_policy - _build stages: - - check-changelog - build - lint - test + - check-changelog - benchmark - deploy - release From 040a980277947ee2eb7ae8b7432a125536dc2f06 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 19 Mar 2024 14:59:25 +0400 Subject: [PATCH 37/42] Add changelog --- changelog.d/fix-bookmark-folder-tests.skip | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 changelog.d/fix-bookmark-folder-tests.skip diff --git a/changelog.d/fix-bookmark-folder-tests.skip b/changelog.d/fix-bookmark-folder-tests.skip new file mode 100644 index 000000000..e69de29bb From 8a14fdbe479ca0ffb3419a83cf3c637b72dd54e0 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 19 Mar 2024 12:03:43 +0000 Subject: [PATCH 38/42] Update transient-validators-defaults.change --- changelog.d/transient-validators-defaults.change | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/transient-validators-defaults.change b/changelog.d/transient-validators-defaults.change index 9da42dca6..225cf4d0c 100644 --- a/changelog.d/transient-validators-defaults.change +++ b/changelog.d/transient-validators-defaults.change @@ -1 +1 @@ -Set defaults values on transient objects (attachment, poll options) validators \ No newline at end of file +Set default values on validators for transient objects (attachment, poll options) From 7dfd148ff8a4f2d349d6d6f92d788effdaab36f3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 9 Dec 2023 18:32:26 -0500 Subject: [PATCH 39/42] Logger metadata for inbound federation requests --- config/config.exs | 4 ++-- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 32c8509be..537517688 100644 --- a/config/config.exs +++ b/config/config.exs @@ -131,13 +131,13 @@ config :logger, :console, level: :debug, format: "\n$time $metadata[$level] $message\n", - metadata: [:request_id] + metadata: [:actor, :request_id, :type] config :logger, :ex_syslogger, level: :debug, ident: "pleroma", format: "$metadata[$level] $message", - metadata: [:request_id] + metadata: [:actor, :request_id, :type] config :mime, :types, %{ "application/xml" => ["xml"], diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index e38a94966..d2b2cae0b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -52,6 +52,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do when action in [:activity, :object] ) + plug(:log_inbox_metadata when action in [:inbox]) plug(:set_requester_reachable when action in [:inbox]) plug(:relay_active? when action in [:relay]) @@ -521,6 +522,13 @@ defp set_requester_reachable(%Plug.Conn{} = conn, _) do conn end + defp log_inbox_metadata(conn = %{params: %{"actor" => actor, "type" => type}}, _) do + Logger.metadata(actor: actor, type: type) + conn + end + + defp log_inbox_metadata(conn, _), do: conn + def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do with {:ok, object} <- ActivityPub.upload( From 40823462e7779fb79a4fcd458daa5e7095a6030b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 17 Dec 2023 18:20:22 -0500 Subject: [PATCH 40/42] Logger metadata for request path and authenticated user --- config/config.exs | 4 ++-- lib/pleroma/web/endpoint.ex | 2 ++ lib/pleroma/web/plugs/logger_metadata_path.ex | 12 ++++++++++++ lib/pleroma/web/plugs/logger_metadata_user.ex | 18 ++++++++++++++++++ lib/pleroma/web/router.ex | 8 ++++++++ 5 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/web/plugs/logger_metadata_path.ex create mode 100644 lib/pleroma/web/plugs/logger_metadata_user.ex diff --git a/config/config.exs b/config/config.exs index 537517688..83e7a33e3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -131,13 +131,13 @@ config :logger, :console, level: :debug, format: "\n$time $metadata[$level] $message\n", - metadata: [:actor, :request_id, :type] + metadata: [:actor, :path, :request_id, :type, :user] config :logger, :ex_syslogger, level: :debug, ident: "pleroma", format: "$metadata[$level] $message", - metadata: [:actor, :request_id, :type] + metadata: [:actor, :path, :request_id, :type, :user] config :mime, :types, %{ "application/xml" => ["xml"], diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 2e2104904..fef907ace 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -38,6 +38,8 @@ defmodule Pleroma.Web.Endpoint do plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint]) + plug(Pleroma.Web.Plugs.LoggerMetadataPath) + plug(Pleroma.Web.Plugs.SetLocalePlug) plug(CORSPlug) plug(Pleroma.Web.Plugs.HTTPSecurityPlug) diff --git a/lib/pleroma/web/plugs/logger_metadata_path.ex b/lib/pleroma/web/plugs/logger_metadata_path.ex new file mode 100644 index 000000000..a5553cfc8 --- /dev/null +++ b/lib/pleroma/web/plugs/logger_metadata_path.ex @@ -0,0 +1,12 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.LoggerMetadataPath do + def init(opts), do: opts + + def call(conn, _) do + Logger.metadata(path: conn.request_path) + conn + end +end diff --git a/lib/pleroma/web/plugs/logger_metadata_user.ex b/lib/pleroma/web/plugs/logger_metadata_user.ex new file mode 100644 index 000000000..6a5c0041d --- /dev/null +++ b/lib/pleroma/web/plugs/logger_metadata_user.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.LoggerMetadataUser do + alias Pleroma.User + + def init(opts), do: opts + + def call(%{assigns: %{user: user = %User{}}} = conn, _) do + Logger.metadata(user: user.nickname) + conn + end + + def call(conn, _) do + conn + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 4fe0cb02f..f0414cc35 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -29,6 +29,7 @@ defmodule Pleroma.Web.Router do pipeline :browser do plug(:accepts, ["html"]) plug(:fetch_session) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :oauth do @@ -67,12 +68,14 @@ defmodule Pleroma.Web.Router do plug(:fetch_session) plug(:authenticate) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :no_auth_or_privacy_expectations_api do plug(:base_api) plug(:after_auth) plug(Pleroma.Web.Plugs.IdempotencyPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end # Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported) @@ -83,12 +86,14 @@ defmodule Pleroma.Web.Router do pipeline :api do plug(:expect_public_instance_or_user_authentication) plug(:no_auth_or_privacy_expectations_api) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :authenticated_api do plug(:expect_user_authentication) plug(:no_auth_or_privacy_expectations_api) plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :admin_api do @@ -99,6 +104,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Web.Plugs.UserIsStaffPlug) plug(Pleroma.Web.Plugs.IdempotencyPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :require_admin do @@ -179,6 +185,7 @@ defmodule Pleroma.Web.Router do plug(:browser) plug(:authenticate) plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :well_known do @@ -193,6 +200,7 @@ defmodule Pleroma.Web.Router do pipeline :pleroma_api do plug(:accepts, ["html", "json"]) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :mailbox_preview do From 99cee755d8798c0743b96fb11e55f283f0195b85 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 17 Dec 2023 18:20:34 -0500 Subject: [PATCH 41/42] Show Logger metadata in dev --- config/dev.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/dev.exs b/config/dev.exs index fe8de5045..f23719fe3 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -35,8 +35,8 @@ # configured to run both http and https servers on # different ports. -# Do not include metadata nor timestamps in development logs -config :logger, :console, format: "[$level] $message\n" +# Do not include timestamps in development logs +config :logger, :console, format: "$metadata[$level] $message\n" # Set a higher stacktrace during development. Avoid configuring such # in production as building large stacktraces may be expensive. From 462d5aa5cbe3194bad56a7c973e9b200742ef6ca Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 19 Mar 2024 20:53:40 -0400 Subject: [PATCH 42/42] logger: remove request_id metadata which is not useful --- config/config.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 83e7a33e3..383eb3768 100644 --- a/config/config.exs +++ b/config/config.exs @@ -131,13 +131,13 @@ config :logger, :console, level: :debug, format: "\n$time $metadata[$level] $message\n", - metadata: [:actor, :path, :request_id, :type, :user] + metadata: [:actor, :path, :type, :user] config :logger, :ex_syslogger, level: :debug, ident: "pleroma", format: "$metadata[$level] $message", - metadata: [:actor, :path, :request_id, :type, :user] + metadata: [:actor, :path, :type, :user] config :mime, :types, %{ "application/xml" => ["xml"],