Merge branch 'develop' into feature/reports-groups-and-multiple-state-update

.gitlab-ci.yml

@@ -29,22 +29,6 @@ build:
   - mix deps.get
   - mix compile --force
 
-docs-build:
-  stage: build
-  only:
-  - master@pleroma/pleroma
-  - develop@pleroma/pleroma
-  variables:
-    MIX_ENV: dev
-    PLEROMA_BUILD_ENV: prod
-  script:
-    - mix deps.get
-    - mix compile
-    - mix docs
-  artifacts:
-    paths:
-      - priv/static/doc
-
 benchmark:
   stage: benchmark
   variables:

CHANGELOG.md

@@ -16,9 +16,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Add `rel="ugc"` to all links in statuses, to prevent SEO spam
 - Extract RSS functionality from OStatus
 - MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities
+- OStatus: Extract RSS functionality
+- Deprecated `User.Info` embedded schema (fields moved to `User`)
+- Store status data inside Flag activity
 <details>
   <summary>API Changes</summary>
 
+- **Breaking** Admin API: `PATCH /api/pleroma/admin/users/:nickname/force_password_reset` is now `PATCH /api/pleroma/admin/users/force_password_reset` (accepts `nicknames` array in the request body)
 - **Breaking:** Admin API: Return link alongside with token on password reset
 - **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string.
 - Admin API: Return `total` when querying for reports
@@ -35,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Authentication: Added rate limit for password-authorized actions / login existence checks
 - Mix task to re-count statuses for all users (`mix pleroma.count_statuses`)
 - Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache).
+- Admin API: Add ability to fetch reports, grouped by status `GET /api/pleroma/admin/grouped_reports`
 <details>
   <summary>API Changes</summary>
 
@@ -53,23 +58,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Admin API: Add `GET /api/pleroma/admin/relay` endpoint - lists all followed relays
 - Pleroma API: `POST /api/v1/pleroma/conversations/read` to mark all conversations as read
 - Mastodon API: Add `/api/v1/markers` for managing timeline read markers
-- Admin API: Add ability to fetch reports, grouped by status `GET /api/pleroma/admin/grouped_reports`
-
-### Changed
-- **Breaking:** Elixir >=1.8 is now required (was >= 1.7)
-- **Breaking:** Admin API: Return link alongside with token on password reset
-- **Breaking:** Admin API: Changing report state now uses `PATCH` (it was `PUT` before) and allows updating multiple reports at once (API changed)
-- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings)
-- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler
-- Admin API: Return `total` when querying for reports
-- Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`)
-- Admin API: Return link alongside with token on password reset
-- MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities
-- OStatus: Extract RSS functionality
-- Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`)
-- Mastodon API: Mark the direct conversation as read for the author when they send a new direct message
-- Deprecated `User.Info` embedded schema (fields moved to `User`)
-- Store status data inside Flag activity
+- Mastodon API: Add the `recipients` parameter to `GET /api/v1/conversations`
 </details>
 
 ### Fixed

config/config.exs

@@ -603,7 +603,7 @@
   activity_pub: nil,
   activity_pub_question: 30_000
 
-config :swarm, node_blacklist: [~r/myhtmlex_.*$/]
+config :swarm, node_blacklist: [~r/myhtml_.*$/]
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

docs/API/admin_api.md

@@ -381,11 +381,12 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 }
 ```
 
-## `PATCH /api/pleroma/admin/users/:nickname/force_password_reset`
+## `PATCH /api/pleroma/admin/users/force_password_reset`
 
 ### Force passord reset for a user with a given nickname
 
-- Params: none
+- Params:
+  - `nicknames`
 - Response: none (code `204`)
 
 ## `GET /api/pleroma/admin/reports`

docs/API/differences_in_mastoapi_responses.md

@@ -72,6 +72,12 @@ Has an additional field under the `pleroma` object:
 
 - `recipients`: The list of the recipients of this Conversation. These will be addressed when replying to this conversation.
 
+## GET `/api/v1/conversations`
+
+Accepts additional parameters:
+
+- `recipients`: Only return conversations with the given recipients (a list of user ids). Usage example: `GET /api/v1/conversations?recipients[]=1&recipients[]=2`
+
 ## Account Search
 
 Behavior has changed:

lib/pleroma/conversation/participation.ex

@@ -122,9 +122,37 @@ def for_user(user, params \\ %{}) do
       order_by: [desc: p.updated_at],
       preload: [conversation: [:users]]
     )
+    |> restrict_recipients(user, params)
     |> Pleroma.Pagination.fetch_paginated(params)
   end
 
+  def restrict_recipients(query, user, %{"recipients" => user_ids}) do
+    user_ids =
+      [user.id | user_ids]
+      |> Enum.uniq()
+      |> Enum.reduce([], fn user_id, acc ->
+        case FlakeId.Ecto.CompatType.dump(user_id) do
+          {:ok, user_id} -> [user_id | acc]
+          _ -> acc
+        end
+      end)
+
+    conversation_subquery =
+      __MODULE__
+      |> group_by([p], p.conversation_id)
+      |> having(
+        [p],
+        count(p.user_id) == ^length(user_ids) and
+          fragment("array_agg(?) @> ?", p.user_id, ^user_ids)
+      )
+      |> select([p], %{id: p.conversation_id})
+
+    query
+    |> join(:inner, [p], c in subquery(conversation_subquery), on: p.conversation_id == c.id)
+  end
+
+  def restrict_recipients(query, _, _), do: query
+
   def for_user_and_conversation(user, conversation) do
     from(p in __MODULE__,
       where: p.user_id == ^user.id,

lib/pleroma/moderation_log.ex

@@ -616,6 +616,17 @@ def get_log_entry_message(%ModerationLog{
     "@#{actor_nickname} deleted status ##{subject_id}"
   end
 
+  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "force_password_reset",
+          "subject" => subjects
+        }
+      }) do
+    "@#{actor_nickname} force password reset for users: #{users_to_nicknames_string(subjects)}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")

lib/pleroma/object/fetcher.ex

@@ -38,7 +38,8 @@ defp reinject_object(struct, data) do
          data <- maybe_reinject_internal_fields(data, struct),
          changeset <- Object.change(struct, %{data: data}),
          changeset <- touch_changeset(changeset),
-         {:ok, object} <- Repo.insert_or_update(changeset) do
+         {:ok, object} <- Repo.insert_or_update(changeset),
+         {:ok, object} <- Object.set_cache(object) do
       {:ok, object}
     else
       e ->
@@ -53,7 +54,7 @@ def refetch_object(%Object{data: %{"id" => id}} = object) do
          {:ok, object} <- reinject_object(object, data) do
       {:ok, object}
     else
-      {:local, true} -> object
+      {:local, true} -> {:ok, object}
       e -> {:error, e}
     end
   end

lib/pleroma/plugs/trailing_format_plug.ex

@@ -24,7 +24,8 @@ defmodule Pleroma.Plugs.TrailingFormatPlug do
     "/api/help",
     "/api/externalprofile",
     "/notice",
-    "/api/pleroma/emoji"
+    "/api/pleroma/emoji",
+    "/api/oauth_tokens"
   ]
 
   def init(opts) do

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -608,10 +608,16 @@ def get_password_reset(conn, %{"nickname" => nickname}) do
   end
 
   @doc "Force password reset for a given user"
-  def force_password_reset(conn, %{"nickname" => nickname}) do
-    (%User{local: true} = user) = User.get_cached_by_nickname(nickname)
+  def force_password_reset(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = nicknames |> Enum.map(&User.get_cached_by_nickname/1)
 
-    User.force_password_reset_async(user)
+    Enum.map(users, &User.force_password_reset_async/1)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "force_password_reset"
+    })
 
     json_response(conn, :no_content, "")
   end

lib/pleroma/web/endpoint.ex

@@ -12,7 +12,7 @@ defmodule Pleroma.Web.Endpoint do
   plug(Pleroma.Plugs.HTTPSecurityPlug)
   plug(Pleroma.Plugs.UploadedMedia)
 
-  @static_cache_control "public, no-cache"
+  @static_cache_control "public max-age=86400 must-revalidate"
 
   # InstanceStatic needs to be before Plug.Static to be able to override shipped-static files
   # If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well

lib/pleroma/web/router.ex

@@ -171,7 +171,7 @@ defmodule Pleroma.Web.Router do
     post("/users/email_invite", AdminAPIController, :email_invite)
 
     get("/users/:nickname/password_reset", AdminAPIController, :get_password_reset)
-    patch("/users/:nickname/force_password_reset", AdminAPIController, :force_password_reset)
+    patch("/users/force_password_reset", AdminAPIController, :force_password_reset)
 
     get("/users", AdminAPIController, :list_users)
     get("/users/:nickname", AdminAPIController, :user_show)

mix.exs

@@ -63,7 +63,7 @@ def copy_nginx_config(%{path: target_path} = release) do
   def application do
     [
       mod: {Pleroma.Application, []},
-      extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :myhtmlex, :swarm],
+      extra_applications: [:logger, :runtime_tools, :comeonin, :quack, :fast_sanitize, :swarm],
       included_applications: [:ex_syslogger]
     ]
   end
@@ -108,10 +108,8 @@ defp deps do
       {:comeonin, "~> 4.1.1"},
       {:pbkdf2_elixir, "~> 0.12.3"},
       {:trailing_format_plug, "~> 0.0.7"},
-      {:fast_sanitize,
-       git: "https://git.pleroma.social/pleroma/fast_sanitize.git",
-       ref: "1af67547a02a104e26c99d03012383e8643bc4c2"},
-      {:html_entities, "~> 0.4"},
+      {:fast_sanitize, "~> 0.1"},
+      {:html_entities, "~> 0.5", override: true},
       {:phoenix_html, "~> 2.10"},
       {:calendar, "~> 0.17.4"},
       {:cachex, "~> 3.0.2"},

mix.lock

@@ -36,7 +36,8 @@
   "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"},
   "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]},
   "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"},
-  "fast_sanitize": {:git, "https://git.pleroma.social/pleroma/fast_sanitize.git", "1af67547a02a104e26c99d03012383e8643bc4c2", [ref: "1af67547a02a104e26c99d03012383e8643bc4c2"]},
+  "fast_html": {:hex, :fast_html, "0.99.0", "ea740358b15c7da6085b421b775f22d4f2c6928a28a15ebb5ad4e8a2ce00350b", [:make, :mix], [], "hexpm"},
+  "fast_sanitize": {:hex, :fast_sanitize, "0.1.1", "a403c3c09369e23423d3e6beb14068ad07be82741d10b293c71abac445dcc636", [:mix], [{:fast_html, "~> 0.99", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
   "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"},
   "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"},
   "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm"},
@@ -44,7 +45,7 @@
   "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"},
   "gettext": {:hex, :gettext, "0.17.1", "8baab33482df4907b3eae22f719da492cee3981a26e649b9c2be1c0192616962", [:mix], [], "hexpm"},
   "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"},
-  "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"},
+  "html_entities": {:hex, :html_entities, "0.5.0", "40f5c5b9cbe23073b48a4e69c67b6c11974f623a76165e2b92d098c0e88ccb1d", [:mix], [], "hexpm"},
   "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"},
   "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "293d77bb6f4a67ac8bde1428735c3b42f22cbb30", [ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"]},
   "httpoison": {:hex, :httpoison, "1.6.1", "2ce5bf6e535cd0ab02e905ba8c276580bab80052c5c549f53ddea52d72e81f33", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},

test/object_test.exs

@@ -124,6 +124,8 @@ test "refetches if the time since the last refetch is greater than the interval"
       %Object{} =
         object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
 
+      Object.set_cache(object)
+
       assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
       assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
 
@@ -133,6 +135,8 @@ test "refetches if the time since the last refetch is greater than the interval"
       })
 
       updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
+      object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
+      assert updated_object == object_in_cache
       assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8
       assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3
     end
@@ -141,6 +145,8 @@ test "returns the old object if refetch fails", %{mock_modified: mock_modified}
       %Object{} =
         object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
 
+      Object.set_cache(object)
+
       assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
       assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
 
@@ -148,6 +154,8 @@ test "returns the old object if refetch fails", %{mock_modified: mock_modified}
                mock_modified.(%Tesla.Env{status: 404, body: ""})
 
                updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
+               object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
+               assert updated_object == object_in_cache
                assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4
                assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0
              end) =~
@@ -160,6 +168,8 @@ test "does not refetch if the time since the last refetch is greater than the in
       %Object{} =
         object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
 
+      Object.set_cache(object)
+
       assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
       assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
 
@@ -169,6 +179,8 @@ test "does not refetch if the time since the last refetch is greater than the in
       })
 
       updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: 100)
+      object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
+      assert updated_object == object_in_cache
       assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4
       assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0
     end
@@ -177,6 +189,8 @@ test "preserves internal fields on refetch", %{mock_modified: mock_modified} do
       %Object{} =
         object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
 
+      Object.set_cache(object)
+
       assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
       assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
 
@@ -192,6 +206,8 @@ test "preserves internal fields on refetch", %{mock_modified: mock_modified} do
       })
 
       updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
+      object_in_cache = Object.get_cached_by_ap_id(object.data["id"])
+      assert updated_object == object_in_cache
       assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8
       assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3
 

test/plugs/cache_control_test.exs

@@ -9,7 +9,7 @@ defmodule Pleroma.Web.CacheControlTest do
   test "Verify Cache-Control header on static assets", %{conn: conn} do
     conn = get(conn, "/index.html")
 
-    assert Conn.get_resp_header(conn, "cache-control") == ["public, no-cache"]
+    assert Conn.get_resp_header(conn, "cache-control") == ["public max-age=86400 must-revalidate"]
   end
 
   test "Verify Cache-Control header on the API", %{conn: conn} do

test/web/admin_api/admin_api_controller_test.exs

@@ -2760,7 +2760,7 @@ test "sets password_reset_pending to true", %{admin: admin, user: user} do
       conn =
         build_conn()
         |> assign(:user, admin)
-        |> patch("/api/pleroma/admin/users/#{user.nickname}/force_password_reset")
+        |> patch("/api/pleroma/admin/users/force_password_reset", %{nicknames: [user.nickname]})
 
       assert json_response(conn, 204) == ""
 

test/web/mastodon_api/controllers/conversation_controller_test.exs

@@ -59,6 +59,59 @@ test "returns a list of conversations", %{conn: conn} do
     assert User.get_cached_by_id(user_one.id).unread_conversation_count == 0
   end
 
+  test "filters conversations by recipients", %{conn: conn} do
+    user_one = insert(:user)
+    user_two = insert(:user)
+    user_three = insert(:user)
+
+    {:ok, direct1} =
+      CommonAPI.post(user_one, %{
+        "status" => "Hi @#{user_two.nickname}!",
+        "visibility" => "direct"
+      })
+
+    {:ok, _direct2} =
+      CommonAPI.post(user_one, %{
+        "status" => "Hi @#{user_three.nickname}!",
+        "visibility" => "direct"
+      })
+
+    {:ok, direct3} =
+      CommonAPI.post(user_one, %{
+        "status" => "Hi @#{user_two.nickname}, @#{user_three.nickname}!",
+        "visibility" => "direct"
+      })
+
+    {:ok, _direct4} =
+      CommonAPI.post(user_two, %{
+        "status" => "Hi @#{user_three.nickname}!",
+        "visibility" => "direct"
+      })
+
+    {:ok, direct5} =
+      CommonAPI.post(user_two, %{
+        "status" => "Hi @#{user_one.nickname}!",
+        "visibility" => "direct"
+      })
+
+    [conversation1, conversation2] =
+      conn
+      |> assign(:user, user_one)
+      |> get("/api/v1/conversations", %{"recipients" => [user_two.id]})
+      |> json_response(200)
+
+    assert conversation1["last_status"]["id"] == direct5.id
+    assert conversation2["last_status"]["id"] == direct1.id
+
+    [conversation1] =
+      conn
+      |> assign(:user, user_one)
+      |> get("/api/v1/conversations", %{"recipients" => [user_two.id, user_three.id]})
+      |> json_response(200)
+
+    assert conversation1["last_status"]["id"] == direct3.id
+  end
+
   test "updates the last_status on reply", %{conn: conn} do
     user_one = insert(:user)
     user_two = insert(:user)