From afa8b469ed0a71247f27efec08d6eeac24b6674f Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Wed, 1 Jul 2020 21:12:59 -0500
Subject: [PATCH 001/104] Allow restricting public timeline by instance

---
 lib/pleroma/web/activity_pub/activity_pub.ex         | 12 ++----------
 .../web/api_spec/operations/timeline_operation.ex    | 10 ++++++++++
 .../mastodon_api/controllers/timeline_controller.ex  |  1 +
 .../controllers/timeline_controller_test.exs         | 12 ++++++++++++
 4 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index bc7b5d95a..9ce2b04dd 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -927,16 +927,8 @@ defp restrict_muted_reblogs(query, %{muting_user: %User{} = user} = opts) do
 
   defp restrict_muted_reblogs(query, _), do: query
 
-  defp restrict_instance(query, %{instance: instance}) do
-    users =
-      from(
-        u in User,
-        select: u.ap_id,
-        where: fragment("? LIKE ?", u.nickname, ^"%@#{instance}")
-      )
-      |> Repo.all()
-
-    from(activity in query, where: activity.actor in ^users)
+  defp restrict_instance(query, %{instance: instance}) when is_binary(instance) do
+    from(activity in query, where: ilike(activity.actor, ^"%://#{instance}/%"))
   end
 
   defp restrict_instance(query, _), do: query
diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex
index 8e19bace7..83cdbad69 100644
--- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex
@@ -59,6 +59,7 @@ def public_operation do
       security: [%{"oAuth" => ["read:statuses"]}],
       parameters: [
         local_param(),
+        instance_param(),
         only_media_param(),
         with_muted_param(),
         exclude_visibilities_param(),
@@ -158,6 +159,15 @@ defp local_param do
     )
   end
 
+  defp instance_param do
+    Operation.parameter(
+      :instance,
+      :query,
+      %Schema{type: :string},
+      "Show only statuses from the given domain"
+    )
+  end
+
   defp with_muted_param do
     Operation.parameter(:with_muted, :query, BooleanLike, "Includeactivities by muted users")
   end
diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
index ab7b1d6aa..7dccc0005 100644
--- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
@@ -110,6 +110,7 @@ def public(%{assigns: %{user: user}} = conn, params) do
         |> Map.put(:blocking_user, user)
         |> Map.put(:muting_user, user)
         |> Map.put(:reply_filtering_user, user)
+        |> Map.put(:instance, params[:instance])
         |> ActivityPub.fetch_public_activities()
 
       conn
diff --git a/test/web/mastodon_api/controllers/timeline_controller_test.exs b/test/web/mastodon_api/controllers/timeline_controller_test.exs
index 50e0d783d..6acd512c7 100644
--- a/test/web/mastodon_api/controllers/timeline_controller_test.exs
+++ b/test/web/mastodon_api/controllers/timeline_controller_test.exs
@@ -140,6 +140,18 @@ test "doesn't return replies if follow is posting with users from blocked domain
       activities = json_response_and_validate_schema(res_conn, 200)
       [%{"id" => ^activity_id}] = activities
     end
+
+    test "can be filtered by instance", %{conn: conn} do
+      user = insert(:user, ap_id: "https://lain.com/users/lain")
+      insert(:note_activity, local: false)
+      insert(:note_activity, local: false)
+
+      {:ok, _} = CommonAPI.post(user, %{status: "test"})
+
+      conn = get(conn, "/api/v1/timelines/public?instance=lain.com")
+
+      assert length(json_response_and_validate_schema(conn, :ok)) == 1
+    end
   end
 
   defp local_and_remote_activities do

From ad9c925efb77287316f5dbac26f6a1b16662910a Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Wed, 5 Aug 2020 13:08:13 -0500
Subject: [PATCH 002/104] Speed up instance timeline query

---
 lib/pleroma/web/activity_pub/activity_pub.ex                 | 5 ++++-
 test/web/admin_api/controllers/admin_api_controller_test.exs | 4 ++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 9ce2b04dd..76fc9c3ee 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -928,7 +928,10 @@ defp restrict_muted_reblogs(query, %{muting_user: %User{} = user} = opts) do
   defp restrict_muted_reblogs(query, _), do: query
 
   defp restrict_instance(query, %{instance: instance}) when is_binary(instance) do
-    from(activity in query, where: ilike(activity.actor, ^"%://#{instance}/%"))
+    from(
+      activity in query,
+      where: fragment("split_part(actor::text, '/'::text, 3) = ?", ^instance)
+    )
   end
 
   defp restrict_instance(query, _), do: query
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index da91cd552..26194eb81 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -1647,8 +1647,8 @@ test "sets password_reset_pending to true", %{conn: conn} do
 
   describe "instances" do
     test "GET /instances/:instance/statuses", %{conn: conn} do
-      user = insert(:user, local: false, nickname: "archaeme@archae.me")
-      user2 = insert(:user, local: false, nickname: "test@test.com")
+      user = insert(:user, local: false, ap_id: "https://archae.me/users/archaeme")
+      user2 = insert(:user, local: false, ap_id: "https://test.com/users/test")
       insert_pair(:note_activity, user: user)
       activity = insert(:note_activity, user: user2)
 

From 24ce9c011caf7401fb261c7df4196b2ef9ba3d90 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Wed, 5 Aug 2020 19:33:51 +0000
Subject: [PATCH 003/104] Apply 1 suggestion(s) to 1 file(s)

---
 lib/pleroma/web/api_spec/operations/timeline_operation.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex
index 83cdbad69..95720df9f 100644
--- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex
@@ -169,7 +169,7 @@ defp instance_param do
   end
 
   defp with_muted_param do
-    Operation.parameter(:with_muted, :query, BooleanLike, "Includeactivities by muted users")
+    Operation.parameter(:with_muted, :query, BooleanLike, "Include activities by muted users")
   end
 
   defp exclude_visibilities_param do

From 0d5088c2b83fafd9d8da1f1b04936f831ac5ee87 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Tue, 1 Sep 2020 09:37:08 +0300
Subject: [PATCH 004/104] remove `unread_conversation_count` from User

---
 lib/pleroma/conversation.ex                   |  6 +--
 lib/pleroma/conversation/participation.ex     | 27 +++++-------
 lib/pleroma/user.ex                           | 42 -------------------
 .../web/mastodon_api/views/account_view.ex    |  2 +-
 ...ve_unread_conversation_count_from_user.exs | 38 +++++++++++++++++
 ...ad_index_to_conversation_participation.exs | 12 ++++++
 test/conversation/participation_test.exs      | 32 +++++++-------
 .../conversation_controller_test.exs          | 23 +++++-----
 .../conversation_controller_test.exs          |  4 +-
 9 files changed, 92 insertions(+), 94 deletions(-)
 create mode 100644 priv/repo/migrations/20200831114918_remove_unread_conversation_count_from_user.exs
 create mode 100644 priv/repo/migrations/20200831115854_add_unread_index_to_conversation_participation.exs

diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex
index e76eb0087..77933f0be 100644
--- a/lib/pleroma/conversation.ex
+++ b/lib/pleroma/conversation.ex
@@ -43,7 +43,7 @@ def get_for_ap_id(ap_id) do
   def maybe_create_recipientships(participation, activity) do
     participation = Repo.preload(participation, :recipients)
 
-    if participation.recipients |> Enum.empty?() do
+    if Enum.empty?(participation.recipients) do
       recipients = User.get_all_by_ap_id(activity.recipients)
       RecipientShip.create(recipients, participation)
     end
@@ -69,10 +69,6 @@ def create_or_bump_for(activity, opts \\ []) do
         Enum.map(users, fn user ->
           invisible_conversation = Enum.any?(users, &User.blocks?(user, &1))
 
-          unless invisible_conversation do
-            User.increment_unread_conversation_count(conversation, user)
-          end
-
           opts = Keyword.put(opts, :invisible_conversation, invisible_conversation)
 
           {:ok, participation} =
diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex
index 8bc3e85d6..4c32b273a 100644
--- a/lib/pleroma/conversation/participation.ex
+++ b/lib/pleroma/conversation/participation.ex
@@ -63,21 +63,10 @@ def mark_as_read(%User{} = user, %Conversation{} = conversation) do
     end
   end
 
-  def mark_as_read(participation) do
-    __MODULE__
-    |> where(id: ^participation.id)
-    |> update(set: [read: true])
-    |> select([p], p)
-    |> Repo.update_all([])
-    |> case do
-      {1, [participation]} ->
-        participation = Repo.preload(participation, :user)
-        User.set_unread_conversation_count(participation.user)
-        {:ok, participation}
-
-      error ->
-        error
-    end
+  def mark_as_read(%__MODULE__{} = participation) do
+    participation
+    |> change(read: true)
+    |> Repo.update()
   end
 
   def mark_all_as_read(%User{local: true} = user, %User{} = target_user) do
@@ -93,7 +82,6 @@ def mark_all_as_read(%User{local: true} = user, %User{} = target_user) do
     |> update([p], set: [read: true])
     |> Repo.update_all([])
 
-    {:ok, user} = User.set_unread_conversation_count(user)
     {:ok, user, []}
   end
 
@@ -108,7 +96,6 @@ def mark_all_as_read(%User{} = user) do
       |> select([p], p)
       |> Repo.update_all([])
 
-    {:ok, user} = User.set_unread_conversation_count(user)
     {:ok, user, participations}
   end
 
@@ -220,6 +207,12 @@ def set_recipients(participation, user_ids) do
     {:ok, Repo.preload(participation, :recipients, force: true)}
   end
 
+  @spec unread_count(User.t()) :: integer()
+  def unread_count(%User{id: user_id}) do
+    from(q in __MODULE__, where: q.user_id == ^user_id and q.read == false)
+    |> Repo.aggregate(:count, :id)
+  end
+
   def unread_conversation_count_for_user(user) do
     from(p in __MODULE__,
       where: p.user_id == ^user.id,
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index d2ad9516f..7fc7a533e 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -129,7 +129,6 @@ defmodule Pleroma.User do
     field(:hide_followers, :boolean, default: false)
     field(:hide_follows, :boolean, default: false)
     field(:hide_favorites, :boolean, default: true)
-    field(:unread_conversation_count, :integer, default: 0)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:email_notifications, :map, default: %{"digest" => false})
     field(:mascot, :map, default: nil)
@@ -1295,47 +1294,6 @@ def update_following_count(%User{local: true} = user) do
     |> update_and_set_cache()
   end
 
-  def set_unread_conversation_count(%User{local: true} = user) do
-    unread_query = Participation.unread_conversation_count_for_user(user)
-
-    User
-    |> join(:inner, [u], p in subquery(unread_query))
-    |> update([u, p],
-      set: [unread_conversation_count: p.count]
-    )
-    |> where([u], u.id == ^user.id)
-    |> select([u], u)
-    |> Repo.update_all([])
-    |> case do
-      {1, [user]} -> set_cache(user)
-      _ -> {:error, user}
-    end
-  end
-
-  def set_unread_conversation_count(user), do: {:ok, user}
-
-  def increment_unread_conversation_count(conversation, %User{local: true} = user) do
-    unread_query =
-      Participation.unread_conversation_count_for_user(user)
-      |> where([p], p.conversation_id == ^conversation.id)
-
-    User
-    |> join(:inner, [u], p in subquery(unread_query))
-    |> update([u, p],
-      inc: [unread_conversation_count: 1]
-    )
-    |> where([u], u.id == ^user.id)
-    |> where([u, p], p.count == 0)
-    |> select([u], u)
-    |> Repo.update_all([])
-    |> case do
-      {1, [user]} -> set_cache(user)
-      _ -> {:error, user}
-    end
-  end
-
-  def increment_unread_conversation_count(_, user), do: {:ok, user}
-
   @spec get_users_from_set([String.t()], keyword()) :: [User.t()]
   def get_users_from_set(ap_ids, opts \\ []) do
     local_only = Keyword.get(opts, :local_only, true)
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 864c0417f..1bf53600c 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -386,7 +386,7 @@ defp maybe_put_unread_conversation_count(data, %User{id: user_id} = user, %User{
     data
     |> Kernel.put_in(
       [:pleroma, :unread_conversation_count],
-      user.unread_conversation_count
+      Pleroma.Conversation.Participation.unread_count(user)
     )
   end
 
diff --git a/priv/repo/migrations/20200831114918_remove_unread_conversation_count_from_user.exs b/priv/repo/migrations/20200831114918_remove_unread_conversation_count_from_user.exs
new file mode 100644
index 000000000..b7bdb9166
--- /dev/null
+++ b/priv/repo/migrations/20200831114918_remove_unread_conversation_count_from_user.exs
@@ -0,0 +1,38 @@
+defmodule Pleroma.Repo.Migrations.RemoveUnreadConversationCountFromUser do
+  use Ecto.Migration
+  import Ecto.Query
+  alias Pleroma.Repo
+
+  def up do
+    alter table(:users) do
+      remove_if_exists(:unread_conversation_count, :integer)
+    end
+  end
+
+  def down do
+    alter table(:users) do
+      add_if_not_exists(:unread_conversation_count, :integer, default: 0)
+    end
+
+    flush()
+    recalc_unread_conversation_count()
+  end
+
+  defp recalc_unread_conversation_count do
+    participations_subquery =
+      from(
+        p in "conversation_participations",
+        where: p.read == false,
+        group_by: p.user_id,
+        select: %{user_id: p.user_id, unread_conversation_count: count(p.id)}
+      )
+
+    from(
+      u in "users",
+      join: p in subquery(participations_subquery),
+      on: p.user_id == u.id,
+      update: [set: [unread_conversation_count: p.unread_conversation_count]]
+    )
+    |> Repo.update_all([])
+  end
+end
diff --git a/priv/repo/migrations/20200831115854_add_unread_index_to_conversation_participation.exs b/priv/repo/migrations/20200831115854_add_unread_index_to_conversation_participation.exs
new file mode 100644
index 000000000..68771c655
--- /dev/null
+++ b/priv/repo/migrations/20200831115854_add_unread_index_to_conversation_participation.exs
@@ -0,0 +1,12 @@
+defmodule Pleroma.Repo.Migrations.AddUnreadIndexToConversationParticipation do
+  use Ecto.Migration
+
+  def change do
+    create(
+      index(:conversation_participations, [:user_id],
+        where: "read = false",
+        name: "unread_conversation_participation_count_index"
+      )
+    )
+  end
+end
diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs
index 59a1b6492..5a603dcc1 100644
--- a/test/conversation/participation_test.exs
+++ b/test/conversation/participation_test.exs
@@ -37,9 +37,8 @@ test "for a new conversation or a reply, it doesn't mark the author's participat
 
     [%{read: true}] = Participation.for_user(user)
     [%{read: false} = participation] = Participation.for_user(other_user)
-
-    assert User.get_cached_by_id(user.id).unread_conversation_count == 0
-    assert User.get_cached_by_id(other_user.id).unread_conversation_count == 1
+    assert Participation.unread_count(user) == 0
+    assert Participation.unread_count(other_user) == 1
 
     {:ok, _} =
       CommonAPI.post(other_user, %{
@@ -54,8 +53,8 @@ test "for a new conversation or a reply, it doesn't mark the author's participat
     [%{read: false}] = Participation.for_user(user)
     [%{read: true}] = Participation.for_user(other_user)
 
-    assert User.get_cached_by_id(user.id).unread_conversation_count == 1
-    assert User.get_cached_by_id(other_user.id).unread_conversation_count == 0
+    assert Participation.unread_count(user) == 1
+    assert Participation.unread_count(other_user) == 0
   end
 
   test "for a new conversation, it sets the recipents of the participation" do
@@ -264,7 +263,7 @@ test "when the user blocks a recipient, the existing conversations with them are
       assert [%{read: false}, %{read: false}, %{read: false}, %{read: false}] =
                Participation.for_user(blocker)
 
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 4
+      assert Participation.unread_count(blocker) == 4
 
       {:ok, _user_relationship} = User.block(blocker, blocked)
 
@@ -272,15 +271,15 @@ test "when the user blocks a recipient, the existing conversations with them are
       assert [%{read: true}, %{read: true}, %{read: true}, %{read: false}] =
                Participation.for_user(blocker)
 
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 1
+      assert Participation.unread_count(blocker) == 1
 
       # The conversation is not marked as read for the blocked user
       assert [_, _, %{read: false}] = Participation.for_user(blocked)
-      assert User.get_cached_by_id(blocked.id).unread_conversation_count == 1
+      assert Participation.unread_count(blocker) == 1
 
       # The conversation is not marked as read for the third user
       assert [%{read: false}, _, _] = Participation.for_user(third_user)
-      assert User.get_cached_by_id(third_user.id).unread_conversation_count == 1
+      assert Participation.unread_count(third_user) == 1
     end
 
     test "the new conversation with the blocked user is not marked as unread " do
@@ -298,7 +297,7 @@ test "the new conversation with the blocked user is not marked as unread " do
         })
 
       assert [%{read: true}] = Participation.for_user(blocker)
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0
+      assert Participation.unread_count(blocker) == 0
 
       # When the blocked user is a recipient
       {:ok, _direct2} =
@@ -308,10 +307,10 @@ test "the new conversation with the blocked user is not marked as unread " do
         })
 
       assert [%{read: true}, %{read: true}] = Participation.for_user(blocker)
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0
+      assert Participation.unread_count(blocker) == 0
 
       assert [%{read: false}, _] = Participation.for_user(blocked)
-      assert User.get_cached_by_id(blocked.id).unread_conversation_count == 1
+      assert Participation.unread_count(blocked) == 1
     end
 
     test "the conversation with the blocked user is not marked as unread on a reply" do
@@ -327,8 +326,8 @@ test "the conversation with the blocked user is not marked as unread on a reply"
 
       {:ok, _user_relationship} = User.block(blocker, blocked)
       assert [%{read: true}] = Participation.for_user(blocker)
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0
 
+      assert Participation.unread_count(blocker) == 0
       assert [blocked_participation] = Participation.for_user(blocked)
 
       # When it's a reply from the blocked user
@@ -340,8 +339,8 @@ test "the conversation with the blocked user is not marked as unread on a reply"
         })
 
       assert [%{read: true}] = Participation.for_user(blocker)
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0
 
+      assert Participation.unread_count(blocker) == 0
       assert [third_user_participation] = Participation.for_user(third_user)
 
       # When it's a reply from the third user
@@ -353,11 +352,12 @@ test "the conversation with the blocked user is not marked as unread on a reply"
         })
 
       assert [%{read: true}] = Participation.for_user(blocker)
-      assert User.get_cached_by_id(blocker.id).unread_conversation_count == 0
+      assert Participation.unread_count(blocker) == 0
 
       # Marked as unread for the blocked user
       assert [%{read: false}] = Participation.for_user(blocked)
-      assert User.get_cached_by_id(blocked.id).unread_conversation_count == 1
+
+      assert Participation.unread_count(blocked) == 1
     end
   end
 end
diff --git a/test/web/mastodon_api/controllers/conversation_controller_test.exs b/test/web/mastodon_api/controllers/conversation_controller_test.exs
index 3e21e6bf1..b23b22752 100644
--- a/test/web/mastodon_api/controllers/conversation_controller_test.exs
+++ b/test/web/mastodon_api/controllers/conversation_controller_test.exs
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.MastodonAPI.ConversationControllerTest do
   use Pleroma.Web.ConnCase
 
+  alias Pleroma.Conversation.Participation
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
 
@@ -28,10 +29,10 @@ test "returns correct conversations", %{
       user_three: user_three,
       conn: conn
     } do
-      assert User.get_cached_by_id(user_two.id).unread_conversation_count == 0
+      assert Participation.unread_count(user_two) == 0
       {:ok, direct} = create_direct_message(user_one, [user_two, user_three])
 
-      assert User.get_cached_by_id(user_two.id).unread_conversation_count == 1
+      assert Participation.unread_count(user_two) == 1
 
       {:ok, _follower_only} =
         CommonAPI.post(user_one, %{
@@ -59,7 +60,7 @@ test "returns correct conversations", %{
       assert is_binary(res_id)
       assert unread == false
       assert res_last_status["id"] == direct.id
-      assert User.get_cached_by_id(user_one.id).unread_conversation_count == 0
+      assert Participation.unread_count(user_one) == 0
     end
 
     test "observes limit params", %{
@@ -134,8 +135,8 @@ test "the user marks a conversation as read", %{user: user_one, conn: conn} do
     user_two = insert(:user)
     {:ok, direct} = create_direct_message(user_one, [user_two])
 
-    assert User.get_cached_by_id(user_one.id).unread_conversation_count == 0
-    assert User.get_cached_by_id(user_two.id).unread_conversation_count == 1
+    assert Participation.unread_count(user_one) == 0
+    assert Participation.unread_count(user_two) == 1
 
     user_two_conn =
       build_conn()
@@ -155,8 +156,8 @@ test "the user marks a conversation as read", %{user: user_one, conn: conn} do
       |> post("/api/v1/conversations/#{direct_conversation_id}/read")
       |> json_response_and_validate_schema(200)
 
-    assert User.get_cached_by_id(user_one.id).unread_conversation_count == 0
-    assert User.get_cached_by_id(user_two.id).unread_conversation_count == 0
+    assert Participation.unread_count(user_one) == 0
+    assert Participation.unread_count(user_two) == 0
 
     # The conversation is marked as unread on reply
     {:ok, _} =
@@ -171,8 +172,8 @@ test "the user marks a conversation as read", %{user: user_one, conn: conn} do
       |> get("/api/v1/conversations")
       |> json_response_and_validate_schema(200)
 
-    assert User.get_cached_by_id(user_one.id).unread_conversation_count == 1
-    assert User.get_cached_by_id(user_two.id).unread_conversation_count == 0
+    assert Participation.unread_count(user_one) == 1
+    assert Participation.unread_count(user_two) == 0
 
     # A reply doesn't increment the user's unread_conversation_count if the conversation is unread
     {:ok, _} =
@@ -182,8 +183,8 @@ test "the user marks a conversation as read", %{user: user_one, conn: conn} do
         in_reply_to_status_id: direct.id
       })
 
-    assert User.get_cached_by_id(user_one.id).unread_conversation_count == 1
-    assert User.get_cached_by_id(user_two.id).unread_conversation_count == 0
+    assert Participation.unread_count(user_one) == 1
+    assert Participation.unread_count(user_two) == 0
   end
 
   test "(vanilla) Mastodon frontend behaviour", %{user: user_one, conn: conn} do
diff --git a/test/web/pleroma_api/controllers/conversation_controller_test.exs b/test/web/pleroma_api/controllers/conversation_controller_test.exs
index e6d0b3e37..f2feeaaef 100644
--- a/test/web/pleroma_api/controllers/conversation_controller_test.exs
+++ b/test/web/pleroma_api/controllers/conversation_controller_test.exs
@@ -121,7 +121,7 @@ test "POST /api/v1/pleroma/conversations/read" do
     [participation2, participation1] = Participation.for_user(other_user)
     assert Participation.get(participation2.id).read == false
     assert Participation.get(participation1.id).read == false
-    assert User.get_cached_by_id(other_user.id).unread_conversation_count == 2
+    assert Participation.unread_count(other_user) == 2
 
     [%{"unread" => false}, %{"unread" => false}] =
       conn
@@ -131,6 +131,6 @@ test "POST /api/v1/pleroma/conversations/read" do
     [participation2, participation1] = Participation.for_user(other_user)
     assert Participation.get(participation2.id).read == true
     assert Participation.get(participation1.id).read == true
-    assert User.get_cached_by_id(other_user.id).unread_conversation_count == 0
+    assert Participation.unread_count(other_user) == 0
   end
 end

From 7efadc3cbd46369e960f31c33a2c555f718ca8c5 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 1 Oct 2020 21:34:45 +0300
Subject: [PATCH 005/104] No auth check in OStatusController, even on
 non-federating instances.

---
 lib/pleroma/web/ostatus/ostatus_controller.ex | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index de1b0b3f0..8646d2c1c 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -16,10 +16,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.Metadata.PlayerView
   alias Pleroma.Web.Router
 
-  plug(Pleroma.Plugs.EnsureAuthenticatedPlug,
-    unless_func: &Pleroma.Web.FederatingPlug.federating?/1
-  )
-
   plug(
     RateLimiter,
     [name: :ap_routes, params: ["uuid"]] when action in [:object, :activity]

From 0d575735bfd280b878bdecc6d018d8cca23ad09f Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 1 Oct 2020 21:41:22 +0300
Subject: [PATCH 006/104] No auth check in UserController.feed_redirect/2, even
 on non-federating instances.

---
 lib/pleroma/web/feed/user_controller.ex | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex
index 71eb1ea7e..09ecdedb4 100644
--- a/lib/pleroma/web/feed/user_controller.ex
+++ b/lib/pleroma/web/feed/user_controller.ex
@@ -23,12 +23,7 @@ def feed_redirect(%{assigns: %{format: "html"}} = conn, %{"nickname" => nickname
 
   def feed_redirect(%{assigns: %{format: format}} = conn, _params)
       when format in ["json", "activity+json"] do
-    with %{halted: false} = conn <-
-           Pleroma.Plugs.EnsureAuthenticatedPlug.call(conn,
-             unless_func: &Pleroma.Web.FederatingPlug.federating?/1
-           ) do
-      ActivityPubController.call(conn, :user)
-    end
+    ActivityPubController.call(conn, :user)
   end
 
   def feed_redirect(conn, %{"nickname" => nickname}) do

From f6024252ae8601d41bea943bb3cae5c656416eb9 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 2 Oct 2020 22:18:02 +0300
Subject: [PATCH 007/104] [#3053] No auth check in StaticFEController, even on
 non-federating instances. Adjusted tests.

---
 lib/pleroma/web/feed/tag_controller.ex        |   4 +-
 lib/pleroma/web/feed/user_controller.ex       |   4 +-
 lib/pleroma/web/router.ex                     |  11 +-
 .../web/static_fe/static_fe_controller.ex     | 176 +++++++++---------
 test/support/conn_case.ex                     |  22 ---
 .../activity_pub_controller_test.exs          |  19 ++
 test/web/feed/user_controller_test.exs        |  12 +-
 test/web/ostatus/ostatus_controller_test.exs  |  24 +--
 .../static_fe/static_fe_controller_test.exs   |  24 ++-
 9 files changed, 162 insertions(+), 134 deletions(-)

diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex
index 93a8294b7..c348b32c2 100644
--- a/lib/pleroma/web/feed/tag_controller.ex
+++ b/lib/pleroma/web/feed/tag_controller.ex
@@ -10,14 +10,14 @@ defmodule Pleroma.Web.Feed.TagController do
   alias Pleroma.Web.Feed.FeedView
 
   def feed(conn, params) do
-    unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do
+    unless Config.restrict_unauthenticated_access?(:activities, :local) do
       render_feed(conn, params)
     else
       render_error(conn, :not_found, "Not found")
     end
   end
 
-  def render_feed(conn, %{"tag" => raw_tag} = params) do
+  defp render_feed(conn, %{"tag" => raw_tag} = params) do
     {format, tag} = parse_tag(raw_tag)
 
     activities =
diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex
index 09ecdedb4..5fbcd82d7 100644
--- a/lib/pleroma/web/feed/user_controller.ex
+++ b/lib/pleroma/web/feed/user_controller.ex
@@ -40,11 +40,11 @@ def feed(conn, params) do
     end
   end
 
-  def render_feed(conn, %{"nickname" => nickname} = params) do
+  defp render_feed(conn, %{"nickname" => nickname} = params) do
     format = get_format(conn)
 
     format =
-      if format in ["rss", "atom"] do
+      if format in ["atom", "rss"] do
         format
       else
         "atom"
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 42a9db21d..e0e92549f 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -561,12 +561,17 @@ defmodule Pleroma.Web.Router do
     plug(Pleroma.Plugs.StaticFEPlug)
   end
 
+  pipeline :ostatus_no_html do
+    plug(:accepts, ["xml", "rss", "atom", "activity+json", "json"])
+  end
+
   pipeline :oembed do
     plug(:accepts, ["json", "xml"])
   end
 
   scope "/", Pleroma.Web do
-    pipe_through([:ostatus, :http_signature])
+    # Note: no authentication plugs, all endpoints below should only yield public objects
+    pipe_through(:ostatus)
 
     get("/objects/:uuid", OStatus.OStatusController, :object)
     get("/activities/:uuid", OStatus.OStatusController, :activity)
@@ -579,6 +584,10 @@ defmodule Pleroma.Web.Router do
 
     get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed)
     get("/users/:nickname", Feed.UserController, :feed_redirect, as: :user_feed)
+  end
+
+  scope "/", Pleroma.Web do
+    pipe_through(:ostatus_no_html)
 
     get("/tags/:tag", Feed.TagController, :feed, as: :tag_feed)
   end
diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex
index a7a891b13..b1c62f5b0 100644
--- a/lib/pleroma/web/static_fe/static_fe_controller.ex
+++ b/lib/pleroma/web/static_fe/static_fe_controller.ex
@@ -17,12 +17,95 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do
   plug(:put_view, Pleroma.Web.StaticFE.StaticFEView)
   plug(:assign_id)
 
-  plug(Pleroma.Plugs.EnsureAuthenticatedPlug,
-    unless_func: &Pleroma.Web.FederatingPlug.federating?/1
-  )
-
   @page_keys ["max_id", "min_id", "limit", "since_id", "order"]
 
+  @doc "Renders requested local public activity"
+  def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do
+    with %Activity{local: true} = activity <-
+           Activity.get_by_id_with_object(notice_id),
+         true <- Visibility.is_public?(activity.object),
+         %User{} = user <- User.get_by_ap_id(activity.object.data["actor"]) do
+      meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user})
+
+      timeline =
+        activity.object.data["context"]
+        |> ActivityPub.fetch_activities_for_context(%{})
+        |> Enum.reverse()
+        |> Enum.map(&represent(&1, &1.object.id == activity.object.id))
+
+      render(conn, "conversation.html", %{activities: timeline, meta: meta})
+    else
+      %Activity{object: %Object{data: data}} ->
+        conn
+        |> put_status(:found)
+        |> redirect(external: data["url"] || data["external_url"] || data["id"])
+
+      _ ->
+        not_found(conn, "Post not found.")
+    end
+  end
+
+  @doc "Renders public activities of requested user"
+  def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do
+    case User.get_cached_by_nickname_or_id(username_or_id) do
+      %User{} = user ->
+        meta = Metadata.build_tags(%{user: user})
+
+        params =
+          params
+          |> Map.take(@page_keys)
+          |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end)
+
+        timeline =
+          user
+          |> ActivityPub.fetch_user_activities(_reading_user = nil, params)
+          |> Enum.map(&represent/1)
+
+        prev_page_id =
+          (params["min_id"] || params["max_id"]) &&
+            List.first(timeline) && List.first(timeline).id
+
+        next_page_id = List.last(timeline) && List.last(timeline).id
+
+        render(conn, "profile.html", %{
+          user: User.sanitize_html(user),
+          timeline: timeline,
+          prev_page_id: prev_page_id,
+          next_page_id: next_page_id,
+          meta: meta
+        })
+
+      _ ->
+        not_found(conn, "User not found.")
+    end
+  end
+
+  def show(%{assigns: %{object_id: _}} = conn, _params) do
+    url = Helpers.url(conn) <> conn.request_path
+
+    case Activity.get_create_by_object_ap_id_with_object(url) do
+      %Activity{} = activity ->
+        to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity)
+        redirect(conn, to: to)
+
+      _ ->
+        not_found(conn, "Post not found.")
+    end
+  end
+
+  def show(%{assigns: %{activity_id: _}} = conn, _params) do
+    url = Helpers.url(conn) <> conn.request_path
+
+    case Activity.get_by_ap_id(url) do
+      %Activity{} = activity ->
+        to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity)
+        redirect(conn, to: to)
+
+      _ ->
+        not_found(conn, "Post not found.")
+    end
+  end
+
   defp get_title(%Object{data: %{"name" => name}}) when is_binary(name),
     do: name
 
@@ -81,91 +164,6 @@ defp represent(%Activity{object: %Object{data: data}} = activity, selected) do
     }
   end
 
-  def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do
-    with %Activity{local: true} = activity <-
-           Activity.get_by_id_with_object(notice_id),
-         true <- Visibility.is_public?(activity.object),
-         %User{} = user <- User.get_by_ap_id(activity.object.data["actor"]) do
-      meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user})
-
-      timeline =
-        activity.object.data["context"]
-        |> ActivityPub.fetch_activities_for_context(%{})
-        |> Enum.reverse()
-        |> Enum.map(&represent(&1, &1.object.id == activity.object.id))
-
-      render(conn, "conversation.html", %{activities: timeline, meta: meta})
-    else
-      %Activity{object: %Object{data: data}} ->
-        conn
-        |> put_status(:found)
-        |> redirect(external: data["url"] || data["external_url"] || data["id"])
-
-      _ ->
-        not_found(conn, "Post not found.")
-    end
-  end
-
-  def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do
-    case User.get_cached_by_nickname_or_id(username_or_id) do
-      %User{} = user ->
-        meta = Metadata.build_tags(%{user: user})
-
-        params =
-          params
-          |> Map.take(@page_keys)
-          |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end)
-
-        timeline =
-          user
-          |> ActivityPub.fetch_user_activities(nil, params)
-          |> Enum.map(&represent/1)
-
-        prev_page_id =
-          (params["min_id"] || params["max_id"]) &&
-            List.first(timeline) && List.first(timeline).id
-
-        next_page_id = List.last(timeline) && List.last(timeline).id
-
-        render(conn, "profile.html", %{
-          user: User.sanitize_html(user),
-          timeline: timeline,
-          prev_page_id: prev_page_id,
-          next_page_id: next_page_id,
-          meta: meta
-        })
-
-      _ ->
-        not_found(conn, "User not found.")
-    end
-  end
-
-  def show(%{assigns: %{object_id: _}} = conn, _params) do
-    url = Helpers.url(conn) <> conn.request_path
-
-    case Activity.get_create_by_object_ap_id_with_object(url) do
-      %Activity{} = activity ->
-        to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity)
-        redirect(conn, to: to)
-
-      _ ->
-        not_found(conn, "Post not found.")
-    end
-  end
-
-  def show(%{assigns: %{activity_id: _}} = conn, _params) do
-    url = Helpers.url(conn) <> conn.request_path
-
-    case Activity.get_by_ap_id(url) do
-      %Activity{} = activity ->
-        to = Helpers.o_status_path(Pleroma.Web.Endpoint, :notice, activity)
-        redirect(conn, to: to)
-
-      _ ->
-        not_found(conn, "Post not found.")
-    end
-  end
-
   defp assign_id(%{path_info: ["notice", notice_id]} = conn, _opts),
     do: assign(conn, :notice_id, notice_id)
 
diff --git a/test/support/conn_case.ex b/test/support/conn_case.ex
index 7ef681258..7b28d70e7 100644
--- a/test/support/conn_case.ex
+++ b/test/support/conn_case.ex
@@ -111,28 +111,6 @@ defp json_response_and_validate_schema(
       defp json_response_and_validate_schema(conn, _status) do
         flunk("Response schema not found for #{conn.method} #{conn.request_path} #{conn.status}")
       end
-
-      defp ensure_federating_or_authenticated(conn, url, user) do
-        initial_setting = Config.get([:instance, :federating])
-        on_exit(fn -> Config.put([:instance, :federating], initial_setting) end)
-
-        Config.put([:instance, :federating], false)
-
-        conn
-        |> get(url)
-        |> response(403)
-
-        conn
-        |> assign(:user, user)
-        |> get(url)
-        |> response(200)
-
-        Config.put([:instance, :federating], true)
-
-        conn
-        |> get(url)
-        |> response(200)
-      end
     end
   end
 
diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs
index 0517571f2..ab57b6523 100644
--- a/test/web/activity_pub/activity_pub_controller_test.exs
+++ b/test/web/activity_pub/activity_pub_controller_test.exs
@@ -33,6 +33,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
 
   setup do: clear_config([:instance, :federating], true)
 
+  defp ensure_federating_or_authenticated(conn, url, user) do
+    Config.put([:instance, :federating], false)
+
+    conn
+    |> get(url)
+    |> response(403)
+
+    conn
+    |> assign(:user, user)
+    |> get(url)
+    |> response(200)
+
+    Config.put([:instance, :federating], true)
+
+    conn
+    |> get(url)
+    |> response(200)
+  end
+
   describe "/relay" do
     setup do: clear_config([:instance, :allow_relay])
 
diff --git a/test/web/feed/user_controller_test.exs b/test/web/feed/user_controller_test.exs
index 9a5610baa..7383e82cc 100644
--- a/test/web/feed/user_controller_test.exs
+++ b/test/web/feed/user_controller_test.exs
@@ -13,7 +13,7 @@ defmodule Pleroma.Web.Feed.UserControllerTest do
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
 
-  setup do: clear_config([:instance, :federating], true)
+  setup do: clear_config([:static_fe, :enabled], false)
 
   describe "feed" do
     setup do: clear_config([:feed])
@@ -192,6 +192,16 @@ test "returns 404 when the user is remote", %{conn: conn} do
              |> get(user_feed_path(conn, :feed, user.nickname))
              |> response(404)
     end
+
+    test "does not require authentication on non-federating instances", %{conn: conn} do
+      clear_config([:instance, :federating], false)
+      user = insert(:user)
+
+      conn
+      |> put_req_header("accept", "application/rss+xml")
+      |> get("/users/#{user.nickname}/feed.rss")
+      |> response(200)
+    end
   end
 
   # Note: see ActivityPubControllerTest for JSON format tests
diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs
index ee498f4b5..65b2c22db 100644
--- a/test/web/ostatus/ostatus_controller_test.exs
+++ b/test/web/ostatus/ostatus_controller_test.exs
@@ -7,7 +7,6 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do
 
   import Pleroma.Factory
 
-  alias Pleroma.Config
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -21,7 +20,7 @@ defmodule Pleroma.Web.OStatus.OStatusControllerTest do
     :ok
   end
 
-  setup do: clear_config([:instance, :federating], true)
+  setup do: clear_config([:static_fe, :enabled], false)
 
   describe "Mastodon compatibility routes" do
     setup %{conn: conn} do
@@ -215,15 +214,16 @@ test "404s a non-existing notice", %{conn: conn} do
       assert response(conn, 404)
     end
 
-    test "it requires authentication if instance is NOT federating", %{
+    test "does not require authentication on non-federating instances", %{
       conn: conn
     } do
-      user = insert(:user)
+      clear_config([:instance, :federating], false)
       note_activity = insert(:note_activity)
 
-      conn = put_req_header(conn, "accept", "text/html")
-
-      ensure_federating_or_authenticated(conn, "/notice/#{note_activity.id}", user)
+      conn
+      |> put_req_header("accept", "text/html")
+      |> get("/notice/#{note_activity.id}")
+      |> response(200)
     end
   end
 
@@ -325,14 +325,16 @@ test "404s when attachment isn't audio or video", %{conn: conn} do
       |> response(404)
     end
 
-    test "it requires authentication if instance is NOT federating", %{
+    test "does not require authentication on non-federating instances", %{
       conn: conn,
       note_activity: note_activity
     } do
-      user = insert(:user)
-      conn = put_req_header(conn, "accept", "text/html")
+      clear_config([:instance, :federating], false)
 
-      ensure_federating_or_authenticated(conn, "/notice/#{note_activity.id}/embed_player", user)
+      conn
+      |> put_req_header("accept", "text/html")
+      |> get("/notice/#{note_activity.id}/embed_player")
+      |> response(200)
     end
   end
 end
diff --git a/test/web/static_fe/static_fe_controller_test.exs b/test/web/static_fe/static_fe_controller_test.exs
index 1598bf675..bab0b0a7b 100644
--- a/test/web/static_fe/static_fe_controller_test.exs
+++ b/test/web/static_fe/static_fe_controller_test.exs
@@ -2,14 +2,12 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do
   use Pleroma.Web.ConnCase
 
   alias Pleroma.Activity
-  alias Pleroma.Config
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.CommonAPI
 
   import Pleroma.Factory
 
   setup_all do: clear_config([:static_fe, :enabled], true)
-  setup do: clear_config([:instance, :federating], true)
 
   setup %{conn: conn} do
     conn = put_req_header(conn, "accept", "text/html")
@@ -70,8 +68,15 @@ test "pagination, page 2", %{conn: conn, user: user} do
       refute html =~ ">test29<"
     end
 
-    test "it requires authentication if instance is NOT federating", %{conn: conn, user: user} do
-      ensure_federating_or_authenticated(conn, "/users/#{user.nickname}", user)
+    test "does not require authentication on non-federating instances", %{
+      conn: conn,
+      user: user
+    } do
+      clear_config([:instance, :federating], false)
+
+      conn = get(conn, "/users/#{user.nickname}")
+
+      assert html_response(conn, 200) =~ user.nickname
     end
   end
 
@@ -183,10 +188,17 @@ test "302 for remote cached status", %{conn: conn, user: user} do
       assert html_response(conn, 302) =~ "redirected"
     end
 
-    test "it requires authentication if instance is NOT federating", %{conn: conn, user: user} do
+    test "does not require authentication on non-federating instances", %{
+      conn: conn,
+      user: user
+    } do
+      clear_config([:instance, :federating], false)
+
       {:ok, activity} = CommonAPI.post(user, %{status: "testing a thing!"})
 
-      ensure_federating_or_authenticated(conn, "/notice/#{activity.id}", user)
+      conn = get(conn, "/notice/#{activity.id}")
+
+      assert html_response(conn, 200) =~ "testing a thing!"
     end
   end
 end

From 094edde7c4ddf65f46e5d692a5ef5b859587d1e1 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Mon, 5 Oct 2020 23:48:00 +0300
Subject: [PATCH 008/104] [#3053] Unauthenticated access control for
 OStatus-related controllers and ActivityPubController (base actions: :user,
 :object, :activity). Tests adjustments.

---
 .../activity_pub/activity_pub_controller.ex   | 56 +++++++++-------
 lib/pleroma/web/activity_pub/visibility.ex    | 39 ++++++++---
 lib/pleroma/web/feed/tag_controller.ex        | 15 +++--
 lib/pleroma/web/feed/user_controller.ex       | 19 +++---
 lib/pleroma/web/ostatus/ostatus_controller.ex | 26 +++----
 lib/pleroma/web/router.ex                     | 52 ++++++++++----
 .../web/static_fe/static_fe_controller.ex     | 48 ++++++-------
 .../activity_pub_controller_test.exs          | 67 -------------------
 test/web/feed/tag_controller_test.exs         | 13 ++--
 9 files changed, 159 insertions(+), 176 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 732c44271..c78edfb4c 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -32,17 +32,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   @federating_only_actions [:internal_fetch, :relay, :relay_following, :relay_followers]
 
+  # Note: :following and :followers must be served even without authentication (as via :api)
+  @auth_only_actions [:read_inbox, :update_outbox, :whoami, :upload_media]
+
+  # Always accessible actions (must perform entity accessibility checks)
+  @no_auth_no_federation_actions [:user, :activity, :object]
+
+  @authenticated_or_federating_actions @federating_only_actions ++
+                                         @auth_only_actions ++ @no_auth_no_federation_actions
+
   plug(FederatingPlug when action in @federating_only_actions)
 
+  plug(EnsureAuthenticatedPlug when action in @auth_only_actions)
+
   plug(
     EnsureAuthenticatedPlug,
-    [unless_func: &FederatingPlug.federating?/1] when action not in @federating_only_actions
-  )
-
-  # Note: :following and :followers must be served even without authentication (as via :api)
-  plug(
-    EnsureAuthenticatedPlug
-    when action in [:read_inbox, :update_outbox, :whoami, :upload_media]
+    [unless_func: &FederatingPlug.federating?/1]
+    when action not in @authenticated_or_federating_actions
   )
 
   plug(
@@ -66,21 +72,22 @@ defp relay_active?(conn, _) do
 
   def user(conn, %{"nickname" => nickname}) do
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname),
+         {_, :visible} <- {:visibility, User.visible_for(user, _reading_user = nil)},
          {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_content_type("application/activity+json")
       |> put_view(UserView)
       |> render("user.json", %{user: user})
     else
-      nil -> {:error, :not_found}
-      %{local: false} -> {:error, :not_found}
+      _ -> {:error, :not_found}
     end
   end
 
   def object(conn, _) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(object)} do
+         {_, true} <- {:public?, Visibility.is_public?(object)},
+         {_, false} <- {:restricted?, Visibility.restrict_unauthenticated_access?(object)} do
       conn
       |> assign(:tracking_fun_data, object.id)
       |> set_cache_ttl_for(object)
@@ -88,25 +95,15 @@ def object(conn, _) do
       |> put_view(ObjectView)
       |> render("object.json", object: object)
     else
-      {:public?, false} ->
-        {:error, :not_found}
+      _ -> {:error, :not_found}
     end
   end
 
-  def track_object_fetch(conn, nil), do: conn
-
-  def track_object_fetch(conn, object_id) do
-    with %{assigns: %{user: %User{id: user_id}}} <- conn do
-      Delivery.create(object_id, user_id)
-    end
-
-    conn
-  end
-
   def activity(conn, _params) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Activity{} = activity <- Activity.normalize(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(activity)} do
+         {_, true} <- {:public?, Visibility.is_public?(activity)},
+         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do
       conn
       |> maybe_set_tracking_data(activity)
       |> set_cache_ttl_for(activity)
@@ -114,8 +111,7 @@ def activity(conn, _params) do
       |> put_view(ObjectView)
       |> render("object.json", object: activity)
     else
-      {:public?, false} -> {:error, :not_found}
-      nil -> {:error, :not_found}
+      _ -> {:error, :not_found}
     end
   end
 
@@ -550,4 +546,14 @@ def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} =
       |> json(object.data)
     end
   end
+
+  def track_object_fetch(conn, nil), do: conn
+
+  def track_object_fetch(conn, object_id) do
+    with %{assigns: %{user: %User{id: user_id}}} <- conn do
+      Delivery.create(object_id, user_id)
+    end
+
+    conn
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex
index 5c349bb7a..76bd54a42 100644
--- a/lib/pleroma/web/activity_pub/visibility.ex
+++ b/lib/pleroma/web/activity_pub/visibility.ex
@@ -44,29 +44,30 @@ def is_direct?(activity) do
   def is_list?(%{data: %{"listMessage" => _}}), do: true
   def is_list?(_), do: false
 
-  @spec visible_for_user?(Activity.t(), User.t() | nil) :: boolean()
-  def visible_for_user?(%{actor: ap_id}, %User{ap_id: ap_id}), do: true
+  @spec visible_for_user?(Activity.t() | nil, User.t() | nil) :: boolean()
+  def visible_for_user?(%Activity{actor: ap_id}, %User{ap_id: ap_id}), do: true
 
   def visible_for_user?(nil, _), do: false
 
-  def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false
+  def visible_for_user?(%Activity{data: %{"listMessage" => _}}, nil), do: false
 
-  def visible_for_user?(%{data: %{"listMessage" => list_ap_id}} = activity, %User{} = user) do
+  def visible_for_user?(
+        %Activity{data: %{"listMessage" => list_ap_id}} = activity,
+        %User{} = user
+      ) do
     user.ap_id in activity.data["to"] ||
       list_ap_id
       |> Pleroma.List.get_by_ap_id()
       |> Pleroma.List.member?(user)
   end
 
-  def visible_for_user?(%{local: local} = activity, nil) do
-    cfg_key = if local, do: :local, else: :remote
-
-    if Pleroma.Config.restrict_unauthenticated_access?(:activities, cfg_key),
+  def visible_for_user?(%Activity{} = activity, nil) do
+    if restrict_unauthenticated_access?(activity),
       do: false,
       else: is_public?(activity)
   end
 
-  def visible_for_user?(activity, user) do
+  def visible_for_user?(%Activity{} = activity, user) do
     x = [user.ap_id | User.following(user)]
     y = [activity.actor] ++ activity.data["to"] ++ (activity.data["cc"] || [])
     is_public?(activity) || Enum.any?(x, &(&1 in y))
@@ -82,6 +83,26 @@ def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do
     result
   end
 
+  def restrict_unauthenticated_access?(%Activity{local: local}) do
+    restrict_unauthenticated_access_to_activity?(local)
+  end
+
+  def restrict_unauthenticated_access?(%Object{} = object) do
+    object
+    |> Object.local?()
+    |> restrict_unauthenticated_access_to_activity?()
+  end
+
+  def restrict_unauthenticated_access?(%User{} = user) do
+    User.visible_for(user, _reading_user = nil)
+  end
+
+  defp restrict_unauthenticated_access_to_activity?(local?) when is_boolean(local?) do
+    cfg_key = if local?, do: :local, else: :remote
+
+    Pleroma.Config.restrict_unauthenticated_access?(:activities, cfg_key)
+  end
+
   def get_visibility(object) do
     to = object.data["to"] || []
     cc = object.data["cc"] || []
diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex
index c348b32c2..218cdbdf3 100644
--- a/lib/pleroma/web/feed/tag_controller.ex
+++ b/lib/pleroma/web/feed/tag_controller.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.Feed.TagController do
   alias Pleroma.Web.Feed.FeedView
 
   def feed(conn, params) do
-    unless Config.restrict_unauthenticated_access?(:activities, :local) do
+    if Config.get!([:instance, :public]) do
       render_feed(conn, params)
     else
       render_error(conn, :not_found, "Not found")
@@ -36,12 +36,13 @@ defp render_feed(conn, %{"tag" => raw_tag} = params) do
   end
 
   @spec parse_tag(binary() | any()) :: {format :: String.t(), tag :: String.t()}
-  defp parse_tag(raw_tag) when is_binary(raw_tag) do
-    case Enum.reverse(String.split(raw_tag, ".")) do
-      [format | tag] when format in ["atom", "rss"] -> {format, Enum.join(tag, ".")}
-      _ -> {"rss", raw_tag}
+  defp parse_tag(raw_tag) do
+    case is_binary(raw_tag) && Enum.reverse(String.split(raw_tag, ".")) do
+      [format | tag] when format in ["rss", "atom"] ->
+        {format, Enum.join(tag, ".")}
+
+      _ ->
+        {"atom", raw_tag}
     end
   end
-
-  defp parse_tag(raw_tag), do: {"rss", raw_tag}
 end
diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex
index 5fbcd82d7..f1d2bb7be 100644
--- a/lib/pleroma/web/feed/user_controller.ex
+++ b/lib/pleroma/web/feed/user_controller.ex
@@ -6,6 +6,8 @@ defmodule Pleroma.Web.Feed.UserController do
   use Pleroma.Web, :controller
 
   alias Fallback.RedirectController
+
+  alias Pleroma.Config
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.ActivityPubController
@@ -32,15 +34,7 @@ def feed_redirect(conn, %{"nickname" => nickname}) do
     end
   end
 
-  def feed(conn, params) do
-    unless Pleroma.Config.restrict_unauthenticated_access?(:profiles, :local) do
-      render_feed(conn, params)
-    else
-      errors(conn, {:error, :not_found})
-    end
-  end
-
-  defp render_feed(conn, %{"nickname" => nickname} = params) do
+  def feed(conn, %{"nickname" => nickname} = params) do
     format = get_format(conn)
 
     format =
@@ -50,7 +44,8 @@ defp render_feed(conn, %{"nickname" => nickname} = params) do
         "atom"
       end
 
-    with {_, %User{local: true} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)} do
+    with {_, %User{local: true} = user} <- {:fetch_user, User.get_cached_by_nickname(nickname)},
+         {_, :visible} <- {:visibility, User.visible_for(user, _reading_user = nil)} do
       activities =
         %{
           type: ["Create"],
@@ -65,7 +60,7 @@ defp render_feed(conn, %{"nickname" => nickname} = params) do
       |> render("user.#{format}",
         user: user,
         activities: activities,
-        feed_config: Pleroma.Config.get([:feed])
+        feed_config: Config.get([:feed])
       )
     end
   end
@@ -77,6 +72,8 @@ def errors(conn, {:error, :not_found}) do
   def errors(conn, {:fetch_user, %User{local: false}}), do: errors(conn, {:error, :not_found})
   def errors(conn, {:fetch_user, nil}), do: errors(conn, {:error, :not_found})
 
+  def errors(conn, {:visibility, _}), do: errors(conn, {:error, :not_found})
+
   def errors(conn, _) do
     render_error(conn, :internal_server_error, "Something went wrong")
   end
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index 8646d2c1c..b4dc2a87f 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -33,16 +33,15 @@ def object(%{assigns: %{format: format}} = conn, _params)
     ActivityPubController.call(conn, :object)
   end
 
-  def object(%{assigns: %{format: format}} = conn, _params) do
+  def object(conn, _params) do
     with id <- Endpoint.url() <> conn.request_path,
          {_, %Activity{} = activity} <-
            {:activity, Activity.get_create_by_object_ap_id_with_object(id)},
-         {_, true} <- {:public?, Visibility.is_public?(activity)} do
-      case format do
-        _ -> redirect(conn, to: "/notice/#{activity.id}")
-      end
+         {_, true} <- {:public?, Visibility.is_public?(activity)},
+         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do
+      redirect(conn, to: "/notice/#{activity.id}")
     else
-      reason when reason in [{:public?, false}, {:activity, nil}] ->
+      reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] ->
         {:error, :not_found}
 
       e ->
@@ -55,15 +54,14 @@ def activity(%{assigns: %{format: format}} = conn, _params)
     ActivityPubController.call(conn, :activity)
   end
 
-  def activity(%{assigns: %{format: format}} = conn, _params) do
+  def activity(conn, _params) do
     with id <- Endpoint.url() <> conn.request_path,
          {_, %Activity{} = activity} <- {:activity, Activity.normalize(id)},
-         {_, true} <- {:public?, Visibility.is_public?(activity)} do
-      case format do
-        _ -> redirect(conn, to: "/notice/#{activity.id}")
-      end
+         {_, true} <- {:public?, Visibility.is_public?(activity)},
+         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do
+      redirect(conn, to: "/notice/#{activity.id}")
     else
-      reason when reason in [{:public?, false}, {:activity, nil}] ->
+      reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] ->
         {:error, :not_found}
 
       e ->
@@ -74,6 +72,7 @@ def activity(%{assigns: %{format: format}} = conn, _params) do
   def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do
     with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)},
          {_, true} <- {:public?, Visibility.is_public?(activity)},
+         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)},
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       cond do
         format in ["json", "activity+json"] ->
@@ -101,7 +100,7 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do
           RedirectController.redirector(conn, nil)
       end
     else
-      reason when reason in [{:public?, false}, {:activity, nil}] ->
+      reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] ->
         conn
         |> put_status(404)
         |> RedirectController.redirector(nil, 404)
@@ -115,6 +114,7 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do
   def notice_player(conn, %{"id" => id}) do
     with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id_with_object(id),
          true <- Visibility.is_public?(activity),
+         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)},
          %Object{} = object <- Object.normalize(activity),
          %{data: %{"attachment" => [%{"url" => [url | _]} | _]}} <- object,
          true <- String.starts_with?(url["mediaType"], ["audio", "video"]) do
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index e0e92549f..6439a1c39 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -5,6 +5,14 @@
 defmodule Pleroma.Web.Router do
   use Pleroma.Web, :router
 
+  pipeline :accepts_html do
+    plug(:accepts, ["html"])
+  end
+
+  pipeline :accepts_xml_rss_atom do
+    plug(:accepts, ["xml", "rss", "atom"])
+  end
+
   pipeline :browser do
     plug(:accepts, ["html"])
     plug(:fetch_session)
@@ -556,39 +564,55 @@ defmodule Pleroma.Web.Router do
     )
   end
 
-  pipeline :ostatus do
+  pipeline :ostatus_html_json do
+    plug(:accepts, ["html", "activity+json", "json"])
+    plug(Pleroma.Plugs.StaticFEPlug)
+  end
+
+  pipeline :ostatus_html_xml do
+    plug(:accepts, ["html", "xml", "rss", "atom"])
+    plug(Pleroma.Plugs.StaticFEPlug)
+  end
+
+  pipeline :ostatus_html_xml_json do
     plug(:accepts, ["html", "xml", "rss", "atom", "activity+json", "json"])
     plug(Pleroma.Plugs.StaticFEPlug)
   end
 
-  pipeline :ostatus_no_html do
-    plug(:accepts, ["xml", "rss", "atom", "activity+json", "json"])
-  end
-
-  pipeline :oembed do
-    plug(:accepts, ["json", "xml"])
-  end
-
   scope "/", Pleroma.Web do
-    # Note: no authentication plugs, all endpoints below should only yield public objects
-    pipe_through(:ostatus)
+    # Note: html format is supported only if static FE is enabled
+    pipe_through(:ostatus_html_json)
 
     get("/objects/:uuid", OStatus.OStatusController, :object)
     get("/activities/:uuid", OStatus.OStatusController, :activity)
     get("/notice/:id", OStatus.OStatusController, :notice)
-    get("/notice/:id/embed_player", OStatus.OStatusController, :notice_player)
 
     # Mastodon compatibility routes
     get("/users/:nickname/statuses/:id", OStatus.OStatusController, :object)
     get("/users/:nickname/statuses/:id/activity", OStatus.OStatusController, :activity)
+  end
 
-    get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed)
+  scope "/", Pleroma.Web do
+    # Note: html format is supported only if static FE is enabled
+    pipe_through(:ostatus_html_xml_json)
+
+    # Note: for json format responds with user profile (not user feed)
     get("/users/:nickname", Feed.UserController, :feed_redirect, as: :user_feed)
   end
 
   scope "/", Pleroma.Web do
-    pipe_through(:ostatus_no_html)
+    # Note: html format is supported only if static FE is enabled
+    pipe_through(:ostatus_html_xml)
+    get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed)
+  end
 
+  scope "/", Pleroma.Web do
+    pipe_through(:accepts_html)
+    get("/notice/:id/embed_player", OStatus.OStatusController, :notice_player)
+  end
+
+  scope "/", Pleroma.Web do
+    pipe_through(:accepts_xml_rss_atom)
     get("/tags/:tag", Feed.TagController, :feed, as: :tag_feed)
   end
 
diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex
index b1c62f5b0..76b82589f 100644
--- a/lib/pleroma/web/static_fe/static_fe_controller.ex
+++ b/lib/pleroma/web/static_fe/static_fe_controller.ex
@@ -24,6 +24,7 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do
     with %Activity{local: true} = activity <-
            Activity.get_by_id_with_object(notice_id),
          true <- Visibility.is_public?(activity.object),
+         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)},
          %User{} = user <- User.get_by_ap_id(activity.object.data["actor"]) do
       meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user})
 
@@ -47,34 +48,35 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do
 
   @doc "Renders public activities of requested user"
   def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do
-    case User.get_cached_by_nickname_or_id(username_or_id) do
-      %User{} = user ->
-        meta = Metadata.build_tags(%{user: user})
+    with {_, %User{local: true} = user} <-
+           {:fetch_user, User.get_cached_by_nickname_or_id(username_or_id)},
+         {_, :visible} <- {:visibility, User.visible_for(user, _reading_user = nil)} do
+      meta = Metadata.build_tags(%{user: user})
 
-        params =
-          params
-          |> Map.take(@page_keys)
-          |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end)
+      params =
+        params
+        |> Map.take(@page_keys)
+        |> Map.new(fn {k, v} -> {String.to_existing_atom(k), v} end)
 
-        timeline =
-          user
-          |> ActivityPub.fetch_user_activities(_reading_user = nil, params)
-          |> Enum.map(&represent/1)
+      timeline =
+        user
+        |> ActivityPub.fetch_user_activities(_reading_user = nil, params)
+        |> Enum.map(&represent/1)
 
-        prev_page_id =
-          (params["min_id"] || params["max_id"]) &&
-            List.first(timeline) && List.first(timeline).id
+      prev_page_id =
+        (params["min_id"] || params["max_id"]) &&
+          List.first(timeline) && List.first(timeline).id
 
-        next_page_id = List.last(timeline) && List.last(timeline).id
-
-        render(conn, "profile.html", %{
-          user: User.sanitize_html(user),
-          timeline: timeline,
-          prev_page_id: prev_page_id,
-          next_page_id: next_page_id,
-          meta: meta
-        })
+      next_page_id = List.last(timeline) && List.last(timeline).id
 
+      render(conn, "profile.html", %{
+        user: User.sanitize_html(user),
+        timeline: timeline,
+        prev_page_id: prev_page_id,
+        next_page_id: next_page_id,
+        meta: meta
+      })
+    else
       _ ->
         not_found(conn, "User not found.")
     end
diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs
index ab57b6523..9ec13b21f 100644
--- a/test/web/activity_pub/activity_pub_controller_test.exs
+++ b/test/web/activity_pub/activity_pub_controller_test.exs
@@ -33,25 +33,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
 
   setup do: clear_config([:instance, :federating], true)
 
-  defp ensure_federating_or_authenticated(conn, url, user) do
-    Config.put([:instance, :federating], false)
-
-    conn
-    |> get(url)
-    |> response(403)
-
-    conn
-    |> assign(:user, user)
-    |> get(url)
-    |> response(200)
-
-    Config.put([:instance, :federating], true)
-
-    conn
-    |> get(url)
-    |> response(200)
-  end
-
   describe "/relay" do
     setup do: clear_config([:instance, :allow_relay])
 
@@ -175,21 +156,6 @@ test "it returns error when user is not found", %{conn: conn} do
 
       assert response == "Not found"
     end
-
-    test "it requires authentication if instance is NOT federating", %{
-      conn: conn
-    } do
-      user = insert(:user)
-
-      conn =
-        put_req_header(
-          conn,
-          "accept",
-          "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""
-        )
-
-      ensure_federating_or_authenticated(conn, "/users/#{user.nickname}.json", user)
-    end
   end
 
   describe "mastodon compatibility routes" do
@@ -357,18 +323,6 @@ test "cached purged after object deletion", %{conn: conn} do
 
       assert "Not found" == json_response(conn2, :not_found)
     end
-
-    test "it requires authentication if instance is NOT federating", %{
-      conn: conn
-    } do
-      user = insert(:user)
-      note = insert(:note)
-      uuid = String.split(note.data["id"], "/") |> List.last()
-
-      conn = put_req_header(conn, "accept", "application/activity+json")
-
-      ensure_federating_or_authenticated(conn, "/objects/#{uuid}", user)
-    end
   end
 
   describe "/activities/:uuid" do
@@ -440,18 +394,6 @@ test "cached purged after activity deletion", %{conn: conn} do
 
       assert "Not found" == json_response(conn2, :not_found)
     end
-
-    test "it requires authentication if instance is NOT federating", %{
-      conn: conn
-    } do
-      user = insert(:user)
-      activity = insert(:note_activity)
-      uuid = String.split(activity.data["id"], "/") |> List.last()
-
-      conn = put_req_header(conn, "accept", "application/activity+json")
-
-      ensure_federating_or_authenticated(conn, "/activities/#{uuid}", user)
-    end
   end
 
   describe "/inbox" do
@@ -912,15 +854,6 @@ test "it returns an announce activity in a collection", %{conn: conn} do
 
       assert response(conn, 200) =~ announce_activity.data["object"]
     end
-
-    test "it requires authentication if instance is NOT federating", %{
-      conn: conn
-    } do
-      user = insert(:user)
-      conn = put_req_header(conn, "accept", "application/activity+json")
-
-      ensure_federating_or_authenticated(conn, "/users/#{user.nickname}/outbox", user)
-    end
   end
 
   describe "POST /users/:nickname/outbox (C2S)" do
diff --git a/test/web/feed/tag_controller_test.exs b/test/web/feed/tag_controller_test.exs
index 868e40965..e4084b0e5 100644
--- a/test/web/feed/tag_controller_test.exs
+++ b/test/web/feed/tag_controller_test.exs
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.Feed.TagControllerTest do
   import Pleroma.Factory
   import SweetXml
 
+  alias Pleroma.Config
   alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.Feed.FeedView
@@ -15,7 +16,7 @@ defmodule Pleroma.Web.Feed.TagControllerTest do
   setup do: clear_config([:feed])
 
   test "gets a feed (ATOM)", %{conn: conn} do
-    Pleroma.Config.put(
+    Config.put(
       [:feed, :post_title],
       %{max_length: 25, omission: "..."}
     )
@@ -82,7 +83,7 @@ test "gets a feed (ATOM)", %{conn: conn} do
   end
 
   test "gets a feed (RSS)", %{conn: conn} do
-    Pleroma.Config.put(
+    Config.put(
       [:feed, :post_title],
       %{max_length: 25, omission: "..."}
     )
@@ -157,7 +158,7 @@ test "gets a feed (RSS)", %{conn: conn} do
     response =
       conn
       |> put_req_header("accept", "application/rss+xml")
-      |> get(tag_feed_path(conn, :feed, "pleromaart"))
+      |> get(tag_feed_path(conn, :feed, "pleromaart.rss"))
       |> response(200)
 
     xml = parse(response)
@@ -183,14 +184,12 @@ test "gets a feed (RSS)", %{conn: conn} do
   end
 
   describe "private instance" do
-    setup do: clear_config([:instance, :public])
+    setup do: clear_config([:instance, :public], false)
 
     test "returns 404 for tags feed", %{conn: conn} do
-      Config.put([:instance, :public], false)
-
       conn
       |> put_req_header("accept", "application/rss+xml")
-      |> get(tag_feed_path(conn, :feed, "pleromaart"))
+      |> get(tag_feed_path(conn, :feed, "pleromaart.rss"))
       |> response(404)
     end
   end

From 257e059e61b89752bcde9544cb5ae645b167c96b Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 19 Aug 2020 15:31:33 +0400
Subject: [PATCH 009/104] Add account export

---
 lib/pleroma/export.ex | 118 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 118 insertions(+)
 create mode 100644 lib/pleroma/export.ex

diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex
new file mode 100644
index 000000000..82a4b7ace
--- /dev/null
+++ b/lib/pleroma/export.ex
@@ -0,0 +1,118 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Export do
+  alias Pleroma.Activity
+  alias Pleroma.Bookmark
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.ActivityPub.UserView
+
+  import Ecto.Query
+
+  def run(user) do
+    with {:ok, dir} <- create_dir(),
+         :ok <- actor(dir, user),
+         :ok <- statuses(dir, user),
+         :ok <- likes(dir, user),
+         :ok <- bookmarks(dir, user) do
+      IO.inspect({"DONE", dir})
+    else
+      err -> IO.inspect({"export error", err})
+    end
+  end
+
+  def actor(dir, user) do
+    with {:ok, json} <-
+           UserView.render("user.json", %{user: user})
+           |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
+           |> Jason.encode() do
+      File.write(dir <> "/actor.json", json)
+    end
+  end
+
+  defp create_dir do
+    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
+    dir = Path.join(System.tmp_dir!(), "archive-" <> datetime)
+
+    with :ok <- File.mkdir(dir), do: {:ok, dir}
+  end
+
+  defp write_header(file, name) do
+    IO.write(
+      file,
+      """
+      {
+        "@context": "https://www.w3.org/ns/activitystreams",
+        "id": "#{name}.json",
+        "type": "OrderedCollection",
+        "orderedItems": [
+      """
+    )
+  end
+
+  defp write(query, dir, name, fun) do
+    path = dir <> "/#{name}.json"
+
+    with {:ok, file} <- File.open(path, [:write, :utf8]),
+         :ok <- write_header(file, name) do
+      counter = :counters.new(1, [])
+
+      query
+      |> Pleroma.RepoStreamer.chunk_stream(100)
+      |> Stream.each(fn items ->
+        Enum.each(items, fn i ->
+          with {:ok, str} <- fun.(i),
+               :ok <- IO.write(file, str <> ",\n") do
+            :counters.add(counter, 1, 1)
+          end
+        end)
+      end)
+      |> Stream.run()
+
+      total = :counters.get(counter, 1)
+
+      with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n  \"totalItems\": #{total}}") do
+        File.close(file)
+      end
+    end
+  end
+
+  def bookmarks(dir, %{id: user_id} = _user) do
+    Bookmark
+    |> where(user_id: ^user_id)
+    |> join(:inner, [b], activity in assoc(b, :activity))
+    |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
+    |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end)
+  end
+
+  def likes(dir, user) do
+    user.ap_id
+    |> Activity.Queries.by_actor()
+    |> Activity.Queries.by_type("Like")
+    |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
+    |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end)
+  end
+
+  def statuses(dir, user) do
+    opts =
+      %{}
+      |> Map.put(:type, ["Create", "Announce"])
+      |> Map.put(:blocking_user, user)
+      |> Map.put(:muting_user, user)
+      |> Map.put(:reply_filtering_user, user)
+      |> Map.put(:announce_filtering_user, user)
+      |> Map.put(:user, user)
+
+    [[user.ap_id], User.following(user), Pleroma.List.memberships(user)]
+    |> Enum.concat()
+    |> ActivityPub.fetch_activities_query(opts)
+    |> write(dir, "outbox", fn a ->
+      with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
+        activity |> Map.delete("@context") |> Jason.encode()
+      end
+    end)
+  end
+end

From 9d564ffc2988f145bc9cf26477eea93b1bf01cb0 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 24 Aug 2020 20:59:57 +0400
Subject: [PATCH 010/104] Zip exported files

---
 lib/pleroma/export.ex | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex
index 82a4b7ace..f0f1ef093 100644
--- a/lib/pleroma/export.ex
+++ b/lib/pleroma/export.ex
@@ -12,15 +12,17 @@ defmodule Pleroma.Export do
 
   import Ecto.Query
 
+  @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
+
   def run(user) do
-    with {:ok, dir} <- create_dir(),
-         :ok <- actor(dir, user),
-         :ok <- statuses(dir, user),
-         :ok <- likes(dir, user),
-         :ok <- bookmarks(dir, user) do
-      IO.inspect({"DONE", dir})
-    else
-      err -> IO.inspect({"export error", err})
+    with {:ok, path} <- create_dir(user),
+         :ok <- actor(path, user),
+         :ok <- statuses(path, user),
+         :ok <- likes(path, user),
+         :ok <- bookmarks(path, user),
+         {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path),
+         {:ok, _} <- File.rm_rf(path) do
+      {:ok, zip_path}
     end
   end
 
@@ -33,9 +35,9 @@ def actor(dir, user) do
     end
   end
 
-  defp create_dir do
+  defp create_dir(user) do
     datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
-    dir = Path.join(System.tmp_dir!(), "archive-" <> datetime)
+    dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}")
 
     with :ok <- File.mkdir(dir), do: {:ok, dir}
   end

From c01a81804835fb92c145b90e3a264c5d4cf9c886 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 25 Aug 2020 18:51:09 +0400
Subject: [PATCH 011/104] Add tests

---
 lib/pleroma/export.ex |   8 +--
 test/export_test.exs  | 111 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 115 insertions(+), 4 deletions(-)
 create mode 100644 test/export_test.exs

diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex
index f0f1ef093..45b8ce749 100644
--- a/lib/pleroma/export.ex
+++ b/lib/pleroma/export.ex
@@ -26,7 +26,7 @@ def run(user) do
     end
   end
 
-  def actor(dir, user) do
+  defp actor(dir, user) do
     with {:ok, json} <-
            UserView.render("user.json", %{user: user})
            |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
@@ -82,7 +82,7 @@ defp write(query, dir, name, fun) do
     end
   end
 
-  def bookmarks(dir, %{id: user_id} = _user) do
+  defp bookmarks(dir, %{id: user_id} = _user) do
     Bookmark
     |> where(user_id: ^user_id)
     |> join(:inner, [b], activity in assoc(b, :activity))
@@ -90,7 +90,7 @@ def bookmarks(dir, %{id: user_id} = _user) do
     |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end)
   end
 
-  def likes(dir, user) do
+  defp likes(dir, user) do
     user.ap_id
     |> Activity.Queries.by_actor()
     |> Activity.Queries.by_type("Like")
@@ -98,7 +98,7 @@ def likes(dir, user) do
     |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end)
   end
 
-  def statuses(dir, user) do
+  defp statuses(dir, user) do
     opts =
       %{}
       |> Map.put(:type, ["Create", "Announce"])
diff --git a/test/export_test.exs b/test/export_test.exs
new file mode 100644
index 000000000..5afd58ccc
--- /dev/null
+++ b/test/export_test.exs
@@ -0,0 +1,111 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ExportTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Bookmark
+
+  test "it exports user data" do
+    user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
+
+    {:ok, %{object: %{data: %{"id" => id1}}} = status1} =
+      CommonAPI.post(user, %{status: "status1"})
+
+    {:ok, %{object: %{data: %{"id" => id2}}} = status2} =
+      CommonAPI.post(user, %{status: "status2"})
+
+    {:ok, %{object: %{data: %{"id" => id3}}} = status3} =
+      CommonAPI.post(user, %{status: "status3"})
+
+    CommonAPI.favorite(user, status1.id)
+    CommonAPI.favorite(user, status2.id)
+
+    Bookmark.create(user.id, status2.id)
+    Bookmark.create(user.id, status3.id)
+
+    assert {:ok, path} = Pleroma.Export.run(user)
+    assert {:ok, zipfile} = :zip.zip_open(path, [:memory])
+    assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
+
+    assert %{
+             "@context" => [
+               "https://www.w3.org/ns/activitystreams",
+               "http://localhost:4001/schemas/litepub-0.1.jsonld",
+               %{"@language" => "und"}
+             ],
+             "bookmarks" => "bookmarks.json",
+             "followers" => "http://cofe.io/users/cofe/followers",
+             "following" => "http://cofe.io/users/cofe/following",
+             "id" => "http://cofe.io/users/cofe",
+             "inbox" => "http://cofe.io/users/cofe/inbox",
+             "likes" => "likes.json",
+             "name" => "Cofe",
+             "outbox" => "http://cofe.io/users/cofe/outbox",
+             "preferredUsername" => "cofe",
+             "publicKey" => %{
+               "id" => "http://cofe.io/users/cofe#main-key",
+               "owner" => "http://cofe.io/users/cofe"
+             },
+             "type" => "Person",
+             "url" => "http://cofe.io/users/cofe"
+           } = Jason.decode!(json)
+
+    assert {:ok, {'outbox.json', json}} = :zip.zip_get('outbox.json', zipfile)
+
+    assert %{
+             "@context" => "https://www.w3.org/ns/activitystreams",
+             "id" => "outbox.json",
+             "orderedItems" => [
+               %{
+                 "object" => %{
+                   "actor" => "http://cofe.io/users/cofe",
+                   "content" => "status1",
+                   "type" => "Note"
+                 },
+                 "type" => "Create"
+               },
+               %{
+                 "object" => %{
+                   "actor" => "http://cofe.io/users/cofe",
+                   "content" => "status2"
+                 }
+               },
+               %{
+                 "actor" => "http://cofe.io/users/cofe",
+                 "object" => %{
+                   "content" => "status3"
+                 }
+               }
+             ],
+             "totalItems" => 3,
+             "type" => "OrderedCollection"
+           } = Jason.decode!(json)
+
+    assert {:ok, {'likes.json', json}} = :zip.zip_get('likes.json', zipfile)
+
+    assert %{
+             "@context" => "https://www.w3.org/ns/activitystreams",
+             "id" => "likes.json",
+             "orderedItems" => [^id1, ^id2],
+             "totalItems" => 2,
+             "type" => "OrderedCollection"
+           } = Jason.decode!(json)
+
+    assert {:ok, {'bookmarks.json', json}} = :zip.zip_get('bookmarks.json', zipfile)
+
+    assert %{
+             "@context" => "https://www.w3.org/ns/activitystreams",
+             "id" => "bookmarks.json",
+             "orderedItems" => [^id2, ^id3],
+             "totalItems" => 2,
+             "type" => "OrderedCollection"
+           } = Jason.decode!(json)
+
+    :zip.zip_close(zipfile)
+    File.rm!(path)
+  end
+end

From c82f9129592553718be4bd4712a2b1848dd0a447 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 25 Aug 2020 19:16:01 +0400
Subject: [PATCH 012/104] Fix credo warning

---
 test/export_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/export_test.exs b/test/export_test.exs
index 5afd58ccc..01ca8e7e8 100644
--- a/test/export_test.exs
+++ b/test/export_test.exs
@@ -6,8 +6,8 @@ defmodule Pleroma.ExportTest do
   use Pleroma.DataCase
   import Pleroma.Factory
 
-  alias Pleroma.Web.CommonAPI
   alias Pleroma.Bookmark
+  alias Pleroma.Web.CommonAPI
 
   test "it exports user data" do
     user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})

From be42ab70dc9538df54ac6f30ee123666223b7287 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 31 Aug 2020 20:31:21 +0400
Subject: [PATCH 013/104] Add backup upload

---
 lib/pleroma/export.ex | 20 +++++++++++++++++++-
 test/export_test.exs  | 17 ++++++++++++++++-
 2 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex
index 45b8ce749..b84eccd78 100644
--- a/lib/pleroma/export.ex
+++ b/lib/pleroma/export.ex
@@ -22,7 +22,25 @@ def run(user) do
          :ok <- bookmarks(path, user),
          {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path),
          {:ok, _} <- File.rm_rf(path) do
-      {:ok, zip_path}
+      {:ok, :binary.list_to_bin(zip_path)}
+    end
+  end
+
+  def upload(zip_path) do
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+    file_name = zip_path |> String.split("/") |> List.last()
+    id = Ecto.UUID.generate()
+
+    upload = %Pleroma.Upload{
+      id: id,
+      name: file_name,
+      tempfile: zip_path,
+      content_type: "application/zip",
+      path: id <> "/" <> file_name
+    }
+
+    with :ok <- uploader.put_file(upload), :ok <- File.rm(zip_path) do
+      {:ok, upload}
     end
   end
 
diff --git a/test/export_test.exs b/test/export_test.exs
index 01ca8e7e8..fae269974 100644
--- a/test/export_test.exs
+++ b/test/export_test.exs
@@ -28,7 +28,7 @@ test "it exports user data" do
     Bookmark.create(user.id, status3.id)
 
     assert {:ok, path} = Pleroma.Export.run(user)
-    assert {:ok, zipfile} = :zip.zip_open(path, [:memory])
+    assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
     assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
 
     assert %{
@@ -108,4 +108,19 @@ test "it exports user data" do
     :zip.zip_close(zipfile)
     File.rm!(path)
   end
+
+  test "it uploads an exported backup archive" do
+    user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
+
+    {:ok, status1} = CommonAPI.post(user, %{status: "status1"})
+    {:ok, status2} = CommonAPI.post(user, %{status: "status2"})
+    {:ok, status3} = CommonAPI.post(user, %{status: "status3"})
+    CommonAPI.favorite(user, status1.id)
+    CommonAPI.favorite(user, status2.id)
+    Bookmark.create(user.id, status2.id)
+    Bookmark.create(user.id, status3.id)
+
+    assert {:ok, path} = Pleroma.Export.run(user)
+    assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+  end
 end

From 75e07ba206b94155c5210151a49e29a11bce6e50 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 31 Aug 2020 23:07:14 +0400
Subject: [PATCH 014/104] Fix tests

---
 lib/pleroma/export.ex |  3 ++-
 test/export_test.exs  | 47 +++++++++++++++++++++++++++++++++----------
 2 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex
index b84eccd78..8b1bfefe2 100644
--- a/lib/pleroma/export.ex
+++ b/lib/pleroma/export.ex
@@ -39,7 +39,8 @@ def upload(zip_path) do
       path: id <> "/" <> file_name
     }
 
-    with :ok <- uploader.put_file(upload), :ok <- File.rm(zip_path) do
+    with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
+         :ok <- File.rm(zip_path) do
       {:ok, upload}
     end
   end
diff --git a/test/export_test.exs b/test/export_test.exs
index fae269974..d7e8f558c 100644
--- a/test/export_test.exs
+++ b/test/export_test.exs
@@ -5,6 +5,7 @@
 defmodule Pleroma.ExportTest do
   use Pleroma.DataCase
   import Pleroma.Factory
+  import Mock
 
   alias Pleroma.Bookmark
   alias Pleroma.Web.CommonAPI
@@ -109,18 +110,42 @@ test "it exports user data" do
     File.rm!(path)
   end
 
-  test "it uploads an exported backup archive" do
-    user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
+  describe "it uploads an exported backup archive" do
+    setup do
+      clear_config(Pleroma.Uploaders.S3,
+        bucket: "test_bucket",
+        public_endpoint: "https://s3.amazonaws.com"
+      )
 
-    {:ok, status1} = CommonAPI.post(user, %{status: "status1"})
-    {:ok, status2} = CommonAPI.post(user, %{status: "status2"})
-    {:ok, status3} = CommonAPI.post(user, %{status: "status3"})
-    CommonAPI.favorite(user, status1.id)
-    CommonAPI.favorite(user, status2.id)
-    Bookmark.create(user.id, status2.id)
-    Bookmark.create(user.id, status3.id)
+      clear_config([Pleroma.Upload, :uploader])
 
-    assert {:ok, path} = Pleroma.Export.run(user)
-    assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+      user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
+
+      {:ok, status1} = CommonAPI.post(user, %{status: "status1"})
+      {:ok, status2} = CommonAPI.post(user, %{status: "status2"})
+      {:ok, status3} = CommonAPI.post(user, %{status: "status3"})
+      CommonAPI.favorite(user, status1.id)
+      CommonAPI.favorite(user, status2.id)
+      Bookmark.create(user.id, status2.id)
+      Bookmark.create(user.id, status3.id)
+
+      assert {:ok, path} = Pleroma.Export.run(user)
+
+      [path: path]
+    end
+
+    test "S3", %{path: path} do
+      Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
+
+      with_mock ExAws, request: fn _ -> {:ok, :ok} end do
+        assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+      end
+    end
+
+    test "Local", %{path: path} do
+      Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+
+      assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+    end
   end
 end

From 4f3a6337454807f4145bbc1830c3d55dd883d46d Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 2 Sep 2020 20:21:33 +0400
Subject: [PATCH 015/104] Add `backups` table

---
 lib/pleroma/{export.ex => backup.ex}          | 110 ++++++++++++++----
 .../20200831192323_create_backups.exs         |  17 +++
 test/{export_test.exs => backup_test.exs}     |  48 ++++++--
 3 files changed, 141 insertions(+), 34 deletions(-)
 rename lib/pleroma/{export.ex => backup.ex} (60%)
 create mode 100644 priv/repo/migrations/20200831192323_create_backups.exs
 rename test/{export_test.exs => backup_test.exs} (75%)

diff --git a/lib/pleroma/export.ex b/lib/pleroma/backup.ex
similarity index 60%
rename from lib/pleroma/export.ex
rename to lib/pleroma/backup.ex
index 8b1bfefe2..4580d8f92 100644
--- a/lib/pleroma/export.ex
+++ b/lib/pleroma/backup.ex
@@ -2,41 +2,110 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Export do
+defmodule Pleroma.Backup do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+
   alias Pleroma.Activity
   alias Pleroma.Bookmark
+  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.UserView
 
-  import Ecto.Query
+  schema "backups" do
+    field(:content_type, :string)
+    field(:file_name, :string)
+    field(:file_size, :integer, default: 0)
+    field(:processed, :boolean, default: false)
+
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+
+    timestamps()
+  end
+
+  def create(user) do
+    with :ok <- validate_limit(user),
+         {:ok, backup} <- user |> new() |> Repo.insert() do
+      {:ok, backup}
+    end
+  end
+
+  def new(user) do
+    rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
+    name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
+
+    %__MODULE__{
+      user_id: user.id,
+      content_type: "application/zip",
+      file_name: name
+    }
+  end
+
+  defp validate_limit(user) do
+    case get_last(user.id) do
+      %__MODULE__{inserted_at: inserted_at} ->
+        days = 7
+        diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
+
+        if diff > days do
+          :ok
+        else
+          {:error, "Last export was less than #{days} days ago"}
+        end
+
+      nil ->
+        :ok
+    end
+  end
+
+  def get_last(user_id) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> limit(1)
+    |> Repo.one()
+  end
+
+  def process(%__MODULE__{} = backup) do
+    with {:ok, zip_file} <- zip(backup),
+         {:ok, %{size: size}} <- File.stat(zip_file),
+         {:ok, _upload} <- upload(backup, zip_file) do
+      backup
+      |> cast(%{file_size: size, processed: true}, [:file_size, :processed])
+      |> Repo.update()
+    end
+  end
 
   @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
+  def zip(%__MODULE__{} = backup) do
+    backup = Repo.preload(backup, :user)
+    name = String.trim_trailing(backup.file_name, ".zip")
+    dir = Path.join(System.tmp_dir!(), name)
 
-  def run(user) do
-    with {:ok, path} <- create_dir(user),
-         :ok <- actor(path, user),
-         :ok <- statuses(path, user),
-         :ok <- likes(path, user),
-         :ok <- bookmarks(path, user),
-         {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path),
-         {:ok, _} <- File.rm_rf(path) do
+    with :ok <- File.mkdir(dir),
+         :ok <- actor(dir, backup.user),
+         :ok <- statuses(dir, backup.user),
+         :ok <- likes(dir, backup.user),
+         :ok <- bookmarks(dir, backup.user),
+         {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
+         {:ok, _} <- File.rm_rf(dir) do
       {:ok, :binary.list_to_bin(zip_path)}
     end
   end
 
-  def upload(zip_path) do
+  def upload(%__MODULE__{} = backup, zip_path) do
     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
-    file_name = zip_path |> String.split("/") |> List.last()
-    id = Ecto.UUID.generate()
 
     upload = %Pleroma.Upload{
-      id: id,
-      name: file_name,
+      name: backup.file_name,
       tempfile: zip_path,
-      content_type: "application/zip",
-      path: id <> "/" <> file_name
+      content_type: backup.content_type,
+      path: "backups/" <> backup.file_name
     }
 
     with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
@@ -54,13 +123,6 @@ defp actor(dir, user) do
     end
   end
 
-  defp create_dir(user) do
-    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
-    dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}")
-
-    with :ok <- File.mkdir(dir), do: {:ok, dir}
-  end
-
   defp write_header(file, name) do
     IO.write(
       file,
diff --git a/priv/repo/migrations/20200831192323_create_backups.exs b/priv/repo/migrations/20200831192323_create_backups.exs
new file mode 100644
index 000000000..3ac5889e2
--- /dev/null
+++ b/priv/repo/migrations/20200831192323_create_backups.exs
@@ -0,0 +1,17 @@
+defmodule Pleroma.Repo.Migrations.CreateBackups do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists table(:backups) do
+      add(:user_id, references(:users, type: :uuid, on_delete: :delete_all))
+      add(:file_name, :string, null: false)
+      add(:content_type, :string, null: false)
+      add(:processed, :boolean, null: false, default: false)
+      add(:file_size, :bigint)
+
+      timestamps()
+    end
+
+    create_if_not_exists(index(:backups, [:user_id]))
+  end
+end
diff --git a/test/export_test.exs b/test/backup_test.exs
similarity index 75%
rename from test/export_test.exs
rename to test/backup_test.exs
index d7e8f558c..27f5cb7f7 100644
--- a/test/export_test.exs
+++ b/test/backup_test.exs
@@ -2,15 +2,41 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.ExportTest do
+defmodule Pleroma.BackupTest do
   use Pleroma.DataCase
   import Pleroma.Factory
   import Mock
 
+  alias Pleroma.Backup
   alias Pleroma.Bookmark
   alias Pleroma.Web.CommonAPI
 
-  test "it exports user data" do
+  test "it creates a backup record" do
+    %{id: user_id} = user = insert(:user)
+    assert {:ok, backup} = Backup.create(user)
+
+    assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
+  end
+
+  test "it return an error if the export limit is over" do
+    %{id: user_id} = user = insert(:user)
+    limit_days = 7
+
+    assert {:ok, backup} = Backup.create(user)
+    assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
+
+    assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"}
+  end
+
+  test "it process a backup record" do
+    %{id: user_id} = user = insert(:user)
+    assert {:ok, %{id: backup_id} = backup} = Backup.create(user)
+    assert {:ok, %Backup{} = backup} = Backup.process(backup)
+    assert backup.file_size > 0
+    assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
+  end
+
+  test "it creates a zip archive with user data" do
     user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
 
     {:ok, %{object: %{data: %{"id" => id1}}} = status1} =
@@ -28,7 +54,8 @@ test "it exports user data" do
     Bookmark.create(user.id, status2.id)
     Bookmark.create(user.id, status3.id)
 
-    assert {:ok, path} = Pleroma.Export.run(user)
+    assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
+    assert {:ok, path} = Backup.zip(backup)
     assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
     assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
 
@@ -110,7 +137,7 @@ test "it exports user data" do
     File.rm!(path)
   end
 
-  describe "it uploads an exported backup archive" do
+  describe "it uploads a backup archive" do
     setup do
       clear_config(Pleroma.Uploaders.S3,
         bucket: "test_bucket",
@@ -129,23 +156,24 @@ test "it exports user data" do
       Bookmark.create(user.id, status2.id)
       Bookmark.create(user.id, status3.id)
 
-      assert {:ok, path} = Pleroma.Export.run(user)
+      assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
+      assert {:ok, path} = Backup.zip(backup)
 
-      [path: path]
+      [path: path, backup: backup]
     end
 
-    test "S3", %{path: path} do
+    test "S3", %{path: path, backup: backup} do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
 
       with_mock ExAws, request: fn _ -> {:ok, :ok} end do
-        assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+        assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
       end
     end
 
-    test "Local", %{path: path} do
+    test "Local", %{path: path, backup: backup} do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
 
-      assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path)
+      assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
     end
   end
 end

From a0ad9bd734e9af0ce912c32c7480a60ff87a4368 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 2 Sep 2020 21:45:22 +0400
Subject: [PATCH 016/104] Add BackupWorker

---
 config/config.exs                    |  1 +
 config/description.exs               |  6 ++++++
 lib/pleroma/backup.ex                | 11 ++++++++++-
 lib/pleroma/workers/backup_worker.ex | 17 +++++++++++++++++
 test/backup_test.exs                 | 20 ++++++++++++++------
 5 files changed, 48 insertions(+), 7 deletions(-)
 create mode 100644 lib/pleroma/workers/backup_worker.ex

diff --git a/config/config.exs b/config/config.exs
index 2e6b0796a..1f10167e5 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -551,6 +551,7 @@
   queues: [
     activity_expiration: 10,
     token_expiration: 5,
+    backup: 1,
     federator_incoming: 50,
     federator_outgoing: 50,
     ingestion_queue: 50,
diff --git a/config/description.exs b/config/description.exs
index 6fa78a5d1..13e44afe8 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -2288,6 +2288,12 @@
             description: "Activity expiration queue",
             suggestions: [10]
           },
+          %{
+            key: :backup,
+            type: :integer,
+            description: "Backup queue",
+            suggestions: [1]
+          },
           %{
             key: :attachments_cleanup,
             type: :integer,
diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 4580d8f92..9b5d2625f 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -30,7 +30,7 @@ defmodule Pleroma.Backup do
   def create(user) do
     with :ok <- validate_limit(user),
          {:ok, backup} <- user |> new() |> Repo.insert() do
-      {:ok, backup}
+      Pleroma.Workers.BackupWorker.enqueue("process", %{"backup_id" => backup.id})
     end
   end
 
@@ -71,6 +71,15 @@ def get_last(user_id) do
     |> Repo.one()
   end
 
+  def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> where([b], b.id != ^latest_id)
+    |> Repo.delete_all()
+  end
+
+  def get(id), do: Repo.get(__MODULE__, id)
+
   def process(%__MODULE__{} = backup) do
     with {:ok, zip_file} <- zip(backup),
          {:ok, %{size: size}} <- File.stat(zip_file),
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
new file mode 100644
index 000000000..c982ffa3a
--- /dev/null
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -0,0 +1,17 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.BackupWorker do
+  alias Pleroma.Backup
+
+  use Pleroma.Workers.WorkerHelper, queue: "backup"
+
+  @impl Oban.Worker
+  def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do
+    with {:ok, %Backup{} = backup} <-
+           backup_id |> Backup.get() |> Backup.process() do
+      {:ok, backup}
+    end
+  end
+end
diff --git a/test/backup_test.exs b/test/backup_test.exs
index 27f5cb7f7..5b1f76dd9 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -3,35 +3,43 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.BackupTest do
+  use Oban.Testing, repo: Pleroma.Repo
   use Pleroma.DataCase
+
   import Pleroma.Factory
   import Mock
 
   alias Pleroma.Backup
   alias Pleroma.Bookmark
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Workers.BackupWorker
 
-  test "it creates a backup record" do
+  setup do: clear_config([Pleroma.Upload, :uploader])
+
+  test "it creates a backup record and an Oban job" do
     %{id: user_id} = user = insert(:user)
-    assert {:ok, backup} = Backup.create(user)
+    assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
+    assert_enqueued(worker: BackupWorker, args: args)
 
+    backup = Backup.get(args["backup_id"])
     assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
   end
 
   test "it return an error if the export limit is over" do
     %{id: user_id} = user = insert(:user)
     limit_days = 7
-
-    assert {:ok, backup} = Backup.create(user)
+    assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
+    backup = Backup.get(args["backup_id"])
     assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
 
     assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"}
   end
 
   test "it process a backup record" do
+    Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
     %{id: user_id} = user = insert(:user)
-    assert {:ok, %{id: backup_id} = backup} = Backup.create(user)
-    assert {:ok, %Backup{} = backup} = Backup.process(backup)
+    assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}} = job} = Backup.create(user)
+    assert {:ok, backup} = BackupWorker.perform(job)
     assert backup.file_size > 0
     assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
   end

From 3ad7492f9dd1c76cdbc64ad2246f8e9c8c5c4ae6 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 4 Sep 2020 18:30:39 +0400
Subject: [PATCH 017/104] Add config for Pleroma.Backup

---
 config/config.exs                |  4 ++++
 config/description.exs           | 20 ++++++++++++++++++++
 docs/configuration/cheatsheet.md |  5 +++++
 lib/pleroma/backup.ex            |  2 +-
 test/backup_test.exs             |  2 +-
 5 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 1f10167e5..09023e2c3 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -818,6 +818,10 @@
 
 config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
 
+config :pleroma, Pleroma.Backup,
+  purge_after_days: 30,
+  limit_days: 7
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/config/description.exs b/config/description.exs
index 13e44afe8..4942e196d 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3712,5 +3712,25 @@
         ]
       }
     ]
+  },
+  %{
+    group: :pleroma,
+    key: Pleroma.Backup,
+    type: :group,
+    description: "Account Backup",
+    children: [
+      %{
+        key: :purge_after_days,
+        type: :integer,
+        description: "Remove backup achives after N days",
+        suggestions: [30]
+      },
+      %{
+        key: :limit_days,
+        type: :integer,
+        description: "Limit user to export not more often than once per N days",
+        suggestions: [7]
+      }
+    ]
   }
 ]
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 42e5fe808..cc4081f14 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -1083,6 +1083,11 @@ Control favicons for instances.
 
 * `enabled`: Allow/disallow displaying and getting instances favicons
 
+## Account Backup
+
+* `:purge_after_days` an integer, remove backup achives after N days.
+* `:limit_days` an integer, limit user to export not more often than once per N days.
+
 ## Frontend management
 
 Frontends in Pleroma are swappable - you can specify which one to use here.
diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 9b5d2625f..e384b6b00 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -49,7 +49,7 @@ def new(user) do
   defp validate_limit(user) do
     case get_last(user.id) do
       %__MODULE__{inserted_at: inserted_at} ->
-        days = 7
+        days = Pleroma.Config.get([Pleroma.Backup, :limit_days])
         diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
 
         if diff > days do
diff --git a/test/backup_test.exs b/test/backup_test.exs
index 5b1f76dd9..f343b0361 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -27,7 +27,7 @@ test "it creates a backup record and an Oban job" do
 
   test "it return an error if the export limit is over" do
     %{id: user_id} = user = insert(:user)
-    limit_days = 7
+    limit_days = Pleroma.Config.get([Pleroma.Backup, :limit_days])
     assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
     backup = Backup.get(args["backup_id"])
     assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup

From 739cb1463ba07513f047b2ac8f7e22a16c89ef4e Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 4 Sep 2020 21:48:52 +0400
Subject: [PATCH 018/104] Add backups deletion

---
 lib/pleroma/backup.ex                | 14 +++++++--
 lib/pleroma/workers/backup_worker.ex | 37 ++++++++++++++++++++--
 test/backup_test.exs                 | 47 +++++++++++++++++++++++++---
 test/support/oban_helpers.ex         |  3 ++
 4 files changed, 91 insertions(+), 10 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index e384b6b00..bd50fd910 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -15,6 +15,7 @@ defmodule Pleroma.Backup do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.UserView
+  alias Pleroma.Workers.BackupWorker
 
   schema "backups" do
     field(:content_type, :string)
@@ -30,7 +31,7 @@ defmodule Pleroma.Backup do
   def create(user) do
     with :ok <- validate_limit(user),
          {:ok, backup} <- user |> new() |> Repo.insert() do
-      Pleroma.Workers.BackupWorker.enqueue("process", %{"backup_id" => backup.id})
+      BackupWorker.process(backup)
     end
   end
 
@@ -46,6 +47,14 @@ def new(user) do
     }
   end
 
+  def delete(backup) do
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+    with :ok <- uploader.delete_file("backups/" <> backup.file_name) do
+      Repo.delete(backup)
+    end
+  end
+
   defp validate_limit(user) do
     case get_last(user.id) do
       %__MODULE__{inserted_at: inserted_at} ->
@@ -75,7 +84,8 @@ def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
     __MODULE__
     |> where(user_id: ^user_id)
     |> where([b], b.id != ^latest_id)
-    |> Repo.delete_all()
+    |> Repo.all()
+    |> Enum.each(&BackupWorker.delete/1)
   end
 
   def get(id), do: Repo.get(__MODULE__, id)
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
index c982ffa3a..f40020794 100644
--- a/lib/pleroma/workers/backup_worker.ex
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -3,15 +3,46 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.BackupWorker do
+  use Oban.Worker, queue: :backup, max_attempts: 1
+
+  alias Oban.Job
   alias Pleroma.Backup
 
-  use Pleroma.Workers.WorkerHelper, queue: "backup"
+  def process(backup) do
+    %{"op" => "process", "backup_id" => backup.id}
+    |> new()
+    |> Oban.insert()
+  end
+
+  def schedule_deletion(backup) do
+    days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days])
+    time = 60 * 60 * 24 * days
+    scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time)
+
+    %{"op" => "delete", "backup_id" => backup.id}
+    |> new(scheduled_at: scheduled_at)
+    |> Oban.insert()
+  end
+
+  def delete(backup) do
+    %{"op" => "delete", "backup_id" => backup.id}
+    |> new()
+    |> Oban.insert()
+  end
 
-  @impl Oban.Worker
   def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do
     with {:ok, %Backup{} = backup} <-
-           backup_id |> Backup.get() |> Backup.process() do
+           backup_id |> Backup.get() |> Backup.process(),
+         {:ok, _job} <- schedule_deletion(backup),
+         :ok <- Backup.remove_outdated(backup) do
       {:ok, backup}
     end
   end
+
+  def perform(%Job{args: %{"op" => "delete", "backup_id" => backup_id}}) do
+    case Backup.get(backup_id) do
+      %Backup{} = backup -> Backup.delete(backup)
+      nil -> :ok
+    end
+  end
 end
diff --git a/test/backup_test.exs b/test/backup_test.exs
index f343b0361..59aebe360 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -13,8 +13,12 @@ defmodule Pleroma.BackupTest do
   alias Pleroma.Bookmark
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Workers.BackupWorker
+  alias Pleroma.Tests.ObanHelpers
 
-  setup do: clear_config([Pleroma.Upload, :uploader])
+  setup do
+    clear_config([Pleroma.Upload, :uploader])
+    clear_config([Pleroma.Backup, :limit_days])
+  end
 
   test "it creates a backup record and an Oban job" do
     %{id: user_id} = user = insert(:user)
@@ -38,10 +42,34 @@ test "it return an error if the export limit is over" do
   test "it process a backup record" do
     Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
     %{id: user_id} = user = insert(:user)
-    assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}} = job} = Backup.create(user)
-    assert {:ok, backup} = BackupWorker.perform(job)
+
+    assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user)
+    assert {:ok, backup} = perform_job(BackupWorker, args)
     assert backup.file_size > 0
     assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
+
+    delete_job_args = %{"op" => "delete", "backup_id" => backup_id}
+
+    assert_enqueued(worker: BackupWorker, args: delete_job_args)
+    assert {:ok, backup} = perform_job(BackupWorker, delete_job_args)
+    refute Backup.get(backup_id)
+  end
+
+  test "it removes outdated backups after creating a fresh one" do
+    Pleroma.Config.put([Pleroma.Backup, :limit_days], -1)
+    Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+    user = insert(:user)
+
+    assert {:ok, job1} = Backup.create(user)
+
+    assert {:ok, %Backup{id: backup1_id}} = ObanHelpers.perform(job1)
+    assert {:ok, job2} = Backup.create(user)
+    assert Pleroma.Repo.aggregate(Backup, :count) == 2
+    assert {:ok, backup2} = ObanHelpers.perform(job2)
+
+    ObanHelpers.perform_all()
+
+    assert [^backup2] = Pleroma.Repo.all(Backup)
   end
 
   test "it creates a zip archive with user data" do
@@ -145,7 +173,7 @@ test "it creates a zip archive with user data" do
     File.rm!(path)
   end
 
-  describe "it uploads a backup archive" do
+  describe "it uploads and deletes a backup archive" do
     setup do
       clear_config(Pleroma.Uploaders.S3,
         bucket: "test_bucket",
@@ -173,8 +201,16 @@ test "it creates a zip archive with user data" do
     test "S3", %{path: path, backup: backup} do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
 
-      with_mock ExAws, request: fn _ -> {:ok, :ok} end do
+      with_mock ExAws,
+        request: fn
+          %{http_method: :put} -> {:ok, :ok}
+          %{http_method: :delete} -> {:ok, %{status_code: 204}}
+        end do
         assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
+        assert {:ok, _backup} = Backup.delete(backup)
+      end
+
+      with_mock ExAws, request: fn %{http_method: :delete} -> {:ok, %{status_code: 204}} end do
       end
     end
 
@@ -182,6 +218,7 @@ test "Local", %{path: path, backup: backup} do
       Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
 
       assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
+      assert {:ok, _backup} = Backup.delete(backup)
     end
   end
 end
diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex
index 9f90a821c..2468f66dc 100644
--- a/test/support/oban_helpers.ex
+++ b/test/support/oban_helpers.ex
@@ -7,6 +7,8 @@ defmodule Pleroma.Tests.ObanHelpers do
   Oban test helpers.
   """
 
+  require Ecto.Query
+
   alias Pleroma.Repo
 
   def wipe_all do
@@ -15,6 +17,7 @@ def wipe_all do
 
   def perform_all do
     Oban.Job
+    |> Ecto.Query.where(state: "available")
     |> Repo.all()
     |> perform()
   end

From abdffc6b8c2eec8f81ffe89f943f11d1f90d7074 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 4 Sep 2020 22:00:26 +0400
Subject: [PATCH 019/104] Fix Credo warning

---
 test/backup_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/backup_test.exs b/test/backup_test.exs
index 59aebe360..5fc519eab 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -11,9 +11,9 @@ defmodule Pleroma.BackupTest do
 
   alias Pleroma.Backup
   alias Pleroma.Bookmark
+  alias Pleroma.Tests.ObanHelpers
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Workers.BackupWorker
-  alias Pleroma.Tests.ObanHelpers
 
   setup do
     clear_config([Pleroma.Upload, :uploader])

From 2c73bfe1227065fa203b0b78c9eb12cf86ab3948 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 9 Sep 2020 01:04:00 +0400
Subject: [PATCH 020/104] Add API endpoints for Backups

---
 lib/pleroma/backup.ex                         |  7 ++
 .../operations/pleroma_backup_operation.ex    | 79 +++++++++++++++++
 .../controllers/backup_controller.ex          | 27 ++++++
 .../web/pleroma_api/views/backup_view.ex      | 24 ++++++
 lib/pleroma/web/router.ex                     |  3 +
 .../controllers/backup_controller_test.exs    | 84 +++++++++++++++++++
 6 files changed, 224 insertions(+)
 create mode 100644 lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
 create mode 100644 lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
 create mode 100644 lib/pleroma/web/pleroma_api/views/backup_view.ex
 create mode 100644 test/web/pleroma_api/controllers/backup_controller_test.exs

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index bd50fd910..348e537a8 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -80,6 +80,13 @@ def get_last(user_id) do
     |> Repo.one()
   end
 
+  def list(%User{id: user_id}) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> Repo.all()
+  end
+
   def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
     __MODULE__
     |> where(user_id: ^user_id)
diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
new file mode 100644
index 000000000..f877ca31b
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
@@ -0,0 +1,79 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Backups"],
+      summary: "List backups",
+      security: [%{"oAuth" => ["read:account"]}],
+      operationId: "PleromaAPI.BackupController.index",
+      responses: %{
+        200 =>
+          Operation.response(
+            "An array of backups",
+            "application/json",
+            %Schema{
+              type: :array,
+              items: backup()
+            }
+          ),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Backups"],
+      summary: "Create a backup",
+      security: [%{"oAuth" => ["read:account"]}],
+      operationId: "PleromaAPI.BackupController.create",
+      responses: %{
+        200 =>
+          Operation.response(
+            "An array of backups",
+            "application/json",
+            %Schema{
+              type: :array,
+              items: backup()
+            }
+          ),
+        400 => Operation.response("Bad Request", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp backup do
+    %Schema{
+      title: "Backup",
+      description: "Response schema for a backup",
+      type: :object,
+      properties: %{
+        inserted_at: %Schema{type: :string, format: :"date-time"},
+        content_type: %Schema{type: :string},
+        file_name: %Schema{type: :string},
+        file_size: %Schema{type: :integer},
+        processed: %Schema{type: :boolean}
+      },
+      example: %{
+        "content_type" => "application/zip",
+        "file_name" =>
+          "archive-cofe-20200908T195819-1lWrJyJqpsj8-KuHFr7N03lfsYYa5nf2NL-7A9-ddFU.zip",
+        "file_size" => 1024,
+        "inserted_at" => "2020-09-08T19:58:20",
+        "processed" => true
+      }
+    }
+  end
+end
diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
new file mode 100644
index 000000000..e52c77ff2
--- /dev/null
+++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
@@ -0,0 +1,27 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.BackupController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Plugs.OAuthScopesPlug
+
+  action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
+  plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create])
+  plug(OpenApiSpex.Plug.CastAndValidate, render_error: Pleroma.Web.ApiSpec.RenderError)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation
+
+  def index(%{assigns: %{user: user}} = conn, _params) do
+    backups = Pleroma.Backup.list(user)
+    render(conn, "index.json", backups: backups)
+  end
+
+  def create(%{assigns: %{user: user}} = conn, _params) do
+    with {:ok, _} <- Pleroma.Backup.create(user) do
+      backups = Pleroma.Backup.list(user)
+      render(conn, "index.json", backups: backups)
+    end
+  end
+end
diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex
new file mode 100644
index 000000000..02b94ce4f
--- /dev/null
+++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex
@@ -0,0 +1,24 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.BackupView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Backup
+  alias Pleroma.Web.CommonAPI.Utils
+
+  def render("show.json", %{backup: %Backup{} = backup}) do
+    %{
+      content_type: backup.content_type,
+      file_name: backup.file_name,
+      file_size: backup.file_size,
+      processed: backup.processed,
+      inserted_at: Utils.to_masto_date(backup.inserted_at)
+    }
+  end
+
+  def render("index.json", %{backups: backups}) do
+    render_many(backups, __MODULE__, "show.json")
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index e22b31b4c..a1a5a1cb5 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -293,6 +293,9 @@ defmodule Pleroma.Web.Router do
     get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup)
     post("/accounts/mfa/confirm/:method", TwoFactorAuthenticationController, :confirm)
     delete("/accounts/mfa/:method", TwoFactorAuthenticationController, :disable)
+
+    get("/backups", BackupController, :index)
+    post("/backups", BackupController, :create)
   end
 
   scope "/oauth", Pleroma.Web.OAuth do
diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs
new file mode 100644
index 000000000..1ad1b63c4
--- /dev/null
+++ b/test/web/pleroma_api/controllers/backup_controller_test.exs
@@ -0,0 +1,84 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do
+  use Pleroma.Web.ConnCase
+
+  alias Pleroma.Backup
+
+  setup do
+    clear_config([Pleroma.Upload, :uploader])
+    clear_config([Backup, :limit_days])
+    oauth_access(["read:accounts"])
+  end
+
+  test "GET /api/pleroma/backups", %{user: user, conn: conn} do
+    assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user)
+
+    backup = Backup.get(backup_id)
+
+    response =
+      conn
+      |> get("/api/pleroma/backups")
+      |> json_response_and_validate_schema(:ok)
+
+    assert [
+             %{
+               "content_type" => "application/zip",
+               "file_name" => file_name,
+               "file_size" => 0,
+               "processed" => false,
+               "inserted_at" => _
+             }
+           ] = response
+
+    assert file_name == backup.file_name
+
+    Pleroma.Tests.ObanHelpers.perform_all()
+
+    assert [
+             %{
+               "file_name" => ^file_name,
+               "processed" => true
+             }
+           ] =
+             conn
+             |> get("/api/pleroma/backups")
+             |> json_response_and_validate_schema(:ok)
+  end
+
+  test "POST /api/pleroma/backups", %{user: _user, conn: conn} do
+    assert [
+             %{
+               "content_type" => "application/zip",
+               "file_name" => file_name,
+               "file_size" => 0,
+               "processed" => false,
+               "inserted_at" => _
+             }
+           ] =
+             conn
+             |> post("/api/pleroma/backups")
+             |> json_response_and_validate_schema(:ok)
+
+    Pleroma.Tests.ObanHelpers.perform_all()
+
+    assert [
+             %{
+               "file_name" => ^file_name,
+               "processed" => true
+             }
+           ] =
+             conn
+             |> get("/api/pleroma/backups")
+             |> json_response_and_validate_schema(:ok)
+
+    days = Pleroma.Config.get([Backup, :limit_days])
+
+    assert %{"error" => "Last export was less than #{days} days ago"} ==
+             conn
+             |> post("/api/pleroma/backups")
+             |> json_response_and_validate_schema(400)
+  end
+end

From 86ce4afd9338d81f741fa57f962509a6f0f50aff Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 9 Sep 2020 20:02:20 +0400
Subject: [PATCH 021/104] Improve backup urls

---
 .../api_spec/operations/pleroma_backup_operation.ex   |  6 +++---
 lib/pleroma/web/pleroma_api/views/backup_view.ex      |  6 +++++-
 .../controllers/backup_controller_test.exs            | 11 ++++++-----
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
index f877ca31b..6993794db 100644
--- a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
@@ -69,9 +69,9 @@ defp backup do
       example: %{
         "content_type" => "application/zip",
         "file_name" =>
-          "archive-cofe-20200908T195819-1lWrJyJqpsj8-KuHFr7N03lfsYYa5nf2NL-7A9-ddFU.zip",
-        "file_size" => 1024,
-        "inserted_at" => "2020-09-08T19:58:20",
+          "https://cofe.fe:4000/media/backups/archive-foobar-20200908T164207-Yr7vuT5Wycv-sN3kSN2iJ0k-9pMo60j9qmvRCdDqIew.zip",
+        "file_size" => 4105,
+        "inserted_at" => "2020-09-08T16:42:07.000Z",
         "processed" => true
       }
     }
diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex
index 02b94ce4f..bf40a001e 100644
--- a/lib/pleroma/web/pleroma_api/views/backup_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do
   def render("show.json", %{backup: %Backup{} = backup}) do
     %{
       content_type: backup.content_type,
-      file_name: backup.file_name,
+      url: download_url(backup),
       file_size: backup.file_size,
       processed: backup.processed,
       inserted_at: Utils.to_masto_date(backup.inserted_at)
@@ -21,4 +21,8 @@ def render("show.json", %{backup: %Backup{} = backup}) do
   def render("index.json", %{backups: backups}) do
     render_many(backups, __MODULE__, "show.json")
   end
+
+  def download_url(%Backup{file_name: file_name}) do
+    Pleroma.Web.Endpoint.url() <> "/media/backups/" <> file_name
+  end
 end
diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs
index 1ad1b63c4..5d2f1206e 100644
--- a/test/web/pleroma_api/controllers/backup_controller_test.exs
+++ b/test/web/pleroma_api/controllers/backup_controller_test.exs
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do
   use Pleroma.Web.ConnCase
 
   alias Pleroma.Backup
+  alias Pleroma.Web.PleromaAPI.BackupView
 
   setup do
     clear_config([Pleroma.Upload, :uploader])
@@ -26,20 +27,20 @@ test "GET /api/pleroma/backups", %{user: user, conn: conn} do
     assert [
              %{
                "content_type" => "application/zip",
-               "file_name" => file_name,
+               "url" => url,
                "file_size" => 0,
                "processed" => false,
                "inserted_at" => _
              }
            ] = response
 
-    assert file_name == backup.file_name
+    assert url == BackupView.download_url(backup)
 
     Pleroma.Tests.ObanHelpers.perform_all()
 
     assert [
              %{
-               "file_name" => ^file_name,
+               "url" => ^url,
                "processed" => true
              }
            ] =
@@ -52,7 +53,7 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do
     assert [
              %{
                "content_type" => "application/zip",
-               "file_name" => file_name,
+               "url" => url,
                "file_size" => 0,
                "processed" => false,
                "inserted_at" => _
@@ -66,7 +67,7 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do
 
     assert [
              %{
-               "file_name" => ^file_name,
+               "url" => ^url,
                "processed" => true
              }
            ] =

From cd13613db3f675b6a9171dea56fc5b03e43ae6b0 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Thu, 10 Sep 2020 20:53:06 +0400
Subject: [PATCH 022/104] Fix query

---
 lib/pleroma/backup.ex | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 348e537a8..ce54a413a 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -8,6 +8,8 @@ defmodule Pleroma.Backup do
   import Ecto.Changeset
   import Ecto.Query
 
+  require Pleroma.Constants
+
   alias Pleroma.Activity
   alias Pleroma.Bookmark
   alias Pleroma.Repo
@@ -158,6 +160,7 @@ defp write_header(file, name) do
         "id": "#{name}.json",
         "type": "OrderedCollection",
         "orderedItems": [
+
       """
     )
   end
@@ -209,13 +212,13 @@ defp statuses(dir, user) do
     opts =
       %{}
       |> Map.put(:type, ["Create", "Announce"])
-      |> Map.put(:blocking_user, user)
-      |> Map.put(:muting_user, user)
-      |> Map.put(:reply_filtering_user, user)
-      |> Map.put(:announce_filtering_user, user)
-      |> Map.put(:user, user)
+      |> Map.put(:actor_id, user.ap_id)
 
-    [[user.ap_id], User.following(user), Pleroma.List.memberships(user)]
+    [
+      [Pleroma.Constants.as_public(), user.ap_id],
+      User.following(user),
+      Pleroma.List.memberships(user)
+    ]
     |> Enum.concat()
     |> ActivityPub.fetch_activities_query(opts)
     |> write(dir, "outbox", fn a ->

From 386199063b9be9fc30ad403f6afb03bf6ca47298 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Thu, 10 Sep 2020 21:09:20 +0400
Subject: [PATCH 023/104] Document `/api/pleroma/backups` API endpoint

---
 docs/API/pleroma_api.md | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md
index 3fd141bd2..aeb266159 100644
--- a/docs/API/pleroma_api.md
+++ b/docs/API/pleroma_api.md
@@ -615,3 +615,41 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
   {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]}
 ]
 ```
+
+## `POST /api/pleroma/backups`
+### Create a user backup archive
+
+* Method: `POST`
+* Authentication: not required
+* Params: none
+* Response: JSON
+* Example response:
+
+```json
+[{
+    "content_type": "application/zip",
+    "file_size": 0,
+    "inserted_at": "2020-09-10T16:18:03.000Z",
+    "processed": false,
+    "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
+}]
+```
+
+## `GET /api/pleroma/backups`
+### Lists user backups
+
+* Method: `GET`
+* Authentication: not required
+* Params: none
+* Response: JSON
+* Example response:
+
+```json
+[{
+    "content_type": "application/zip",
+    "file_size": 55457,
+    "inserted_at": "2020-09-10T16:18:03.000Z",
+    "processed": true,
+    "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
+}]
+```

From 27bc121ec00a7b088030d6fb36c7e731f5b072b6 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 15 Sep 2020 18:07:28 +0400
Subject: [PATCH 024/104] Require email

---
 docs/configuration/cheatsheet.md |  3 +++
 lib/pleroma/backup.ex            | 19 ++++++++++++++++---
 test/backup_test.exs             | 16 ++++++++++++++--
 3 files changed, 33 insertions(+), 5 deletions(-)

diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index cc4081f14..8da8a7bd6 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -1085,6 +1085,9 @@ Control favicons for instances.
 
 ## Account Backup
 
+!!! note
+    Requires enabled email
+
 * `:purge_after_days` an integer, remove backup achives after N days.
 * `:limit_days` an integer, limit user to export not more often than once per N days.
 
diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index ce54a413a..3b85dd1c1 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -31,7 +31,9 @@ defmodule Pleroma.Backup do
   end
 
   def create(user) do
-    with :ok <- validate_limit(user),
+    with :ok <- validate_email_enabled(),
+         :ok <- validate_user_email(user),
+         :ok <- validate_limit(user),
          {:ok, backup} <- user |> new() |> Repo.insert() do
       BackupWorker.process(backup)
     end
@@ -74,6 +76,17 @@ defp validate_limit(user) do
     end
   end
 
+  defp validate_email_enabled do
+    if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
+      :ok
+    else
+      {:error, "Backups require enabled email"}
+    end
+  end
+
+  defp validate_user_email(%User{email: nil}), do: {:error, "Email is required"}
+  defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
+
   def get_last(user_id) do
     __MODULE__
     |> where(user_id: ^user_id)
@@ -100,7 +113,7 @@ def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
   def get(id), do: Repo.get(__MODULE__, id)
 
   def process(%__MODULE__{} = backup) do
-    with {:ok, zip_file} <- zip(backup),
+    with {:ok, zip_file} <- export(backup),
          {:ok, %{size: size}} <- File.stat(zip_file),
          {:ok, _upload} <- upload(backup, zip_file) do
       backup
@@ -110,7 +123,7 @@ def process(%__MODULE__{} = backup) do
   end
 
   @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
-  def zip(%__MODULE__{} = backup) do
+  def export(%__MODULE__{} = backup) do
     backup = Repo.preload(backup, :user)
     name = String.trim_trailing(backup.file_name, ".zip")
     dir = Path.join(System.tmp_dir!(), name)
diff --git a/test/backup_test.exs b/test/backup_test.exs
index 5fc519eab..318c8c419 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -18,6 +18,18 @@ defmodule Pleroma.BackupTest do
   setup do
     clear_config([Pleroma.Upload, :uploader])
     clear_config([Pleroma.Backup, :limit_days])
+    clear_config([Pleroma.Emails.Mailer, :enabled])
+  end
+
+  test "it requries enabled email" do
+    Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], false)
+    user = insert(:user)
+    assert {:error, "Backups require enabled email"} == Backup.create(user)
+  end
+
+  test "it requries user's email" do
+    user = insert(:user, %{email: nil})
+    assert {:error, "Email is required"} == Backup.create(user)
   end
 
   test "it creates a backup record and an Oban job" do
@@ -91,7 +103,7 @@ test "it creates a zip archive with user data" do
     Bookmark.create(user.id, status3.id)
 
     assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
-    assert {:ok, path} = Backup.zip(backup)
+    assert {:ok, path} = Backup.export(backup)
     assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
     assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
 
@@ -193,7 +205,7 @@ test "it creates a zip archive with user data" do
       Bookmark.create(user.id, status3.id)
 
       assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
-      assert {:ok, path} = Backup.zip(backup)
+      assert {:ok, path} = Backup.export(backup)
 
       [path: path, backup: backup]
     end

From e52dd62e14a956a28a706124464f3ac4b985080d Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 16 Sep 2020 23:21:13 +0400
Subject: [PATCH 025/104] Add configurable temporary directory

---
 config/config.exs                | 3 ++-
 docs/configuration/cheatsheet.md | 6 ++++++
 lib/pleroma/backup.ex            | 7 ++++++-
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 09023e2c3..0e12d6e15 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -820,7 +820,8 @@
 
 config :pleroma, Pleroma.Backup,
   purge_after_days: 30,
-  limit_days: 7
+  limit_days: 7,
+  dir: nil
 
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 8da8a7bd6..9271964f1 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -1090,6 +1090,12 @@ Control favicons for instances.
 
 * `:purge_after_days` an integer, remove backup achives after N days.
 * `:limit_days` an integer, limit user to export not more often than once per N days.
+* `:dir` a string with a path to backup temporary directory or `nil` to let Pleroma choose temporary directory in the following order:
+    1. the directory named by the TMPDIR environment variable
+    2. the directory named by the TEMP environment variable
+    3. the directory named by the TMP environment variable
+    4. C:\TMP on Windows or /tmp on Unix-like operating systems
+    5. as a last resort, the current working directory
 
 ## Frontend management
 
diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 3b85dd1c1..450dd5b84 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -126,7 +126,7 @@ def process(%__MODULE__{} = backup) do
   def export(%__MODULE__{} = backup) do
     backup = Repo.preload(backup, :user)
     name = String.trim_trailing(backup.file_name, ".zip")
-    dir = Path.join(System.tmp_dir!(), name)
+    dir = dir(name)
 
     with :ok <- File.mkdir(dir),
          :ok <- actor(dir, backup.user),
@@ -139,6 +139,11 @@ def export(%__MODULE__{} = backup) do
     end
   end
 
+  def dir(name) do
+    dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!()
+    Path.join(dir, name)
+  end
+
   def upload(%__MODULE__{} = backup, zip_path) do
     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
 

From 7fdd81d000d857cbcd5bf442f68c91b1c5b1cebb Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Thu, 17 Sep 2020 18:42:24 +0400
Subject: [PATCH 026/104] Add "Your backup is ready" email

---
 lib/pleroma/emails/user_email.ex     | 16 ++++++++++++++++
 lib/pleroma/workers/backup_worker.ex |  6 +++++-
 test/backup_test.exs                 |  5 ++++-
 3 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex
index 1d8c72ae9..f943dda0d 100644
--- a/lib/pleroma/emails/user_email.ex
+++ b/lib/pleroma/emails/user_email.ex
@@ -189,4 +189,20 @@ def unsubscribe_url(user, notifications_type) do
 
     Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
   end
+
+  def backup_is_ready_email(backup) do
+    %{user: user} = Pleroma.Repo.preload(backup, :user)
+    download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
+
+    html_body = """
+    <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
+    <p><a href="#{download_url}"></a></p>
+    """
+
+    new()
+    |> to(recipient(user))
+    |> from(sender())
+    |> subject("Your account archive is ready")
+    |> html_body(html_body)
+  end
 end
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
index f40020794..405d55269 100644
--- a/lib/pleroma/workers/backup_worker.ex
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -34,7 +34,11 @@ def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do
     with {:ok, %Backup{} = backup} <-
            backup_id |> Backup.get() |> Backup.process(),
          {:ok, _job} <- schedule_deletion(backup),
-         :ok <- Backup.remove_outdated(backup) do
+         :ok <- Backup.remove_outdated(backup),
+         {:ok, _} <-
+           backup
+           |> Pleroma.Emails.UserEmail.backup_is_ready_email()
+           |> Pleroma.Emails.Mailer.deliver() do
       {:ok, backup}
     end
   end
diff --git a/test/backup_test.exs b/test/backup_test.exs
index 318c8c419..0ea40e6fd 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -6,8 +6,9 @@ defmodule Pleroma.BackupTest do
   use Oban.Testing, repo: Pleroma.Repo
   use Pleroma.DataCase
 
-  import Pleroma.Factory
   import Mock
+  import Pleroma.Factory
+  import Swoosh.TestAssertions
 
   alias Pleroma.Backup
   alias Pleroma.Bookmark
@@ -65,6 +66,8 @@ test "it process a backup record" do
     assert_enqueued(worker: BackupWorker, args: delete_job_args)
     assert {:ok, backup} = perform_job(BackupWorker, delete_job_args)
     refute Backup.get(backup_id)
+
+    assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup))
   end
 
   test "it removes outdated backups after creating a fresh one" do

From 7c22c9afb410668d87dcd4a90651d62d9a1e9e4d Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 18 Sep 2020 22:18:34 +0400
Subject: [PATCH 027/104] Allow admins request user backups

---
 lib/pleroma/backup.ex                         |  4 ++--
 lib/pleroma/emails/user_email.ex              | 20 +++++++++++++-----
 .../controllers/admin_api_controller.ex       | 12 ++++++++++-
 lib/pleroma/web/router.ex                     |  2 ++
 lib/pleroma/workers/backup_worker.ex          | 10 +++++----
 .../controllers/admin_api_controller_test.exs | 21 +++++++++++++++++++
 6 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 450dd5b84..d589f12f1 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -30,12 +30,12 @@ defmodule Pleroma.Backup do
     timestamps()
   end
 
-  def create(user) do
+  def create(user, admin_user_id \\ nil) do
     with :ok <- validate_email_enabled(),
          :ok <- validate_user_email(user),
          :ok <- validate_limit(user),
          {:ok, backup} <- user |> new() |> Repo.insert() do
-      BackupWorker.process(backup)
+      BackupWorker.process(backup, admin_user_id)
     end
   end
 
diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex
index f943dda0d..5745794ec 100644
--- a/lib/pleroma/emails/user_email.ex
+++ b/lib/pleroma/emails/user_email.ex
@@ -190,14 +190,24 @@ def unsubscribe_url(user, notifications_type) do
     Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
   end
 
-  def backup_is_ready_email(backup) do
+  def backup_is_ready_email(backup, admin_user_id \\ nil) do
     %{user: user} = Pleroma.Repo.preload(backup, :user)
     download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
 
-    html_body = """
-    <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
-    <p><a href="#{download_url}"></a></p>
-    """
+    html_body =
+      if is_nil(admin_user_id) do
+        """
+        <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
+        <p><a href="#{download_url}"></a></p>
+        """
+      else
+        admin = Pleroma.Repo.get(User, admin_user_id)
+
+        """
+        <p>Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
+        <p><a href="#{download_url}"></a></p>
+        """
+      end
 
     new()
     |> to(recipient(user))
diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index d5713c3dd..f7d2fe5b1 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -23,12 +23,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.Router
 
+  require Logger
+
   @users_page_size 50
 
   plug(
     OAuthScopesPlug,
     %{scopes: ["read:accounts"], admin: true}
-    when action in [:list_users, :user_show, :right_get, :show_user_credentials]
+    when action in [:list_users, :user_show, :right_get, :show_user_credentials, :create_backup]
   )
 
   plug(
@@ -681,6 +683,14 @@ def stats(conn, params) do
     json(conn, %{"status_visibility" => counters})
   end
 
+  def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    with %User{} = user <- User.get_by_nickname(nickname),
+         {:ok, _} <- Pleroma.Backup.create(user, admin.id) do
+      Logger.info("Admin @#{admin.nickname} requested account backup for @{nickname}")
+      json(conn, "")
+    end
+  end
+
   defp page_params(params) do
     {get_page(params["page"]), get_page_size(params["page_size"])}
   end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index a1a5a1cb5..e539eeeeb 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -129,6 +129,8 @@ defmodule Pleroma.Web.Router do
   scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
     pipe_through(:admin_api)
 
+    post("/backups", AdminAPIController, :create_backup)
+
     post("/users/follow", AdminAPIController, :user_follow)
     post("/users/unfollow", AdminAPIController, :user_unfollow)
 
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
index 405d55269..65754b6a2 100644
--- a/lib/pleroma/workers/backup_worker.ex
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -8,8 +8,8 @@ defmodule Pleroma.Workers.BackupWorker do
   alias Oban.Job
   alias Pleroma.Backup
 
-  def process(backup) do
-    %{"op" => "process", "backup_id" => backup.id}
+  def process(backup, admin_user_id \\ nil) do
+    %{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id}
     |> new()
     |> Oban.insert()
   end
@@ -30,14 +30,16 @@ def delete(backup) do
     |> Oban.insert()
   end
 
-  def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do
+  def perform(%Job{
+        args: %{"op" => "process", "backup_id" => backup_id, "admin_user_id" => admin_user_id}
+      }) do
     with {:ok, %Backup{} = backup} <-
            backup_id |> Backup.get() |> Backup.process(),
          {:ok, _job} <- schedule_deletion(backup),
          :ok <- Backup.remove_outdated(backup),
          {:ok, _} <-
            backup
-           |> Pleroma.Emails.UserEmail.backup_is_ready_email()
+           |> Pleroma.Emails.UserEmail.backup_is_ready_email(admin_user_id)
            |> Pleroma.Emails.Mailer.deliver() do
       {:ok, backup}
     end
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index cba6b43d3..4d331779e 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -2024,6 +2024,27 @@ test "by instance", %{conn: conn} do
                response["status_visibility"]
     end
   end
+
+  describe "/api/pleroma/backups" do
+    test "it creates a backup", %{conn: conn} do
+      admin = insert(:user, is_admin: true)
+      token = insert(:oauth_admin_token, user: admin)
+      user = insert(:user)
+
+      assert "" ==
+               conn
+               |> assign(:user, admin)
+               |> assign(:token, token)
+               |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
+               |> json_response(200)
+
+      assert [backup] = Repo.all(Pleroma.Backup)
+
+      ObanHelpers.perform_all()
+
+      assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id))
+    end
+  end
 end
 
 # Needed for testing

From 563801716a0aa54e30f680b4e985d4b8c79578fb Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 18 Sep 2020 22:01:46 +0400
Subject: [PATCH 028/104] Update changelog

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8fc1750d1..04b49d80a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
 - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
 - Mix task option for force-unfollowing relays
+- Account backup
 
 ### Changed
 

From e50314d9d342dbf9a03ca484654b07717592d4bd Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 18 Sep 2020 22:33:12 +0400
Subject: [PATCH 029/104] Fix export

---
 lib/pleroma/backup.ex | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index d589f12f1..242773bdb 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -191,16 +191,13 @@ defp write(query, dir, name, fun) do
       counter = :counters.new(1, [])
 
       query
-      |> Pleroma.RepoStreamer.chunk_stream(100)
-      |> Stream.each(fn items ->
-        Enum.each(items, fn i ->
-          with {:ok, str} <- fun.(i),
-               :ok <- IO.write(file, str <> ",\n") do
-            :counters.add(counter, 1, 1)
-          end
-        end)
+      |> Pleroma.Repo.chunk_stream(100)
+      |> Enum.each(fn i ->
+        with {:ok, str} <- fun.(i),
+             :ok <- IO.write(file, str <> ",\n") do
+          :counters.add(counter, 1, 1)
+        end
       end)
-      |> Stream.run()
 
       total = :counters.get(counter, 1)
 

From a9efd441e242f1d8ac608b866d0cfafe4833243a Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sun, 20 Sep 2020 19:57:09 +0400
Subject: [PATCH 030/104] Use `Pleroma.Repo.chunk_stream/2` instead of
 `Pleroma.RepoStreamer.chunk_stream/2`

---
 lib/pleroma/backup.ex | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 242773bdb..f5f39431d 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -188,18 +188,17 @@ defp write(query, dir, name, fun) do
 
     with {:ok, file} <- File.open(path, [:write, :utf8]),
          :ok <- write_header(file, name) do
-      counter = :counters.new(1, [])
-
-      query
-      |> Pleroma.Repo.chunk_stream(100)
-      |> Enum.each(fn i ->
-        with {:ok, str} <- fun.(i),
-             :ok <- IO.write(file, str <> ",\n") do
-          :counters.add(counter, 1, 1)
-        end
-      end)
-
-      total = :counters.get(counter, 1)
+      total =
+        query
+        |> Pleroma.Repo.chunk_stream(100)
+        |> Enum.reduce(0, fn i, acc ->
+          with {:ok, str} <- fun.(i),
+               :ok <- IO.write(file, str <> ",\n") do
+            acc + 1
+          else
+            _ -> acc
+          end
+        end)
 
       with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n  \"totalItems\": #{total}}") do
         File.close(file)

From 17562bf4147ab03e171b1f1d365a512f2e5b3202 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sun, 20 Sep 2020 20:43:27 +0400
Subject: [PATCH 031/104] Move API endpoints to `/api/v1/pleroma/backups`

---
 docs/API/pleroma_api.md                            |  4 ++--
 lib/pleroma/web/router.ex                          |  6 +++---
 .../controllers/backup_controller_test.exs         | 14 +++++++-------
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md
index aeb266159..fa3a9a449 100644
--- a/docs/API/pleroma_api.md
+++ b/docs/API/pleroma_api.md
@@ -616,7 +616,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
 ]
 ```
 
-## `POST /api/pleroma/backups`
+## `POST /api/v1/pleroma/backups`
 ### Create a user backup archive
 
 * Method: `POST`
@@ -635,7 +635,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
 }]
 ```
 
-## `GET /api/pleroma/backups`
+## `GET /api/v1/pleroma/backups`
 ### Lists user backups
 
 * Method: `GET`
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index e539eeeeb..ad7e315c7 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -295,9 +295,6 @@ defmodule Pleroma.Web.Router do
     get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup)
     post("/accounts/mfa/confirm/:method", TwoFactorAuthenticationController, :confirm)
     delete("/accounts/mfa/:method", TwoFactorAuthenticationController, :disable)
-
-    get("/backups", BackupController, :index)
-    post("/backups", BackupController, :create)
   end
 
   scope "/oauth", Pleroma.Web.OAuth do
@@ -358,6 +355,9 @@ defmodule Pleroma.Web.Router do
       put("/mascot", MascotController, :update)
 
       post("/scrobble", ScrobbleController, :create)
+
+      get("/backups", BackupController, :index)
+      post("/backups", BackupController, :create)
     end
 
     scope [] do
diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/web/pleroma_api/controllers/backup_controller_test.exs
index 5d2f1206e..b2ac74c7d 100644
--- a/test/web/pleroma_api/controllers/backup_controller_test.exs
+++ b/test/web/pleroma_api/controllers/backup_controller_test.exs
@@ -14,14 +14,14 @@ defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do
     oauth_access(["read:accounts"])
   end
 
-  test "GET /api/pleroma/backups", %{user: user, conn: conn} do
+  test "GET /api/v1/pleroma/backups", %{user: user, conn: conn} do
     assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user)
 
     backup = Backup.get(backup_id)
 
     response =
       conn
-      |> get("/api/pleroma/backups")
+      |> get("/api/v1/pleroma/backups")
       |> json_response_and_validate_schema(:ok)
 
     assert [
@@ -45,11 +45,11 @@ test "GET /api/pleroma/backups", %{user: user, conn: conn} do
              }
            ] =
              conn
-             |> get("/api/pleroma/backups")
+             |> get("/api/v1/pleroma/backups")
              |> json_response_and_validate_schema(:ok)
   end
 
-  test "POST /api/pleroma/backups", %{user: _user, conn: conn} do
+  test "POST /api/v1/pleroma/backups", %{user: _user, conn: conn} do
     assert [
              %{
                "content_type" => "application/zip",
@@ -60,7 +60,7 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do
              }
            ] =
              conn
-             |> post("/api/pleroma/backups")
+             |> post("/api/v1/pleroma/backups")
              |> json_response_and_validate_schema(:ok)
 
     Pleroma.Tests.ObanHelpers.perform_all()
@@ -72,14 +72,14 @@ test "POST /api/pleroma/backups", %{user: _user, conn: conn} do
              }
            ] =
              conn
-             |> get("/api/pleroma/backups")
+             |> get("/api/v1/pleroma/backups")
              |> json_response_and_validate_schema(:ok)
 
     days = Pleroma.Config.get([Backup, :limit_days])
 
     assert %{"error" => "Last export was less than #{days} days ago"} ==
              conn
-             |> post("/api/pleroma/backups")
+             |> post("/api/v1/pleroma/backups")
              |> json_response_and_validate_schema(400)
   end
 end

From e4792ce76af3094d378a3a201ca429ae38203696 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sun, 20 Sep 2020 21:06:16 +0400
Subject: [PATCH 032/104] Do not limit admins

---
 lib/pleroma/backup.ex                         | 10 ++++----
 .../controllers/admin_api_controller_test.exs | 24 +++++++++++++++++++
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index f5f39431d..e2673db80 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -30,12 +30,12 @@ defmodule Pleroma.Backup do
     timestamps()
   end
 
-  def create(user, admin_user_id \\ nil) do
+  def create(user, admin_id \\ nil) do
     with :ok <- validate_email_enabled(),
          :ok <- validate_user_email(user),
-         :ok <- validate_limit(user),
+         :ok <- validate_limit(user, admin_id),
          {:ok, backup} <- user |> new() |> Repo.insert() do
-      BackupWorker.process(backup, admin_user_id)
+      BackupWorker.process(backup, admin_id)
     end
   end
 
@@ -59,7 +59,9 @@ def delete(backup) do
     end
   end
 
-  defp validate_limit(user) do
+  defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok
+
+  defp validate_limit(user, nil) do
     case get_last(user.id) do
       %__MODULE__{inserted_at: inserted_at} ->
         days = Pleroma.Config.get([Pleroma.Backup, :limit_days])
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index 4d331779e..4b3abce0d 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -2044,6 +2044,30 @@ test "it creates a backup", %{conn: conn} do
 
       assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id))
     end
+
+    test "it doesn't limit admins", %{conn: conn} do
+      admin = insert(:user, is_admin: true)
+      token = insert(:oauth_admin_token, user: admin)
+      user = insert(:user)
+
+      assert "" ==
+               conn
+               |> assign(:user, admin)
+               |> assign(:token, token)
+               |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
+               |> json_response(200)
+
+      assert [_backup] = Repo.all(Pleroma.Backup)
+
+      assert "" ==
+               conn
+               |> assign(:user, admin)
+               |> assign(:token, token)
+               |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
+               |> json_response(200)
+
+      assert Repo.aggregate(Pleroma.Backup, :count) == 2
+    end
   end
 end
 

From 8baee855d90530def46dc62b81e6a0cb0c315914 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 21 Sep 2020 21:47:36 +0400
Subject: [PATCH 033/104] Fix emails

---
 lib/pleroma/emails/user_email.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex
index 5745794ec..806a61fd2 100644
--- a/lib/pleroma/emails/user_email.ex
+++ b/lib/pleroma/emails/user_email.ex
@@ -198,14 +198,14 @@ def backup_is_ready_email(backup, admin_user_id \\ nil) do
       if is_nil(admin_user_id) do
         """
         <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
-        <p><a href="#{download_url}"></a></p>
+        <p><a href="#{download_url}">#{download_url}</a></p>
         """
       else
         admin = Pleroma.Repo.get(User, admin_user_id)
 
         """
         <p>Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
-        <p><a href="#{download_url}"></a></p>
+        <p><a href="#{download_url}">#{download_url}</a></p>
         """
       end
 

From f1e4333dd7976e8cbef44a3bcfe5c96bef177c6f Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 23 Sep 2020 20:23:11 +0400
Subject: [PATCH 034/104] Fix test

---
 test/backup_test.exs                                       | 7 ++++++-
 .../admin_api/controllers/admin_api_controller_test.exs    | 5 ++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/test/backup_test.exs b/test/backup_test.exs
index 0ea40e6fd..23c08b680 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -67,7 +67,12 @@ test "it process a backup record" do
     assert {:ok, backup} = perform_job(BackupWorker, delete_job_args)
     refute Backup.get(backup_id)
 
-    assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup))
+    email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup)
+
+    assert_email_sent(
+      to: {user.name, user.email},
+      html_body: email.html_body
+    )
   end
 
   test "it removes outdated backups after creating a fresh one" do
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index 4b3abce0d..a6dc4f62d 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -2042,7 +2042,10 @@ test "it creates a backup", %{conn: conn} do
 
       ObanHelpers.perform_all()
 
-      assert_email_sent(Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id))
+      email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)
+
+      assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup")
+      assert_email_sent(to: {user.name, user.email}, html_body: email.html_body)
     end
 
     test "it doesn't limit admins", %{conn: conn} do

From 6d5f02a1da81ed7693c5ae364a25bc0b54ee1a38 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 26 Sep 2020 20:34:44 +0400
Subject: [PATCH 035/104] Fix API documentation

---
 docs/API/pleroma_api.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md
index fa3a9a449..7a0a80dad 100644
--- a/docs/API/pleroma_api.md
+++ b/docs/API/pleroma_api.md
@@ -620,7 +620,7 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
 ### Create a user backup archive
 
 * Method: `POST`
-* Authentication: not required
+* Authentication: required
 * Params: none
 * Response: JSON
 * Example response:

From d7a5291b4fa3b7568674c0f7643fe287fcd21eff Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 26 Sep 2020 21:24:35 +0400
Subject: [PATCH 036/104] Use `Jason.encode/1` for likes and bookmarks

---
 lib/pleroma/backup.ex | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index e2673db80..b43dc94d6 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -194,7 +194,8 @@ defp write(query, dir, name, fun) do
         query
         |> Pleroma.Repo.chunk_stream(100)
         |> Enum.reduce(0, fn i, acc ->
-          with {:ok, str} <- fun.(i),
+          with {:ok, data} <- fun.(i),
+               {:ok, str} <- Jason.encode(data),
                :ok <- IO.write(file, str <> ",\n") do
             acc + 1
           else
@@ -213,7 +214,7 @@ defp bookmarks(dir, %{id: user_id} = _user) do
     |> where(user_id: ^user_id)
     |> join(:inner, [b], activity in assoc(b, :activity))
     |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
-    |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end)
+    |> write(dir, "bookmarks", fn a -> {:ok, a.object} end)
   end
 
   defp likes(dir, user) do
@@ -221,7 +222,7 @@ defp likes(dir, user) do
     |> Activity.Queries.by_actor()
     |> Activity.Queries.by_type("Like")
     |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
-    |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end)
+    |> write(dir, "likes", fn a -> {:ok, a.object} end)
   end
 
   defp statuses(dir, user) do
@@ -239,7 +240,7 @@ defp statuses(dir, user) do
     |> ActivityPub.fetch_activities_query(opts)
     |> write(dir, "outbox", fn a ->
       with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
-        activity |> Map.delete("@context") |> Jason.encode()
+        {:ok, Map.delete(activity, "@context")}
       end
     end)
   end

From 9af9f02f4b3c4eac859a69ab9b2f546a91110287 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 26 Sep 2020 21:45:03 +0400
Subject: [PATCH 037/104] Use Gettext for error messages

---
 lib/pleroma/backup.ex | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index b43dc94d6..0ebaf02e5 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Backup do
 
   import Ecto.Changeset
   import Ecto.Query
+  import Pleroma.Web.Gettext
 
   require Pleroma.Constants
 
@@ -70,7 +71,14 @@ defp validate_limit(user, nil) do
         if diff > days do
           :ok
         else
-          {:error, "Last export was less than #{days} days ago"}
+          {:error,
+           dngettext(
+             "errors",
+             "Last export was less than a day ago",
+             "Last export was less than %{days} days ago",
+             days,
+             days: days
+           )}
         end
 
       nil ->
@@ -82,11 +90,14 @@ defp validate_email_enabled do
     if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
       :ok
     else
-      {:error, "Backups require enabled email"}
+      {:error, dgettext("errors", "Backups require enabled email")}
     end
   end
 
-  defp validate_user_email(%User{email: nil}), do: {:error, "Email is required"}
+  defp validate_user_email(%User{email: nil}) do
+    {:error, dgettext("errors", "Email is required")}
+  end
+
   defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
 
   def get_last(user_id) do

From 08972dd135c200073f5de0c8731b886cc2e72eeb Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 26 Sep 2020 21:50:31 +0400
Subject: [PATCH 038/104] Use Path.join/2

---
 lib/pleroma/backup.ex | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index 0ebaf02e5..cee51d7c1 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -55,7 +55,7 @@ def new(user) do
   def delete(backup) do
     uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
 
-    with :ok <- uploader.delete_file("backups/" <> backup.file_name) do
+    with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do
       Repo.delete(backup)
     end
   end
@@ -164,7 +164,7 @@ def upload(%__MODULE__{} = backup, zip_path) do
       name: backup.file_name,
       tempfile: zip_path,
       content_type: backup.content_type,
-      path: "backups/" <> backup.file_name
+      path: Path.join("backups", backup.file_name)
     }
 
     with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
@@ -178,7 +178,7 @@ defp actor(dir, user) do
            UserView.render("user.json", %{user: user})
            |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
            |> Jason.encode() do
-      File.write(dir <> "/actor.json", json)
+      File.write(Path.join(dir, "actor.json"), json)
     end
   end
 
@@ -197,7 +197,7 @@ defp write_header(file, name) do
   end
 
   defp write(query, dir, name, fun) do
-    path = dir <> "/#{name}.json"
+    path = Path.join(dir, "#{name}.json")
 
     with {:ok, file} <- File.open(path, [:write, :utf8]),
          :ok <- write_header(file, name) do

From 8545d533ddee2978e9bf7f3284cc7dcb822a77e6 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 26 Sep 2020 21:53:04 +0400
Subject: [PATCH 039/104] Use to_string/1 instead of :binary.list_to_bin/1

---
 lib/pleroma/backup.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
index cee51d7c1..629e879a7 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/backup.ex
@@ -148,7 +148,7 @@ def export(%__MODULE__{} = backup) do
          :ok <- bookmarks(dir, backup.user),
          {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
          {:ok, _} <- File.rm_rf(dir) do
-      {:ok, :binary.list_to_bin(zip_path)}
+      {:ok, to_string(zip_path)}
     end
   end
 

From bc3db724030707e9903d161a70b10fe217a83212 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 26 Sep 2020 23:16:56 +0400
Subject: [PATCH 040/104] Use ModerationLog instead of Logger

---
 lib/pleroma/moderation_log.ex                 | 10 ++++++++
 .../controllers/admin_api_controller.ex       |  3 ++-
 .../controllers/admin_api_controller_test.exs | 23 +++++++++++++++++--
 3 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex
index 47036a6f6..be1e81467 100644
--- a/lib/pleroma/moderation_log.ex
+++ b/lib/pleroma/moderation_log.ex
@@ -651,6 +651,16 @@ def get_log_entry_message(%ModerationLog{
     "@#{actor_nickname} deleted chat message ##{subject_id}"
   end
 
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "create_backup",
+          "subject" => %{"nickname" => user_nickname}
+        }
+      }) do
+    "@#{actor_nickname} requested account backup for @#{user_nickname}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")
diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index f7d2fe5b1..8b5310d80 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -686,7 +686,8 @@ def stats(conn, params) do
   def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
     with %User{} = user <- User.get_by_nickname(nickname),
          {:ok, _} <- Pleroma.Backup.create(user, admin.id) do
-      Logger.info("Admin @#{admin.nickname} requested account backup for @{nickname}")
+      ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"})
+
       json(conn, "")
     end
   end
diff --git a/test/web/admin_api/controllers/admin_api_controller_test.exs b/test/web/admin_api/controllers/admin_api_controller_test.exs
index a6dc4f62d..34d48c2c1 100644
--- a/test/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/web/admin_api/controllers/admin_api_controller_test.exs
@@ -2027,9 +2027,9 @@ test "by instance", %{conn: conn} do
 
   describe "/api/pleroma/backups" do
     test "it creates a backup", %{conn: conn} do
-      admin = insert(:user, is_admin: true)
+      admin = %{id: admin_id, nickname: admin_nickname} = insert(:user, is_admin: true)
       token = insert(:oauth_admin_token, user: admin)
-      user = insert(:user)
+      user = %{id: user_id, nickname: user_nickname} = insert(:user)
 
       assert "" ==
                conn
@@ -2046,6 +2046,25 @@ test "it creates a backup", %{conn: conn} do
 
       assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup")
       assert_email_sent(to: {user.name, user.email}, html_body: email.html_body)
+
+      log_message = "@#{admin_nickname} requested account backup for @#{user_nickname}"
+
+      assert [
+               %{
+                 data: %{
+                   "action" => "create_backup",
+                   "actor" => %{
+                     "id" => ^admin_id,
+                     "nickname" => ^admin_nickname
+                   },
+                   "message" => ^log_message,
+                   "subject" => %{
+                     "id" => ^user_id,
+                     "nickname" => ^user_nickname
+                   }
+                 }
+               }
+             ] = Pleroma.ModerationLog |> Repo.all()
     end
 
     test "it doesn't limit admins", %{conn: conn} do

From 9c672ecbb5d4477cd16d2139a2cb66d3923ac5c8 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Thu, 8 Oct 2020 20:01:48 -0500
Subject: [PATCH 041/104] Remote Timeline: add Streaming support

---
 CHANGELOG.md                                  |  1 +
 docs/API/differences_in_mastoapi_responses.md |  6 ++++++
 lib/pleroma/activity/ir/topics.ex             | 13 ++++++++++++-
 lib/pleroma/web/streamer/streamer.ex          |  9 +++++++++
 test/activity/ir/topics_test.exs              | 14 ++++++++++++++
 test/integration/mastodon_websocket_test.exs  |  1 +
 test/web/streamer/streamer_test.exs           |  8 ++++++++
 7 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8fc1750d1..0eeffb72f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
 - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
 - Mix task option for force-unfollowing relays
+- Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media`
 
 ### Changed
 
diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md
index 38865dc68..bb1000b0b 100644
--- a/docs/API/differences_in_mastoapi_responses.md
+++ b/docs/API/differences_in_mastoapi_responses.md
@@ -9,9 +9,13 @@ Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mas
 ## Timelines
 
 Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
+
 Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`.
+
 Adding the parameter `reply_visibility` to the public and home timelines queries will filter replies. Possible values: without parameter (default) shows all replies, `following` - replies directed to you or users you follow, `self` - replies directed to you.
 
+Adding the parameter `instance=lain.com` to the public timeline will show only statuses originating from `lain.com` (or any remote instance).
+
 ## Statuses
 
 - `visibility`: has an additional possible value `list`
@@ -249,6 +253,8 @@ Has these additional fields under the `pleroma` object:
 
 There is an additional `user:pleroma_chat` stream. Incoming chat messages will make the current chat be sent to this `user` stream. The `event` of an incoming chat message is `pleroma:chat_update`. The payload is the updated chat with the incoming chat message in the `last_message` field.
 
+For viewing remote server timelines, there are `public:remote` and `public:remote:media` streams. Each of these accept a parameter like `?instance=lain.com`.
+
 ## Not implemented
 
 Pleroma is generally compatible with the Mastodon 2.7.2 API, but some newer features and non-essential features are omitted. These features usually return an HTTP 200 status code, but with an empty response. While they may be added in the future, they are considered low priority.
diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex
index 9e65bedad..fe2e8cb5c 100644
--- a/lib/pleroma/activity/ir/topics.ex
+++ b/lib/pleroma/activity/ir/topics.ex
@@ -40,7 +40,8 @@ defp visibility_tags(object, activity) do
   end
 
   defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do
-    tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
+    tags ++
+      remote_topics(activity) ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
   end
 
   defp item_creation_tags(tags, _, _) do
@@ -55,9 +56,19 @@ defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
 
   defp hashtags_to_topics(_), do: []
 
+  defp remote_topics(%{local: true}), do: []
+
+  defp remote_topics(%{actor: actor}) when is_binary(actor),
+    do: ["public:remote:" <> URI.parse(actor).host]
+
+  defp remote_topics(_), do: []
+
   defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: []
 
   defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"]
 
+  defp attachment_topics(_object, %{actor: actor}) when is_binary(actor),
+    do: ["public:media", "public:remote:media:" <> URI.parse(actor).host]
+
   defp attachment_topics(_object, _act), do: ["public:media"]
 end
diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex
index 5475f18a6..d774f0dd9 100644
--- a/lib/pleroma/web/streamer/streamer.ex
+++ b/lib/pleroma/web/streamer/streamer.ex
@@ -57,6 +57,15 @@ def get_topic("hashtag", _user, _oauth_token, %{"tag" => tag} = _params) do
     {:ok, "hashtag:" <> tag}
   end
 
+  # Allow remote instance streams.
+  def get_topic("public:remote", _user, _oauth_token, %{"instance" => instance} = _params) do
+    {:ok, "public:remote:" <> instance}
+  end
+
+  def get_topic("public:remote:media", _user, _oauth_token, %{"instance" => instance} = _params) do
+    {:ok, "public:remote:media:" <> instance}
+  end
+
   # Expand user streams.
   def get_topic(
         stream,
diff --git a/test/activity/ir/topics_test.exs b/test/activity/ir/topics_test.exs
index 14a6e6b71..c8dcb28cc 100644
--- a/test/activity/ir/topics_test.exs
+++ b/test/activity/ir/topics_test.exs
@@ -93,6 +93,13 @@ test "only converts strings to hash tags", %{
 
       refute Enum.member?(topics, "hashtag:2")
     end
+
+    test "non-local action produces public:remote topic", %{activity: activity} do
+      activity = %{activity | local: false, actor: "https://lain.com/users/lain"}
+      topics = Topics.get_activity_topics(activity)
+
+      assert Enum.member?(topics, "public:remote:lain.com")
+    end
   end
 
   describe "public visibility create events with attachments" do
@@ -124,6 +131,13 @@ test "non-local doesn't produce public:local:media topics", %{activity: activity
 
       refute Enum.member?(topics, "public:local:media")
     end
+
+    test "non-local action produces public:remote:media topic", %{activity: activity} do
+      activity = %{activity | local: false, actor: "https://lain.com/users/lain"}
+      topics = Topics.get_activity_topics(activity)
+
+      assert Enum.member?(topics, "public:remote:media:lain.com")
+    end
   end
 
   describe "non-public visibility" do
diff --git a/test/integration/mastodon_websocket_test.exs b/test/integration/mastodon_websocket_test.exs
index 0f2e6cc2b..bb8e795b7 100644
--- a/test/integration/mastodon_websocket_test.exs
+++ b/test/integration/mastodon_websocket_test.exs
@@ -49,6 +49,7 @@ test "requires authentication and a valid token for protected streams" do
   test "allows public streams without authentication" do
     assert {:ok, _} = start_socket("?stream=public")
     assert {:ok, _} = start_socket("?stream=public:local")
+    assert {:ok, _} = start_socket("?stream=public:remote&instance=lain.com")
     assert {:ok, _} = start_socket("?stream=hashtag&tag=lain")
   end
 
diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs
index 185724a9f..1495ed124 100644
--- a/test/web/streamer/streamer_test.exs
+++ b/test/web/streamer/streamer_test.exs
@@ -29,6 +29,14 @@ test "allows public" do
       assert {:ok, "public:local:media"} = Streamer.get_topic("public:local:media", nil, nil)
     end
 
+    test "allows instance streams" do
+      assert {:ok, "public:remote:lain.com"} =
+               Streamer.get_topic("public:remote", nil, nil, %{"instance" => "lain.com"})
+
+      assert {:ok, "public:remote:media:lain.com"} =
+               Streamer.get_topic("public:remote:media", nil, nil, %{"instance" => "lain.com"})
+    end
+
     test "allows hashtag streams" do
       assert {:ok, "hashtag:cofe"} = Streamer.get_topic("hashtag", nil, nil, %{"tag" => "cofe"})
     end

From e1eb54d3899883b5af6a43687a2345543d69ad4a Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sun, 11 Oct 2020 13:37:19 +0300
Subject: [PATCH 042/104] [#3053] Rollback of access control changes in
 ActivityPubController (base actions: :user, :object, :activity).

---
 .../activity_pub/activity_pub_controller.ex   | 56 +++++++++----------
 1 file changed, 25 insertions(+), 31 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index c78edfb4c..732c44271 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -32,23 +32,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   @federating_only_actions [:internal_fetch, :relay, :relay_following, :relay_followers]
 
-  # Note: :following and :followers must be served even without authentication (as via :api)
-  @auth_only_actions [:read_inbox, :update_outbox, :whoami, :upload_media]
-
-  # Always accessible actions (must perform entity accessibility checks)
-  @no_auth_no_federation_actions [:user, :activity, :object]
-
-  @authenticated_or_federating_actions @federating_only_actions ++
-                                         @auth_only_actions ++ @no_auth_no_federation_actions
-
   plug(FederatingPlug when action in @federating_only_actions)
 
-  plug(EnsureAuthenticatedPlug when action in @auth_only_actions)
-
   plug(
     EnsureAuthenticatedPlug,
-    [unless_func: &FederatingPlug.federating?/1]
-    when action not in @authenticated_or_federating_actions
+    [unless_func: &FederatingPlug.federating?/1] when action not in @federating_only_actions
+  )
+
+  # Note: :following and :followers must be served even without authentication (as via :api)
+  plug(
+    EnsureAuthenticatedPlug
+    when action in [:read_inbox, :update_outbox, :whoami, :upload_media]
   )
 
   plug(
@@ -72,22 +66,21 @@ defp relay_active?(conn, _) do
 
   def user(conn, %{"nickname" => nickname}) do
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname),
-         {_, :visible} <- {:visibility, User.visible_for(user, _reading_user = nil)},
          {:ok, user} <- User.ensure_keys_present(user) do
       conn
       |> put_resp_content_type("application/activity+json")
       |> put_view(UserView)
       |> render("user.json", %{user: user})
     else
-      _ -> {:error, :not_found}
+      nil -> {:error, :not_found}
+      %{local: false} -> {:error, :not_found}
     end
   end
 
   def object(conn, _) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(object)},
-         {_, false} <- {:restricted?, Visibility.restrict_unauthenticated_access?(object)} do
+         {_, true} <- {:public?, Visibility.is_public?(object)} do
       conn
       |> assign(:tracking_fun_data, object.id)
       |> set_cache_ttl_for(object)
@@ -95,15 +88,25 @@ def object(conn, _) do
       |> put_view(ObjectView)
       |> render("object.json", object: object)
     else
-      _ -> {:error, :not_found}
+      {:public?, false} ->
+        {:error, :not_found}
     end
   end
 
+  def track_object_fetch(conn, nil), do: conn
+
+  def track_object_fetch(conn, object_id) do
+    with %{assigns: %{user: %User{id: user_id}}} <- conn do
+      Delivery.create(object_id, user_id)
+    end
+
+    conn
+  end
+
   def activity(conn, _params) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Activity{} = activity <- Activity.normalize(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(activity)},
-         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do
+         {_, true} <- {:public?, Visibility.is_public?(activity)} do
       conn
       |> maybe_set_tracking_data(activity)
       |> set_cache_ttl_for(activity)
@@ -111,7 +114,8 @@ def activity(conn, _params) do
       |> put_view(ObjectView)
       |> render("object.json", object: activity)
     else
-      _ -> {:error, :not_found}
+      {:public?, false} -> {:error, :not_found}
+      nil -> {:error, :not_found}
     end
   end
 
@@ -546,14 +550,4 @@ def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} =
       |> json(object.data)
     end
   end
-
-  def track_object_fetch(conn, nil), do: conn
-
-  def track_object_fetch(conn, object_id) do
-    with %{assigns: %{user: %User{id: user_id}}} <- conn do
-      Delivery.create(object_id, user_id)
-    end
-
-    conn
-  end
 end

From 89c595b772eaaa8809f5339d708d7dc22e51b662 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sun, 11 Oct 2020 22:34:28 +0300
Subject: [PATCH 043/104] [#3053] Removed target accessibility checks for
 OStatus endpoints delegating to RedirectController. Added tests.

---
 CHANGELOG.md                                  |  1 +
 lib/pleroma/web/ostatus/ostatus_controller.ex | 13 +++----
 lib/pleroma/web/router.ex                     | 38 +++++++++----------
 .../static_fe/static_fe_controller_test.exs   | 23 +++++++++++
 4 files changed, 48 insertions(+), 27 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9ae5d0eda..1e7bcca08 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,6 +47,7 @@ switched to a new configuration mechanism, however it was not officially removed
 
 - Add documented-but-missing chat pagination.
 - Allow sending out emails again.
+- OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting.
 
 ## Unreleased (Patch)
 
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index b4dc2a87f..e03ca8c0a 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -37,11 +37,10 @@ def object(conn, _params) do
     with id <- Endpoint.url() <> conn.request_path,
          {_, %Activity{} = activity} <-
            {:activity, Activity.get_create_by_object_ap_id_with_object(id)},
-         {_, true} <- {:public?, Visibility.is_public?(activity)},
-         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do
+         {_, true} <- {:public?, Visibility.is_public?(activity)} do
       redirect(conn, to: "/notice/#{activity.id}")
     else
-      reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] ->
+      reason when reason in [{:public?, false}, {:activity, nil}] ->
         {:error, :not_found}
 
       e ->
@@ -57,11 +56,10 @@ def activity(%{assigns: %{format: format}} = conn, _params)
   def activity(conn, _params) do
     with id <- Endpoint.url() <> conn.request_path,
          {_, %Activity{} = activity} <- {:activity, Activity.normalize(id)},
-         {_, true} <- {:public?, Visibility.is_public?(activity)},
-         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)} do
+         {_, true} <- {:public?, Visibility.is_public?(activity)} do
       redirect(conn, to: "/notice/#{activity.id}")
     else
-      reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] ->
+      reason when reason in [{:public?, false}, {:activity, nil}] ->
         {:error, :not_found}
 
       e ->
@@ -72,7 +70,6 @@ def activity(conn, _params) do
   def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do
     with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)},
          {_, true} <- {:public?, Visibility.is_public?(activity)},
-         {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)},
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       cond do
         format in ["json", "activity+json"] ->
@@ -100,7 +97,7 @@ def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do
           RedirectController.redirector(conn, nil)
       end
     else
-      reason when reason in [{:public?, false}, {:visible?, false}, {:activity, nil}] ->
+      reason when reason in [{:public?, false}, {:activity, nil}] ->
         conn
         |> put_status(404)
         |> RedirectController.redirector(nil, 404)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 97fcaafd5..ef56360ed 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -9,6 +9,18 @@ defmodule Pleroma.Web.Router do
     plug(:accepts, ["html"])
   end
 
+  pipeline :accepts_html_xml do
+    plug(:accepts, ["html", "xml", "rss", "atom"])
+  end
+
+  pipeline :accepts_html_json do
+    plug(:accepts, ["html", "activity+json", "json"])
+  end
+
+  pipeline :accepts_html_xml_json do
+    plug(:accepts, ["html", "xml", "rss", "atom", "activity+json", "json"])
+  end
+
   pipeline :accepts_xml_rss_atom do
     plug(:accepts, ["xml", "rss", "atom"])
   end
@@ -574,24 +586,10 @@ defmodule Pleroma.Web.Router do
     )
   end
 
-  pipeline :ostatus_html_json do
-    plug(:accepts, ["html", "activity+json", "json"])
-    plug(Pleroma.Plugs.StaticFEPlug)
-  end
-
-  pipeline :ostatus_html_xml do
-    plug(:accepts, ["html", "xml", "rss", "atom"])
-    plug(Pleroma.Plugs.StaticFEPlug)
-  end
-
-  pipeline :ostatus_html_xml_json do
-    plug(:accepts, ["html", "xml", "rss", "atom", "activity+json", "json"])
-    plug(Pleroma.Plugs.StaticFEPlug)
-  end
-
   scope "/", Pleroma.Web do
     # Note: html format is supported only if static FE is enabled
-    pipe_through(:ostatus_html_json)
+    # Note: http signature is only considered for json requests (no auth for non-json requests)
+    pipe_through([:accepts_html_json, :http_signature, Pleroma.Plugs.StaticFEPlug])
 
     get("/objects/:uuid", OStatus.OStatusController, :object)
     get("/activities/:uuid", OStatus.OStatusController, :activity)
@@ -604,15 +602,17 @@ defmodule Pleroma.Web.Router do
 
   scope "/", Pleroma.Web do
     # Note: html format is supported only if static FE is enabled
-    pipe_through(:ostatus_html_xml_json)
+    # Note: http signature is only considered for json requests (no auth for non-json requests)
+    pipe_through([:accepts_html_xml_json, :http_signature, Pleroma.Plugs.StaticFEPlug])
 
-    # Note: for json format responds with user profile (not user feed)
+    # Note: returns user _profile_ for json requests, redirects to user _feed_ for non-json ones
     get("/users/:nickname", Feed.UserController, :feed_redirect, as: :user_feed)
   end
 
   scope "/", Pleroma.Web do
     # Note: html format is supported only if static FE is enabled
-    pipe_through(:ostatus_html_xml)
+    pipe_through([:accepts_html_xml, Pleroma.Plugs.StaticFEPlug])
+
     get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed)
   end
 
diff --git a/test/web/static_fe/static_fe_controller_test.exs b/test/web/static_fe/static_fe_controller_test.exs
index bab0b0a7b..8baf5b1ce 100644
--- a/test/web/static_fe/static_fe_controller_test.exs
+++ b/test/web/static_fe/static_fe_controller_test.exs
@@ -78,6 +78,18 @@ test "does not require authentication on non-federating instances", %{
 
       assert html_response(conn, 200) =~ user.nickname
     end
+
+    test "returns 404 for local user with `restrict_unauthenticated/profiles/local` setting", %{
+      conn: conn
+    } do
+      clear_config([:restrict_unauthenticated, :profiles, :local], true)
+
+      local_user = insert(:user, local: true)
+
+      conn
+      |> get("/users/#{local_user.nickname}")
+      |> html_response(404)
+    end
   end
 
   describe "notice html" do
@@ -200,5 +212,16 @@ test "does not require authentication on non-federating instances", %{
 
       assert html_response(conn, 200) =~ "testing a thing!"
     end
+
+    test "returns 404 for local public activity with `restrict_unauthenticated/activities/local` setting",
+         %{conn: conn, user: user} do
+      clear_config([:restrict_unauthenticated, :activities, :local], true)
+
+      {:ok, activity} = CommonAPI.post(user, %{status: "testing a thing!"})
+
+      conn
+      |> get("/notice/#{activity.id}")
+      |> html_response(404)
+    end
   end
 end

From 33f4f39b1cf3a6d8ce350da194696b19ca6f3a05 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 13 Oct 2020 21:39:41 +0400
Subject: [PATCH 044/104] Add pagination for Blocks

---
 .../api_spec/operations/account_operation.ex  |  1 +
 .../controllers/account_controller.ex         | 12 +++--
 .../controllers/account_controller_test.exs   | 46 +++++++++++++++++--
 3 files changed, 51 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index d90ddb787..9cd516f05 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -348,6 +348,7 @@ def blocks_operation do
       operationId: "AccountController.blocks",
       description: "View your blocks. See also accounts/:id/{block,unblock}",
       security: [%{"oAuth" => ["read:blocks"]}],
+      parameters: pagination_params(),
       responses: %{
         200 => Operation.response("Accounts", "application/json", array_of_accounts())
       }
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 4f9696d52..1b221e3a1 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -448,9 +448,15 @@ def mutes(%{assigns: %{user: user}} = conn, _) do
   end
 
   @doc "GET /api/v1/blocks"
-  def blocks(%{assigns: %{user: user}} = conn, _) do
-    users = User.blocked_users(user, _restrict_deactivated = true)
-    render(conn, "index.json", users: users, for: user, as: :user)
+  def blocks(%{assigns: %{user: user}} = conn, params) do
+    users =
+      user
+      |> User.blocked_users_relation(_restrict_deactivated = true)
+      |> Pleroma.Pagination.fetch_paginated(Map.put(params, :skip_order, true))
+
+    conn
+    |> add_link_headers(users)
+    |> render("index.json", users: users, for: user, as: :user)
   end
 
   @doc "GET /api/v1/endorsements"
diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
index f7f1369e4..6ad9dfc39 100644
--- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
@@ -1521,16 +1521,52 @@ test "getting a list of mutes" do
 
   test "getting a list of blocks" do
     %{user: user, conn: conn} = oauth_access(["read:blocks"])
-    other_user = insert(:user)
+    %{id: id1} = other_user1 = insert(:user)
+    %{id: id2} = other_user2 = insert(:user)
+    %{id: id3} = other_user3 = insert(:user)
 
-    {:ok, _user_relationship} = User.block(user, other_user)
+    {:ok, _user_relationship} = User.block(user, other_user1)
+    {:ok, _user_relationship} = User.block(user, other_user3)
+    {:ok, _user_relationship} = User.block(user, other_user2)
 
-    conn =
+    result =
       conn
       |> assign(:user, user)
       |> get("/api/v1/blocks")
+      |> json_response_and_validate_schema(200)
 
-    other_user_id = to_string(other_user.id)
-    assert [%{"id" => ^other_user_id}] = json_response_and_validate_schema(conn, 200)
+    assert [id1, id2, id3] == Enum.map(result, & &1["id"])
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/blocks?limit=1")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id1}] = result
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/blocks?since_id=#{id1}")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id2}, %{"id" => ^id3}] = result
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/blocks?since_id=#{id1}&max_id=#{id3}")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id2}] = result
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/blocks?since_id=#{id1}&limit=1")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id2}] = result
   end
 end

From 6734abcbd448b92d57ec376b796fea1fad18b792 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 13 Oct 2020 21:58:18 +0400
Subject: [PATCH 045/104] Add pagination for Mutes

---
 .../api_spec/operations/account_operation.ex  |  1 +
 .../controllers/account_controller.ex         | 12 +++--
 .../controllers/account_controller_test.exs   | 49 +++++++++++++++++--
 3 files changed, 54 insertions(+), 8 deletions(-)

diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index 9cd516f05..4934b7788 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -335,6 +335,7 @@ def mutes_operation do
       operationId: "AccountController.mutes",
       description: "Accounts the user has muted.",
       security: [%{"oAuth" => ["follow", "read:mutes"]}],
+      parameters: pagination_params(),
       responses: %{
         200 => Operation.response("Accounts", "application/json", array_of_accounts())
       }
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 1b221e3a1..c8606e5d6 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -442,9 +442,15 @@ def follow_by_uri(%{body_params: %{uri: uri}} = conn, _) do
   end
 
   @doc "GET /api/v1/mutes"
-  def mutes(%{assigns: %{user: user}} = conn, _) do
-    users = User.muted_users(user, _restrict_deactivated = true)
-    render(conn, "index.json", users: users, for: user, as: :user)
+  def mutes(%{assigns: %{user: user}} = conn, params) do
+    users =
+      user
+      |> User.muted_users_relation(_restrict_deactivated = true)
+      |> Pleroma.Pagination.fetch_paginated(Map.put(params, :skip_order, true))
+
+    conn
+    |> add_link_headers(users)
+    |> render("index.json", users: users, for: user, as: :user)
   end
 
   @doc "GET /api/v1/blocks"
diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
index 6ad9dfc39..69f2b6f4a 100644
--- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs
@@ -1509,14 +1509,53 @@ test "returns an empty list on a bad request", %{conn: conn} do
 
   test "getting a list of mutes" do
     %{user: user, conn: conn} = oauth_access(["read:mutes"])
-    other_user = insert(:user)
+    %{id: id1} = other_user1 = insert(:user)
+    %{id: id2} = other_user2 = insert(:user)
+    %{id: id3} = other_user3 = insert(:user)
 
-    {:ok, _user_relationships} = User.mute(user, other_user)
+    {:ok, _user_relationships} = User.mute(user, other_user1)
+    {:ok, _user_relationships} = User.mute(user, other_user2)
+    {:ok, _user_relationships} = User.mute(user, other_user3)
 
-    conn = get(conn, "/api/v1/mutes")
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/mutes")
+      |> json_response_and_validate_schema(200)
 
-    other_user_id = to_string(other_user.id)
-    assert [%{"id" => ^other_user_id}] = json_response_and_validate_schema(conn, 200)
+    assert [id1, id2, id3] == Enum.map(result, & &1["id"])
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/mutes?limit=1")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id1}] = result
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/mutes?since_id=#{id1}")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id2}, %{"id" => ^id3}] = result
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/mutes?since_id=#{id1}&max_id=#{id3}")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id2}] = result
+
+    result =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/mutes?since_id=#{id1}&limit=1")
+      |> json_response_and_validate_schema(200)
+
+    assert [%{"id" => ^id2}] = result
   end
 
   test "getting a list of blocks" do

From 2b58b0dbce36042a8acbb41a8ddc8696d2e00d0e Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 13 Oct 2020 21:58:26 +0400
Subject: [PATCH 046/104] Update CHANGELOG

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8fc1750d1..216d7bb32 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
 - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
 - Mix task option for force-unfollowing relays
+- Support pagination of blocks and mutes
 
 ### Changed
 

From 3b5a7a6b14f4c09d1d371d6fcb49bece84d6c3e1 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 16 Oct 2020 00:32:20 +0200
Subject: [PATCH 047/104] federation_status: New endpoint showing unreachable
 instances

---
 lib/pleroma/instances.ex                      |  1 +
 lib/pleroma/instances/instance.ex             | 11 +++++
 .../controllers/instances_controller.ex       | 19 +++++++++
 lib/pleroma/web/router.ex                     |  1 +
 .../controllers/instances_controller_test.exs | 40 +++++++++++++++++++
 5 files changed, 72 insertions(+)
 create mode 100644 lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
 create mode 100644 test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs

diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
index 557e8decf..7315bd7cb 100644
--- a/lib/pleroma/instances.ex
+++ b/lib/pleroma/instances.ex
@@ -11,6 +11,7 @@ defmodule Pleroma.Instances do
   defdelegate reachable?(url_or_host), to: @adapter
   defdelegate set_reachable(url_or_host), to: @adapter
   defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
+  defdelegate get_consistently_unreachable(), to: @adapter
 
   def set_consistently_unreachable(url_or_host),
     do: set_unreachable(url_or_host, reachability_datetime_threshold())
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index f0f601469..df471a39d 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -119,6 +119,17 @@ def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host)
 
   def set_unreachable(_, _), do: {:error, nil}
 
+  def get_consistently_unreachable do
+    reachability_datetime_threshold = Instances.reachability_datetime_threshold()
+
+    from(i in Instance,
+      where: ^reachability_datetime_threshold > i.unreachable_since,
+      order_by: i.unreachable_since,
+      select: {i.host, i.unreachable_since}
+    )
+    |> Repo.all()
+  end
+
   defp parse_datetime(datetime) when is_binary(datetime) do
     NaiveDateTime.from_iso8601(datetime)
   end
diff --git a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
new file mode 100644
index 000000000..bd95cb523
--- /dev/null
+++ b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.InstancesController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Instances
+
+  # defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaInstancesController
+
+  def show(conn, _params) do
+    unreachable =
+      Instances.get_consistently_unreachable()
+      |> Enum.reduce(%{}, fn {host, date}, acc -> Map.put(acc, host, to_string(date)) end)
+
+    json(conn, %{"unreachable" => unreachable})
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index d2d939989..5f9a749e4 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -373,6 +373,7 @@ defmodule Pleroma.Web.Router do
   scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do
     pipe_through(:api)
     get("/accounts/:id/scrobbles", ScrobbleController, :index)
+    get("/federation_status", InstancesController, :show)
   end
 
   scope "/api/v1", Pleroma.Web.MastodonAPI do
diff --git a/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs
new file mode 100644
index 000000000..9ce901ce3
--- /dev/null
+++ b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs
@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaApi.InstancesControllerTest do
+  use Pleroma.Web.ConnCase
+
+  alias Pleroma.Instances
+
+  setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1)
+
+  setup do
+    constant = "http://consistently-unreachable.name/"
+    eventual = "http://eventually-unreachable.com/path"
+
+    {:ok, %Pleroma.Instances.Instance{unreachable_since: constant_unreachable}} =
+      Instances.set_consistently_unreachable(constant)
+
+    _eventual_unrechable = Instances.set_unreachable(eventual)
+
+    %{constant_unreachable: constant_unreachable, constant: constant}
+  end
+
+  test "GET /api/v1/pleroma/federation_status", %{
+    conn: conn,
+    constant_unreachable: constant_unreachable,
+    constant: constant
+  } do
+    constant_host = URI.parse(constant).host
+
+    assert conn
+           |> put_req_header("content-type", "application/json")
+           |> get("/api/v1/pleroma/federation_status")
+           |> json_response(200) == %{
+             "unreachable" => %{constant_host => to_string(constant_unreachable)}
+           }
+
+    # |> json_response_and_validate_schema(200)
+  end
+end

From aafdc975bdd38f74cdf5d3f8517d41c5dd76c56b Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 16 Oct 2020 01:13:52 +0200
Subject: [PATCH 048/104] federation_status: Add ApiSpec

---
 .../operations/pleroma_instances_operation.ex | 40 +++++++++++++++++++
 .../controllers/instances_controller.ex       |  4 +-
 .../controllers/instances_controller_test.exs |  4 +-
 3 files changed, 44 insertions(+), 4 deletions(-)
 create mode 100644 lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex

diff --git a/lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex
new file mode 100644
index 000000000..2c455b0df
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex
@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaInstancesOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["PleromaInstances"],
+      summary: "Instances federation status",
+      description: "Information about instances deemed unreachable by the server",
+      operationId: "PleromaInstances.show",
+      responses: %{
+        200 => Operation.response("PleromaInstances", "application/json", pleroma_instances())
+      }
+    }
+  end
+
+  def pleroma_instances do
+    %Schema{
+      type: :object,
+      properties: %{
+        unreachable: %Schema{
+          type: :object,
+          properties: %{hostname: %Schema{type: :string, format: :"date-time"}}
+        }
+      },
+      example: %{
+        "unreachable" => %{"consistently-unreachable.name" => "2020-10-14 22:07:58.216473"}
+      }
+    }
+  end
+end
diff --git a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
index bd95cb523..c577f1d1e 100644
--- a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
@@ -7,7 +7,9 @@ defmodule Pleroma.Web.PleromaAPI.InstancesController do
 
   alias Pleroma.Instances
 
-  # defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaInstancesController
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaInstancesOperation
 
   def show(conn, _params) do
     unreachable =
diff --git a/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs
index 9ce901ce3..13491ed9c 100644
--- a/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/instances_controller_test.exs
@@ -31,10 +31,8 @@ test "GET /api/v1/pleroma/federation_status", %{
     assert conn
            |> put_req_header("content-type", "application/json")
            |> get("/api/v1/pleroma/federation_status")
-           |> json_response(200) == %{
+           |> json_response_and_validate_schema(200) == %{
              "unreachable" => %{constant_host => to_string(constant_unreachable)}
            }
-
-    # |> json_response_and_validate_schema(200)
   end
 end

From 1b8fd7e65af980c42b72f584c2a957b12ca5c78b Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Fri, 16 Oct 2020 17:32:05 +0000
Subject: [PATCH 049/104] Adds feature to permit e.g., local admins and
 community moderators to automatically follow all newly registered accounts

---
 config/config.exs                |  1 +
 config/description.exs           | 11 +++++++++++
 docs/configuration/cheatsheet.md |  1 +
 lib/pleroma/user.ex              | 11 +++++++++++
 test/pleroma/user_test.exs       | 18 ++++++++++++++++++
 5 files changed, 42 insertions(+)

diff --git a/config/config.exs b/config/config.exs
index 2c6142360..0f3785d12 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -235,6 +235,7 @@
     "text/bbcode"
   ],
   autofollowed_nicknames: [],
+  autofollowing_nicknames: [],
   max_pinned_statuses: 1,
   attachment_links: false,
   max_report_comment_size: 1000,
diff --git a/config/description.exs b/config/description.exs
index 2a1898922..2bbb12540 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -837,6 +837,17 @@
           "rinpatch"
         ]
       },
+      %{
+        key: :autofollowing_nicknames,
+        type: {:list, :string},
+        description:
+          "Set to nicknames of (local) users that automatically follows every newly registered user",
+        suggestions: [
+          "admin",
+          "info",
+          "moderator",
+        ]
+      },
       %{
         key: :attachment_links,
         type: :boolean,
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index 0b13d7e88..f4b4b6c3c 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -45,6 +45,7 @@ To add configuration to your config file, you can copy it from the base config.
     older software for theses nicknames.
 * `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature.
 * `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow.
+* `autofollowing_nicknames`: Set to nicknames of (local) users that automatically follows every newly registered user.
 * `attachment_links`: Set to true to enable automatically adding attachment link text to statuses.
 * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`).
 * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`.
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index dc41d0001..2a3495103 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -765,6 +765,16 @@ defp autofollow_users(user) do
     follow_all(user, autofollowed_users)
   end
 
+  defp autofollowing_users(user) do
+    candidates = Config.get([:instance, :autofollowing_nicknames])
+
+    User.Query.build(%{nickname: candidates, local: true, deactivated: false})
+    |> Repo.all()
+    |> Enum.each(&follow(&1, user, :follow_accept))
+
+    {:ok, :success}
+  end
+
   @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)"
   def register(%Ecto.Changeset{} = changeset) do
     with {:ok, user} <- Repo.insert(changeset) do
@@ -774,6 +784,7 @@ def register(%Ecto.Changeset{} = changeset) do
 
   def post_register_action(%User{} = user) do
     with {:ok, user} <- autofollow_users(user),
+         {:ok, _} <- autofollowing_users(user),
          {:ok, user} <- set_cache(user),
          {:ok, _} <- send_welcome_email(user),
          {:ok, _} <- send_welcome_message(user),
diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs
index 7220ce846..9ae52d594 100644
--- a/test/pleroma/user_test.exs
+++ b/test/pleroma/user_test.exs
@@ -388,6 +388,7 @@ test "fetches correct profile for nickname beginning with number" do
     }
 
     setup do: clear_config([:instance, :autofollowed_nicknames])
+    setup do: clear_config([:instance, :autofollowing_nicknames])
     setup do: clear_config([:welcome])
     setup do: clear_config([:instance, :account_activation_required])
 
@@ -408,6 +409,23 @@ test "it autofollows accounts that are set for it" do
       refute User.following?(registered_user, remote_user)
     end
 
+    test "it adds automatic followers for new registered accounts" do
+      user1 = insert(:user)
+      user2 = insert(:user)
+
+      Pleroma.Config.put([:instance, :autofollowing_nicknames], [
+        user1.nickname,
+        user2.nickname
+      ])
+
+      cng = User.register_changeset(%User{}, @full_user_data)
+
+      {:ok, registered_user} = User.register(cng)
+
+      assert User.following?(user1, registered_user)
+      assert User.following?(user2, registered_user)
+    end
+
     test "it sends a welcome message if it is set" do
       welcome_user = insert(:user)
       Pleroma.Config.put([:welcome, :direct_message, :enabled], true)

From efd6572ffbf4cce86e28d351d3cbddd0a5334980 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Fri, 16 Oct 2020 17:43:44 +0000
Subject: [PATCH 050/104] Remove suggestions

---
 config/description.exs | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/config/description.exs b/config/description.exs
index 2bbb12540..79e1368d0 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -841,12 +841,7 @@
         key: :autofollowing_nicknames,
         type: {:list, :string},
         description:
-          "Set to nicknames of (local) users that automatically follows every newly registered user",
-        suggestions: [
-          "admin",
-          "info",
-          "moderator",
-        ]
+          "Set to nicknames of (local) users that automatically follows every newly registered user"
       },
       %{
         key: :attachment_links,

From d54233760f4c006d89aa80e0ae78cb6910fc74ab Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sat, 17 Oct 2020 13:33:57 +0300
Subject: [PATCH 051/104] [#3053] Post-merge fix.

---
 lib/pleroma/web/feed/user_controller.ex | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex
index b66fdf275..a5013d2c0 100644
--- a/lib/pleroma/web/feed/user_controller.ex
+++ b/lib/pleroma/web/feed/user_controller.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Feed.UserController do
   use Pleroma.Web, :controller
 
+  alias Pleroma.Config
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.ActivityPubController

From d16336e7fbf7851ee5f50a17747067f2be741581 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@feld.me>
Date: Sat, 17 Oct 2020 16:54:05 +0000
Subject: [PATCH 052/104] Document autofollowing_nicknames

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 36a84b1a8..fd5b9034d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
 - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
 - Mix task option for force-unfollowing relays
+- Setting `:instance, autofollowing_nicknames` to provide a way to make accounts automatically follow new users that register on the local Pleroma instance.
 
 ### Changed
 

From 524fb0e4c2561f4a2e4c8e58519df991f034c901 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sun, 18 Oct 2020 21:22:21 +0300
Subject: [PATCH 053/104] [#1668] Restricted access to app metrics endpoint by
 default. Added ability to configure IP whitelist for this endpoint. Added
 tests and documentation.

---
 CHANGELOG.md                                  |  2 +
 config/config.exs                             |  7 +-
 docs/API/prometheus.md                        | 26 ++++++-
 lib/pleroma/helpers/inet_helper.ex            | 19 +++++
 lib/pleroma/web/endpoint.ex                   | 40 +++++++++--
 .../web/endpoint/metrics_exporter_test.exs    | 69 +++++++++++++++++++
 6 files changed, 154 insertions(+), 9 deletions(-)
 create mode 100644 lib/pleroma/helpers/inet_helper.ex
 create mode 100644 test/pleroma/web/endpoint/metrics_exporter_test.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 05e94581a..9f6a31f23 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,12 +12,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
 - Pleroma API: Importing the mutes users from CSV files.
 - Experimental websocket-based federation between Pleroma instances.
+- App metrics: ability to restrict access to specified IP whitelist.
 
 ### Changed
 
 - **Breaking** Requires `libmagic` (or `file`) to guess file types.
 - **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
 - **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
+- **Breaking:** App metrics endpoint (`/api/pleroma/app_metrics`) is disabled by default, check `docs/API/prometheus.md` on enabling and configuring. 
 - Search: Users are now findable by their urls.
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
diff --git a/config/config.exs b/config/config.exs
index 2c6142360..a7aae5802 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -636,7 +636,12 @@
 
 config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: false
 
-config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics"
+config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
+  enabled: false,
+  auth: false,
+  ip_whitelist: [],
+  path: "/api/pleroma/app_metrics",
+  format: :text
 
 config :pleroma, Pleroma.ScheduledActivity,
   daily_user_limit: 25,
diff --git a/docs/API/prometheus.md b/docs/API/prometheus.md
index 19c564e3c..a5158d905 100644
--- a/docs/API/prometheus.md
+++ b/docs/API/prometheus.md
@@ -2,15 +2,37 @@
 
 Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library.
 
+Config example:
+
+```
+config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
+  enabled: true,
+  auth: {:basic, "myusername", "mypassword"},
+  ip_whitelist: ["127.0.0.1"],
+  path: "/api/pleroma/app_metrics",
+  format: :text
+```
+
+* `enabled` (Pleroma extension) enables the endpoint
+* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs
+* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation)
+* `format` sets the output format (`:text` or `:protobuf`)
+* `path` sets the path to app metrics page 
+
+
 ## `/api/pleroma/app_metrics`
+
 ### Exports Prometheus application metrics
+
 * Method: `GET`
-* Authentication: not required
+* Authentication: not required by default (see configuration options above)
 * Params: none
-* Response: JSON
+* Response: text
 
 ## Grafana
+
 ### Config example
+
 The following is a config example to use with [Grafana](https://grafana.com)
 
 ```
diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex
new file mode 100644
index 000000000..126f82381
--- /dev/null
+++ b/lib/pleroma/helpers/inet_helper.ex
@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.InetHelper do
+  def parse_address(ip) when is_tuple(ip) do
+    {:ok, ip}
+  end
+
+  def parse_address(ip) when is_binary(ip) do
+    ip
+    |> String.to_charlist()
+    |> parse_address()
+  end
+
+  def parse_address(ip) do
+    :inet.parse_address(ip)
+  end
+end
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 56562c12f..1a8fdd8b9 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -7,6 +7,8 @@ defmodule Pleroma.Web.Endpoint do
 
   require Pleroma.Constants
 
+  alias Pleroma.Config
+
   socket("/socket", Pleroma.Web.UserSocket)
 
   plug(Pleroma.Web.Plugs.SetLocalePlug)
@@ -86,19 +88,19 @@ defmodule Pleroma.Web.Endpoint do
   plug(Plug.Parsers,
     parsers: [
       :urlencoded,
-      {:multipart, length: {Pleroma.Config, :get, [[:instance, :upload_limit]]}},
+      {:multipart, length: {Config, :get, [[:instance, :upload_limit]]}},
       :json
     ],
     pass: ["*/*"],
     json_decoder: Jason,
-    length: Pleroma.Config.get([:instance, :upload_limit]),
+    length: Config.get([:instance, :upload_limit]),
     body_reader: {Pleroma.Web.Plugs.DigestPlug, :read_body, []}
   )
 
   plug(Plug.MethodOverride)
   plug(Plug.Head)
 
-  secure_cookies = Pleroma.Config.get([__MODULE__, :secure_cookie_flag])
+  secure_cookies = Config.get([__MODULE__, :secure_cookie_flag])
 
   cookie_name =
     if secure_cookies,
@@ -106,7 +108,7 @@ defmodule Pleroma.Web.Endpoint do
       else: "pleroma_key"
 
   extra =
-    Pleroma.Config.get([__MODULE__, :extra_cookie_attrs])
+    Config.get([__MODULE__, :extra_cookie_attrs])
     |> Enum.join(";")
 
   # The session will be stored in the cookie and signed,
@@ -116,7 +118,7 @@ defmodule Pleroma.Web.Endpoint do
     Plug.Session,
     store: :cookie,
     key: cookie_name,
-    signing_salt: Pleroma.Config.get([__MODULE__, :signing_salt], "CqaoopA2"),
+    signing_salt: Config.get([__MODULE__, :signing_salt], "CqaoopA2"),
     http_only: true,
     secure: secure_cookies,
     extra: extra
@@ -136,8 +138,34 @@ defmodule MetricsExporter do
     use Prometheus.PlugExporter
   end
 
+  defmodule MetricsExporterCaller do
+    @behaviour Plug
+
+    def init(opts), do: opts
+
+    def call(conn, opts) do
+      prometheus_config = Application.get_env(:prometheus, MetricsExporter, [])
+      ip_whitelist = List.wrap(prometheus_config[:ip_whitelist])
+
+      cond do
+        !prometheus_config[:enabled] ->
+          conn
+
+        ip_whitelist != [] and
+            !Enum.find(ip_whitelist, fn ip ->
+              Pleroma.Helpers.InetHelper.parse_address(ip) == {:ok, conn.remote_ip}
+            end) ->
+          conn
+
+        true ->
+          MetricsExporter.call(conn, opts)
+      end
+    end
+  end
+
   plug(PipelineInstrumenter)
-  plug(MetricsExporter)
+
+  plug(MetricsExporterCaller)
 
   plug(Pleroma.Web.Router)
 
diff --git a/test/pleroma/web/endpoint/metrics_exporter_test.exs b/test/pleroma/web/endpoint/metrics_exporter_test.exs
new file mode 100644
index 000000000..f954cc1e7
--- /dev/null
+++ b/test/pleroma/web/endpoint/metrics_exporter_test.exs
@@ -0,0 +1,69 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Endpoint.MetricsExporterTest do
+  use Pleroma.Web.ConnCase
+
+  alias Pleroma.Web.Endpoint.MetricsExporter
+
+  defp config do
+    Application.get_env(:prometheus, MetricsExporter)
+  end
+
+  describe "with default config" do
+    test "does NOT expose app metrics", %{conn: conn} do
+      conn
+      |> get(config()[:path])
+      |> json_response(404)
+    end
+  end
+
+  describe "when enabled" do
+    setup do
+      initial_config = config()
+      on_exit(fn -> Application.put_env(:prometheus, MetricsExporter, initial_config) end)
+
+      Application.put_env(
+        :prometheus,
+        MetricsExporter,
+        Keyword.put(initial_config, :enabled, true)
+      )
+    end
+
+    test "serves app metrics", %{conn: conn} do
+      conn = get(conn, config()[:path])
+      assert response = response(conn, 200)
+
+      for metric <- [
+            "http_requests_total",
+            "http_request_duration_microseconds",
+            "phoenix_controller_render_duration",
+            "phoenix_controller_call_duration",
+            "telemetry_scrape_duration",
+            "erlang_vm_memory_atom_bytes_total"
+          ] do
+        assert response =~ ~r/#{metric}/
+      end
+    end
+
+    test "when IP whitelist configured, " <>
+           "serves app metrics only if client IP is whitelisted",
+         %{conn: conn} do
+      Application.put_env(
+        :prometheus,
+        MetricsExporter,
+        Keyword.put(config(), :ip_whitelist, ["127.127.127.127", {1, 1, 1, 1}, '255.255.255.255'])
+      )
+
+      conn
+      |> get(config()[:path])
+      |> json_response(404)
+
+      conn
+      |> Map.put(:remote_ip, {127, 127, 127, 127})
+      |> get(config()[:path])
+      |> response(200)
+    end
+  end
+end

From 98f32cf8204113c6d019653c22e446e558147248 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 19 Oct 2020 15:30:32 +0400
Subject: [PATCH 054/104] Fix tests

---
 lib/pleroma/web/pleroma_api/controllers/backup_controller.ex | 2 +-
 test/backup_test.exs                                         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
index e52c77ff2..8e3d081f3 100644
--- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.PleromaAPI.BackupController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
   plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create])
diff --git a/test/backup_test.exs b/test/backup_test.exs
index 23c08b680..078e03621 100644
--- a/test/backup_test.exs
+++ b/test/backup_test.exs
@@ -19,7 +19,7 @@ defmodule Pleroma.BackupTest do
   setup do
     clear_config([Pleroma.Upload, :uploader])
     clear_config([Pleroma.Backup, :limit_days])
-    clear_config([Pleroma.Emails.Mailer, :enabled])
+    clear_config([Pleroma.Emails.Mailer, :enabled], true)
   end
 
   test "it requries enabled email" do

From c1976d5b19fbceaecf1f52711fe35e1c7d5312aa Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 19 Oct 2020 18:14:39 +0400
Subject: [PATCH 055/104] Fix credo warnings

---
 test/{ => pleroma}/backup_test.exs                                | 0
 .../web/pleroma_api/controllers/backup_controller_test.exs        | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename test/{ => pleroma}/backup_test.exs (100%)
 rename test/{ => pleroma}/web/pleroma_api/controllers/backup_controller_test.exs (100%)

diff --git a/test/backup_test.exs b/test/pleroma/backup_test.exs
similarity index 100%
rename from test/backup_test.exs
rename to test/pleroma/backup_test.exs
diff --git a/test/web/pleroma_api/controllers/backup_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
similarity index 100%
rename from test/web/pleroma_api/controllers/backup_controller_test.exs
rename to test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs

From 3a28aa8814e186eea67df7da00df1636823291f3 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Tue, 20 Oct 2020 15:13:20 +0300
Subject: [PATCH 056/104] [#1668] Added :prometheus group config to
 config/description.exs.

---
 config/description.exs | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/config/description.exs b/config/description.exs
index 2a1898922..0fe86ded7 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3722,5 +3722,41 @@
         suggestions: [2]
       }
     ]
+  },
+  %{
+    group: :prometheus,
+    key: Pleroma.Web.Endpoint.MetricsExporter,
+    type: :group,
+    description: "Prometheus app metrics endpoint configuration",
+    children: [
+      %{
+        key: :enabled,
+        type: :boolean,
+        description: "[Pleroma extension] Enables app metrics endpoint."
+      },
+      %{
+        key: :ip_whitelist,
+        type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}],
+        description: "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses."
+      },
+      %{
+        key: :auth,
+        type: [:boolean, :tuple],
+        description: "Enables HTTP Basic Auth for app metrics endpoint.",
+        suggestion: [false, {:basic, "myusername", "mypassword"}]
+      },
+      %{
+        key: :path,
+        type: :string,
+        description: "App metrics endpoint URI path.",
+        suggestions: ["/api/pleroma/app_metrics"]
+      },
+      %{
+        key: :format,
+        type: :atom,
+        description: "App metrics endpoint output format.",
+        suggestions: [:text, :protobuf]
+      }
+    ]
   }
 ]

From ad605e3e16ba3f6ee3df7a0a3e6705036fef369f Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 20 Oct 2020 17:16:58 +0400
Subject: [PATCH 057/104] Rename `Pleroma.Backup` to `Pleroma.User.Backup`

---
 config/config.exs                                      |  2 +-
 config/description.exs                                 |  2 +-
 docs/configuration/cheatsheet.md                       |  2 +-
 lib/pleroma/{ => user}/backup.ex                       |  4 ++--
 .../web/admin_api/controllers/admin_api_controller.ex  |  2 +-
 .../web/pleroma_api/controllers/backup_controller.ex   |  7 ++++---
 lib/pleroma/web/pleroma_api/views/backup_view.ex       |  2 +-
 lib/pleroma/workers/backup_worker.ex                   |  4 ++--
 test/pleroma/{ => user}/backup_test.exs                | 10 +++++-----
 .../controllers/admin_api_controller_test.exs          |  6 +++---
 .../pleroma_api/controllers/backup_controller_test.exs |  2 +-
 11 files changed, 22 insertions(+), 21 deletions(-)
 rename lib/pleroma/{ => user}/backup.ex (98%)
 rename test/pleroma/{ => user}/backup_test.exs (97%)

diff --git a/config/config.exs b/config/config.exs
index 63e386250..c758c818c 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -831,7 +831,7 @@
 
 config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
 
-config :pleroma, Pleroma.Backup,
+config :pleroma, Pleroma.User.Backup,
   purge_after_days: 30,
   limit_days: 7,
   dir: nil
diff --git a/config/description.exs b/config/description.exs
index 88f2a6133..9f23b6d3d 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3731,7 +3731,7 @@
   },
   %{
     group: :pleroma,
-    key: Pleroma.Backup,
+    key: Pleroma.User.Backup,
     type: :group,
     description: "Account Backup",
     children: [
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
index aafc43f3d..b40a2aebf 100644
--- a/docs/configuration/cheatsheet.md
+++ b/docs/configuration/cheatsheet.md
@@ -1077,7 +1077,7 @@ Control favicons for instances.
 
 * `enabled`: Allow/disallow displaying and getting instances favicons
 
-## Account Backup
+## Pleroma.User.Backup
 
 !!! note
     Requires enabled email
diff --git a/lib/pleroma/backup.ex b/lib/pleroma/user/backup.ex
similarity index 98%
rename from lib/pleroma/backup.ex
rename to lib/pleroma/user/backup.ex
index 629e879a7..a9041fd94 100644
--- a/lib/pleroma/backup.ex
+++ b/lib/pleroma/user/backup.ex
@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Backup do
+defmodule Pleroma.User.Backup do
   use Ecto.Schema
 
   import Ecto.Changeset
@@ -65,7 +65,7 @@ defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok
   defp validate_limit(user, nil) do
     case get_last(user.id) do
       %__MODULE__{inserted_at: inserted_at} ->
-        days = Pleroma.Config.get([Pleroma.Backup, :limit_days])
+        days = Pleroma.Config.get([__MODULE__, :limit_days])
         diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
 
         if diff > days do
diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index a4f0d7d34..0a27c5861 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -685,7 +685,7 @@ def stats(conn, params) do
 
   def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
     with %User{} = user <- User.get_by_nickname(nickname),
-         {:ok, _} <- Pleroma.Backup.create(user, admin.id) do
+         {:ok, _} <- Pleroma.User.Backup.create(user, admin.id) do
       ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"})
 
       json(conn, "")
diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
index 8e3d081f3..bd7b36880 100644
--- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do
   use Pleroma.Web, :controller
 
   alias Pleroma.Web.Plugs.OAuthScopesPlug
+  alias Pleroma.User.Backup
 
   action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
   plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create])
@@ -14,13 +15,13 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation
 
   def index(%{assigns: %{user: user}} = conn, _params) do
-    backups = Pleroma.Backup.list(user)
+    backups = Backup.list(user)
     render(conn, "index.json", backups: backups)
   end
 
   def create(%{assigns: %{user: user}} = conn, _params) do
-    with {:ok, _} <- Pleroma.Backup.create(user) do
-      backups = Pleroma.Backup.list(user)
+    with {:ok, _} <- Backup.create(user) do
+      backups = Backup.list(user)
       render(conn, "index.json", backups: backups)
     end
   end
diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex
index bf40a001e..af75876aa 100644
--- a/lib/pleroma/web/pleroma_api/views/backup_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.PleromaAPI.BackupView do
   use Pleroma.Web, :view
 
-  alias Pleroma.Backup
+  alias Pleroma.User.Backup
   alias Pleroma.Web.CommonAPI.Utils
 
   def render("show.json", %{backup: %Backup{} = backup}) do
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
index 65754b6a2..5b4985983 100644
--- a/lib/pleroma/workers/backup_worker.ex
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Workers.BackupWorker do
   use Oban.Worker, queue: :backup, max_attempts: 1
 
   alias Oban.Job
-  alias Pleroma.Backup
+  alias Pleroma.User.Backup
 
   def process(backup, admin_user_id \\ nil) do
     %{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id}
@@ -15,7 +15,7 @@ def process(backup, admin_user_id \\ nil) do
   end
 
   def schedule_deletion(backup) do
-    days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days])
+    days = Pleroma.Config.get([Backup, :purge_after_days])
     time = 60 * 60 * 24 * days
     scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time)
 
diff --git a/test/pleroma/backup_test.exs b/test/pleroma/user/backup_test.exs
similarity index 97%
rename from test/pleroma/backup_test.exs
rename to test/pleroma/user/backup_test.exs
index 078e03621..5ad587833 100644
--- a/test/pleroma/backup_test.exs
+++ b/test/pleroma/user/backup_test.exs
@@ -2,7 +2,7 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.BackupTest do
+defmodule Pleroma.User.BackupTest do
   use Oban.Testing, repo: Pleroma.Repo
   use Pleroma.DataCase
 
@@ -10,7 +10,7 @@ defmodule Pleroma.BackupTest do
   import Pleroma.Factory
   import Swoosh.TestAssertions
 
-  alias Pleroma.Backup
+  alias Pleroma.User.Backup
   alias Pleroma.Bookmark
   alias Pleroma.Tests.ObanHelpers
   alias Pleroma.Web.CommonAPI
@@ -18,7 +18,7 @@ defmodule Pleroma.BackupTest do
 
   setup do
     clear_config([Pleroma.Upload, :uploader])
-    clear_config([Pleroma.Backup, :limit_days])
+    clear_config([Backup, :limit_days])
     clear_config([Pleroma.Emails.Mailer, :enabled], true)
   end
 
@@ -44,7 +44,7 @@ test "it creates a backup record and an Oban job" do
 
   test "it return an error if the export limit is over" do
     %{id: user_id} = user = insert(:user)
-    limit_days = Pleroma.Config.get([Pleroma.Backup, :limit_days])
+    limit_days = Pleroma.Config.get([Backup, :limit_days])
     assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
     backup = Backup.get(args["backup_id"])
     assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
@@ -76,7 +76,7 @@ test "it process a backup record" do
   end
 
   test "it removes outdated backups after creating a fresh one" do
-    Pleroma.Config.put([Pleroma.Backup, :limit_days], -1)
+    Pleroma.Config.put([Backup, :limit_days], -1)
     Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
     user = insert(:user)
 
diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
index 34d48c2c1..5efe8ef71 100644
--- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
@@ -2038,7 +2038,7 @@ test "it creates a backup", %{conn: conn} do
                |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
                |> json_response(200)
 
-      assert [backup] = Repo.all(Pleroma.Backup)
+      assert [backup] = Repo.all(Pleroma.User.Backup)
 
       ObanHelpers.perform_all()
 
@@ -2079,7 +2079,7 @@ test "it doesn't limit admins", %{conn: conn} do
                |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
                |> json_response(200)
 
-      assert [_backup] = Repo.all(Pleroma.Backup)
+      assert [_backup] = Repo.all(Pleroma.User.Backup)
 
       assert "" ==
                conn
@@ -2088,7 +2088,7 @@ test "it doesn't limit admins", %{conn: conn} do
                |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
                |> json_response(200)
 
-      assert Repo.aggregate(Pleroma.Backup, :count) == 2
+      assert Repo.aggregate(Pleroma.User.Backup, :count) == 2
     end
   end
 end
diff --git a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
index b2ac74c7d..f1941f6dd 100644
--- a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do
   use Pleroma.Web.ConnCase
 
-  alias Pleroma.Backup
+  alias Pleroma.User.Backup
   alias Pleroma.Web.PleromaAPI.BackupView
 
   setup do

From 50d428088017e0383d8b35d4ab1b831f40646ab0 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Tue, 20 Oct 2020 16:18:24 +0300
Subject: [PATCH 058/104] [#1668] Formatting fix.

---
 config/description.exs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/description.exs b/config/description.exs
index 0fe86ded7..11755d757 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -3737,7 +3737,8 @@
       %{
         key: :ip_whitelist,
         type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}],
-        description: "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses."
+        description:
+          "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses."
       },
       %{
         key: :auth,

From 034ac43f3a91178694d3c621c52ce68207ec4f69 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 20 Oct 2020 17:47:04 +0400
Subject: [PATCH 059/104] Fix credo warnings

---
 lib/pleroma/web/pleroma_api/controllers/backup_controller.ex | 2 +-
 test/pleroma/user/backup_test.exs                            | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
index bd7b36880..dd0a2e22f 100644
--- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex
@@ -5,8 +5,8 @@
 defmodule Pleroma.Web.PleromaAPI.BackupController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.Web.Plugs.OAuthScopesPlug
   alias Pleroma.User.Backup
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
   plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create])
diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs
index 5ad587833..513798911 100644
--- a/test/pleroma/user/backup_test.exs
+++ b/test/pleroma/user/backup_test.exs
@@ -10,9 +10,9 @@ defmodule Pleroma.User.BackupTest do
   import Pleroma.Factory
   import Swoosh.TestAssertions
 
-  alias Pleroma.User.Backup
   alias Pleroma.Bookmark
   alias Pleroma.Tests.ObanHelpers
+  alias Pleroma.User.Backup
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Workers.BackupWorker
 

From b18b93bbed925e5058d941662e93f0a46a27c325 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 21 Oct 2020 21:42:21 +0400
Subject: [PATCH 060/104] Fix formatting and typos in "Managing frontends"
 guide

---
 docs/administration/CLI_tasks/frontend.md | 93 +++++++++++++++--------
 1 file changed, 60 insertions(+), 33 deletions(-)

diff --git a/docs/administration/CLI_tasks/frontend.md b/docs/administration/CLI_tasks/frontend.md
index 7d1c1e937..d4a48cb56 100644
--- a/docs/administration/CLI_tasks/frontend.md
+++ b/docs/administration/CLI_tasks/frontend.md
@@ -1,12 +1,23 @@
 # Managing frontends
 
-`mix pleroma.frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]`
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]
+    ```
 
 Frontend can be installed either from local zip file, or automatically downloaded from the web.
 
-You can give all the options directly on the command like, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files.
+You can give all the options directly on the command line, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files.
+
+Currently, known `<frontend>` values are:
 
-Currently known `<frontend>` values are:
 - [admin-fe](https://git.pleroma.social/pleroma/admin-fe)
 - [kenoma](http://git.pleroma.social/lambadalambda/kenoma)
 - [pleroma-fe](http://git.pleroma.social/pleroma/pleroma-fe)
@@ -19,51 +30,67 @@ You can still install frontends that are not configured, see below.
 
 For a frontend configured under the `available` key, it's enough to install it by name.
 
-```sh tab="OTP"
-./bin/pleroma_ctl frontend install pleroma
-```
+=== "OTP"
 
-```sh tab="From Source"
-mix pleroma.frontend install pleroma
-```
+    ```sh
+    ./bin/pleroma_ctl frontend install pleroma
+    ```
 
-This will download the latest build for the the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`).
+=== "From Source"
 
-You can override any of the details. To install a pleroma build from a different url, you could do this:
+    ```sh
+    mix pleroma.frontend install pleroma
+    ```
 
-```sh tab="OPT"
-./bin/pleroma_ctl frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
-```
+This will download the latest build for the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`).
 
-```sh tab="From Source"
-mix pleroma.frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
-```
+You can override any of the details. To install a pleroma build from a different URL, you could do this:
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
+    ```
 
 Similarly, you can also install from a local zip file.
 
-```sh tab="OTP"
-./bin/pleroma_ctl frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
-```
+=== "OTP"
 
-```sh tab="From Source"
-mix pleroma.frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
-```
+    ```sh
+    ./bin/pleroma_ctl frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
+    ```
 
-The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+=== "From Source"
 
-Careful: This folder will be completely replaced on installation
+    ```sh
+    mix pleroma.frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
+    ```
+
+The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`.
+
+Careful: This folder will be completely replaced on installation.
 
 ## Example installation for an unknown frontend
 
-The installation process is the same, but you will have to give all the needed options on the commond line. For example:
+The installation process is the same, but you will have to give all the needed options on the command line. For example:
 
-```sh tab="OTP"
-./bin/pleroma_ctl frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
-```
+=== "OTP"
 
-```sh tab="From Source"
-mix pleroma.frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
-```
+    ```sh
+    ./bin/pleroma_ctl frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
+    ```
 
-If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+=== "From Source"
+
+    ```sh
+    mix pleroma.frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
+    ```
+
+If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`.
 

From 2ca98f2d94e2976ae35998aecff27809d4b066cf Mon Sep 17 00:00:00 2001
From: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Wed, 21 Oct 2020 19:40:37 +0000
Subject: [PATCH 061/104] Apply 1 suggestion(s) to 1 file(s)

---
 lib/pleroma/web/pleroma_api/controllers/instances_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
index c577f1d1e..9e97480df 100644
--- a/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/instances_controller.ex
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.PleromaAPI.InstancesController do
   def show(conn, _params) do
     unreachable =
       Instances.get_consistently_unreachable()
-      |> Enum.reduce(%{}, fn {host, date}, acc -> Map.put(acc, host, to_string(date)) end)
+      |> Map.new(fn {host, date} -> {host, to_string(date)} end)
 
     json(conn, %{"unreachable" => unreachable})
   end

From 9ef46ce4103f338d5bb75278013a63d3ae418d7e Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Mon, 21 Sep 2020 09:33:51 +0300
Subject: [PATCH 062/104] added 'unconfirmed' filter to admin/users

---
 lib/pleroma/user/query.ex                     |   5 +
 .../controllers/admin_api_controller.ex       |  55 +--
 .../controllers/admin_api_controller_test.exs | 399 +++++-------------
 test/pleroma/web/admin_api/search_test.exs    |  11 +
 4 files changed, 131 insertions(+), 339 deletions(-)

diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex
index 2440bf890..2933e7fb4 100644
--- a/lib/pleroma/user/query.ex
+++ b/lib/pleroma/user/query.ex
@@ -43,6 +43,7 @@ defmodule Pleroma.User.Query do
             active: boolean(),
             deactivated: boolean(),
             need_approval: boolean(),
+            need_confirmed: boolean(),
             is_admin: boolean(),
             is_moderator: boolean(),
             super_users: boolean(),
@@ -156,6 +157,10 @@ defp compose_query({:need_approval, _}, query) do
     where(query, [u], u.approval_pending)
   end
 
+  defp compose_query({:need_confirmed, _}, query) do
+    where(query, [u], u.confirmation_pending)
+  end
+
   defp compose_query({:followers, %User{id: id}}, query) do
     query
     |> where([u], u.id != ^id)
diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index bdd3e195d..acfbeb0c8 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -5,7 +5,8 @@
 defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   use Pleroma.Web, :controller
 
-  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
+  import Pleroma.Web.ControllerHelper,
+    only: [json_response: 3, fetch_integer_param: 3]
 
   alias Pleroma.Config
   alias Pleroma.MFA
@@ -100,12 +101,9 @@ def user_delete(conn, %{"nickname" => nickname}) do
   end
 
   def user_delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users =
-      nicknames
-      |> Enum.map(&User.get_cached_by_nickname/1)
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
 
-    users
-    |> Enum.each(fn user ->
+    Enum.each(users, fn user ->
       {:ok, delete_data, _} = Builder.delete(admin, user.ap_id)
       Pipeline.common_pipeline(delete_data, local: true)
     end)
@@ -367,16 +365,18 @@ def list_users(conn, params) do
     {page, page_size} = page_params(params)
     filters = maybe_parse_filters(params["filters"])
 
-    search_params = %{
-      query: params["query"],
-      page: page,
-      page_size: page_size,
-      tags: params["tags"],
-      name: params["name"],
-      email: params["email"]
-    }
+    search_params =
+      %{
+        query: params["query"],
+        page: page,
+        page_size: page_size,
+        tags: params["tags"],
+        name: params["name"],
+        email: params["email"]
+      }
+      |> Map.merge(filters)
 
-    with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)) do
+    with {:ok, users, count} <- Search.user(search_params) do
       json(
         conn,
         AccountView.render("index.json",
@@ -388,7 +388,7 @@ def list_users(conn, params) do
     end
   end
 
-  @filters ~w(local external active deactivated need_approval is_admin is_moderator)
+  @filters ~w(local external active deactivated need_approval need_confirmed is_admin is_moderator)
 
   @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
   defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
@@ -682,24 +682,9 @@ def stats(conn, params) do
   end
 
   defp page_params(params) do
-    {get_page(params["page"]), get_page_size(params["page_size"])}
-  end
-
-  defp get_page(page_string) when is_nil(page_string), do: 1
-
-  defp get_page(page_string) do
-    case Integer.parse(page_string) do
-      {page, _} -> page
-      :error -> 1
-    end
-  end
-
-  defp get_page_size(page_size_string) when is_nil(page_size_string), do: @users_page_size
-
-  defp get_page_size(page_size_string) do
-    case Integer.parse(page_size_string) do
-      {page_size, _} -> page_size
-      :error -> @users_page_size
-    end
+    {
+      fetch_integer_param(params, "page", 1),
+      fetch_integer_param(params, "page_size", @users_page_size)
+    }
   end
 end
diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
index cba6b43d3..686b53a80 100644
--- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
@@ -369,23 +369,7 @@ test "Show", %{conn: conn} do
 
       conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}")
 
-      expected = %{
-        "deactivated" => false,
-        "id" => to_string(user.id),
-        "local" => true,
-        "nickname" => user.nickname,
-        "roles" => %{"admin" => false, "moderator" => false},
-        "tags" => [],
-        "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-        "display_name" => HTML.strip_tags(user.name || user.nickname),
-        "confirmation_pending" => false,
-        "approval_pending" => false,
-        "url" => user.ap_id,
-        "registration_reason" => nil,
-        "actor_type" => "Person"
-      }
-
-      assert expected == json_response(conn, 200)
+      assert user_response(user) == json_response(conn, 200)
     end
 
     test "when the user doesn't exist", %{conn: conn} do
@@ -652,51 +636,20 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do
 
       users =
         [
-          %{
-            "deactivated" => admin.deactivated,
-            "id" => admin.id,
-            "nickname" => admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "local" => true,
-            "tags" => [],
-            "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
-          %{
-            "deactivated" => user.deactivated,
-            "id" => user.id,
-            "nickname" => user.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => false,
-            "tags" => ["foo", "bar"],
-            "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user.name || user.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => user.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
-          %{
-            "deactivated" => user2.deactivated,
-            "id" => user2.id,
-            "nickname" => user2.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => true,
-            "tags" => [],
-            "avatar" => User.avatar_url(user2) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user2.name || user2.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => true,
-            "url" => user2.ap_id,
-            "registration_reason" => "I'm a chill dude",
-            "actor_type" => "Person"
-          }
+          user_response(
+            admin,
+            %{"roles" => %{"admin" => true, "moderator" => false}}
+          ),
+          user_response(user, %{"local" => false, "tags" => ["foo", "bar"]}),
+          user_response(
+            user2,
+            %{
+              "local" => true,
+              "approval_pending" => true,
+              "registration_reason" => "I'm a chill dude",
+              "actor_type" => "Person"
+            }
+          )
         ]
         |> Enum.sort_by(& &1["nickname"])
 
@@ -757,23 +710,7 @@ test "regular search", %{conn: conn} do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user, %{"local" => true})]
              }
     end
 
@@ -786,23 +723,7 @@ test "search by domain", %{conn: conn} do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
     end
 
@@ -815,23 +736,7 @@ test "search by full nickname", %{conn: conn} do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
     end
 
@@ -844,23 +749,7 @@ test "search by display name", %{conn: conn} do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
     end
 
@@ -873,23 +762,7 @@ test "search by email", %{conn: conn} do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
     end
 
@@ -902,23 +775,7 @@ test "regular search with page size", %{conn: conn} do
       assert json_response(conn1, 200) == %{
                "count" => 2,
                "page_size" => 1,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
 
       conn2 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=2")
@@ -926,23 +783,7 @@ test "regular search with page size", %{conn: conn} do
       assert json_response(conn2, 200) == %{
                "count" => 2,
                "page_size" => 1,
-               "users" => [
-                 %{
-                   "deactivated" => user2.deactivated,
-                   "id" => user2.id,
-                   "nickname" => user2.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user2) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user2.name || user2.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user2.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user2)]
              }
     end
 
@@ -962,23 +803,7 @@ test "only local users" do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
     end
 
@@ -992,51 +817,14 @@ test "only local users with no query", %{conn: conn, admin: old_admin} do
 
       users =
         [
-          %{
-            "deactivated" => user.deactivated,
-            "id" => user.id,
-            "nickname" => user.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => true,
-            "tags" => [],
-            "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user.name || user.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => user.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
-          %{
-            "deactivated" => admin.deactivated,
-            "id" => admin.id,
-            "nickname" => admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "local" => true,
-            "tags" => [],
-            "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
-          %{
+          user_response(user),
+          user_response(admin, %{
+            "roles" => %{"admin" => true, "moderator" => false}
+          }),
+          user_response(old_admin, %{
             "deactivated" => false,
-            "id" => old_admin.id,
-            "local" => true,
-            "nickname" => old_admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "tags" => [],
-            "avatar" => User.avatar_url(old_admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(old_admin.name || old_admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => old_admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          }
+            "roles" => %{"admin" => true, "moderator" => false}
+          })
         ]
         |> Enum.sort_by(& &1["nickname"])
 
@@ -1047,6 +835,30 @@ test "only local users with no query", %{conn: conn, admin: old_admin} do
              }
     end
 
+    test "only unconfirmed users", %{conn: conn} do
+      sad_user = insert(:user, nickname: "sadboy", confirmation_pending: true)
+      old_user = insert(:user, nickname: "oldboy", confirmation_pending: true)
+
+      insert(:user, nickname: "happyboy", approval_pending: false)
+      insert(:user, confirmation_pending: false)
+
+      result =
+        conn
+        |> get("/api/pleroma/admin/users?filters=need_confirmed")
+        |> json_response(200)
+
+      users =
+        Enum.map([old_user, sad_user], fn user ->
+          user_response(user, %{
+            "confirmation_pending" => true,
+            "approval_pending" => false
+          })
+        end)
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert result == %{"count" => 2, "page_size" => 50, "users" => users}
+    end
+
     test "only unapproved users", %{conn: conn} do
       user =
         insert(:user,
@@ -1175,21 +987,22 @@ test "load users with tags list", %{conn: conn} do
 
       users =
         [
-          %{
-            "deactivated" => false,
-            "id" => user1.id,
-            "nickname" => user1.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => user1.local,
-            "tags" => ["first"],
-            "avatar" => User.avatar_url(user1) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user1.name || user1.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => user1.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
+          user_response(
+            user1,
+            %{
+              "deactivated" => false,
+              "roles" => %{"admin" => false, "moderator" => false},
+              "local" => user1.local,
+              "tags" => ["first"],
+              "avatar" => User.avatar_url(user1) |> MediaProxy.url(),
+              "display_name" => HTML.strip_tags(user1.name || user1.nickname),
+              "confirmation_pending" => false,
+              "approval_pending" => false,
+              "url" => user1.ap_id,
+              "registration_reason" => nil,
+              "actor_type" => "Person"
+            }
+          ),
           %{
             "deactivated" => false,
             "id" => user2.id,
@@ -1253,23 +1066,7 @@ test "it works with multiple filters" do
       assert json_response(conn, 200) == %{
                "count" => 1,
                "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => user.deactivated,
-                   "id" => user.id,
-                   "nickname" => user.nickname,
-                   "roles" => %{"admin" => false, "moderator" => false},
-                   "local" => user.local,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(user.name || user.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => user.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
+               "users" => [user_response(user)]
              }
     end
 
@@ -1282,21 +1079,7 @@ test "it omits relay user", %{admin: admin, conn: conn} do
                "count" => 1,
                "page_size" => 50,
                "users" => [
-                 %{
-                   "deactivated" => admin.deactivated,
-                   "id" => admin.id,
-                   "nickname" => admin.nickname,
-                   "roles" => %{"admin" => true, "moderator" => false},
-                   "local" => true,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => admin.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
+                 user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}})
                ]
              }
     end
@@ -1368,21 +1151,10 @@ test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation", %{admin: admi
     conn = patch(conn, "/api/pleroma/admin/users/#{user.nickname}/toggle_activation")
 
     assert json_response(conn, 200) ==
-             %{
-               "deactivated" => !user.deactivated,
-               "id" => user.id,
-               "nickname" => user.nickname,
-               "roles" => %{"admin" => false, "moderator" => false},
-               "local" => true,
-               "tags" => [],
-               "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-               "display_name" => HTML.strip_tags(user.name || user.nickname),
-               "confirmation_pending" => false,
-               "approval_pending" => false,
-               "url" => user.ap_id,
-               "registration_reason" => nil,
-               "actor_type" => "Person"
-             }
+             user_response(
+               user,
+               %{"deactivated" => !user.deactivated}
+             )
 
     log_entry = Repo.one(ModerationLog)
 
@@ -2024,6 +1796,25 @@ test "by instance", %{conn: conn} do
                response["status_visibility"]
     end
   end
+
+  defp user_response(user, attrs \\ %{}) do
+    %{
+      "deactivated" => user.deactivated,
+      "id" => user.id,
+      "nickname" => user.nickname,
+      "roles" => %{"admin" => false, "moderator" => false},
+      "local" => user.local,
+      "tags" => [],
+      "avatar" => User.avatar_url(user) |> MediaProxy.url(),
+      "display_name" => HTML.strip_tags(user.name || user.nickname),
+      "confirmation_pending" => false,
+      "approval_pending" => false,
+      "url" => user.ap_id,
+      "registration_reason" => nil,
+      "actor_type" => "Person"
+    }
+    |> Map.merge(attrs)
+  end
 end
 
 # Needed for testing
diff --git a/test/pleroma/web/admin_api/search_test.exs b/test/pleroma/web/admin_api/search_test.exs
index ceec64f1e..27ca396e6 100644
--- a/test/pleroma/web/admin_api/search_test.exs
+++ b/test/pleroma/web/admin_api/search_test.exs
@@ -178,6 +178,17 @@ test "it returns unapproved user" do
       assert count == 1
     end
 
+    test "it returns unconfirmed user" do
+      unconfirmed = insert(:user, confirmation_pending: true)
+      insert(:user)
+      insert(:user)
+
+      {:ok, _results, total} = Search.user()
+      {:ok, [^unconfirmed], count} = Search.user(%{need_confirmed: true})
+      assert total == 3
+      assert count == 1
+    end
+
     test "it returns non-discoverable users" do
       insert(:user)
       insert(:user, is_discoverable: false)

From cf4f3937946eb0232be8bf8ddba08d365b4e17ee Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Mon, 21 Sep 2020 15:01:03 +0300
Subject: [PATCH 063/104] added AdminApi.UserController

---
 .../controllers/admin_api_controller.ex       | 242 +----
 .../admin_api/controllers/user_controller.ex  | 280 +++++
 lib/pleroma/web/router.ex                     |  22 +-
 .../controllers/admin_api_controller_test.exs | 837 ---------------
 .../controllers/user_controller_test.exs      | 983 ++++++++++++++++++
 5 files changed, 1275 insertions(+), 1089 deletions(-)
 create mode 100644 lib/pleroma/web/admin_api/controllers/user_controller.ex
 create mode 100644 test/web/admin_api/controllers/user_controller_test.exs

diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index acfbeb0c8..df5817cfa 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -14,12 +14,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.Stats
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Builder
-  alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.AdminAPI.AccountView
   alias Pleroma.Web.AdminAPI.ModerationLogView
-  alias Pleroma.Web.AdminAPI.Search
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.Plugs.OAuthScopesPlug
   alias Pleroma.Web.Router
@@ -29,7 +26,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   plug(
     OAuthScopesPlug,
     %{scopes: ["read:accounts"], admin: true}
-    when action in [:list_users, :user_show, :right_get, :show_user_credentials]
+    when action in [:right_get, :show_user_credentials]
   )
 
   plug(
@@ -38,12 +35,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     when action in [
            :get_password_reset,
            :force_password_reset,
-           :user_delete,
-           :users_create,
-           :user_toggle_activation,
-           :user_activate,
-           :user_deactivate,
-           :user_approve,
            :tag_users,
            :untag_users,
            :right_add,
@@ -55,12 +46,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
          ]
   )
 
-  plug(
-    OAuthScopesPlug,
-    %{scopes: ["write:follows"], admin: true}
-    when action in [:user_follow, :user_unfollow]
-  )
-
   plug(
     OAuthScopesPlug,
     %{scopes: ["read:statuses"], admin: true}
@@ -96,129 +81,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
   action_fallback(AdminAPI.FallbackController)
 
-  def user_delete(conn, %{"nickname" => nickname}) do
-    user_delete(conn, %{"nicknames" => [nickname]})
-  end
-
-  def user_delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
-
-    Enum.each(users, fn user ->
-      {:ok, delete_data, _} = Builder.delete(admin, user.ap_id)
-      Pipeline.common_pipeline(delete_data, local: true)
-    end)
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "delete"
-    })
-
-    json(conn, nicknames)
-  end
-
-  def user_follow(%{assigns: %{user: admin}} = conn, %{
-        "follower" => follower_nick,
-        "followed" => followed_nick
-      }) do
-    with %User{} = follower <- User.get_cached_by_nickname(follower_nick),
-         %User{} = followed <- User.get_cached_by_nickname(followed_nick) do
-      User.follow(follower, followed)
-
-      ModerationLog.insert_log(%{
-        actor: admin,
-        followed: followed,
-        follower: follower,
-        action: "follow"
-      })
-    end
-
-    json(conn, "ok")
-  end
-
-  def user_unfollow(%{assigns: %{user: admin}} = conn, %{
-        "follower" => follower_nick,
-        "followed" => followed_nick
-      }) do
-    with %User{} = follower <- User.get_cached_by_nickname(follower_nick),
-         %User{} = followed <- User.get_cached_by_nickname(followed_nick) do
-      User.unfollow(follower, followed)
-
-      ModerationLog.insert_log(%{
-        actor: admin,
-        followed: followed,
-        follower: follower,
-        action: "unfollow"
-      })
-    end
-
-    json(conn, "ok")
-  end
-
-  def users_create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do
-    changesets =
-      Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} ->
-        user_data = %{
-          nickname: nickname,
-          name: nickname,
-          email: email,
-          password: password,
-          password_confirmation: password,
-          bio: "."
-        }
-
-        User.register_changeset(%User{}, user_data, need_confirmation: false)
-      end)
-      |> Enum.reduce(Ecto.Multi.new(), fn changeset, multi ->
-        Ecto.Multi.insert(multi, Ecto.UUID.generate(), changeset)
-      end)
-
-    case Pleroma.Repo.transaction(changesets) do
-      {:ok, users} ->
-        res =
-          users
-          |> Map.values()
-          |> Enum.map(fn user ->
-            {:ok, user} = User.post_register_action(user)
-
-            user
-          end)
-          |> Enum.map(&AccountView.render("created.json", %{user: &1}))
-
-        ModerationLog.insert_log(%{
-          actor: admin,
-          subjects: Map.values(users),
-          action: "create"
-        })
-
-        json(conn, res)
-
-      {:error, id, changeset, _} ->
-        res =
-          Enum.map(changesets.operations, fn
-            {current_id, {:changeset, _current_changeset, _}} when current_id == id ->
-              AccountView.render("create-error.json", %{changeset: changeset})
-
-            {_, {:changeset, current_changeset, _}} ->
-              AccountView.render("create-error.json", %{changeset: current_changeset})
-          end)
-
-        conn
-        |> put_status(:conflict)
-        |> json(res)
-    end
-  end
-
-  def user_show(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
-    with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do
-      conn
-      |> put_view(AccountView)
-      |> render("show.json", %{user: user})
-    else
-      _ -> {:error, :not_found}
-    end
-  end
-
   def list_instance_statuses(conn, %{"instance" => instance} = params) do
     with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true
     {page, page_size} = page_params(params)
@@ -272,69 +134,6 @@ def list_user_chats(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}
     end
   end
 
-  def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
-    user = User.get_cached_by_nickname(nickname)
-
-    {:ok, updated_user} = User.deactivate(user, !user.deactivated)
-
-    action = if user.deactivated, do: "activate", else: "deactivate"
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: [user],
-      action: action
-    })
-
-    conn
-    |> put_view(AccountView)
-    |> render("show.json", %{user: updated_user})
-  end
-
-  def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
-    {:ok, updated_users} = User.deactivate(users, false)
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "activate"
-    })
-
-    conn
-    |> put_view(AccountView)
-    |> render("index.json", %{users: Keyword.values(updated_users)})
-  end
-
-  def user_deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
-    {:ok, updated_users} = User.deactivate(users, true)
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "deactivate"
-    })
-
-    conn
-    |> put_view(AccountView)
-    |> render("index.json", %{users: Keyword.values(updated_users)})
-  end
-
-  def user_approve(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
-    {:ok, updated_users} = User.approve(users)
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "approve"
-    })
-
-    conn
-    |> put_view(AccountView)
-    |> render("index.json", %{users: updated_users})
-  end
-
   def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do
     with {:ok, _} <- User.tag(nicknames, tags) do
       ModerationLog.insert_log(%{
@@ -361,45 +160,6 @@ def untag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "
     end
   end
 
-  def list_users(conn, params) do
-    {page, page_size} = page_params(params)
-    filters = maybe_parse_filters(params["filters"])
-
-    search_params =
-      %{
-        query: params["query"],
-        page: page,
-        page_size: page_size,
-        tags: params["tags"],
-        name: params["name"],
-        email: params["email"]
-      }
-      |> Map.merge(filters)
-
-    with {:ok, users, count} <- Search.user(search_params) do
-      json(
-        conn,
-        AccountView.render("index.json",
-          users: users,
-          count: count,
-          page_size: page_size
-        )
-      )
-    end
-  end
-
-  @filters ~w(local external active deactivated need_approval need_confirmed is_admin is_moderator)
-
-  @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
-  defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
-
-  defp maybe_parse_filters(filters) do
-    filters
-    |> String.split(",")
-    |> Enum.filter(&Enum.member?(@filters, &1))
-    |> Map.new(&{String.to_existing_atom(&1), true})
-  end
-
   def right_add_multiple(%{assigns: %{user: admin}} = conn, %{
         "permission_group" => permission_group,
         "nicknames" => nicknames
diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex
new file mode 100644
index 000000000..5e049aa0f
--- /dev/null
+++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex
@@ -0,0 +1,280 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.UserController do
+  use Pleroma.Web, :controller
+
+  import Pleroma.Web.ControllerHelper,
+    only: [fetch_integer_param: 3]
+
+  alias Pleroma.ModerationLog
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.Builder
+  alias Pleroma.Web.ActivityPub.Pipeline
+  alias Pleroma.Web.AdminAPI
+  alias Pleroma.Web.AdminAPI.AccountView
+  alias Pleroma.Web.AdminAPI.Search
+
+  @users_page_size 50
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:accounts"], admin: true}
+    when action in [:list, :show]
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:accounts"], admin: true}
+    when action in [
+           :delete,
+           :create,
+           :toggle_activation,
+           :activate,
+           :deactivate,
+           :approve
+         ]
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:follows"], admin: true}
+    when action in [:follow, :unfollow]
+  )
+
+  action_fallback(AdminAPI.FallbackController)
+
+  def delete(conn, %{"nickname" => nickname}) do
+    delete(conn, %{"nicknames" => [nickname]})
+  end
+
+  def delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+
+    Enum.each(users, fn user ->
+      {:ok, delete_data, _} = Builder.delete(admin, user.ap_id)
+      Pipeline.common_pipeline(delete_data, local: true)
+    end)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "delete"
+    })
+
+    json(conn, nicknames)
+  end
+
+  def follow(%{assigns: %{user: admin}} = conn, %{
+        "follower" => follower_nick,
+        "followed" => followed_nick
+      }) do
+    with %User{} = follower <- User.get_cached_by_nickname(follower_nick),
+         %User{} = followed <- User.get_cached_by_nickname(followed_nick) do
+      User.follow(follower, followed)
+
+      ModerationLog.insert_log(%{
+        actor: admin,
+        followed: followed,
+        follower: follower,
+        action: "follow"
+      })
+    end
+
+    json(conn, "ok")
+  end
+
+  def unfollow(%{assigns: %{user: admin}} = conn, %{
+        "follower" => follower_nick,
+        "followed" => followed_nick
+      }) do
+    with %User{} = follower <- User.get_cached_by_nickname(follower_nick),
+         %User{} = followed <- User.get_cached_by_nickname(followed_nick) do
+      User.unfollow(follower, followed)
+
+      ModerationLog.insert_log(%{
+        actor: admin,
+        followed: followed,
+        follower: follower,
+        action: "unfollow"
+      })
+    end
+
+    json(conn, "ok")
+  end
+
+  def create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do
+    changesets =
+      Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} ->
+        user_data = %{
+          nickname: nickname,
+          name: nickname,
+          email: email,
+          password: password,
+          password_confirmation: password,
+          bio: "."
+        }
+
+        User.register_changeset(%User{}, user_data, need_confirmation: false)
+      end)
+      |> Enum.reduce(Ecto.Multi.new(), fn changeset, multi ->
+        Ecto.Multi.insert(multi, Ecto.UUID.generate(), changeset)
+      end)
+
+    case Pleroma.Repo.transaction(changesets) do
+      {:ok, users} ->
+        res =
+          users
+          |> Map.values()
+          |> Enum.map(fn user ->
+            {:ok, user} = User.post_register_action(user)
+
+            user
+          end)
+          |> Enum.map(&AccountView.render("created.json", %{user: &1}))
+
+        ModerationLog.insert_log(%{
+          actor: admin,
+          subjects: Map.values(users),
+          action: "create"
+        })
+
+        json(conn, res)
+
+      {:error, id, changeset, _} ->
+        res =
+          Enum.map(changesets.operations, fn
+            {current_id, {:changeset, _current_changeset, _}} when current_id == id ->
+              AccountView.render("create-error.json", %{changeset: changeset})
+
+            {_, {:changeset, current_changeset, _}} ->
+              AccountView.render("create-error.json", %{changeset: current_changeset})
+          end)
+
+        conn
+        |> put_status(:conflict)
+        |> json(res)
+    end
+  end
+
+  def show(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do
+      conn
+      |> put_view(AccountView)
+      |> render("show.json", %{user: user})
+    else
+      _ -> {:error, :not_found}
+    end
+  end
+
+  def toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    user = User.get_cached_by_nickname(nickname)
+
+    {:ok, updated_user} = User.deactivate(user, !user.deactivated)
+
+    action = if user.deactivated, do: "activate", else: "deactivate"
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: [user],
+      action: action
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("show.json", %{user: updated_user})
+  end
+
+  def activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.deactivate(users, false)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "activate"
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: Keyword.values(updated_users)})
+  end
+
+  def deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.deactivate(users, true)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "deactivate"
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: Keyword.values(updated_users)})
+  end
+
+  def approve(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.approve(users)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "approve"
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: updated_users})
+  end
+
+  def list(conn, params) do
+    {page, page_size} = page_params(params)
+    filters = maybe_parse_filters(params["filters"])
+
+    search_params =
+      %{
+        query: params["query"],
+        page: page,
+        page_size: page_size,
+        tags: params["tags"],
+        name: params["name"],
+        email: params["email"]
+      }
+      |> Map.merge(filters)
+
+    with {:ok, users, count} <- Search.user(search_params) do
+      json(
+        conn,
+        AccountView.render("index.json",
+          users: users,
+          count: count,
+          page_size: page_size
+        )
+      )
+    end
+  end
+
+  @filters ~w(local external active deactivated need_approval need_confirmed is_admin is_moderator)
+
+  @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
+  defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
+
+  defp maybe_parse_filters(filters) do
+    filters
+    |> String.split(",")
+    |> Enum.filter(&Enum.member?(@filters, &1))
+    |> Map.new(&{String.to_existing_atom(&1), true})
+  end
+
+  defp page_params(params) do
+    {
+      fetch_integer_param(params, "page", 1),
+      fetch_integer_param(params, "page_size", @users_page_size)
+    }
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index d2d939989..3a9605778 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -129,16 +129,7 @@ defmodule Pleroma.Web.Router do
   scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
     pipe_through(:admin_api)
 
-    post("/users/follow", AdminAPIController, :user_follow)
-    post("/users/unfollow", AdminAPIController, :user_unfollow)
-
     put("/users/disable_mfa", AdminAPIController, :disable_mfa)
-    delete("/users", AdminAPIController, :user_delete)
-    post("/users", AdminAPIController, :users_create)
-    patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
-    patch("/users/activate", AdminAPIController, :user_activate)
-    patch("/users/deactivate", AdminAPIController, :user_deactivate)
-    patch("/users/approve", AdminAPIController, :user_approve)
     put("/users/tag", AdminAPIController, :tag_users)
     delete("/users/tag", AdminAPIController, :untag_users)
 
@@ -161,6 +152,15 @@ defmodule Pleroma.Web.Router do
       :right_delete_multiple
     )
 
+    post("/users/follow", UserController, :follow)
+    post("/users/unfollow", UserController, :unfollow)
+    delete("/users", UserController, :delete)
+    post("/users", UserController, :create)
+    patch("/users/:nickname/toggle_activation", UserController, :toggle_activation)
+    patch("/users/activate", UserController, :activate)
+    patch("/users/deactivate", UserController, :deactivate)
+    patch("/users/approve", UserController, :approve)
+
     get("/relay", RelayController, :index)
     post("/relay", RelayController, :follow)
     delete("/relay", RelayController, :unfollow)
@@ -175,8 +175,8 @@ defmodule Pleroma.Web.Router do
     get("/users/:nickname/credentials", AdminAPIController, :show_user_credentials)
     patch("/users/:nickname/credentials", AdminAPIController, :update_user_credentials)
 
-    get("/users", AdminAPIController, :list_users)
-    get("/users/:nickname", AdminAPIController, :user_show)
+    get("/users", UserController, :list)
+    get("/users/:nickname", UserController, :show)
     get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses)
     get("/users/:nickname/chats", AdminAPIController, :list_user_chats)
 
diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
index 686b53a80..34b26dddf 100644
--- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
@@ -7,22 +7,17 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
   use Oban.Testing, repo: Pleroma.Repo
 
   import ExUnit.CaptureLog
-  import Mock
   import Pleroma.Factory
   import Swoosh.TestAssertions
 
   alias Pleroma.Activity
   alias Pleroma.Config
-  alias Pleroma.HTML
   alias Pleroma.MFA
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
   alias Pleroma.Tests.ObanHelpers
   alias Pleroma.User
-  alias Pleroma.Web
-  alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.MediaProxy
 
   setup_all do
     Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
@@ -153,284 +148,6 @@ test "GET /api/pleroma/admin/users/:nickname requires " <>
     end
   end
 
-  describe "DELETE /api/pleroma/admin/users" do
-    test "single user", %{admin: admin, conn: conn} do
-      clear_config([:instance, :federating], true)
-
-      user =
-        insert(:user,
-          avatar: %{"url" => [%{"href" => "https://someurl"}]},
-          banner: %{"url" => [%{"href" => "https://somebanner"}]},
-          bio: "Hello world!",
-          name: "A guy"
-        )
-
-      # Create some activities to check they got deleted later
-      follower = insert(:user)
-      {:ok, _} = CommonAPI.post(user, %{status: "test"})
-      {:ok, _, _, _} = CommonAPI.follow(user, follower)
-      {:ok, _, _, _} = CommonAPI.follow(follower, user)
-      user = Repo.get(User, user.id)
-      assert user.note_count == 1
-      assert user.follower_count == 1
-      assert user.following_count == 1
-      refute user.deactivated
-
-      with_mock Pleroma.Web.Federator,
-        publish: fn _ -> nil end,
-        perform: fn _, _ -> nil end do
-        conn =
-          conn
-          |> put_req_header("accept", "application/json")
-          |> delete("/api/pleroma/admin/users?nickname=#{user.nickname}")
-
-        ObanHelpers.perform_all()
-
-        assert User.get_by_nickname(user.nickname).deactivated
-
-        log_entry = Repo.one(ModerationLog)
-
-        assert ModerationLog.get_log_entry_message(log_entry) ==
-                 "@#{admin.nickname} deleted users: @#{user.nickname}"
-
-        assert json_response(conn, 200) == [user.nickname]
-
-        user = Repo.get(User, user.id)
-        assert user.deactivated
-
-        assert user.avatar == %{}
-        assert user.banner == %{}
-        assert user.note_count == 0
-        assert user.follower_count == 0
-        assert user.following_count == 0
-        assert user.bio == ""
-        assert user.name == nil
-
-        assert called(Pleroma.Web.Federator.publish(:_))
-      end
-    end
-
-    test "multiple users", %{admin: admin, conn: conn} do
-      user_one = insert(:user)
-      user_two = insert(:user)
-
-      conn =
-        conn
-        |> put_req_header("accept", "application/json")
-        |> delete("/api/pleroma/admin/users", %{
-          nicknames: [user_one.nickname, user_two.nickname]
-        })
-
-      log_entry = Repo.one(ModerationLog)
-
-      assert ModerationLog.get_log_entry_message(log_entry) ==
-               "@#{admin.nickname} deleted users: @#{user_one.nickname}, @#{user_two.nickname}"
-
-      response = json_response(conn, 200)
-      assert response -- [user_one.nickname, user_two.nickname] == []
-    end
-  end
-
-  describe "/api/pleroma/admin/users" do
-    test "Create", %{conn: conn} do
-      conn =
-        conn
-        |> put_req_header("accept", "application/json")
-        |> post("/api/pleroma/admin/users", %{
-          "users" => [
-            %{
-              "nickname" => "lain",
-              "email" => "lain@example.org",
-              "password" => "test"
-            },
-            %{
-              "nickname" => "lain2",
-              "email" => "lain2@example.org",
-              "password" => "test"
-            }
-          ]
-        })
-
-      response = json_response(conn, 200) |> Enum.map(&Map.get(&1, "type"))
-      assert response == ["success", "success"]
-
-      log_entry = Repo.one(ModerationLog)
-
-      assert ["lain", "lain2"] -- Enum.map(log_entry.data["subjects"], & &1["nickname"]) == []
-    end
-
-    test "Cannot create user with existing email", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> put_req_header("accept", "application/json")
-        |> post("/api/pleroma/admin/users", %{
-          "users" => [
-            %{
-              "nickname" => "lain",
-              "email" => user.email,
-              "password" => "test"
-            }
-          ]
-        })
-
-      assert json_response(conn, 409) == [
-               %{
-                 "code" => 409,
-                 "data" => %{
-                   "email" => user.email,
-                   "nickname" => "lain"
-                 },
-                 "error" => "email has already been taken",
-                 "type" => "error"
-               }
-             ]
-    end
-
-    test "Cannot create user with existing nickname", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> put_req_header("accept", "application/json")
-        |> post("/api/pleroma/admin/users", %{
-          "users" => [
-            %{
-              "nickname" => user.nickname,
-              "email" => "someuser@plerama.social",
-              "password" => "test"
-            }
-          ]
-        })
-
-      assert json_response(conn, 409) == [
-               %{
-                 "code" => 409,
-                 "data" => %{
-                   "email" => "someuser@plerama.social",
-                   "nickname" => user.nickname
-                 },
-                 "error" => "nickname has already been taken",
-                 "type" => "error"
-               }
-             ]
-    end
-
-    test "Multiple user creation works in transaction", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> put_req_header("accept", "application/json")
-        |> post("/api/pleroma/admin/users", %{
-          "users" => [
-            %{
-              "nickname" => "newuser",
-              "email" => "newuser@pleroma.social",
-              "password" => "test"
-            },
-            %{
-              "nickname" => "lain",
-              "email" => user.email,
-              "password" => "test"
-            }
-          ]
-        })
-
-      assert json_response(conn, 409) == [
-               %{
-                 "code" => 409,
-                 "data" => %{
-                   "email" => user.email,
-                   "nickname" => "lain"
-                 },
-                 "error" => "email has already been taken",
-                 "type" => "error"
-               },
-               %{
-                 "code" => 409,
-                 "data" => %{
-                   "email" => "newuser@pleroma.social",
-                   "nickname" => "newuser"
-                 },
-                 "error" => "",
-                 "type" => "error"
-               }
-             ]
-
-      assert User.get_by_nickname("newuser") === nil
-    end
-  end
-
-  describe "/api/pleroma/admin/users/:nickname" do
-    test "Show", %{conn: conn} do
-      user = insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}")
-
-      assert user_response(user) == json_response(conn, 200)
-    end
-
-    test "when the user doesn't exist", %{conn: conn} do
-      user = build(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}")
-
-      assert %{"error" => "Not found"} == json_response(conn, 404)
-    end
-  end
-
-  describe "/api/pleroma/admin/users/follow" do
-    test "allows to force-follow another user", %{admin: admin, conn: conn} do
-      user = insert(:user)
-      follower = insert(:user)
-
-      conn
-      |> put_req_header("accept", "application/json")
-      |> post("/api/pleroma/admin/users/follow", %{
-        "follower" => follower.nickname,
-        "followed" => user.nickname
-      })
-
-      user = User.get_cached_by_id(user.id)
-      follower = User.get_cached_by_id(follower.id)
-
-      assert User.following?(follower, user)
-
-      log_entry = Repo.one(ModerationLog)
-
-      assert ModerationLog.get_log_entry_message(log_entry) ==
-               "@#{admin.nickname} made @#{follower.nickname} follow @#{user.nickname}"
-    end
-  end
-
-  describe "/api/pleroma/admin/users/unfollow" do
-    test "allows to force-unfollow another user", %{admin: admin, conn: conn} do
-      user = insert(:user)
-      follower = insert(:user)
-
-      User.follow(follower, user)
-
-      conn
-      |> put_req_header("accept", "application/json")
-      |> post("/api/pleroma/admin/users/unfollow", %{
-        "follower" => follower.nickname,
-        "followed" => user.nickname
-      })
-
-      user = User.get_cached_by_id(user.id)
-      follower = User.get_cached_by_id(follower.id)
-
-      refute User.following?(follower, user)
-
-      log_entry = Repo.one(ModerationLog)
-
-      assert ModerationLog.get_log_entry_message(log_entry) ==
-               "@#{admin.nickname} made @#{follower.nickname} unfollow @#{user.nickname}"
-    end
-  end
-
   describe "PUT /api/pleroma/admin/users/tag" do
     setup %{conn: conn} do
       user1 = insert(:user, %{tags: ["x"]})
@@ -627,541 +344,6 @@ test "/api/pleroma/admin/users/:nickname/password_reset", %{conn: conn} do
     assert Regex.match?(~r/(http:\/\/|https:\/\/)/, resp["link"])
   end
 
-  describe "GET /api/pleroma/admin/users" do
-    test "renders users array for the first page", %{conn: conn, admin: admin} do
-      user = insert(:user, local: false, tags: ["foo", "bar"])
-      user2 = insert(:user, approval_pending: true, registration_reason: "I'm a chill dude")
-
-      conn = get(conn, "/api/pleroma/admin/users?page=1")
-
-      users =
-        [
-          user_response(
-            admin,
-            %{"roles" => %{"admin" => true, "moderator" => false}}
-          ),
-          user_response(user, %{"local" => false, "tags" => ["foo", "bar"]}),
-          user_response(
-            user2,
-            %{
-              "local" => true,
-              "approval_pending" => true,
-              "registration_reason" => "I'm a chill dude",
-              "actor_type" => "Person"
-            }
-          )
-        ]
-        |> Enum.sort_by(& &1["nickname"])
-
-      assert json_response(conn, 200) == %{
-               "count" => 3,
-               "page_size" => 50,
-               "users" => users
-             }
-    end
-
-    test "pagination works correctly with service users", %{conn: conn} do
-      service1 = User.get_or_create_service_actor_by_ap_id(Web.base_url() <> "/meido", "meido")
-
-      insert_list(25, :user)
-
-      assert %{"count" => 26, "page_size" => 10, "users" => users1} =
-               conn
-               |> get("/api/pleroma/admin/users?page=1&filters=", %{page_size: "10"})
-               |> json_response(200)
-
-      assert Enum.count(users1) == 10
-      assert service1 not in users1
-
-      assert %{"count" => 26, "page_size" => 10, "users" => users2} =
-               conn
-               |> get("/api/pleroma/admin/users?page=2&filters=", %{page_size: "10"})
-               |> json_response(200)
-
-      assert Enum.count(users2) == 10
-      assert service1 not in users2
-
-      assert %{"count" => 26, "page_size" => 10, "users" => users3} =
-               conn
-               |> get("/api/pleroma/admin/users?page=3&filters=", %{page_size: "10"})
-               |> json_response(200)
-
-      assert Enum.count(users3) == 6
-      assert service1 not in users3
-    end
-
-    test "renders empty array for the second page", %{conn: conn} do
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?page=2")
-
-      assert json_response(conn, 200) == %{
-               "count" => 2,
-               "page_size" => 50,
-               "users" => []
-             }
-    end
-
-    test "regular search", %{conn: conn} do
-      user = insert(:user, nickname: "bob")
-
-      conn = get(conn, "/api/pleroma/admin/users?query=bo")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user, %{"local" => true})]
-             }
-    end
-
-    test "search by domain", %{conn: conn} do
-      user = insert(:user, nickname: "nickname@domain.com")
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?query=domain.com")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user)]
-             }
-    end
-
-    test "search by full nickname", %{conn: conn} do
-      user = insert(:user, nickname: "nickname@domain.com")
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?query=nickname@domain.com")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user)]
-             }
-    end
-
-    test "search by display name", %{conn: conn} do
-      user = insert(:user, name: "Display name")
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?name=display")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user)]
-             }
-    end
-
-    test "search by email", %{conn: conn} do
-      user = insert(:user, email: "email@example.com")
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?email=email@example.com")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user)]
-             }
-    end
-
-    test "regular search with page size", %{conn: conn} do
-      user = insert(:user, nickname: "aalice")
-      user2 = insert(:user, nickname: "alice")
-
-      conn1 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=1")
-
-      assert json_response(conn1, 200) == %{
-               "count" => 2,
-               "page_size" => 1,
-               "users" => [user_response(user)]
-             }
-
-      conn2 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=2")
-
-      assert json_response(conn2, 200) == %{
-               "count" => 2,
-               "page_size" => 1,
-               "users" => [user_response(user2)]
-             }
-    end
-
-    test "only local users" do
-      admin = insert(:user, is_admin: true, nickname: "john")
-      token = insert(:oauth_admin_token, user: admin)
-      user = insert(:user, nickname: "bob")
-
-      insert(:user, nickname: "bobb", local: false)
-
-      conn =
-        build_conn()
-        |> assign(:user, admin)
-        |> assign(:token, token)
-        |> get("/api/pleroma/admin/users?query=bo&filters=local")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user)]
-             }
-    end
-
-    test "only local users with no query", %{conn: conn, admin: old_admin} do
-      admin = insert(:user, is_admin: true, nickname: "john")
-      user = insert(:user, nickname: "bob")
-
-      insert(:user, nickname: "bobb", local: false)
-
-      conn = get(conn, "/api/pleroma/admin/users?filters=local")
-
-      users =
-        [
-          user_response(user),
-          user_response(admin, %{
-            "roles" => %{"admin" => true, "moderator" => false}
-          }),
-          user_response(old_admin, %{
-            "deactivated" => false,
-            "roles" => %{"admin" => true, "moderator" => false}
-          })
-        ]
-        |> Enum.sort_by(& &1["nickname"])
-
-      assert json_response(conn, 200) == %{
-               "count" => 3,
-               "page_size" => 50,
-               "users" => users
-             }
-    end
-
-    test "only unconfirmed users", %{conn: conn} do
-      sad_user = insert(:user, nickname: "sadboy", confirmation_pending: true)
-      old_user = insert(:user, nickname: "oldboy", confirmation_pending: true)
-
-      insert(:user, nickname: "happyboy", approval_pending: false)
-      insert(:user, confirmation_pending: false)
-
-      result =
-        conn
-        |> get("/api/pleroma/admin/users?filters=need_confirmed")
-        |> json_response(200)
-
-      users =
-        Enum.map([old_user, sad_user], fn user ->
-          user_response(user, %{
-            "confirmation_pending" => true,
-            "approval_pending" => false
-          })
-        end)
-        |> Enum.sort_by(& &1["nickname"])
-
-      assert result == %{"count" => 2, "page_size" => 50, "users" => users}
-    end
-
-    test "only unapproved users", %{conn: conn} do
-      user =
-        insert(:user,
-          nickname: "sadboy",
-          approval_pending: true,
-          registration_reason: "Plz let me in!"
-        )
-
-      insert(:user, nickname: "happyboy", approval_pending: false)
-
-      conn = get(conn, "/api/pleroma/admin/users?filters=need_approval")
-
-      users =
-        [
-          %{
-            "deactivated" => user.deactivated,
-            "id" => user.id,
-            "nickname" => user.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => true,
-            "tags" => [],
-            "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user.name || user.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => true,
-            "url" => user.ap_id,
-            "registration_reason" => "Plz let me in!",
-            "actor_type" => "Person"
-          }
-        ]
-        |> Enum.sort_by(& &1["nickname"])
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => users
-             }
-    end
-
-    test "load only admins", %{conn: conn, admin: admin} do
-      second_admin = insert(:user, is_admin: true)
-      insert(:user)
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?filters=is_admin")
-
-      users =
-        [
-          %{
-            "deactivated" => false,
-            "id" => admin.id,
-            "nickname" => admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "local" => admin.local,
-            "tags" => [],
-            "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
-          %{
-            "deactivated" => false,
-            "id" => second_admin.id,
-            "nickname" => second_admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "local" => second_admin.local,
-            "tags" => [],
-            "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => second_admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          }
-        ]
-        |> Enum.sort_by(& &1["nickname"])
-
-      assert json_response(conn, 200) == %{
-               "count" => 2,
-               "page_size" => 50,
-               "users" => users
-             }
-    end
-
-    test "load only moderators", %{conn: conn} do
-      moderator = insert(:user, is_moderator: true)
-      insert(:user)
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?filters=is_moderator")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [
-                 %{
-                   "deactivated" => false,
-                   "id" => moderator.id,
-                   "nickname" => moderator.nickname,
-                   "roles" => %{"admin" => false, "moderator" => true},
-                   "local" => moderator.local,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(moderator) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(moderator.name || moderator.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => moderator.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
-               ]
-             }
-    end
-
-    test "load users with tags list", %{conn: conn} do
-      user1 = insert(:user, tags: ["first"])
-      user2 = insert(:user, tags: ["second"])
-      insert(:user)
-      insert(:user)
-
-      conn = get(conn, "/api/pleroma/admin/users?tags[]=first&tags[]=second")
-
-      users =
-        [
-          user_response(
-            user1,
-            %{
-              "deactivated" => false,
-              "roles" => %{"admin" => false, "moderator" => false},
-              "local" => user1.local,
-              "tags" => ["first"],
-              "avatar" => User.avatar_url(user1) |> MediaProxy.url(),
-              "display_name" => HTML.strip_tags(user1.name || user1.nickname),
-              "confirmation_pending" => false,
-              "approval_pending" => false,
-              "url" => user1.ap_id,
-              "registration_reason" => nil,
-              "actor_type" => "Person"
-            }
-          ),
-          %{
-            "deactivated" => false,
-            "id" => user2.id,
-            "nickname" => user2.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => user2.local,
-            "tags" => ["second"],
-            "avatar" => User.avatar_url(user2) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user2.name || user2.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => user2.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          }
-        ]
-        |> Enum.sort_by(& &1["nickname"])
-
-      assert json_response(conn, 200) == %{
-               "count" => 2,
-               "page_size" => 50,
-               "users" => users
-             }
-    end
-
-    test "`active` filters out users pending approval", %{token: token} do
-      insert(:user, approval_pending: true)
-      %{id: user_id} = insert(:user, approval_pending: false)
-      %{id: admin_id} = token.user
-
-      conn =
-        build_conn()
-        |> assign(:user, token.user)
-        |> assign(:token, token)
-        |> get("/api/pleroma/admin/users?filters=active")
-
-      assert %{
-               "count" => 2,
-               "page_size" => 50,
-               "users" => [
-                 %{"id" => ^admin_id},
-                 %{"id" => ^user_id}
-               ]
-             } = json_response(conn, 200)
-    end
-
-    test "it works with multiple filters" do
-      admin = insert(:user, nickname: "john", is_admin: true)
-      token = insert(:oauth_admin_token, user: admin)
-      user = insert(:user, nickname: "bob", local: false, deactivated: true)
-
-      insert(:user, nickname: "ken", local: true, deactivated: true)
-      insert(:user, nickname: "bobb", local: false, deactivated: false)
-
-      conn =
-        build_conn()
-        |> assign(:user, admin)
-        |> assign(:token, token)
-        |> get("/api/pleroma/admin/users?filters=deactivated,external")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [user_response(user)]
-             }
-    end
-
-    test "it omits relay user", %{admin: admin, conn: conn} do
-      assert %User{} = Relay.get_actor()
-
-      conn = get(conn, "/api/pleroma/admin/users")
-
-      assert json_response(conn, 200) == %{
-               "count" => 1,
-               "page_size" => 50,
-               "users" => [
-                 user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}})
-               ]
-             }
-    end
-  end
-
-  test "PATCH /api/pleroma/admin/users/activate", %{admin: admin, conn: conn} do
-    user_one = insert(:user, deactivated: true)
-    user_two = insert(:user, deactivated: true)
-
-    conn =
-      patch(
-        conn,
-        "/api/pleroma/admin/users/activate",
-        %{nicknames: [user_one.nickname, user_two.nickname]}
-      )
-
-    response = json_response(conn, 200)
-    assert Enum.map(response["users"], & &1["deactivated"]) == [false, false]
-
-    log_entry = Repo.one(ModerationLog)
-
-    assert ModerationLog.get_log_entry_message(log_entry) ==
-             "@#{admin.nickname} activated users: @#{user_one.nickname}, @#{user_two.nickname}"
-  end
-
-  test "PATCH /api/pleroma/admin/users/deactivate", %{admin: admin, conn: conn} do
-    user_one = insert(:user, deactivated: false)
-    user_two = insert(:user, deactivated: false)
-
-    conn =
-      patch(
-        conn,
-        "/api/pleroma/admin/users/deactivate",
-        %{nicknames: [user_one.nickname, user_two.nickname]}
-      )
-
-    response = json_response(conn, 200)
-    assert Enum.map(response["users"], & &1["deactivated"]) == [true, true]
-
-    log_entry = Repo.one(ModerationLog)
-
-    assert ModerationLog.get_log_entry_message(log_entry) ==
-             "@#{admin.nickname} deactivated users: @#{user_one.nickname}, @#{user_two.nickname}"
-  end
-
-  test "PATCH /api/pleroma/admin/users/approve", %{admin: admin, conn: conn} do
-    user_one = insert(:user, approval_pending: true)
-    user_two = insert(:user, approval_pending: true)
-
-    conn =
-      patch(
-        conn,
-        "/api/pleroma/admin/users/approve",
-        %{nicknames: [user_one.nickname, user_two.nickname]}
-      )
-
-    response = json_response(conn, 200)
-    assert Enum.map(response["users"], & &1["approval_pending"]) == [false, false]
-
-    log_entry = Repo.one(ModerationLog)
-
-    assert ModerationLog.get_log_entry_message(log_entry) ==
-             "@#{admin.nickname} approved users: @#{user_one.nickname}, @#{user_two.nickname}"
-  end
-
-  test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation", %{admin: admin, conn: conn} do
-    user = insert(:user)
-
-    conn = patch(conn, "/api/pleroma/admin/users/#{user.nickname}/toggle_activation")
-
-    assert json_response(conn, 200) ==
-             user_response(
-               user,
-               %{"deactivated" => !user.deactivated}
-             )
-
-    log_entry = Repo.one(ModerationLog)
-
-    assert ModerationLog.get_log_entry_message(log_entry) ==
-             "@#{admin.nickname} deactivated users: @#{user.nickname}"
-  end
-
   describe "PUT disable_mfa" do
     test "returns 200 and disable 2fa", %{conn: conn} do
       user =
@@ -1796,25 +978,6 @@ test "by instance", %{conn: conn} do
                response["status_visibility"]
     end
   end
-
-  defp user_response(user, attrs \\ %{}) do
-    %{
-      "deactivated" => user.deactivated,
-      "id" => user.id,
-      "nickname" => user.nickname,
-      "roles" => %{"admin" => false, "moderator" => false},
-      "local" => user.local,
-      "tags" => [],
-      "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-      "display_name" => HTML.strip_tags(user.name || user.nickname),
-      "confirmation_pending" => false,
-      "approval_pending" => false,
-      "url" => user.ap_id,
-      "registration_reason" => nil,
-      "actor_type" => "Person"
-    }
-    |> Map.merge(attrs)
-  end
 end
 
 # Needed for testing
diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/web/admin_api/controllers/user_controller_test.exs
new file mode 100644
index 000000000..347258058
--- /dev/null
+++ b/test/web/admin_api/controllers/user_controller_test.exs
@@ -0,0 +1,983 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.UserControllerTest do
+  use Pleroma.Web.ConnCase
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Mock
+  import Pleroma.Factory
+
+  alias Pleroma.Config
+  alias Pleroma.HTML
+  alias Pleroma.ModerationLog
+  alias Pleroma.Repo
+  alias Pleroma.Tests.ObanHelpers
+  alias Pleroma.User
+  alias Pleroma.Web
+  alias Pleroma.Web.ActivityPub.Relay
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.MediaProxy
+
+  setup_all do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+
+    :ok
+  end
+
+  setup do
+    admin = insert(:user, is_admin: true)
+    token = insert(:oauth_admin_token, user: admin)
+
+    conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> assign(:token, token)
+
+    {:ok, %{admin: admin, token: token, conn: conn}}
+  end
+
+  test "with valid `admin_token` query parameter, skips OAuth scopes check" do
+    clear_config([:admin_token], "password123")
+
+    user = insert(:user)
+
+    conn = get(build_conn(), "/api/pleroma/admin/users/#{user.nickname}?admin_token=password123")
+
+    assert json_response(conn, 200)
+  end
+
+  describe "with [:auth, :enforce_oauth_admin_scope_usage]," do
+    setup do: clear_config([:auth, :enforce_oauth_admin_scope_usage], true)
+
+    test "GET /api/pleroma/admin/users/:nickname requires admin:read:accounts or broader scope",
+         %{admin: admin} do
+      user = insert(:user)
+      url = "/api/pleroma/admin/users/#{user.nickname}"
+
+      good_token1 = insert(:oauth_token, user: admin, scopes: ["admin"])
+      good_token2 = insert(:oauth_token, user: admin, scopes: ["admin:read"])
+      good_token3 = insert(:oauth_token, user: admin, scopes: ["admin:read:accounts"])
+
+      bad_token1 = insert(:oauth_token, user: admin, scopes: ["read:accounts"])
+      bad_token2 = insert(:oauth_token, user: admin, scopes: ["admin:read:accounts:partial"])
+      bad_token3 = nil
+
+      for good_token <- [good_token1, good_token2, good_token3] do
+        conn =
+          build_conn()
+          |> assign(:user, admin)
+          |> assign(:token, good_token)
+          |> get(url)
+
+        assert json_response(conn, 200)
+      end
+
+      for good_token <- [good_token1, good_token2, good_token3] do
+        conn =
+          build_conn()
+          |> assign(:user, nil)
+          |> assign(:token, good_token)
+          |> get(url)
+
+        assert json_response(conn, :forbidden)
+      end
+
+      for bad_token <- [bad_token1, bad_token2, bad_token3] do
+        conn =
+          build_conn()
+          |> assign(:user, admin)
+          |> assign(:token, bad_token)
+          |> get(url)
+
+        assert json_response(conn, :forbidden)
+      end
+    end
+  end
+
+  describe "unless [:auth, :enforce_oauth_admin_scope_usage]," do
+    setup do: clear_config([:auth, :enforce_oauth_admin_scope_usage], false)
+
+    test "GET /api/pleroma/admin/users/:nickname requires " <>
+           "read:accounts or admin:read:accounts or broader scope",
+         %{admin: admin} do
+      user = insert(:user)
+      url = "/api/pleroma/admin/users/#{user.nickname}"
+
+      good_token1 = insert(:oauth_token, user: admin, scopes: ["admin"])
+      good_token2 = insert(:oauth_token, user: admin, scopes: ["admin:read"])
+      good_token3 = insert(:oauth_token, user: admin, scopes: ["admin:read:accounts"])
+      good_token4 = insert(:oauth_token, user: admin, scopes: ["read:accounts"])
+      good_token5 = insert(:oauth_token, user: admin, scopes: ["read"])
+
+      good_tokens = [good_token1, good_token2, good_token3, good_token4, good_token5]
+
+      bad_token1 = insert(:oauth_token, user: admin, scopes: ["read:accounts:partial"])
+      bad_token2 = insert(:oauth_token, user: admin, scopes: ["admin:read:accounts:partial"])
+      bad_token3 = nil
+
+      for good_token <- good_tokens do
+        conn =
+          build_conn()
+          |> assign(:user, admin)
+          |> assign(:token, good_token)
+          |> get(url)
+
+        assert json_response(conn, 200)
+      end
+
+      for good_token <- good_tokens do
+        conn =
+          build_conn()
+          |> assign(:user, nil)
+          |> assign(:token, good_token)
+          |> get(url)
+
+        assert json_response(conn, :forbidden)
+      end
+
+      for bad_token <- [bad_token1, bad_token2, bad_token3] do
+        conn =
+          build_conn()
+          |> assign(:user, admin)
+          |> assign(:token, bad_token)
+          |> get(url)
+
+        assert json_response(conn, :forbidden)
+      end
+    end
+  end
+
+  describe "DELETE /api/pleroma/admin/users" do
+    test "single user", %{admin: admin, conn: conn} do
+      clear_config([:instance, :federating], true)
+
+      user =
+        insert(:user,
+          avatar: %{"url" => [%{"href" => "https://someurl"}]},
+          banner: %{"url" => [%{"href" => "https://somebanner"}]},
+          bio: "Hello world!",
+          name: "A guy"
+        )
+
+      # Create some activities to check they got deleted later
+      follower = insert(:user)
+      {:ok, _} = CommonAPI.post(user, %{status: "test"})
+      {:ok, _, _, _} = CommonAPI.follow(user, follower)
+      {:ok, _, _, _} = CommonAPI.follow(follower, user)
+      user = Repo.get(User, user.id)
+      assert user.note_count == 1
+      assert user.follower_count == 1
+      assert user.following_count == 1
+      refute user.deactivated
+
+      with_mock Pleroma.Web.Federator,
+        publish: fn _ -> nil end,
+        perform: fn _, _ -> nil end do
+        conn =
+          conn
+          |> put_req_header("accept", "application/json")
+          |> delete("/api/pleroma/admin/users?nickname=#{user.nickname}")
+
+        ObanHelpers.perform_all()
+
+        assert User.get_by_nickname(user.nickname).deactivated
+
+        log_entry = Repo.one(ModerationLog)
+
+        assert ModerationLog.get_log_entry_message(log_entry) ==
+                 "@#{admin.nickname} deleted users: @#{user.nickname}"
+
+        assert json_response(conn, 200) == [user.nickname]
+
+        user = Repo.get(User, user.id)
+        assert user.deactivated
+
+        assert user.avatar == %{}
+        assert user.banner == %{}
+        assert user.note_count == 0
+        assert user.follower_count == 0
+        assert user.following_count == 0
+        assert user.bio == ""
+        assert user.name == nil
+
+        assert called(Pleroma.Web.Federator.publish(:_))
+      end
+    end
+
+    test "multiple users", %{admin: admin, conn: conn} do
+      user_one = insert(:user)
+      user_two = insert(:user)
+
+      conn =
+        conn
+        |> put_req_header("accept", "application/json")
+        |> delete("/api/pleroma/admin/users", %{
+          nicknames: [user_one.nickname, user_two.nickname]
+        })
+
+      log_entry = Repo.one(ModerationLog)
+
+      assert ModerationLog.get_log_entry_message(log_entry) ==
+               "@#{admin.nickname} deleted users: @#{user_one.nickname}, @#{user_two.nickname}"
+
+      response = json_response(conn, 200)
+      assert response -- [user_one.nickname, user_two.nickname] == []
+    end
+  end
+
+  describe "/api/pleroma/admin/users" do
+    test "Create", %{conn: conn} do
+      conn =
+        conn
+        |> put_req_header("accept", "application/json")
+        |> post("/api/pleroma/admin/users", %{
+          "users" => [
+            %{
+              "nickname" => "lain",
+              "email" => "lain@example.org",
+              "password" => "test"
+            },
+            %{
+              "nickname" => "lain2",
+              "email" => "lain2@example.org",
+              "password" => "test"
+            }
+          ]
+        })
+
+      response = json_response(conn, 200) |> Enum.map(&Map.get(&1, "type"))
+      assert response == ["success", "success"]
+
+      log_entry = Repo.one(ModerationLog)
+
+      assert ["lain", "lain2"] -- Enum.map(log_entry.data["subjects"], & &1["nickname"]) == []
+    end
+
+    test "Cannot create user with existing email", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> put_req_header("accept", "application/json")
+        |> post("/api/pleroma/admin/users", %{
+          "users" => [
+            %{
+              "nickname" => "lain",
+              "email" => user.email,
+              "password" => "test"
+            }
+          ]
+        })
+
+      assert json_response(conn, 409) == [
+               %{
+                 "code" => 409,
+                 "data" => %{
+                   "email" => user.email,
+                   "nickname" => "lain"
+                 },
+                 "error" => "email has already been taken",
+                 "type" => "error"
+               }
+             ]
+    end
+
+    test "Cannot create user with existing nickname", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> put_req_header("accept", "application/json")
+        |> post("/api/pleroma/admin/users", %{
+          "users" => [
+            %{
+              "nickname" => user.nickname,
+              "email" => "someuser@plerama.social",
+              "password" => "test"
+            }
+          ]
+        })
+
+      assert json_response(conn, 409) == [
+               %{
+                 "code" => 409,
+                 "data" => %{
+                   "email" => "someuser@plerama.social",
+                   "nickname" => user.nickname
+                 },
+                 "error" => "nickname has already been taken",
+                 "type" => "error"
+               }
+             ]
+    end
+
+    test "Multiple user creation works in transaction", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> put_req_header("accept", "application/json")
+        |> post("/api/pleroma/admin/users", %{
+          "users" => [
+            %{
+              "nickname" => "newuser",
+              "email" => "newuser@pleroma.social",
+              "password" => "test"
+            },
+            %{
+              "nickname" => "lain",
+              "email" => user.email,
+              "password" => "test"
+            }
+          ]
+        })
+
+      assert json_response(conn, 409) == [
+               %{
+                 "code" => 409,
+                 "data" => %{
+                   "email" => user.email,
+                   "nickname" => "lain"
+                 },
+                 "error" => "email has already been taken",
+                 "type" => "error"
+               },
+               %{
+                 "code" => 409,
+                 "data" => %{
+                   "email" => "newuser@pleroma.social",
+                   "nickname" => "newuser"
+                 },
+                 "error" => "",
+                 "type" => "error"
+               }
+             ]
+
+      assert User.get_by_nickname("newuser") === nil
+    end
+  end
+
+  describe "/api/pleroma/admin/users/:nickname" do
+    test "Show", %{conn: conn} do
+      user = insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}")
+
+      assert user_response(user) == json_response(conn, 200)
+    end
+
+    test "when the user doesn't exist", %{conn: conn} do
+      user = build(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users/#{user.nickname}")
+
+      assert %{"error" => "Not found"} == json_response(conn, 404)
+    end
+  end
+
+  describe "/api/pleroma/admin/users/follow" do
+    test "allows to force-follow another user", %{admin: admin, conn: conn} do
+      user = insert(:user)
+      follower = insert(:user)
+
+      conn
+      |> put_req_header("accept", "application/json")
+      |> post("/api/pleroma/admin/users/follow", %{
+        "follower" => follower.nickname,
+        "followed" => user.nickname
+      })
+
+      user = User.get_cached_by_id(user.id)
+      follower = User.get_cached_by_id(follower.id)
+
+      assert User.following?(follower, user)
+
+      log_entry = Repo.one(ModerationLog)
+
+      assert ModerationLog.get_log_entry_message(log_entry) ==
+               "@#{admin.nickname} made @#{follower.nickname} follow @#{user.nickname}"
+    end
+  end
+
+  describe "/api/pleroma/admin/users/unfollow" do
+    test "allows to force-unfollow another user", %{admin: admin, conn: conn} do
+      user = insert(:user)
+      follower = insert(:user)
+
+      User.follow(follower, user)
+
+      conn
+      |> put_req_header("accept", "application/json")
+      |> post("/api/pleroma/admin/users/unfollow", %{
+        "follower" => follower.nickname,
+        "followed" => user.nickname
+      })
+
+      user = User.get_cached_by_id(user.id)
+      follower = User.get_cached_by_id(follower.id)
+
+      refute User.following?(follower, user)
+
+      log_entry = Repo.one(ModerationLog)
+
+      assert ModerationLog.get_log_entry_message(log_entry) ==
+               "@#{admin.nickname} made @#{follower.nickname} unfollow @#{user.nickname}"
+    end
+  end
+
+  describe "GET /api/pleroma/admin/users" do
+    test "renders users array for the first page", %{conn: conn, admin: admin} do
+      user = insert(:user, local: false, tags: ["foo", "bar"])
+      user2 = insert(:user, approval_pending: true, registration_reason: "I'm a chill dude")
+
+      conn = get(conn, "/api/pleroma/admin/users?page=1")
+
+      users =
+        [
+          user_response(
+            admin,
+            %{"roles" => %{"admin" => true, "moderator" => false}}
+          ),
+          user_response(user, %{"local" => false, "tags" => ["foo", "bar"]}),
+          user_response(
+            user2,
+            %{
+              "local" => true,
+              "approval_pending" => true,
+              "registration_reason" => "I'm a chill dude",
+              "actor_type" => "Person"
+            }
+          )
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert json_response(conn, 200) == %{
+               "count" => 3,
+               "page_size" => 50,
+               "users" => users
+             }
+    end
+
+    test "pagination works correctly with service users", %{conn: conn} do
+      service1 = User.get_or_create_service_actor_by_ap_id(Web.base_url() <> "/meido", "meido")
+
+      insert_list(25, :user)
+
+      assert %{"count" => 26, "page_size" => 10, "users" => users1} =
+               conn
+               |> get("/api/pleroma/admin/users?page=1&filters=", %{page_size: "10"})
+               |> json_response(200)
+
+      assert Enum.count(users1) == 10
+      assert service1 not in users1
+
+      assert %{"count" => 26, "page_size" => 10, "users" => users2} =
+               conn
+               |> get("/api/pleroma/admin/users?page=2&filters=", %{page_size: "10"})
+               |> json_response(200)
+
+      assert Enum.count(users2) == 10
+      assert service1 not in users2
+
+      assert %{"count" => 26, "page_size" => 10, "users" => users3} =
+               conn
+               |> get("/api/pleroma/admin/users?page=3&filters=", %{page_size: "10"})
+               |> json_response(200)
+
+      assert Enum.count(users3) == 6
+      assert service1 not in users3
+    end
+
+    test "renders empty array for the second page", %{conn: conn} do
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?page=2")
+
+      assert json_response(conn, 200) == %{
+               "count" => 2,
+               "page_size" => 50,
+               "users" => []
+             }
+    end
+
+    test "regular search", %{conn: conn} do
+      user = insert(:user, nickname: "bob")
+
+      conn = get(conn, "/api/pleroma/admin/users?query=bo")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user, %{"local" => true})]
+             }
+    end
+
+    test "search by domain", %{conn: conn} do
+      user = insert(:user, nickname: "nickname@domain.com")
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?query=domain.com")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user)]
+             }
+    end
+
+    test "search by full nickname", %{conn: conn} do
+      user = insert(:user, nickname: "nickname@domain.com")
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?query=nickname@domain.com")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user)]
+             }
+    end
+
+    test "search by display name", %{conn: conn} do
+      user = insert(:user, name: "Display name")
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?name=display")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user)]
+             }
+    end
+
+    test "search by email", %{conn: conn} do
+      user = insert(:user, email: "email@example.com")
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?email=email@example.com")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user)]
+             }
+    end
+
+    test "regular search with page size", %{conn: conn} do
+      user = insert(:user, nickname: "aalice")
+      user2 = insert(:user, nickname: "alice")
+
+      conn1 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=1")
+
+      assert json_response(conn1, 200) == %{
+               "count" => 2,
+               "page_size" => 1,
+               "users" => [user_response(user)]
+             }
+
+      conn2 = get(conn, "/api/pleroma/admin/users?query=a&page_size=1&page=2")
+
+      assert json_response(conn2, 200) == %{
+               "count" => 2,
+               "page_size" => 1,
+               "users" => [user_response(user2)]
+             }
+    end
+
+    test "only local users" do
+      admin = insert(:user, is_admin: true, nickname: "john")
+      token = insert(:oauth_admin_token, user: admin)
+      user = insert(:user, nickname: "bob")
+
+      insert(:user, nickname: "bobb", local: false)
+
+      conn =
+        build_conn()
+        |> assign(:user, admin)
+        |> assign(:token, token)
+        |> get("/api/pleroma/admin/users?query=bo&filters=local")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user)]
+             }
+    end
+
+    test "only local users with no query", %{conn: conn, admin: old_admin} do
+      admin = insert(:user, is_admin: true, nickname: "john")
+      user = insert(:user, nickname: "bob")
+
+      insert(:user, nickname: "bobb", local: false)
+
+      conn = get(conn, "/api/pleroma/admin/users?filters=local")
+
+      users =
+        [
+          user_response(user),
+          user_response(admin, %{
+            "roles" => %{"admin" => true, "moderator" => false}
+          }),
+          user_response(old_admin, %{
+            "deactivated" => false,
+            "roles" => %{"admin" => true, "moderator" => false}
+          })
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert json_response(conn, 200) == %{
+               "count" => 3,
+               "page_size" => 50,
+               "users" => users
+             }
+    end
+
+    test "only unconfirmed users", %{conn: conn} do
+      sad_user = insert(:user, nickname: "sadboy", confirmation_pending: true)
+      old_user = insert(:user, nickname: "oldboy", confirmation_pending: true)
+
+      insert(:user, nickname: "happyboy", approval_pending: false)
+      insert(:user, confirmation_pending: false)
+
+      result =
+        conn
+        |> get("/api/pleroma/admin/users?filters=need_confirmed")
+        |> json_response(200)
+
+      users =
+        Enum.map([old_user, sad_user], fn user ->
+          user_response(user, %{
+            "confirmation_pending" => true,
+            "approval_pending" => false
+          })
+        end)
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert result == %{"count" => 2, "page_size" => 50, "users" => users}
+    end
+
+    test "only unapproved users", %{conn: conn} do
+      user =
+        insert(:user,
+          nickname: "sadboy",
+          approval_pending: true,
+          registration_reason: "Plz let me in!"
+        )
+
+      insert(:user, nickname: "happyboy", approval_pending: false)
+
+      conn = get(conn, "/api/pleroma/admin/users?filters=need_approval")
+
+      users =
+        [
+          %{
+            "deactivated" => user.deactivated,
+            "id" => user.id,
+            "nickname" => user.nickname,
+            "roles" => %{"admin" => false, "moderator" => false},
+            "local" => true,
+            "tags" => [],
+            "avatar" => User.avatar_url(user) |> MediaProxy.url(),
+            "display_name" => HTML.strip_tags(user.name || user.nickname),
+            "confirmation_pending" => false,
+            "approval_pending" => true,
+            "url" => user.ap_id,
+            "registration_reason" => "Plz let me in!",
+            "actor_type" => "Person"
+          }
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => users
+             }
+    end
+
+    test "load only admins", %{conn: conn, admin: admin} do
+      second_admin = insert(:user, is_admin: true)
+      insert(:user)
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?filters=is_admin")
+
+      users =
+        [
+          %{
+            "deactivated" => false,
+            "id" => admin.id,
+            "nickname" => admin.nickname,
+            "roles" => %{"admin" => true, "moderator" => false},
+            "local" => admin.local,
+            "tags" => [],
+            "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
+            "display_name" => HTML.strip_tags(admin.name || admin.nickname),
+            "confirmation_pending" => false,
+            "approval_pending" => false,
+            "url" => admin.ap_id,
+            "registration_reason" => nil,
+            "actor_type" => "Person"
+          },
+          %{
+            "deactivated" => false,
+            "id" => second_admin.id,
+            "nickname" => second_admin.nickname,
+            "roles" => %{"admin" => true, "moderator" => false},
+            "local" => second_admin.local,
+            "tags" => [],
+            "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(),
+            "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname),
+            "confirmation_pending" => false,
+            "approval_pending" => false,
+            "url" => second_admin.ap_id,
+            "registration_reason" => nil,
+            "actor_type" => "Person"
+          }
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert json_response(conn, 200) == %{
+               "count" => 2,
+               "page_size" => 50,
+               "users" => users
+             }
+    end
+
+    test "load only moderators", %{conn: conn} do
+      moderator = insert(:user, is_moderator: true)
+      insert(:user)
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?filters=is_moderator")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [
+                 %{
+                   "deactivated" => false,
+                   "id" => moderator.id,
+                   "nickname" => moderator.nickname,
+                   "roles" => %{"admin" => false, "moderator" => true},
+                   "local" => moderator.local,
+                   "tags" => [],
+                   "avatar" => User.avatar_url(moderator) |> MediaProxy.url(),
+                   "display_name" => HTML.strip_tags(moderator.name || moderator.nickname),
+                   "confirmation_pending" => false,
+                   "approval_pending" => false,
+                   "url" => moderator.ap_id,
+                   "registration_reason" => nil,
+                   "actor_type" => "Person"
+                 }
+               ]
+             }
+    end
+
+    test "load users with tags list", %{conn: conn} do
+      user1 = insert(:user, tags: ["first"])
+      user2 = insert(:user, tags: ["second"])
+      insert(:user)
+      insert(:user)
+
+      conn = get(conn, "/api/pleroma/admin/users?tags[]=first&tags[]=second")
+
+      users =
+        [
+          user_response(
+            user1,
+            %{
+              "deactivated" => false,
+              "roles" => %{"admin" => false, "moderator" => false},
+              "local" => user1.local,
+              "tags" => ["first"],
+              "avatar" => User.avatar_url(user1) |> MediaProxy.url(),
+              "display_name" => HTML.strip_tags(user1.name || user1.nickname),
+              "confirmation_pending" => false,
+              "approval_pending" => false,
+              "url" => user1.ap_id,
+              "registration_reason" => nil,
+              "actor_type" => "Person"
+            }
+          ),
+          %{
+            "deactivated" => false,
+            "id" => user2.id,
+            "nickname" => user2.nickname,
+            "roles" => %{"admin" => false, "moderator" => false},
+            "local" => user2.local,
+            "tags" => ["second"],
+            "avatar" => User.avatar_url(user2) |> MediaProxy.url(),
+            "display_name" => HTML.strip_tags(user2.name || user2.nickname),
+            "confirmation_pending" => false,
+            "approval_pending" => false,
+            "url" => user2.ap_id,
+            "registration_reason" => nil,
+            "actor_type" => "Person"
+          }
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert json_response(conn, 200) == %{
+               "count" => 2,
+               "page_size" => 50,
+               "users" => users
+             }
+    end
+
+    test "`active` filters out users pending approval", %{token: token} do
+      insert(:user, approval_pending: true)
+      %{id: user_id} = insert(:user, approval_pending: false)
+      %{id: admin_id} = token.user
+
+      conn =
+        build_conn()
+        |> assign(:user, token.user)
+        |> assign(:token, token)
+        |> get("/api/pleroma/admin/users?filters=active")
+
+      assert %{
+               "count" => 2,
+               "page_size" => 50,
+               "users" => [
+                 %{"id" => ^admin_id},
+                 %{"id" => ^user_id}
+               ]
+             } = json_response(conn, 200)
+    end
+
+    test "it works with multiple filters" do
+      admin = insert(:user, nickname: "john", is_admin: true)
+      token = insert(:oauth_admin_token, user: admin)
+      user = insert(:user, nickname: "bob", local: false, deactivated: true)
+
+      insert(:user, nickname: "ken", local: true, deactivated: true)
+      insert(:user, nickname: "bobb", local: false, deactivated: false)
+
+      conn =
+        build_conn()
+        |> assign(:user, admin)
+        |> assign(:token, token)
+        |> get("/api/pleroma/admin/users?filters=deactivated,external")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [user_response(user)]
+             }
+    end
+
+    test "it omits relay user", %{admin: admin, conn: conn} do
+      assert %User{} = Relay.get_actor()
+
+      conn = get(conn, "/api/pleroma/admin/users")
+
+      assert json_response(conn, 200) == %{
+               "count" => 1,
+               "page_size" => 50,
+               "users" => [
+                 user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}})
+               ]
+             }
+    end
+  end
+
+  test "PATCH /api/pleroma/admin/users/activate", %{admin: admin, conn: conn} do
+    user_one = insert(:user, deactivated: true)
+    user_two = insert(:user, deactivated: true)
+
+    conn =
+      patch(
+        conn,
+        "/api/pleroma/admin/users/activate",
+        %{nicknames: [user_one.nickname, user_two.nickname]}
+      )
+
+    response = json_response(conn, 200)
+    assert Enum.map(response["users"], & &1["deactivated"]) == [false, false]
+
+    log_entry = Repo.one(ModerationLog)
+
+    assert ModerationLog.get_log_entry_message(log_entry) ==
+             "@#{admin.nickname} activated users: @#{user_one.nickname}, @#{user_two.nickname}"
+  end
+
+  test "PATCH /api/pleroma/admin/users/deactivate", %{admin: admin, conn: conn} do
+    user_one = insert(:user, deactivated: false)
+    user_two = insert(:user, deactivated: false)
+
+    conn =
+      patch(
+        conn,
+        "/api/pleroma/admin/users/deactivate",
+        %{nicknames: [user_one.nickname, user_two.nickname]}
+      )
+
+    response = json_response(conn, 200)
+    assert Enum.map(response["users"], & &1["deactivated"]) == [true, true]
+
+    log_entry = Repo.one(ModerationLog)
+
+    assert ModerationLog.get_log_entry_message(log_entry) ==
+             "@#{admin.nickname} deactivated users: @#{user_one.nickname}, @#{user_two.nickname}"
+  end
+
+  test "PATCH /api/pleroma/admin/users/approve", %{admin: admin, conn: conn} do
+    user_one = insert(:user, approval_pending: true)
+    user_two = insert(:user, approval_pending: true)
+
+    conn =
+      patch(
+        conn,
+        "/api/pleroma/admin/users/approve",
+        %{nicknames: [user_one.nickname, user_two.nickname]}
+      )
+
+    response = json_response(conn, 200)
+    assert Enum.map(response["users"], & &1["approval_pending"]) == [false, false]
+
+    log_entry = Repo.one(ModerationLog)
+
+    assert ModerationLog.get_log_entry_message(log_entry) ==
+             "@#{admin.nickname} approved users: @#{user_one.nickname}, @#{user_two.nickname}"
+  end
+
+  test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation", %{admin: admin, conn: conn} do
+    user = insert(:user)
+
+    conn = patch(conn, "/api/pleroma/admin/users/#{user.nickname}/toggle_activation")
+
+    assert json_response(conn, 200) ==
+             user_response(
+               user,
+               %{"deactivated" => !user.deactivated}
+             )
+
+    log_entry = Repo.one(ModerationLog)
+
+    assert ModerationLog.get_log_entry_message(log_entry) ==
+             "@#{admin.nickname} deactivated users: @#{user.nickname}"
+  end
+
+  defp user_response(user, attrs \\ %{}) do
+    %{
+      "deactivated" => user.deactivated,
+      "id" => user.id,
+      "nickname" => user.nickname,
+      "roles" => %{"admin" => false, "moderator" => false},
+      "local" => user.local,
+      "tags" => [],
+      "avatar" => User.avatar_url(user) |> MediaProxy.url(),
+      "display_name" => HTML.strip_tags(user.name || user.nickname),
+      "confirmation_pending" => false,
+      "approval_pending" => false,
+      "url" => user.ap_id,
+      "registration_reason" => nil,
+      "actor_type" => "Person"
+    }
+    |> Map.merge(attrs)
+  end
+end

From 46b420aa602050d7b3bff33a6b51d54852b2adb3 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 25 Sep 2020 09:39:49 +0300
Subject: [PATCH 064/104] need_confirmed -> unconfirmed

---
 lib/pleroma/user/query.ex                                | 4 ++--
 lib/pleroma/web/admin_api/controllers/user_controller.ex | 2 +-
 test/pleroma/web/admin_api/search_test.exs               | 2 +-
 test/web/admin_api/controllers/user_controller_test.exs  | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex
index 2933e7fb4..711439eef 100644
--- a/lib/pleroma/user/query.ex
+++ b/lib/pleroma/user/query.ex
@@ -43,7 +43,7 @@ defmodule Pleroma.User.Query do
             active: boolean(),
             deactivated: boolean(),
             need_approval: boolean(),
-            need_confirmed: boolean(),
+            unconfirmed: boolean(),
             is_admin: boolean(),
             is_moderator: boolean(),
             super_users: boolean(),
@@ -157,7 +157,7 @@ defp compose_query({:need_approval, _}, query) do
     where(query, [u], u.approval_pending)
   end
 
-  defp compose_query({:need_confirmed, _}, query) do
+  defp compose_query({:unconfirmed, _}, query) do
     where(query, [u], u.confirmation_pending)
   end
 
diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex
index 5e049aa0f..db6ef3220 100644
--- a/lib/pleroma/web/admin_api/controllers/user_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex
@@ -259,7 +259,7 @@ def list(conn, params) do
     end
   end
 
-  @filters ~w(local external active deactivated need_approval need_confirmed is_admin is_moderator)
+  @filters ~w(local external active deactivated need_approval unconfirmed is_admin is_moderator)
 
   @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
   defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
diff --git a/test/pleroma/web/admin_api/search_test.exs b/test/pleroma/web/admin_api/search_test.exs
index 27ca396e6..82da86f7f 100644
--- a/test/pleroma/web/admin_api/search_test.exs
+++ b/test/pleroma/web/admin_api/search_test.exs
@@ -184,7 +184,7 @@ test "it returns unconfirmed user" do
       insert(:user)
 
       {:ok, _results, total} = Search.user()
-      {:ok, [^unconfirmed], count} = Search.user(%{need_confirmed: true})
+      {:ok, [^unconfirmed], count} = Search.user(%{unconfirmed: true})
       assert total == 3
       assert count == 1
     end
diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/web/admin_api/controllers/user_controller_test.exs
index 347258058..6384f8ddc 100644
--- a/test/web/admin_api/controllers/user_controller_test.exs
+++ b/test/web/admin_api/controllers/user_controller_test.exs
@@ -644,7 +644,7 @@ test "only unconfirmed users", %{conn: conn} do
 
       result =
         conn
-        |> get("/api/pleroma/admin/users?filters=need_confirmed")
+        |> get("/api/pleroma/admin/users?filters=unconfirmed")
         |> json_response(200)
 
       users =

From 60663150b5b936c831bdc0cfeade30867e536317 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 25 Sep 2020 16:04:01 +0300
Subject: [PATCH 065/104] admin user search: added filter by `actor_type`

---
 lib/pleroma/user/query.ex                  |  7 ++++++-
 test/pleroma/web/admin_api/search_test.exs | 14 ++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex
index 711439eef..7ef2a1455 100644
--- a/lib/pleroma/user/query.ex
+++ b/lib/pleroma/user/query.ex
@@ -56,7 +56,8 @@ defmodule Pleroma.User.Query do
             ap_id: [String.t()],
             order_by: term(),
             select: term(),
-            limit: pos_integer()
+            limit: pos_integer(),
+            actor_types: [String.t()]
           }
           | map()
 
@@ -115,6 +116,10 @@ defp compose_query({:is_admin, bool}, query) do
     where(query, [u], u.is_admin == ^bool)
   end
 
+  defp compose_query({:actor_types, actor_types}, query) when is_list(actor_types) do
+    where(query, [u], u.actor_type in ^actor_types)
+  end
+
   defp compose_query({:is_moderator, bool}, query) do
     where(query, [u], u.is_moderator == ^bool)
   end
diff --git a/test/pleroma/web/admin_api/search_test.exs b/test/pleroma/web/admin_api/search_test.exs
index 82da86f7f..92a116c65 100644
--- a/test/pleroma/web/admin_api/search_test.exs
+++ b/test/pleroma/web/admin_api/search_test.exs
@@ -143,6 +143,20 @@ test "it returns users with tags" do
       assert user2 in users
     end
 
+    test "it returns users by actor_types" do
+      user_service = insert(:user, actor_type: "Service")
+      user_application = insert(:user, actor_type: "Application")
+      user1 = insert(:user)
+      user2 = insert(:user)
+
+      {:ok, [^user_service], 1} = Search.user(%{actor_types: ["Service"]})
+      {:ok, [^user_application], 1} = Search.user(%{actor_types: ["Application"]})
+      {:ok, [^user1, ^user2], 2} = Search.user(%{actor_types: ["Person"]})
+
+      {:ok, [^user_service, ^user1, ^user2], 3} =
+        Search.user(%{actor_types: ["Person", "Service"]})
+    end
+
     test "it returns user by display name" do
       user = insert(:user, name: "Display name")
       insert(:user)

From 44e5a57d1a1b36ad2971763f3a8cab82bdb94b08 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 25 Sep 2020 16:37:55 +0300
Subject: [PATCH 066/104] admin api: added user filters by `actor_types`

---
 .../admin_api/controllers/user_controller.ex  |   3 +-
 .../controllers/user_controller_test.exs      | 164 ++++++++----------
 2 files changed, 76 insertions(+), 91 deletions(-)

diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex
index db6ef3220..15cec9b09 100644
--- a/lib/pleroma/web/admin_api/controllers/user_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex
@@ -243,7 +243,8 @@ def list(conn, params) do
         page_size: page_size,
         tags: params["tags"],
         name: params["name"],
-        email: params["email"]
+        email: params["email"],
+        actor_types: params["actor_types"]
       }
       |> Map.merge(filters)
 
diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/web/admin_api/controllers/user_controller_test.exs
index 6384f8ddc..b8a927e56 100644
--- a/test/web/admin_api/controllers/user_controller_test.exs
+++ b/test/web/admin_api/controllers/user_controller_test.exs
@@ -673,23 +673,10 @@ test "only unapproved users", %{conn: conn} do
 
       users =
         [
-          %{
-            "deactivated" => user.deactivated,
-            "id" => user.id,
-            "nickname" => user.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => true,
-            "tags" => [],
-            "avatar" => User.avatar_url(user) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user.name || user.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => true,
-            "url" => user.ap_id,
-            "registration_reason" => "Plz let me in!",
-            "actor_type" => "Person"
-          }
+          user_response(user,
+            %{"approval_pending" => true, "registration_reason" => "Plz let me in!"}
+          )
         ]
-        |> Enum.sort_by(& &1["nickname"])
 
       assert json_response(conn, 200) == %{
                "count" => 1,
@@ -707,36 +694,14 @@ test "load only admins", %{conn: conn, admin: admin} do
 
       users =
         [
-          %{
+          user_response(admin, %{
             "deactivated" => false,
-            "id" => admin.id,
-            "nickname" => admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "local" => admin.local,
-            "tags" => [],
-            "avatar" => User.avatar_url(admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(admin.name || admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          },
-          %{
+            "roles" => %{"admin" => true, "moderator" => false}
+          }),
+          user_response(second_admin, %{
             "deactivated" => false,
-            "id" => second_admin.id,
-            "nickname" => second_admin.nickname,
-            "roles" => %{"admin" => true, "moderator" => false},
-            "local" => second_admin.local,
-            "tags" => [],
-            "avatar" => User.avatar_url(second_admin) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => second_admin.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          }
+            "roles" => %{"admin" => true, "moderator" => false}
+          })
         ]
         |> Enum.sort_by(& &1["nickname"])
 
@@ -758,25 +723,73 @@ test "load only moderators", %{conn: conn} do
                "count" => 1,
                "page_size" => 50,
                "users" => [
-                 %{
-                   "deactivated" => false,
-                   "id" => moderator.id,
-                   "nickname" => moderator.nickname,
-                   "roles" => %{"admin" => false, "moderator" => true},
-                   "local" => moderator.local,
-                   "tags" => [],
-                   "avatar" => User.avatar_url(moderator) |> MediaProxy.url(),
-                   "display_name" => HTML.strip_tags(moderator.name || moderator.nickname),
-                   "confirmation_pending" => false,
-                   "approval_pending" => false,
-                   "url" => moderator.ap_id,
-                   "registration_reason" => nil,
-                   "actor_type" => "Person"
-                 }
+                 user_response(moderator, %{
+                       "deactivated" => false,
+                       "roles" => %{"admin" => false, "moderator" => true}
+                               })
                ]
              }
     end
 
+    test "load users with actor_type is Person", %{admin: admin, conn: conn} do
+      insert(:user, actor_type: "Service")
+      insert(:user, actor_type: "Application")
+
+      user1 = insert(:user)
+      user2 = insert(:user)
+
+      response = conn
+      |> get(user_path(conn, :list), %{actor_types: ["Person"]})
+      |> json_response(200)
+
+      users =
+        [
+          user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}}),
+          user_response(user1),
+          user_response(user2)
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert response == %{"count" => 3, "page_size" => 50, "users" => users}
+    end
+
+    test "load users with actor_type is Person and Service", %{admin: admin, conn: conn} do
+      user_service = insert(:user, actor_type: "Service")
+      insert(:user, actor_type: "Application")
+
+      user1 = insert(:user)
+      user2 = insert(:user)
+
+      response  = conn
+      |> get(user_path(conn, :list), %{actor_types: ["Person", "Service"]})
+      |> json_response(200)
+
+      users =
+        [
+          user_response(admin, %{"roles" => %{"admin" => true, "moderator" => false}}),
+          user_response(user1),
+          user_response(user2),
+          user_response(user_service, %{"actor_type" => "Service"})
+        ]
+        |> Enum.sort_by(& &1["nickname"])
+
+      assert response == %{"count" => 4, "page_size" => 50, "users" => users}
+    end
+
+    test "load users with actor_type is Service", %{conn: conn} do
+      user_service = insert(:user, actor_type: "Service")
+      insert(:user, actor_type: "Application")
+      insert(:user)
+      insert(:user)
+
+      response = conn
+      |> get(user_path(conn, :list), %{actor_types: ["Service"]})
+      |> json_response(200)
+      users = [user_response(user_service, %{"actor_type" => "Service"})]
+
+      assert response == %{"count" => 1, "page_size" => 50, "users" => users }
+    end
+
     test "load users with tags list", %{conn: conn} do
       user1 = insert(:user, tags: ["first"])
       user2 = insert(:user, tags: ["second"])
@@ -787,37 +800,8 @@ test "load users with tags list", %{conn: conn} do
 
       users =
         [
-          user_response(
-            user1,
-            %{
-              "deactivated" => false,
-              "roles" => %{"admin" => false, "moderator" => false},
-              "local" => user1.local,
-              "tags" => ["first"],
-              "avatar" => User.avatar_url(user1) |> MediaProxy.url(),
-              "display_name" => HTML.strip_tags(user1.name || user1.nickname),
-              "confirmation_pending" => false,
-              "approval_pending" => false,
-              "url" => user1.ap_id,
-              "registration_reason" => nil,
-              "actor_type" => "Person"
-            }
-          ),
-          %{
-            "deactivated" => false,
-            "id" => user2.id,
-            "nickname" => user2.nickname,
-            "roles" => %{"admin" => false, "moderator" => false},
-            "local" => user2.local,
-            "tags" => ["second"],
-            "avatar" => User.avatar_url(user2) |> MediaProxy.url(),
-            "display_name" => HTML.strip_tags(user2.name || user2.nickname),
-            "confirmation_pending" => false,
-            "approval_pending" => false,
-            "url" => user2.ap_id,
-            "registration_reason" => nil,
-            "actor_type" => "Person"
-          }
+          user_response(user1, %{"tags" => ["first"]}),
+          user_response(user2, %{"tags" => ["second"]})
         ]
         |> Enum.sort_by(& &1["nickname"])
 

From 51189ad365a462163a4c98a7b5cb7ae74f2d634a Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 25 Sep 2020 16:40:50 +0300
Subject: [PATCH 067/104] update docs

---
 docs/API/admin_api.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md
index 7bf13daef..f7b5bcae7 100644
--- a/docs/API/admin_api.md
+++ b/docs/API/admin_api.md
@@ -20,12 +20,14 @@ Configuration options:
     - `external`: only external users
     - `active`: only active users
     - `need_approval`: only unapproved users
+    - `unconfirmed`: only unconfirmed users
     - `deactivated`: only deactivated users
     - `is_admin`: users with admin role
     - `is_moderator`: users with moderator role
   - *optional* `page`: **integer** page number
   - *optional* `page_size`: **integer** number of users per page (default is `50`)
   - *optional* `tags`: **[string]** tags list
+  - *optional* `actor_types`: **[string]** actor type list (`Person`, `Service`, `Application`)
   - *optional* `name`: **string** user display name
   - *optional* `email`: **string** user email
 - Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com`

From add26817e3a8756d2f6aea256c6d4b2070a8070b Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 25 Sep 2020 16:50:47 +0300
Subject: [PATCH 068/104] update changelog

---
 CHANGELOG.md                                  |  2 +
 .../controllers/user_controller_test.exs      | 42 ++++++++++---------
 2 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 05e94581a..82c5eaf8c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -32,6 +32,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Pleroma API: Importing the mutes users from CSV files.
 - Admin API: Importing emoji from a zip file
 - Pleroma API: Pagination for remote/local packs and emoji.
+- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status
+- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type`
 
 </details>
 
diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/web/admin_api/controllers/user_controller_test.exs
index b8a927e56..da26caf25 100644
--- a/test/web/admin_api/controllers/user_controller_test.exs
+++ b/test/web/admin_api/controllers/user_controller_test.exs
@@ -671,12 +671,12 @@ test "only unapproved users", %{conn: conn} do
 
       conn = get(conn, "/api/pleroma/admin/users?filters=need_approval")
 
-      users =
-        [
-          user_response(user,
-            %{"approval_pending" => true, "registration_reason" => "Plz let me in!"}
-          )
-        ]
+      users = [
+        user_response(
+          user,
+          %{"approval_pending" => true, "registration_reason" => "Plz let me in!"}
+        )
+      ]
 
       assert json_response(conn, 200) == %{
                "count" => 1,
@@ -724,9 +724,9 @@ test "load only moderators", %{conn: conn} do
                "page_size" => 50,
                "users" => [
                  user_response(moderator, %{
-                       "deactivated" => false,
-                       "roles" => %{"admin" => false, "moderator" => true}
-                               })
+                   "deactivated" => false,
+                   "roles" => %{"admin" => false, "moderator" => true}
+                 })
                ]
              }
     end
@@ -738,9 +738,10 @@ test "load users with actor_type is Person", %{admin: admin, conn: conn} do
       user1 = insert(:user)
       user2 = insert(:user)
 
-      response = conn
-      |> get(user_path(conn, :list), %{actor_types: ["Person"]})
-      |> json_response(200)
+      response =
+        conn
+        |> get(user_path(conn, :list), %{actor_types: ["Person"]})
+        |> json_response(200)
 
       users =
         [
@@ -760,9 +761,10 @@ test "load users with actor_type is Person and Service", %{admin: admin, conn: c
       user1 = insert(:user)
       user2 = insert(:user)
 
-      response  = conn
-      |> get(user_path(conn, :list), %{actor_types: ["Person", "Service"]})
-      |> json_response(200)
+      response =
+        conn
+        |> get(user_path(conn, :list), %{actor_types: ["Person", "Service"]})
+        |> json_response(200)
 
       users =
         [
@@ -782,12 +784,14 @@ test "load users with actor_type is Service", %{conn: conn} do
       insert(:user)
       insert(:user)
 
-      response = conn
-      |> get(user_path(conn, :list), %{actor_types: ["Service"]})
-      |> json_response(200)
+      response =
+        conn
+        |> get(user_path(conn, :list), %{actor_types: ["Service"]})
+        |> json_response(200)
+
       users = [user_response(user_service, %{"actor_type" => "Service"})]
 
-      assert response == %{"count" => 1, "page_size" => 50, "users" => users }
+      assert response == %{"count" => 1, "page_size" => 50, "users" => users}
     end
 
     test "load users with tags list", %{conn: conn} do

From ef627b9391e0dde8adf03c0132fbc2eeac6bdede Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Thu, 22 Oct 2020 12:04:23 +0300
Subject: [PATCH 069/104] fix module name

---
 lib/pleroma/web/admin_api/controllers/user_controller.ex        | 2 +-
 .../web/admin_api/controllers/user_controller_test.exs          | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename test/{ => pleroma}/web/admin_api/controllers/user_controller_test.exs (100%)

diff --git a/lib/pleroma/web/admin_api/controllers/user_controller.ex b/lib/pleroma/web/admin_api/controllers/user_controller.ex
index 15cec9b09..a2a1c875d 100644
--- a/lib/pleroma/web/admin_api/controllers/user_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/user_controller.ex
@@ -9,13 +9,13 @@ defmodule Pleroma.Web.AdminAPI.UserController do
     only: [fetch_integer_param: 3]
 
   alias Pleroma.ModerationLog
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.AdminAPI.AccountView
   alias Pleroma.Web.AdminAPI.Search
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   @users_page_size 50
 
diff --git a/test/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs
similarity index 100%
rename from test/web/admin_api/controllers/user_controller_test.exs
rename to test/pleroma/web/admin_api/controllers/user_controller_test.exs

From 60e379ce0b74bbe1b0f40a954aec040beab20e4e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 23 Oct 2020 13:53:01 +0200
Subject: [PATCH 070/104] User: Correctly handle whitespace names.

---
 lib/pleroma/user.ex                           |  5 +-
 lib/pleroma/web/activity_pub/activity_pub.ex  |  4 --
 test/fixtures/mewmew_no_name.json             | 46 +++++++++++++++++++
 .../web/activity_pub/activity_pub_test.exs    | 11 +++++
 4 files changed, 60 insertions(+), 6 deletions(-)
 create mode 100644 test/fixtures/mewmew_no_name.json

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index dc41d0001..72f507f1e 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -426,7 +426,6 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
       params,
       [
         :bio,
-        :name,
         :emoji,
         :ap_id,
         :inbox,
@@ -455,7 +454,9 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
         :accepts_chat_messages
       ]
     )
-    |> validate_required([:name, :ap_id])
+    |> cast(params, [:name], empty_values: [])
+    |> validate_required([:ap_id])
+    |> validate_required([:name], trim: false)
     |> unique_constraint(:nickname)
     |> validate_format(:nickname, @email_regex)
     |> validate_length(:bio, max: bio_limit)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index d17c892a7..df18db603 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1378,10 +1378,6 @@ def fetch_and_prepare_user_from_ap_id(ap_id, opts \\ []) do
          {:ok, data} <- user_data_from_user_object(data) do
       {:ok, maybe_update_follow_information(data)}
     else
-      {:error, "Object has been deleted" = e} ->
-        Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
-        {:error, e}
-
       {:error, {:reject, reason} = e} ->
         Logger.info("Rejected user #{ap_id}: #{inspect(reason)}")
         {:error, e}
diff --git a/test/fixtures/mewmew_no_name.json b/test/fixtures/mewmew_no_name.json
new file mode 100644
index 000000000..532d4cf70
--- /dev/null
+++ b/test/fixtures/mewmew_no_name.json
@@ -0,0 +1,46 @@
+{
+   "@context" : [
+      "https://www.w3.org/ns/activitystreams",
+      "https://princess.cat/schemas/litepub-0.1.jsonld",
+      {
+         "@language" : "und"
+      }
+   ],
+   "attachment" : [],
+   "capabilities" : {
+      "acceptsChatMessages" : true
+   },
+   "discoverable" : false,
+   "endpoints" : {
+      "oauthAuthorizationEndpoint" : "https://princess.cat/oauth/authorize",
+      "oauthRegistrationEndpoint" : "https://princess.cat/api/v1/apps",
+      "oauthTokenEndpoint" : "https://princess.cat/oauth/token",
+      "sharedInbox" : "https://princess.cat/inbox",
+      "uploadMedia" : "https://princess.cat/api/ap/upload_media"
+   },
+   "followers" : "https://princess.cat/users/mewmew/followers",
+   "following" : "https://princess.cat/users/mewmew/following",
+   "icon" : {
+      "type" : "Image",
+      "url" : "https://princess.cat/media/12794fb50e86911e65be97f69196814049dcb398a2f8b58b99bb6591576e648c.png?name=blobcatpresentpink.png"
+   },
+   "id" : "https://princess.cat/users/mewmew",
+   "image" : {
+      "type" : "Image",
+      "url" : "https://princess.cat/media/05d8bf3953ab6028fc920494ffc643fbee9dcef40d7bdd06f107e19acbfbd7f9.png"
+   },
+   "inbox" : "https://princess.cat/users/mewmew/inbox",
+   "manuallyApprovesFollowers" : true,
+   "name" : " ",
+   "outbox" : "https://princess.cat/users/mewmew/outbox",
+   "preferredUsername" : "mewmew",
+   "publicKey" : {
+      "id" : "https://princess.cat/users/mewmew#main-key",
+      "owner" : "https://princess.cat/users/mewmew",
+      "publicKeyPem" : "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAru7VpygVef4zrFwnj0Mh\nrbO/2z2EdKN3rERtNrT8zWsLXNLQ50lfpRPnGDrd+xq7Rva4EIu0d5KJJ9n4vtY0\nuxK3On9vA2oyjLlR9O0lI3XTrHJborG3P7IPXrmNUMFpHiFHNqHp5tugUrs1gUFq\n7tmOmM92IP4Wjk8qNHFcsfnUbaPTX7sNIhteQKdi5HrTb/6lrEIe4G/FlMKRqxo3\nRNHuv6SNFQuiUKvFzjzazvjkjvBSm+aFROgdHa2tKl88StpLr7xmuY8qNFCRT6W0\nLacRp6c8ah5f03Kd+xCBVhCKvKaF1K0ERnQTBiitUh85md+Mtx/CoDoLnmpnngR3\nvQIDAQAB\n-----END PUBLIC KEY-----\n\n"
+   },
+   "summary" : "please reply to my posts as direct messages if you have many followers",
+   "tag" : [],
+   "type" : "Person",
+   "url" : "https://princess.cat/users/mewmew"
+}
diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs
index 6ac883b23..43bd14ee6 100644
--- a/test/pleroma/web/activity_pub/activity_pub_test.exs
+++ b/test/pleroma/web/activity_pub/activity_pub_test.exs
@@ -2273,4 +2273,15 @@ test "`following` still contains self-replies by friends" do
       assert length(activities) == 2
     end
   end
+
+  test "allow fetching of accounts with an empty string name field" do
+    Tesla.Mock.mock(fn
+      %{method: :get, url: "https://princess.cat/users/mewmew"} ->
+        file = File.read!("test/fixtures/mewmew_no_name.json")
+        %Tesla.Env{status: 200, body: file}
+    end)
+
+    {:ok, user} = ActivityPub.make_user_from_ap_id("https://princess.cat/users/mewmew")
+    assert user.name == " "
+  end
 end

From a999e8b0b73372ed129c5a26acfe73e5f7478c5b Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 23 Oct 2020 13:55:08 +0200
Subject: [PATCH 071/104] Changelog: Add info about whitespace name remote
 users.

---
 CHANGELOG.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index afeaa930b..1f15a8b95 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -47,7 +47,8 @@ switched to a new configuration mechanism, however it was not officially removed
 
 - Add documented-but-missing chat pagination.
 - Allow sending out emails again.
-- Allow sending chat messages to yourself
+- Allow sending chat messages to yourself.
+- Fix remote users with a whitespace name.
 
 ## Unreleased (Patch)
 

From de6d49c8cec84a530f2835313c95064ae8df3604 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 26 Oct 2020 16:33:26 +0100
Subject: [PATCH 072/104] ActivityPub: Add back debug call + explanation.

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index df18db603..13869f897 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1378,6 +1378,11 @@ def fetch_and_prepare_user_from_ap_id(ap_id, opts \\ []) do
          {:ok, data} <- user_data_from_user_object(data) do
       {:ok, maybe_update_follow_information(data)}
     else
+      # If this has been deleted, only log a debug and not an error
+      {:error, "Object has been deleted" = e} ->
+        Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
+        {:error, e}
+
       {:error, {:reject, reason} = e} ->
         Logger.info("Rejected user #{ap_id}: #{inspect(reason)}")
         {:error, e}

From cbe41408e4f77da55d59ee3d4d26d002a1f20f02 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 27 Oct 2020 14:37:48 -0500
Subject: [PATCH 073/104] phoenix_controller_render_duration is no longer
 available in telemetry of Phoenix 1.5+

---
 test/pleroma/web/endpoint/metrics_exporter_test.exs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/pleroma/web/endpoint/metrics_exporter_test.exs b/test/pleroma/web/endpoint/metrics_exporter_test.exs
index f954cc1e7..875addc96 100644
--- a/test/pleroma/web/endpoint/metrics_exporter_test.exs
+++ b/test/pleroma/web/endpoint/metrics_exporter_test.exs
@@ -38,7 +38,6 @@ test "serves app metrics", %{conn: conn} do
       for metric <- [
             "http_requests_total",
             "http_request_duration_microseconds",
-            "phoenix_controller_render_duration",
             "phoenix_controller_call_duration",
             "telemetry_scrape_duration",
             "erlang_vm_memory_atom_bytes_total"

From d28f72a55af9442719ff01fe7052802c285f6ea8 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Tue, 27 Oct 2020 22:58:55 +0300
Subject: [PATCH 074/104] FrontStatic plug: excluded invalid url

---
 lib/pleroma/web/plugs/frontend_static.ex      | 26 +++++++++++--------
 .../web/plugs/frontend_static_plug_test.exs   | 21 +++++++++++++++
 2 files changed, 36 insertions(+), 11 deletions(-)

diff --git a/lib/pleroma/web/plugs/frontend_static.ex b/lib/pleroma/web/plugs/frontend_static.ex
index ceb10dcf8..1b0b36813 100644
--- a/lib/pleroma/web/plugs/frontend_static.ex
+++ b/lib/pleroma/web/plugs/frontend_static.ex
@@ -34,22 +34,26 @@ def init(opts) do
   end
 
   def call(conn, opts) do
-    frontend_type = Map.get(opts, :frontend_type, :primary)
-    path = file_path("", frontend_type)
-
-    if path do
-      conn
-      |> call_static(opts, path)
+    with false <- invalid_path?(conn.path_info),
+         frontend_type <- Map.get(opts, :frontend_type, :primary),
+         path when not is_nil(path) <- file_path("", frontend_type) do
+      call_static(conn, opts, path)
     else
-      conn
+      _ ->
+        conn
     end
   end
 
-  defp call_static(conn, opts, from) do
-    opts =
-      opts
-      |> Map.put(:from, from)
+  defp invalid_path?(list) do
+    invalid_path?(list, :binary.compile_pattern(["/", "\\", ":", "\0"]))
+  end
 
+  defp invalid_path?([h | _], _match) when h in [".", "..", ""], do: true
+  defp invalid_path?([h | t], match), do: String.contains?(h, match) or invalid_path?(t)
+  defp invalid_path?([], _match), do: false
+
+  defp call_static(conn, opts, from) do
+    opts = Map.put(opts, :from, from)
     Plug.Static.call(conn, opts)
   end
 end
diff --git a/test/pleroma/web/plugs/frontend_static_plug_test.exs b/test/pleroma/web/plugs/frontend_static_plug_test.exs
index f6f7d7bdb..8b7b022fc 100644
--- a/test/pleroma/web/plugs/frontend_static_plug_test.exs
+++ b/test/pleroma/web/plugs/frontend_static_plug_test.exs
@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.Plugs.FrontendStaticPlugTest do
   use Pleroma.Web.ConnCase
+  import Mock
 
   @dir "test/tmp/instance_static"
 
@@ -53,4 +54,24 @@ test "overrides existing static files for the `pleroma/admin` path", %{conn: con
     index = get(conn, "/pleroma/admin/")
     assert html_response(index, 200) == "from frontend plug"
   end
+
+  test "exclude invalid path", %{conn: conn} do
+    name = "pleroma-fe"
+    ref = "dist"
+    clear_config([:media_proxy, :enabled], true)
+    clear_config([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
+    clear_config([:frontends, :primary], %{"name" => name, "ref" => ref})
+    path = "#{@dir}/frontends/#{name}/#{ref}"
+
+    File.mkdir_p!("#{path}/proxy/rr/ss")
+    File.write!("#{path}/proxy/rr/ss/Ek7w8WPVcAApOvN.jpg:large", "FB image")
+
+    url =
+      Pleroma.Web.MediaProxy.encode_url("https://pbs.twimg.com/media/Ek7w8WPVcAApOvN.jpg:large")
+
+    with_mock Pleroma.ReverseProxy,
+      call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do
+      assert %Plug.Conn{status: :success} = get(conn, url)
+    end
+  end
 end

From 4f90077767b416f3469fe7c8acfaa6932c579ec2 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 28 Oct 2020 15:32:44 +0400
Subject: [PATCH 075/104] Fix warning

---
 test/pleroma/user/backup_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs
index 513798911..f68e4a029 100644
--- a/test/pleroma/user/backup_test.exs
+++ b/test/pleroma/user/backup_test.exs
@@ -82,7 +82,7 @@ test "it removes outdated backups after creating a fresh one" do
 
     assert {:ok, job1} = Backup.create(user)
 
-    assert {:ok, %Backup{id: backup1_id}} = ObanHelpers.perform(job1)
+    assert {:ok, %Backup{}} = ObanHelpers.perform(job1)
     assert {:ok, job2} = Backup.create(user)
     assert Pleroma.Repo.aggregate(Backup, :count) == 2
     assert {:ok, backup2} = ObanHelpers.perform(job2)

From da4a1e57b11d5600788b90f21d18bbcd97f6849f Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 28 Oct 2020 19:09:38 +0300
Subject: [PATCH 076/104] @doc fix.

---
 lib/pleroma/web/static_fe/static_fe_controller.ex | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex
index 76b82589f..bdec0897a 100644
--- a/lib/pleroma/web/static_fe/static_fe_controller.ex
+++ b/lib/pleroma/web/static_fe/static_fe_controller.ex
@@ -19,7 +19,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do
 
   @page_keys ["max_id", "min_id", "limit", "since_id", "order"]
 
-  @doc "Renders requested local public activity"
+  @doc "Renders requested local public activity or public activities of requested user"
   def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do
     with %Activity{local: true} = activity <-
            Activity.get_by_id_with_object(notice_id),
@@ -46,7 +46,6 @@ def show(%{assigns: %{notice_id: notice_id}} = conn, _params) do
     end
   end
 
-  @doc "Renders public activities of requested user"
   def show(%{assigns: %{username_or_id: username_or_id}} = conn, params) do
     with {_, %User{local: true} = user} <-
            {:fetch_user, User.get_cached_by_nickname_or_id(username_or_id)},

From 9f5f7dc9f956359204fc44a0627e20fd9765d8bd Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 28 Oct 2020 22:29:52 +0300
Subject: [PATCH 077/104] Fixed User.is_discoverable attribute rendering in
 Admin API User view.

---
 lib/pleroma/web/admin_api/views/account_view.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index bda7ea19c..8bac24d3e 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -52,7 +52,7 @@ def render("credentials.json", %{user: user, for: for_user}) do
       :skip_thread_containment,
       :pleroma_settings_store,
       :raw_fields,
-      :discoverable,
+      :is_discoverable,
       :actor_type
     ])
     |> Map.merge(%{

From 89c356d19fde3349a741522e7940a281487d9b08 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 29 Oct 2020 14:22:07 -0500
Subject: [PATCH 078/104] Improve Keyword descriptions for AdminFE

---
 config/description.exs | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/config/description.exs b/config/description.exs
index 0bfa9979f..798cbe2ad 100644
--- a/config/description.exs
+++ b/config/description.exs
@@ -1757,28 +1757,37 @@
     related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy",
     label: "MRF Keyword",
     type: :group,
-    description: "Reject or Word-Replace messages with a keyword or regex",
+    description:
+      "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).",
     children: [
       %{
         key: :reject,
         type: {:list, :string},
-        description:
-          "A list of patterns which result in message being rejected. Each pattern can be a string or a regular expression.",
+        description: """
+          A list of patterns which result in message being rejected.
+
+          Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+        """,
         suggestions: ["foo", ~r/foo/iu]
       },
       %{
         key: :federated_timeline_removal,
         type: {:list, :string},
-        description:
-          "A list of patterns which result in message being removed from federated timelines (a.k.a unlisted). Each pattern can be a string or a regular expression.",
+        description: """
+          A list of patterns which result in message being removed from federated timelines (a.k.a unlisted).
+
+          Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+        """,
         suggestions: ["foo", ~r/foo/iu]
       },
       %{
         key: :replace,
         type: {:list, :tuple},
-        description:
-          "A list of tuples containing {pattern, replacement}. Each pattern can be a string or a regular expression.",
-        suggestions: [{"foo", "bar"}, {~r/foo/iu, "bar"}]
+        description: """
+          **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+
+          **Replacement**: a string. Leaving the field empty is permitted.
+        """
       }
     ]
   },

From 241bd061fc60a5c90c172f46f3b4e576ba660aaf Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Fri, 16 Oct 2020 18:28:27 +0000
Subject: [PATCH 079/104] ConversationView: add current user to conversations,
 according to Mastodon behaviour

---
 lib/pleroma/web/mastodon_api/views/conversation_view.ex | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
index a91994915..cf34933ab 100644
--- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
@@ -33,12 +33,10 @@ def render("participation.json", %{participation: participation, for: user}) do
       end
 
     activity = Activity.get_by_id_with_object(last_activity_id)
-    # Conversations return all users except the current user.
-    users = Enum.reject(participation.recipients, &(&1.id == user.id))
 
     %{
       id: participation.id |> to_string(),
-      accounts: render(AccountView, "index.json", users: users, for: user),
+      accounts: render(AccountView, "index.json", users: participation.recipients, for: user),
       unread: !participation.read,
       last_status:
         render(StatusView, "show.json",

From 390a12d4c892e58e12546a78bc02dcc0e3a3484b Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Sun, 18 Oct 2020 15:58:06 +0000
Subject: [PATCH 080/104] ConversationControllerTest: fix test

---
 .../mastodon_api/controllers/conversation_controller_test.exs  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
index b23b22752..afc24027b 100644
--- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
@@ -54,7 +54,8 @@ test "returns correct conversations", %{
              ] = response
 
       account_ids = Enum.map(res_accounts, & &1["id"])
-      assert length(res_accounts) == 2
+      assert length(res_accounts) == 3
+      assert user_one.id in account_ids
       assert user_two.id in account_ids
       assert user_three.id in account_ids
       assert is_binary(res_id)

From 149589c842e677a082436db927834dd6f1b10cb5 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Sun, 18 Oct 2020 16:01:17 +0000
Subject: [PATCH 081/104] ConversationViewTest: fix test

---
 .../web/mastodon_api/views/conversation_view_test.exs       | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
index 2e8203c9b..bd58fb254 100644
--- a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
@@ -37,8 +37,10 @@ test "represents a Mastodon Conversation entity" do
     assert conversation.id == participation.id |> to_string()
     assert conversation.last_status.id == activity.id
 
-    assert [account] = conversation.accounts
-    assert account.id == other_user.id
+    account_ids = Enum.map(conversation.accounts, & &1["id"])
+    assert length(conversation.accounts) == 2
+    assert user.id in account_ids
+    assert other_user.id in account_ids
     assert conversation.last_status.pleroma.direct_conversation_id == participation.id
   end
 end

From 630eb0f939013db721c78e9b33e4e8bdc8232834 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Sun, 18 Oct 2020 19:12:42 +0000
Subject: [PATCH 082/104] ConversationViewTest: fix test #2

---
 test/pleroma/web/mastodon_api/views/conversation_view_test.exs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
index bd58fb254..81a471cb5 100644
--- a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
@@ -37,7 +37,7 @@ test "represents a Mastodon Conversation entity" do
     assert conversation.id == participation.id |> to_string()
     assert conversation.last_status.id == activity.id
 
-    account_ids = Enum.map(conversation.accounts, & &1["id"])
+    account_ids = Enum.map(conversation.accounts, & &1.id)
     assert length(conversation.accounts) == 2
     assert user.id in account_ids
     assert other_user.id in account_ids

From 9b93eef71550eabf55b9728b6c8925a4dede222d Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Fri, 30 Oct 2020 13:01:58 +0100
Subject: [PATCH 083/104] ConversationView: fix last_status.account being
 empty, fix current user being included in group conversations

---
 .../mastodon_api/views/conversation_view.ex   | 12 +++++++--
 .../conversation_controller_test.exs          | 25 +++++++++++++++++--
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
index cf34933ab..4636c00e3 100644
--- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
@@ -34,14 +34,22 @@ def render("participation.json", %{participation: participation, for: user}) do
 
     activity = Activity.get_by_id_with_object(last_activity_id)
 
+    # Conversations return all users except current user when current user is not only participant
+    users = if length(participation.recipients) > 1 do
+      Enum.reject(participation.recipients, &(&1.id == user.id))
+    else
+      participation.recipients
+    end
+
     %{
       id: participation.id |> to_string(),
-      accounts: render(AccountView, "index.json", users: participation.recipients, for: user),
+      accounts: render(AccountView, "index.json", users: users, for: user),
       unread: !participation.read,
       last_status:
         render(StatusView, "show.json",
           activity: activity,
-          direct_conversation_id: participation.id
+          direct_conversation_id: participation.id,
+          for: user
         )
     }
   end
diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
index afc24027b..8d07cff3f 100644
--- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
@@ -54,16 +54,37 @@ test "returns correct conversations", %{
              ] = response
 
       account_ids = Enum.map(res_accounts, & &1["id"])
-      assert length(res_accounts) == 3
-      assert user_one.id in account_ids
+      assert length(res_accounts) == 2
+      assert user_one.id not in account_ids
       assert user_two.id in account_ids
       assert user_three.id in account_ids
       assert is_binary(res_id)
       assert unread == false
       assert res_last_status["id"] == direct.id
+      assert res_last_status["account"]["id"] == user_one.id
       assert Participation.unread_count(user_one) == 0
     end
 
+    test "special behaviour when conversation have only one user", %{
+      user: user_one,
+      user_two: user_two,
+      conn: conn
+    } do
+      {:ok, direct} = create_direct_message(user_one, [])
+
+      res_conn = get(conn, "/api/v1/conversations")
+
+      assert response = json_response_and_validate_schema(res_conn, 200)
+      assert [
+               %{
+                 "accounts" => res_accounts,
+                 "last_status" => res_last_status
+               }
+             ] = response
+      assert length(res_accounts) == 1
+      assert res_accounts[0]["id"] == user_one.id
+    end
+
     test "observes limit params", %{
       user: user_one,
       user_two: user_two,

From 5591dc02486c30e4b80061706f7368d4b788b431 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Fri, 30 Oct 2020 13:07:01 +0100
Subject: [PATCH 084/104] Add entry in changelog

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 11820d313..c62d20868 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -56,6 +56,8 @@ switched to a new configuration mechanism, however it was not officially removed
 - Allow sending chat messages to yourself.
 - Fix remote users with a whitespace name.
 - OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting.
+- Mastodon API: Current user is now included in conversation if it's the only participant
+- Mastodon API: Fixed last_status.account being not filled with account data
 
 ## Unreleased (Patch)
 

From 0552a08dfd4daeca69abca0274bbd6db018e5edb Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Fri, 30 Oct 2020 13:10:19 +0100
Subject: [PATCH 085/104] ConversationControllerTest: fix test, fix formatting

---
 .../controllers/conversation_controller_test.exs             | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
index 8d07cff3f..291b6b295 100644
--- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
@@ -75,14 +75,17 @@ test "special behaviour when conversation have only one user", %{
       res_conn = get(conn, "/api/v1/conversations")
 
       assert response = json_response_and_validate_schema(res_conn, 200)
+
       assert [
                %{
                  "accounts" => res_accounts,
                  "last_status" => res_last_status
                }
              ] = response
+
+      account_ids = Enum.map(res_accounts, & &1["id"])
       assert length(res_accounts) == 1
-      assert res_accounts[0]["id"] == user_one.id
+      assert user_one.id in account_ids
     end
 
     test "observes limit params", %{

From d63ec02f31e5ee7bb278c4247a83900aceb9193a Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Fri, 30 Oct 2020 13:25:13 +0100
Subject: [PATCH 086/104] ConversationView: fix formatting

---
 .../web/mastodon_api/views/conversation_view.ex       | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
index 4636c00e3..545778165 100644
--- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
@@ -35,11 +35,12 @@ def render("participation.json", %{participation: participation, for: user}) do
     activity = Activity.get_by_id_with_object(last_activity_id)
 
     # Conversations return all users except current user when current user is not only participant
-    users = if length(participation.recipients) > 1 do
-      Enum.reject(participation.recipients, &(&1.id == user.id))
-    else
-      participation.recipients
-    end
+    users =
+      if length(participation.recipients) > 1 do
+        Enum.reject(participation.recipients, &(&1.id == user.id))
+      else
+        participation.recipients
+      end
 
     %{
       id: participation.id |> to_string(),

From 1042c30fa53e838f3acae2c176f47997fa425755 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Fri, 30 Oct 2020 13:37:15 +0100
Subject: [PATCH 087/104] ConversationViewTest: fix test

---
 .../pleroma/web/mastodon_api/views/conversation_view_test.exs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
index 81a471cb5..cd02158f9 100644
--- a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
@@ -36,10 +36,10 @@ test "represents a Mastodon Conversation entity" do
 
     assert conversation.id == participation.id |> to_string()
     assert conversation.last_status.id == activity.id
+    assert conversation.last_status.account.id == user.id
 
     account_ids = Enum.map(conversation.accounts, & &1.id)
-    assert length(conversation.accounts) == 2
-    assert user.id in account_ids
+    assert length(conversation.accounts) == 1
     assert other_user.id in account_ids
     assert conversation.last_status.pleroma.direct_conversation_id == participation.id
   end

From 1a98476f48db380b5d26b882dad7fba745b3f160 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 30 Oct 2020 16:39:14 +0400
Subject: [PATCH 088/104] Remove unused aliases

---
 .../web/admin_api/controllers/admin_api_controller_test.exs      | 1 -
 test/pleroma/web/admin_api/controllers/chat_controller_test.exs  | 1 -
 .../admin_api/controllers/instance_document_controller_test.exs  | 1 -
 .../web/admin_api/controllers/o_auth_app_controller_test.exs     | 1 -
 test/pleroma/web/admin_api/controllers/relay_controller_test.exs | 1 -
 .../pleroma/web/admin_api/controllers/report_controller_test.exs | 1 -
 .../pleroma/web/admin_api/controllers/status_controller_test.exs | 1 -
 test/pleroma/web/admin_api/controllers/user_controller_test.exs  | 1 -
 .../web/mastodon_api/controllers/timeline_controller_test.exs    | 1 -
 .../web/pleroma_api/controllers/user_import_controller_test.exs  | 1 -
 test/pleroma/web/plugs/http_security_plug_test.exs               | 1 -
 test/pleroma/web/twitter_api/remote_follow_controller_test.exs   | 1 -
 12 files changed, 12 deletions(-)

diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
index 34b26dddf..e2d6d5f6e 100644
--- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
@@ -11,7 +11,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
   import Swoosh.TestAssertions
 
   alias Pleroma.Activity
-  alias Pleroma.Config
   alias Pleroma.MFA
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
diff --git a/test/pleroma/web/admin_api/controllers/chat_controller_test.exs b/test/pleroma/web/admin_api/controllers/chat_controller_test.exs
index bd4c9c9d1..5aefa1e60 100644
--- a/test/pleroma/web/admin_api/controllers/chat_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/chat_controller_test.exs
@@ -9,7 +9,6 @@ defmodule Pleroma.Web.AdminAPI.ChatControllerTest do
 
   alias Pleroma.Chat
   alias Pleroma.Chat.MessageReference
-  alias Pleroma.Config
   alias Pleroma.ModerationLog
   alias Pleroma.Object
   alias Pleroma.Repo
diff --git a/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs b/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs
index 5f7b042f6..ce867dd0e 100644
--- a/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/instance_document_controller_test.exs
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.AdminAPI.InstanceDocumentControllerTest do
   use Pleroma.Web.ConnCase, async: true
   import Pleroma.Factory
-  alias Pleroma.Config
 
   @dir "test/tmp/instance_static"
   @default_instance_panel ~s(<p>Welcome to <a href="https://pleroma.social" target="_blank">Pleroma!</a></p>)
diff --git a/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs b/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs
index ed7c4172c..f388375d1 100644
--- a/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/o_auth_app_controller_test.exs
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.OAuthAppControllerTest do
 
   import Pleroma.Factory
 
-  alias Pleroma.Config
   alias Pleroma.Web
 
   setup do
diff --git a/test/pleroma/web/admin_api/controllers/relay_controller_test.exs b/test/pleroma/web/admin_api/controllers/relay_controller_test.exs
index adadf2b5c..b4c5e7567 100644
--- a/test/pleroma/web/admin_api/controllers/relay_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/relay_controller_test.exs
@@ -7,7 +7,6 @@ defmodule Pleroma.Web.AdminAPI.RelayControllerTest do
 
   import Pleroma.Factory
 
-  alias Pleroma.Config
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
   alias Pleroma.User
diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs
index 57946e6bb..fa746d6ea 100644
--- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do
   import Pleroma.Factory
 
   alias Pleroma.Activity
-  alias Pleroma.Config
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
   alias Pleroma.ReportNote
diff --git a/test/pleroma/web/admin_api/controllers/status_controller_test.exs b/test/pleroma/web/admin_api/controllers/status_controller_test.exs
index eff78fb0a..a18ef9e4b 100644
--- a/test/pleroma/web/admin_api/controllers/status_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/status_controller_test.exs
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.StatusControllerTest do
   import Pleroma.Factory
 
   alias Pleroma.Activity
-  alias Pleroma.Config
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
   alias Pleroma.User
diff --git a/test/pleroma/web/admin_api/controllers/user_controller_test.exs b/test/pleroma/web/admin_api/controllers/user_controller_test.exs
index da26caf25..5705306c7 100644
--- a/test/pleroma/web/admin_api/controllers/user_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/user_controller_test.exs
@@ -9,7 +9,6 @@ defmodule Pleroma.Web.AdminAPI.UserControllerTest do
   import Mock
   import Pleroma.Factory
 
-  alias Pleroma.Config
   alias Pleroma.HTML
   alias Pleroma.ModerationLog
   alias Pleroma.Repo
diff --git a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs
index c6e0268fd..9f1ee0424 100644
--- a/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/timeline_controller_test.exs
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do
   import Pleroma.Factory
   import Tesla.Mock
 
-  alias Pleroma.Config
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
 
diff --git a/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs
index 433c97e81..68723de71 100644
--- a/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/user_import_controller_test.exs
@@ -6,7 +6,6 @@ defmodule Pleroma.Web.PleromaAPI.UserImportControllerTest do
   use Pleroma.Web.ConnCase
   use Oban.Testing, repo: Pleroma.Repo
 
-  alias Pleroma.Config
   alias Pleroma.Tests.ObanHelpers
 
   import Pleroma.Factory
diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs
index 2297e3dac..df2b5ebb3 100644
--- a/test/pleroma/web/plugs/http_security_plug_test.exs
+++ b/test/pleroma/web/plugs/http_security_plug_test.exs
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
   use Pleroma.Web.ConnCase
 
-  alias Pleroma.Config
   alias Plug.Conn
 
   describe "http security enabled" do
diff --git a/test/pleroma/web/twitter_api/remote_follow_controller_test.exs b/test/pleroma/web/twitter_api/remote_follow_controller_test.exs
index 3852c7ce9..a3e784d13 100644
--- a/test/pleroma/web/twitter_api/remote_follow_controller_test.exs
+++ b/test/pleroma/web/twitter_api/remote_follow_controller_test.exs
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.TwitterAPI.RemoteFollowControllerTest do
   use Pleroma.Web.ConnCase
 
-  alias Pleroma.Config
   alias Pleroma.MFA
   alias Pleroma.MFA.TOTP
   alias Pleroma.User

From d1698267a27bd5084916f5f6f36d66b1ff2ffc5f Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Sat, 31 Oct 2020 00:26:11 +0400
Subject: [PATCH 089/104] Fix credo warning

---
 lib/pleroma/web/router.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 9592d0f38..efe67ad7a 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -148,7 +148,7 @@ defmodule Pleroma.Web.Router do
 
   scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do
     pipe_through(:admin_api)
-    
+
     put("/users/disable_mfa", AdminAPIController, :disable_mfa)
     put("/users/tag", AdminAPIController, :tag_users)
     delete("/users/tag", AdminAPIController, :untag_users)

From 8e41baff40555ef7c74c8842d6fbfebc2368631a Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Sat, 31 Oct 2020 05:50:48 +0300
Subject: [PATCH 090/104] Add idempotency_key to the chat_message entity.

---
 CHANGELOG.md                                          |  1 +
 docs/API/chats.md                                     |  5 ++++-
 lib/pleroma/application.ex                            |  9 ++++++++-
 lib/pleroma/web/activity_pub/side_effects.ex          |  6 ++++++
 lib/pleroma/web/common_api.ex                         |  3 ++-
 .../web/pleroma_api/controllers/chat_controller.ex    | 10 +++++++++-
 .../pleroma_api/views/chat/message_reference_view.ex  | 11 +++++++++++
 .../pleroma_api/controllers/chat_controller_test.exs  |  2 ++
 .../views/chat_message_reference_view_test.exs        |  5 ++++-
 test/pleroma/web/streamer_test.exs                    |  4 +++-
 10 files changed, 50 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 11820d313..bb02d7b32 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -38,6 +38,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Pleroma API: Pagination for remote/local packs and emoji.
 - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status
 - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type`
+- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
 
 </details>
 
diff --git a/docs/API/chats.md b/docs/API/chats.md
index aa6119670..9857aac67 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -173,11 +173,14 @@ Returned data:
     "created_at": "2020-04-21T15:06:45.000Z",
     "emojis": [],
     "id": "12",
-    "unread": false
+    "unread": false,
+    "idempotency_key": "75442486-0874-440c-9db1-a7006c25a31f"
   }
 ]
 ```
 
+- idempotency_key: The copy of the `idempotency-key` HTTP request header that can be used for optimistic message sending. Included only during the first few minutes after the message creation.
+
 ### Posting a chat message
 
 Posting a chat message for given Chat id works like this:
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 51e9dda3b..7c4cd9626 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -168,7 +168,11 @@ defp cachex_children do
       build_cachex("web_resp", limit: 2500),
       build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
       build_cachex("failed_proxy_url", limit: 2500),
-      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
+      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
+      build_cachex("chat_message_id_idempotency_key",
+        expiration: chat_message_id_idempotency_key_expiration(),
+        limit: 500_000
+      )
     ]
   end
 
@@ -178,6 +182,9 @@ defp emoji_packs_expiration,
   defp idempotency_expiration,
     do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
 
+  defp chat_message_id_idempotency_key_expiration,
+    do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60))
+
   defp seconds_valid_interval,
     do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
 
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 0fff5faf2..d552e91fc 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -312,6 +312,12 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
             {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
             {:ok, cm_ref} = MessageReference.create(chat, object, user.ap_id != actor.ap_id)
 
+            Cachex.put(
+              :chat_message_id_idempotency_key_cache,
+              cm_ref.id,
+              meta[:idempotency_key]
+            )
+
             {
               ["user", "user:pleroma_chat"],
               {user, %{cm_ref | chat: chat, object: object}}
diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex
index 60a50b027..318ffc5d0 100644
--- a/lib/pleroma/web/common_api.ex
+++ b/lib/pleroma/web/common_api.ex
@@ -45,7 +45,8 @@ def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ [])
          {_, {:ok, %Activity{} = activity, _meta}} <-
            {:common_pipeline,
             Pipeline.common_pipeline(create_activity_data,
-              local: true
+              local: true,
+              idempotency_key: opts[:idempotency_key]
             )} do
       {:ok, activity}
     else
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 6357148d0..2c4d3f135 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -80,7 +80,8 @@ def post_chat_message(
          %User{} = recipient <- User.get_cached_by_ap_id(chat.recipient),
          {:ok, activity} <-
            CommonAPI.post_chat_message(user, recipient, params[:content],
-             media_id: params[:media_id]
+             media_id: params[:media_id],
+             idempotency_key: idempotency_key(conn)
            ),
          message <- Object.normalize(activity, false),
          cm_ref <- MessageReference.for_chat_and_object(chat, message) do
@@ -169,4 +170,11 @@ def show(%{assigns: %{user: user}} = conn, %{id: id}) do
       |> render("show.json", chat: chat)
     end
   end
+
+  defp idempotency_key(conn) do
+    case get_req_header(conn, "idempotency-key") do
+      [key] -> key
+      _ -> nil
+    end
+  end
 end
diff --git a/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex
index d4e08b50d..c058fb340 100644
--- a/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do
   use Pleroma.Web, :view
 
+  alias Pleroma.Maps
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.StatusView
@@ -37,6 +38,7 @@ def render(
           Pleroma.Web.RichMedia.Helpers.fetch_data_for_object(object)
         )
     }
+    |> put_idempotency_key()
   end
 
   def render("index.json", opts) do
@@ -47,4 +49,13 @@ def render("index.json", opts) do
       Map.put(opts, :as, :chat_message_reference)
     )
   end
+
+  defp put_idempotency_key(data) do
+    with {:ok, idempotency_key} <- Cachex.get(:chat_message_id_idempotency_key_cache, data.id) do
+      data
+      |> Maps.put_if_present(:idempotency_key, idempotency_key)
+    else
+      _ -> data
+    end
+  end
 end
diff --git a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
index 6381f9757..fa6b9db65 100644
--- a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
@@ -82,11 +82,13 @@ test "it posts a message to the chat", %{conn: conn, user: user} do
       result =
         conn
         |> put_req_header("content-type", "application/json")
+        |> put_req_header("idempotency-key", "123")
         |> post("/api/v1/pleroma/chats/#{chat.id}/messages", %{"content" => "Hallo!!"})
         |> json_response_and_validate_schema(200)
 
       assert result["content"] == "Hallo!!"
       assert result["chat_id"] == chat.id |> to_string()
+      assert result["idempotency_key"] == "123"
     end
 
     test "it fails if there is no content", %{conn: conn, user: user} do
diff --git a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
index f171a1e55..ae8257870 100644
--- a/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
+++ b/test/pleroma/web/pleroma_api/views/chat_message_reference_view_test.exs
@@ -25,7 +25,9 @@ test "it displays a chat message" do
     }
 
     {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id)
-    {:ok, activity} = CommonAPI.post_chat_message(user, recipient, "kippis :firefox:")
+
+    {:ok, activity} =
+      CommonAPI.post_chat_message(user, recipient, "kippis :firefox:", idempotency_key: "123")
 
     chat = Chat.get(user.id, recipient.ap_id)
 
@@ -42,6 +44,7 @@ test "it displays a chat message" do
     assert chat_message[:created_at]
     assert chat_message[:unread] == false
     assert match?([%{shortcode: "firefox"}], chat_message[:emojis])
+    assert chat_message[:idempotency_key] == "123"
 
     clear_config([:rich_media, :enabled], true)
 
diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs
index 185724a9f..395016da2 100644
--- a/test/pleroma/web/streamer_test.exs
+++ b/test/pleroma/web/streamer_test.exs
@@ -255,7 +255,9 @@ test "it sends chat messages to the 'user:pleroma_chat' stream", %{
     } do
       other_user = insert(:user)
 
-      {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno")
+      {:ok, create_activity} =
+        CommonAPI.post_chat_message(other_user, user, "hey cirno", idempotency_key: "123")
+
       object = Object.normalize(create_activity, false)
       chat = Chat.get(user.id, other_user.ap_id)
       cm_ref = MessageReference.for_chat_and_object(chat, object)

From 8f00d90f9199e384fb1befb677c1c0595a0c854c Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis <vaartis@cock.li>
Date: Sun, 1 Nov 2020 12:05:39 +0300
Subject: [PATCH 091/104] Use Pleroma.HTTP instead of Tesla

Closes #2275

As discovered in the issue, captcha used Tesla.get instead of
Pleroma.HTTP. I've also grep'ed the repo and changed the other place
where this was used.
---
 lib/pleroma/captcha/kocaptcha.ex | 2 +-
 lib/pleroma/emoji/pack.ex        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/captcha/kocaptcha.ex b/lib/pleroma/captcha/kocaptcha.ex
index 337506647..201b55ab4 100644
--- a/lib/pleroma/captcha/kocaptcha.ex
+++ b/lib/pleroma/captcha/kocaptcha.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Captcha.Kocaptcha do
   def new do
     endpoint = Pleroma.Config.get!([__MODULE__, :endpoint])
 
-    case Tesla.get(endpoint <> "/new") do
+    case Pleroma.HTTP.get(endpoint <> "/new") do
       {:error, _} ->
         %{error: :kocaptcha_service_unavailable}
 
diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex
index 0670f29f1..ca58e5432 100644
--- a/lib/pleroma/emoji/pack.ex
+++ b/lib/pleroma/emoji/pack.ex
@@ -594,7 +594,7 @@ defp fetch_pack_info(remote_pack, uri, name) do
   end
 
   defp download_archive(url, sha) do
-    with {:ok, %{body: archive}} <- Tesla.get(url) do
+    with {:ok, %{body: archive}} <- Pleroma.HTTP.get(url) do
       if Base.decode16!(sha) == :crypto.hash(:sha256, archive) do
         {:ok, archive}
       else
@@ -617,7 +617,7 @@ defp fallback_sha_changed?(pack, data) do
   end
 
   defp update_sha_and_save_metadata(pack, data) do
-    with {:ok, %{body: zip}} <- Tesla.get(data[:"fallback-src"]),
+    with {:ok, %{body: zip}} <- Pleroma.HTTP.get(data[:"fallback-src"]),
          :ok <- validate_has_all_files(pack, zip) do
       fallback_sha = :sha256 |> :crypto.hash(zip) |> Base.encode16()
 

From 4caad4e9101c34debfa90d2e89850d4125a471b3 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 2 Nov 2020 05:43:06 +0100
Subject: [PATCH 092/104] =?UTF-8?q?side=5Feffects:=20Don=E2=80=99t=20incre?=
 =?UTF-8?q?ase=5Freplies=5Fcount=20when=20it=E2=80=99s=20an=20Answer?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 lib/pleroma/web/activity_pub/side_effects.ex                   | 2 +-
 .../web/activity_pub/transmogrifier/answer_handling_test.exs   | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 0fff5faf2..9b1171d07 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -187,7 +187,7 @@ def handle(%{data: %{"type" => "Create"}} = activity, meta) do
       {:ok, notifications} = Notification.create_notifications(activity, do_send: false)
       {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object)
 
-      if in_reply_to = object.data["inReplyTo"] do
+      if in_reply_to = object.data["inReplyTo"] && object.data["type"] != "Answer" do
         Object.increase_replies_count(in_reply_to)
       end
 
diff --git a/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs
index 0f6605c3f..e7d85a2c5 100644
--- a/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs
+++ b/test/pleroma/web/activity_pub/transmogrifier/answer_handling_test.exs
@@ -27,6 +27,7 @@ test "incoming, rewrites Note to Answer and increments vote counters" do
       })
 
     object = Object.normalize(activity)
+    assert object.data["repliesCount"] == nil
 
     data =
       File.read!("test/fixtures/mastodon-vote.json")
@@ -41,7 +42,7 @@ test "incoming, rewrites Note to Answer and increments vote counters" do
     assert answer_object.data["inReplyTo"] == object.data["id"]
 
     new_object = Object.get_by_ap_id(object.data["id"])
-    assert new_object.data["replies_count"] == object.data["replies_count"]
+    assert new_object.data["repliesCount"] == nil
 
     assert Enum.any?(
              new_object.data["oneOf"],

From be52819a112abb66032a56d613eed0233995eef4 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Mon, 2 Nov 2020 17:51:54 +0400
Subject: [PATCH 093/104] Hide chats from muted users

---
 .../controllers/chat_controller.ex            | 27 ++++++++-----------
 .../controllers/chat_controller_test.exs      | 22 +++++++++++++++
 2 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 2c4d3f135..8fc70c15a 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -15,7 +15,6 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
-  alias Pleroma.Web.PleromaAPI.ChatView
   alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   import Ecto.Query
@@ -121,9 +120,7 @@ def mark_as_read(
       ) do
     with {:ok, chat} <- Chat.get_by_user_and_id(user, id),
          {_n, _} <- MessageReference.set_all_seen_for_chat(chat, last_read_id) do
-      conn
-      |> put_view(ChatView)
-      |> render("show.json", chat: chat)
+      render(conn, "show.json", chat: chat)
     end
   end
 
@@ -142,32 +139,30 @@ def messages(%{assigns: %{user: user}} = conn, %{id: id} = params) do
   end
 
   def index(%{assigns: %{user: %{id: user_id} = user}} = conn, _params) do
-    blocked_ap_ids = User.blocked_users_ap_ids(user)
+    exclude_users =
+      user
+      |> User.blocked_users_ap_ids()
+      |> Enum.concat(User.muted_users_ap_ids(user))
 
     chats =
-      Chat.for_user_query(user_id)
-      |> where([c], c.recipient not in ^blocked_ap_ids)
+      user_id
+      |> Chat.for_user_query()
+      |> where([c], c.recipient not in ^exclude_users)
       |> Repo.all()
 
-    conn
-    |> put_view(ChatView)
-    |> render("index.json", chats: chats)
+    render(conn, "index.json", chats: chats)
   end
 
   def create(%{assigns: %{user: user}} = conn, %{id: id}) do
     with %User{ap_id: recipient} <- User.get_cached_by_id(id),
          {:ok, %Chat{} = chat} <- Chat.get_or_create(user.id, recipient) do
-      conn
-      |> put_view(ChatView)
-      |> render("show.json", chat: chat)
+      render(conn, "show.json", chat: chat)
     end
   end
 
   def show(%{assigns: %{user: user}} = conn, %{id: id}) do
     with {:ok, chat} <- Chat.get_by_user_and_id(user, id) do
-      conn
-      |> put_view(ChatView)
-      |> render("show.json", chat: chat)
+      render(conn, "show.json", chat: chat)
     end
   end
 
diff --git a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
index fa6b9db65..b0498df2b 100644
--- a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
@@ -343,6 +343,28 @@ test "it does not return chats with users you blocked", %{conn: conn, user: user
       assert length(result) == 0
     end
 
+    test "it does not return chats with users you muted", %{conn: conn, user: user} do
+      recipient = insert(:user)
+
+      {:ok, _} = Chat.get_or_create(user.id, recipient.ap_id)
+
+      result =
+        conn
+        |> get("/api/v1/pleroma/chats")
+        |> json_response_and_validate_schema(200)
+
+      assert length(result) == 1
+
+      User.mute(user, recipient)
+
+      result =
+        conn
+        |> get("/api/v1/pleroma/chats")
+        |> json_response_and_validate_schema(200)
+
+      assert length(result) == 0
+    end
+
     test "it returns all chats", %{conn: conn, user: user} do
       Enum.each(1..30, fn _ ->
         recipient = insert(:user)

From 7efc074eadae9b3d6d351e769ead0661f1f4c89c Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 2 Nov 2020 12:19:44 -0600
Subject: [PATCH 094/104] Permit fetching individual reports with notes
 preloaded

---
 lib/pleroma/activity.ex                             | 13 +++++++++++++
 .../web/admin_api/controllers/report_controller.ex  |  2 +-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 17af04257..553834da0 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -14,6 +14,7 @@ defmodule Pleroma.Activity do
   alias Pleroma.ReportNote
   alias Pleroma.ThreadMute
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
 
   import Ecto.Changeset
   import Ecto.Query
@@ -153,6 +154,18 @@ def get_bookmark(%Activity{} = activity, %User{} = user) do
 
   def get_bookmark(_, _), do: nil
 
+  def get_report(activity_id) do
+    opts = %{
+      type: "Flag",
+      skip_preload: true,
+      preload_report_notes: true
+    }
+
+    ActivityPub.fetch_activities_query([], opts)
+    |> where(id: ^activity_id)
+    |> Repo.one()
+  end
+
   def change(struct, params \\ %{}) do
     struct
     |> cast(params, [:data, :recipients])
diff --git a/lib/pleroma/web/admin_api/controllers/report_controller.ex b/lib/pleroma/web/admin_api/controllers/report_controller.ex
index 86da93893..6a0e56f5f 100644
--- a/lib/pleroma/web/admin_api/controllers/report_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/report_controller.ex
@@ -38,7 +38,7 @@ def index(conn, params) do
   end
 
   def show(conn, %{id: id}) do
-    with %Activity{} = report <- Activity.get_by_id(id) do
+    with %Activity{} = report <- Activity.get_report(id) do
       render(conn, "show.json", Report.extract_report_info(report))
     else
       _ -> {:error, :not_found}

From 53dd048590b93da67f9d4abac8cc111424c4d5c0 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 2 Nov 2020 15:49:07 -0600
Subject: [PATCH 095/104] Test the note is returned when fetching a single
 report

---
 .../web/admin_api/controllers/report_controller_test.exs | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs
index fa746d6ea..958e1d3ab 100644
--- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs
@@ -37,12 +37,21 @@ test "returns report by its id", %{conn: conn} do
           status_ids: [activity.id]
         })
 
+      conn
+      |> put_req_header("content-type", "application/json")
+      |> post("/api/pleroma/admin/reports/#{report_id}/notes", %{
+        content: "this is an admin note"
+      })
+
       response =
         conn
         |> get("/api/pleroma/admin/reports/#{report_id}")
         |> json_response_and_validate_schema(:ok)
 
       assert response["id"] == report_id
+
+      [notes] = response["notes"]
+      assert notes["content"] == "this is an admin note"
     end
 
     test "returns 404 when report id is invalid", %{conn: conn} do

From 2f2281fdf1bd3fbd5d82bb437ce3d43ff9043862 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 2 Nov 2020 17:09:56 -0600
Subject: [PATCH 096/104] Ensure URLs for git repos end in .git for older git
 clients like on CentOS 7

---
 mix.exs  | 4 ++--
 mix.lock | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/mix.exs b/mix.exs
index e0da696ce..0691902a6 100644
--- a/mix.exs
+++ b/mix.exs
@@ -134,7 +134,7 @@ defp deps do
       {:cachex, "~> 3.2"},
       {:poison, "~> 3.0", override: true},
       {:tesla,
-       git: "https://github.com/teamon/tesla/",
+       git: "https://github.com/teamon/tesla.git",
        ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30",
        override: true},
       {:castore, "~> 0.1"},
@@ -196,7 +196,7 @@ defp deps do
        ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"},
       {:restarter, path: "./restarter"},
       {:majic,
-       git: "https://git.pleroma.social/pleroma/elixir-libraries/majic", branch: "develop"},
+       git: "https://git.pleroma.social/pleroma/elixir-libraries/majic.git", branch: "develop"},
       {:open_api_spex,
        git: "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git",
        ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"},
diff --git a/mix.lock b/mix.lock
index 07238f550..e5d9bc693 100644
--- a/mix.lock
+++ b/mix.lock
@@ -66,7 +66,7 @@
   "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"},
   "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"},
   "linkify": {:hex, :linkify, "0.2.0", "2518bbbea21d2caa9d372424e1ad845b640c6630e2d016f1bd1f518f9ebcca28", [:mix], [], "hexpm", "b8ca8a68b79e30b7938d6c996085f3db14939f29538a59ca5101988bb7f917f6"},
-  "majic": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/majic", "4c692e544b28d1f5e543fb8a44be090f8cd96f80", [branch: "develop"]},
+  "majic": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/majic.git", "4c692e544b28d1f5e543fb8a44be090f8cd96f80", [branch: "develop"]},
   "makeup": {:hex, :makeup, "1.0.3", "e339e2f766d12e7260e6672dd4047405963c5ec99661abdc432e6ec67d29ef95", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "2e9b4996d11832947731f7608fed7ad2f9443011b3b479ae288011265cdd3dad"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.14.1", "4f0e96847c63c17841d42c08107405a005a2680eb9c7ccadfd757bd31dabccfb", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f2438b1a80eaec9ede832b5c41cd4f373b38fd7aa33e3b22d9db79e640cbde11"},
   "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"},
@@ -115,7 +115,7 @@
   "swoosh": {:hex, :swoosh, "1.0.6", "6765e334c67dacabe721f0d701c7e5a6f06e4595c90df6f91e73ebd54d555833", [:mix], [{:cowboy, "~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13 or ~> 1.0", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "7c50ef78e4acfd1cbd4907dc1fa87b5540675a6be9dc979d04890f49d7ec1830"},
   "syslog": {:hex, :syslog, "1.1.0", "6419a232bea84f07b56dc575225007ffe34d9fdc91abe6f1b2f254fd71d8efc2", [:rebar3], [], "hexpm", "4c6a41373c7e20587be33ef841d3de6f3beba08519809329ecc4d27b15b659e1"},
   "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"},
-  "tesla": {:git, "https://github.com/teamon/tesla/", "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", [ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30"]},
+  "tesla": {:git, "https://github.com/teamon/tesla.git", "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30", [ref: "9f7261ca49f9f901ceb73b60219ad6f8a9f6aa30"]},
   "timex": {:hex, :timex, "3.6.2", "845cdeb6119e2fef10751c0b247b6c59d86d78554c83f78db612e3290f819bc2", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "26030b46199d02a590be61c2394b37ea25a3664c02fafbeca0b24c972025d47a"},
   "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"},
   "tzdata": {:hex, :tzdata, "1.0.4", "a3baa4709ea8dba552dca165af6ae97c624a2d6ac14bd265165eaa8e8af94af6", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "b02637db3df1fd66dd2d3c4f194a81633d0e4b44308d36c1b2fdfd1e4e6f169b"},

From 179936609f4fbf51575fabd7af11cf14ba570c0c Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 2 Nov 2020 06:11:14 +0100
Subject: [PATCH 097/104] favicon: Update to pleroma logo, provided by @shpuld

Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/2270
---
 priv/static/favicon.png | Bin 1603 -> 1583 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

diff --git a/priv/static/favicon.png b/priv/static/favicon.png
index a96d5d25225abd92a95b35c58b301463752cf335..098040a00d025387fc376f534fb6cb5aef842761 100644
GIT binary patch
literal 1583
zcmds2`#TeQ9R6+=vrN&1qgYOg43(tl%qEv6*=#G4TS}C)b6ldcu}eZO;gEK6O_v8n
ziaD&Ba>>eNbsmbbxnFXZT*5f(=l+B9d7k(4KF|App67kupFTH9E{;gVF9-ktl5ot9
zEN9jaz!l}Y4tusqPOwm00u?SF2K;<H02_u0c6e%}_hb$+{$x0+ahgscPKx3WSvkd+
z9!U1!qG1MXV|>g(lWdQw4<~~(U8_<H{Lr<$D-`Z&J5tp#E`D2Z``cKELLF!8Kt-{`
z^xP|cKp*=0nqX1TGo4GH>RahGc^eu%ojoUup2~}eoBjMQ%%>|MWF>KNyn#-f%{7#Y
z#wvSD#nfgyQ9Z`A`jkp^d1PKer-25=`LPbmGU0TqVQU>(L!)pbI6O8KZ{zt)?~$=F
zNeN^nXk$&J*0%%BR;!^#Z<Xz;u)D>$t49{PV=O?3cqEP1JV>%Of~4wTLqD%xsgKzg
zbhIGOPi#SvC<u#czDd?z&}#UTbpvtUhsz=w2Zu)vUV$sC>x^&zE#hLENK3UGQxmdK
zUpP*AlK7>@Sw;oIMaGi2dcHxr>b5&&BDeebhF#`(d$!!**(_OAMKo@-Y|X<;zl3qF
zsf_K}{KpN?Agb=PqTa|Y+E}s)BWhWz<+H^<&O8(lmdkp_m%mdS`R8&_JWjaQlTsXl
zGN@@yjLNw-T_RcW&Q0}Mp7AHKQD8#3Wxkc|m45Hgxt2YtP*(3pnYeWFO=S5%`JV{D
zv1FN@NJ^nxc!78TUQdNhsO-Ukd^^DwIE%*Y3xfftL2EWxTF`WH{C~)8p0-lGBe3=r
zv}&EaU7KfZzhsD>Z<TO;=(K~pVv9j@vn)i3x~@@wmJ4t<?-G^*czHyT=Q@&Fv!Q>l
z^N*XIjG3Vc#?AM~4MI3ZFtj1j{7s6P_p4c1OV{YKsU@rF+4e_bbHmC_rDmulx`l}7
z!n9tO)T;0I`NmJRJpfI$ZHO0Dt>^dnEDt=r{de`52q(`uy{BR31ruU26i^v!9RYXn
zPDL}ZQujqXR%=1&`nr<|eD?{>zR>_c=75+Rt?JsD^KMECq~gm<9%)OVo#!(evv3bm
zOP2ENtqG;Fl(+hzQbjfV*I_34WqS6qo<_d`dS`VGH+x}7NW3ISas(6H)e4uvShG$G
zW|SJV%v(*zrYX(T@g<}EcyyZ$Rv!{5{e7Ri?yAu2>R_kfh%`C6*6k_=n(9ySZ}`e>
zq(OAOO7?Dn6rwX5snFh!?Fe%<7Q*kEvjO<Ym_3ZkjH2Mp5{q0#`yv&~0*m~JFr4e<
z#D{Q}y&H<I4-)*&VC(^gU(m4yLPB-<89{xUZ%<h{ep=7wdtsG8!mwLwE->Q7(j7%!
z#pOT*!i&2&qFumbZ&B-3WgJeze_F%-H|QRpjs_6T*g71^h#Ij8hI@DHu|J%3G9Y7F
zA2`{bF*ypJSlw^agpkz5m$ae2?b;6oaMoWRPBm=?k6R7r?LkINrAI7q4|gasgXZt7
zXx|TeU2N_Yr3MV{m1OP!r*d4!5`m#|Fi!<iHkL&8f#C>&KSxl$HAE>Ov+jQs_e<!3
z<65Y=;IAsc;F2l6G7ZvRO+#-k<n=M9LwjC-x_hT!wn!8(;>1v0L}GT3$H`d!%s}lX
z8;Lh0IkX3dCXLsIh?(n&o!`~79Nu3hEU5P9#aSj1a&5L@)&$SH+XU<!*j&~cb73&W
z*ip8^=kk=pkWqHegU@#}k=xoCt~M8fg|CEKi^wEYSxZwj%{NPuIFYlW=z!=L9_qF%
zYnifN$eoCtCPd8y^`=uVuJu&WeCKov!#|7)*y{?y^p5PX3_L{s0RX|?#je!Wm-8RQ
CN89cI

literal 1603
zcmV-J2E6%+P)<h;3K|Lk000e1NJLTq001Tc001%w1ONa4nsZ^O000IGNkl<Zcmb`!
z3(ydP0SEB^*Z1*Nx}>M3rg3eNl1HVk(c>Z|vaX`mV@f0+I+dn}9<xLbX_D$A^>wE;
ztwfg|tVJYaxe1%cj7~P@+Og5aZ|CblDYV<~mm?u&nW0_&H=?q_loJD-?8mk^AxBCS
zX>OZSl0-CD<Sxq$vRE|MDc_B_$U`1?g;FDuM4aO(g<4zXr=p?-=H%OmLK~c-h9VE^
zZ@3moBHCMGrI{%whH05^A}V=Eg+$CW+i^x3m~x_qhN88_i5ROxzKIx_h$=3(T3^w_
zyYA3gVakcR$~4u-#(X=`UU%yhn`(~d+^@Md)+tCi(aBQl6pLchot`g=y4g>Z8sTH>
z6EQ{;T}=0?X)5FZqS(c1S?HMTA&RY3IpxHadU)5F3KeN=oQXDiUE3T)lsU#@&a*_*
z>>*0LW~%<K)?F1HJ*HwJ>buP;##*bo%{hp;LrW)FYqGlelIUoSxsG+I*pMWmjY*ER
zz-EKImV=0)R%qi9r7GmBL~S>UX-@K#sHBcghB?=r{%M)Y`5MvDSeN8bL|x~bVtXRC
znXRo}%KQo~73gf7!RDK+UdoB%Ra4us>f}q}Y*F0+)eN)JG|lXAy<#!UUo7#F5!M*w
zT94~xRFa69Hk;~M?_>`#%3Pxp(Zx3XEHcDRs;Xm>yF4$dxlyYm5e070OV=b31;*K;
zb`mkbJ*GQ3<-}nBu}&|;Q%=kiRc!D?_7Y`wIyw=9ZE<40;6z&u%~yz}#+vF@Rg*-#
zE4q5!2P!0@$T&OA(jpO^lzP$}%_3@Bs(ZddbhFjninVd7C3g9dKYPI@)l^nXXYIYA
zt==goPP0;3z92T4=jiMu8lg-yRBD>FF4MxN`Z(RC)~Kb#W{q5%B;qs^P0BuEvMSk2
z^zgO|^-RPAK5?Ia`I9r8sfSCvAZqF0vP2B`l1e!o(c3b$y)IfAZGx+m_*_(UzpVy|
z-&>+pA_{XL(I)$eZpJtcGu)=C4&KuZTXa`5NklDFO~m!OC^1F%B%->BZpwb5w#)VK
zoKcP!*IH;i8i-m>P)T)f>z;@jN)>ufQ*)zGYm2PT0YnQkZL(Ec7pmt0zxJ^Ox;x82
zy=~VkNkn5a)b>}gJhphzts3P3qOJbcSmiv;6Y-W+c*RB=T#~&+ffXLM!zkCqP@_Dk
zK@K2#d(mnG%+c2zW8LBd^vprT(eAZTmn7m`TiuZZh;vQ$sEZw?pWj%m#GAG$%Qq7Z
zy{uHs(KY2nGru(7EOp%O6<Y+Kc{T?SHB3yzWX-dO_`XJ#8)l#qQx%C)eXSCAdeSnn
z8=vO@;%svhG2Ha*BSu<jwG-60$didU)^ah&)8>jV>{2NaH4RG=(cBc(6u8pF>>(Ps
zOcT_zPFr*k_01Ngwwf*$+ijQcC8DxVd}zM&#d^PRvuCo8*l4R>{w<oh*mlE2Q}a|%
z=2PqS^R&-!d?FU9pn|wr1yQIl`-rLvL=$6OsGp&BnjmW1VYM}0_LisJj!zRY+sWd0
zHffi`5j7oWuEn++pNN-qH3Dl5aRnA8VyS8VX1*bM<?uv-1(tZ<@I=fM({Pb{mHOO>
zL@d_RZDOYW`8rW#d?Jo<ohN+cnncV~!$$9E;t$U8j=qVQXqKx)pB#!9p=TnhTHqcp
zxhfGe#ZFXmyQV(YGZDAA!NWN;QS5;v5f!bl!VaSo@kcGOOH5YRZdWJbZqvP*LlKqz
z%uU8yrk$8(p(%EY{yLk631Xm{{_aiD*frReLlH$<J4+{<R1w7%c+dpBP0`Ip)Dp!W
z@k2vgZJ=)6wm63;N^}<O#SJbs+M_yKjGrWlIL|%0i2MA*PCv-ui2l}?ZoOMI61SLd
zIo{VPdx;XP^SlrB%i)RfCO9Yii3a%^QO{91GNO9^UqnX_Iw=to5>cc;E6vq6Us;ZX
zXk&3ADx2p5QDlvdMrvb_pXW%3J|-q&j0ILEV!A?OG}Y3ee0SnZj~Of)dd5(9DG@83
zp`8mXbb5}2I8}Xh{nm?Oi5c2C%6w0G!=xO>e*siY3nVrpr!xQm002ovPDHLkV1irG
B2owMS


From c37118e6f26f0305d540047e4ccb8d594d2c0e6b Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 3 Nov 2020 13:56:12 +0100
Subject: [PATCH 098/104] Conversations: A few refactors

---
 .../web/mastodon_api/views/conversation_view.ex      |  3 ++-
 .../controllers/conversation_controller_test.exs     | 12 ++++--------
 .../mastodon_api/views/conversation_view_test.exs    |  6 +++---
 3 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
index 545778165..82fcff062 100644
--- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
@@ -34,7 +34,8 @@ def render("participation.json", %{participation: participation, for: user}) do
 
     activity = Activity.get_by_id_with_object(last_activity_id)
 
-    # Conversations return all users except current user when current user is not only participant
+    # Conversations return all users except the current user,
+    # except when the current user is the only participant
     users =
       if length(participation.recipients) > 1 do
         Enum.reject(participation.recipients, &(&1.id == user.id))
diff --git a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
index 291b6b295..c67e584dd 100644
--- a/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
+++ b/test/pleroma/web/mastodon_api/controllers/conversation_controller_test.exs
@@ -65,12 +65,11 @@ test "returns correct conversations", %{
       assert Participation.unread_count(user_one) == 0
     end
 
-    test "special behaviour when conversation have only one user", %{
+    test "includes the user if the user is the only participant", %{
       user: user_one,
-      user_two: user_two,
       conn: conn
     } do
-      {:ok, direct} = create_direct_message(user_one, [])
+      {:ok, _direct} = create_direct_message(user_one, [])
 
       res_conn = get(conn, "/api/v1/conversations")
 
@@ -78,14 +77,11 @@ test "special behaviour when conversation have only one user", %{
 
       assert [
                %{
-                 "accounts" => res_accounts,
-                 "last_status" => res_last_status
+                 "accounts" => [account]
                }
              ] = response
 
-      account_ids = Enum.map(res_accounts, & &1["id"])
-      assert length(res_accounts) == 1
-      assert user_one.id in account_ids
+      assert user_one.id == account["id"]
     end
 
     test "observes limit params", %{
diff --git a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
index cd02158f9..20c10ba3d 100644
--- a/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/conversation_view_test.exs
@@ -38,9 +38,9 @@ test "represents a Mastodon Conversation entity" do
     assert conversation.last_status.id == activity.id
     assert conversation.last_status.account.id == user.id
 
-    account_ids = Enum.map(conversation.accounts, & &1.id)
-    assert length(conversation.accounts) == 1
-    assert other_user.id in account_ids
+    assert [account] = conversation.accounts
+    assert account.id == other_user.id
+
     assert conversation.last_status.pleroma.direct_conversation_id == participation.id
   end
 end

From 1cfc3278c086c9eaa7b2d1bd170e82c8b2aebd78 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 4 Nov 2020 10:14:00 +0100
Subject: [PATCH 099/104] Poll View: Always return `voters_count`.

---
 lib/pleroma/web/mastodon_api/views/poll_view.ex        | 2 +-
 test/pleroma/web/mastodon_api/views/poll_view_test.exs | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex
index 1208dc9a0..4101f21d0 100644
--- a/lib/pleroma/web/mastodon_api/views/poll_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex
@@ -19,7 +19,7 @@ def render("show.json", %{object: object, multiple: multiple, options: options}
       expired: expired,
       multiple: multiple,
       votes_count: votes_count,
-      voters_count: (multiple || nil) && voters_count(object),
+      voters_count: voters_count(object),
       options: options,
       voted: voted?(params),
       emojis: Pleroma.Web.MastodonAPI.StatusView.build_emojis(object.data["emoji"])
diff --git a/test/pleroma/web/mastodon_api/views/poll_view_test.exs b/test/pleroma/web/mastodon_api/views/poll_view_test.exs
index b7e2f17ef..c655ca438 100644
--- a/test/pleroma/web/mastodon_api/views/poll_view_test.exs
+++ b/test/pleroma/web/mastodon_api/views/poll_view_test.exs
@@ -44,7 +44,7 @@ test "renders a poll" do
       ],
       voted: false,
       votes_count: 0,
-      voters_count: nil
+      voters_count: 0
     }
 
     result = PollView.render("show.json", %{object: object})

From f09bb814a96c71f24fcf6e403a25e90be9cc684e Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 4 Nov 2020 10:14:48 +0100
Subject: [PATCH 100/104] Changelog: Add info about poll view changes

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2ee17d239..8c5a9f9dc 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Users with the `discoverable` field set to false will not show up in searches.
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
 - Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`.
+- Polls now always return a `voters_count`, even if they are single-choice
 
 <details>
   <summary>API Changes</summary>

From 92d252f364ed421f2afcdd135507ced3554eb3f0 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 4 Nov 2020 10:20:09 +0100
Subject: [PATCH 101/104] Poll Schema: Update and fix.

---
 lib/pleroma/web/api_spec/schemas/poll.ex | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/lib/pleroma/web/api_spec/schemas/poll.ex b/lib/pleroma/web/api_spec/schemas/poll.ex
index c62096db0..0dfa60b97 100644
--- a/lib/pleroma/web/api_spec/schemas/poll.ex
+++ b/lib/pleroma/web/api_spec/schemas/poll.ex
@@ -28,8 +28,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do
       },
       votes_count: %Schema{
         type: :integer,
-        nullable: true,
-        description: "How many votes have been received. Number, or null if `multiple` is false."
+        description: "How many votes have been received. Number."
+      },
+      voters_count: %Schema{
+        type: :integer,
+        description: "How many unique accounts have voted. Number."
       },
       voted: %Schema{
         type: :boolean,
@@ -61,7 +64,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do
       expired: true,
       multiple: false,
       votes_count: 10,
-      voters_count: nil,
+      voters_count: 10,
       voted: true,
       own_votes: [
         1

From ca95cbe0b48b6c64e6e33addf79e4d212d5f9872 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 4 Nov 2020 16:40:12 +0400
Subject: [PATCH 102/104] Add `with_muted` param to ChatController.index/2

---
 docs/API/chats.md                                          | 4 ++++
 lib/pleroma/web/api_spec/operations/chat_operation.ex      | 6 +++++-
 lib/pleroma/web/api_spec/operations/timeline_operation.ex  | 2 +-
 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex | 7 +++----
 .../web/pleroma_api/controllers/chat_controller_test.exs   | 7 +++++++
 5 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/docs/API/chats.md b/docs/API/chats.md
index 9857aac67..f50144c86 100644
--- a/docs/API/chats.md
+++ b/docs/API/chats.md
@@ -116,6 +116,10 @@ The modified chat message
 This will return a list of chats that you have been involved in, sorted by their
 last update (so new chats will be at the top).
 
+Parameters:
+
+- with_muted: Include chats from muted users (boolean).
+
 Returned data:
 
 ```json
diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 0dcfdb354..560b81f17 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do
   alias OpenApiSpex.Operation
   alias OpenApiSpex.Schema
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
+  alias Pleroma.Web.ApiSpec.Schemas.BooleanLike
   alias Pleroma.Web.ApiSpec.Schemas.Chat
   alias Pleroma.Web.ApiSpec.Schemas.ChatMessage
 
@@ -132,7 +133,10 @@ def index_operation do
       tags: ["chat"],
       summary: "Get a list of chats that you participated in",
       operationId: "ChatController.index",
-      parameters: pagination_params(),
+      parameters: [
+        Operation.parameter(:with_muted, :query, BooleanLike, "Include chats from muted users")
+        | pagination_params()
+      ],
       responses: %{
         200 => Operation.response("The chats of the user", "application/json", chats_response())
       },
diff --git a/lib/pleroma/web/api_spec/operations/timeline_operation.ex b/lib/pleroma/web/api_spec/operations/timeline_operation.ex
index 8e19bace7..1b5ad796f 100644
--- a/lib/pleroma/web/api_spec/operations/timeline_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/timeline_operation.ex
@@ -159,7 +159,7 @@ defp local_param do
   end
 
   defp with_muted_param do
-    Operation.parameter(:with_muted, :query, BooleanLike, "Includeactivities by muted users")
+    Operation.parameter(:with_muted, :query, BooleanLike, "Include activities by muted users")
   end
 
   defp exclude_visibilities_param do
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 8fc70c15a..77564b342 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -138,11 +138,10 @@ def messages(%{assigns: %{user: user}} = conn, %{id: id} = params) do
     end
   end
 
-  def index(%{assigns: %{user: %{id: user_id} = user}} = conn, _params) do
+  def index(%{assigns: %{user: %{id: user_id} = user}} = conn, params) do
     exclude_users =
-      user
-      |> User.blocked_users_ap_ids()
-      |> Enum.concat(User.muted_users_ap_ids(user))
+      User.blocked_users_ap_ids(user) ++
+        if params[:with_muted], do: [], else: User.muted_users_ap_ids(user)
 
     chats =
       user_id
diff --git a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
index b0498df2b..c1e6a8cc5 100644
--- a/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
+++ b/test/pleroma/web/pleroma_api/controllers/chat_controller_test.exs
@@ -363,6 +363,13 @@ test "it does not return chats with users you muted", %{conn: conn, user: user}
         |> json_response_and_validate_schema(200)
 
       assert length(result) == 0
+
+      result =
+        conn
+        |> get("/api/v1/pleroma/chats?with_muted=true")
+        |> json_response_and_validate_schema(200)
+
+      assert length(result) == 1
     end
 
     test "it returns all chats", %{conn: conn, user: user} do

From cb3cd3a761d96a08b1b55f5b277795822aa7e1d7 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 4 Nov 2020 15:24:10 +0100
Subject: [PATCH 103/104] TopicsTest: Small addition.

---
 test/pleroma/activity/ir/topics_test.exs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/test/pleroma/activity/ir/topics_test.exs b/test/pleroma/activity/ir/topics_test.exs
index 2e5655334..5e5c2f8da 100644
--- a/test/pleroma/activity/ir/topics_test.exs
+++ b/test/pleroma/activity/ir/topics_test.exs
@@ -104,6 +104,13 @@ test "non-local action produces public:remote topic", %{activity: activity} do
 
       assert Enum.member?(topics, "public:remote:lain.com")
     end
+
+    test "local action doesn't produce public:remote topic", %{activity: activity} do
+      activity = %{activity | local: true, actor: "https://lain.com/users/lain"}
+      topics = Topics.get_activity_topics(activity)
+
+      refute Enum.member?(topics, "public:remote:lain.com")
+    end
   end
 
   describe "public visibility create events with attachments" do

From eb1e1e74945a55c37e4b0dc7cea1a7c926ff981a Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 4 Nov 2020 15:39:32 +0100
Subject: [PATCH 104/104] Changelog: Add info about federation status endpoint

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2ee17d239..8951b4523 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -40,6 +40,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status
 - Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type`
 - Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
+- Pleroma API: (`GET /api/v1/pleroma/federation_status`) Add a way to get a list of unreachable instances.
 
 </details>