From d4270397dcb2aebde8ed14fd89998ab57aaae545 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 22 Oct 2019 13:42:59 +0300 Subject: [PATCH 01/84] Marker: added unread_count field --- lib/pleroma/marker.ex | 5 +- .../web/mastodon_api/views/marker_view.ex | 1 + .../20191021113356_add_unread_to_marker.exs | 49 +++++++++++++++++++ .../controllers/marker_controller_test.exs | 7 ++- .../mastodon_api/views/marker_view_test.exs | 4 +- 5 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 priv/repo/migrations/20191021113356_add_unread_to_marker.exs diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index 7f87c86c3..c4d554980 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -18,6 +18,7 @@ defmodule Pleroma.Marker do field(:last_read_id, :string, default: "") field(:timeline, :string, default: "") field(:lock_version, :integer, default: 0) + field(:unread_count, :integer, default: 0) belongs_to(:user, User, type: FlakeId.Ecto.CompatType) timestamps() @@ -38,7 +39,7 @@ def upsert(%User{} = user, attrs) do Multi.insert(multi, timeline, marker, returning: true, - on_conflict: {:replace, [:last_read_id]}, + on_conflict: {:replace, [:last_read_id, :unread_count]}, conflict_target: [:user_id, :timeline] ) end) @@ -55,7 +56,7 @@ defp get_marker(user, timeline) do @doc false defp changeset(marker, attrs) do marker - |> cast(attrs, [:last_read_id]) + |> cast(attrs, [:last_read_id, :unread_count]) |> validate_required([:user_id, :timeline, :last_read_id]) |> validate_inclusion(:timeline, @timelines) end diff --git a/lib/pleroma/web/mastodon_api/views/marker_view.ex b/lib/pleroma/web/mastodon_api/views/marker_view.ex index 38fbeed5f..1501c2a30 100644 --- a/lib/pleroma/web/mastodon_api/views/marker_view.ex +++ b/lib/pleroma/web/mastodon_api/views/marker_view.ex @@ -10,6 +10,7 @@ def render("markers.json", %{markers: markers}) do Map.put_new(acc, m.timeline, %{ last_read_id: m.last_read_id, version: m.lock_version, + unread_count: m.unread_count, updated_at: NaiveDateTime.to_iso8601(m.updated_at) }) end) diff --git a/priv/repo/migrations/20191021113356_add_unread_to_marker.exs b/priv/repo/migrations/20191021113356_add_unread_to_marker.exs new file mode 100644 index 000000000..32789b7f9 --- /dev/null +++ b/priv/repo/migrations/20191021113356_add_unread_to_marker.exs @@ -0,0 +1,49 @@ +defmodule Pleroma.Repo.Migrations.AddUnreadToMarker do + use Ecto.Migration + import Ecto.Query + alias Pleroma.Repo + alias Pleroma.Notification + + def up do + alter table(:markers) do + add_if_not_exists(:unread_count, :integer, default: 0) + end + + flush() + + update_markers() + end + + def down do + alter table(:markers) do + remove_if_exists(:unread_count, :integer) + end + end + + def update_markers do + from(q in Notification, + select: %{ + timeline: "notifications", + user_id: q.user_id, + unread_count: fragment("COUNT(*) FILTER (WHERE seen = false) as unread_count"), + last_read_id: fragment("(MAX(id) FILTER (WHERE seen = true)::text) as last_read_id ") + }, + group_by: [q.user_id] + ) + |> Repo.all() + |> Enum.reduce(Ecto.Multi.new(), fn attrs, multi -> + marker = + Pleroma.Marker + |> struct(attrs) + |> Ecto.Changeset.change() + + multi + |> Ecto.Multi.insert(attrs[:user_id], marker, + returning: true, + on_conflict: {:replace, [:last_read_id, :unread_count]}, + conflict_target: [:user_id, :timeline] + ) + end) + |> Pleroma.Repo.transaction() + end +end diff --git a/test/web/mastodon_api/controllers/marker_controller_test.exs b/test/web/mastodon_api/controllers/marker_controller_test.exs index 1fcad873d..5e7b4001f 100644 --- a/test/web/mastodon_api/controllers/marker_controller_test.exs +++ b/test/web/mastodon_api/controllers/marker_controller_test.exs @@ -15,7 +15,7 @@ test "gets markers with correct scopes", %{conn: conn} do {:ok, %{"notifications" => marker}} = Pleroma.Marker.upsert( user, - %{"notifications" => %{"last_read_id" => "69420"}} + %{"notifications" => %{"last_read_id" => "69420", "unread_count" => 7}} ) response = @@ -28,6 +28,7 @@ test "gets markers with correct scopes", %{conn: conn} do assert response == %{ "notifications" => %{ "last_read_id" => "69420", + "unread_count" => 7, "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at), "version" => 0 } @@ -70,7 +71,8 @@ test "creates a marker with correct scopes", %{conn: conn} do "notifications" => %{ "last_read_id" => "69420", "updated_at" => _, - "version" => 0 + "version" => 0, + "unread_count" => 0 } } = response end @@ -98,6 +100,7 @@ test "updates exist marker", %{conn: conn} do assert response == %{ "notifications" => %{ "last_read_id" => "69888", + "unread_count" => 0, "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at), "version" => 0 } diff --git a/test/web/mastodon_api/views/marker_view_test.exs b/test/web/mastodon_api/views/marker_view_test.exs index 8a5c89d56..3ce794617 100644 --- a/test/web/mastodon_api/views/marker_view_test.exs +++ b/test/web/mastodon_api/views/marker_view_test.exs @@ -8,17 +8,19 @@ defmodule Pleroma.Web.MastodonAPI.MarkerViewTest do import Pleroma.Factory test "returns markers" do - marker1 = insert(:marker, timeline: "notifications", last_read_id: "17") + marker1 = insert(:marker, timeline: "notifications", last_read_id: "17", unread_count: 5) marker2 = insert(:marker, timeline: "home", last_read_id: "42") assert MarkerView.render("markers.json", %{markers: [marker1, marker2]}) == %{ "home" => %{ last_read_id: "42", + unread_count: 0, updated_at: NaiveDateTime.to_iso8601(marker2.updated_at), version: 0 }, "notifications" => %{ last_read_id: "17", + unread_count: 5, updated_at: NaiveDateTime.to_iso8601(marker1.updated_at), version: 0 } From 9a4afbd2a0486238bfaf4047d91376d32635514a Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 22 Oct 2019 16:13:22 +0300 Subject: [PATCH 02/84] added update unread_count for notifications --- lib/pleroma/marker.ex | 36 ++++++++++++++++++++++++++ lib/pleroma/notification.ex | 50 ++++++++++++++++++++++++------------- test/notification_test.exs | 7 ++++++ 3 files changed, 76 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index c4d554980..4b8198690 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Marker do alias Ecto.Multi alias Pleroma.Repo alias Pleroma.User + alias __MODULE__ @timelines ["notifications"] @@ -46,6 +47,41 @@ def upsert(%User{} = user, attrs) do |> Repo.transaction() end + @spec multi_set_unread_count(Multi.t(), User.t(), String.t()) :: Multi.t() + def multi_set_unread_count(multi, %User{} = user, "notifications") do + multi + |> Multi.run(:counters, fn _repo, _changes -> + query = + from(q in Pleroma.Notification, + where: q.user_id == ^user.id, + select: %{ + timeline: "notifications", + user_id: ^user.id, + unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END ) as unread_count") + } + ) + + {:ok, Repo.one(query)} + end) + |> Multi.insert( + :marker, + fn %{counters: attrs} -> + Marker + |> struct(attrs) + |> Ecto.Changeset.change() + end, + returning: true, + on_conflict: {:replace, [:last_read_id, :unread_count]}, + conflict_target: [:user_id, :timeline] + ) + end + + def set_unread_count(%User{} = user, timeline) do + Multi.new() + |> multi_set_unread_count(user, timeline) + |> Repo.transaction() + end + defp get_marker(user, timeline) do case Repo.find_resource(get_query(user, timeline)) do {:ok, marker} -> %__MODULE__{marker | user: user} diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index e5da1492b..d339fdf64 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -5,7 +5,9 @@ defmodule Pleroma.Notification do use Ecto.Schema + alias Ecto.Multi alias Pleroma.Activity + alias Pleroma.Marker alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Pagination @@ -151,25 +153,23 @@ def for_user_since(user, date) do |> Repo.all() end - def set_read_up_to(%{id: user_id} = _user, id) do + def set_read_up_to(%{id: user_id} = user, id) do query = from( n in Notification, where: n.user_id == ^user_id, where: n.id <= ^id, where: n.seen == false, - update: [ - set: [ - seen: true, - updated_at: ^NaiveDateTime.utc_now() - ] - ], # Ideally we would preload object and activities here # but Ecto does not support preloads in update_all select: n.id ) - {_, notification_ids} = Repo.update_all(query, []) + {:ok, %{ids: {_, notification_ids}}} = + Multi.new() + |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) + |> Marker.multi_set_unread_count(user, "notifications") + |> Repo.transaction() Notification |> where([n], n.id in ^notification_ids) @@ -186,11 +186,18 @@ def set_read_up_to(%{id: user_id} = _user, id) do |> Repo.all() end + @spec read_one(User.t(), String.t()) :: + {:ok, Notification.t()} | {:error, Ecto.Changeset.t()} | nil def read_one(%User{} = user, notification_id) do with {:ok, %Notification{} = notification} <- get(user, notification_id) do - notification - |> changeset(%{seen: true}) - |> Repo.update() + Multi.new() + |> Multi.update(:update, changeset(notification, %{seen: true})) + |> Marker.multi_set_unread_count(user, "notifications") + |> Repo.transaction() + |> case do + {:ok, %{update: notification}} -> {:ok, notification} + {:error, :update, changeset, _} -> {:error, changeset} + end end end @@ -243,8 +250,11 @@ def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = act object = Object.normalize(activity) unless object && object.data["type"] == "Answer" do - users = get_notified_from_activity(activity) - notifications = Enum.map(users, fn user -> create_notification(activity, user) end) + notifications = + activity + |> get_notified_from_activity() + |> Enum.map(&create_notification(activity, &1)) + {:ok, notifications} else {:ok, []} @@ -253,8 +263,11 @@ def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = act def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity) when type in ["Like", "Announce", "Follow"] do - users = get_notified_from_activity(activity) - notifications = Enum.map(users, fn user -> create_notification(activity, user) end) + notifications = + activity + |> get_notified_from_activity + |> Enum.map(&create_notification(activity, &1)) + {:ok, notifications} end @@ -263,8 +276,11 @@ def create_notifications(_), do: {:ok, []} # TODO move to sql, too. def create_notification(%Activity{} = activity, %User{} = user) do unless skip?(activity, user) do - notification = %Notification{user_id: user.id, activity: activity} - {:ok, notification} = Repo.insert(notification) + {:ok, %{notification: notification}} = + Multi.new() + |> Multi.insert(:notification, %Notification{user_id: user.id, activity: activity}) + |> Marker.multi_set_unread_count(user, "notifications") + |> Repo.transaction() ["user", "user:notification"] |> Streamer.stream(notification) diff --git a/test/notification_test.exs b/test/notification_test.exs index 96316f8dd..558ac358c 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -310,6 +310,13 @@ test "it sets all notifications as read up to a specified notification ID" do assert n1.seen == true assert n2.seen == true assert n3.seen == false + + assert %Pleroma.Marker{unread_count: 1} = + Pleroma.Repo.get_by( + Pleroma.Marker, + user_id: other_user.id, + timeline: "notifications" + ) end end From aa64b3108ba6aa4294e541e86da323ba1e1a7243 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 23 Oct 2019 11:54:52 +0300 Subject: [PATCH 03/84] fix migrate --- lib/pleroma/marker.ex | 5 +++-- .../20191021113356_add_unread_to_marker.exs | 18 +++++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index 4b8198690..098fe3bbd 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -56,8 +56,9 @@ def multi_set_unread_count(multi, %User{} = user, "notifications") do where: q.user_id == ^user.id, select: %{ timeline: "notifications", - user_id: ^user.id, - unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END ) as unread_count") + user_id: type(^user.id, :string), + unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), + last_read_id: type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) } ) diff --git a/priv/repo/migrations/20191021113356_add_unread_to_marker.exs b/priv/repo/migrations/20191021113356_add_unread_to_marker.exs index 32789b7f9..964c7fb98 100644 --- a/priv/repo/migrations/20191021113356_add_unread_to_marker.exs +++ b/priv/repo/migrations/20191021113356_add_unread_to_marker.exs @@ -25,25 +25,21 @@ def update_markers do select: %{ timeline: "notifications", user_id: q.user_id, - unread_count: fragment("COUNT(*) FILTER (WHERE seen = false) as unread_count"), - last_read_id: fragment("(MAX(id) FILTER (WHERE seen = true)::text) as last_read_id ") + unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), + last_read_id: type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) }, group_by: [q.user_id] ) |> Repo.all() - |> Enum.reduce(Ecto.Multi.new(), fn attrs, multi -> - marker = - Pleroma.Marker - |> struct(attrs) - |> Ecto.Changeset.change() - - multi - |> Ecto.Multi.insert(attrs[:user_id], marker, + |> Enum.each(fn attrs -> + Pleroma.Marker + |> struct(attrs) + |> Ecto.Changeset.change() + |> Pleroma.Repo.insert( returning: true, on_conflict: {:replace, [:last_read_id, :unread_count]}, conflict_target: [:user_id, :timeline] ) end) - |> Pleroma.Repo.transaction() end end From d3fb9e02cc0ce7dc462e587e639e117aaef5fbc5 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 23 Oct 2019 22:48:04 +0300 Subject: [PATCH 04/84] add tests --- lib/pleroma/marker.ex | 9 +++------ .../20191021113356_add_unread_to_marker.exs | 3 ++- test/marker_test.exs | 15 +++++++++++++++ test/notification_test.exs | 3 +++ 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index 098fe3bbd..5f6a47f38 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -58,7 +58,8 @@ def multi_set_unread_count(multi, %User{} = user, "notifications") do timeline: "notifications", user_id: type(^user.id, :string), unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), - last_read_id: type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) + last_read_id: + type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) } ) @@ -77,11 +78,7 @@ def multi_set_unread_count(multi, %User{} = user, "notifications") do ) end - def set_unread_count(%User{} = user, timeline) do - Multi.new() - |> multi_set_unread_count(user, timeline) - |> Repo.transaction() - end + def multi_set_unread_count(multi, _, _), do: multi defp get_marker(user, timeline) do case Repo.find_resource(get_query(user, timeline)) do diff --git a/priv/repo/migrations/20191021113356_add_unread_to_marker.exs b/priv/repo/migrations/20191021113356_add_unread_to_marker.exs index 964c7fb98..c15e2ff13 100644 --- a/priv/repo/migrations/20191021113356_add_unread_to_marker.exs +++ b/priv/repo/migrations/20191021113356_add_unread_to_marker.exs @@ -26,7 +26,8 @@ def update_markers do timeline: "notifications", user_id: q.user_id, unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), - last_read_id: type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) + last_read_id: + type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) }, group_by: [q.user_id] ) diff --git a/test/marker_test.exs b/test/marker_test.exs index 04bd67fe6..1900ed08b 100644 --- a/test/marker_test.exs +++ b/test/marker_test.exs @@ -8,6 +8,21 @@ defmodule Pleroma.MarkerTest do import Pleroma.Factory + describe "multi_set_unread_count/3" do + test "returns multi" do + user = insert(:user) + + assert %Ecto.Multi{ + operations: [marker: {:run, _}, counters: {:run, _}] + } = + Marker.multi_set_unread_count( + Ecto.Multi.new(), + user, + "notifications" + ) + end + end + describe "get_markers/2" do test "returns user markers" do user = insert(:user) diff --git a/test/notification_test.exs b/test/notification_test.exs index 558ac358c..1e8a9ca98 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -31,6 +31,9 @@ test "notifies someone when they are directly addressed" do assert notified_ids == [other_user.id, third_user.id] assert notification.activity_id == activity.id assert other_notification.activity_id == activity.id + + assert [%Pleroma.Marker{unread_count: 2}] = + Pleroma.Marker.get_markers(other_user, ["notifications"]) end test "it creates a notification for subscribed users" do From 922e3d082c38ccd108710e21d4bda8e65b551f9c Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 24 Oct 2019 09:50:41 +0300 Subject: [PATCH 05/84] add test --- lib/pleroma/marker.ex | 14 +------------- lib/pleroma/notification.ex | 14 ++++++++++++++ test/marker_test.exs | 6 ++++++ 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index 5f6a47f38..a7ea542dd 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -51,19 +51,7 @@ def upsert(%User{} = user, attrs) do def multi_set_unread_count(multi, %User{} = user, "notifications") do multi |> Multi.run(:counters, fn _repo, _changes -> - query = - from(q in Pleroma.Notification, - where: q.user_id == ^user.id, - select: %{ - timeline: "notifications", - user_id: type(^user.id, :string), - unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), - last_read_id: - type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) - } - ) - - {:ok, Repo.one(query)} + {:ok, Repo.one(Pleroma.Notification.notifications_info_query(user))} end) |> Multi.insert( :marker, diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index af56cc667..373f9b06a 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -36,6 +36,20 @@ def changeset(%Notification{} = notification, attrs) do |> cast(attrs, [:seen]) end + @spec notifications_info_query(User.t()) :: Ecto.Queryable.t() + def notifications_info_query(user) do + from(q in Pleroma.Notification, + where: q.user_id == ^user.id, + select: %{ + timeline: "notifications", + user_id: type(^user.id, :string), + unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), + last_read_id: + type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) + } + ) + end + def for_user_query(user, opts \\ []) do Notification |> where(user_id: ^user.id) diff --git a/test/marker_test.exs b/test/marker_test.exs index 1900ed08b..5d03db48e 100644 --- a/test/marker_test.exs +++ b/test/marker_test.exs @@ -21,6 +21,12 @@ test "returns multi" do "notifications" ) end + + test "return empty multi" do + user = insert(:user) + multi = Ecto.Multi.new() + assert Marker.multi_set_unread_count(multi, user, "home") == multi + end end describe "get_markers/2" do From 1b82eb6d4102bc2d7acec0a905e7714c95eadc94 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 30 Oct 2019 23:22:38 +0300 Subject: [PATCH 06/84] move sql (update_markers) from migrate to mix task --- lib/mix/tasks/pleroma/marker.ex | 36 +++++++++++++++ .../20191021113356_add_unread_to_marker.exs | 46 ------------------- .../20191030202008_add_unread_to_marker.exs | 18 ++++++++ 3 files changed, 54 insertions(+), 46 deletions(-) create mode 100644 lib/mix/tasks/pleroma/marker.ex delete mode 100644 priv/repo/migrations/20191021113356_add_unread_to_marker.exs create mode 100644 priv/repo/migrations/20191030202008_add_unread_to_marker.exs diff --git a/lib/mix/tasks/pleroma/marker.ex b/lib/mix/tasks/pleroma/marker.ex new file mode 100644 index 000000000..1d5be11de --- /dev/null +++ b/lib/mix/tasks/pleroma/marker.ex @@ -0,0 +1,36 @@ +defmodule Mix.Tasks.Pleroma.Marker do + use Mix.Task + import Mix.Pleroma + + import Ecto.Query + alias Pleroma.Repo + alias Pleroma.Notification + + def run(["update_markers"]) do + start_pleroma() + + from(q in Notification, + select: %{ + timeline: "notifications", + user_id: q.user_id, + unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), + last_read_id: + type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) + }, + group_by: [q.user_id] + ) + |> Repo.all() + |> Enum.each(fn attrs -> + Pleroma.Marker + |> struct(attrs) + |> Ecto.Changeset.change() + |> Pleroma.Repo.insert( + returning: true, + on_conflict: {:replace, [:last_read_id, :unread_count]}, + conflict_target: [:user_id, :timeline] + ) + end) + + shell_info("Done") + end +end diff --git a/priv/repo/migrations/20191021113356_add_unread_to_marker.exs b/priv/repo/migrations/20191021113356_add_unread_to_marker.exs deleted file mode 100644 index c15e2ff13..000000000 --- a/priv/repo/migrations/20191021113356_add_unread_to_marker.exs +++ /dev/null @@ -1,46 +0,0 @@ -defmodule Pleroma.Repo.Migrations.AddUnreadToMarker do - use Ecto.Migration - import Ecto.Query - alias Pleroma.Repo - alias Pleroma.Notification - - def up do - alter table(:markers) do - add_if_not_exists(:unread_count, :integer, default: 0) - end - - flush() - - update_markers() - end - - def down do - alter table(:markers) do - remove_if_exists(:unread_count, :integer) - end - end - - def update_markers do - from(q in Notification, - select: %{ - timeline: "notifications", - user_id: q.user_id, - unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), - last_read_id: - type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) - }, - group_by: [q.user_id] - ) - |> Repo.all() - |> Enum.each(fn attrs -> - Pleroma.Marker - |> struct(attrs) - |> Ecto.Changeset.change() - |> Pleroma.Repo.insert( - returning: true, - on_conflict: {:replace, [:last_read_id, :unread_count]}, - conflict_target: [:user_id, :timeline] - ) - end) - end -end diff --git a/priv/repo/migrations/20191030202008_add_unread_to_marker.exs b/priv/repo/migrations/20191030202008_add_unread_to_marker.exs new file mode 100644 index 000000000..f81339c9f --- /dev/null +++ b/priv/repo/migrations/20191030202008_add_unread_to_marker.exs @@ -0,0 +1,18 @@ +defmodule Pleroma.Repo.Migrations.AddUnreadToMarker do + use Ecto.Migration + import Ecto.Query + alias Pleroma.Repo + alias Pleroma.Notification + + def up do + alter table(:markers) do + add_if_not_exists(:unread_count, :integer, default: 0) + end + end + + def down do + alter table(:markers) do + remove_if_exists(:unread_count, :integer) + end + end +end From 209319c8d289564653f73cbf15fb6449d91cf3ca Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 30 Oct 2019 23:49:05 +0300 Subject: [PATCH 07/84] update marker api --- .../web/mastodon_api/views/marker_view.ex | 6 ++++-- .../controllers/marker_controller_test.exs | 20 +++++++++---------- .../mastodon_api/views/marker_view_test.exs | 8 ++++---- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/marker_view.ex b/lib/pleroma/web/mastodon_api/views/marker_view.ex index 1501c2a30..81545cff0 100644 --- a/lib/pleroma/web/mastodon_api/views/marker_view.ex +++ b/lib/pleroma/web/mastodon_api/views/marker_view.ex @@ -10,8 +10,10 @@ def render("markers.json", %{markers: markers}) do Map.put_new(acc, m.timeline, %{ last_read_id: m.last_read_id, version: m.lock_version, - unread_count: m.unread_count, - updated_at: NaiveDateTime.to_iso8601(m.updated_at) + updated_at: NaiveDateTime.to_iso8601(m.updated_at), + pleroma: %{ + unread_count: m.unread_count + } }) end) end diff --git a/test/web/mastodon_api/controllers/marker_controller_test.exs b/test/web/mastodon_api/controllers/marker_controller_test.exs index 5e7b4001f..e0aacccb4 100644 --- a/test/web/mastodon_api/controllers/marker_controller_test.exs +++ b/test/web/mastodon_api/controllers/marker_controller_test.exs @@ -26,13 +26,13 @@ test "gets markers with correct scopes", %{conn: conn} do |> json_response(200) assert response == %{ - "notifications" => %{ - "last_read_id" => "69420", - "unread_count" => 7, - "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at), - "version" => 0 - } - } + "notifications" => %{ + "last_read_id" => "69420", + "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at), + "version" => 0, + "pleroma" => %{ "unread_count" => 7 } + } + } end test "gets markers with missed scopes", %{conn: conn} do @@ -72,7 +72,7 @@ test "creates a marker with correct scopes", %{conn: conn} do "last_read_id" => "69420", "updated_at" => _, "version" => 0, - "unread_count" => 0 + "pleroma" => %{ "unread_count" => 0 } } } = response end @@ -100,9 +100,9 @@ test "updates exist marker", %{conn: conn} do assert response == %{ "notifications" => %{ "last_read_id" => "69888", - "unread_count" => 0, "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at), - "version" => 0 + "version" => 0, + "pleroma" => %{ "unread_count" => 0 } } } end diff --git a/test/web/mastodon_api/views/marker_view_test.exs b/test/web/mastodon_api/views/marker_view_test.exs index 3ce794617..f172e5023 100644 --- a/test/web/mastodon_api/views/marker_view_test.exs +++ b/test/web/mastodon_api/views/marker_view_test.exs @@ -14,15 +14,15 @@ test "returns markers" do assert MarkerView.render("markers.json", %{markers: [marker1, marker2]}) == %{ "home" => %{ last_read_id: "42", - unread_count: 0, updated_at: NaiveDateTime.to_iso8601(marker2.updated_at), - version: 0 + version: 0, + pleroma: %{unread_count: 0} }, "notifications" => %{ last_read_id: "17", - unread_count: 5, updated_at: NaiveDateTime.to_iso8601(marker1.updated_at), - version: 0 + version: 0, + pleroma: %{unread_count: 5} } } end From 58da7f66202f3f2e90d4954649c3e5a0e3a7dc32 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 31 Oct 2019 17:34:17 +0300 Subject: [PATCH 08/84] updated docs\changelog --- CHANGELOG.md | 1 + docs/API/differences_in_mastoapi_responses.md | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51e5424c6..c14c8c0e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`) - Mastodon API: `pleroma.thread_muted` to the Status entity - Mastodon API: Mark the direct conversation as read for the author when they send a new direct message +- Mastodon API: Add `pleroma.unread_count` to the Marker entity ### Added diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index aca0f5e0e..d18b976b6 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -155,3 +155,9 @@ Has theses additionnal parameters (which are the same as in Pleroma-API): * `captcha_solution`: optional, contains provider-specific captcha solution, * `captcha_token`: optional, contains provider-specific captcha token * `token`: invite token required when the registerations aren't public. + +## Markers + +Has these additional fields under the `pleroma` object: + +- `unread_count`: contains number unread notifications From 1b3a942a84b8b612e07e3bf34801137741926911 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 31 Oct 2019 17:36:59 +0300 Subject: [PATCH 09/84] fix format --- lib/mix/tasks/pleroma/marker.ex | 10 +++++----- .../controllers/marker_controller_test.exs | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/mix/tasks/pleroma/marker.ex b/lib/mix/tasks/pleroma/marker.ex index 1d5be11de..bebef0d6a 100644 --- a/lib/mix/tasks/pleroma/marker.ex +++ b/lib/mix/tasks/pleroma/marker.ex @@ -1,10 +1,10 @@ defmodule Mix.Tasks.Pleroma.Marker do use Mix.Task import Mix.Pleroma - import Ecto.Query - alias Pleroma.Repo + alias Pleroma.Notification + alias Pleroma.Repo def run(["update_markers"]) do start_pleroma() @@ -15,7 +15,7 @@ def run(["update_markers"]) do user_id: q.user_id, unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), last_read_id: - type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) + type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) }, group_by: [q.user_id] ) @@ -26,8 +26,8 @@ def run(["update_markers"]) do |> Ecto.Changeset.change() |> Pleroma.Repo.insert( returning: true, - on_conflict: {:replace, [:last_read_id, :unread_count]}, - conflict_target: [:user_id, :timeline] + on_conflict: {:replace, [:last_read_id, :unread_count]}, + conflict_target: [:user_id, :timeline] ) end) diff --git a/test/web/mastodon_api/controllers/marker_controller_test.exs b/test/web/mastodon_api/controllers/marker_controller_test.exs index e0aacccb4..8bcfcb7e1 100644 --- a/test/web/mastodon_api/controllers/marker_controller_test.exs +++ b/test/web/mastodon_api/controllers/marker_controller_test.exs @@ -26,13 +26,13 @@ test "gets markers with correct scopes", %{conn: conn} do |> json_response(200) assert response == %{ - "notifications" => %{ - "last_read_id" => "69420", - "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at), - "version" => 0, - "pleroma" => %{ "unread_count" => 7 } - } - } + "notifications" => %{ + "last_read_id" => "69420", + "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at), + "version" => 0, + "pleroma" => %{"unread_count" => 7} + } + } end test "gets markers with missed scopes", %{conn: conn} do @@ -72,7 +72,7 @@ test "creates a marker with correct scopes", %{conn: conn} do "last_read_id" => "69420", "updated_at" => _, "version" => 0, - "pleroma" => %{ "unread_count" => 0 } + "pleroma" => %{"unread_count" => 0} } } = response end @@ -102,7 +102,7 @@ test "updates exist marker", %{conn: conn} do "last_read_id" => "69888", "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at), "version" => 0, - "pleroma" => %{ "unread_count" => 0 } + "pleroma" => %{"unread_count" => 0} } } end From 57995fa8cf26c9d5cd31969b59dbafb9f8c8fdc7 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Sat, 2 Nov 2019 21:19:01 +0300 Subject: [PATCH 10/84] fix migrate update migrate --- lib/mix/tasks/pleroma/marker.ex | 36 ------------------- .../20191030202008_add_unread_to_marker.exs | 32 ++++++++++++++++- 2 files changed, 31 insertions(+), 37 deletions(-) delete mode 100644 lib/mix/tasks/pleroma/marker.ex diff --git a/lib/mix/tasks/pleroma/marker.ex b/lib/mix/tasks/pleroma/marker.ex deleted file mode 100644 index bebef0d6a..000000000 --- a/lib/mix/tasks/pleroma/marker.ex +++ /dev/null @@ -1,36 +0,0 @@ -defmodule Mix.Tasks.Pleroma.Marker do - use Mix.Task - import Mix.Pleroma - import Ecto.Query - - alias Pleroma.Notification - alias Pleroma.Repo - - def run(["update_markers"]) do - start_pleroma() - - from(q in Notification, - select: %{ - timeline: "notifications", - user_id: q.user_id, - unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), - last_read_id: - type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) - }, - group_by: [q.user_id] - ) - |> Repo.all() - |> Enum.each(fn attrs -> - Pleroma.Marker - |> struct(attrs) - |> Ecto.Changeset.change() - |> Pleroma.Repo.insert( - returning: true, - on_conflict: {:replace, [:last_read_id, :unread_count]}, - conflict_target: [:user_id, :timeline] - ) - end) - - shell_info("Done") - end -end diff --git a/priv/repo/migrations/20191030202008_add_unread_to_marker.exs b/priv/repo/migrations/20191030202008_add_unread_to_marker.exs index f81339c9f..2b3abc682 100644 --- a/priv/repo/migrations/20191030202008_add_unread_to_marker.exs +++ b/priv/repo/migrations/20191030202008_add_unread_to_marker.exs @@ -2,12 +2,15 @@ defmodule Pleroma.Repo.Migrations.AddUnreadToMarker do use Ecto.Migration import Ecto.Query alias Pleroma.Repo - alias Pleroma.Notification def up do alter table(:markers) do add_if_not_exists(:unread_count, :integer, default: 0) end + + flush() + + update_markers() end def down do @@ -15,4 +18,31 @@ def down do remove_if_exists(:unread_count, :integer) end end + + def update_markers do + now = NaiveDateTime.utc_now() + + markers_attrs = + from(q in "notifications", + select: %{ + timeline: "notifications", + user_id: q.user_id, + unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), + last_read_id: + type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) + }, + group_by: [q.user_id] + ) + |> Repo.all() + |> Enum.map(fn attrs -> + attrs + |> Map.put_new(:inserted_at, now) + |> Map.put_new(:updated_at, now) + end) + + Repo.insert_all("markers", markers_attrs, + on_conflict: {:replace, [:last_read_id, :unread_count]}, + conflict_target: [:user_id, :timeline] + ) + end end From ddbfc995ac40db9bd1da137b03e5acf6d050ddc5 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 11 Nov 2019 17:06:41 +0300 Subject: [PATCH 11/84] clean sql query --- lib/pleroma/marker.ex | 2 +- lib/pleroma/notification.ex | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index a7ea542dd..d5ca27bf2 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -56,7 +56,7 @@ def multi_set_unread_count(multi, %User{} = user, "notifications") do |> Multi.insert( :marker, fn %{counters: attrs} -> - Marker + %Marker{timeline: "notifications", user_id: user.id} |> struct(attrs) |> Ecto.Changeset.change() end, diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 373f9b06a..158903c4b 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -41,8 +41,6 @@ def notifications_info_query(user) do from(q in Pleroma.Notification, where: q.user_id == ^user.id, select: %{ - timeline: "notifications", - user_id: type(^user.id, :string), unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), last_read_id: type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) From b5b62f42b2864dc8b95c8ba7d650321ebcc332ad Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 12 Nov 2019 15:59:34 +0300 Subject: [PATCH 12/84] update Marker.multi_set_unread_count --- lib/pleroma/marker.ex | 7 ++++++- lib/pleroma/notification.ex | 21 ++++++++++++++------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index d5ca27bf2..a32546094 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Marker do import Ecto.Query alias Ecto.Multi + alias Pleroma.Notification alias Pleroma.Repo alias Pleroma.User alias __MODULE__ @@ -51,7 +52,11 @@ def upsert(%User{} = user, attrs) do def multi_set_unread_count(multi, %User{} = user, "notifications") do multi |> Multi.run(:counters, fn _repo, _changes -> - {:ok, Repo.one(Pleroma.Notification.notifications_info_query(user))} + {:ok, + %{ + unread_count: Repo.aggregate(Notification.unread_count_query(user), :count, :id), + last_read_id: Repo.one(Notification.last_read_query(user)) + }} end) |> Multi.insert( :marker, diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 158903c4b..1cc6a4735 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -36,15 +36,22 @@ def changeset(%Notification{} = notification, attrs) do |> cast(attrs, [:seen]) end - @spec notifications_info_query(User.t()) :: Ecto.Queryable.t() - def notifications_info_query(user) do + @spec unread_count_query(User.t()) :: Ecto.Queryable.t() + def unread_count_query(user) do from(q in Pleroma.Notification, where: q.user_id == ^user.id, - select: %{ - unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), - last_read_id: - type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) - } + where: q.seen == false + ) + end + + @spec last_read_query(User.t()) :: Ecto.Queryable.t() + def last_read_query(user) do + from(q in Pleroma.Notification, + where: q.user_id == ^user.id, + where: q.seen == true, + select: type(q.id, :string), + limit: 1, + order_by: [desc: :id] ) end From b9041c209787dc279d4dc5194d65dff73684cdb9 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 15 Nov 2019 22:10:41 +0300 Subject: [PATCH 13/84] added recount unread notifications to markers --- lib/pleroma/marker.ex | 29 +++++++++++++++++-- .../controllers/marker_controller.ex | 8 ++++- test/marker_test.exs | 14 +++++++++ .../controllers/marker_controller_test.exs | 3 +- 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index a32546094..2d217a0b7 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Marker do alias __MODULE__ @timelines ["notifications"] + @type t :: %__MODULE__{} schema "markers" do field(:last_read_id, :string, default: "") @@ -26,8 +27,18 @@ defmodule Pleroma.Marker do timestamps() end - def get_markers(user, timelines \\ []) do - Repo.all(get_query(user, timelines)) + @doc """ + Gets markers by user and timeline. + + opts: + `recount_unread` - run force recount unread notifications for `true` value + """ + @spec get_markers(User.t(), list(String), map()) :: list(t()) + def get_markers(user, timelines \\ [], opts \\ %{}) do + user + |> get_query(timelines) + |> recount_unread_notifications(opts[:recount_unread]) + |> Repo.all() end def upsert(%User{} = user, attrs) do @@ -99,4 +110,18 @@ defp get_query(user, timelines) do |> by_user_id(user.id) |> by_timeline(timelines) end + + defp recount_unread_notifications(query, true) do + from( + q in query, + left_join: n in "notifications", + on: n.user_id == q.user_id and n.seen == false, + group_by: [:id], + select_merge: %{ + unread_count: fragment("count(?)", n.id) + } + ) + end + + defp recount_unread_notifications(query, _), do: query end diff --git a/lib/pleroma/web/mastodon_api/controllers/marker_controller.ex b/lib/pleroma/web/mastodon_api/controllers/marker_controller.ex index ce025624d..6649ffbda 100644 --- a/lib/pleroma/web/mastodon_api/controllers/marker_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/marker_controller.ex @@ -18,7 +18,13 @@ defmodule Pleroma.Web.MastodonAPI.MarkerController do # GET /api/v1/markers def index(%{assigns: %{user: user}} = conn, params) do - markers = Pleroma.Marker.get_markers(user, params["timeline"]) + markers = + Pleroma.Marker.get_markers( + user, + params["timeline"], + %{recount_unread: true} + ) + render(conn, "markers.json", %{markers: markers}) end diff --git a/test/marker_test.exs b/test/marker_test.exs index 5d03db48e..7b1d2218a 100644 --- a/test/marker_test.exs +++ b/test/marker_test.exs @@ -36,6 +36,20 @@ test "returns user markers" do insert(:marker, timeline: "home", user: user) assert Marker.get_markers(user, ["notifications"]) == [refresh_record(marker)] end + + test "returns user markers with recount unread notifications" do + user = insert(:user) + marker = insert(:marker, user: user) + insert(:notification, user: user) + insert(:notification, user: user) + insert(:marker, timeline: "home", user: user) + + assert Marker.get_markers( + user, + ["notifications"], + %{recount_unread: true} + ) == [%Marker{refresh_record(marker) | unread_count: 2}] + end end describe "upsert/2" do diff --git a/test/web/mastodon_api/controllers/marker_controller_test.exs b/test/web/mastodon_api/controllers/marker_controller_test.exs index 8bcfcb7e1..64bf79bb1 100644 --- a/test/web/mastodon_api/controllers/marker_controller_test.exs +++ b/test/web/mastodon_api/controllers/marker_controller_test.exs @@ -11,11 +11,12 @@ defmodule Pleroma.Web.MastodonAPI.MarkerControllerTest do test "gets markers with correct scopes", %{conn: conn} do user = insert(:user) token = insert(:oauth_token, user: user, scopes: ["read:statuses"]) + insert_list(7, :notification, user: user) {:ok, %{"notifications" => marker}} = Pleroma.Marker.upsert( user, - %{"notifications" => %{"last_read_id" => "69420", "unread_count" => 7}} + %{"notifications" => %{"last_read_id" => "69420"}} ) response = From cd040691bd28fea1437b8f1c39bb914465e1ff46 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 10 Feb 2020 09:01:45 +0300 Subject: [PATCH 14/84] maked `unread_count` as virtual field --- lib/pleroma/marker.ex | 31 ++- lib/pleroma/notification.ex | 14 +- .../controllers/marker_controller.ex | 8 +- mix.lock | 192 +++++++++--------- ....exs => 20200210050658_update_markers.exs} | 15 +- test/marker_test.exs | 14 +- test/notification_test.exs | 5 +- 7 files changed, 123 insertions(+), 156 deletions(-) rename priv/repo/migrations/{20191030202008_add_unread_to_marker.exs => 20200210050658_update_markers.exs} (68%) diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index 2d217a0b7..dab97d8b6 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Marker do field(:last_read_id, :string, default: "") field(:timeline, :string, default: "") field(:lock_version, :integer, default: 0) - field(:unread_count, :integer, default: 0) + field(:unread_count, :integer, default: 0, virtual: true) belongs_to(:user, User, type: FlakeId.Ecto.CompatType) timestamps() @@ -33,14 +33,15 @@ defmodule Pleroma.Marker do opts: `recount_unread` - run force recount unread notifications for `true` value """ - @spec get_markers(User.t(), list(String), map()) :: list(t()) - def get_markers(user, timelines \\ [], opts \\ %{}) do + @spec get_markers(User.t(), list(String)) :: list(t()) + def get_markers(user, timelines \\ []) do user |> get_query(timelines) - |> recount_unread_notifications(opts[:recount_unread]) + |> unread_count_query() |> Repo.all() end + @spec upsert(User.t(), map()) :: {:ok | :error, any()} def upsert(%User{} = user, attrs) do attrs |> Map.take(@timelines) @@ -52,22 +53,18 @@ def upsert(%User{} = user, attrs) do Multi.insert(multi, timeline, marker, returning: true, - on_conflict: {:replace, [:last_read_id, :unread_count]}, + on_conflict: {:replace, [:last_read_id]}, conflict_target: [:user_id, :timeline] ) end) |> Repo.transaction() end - @spec multi_set_unread_count(Multi.t(), User.t(), String.t()) :: Multi.t() - def multi_set_unread_count(multi, %User{} = user, "notifications") do + @spec multi_set_last_read_id(Multi.t(), User.t(), String.t()) :: Multi.t() + def multi_set_last_read_id(multi, %User{} = user, "notifications") do multi |> Multi.run(:counters, fn _repo, _changes -> - {:ok, - %{ - unread_count: Repo.aggregate(Notification.unread_count_query(user), :count, :id), - last_read_id: Repo.one(Notification.last_read_query(user)) - }} + {:ok, %{last_read_id: Repo.one(Notification.last_read_query(user))}} end) |> Multi.insert( :marker, @@ -77,12 +74,12 @@ def multi_set_unread_count(multi, %User{} = user, "notifications") do |> Ecto.Changeset.change() end, returning: true, - on_conflict: {:replace, [:last_read_id, :unread_count]}, + on_conflict: {:replace, [:last_read_id]}, conflict_target: [:user_id, :timeline] ) end - def multi_set_unread_count(multi, _, _), do: multi + def multi_set_last_read_id(multi, _, _), do: multi defp get_marker(user, timeline) do case Repo.find_resource(get_query(user, timeline)) do @@ -94,7 +91,7 @@ defp get_marker(user, timeline) do @doc false defp changeset(marker, attrs) do marker - |> cast(attrs, [:last_read_id, :unread_count]) + |> cast(attrs, [:last_read_id]) |> validate_required([:user_id, :timeline, :last_read_id]) |> validate_inclusion(:timeline, @timelines) end @@ -111,7 +108,7 @@ defp get_query(user, timelines) do |> by_timeline(timelines) end - defp recount_unread_notifications(query, true) do + defp unread_count_query(query) do from( q in query, left_join: n in "notifications", @@ -122,6 +119,4 @@ defp recount_unread_notifications(query, true) do } ) end - - defp recount_unread_notifications(query, _), do: query end diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 2e4fe2edb..70fd97bfa 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -38,14 +38,6 @@ def changeset(%Notification{} = notification, attrs) do |> cast(attrs, [:seen]) end - @spec unread_count_query(User.t()) :: Ecto.Queryable.t() - def unread_count_query(user) do - from(q in Pleroma.Notification, - where: q.user_id == ^user.id, - where: q.seen == false - ) - end - @spec last_read_query(User.t()) :: Ecto.Queryable.t() def last_read_query(user) do from(q in Pleroma.Notification, @@ -229,7 +221,7 @@ def set_read_up_to(%{id: user_id} = user, id) do {:ok, %{ids: {_, notification_ids}}} = Multi.new() |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) - |> Marker.multi_set_unread_count(user, "notifications") + |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() Notification @@ -253,7 +245,7 @@ def read_one(%User{} = user, notification_id) do with {:ok, %Notification{} = notification} <- get(user, notification_id) do Multi.new() |> Multi.update(:update, changeset(notification, %{seen: true})) - |> Marker.multi_set_unread_count(user, "notifications") + |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() |> case do {:ok, %{update: notification}} -> {:ok, notification} @@ -340,7 +332,7 @@ def create_notification(%Activity{} = activity, %User{} = user) do {:ok, %{notification: notification}} = Multi.new() |> Multi.insert(:notification, %Notification{user_id: user.id, activity: activity}) - |> Marker.multi_set_unread_count(user, "notifications") + |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() ["user", "user:notification"] diff --git a/lib/pleroma/web/mastodon_api/controllers/marker_controller.ex b/lib/pleroma/web/mastodon_api/controllers/marker_controller.ex index 6649ffbda..ce025624d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/marker_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/marker_controller.ex @@ -18,13 +18,7 @@ defmodule Pleroma.Web.MastodonAPI.MarkerController do # GET /api/v1/markers def index(%{assigns: %{user: user}} = conn, params) do - markers = - Pleroma.Marker.get_markers( - user, - params["timeline"], - %{recount_unread: true} - ) - + markers = Pleroma.Marker.get_markers(user, params["timeline"]) render(conn, "markers.json", %{markers: markers}) end diff --git a/mix.lock b/mix.lock index b8a35a795..1893e7e41 100644 --- a/mix.lock +++ b/mix.lock @@ -1,112 +1,112 @@ %{ - "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"}, + "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"}, "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "95e8188490e97505c56636c1379ffdf036c1fdde", [ref: "95e8188490e97505c56636c1379ffdf036c1fdde"]}, - "base62": {:hex, :base62, "1.2.1", "4866763e08555a7b3917064e9eef9194c41667276c51b59de2bc42c6ea65f806", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm"}, - "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, - "bbcode": {:hex, :bbcode, "0.1.1", "0023e2c7814119b2e620b7add67182e3f6019f92bfec9a22da7e99821aceba70", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, - "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm"}, - "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"}, - "cachex": {:hex, :cachex, "3.0.3", "4e2d3e05814a5738f5ff3903151d5c25636d72a3527251b753f501ad9c657967", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"}, - "calendar": {:hex, :calendar, "0.17.6", "ec291cb2e4ba499c2e8c0ef5f4ace974e2f9d02ae9e807e711a9b0c7850b9aee", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, + "base62": {:hex, :base62, "1.2.1", "4866763e08555a7b3917064e9eef9194c41667276c51b59de2bc42c6ea65f806", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "3b29948de2013d3f93aa898c884a9dff847e7aec75d9d6d8c1dc4c61c2716c42"}, + "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm", "fab09b20e3f5db886725544cbcf875b8e73ec93363954eb8a1a9ed834aa8c1f9"}, + "bbcode": {:hex, :bbcode, "0.1.1", "0023e2c7814119b2e620b7add67182e3f6019f92bfec9a22da7e99821aceba70", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5a981b98ac7d366a9b6bf40eac389aaf4d6e623c631e6b6f8a6b571efaafd338"}, + "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm", "3ad58ae787e9c7c94dd7ceda3b587ec2c64604563e049b2a0e8baafae832addb"}, + "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, + "cachex": {:hex, :cachex, "3.0.3", "4e2d3e05814a5738f5ff3903151d5c25636d72a3527251b753f501ad9c657967", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "3aadb1e605747122f60aa7b0b121cca23c14868558157563b3f3e19ea929f7d0"}, + "calendar": {:hex, :calendar, "0.17.6", "ec291cb2e4ba499c2e8c0ef5f4ace974e2f9d02ae9e807e711a9b0c7850b9aee", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "738d0e17a93c2ccfe4ddc707bdc8e672e9074c8569498483feb1c4530fb91b2b"}, "captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "e0f16822d578866e186a0974d65ad58cddc1e2ab", [ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"]}, - "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"}, - "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm"}, - "comeonin": {:hex, :comeonin, "4.1.2", "3eb5620fd8e35508991664b4c2b04dd41e52f1620b36957be837c1d7784b7592", [:mix], [{:argon2_elixir, "~> 1.2", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:bcrypt_elixir, "~> 0.12.1 or ~> 1.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: true]}, {:pbkdf2_elixir, "~> 0.12", [hex: :pbkdf2_elixir, repo: "hexpm", optional: true]}], "hexpm"}, - "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, - "cors_plug": {:hex, :cors_plug, "1.5.2", "72df63c87e4f94112f458ce9d25800900cc88608c1078f0e4faddf20933eda6e", [:mix], [{:plug, "~> 1.3 or ~> 1.4 or ~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "cowboy": {:hex, :cowboy, "2.7.0", "91ed100138a764355f43316b1d23d7ff6bdb0de4ea618cb5d8677c93a7a2f115", [:rebar3], [{:cowlib, "~> 2.8.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, - "cowlib": {:hex, :cowlib, "2.8.0", "fd0ff1787db84ac415b8211573e9a30a3ebe71b5cbff7f720089972b2319c8a4", [:rebar3], [], "hexpm"}, - "credo": {:hex, :credo, "1.1.5", "caec7a3cadd2e58609d7ee25b3931b129e739e070539ad1a0cd7efeeb47014f4", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, - "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, + "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "805abd97539caf89ec6d4732c91e62ba9da0cda51ac462380bbd28ee697a8c42"}, + "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, + "comeonin": {:hex, :comeonin, "4.1.2", "3eb5620fd8e35508991664b4c2b04dd41e52f1620b36957be837c1d7784b7592", [:mix], [{:argon2_elixir, "~> 1.2", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:bcrypt_elixir, "~> 0.12.1 or ~> 1.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: true]}, {:pbkdf2_elixir, "~> 0.12", [hex: :pbkdf2_elixir, repo: "hexpm", optional: true]}], "hexpm", "d8700a0ca4dbb616c22c9b3f6dd539d88deaafec3efe66869d6370c9a559b3e9"}, + "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"}, + "cors_plug": {:hex, :cors_plug, "1.5.2", "72df63c87e4f94112f458ce9d25800900cc88608c1078f0e4faddf20933eda6e", [:mix], [{:plug, "~> 1.3 or ~> 1.4 or ~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "9af027d20dc12dd0c4345a6b87247e0c62965871feea0bfecf9764648b02cc69"}, + "cowboy": {:hex, :cowboy, "2.7.0", "91ed100138a764355f43316b1d23d7ff6bdb0de4ea618cb5d8677c93a7a2f115", [:rebar3], [{:cowlib, "~> 2.8.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "04fd8c6a39edc6aaa9c26123009200fc61f92a3a94f3178c527b70b767c6e605"}, + "cowlib": {:hex, :cowlib, "2.8.0", "fd0ff1787db84ac415b8211573e9a30a3ebe71b5cbff7f720089972b2319c8a4", [:rebar3], [], "hexpm", "79f954a7021b302186a950a32869dbc185523d99d3e44ce430cd1f3289f41ed4"}, + "credo": {:hex, :credo, "1.1.5", "caec7a3cadd2e58609d7ee25b3931b129e739e070539ad1a0cd7efeeb47014f4", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d0bbd3222607ccaaac5c0340f7f525c627ae4d7aee6c8c8c108922620c5b6446"}, + "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "48e513299cd28b12c77266c0ed5b1c844368e5c1823724994ae84834f43d6bbe"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, - "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm"}, - "db_connection": {:hex, :db_connection, "2.2.0", "e923e88887cd60f9891fd324ac5e0290954511d090553c415fbf54be4c57ee63", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, - "decimal": {:hex, :decimal, "1.8.1", "a4ef3f5f3428bdbc0d35374029ffcf4ede8533536fa79896dd450168d9acdf3c", [:mix], [], "hexpm"}, - "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"}, - "earmark": {:hex, :earmark, "1.4.2", "3aa0bd23bc4c61cf2f1e5d752d1bb470560a6f8539974f767a38923bb20e1d7f", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.3.1", "82ab74298065bf0c64ca299f6c6785e68ea5d6b980883ee80b044499df35aba1", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm"}, - "ecto_sql": {:hex, :ecto_sql, "3.3.2", "92804e0de69bb63e621273c3492252cb08a29475c05d40eeb6f41ad2d483cfd3", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, - "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"}, - "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm"}, + "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, + "db_connection": {:hex, :db_connection, "2.2.0", "e923e88887cd60f9891fd324ac5e0290954511d090553c415fbf54be4c57ee63", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "bdf196feedfa6b83071e808b2b086fb113f8a1c4c7761f6eff6fe4b96aba0086"}, + "decimal": {:hex, :decimal, "1.8.1", "a4ef3f5f3428bdbc0d35374029ffcf4ede8533536fa79896dd450168d9acdf3c", [:mix], [], "hexpm", "3cb154b00225ac687f6cbd4acc4b7960027c757a5152b369923ead9ddbca7aec"}, + "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, + "earmark": {:hex, :earmark, "1.4.2", "3aa0bd23bc4c61cf2f1e5d752d1bb470560a6f8539974f767a38923bb20e1d7f", [:mix], [], "hexpm", "5e8806285d8a3a8999bd38e4a73c58d28534c856bc38c44818e5ba85bbda16fb"}, + "ecto": {:hex, :ecto, "3.3.1", "82ab74298065bf0c64ca299f6c6785e68ea5d6b980883ee80b044499df35aba1", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "e6c614dfe3bcff2d575ce16d815dbd43f4ee1844599a83de1eea81976a31c174"}, + "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, + "ecto_sql": {:hex, :ecto_sql, "3.3.2", "92804e0de69bb63e621273c3492252cb08a29475c05d40eeb6f41ad2d483cfd3", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b82d89d4e6a9f7f7f04783b07e8b0af968e0be2f01ee4b39047fe727c5c07471"}, + "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm", "98d0f3c6f4b8a0333170df770c6fe772b3d04564fb514c1a09504cf5ab2f48a5"}, + "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm", "b14f1dc204321429479c569cfbe8fb287541184ed040956c8862cb7a677b8406"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, - "ex_aws": {:hex, :ex_aws, "2.1.1", "1e4de2106cfbf4e837de41be41cd15813eabc722315e388f0d6bb3732cec47cd", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"}, - "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.2", "c0258bbdfea55de4f98f0b2f0ca61fe402cc696f573815134beb1866e778f47b", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"}, - "ex_const": {:hex, :ex_const, "0.2.4", "d06e540c9d834865b012a17407761455efa71d0ce91e5831e86881b9c9d82448", [:mix], [], "hexpm"}, - "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, - "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, + "ex_aws": {:hex, :ex_aws, "2.1.1", "1e4de2106cfbf4e837de41be41cd15813eabc722315e388f0d6bb3732cec47cd", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "06b6fde12b33bb6d65d5d3493e903ba5a56d57a72350c15285a4298338089e10"}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.2", "c0258bbdfea55de4f98f0b2f0ca61fe402cc696f573815134beb1866e778f47b", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "0569f5b211b1a3b12b705fe2a9d0e237eb1360b9d76298028df2346cad13097a"}, + "ex_const": {:hex, :ex_const, "0.2.4", "d06e540c9d834865b012a17407761455efa71d0ce91e5831e86881b9c9d82448", [:mix], [], "hexpm", "96fd346610cc992b8f896ed26a98be82ac4efb065a0578f334a32d60a3ba9767"}, + "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f1155337ae17ff7a1255217b4c1ceefcd1860b7ceb1a1874031e7a861b052e39"}, + "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "b84f6af156264530b312a8ab98ac6088f6b77ae5fe2058305c81434aa01fbaf9"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, - "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, - "fast_html": {:hex, :fast_html, "1.0.1", "5bc7df4dc4607ec2c314c16414e4111d79a209956c4f5df96602d194c61197f9", [:make, :mix], [], "hexpm"}, - "fast_sanitize": {:hex, :fast_sanitize, "0.1.6", "60a5ae96879956dea409a91a77f5dd2994c24cc10f80eefd8f9892ee4c0c7b25", [:mix], [{:fast_html, "~> 1.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, - "floki": {:hex, :floki, "0.23.1", "e100306ce7d8841d70a559748e5091542e2cfc67ffb3ade92b89a8435034dab1", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, - "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm"}, - "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"}, - "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, - "gettext": {:hex, :gettext, "0.17.1", "8baab33482df4907b3eae22f719da492cee3981a26e649b9c2be1c0192616962", [:mix], [], "hexpm"}, - "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, - "html_entities": {:hex, :html_entities, "0.5.0", "40f5c5b9cbe23073b48a4e69c67b6c11974f623a76165e2b92d098c0e88ccb1d", [:mix], [], "hexpm"}, + "excoveralls": {:hex, :excoveralls, "0.11.2", "0c6f2c8db7683b0caa9d490fb8125709c54580b4255ffa7ad35f3264b075a643", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "e11a4490976aabeed3eb9dc70ec94a4f2d11fed5c9d4b5dc5d89bfa0a215abb5"}, + "fast_html": {:hex, :fast_html, "1.0.1", "5bc7df4dc4607ec2c314c16414e4111d79a209956c4f5df96602d194c61197f9", [:make, :mix], [], "hexpm", "18e627dd62051a375ef94b197f41e8027c3e8eef0180ab8f81e0543b3dc6900a"}, + "fast_sanitize": {:hex, :fast_sanitize, "0.1.6", "60a5ae96879956dea409a91a77f5dd2994c24cc10f80eefd8f9892ee4c0c7b25", [:mix], [{:fast_html, "~> 1.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "b73f50f0cb522dd0331ea8e8c90b408de42c50f37641219d6364f0e3e7efd22c"}, + "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"}, + "floki": {:hex, :floki, "0.23.1", "e100306ce7d8841d70a559748e5091542e2cfc67ffb3ade92b89a8435034dab1", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "39b431b6330206cadee418e793177401ebedf2e86abc945ddd545aedb37dfc19"}, + "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"}, + "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm", "8453e2289d94c3199396eb517d65d6715ef26bcae0ee83eb5ff7a84445458d76"}, + "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm", "5cacd405e72b2609a7e1f891bddb80c53d0b3b7b0036d1648e7382ca108c41c8"}, + "gettext": {:hex, :gettext, "0.17.1", "8baab33482df4907b3eae22f719da492cee3981a26e649b9c2be1c0192616962", [:mix], [], "hexpm", "f7d97341e536f95b96eef2988d6d4230f7262cf239cda0e2e63123ee0b717222"}, + "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"}, + "html_entities": {:hex, :html_entities, "0.5.0", "40f5c5b9cbe23073b48a4e69c67b6c11974f623a76165e2b92d098c0e88ccb1d", [:mix], [], "hexpm", "8e9186e1873bea1067895f6a542b59df6c9fcf3b516ba272eeff3ea0c7b755cd"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "293d77bb6f4a67ac8bde1428735c3b42f22cbb30", [ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"]}, - "httpoison": {:hex, :httpoison, "1.6.1", "2ce5bf6e535cd0ab02e905ba8c276580bab80052c5c549f53ddea52d72e81f33", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, - "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, - "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm"}, - "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, - "joken": {:hex, :joken, "2.1.0", "bf21a73105d82649f617c5e59a7f8919aa47013d2519ebcc39d998d8d12adda9", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm"}, - "jose": {:hex, :jose, "1.9.0", "4167c5f6d06ffaebffd15cdb8da61a108445ef5e85ab8f5a7ad926fdf3ada154", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm"}, - "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, - "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, - "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm"}, - "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, - "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"}, - "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"}, + "httpoison": {:hex, :httpoison, "1.6.1", "2ce5bf6e535cd0ab02e905ba8c276580bab80052c5c549f53ddea52d72e81f33", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "89149056039084024a284cd703b2d1900d584958dba432132cb21ef35aed7487"}, + "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"}, + "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm", "64a2d30189704ae41ca7dbdd587f5291db5d1dda1414e0774c29ffc81088c1bc"}, + "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fdf843bca858203ae1de16da2ee206f53416bbda5dc8c9e78f43243de4bc3afe"}, + "joken": {:hex, :joken, "2.1.0", "bf21a73105d82649f617c5e59a7f8919aa47013d2519ebcc39d998d8d12adda9", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "eb02df7d5526df13063397e051b926b7006d5986d66f399eefc474f560cdad6a"}, + "jose": {:hex, :jose, "1.9.0", "4167c5f6d06ffaebffd15cdb8da61a108445ef5e85ab8f5a7ad926fdf3ada154", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm", "6429c4fee52b2dda7861ee19a4f09c8c1ffa213bee3a1ec187828fde95d447ed"}, + "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm", "1feaf05ee886815ad047cad7ede17d6910710986148ae09cf73eee2989717b81"}, + "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"}, + "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"}, + "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, + "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm", "6cbe761d6a0ca5a31a0931bf4c63204bceb64538e664a8ecf784a9a6f3b875f1"}, + "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"}, - "mock": {:hex, :mock, "0.3.4", "c5862eb3b8c64237f45f586cf00c9d892ba07bb48305a43319d428ce3c2897dd", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, - "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, - "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm"}, + "mock": {:hex, :mock, "0.3.4", "c5862eb3b8c64237f45f586cf00c9d892ba07bb48305a43319d428ce3c2897dd", [:mix], [{:meck, "~> 0.8.13", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "e6d886252f1a41f4ba06ecf2b4c8d38760b34b1c08a11c28f7397b2e03995964"}, + "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm", "3bc928d817974fa10cc11e6c89b9a9361e37e96dbbf3d868c41094ec05745dcd"}, + "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm", "052346cf322311c49a0f22789f3698eea030eec09b8c47367f0686ef2634ae14"}, "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]}, - "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, + "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm", "00e3ebdc821fb3a36957320d49e8f4bfa310d73ea31c90e5f925dc75e030da8f"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, - "oban": {:hex, :oban, "0.12.1", "695e9490c6e0edfca616d80639528e448bd29b3bff7b7dd10a56c79b00a5d7fb", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, - "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, - "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm"}, - "phoenix": {:hex, :phoenix, "1.4.10", "619e4a545505f562cd294df52294372d012823f4fd9d34a6657a8b242898c255", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"}, - "phoenix_ecto": {:hex, :phoenix_ecto, "4.0.0", "c43117a136e7399ea04ecaac73f8f23ee0ffe3e07acfcb8062fe5f4c9f0f6531", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "phoenix_html": {:hex, :phoenix_html, "2.13.3", "850e292ff6e204257f5f9c4c54a8cb1f6fbc16ed53d360c2b780a3d0ba333867", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.2", "496c303bdf1b2e98a9d26e89af5bba3ab487ba3a3735f74bf1f4064d2a845a3e", [:mix], [], "hexpm"}, - "phoenix_swoosh": {:hex, :phoenix_swoosh, "0.2.0", "a7e0b32077cd6d2323ae15198839b05d9caddfa20663fd85787479e81f89520e", [:mix], [{:phoenix, "~> 1.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.2", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:swoosh, "~> 0.1", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm"}, - "plug": {:hex, :plug, "1.9.0", "8d7c4e26962283ff9f8f3347bd73838e2413fbc38b7bb5467d5924f68f3a5a4a", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.1.0", "b75768153c3a8a9e8039d4b25bb9b14efbc58e9c4a6e6a270abff1cd30cbe320", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, - "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, - "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, - "postgrex": {:hex, :postgrex, "0.15.3", "5806baa8a19a68c4d07c7a624ccdb9b57e89cbc573f1b98099e3741214746ae4", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm"}, - "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.3", "3dd4da1812b8e0dbee81ea58bb3b62ed7588f2eae0c9e97e434c46807ff82311", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, - "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, - "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, - "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"}, - "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"}, - "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm"}, - "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, + "oban": {:hex, :oban, "0.12.1", "695e9490c6e0edfca616d80639528e448bd29b3bff7b7dd10a56c79b00a5d7fb", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c1d58d69b8b5a86e7167abbb8cc92764a66f25f12f6172052595067fc6a30a17"}, + "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, + "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm", "595d09db74cb093b1903381c9de423276a931a2480a46a1a5dc7f932a2a6375b"}, + "phoenix": {:hex, :phoenix, "1.4.10", "619e4a545505f562cd294df52294372d012823f4fd9d34a6657a8b242898c255", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "256ad7a140efadc3f0290470369da5bd3de985ec7c706eba07c2641b228974be"}, + "phoenix_ecto": {:hex, :phoenix_ecto, "4.0.0", "c43117a136e7399ea04ecaac73f8f23ee0ffe3e07acfcb8062fe5f4c9f0f6531", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "fe15d9fee5b82f5e64800502011ffe530650d42e1710ae9b14bc4c9be38bf303"}, + "phoenix_html": {:hex, :phoenix_html, "2.13.3", "850e292ff6e204257f5f9c4c54a8cb1f6fbc16ed53d360c2b780a3d0ba333867", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "8b01b3d6d39731ab18aa548d928b5796166d2500755f553725cfe967bafba7d9"}, + "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.2", "496c303bdf1b2e98a9d26e89af5bba3ab487ba3a3735f74bf1f4064d2a845a3e", [:mix], [], "hexpm", "1f13f9f0f3e769a667a6b6828d29dec37497a082d195cc52dbef401a9b69bf38"}, + "phoenix_swoosh": {:hex, :phoenix_swoosh, "0.2.0", "a7e0b32077cd6d2323ae15198839b05d9caddfa20663fd85787479e81f89520e", [:mix], [{:phoenix, "~> 1.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.2", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:swoosh, "~> 0.1", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "ebf1bfa7b3c1c850c04929afe02e2e0d7ab135e0706332c865de03e761676b1f"}, + "plug": {:hex, :plug, "1.9.0", "8d7c4e26962283ff9f8f3347bd73838e2413fbc38b7bb5467d5924f68f3a5a4a", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "9902eda2c52ada2a096434682e99a2493f5d06a94d6ac6bcfff9805f952350f1"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.1.0", "b75768153c3a8a9e8039d4b25bb9b14efbc58e9c4a6e6a270abff1cd30cbe320", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "6cd8ddd1bd1fbfa54d3fc61d4719c2057dae67615395d58d40437a919a46f132"}, + "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm", "73c1682f0e414cfb5d9b95c8e8cd6ffcfdae699e3b05e1db744e58b7be857759"}, + "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"}, + "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, + "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, + "postgrex": {:hex, :postgrex, "0.15.3", "5806baa8a19a68c4d07c7a624ccdb9b57e89cbc573f1b98099e3741214746ae4", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "4737ce62a31747b4c63c12b20c62307e51bb4fcd730ca0c32c280991e0606c90"}, + "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm", "d39f2ce1f3f29f3bf04f915aa3cf9c7cd4d2cee2f975e05f526e06cae9b7c902"}, + "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.3", "3dd4da1812b8e0dbee81ea58bb3b62ed7588f2eae0c9e97e434c46807ff82311", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "8d66289f77f913b37eda81fd287340c17e61a447549deb28efc254532b2bed82"}, + "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm", "9fd13404a48437e044b288b41f76e64acd9735fb8b0e3809f494811dfa66d0fb"}, + "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "c4d1404ac4e9d3d963da601db2a7d8ea31194f0017057fabf0cfb9bf5a6c8c75"}, + "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm", "0273a6483ccb936d79ca19b0ab629aef0dba958697c94782bb728b920dfc6a79"}, + "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "d736bfa7444112eb840027bb887832a0e403a4a3437f48028c3b29a2dbbd2543"}, + "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm", "6de553ba9ac0668d3728b699d5065543f3e40c854154017461ee8c09038752da"}, + "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"}, "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "825dc00aaba5a1b7c4202a532b696b595dd3bcb3", [ref: "825dc00aaba5a1b7c4202a532b696b595dd3bcb3"]}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm"}, - "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm"}, - "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"}, - "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm", "13104d7897e38ed7f044c4de953a6c28597d1c952075eb2e328bc6d6f2bfc496"}, + "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm", "94884f84783fc1ba027aba8fe8a7dae4aad78c98e9f9c76667ec3471585c08c6"}, + "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm", "2e1ec458f892ffa81f9f8386e3f35a1af6db7a7a37748a64478f13163a1f3573"}, + "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "e3928e1d2889a308aaf3e42755809ac21cffd77cb58eef01cbfdab4ce2fd1e21"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, - "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm"}, - "tesla": {:hex, :tesla, "1.3.0", "f35d72f029e608f9cdc6f6d6fcc7c66cf6d6512a70cfef9206b21b8bd0203a30", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 0.4", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, - "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, - "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "tzdata": {:hex, :tzdata, "0.5.22", "f2ba9105117ee0360eae2eca389783ef7db36d533899b2e84559404dbc77ebb8", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, - "ueberauth": {:hex, :ueberauth, "0.6.2", "25a31111249d60bad8b65438b2306a4dc91f3208faa62f5a8c33e8713989b2e8", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, - "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm"}, - "web_push_encryption": {:hex, :web_push_encryption, "0.2.3", "a0ceab85a805a30852f143d22d71c434046fbdbafbc7292e7887cec500826a80", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, + "telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm", "4738382e36a0a9a2b6e25d67c960e40e1a2c95560b9f936d8e29de8cd858480f"}, + "tesla": {:hex, :tesla, "1.3.0", "f35d72f029e608f9cdc6f6d6fcc7c66cf6d6512a70cfef9206b21b8bd0203a30", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 0.4", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "93a7cacc5ca47997759cfa1d3ab25501d291e490908006d5be56f37f89d96693"}, + "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "f354efb2400dd7a80fd9eb6c8419068c4f632da4ac47f3d8822d6e33f08bc852"}, + "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, + "tzdata": {:hex, :tzdata, "0.5.22", "f2ba9105117ee0360eae2eca389783ef7db36d533899b2e84559404dbc77ebb8", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "cd66c8a1e6a9e121d1f538b01bef459334bb4029a1ffb4eeeb5e4eae0337e7b6"}, + "ueberauth": {:hex, :ueberauth, "0.6.2", "25a31111249d60bad8b65438b2306a4dc91f3208faa62f5a8c33e8713989b2e8", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "db9fbfb5ac707bc4f85a297758406340bf0358b4af737a88113c1a9eee120ac7"}, + "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm", "1d1848c40487cdb0b30e8ed975e34e025860c02e419cb615d255849f3427439d"}, + "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"}, + "web_push_encryption": {:hex, :web_push_encryption, "0.2.3", "a0ceab85a805a30852f143d22d71c434046fbdbafbc7292e7887cec500826a80", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "9315c8f37c108835cf3f8e9157d7a9b8f420a34f402d1b1620a31aed5b93ecdf"}, "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []}, } diff --git a/priv/repo/migrations/20191030202008_add_unread_to_marker.exs b/priv/repo/migrations/20200210050658_update_markers.exs similarity index 68% rename from priv/repo/migrations/20191030202008_add_unread_to_marker.exs rename to priv/repo/migrations/20200210050658_update_markers.exs index 2b3abc682..b280e156c 100644 --- a/priv/repo/migrations/20191030202008_add_unread_to_marker.exs +++ b/priv/repo/migrations/20200210050658_update_markers.exs @@ -1,25 +1,17 @@ -defmodule Pleroma.Repo.Migrations.AddUnreadToMarker do +defmodule Pleroma.Repo.Migrations.UpdateMarkers do use Ecto.Migration import Ecto.Query alias Pleroma.Repo def up do - alter table(:markers) do - add_if_not_exists(:unread_count, :integer, default: 0) - end - - flush() - update_markers() end def down do - alter table(:markers) do - remove_if_exists(:unread_count, :integer) - end + :ok end - def update_markers do + defp update_markers do now = NaiveDateTime.utc_now() markers_attrs = @@ -27,7 +19,6 @@ def update_markers do select: %{ timeline: "notifications", user_id: q.user_id, - unread_count: fragment("SUM( CASE WHEN seen = false THEN 1 ELSE 0 END )"), last_read_id: type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) }, diff --git a/test/marker_test.exs b/test/marker_test.exs index 7b1d2218a..54c710691 100644 --- a/test/marker_test.exs +++ b/test/marker_test.exs @@ -15,7 +15,7 @@ test "returns multi" do assert %Ecto.Multi{ operations: [marker: {:run, _}, counters: {:run, _}] } = - Marker.multi_set_unread_count( + Marker.multi_set_last_read_id( Ecto.Multi.new(), user, "notifications" @@ -25,19 +25,12 @@ test "returns multi" do test "return empty multi" do user = insert(:user) multi = Ecto.Multi.new() - assert Marker.multi_set_unread_count(multi, user, "home") == multi + assert Marker.multi_set_last_read_id(multi, user, "home") == multi end end describe "get_markers/2" do test "returns user markers" do - user = insert(:user) - marker = insert(:marker, user: user) - insert(:marker, timeline: "home", user: user) - assert Marker.get_markers(user, ["notifications"]) == [refresh_record(marker)] - end - - test "returns user markers with recount unread notifications" do user = insert(:user) marker = insert(:marker, user: user) insert(:notification, user: user) @@ -46,8 +39,7 @@ test "returns user markers with recount unread notifications" do assert Marker.get_markers( user, - ["notifications"], - %{recount_unread: true} + ["notifications"] ) == [%Marker{refresh_record(marker) | unread_count: 2}] end end diff --git a/test/notification_test.exs b/test/notification_test.exs index c9b352097..49a79b2d3 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -338,12 +338,15 @@ test "it sets all notifications as read up to a specified notification ID" do assert n2.seen == true assert n3.seen == false - assert %Pleroma.Marker{unread_count: 1} = + assert %Pleroma.Marker{} = + m = Pleroma.Repo.get_by( Pleroma.Marker, user_id: other_user.id, timeline: "notifications" ) + + assert m.last_read_id == to_string(n2.id) end end From 3830cb538bd3aaee3fc48bc97b57230a558b98cf Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 10 Feb 2020 09:14:15 +0300 Subject: [PATCH 15/84] removed a comments --- lib/pleroma/marker.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index dab97d8b6..ff5f60351 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -27,12 +27,7 @@ defmodule Pleroma.Marker do timestamps() end - @doc """ - Gets markers by user and timeline. - - opts: - `recount_unread` - run force recount unread notifications for `true` value - """ + @doc "Gets markers by user and timeline." @spec get_markers(User.t(), list(String)) :: list(t()) def get_markers(user, timelines \\ []) do user From d3cf7e19fbe089b3a6d62d6a26f3dfc866a6669d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 17 Mar 2020 13:02:10 +0100 Subject: [PATCH 16/84] activity_pub_controller_test.exs: test posting with AP C2S uploaded media --- .../activity_pub_controller_test.exs | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index bd8e0b5cc..2bd494a37 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -1241,16 +1241,46 @@ test "POST /api/ap/upload_media", %{conn: conn} do filename: "an_image.jpg" } - conn = + object = conn |> assign(:user, user) |> post("/api/ap/upload_media", %{"file" => image, "description" => desc}) + |> json_response(:created) - assert object = json_response(conn, :created) assert object["name"] == desc assert object["type"] == "Document" assert object["actor"] == user.ap_id + assert [%{"href" => object_href}] = object["url"] + activity_request = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "AP C2S test, attachment", + "attachment" => [object] + }, + "to" => "https://www.w3.org/ns/activitystreams#Public", + "cc" => [] + } + + activity_response = + conn + |> assign(:user, user) + |> post("/users/#{user.nickname}/outbox", activity_request) + |> json_response(:created) + + assert activity_response["id"] + assert activity_response["object"] + assert activity_response["actor"] == user.ap_id + + assert %Object{data: %{"attachment" => [attachment]}} = Object.normalize(activity_response["object"]) + assert attachment["type"] == "Document" + assert attachment["name"] == desc + assert [%{"href" => attachment_href}] = attachment["url"] + assert attachment_href == object_href + + # Fails if unauthenticated conn |> post("/api/ap/upload_media", %{"file" => image, "description" => desc}) |> json_response(403) From f9d622d25a744f58fbaf8370ad4435597bb15bf0 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 19 Mar 2020 15:08:49 +0100 Subject: [PATCH 17/84] WIP --- lib/pleroma/web/activity_pub/transmogrifier.ex | 15 --------------- .../activity_pub_controller_test.exs | 18 ++++++++++++++---- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9cd3de705..db848f657 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -202,21 +202,6 @@ def fix_context(object) do |> Map.put("conversation", context) end - def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do - attachments = - Enum.map(attachment, fn data -> - media_type = data["mediaType"] || data["mimeType"] - href = data["url"] || data["href"] - url = [%{"type" => "Link", "mediaType" => media_type, "href" => href}] - - data - |> Map.put("mediaType", media_type) - |> Map.put("url", url) - end) - - Map.put(object, "attachment", attachments) - end - def fix_attachments(%{"attachment" => attachment} = object) when is_map(attachment) do object |> Map.put("attachment", [attachment]) diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index 2bd494a37..01c955c0a 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -1250,7 +1250,9 @@ test "POST /api/ap/upload_media", %{conn: conn} do assert object["name"] == desc assert object["type"] == "Document" assert object["actor"] == user.ap_id - assert [%{"href" => object_href}] = object["url"] + assert [%{"href" => object_href, "mediaType" => object_mediatype}] = object["url"] + assert is_binary(object_href) + assert object_mediatype == "image/jpeg" activity_request = %{ "@context" => "https://www.w3.org/ns/activitystreams", @@ -1274,11 +1276,19 @@ test "POST /api/ap/upload_media", %{conn: conn} do assert activity_response["object"] assert activity_response["actor"] == user.ap_id - assert %Object{data: %{"attachment" => [attachment]}} = Object.normalize(activity_response["object"]) + assert %Object{data: %{"attachment" => [attachment]}} = + Object.normalize(activity_response["object"]) + assert attachment["type"] == "Document" assert attachment["name"] == desc - assert [%{"href" => attachment_href}] = attachment["url"] - assert attachment_href == object_href + + assert [ + %{ + "href" => ^object_href, + "type" => "Link", + "mediaType" => ^object_mediatype + } + ] = attachment["url"] # Fails if unauthenticated conn From c1fd4f665335ba67336bd1b2fab2d9df5e247e08 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 19 Mar 2020 19:10:03 +0100 Subject: [PATCH 18/84] transmogrifier.ex: rework fix_attachment for better IR --- .../web/activity_pub/transmogrifier.ex | 45 +++++++++++++++++++ test/web/activity_pub/transmogrifier_test.exs | 30 +++---------- 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index db848f657..df5ca0239 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -202,6 +202,51 @@ def fix_context(object) do |> Map.put("conversation", context) end + defp add_if_present(map, _key, nil), do: map + + defp add_if_present(map, key, value) do + Map.put(map, key, value) + end + + def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachment) do + attachments = + Enum.map(attachment, fn data -> + url = + cond do + is_list(data["url"]) -> List.first(data["url"]) + is_map(data["url"]) -> data["url"] + true -> nil + end + + media_type = + cond do + is_map(url) && is_binary(url["mediaType"]) -> url["mediaType"] + is_binary(data["mediaType"]) -> data["mediaType"] + is_binary(data["mimeType"]) -> data["mimeType"] + true -> nil + end + + href = + cond do + is_map(url) && is_binary(url["href"]) -> url["href"] + is_binary(data["url"]) -> data["url"] + is_binary(data["href"]) -> data["href"] + end + + attachment_url = + %{"href" => href} + |> add_if_present("mediaType", media_type) + |> add_if_present("type", Map.get(url || %{}, "type")) + + %{"url" => [attachment_url]} + |> add_if_present("mediaType", media_type) + |> add_if_present("type", data["type"]) + |> add_if_present("name", data["name"]) + end) + + Map.put(object, "attachment", attachments) + end + def fix_attachments(%{"attachment" => attachment} = object) when is_map(attachment) do object |> Map.put("attachment", [attachment]) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index efbca82f6..242d933e7 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -1228,19 +1228,13 @@ test "it remaps video URLs as attachments if necessary" do attachment = %{ "type" => "Link", "mediaType" => "video/mp4", - "href" => - "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4", - "mimeType" => "video/mp4", - "size" => 5_015_880, "url" => [ %{ "href" => "https://peertube.moe/static/webseed/df5f464b-be8d-46fb-ad81-2d4c2d1630e3-480.mp4", - "mediaType" => "video/mp4", - "type" => "Link" + "mediaType" => "video/mp4" } - ], - "width" => 480 + ] } assert object.data["url"] == @@ -2067,11 +2061,7 @@ test "returns modified object when attachment is map" do %{ "mediaType" => "video/mp4", "url" => [ - %{ - "href" => "https://peertube.moe/stat-480.mp4", - "mediaType" => "video/mp4", - "type" => "Link" - } + %{"href" => "https://peertube.moe/stat-480.mp4", "mediaType" => "video/mp4"} ] } ] @@ -2089,23 +2079,13 @@ test "returns modified object when attachment is list" do %{ "mediaType" => "video/mp4", "url" => [ - %{ - "href" => "https://pe.er/stat-480.mp4", - "mediaType" => "video/mp4", - "type" => "Link" - } + %{"href" => "https://pe.er/stat-480.mp4", "mediaType" => "video/mp4"} ] }, %{ - "href" => "https://pe.er/stat-480.mp4", "mediaType" => "video/mp4", - "mimeType" => "video/mp4", "url" => [ - %{ - "href" => "https://pe.er/stat-480.mp4", - "mediaType" => "video/mp4", - "type" => "Link" - } + %{"href" => "https://pe.er/stat-480.mp4", "mediaType" => "video/mp4"} ] } ] From a6ee6784bc74b311d454112c427f41b1fdec6ce0 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 28 Feb 2020 11:16:40 +0300 Subject: [PATCH 19/84] creating trusted app from adminFE & mix task --- CHANGELOG.md | 2 + docs/API/admin_api.md | 101 ++++++++++ docs/administration/CLI_tasks/oauth_app.md | 16 ++ lib/mix/tasks/pleroma/app.ex | 49 +++++ .../web/admin_api/admin_api_controller.ex | 79 ++++++++ .../controllers/account_controller.ex | 1 + .../web/mastodon_api/views/app_view.ex | 15 ++ lib/pleroma/web/oauth/app.ex | 82 +++++++- lib/pleroma/web/router.ex | 5 + lib/pleroma/web/twitter_api/twitter_api.ex | 3 +- .../20200227122417_add_trusted_to_apps.exs | 9 + test/support/factory.ex | 2 +- test/tasks/app_test.exs | 65 ++++++ .../admin_api/admin_api_controller_test.exs | 185 ++++++++++++++++++ .../controllers/account_controller_test.exs | 67 +++++++ 15 files changed, 678 insertions(+), 3 deletions(-) create mode 100644 docs/administration/CLI_tasks/oauth_app.md create mode 100644 lib/mix/tasks/pleroma/app.ex create mode 100644 priv/repo/migrations/20200227122417_add_trusted_to_apps.exs create mode 100644 test/tasks/app_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 15a073c64..a1271cbca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Add an option `authorized_fetch_mode` to require HTTP signatures for AP fetches. - ActivityPub: support for `replies` collection (output for outgoing federation & fetching on incoming federation). - Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`) +- Mix task to create trusted OAuth App.
API Changes @@ -145,6 +146,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - ActivityPub: `[:activitypub, :note_replies_output_limit]` setting sets the number of note self-replies to output on outgoing federation. - Admin API: `GET /api/pleroma/admin/stats` to get status count by visibility scope - Admin API: `GET /api/pleroma/admin/statuses` - list all statuses (accepts `godmode` and `local_only`) +- Admin API: endpoints for create/update/delete OAuth Apps.
### Fixed diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 47afdfba5..4d12698ec 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -983,3 +983,104 @@ Loads json generated from `config/descriptions.exs`. } } ``` + +## `GET /api/pleroma/admin/oauth_app` + +### List OAuth app + +- Params: + - *optional* `name` + - *optional* `client_id` + - *optional* `page` + - *optional* `page_size` + - *optional* `trusted` + +- Response: + +```json +{ + "apps": [ + { + "id": 1, + "name": "App name", + "client_id": "yHoDSiWYp5mPV6AfsaVOWjdOyt5PhWRiafi6MRd1lSk", + "client_secret": "nLmis486Vqrv2o65eM9mLQx_m_4gH-Q6PcDpGIMl6FY", + "redirect_uri": "https://example.com/oauth-callback", + "website": "https://example.com", + "trusted": true + } + ], + "count": 17, + "page_size": 50 +} +``` + + +## `POST /api/pleroma/admin/oauth_app` + +### Create OAuth App + +- Params: + - `name` + - `redirect_uris` + - `scopes` + - *optional* `website` + - *optional* `trusted` + +- Response: + +```json +{ + "id": 1, + "name": "App name", + "client_id": "yHoDSiWYp5mPV6AfsaVOWjdOyt5PhWRiafi6MRd1lSk", + "client_secret": "nLmis486Vqrv2o65eM9mLQx_m_4gH-Q6PcDpGIMl6FY", + "redirect_uri": "https://example.com/oauth-callback", + "website": "https://example.com", + "trusted": true +} +``` + +- On failure: +```json +{ + "redirect_uris": "can't be blank", + "name": "can't be blank" +} +``` + +## `PATCH /api/pleroma/admin/oauth_app/:id` + +### Update OAuth App + +- Params: + - *optional* `name` + - *optional* `redirect_uris` + - *optional* `scopes` + - *optional* `website` + - *optional* `trusted` + +- Response: + +```json +{ + "id": 1, + "name": "App name", + "client_id": "yHoDSiWYp5mPV6AfsaVOWjdOyt5PhWRiafi6MRd1lSk", + "client_secret": "nLmis486Vqrv2o65eM9mLQx_m_4gH-Q6PcDpGIMl6FY", + "redirect_uri": "https://example.com/oauth-callback", + "website": "https://example.com", + "trusted": true +} +``` + +## `DELETE /api/pleroma/admin/oauth_app/:id` + +### Delete OAuth App + +- Params: None + +- Response: + - On success: `204`, empty response + - On failure: + - 400 Bad Request `"Invalid parameters"` when `status` is missing \ No newline at end of file diff --git a/docs/administration/CLI_tasks/oauth_app.md b/docs/administration/CLI_tasks/oauth_app.md new file mode 100644 index 000000000..4d6bfc25a --- /dev/null +++ b/docs/administration/CLI_tasks/oauth_app.md @@ -0,0 +1,16 @@ +# Creating trusted OAuth App + +{! backend/administration/CLI_tasks/general_cli_task_info.include !} + +## Create trusted OAuth App. + +Optional params: + * `-s SCOPES` - scopes for app, e.g. `read,write,follow,push`. + +```sh tab="OTP" + ./bin/pleroma_ctl app create -n APP_NAME -r REDIRECT_URI +``` + +```sh tab="From Source" +mix pleroma.app create -n APP_NAME -r REDIRECT_URI +``` \ No newline at end of file diff --git a/lib/mix/tasks/pleroma/app.ex b/lib/mix/tasks/pleroma/app.ex new file mode 100644 index 000000000..463e2449f --- /dev/null +++ b/lib/mix/tasks/pleroma/app.ex @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.App do + @moduledoc File.read!("docs/administration/CLI_tasks/oauth_app.md") + use Mix.Task + + import Mix.Pleroma + + @shortdoc "Creates trusted OAuth App" + + def run(["create" | options]) do + start_pleroma() + + {opts, _} = + OptionParser.parse!(options, + strict: [name: :string, redirect_uri: :string, scopes: :string], + aliases: [n: :name, r: :redirect_uri, s: :scopes] + ) + + scopes = + if opts[:scopes] do + String.split(opts[:scopes], ",") + else + ["read", "write", "follow", "push"] + end + + params = %{ + client_name: opts[:name], + redirect_uris: opts[:redirect_uri], + trusted: true, + scopes: scopes + } + + with {:ok, app} <- Pleroma.Web.OAuth.App.create(params) do + shell_info("#{app.client_name} successfully created:") + shell_info("App client_id: " <> app.client_id) + shell_info("App client_secret: " <> app.client_secret) + else + {:error, changeset} -> + shell_error("Creating failed:") + + Enum.each(Pleroma.Web.OAuth.App.errors(changeset), fn {key, error} -> + shell_error("#{key}: #{error}") + end) + end + end +end diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 175260bc2..b03fa7169 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -27,7 +27,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.Web.AdminAPI.Search alias Pleroma.Web.CommonAPI alias Pleroma.Web.Endpoint + alias Pleroma.Web.MastodonAPI.AppView alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Web.OAuth.App alias Pleroma.Web.Router require Logger @@ -978,6 +980,83 @@ def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" = conn |> json("") end + def oauth_app_create(conn, params) do + params = + if params["name"] do + Map.put(params, "client_name", params["name"]) + else + params + end + + result = + case App.create(params) do + {:ok, app} -> + AppView.render("show.json", %{app: app, admin: true}) + + {:error, changeset} -> + App.errors(changeset) + end + + json(conn, result) + end + + def oauth_app_update(conn, params) do + params = + if params["name"] do + Map.put(params, "client_name", params["name"]) + else + params + end + + with {:ok, app} <- App.update(params) do + json(conn, AppView.render("show.json", %{app: app, admin: true})) + else + {:error, changeset} -> + json(conn, App.errors(changeset)) + + nil -> + json_response(conn, :bad_request, "") + end + end + + def oauth_app_list(conn, params) do + {page, page_size} = page_params(params) + + search_params = %{ + client_name: params["name"], + client_id: params["client_id"], + page: page, + page_size: page_size + } + + search_params = + if Map.has_key?(params, "trusted") do + Map.put(search_params, :trusted, params["trusted"]) + else + search_params + end + + with {:ok, apps, count} <- App.search(search_params) do + json( + conn, + AppView.render("index.json", + apps: apps, + count: count, + page_size: page_size, + admin: true + ) + ) + end + end + + def oauth_app_delete(conn, params) do + with {:ok, _app} <- App.destroy(params["id"]) do + json_response(conn, :no_content, "") + else + _ -> json_response(conn, :bad_request, "") + end + end + def stats(conn, _) do count = Stats.get_status_visibility_count() diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 6dbf11ac9..5f8aa2e3e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -92,6 +92,7 @@ def create( |> Map.put("fullname", params["fullname"] || nickname) |> Map.put("bio", params["bio"] || "") |> Map.put("confirm", params["password"]) + |> Map.put("trusted_app", app.trusted) with :ok <- validate_email_param(params), {:ok, user} <- TwitterAPI.register_user(params, need_confirmation: true), diff --git a/lib/pleroma/web/mastodon_api/views/app_view.ex b/lib/pleroma/web/mastodon_api/views/app_view.ex index d934e2107..36071cd25 100644 --- a/lib/pleroma/web/mastodon_api/views/app_view.ex +++ b/lib/pleroma/web/mastodon_api/views/app_view.ex @@ -7,6 +7,21 @@ defmodule Pleroma.Web.MastodonAPI.AppView do alias Pleroma.Web.OAuth.App + def render("index.json", %{apps: apps, count: count, page_size: page_size, admin: true}) do + %{ + apps: render_many(apps, Pleroma.Web.MastodonAPI.AppView, "show.json", %{admin: true}), + count: count, + page_size: page_size + } + end + + def render("show.json", %{admin: true, app: %App{} = app} = assigns) do + "show.json" + |> render(Map.delete(assigns, :admin)) + |> Map.put(:trusted, app.trusted) + |> Map.put(:id, app.id) + end + def render("show.json", %{app: %App{} = app}) do %{ id: app.id |> to_string, diff --git a/lib/pleroma/web/oauth/app.ex b/lib/pleroma/web/oauth/app.ex index 01ed326f4..6a6d5f2e2 100644 --- a/lib/pleroma/web/oauth/app.ex +++ b/lib/pleroma/web/oauth/app.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.OAuth.App do use Ecto.Schema import Ecto.Changeset + import Ecto.Query alias Pleroma.Repo @type t :: %__MODULE__{} @@ -16,14 +17,24 @@ defmodule Pleroma.Web.OAuth.App do field(:website, :string) field(:client_id, :string) field(:client_secret, :string) + field(:trusted, :boolean, default: false) + + has_many(:oauth_authorizations, Pleroma.Web.OAuth.Authorization, on_delete: :delete_all) + has_many(:oauth_tokens, Pleroma.Web.OAuth.Token, on_delete: :delete_all) timestamps() end + @spec changeset(App.t(), map()) :: Ecto.Changeset.t() + def changeset(struct, params) do + cast(struct, params, [:client_name, :redirect_uris, :scopes, :website, :trusted]) + end + + @spec register_changeset(App.t(), map()) :: Ecto.Changeset.t() def register_changeset(struct, params \\ %{}) do changeset = struct - |> cast(params, [:client_name, :redirect_uris, :scopes, :website]) + |> changeset(params) |> validate_required([:client_name, :redirect_uris, :scopes]) if changeset.valid? do @@ -41,6 +52,21 @@ def register_changeset(struct, params \\ %{}) do end end + @spec create(map()) :: {:ok, App.t()} | {:error, Ecto.Changeset.t()} + def create(params) do + with changeset <- __MODULE__.register_changeset(%__MODULE__{}, params) do + Repo.insert(changeset) + end + end + + @spec update(map()) :: {:ok, App.t()} | {:error, Ecto.Changeset.t()} + def update(params) do + with %__MODULE__{} = app <- Repo.get(__MODULE__, params["id"]), + changeset <- changeset(app, params) do + Repo.update(changeset) + end + end + @doc """ Gets app by attrs or create new with attrs. And updates the scopes if need. @@ -65,4 +91,58 @@ defp update_scopes(%__MODULE__{} = app, scopes) do |> change(%{scopes: scopes}) |> Repo.update() end + + @spec search(map()) :: {:ok, [App.t()], non_neg_integer()} + def search(params) do + query = from(a in __MODULE__) + + query = + if params[:client_name] do + from(a in query, where: a.client_name == ^params[:client_name]) + else + query + end + + query = + if params[:client_id] do + from(a in query, where: a.client_id == ^params[:client_id]) + else + query + end + + query = + if Map.has_key?(params, :trusted) do + from(a in query, where: a.trusted == ^params[:trusted]) + else + query + end + + query = + from(u in query, + limit: ^params[:page_size], + offset: ^((params[:page] - 1) * params[:page_size]) + ) + + count = Repo.aggregate(__MODULE__, :count, :id) + + {:ok, Repo.all(query), count} + end + + @spec destroy(pos_integer()) :: {:ok, App.t()} | {:error, Ecto.Changeset.t()} + def destroy(id) do + with %__MODULE__{} = app <- Repo.get(__MODULE__, id) do + Repo.delete(app) + end + end + + @spec errors(Ecto.Changeset.t()) :: map() + def errors(changeset) do + Enum.reduce(changeset.errors, %{}, fn + {:client_name, {error, _}}, acc -> + Map.put(acc, :name, error) + + {key, {error, _}}, acc -> + Map.put(acc, key, error) + end) + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 3f36f6c1a..c37ef59a0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -203,6 +203,11 @@ defmodule Pleroma.Web.Router do post("/reload_emoji", AdminAPIController, :reload_emoji) get("/stats", AdminAPIController, :stats) + + get("/oauth_app", AdminAPIController, :oauth_app_list) + post("/oauth_app", AdminAPIController, :oauth_app_create) + patch("/oauth_app/:id", AdminAPIController, :oauth_app_update) + delete("/oauth_app/:id", AdminAPIController, :oauth_app_delete) end scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index f9c0994da..7a1ba6936 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do def register_user(params, opts \\ []) do token = params["token"] + trusted_app? = params["trusted_app"] params = %{ nickname: params["nickname"], @@ -29,7 +30,7 @@ def register_user(params, opts \\ []) do captcha_enabled = Pleroma.Config.get([Pleroma.Captcha, :enabled]) # true if captcha is disabled or enabled and valid, false otherwise captcha_ok = - if not captcha_enabled do + if trusted_app? || not captcha_enabled do :ok else Pleroma.Captcha.validate( diff --git a/priv/repo/migrations/20200227122417_add_trusted_to_apps.exs b/priv/repo/migrations/20200227122417_add_trusted_to_apps.exs new file mode 100644 index 000000000..4e2a62af0 --- /dev/null +++ b/priv/repo/migrations/20200227122417_add_trusted_to_apps.exs @@ -0,0 +1,9 @@ +defmodule Pleroma.Repo.Migrations.AddTrustedToApps do + use Ecto.Migration + + def change do + alter table(:apps) do + add(:trusted, :boolean, default: false) + end + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index af639b6cd..f0b797fd4 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -294,7 +294,7 @@ def follow_activity_factory do def oauth_app_factory do %Pleroma.Web.OAuth.App{ - client_name: "Some client", + client_name: sequence(:client_name, &"Some client #{&1}"), redirect_uris: "https://example.com/callback", scopes: ["read", "write", "follow", "push", "admin"], website: "https://example.com", diff --git a/test/tasks/app_test.exs b/test/tasks/app_test.exs new file mode 100644 index 000000000..b8f03566d --- /dev/null +++ b/test/tasks/app_test.exs @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.AppTest do + use Pleroma.DataCase, async: true + + setup_all do + Mix.shell(Mix.Shell.Process) + + on_exit(fn -> + Mix.shell(Mix.Shell.IO) + end) + end + + describe "creates new app" do + test "with default scopes" do + name = "Some name" + redirect = "https://example.com" + Mix.Tasks.Pleroma.App.run(["create", "-n", name, "-r", redirect]) + + assert_app(name, redirect, ["read", "write", "follow", "push"]) + end + + test "with custom scopes" do + name = "Another name" + redirect = "https://example.com" + + Mix.Tasks.Pleroma.App.run([ + "create", + "-n", + name, + "-r", + redirect, + "-s", + "read,write,follow,push,admin" + ]) + + assert_app(name, redirect, ["read", "write", "follow", "push", "admin"]) + end + end + + test "with errors" do + Mix.Tasks.Pleroma.App.run(["create"]) + {:mix_shell, :error, ["Creating failed:"]} + {:mix_shell, :error, ["name: can't be blank"]} + {:mix_shell, :error, ["redirect_uris: can't be blank"]} + end + + defp assert_app(name, redirect, scopes) do + app = Repo.get_by(Pleroma.Web.OAuth.App, client_name: name) + + assert_received {:mix_shell, :info, [message]} + assert message == "#{name} successfully created:" + + assert_received {:mix_shell, :info, [message]} + assert message == "App client_id: #{app.client_id}" + + assert_received {:mix_shell, :info, [message]} + assert message == "App client_secret: #{app.client_secret}" + + assert app.scopes == scopes + assert app.redirect_uris == redirect + end +end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 0a902585d..d77e8d1d2 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -3623,6 +3623,191 @@ test "status visibility count", %{conn: conn} do response["status_visibility"] end end + + describe "POST /api/pleroma/admin/oauth_app" do + test "errors", %{conn: conn} do + response = conn |> post("/api/pleroma/admin/oauth_app", %{}) |> json_response(200) + + assert response == %{"name" => "can't be blank", "redirect_uris" => "can't be blank"} + end + + test "success", %{conn: conn} do + base_url = Pleroma.Web.base_url() + app_name = "Trusted app" + + response = + conn + |> post("/api/pleroma/admin/oauth_app", %{ + name: app_name, + redirect_uris: base_url + }) + |> json_response(200) + + assert %{ + "client_id" => _, + "client_secret" => _, + "name" => ^app_name, + "redirect_uri" => ^base_url, + "trusted" => false + } = response + end + + test "with trusted", %{conn: conn} do + base_url = Pleroma.Web.base_url() + app_name = "Trusted app" + + response = + conn + |> post("/api/pleroma/admin/oauth_app", %{ + name: app_name, + redirect_uris: base_url, + trusted: true + }) + |> json_response(200) + + assert %{ + "client_id" => _, + "client_secret" => _, + "name" => ^app_name, + "redirect_uri" => ^base_url, + "trusted" => true + } = response + end + end + + describe "GET /api/pleroma/admin/oauth_app" do + setup do + app = insert(:oauth_app) + {:ok, app: app} + end + + test "list", %{conn: conn} do + response = + conn + |> get("/api/pleroma/admin/oauth_app") + |> json_response(200) + + assert %{"apps" => apps, "count" => count, "page_size" => _} = response + + assert length(apps) == count + end + + test "with page size", %{conn: conn} do + insert(:oauth_app) + page_size = 1 + + response = + conn + |> get("/api/pleroma/admin/oauth_app", %{page_size: to_string(page_size)}) + |> json_response(200) + + assert %{"apps" => apps, "count" => _, "page_size" => ^page_size} = response + + assert length(apps) == page_size + end + + test "search by client name", %{conn: conn, app: app} do + response = + conn + |> get("/api/pleroma/admin/oauth_app", %{name: app.client_name}) + |> json_response(200) + + assert %{"apps" => [returned], "count" => _, "page_size" => _} = response + + assert returned["client_id"] == app.client_id + assert returned["name"] == app.client_name + end + + test "search by client id", %{conn: conn, app: app} do + response = + conn + |> get("/api/pleroma/admin/oauth_app", %{client_id: app.client_id}) + |> json_response(200) + + assert %{"apps" => [returned], "count" => _, "page_size" => _} = response + + assert returned["client_id"] == app.client_id + assert returned["name"] == app.client_name + end + + test "only trusted", %{conn: conn} do + app = insert(:oauth_app, trusted: true) + + response = + conn + |> get("/api/pleroma/admin/oauth_app", %{trusted: true}) + |> json_response(200) + + assert %{"apps" => [returned], "count" => _, "page_size" => _} = response + + assert returned["client_id"] == app.client_id + assert returned["name"] == app.client_name + end + end + + describe "DELETE /api/pleroma/admin/oauth_app/:id" do + test "with id", %{conn: conn} do + app = insert(:oauth_app) + + response = + conn + |> delete("/api/pleroma/admin/oauth_app/" <> to_string(app.id)) + |> json_response(:no_content) + + assert response == "" + end + + test "with non existance id", %{conn: conn} do + response = + conn + |> delete("/api/pleroma/admin/oauth_app/0") + |> json_response(:bad_request) + + assert response == "" + end + end + + describe "PATCH /api/pleroma/admin/oauth_app/:id" do + test "with id", %{conn: conn} do + app = insert(:oauth_app) + + name = "another name" + url = "https://example.com" + scopes = ["admin"] + id = app.id + website = "http://website.com" + + response = + conn + |> patch("/api/pleroma/admin/oauth_app/" <> to_string(app.id), %{ + name: name, + trusted: true, + redirect_uris: url, + scopes: scopes, + website: website + }) + |> json_response(200) + + assert %{ + "client_id" => _, + "client_secret" => _, + "id" => ^id, + "name" => ^name, + "redirect_uri" => ^url, + "trusted" => true, + "website" => ^website + } = response + end + + test "without id", %{conn: conn} do + response = + conn + |> patch("/api/pleroma/admin/oauth_app/0") + |> json_response(:bad_request) + + assert response == "" + end + end end # Needed for testing diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index a9fa0ce48..f770232df 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -942,6 +942,73 @@ test "returns forbidden if token is invalid", %{conn: conn, valid_params: valid_ res = post(conn, "/api/v1/accounts", valid_params) assert json_response(res, 403) == %{"error" => "Invalid credentials"} end + + test "registration from trusted app" do + clear_config([Pleroma.Captcha, :enabled], true) + app = insert(:oauth_app, trusted: true, scopes: ["read", "write", "follow", "push"]) + + conn = + build_conn() + |> post("/oauth/token", %{ + "grant_type" => "client_credentials", + "client_id" => app.client_id, + "client_secret" => app.client_secret + }) + + assert %{"access_token" => token, "token_type" => "Bearer"} = json_response(conn, 200) + + response = + build_conn() + |> Plug.Conn.put_req_header("authorization", "Bearer " <> token) + |> post("/api/v1/accounts", %{ + nickname: "nickanme", + agreement: true, + email: "email@example.com", + fullname: "Lain", + username: "Lain", + password: "some_password", + confirm: "some_password" + }) + |> json_response(200) + + assert %{ + "access_token" => access_token, + "created_at" => _, + "scope" => ["read", "write", "follow", "push"], + "token_type" => "Bearer" + } = response + + response = + build_conn() + |> Plug.Conn.put_req_header("authorization", "Bearer " <> access_token) + |> get("/api/v1/accounts/verify_credentials") + |> json_response(200) + + assert %{ + "acct" => "Lain", + "bot" => false, + "display_name" => "Lain", + "follow_requests_count" => 0, + "followers_count" => 0, + "following_count" => 0, + "locked" => false, + "note" => "", + "source" => %{ + "fields" => [], + "note" => "", + "pleroma" => %{ + "actor_type" => "Person", + "discoverable" => false, + "no_rich_text" => false, + "show_role" => true + }, + "privacy" => "public", + "sensitive" => false + }, + "statuses_count" => 0, + "username" => "Lain" + } = response + end end describe "create account by app / rate limit" do From 4a2538967caf5b0f9970cc5f973c16ea5d776aa3 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 24 Mar 2020 20:18:27 +0400 Subject: [PATCH 20/84] Support pagination in conversations --- CHANGELOG.md | 3 +++ lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- .../controllers/pleroma_api_controller.ex | 10 +++++----- .../controllers/pleroma_api_controller_test.exs | 17 +++++++++++++++++ 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15a073c64..905364d7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Support for `include_types` in `/api/v1/notifications`. +### Fixed +- Support pagination in conversations API + ## [2.0.0] - 2019-03-08 ### Security - Mastodon API: Fix being able to request enourmous amount of statuses in timelines leading to DoS. Now limited to 40 per request. diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 30e282840..351d1bdb8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -696,7 +696,7 @@ def move(%User{} = origin, %User{} = target, local \\ true) do end end - defp fetch_activities_for_context_query(context, opts) do + def fetch_activities_for_context_query(context, opts) do public = [Constants.as_public()] recipients = diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex index dae7f0f2f..edb071baa 100644 --- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex @@ -110,12 +110,11 @@ def conversation(%{assigns: %{user: user}} = conn, %{"id" => participation_id}) end def conversation_statuses( - %{assigns: %{user: user}} = conn, + %{assigns: %{user: %{id: user_id} = user}} = conn, %{"id" => participation_id} = params ) do - with %Participation{} = participation <- - Participation.get(participation_id, preload: [:conversation]), - true <- user.id == participation.user_id do + with %Participation{user_id: ^user_id} = participation <- + Participation.get(participation_id, preload: [:conversation]) do params = params |> Map.put("blocking_user", user) @@ -124,7 +123,8 @@ def conversation_statuses( activities = participation.conversation.ap_id - |> ActivityPub.fetch_activities_for_context(params) + |> ActivityPub.fetch_activities_for_context_query(params) + |> Pleroma.Pagination.fetch_paginated(Map.put(params, "total", false)) |> Enum.reverse() conn diff --git a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs index 32250f06f..8bf7eb3be 100644 --- a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs @@ -169,6 +169,23 @@ test "/api/v1/pleroma/conversations/:id/statuses" do id_one = activity.id id_two = activity_two.id assert [%{"id" => ^id_one}, %{"id" => ^id_two}] = result + + {:ok, %{id: id_three}} = + CommonAPI.post(other_user, %{ + "status" => "Bye!", + "in_reply_to_status_id" => activity.id, + "in_reply_to_conversation_id" => participation.id + }) + + assert [%{"id" => ^id_two}, %{"id" => ^id_three}] = + conn + |> get("/api/v1/pleroma/conversations/#{participation.id}/statuses?limit=2") + |> json_response(:ok) + + assert [%{"id" => ^id_three}] = + conn + |> get("/api/v1/pleroma/conversations/#{participation.id}/statuses?min_id=#{id_two}") + |> json_response(:ok) end test "PATCH /api/v1/pleroma/conversations/:id" do From be9d18461a5ed6bd835e2eba8d3b54ba61fc51fb Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sat, 28 Mar 2020 18:49:03 +0300 Subject: [PATCH 21/84] FollowingRelationship storage & performance optimizations (state turned `ecto_enum`-driven integer, reorganized indices etc.). --- lib/pleroma/ecto_enums.ex | 6 +++ lib/pleroma/following_relationship.ex | 43 ++++++++++++++++--- lib/pleroma/user.ex | 18 +++++--- lib/pleroma/user/query.ex | 6 +-- lib/pleroma/web/activity_pub/mrf.ex | 2 +- .../web/activity_pub/transmogrifier.ex | 13 +++--- lib/pleroma/web/common_api/common_api.ex | 4 +- .../web/mastodon_api/views/account_view.ex | 6 +-- ...llowing_relationships_state_to_integer.exs | 29 +++++++++++++ ...owing_relationships_following_id_index.exs | 11 +++++ test/following_relationship_test.exs | 8 ++-- test/tasks/user_test.exs | 2 +- test/user_test.exs | 9 ++-- test/web/activity_pub/transmogrifier_test.exs | 2 +- test/web/common_api/common_api_test.exs | 4 +- .../follow_request_controller_test.exs | 4 +- test/web/streamer/streamer_test.exs | 6 +-- 17 files changed, 128 insertions(+), 45 deletions(-) create mode 100644 priv/repo/migrations/20200328124805_change_following_relationships_state_to_integer.exs create mode 100644 priv/repo/migrations/20200328130139_add_following_relationships_following_id_index.exs diff --git a/lib/pleroma/ecto_enums.ex b/lib/pleroma/ecto_enums.ex index d9b601223..b98ac4ba1 100644 --- a/lib/pleroma/ecto_enums.ex +++ b/lib/pleroma/ecto_enums.ex @@ -11,3 +11,9 @@ notification_mute: 4, inverse_subscription: 5 ) + +defenum(FollowingRelationshipStateEnum, + follow_pending: 1, + follow_accept: 2, + follow_reject: 3 +) diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index a9538ea4e..a28da8bec 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -13,7 +13,7 @@ defmodule Pleroma.FollowingRelationship do alias Pleroma.User schema "following_relationships" do - field(:state, :string, default: "accept") + field(:state, FollowingRelationshipStateEnum, default: :follow_pending) belongs_to(:follower, User, type: CompatType) belongs_to(:following, User, type: CompatType) @@ -27,6 +27,19 @@ def changeset(%__MODULE__{} = following_relationship, attrs) do |> put_assoc(:follower, attrs.follower) |> put_assoc(:following, attrs.following) |> validate_required([:state, :follower, :following]) + |> unique_constraint(:follower_id, + name: :following_relationships_follower_id_following_id_index + ) + |> validate_not_self_relationship() + end + + def state_to_enum(state) when is_binary(state) do + case state do + "pending" -> :follow_pending + "accept" -> :follow_accept + "reject" -> :follow_reject + _ -> raise "State is not convertible to FollowingRelationshipStateEnum: #{state}" + end end def get(%User{} = follower, %User{} = following) do @@ -35,7 +48,7 @@ def get(%User{} = follower, %User{} = following) do |> Repo.one() end - def update(follower, following, "reject"), do: unfollow(follower, following) + def update(follower, following, :follow_reject), do: unfollow(follower, following) def update(%User{} = follower, %User{} = following, state) do case get(follower, following) do @@ -50,7 +63,7 @@ def update(%User{} = follower, %User{} = following, state) do end end - def follow(%User{} = follower, %User{} = following, state \\ "accept") do + def follow(%User{} = follower, %User{} = following, state \\ :follow_accept) do %__MODULE__{} |> changeset(%{follower: follower, following: following, state: state}) |> Repo.insert(on_conflict: :nothing) @@ -80,7 +93,7 @@ def following_count(%User{} = user) do def get_follow_requests(%User{id: id}) do __MODULE__ |> join(:inner, [r], f in assoc(r, :follower)) - |> where([r], r.state == "pending") + |> where([r], r.state == ^:follow_pending) |> where([r], r.following_id == ^id) |> select([r, f], f) |> Repo.all() @@ -88,7 +101,7 @@ def get_follow_requests(%User{id: id}) do def following?(%User{id: follower_id}, %User{id: followed_id}) do __MODULE__ - |> where(follower_id: ^follower_id, following_id: ^followed_id, state: "accept") + |> where(follower_id: ^follower_id, following_id: ^followed_id, state: ^:follow_accept) |> Repo.exists?() end @@ -97,7 +110,7 @@ def following(%User{} = user) do __MODULE__ |> join(:inner, [r], u in User, on: r.following_id == u.id) |> where([r], r.follower_id == ^user.id) - |> where([r], r.state == "accept") + |> where([r], r.state == ^:follow_accept) |> select([r, u], u.follower_address) |> Repo.all() @@ -157,4 +170,22 @@ def find(following_relationships, follower, following) do fr -> fr.follower_id == follower.id and fr.following_id == following.id end) end + + defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do + changeset + |> validate_change(:following_id, fn _, following_id -> + if following_id == get_field(changeset, :follower_id) do + [target_id: "can't be equal to follower_id"] + else + [] + end + end) + |> validate_change(:follower_id, fn _, follower_id -> + if follower_id == get_field(changeset, :following_id) do + [source_id: "can't be equal to following_id"] + else + [] + end + end) + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d9aa54057..6ffb82045 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -697,7 +697,7 @@ def needs_update?(_), do: true @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()} def maybe_direct_follow(%User{} = follower, %User{local: true, locked: true} = followed) do - follow(follower, followed, "pending") + follow(follower, followed, :follow_pending) end def maybe_direct_follow(%User{} = follower, %User{local: true} = followed) do @@ -717,14 +717,14 @@ def maybe_direct_follow(%User{} = follower, %User{} = followed) do def follow_all(follower, followeds) do followeds |> Enum.reject(fn followed -> blocks?(follower, followed) || blocks?(followed, follower) end) - |> Enum.each(&follow(follower, &1, "accept")) + |> Enum.each(&follow(follower, &1, :follow_accept)) set_cache(follower) end defdelegate following(user), to: FollowingRelationship - def follow(%User{} = follower, %User{} = followed, state \\ "accept") do + def follow(%User{} = follower, %User{} = followed, state \\ :follow_accept) do deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked]) cond do @@ -751,7 +751,7 @@ def unfollow(%User{ap_id: ap_id}, %User{ap_id: ap_id}) do def unfollow(%User{} = follower, %User{} = followed) do case get_follow_state(follower, followed) do - state when state in ["accept", "pending"] -> + state when state in [:follow_pending, :follow_accept] -> FollowingRelationship.unfollow(follower, followed) {:ok, followed} = update_follower_count(followed) @@ -769,6 +769,7 @@ def unfollow(%User{} = follower, %User{} = followed) do defdelegate following?(follower, followed), to: FollowingRelationship + @doc "Returns follow state as FollowingRelationshipStateEnum value" def get_follow_state(%User{} = follower, %User{} = following) do following_relationship = FollowingRelationship.get(follower, following) get_follow_state(follower, following, following_relationship) @@ -782,8 +783,11 @@ def get_follow_state( case {following_relationship, following.local} do {nil, false} -> case Utils.fetch_latest_follow(follower, following) do - %{data: %{"state" => state}} when state in ["pending", "accept"] -> state - _ -> nil + %Activity{data: %{"state" => state}} when state in ["pending", "accept"] -> + FollowingRelationship.state_to_enum(state) + + _ -> + nil end {%{state: state}, _} -> @@ -1282,7 +1286,7 @@ def blocks?(nil, _), do: false def blocks?(%User{} = user, %User{} = target) do blocks_user?(user, target) || - (!User.following?(user, target) && blocks_domain?(user, target)) + (blocks_domain?(user, target) and not User.following?(user, target)) end def blocks_user?(%User{} = user, %User{} = target) do diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index 884e33039..ec88088cf 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -148,7 +148,7 @@ defp compose_query({:followers, %User{id: id}}, query) do as: :relationships, on: r.following_id == ^id and r.follower_id == u.id ) - |> where([relationships: r], r.state == "accept") + |> where([relationships: r], r.state == ^:follow_accept) end defp compose_query({:friends, %User{id: id}}, query) do @@ -158,7 +158,7 @@ defp compose_query({:friends, %User{id: id}}, query) do as: :relationships, on: r.following_id == u.id and r.follower_id == ^id ) - |> where([relationships: r], r.state == "accept") + |> where([relationships: r], r.state == ^:follow_accept) end defp compose_query({:recipients_from_activity, to}, query) do @@ -173,7 +173,7 @@ defp compose_query({:recipients_from_activity, to}, query) do ) |> where( [u, following: f, relationships: r], - u.ap_id in ^to or (f.follower_address in ^to and r.state == "accept") + u.ap_id in ^to or (f.follower_address in ^to and r.state == ^:follow_accept) ) |> distinct(true) end diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index a0b3af432..f54647945 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -33,7 +33,7 @@ def subdomains_regex(domains) when is_list(domains) do @spec subdomain_match?([Regex.t()], String.t()) :: boolean() def subdomain_match?(domains, host) do - Enum.any?(domains, fn domain -> Regex.match?(domain, host) end) + !!Enum.find(domains, fn domain -> Regex.match?(domain, host) end) end @callback describe() :: {:ok | :error, Map.t()} diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index d6549a932..37e485741 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -490,7 +490,8 @@ def handle_incoming( {_, {:ok, follower}} <- {:follow, User.follow(follower, followed)}, {_, {:ok, _}} <- {:follow_state_update, Utils.update_follow_state_for_all(activity, "accept")}, - {:ok, _relationship} <- FollowingRelationship.update(follower, followed, "accept") do + {:ok, _relationship} <- + FollowingRelationship.update(follower, followed, :follow_accept) do ActivityPub.accept(%{ to: [follower.ap_id], actor: followed, @@ -500,7 +501,7 @@ def handle_incoming( else {:user_blocked, true} -> {:ok, _} = Utils.update_follow_state_for_all(activity, "reject") - {:ok, _relationship} = FollowingRelationship.update(follower, followed, "reject") + {:ok, _relationship} = FollowingRelationship.update(follower, followed, :follow_reject) ActivityPub.reject(%{ to: [follower.ap_id], @@ -511,7 +512,7 @@ def handle_incoming( {:follow, {:error, _}} -> {:ok, _} = Utils.update_follow_state_for_all(activity, "reject") - {:ok, _relationship} = FollowingRelationship.update(follower, followed, "reject") + {:ok, _relationship} = FollowingRelationship.update(follower, followed, :follow_reject) ActivityPub.reject(%{ to: [follower.ap_id], @@ -521,7 +522,7 @@ def handle_incoming( }) {:user_locked, true} -> - {:ok, _relationship} = FollowingRelationship.update(follower, followed, "pending") + {:ok, _relationship} = FollowingRelationship.update(follower, followed, :follow_pending) :noop end @@ -541,7 +542,7 @@ def handle_incoming( {:ok, follow_activity} <- get_follow_activity(follow_object, followed), {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"), %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]), - {:ok, _relationship} <- FollowingRelationship.update(follower, followed, "accept") do + {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_accept) do ActivityPub.accept(%{ to: follow_activity.data["to"], type: "Accept", @@ -564,7 +565,7 @@ def handle_incoming( {:ok, follow_activity} <- get_follow_activity(follow_object, followed), {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject"), %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]), - {:ok, _relationship} <- FollowingRelationship.update(follower, followed, "reject"), + {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_reject), {:ok, activity} <- ActivityPub.reject(%{ to: follow_activity.data["to"], diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 2646b9f7b..d530da42c 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -42,7 +42,7 @@ def accept_follow_request(follower, followed) do with {:ok, follower} <- User.follow(follower, followed), %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed), {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"), - {:ok, _relationship} <- FollowingRelationship.update(follower, followed, "accept"), + {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_accept), {:ok, _activity} <- ActivityPub.accept(%{ to: [follower.ap_id], @@ -57,7 +57,7 @@ def accept_follow_request(follower, followed) do def reject_follow_request(follower, followed) do with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed), {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject"), - {:ok, _relationship} <- FollowingRelationship.update(follower, followed, "reject"), + {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_reject), {:ok, _activity} <- ActivityPub.reject(%{ to: [follower.ap_id], diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 0efcabc01..f2dc2a9bd 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -71,7 +71,7 @@ def render( followed_by = if following_relationships do case FollowingRelationship.find(following_relationships, target, reading_user) do - %{state: "accept"} -> true + %{state: :follow_accept} -> true _ -> false end else @@ -81,7 +81,7 @@ def render( # NOTE: adjust UserRelationship.view_relationships_option/2 on new relation-related flags %{ id: to_string(target.id), - following: follow_state == "accept", + following: follow_state == :follow_accept, followed_by: followed_by, blocking: UserRelationship.exists?( @@ -123,7 +123,7 @@ def render( reading_user, &User.subscribed_to?(&2, &1) ), - requested: follow_state == "pending", + requested: follow_state == :follow_pending, domain_blocking: User.blocks_domain?(reading_user, target), showing_reblogs: not UserRelationship.exists?( diff --git a/priv/repo/migrations/20200328124805_change_following_relationships_state_to_integer.exs b/priv/repo/migrations/20200328124805_change_following_relationships_state_to_integer.exs new file mode 100644 index 000000000..d5a431c00 --- /dev/null +++ b/priv/repo/migrations/20200328124805_change_following_relationships_state_to_integer.exs @@ -0,0 +1,29 @@ +defmodule Pleroma.Repo.Migrations.ChangeFollowingRelationshipsStateToInteger do + use Ecto.Migration + + @alter_apps_scopes "ALTER TABLE following_relationships ALTER COLUMN state" + + def up do + execute(""" + #{@alter_apps_scopes} TYPE integer USING + CASE + WHEN state = 'pending' THEN 1 + WHEN state = 'accept' THEN 2 + WHEN state = 'reject' THEN 3 + ELSE 0 + END; + """) + end + + def down do + execute(""" + #{@alter_apps_scopes} TYPE varchar(255) USING + CASE + WHEN state = 1 THEN 'pending' + WHEN state = 2 THEN 'accept' + WHEN state = 3 THEN 'reject' + ELSE '' + END; + """) + end +end diff --git a/priv/repo/migrations/20200328130139_add_following_relationships_following_id_index.exs b/priv/repo/migrations/20200328130139_add_following_relationships_following_id_index.exs new file mode 100644 index 000000000..4c9faf48f --- /dev/null +++ b/priv/repo/migrations/20200328130139_add_following_relationships_following_id_index.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.AddFollowingRelationshipsFollowingIdIndex do + use Ecto.Migration + + # [:follower_index] index is useless because of [:follower_id, :following_id] index + # [:following_id] index makes sense because of user's followers-targeted queries + def change do + drop_if_exists(index(:following_relationships, [:follower_id])) + + create_if_not_exists(drop_if_exists(index(:following_relationships, [:following_id]))) + end +end diff --git a/test/following_relationship_test.exs b/test/following_relationship_test.exs index 865bb3838..17a468abb 100644 --- a/test/following_relationship_test.exs +++ b/test/following_relationship_test.exs @@ -15,28 +15,28 @@ defmodule Pleroma.FollowingRelationshipTest do test "returns following addresses without internal.fetch" do user = insert(:user) fetch_actor = InternalFetchActor.get_actor() - FollowingRelationship.follow(fetch_actor, user, "accept") + FollowingRelationship.follow(fetch_actor, user, :follow_accept) assert FollowingRelationship.following(fetch_actor) == [user.follower_address] end test "returns following addresses without relay" do user = insert(:user) relay_actor = Relay.get_actor() - FollowingRelationship.follow(relay_actor, user, "accept") + FollowingRelationship.follow(relay_actor, user, :follow_accept) assert FollowingRelationship.following(relay_actor) == [user.follower_address] end test "returns following addresses without remote user" do user = insert(:user) actor = insert(:user, local: false) - FollowingRelationship.follow(actor, user, "accept") + FollowingRelationship.follow(actor, user, :follow_accept) assert FollowingRelationship.following(actor) == [user.follower_address] end test "returns following addresses with local user" do user = insert(:user) actor = insert(:user, local: true) - FollowingRelationship.follow(actor, user, "accept") + FollowingRelationship.follow(actor, user, :follow_accept) assert FollowingRelationship.following(actor) == [ actor.follower_address, diff --git a/test/tasks/user_test.exs b/test/tasks/user_test.exs index b45f37263..8df835b56 100644 --- a/test/tasks/user_test.exs +++ b/test/tasks/user_test.exs @@ -140,7 +140,7 @@ test "no user to toggle" do test "user is unsubscribed" do followed = insert(:user) user = insert(:user) - User.follow(user, followed, "accept") + User.follow(user, followed, :follow_accept) Mix.Tasks.Pleroma.User.run(["unsubscribe", user.nickname]) diff --git a/test/user_test.exs b/test/user_test.exs index 8055ebd08..e7dfc5980 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -194,7 +194,8 @@ test "doesn't return already accepted or duplicate follow requests" do CommonAPI.follow(pending_follower, locked) CommonAPI.follow(pending_follower, locked) CommonAPI.follow(accepted_follower, locked) - Pleroma.FollowingRelationship.update(accepted_follower, locked, "accept") + + Pleroma.FollowingRelationship.update(accepted_follower, locked, :follow_accept) assert [^pending_follower] = User.get_follow_requests(locked) end @@ -319,7 +320,7 @@ test "unfollow with syncronizes external user" do following_address: "http://localhost:4001/users/fuser2/following" }) - {:ok, user} = User.follow(user, followed, "accept") + {:ok, user} = User.follow(user, followed, :follow_accept) {:ok, user, _activity} = User.unfollow(user, followed) @@ -332,7 +333,7 @@ test "unfollow takes a user and another user" do followed = insert(:user) user = insert(:user) - {:ok, user} = User.follow(user, followed, "accept") + {:ok, user} = User.follow(user, followed, :follow_accept) assert User.following(user) == [user.follower_address, followed.follower_address] @@ -353,7 +354,7 @@ test "unfollow doesn't unfollow yourself" do test "test if a user is following another user" do followed = insert(:user) user = insert(:user) - User.follow(user, followed, "accept") + User.follow(user, followed, :follow_accept) assert User.following?(user, followed) refute User.following?(followed, user) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index b2cabbd30..b998f0d78 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -1622,7 +1622,7 @@ test "it upgrades a user to activitypub" do }) user_two = insert(:user) - Pleroma.FollowingRelationship.follow(user_two, user, "accept") + Pleroma.FollowingRelationship.follow(user_two, user, :follow_accept) {:ok, activity} = CommonAPI.post(user, %{"status" => "test"}) {:ok, unrelated_activity} = CommonAPI.post(user_two, %{"status" => "test"}) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index 0da0bd2e2..e53a7cedd 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -562,7 +562,7 @@ test "cancels a pending follow for a local user" do assert {:ok, follower, followed, %{id: activity_id, data: %{"state" => "pending"}}} = CommonAPI.follow(follower, followed) - assert User.get_follow_state(follower, followed) == "pending" + assert User.get_follow_state(follower, followed) == :follow_pending assert {:ok, follower} = CommonAPI.unfollow(follower, followed) assert User.get_follow_state(follower, followed) == nil @@ -584,7 +584,7 @@ test "cancels a pending follow for a remote user" do assert {:ok, follower, followed, %{id: activity_id, data: %{"state" => "pending"}}} = CommonAPI.follow(follower, followed) - assert User.get_follow_state(follower, followed) == "pending" + assert User.get_follow_state(follower, followed) == :follow_pending assert {:ok, follower} = CommonAPI.unfollow(follower, followed) assert User.get_follow_state(follower, followed) == nil diff --git a/test/web/mastodon_api/controllers/follow_request_controller_test.exs b/test/web/mastodon_api/controllers/follow_request_controller_test.exs index dd848821a..d8dbe4800 100644 --- a/test/web/mastodon_api/controllers/follow_request_controller_test.exs +++ b/test/web/mastodon_api/controllers/follow_request_controller_test.exs @@ -21,7 +21,7 @@ test "/api/v1/follow_requests works", %{user: user, conn: conn} do other_user = insert(:user) {:ok, _activity} = ActivityPub.follow(other_user, user) - {:ok, other_user} = User.follow(other_user, user, "pending") + {:ok, other_user} = User.follow(other_user, user, :follow_pending) assert User.following?(other_user, user) == false @@ -35,7 +35,7 @@ test "/api/v1/follow_requests/:id/authorize works", %{user: user, conn: conn} do other_user = insert(:user) {:ok, _activity} = ActivityPub.follow(other_user, user) - {:ok, other_user} = User.follow(other_user, user, "pending") + {:ok, other_user} = User.follow(other_user, user, :follow_pending) user = User.get_cached_by_id(user.id) other_user = User.get_cached_by_id(other_user.id) diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs index a5d6e8ecf..ad8ce030b 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -197,7 +197,7 @@ test "it doesn't send to user if recipients invalid and thread containment is en Pleroma.Config.put([:instance, :skip_thread_containment], false) author = insert(:user) user = insert(:user) - User.follow(user, author, "accept") + User.follow(user, author, :follow_accept) activity = insert(:note_activity, @@ -220,7 +220,7 @@ test "it sends message if recipients invalid and thread containment is disabled" Pleroma.Config.put([:instance, :skip_thread_containment], true) author = insert(:user) user = insert(:user) - User.follow(user, author, "accept") + User.follow(user, author, :follow_accept) activity = insert(:note_activity, @@ -243,7 +243,7 @@ test "it sends message if recipients invalid and thread containment is enabled b Pleroma.Config.put([:instance, :skip_thread_containment], false) author = insert(:user) user = insert(:user, skip_thread_containment: true) - User.follow(user, author, "accept") + User.follow(user, author, :follow_accept) activity = insert(:note_activity, From 9c94b6a327118d8c7ea21355d6c378ef31c54321 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 30 Mar 2020 19:08:37 +0300 Subject: [PATCH 22/84] [#2332] Misc. fixes per code change requests. --- lib/pleroma/web/activity_pub/mrf.ex | 2 +- ...328130139_add_following_relationships_following_id_index.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index f54647945..a0b3af432 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -33,7 +33,7 @@ def subdomains_regex(domains) when is_list(domains) do @spec subdomain_match?([Regex.t()], String.t()) :: boolean() def subdomain_match?(domains, host) do - !!Enum.find(domains, fn domain -> Regex.match?(domain, host) end) + Enum.any?(domains, fn domain -> Regex.match?(domain, host) end) end @callback describe() :: {:ok | :error, Map.t()} diff --git a/priv/repo/migrations/20200328130139_add_following_relationships_following_id_index.exs b/priv/repo/migrations/20200328130139_add_following_relationships_following_id_index.exs index 4c9faf48f..884832f84 100644 --- a/priv/repo/migrations/20200328130139_add_following_relationships_following_id_index.exs +++ b/priv/repo/migrations/20200328130139_add_following_relationships_following_id_index.exs @@ -6,6 +6,6 @@ defmodule Pleroma.Repo.Migrations.AddFollowingRelationshipsFollowingIdIndex do def change do drop_if_exists(index(:following_relationships, [:follower_id])) - create_if_not_exists(drop_if_exists(index(:following_relationships, [:following_id]))) + create_if_not_exists(index(:following_relationships, [:following_id])) end end From ea9c57b26ed463622e4489736fcddb8fca1b3341 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 31 Mar 2020 09:21:42 +0300 Subject: [PATCH 23/84] [#2332] Misc. improvements per code change requests. --- lib/pleroma/ecto_enums.ex | 4 +- lib/pleroma/following_relationship.ex | 42 +++++++++++-------- lib/pleroma/user.ex | 2 +- lib/pleroma/user_relationship.ex | 33 +++++++++------ ...llowing_relationships_state_to_integer.exs | 6 +-- 5 files changed, 52 insertions(+), 35 deletions(-) diff --git a/lib/pleroma/ecto_enums.ex b/lib/pleroma/ecto_enums.ex index b98ac4ba1..6fc47620c 100644 --- a/lib/pleroma/ecto_enums.ex +++ b/lib/pleroma/ecto_enums.ex @@ -4,7 +4,7 @@ import EctoEnum -defenum(UserRelationshipTypeEnum, +defenum(Pleroma.UserRelationship.Type, block: 1, mute: 2, reblog_mute: 3, @@ -12,7 +12,7 @@ inverse_subscription: 5 ) -defenum(FollowingRelationshipStateEnum, +defenum(Pleroma.FollowingRelationship.State, follow_pending: 1, follow_accept: 2, follow_reject: 3 diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index a28da8bec..9ccf40495 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -8,12 +8,13 @@ defmodule Pleroma.FollowingRelationship do import Ecto.Changeset import Ecto.Query + alias Ecto.Changeset alias FlakeId.Ecto.CompatType alias Pleroma.Repo alias Pleroma.User schema "following_relationships" do - field(:state, FollowingRelationshipStateEnum, default: :follow_pending) + field(:state, Pleroma.FollowingRelationship.State, default: :follow_pending) belongs_to(:follower, User, type: CompatType) belongs_to(:following, User, type: CompatType) @@ -33,13 +34,12 @@ def changeset(%__MODULE__{} = following_relationship, attrs) do |> validate_not_self_relationship() end - def state_to_enum(state) when is_binary(state) do - case state do - "pending" -> :follow_pending - "accept" -> :follow_accept - "reject" -> :follow_reject - _ -> raise "State is not convertible to FollowingRelationshipStateEnum: #{state}" - end + def state_to_enum(state) when state in ["pending", "accept", "reject"] do + String.to_existing_atom("follow_#{state}") + end + + def state_to_enum(state) do + raise "State is not convertible to Pleroma.FollowingRelationship.State: #{state}" end def get(%User{} = follower, %User{} = following) do @@ -171,16 +171,14 @@ def find(following_relationships, follower, following) do end) end - defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do + defp validate_not_self_relationship(%Changeset{} = changeset) do changeset - |> validate_change(:following_id, fn _, following_id -> - if following_id == get_field(changeset, :follower_id) do - [target_id: "can't be equal to follower_id"] - else - [] - end - end) - |> validate_change(:follower_id, fn _, follower_id -> + |> validate_follower_id_following_id_inequality() + |> validate_following_id_follower_id_inequality() + end + + defp validate_follower_id_following_id_inequality(%Changeset{} = changeset) do + validate_change(changeset, :follower_id, fn _, follower_id -> if follower_id == get_field(changeset, :following_id) do [source_id: "can't be equal to following_id"] else @@ -188,4 +186,14 @@ defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do end end) end + + defp validate_following_id_follower_id_inequality(%Changeset{} = changeset) do + validate_change(changeset, :following_id, fn _, following_id -> + if following_id == get_field(changeset, :follower_id) do + [target_id: "can't be equal to follower_id"] + else + [] + end + end) + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 6ffb82045..4f3abd7d5 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -769,7 +769,7 @@ def unfollow(%User{} = follower, %User{} = followed) do defdelegate following?(follower, followed), to: FollowingRelationship - @doc "Returns follow state as FollowingRelationshipStateEnum value" + @doc "Returns follow state as Pleroma.FollowingRelationship.State value" def get_follow_state(%User{} = follower, %User{} = following) do following_relationship = FollowingRelationship.get(follower, following) get_follow_state(follower, following, following_relationship) diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 18a5eec72..ad0d303b1 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -8,6 +8,7 @@ defmodule Pleroma.UserRelationship do import Ecto.Changeset import Ecto.Query + alias Ecto.Changeset alias Pleroma.FollowingRelationship alias Pleroma.Repo alias Pleroma.User @@ -16,12 +17,12 @@ defmodule Pleroma.UserRelationship do schema "user_relationships" do belongs_to(:source, User, type: FlakeId.Ecto.CompatType) belongs_to(:target, User, type: FlakeId.Ecto.CompatType) - field(:relationship_type, UserRelationshipTypeEnum) + field(:relationship_type, Pleroma.UserRelationship.Type) timestamps(updated_at: false) end - for relationship_type <- Keyword.keys(UserRelationshipTypeEnum.__enum_map__()) do + for relationship_type <- Keyword.keys(Pleroma.UserRelationship.Type.__enum_map__()) do # `def create_block/2`, `def create_mute/2`, `def create_reblog_mute/2`, # `def create_notification_mute/2`, `def create_inverse_subscription/2` def unquote(:"create_#{relationship_type}")(source, target), @@ -40,7 +41,7 @@ def unquote(:"#{relationship_type}_exists?")(source, target), def user_relationship_types, do: Keyword.keys(user_relationship_mappings()) - def user_relationship_mappings, do: UserRelationshipTypeEnum.__enum_map__() + def user_relationship_mappings, do: Pleroma.UserRelationship.Type.__enum_map__() def changeset(%UserRelationship{} = user_relationship, params \\ %{}) do user_relationship @@ -147,16 +148,14 @@ def view_relationships_option(%User{} = reading_user, actors) do %{user_relationships: user_relationships, following_relationships: following_relationships} end - defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do + defp validate_not_self_relationship(%Changeset{} = changeset) do changeset - |> validate_change(:target_id, fn _, target_id -> - if target_id == get_field(changeset, :source_id) do - [target_id: "can't be equal to source_id"] - else - [] - end - end) - |> validate_change(:source_id, fn _, source_id -> + |> validate_source_id_target_id_inequality() + |> validate_target_id_source_id_inequality() + end + + defp validate_source_id_target_id_inequality(%Changeset{} = changeset) do + validate_change(changeset, :source_id, fn _, source_id -> if source_id == get_field(changeset, :target_id) do [source_id: "can't be equal to target_id"] else @@ -164,4 +163,14 @@ defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do end end) end + + defp validate_target_id_source_id_inequality(%Changeset{} = changeset) do + validate_change(changeset, :target_id, fn _, target_id -> + if target_id == get_field(changeset, :source_id) do + [target_id: "can't be equal to source_id"] + else + [] + end + end) + end end diff --git a/priv/repo/migrations/20200328124805_change_following_relationships_state_to_integer.exs b/priv/repo/migrations/20200328124805_change_following_relationships_state_to_integer.exs index d5a431c00..2b0820f3f 100644 --- a/priv/repo/migrations/20200328124805_change_following_relationships_state_to_integer.exs +++ b/priv/repo/migrations/20200328124805_change_following_relationships_state_to_integer.exs @@ -1,11 +1,11 @@ defmodule Pleroma.Repo.Migrations.ChangeFollowingRelationshipsStateToInteger do use Ecto.Migration - @alter_apps_scopes "ALTER TABLE following_relationships ALTER COLUMN state" + @alter_following_relationship_state "ALTER TABLE following_relationships ALTER COLUMN state" def up do execute(""" - #{@alter_apps_scopes} TYPE integer USING + #{@alter_following_relationship_state} TYPE integer USING CASE WHEN state = 'pending' THEN 1 WHEN state = 'accept' THEN 2 @@ -17,7 +17,7 @@ def up do def down do execute(""" - #{@alter_apps_scopes} TYPE varchar(255) USING + #{@alter_following_relationship_state} TYPE varchar(255) USING CASE WHEN state = 1 THEN 'pending' WHEN state = 2 THEN 'accept' From 2f2bd7fe72f474b7177c751a2dc3af716622ba91 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 1 Apr 2020 19:49:09 +0300 Subject: [PATCH 24/84] Ability to control the output of account/pleroma/relationship in statuses in order to improve the rendering performance. See `[:extensions, output_relationships_in_statuses_by_default]` setting and `with_relationships` param. --- CHANGELOG.md | 3 +- benchmarks/load_testing/fetcher.ex | 21 +++++++--- config/config.exs | 2 + config/description.exs | 16 ++++++++ lib/mix/tasks/pleroma/benchmark.ex | 3 +- lib/pleroma/user_relationship.ex | 18 +++++++-- .../web/admin_api/admin_api_controller.ex | 6 +-- .../web/admin_api/views/report_view.ex | 7 +++- lib/pleroma/web/common_api/activity_draft.ex | 2 +- lib/pleroma/web/controller_helper.ex | 24 +++++++++-- .../controllers/account_controller.ex | 15 ++++++- .../controllers/notification_controller.ex | 8 +++- .../controllers/search_controller.ex | 24 ++++++++--- .../controllers/status_controller.ex | 26 +++++++++--- .../controllers/timeline_controller.ex | 40 +++++++++++++++---- .../web/mastodon_api/views/account_view.ex | 15 ++++--- .../mastodon_api/views/notification_view.ex | 30 ++++++++------ .../web/mastodon_api/views/status_view.ex | 11 +++-- .../controllers/account_controller.ex | 9 ++++- .../controllers/pleroma_api_controller.ex | 17 ++++++-- .../20190414125034_migrate_old_bookmarks.exs | 1 - .../20190711042021_create_safe_jsonb_set.exs | 1 - .../notification_controller_test.exs | 20 ++++++++++ .../controllers/status_controller_test.exs | 6 ++- .../controllers/timeline_controller_test.exs | 29 ++++++++++++-- 25 files changed, 278 insertions(+), 76 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 350e03894..a391bf1fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list. - Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses. +- Configuration: `:extensions/:output_relationships_in_statuses_by_default` option (if `false`, disables the output of account/pleroma/relationship for statuses and notifications by default, breaking the compatibility with older PleromaFE versions).
API Changes - Mastodon API: Support for `include_types` in `/api/v1/notifications`. @@ -20,7 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [2.0.0] - 2019-03-08 ### Security -- Mastodon API: Fix being able to request enourmous amount of statuses in timelines leading to DoS. Now limited to 40 per request. +- Mastodon API: Fix being able to request enormous amount of statuses in timelines leading to DoS. Now limited to 40 per request. ### Removed - **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media` diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index bd65ac84f..786929ace 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -386,47 +386,56 @@ defp render_timelines(user) do favourites = ActivityPub.fetch_favourites(user) + output_relationships = + !!Pleroma.Config.get([:extensions, :output_relationships_in_statuses_by_default]) + Benchee.run( %{ "Rendering home timeline" => fn -> StatusView.render("index.json", %{ activities: home_activities, for: user, - as: :activity + as: :activity, + skip_relationships: !output_relationships }) end, "Rendering direct timeline" => fn -> StatusView.render("index.json", %{ activities: direct_activities, for: user, - as: :activity + as: :activity, + skip_relationships: !output_relationships }) end, "Rendering public timeline" => fn -> StatusView.render("index.json", %{ activities: public_activities, for: user, - as: :activity + as: :activity, + skip_relationships: !output_relationships }) end, "Rendering tag timeline" => fn -> StatusView.render("index.json", %{ activities: tag_activities, for: user, - as: :activity + as: :activity, + skip_relationships: !output_relationships }) end, "Rendering notifications" => fn -> Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{ notifications: notifications, - for: user + for: user, + skip_relationships: !output_relationships }) end, "Rendering favourites timeline" => fn -> StatusView.render("index.json", %{ activities: favourites, for: user, - as: :activity + as: :activity, + skip_relationships: !output_relationships }) end }, diff --git a/config/config.exs b/config/config.exs index 2ab939107..73bf658fe 100644 --- a/config/config.exs +++ b/config/config.exs @@ -262,6 +262,8 @@ extended_nickname_format: true, cleanup_attachments: false +config :pleroma, :extensions, output_relationships_in_statuses_by_default: true + config :pleroma, :feed, post_title: %{ max_length: 100, diff --git a/config/description.exs b/config/description.exs index 9612adba7..d127f8f20 100644 --- a/config/description.exs +++ b/config/description.exs @@ -121,6 +121,22 @@ } ] }, + %{ + group: :pleroma, + key: :extensions, + type: :group, + description: "Pleroma-specific extensions", + children: [ + %{ + key: :output_relationships_in_statuses_by_default, + type: :beeolean, + description: + "If `true`, outputs account/pleroma/relationship map for each rendered status / notification (for all clients). " <> + "If `false`, outputs the above only if `with_relationships` param is tru-ish " <> + "(that breaks compatibility with older PleromaFE versions which do not send this param but expect the output)." + } + ] + }, %{ group: :pleroma, key: Pleroma.Uploaders.Local, diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex index a4885b70c..b2bbe40ac 100644 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ b/lib/mix/tasks/pleroma/benchmark.ex @@ -67,7 +67,8 @@ def run(["render_timeline", nickname | _] = args) do Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ activities: activities, for: user, - as: :activity + as: :activity, + skip_relationships: true }) end }, diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index 18a5eec72..d42dc250e 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -129,17 +129,27 @@ def exists?(dictionary, rel_type, source, target, func) do end @doc ":relationships option for StatusView / AccountView / NotificationView" - def view_relationships_option(nil = _reading_user, _actors) do + def view_relationships_option(reading_user, actors, opts \\ []) + + def view_relationships_option(nil = _reading_user, _actors, _opts) do %{user_relationships: [], following_relationships: []} end - def view_relationships_option(%User{} = reading_user, actors) do + def view_relationships_option(%User{} = reading_user, actors, opts) do + {source_to_target_rel_types, target_to_source_rel_types} = + if opts[:source_mutes_only] do + # This option is used for rendering statuses (FE needs `muted` flag for each one anyways) + {[:mute], []} + else + {[:block, :mute, :notification_mute, :reblog_mute], [:block, :inverse_subscription]} + end + user_relationships = UserRelationship.dictionary( [reading_user], actors, - [:block, :mute, :notification_mute, :reblog_mute], - [:block, :inverse_subscription] + source_to_target_rel_types, + target_to_source_rel_types ) following_relationships = FollowingRelationship.all_between_user_sets([reading_user], actors) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index ca5439920..747d97f80 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -258,7 +258,7 @@ def list_instance_statuses(conn, %{"instance" => instance} = params) do conn |> put_view(Pleroma.Web.AdminAPI.StatusView) - |> render("index.json", %{activities: activities, as: :activity}) + |> render("index.json", %{activities: activities, as: :activity, skip_relationships: false}) end def list_user_statuses(conn, %{"nickname" => nickname} = params) do @@ -277,7 +277,7 @@ def list_user_statuses(conn, %{"nickname" => nickname} = params) do conn |> put_view(StatusView) - |> render("index.json", %{activities: activities, as: :activity}) + |> render("index.json", %{activities: activities, as: :activity, skip_relationships: false}) else _ -> {:error, :not_found} end @@ -801,7 +801,7 @@ def list_statuses(%{assigns: %{user: _admin}} = conn, params) do conn |> put_view(Pleroma.Web.AdminAPI.StatusView) - |> render("index.json", %{activities: activities, as: :activity}) + |> render("index.json", %{activities: activities, as: :activity, skip_relationships: false}) end def status_update(%{assigns: %{user: admin}} = conn, %{"id" => id} = params) do diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index ca0bcebc7..d50969b2a 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -38,7 +38,12 @@ def render("show.json", %{report: report, user: user, account: account, statuses actor: merge_account_views(user), content: content, created_at: created_at, - statuses: StatusView.render("index.json", %{activities: statuses, as: :activity}), + statuses: + StatusView.render("index.json", %{ + activities: statuses, + as: :activity, + skip_relationships: false + }), state: report.data["state"], notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}) } diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index c4356f93b..c1cd15bb2 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -187,7 +187,7 @@ defp object(draft) do end defp preview?(draft) do - preview? = Pleroma.Web.ControllerHelper.truthy_param?(draft.params["preview"]) || false + preview? = Pleroma.Web.ControllerHelper.truthy_param?(draft.params["preview"]) %__MODULE__{draft | preview?: preview?} end diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex index b49523ec3..4780081b2 100644 --- a/lib/pleroma/web/controller_helper.ex +++ b/lib/pleroma/web/controller_helper.ex @@ -5,10 +5,18 @@ defmodule Pleroma.Web.ControllerHelper do use Pleroma.Web, :controller - # As in MastoAPI, per https://api.rubyonrails.org/classes/ActiveModel/Type/Boolean.html + alias Pleroma.Config + + # As in Mastodon API, per https://api.rubyonrails.org/classes/ActiveModel/Type/Boolean.html @falsy_param_values [false, 0, "0", "f", "F", "false", "False", "FALSE", "off", "OFF"] - def truthy_param?(blank_value) when blank_value in [nil, ""], do: nil - def truthy_param?(value), do: value not in @falsy_param_values + + def explicitly_falsy_param?(value), do: value in @falsy_param_values + + # Note: `nil` and `""` are considered falsy values in Pleroma + def falsy_param?(value), + do: explicitly_falsy_param?(value) or value in [nil, ""] + + def truthy_param?(value), do: not falsy_param?(value) def json_response(conn, status, json) do conn @@ -96,4 +104,14 @@ def try_render(conn, _, _) do def put_if_exist(map, _key, nil), do: map def put_if_exist(map, key, value), do: Map.put(map, key, value) + + @doc "Whether to skip rendering `[:account][:pleroma][:relationship]`for statuses/notifications" + def skip_relationships?(params) do + if Config.get([:extensions, :output_relationships_in_statuses_by_default]) do + false + else + # BREAKING: older PleromaFE versions do not send this param but _do_ expect relationships. + not truthy_param?(params["with_relationships"]) + end + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 21bc3d5a5..7da1a11f6 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -6,7 +6,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do use Pleroma.Web, :controller import Pleroma.Web.ControllerHelper, - only: [add_link_headers: 2, truthy_param?: 1, assign_account_by_id: 2, json_response: 3] + only: [ + add_link_headers: 2, + truthy_param?: 1, + assign_account_by_id: 2, + json_response: 3, + skip_relationships?: 1 + ] alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Plugs.RateLimiter @@ -237,7 +243,12 @@ def statuses(%{assigns: %{user: reading_user}} = conn, params) do conn |> add_link_headers(activities) |> put_view(StatusView) - |> render("index.json", activities: activities, for: reading_user, as: :activity) + |> render("index.json", + activities: activities, + for: reading_user, + as: :activity, + skip_relationships: skip_relationships?(params) + ) else _e -> render_error(conn, :not_found, "Can't find user") end diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index 0c9218454..c7e808253 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do use Pleroma.Web, :controller - import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2, skip_relationships?: 1] alias Pleroma.Notification alias Pleroma.Plugs.OAuthScopesPlug @@ -45,7 +45,11 @@ def index(%{assigns: %{user: user}} = conn, params) do conn |> add_link_headers(notifications) - |> render("index.json", notifications: notifications, for: user) + |> render("index.json", + notifications: notifications, + for: user, + skip_relationships: skip_relationships?(params) + ) end # GET /api/v1/notifications/:id diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index fcab4ef63..c258742dd 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -5,13 +5,14 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do use Pleroma.Web, :controller + import Pleroma.Web.ControllerHelper, only: [fetch_integer_param: 2, skip_relationships?: 1] + alias Pleroma.Activity alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Plugs.RateLimiter alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web - alias Pleroma.Web.ControllerHelper alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MastodonAPI.StatusView @@ -66,10 +67,11 @@ defp do_search(version, %{assigns: %{user: user}} = conn, %{"q" => query} = para defp search_options(params, user) do [ + skip_relationships: skip_relationships?(params), resolve: params["resolve"] == "true", following: params["following"] == "true", - limit: ControllerHelper.fetch_integer_param(params, "limit"), - offset: ControllerHelper.fetch_integer_param(params, "offset"), + limit: fetch_integer_param(params, "limit"), + offset: fetch_integer_param(params, "offset"), type: params["type"], author: get_author(params), for_user: user @@ -79,12 +81,24 @@ defp search_options(params, user) do defp resource_search(_, "accounts", query, options) do accounts = with_fallback(fn -> User.search(query, options) end) - AccountView.render("index.json", users: accounts, for: options[:for_user], as: :user) + + AccountView.render("index.json", + users: accounts, + for: options[:for_user], + as: :user, + skip_relationships: false + ) end defp resource_search(_, "statuses", query, options) do statuses = with_fallback(fn -> Activity.search(options[:for_user], query, options) end) - StatusView.render("index.json", activities: statuses, for: options[:for_user], as: :activity) + + StatusView.render("index.json", + activities: statuses, + for: options[:for_user], + as: :activity, + skip_relationships: options[:skip_relationships] + ) end defp resource_search(:v2, "hashtags", query, _options) do diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 37afe6949..eb3d90aeb 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -5,7 +5,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do use Pleroma.Web, :controller - import Pleroma.Web.ControllerHelper, only: [try_render: 3, add_link_headers: 2] + import Pleroma.Web.ControllerHelper, + only: [try_render: 3, add_link_headers: 2, skip_relationships?: 1] require Ecto.Query @@ -101,7 +102,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do `ids` query param is required """ - def index(%{assigns: %{user: user}} = conn, %{"ids" => ids}) do + def index(%{assigns: %{user: user}} = conn, %{"ids" => ids} = params) do limit = 100 activities = @@ -110,7 +111,12 @@ def index(%{assigns: %{user: user}} = conn, %{"ids" => ids}) do |> Activity.all_by_ids_with_object() |> Enum.filter(&Visibility.visible_for_user?(&1, user)) - render(conn, "index.json", activities: activities, for: user, as: :activity) + render(conn, "index.json", + activities: activities, + for: user, + as: :activity, + skip_relationships: skip_relationships?(params) + ) end @doc """ @@ -360,7 +366,12 @@ def favourites(%{assigns: %{user: user}} = conn, params) do conn |> add_link_headers(activities) - |> render("index.json", activities: activities, for: user, as: :activity) + |> render("index.json", + activities: activities, + for: user, + as: :activity, + skip_relationships: skip_relationships?(params) + ) end @doc "GET /api/v1/bookmarks" @@ -378,6 +389,11 @@ def bookmarks(%{assigns: %{user: user}} = conn, params) do conn |> add_link_headers(bookmarks) - |> render("index.json", %{activities: activities, for: user, as: :activity}) + |> render("index.json", + activities: activities, + for: user, + as: :activity, + skip_relationships: skip_relationships?(params) + ) end end diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index 91f41416d..b3c58005e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do use Pleroma.Web, :controller import Pleroma.Web.ControllerHelper, - only: [add_link_headers: 2, add_link_headers: 3, truthy_param?: 1] + only: [add_link_headers: 2, add_link_headers: 3, truthy_param?: 1, skip_relationships?: 1] alias Pleroma.Pagination alias Pleroma.Plugs.OAuthScopesPlug @@ -14,9 +14,8 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub - # TODO: Replace with a macro when there is a Phoenix release with + # TODO: Replace with a macro when there is a Phoenix release with the following commit in it: # https://github.com/phoenixframework/phoenix/commit/2e8c63c01fec4dde5467dbbbf9705ff9e780735e - # in it plug(RateLimiter, [name: :timeline, bucket_name: :direct_timeline] when action == :direct) plug(RateLimiter, [name: :timeline, bucket_name: :public_timeline] when action == :public) @@ -49,7 +48,12 @@ def home(%{assigns: %{user: user}} = conn, params) do conn |> add_link_headers(activities) - |> render("index.json", activities: activities, for: user, as: :activity) + |> render("index.json", + activities: activities, + for: user, + as: :activity, + skip_relationships: skip_relationships?(params) + ) end # GET /api/v1/timelines/direct @@ -68,7 +72,12 @@ def direct(%{assigns: %{user: user}} = conn, params) do conn |> add_link_headers(activities) - |> render("index.json", activities: activities, for: user, as: :activity) + |> render("index.json", + activities: activities, + for: user, + as: :activity, + skip_relationships: skip_relationships?(params) + ) end # GET /api/v1/timelines/public @@ -95,7 +104,12 @@ def public(%{assigns: %{user: user}} = conn, params) do conn |> add_link_headers(activities, %{"local" => local_only}) - |> render("index.json", activities: activities, for: user, as: :activity) + |> render("index.json", + activities: activities, + for: user, + as: :activity, + skip_relationships: skip_relationships?(params) + ) else render_error(conn, :unauthorized, "authorization required for timeline view") end @@ -140,7 +154,12 @@ def hashtag(%{assigns: %{user: user}} = conn, params) do conn |> add_link_headers(activities, %{"local" => local_only}) - |> render("index.json", activities: activities, for: user, as: :activity) + |> render("index.json", + activities: activities, + for: user, + as: :activity, + skip_relationships: skip_relationships?(params) + ) end # GET /api/v1/timelines/list/:list_id @@ -164,7 +183,12 @@ def list(%{assigns: %{user: user}} = conn, %{"list_id" => id} = params) do |> ActivityPub.fetch_activities_bounded(following, params) |> Enum.reverse() - render(conn, "index.json", activities: activities, for: user, as: :activity) + render(conn, "index.json", + activities: activities, + for: user, + as: :activity, + skip_relationships: skip_relationships?(params) + ) else _e -> render_error(conn, :forbidden, "Error.") end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index c482bba64..b20a00a89 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do alias Pleroma.Web.MediaProxy def render("index.json", %{users: users} = opts) do + # Note: :skip_relationships option is currently intentionally not supported for accounts relationships_opt = cond do Map.has_key?(opts, :relationships) -> @@ -190,11 +191,15 @@ defp do_render("show.json", %{user: user} = opts) do end) relationship = - render("relationship.json", %{ - user: opts[:for], - target: user, - relationships: opts[:relationships] - }) + if opts[:skip_relationships] do + %{} + else + render("relationship.json", %{ + user: opts[:for], + target: user, + relationships: opts[:relationships] + }) + end %{ id: to_string(user.id), diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 89f5734ff..78d187f9a 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -51,14 +51,15 @@ def render("index.json", %{notifications: notifications, for: reading_user} = op |> Enum.filter(& &1) |> Kernel.++(move_activities_targets) - UserRelationship.view_relationships_option(reading_user, actors) + UserRelationship.view_relationships_option(reading_user, actors, + source_mutes_only: opts[:skip_relationships] + ) end - opts = %{ - for: reading_user, - parent_activities: parent_activities, - relationships: relationships_opt - } + opts = + opts + |> Map.put(:parent_activities, parent_activities) + |> Map.put(:relationships, relationships_opt) safe_render_many(notifications, NotificationView, "show.json", opts) end @@ -82,12 +83,16 @@ def render( mastodon_type = Activity.mastodon_notification_type(activity) + render_opts = %{ + relationships: opts[:relationships], + skip_relationships: opts[:skip_relationships] + } + with %{id: _} = account <- - AccountView.render("show.json", %{ - user: actor, - for: reading_user, - relationships: opts[:relationships] - }) do + AccountView.render( + "show.json", + Map.merge(render_opts, %{user: actor, for: reading_user}) + ) do response = %{ id: to_string(notification.id), type: mastodon_type, @@ -98,8 +103,6 @@ def render( } } - render_opts = %{relationships: opts[:relationships]} - case mastodon_type do "mention" -> put_status(response, activity, reading_user, render_opts) @@ -111,6 +114,7 @@ def render( put_status(response, parent_activity_fn.(), reading_user, render_opts) "move" -> + # Note: :skip_relationships option being applied to _account_ rendering (here) put_target(response, activity, reading_user, render_opts) "follow" -> diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 82326986c..9cbd31878 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -97,7 +97,9 @@ def render("index.json", opts) do true -> actors = Enum.map(activities ++ parent_activities, &get_user(&1.data["actor"])) - UserRelationship.view_relationships_option(opts[:for], actors) + UserRelationship.view_relationships_option(opts[:for], actors, + source_mutes_only: opts[:skip_relationships] + ) end opts = @@ -151,7 +153,8 @@ def render( AccountView.render("show.json", %{ user: user, for: opts[:for], - relationships: opts[:relationships] + relationships: opts[:relationships], + skip_relationships: opts[:skip_relationships] }), in_reply_to_id: nil, in_reply_to_account_id: nil, @@ -299,6 +302,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} _ -> [] end + # Status muted state (would do 1 request per status unless user mutes are preloaded) muted = thread_muted? || UserRelationship.exists?( @@ -317,7 +321,8 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} AccountView.render("show.json", %{ user: user, for: opts[:for], - relationships: opts[:relationships] + relationships: opts[:relationships], + skip_relationships: opts[:skip_relationships] }), in_reply_to_id: reply_to && to_string(reply_to.id), in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), diff --git a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex index dcba67d03..9d0b3b1e4 100644 --- a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do use Pleroma.Web, :controller import Pleroma.Web.ControllerHelper, - only: [json_response: 3, add_link_headers: 2, assign_account_by_id: 2] + only: [json_response: 3, add_link_headers: 2, assign_account_by_id: 2, skip_relationships?: 1] alias Ecto.Changeset alias Pleroma.Plugs.OAuthScopesPlug @@ -139,7 +139,12 @@ def favourites(%{assigns: %{user: for_user, account: user}} = conn, params) do conn |> add_link_headers(activities) |> put_view(StatusView) - |> render("index.json", activities: activities, for: for_user, as: :activity) + |> render("index.json", + activities: activities, + for: for_user, + as: :activity, + skip_relationships: skip_relationships?(params) + ) end @doc "POST /api/v1/pleroma/accounts/:id/subscribe" diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex index dae7f0f2f..83983b576 100644 --- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do use Pleroma.Web, :controller - import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2, skip_relationships?: 1] alias Pleroma.Activity alias Pleroma.Conversation.Participation @@ -130,7 +130,12 @@ def conversation_statuses( conn |> add_link_headers(activities) |> put_view(StatusView) - |> render("index.json", %{activities: activities, for: user, as: :activity}) + |> render("index.json", + activities: activities, + for: user, + as: :activity, + skip_relationships: skip_relationships?(params) + ) else _error -> conn @@ -184,13 +189,17 @@ def read_notification(%{assigns: %{user: user}} = conn, %{"id" => notification_i end end - def read_notification(%{assigns: %{user: user}} = conn, %{"max_id" => max_id}) do + def read_notification(%{assigns: %{user: user}} = conn, %{"max_id" => max_id} = params) do with notifications <- Notification.set_read_up_to(user, max_id) do notifications = Enum.take(notifications, 80) conn |> put_view(NotificationView) - |> render("index.json", %{notifications: notifications, for: user}) + |> render("index.json", + notifications: notifications, + for: user, + skip_relationships: skip_relationships?(params) + ) end end end diff --git a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs index c618ea381..b6f0ac66b 100644 --- a/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs +++ b/priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs @@ -3,7 +3,6 @@ defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do import Ecto.Query alias Pleroma.Activity alias Pleroma.Bookmark - alias Pleroma.User alias Pleroma.Repo def up do diff --git a/priv/repo/migrations/20190711042021_create_safe_jsonb_set.exs b/priv/repo/migrations/20190711042021_create_safe_jsonb_set.exs index 2f336a5e8..43d616705 100644 --- a/priv/repo/migrations/20190711042021_create_safe_jsonb_set.exs +++ b/priv/repo/migrations/20190711042021_create_safe_jsonb_set.exs @@ -1,6 +1,5 @@ defmodule Pleroma.Repo.Migrations.CreateSafeJsonbSet do use Ecto.Migration - alias Pleroma.User def change do execute(""" diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs index 7a0011646..42a311f99 100644 --- a/test/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/web/mastodon_api/controllers/notification_controller_test.exs @@ -12,6 +12,26 @@ defmodule Pleroma.Web.MastodonAPI.NotificationControllerTest do import Pleroma.Factory + test "does NOT render account/pleroma/relationship if this is disabled by default" do + clear_config([:extensions, :output_relationships_in_statuses_by_default], false) + + %{user: user, conn: conn} = oauth_access(["read:notifications"]) + other_user = insert(:user) + + {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + {:ok, [_notification]} = Notification.create_notifications(activity) + + response = + conn + |> assign(:user, user) + |> get("/api/v1/notifications") + |> json_response(200) + + assert Enum.all?(response, fn n -> + get_in(n, ["account", "pleroma", "relationship"]) == %{} + end) + end + test "list of notifications" do %{user: user, conn: conn} = oauth_access(["read:notifications"]) other_user = insert(:user) diff --git a/test/web/mastodon_api/controllers/status_controller_test.exs b/test/web/mastodon_api/controllers/status_controller_test.exs index d59974d50..6b126217a 100644 --- a/test/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/web/mastodon_api/controllers/status_controller_test.exs @@ -1043,6 +1043,8 @@ test "replaces missing description with an empty string", %{conn: conn, user: us end test "bookmarks" do + bookmarks_uri = "/api/v1/bookmarks?with_relationships=true" + %{conn: conn} = oauth_access(["write:bookmarks", "read:bookmarks"]) author = insert(:user) @@ -1064,7 +1066,7 @@ test "bookmarks" do assert json_response(response2, 200)["bookmarked"] == true - bookmarks = get(conn, "/api/v1/bookmarks") + bookmarks = get(conn, bookmarks_uri) assert [json_response(response2, 200), json_response(response1, 200)] == json_response(bookmarks, 200) @@ -1073,7 +1075,7 @@ test "bookmarks" do assert json_response(response1, 200)["bookmarked"] == false - bookmarks = get(conn, "/api/v1/bookmarks") + bookmarks = get(conn, bookmarks_uri) assert [json_response(response2, 200)] == json_response(bookmarks, 200) end diff --git a/test/web/mastodon_api/controllers/timeline_controller_test.exs b/test/web/mastodon_api/controllers/timeline_controller_test.exs index 97b1c3e66..06efdc901 100644 --- a/test/web/mastodon_api/controllers/timeline_controller_test.exs +++ b/test/web/mastodon_api/controllers/timeline_controller_test.exs @@ -20,7 +20,30 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do describe "home" do setup do: oauth_access(["read:statuses"]) + test "does NOT render account/pleroma/relationship if this is disabled by default", %{ + user: user, + conn: conn + } do + clear_config([:extensions, :output_relationships_in_statuses_by_default], false) + + other_user = insert(:user) + + {:ok, _} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + + response = + conn + |> assign(:user, user) + |> get("/api/v1/timelines/home") + |> json_response(200) + + assert Enum.all?(response, fn n -> + get_in(n, ["account", "pleroma", "relationship"]) == %{} + end) + end + test "the home timeline", %{user: user, conn: conn} do + uri = "/api/v1/timelines/home?with_relationships=true" + following = insert(:user, nickname: "followed") third_user = insert(:user, nickname: "repeated") @@ -28,13 +51,13 @@ test "the home timeline", %{user: user, conn: conn} do {:ok, activity} = CommonAPI.post(third_user, %{"status" => "repeated post"}) {:ok, _, _} = CommonAPI.repeat(activity.id, following) - ret_conn = get(conn, "/api/v1/timelines/home") + ret_conn = get(conn, uri) assert Enum.empty?(json_response(ret_conn, :ok)) {:ok, _user} = User.follow(user, following) - ret_conn = get(conn, "/api/v1/timelines/home") + ret_conn = get(conn, uri) assert [ %{ @@ -59,7 +82,7 @@ test "the home timeline", %{user: user, conn: conn} do {:ok, _user} = User.follow(third_user, user) - ret_conn = get(conn, "/api/v1/timelines/home") + ret_conn = get(conn, uri) assert [ %{ From 2d64500a9dee8bc53c988719bde1c1f4f41575b7 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 1 Apr 2020 20:26:33 +0300 Subject: [PATCH 25/84] error improvement for email_invite endpoint --- docs/API/admin_api.md | 13 +++++++ .../web/admin_api/admin_api_controller.ex | 17 ++++++-- .../admin_api/admin_api_controller_test.exs | 39 ++++++++++++++++++- 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index edcf73e14..179d8c451 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -392,6 +392,19 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - `email` - `name`, optional +- Response: + - On success: `204`, empty response + - On failure: + - 400 Bad Request, JSON: + + ```json + [ + { + `error` // error message + } + ] + ``` + ## `GET /api/pleroma/admin/users/:nickname/password_reset` ### Get a password reset token for a given nickname diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index ca5439920..7b442f6e1 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -576,9 +576,8 @@ def relay_unfollow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) @doc "Sends registration invite via email" def email_invite(%{assigns: %{user: user}} = conn, %{"email" => email} = params) do - with true <- - Config.get([:instance, :invites_enabled]) && - !Config.get([:instance, :registrations_open]), + with {_, false} <- {:registrations_open, Config.get([:instance, :registrations_open])}, + {_, true} <- {:invites_enabled, Config.get([:instance, :invites_enabled])}, {:ok, invite_token} <- UserInviteToken.create_invite(), email <- Pleroma.Emails.UserEmail.user_invitation_email( @@ -589,6 +588,18 @@ def email_invite(%{assigns: %{user: user}} = conn, %{"email" => email} = params) ), {:ok, _} <- Pleroma.Emails.Mailer.deliver(email) do json_response(conn, :no_content, "") + else + {:registrations_open, _} -> + errors( + conn, + {:error, "To send invites you need set `registrations_open` option to false."} + ) + + {:invites_enabled, _} -> + errors( + conn, + {:error, "To send invites you need set `invites_enabled` option to true."} + ) end end diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index ea0c92502..32fe69d19 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -625,6 +625,39 @@ test "it returns 403 if requested by a non-admin" do assert json_response(conn, :forbidden) end + + test "email with +", %{conn: conn, admin: admin} do + recipient_email = "foo+bar@baz.com" + + conn + |> put_req_header("content-type", "application/json;charset=utf-8") + |> post("/api/pleroma/admin/users/email_invite", %{email: recipient_email}) + |> json_response(:no_content) + + token_record = + Pleroma.UserInviteToken + |> Repo.all() + |> List.last() + + assert token_record + refute token_record.used + + notify_email = Config.get([:instance, :notify_email]) + instance_name = Config.get([:instance, :name]) + + email = + Pleroma.Emails.UserEmail.user_invitation_email( + admin, + token_record, + recipient_email + ) + + Swoosh.TestAssertions.assert_email_sent( + from: {instance_name, notify_email}, + to: recipient_email, + html_body: email.html_body + ) + end end describe "POST /api/pleroma/admin/users/email_invite, with invalid config" do @@ -637,7 +670,8 @@ test "it returns 500 if `invites_enabled` is not enabled", %{conn: conn} do conn = post(conn, "/api/pleroma/admin/users/email_invite?email=foo@bar.com&name=JD") - assert json_response(conn, :internal_server_error) + assert json_response(conn, :bad_request) == + "To send invites you need set `invites_enabled` option to true." end test "it returns 500 if `registrations_open` is enabled", %{conn: conn} do @@ -646,7 +680,8 @@ test "it returns 500 if `registrations_open` is enabled", %{conn: conn} do conn = post(conn, "/api/pleroma/admin/users/email_invite?email=foo@bar.com&name=JD") - assert json_response(conn, :internal_server_error) + assert json_response(conn, :bad_request) == + "To send invites you need set `registrations_open` option to false." end end From 23219e6fb3163bfac07fb5fb1b2602dcd27e47c2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 1 Apr 2020 23:00:59 +0400 Subject: [PATCH 26/84] Add OpenAPI --- lib/pleroma/web/api_spec.ex | 30 ++++++ .../web/api_spec/operations/app_operation.ex | 94 +++++++++++++++++++ .../api_spec/schemas/app_create_request.ex | 33 +++++++ .../api_spec/schemas/app_create_response.ex | 33 +++++++ .../controllers/app_controller.ex | 9 +- lib/pleroma/web/oauth/scopes.ex | 7 +- lib/pleroma/web/router.ex | 11 +++ mix.exs | 3 +- mix.lock | 1 + test/web/api_spec/app_operation_test.exs | 45 +++++++++ .../controllers/account_controller_test.exs | 4 +- .../controllers/app_controller_test.exs | 4 +- 12 files changed, 266 insertions(+), 8 deletions(-) create mode 100644 lib/pleroma/web/api_spec.ex create mode 100644 lib/pleroma/web/api_spec/operations/app_operation.ex create mode 100644 lib/pleroma/web/api_spec/schemas/app_create_request.ex create mode 100644 lib/pleroma/web/api_spec/schemas/app_create_response.ex create mode 100644 test/web/api_spec/app_operation_test.exs diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex new file mode 100644 index 000000000..22f76d4bf --- /dev/null +++ b/lib/pleroma/web/api_spec.ex @@ -0,0 +1,30 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec do + alias OpenApiSpex.OpenApi + alias Pleroma.Web.Endpoint + alias Pleroma.Web.Router + + @behaviour OpenApi + + @impl OpenApi + def spec do + %OpenApi{ + servers: [ + # Populate the Server info from a phoenix endpoint + OpenApiSpex.Server.from_endpoint(Endpoint) + ], + info: %OpenApiSpex.Info{ + title: "Pleroma", + description: Application.spec(:pleroma, :description) |> to_string(), + version: Application.spec(:pleroma, :vsn) |> to_string() + }, + # populate the paths from a phoenix router + paths: OpenApiSpex.Paths.from_router(Router) + } + # discover request/response schemas from path specs + |> OpenApiSpex.resolve_schema_modules() + end +end diff --git a/lib/pleroma/web/api_spec/operations/app_operation.ex b/lib/pleroma/web/api_spec/operations/app_operation.ex new file mode 100644 index 000000000..2a4958acf --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/app_operation.ex @@ -0,0 +1,94 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.AppOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.AppCreateRequest + alias Pleroma.Web.ApiSpec.Schemas.AppCreateResponse + + @spec open_api_operation(atom) :: Operation.t() + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + @spec create_operation() :: Operation.t() + def create_operation do + %Operation{ + tags: ["apps"], + summary: "Create an application", + description: "Create a new application to obtain OAuth2 credentials", + operationId: "AppController.create", + requestBody: + Operation.request_body("Parameters", "application/json", AppCreateRequest, required: true), + responses: %{ + 200 => Operation.response("App", "application/json", AppCreateResponse), + 422 => + Operation.response( + "Unprocessable Entity", + "application/json", + %Schema{ + type: :object, + description: + "If a required parameter is missing or improperly formatted, the request will fail.", + properties: %{ + error: %Schema{type: :string} + }, + example: %{ + "error" => "Validation failed: Redirect URI must be an absolute URI." + } + } + ) + } + } + end + + def verify_credentials_operation do + %Operation{ + tags: ["apps"], + summary: "Verify your app works", + description: "Confirm that the app's OAuth2 credentials work.", + operationId: "AppController.verify_credentials", + parameters: [ + Operation.parameter(:authorization, :header, :string, "Bearer ", required: true) + ], + responses: %{ + 200 => + Operation.response("App", "application/json", %Schema{ + type: :object, + description: + "If the Authorization header was provided with a valid token, you should see your app returned as an Application entity.", + properties: %{ + name: %Schema{type: :string}, + vapid_key: %Schema{type: :string}, + website: %Schema{type: :string, nullable: true} + }, + example: %{ + "name" => "My App", + "vapid_key" => + "BCk-QqERU0q-CfYZjcuB6lnyyOYfJ2AifKqfeGIm7Z-HiTU5T9eTG5GxVA0_OH5mMlI4UkkDTpaZwozy0TzdZ2M=", + "website" => "https://myapp.com/" + } + }), + 422 => + Operation.response( + "Unauthorized", + "application/json", + %Schema{ + type: :object, + description: + "If the Authorization header contains an invalid token, is malformed, or is not present, an error will be returned indicating an authorization failure.", + properties: %{ + error: %Schema{type: :string} + }, + example: %{ + "error" => "The access token is invalid." + } + } + ) + } + } + end +end diff --git a/lib/pleroma/web/api_spec/schemas/app_create_request.ex b/lib/pleroma/web/api_spec/schemas/app_create_request.ex new file mode 100644 index 000000000..8a83abef3 --- /dev/null +++ b/lib/pleroma/web/api_spec/schemas/app_create_request.ex @@ -0,0 +1,33 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Schemas.AppCreateRequest do + alias OpenApiSpex.Schema + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "AppCreateRequest", + description: "POST body for creating an app", + type: :object, + properties: %{ + client_name: %Schema{type: :string, description: "A name for your application."}, + redirect_uris: %Schema{ + type: :string, + description: + "Where the user should be redirected after authorization. To display the authorization code to the user instead of redirecting to a web page, use `urn:ietf:wg:oauth:2.0:oob` in this parameter." + }, + scopes: %Schema{ + type: :string, + description: "Space separated list of scopes. If none is provided, defaults to `read`." + }, + website: %Schema{type: :string, description: "A URL to the homepage of your app"} + }, + required: [:client_name, :redirect_uris], + example: %{ + "client_name" => "My App", + "redirect_uris" => "https://myapp.com/auth/callback", + "website" => "https://myapp.com/" + } + }) +end diff --git a/lib/pleroma/web/api_spec/schemas/app_create_response.ex b/lib/pleroma/web/api_spec/schemas/app_create_response.ex new file mode 100644 index 000000000..f290fb031 --- /dev/null +++ b/lib/pleroma/web/api_spec/schemas/app_create_response.ex @@ -0,0 +1,33 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Schemas.AppCreateResponse do + alias OpenApiSpex.Schema + + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "AppCreateResponse", + description: "Response schema for an app", + type: :object, + properties: %{ + id: %Schema{type: :string}, + name: %Schema{type: :string}, + client_id: %Schema{type: :string}, + client_secret: %Schema{type: :string}, + redirect_uri: %Schema{type: :string}, + vapid_key: %Schema{type: :string}, + website: %Schema{type: :string, nullable: true} + }, + example: %{ + "id" => "123", + "name" => "My App", + "client_id" => "TWhM-tNSuncnqN7DBJmoyeLnk6K3iJJ71KKXxgL1hPM", + "client_secret" => "ZEaFUFmF0umgBX1qKJDjaU99Q31lDkOU8NutzTOoliw", + "vapid_key" => + "BCk-QqERU0q-CfYZjcuB6lnyyOYfJ2AifKqfeGIm7Z-HiTU5T9eTG5GxVA0_OH5mMlI4UkkDTpaZwozy0TzdZ2M=", + "website" => "https://myapp.com/" + } + }) +end diff --git a/lib/pleroma/web/mastodon_api/controllers/app_controller.ex b/lib/pleroma/web/mastodon_api/controllers/app_controller.ex index 5e2871f18..005c60444 100644 --- a/lib/pleroma/web/mastodon_api/controllers/app_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/app_controller.ex @@ -14,17 +14,20 @@ defmodule Pleroma.Web.MastodonAPI.AppController do action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read"]} when action == :verify_credentials) + plug(OpenApiSpex.Plug.CastAndValidate) @local_mastodon_name "Mastodon-Local" + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.AppOperation + @doc "POST /api/v1/apps" - def create(conn, params) do + def create(%{body_params: params} = conn, _params) do scopes = Scopes.fetch_scopes(params, ["read"]) app_attrs = params - |> Map.drop(["scope", "scopes"]) - |> Map.put("scopes", scopes) + |> Map.take([:client_name, :redirect_uris, :website]) + |> Map.put(:scopes, scopes) with cs <- App.register_changeset(%App{}, app_attrs), false <- cs.changes[:client_name] == @local_mastodon_name, diff --git a/lib/pleroma/web/oauth/scopes.ex b/lib/pleroma/web/oauth/scopes.ex index 8ecf901f3..1023f16d4 100644 --- a/lib/pleroma/web/oauth/scopes.ex +++ b/lib/pleroma/web/oauth/scopes.ex @@ -15,7 +15,12 @@ defmodule Pleroma.Web.OAuth.Scopes do Note: `scopes` is used by Mastodon — supporting it but sticking to OAuth's standard `scope` wherever we control it """ - @spec fetch_scopes(map(), list()) :: list() + @spec fetch_scopes(map() | struct(), list()) :: list() + + def fetch_scopes(%Pleroma.Web.ApiSpec.Schemas.AppCreateRequest{scopes: scopes}, default) do + parse_scopes(scopes, default) + end + def fetch_scopes(params, default) do parse_scopes(params["scope"] || params["scopes"], default) end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 5a0902739..3ecd59cd1 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -29,6 +29,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Plugs.SetUserSessionIdPlug) plug(Pleroma.Plugs.EnsureUserKeyPlug) plug(Pleroma.Plugs.IdempotencyPlug) + plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) end pipeline :authenticated_api do @@ -44,6 +45,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Plugs.SetUserSessionIdPlug) plug(Pleroma.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Plugs.IdempotencyPlug) + plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) end pipeline :admin_api do @@ -61,6 +63,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Plugs.UserIsAdminPlug) plug(Pleroma.Plugs.IdempotencyPlug) + plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) end pipeline :mastodon_html do @@ -94,10 +97,12 @@ defmodule Pleroma.Web.Router do pipeline :config do plug(:accepts, ["json", "xml"]) + plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) end pipeline :pleroma_api do plug(:accepts, ["html", "json"]) + plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) end pipeline :mailbox_preview do @@ -500,6 +505,12 @@ defmodule Pleroma.Web.Router do ) end + scope "/api" do + pipe_through(:api) + + get("/openapi", OpenApiSpex.Plug.RenderSpec, []) + end + scope "/api", Pleroma.Web, as: :authenticated_twitter_api do pipe_through(:authenticated_api) diff --git a/mix.exs b/mix.exs index 890979f8b..ebd4a5ea6 100644 --- a/mix.exs +++ b/mix.exs @@ -171,7 +171,8 @@ defp deps do git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, {:mox, "~> 0.5", only: :test}, - {:restarter, path: "./restarter"} + {:restarter, path: "./restarter"}, + {:open_api_spex, "~> 3.6"} ] ++ oauth_deps() end diff --git a/mix.lock b/mix.lock index 62e14924a..fd26ca01b 100644 --- a/mix.lock +++ b/mix.lock @@ -72,6 +72,7 @@ "nimble_parsec": {:hex, :nimble_parsec, "0.5.3", "def21c10a9ed70ce22754fdeea0810dafd53c2db3219a0cd54cf5526377af1c6", [:mix], [], "hexpm", "589b5af56f4afca65217a1f3eb3fee7e79b09c40c742fddc1c312b3ac0b3399f"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, "oban": {:hex, :oban, "0.12.1", "695e9490c6e0edfca616d80639528e448bd29b3bff7b7dd10a56c79b00a5d7fb", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c1d58d69b8b5a86e7167abbb8cc92764a66f25f12f6172052595067fc6a30a17"}, + "open_api_spex": {:hex, :open_api_spex, "3.6.0", "64205aba9f2607f71b08fd43e3351b9c5e9898ec5ef49fc0ae35890da502ade9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.1", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm", "126ba3473966277132079cb1d5bf1e3df9e36fe2acd00166e75fd125cecb59c5"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm", "595d09db74cb093b1903381c9de423276a931a2480a46a1a5dc7f932a2a6375b"}, "phoenix": {:hex, :phoenix, "1.4.13", "67271ad69b51f3719354604f4a3f968f83aa61c19199343656c9caee057ff3b8", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ab765a0feddb81fc62e2116c827b5f068df85159c162bee760745276ad7ddc1b"}, diff --git a/test/web/api_spec/app_operation_test.exs b/test/web/api_spec/app_operation_test.exs new file mode 100644 index 000000000..5b96abb44 --- /dev/null +++ b/test/web/api_spec/app_operation_test.exs @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.AppOperationTest do + use Pleroma.Web.ConnCase, async: true + + alias Pleroma.Web.ApiSpec + alias Pleroma.Web.ApiSpec.Schemas.AppCreateRequest + alias Pleroma.Web.ApiSpec.Schemas.AppCreateResponse + + import OpenApiSpex.TestAssertions + import Pleroma.Factory + + test "AppCreateRequest example matches schema" do + api_spec = ApiSpec.spec() + schema = AppCreateRequest.schema() + assert_schema(schema.example, "AppCreateRequest", api_spec) + end + + test "AppCreateResponse example matches schema" do + api_spec = ApiSpec.spec() + schema = AppCreateResponse.schema() + assert_schema(schema.example, "AppCreateResponse", api_spec) + end + + test "AppController produces a AppCreateResponse", %{conn: conn} do + api_spec = ApiSpec.spec() + app_attrs = build(:oauth_app) + + json = + conn + |> put_req_header("content-type", "application/json") + |> post( + "/api/v1/apps", + Jason.encode!(%{ + client_name: app_attrs.client_name, + redirect_uris: app_attrs.redirect_uris + }) + ) + |> json_response(200) + + assert_schema(json, "AppCreateResponse", api_spec) + end +end diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index a9fa0ce48..a450a732c 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -794,7 +794,9 @@ test "blocking / unblocking a user" do test "Account registration via Application", %{conn: conn} do conn = - post(conn, "/api/v1/apps", %{ + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/apps", %{ client_name: "client_name", redirect_uris: "urn:ietf:wg:oauth:2.0:oob", scopes: "read, write, follow" diff --git a/test/web/mastodon_api/controllers/app_controller_test.exs b/test/web/mastodon_api/controllers/app_controller_test.exs index 77d234d67..e7b11d14e 100644 --- a/test/web/mastodon_api/controllers/app_controller_test.exs +++ b/test/web/mastodon_api/controllers/app_controller_test.exs @@ -16,8 +16,7 @@ test "apps/verify_credentials", %{conn: conn} do conn = conn - |> assign(:user, token.user) - |> assign(:token, token) + |> put_req_header("authorization", "Bearer #{token.token}") |> get("/api/v1/apps/verify_credentials") app = Repo.preload(token, :app).app @@ -37,6 +36,7 @@ test "creates an oauth app", %{conn: conn} do conn = conn + |> put_req_header("content-type", "application/json") |> assign(:user, user) |> post("/api/v1/apps", %{ client_name: app_attrs.client_name, From 591f7015d91b383dae1ee29576d13c0fad65cad6 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 2 Apr 2020 09:34:11 +0300 Subject: [PATCH 27/84] update Oban package --- mix.exs | 2 +- mix.lock | 6 +++--- .../20200402063221_update_oban_jobs_table.exs | 11 +++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 priv/repo/migrations/20200402063221_update_oban_jobs_table.exs diff --git a/mix.exs b/mix.exs index 87c025d89..375bc67c1 100644 --- a/mix.exs +++ b/mix.exs @@ -108,7 +108,7 @@ defp deps do {:ecto_enum, "~> 1.4"}, {:ecto_sql, "~> 3.3.2"}, {:postgrex, ">= 0.13.5"}, - {:oban, "~> 0.12.1"}, + {:oban, "~> 1.2"}, {:gettext, "~> 0.15"}, {:comeonin, "~> 4.1.1"}, {:pbkdf2_elixir, "~> 0.12.3"}, diff --git a/mix.lock b/mix.lock index 6cca578d6..50be45a4d 100644 --- a/mix.lock +++ b/mix.lock @@ -26,7 +26,7 @@ "decimal": {:hex, :decimal, "1.8.1", "a4ef3f5f3428bdbc0d35374029ffcf4ede8533536fa79896dd450168d9acdf3c", [:mix], [], "hexpm", "3cb154b00225ac687f6cbd4acc4b7960027c757a5152b369923ead9ddbca7aec"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, - "ecto": {:hex, :ecto, "3.3.3", "0830bf3aebcbf3d8c1a1811cd581773b6866886c012f52c0f027031fa96a0b53", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "12e368e3c2a2938d7776defaabdae40e82900fc4d8d66120ec1e01dfd8b93c3a"}, + "ecto": {:hex, :ecto, "3.4.0", "a7a83ab8359bf816ce729e5e65981ce25b9fc5adfc89c2ea3980f4fed0bfd7c1", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "5eed18252f5b5bbadec56a24112b531343507dbe046273133176b12190ce19cc"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_sql": {:hex, :ecto_sql, "3.3.4", "aa18af12eb875fbcda2f75e608b3bd534ebf020fc4f6448e4672fcdcbb081244", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4 or ~> 3.3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5eccbdbf92e3c6f213007a82d5dbba4cd9bb659d1a21331f89f408e4c0efd7a8"}, "esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm", "d73e341e3009d390aa36387dc8862860bf9f874c94d9fd92ade2926376f49981"}, @@ -55,7 +55,7 @@ "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"}, "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm", "64a2d30189704ae41ca7dbdd587f5291db5d1dda1414e0774c29ffc81088c1bc"}, - "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fdf843bca858203ae1de16da2ee206f53416bbda5dc8c9e78f43243de4bc3afe"}, + "jason": {:hex, :jason, "1.2.0", "10043418c42d2493d0ee212d3fddd25d7ffe484380afad769a0a38795938e448", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "116747dbe057794c3a3e4e143b7c8390b29f634e16c78a7f59ba75bfa6852e7f"}, "joken": {:hex, :joken, "2.2.0", "2daa1b12be05184aff7b5ace1d43ca1f81345962285fff3f88db74927c954d3a", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "b4f92e30388206f869dd25d1af628a1d99d7586e5cf0672f64d4df84c4d2f5e9"}, "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm", "3c7ddc8a9394b92891db7c2771da94bf819834a1a4c92e30857b7d582e2f8257"}, "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, @@ -73,7 +73,7 @@ "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]}, "nimble_parsec": {:hex, :nimble_parsec, "0.5.3", "def21c10a9ed70ce22754fdeea0810dafd53c2db3219a0cd54cf5526377af1c6", [:mix], [], "hexpm", "589b5af56f4afca65217a1f3eb3fee7e79b09c40c742fddc1c312b3ac0b3399f"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, - "oban": {:hex, :oban, "0.12.1", "695e9490c6e0edfca616d80639528e448bd29b3bff7b7dd10a56c79b00a5d7fb", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c1d58d69b8b5a86e7167abbb8cc92764a66f25f12f6172052595067fc6a30a17"}, + "oban": {:hex, :oban, "1.2.0", "7cca94d341be43d220571e28f69131c4afc21095b25257397f50973d3fc59b07", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ba5f8b3f7d76967b3e23cf8014f6a13e4ccb33431e4808f036709a7f822362ee"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm", "595d09db74cb093b1903381c9de423276a931a2480a46a1a5dc7f932a2a6375b"}, "phoenix": {:hex, :phoenix, "1.4.13", "67271ad69b51f3719354604f4a3f968f83aa61c19199343656c9caee057ff3b8", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ab765a0feddb81fc62e2116c827b5f068df85159c162bee760745276ad7ddc1b"}, diff --git a/priv/repo/migrations/20200402063221_update_oban_jobs_table.exs b/priv/repo/migrations/20200402063221_update_oban_jobs_table.exs new file mode 100644 index 000000000..c8ee12192 --- /dev/null +++ b/priv/repo/migrations/20200402063221_update_oban_jobs_table.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.UpdateObanJobsTable do + use Ecto.Migration + + def up do + Oban.Migrations.up() + end + + def down do + Oban.Migrations.down(version: 1) + end +end From 0aa24a150bbb153f55ca92dfb595385b4fe3839c Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 2 Apr 2020 17:33:23 +0400 Subject: [PATCH 28/84] Add oAuth --- lib/pleroma/web/api_spec.ex | 16 +++++++++++++++- .../web/api_spec/operations/app_operation.ex | 6 ++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 22f76d4bf..41e48a085 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -22,7 +22,21 @@ def spec do version: Application.spec(:pleroma, :vsn) |> to_string() }, # populate the paths from a phoenix router - paths: OpenApiSpex.Paths.from_router(Router) + paths: OpenApiSpex.Paths.from_router(Router), + components: %OpenApiSpex.Components{ + securitySchemes: %{ + "oAuth" => %OpenApiSpex.SecurityScheme{ + type: "oauth2", + flows: %OpenApiSpex.OAuthFlows{ + password: %OpenApiSpex.OAuthFlow{ + authorizationUrl: "/oauth/authorize", + tokenUrl: "/oauth/token", + scopes: %{"read" => "read"} + } + } + } + } + } } # discover request/response schemas from path specs |> OpenApiSpex.resolve_schema_modules() diff --git a/lib/pleroma/web/api_spec/operations/app_operation.ex b/lib/pleroma/web/api_spec/operations/app_operation.ex index 2a4958acf..41d56693a 100644 --- a/lib/pleroma/web/api_spec/operations/app_operation.ex +++ b/lib/pleroma/web/api_spec/operations/app_operation.ex @@ -51,8 +51,10 @@ def verify_credentials_operation do summary: "Verify your app works", description: "Confirm that the app's OAuth2 credentials work.", operationId: "AppController.verify_credentials", - parameters: [ - Operation.parameter(:authorization, :header, :string, "Bearer ", required: true) + security: [ + %{ + "oAuth" => ["read"] + } ], responses: %{ 200 => From fc81e5a49c34224e07e85f490a30f92db0835d45 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 6 Apr 2020 10:20:44 +0300 Subject: [PATCH 29/84] Enforcement of OAuth scopes check for authenticated API endpoints, :skip_plug plug to mark a plug explicitly skipped (disabled). --- lib/pleroma/plugs/auth_expected_plug.ex | 13 +++++++ lib/pleroma/plugs/oauth_scopes_plug.ex | 3 ++ lib/pleroma/plugs/plug_helper.ex | 38 +++++++++++++++++++ lib/pleroma/web/masto_fe_controller.ex | 2 +- .../controllers/account_controller.ex | 9 ++++- .../controllers/mastodon_api_controller.ex | 18 +++++++-- .../controllers/suggestion_controller.ex | 9 +++-- lib/pleroma/web/oauth/oauth_controller.ex | 2 + .../controllers/pleroma_api_controller.ex | 2 +- lib/pleroma/web/router.ex | 3 +- .../web/twitter_api/twitter_api_controller.ex | 2 + lib/pleroma/web/web.ex | 23 +++++++++++ .../suggestion_controller_test.exs | 26 ------------- .../pleroma_api_controller_test.exs | 2 +- 14 files changed, 113 insertions(+), 39 deletions(-) create mode 100644 lib/pleroma/plugs/auth_expected_plug.ex create mode 100644 lib/pleroma/plugs/plug_helper.ex diff --git a/lib/pleroma/plugs/auth_expected_plug.ex b/lib/pleroma/plugs/auth_expected_plug.ex new file mode 100644 index 000000000..9e4a4bec8 --- /dev/null +++ b/lib/pleroma/plugs/auth_expected_plug.ex @@ -0,0 +1,13 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Plugs.AuthExpectedPlug do + import Plug.Conn + + def init(options), do: options + + def call(conn, _) do + put_private(conn, :auth_expected, true) + end +end diff --git a/lib/pleroma/plugs/oauth_scopes_plug.ex b/lib/pleroma/plugs/oauth_scopes_plug.ex index 38df074ad..b09e1bb4d 100644 --- a/lib/pleroma/plugs/oauth_scopes_plug.ex +++ b/lib/pleroma/plugs/oauth_scopes_plug.ex @@ -8,12 +8,15 @@ defmodule Pleroma.Plugs.OAuthScopesPlug do alias Pleroma.Config alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug + alias Pleroma.Plugs.PlugHelper @behaviour Plug def init(%{scopes: _} = options), do: options def call(%Plug.Conn{assigns: assigns} = conn, %{scopes: scopes} = options) do + conn = PlugHelper.append_to_called_plugs(conn, __MODULE__) + op = options[:op] || :| token = assigns[:token] diff --git a/lib/pleroma/plugs/plug_helper.ex b/lib/pleroma/plugs/plug_helper.ex new file mode 100644 index 000000000..4f83e9414 --- /dev/null +++ b/lib/pleroma/plugs/plug_helper.ex @@ -0,0 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Plugs.PlugHelper do + @moduledoc "Pleroma Plug helper" + + def append_to_called_plugs(conn, plug_module) do + append_to_private_list(conn, :called_plugs, plug_module) + end + + def append_to_skipped_plugs(conn, plug_module) do + append_to_private_list(conn, :skipped_plugs, plug_module) + end + + def plug_called?(conn, plug_module) do + contained_in_private_list?(conn, :called_plugs, plug_module) + end + + def plug_skipped?(conn, plug_module) do + contained_in_private_list?(conn, :skipped_plugs, plug_module) + end + + def plug_called_or_skipped?(conn, plug_module) do + plug_called?(conn, plug_module) || plug_skipped?(conn, plug_module) + end + + defp append_to_private_list(conn, private_variable, value) do + list = conn.private[private_variable] || [] + modified_list = Enum.uniq(list ++ [value]) + Plug.Conn.put_private(conn, private_variable, modified_list) + end + + defp contained_in_private_list?(conn, private_variable, value) do + list = conn.private[private_variable] || [] + value in list + end +end diff --git a/lib/pleroma/web/masto_fe_controller.ex b/lib/pleroma/web/masto_fe_controller.ex index 43649ad26..557cde328 100644 --- a/lib/pleroma/web/masto_fe_controller.ex +++ b/lib/pleroma/web/masto_fe_controller.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Web.MastoFEController do when action == :index ) - plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug when action != :index) + plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug when action not in [:index, :manifest]) @doc "GET /web/*path" def index(%{assigns: %{user: user, token: token}} = conn, _params) diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 21bc3d5a5..bd6853d12 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -15,10 +15,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.ListView alias Pleroma.Web.MastodonAPI.MastodonAPI + alias Pleroma.Web.MastodonAPI.MastodonAPIController alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.OAuth.Token alias Pleroma.Web.TwitterAPI.TwitterAPI + plug(:skip_plug, OAuthScopesPlug when action == :identity_proofs) + plug( OAuthScopesPlug, %{fallback: :proceed_unauthenticated, scopes: ["read:accounts"]} @@ -369,6 +372,8 @@ def blocks(%{assigns: %{user: user}} = conn, _) do end @doc "GET /api/v1/endorsements" - def endorsements(conn, params), - do: Pleroma.Web.MastodonAPI.MastodonAPIController.empty_array(conn, params) + def endorsements(conn, params), do: MastodonAPIController.empty_array(conn, params) + + @doc "GET /api/v1/identity_proofs" + def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params) end diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex index 14075307d..ac8c18f24 100644 --- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex @@ -3,21 +3,31 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do + @moduledoc """ + Contains stubs for unimplemented Mastodon API endpoints. + + Note: instead of routing directly to this controller's action, + it's preferable to define an action in relevant (non-generic) controller, + set up OAuth rules for it and call this controller's function from it. + """ + use Pleroma.Web, :controller require Logger + plug(:skip_plug, Pleroma.Plugs.OAuthScopesPlug when action in [:empty_array, :empty_object]) + + plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug) + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) - # Stubs for unimplemented mastodon api - # def empty_array(conn, _) do - Logger.debug("Unimplemented, returning an empty array") + Logger.debug("Unimplemented, returning an empty array (list)") json(conn, []) end def empty_object(conn, _) do - Logger.debug("Unimplemented, returning an empty object") + Logger.debug("Unimplemented, returning an empty object (map)") json(conn, %{}) end end diff --git a/lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex b/lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex index 0cdc7bd8d..c93a43969 100644 --- a/lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex @@ -5,10 +5,13 @@ defmodule Pleroma.Web.MastodonAPI.SuggestionController do use Pleroma.Web, :controller + alias Pleroma.Plugs.OAuthScopesPlug + require Logger + plug(OAuthScopesPlug, %{scopes: ["read"]} when action == :index) + @doc "GET /api/v1/suggestions" - def index(conn, _) do - json(conn, []) - end + def index(conn, params), + do: Pleroma.Web.MastodonAPI.MastodonAPIController.empty_array(conn, params) end diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 46688db7e..0121cd661 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -27,6 +27,8 @@ defmodule Pleroma.Web.OAuth.OAuthController do plug(:fetch_flash) plug(RateLimiter, [name: :authentication] when action == :create_authorization) + plug(:skip_plug, Pleroma.Plugs.OAuthScopesPlug) + action_fallback(Pleroma.Web.OAuth.FallbackController) @oob_token_redirect_uri "urn:ietf:wg:oauth:2.0:oob" diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex index dae7f0f2f..75f61b675 100644 --- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex @@ -34,7 +34,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do plug( OAuthScopesPlug, - %{scopes: ["write:conversations"]} when action == :update_conversation + %{scopes: ["write:conversations"]} when action in [:update_conversation, :read_conversations] ) plug(OAuthScopesPlug, %{scopes: ["write:notifications"]} when action == :read_notification) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 5a0902739..3d57073d0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -34,6 +34,7 @@ defmodule Pleroma.Web.Router do pipeline :authenticated_api do plug(:accepts, ["json"]) plug(:fetch_session) + plug(Pleroma.Plugs.AuthExpectedPlug) plug(Pleroma.Plugs.OAuthPlug) plug(Pleroma.Plugs.BasicAuthDecoderPlug) plug(Pleroma.Plugs.UserFetcherPlug) @@ -333,7 +334,7 @@ defmodule Pleroma.Web.Router do get("/accounts/relationships", AccountController, :relationships) get("/accounts/:id/lists", AccountController, :lists) - get("/accounts/:id/identity_proofs", MastodonAPIController, :empty_array) + get("/accounts/:id/identity_proofs", AccountController, :identity_proofs) get("/follow_requests", FollowRequestController, :index) get("/blocks", AccountController, :blocks) diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 0229aea97..31adc2817 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -15,6 +15,8 @@ defmodule Pleroma.Web.TwitterAPI.Controller do plug(OAuthScopesPlug, %{scopes: ["write:notifications"]} when action == :notifications_read) + plug(:skip_plug, OAuthScopesPlug when action in [:oauth_tokens, :revoke_token]) + plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug) action_fallback(:errors) diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex index cf3ac1287..1af29ce78 100644 --- a/lib/pleroma/web/web.ex +++ b/lib/pleroma/web/web.ex @@ -29,11 +29,34 @@ def controller do import Pleroma.Web.Router.Helpers import Pleroma.Web.TranslationHelpers + alias Pleroma.Plugs.PlugHelper + plug(:set_put_layout) defp set_put_layout(conn, _) do put_layout(conn, Pleroma.Config.get(:app_layout, "app.html")) end + + # Marks a plug as intentionally skipped + # (states that the plug is not called for a good reason, not by a mistake) + defp skip_plug(conn, plug_module) do + PlugHelper.append_to_skipped_plugs(conn, plug_module) + end + + # Here we can apply before-action hooks (e.g. verify whether auth checks were preformed) + defp action(conn, params) do + if conn.private[:auth_expected] && + not PlugHelper.plug_called_or_skipped?(conn, Pleroma.Plugs.OAuthScopesPlug) do + conn + |> render_error( + :forbidden, + "Security violation: OAuth scopes check was neither handled nor explicitly skipped." + ) + |> halt() + else + super(conn, params) + end + end end end diff --git a/test/web/mastodon_api/controllers/suggestion_controller_test.exs b/test/web/mastodon_api/controllers/suggestion_controller_test.exs index c697a39f8..8d0e70db8 100644 --- a/test/web/mastodon_api/controllers/suggestion_controller_test.exs +++ b/test/web/mastodon_api/controllers/suggestion_controller_test.exs @@ -7,34 +7,8 @@ defmodule Pleroma.Web.MastodonAPI.SuggestionControllerTest do alias Pleroma.Config - import Pleroma.Factory - import Tesla.Mock - setup do: oauth_access(["read"]) - setup %{user: user} do - other_user = insert(:user) - host = Config.get([Pleroma.Web.Endpoint, :url, :host]) - url500 = "http://test500?#{host}&#{user.nickname}" - url200 = "http://test200?#{host}&#{user.nickname}" - - mock(fn - %{method: :get, url: ^url500} -> - %Tesla.Env{status: 500, body: "bad request"} - - %{method: :get, url: ^url200} -> - %Tesla.Env{ - status: 200, - body: - ~s([{"acct":"yj455","avatar":"https://social.heldscal.la/avatar/201.jpeg","avatar_static":"https://social.heldscal.la/avatar/s/201.jpeg"}, {"acct":"#{ - other_user.ap_id - }","avatar":"https://social.heldscal.la/avatar/202.jpeg","avatar_static":"https://social.heldscal.la/avatar/s/202.jpeg"}]) - } - end) - - [other_user: other_user] - end - test "returns empty result", %{conn: conn} do res = conn diff --git a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs index 32250f06f..8f0cbe9b2 100644 --- a/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs +++ b/test/web/pleroma_api/controllers/pleroma_api_controller_test.exs @@ -203,7 +203,7 @@ test "PATCH /api/v1/pleroma/conversations/:id" do test "POST /api/v1/pleroma/conversations/read" do user = insert(:user) - %{user: other_user, conn: conn} = oauth_access(["write:notifications"]) + %{user: other_user, conn: conn} = oauth_access(["write:conversations"]) {:ok, _activity} = CommonAPI.post(user, %{"status" => "Hi @#{other_user.nickname}", "visibility" => "direct"}) From 03eebabe8e5b2e3f96f6ffe51a6f063a42f6a5d2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 3 Apr 2020 22:52:25 +0400 Subject: [PATCH 30/84] Add Pleroma.Web.ApiSpec.Helpers --- lib/pleroma/web/api_spec/helpers.ex | 27 +++++++++++++++++++ .../web/api_spec/operations/app_operation.ex | 4 +-- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/web/api_spec/helpers.ex diff --git a/lib/pleroma/web/api_spec/helpers.ex b/lib/pleroma/web/api_spec/helpers.ex new file mode 100644 index 000000000..35cf4c0d8 --- /dev/null +++ b/lib/pleroma/web/api_spec/helpers.ex @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Helpers do + def request_body(description, schema_ref, opts \\ []) do + media_types = ["application/json", "multipart/form-data"] + + content = + media_types + |> Enum.map(fn type -> + {type, + %OpenApiSpex.MediaType{ + schema: schema_ref, + example: opts[:example], + examples: opts[:examples] + }} + end) + |> Enum.into(%{}) + + %OpenApiSpex.RequestBody{ + description: description, + content: content, + required: opts[:required] || false + } + end +end diff --git a/lib/pleroma/web/api_spec/operations/app_operation.ex b/lib/pleroma/web/api_spec/operations/app_operation.ex index 41d56693a..26d8dbd42 100644 --- a/lib/pleroma/web/api_spec/operations/app_operation.ex +++ b/lib/pleroma/web/api_spec/operations/app_operation.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ApiSpec.AppOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Helpers alias Pleroma.Web.ApiSpec.Schemas.AppCreateRequest alias Pleroma.Web.ApiSpec.Schemas.AppCreateResponse @@ -21,8 +22,7 @@ def create_operation do summary: "Create an application", description: "Create a new application to obtain OAuth2 credentials", operationId: "AppController.create", - requestBody: - Operation.request_body("Parameters", "application/json", AppCreateRequest, required: true), + requestBody: Helpers.request_body("Parameters", AppCreateRequest, required: true), responses: %{ 200 => Operation.response("App", "application/json", AppCreateResponse), 422 => From 5739c498c029914c446656244cdd213a3e358fec Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 8 Apr 2020 18:46:01 +0300 Subject: [PATCH 31/84] fix for gun connections pool --- CHANGELOG.md | 3 +++ lib/pleroma/gun/conn.ex | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6e5d807c..92d1abc4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Support for `include_types` in `/api/v1/notifications`.
+### Fixed +- Gun connections pool `max_connections` option. + ## [2.0.0] - 2019-03-08 ### Security - Mastodon API: Fix being able to request enourmous amount of statuses in timelines leading to DoS. Now limited to 40 per request. diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 20823a765..cd25a2e74 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -49,8 +49,10 @@ def open(%URI{} = uri, name, opts) do key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + max_connections = pool_opts[:max_connections] || 250 + conn_pid = - if Connections.count(name) < opts[:max_connection] do + if Connections.count(name) < max_connections do do_open(uri, opts) else close_least_used_and_do_open(name, uri, opts) From d067eaa7b3bb76e7fc5ae019d6e00510b657171d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 8 Apr 2020 22:58:31 +0300 Subject: [PATCH 32/84] formatter.ex: Use Phoenix.HTML for mention/hashtag generation Unlike concatenating strings, this makes sure everything is escaped. Tests had to be changed because Phoenix.HTML runs attributes through Enum.sort before generation for whatever reason. --- lib/pleroma/formatter.ex | 26 ++++++++++++++++--- test/formatter_test.exs | 24 +++++++---------- test/user_test.exs | 2 +- test/web/common_api/common_api_utils_test.exs | 6 ++--- .../update_credentials_test.exs | 4 +-- .../notification_controller_test.exs | 4 +-- test/web/twitter_api/twitter_api_test.exs | 2 +- 7 files changed, 41 insertions(+), 27 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index e2a658cb3..c44e7fc8b 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -35,9 +35,19 @@ def mention_handler("@" <> nickname, buffer, opts, acc) do nickname_text = get_nickname_text(nickname, opts) link = - ~s(@#{ - nickname_text - }) + Phoenix.HTML.Tag.content_tag( + :span, + Phoenix.HTML.Tag.content_tag( + :a, + ["@", Phoenix.HTML.Tag.content_tag(:span, nickname_text)], + "data-user": id, + class: "u-url mention", + href: ap_id, + rel: "ugc" + ), + class: "h-card" + ) + |> Phoenix.HTML.safe_to_string() {link, %{acc | mentions: MapSet.put(acc.mentions, {"@" <> nickname, user})}} @@ -49,7 +59,15 @@ def mention_handler("@" <> nickname, buffer, opts, acc) do def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do tag = String.downcase(tag) url = "#{Pleroma.Web.base_url()}/tag/#{tag}" - link = ~s(#{tag_text}) + + link = + Phoenix.HTML.Tag.content_tag(:a, tag_text, + class: "hashtag", + "data-tag": tag, + href: url, + rel: "tag ugc" + ) + |> Phoenix.HTML.safe_to_string() {link, %{acc | tags: MapSet.put(acc.tags, {tag_text, tag})}} end diff --git a/test/formatter_test.exs b/test/formatter_test.exs index cf8441cf6..93fd8eab7 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -150,13 +150,13 @@ test "gives a replacement for user links, using local nicknames in user links te assert length(mentions) == 3 expected_text = - ~s(@gsimg According to @gsimg According to @archa_eme_, that is @daggsy. Also hello @archa_eme_, that is @daggsy. Also hello @archaeme) + }" href="#{archaeme_remote.ap_id}" rel="ugc">@archaeme) assert expected_text == text end @@ -171,7 +171,7 @@ test "gives a replacement for user links when the user is using Osada" do assert length(mentions) == 1 expected_text = - ~s(@mike test) @@ -187,7 +187,7 @@ test "gives a replacement for single-character local nicknames" do assert length(mentions) == 1 expected_text = - ~s(@o hi) + ~s(@o hi) assert expected_text == text end @@ -209,17 +209,13 @@ test "given the 'safe_mention' option, it will only mention people in the beginn assert mentions == [{"@#{user.nickname}", user}, {"@#{other_user.nickname}", other_user}] assert expected_text == - ~s(@#{user.nickname} @#{user.nickname} @#{ - other_user.nickname - } hey dudes i hate @#{other_user.nickname} hey dudes i hate @#{ - third_user.nickname - }) + }" href="#{third_user.ap_id}" rel="ugc">@#{third_user.nickname}) end test "given the 'safe_mention' option, it will still work without any mention" do diff --git a/test/user_test.exs b/test/user_test.exs index 0479f294d..d39787f35 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1404,7 +1404,7 @@ test "preserves hosts in user links text" do bio = "A.k.a. @nick@domain.com" expected_text = - ~s(A.k.a. @nick@domain.com) diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs index d383d1714..98cf02d49 100644 --- a/test/web/common_api/common_api_utils_test.exs +++ b/test/web/common_api/common_api_utils_test.exs @@ -159,11 +159,11 @@ test "works for text/markdown with mentions" do {output, _, _} = Utils.format_input(text, "text/markdown") assert output == - ~s(

hello world

another @user__test and @user__test and @user__test google.com paragraph

) + }" href="http://foo.com/user__test" rel="ugc">@user__test google.com paragraph

) end end diff --git a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs index d78fbc5a1..2d256f63c 100644 --- a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs @@ -82,9 +82,9 @@ test "updates the user's bio", %{conn: conn} do assert user_data = json_response(conn, 200) assert user_data["note"] == - ~s(I drink #cofe with #cofe with @#{user2.nickname}

suya..) + }" href="#{user2.ap_id}" rel="ugc">@#{user2.nickname}

suya..) end test "updates the user's locking status", %{conn: conn} do diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs index 344eabb4a..6f1fab069 100644 --- a/test/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/web/mastodon_api/controllers/notification_controller_test.exs @@ -26,7 +26,7 @@ test "list of notifications" do |> get("/api/v1/notifications") expected_response = - "hi @#{user.nickname}" @@ -45,7 +45,7 @@ test "getting a single notification" do conn = get(conn, "/api/v1/notifications/#{notification.id}") expected_response = - "hi @#{user.nickname}" diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 92f9aa0f5..f6e13b661 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -109,7 +109,7 @@ test "it registers a new user and parses mentions in the bio" do {:ok, user2} = TwitterAPI.register_user(data2) expected_text = - ~s(@john test) From c401b00c7885823744183dbd077db9239585d20d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 9 Apr 2020 04:36:39 +0200 Subject: [PATCH 33/84] ObjectValidators.Types.ObjectID: Fix when URI.parse returns %URL{host: ""} --- .../object_validators/types/object_id.ex | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex index ee10be0b0..f6e749b33 100644 --- a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex +++ b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex @@ -6,14 +6,10 @@ def type, do: :string def cast(object) when is_binary(object) do # Host has to be present and scheme has to be an http scheme (for now) case URI.parse(object) do - %URI{host: nil} -> - :error - - %URI{scheme: scheme} when scheme in ["https", "http"] -> - {:ok, object} - - _ -> - :error + %URI{host: nil} -> :error + %URI{host: ""} -> :error + %URI{scheme: scheme} when scheme in ["https", "http"] -> {:ok, object} + _ -> :error end end From 73134e248a031613151df87fdd406580d16dc6b9 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Apr 2020 08:03:21 +0300 Subject: [PATCH 34/84] no changelog entry - bug fixed only in develop --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92d1abc4e..b6e5d807c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,9 +20,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Support for `include_types` in `/api/v1/notifications`. -### Fixed -- Gun connections pool `max_connections` option. - ## [2.0.0] - 2019-03-08 ### Security - Mastodon API: Fix being able to request enourmous amount of statuses in timelines leading to DoS. Now limited to 40 per request. From c8bfbf511eeca2045267ad4792c35648625788cf Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Apr 2020 10:17:24 +0000 Subject: [PATCH 35/84] Apply suggestion to docs/API/admin_api.md --- docs/API/admin_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 179d8c451..b3cf89818 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -400,7 +400,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret ```json [ { - `error` // error message + "error": "Appropriate error message here" } ] ``` From 4c60fdcbb1ab06183b8e300cbbb84d70ecd3e25b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Apr 2020 10:17:31 +0000 Subject: [PATCH 36/84] Apply suggestion to lib/pleroma/web/admin_api/admin_api_controller.ex --- lib/pleroma/web/admin_api/admin_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 7b442f6e1..a66db68f3 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -592,7 +592,7 @@ def email_invite(%{assigns: %{user: user}} = conn, %{"email" => email} = params) {:registrations_open, _} -> errors( conn, - {:error, "To send invites you need set `registrations_open` option to false."} + {:error, "To send invites you need to set the `registrations_open` option to false."} ) {:invites_enabled, _} -> From 1cf0d5ab0d579ee4a1a779c308fedb0ab8ec3884 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Apr 2020 10:17:36 +0000 Subject: [PATCH 37/84] Apply suggestion to lib/pleroma/web/admin_api/admin_api_controller.ex --- lib/pleroma/web/admin_api/admin_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index a66db68f3..09959b3bf 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -598,7 +598,7 @@ def email_invite(%{assigns: %{user: user}} = conn, %{"email" => email} = params) {:invites_enabled, _} -> errors( conn, - {:error, "To send invites you need set `invites_enabled` option to true."} + {:error, "To send invites you need set to set the `invites_enabled` option to true."} ) end end From 365c34a7a96a9cbd5acb30eb6eedf195eeaff131 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Apr 2020 10:17:44 +0000 Subject: [PATCH 38/84] Apply suggestion to test/web/admin_api/admin_api_controller_test.exs --- test/web/admin_api/admin_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 32fe69d19..afd894269 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -671,7 +671,7 @@ test "it returns 500 if `invites_enabled` is not enabled", %{conn: conn} do conn = post(conn, "/api/pleroma/admin/users/email_invite?email=foo@bar.com&name=JD") assert json_response(conn, :bad_request) == - "To send invites you need set `invites_enabled` option to true." + "To send invites you need to set the `invites_enabled` option to true." end test "it returns 500 if `registrations_open` is enabled", %{conn: conn} do From 9795ff5b016e74c0e7b94ac2ea28023208d1f8ee Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Apr 2020 10:17:50 +0000 Subject: [PATCH 39/84] Apply suggestion to test/web/admin_api/admin_api_controller_test.exs --- test/web/admin_api/admin_api_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index afd894269..e8d11b88c 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -681,7 +681,7 @@ test "it returns 500 if `registrations_open` is enabled", %{conn: conn} do conn = post(conn, "/api/pleroma/admin/users/email_invite?email=foo@bar.com&name=JD") assert json_response(conn, :bad_request) == - "To send invites you need set `registrations_open` option to false." + "To send invites you need to set the `registrations_open` option to false." end end From f20a19de853e8834f7774ee0098a14213bc7427f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Apr 2020 13:28:54 +0300 Subject: [PATCH 40/84] typo fix --- lib/pleroma/web/admin_api/admin_api_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 09959b3bf..fdbd24acb 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -598,7 +598,7 @@ def email_invite(%{assigns: %{user: user}} = conn, %{"email" => email} = params) {:invites_enabled, _} -> errors( conn, - {:error, "To send invites you need set to set the `invites_enabled` option to true."} + {:error, "To send invites you need to set the `invites_enabled` option to true."} ) end end From d37a102933dbfbb0996546b4d148bbe36fbd4220 Mon Sep 17 00:00:00 2001 From: kPherox Date: Thu, 9 Apr 2020 21:16:29 +0900 Subject: [PATCH 41/84] Fix OTP_VERSION file in docker --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 29931a5e3..c2f3ad98c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,8 @@ RUN apk add git gcc g++ musl-dev make &&\ mkdir release &&\ mix release --path release +RUN echo "${OTP_VERSION}" > release/OTP_VERSION + FROM alpine:3.11 ARG BUILD_DATE From d545b883eb3c5b79b89a49ccaf9256c31b401145 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 9 Apr 2020 17:08:43 +0400 Subject: [PATCH 42/84] Add `/api/v1/notifications/:id/dismiss` endpoint --- .../controllers/notification_controller.ex | 3 ++- lib/pleroma/web/router.ex | 4 +++- .../notification_controller_test.exs | 18 +++++++++++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index 0c9218454..a6b4096ec 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -66,7 +66,8 @@ def clear(%{assigns: %{user: user}} = conn, _params) do json(conn, %{}) end - # POST /api/v1/notifications/dismiss + # POST /api/v1/notifications/:id/dismiss + # POST /api/v1/notifications/dismiss (deprecated) def dismiss(%{assigns: %{user: user}} = conn, %{"id" => id} = _params) do with {:ok, _notif} <- Notification.dismiss(user, id) do json(conn, %{}) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 3ecd59cd1..5f5ec1c81 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -352,9 +352,11 @@ defmodule Pleroma.Web.Router do get("/notifications", NotificationController, :index) get("/notifications/:id", NotificationController, :show) + post("/notifications/:id/dismiss", NotificationController, :dismiss) post("/notifications/clear", NotificationController, :clear) - post("/notifications/dismiss", NotificationController, :dismiss) delete("/notifications/destroy_multiple", NotificationController, :destroy_multiple) + # Deprecated: was removed in Mastodon v3, use `/notifications/:id/dismiss` instead + post("/notifications/dismiss", NotificationController, :dismiss) get("/scheduled_statuses", ScheduledActivityController, :index) get("/scheduled_statuses/:id", ScheduledActivityController, :show) diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs index 6f1fab069..1557937d8 100644 --- a/test/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/web/mastodon_api/controllers/notification_controller_test.exs @@ -53,7 +53,7 @@ test "getting a single notification" do assert response == expected_response end - test "dismissing a single notification" do + test "dismissing a single notification (deprecated endpoint)" do %{user: user, conn: conn} = oauth_access(["write:notifications"]) other_user = insert(:user) @@ -69,6 +69,22 @@ test "dismissing a single notification" do assert %{} = json_response(conn, 200) end + test "dismissing a single notification" do + %{user: user, conn: conn} = oauth_access(["write:notifications"]) + other_user = insert(:user) + + {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hi @#{user.nickname}"}) + + {:ok, [notification]} = Notification.create_notifications(activity) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/notifications/#{notification.id}/dismiss") + + assert %{} = json_response(conn, 200) + end + test "clearing all notifications" do %{user: user, conn: conn} = oauth_access(["write:notifications", "read:notifications"]) other_user = insert(:user) From 0e8f6d24b87812664d3bb021d17f120686cf2401 Mon Sep 17 00:00:00 2001 From: kPherox Date: Fri, 10 Apr 2020 00:19:09 +0900 Subject: [PATCH 43/84] Create OTP_VERSION file by `mix release` --- Dockerfile | 2 -- mix.exs | 11 ++++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c2f3ad98c..29931a5e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,6 @@ RUN apk add git gcc g++ musl-dev make &&\ mkdir release &&\ mix release --path release -RUN echo "${OTP_VERSION}" > release/OTP_VERSION - FROM alpine:3.11 ARG BUILD_DATE diff --git a/mix.exs b/mix.exs index 3e4c7cbd8..ad2029518 100644 --- a/mix.exs +++ b/mix.exs @@ -37,12 +37,21 @@ def project do pleroma: [ include_executables_for: [:unix], applications: [ex_syslogger: :load, syslog: :load], - steps: [:assemble, ©_files/1, ©_nginx_config/1] + steps: [:assemble, &put_files/1, ©_files/1, ©_nginx_config/1] ] ] ] end + def put_files(%{path: target_path} = release) do + File.write!( + Path.join([target_path, "OTP_VERSION"]), + Pleroma.OTPVersion.version() + ) + + release + end + def copy_files(%{path: target_path} = release) do File.cp_r!("./rel/files", target_path) release From c826d5195f1746449eb369e86a730f14de9fa267 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 9 Apr 2020 23:36:17 +0400 Subject: [PATCH 44/84] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6e5d807c..2f5d8f612 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
API Changes - Mastodon API: Support for `include_types` in `/api/v1/notifications`. +- Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint.
## [2.0.0] - 2019-03-08 From 781ac28859596fce5f2fd24ffe1cdf24caaaa2fc Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 15 Mar 2020 17:26:58 +0300 Subject: [PATCH 45/84] changelog.md: add 2.0.1 entry --- CHANGELOG.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f5d8f612..15f0463b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,9 +3,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [unreleased] +## [2.0.1] - 2020-03-15 +### Fixed +- 500 errors when no `Accept` header is present if Static-FE is enabled +- Instance panel not being updated immediately due to wrong `Cache-Control` headers +- Statuses posted with BBCode/Markdown having unncessary newlines in Pleroma-FE +- OTP: Fix some settings not being migrated to in-database config properly +- No `Cache-Control` headers on attachment/media proxy requests +- Character limit enforcement being off by 1 +- Mastodon Streaming API: hashtag timelines not working + ### Changed -- **Breaking:** BBCode and Markdown formatters will no longer return any `\n` and only use `
` for newlines +- BBCode and Markdown formatters will no longer return any `\n` and only use `
` for newlines +- Mastodon API: Allow registration without email if email verification is not enabled ### Removed - **Breaking:** removed `with_move` parameter from notifications timeline. From 2a08f44b026bae611064b6ac459e7df16e4a36f9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 16 Mar 2020 00:50:03 +0300 Subject: [PATCH 46/84] CHANGELOG.md: Add upgrade notes for 2.0.1 --- CHANGELOG.md | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15f0463b2..8c976228c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## [unreleased] +### Removed +- **Breaking:** removed `with_move` parameter from notifications timeline. + +### Added +- NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list. +- NodeInfo: `pleroma_emoji_reactions` to the `features` list. +- Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses. +- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required. +
+ API Changes +- Mastodon API: Support for `include_types` in `/api/v1/notifications`. +- Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint. +
+ ## [2.0.1] - 2020-03-15 ### Fixed - 500 errors when no `Accept` header is present if Static-FE is enabled @@ -17,19 +32,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - BBCode and Markdown formatters will no longer return any `\n` and only use `
` for newlines - Mastodon API: Allow registration without email if email verification is not enabled -### Removed -- **Breaking:** removed `with_move` parameter from notifications timeline. +### Upgrade notes +#### Nginx only +1. Remove `proxy_ignore_headers Cache-Control;` and `proxy_hide_header Cache-Control;` from your config. -### Added -- NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list. -- NodeInfo: `pleroma_emoji_reactions` to the `features` list. -- Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses. -- New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required. -
- API Changes -- Mastodon API: Support for `include_types` in `/api/v1/notifications`. -- Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint. -
+#### Everyone +1. Run database migrations (inside Pleroma directory): + - OTP: `./bin/pleroma_ctl migrate` + - From Source: `mix ecto.migrate` +2. Restart Pleroma ## [2.0.0] - 2019-03-08 ### Security From 7306d2d06942f7912fd42809b1feb9ac43089012 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 31 Mar 2020 13:59:26 +0300 Subject: [PATCH 47/84] CHANGELOG.md: Add 2.0.2 entry --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c976228c..6942ad0bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint. +## [2.0.2] - 2020-03-31 +### Fixed +- Blocked/muted users still generating push notifications +- Input textbox for bio ignoring newlines +- OTP: Inability to use PostgreSQL databases with SSL +- `user delete_activities` breaking when trying to delete already deleted posts + +### Added +- Admin API: `PATCH /api/pleroma/admin/users/:nickname/update_credentials` + ## [2.0.1] - 2020-03-15 +### Security +- Static-FE: Fix remote posts not being sanitized + ### Fixed - 500 errors when no `Accept` header is present if Static-FE is enabled - Instance panel not being updated immediately due to wrong `Cache-Control` headers From 0b8f9a66aefdf4c9e2b7c1fa931e19cd724b6b4b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 2 Apr 2020 23:37:14 +0300 Subject: [PATCH 48/84] CHANGELOG.md: add entries for funkwhale-related changes --- CHANGELOG.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6942ad0bf..8eed9cf7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,14 +19,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [2.0.2] - 2020-03-31 +### Added +- Support for Funkwhale's `Audio` activity +- Admin API: `PATCH /api/pleroma/admin/users/:nickname/update_credentials` + ### Fixed - Blocked/muted users still generating push notifications - Input textbox for bio ignoring newlines - OTP: Inability to use PostgreSQL databases with SSL - `user delete_activities` breaking when trying to delete already deleted posts - -### Added -- Admin API: `PATCH /api/pleroma/admin/users/:nickname/update_credentials` +- Incorrect URL for Funkwhale channels ## [2.0.1] - 2020-03-15 ### Security From adeb82e4966a505e9ac65743e6336db27558e38f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 8 Apr 2020 00:38:48 +0300 Subject: [PATCH 49/84] CHANGELOG.md: add 2.0.2 update notes --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eed9cf7d..408b932b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - `user delete_activities` breaking when trying to delete already deleted posts - Incorrect URL for Funkwhale channels +### Upgrade notes +1. Restart Pleroma + ## [2.0.1] - 2020-03-15 ### Security - Static-FE: Fix remote posts not being sanitized From 9abf13abe05f3f53bdf21d4d97242e571b1767c6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 8 Apr 2020 00:39:55 +0300 Subject: [PATCH 50/84] CHANGELOG.md: update 2.0.2 release date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 408b932b8..bac69ad6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint. -## [2.0.2] - 2020-03-31 +## [2.0.2] - 2020-04-08 ### Added - Support for Funkwhale's `Audio` activity - Admin API: `PATCH /api/pleroma/admin/users/:nickname/update_credentials` From c2aad36aa86694d4131adb2ed47441beca2ab2e8 Mon Sep 17 00:00:00 2001 From: kPherox Date: Thu, 9 Apr 2020 23:19:41 +0000 Subject: [PATCH 51/84] Rename function --- mix.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index ad2029518..a1fcde564 100644 --- a/mix.exs +++ b/mix.exs @@ -37,13 +37,13 @@ def project do pleroma: [ include_executables_for: [:unix], applications: [ex_syslogger: :load, syslog: :load], - steps: [:assemble, &put_files/1, ©_files/1, ©_nginx_config/1] + steps: [:assemble, &put_otp_version/1, ©_files/1, ©_nginx_config/1] ] ] ] end - def put_files(%{path: target_path} = release) do + def put_otp_version(%{path: target_path} = release) do File.write!( Path.join([target_path, "OTP_VERSION"]), Pleroma.OTPVersion.version() From 5628984df4809888746ea005decf3856ca929858 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 1 Apr 2020 01:50:53 +0200 Subject: [PATCH 52/84] User: remove source_data use for follower_address and following_address --- lib/pleroma/user.ex | 91 ++++++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 71c8c3a4e..d030c7314 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -306,6 +306,7 @@ def banner_url(user, options \\ []) do end end + # Should probably be renamed or removed def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}" def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa @@ -339,6 +340,13 @@ defp truncate_if_exists(params, key, max_length) do end end + defp fix_follower_address(%{follower_address: _, following_address: _} = params), do: params + + defp fix_follower_address(%{nickname: nickname} = params), + do: Map.put(params, :follower_address, ap_followers(%User{nickname: nickname})) + + defp fix_follower_address(params), do: params + def remote_user_creation(params) do bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) @@ -348,53 +356,44 @@ def remote_user_creation(params) do |> truncate_if_exists(:name, name_limit) |> truncate_if_exists(:bio, bio_limit) |> truncate_fields_param() + |> fix_follower_address() - changeset = - %User{local: false} - |> cast( - params, - [ - :bio, - :name, - :ap_id, - :nickname, - :avatar, - :ap_enabled, - :source_data, - :banner, - :locked, - :magic_key, - :uri, - :hide_followers, - :hide_follows, - :hide_followers_count, - :hide_follows_count, - :follower_count, - :fields, - :following_count, - :discoverable, - :invisible, - :actor_type, - :also_known_as - ] - ) - |> validate_required([:name, :ap_id]) - |> unique_constraint(:nickname) - |> validate_format(:nickname, @email_regex) - |> validate_length(:bio, max: bio_limit) - |> validate_length(:name, max: name_limit) - |> validate_fields(true) - - case params[:source_data] do - %{"followers" => followers, "following" => following} -> - changeset - |> put_change(:follower_address, followers) - |> put_change(:following_address, following) - - _ -> - followers = ap_followers(%User{nickname: get_field(changeset, :nickname)}) - put_change(changeset, :follower_address, followers) - end + %User{local: false} + |> cast( + params, + [ + :bio, + :name, + :ap_id, + :nickname, + :avatar, + :ap_enabled, + :source_data, + :banner, + :locked, + :magic_key, + :uri, + :follower_address, + :following_address, + :hide_followers, + :hide_follows, + :hide_followers_count, + :hide_follows_count, + :follower_count, + :fields, + :following_count, + :discoverable, + :invisible, + :actor_type, + :also_known_as + ] + ) + |> validate_required([:name, :ap_id]) + |> unique_constraint(:nickname) + |> validate_format(:nickname, @email_regex) + |> validate_length(:bio, max: bio_limit) + |> validate_length(:name, max: name_limit) + |> validate_fields(true) end def update_changeset(struct, params \\ %{}) do From 19eedb3d0424abb235eec1a51457ed0bf3a0e95d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 1 Apr 2020 06:58:48 +0200 Subject: [PATCH 53/84] User: Move public_key from source_data to own field --- lib/pleroma/user.ex | 9 ++++++--- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +++- .../20200401030751_users_add_public_key.exs | 17 +++++++++++++++++ test/signature_test.exs | 13 ++++--------- 4 files changed, 30 insertions(+), 13 deletions(-) create mode 100644 priv/repo/migrations/20200401030751_users_add_public_key.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index d030c7314..0adea42ec 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -82,6 +82,7 @@ defmodule Pleroma.User do field(:password, :string, virtual: true) field(:password_confirmation, :string, virtual: true) field(:keys, :string) + field(:public_key, :string) field(:ap_id, :string) field(:avatar, :map) field(:local, :boolean, default: true) @@ -366,6 +367,7 @@ def remote_user_creation(params) do :name, :ap_id, :nickname, + :public_key, :avatar, :ap_enabled, :source_data, @@ -407,6 +409,7 @@ def update_changeset(struct, params \\ %{}) do :bio, :name, :avatar, + :public_key, :locked, :no_rich_text, :default_scope, @@ -503,6 +506,7 @@ def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do :name, :follower_address, :following_address, + :public_key, :avatar, :last_refreshed_at, :ap_enabled, @@ -1616,8 +1620,7 @@ defp create_service_actor(uri, nickname) do |> set_cache() end - # AP style - def public_key(%{source_data: %{"publicKey" => %{"publicKeyPem" => public_key_pem}}}) do + def public_key(%{public_key: public_key_pem}) when is_binary(public_key_pem) do key = public_key_pem |> :public_key.pem_decode() @@ -1627,7 +1630,7 @@ def public_key(%{source_data: %{"publicKey" => %{"publicKeyPem" => public_key_pe {:ok, key} end - def public_key(_), do: {:error, "not found key"} + def public_key(_), do: {:error, "key not found"} def get_public_key_for_ap_id(ap_id) do with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id), diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 19286fd01..0e4a9d842 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1432,6 +1432,7 @@ defp object_to_user_data(data) do discoverable = data["discoverable"] || false invisible = data["invisible"] || false actor_type = data["type"] || "Person" + public_key = data["publicKey"]["publicKeyPem"] user_data = %{ ap_id: data["id"], @@ -1449,7 +1450,8 @@ defp object_to_user_data(data) do following_address: data["following"], bio: data["summary"], actor_type: actor_type, - also_known_as: Map.get(data, "alsoKnownAs", []) + also_known_as: Map.get(data, "alsoKnownAs", []), + public_key: public_key } # nickname can be nil because of virtual actors diff --git a/priv/repo/migrations/20200401030751_users_add_public_key.exs b/priv/repo/migrations/20200401030751_users_add_public_key.exs new file mode 100644 index 000000000..04e5ad1e2 --- /dev/null +++ b/priv/repo/migrations/20200401030751_users_add_public_key.exs @@ -0,0 +1,17 @@ +defmodule Pleroma.Repo.Migrations.UsersAddPublicKey do + use Ecto.Migration + + def up do + alter table(:users) do + add_if_not_exists(:public_key, :text) + end + + execute("UPDATE users SET public_key = source_data->'publicKey'->>'publicKeyPem'") + end + + def down do + alter table(:users) do + remove_if_exists(:public_key, :text) + end + end +end diff --git a/test/signature_test.exs b/test/signature_test.exs index 04736d8b9..d5a2a62c4 100644 --- a/test/signature_test.exs +++ b/test/signature_test.exs @@ -19,12 +19,7 @@ defmodule Pleroma.SignatureTest do @private_key "-----BEGIN RSA PRIVATE KEY-----\nMIIEpQIBAAKCAQEA48qb4v6kqigZutO9Ot0wkp27GIF2LiVaADgxQORZozZR63jH\nTaoOrS3Xhngbgc8SSOhfXET3omzeCLqaLNfXnZ8OXmuhJfJSU6mPUvmZ9QdT332j\nfN/g3iWGhYMf/M9ftCKh96nvFVO/tMruzS9xx7tkrfJjehdxh/3LlJMMImPtwcD7\nkFXwyt1qZTAU6Si4oQAJxRDQXHp1ttLl3Ob829VM7IKkrVmY8TD+JSlV0jtVJPj6\n1J19ytKTx/7UaucYvb9HIiBpkuiy5n/irDqKLVf5QEdZoNCdojOZlKJmTLqHhzKP\n3E9TxsUjhrf4/EqegNc/j982RvOxeu4i40zMQwIDAQABAoIBAQDH5DXjfh21i7b4\ncXJuw0cqget617CDUhemdakTDs9yH+rHPZd3mbGDWuT0hVVuFe4vuGpmJ8c+61X0\nRvugOlBlavxK8xvYlsqTzAmPgKUPljyNtEzQ+gz0I+3mH2jkin2rL3D+SksZZgKm\nfiYMPIQWB2WUF04gB46DDb2mRVuymGHyBOQjIx3WC0KW2mzfoFUFRlZEF+Nt8Ilw\nT+g/u0aZ1IWoszbsVFOEdghgZET0HEarum0B2Je/ozcPYtwmU10iBANGMKdLqaP/\nj954BPunrUf6gmlnLZKIKklJj0advx0NA+cL79+zeVB3zexRYSA5o9q0WPhiuTwR\n/aedWHnBAoGBAP0sDWBAM1Y4TRAf8ZI9PcztwLyHPzfEIqzbObJJnx1icUMt7BWi\n+/RMOnhrlPGE1kMhOqSxvXYN3u+eSmWTqai2sSH5Hdw2EqnrISSTnwNUPINX7fHH\njEkgmXQ6ixE48SuBZnb4w1EjdB/BA6/sjL+FNhggOc87tizLTkMXmMtTAoGBAOZV\n+wPuAMBDBXmbmxCuDIjoVmgSlgeRunB1SA8RCPAFAiUo3+/zEgzW2Oz8kgI+xVwM\n33XkLKrWG1Orhpp6Hm57MjIc5MG+zF4/YRDpE/KNG9qU1tiz0UD5hOpIU9pP4bR/\ngxgPxZzvbk4h5BfHWLpjlk8UUpgk6uxqfti48c1RAoGBALBOKDZ6HwYRCSGMjUcg\n3NPEUi84JD8qmFc2B7Tv7h2he2ykIz9iFAGpwCIyETQsJKX1Ewi0OlNnD3RhEEAy\nl7jFGQ+mkzPSeCbadmcpYlgIJmf1KN/x7fDTAepeBpCEzfZVE80QKbxsaybd3Dp8\nCfwpwWUFtBxr4c7J+gNhAGe/AoGAPn8ZyqkrPv9wXtyfqFjxQbx4pWhVmNwrkBPi\nZ2Qh3q4dNOPwTvTO8vjghvzIyR8rAZzkjOJKVFgftgYWUZfM5gE7T2mTkBYq8W+U\n8LetF+S9qAM2gDnaDx0kuUTCq7t87DKk6URuQ/SbI0wCzYjjRD99KxvChVGPBHKo\n1DjqMuECgYEAgJGNm7/lJCS2wk81whfy/ttKGsEIkyhPFYQmdGzSYC5aDc2gp1R3\nxtOkYEvdjfaLfDGEa4UX8CHHF+w3t9u8hBtcdhMH6GYb9iv6z0VBTt4A/11HUR49\n3Z7TQ18Iyh3jAUCzFV9IJlLIExq5Y7P4B3ojWFBN607sDCt8BMPbDYs=\n-----END RSA PRIVATE KEY-----" - @public_key %{ - "id" => "https://mastodon.social/users/lambadalambda#main-key", - "owner" => "https://mastodon.social/users/lambadalambda", - "publicKeyPem" => - "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw0P/Tq4gb4G/QVuMGbJo\nC/AfMNcv+m7NfrlOwkVzcU47jgESuYI4UtJayissCdBycHUnfVUd9qol+eznSODz\nCJhfJloqEIC+aSnuEPGA0POtWad6DU0E6/Ho5zQn5WAWUwbRQqowbrsm/GHo2+3v\neR5jGenwA6sYhINg/c3QQbksyV0uJ20Umyx88w8+TJuv53twOfmyDWuYNoQ3y5cc\nHKOZcLHxYOhvwg3PFaGfFHMFiNmF40dTXt9K96r7sbzc44iLD+VphbMPJEjkMuf8\nPGEFOBzy8pm3wJZw2v32RNW2VESwMYyqDzwHXGSq1a73cS7hEnc79gXlELsK04L9\nQQIDAQAB\n-----END PUBLIC KEY-----\n" - } + @public_key "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw0P/Tq4gb4G/QVuMGbJo\nC/AfMNcv+m7NfrlOwkVzcU47jgESuYI4UtJayissCdBycHUnfVUd9qol+eznSODz\nCJhfJloqEIC+aSnuEPGA0POtWad6DU0E6/Ho5zQn5WAWUwbRQqowbrsm/GHo2+3v\neR5jGenwA6sYhINg/c3QQbksyV0uJ20Umyx88w8+TJuv53twOfmyDWuYNoQ3y5cc\nHKOZcLHxYOhvwg3PFaGfFHMFiNmF40dTXt9K96r7sbzc44iLD+VphbMPJEjkMuf8\nPGEFOBzy8pm3wJZw2v32RNW2VESwMYyqDzwHXGSq1a73cS7hEnc79gXlELsK04L9\nQQIDAQAB\n-----END PUBLIC KEY-----\n" @rsa_public_key { :RSAPublicKey, @@ -42,7 +37,7 @@ defp make_fake_conn(key_id), test "it returns key" do expected_result = {:ok, @rsa_public_key} - user = insert(:user, source_data: %{"publicKey" => @public_key}) + user = insert(:user, public_key: @public_key) assert Signature.fetch_public_key(make_fake_conn(user.ap_id)) == expected_result end @@ -53,8 +48,8 @@ test "it returns error when not found user" do end) =~ "[error] Could not decode user" end - test "it returns error if public key is empty" do - user = insert(:user, source_data: %{"publicKey" => %{}}) + test "it returns error if public key is nil" do + user = insert(:user, public_key: nil) assert Signature.fetch_public_key(make_fake_conn(user.ap_id)) == {:error, :error} end From b6bed1a284ce07359642e0a884d2476ca387439d Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 9 Apr 2020 13:01:35 +0200 Subject: [PATCH 54/84] Types.URI: New --- lib/pleroma/user.ex | 3 ++- .../object_validators/note_validator.ex | 1 + .../object_validators/types/object_id.ex | 12 +++-------- .../object_validators/types/uri.ex | 20 +++++++++++++++++++ 4 files changed, 26 insertions(+), 10 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/object_validators/types/uri.ex diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 0adea42ec..027386a22 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -28,6 +28,7 @@ defmodule Pleroma.User do alias Pleroma.UserRelationship alias Pleroma.Web alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils, as: CommonUtils @@ -113,7 +114,7 @@ defmodule Pleroma.User do field(:show_role, :boolean, default: true) field(:settings, :map, default: nil) field(:magic_key, :string, default: nil) - field(:uri, :string, default: nil) + field(:uri, Types.Uri, default: nil) field(:hide_followers_count, :boolean, default: false) field(:hide_follows_count, :boolean, default: false) field(:hide_followers, :boolean, default: false) diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex index c95b622e4..462a5620a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -35,6 +35,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) field(:inRepyTo, :string) + field(:uri, Types.Uri) field(:likes, {:array, :string}, default: []) field(:announcements, {:array, :string}, default: []) diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex index f6e749b33..f71f76370 100644 --- a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex +++ b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex @@ -15,15 +15,9 @@ def cast(object) when is_binary(object) do def cast(%{"id" => object}), do: cast(object) - def cast(_) do - :error - end + def cast(_), do: :error - def dump(data) do - {:ok, data} - end + def dump(data), do: {:ok, data} - def load(data) do - {:ok, data} - end + def load(data), do: {:ok, data} end diff --git a/lib/pleroma/web/activity_pub/object_validators/types/uri.ex b/lib/pleroma/web/activity_pub/object_validators/types/uri.ex new file mode 100644 index 000000000..24845bcc0 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/types/uri.ex @@ -0,0 +1,20 @@ +defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.Uri do + use Ecto.Type + + def type, do: :string + + def cast(uri) when is_binary(uri) do + case URI.parse(uri) do + %URI{host: nil} -> :error + %URI{host: ""} -> :error + %URI{scheme: scheme} when scheme in ["https", "http"] -> {:ok, uri} + _ -> :error + end + end + + def cast(_), do: :error + + def dump(data), do: {:ok, data} + + def load(data), do: {:ok, data} +end From 369c03834c5f2638080ff515055723e6c1c716bf Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 1 Apr 2020 07:15:38 +0200 Subject: [PATCH 55/84] formatter: Use user.uri instead of user.source_data.uri --- lib/pleroma/formatter.ex | 7 ++----- test/formatter_test.exs | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index c44e7fc8b..02a93a8dc 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -31,7 +31,7 @@ def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do def mention_handler("@" <> nickname, buffer, opts, acc) do case User.get_cached_by_nickname(nickname) do %User{id: id} = user -> - ap_id = get_ap_id(user) + user_url = user.uri || user.ap_id nickname_text = get_nickname_text(nickname, opts) link = @@ -42,7 +42,7 @@ def mention_handler("@" <> nickname, buffer, opts, acc) do ["@", Phoenix.HTML.Tag.content_tag(:span, nickname_text)], "data-user": id, class: "u-url mention", - href: ap_id, + href: user_url, rel: "ugc" ), class: "h-card" @@ -146,9 +146,6 @@ def truncate(text, max_length \\ 200, omission \\ "...") do end end - defp get_ap_id(%User{source_data: %{"url" => url}}) when is_binary(url), do: url - defp get_ap_id(%User{ap_id: ap_id}), do: ap_id - defp get_nickname_text(nickname, %{mentions_format: :full}), do: User.full_nickname(nickname) defp get_nickname_text(nickname, _), do: User.local_nickname(nickname) end diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 93fd8eab7..bef5a2c28 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -140,7 +140,7 @@ test "gives a replacement for user links, using local nicknames in user links te archaeme = insert(:user, nickname: "archa_eme_", - source_data: %{"url" => "https://archeme/@archa_eme_"} + uri: "https://archeme/@archa_eme_" ) archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"}) From 62656ab259cec1a8585abecf45096b283fa4c60a Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 1 Apr 2020 07:47:07 +0200 Subject: [PATCH 56/84] User: Move inbox & shared_inbox to own fields --- lib/pleroma/user.ex | 8 +++ lib/pleroma/web/activity_pub/activity_pub.ex | 19 ++++++- lib/pleroma/web/activity_pub/publisher.ex | 13 +++-- .../20200401072456_users_add_inboxes.exs | 20 +++++++ test/web/activity_pub/publisher_test.exs | 52 ++++++------------- test/web/federator_test.exs | 4 +- 6 files changed, 70 insertions(+), 46 deletions(-) create mode 100644 priv/repo/migrations/20200401072456_users_add_inboxes.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 027386a22..7d8f3a76b 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -134,6 +134,8 @@ defmodule Pleroma.User do field(:skip_thread_containment, :boolean, default: false) field(:actor_type, :string, default: "Person") field(:also_known_as, {:array, :string}, default: []) + field(:inbox, :string) + field(:shared_inbox, :string) embeds_one( :notification_settings, @@ -367,6 +369,8 @@ def remote_user_creation(params) do :bio, :name, :ap_id, + :inbox, + :shared_inbox, :nickname, :public_key, :avatar, @@ -411,6 +415,8 @@ def update_changeset(struct, params \\ %{}) do :name, :avatar, :public_key, + :inbox, + :shared_inbox, :locked, :no_rich_text, :default_scope, @@ -508,6 +514,8 @@ def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do :follower_address, :following_address, :public_key, + :inbox, + :shared_inbox, :avatar, :last_refreshed_at, :ap_enabled, diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 0e4a9d842..f0bbecc9b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1432,7 +1432,20 @@ defp object_to_user_data(data) do discoverable = data["discoverable"] || false invisible = data["invisible"] || false actor_type = data["type"] || "Person" - public_key = data["publicKey"]["publicKeyPem"] + + public_key = + if is_map(data["publicKey"]) && is_binary(data["publicKey"]["publicKeyPem"]) do + data["publicKey"]["publicKeyPem"] + else + nil + end + + shared_inbox = + if is_map(data["endpoints"]) && is_binary(data["endpoints"]["sharedInbox"]) do + data["endpoints"]["sharedInbox"] + else + nil + end user_data = %{ ap_id: data["id"], @@ -1451,7 +1464,9 @@ defp object_to_user_data(data) do bio: data["summary"], actor_type: actor_type, also_known_as: Map.get(data, "alsoKnownAs", []), - public_key: public_key + public_key: public_key, + inbox: data["inbox"], + shared_inbox: shared_inbox } # nickname can be nil because of virtual actors diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 6c558e7f0..b70cbd043 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -141,8 +141,8 @@ defp get_cc_ap_ids(ap_id, recipients) do |> Enum.map(& &1.ap_id) end - defp maybe_use_sharedinbox(%User{source_data: data}), - do: (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"] + defp maybe_use_sharedinbox(%User{shared_inbox: nil, inbox: inbox}), do: inbox + defp maybe_use_sharedinbox(%User{shared_inbox: shared_inbox}), do: shared_inbox @doc """ Determine a user inbox to use based on heuristics. These heuristics @@ -157,7 +157,7 @@ defp maybe_use_sharedinbox(%User{source_data: data}), """ def determine_inbox( %Activity{data: activity_data}, - %User{source_data: data} = user + %User{inbox: inbox} = user ) do to = activity_data["to"] || [] cc = activity_data["cc"] || [] @@ -174,7 +174,7 @@ def determine_inbox( maybe_use_sharedinbox(user) true -> - data["inbox"] + inbox end end @@ -192,14 +192,13 @@ def publish(%User{} = actor, %{data: %{"bcc" => bcc}} = activity) inboxes = recipients |> Enum.filter(&User.ap_enabled?/1) - |> Enum.map(fn %{source_data: data} -> data["inbox"] end) + |> Enum.map(fn actor -> actor.inbox end) |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) |> Instances.filter_reachable() Repo.checkout(fn -> Enum.each(inboxes, fn {inbox, unreachable_since} -> - %User{ap_id: ap_id} = - Enum.find(recipients, fn %{source_data: data} -> data["inbox"] == inbox end) + %User{ap_id: ap_id} = Enum.find(recipients, fn actor -> actor.inbox == inbox end) # Get all the recipients on the same host and add them to cc. Otherwise, a remote # instance would only accept a first message for the first recipient and ignore the rest. diff --git a/priv/repo/migrations/20200401072456_users_add_inboxes.exs b/priv/repo/migrations/20200401072456_users_add_inboxes.exs new file mode 100644 index 000000000..0947f0ab2 --- /dev/null +++ b/priv/repo/migrations/20200401072456_users_add_inboxes.exs @@ -0,0 +1,20 @@ +defmodule Pleroma.Repo.Migrations.UsersAddInboxes do + use Ecto.Migration + + def up do + alter table(:users) do + add_if_not_exists(:inbox, :text) + add_if_not_exists(:shared_inbox, :text) + end + + execute("UPDATE users SET inbox = source_data->>'inbox'") + execute("UPDATE users SET shared_inbox = source_data->'endpoints'->>'sharedInbox'") + end + + def down do + alter table(:users) do + remove_if_exists(:inbox, :text) + remove_if_exists(:shared_inbox, :text) + end + end +end diff --git a/test/web/activity_pub/publisher_test.exs b/test/web/activity_pub/publisher_test.exs index 801da03c1..c2bc38d52 100644 --- a/test/web/activity_pub/publisher_test.exs +++ b/test/web/activity_pub/publisher_test.exs @@ -48,10 +48,7 @@ test "it returns links" do describe "determine_inbox/2" do test "it returns sharedInbox for messages involving as:Public in to" do - user = - insert(:user, %{ - source_data: %{"endpoints" => %{"sharedInbox" => "http://example.com/inbox"}} - }) + user = insert(:user, %{shared_inbox: "http://example.com/inbox"}) activity = %Activity{ data: %{"to" => [@as_public], "cc" => [user.follower_address]} @@ -61,10 +58,7 @@ test "it returns sharedInbox for messages involving as:Public in to" do end test "it returns sharedInbox for messages involving as:Public in cc" do - user = - insert(:user, %{ - source_data: %{"endpoints" => %{"sharedInbox" => "http://example.com/inbox"}} - }) + user = insert(:user, %{shared_inbox: "http://example.com/inbox"}) activity = %Activity{ data: %{"cc" => [@as_public], "to" => [user.follower_address]} @@ -74,11 +68,7 @@ test "it returns sharedInbox for messages involving as:Public in cc" do end test "it returns sharedInbox for messages involving multiple recipients in to" do - user = - insert(:user, %{ - source_data: %{"endpoints" => %{"sharedInbox" => "http://example.com/inbox"}} - }) - + user = insert(:user, %{shared_inbox: "http://example.com/inbox"}) user_two = insert(:user) user_three = insert(:user) @@ -90,11 +80,7 @@ test "it returns sharedInbox for messages involving multiple recipients in to" d end test "it returns sharedInbox for messages involving multiple recipients in cc" do - user = - insert(:user, %{ - source_data: %{"endpoints" => %{"sharedInbox" => "http://example.com/inbox"}} - }) - + user = insert(:user, %{shared_inbox: "http://example.com/inbox"}) user_two = insert(:user) user_three = insert(:user) @@ -107,12 +93,10 @@ test "it returns sharedInbox for messages involving multiple recipients in cc" d test "it returns sharedInbox for messages involving multiple recipients in total" do user = - insert(:user, - source_data: %{ - "inbox" => "http://example.com/personal-inbox", - "endpoints" => %{"sharedInbox" => "http://example.com/inbox"} - } - ) + insert(:user, %{ + shared_inbox: "http://example.com/inbox", + inbox: "http://example.com/personal-inbox" + }) user_two = insert(:user) @@ -125,12 +109,10 @@ test "it returns sharedInbox for messages involving multiple recipients in total test "it returns inbox for messages involving single recipients in total" do user = - insert(:user, - source_data: %{ - "inbox" => "http://example.com/personal-inbox", - "endpoints" => %{"sharedInbox" => "http://example.com/inbox"} - } - ) + insert(:user, %{ + shared_inbox: "http://example.com/inbox", + inbox: "http://example.com/personal-inbox" + }) activity = %Activity{ data: %{"to" => [user.ap_id], "cc" => []} @@ -258,11 +240,11 @@ test "it returns inbox for messages involving single recipients in total" do [:passthrough], [] do follower = - insert(:user, + insert(:user, %{ local: false, - source_data: %{"inbox" => "https://domain.com/users/nick1/inbox"}, + inbox: "https://domain.com/users/nick1/inbox", ap_enabled: true - ) + }) actor = insert(:user, follower_address: follower.ap_id) user = insert(:user) @@ -295,14 +277,14 @@ test "it returns inbox for messages involving single recipients in total" do fetcher = insert(:user, local: false, - source_data: %{"inbox" => "https://domain.com/users/nick1/inbox"}, + inbox: "https://domain.com/users/nick1/inbox", ap_enabled: true ) another_fetcher = insert(:user, local: false, - source_data: %{"inbox" => "https://domain2.com/users/nick1/inbox"}, + inbox: "https://domain2.com/users/nick1/inbox", ap_enabled: true ) diff --git a/test/web/federator_test.exs b/test/web/federator_test.exs index da844c24c..59e53bb03 100644 --- a/test/web/federator_test.exs +++ b/test/web/federator_test.exs @@ -78,7 +78,7 @@ test "it federates only to reachable instances via AP" do local: false, nickname: "nick1@domain.com", ap_id: "https://domain.com/users/nick1", - source_data: %{"inbox" => inbox1}, + inbox: inbox1, ap_enabled: true }) @@ -86,7 +86,7 @@ test "it federates only to reachable instances via AP" do local: false, nickname: "nick2@domain2.com", ap_id: "https://domain2.com/users/nick2", - source_data: %{"inbox" => inbox2}, + inbox: inbox2, ap_enabled: true }) From 9172d719ccbf84d55236007d329fc880db69fe42 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 3 Apr 2020 13:03:32 +0200 Subject: [PATCH 57/84] profile emojis in User.emoji instead of source_data --- lib/pleroma/emoji/formatter.ex | 14 ++------ lib/pleroma/user.ex | 27 +++++++++----- lib/pleroma/web/activity_pub/activity_pub.ex | 9 +++++ .../web/activity_pub/transmogrifier.ex | 2 +- .../web/activity_pub/views/user_view.ex | 2 +- lib/pleroma/web/common_api/common_api.ex | 20 ----------- lib/pleroma/web/common_api/utils.ex | 17 +-------- .../controllers/account_controller.ex | 6 +--- .../web/mastodon_api/views/account_view.ex | 10 +++--- .../controllers/account_controller.ex | 15 +++----- lib/pleroma/web/static_fe/static_fe_view.ex | 9 ----- .../static_fe/static_fe/_user_card.html.eex | 2 +- .../static_fe/static_fe/profile.html.eex | 2 +- .../20200406100225_users_add_emoji.exs | 35 +++++++++++++++++++ test/emoji/formatter_test.exs | 24 ++++--------- test/web/activity_pub/transmogrifier_test.exs | 14 ++++++++ .../web/activity_pub/views/user_view_test.exs | 2 +- test/web/common_api/common_api_test.exs | 12 ------- test/web/common_api/common_api_utils_test.exs | 23 ------------ .../mastodon_api/views/account_view_test.exs | 16 ++------- 20 files changed, 103 insertions(+), 158 deletions(-) create mode 100644 priv/repo/migrations/20200406100225_users_add_emoji.exs diff --git a/lib/pleroma/emoji/formatter.ex b/lib/pleroma/emoji/formatter.ex index 59ff2cac3..dc45b8a38 100644 --- a/lib/pleroma/emoji/formatter.ex +++ b/lib/pleroma/emoji/formatter.ex @@ -38,22 +38,14 @@ def demojify(text) do def demojify(text, nil), do: text - @doc "Outputs a list of the emoji-shortcodes in a text" - def get_emoji(text) when is_binary(text) do - Enum.filter(Emoji.get_all(), fn {emoji, %Emoji{}} -> - String.contains?(text, ":#{emoji}:") - end) - end - - def get_emoji(_), do: [] - @doc "Outputs a list of the emoji-Maps in a text" def get_emoji_map(text) when is_binary(text) do - get_emoji(text) + Emoji.get_all() + |> Enum.filter(fn {emoji, %Emoji{}} -> String.contains?(text, ":#{emoji}:") end) |> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc -> Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") end) end - def get_emoji_map(_), do: [] + def get_emoji_map(_), do: %{} end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 7d8f3a76b..cd3551e11 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -15,6 +15,7 @@ defmodule Pleroma.User do alias Pleroma.Config alias Pleroma.Conversation.Participation alias Pleroma.Delivery + alias Pleroma.Emoji alias Pleroma.FollowingRelationship alias Pleroma.Formatter alias Pleroma.HTML @@ -124,7 +125,7 @@ defmodule Pleroma.User do field(:pinned_activities, {:array, :string}, default: []) field(:email_notifications, :map, default: %{"digest" => false}) field(:mascot, :map, default: nil) - field(:emoji, {:array, :map}, default: []) + field(:emoji, :map, default: %{}) field(:pleroma_settings_store, :map, default: %{}) field(:fields, {:array, :map}, default: []) field(:raw_fields, {:array, :map}, default: []) @@ -368,6 +369,7 @@ def remote_user_creation(params) do [ :bio, :name, + :emoji, :ap_id, :inbox, :shared_inbox, @@ -413,6 +415,7 @@ def update_changeset(struct, params \\ %{}) do [ :bio, :name, + :emoji, :avatar, :public_key, :inbox, @@ -443,6 +446,7 @@ def update_changeset(struct, params \\ %{}) do |> validate_length(:bio, max: bio_limit) |> validate_length(:name, min: 1, max: name_limit) |> put_fields() + |> put_emoji() |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)}) |> put_change_if_present(:avatar, &put_upload(&1, :avatar)) |> put_change_if_present(:banner, &put_upload(&1, :banner)) @@ -478,6 +482,18 @@ defp parse_fields(value) do |> elem(0) end + defp put_emoji(changeset) do + bio = get_change(changeset, :bio) + name = get_change(changeset, :name) + + if bio || name do + emoji = Map.merge(Emoji.Formatter.get_emoji_map(bio), Emoji.Formatter.get_emoji_map(name)) + put_change(changeset, :emoji, emoji) + else + changeset + end + end + defp put_change_if_present(changeset, map_field, value_function) do if value = get_change(changeset, map_field) do with {:ok, new_value} <- value_function.(value) do @@ -511,6 +527,7 @@ def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do [ :bio, :name, + :emoji, :follower_address, :following_address, :public_key, @@ -618,7 +635,7 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do struct |> confirmation_changeset(need_confirmation: need_confirmation?) - |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation]) + |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation, :emoji]) |> validate_required([:name, :nickname, :password, :password_confirmation]) |> validate_confirmation(:password) |> unique_constraint(:email) @@ -1969,12 +1986,6 @@ def update_background(user, background) do |> update_and_set_cache() end - def update_source_data(user, source_data) do - user - |> cast(%{source_data: source_data}, [:source_data]) - |> update_and_set_cache() - end - def roles(%{is_moderator: is_moderator, is_admin: is_admin}) do %{ admin: is_admin, diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f0bbecc9b..63502b484 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1427,6 +1427,14 @@ defp object_to_user_data(data) do |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) + emojis = + data + |> Map.get("tag", []) + |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end) + |> Enum.reduce(%{}, fn %{"icon" => %{"url" => url}, "name" => name}, acc -> + Map.put(acc, String.trim(name, ":"), url) + end) + locked = data["manuallyApprovesFollowers"] || false data = Transmogrifier.maybe_fix_user_object(data) discoverable = data["discoverable"] || false @@ -1454,6 +1462,7 @@ defp object_to_user_data(data) do source_data: data, banner: banner, fields: fields, + emoji: emojis, locked: locked, discoverable: discoverable, invisible: invisible, diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 0a8ad62ad..3d4070fd5 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -1129,7 +1129,7 @@ defp build_mention_tag(%{ap_id: ap_id, nickname: nickname} = _) do def take_emoji_tags(%User{emoji: emoji}) do emoji - |> Enum.flat_map(&Map.to_list/1) + |> Map.to_list() |> Enum.map(&build_emoji_tag/1) end diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index bc21ac6c7..d3d79dd5e 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -103,7 +103,7 @@ def render("user.json", %{user: user}) do }, "endpoints" => endpoints, "attachment" => fields, - "tag" => (user.source_data["tag"] || []) ++ emoji_tags, + "tag" => emoji_tags, "discoverable" => user.discoverable } |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 636cf3301..952a8d8cb 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -332,26 +332,6 @@ defp maybe_create_activity_expiration({:ok, activity}, %NaiveDateTime{} = expire defp maybe_create_activity_expiration(result, _), do: result - # Updates the emojis for a user based on their profile - def update(user) do - emoji = emoji_from_profile(user) - source_data = Map.put(user.source_data, "tag", emoji) - - user = - case User.update_source_data(user, source_data) do - {:ok, user} -> user - _ -> user - end - - ActivityPub.update(%{ - local: true, - to: [Pleroma.Constants.as_public(), user.follower_address], - cc: [], - actor: user.ap_id, - object: Pleroma.Web.ActivityPub.UserView.render("user.json", %{user: user}) - }) - end - def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do with %Activity{ actor: ^user_ap_id, diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 635e7cd38..7eec5aa09 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -10,7 +10,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Conversation.Participation - alias Pleroma.Emoji alias Pleroma.Formatter alias Pleroma.Object alias Pleroma.Plugs.AuthenticationPlug @@ -18,7 +17,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do alias Pleroma.User alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Web.Endpoint alias Pleroma.Web.MediaProxy require Logger @@ -175,7 +173,7 @@ def make_poll_data(%{"poll" => %{"options" => options, "expires_in" => expires_i "replies" => %{"type" => "Collection", "totalItems" => 0} } - {note, Map.merge(emoji, Emoji.Formatter.get_emoji_map(option))} + {note, Map.merge(emoji, Pleroma.Emoji.Formatter.get_emoji_map(option))} end) end_time = @@ -431,19 +429,6 @@ def confirm_current_password(user, password) do end end - def emoji_from_profile(%User{bio: bio, name: name}) do - [bio, name] - |> Enum.map(&Emoji.Formatter.get_emoji/1) - |> Enum.concat() - |> Enum.map(fn {shortcode, %Emoji{file: path}} -> - %{ - "type" => "Emoji", - "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{path}"}, - "name" => ":#{shortcode}:" - } - end) - end - def maybe_notify_to_recipients( recipients, %Activity{data: %{"to" => to, "type" => _type}} = _activity diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 21bc3d5a5..3fcaa6be6 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -140,9 +140,7 @@ def verify_credentials(%{assigns: %{user: user}} = conn, _) do end @doc "PATCH /api/v1/accounts/update_credentials" - def update_credentials(%{assigns: %{user: original_user}} = conn, params) do - user = original_user - + def update_credentials(%{assigns: %{user: user}} = conn, params) do user_params = [ :no_rich_text, @@ -178,8 +176,6 @@ def update_credentials(%{assigns: %{user: original_user}} = conn, params) do changeset = User.update_changeset(user, user_params) with {:ok, user} <- User.update_and_set_cache(changeset) do - if original_user != user, do: CommonAPI.update(user) - render(conn, "show.json", user: user, for: user, with_pleroma_settings: true) else _e -> render_error(conn, :forbidden, "Invalid request") diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 99e62f580..966032b69 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -180,13 +180,11 @@ defp do_render("show.json", %{user: user} = opts) do bot = user.actor_type in ["Application", "Service"] emojis = - (user.source_data["tag"] || []) - |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end) - |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} -> + Enum.map(user.emoji, fn {shortcode, url} -> %{ - "shortcode" => String.trim(name, ":"), - "url" => MediaProxy.url(url), - "static_url" => MediaProxy.url(url), + "shortcode" => shortcode, + "url" => url, + "static_url" => url, "visible_in_picker" => false } end) diff --git a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex index dcba67d03..ed4fdfdba 100644 --- a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex @@ -13,7 +13,6 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do alias Pleroma.Plugs.RateLimiter alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.StatusView require Pleroma.Constants @@ -58,38 +57,32 @@ def confirmation_resend(conn, params) do @doc "PATCH /api/v1/pleroma/accounts/update_avatar" def update_avatar(%{assigns: %{user: user}} = conn, %{"img" => ""}) do - {:ok, user} = + {:ok, _user} = user |> Changeset.change(%{avatar: nil}) |> User.update_and_set_cache() - CommonAPI.update(user) - json(conn, %{url: nil}) end def update_avatar(%{assigns: %{user: user}} = conn, params) do {:ok, %{data: data}} = ActivityPub.upload(params, type: :avatar) - {:ok, user} = user |> Changeset.change(%{avatar: data}) |> User.update_and_set_cache() + {:ok, _user} = user |> Changeset.change(%{avatar: data}) |> User.update_and_set_cache() %{"url" => [%{"href" => href} | _]} = data - CommonAPI.update(user) - json(conn, %{url: href}) end @doc "PATCH /api/v1/pleroma/accounts/update_banner" def update_banner(%{assigns: %{user: user}} = conn, %{"banner" => ""}) do - with {:ok, user} <- User.update_banner(user, %{}) do - CommonAPI.update(user) + with {:ok, _user} <- User.update_banner(user, %{}) do json(conn, %{url: nil}) end end def update_banner(%{assigns: %{user: user}} = conn, params) do with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}, type: :banner), - {:ok, user} <- User.update_banner(user, object.data) do - CommonAPI.update(user) + {:ok, _user} <- User.update_banner(user, object.data) do %{"url" => [%{"href" => href} | _]} = object.data json(conn, %{url: href}) diff --git a/lib/pleroma/web/static_fe/static_fe_view.ex b/lib/pleroma/web/static_fe/static_fe_view.ex index 66d87620c..b3d1d1ec8 100644 --- a/lib/pleroma/web/static_fe/static_fe_view.ex +++ b/lib/pleroma/web/static_fe/static_fe_view.ex @@ -18,15 +18,6 @@ defmodule Pleroma.Web.StaticFE.StaticFEView do @media_types ["image", "audio", "video"] - def emoji_for_user(%User{} = user) do - user.source_data - |> Map.get("tag", []) - |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end) - |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} -> - {String.trim(name, ":"), url} - end) - end - def fetch_media_type(%{"mediaType" => mediaType}) do Utils.fetch_media_type(@media_types, mediaType) end diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex index 2a7582d45..56f3a1524 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex @@ -4,7 +4,7 @@ - <%= raw (@user.name |> Formatter.emojify(emoji_for_user(@user))) %> + <%= raw Formatter.emojify(@user.name, @user.emoji) %> <%= @user.nickname %> diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex index e7d2aecad..3191bf450 100644 --- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex +++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex @@ -7,7 +7,7 @@ - <%= raw Formatter.emojify(@user.name, emoji_for_user(@user)) %> | + <%= raw Formatter.emojify(@user.name, @user.emoji) %> | <%= link "@#{@user.nickname}@#{Endpoint.host()}", to: (@user.uri || @user.ap_id) %>

<%= raw @user.bio %>

diff --git a/priv/repo/migrations/20200406100225_users_add_emoji.exs b/priv/repo/migrations/20200406100225_users_add_emoji.exs new file mode 100644 index 000000000..d0254c170 --- /dev/null +++ b/priv/repo/migrations/20200406100225_users_add_emoji.exs @@ -0,0 +1,35 @@ +defmodule Pleroma.Repo.Migrations.UsersPopulateEmoji do + use Ecto.Migration + + import Ecto.Query + + alias Pleroma.User + alias Pleroma.Repo + + def up do + execute("ALTER TABLE users ALTER COLUMN emoji SET DEFAULT '{}'::jsonb") + execute("UPDATE users SET emoji = DEFAULT WHERE emoji = '[]'::jsonb") + + from(u in User) + |> select([u], struct(u, [:id, :ap_id, :source_data])) + |> Repo.stream() + |> Enum.each(fn user -> + emoji = + user.source_data + |> Map.get("tag", []) + |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end) + |> Enum.reduce(%{}, fn %{"icon" => %{"url" => url}, "name" => name}, acc -> + Map.put(acc, String.trim(name, ":"), url) + end) + + user + |> Ecto.Changeset.cast(%{emoji: emoji}, [:emoji]) + |> Repo.update() + end) + end + + def down do + execute("ALTER TABLE users ALTER COLUMN emoji SET DEFAULT '[]'::jsonb") + execute("UPDATE users SET emoji = DEFAULT WHERE emoji = '{}'::jsonb") + end +end diff --git a/test/emoji/formatter_test.exs b/test/emoji/formatter_test.exs index 3bfee9420..12af6cd8b 100644 --- a/test/emoji/formatter_test.exs +++ b/test/emoji/formatter_test.exs @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Emoji.FormatterTest do - alias Pleroma.Emoji alias Pleroma.Emoji.Formatter use Pleroma.DataCase @@ -32,30 +31,19 @@ test "it does not add XSS emoji" do end end - describe "get_emoji" do + describe "get_emoji_map" do test "it returns the emoji used in the text" do - text = "I love :firefox:" - - assert Formatter.get_emoji(text) == [ - {"firefox", - %Emoji{ - code: "firefox", - file: "/emoji/Firefox.gif", - tags: ["Gif", "Fun"], - safe_code: "firefox", - safe_file: "/emoji/Firefox.gif" - }} - ] + assert Formatter.get_emoji_map("I love :firefox:") == %{ + "firefox" => "http://localhost:4001/emoji/Firefox.gif" + } end test "it returns a nice empty result when no emojis are present" do - text = "I love moominamma" - assert Formatter.get_emoji(text) == [] + assert Formatter.get_emoji_map("I love moominamma") == %{} end test "it doesn't die when text is absent" do - text = nil - assert Formatter.get_emoji(text) == [] + assert Formatter.get_emoji_map(nil) == %{} end end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 6dfd823f7..d7f11d1d7 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -2182,4 +2182,18 @@ test "sets `replies` collection with a limited number of self-replies" do Transmogrifier.set_replies(object.data)["replies"] end end + + test "take_emoji_tags/1" do + user = insert(:user, %{emoji: %{"firefox" => "https://example.org/firefox.png"}}) + + assert Transmogrifier.take_emoji_tags(user) == [ + %{ + "icon" => %{"type" => "Image", "url" => "https://example.org/firefox.png"}, + "id" => "https://example.org/firefox.png", + "name" => ":firefox:", + "type" => "Emoji", + "updated" => "1970-01-01T00:00:00Z" + } + ] + end end diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index ecb2dc386..20578161b 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -38,7 +38,7 @@ test "Renders profile fields" do end test "Renders with emoji tags" do - user = insert(:user, emoji: [%{"bib" => "/test"}]) + user = insert(:user, emoji: %{"bib" => "/test"}) assert %{ "tag" => [ diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs index f46ad0272..5e78c5758 100644 --- a/test/web/common_api/common_api_test.exs +++ b/test/web/common_api/common_api_test.exs @@ -97,18 +97,6 @@ test "it adds emoji in the object" do assert Object.normalize(activity).data["emoji"]["firefox"] end - test "it adds emoji when updating profiles" do - user = insert(:user, %{name: ":firefox:"}) - - {:ok, activity} = CommonAPI.update(user) - user = User.get_cached_by_ap_id(user.ap_id) - [firefox] = user.source_data["tag"] - - assert firefox["name"] == ":firefox:" - - assert Pleroma.Constants.as_public() in activity.recipients - end - describe "posting" do test "it supports explicit addressing" do user = insert(:user) diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs index 98cf02d49..b21445fe9 100644 --- a/test/web/common_api/common_api_utils_test.exs +++ b/test/web/common_api/common_api_utils_test.exs @@ -7,7 +7,6 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do alias Pleroma.Object alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils - alias Pleroma.Web.Endpoint use Pleroma.DataCase import ExUnit.CaptureLog @@ -42,28 +41,6 @@ test "correct password given" do end end - test "parses emoji from name and bio" do - {:ok, user} = UserBuilder.insert(%{name: ":blank:", bio: ":firefox:"}) - - expected = [ - %{ - "type" => "Emoji", - "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}/emoji/Firefox.gif"}, - "name" => ":firefox:" - }, - %{ - "type" => "Emoji", - "icon" => %{ - "type" => "Image", - "url" => "#{Endpoint.url()}/emoji/blank.png" - }, - "name" => ":blank:" - } - ] - - assert expected == Utils.emoji_from_profile(user) - end - describe "format_input/3" do test "works for bare text/plain" do text = "hello world!" diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index 4435f69ff..85fa4f6a2 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -19,16 +19,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do end test "Represent a user account" do - source_data = %{ - "tag" => [ - %{ - "type" => "Emoji", - "icon" => %{"url" => "/file.png"}, - "name" => ":karjalanpiirakka:" - } - ] - } - background_image = %{ "url" => [%{"href" => "https://example.com/images/asuka_hospital.png"}] } @@ -37,13 +27,13 @@ test "Represent a user account" do insert(:user, %{ follower_count: 3, note_count: 5, - source_data: source_data, background: background_image, nickname: "shp@shitposter.club", name: ":karjalanpiirakka: shp", bio: "valid html. a
b
c
d
f", - inserted_at: ~N[2017-08-15 15:47:06.597036] + inserted_at: ~N[2017-08-15 15:47:06.597036], + emoji: %{"karjalanpiirakka" => "/file.png"} }) expected = %{ @@ -117,7 +107,6 @@ test "Represent a Service(bot) account" do insert(:user, %{ follower_count: 3, note_count: 5, - source_data: %{}, actor_type: "Service", nickname: "shp@shitposter.club", inserted_at: ~N[2017-08-15 15:47:06.597036] @@ -311,7 +300,6 @@ test "represent an embedded relationship" do insert(:user, %{ follower_count: 0, note_count: 5, - source_data: %{}, actor_type: "Service", nickname: "shp@shitposter.club", inserted_at: ~N[2017-08-15 15:47:06.597036] From 3420dec494203b46d37ddc17f7e1235dc908a5b3 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 6 Apr 2020 10:44:48 +0200 Subject: [PATCH 58/84] Remove User.fields/1 --- lib/pleroma/user.ex | 19 +------------------ .../web/activity_pub/views/user_view.ex | 5 +---- test/web/activity_pub/transmogrifier_test.exs | 8 ++++---- 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index cd3551e11..79e9b2c86 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1993,21 +1993,6 @@ def roles(%{is_moderator: is_moderator, is_admin: is_admin}) do } end - # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``. - # For example: [{"name": "Pronoun", "value": "she/her"}, …] - def fields(%{fields: nil, source_data: %{"attachment" => attachment}}) do - limit = Pleroma.Config.get([:instance, :max_remote_account_fields], 0) - - attachment - |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end) - |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end) - |> Enum.take(limit) - end - - def fields(%{fields: nil}), do: [] - - def fields(%{fields: fields}), do: fields - def validate_fields(changeset, remote? \\ false) do limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields limit = Pleroma.Config.get([:instance, limit_name], 0) @@ -2195,9 +2180,7 @@ def sanitize_html(%User{} = user) do # - display name def sanitize_html(%User{} = user, filter) do fields = - user - |> User.fields() - |> Enum.map(fn %{"name" => name, "value" => value} -> + Enum.map(user.fields, fn %{"name" => name, "value" => value} -> %{ "name" => name, "value" => HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index d3d79dd5e..34590b16d 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -79,10 +79,7 @@ def render("user.json", %{user: user}) do emoji_tags = Transmogrifier.take_emoji_tags(user) - fields = - user - |> User.fields() - |> Enum.map(&Map.put(&1, "type", "PropertyValue")) + fields = Enum.map(user.fields, &Map.put(&1, "type", "PropertyValue")) %{ "id" => user.ap_id, diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index d7f11d1d7..8ddc75669 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -746,7 +746,7 @@ test "it works with custom profile fields" do user = User.get_cached_by_ap_id(activity.actor) - assert User.fields(user) == [ + assert user.fields == [ %{"name" => "foo", "value" => "bar"}, %{"name" => "foo1", "value" => "bar1"} ] @@ -767,7 +767,7 @@ test "it works with custom profile fields" do user = User.get_cached_by_ap_id(user.ap_id) - assert User.fields(user) == [ + assert user.fields == [ %{"name" => "foo", "value" => "updated"}, %{"name" => "foo1", "value" => "updated"} ] @@ -785,7 +785,7 @@ test "it works with custom profile fields" do user = User.get_cached_by_ap_id(user.ap_id) - assert User.fields(user) == [ + assert user.fields == [ %{"name" => "foo", "value" => "updated"}, %{"name" => "foo1", "value" => "updated"} ] @@ -796,7 +796,7 @@ test "it works with custom profile fields" do user = User.get_cached_by_ap_id(user.ap_id) - assert User.fields(user) == [] + assert user.fields == [] end test "it works for incoming update activities which lock the account" do From e89078ac2a27bb0a833c982dbb5eef63ddea3cc0 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 6 Apr 2020 10:59:35 +0200 Subject: [PATCH 59/84] User: remove source_data --- lib/pleroma/user.ex | 3 --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 - .../20200406105422_users_remove_source_data.exs | 15 +++++++++++++++ test/user_test.exs | 2 +- test/web/activity_pub/activity_pub_test.exs | 1 - 5 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 priv/repo/migrations/20200406105422_users_remove_source_data.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 79e9b2c86..d05dfb480 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -97,7 +97,6 @@ defmodule Pleroma.User do field(:last_digest_emailed_at, :naive_datetime) field(:banner, :map, default: %{}) field(:background, :map, default: %{}) - field(:source_data, :map, default: %{}) field(:note_count, :integer, default: 0) field(:follower_count, :integer, default: 0) field(:following_count, :integer, default: 0) @@ -377,7 +376,6 @@ def remote_user_creation(params) do :public_key, :avatar, :ap_enabled, - :source_data, :banner, :locked, :magic_key, @@ -536,7 +534,6 @@ def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do :avatar, :last_refreshed_at, :ap_enabled, - :source_data, :banner, :locked, :magic_key, diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 63502b484..9b832f4cb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1459,7 +1459,6 @@ defp object_to_user_data(data) do ap_id: data["id"], uri: get_actor_url(data["url"]), ap_enabled: true, - source_data: data, banner: banner, fields: fields, emoji: emojis, diff --git a/priv/repo/migrations/20200406105422_users_remove_source_data.exs b/priv/repo/migrations/20200406105422_users_remove_source_data.exs new file mode 100644 index 000000000..9812d480f --- /dev/null +++ b/priv/repo/migrations/20200406105422_users_remove_source_data.exs @@ -0,0 +1,15 @@ +defmodule Pleroma.Repo.Migrations.UsersRemoveSourceData do + use Ecto.Migration + + def up do + alter table(:users) do + remove_if_exists(:source_data, :map) + end + end + + def down do + alter table(:users) do + add_if_not_exists(:source_data, :map, default: %{}) + end + end +end diff --git a/test/user_test.exs b/test/user_test.exs index d39787f35..d35005353 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -581,7 +581,7 @@ test "updates an existing user, if stale" do {:ok, user} = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/admin") - assert user.source_data["endpoints"] + assert user.inbox refute user.last_refreshed_at == orig_user.last_refreshed_at end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 17e7b97de..6410df49b 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -180,7 +180,6 @@ test "it returns a user" do {:ok, user} = ActivityPub.make_user_from_ap_id(user_id) assert user.ap_id == user_id assert user.nickname == "admin@mastodon.example.org" - assert user.source_data assert user.ap_enabled assert user.follower_address == "http://mastodon.example.org/users/admin/followers" end From 6ff8812ea3403a2f4a31206a96a58fad93fff51f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 10 Apr 2020 11:37:02 -0500 Subject: [PATCH 60/84] Add a section for changelog entries that pertain to the next patch release. This will make it easier to keep changelogs synced between develop and stable branches. --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd5d5f800..36897503a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Support pagination in conversations API +## [unreleased-patch] + ## [2.0.2] - 2020-04-08 ### Added - Support for Funkwhale's `Audio` activity From ad92cef844d4f4211a65fd37b08f8bd8abea8dda Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 10 Apr 2020 21:27:50 +0300 Subject: [PATCH 61/84] fix Oban migration --- .../repo/migrations/20200402063221_update_oban_jobs_table.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/priv/repo/migrations/20200402063221_update_oban_jobs_table.exs b/priv/repo/migrations/20200402063221_update_oban_jobs_table.exs index c8ee12192..e7ff04008 100644 --- a/priv/repo/migrations/20200402063221_update_oban_jobs_table.exs +++ b/priv/repo/migrations/20200402063221_update_oban_jobs_table.exs @@ -2,10 +2,10 @@ defmodule Pleroma.Repo.Migrations.UpdateObanJobsTable do use Ecto.Migration def up do - Oban.Migrations.up() + Oban.Migrations.up(version: 8) end def down do - Oban.Migrations.down(version: 1) + Oban.Migrations.down(version: 7) end end From c077ad0b3305e74f5b8d1b9bf38d4f480d76c1a6 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 11 Apr 2020 21:44:52 +0300 Subject: [PATCH 62/84] Remove User.upgrade_changeset in favor of remote_user_creation The two changesets had the same purpose, yet some changes were updated in one, but not the other (`uri`, for example). Also makes `Transmogrifier.upgrade_user_from_ap_id` be called from `ActivityPub.make_user_from_ap_id` only when the user is actually not AP enabled yet. I did not bother rewriting tests that used `User.insert_or_update` to use the changeset instead because they seemed to just test the implementation, rather than behavior. --- lib/pleroma/user.ex | 60 ++--------------- lib/pleroma/web/activity_pub/activity_pub.ex | 15 ++++- .../web/activity_pub/transmogrifier.ex | 14 ++-- test/user_test.exs | 64 ++----------------- .../web/activity_pub/views/user_view_test.exs | 2 +- 5 files changed, 31 insertions(+), 124 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 71c8c3a4e..fab405233 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -339,18 +339,20 @@ defp truncate_if_exists(params, key, max_length) do end end - def remote_user_creation(params) do + def remote_user_changeset(struct \\ %User{local: false}, params) do bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) params = params + |> Map.put(:name, blank?(params[:name]) || params[:nickname]) + |> Map.put_new(:last_refreshed_at, NaiveDateTime.utc_now()) |> truncate_if_exists(:name, name_limit) |> truncate_if_exists(:bio, bio_limit) |> truncate_fields_param() changeset = - %User{local: false} + struct |> cast( params, [ @@ -375,7 +377,8 @@ def remote_user_creation(params) do :discoverable, :invisible, :actor_type, - :also_known_as + :also_known_as, + :last_refreshed_at ] ) |> validate_required([:name, :ap_id]) @@ -488,49 +491,6 @@ defp put_upload(value, type) do end end - def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do - bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) - name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) - - params = Map.put(params, :last_refreshed_at, NaiveDateTime.utc_now()) - - params = if remote?, do: truncate_fields_param(params), else: params - - struct - |> cast( - params, - [ - :bio, - :name, - :follower_address, - :following_address, - :avatar, - :last_refreshed_at, - :ap_enabled, - :source_data, - :banner, - :locked, - :magic_key, - :follower_count, - :following_count, - :hide_follows, - :fields, - :hide_followers, - :allow_following_move, - :discoverable, - :hide_followers_count, - :hide_follows_count, - :actor_type, - :also_known_as - ] - ) - |> unique_constraint(:nickname) - |> validate_format(:nickname, local_nickname_regex()) - |> validate_length(:bio, max: bio_limit) - |> validate_length(:name, max: name_limit) - |> validate_fields(remote?) - end - def update_as_admin_changeset(struct, params) do struct |> update_changeset(params) @@ -1642,14 +1602,6 @@ def get_public_key_for_ap_id(ap_id) do defp blank?(""), do: nil defp blank?(n), do: n - def insert_or_update_user(data) do - data - |> Map.put(:name, blank?(data[:name]) || data[:nickname]) - |> remote_user_creation() - |> Repo.insert(on_conflict: {:replace_all_except, [:id]}, conflict_target: :nickname) - |> set_cache() - end - def ap_enabled?(%User{local: true}), do: true def ap_enabled?(%User{ap_enabled: ap_enabled}), do: ap_enabled def ap_enabled?(_), do: false diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 86b105b7f..2602b966b 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1551,11 +1551,22 @@ def fetch_and_prepare_user_from_ap_id(ap_id) do end def make_user_from_ap_id(ap_id) do - if _user = User.get_cached_by_ap_id(ap_id) do + user = User.get_cached_by_ap_id(ap_id) + + if user && !User.ap_enabled?(user) do Transmogrifier.upgrade_user_from_ap_id(ap_id) else with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do - User.insert_or_update_user(data) + if user do + user + |> User.remote_user_changeset(data) + |> User.update_and_set_cache() + else + data + |> User.remote_user_changeset() + |> Repo.insert() + |> User.set_cache() + end else e -> {:error, e} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index f9951cc5d..18fd56bed 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -710,7 +710,7 @@ def handle_incoming( {:ok, new_user_data} = ActivityPub.user_data_from_user_object(object) actor - |> User.upgrade_changeset(new_user_data, true) + |> User.remote_user_changeset(new_user_data) |> User.update_and_set_cache() ActivityPub.update(%{ @@ -1253,12 +1253,8 @@ def perform(:user_upgrade, user) do def upgrade_user_from_ap_id(ap_id) do with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id), {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id), - already_ap <- User.ap_enabled?(user), - {:ok, user} <- upgrade_user(user, data) do - if not already_ap do - TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id}) - end - + {:ok, user} <- update_user(user, data) do + TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id}) {:ok, user} else %User{} = user -> {:ok, user} @@ -1266,9 +1262,9 @@ def upgrade_user_from_ap_id(ap_id) do end end - defp upgrade_user(user, data) do + defp update_user(user, data) do user - |> User.upgrade_changeset(data, true) + |> User.remote_user_changeset(data) |> User.update_and_set_cache() end diff --git a/test/user_test.exs b/test/user_test.exs index d39787f35..5c24955c2 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -609,7 +609,7 @@ test "returns an ap_followers link for a user" do ) <> "/followers" end - describe "remote user creation changeset" do + describe "remote user changeset" do @valid_remote %{ bio: "hello", name: "Someone", @@ -621,28 +621,28 @@ test "returns an ap_followers link for a user" do setup do: clear_config([:instance, :user_name_length]) test "it confirms validity" do - cs = User.remote_user_creation(@valid_remote) + cs = User.remote_user_changeset(@valid_remote) assert cs.valid? end test "it sets the follower_adress" do - cs = User.remote_user_creation(@valid_remote) + cs = User.remote_user_changeset(@valid_remote) # remote users get a fake local follower address assert cs.changes.follower_address == User.ap_followers(%User{nickname: @valid_remote[:nickname]}) end test "it enforces the fqn format for nicknames" do - cs = User.remote_user_creation(%{@valid_remote | nickname: "bla"}) + cs = User.remote_user_changeset(%{@valid_remote | nickname: "bla"}) assert Ecto.Changeset.get_field(cs, :local) == false assert cs.changes.avatar refute cs.valid? end test "it has required fields" do - [:name, :ap_id] + [:ap_id] |> Enum.each(fn field -> - cs = User.remote_user_creation(Map.delete(@valid_remote, field)) + cs = User.remote_user_changeset(Map.delete(@valid_remote, field)) refute cs.valid? end) end @@ -1198,58 +1198,6 @@ test "get_public_key_for_ap_id fetches a user that's not in the db" do assert {:ok, _key} = User.get_public_key_for_ap_id("http://mastodon.example.org/users/admin") end - describe "insert or update a user from given data" do - test "with normal data" do - user = insert(:user, %{nickname: "nick@name.de"}) - data = %{ap_id: user.ap_id <> "xxx", name: user.name, nickname: user.nickname} - - assert {:ok, %User{}} = User.insert_or_update_user(data) - end - - test "with overly long fields" do - current_max_length = Pleroma.Config.get([:instance, :account_field_value_length], 255) - user = insert(:user, nickname: "nickname@supergood.domain") - - data = %{ - ap_id: user.ap_id, - name: user.name, - nickname: user.nickname, - fields: [ - %{"name" => "myfield", "value" => String.duplicate("h", current_max_length + 1)} - ] - } - - assert {:ok, %User{}} = User.insert_or_update_user(data) - end - - test "with an overly long bio" do - current_max_length = Pleroma.Config.get([:instance, :user_bio_length], 5000) - user = insert(:user, nickname: "nickname@supergood.domain") - - data = %{ - ap_id: user.ap_id, - name: user.name, - nickname: user.nickname, - bio: String.duplicate("h", current_max_length + 1) - } - - assert {:ok, %User{}} = User.insert_or_update_user(data) - end - - test "with an overly long display name" do - current_max_length = Pleroma.Config.get([:instance, :user_name_length], 100) - user = insert(:user, nickname: "nickname@supergood.domain") - - data = %{ - ap_id: user.ap_id, - name: String.duplicate("h", current_max_length + 1), - nickname: user.nickname - } - - assert {:ok, %User{}} = User.insert_or_update_user(data) - end - end - describe "per-user rich-text filtering" do test "html_filter_policy returns default policies, when rich-text is enabled" do user = insert(:user) diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index ecb2dc386..514fd97b8 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -29,7 +29,7 @@ test "Renders profile fields" do {:ok, user} = insert(:user) - |> User.upgrade_changeset(%{fields: fields}) + |> User.update_changeset(%{fields: fields}) |> User.update_and_set_cache() assert %{ From 2ba754ffe11b98305e0c0607fec7ca4d510aa67f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 12 Apr 2020 18:49:31 +0300 Subject: [PATCH 63/84] Fix mix tasks failing on OTP releases No idea why this was even added. Closes #1678 --- lib/mix/pleroma.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 4dfcc32e7..3ad6edbfb 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -5,7 +5,6 @@ defmodule Mix.Pleroma do @doc "Common functions to be reused in mix tasks" def start_pleroma do - Mix.Task.run("app.start") Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) if Pleroma.Config.get(:env) != :test do From a050f3e015a6c5c8d38d535692d4da7a6b1e9c60 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 10 Apr 2020 14:42:52 +0300 Subject: [PATCH 64/84] fix for logger configuration through admin-fe --- lib/pleroma/config/transfer_task.ex | 104 +++++++++++------- .../admin_api/admin_api_controller_test.exs | 17 ++- 2 files changed, 71 insertions(+), 50 deletions(-) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 936bc9ab1..3871e1cbb 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -54,10 +54,19 @@ def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do [:pleroma, nil, :prometheus] end + {logger, other} = + (Repo.all(ConfigDB) ++ deleted_settings) + |> Enum.map(&transform_and_merge/1) + |> Enum.split_with(fn {group, _, _, _} -> group in [:logger, :quack] end) + + logger + |> Enum.sort() + |> Enum.each(&configure/1) + started_applications = Application.started_applications() - (Repo.all(ConfigDB) ++ deleted_settings) - |> Enum.map(&merge_and_update/1) + other + |> Enum.map(&update/1) |> Enum.uniq() |> Enum.reject(&(&1 in reject_restart)) |> maybe_set_pleroma_last() @@ -81,51 +90,66 @@ defp maybe_set_pleroma_last(apps) do end end - defp group_for_restart(:logger, key, _, merged_value) do - # change logger configuration in runtime, without restart - if Keyword.keyword?(merged_value) and - key not in [:compile_time_application, :backends, :compile_time_purge_matching] do - Logger.configure_backend(key, merged_value) - else - Logger.configure([{key, merged_value}]) - end + defp transform_and_merge(%{group: group, key: key, value: value} = setting) do + group = ConfigDB.from_string(group) + key = ConfigDB.from_string(key) + value = ConfigDB.from_binary(value) - nil + default = Config.Holder.default_config(group, key) + + merged = + cond do + Ecto.get_meta(setting, :state) == :deleted -> default + can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value) + true -> value + end + + {group, key, value, merged} end - defp group_for_restart(group, _, _, _) when group != :pleroma, do: group - - defp group_for_restart(group, key, value, _) do - if pleroma_need_restart?(group, key, value), do: group + # change logger configuration in runtime, without restart + defp configure({:quack, key, _, merged}) do + Logger.configure_backend(Quack.Logger, [{key, merged}]) + :ok = update_env(:quack, key, merged) end - defp merge_and_update(setting) do + defp configure({_, :backends, _, merged}) do + # removing current backends + Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1) + + Enum.each(merged, &Logger.add_backend/1) + + :ok = update_env(:logger, :backends, merged) + end + + defp configure({group, key, _, merged}) do + merged = + if key == :console do + put_in(merged[:format], merged[:format] <> "\n") + else + merged + end + + backend = + if key == :ex_syslogger, + do: {ExSyslogger, :ex_syslogger}, + else: key + + Logger.configure_backend(backend, merged) + :ok = update_env(:logger, group, merged) + end + + defp update({group, key, value, merged}) do try do - key = ConfigDB.from_string(setting.key) - group = ConfigDB.from_string(setting.group) + :ok = update_env(group, key, merged) - default = Config.Holder.default_config(group, key) - value = ConfigDB.from_binary(setting.value) - - merged_value = - cond do - Ecto.get_meta(setting, :state) == :deleted -> default - can_be_merged?(default, value) -> ConfigDB.merge_group(group, key, default, value) - true -> value - end - - :ok = update_env(group, key, merged_value) - - group_for_restart(group, key, value, merged_value) + if group != :pleroma or pleroma_need_restart?(group, key, value), do: group rescue error -> error_msg = - "updating env causes error, group: " <> - inspect(setting.group) <> - " key: " <> - inspect(setting.key) <> - " value: " <> - inspect(ConfigDB.from_binary(setting.value)) <> " error: " <> inspect(error) + "updating env causes error, group: #{inspect(group)}, key: #{inspect(key)}, value: #{ + inspect(value) + } error: #{inspect(error)}" Logger.warn(error_msg) @@ -133,6 +157,9 @@ defp merge_and_update(setting) do end end + defp update_env(group, key, nil), do: Application.delete_env(group, key) + defp update_env(group, key, value), do: Application.put_env(group, key, value) + @spec pleroma_need_restart?(atom(), atom(), any()) :: boolean() def pleroma_need_restart?(group, key, value) do group_and_key_need_reboot?(group, key) or group_and_subkey_need_reboot?(group, key, value) @@ -150,9 +177,6 @@ defp group_and_subkey_need_reboot?(group, key, value) do end) end - defp update_env(group, key, nil), do: Application.delete_env(group, key) - defp update_env(group, key, value), do: Application.put_env(group, key, value) - defp restart(_, :pleroma, env), do: Restarter.Pleroma.restart_after_boot(env) defp restart(started_applications, app, _) do diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index f02f6ae7a..60ec895f5 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -2273,13 +2273,17 @@ test "saving full setting if value is in full_key_update list", %{conn: conn} do value: :erlang.term_to_binary([]) ) + Pleroma.Config.TransferTask.load_and_update_env([], false) + + assert Application.get_env(:logger, :backends) == [] + conn = post(conn, "/api/pleroma/admin/config", %{ configs: [ %{ group: config.group, key: config.key, - value: [":console", %{"tuple" => ["ExSyslogger", ":ex_syslogger"]}] + value: [":console"] } ] }) @@ -2290,8 +2294,7 @@ test "saving full setting if value is in full_key_update list", %{conn: conn} do "group" => ":logger", "key" => ":backends", "value" => [ - ":console", - %{"tuple" => ["ExSyslogger", ":ex_syslogger"]} + ":console" ], "db" => [":backends"] } @@ -2299,14 +2302,8 @@ test "saving full setting if value is in full_key_update list", %{conn: conn} do } assert Application.get_env(:logger, :backends) == [ - :console, - {ExSyslogger, :ex_syslogger} + :console ] - - capture_log(fn -> - require Logger - Logger.warn("Ooops...") - end) =~ "Ooops..." end test "saving full setting if value is not keyword", %{conn: conn} do From de34c4ee6b0487941bfcf02a48b45a578ae329af Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 13 Apr 2020 08:59:06 +0300 Subject: [PATCH 65/84] changelog entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36897503a..e29be28d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Support pagination in conversations API ## [unreleased-patch] +### Fixed +- Logger configuration through AdminFE ## [2.0.2] - 2020-04-08 ### Added From dc2637c18880160286f50505b1140a58fdfdf7d1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 13 Apr 2020 09:16:35 +0300 Subject: [PATCH 66/84] [#2342] Removed changelog entry for temporary configuration option. --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b41502a27..7d9b10b28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - NodeInfo: `pleroma:api/v1/notifications:include_types_filter` to the `features` list. - Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses. - New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required. -- Configuration: `:extensions/:output_relationships_in_statuses_by_default` option (if `false`, disables the output of account/pleroma/relationship for statuses and notifications by default, breaking the compatibility with older PleromaFE versions).
API Changes - Mastodon API: Support for `include_types` in `/api/v1/notifications`. From 5c76afb06c731557b537f928296e0b5c259f8d5e Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 13 Apr 2020 15:38:50 +0300 Subject: [PATCH 67/84] [#2342] Removed description.exs entry for temporary configuration option. --- config/description.exs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/config/description.exs b/config/description.exs index 1b450db58..642f1a3ce 100644 --- a/config/description.exs +++ b/config/description.exs @@ -121,22 +121,6 @@ } ] }, - %{ - group: :pleroma, - key: :extensions, - type: :group, - description: "Pleroma-specific extensions", - children: [ - %{ - key: :output_relationships_in_statuses_by_default, - type: :beeolean, - description: - "If `true`, outputs account/pleroma/relationship map for each rendered status / notification (for all clients). " <> - "If `false`, outputs the above only if `with_relationships` param is tru-ish " <> - "(that breaks compatibility with older PleromaFE versions which do not send this param but expect the output)." - } - ] - }, %{ group: :pleroma, key: Pleroma.Uploaders.Local, From 4dca712e90a81a1a754608eb4f8e22dae99eb755 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 13 Apr 2020 22:44:52 +0400 Subject: [PATCH 68/84] Add OpenAPI spec for DomainBlockController --- lib/pleroma/web/api_spec.ex | 2 +- .../operations/domain_block_operation.ex | 64 +++++++++++++++++++ .../api_spec/schemas/domain_block_request.ex | 20 ++++++ .../schemas/domain_blocks_response.ex | 16 +++++ .../controllers/domain_block_controller.ex | 7 +- .../domain_block_controller_test.exs | 20 +++++- 6 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 lib/pleroma/web/api_spec/operations/domain_block_operation.ex create mode 100644 lib/pleroma/web/api_spec/schemas/domain_block_request.ex create mode 100644 lib/pleroma/web/api_spec/schemas/domain_blocks_response.ex diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 41e48a085..3890489e3 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -31,7 +31,7 @@ def spec do password: %OpenApiSpex.OAuthFlow{ authorizationUrl: "/oauth/authorize", tokenUrl: "/oauth/token", - scopes: %{"read" => "read"} + scopes: %{"read" => "read", "write" => "write", "follow" => "follow"} } } } diff --git a/lib/pleroma/web/api_spec/operations/domain_block_operation.ex b/lib/pleroma/web/api_spec/operations/domain_block_operation.ex new file mode 100644 index 000000000..dd14837c3 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/domain_block_operation.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Helpers + alias Pleroma.Web.ApiSpec.Schemas.DomainBlockRequest + alias Pleroma.Web.ApiSpec.Schemas.DomainBlocksResponse + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["domain_blocks"], + summary: "Fetch domain blocks", + description: "View domains the user has blocked.", + security: [%{"oAuth" => ["follow", "read:blocks"]}], + operationId: "DomainBlockController.index", + responses: %{ + 200 => Operation.response("Domain blocks", "application/json", DomainBlocksResponse) + } + } + end + + def create_operation do + %Operation{ + tags: ["domain_blocks"], + summary: "Block a domain", + description: """ + Block a domain to: + + - hide all public posts from it + - hide all notifications from it + - remove all followers from it + - prevent following new users from it (but does not remove existing follows) + """, + operationId: "DomainBlockController.create", + requestBody: Helpers.request_body("Parameters", DomainBlockRequest, required: true), + security: [%{"oAuth" => ["follow", "write:blocks"]}], + responses: %{ + 200 => Operation.response("Empty object", "application/json", %Schema{type: :object}) + } + } + end + + def delete_operation do + %Operation{ + tags: ["domain_blocks"], + summary: "Unblock a domain", + description: "Remove a domain block, if it exists in the user's array of blocked domains.", + operationId: "DomainBlockController.delete", + requestBody: Helpers.request_body("Parameters", DomainBlockRequest, required: true), + security: [%{"oAuth" => ["follow", "write:blocks"]}], + responses: %{ + 200 => Operation.response("Empty object", "application/json", %Schema{type: :object}) + } + } + end +end diff --git a/lib/pleroma/web/api_spec/schemas/domain_block_request.ex b/lib/pleroma/web/api_spec/schemas/domain_block_request.ex new file mode 100644 index 000000000..ee9238361 --- /dev/null +++ b/lib/pleroma/web/api_spec/schemas/domain_block_request.ex @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Schemas.DomainBlockRequest do + alias OpenApiSpex.Schema + require OpenApiSpex + + OpenApiSpex.schema(%{ + title: "DomainBlockRequest", + type: :object, + properties: %{ + domain: %Schema{type: :string} + }, + required: [:domain], + example: %{ + "domain" => "facebook.com" + } + }) +end diff --git a/lib/pleroma/web/api_spec/schemas/domain_blocks_response.ex b/lib/pleroma/web/api_spec/schemas/domain_blocks_response.ex new file mode 100644 index 000000000..d895aca4e --- /dev/null +++ b/lib/pleroma/web/api_spec/schemas/domain_blocks_response.ex @@ -0,0 +1,16 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Schemas.DomainBlocksResponse do + require OpenApiSpex + alias OpenApiSpex.Schema + + OpenApiSpex.schema(%{ + title: "DomainBlocksResponse", + description: "Response schema for domain blocks", + type: :array, + items: %Schema{type: :string}, + example: ["google.com", "facebook.com"] + }) +end diff --git a/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex b/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex index e4156cbe6..84de79413 100644 --- a/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex @@ -8,6 +8,9 @@ defmodule Pleroma.Web.MastodonAPI.DomainBlockController do alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.User + plug(OpenApiSpex.Plug.CastAndValidate) + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.DomainBlockOperation + plug( OAuthScopesPlug, %{scopes: ["follow", "read:blocks"]} when action == :index @@ -26,13 +29,13 @@ def index(%{assigns: %{user: user}} = conn, _) do end @doc "POST /api/v1/domain_blocks" - def create(%{assigns: %{user: blocker}} = conn, %{"domain" => domain}) do + def create(%{assigns: %{user: blocker}, body_params: %{domain: domain}} = conn, _params) do User.block_domain(blocker, domain) json(conn, %{}) end @doc "DELETE /api/v1/domain_blocks" - def delete(%{assigns: %{user: blocker}} = conn, %{"domain" => domain}) do + def delete(%{assigns: %{user: blocker}, body_params: %{domain: domain}} = conn, _params) do User.unblock_domain(blocker, domain) json(conn, %{}) end diff --git a/test/web/mastodon_api/controllers/domain_block_controller_test.exs b/test/web/mastodon_api/controllers/domain_block_controller_test.exs index 8d24b3b88..d66190c90 100644 --- a/test/web/mastodon_api/controllers/domain_block_controller_test.exs +++ b/test/web/mastodon_api/controllers/domain_block_controller_test.exs @@ -6,20 +6,29 @@ defmodule Pleroma.Web.MastodonAPI.DomainBlockControllerTest do use Pleroma.Web.ConnCase alias Pleroma.User + alias Pleroma.Web.ApiSpec + alias Pleroma.Web.ApiSpec.Schemas.DomainBlocksResponse import Pleroma.Factory + import OpenApiSpex.TestAssertions test "blocking / unblocking a domain" do %{user: user, conn: conn} = oauth_access(["write:blocks"]) other_user = insert(:user, %{ap_id: "https://dogwhistle.zone/@pundit"}) - ret_conn = post(conn, "/api/v1/domain_blocks", %{"domain" => "dogwhistle.zone"}) + ret_conn = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/domain_blocks", %{"domain" => "dogwhistle.zone"}) assert %{} = json_response(ret_conn, 200) user = User.get_cached_by_ap_id(user.ap_id) assert User.blocks?(user, other_user) - ret_conn = delete(conn, "/api/v1/domain_blocks", %{"domain" => "dogwhistle.zone"}) + ret_conn = + conn + |> put_req_header("content-type", "application/json") + |> delete("/api/v1/domain_blocks", %{"domain" => "dogwhistle.zone"}) assert %{} = json_response(ret_conn, 200) user = User.get_cached_by_ap_id(user.ap_id) @@ -41,5 +50,12 @@ test "getting a list of domain blocks" do assert "bad.site" in domain_blocks assert "even.worse.site" in domain_blocks + assert_schema(domain_blocks, "DomainBlocksResponse", ApiSpec.spec()) + end + + test "DomainBlocksResponse example matches schema" do + api_spec = ApiSpec.spec() + schema = DomainBlocksResponse.schema() + assert_schema(schema.example, "DomainBlocksResponse", api_spec) end end From f3725b8fc4506e4bb400878214b9886ed954590f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 13 Apr 2020 17:04:43 -0500 Subject: [PATCH 69/84] Fix spelling --- lib/pleroma/pool/connections.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex index 4d4ba913c..acafe1bea 100644 --- a/lib/pleroma/pool/connections.ex +++ b/lib/pleroma/pool/connections.ex @@ -243,7 +243,7 @@ def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do @impl true def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do - Logger.debug("received DOWM message for #{inspect(conn_pid)} reason -> #{inspect(reason)}") + Logger.debug("received DOWN message for #{inspect(conn_pid)} reason -> #{inspect(reason)}") state = with {key, conn} <- find_conn(state.conns, conn_pid) do From c4e7ed660c0c561d7b014664ca393d39dc7ee29a Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 14 Apr 2020 08:43:47 +0300 Subject: [PATCH 70/84] fix logger message --- lib/pleroma/plugs/mapped_signature_to_identity_plug.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex index 4f124ed4d..84b7c5d83 100644 --- a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex +++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex @@ -42,13 +42,13 @@ def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = con else {:user_match, false} -> Logger.debug("Failed to map identity from signature (payload actor mismatch)") - Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}") + Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}, actor=#{inspect(actor)}") assign(conn, :valid_signature, false) # remove me once testsuite uses mapped capabilities instead of what we do now {:user, nil} -> Logger.debug("Failed to map identity from signature (lookup failure)") - Logger.debug("key_id=#{key_id_from_conn(conn)}, actor=#{actor}") + Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}, actor=#{actor}") conn end end @@ -60,7 +60,7 @@ def call(%{assigns: %{valid_signature: true}} = conn, _opts) do else _ -> Logger.debug("Failed to map identity from signature (no payload actor mismatch)") - Logger.debug("key_id=#{key_id_from_conn(conn)}") + Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}") assign(conn, :valid_signature, false) end end From d8b12ffd5909a2698cce50d81b69f00b8893d41b Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 14 Apr 2020 15:06:09 +0200 Subject: [PATCH 71/84] Marker update migration: Don't try to update virtual field. --- priv/repo/migrations/20200210050658_update_markers.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/repo/migrations/20200210050658_update_markers.exs b/priv/repo/migrations/20200210050658_update_markers.exs index b280e156c..db7a355ec 100644 --- a/priv/repo/migrations/20200210050658_update_markers.exs +++ b/priv/repo/migrations/20200210050658_update_markers.exs @@ -32,7 +32,7 @@ defp update_markers do end) Repo.insert_all("markers", markers_attrs, - on_conflict: {:replace, [:last_read_id, :unread_count]}, + on_conflict: {:replace, [:last_read_id]}, conflict_target: [:user_id, :timeline] ) end From 7c060432fcac294269742ac3f452beb3f9a2fdab Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 14 Apr 2020 16:31:30 +0000 Subject: [PATCH 72/84] Revert "Merge branch 'marker-update-fix' into 'develop'" This reverts merge request !2380 --- priv/repo/migrations/20200210050658_update_markers.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/repo/migrations/20200210050658_update_markers.exs b/priv/repo/migrations/20200210050658_update_markers.exs index db7a355ec..b280e156c 100644 --- a/priv/repo/migrations/20200210050658_update_markers.exs +++ b/priv/repo/migrations/20200210050658_update_markers.exs @@ -32,7 +32,7 @@ defp update_markers do end) Repo.insert_all("markers", markers_attrs, - on_conflict: {:replace, [:last_read_id]}, + on_conflict: {:replace, [:last_read_id, :unread_count]}, conflict_target: [:user_id, :timeline] ) end From 4576520461e2e3a1c78133aaf31cb742a2a1a689 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 14 Apr 2020 16:32:22 +0000 Subject: [PATCH 73/84] Revert "Merge branch 'issue/1276' into 'develop'" This reverts merge request !1877 --- CHANGELOG.md | 1 - docs/API/differences_in_mastoapi_responses.md | 17 ++----- lib/pleroma/marker.ex | 45 +---------------- lib/pleroma/notification.ex | 47 +++++------------ .../web/mastodon_api/views/marker_view.ex | 5 +- mix.lock | 50 +++++++++---------- .../20200210050658_update_markers.exs | 39 --------------- test/marker_test.exs | 29 +---------- test/notification_test.exs | 13 ----- .../controllers/marker_controller_test.exs | 10 ++-- .../mastodon_api/views/marker_view_test.exs | 8 ++- 11 files changed, 51 insertions(+), 213 deletions(-) delete mode 100644 priv/repo/migrations/20200210050658_update_markers.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f7fc1802..56b235f6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,7 +123,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: `pleroma.thread_muted` to the Status entity - Mastodon API: Mark the direct conversation as read for the author when they send a new direct message - Mastodon API, streaming: Add `pleroma.direct_conversation_id` to the `conversation` stream event payload. -- Mastodon API: Add `pleroma.unread_count` to the Marker entity - Admin API: Render whole status in grouped reports - Mastodon API: User timelines will now respect blocks, unless you are getting the user timeline of somebody you blocked (which would be empty otherwise). - Mastodon API: Favoriting / Repeating a post multiple times will now return the identical response every time. Before, executing that action twice would return an error ("already favorited") on the second try. diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index 0a7520f9e..1059155cf 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -185,15 +185,8 @@ Post here request with `grant_type=refresh_token` to obtain new access token. Re Has theses additional parameters (which are the same as in Pleroma-API): - `fullname`: optional - `bio`: optional - `captcha_solution`: optional, contains provider-specific captcha solution, - `captcha_token`: optional, contains provider-specific captcha token - `token`: invite token required when the registrations aren't public. - - -## Markers - -Has these additional fields under the `pleroma` object: - -- `unread_count`: contains number unread notifications +- `fullname`: optional +- `bio`: optional +- `captcha_solution`: optional, contains provider-specific captcha solution, +- `captcha_token`: optional, contains provider-specific captcha token +- `token`: invite token required when the registrations aren't public. diff --git a/lib/pleroma/marker.ex b/lib/pleroma/marker.ex index 4d82860f5..443927392 100644 --- a/lib/pleroma/marker.ex +++ b/lib/pleroma/marker.ex @@ -9,34 +9,24 @@ defmodule Pleroma.Marker do import Ecto.Query alias Ecto.Multi - alias Pleroma.Notification alias Pleroma.Repo alias Pleroma.User - alias __MODULE__ @timelines ["notifications"] - @type t :: %__MODULE__{} schema "markers" do field(:last_read_id, :string, default: "") field(:timeline, :string, default: "") field(:lock_version, :integer, default: 0) - field(:unread_count, :integer, default: 0, virtual: true) belongs_to(:user, User, type: FlakeId.Ecto.CompatType) timestamps() end - @doc "Gets markers by user and timeline." - @spec get_markers(User.t(), list(String)) :: list(t()) def get_markers(user, timelines \\ []) do - user - |> get_query(timelines) - |> unread_count_query() - |> Repo.all() + Repo.all(get_query(user, timelines)) end - @spec upsert(User.t(), map()) :: {:ok | :error, any()} def upsert(%User{} = user, attrs) do attrs |> Map.take(@timelines) @@ -55,27 +45,6 @@ def upsert(%User{} = user, attrs) do |> Repo.transaction() end - @spec multi_set_last_read_id(Multi.t(), User.t(), String.t()) :: Multi.t() - def multi_set_last_read_id(multi, %User{} = user, "notifications") do - multi - |> Multi.run(:counters, fn _repo, _changes -> - {:ok, %{last_read_id: Repo.one(Notification.last_read_query(user))}} - end) - |> Multi.insert( - :marker, - fn %{counters: attrs} -> - %Marker{timeline: "notifications", user_id: user.id} - |> struct(attrs) - |> Ecto.Changeset.change() - end, - returning: true, - on_conflict: {:replace, [:last_read_id]}, - conflict_target: [:user_id, :timeline] - ) - end - - def multi_set_last_read_id(multi, _, _), do: multi - defp get_marker(user, timeline) do case Repo.find_resource(get_query(user, timeline)) do {:ok, marker} -> %__MODULE__{marker | user: user} @@ -102,16 +71,4 @@ defp get_query(user, timelines) do |> by_user_id(user.id) |> by_timeline(timelines) end - - defp unread_count_query(query) do - from( - q in query, - left_join: n in "notifications", - on: n.user_id == q.user_id and n.seen == false, - group_by: [:id], - select_merge: %{ - unread_count: fragment("count(?)", n.id) - } - ) - end end diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 3084bac3b..04ee510b9 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -5,9 +5,7 @@ defmodule Pleroma.Notification do use Ecto.Schema - alias Ecto.Multi alias Pleroma.Activity - alias Pleroma.Marker alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Pagination @@ -40,17 +38,6 @@ def changeset(%Notification{} = notification, attrs) do |> cast(attrs, [:seen]) end - @spec last_read_query(User.t()) :: Ecto.Queryable.t() - def last_read_query(user) do - from(q in Pleroma.Notification, - where: q.user_id == ^user.id, - where: q.seen == true, - select: type(q.id, :string), - limit: 1, - order_by: [desc: :id] - ) - end - defp for_user_query_ap_id_opts(user, opts) do ap_id_relationships = [:block] ++ @@ -199,23 +186,25 @@ def for_user_since(user, date) do |> Repo.all() end - def set_read_up_to(%{id: user_id} = user, id) do + def set_read_up_to(%{id: user_id} = _user, id) do query = from( n in Notification, where: n.user_id == ^user_id, where: n.id <= ^id, where: n.seen == false, + update: [ + set: [ + seen: true, + updated_at: ^NaiveDateTime.utc_now() + ] + ], # Ideally we would preload object and activities here # but Ecto does not support preloads in update_all select: n.id ) - {:ok, %{ids: {_, notification_ids}}} = - Multi.new() - |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) - |> Marker.multi_set_last_read_id(user, "notifications") - |> Repo.transaction() + {_, notification_ids} = Repo.update_all(query, []) Notification |> where([n], n.id in ^notification_ids) @@ -232,18 +221,11 @@ def set_read_up_to(%{id: user_id} = user, id) do |> Repo.all() end - @spec read_one(User.t(), String.t()) :: - {:ok, Notification.t()} | {:error, Ecto.Changeset.t()} | nil def read_one(%User{} = user, notification_id) do with {:ok, %Notification{} = notification} <- get(user, notification_id) do - Multi.new() - |> Multi.update(:update, changeset(notification, %{seen: true})) - |> Marker.multi_set_last_read_id(user, "notifications") - |> Repo.transaction() - |> case do - {:ok, %{update: notification}} -> {:ok, notification} - {:error, :update, changeset, _} -> {:error, changeset} - end + notification + |> changeset(%{seen: true}) + |> Repo.update() end end @@ -325,11 +307,8 @@ defp do_create_notifications(%Activity{} = activity) do # TODO move to sql, too. def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true) do unless skip?(activity, user) do - {:ok, %{notification: notification}} = - Multi.new() - |> Multi.insert(:notification, %Notification{user_id: user.id, activity: activity}) - |> Marker.multi_set_last_read_id(user, "notifications") - |> Repo.transaction() + notification = %Notification{user_id: user.id, activity: activity} + {:ok, notification} = Repo.insert(notification) if do_send do Streamer.stream(["user", "user:notification"], notification) diff --git a/lib/pleroma/web/mastodon_api/views/marker_view.ex b/lib/pleroma/web/mastodon_api/views/marker_view.ex index 415dae93b..985368fe5 100644 --- a/lib/pleroma/web/mastodon_api/views/marker_view.ex +++ b/lib/pleroma/web/mastodon_api/views/marker_view.ex @@ -10,10 +10,7 @@ def render("markers.json", %{markers: markers}) do Map.put_new(acc, m.timeline, %{ last_read_id: m.last_read_id, version: m.lock_version, - updated_at: NaiveDateTime.to_iso8601(m.updated_at), - pleroma: %{ - unread_count: m.unread_count - } + updated_at: NaiveDateTime.to_iso8601(m.updated_at) }) end) end diff --git a/mix.lock b/mix.lock index 23467bbb4..ba4e3ac44 100644 --- a/mix.lock +++ b/mix.lock @@ -2,8 +2,8 @@ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"}, "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "95e8188490e97505c56636c1379ffdf036c1fdde", [ref: "95e8188490e97505c56636c1379ffdf036c1fdde"]}, "base62": {:hex, :base62, "1.2.1", "4866763e08555a7b3917064e9eef9194c41667276c51b59de2bc42c6ea65f806", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "3b29948de2013d3f93aa898c884a9dff847e7aec75d9d6d8c1dc4c61c2716c42"}, - "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm", "fab09b20e3f5db886725544cbcf875b8e73ec93363954eb8a1a9ed834aa8c1f9"}, - "bbcode": {:hex, :bbcode, "0.1.1", "0023e2c7814119b2e620b7add67182e3f6019f92bfec9a22da7e99821aceba70", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5a981b98ac7d366a9b6bf40eac389aaf4d6e623c631e6b6f8a6b571efaafd338"}, + "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, + "bbcode": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/bbcode.git", "f2d267675e9a7e1ad1ea9beb4cc23382762b66c2", [ref: "v0.2.0"]}, "bbcode_pleroma": {:hex, :bbcode_pleroma, "0.2.0", "d36f5bca6e2f62261c45be30fa9b92725c0655ad45c99025cb1c3e28e25803ef", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "19851074419a5fedb4ef49e1f01b30df504bb5dbb6d6adfc135238063bebd1c3"}, "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm", "3ad58ae787e9c7c94dd7ceda3b587ec2c64604563e049b2a0e8baafae832addb"}, "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm", "7af5c7e09fe1d40f76c8e4f9dd2be7cebd83909f31fee7cd0e9eadc567da8353"}, @@ -19,47 +19,47 @@ "cowboy": {:hex, :cowboy, "2.7.0", "91ed100138a764355f43316b1d23d7ff6bdb0de4ea618cb5d8677c93a7a2f115", [:rebar3], [{:cowlib, "~> 2.8.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "04fd8c6a39edc6aaa9c26123009200fc61f92a3a94f3178c527b70b767c6e605"}, "cowlib": {:hex, :cowlib, "2.8.0", "fd0ff1787db84ac415b8211573e9a30a3ebe71b5cbff7f720089972b2319c8a4", [:rebar3], [], "hexpm", "79f954a7021b302186a950a32869dbc185523d99d3e44ce430cd1f3289f41ed4"}, "credo": {:hex, :credo, "1.1.5", "caec7a3cadd2e58609d7ee25b3931b129e739e070539ad1a0cd7efeeb47014f4", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d0bbd3222607ccaaac5c0340f7f525c627ae4d7aee6c8c8c108922620c5b6446"}, - "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "48e513299cd28b12c77266c0ed5b1c844368e5c1823724994ae84834f43d6bbe"}, + "crontab": {:hex, :crontab, "1.1.8", "2ce0e74777dfcadb28a1debbea707e58b879e6aa0ffbf9c9bb540887bce43617", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm"}, "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, "db_connection": {:hex, :db_connection, "2.2.1", "caee17725495f5129cb7faebde001dc4406796f12a62b8949f4ac69315080566", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "2b02ece62d9f983fcd40954e443b7d9e6589664380e5546b2b9b523cd0fb59e1"}, "decimal": {:hex, :decimal, "1.8.1", "a4ef3f5f3428bdbc0d35374029ffcf4ede8533536fa79896dd450168d9acdf3c", [:mix], [], "hexpm", "3cb154b00225ac687f6cbd4acc4b7960027c757a5152b369923ead9ddbca7aec"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "earmark": {:hex, :earmark, "1.4.2", "3aa0bd23bc4c61cf2f1e5d752d1bb470560a6f8539974f767a38923bb20e1d7f", [:mix], [], "hexpm", "5e8806285d8a3a8999bd38e4a73c58d28534c856bc38c44818e5ba85bbda16fb"}, - "ecto": {:hex, :ecto, "3.4.2", "6890af71025769bd27ef62b1ed1925cfe23f7f0460bcb3041da4b705215ff23e", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3959b8a83e086202a4bd86b4b5e6e71f9f1840813de14a57d502d3fc2ef7132"}, + "earmark": {:hex, :earmark, "1.4.3", "364ca2e9710f6bff494117dbbd53880d84bebb692dafc3a78eb50aa3183f2bfd", [:mix], [], "hexpm", "8cf8a291ebf1c7b9539e3cddb19e9cef066c2441b1640f13c34c1d3cfc825fec"}, + "ecto": {:hex, :ecto, "3.4.0", "a7a83ab8359bf816ce729e5e65981ce25b9fc5adfc89c2ea3980f4fed0bfd7c1", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "5eed18252f5b5bbadec56a24112b531343507dbe046273133176b12190ce19cc"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_sql": {:hex, :ecto_sql, "3.3.4", "aa18af12eb875fbcda2f75e608b3bd534ebf020fc4f6448e4672fcdcbb081244", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4 or ~> 3.3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5eccbdbf92e3c6f213007a82d5dbba4cd9bb659d1a21331f89f408e4c0efd7a8"}, - "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm", "98d0f3c6f4b8a0333170df770c6fe772b3d04564fb514c1a09504cf5ab2f48a5"}, + "esshd": {:hex, :esshd, "0.1.1", "d4dd4c46698093a40a56afecce8a46e246eb35463c457c246dacba2e056f31b5", [:mix], [], "hexpm", "d73e341e3009d390aa36387dc8862860bf9f874c94d9fd92ade2926376f49981"}, "eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm", "b14f1dc204321429479c569cfbe8fb287541184ed040956c8862cb7a677b8406"}, "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"}, "ex_aws": {:hex, :ex_aws, "2.1.1", "1e4de2106cfbf4e837de41be41cd15813eabc722315e388f0d6bb3732cec47cd", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "06b6fde12b33bb6d65d5d3493e903ba5a56d57a72350c15285a4298338089e10"}, "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.2", "c0258bbdfea55de4f98f0b2f0ca61fe402cc696f573815134beb1866e778f47b", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "0569f5b211b1a3b12b705fe2a9d0e237eb1360b9d76298028df2346cad13097a"}, "ex_const": {:hex, :ex_const, "0.2.4", "d06e540c9d834865b012a17407761455efa71d0ce91e5831e86881b9c9d82448", [:mix], [], "hexpm", "96fd346610cc992b8f896ed26a98be82ac4efb065a0578f334a32d60a3ba9767"}, - "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f1155337ae17ff7a1255217b4c1ceefcd1860b7ceb1a1874031e7a861b052e39"}, + "ex_doc": {:hex, :ex_doc, "0.21.3", "857ec876b35a587c5d9148a2512e952e24c24345552259464b98bfbb883c7b42", [:mix], [{:earmark, "~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "0db1ee8d1547ab4877c5b5dffc6604ef9454e189928d5ba8967d4a58a801f161"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "b84f6af156264530b312a8ab98ac6088f6b77ae5fe2058305c81434aa01fbaf9"}, "ex_syslogger": {:hex, :ex_syslogger, "1.5.0", "bc936ee3fd13d9e592cb4c3a1e8a55fccd33b05e3aa7b185f211f3ed263ff8f0", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.0.5", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "f3b4b184dcdd5f356b7c26c6cd72ab0918ba9dfb4061ccfaf519e562942af87b"}, "excoveralls": {:hex, :excoveralls, "0.12.2", "a513defac45c59e310ac42fcf2b8ae96f1f85746410f30b1ff2b710a4b6cd44b", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "151c476331d49b45601ffc45f43cb3a8beb396b02a34e3777fea0ad34ae57d89"}, - "fast_html": {:hex, :fast_html, "1.0.1", "5bc7df4dc4607ec2c314c16414e4111d79a209956c4f5df96602d194c61197f9", [:make, :mix], [], "hexpm", "18e627dd62051a375ef94b197f41e8027c3e8eef0180ab8f81e0543b3dc6900a"}, - "fast_sanitize": {:hex, :fast_sanitize, "0.1.6", "60a5ae96879956dea409a91a77f5dd2994c24cc10f80eefd8f9892ee4c0c7b25", [:mix], [{:fast_html, "~> 1.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "b73f50f0cb522dd0331ea8e8c90b408de42c50f37641219d6364f0e3e7efd22c"}, + "fast_html": {:hex, :fast_html, "1.0.3", "2cc0d4b68496266a1530e0c852cafeaede0bd10cfdee26fda50dc696c203162f", [:make, :mix], [], "hexpm", "ab3d782b639d3c4655fbaec0f9d032c91f8cab8dd791ac7469c2381bc7c32f85"}, + "fast_sanitize": {:hex, :fast_sanitize, "0.1.7", "2a7cd8734c88a2de6de55022104f8a3b87f1fdbe8bbf131d9049764b53d50d0d", [:mix], [{:fast_html, "~> 1.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f39fe8ea08fbac17487c30bf09b7d9f3e12472e51fb07a88ffeb8fd17da8ab67"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"}, - "floki": {:hex, :floki, "0.26.0", "4df88977e2e357c6720e1b650f613444bfb48c5acfc6a0c646ab007d08ad13bf", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "e7b66ce7feef5518a9cd9fc7b52dd62a64028bd9cb6d6ad282a0f0fc90a4ae52"}, + "floki": {:hex, :floki, "0.25.0", "b1c9ddf5f32a3a90b43b76f3386ca054325dc2478af020e87b5111c19f2284ac", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "631f4e627c46d5ecd347df5a2accdaf0621c77c3693c5b75a8ad58e84c61f242"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"}, - "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm", "8453e2289d94c3199396eb517d65d6715ef26bcae0ee83eb5ff7a84445458d76"}, - "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm", "5cacd405e72b2609a7e1f891bddb80c53d0b3b7b0036d1648e7382ca108c41c8"}, - "gettext": {:hex, :gettext, "0.17.1", "8baab33482df4907b3eae22f719da492cee3981a26e649b9c2be1c0192616962", [:mix], [], "hexpm", "f7d97341e536f95b96eef2988d6d4230f7262cf239cda0e2e63123ee0b717222"}, + "gen_stage": {:hex, :gen_stage, "0.14.3", "d0c66f1c87faa301c1a85a809a3ee9097a4264b2edf7644bf5c123237ef732bf", [:mix], [], "hexpm"}, + "gen_state_machine": {:hex, :gen_state_machine, "2.0.5", "9ac15ec6e66acac994cc442dcc2c6f9796cf380ec4b08267223014be1c728a95", [:mix], [], "hexpm"}, + "gettext": {:hex, :gettext, "0.17.4", "f13088e1ec10ce01665cf25f5ff779e7df3f2dc71b37084976cf89d1aa124d5c", [:mix], [], "hexpm", "3c75b5ea8288e2ee7ea503ff9e30dfe4d07ad3c054576a6e60040e79a801e14d"}, "gun": {:git, "https://github.com/ninenines/gun.git", "e1a69b36b180a574c0ac314ced9613fdd52312cc", [ref: "e1a69b36b180a574c0ac314ced9613fdd52312cc"]}, "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"}, "html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "http_signatures": {:git, "https://git.pleroma.social/pleroma/http_signatures.git", "293d77bb6f4a67ac8bde1428735c3b42f22cbb30", [ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"]}, - "httpoison": {:hex, :httpoison, "1.6.1", "2ce5bf6e535cd0ab02e905ba8c276580bab80052c5c549f53ddea52d72e81f33", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "89149056039084024a284cd703b2d1900d584958dba432132cb21ef35aed7487"}, + "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "4bdd305eb64e18b0273864920695cb18d7a2021f31a11b9c5fbcd9a253f936e2"}, "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm", "64a2d30189704ae41ca7dbdd587f5291db5d1dda1414e0774c29ffc81088c1bc"}, "jason": {:hex, :jason, "1.2.0", "10043418c42d2493d0ee212d3fddd25d7ffe484380afad769a0a38795938e448", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "116747dbe057794c3a3e4e143b7c8390b29f634e16c78a7f59ba75bfa6852e7f"}, - "joken": {:hex, :joken, "2.1.0", "bf21a73105d82649f617c5e59a7f8919aa47013d2519ebcc39d998d8d12adda9", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "eb02df7d5526df13063397e051b926b7006d5986d66f399eefc474f560cdad6a"}, - "jose": {:hex, :jose, "1.9.0", "4167c5f6d06ffaebffd15cdb8da61a108445ef5e85ab8f5a7ad926fdf3ada154", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm", "6429c4fee52b2dda7861ee19a4f09c8c1ffa213bee3a1ec187828fde95d447ed"}, + "joken": {:hex, :joken, "2.2.0", "2daa1b12be05184aff7b5ace1d43ca1f81345962285fff3f88db74927c954d3a", [:mix], [{:jose, "~> 1.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "b4f92e30388206f869dd25d1af628a1d99d7586e5cf0672f64d4df84c4d2f5e9"}, + "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm", "3c7ddc8a9394b92891db7c2771da94bf819834a1a4c92e30857b7d582e2f8257"}, "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, - "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm", "1feaf05ee886815ad047cad7ede17d6910710986148ae09cf73eee2989717b81"}, + "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"}, "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"}, "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"}, "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"}, @@ -71,37 +71,35 @@ "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm", "3bc928d817974fa10cc11e6c89b9a9361e37e96dbbf3d868c41094ec05745dcd"}, "mox": {:hex, :mox, "0.5.1", "f86bb36026aac1e6f924a4b6d024b05e9adbed5c63e8daa069bd66fb3292165b", [:mix], [], "hexpm", "052346cf322311c49a0f22789f3698eea030eec09b8c47367f0686ef2634ae14"}, "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]}, - "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm", "00e3ebdc821fb3a36957320d49e8f4bfa310d73ea31c90e5f925dc75e030da8f"}, + "nimble_parsec": {:hex, :nimble_parsec, "0.5.3", "def21c10a9ed70ce22754fdeea0810dafd53c2db3219a0cd54cf5526377af1c6", [:mix], [], "hexpm", "589b5af56f4afca65217a1f3eb3fee7e79b09c40c742fddc1c312b3ac0b3399f"}, "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]}, "oban": {:hex, :oban, "1.2.0", "7cca94d341be43d220571e28f69131c4afc21095b25257397f50973d3fc59b07", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ba5f8b3f7d76967b3e23cf8014f6a13e4ccb33431e4808f036709a7f822362ee"}, "open_api_spex": {:hex, :open_api_spex, "3.6.0", "64205aba9f2607f71b08fd43e3351b9c5e9898ec5ef49fc0ae35890da502ade9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.1", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm", "126ba3473966277132079cb1d5bf1e3df9e36fe2acd00166e75fd125cecb59c5"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm", "595d09db74cb093b1903381c9de423276a931a2480a46a1a5dc7f932a2a6375b"}, - "phoenix": {:hex, :phoenix, "1.4.10", "619e4a545505f562cd294df52294372d012823f4fd9d34a6657a8b242898c255", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "256ad7a140efadc3f0290470369da5bd3de985ec7c706eba07c2641b228974be"}, - "phoenix_ecto": {:hex, :phoenix_ecto, "4.0.0", "c43117a136e7399ea04ecaac73f8f23ee0ffe3e07acfcb8062fe5f4c9f0f6531", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "fe15d9fee5b82f5e64800502011ffe530650d42e1710ae9b14bc4c9be38bf303"}, - "phoenix_html": {:hex, :phoenix_html, "2.13.3", "850e292ff6e204257f5f9c4c54a8cb1f6fbc16ed53d360c2b780a3d0ba333867", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "8b01b3d6d39731ab18aa548d928b5796166d2500755f553725cfe967bafba7d9"}, + "phoenix": {:hex, :phoenix, "1.4.13", "67271ad69b51f3719354604f4a3f968f83aa61c19199343656c9caee057ff3b8", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ab765a0feddb81fc62e2116c827b5f068df85159c162bee760745276ad7ddc1b"}, + "phoenix_ecto": {:hex, :phoenix_ecto, "4.1.0", "a044d0756d0464c5a541b4a0bf4bcaf89bffcaf92468862408290682c73ae50d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "c5e666a341ff104d0399d8f0e4ff094559b2fde13a5985d4cb5023b2c2ac558b"}, + "phoenix_html": {:hex, :phoenix_html, "2.14.0", "d8c6bc28acc8e65f8ea0080ee05aa13d912c8758699283b8d3427b655aabe284", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "b0bb30eda478a06dbfbe96728061a93833db3861a49ccb516f839ecb08493fbb"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.2", "496c303bdf1b2e98a9d26e89af5bba3ab487ba3a3735f74bf1f4064d2a845a3e", [:mix], [], "hexpm", "1f13f9f0f3e769a667a6b6828d29dec37497a082d195cc52dbef401a9b69bf38"}, "phoenix_swoosh": {:hex, :phoenix_swoosh, "0.2.0", "a7e0b32077cd6d2323ae15198839b05d9caddfa20663fd85787479e81f89520e", [:mix], [{:phoenix, "~> 1.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.2", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:swoosh, "~> 0.1", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "ebf1bfa7b3c1c850c04929afe02e2e0d7ab135e0706332c865de03e761676b1f"}, "plug": {:hex, :plug, "1.9.0", "8d7c4e26962283ff9f8f3347bd73838e2413fbc38b7bb5467d5924f68f3a5a4a", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "9902eda2c52ada2a096434682e99a2493f5d06a94d6ac6bcfff9805f952350f1"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.1.0", "b75768153c3a8a9e8039d4b25bb9b14efbc58e9c4a6e6a270abff1cd30cbe320", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "6cd8ddd1bd1fbfa54d3fc61d4719c2057dae67615395d58d40437a919a46f132"}, - "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm", "73c1682f0e414cfb5d9b95c8e8cd6ffcfdae699e3b05e1db744e58b7be857759"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.1.2", "8b0addb5908c5238fac38e442e81b6fcd32788eaa03246b4d55d147c47c5805e", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "7d722581ce865a237e14da6d946f92704101740a256bd13ec91e63c0b122fc70"}, + "plug_crypto": {:hex, :plug_crypto, "1.1.2", "bdd187572cc26dbd95b87136290425f2b580a116d3fb1f564216918c9730d227", [:mix], [], "hexpm", "6b8b608f895b6ffcfad49c37c7883e8df98ae19c6a28113b02aa1e9c5b22d6b5"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, "postgrex": {:hex, :postgrex, "0.15.3", "5806baa8a19a68c4d07c7a624ccdb9b57e89cbc573f1b98099e3741214746ae4", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "4737ce62a31747b4c63c12b20c62307e51bb4fcd730ca0c32c280991e0606c90"}, - "prometheus": {:hex, :prometheus, "4.4.1", "1e96073b3ed7788053768fea779cbc896ddc3bdd9ba60687f2ad50b252ac87d6", [:mix, :rebar3], [], "hexpm", "d39f2ce1f3f29f3bf04f915aa3cf9c7cd4d2cee2f975e05f526e06cae9b7c902"}, + "prometheus": {:hex, :prometheus, "4.5.0", "8f4a2246fe0beb50af0f77c5e0a5bb78fe575c34a9655d7f8bc743aad1c6bf76", [:mix, :rebar3], [], "hexpm", "679b5215480fff612b8351f45c839d995a07ce403e42ff02f1c6b20960d41a4e"}, "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.3", "3dd4da1812b8e0dbee81ea58bb3b62ed7588f2eae0c9e97e434c46807ff82311", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "8d66289f77f913b37eda81fd287340c17e61a447549deb28efc254532b2bed82"}, "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm", "9fd13404a48437e044b288b41f76e64acd9735fb8b0e3809f494811dfa66d0fb"}, "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.3.0", "c4b527e0b3a9ef1af26bdcfbfad3998f37795b9185d475ca610fe4388fdd3bb5", [:mix], [{:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm", "c4d1404ac4e9d3d963da601db2a7d8ea31194f0017057fabf0cfb9bf5a6c8c75"}, "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm", "0273a6483ccb936d79ca19b0ab629aef0dba958697c94782bb728b920dfc6a79"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "d736bfa7444112eb840027bb887832a0e403a4a3437f48028c3b29a2dbbd2543"}, - "quantum": {:hex, :quantum, "2.3.4", "72a0e8855e2adc101459eac8454787cb74ab4169de6ca50f670e72142d4960e9", [:mix], [{:calendar, "~> 0.17", [hex: :calendar, repo: "hexpm", optional: true]}, {:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.12", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:swarm, "~> 3.3", [hex: :swarm, repo: "hexpm", optional: false]}, {:timex, "~> 3.1", [hex: :timex, repo: "hexpm", optional: true]}], "hexpm", "6de553ba9ac0668d3728b699d5065543f3e40c854154017461ee8c09038752da"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"}, "recon": {:hex, :recon, "2.5.0", "2f7fcbec2c35034bade2f9717f77059dc54eb4e929a3049ca7ba6775c0bd66cd", [:mix, :rebar3], [], "hexpm", "72f3840fedd94f06315c523f6cecf5b4827233bed7ae3fe135b2a0ebeab5e196"}, "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "825dc00aaba5a1b7c4202a532b696b595dd3bcb3", [ref: "825dc00aaba5a1b7c4202a532b696b595dd3bcb3"]}, "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm", "13104d7897e38ed7f044c4de953a6c28597d1c952075eb2e328bc6d6f2bfc496"}, - "swarm": {:hex, :swarm, "3.4.0", "64f8b30055d74640d2186c66354b33b999438692a91be275bb89cdc7e401f448", [:mix], [{:gen_state_machine, "~> 2.0", [hex: :gen_state_machine, repo: "hexpm", optional: false]}, {:libring, "~> 1.0", [hex: :libring, repo: "hexpm", optional: false]}], "hexpm", "94884f84783fc1ba027aba8fe8a7dae4aad78c98e9f9c76667ec3471585c08c6"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm", "2e1ec458f892ffa81f9f8386e3f35a1af6db7a7a37748a64478f13163a1f3573"}, "swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "e3928e1d2889a308aaf3e42755809ac21cffd77cb58eef01cbfdab4ce2fd1e21"}, "syslog": {:hex, :syslog, "1.0.6", "995970c9aa7feb380ac493302138e308d6e04fd57da95b439a6df5bb3bf75076", [:rebar3], [], "hexpm", "769ddfabd0d2a16f3f9c17eb7509951e0ca4f68363fb26f2ee51a8ec4a49881a"}, diff --git a/priv/repo/migrations/20200210050658_update_markers.exs b/priv/repo/migrations/20200210050658_update_markers.exs deleted file mode 100644 index b280e156c..000000000 --- a/priv/repo/migrations/20200210050658_update_markers.exs +++ /dev/null @@ -1,39 +0,0 @@ -defmodule Pleroma.Repo.Migrations.UpdateMarkers do - use Ecto.Migration - import Ecto.Query - alias Pleroma.Repo - - def up do - update_markers() - end - - def down do - :ok - end - - defp update_markers do - now = NaiveDateTime.utc_now() - - markers_attrs = - from(q in "notifications", - select: %{ - timeline: "notifications", - user_id: q.user_id, - last_read_id: - type(fragment("MAX( CASE WHEN seen = true THEN id ELSE null END )"), :string) - }, - group_by: [q.user_id] - ) - |> Repo.all() - |> Enum.map(fn attrs -> - attrs - |> Map.put_new(:inserted_at, now) - |> Map.put_new(:updated_at, now) - end) - - Repo.insert_all("markers", markers_attrs, - on_conflict: {:replace, [:last_read_id, :unread_count]}, - conflict_target: [:user_id, :timeline] - ) - end -end diff --git a/test/marker_test.exs b/test/marker_test.exs index 5b6d0b4a4..c80ae16b6 100644 --- a/test/marker_test.exs +++ b/test/marker_test.exs @@ -8,39 +8,12 @@ defmodule Pleroma.MarkerTest do import Pleroma.Factory - describe "multi_set_unread_count/3" do - test "returns multi" do - user = insert(:user) - - assert %Ecto.Multi{ - operations: [marker: {:run, _}, counters: {:run, _}] - } = - Marker.multi_set_last_read_id( - Ecto.Multi.new(), - user, - "notifications" - ) - end - - test "return empty multi" do - user = insert(:user) - multi = Ecto.Multi.new() - assert Marker.multi_set_last_read_id(multi, user, "home") == multi - end - end - describe "get_markers/2" do test "returns user markers" do user = insert(:user) marker = insert(:marker, user: user) - insert(:notification, user: user) - insert(:notification, user: user) insert(:marker, timeline: "home", user: user) - - assert Marker.get_markers( - user, - ["notifications"] - ) == [%Marker{refresh_record(marker) | unread_count: 2}] + assert Marker.get_markers(user, ["notifications"]) == [refresh_record(marker)] end end diff --git a/test/notification_test.exs b/test/notification_test.exs index f78a47af6..837a9dacd 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -45,9 +45,6 @@ test "notifies someone when they are directly addressed" do assert notified_ids == [other_user.id, third_user.id] assert notification.activity_id == activity.id assert other_notification.activity_id == activity.id - - assert [%Pleroma.Marker{unread_count: 2}] = - Pleroma.Marker.get_markers(other_user, ["notifications"]) end test "it creates a notification for subscribed users" do @@ -413,16 +410,6 @@ test "it sets all notifications as read up to a specified notification ID" do assert n1.seen == true assert n2.seen == true assert n3.seen == false - - assert %Pleroma.Marker{} = - m = - Pleroma.Repo.get_by( - Pleroma.Marker, - user_id: other_user.id, - timeline: "notifications" - ) - - assert m.last_read_id == to_string(n2.id) end end diff --git a/test/web/mastodon_api/controllers/marker_controller_test.exs b/test/web/mastodon_api/controllers/marker_controller_test.exs index 7280abd10..919f295bd 100644 --- a/test/web/mastodon_api/controllers/marker_controller_test.exs +++ b/test/web/mastodon_api/controllers/marker_controller_test.exs @@ -11,7 +11,6 @@ defmodule Pleroma.Web.MastodonAPI.MarkerControllerTest do test "gets markers with correct scopes", %{conn: conn} do user = insert(:user) token = insert(:oauth_token, user: user, scopes: ["read:statuses"]) - insert_list(7, :notification, user: user) {:ok, %{"notifications" => marker}} = Pleroma.Marker.upsert( @@ -30,8 +29,7 @@ test "gets markers with correct scopes", %{conn: conn} do "notifications" => %{ "last_read_id" => "69420", "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at), - "version" => 0, - "pleroma" => %{"unread_count" => 7} + "version" => 0 } } end @@ -72,8 +70,7 @@ test "creates a marker with correct scopes", %{conn: conn} do "notifications" => %{ "last_read_id" => "69420", "updated_at" => _, - "version" => 0, - "pleroma" => %{"unread_count" => 0} + "version" => 0 } } = response end @@ -102,8 +99,7 @@ test "updates exist marker", %{conn: conn} do "notifications" => %{ "last_read_id" => "69888", "updated_at" => NaiveDateTime.to_iso8601(marker.updated_at), - "version" => 0, - "pleroma" => %{"unread_count" => 0} + "version" => 0 } } end diff --git a/test/web/mastodon_api/views/marker_view_test.exs b/test/web/mastodon_api/views/marker_view_test.exs index 48a0a6d33..893cf8857 100644 --- a/test/web/mastodon_api/views/marker_view_test.exs +++ b/test/web/mastodon_api/views/marker_view_test.exs @@ -8,21 +8,19 @@ defmodule Pleroma.Web.MastodonAPI.MarkerViewTest do import Pleroma.Factory test "returns markers" do - marker1 = insert(:marker, timeline: "notifications", last_read_id: "17", unread_count: 5) + marker1 = insert(:marker, timeline: "notifications", last_read_id: "17") marker2 = insert(:marker, timeline: "home", last_read_id: "42") assert MarkerView.render("markers.json", %{markers: [marker1, marker2]}) == %{ "home" => %{ last_read_id: "42", updated_at: NaiveDateTime.to_iso8601(marker2.updated_at), - version: 0, - pleroma: %{unread_count: 0} + version: 0 }, "notifications" => %{ last_read_id: "17", updated_at: NaiveDateTime.to_iso8601(marker1.updated_at), - version: 0, - pleroma: %{unread_count: 5} + version: 0 } } end From 3bf78f2be7c151cd64ed954570dbf8592e836f56 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 14 Apr 2020 11:43:53 -0500 Subject: [PATCH 74/84] Fix Oban not receiving :ok from RichMediaHelper job --- lib/pleroma/web/rich_media/helpers.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 0314535d2..9d3d7f978 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -64,5 +64,8 @@ def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) d def fetch_data_for_activity(_), do: %{} - def perform(:fetch, %Activity{} = activity), do: fetch_data_for_activity(activity) + def perform(:fetch, %Activity{} = activity) do + fetch_data_for_activity(activity) + :ok + end end From cc4ff19e34fae2c4ba944e235861b6cb800b7c86 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 15 Apr 2020 00:49:21 +0300 Subject: [PATCH 75/84] openapi: add application/x-www-form-urlencoded to body types Closes #1683 --- lib/pleroma/web/api_spec/helpers.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/api_spec/helpers.ex b/lib/pleroma/web/api_spec/helpers.ex index 35cf4c0d8..7348dcbee 100644 --- a/lib/pleroma/web/api_spec/helpers.ex +++ b/lib/pleroma/web/api_spec/helpers.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.ApiSpec.Helpers do def request_body(description, schema_ref, opts \\ []) do - media_types = ["application/json", "multipart/form-data"] + media_types = ["application/json", "multipart/form-data", "application/x-www-form-urlencoded"] content = media_types From 6bc76df287d7f4beb35c3a55b784b07ce9d833ff Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 15 Apr 2020 12:05:22 +0200 Subject: [PATCH 76/84] Uploads: Sandbox them in the CSP. --- lib/pleroma/plugs/uploaded_media.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex index 36ff024a7..94147e0c4 100644 --- a/lib/pleroma/plugs/uploaded_media.ex +++ b/lib/pleroma/plugs/uploaded_media.ex @@ -41,6 +41,7 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do conn -> conn end + |> merge_resp_headers([{"content-security-policy", "sandbox"}]) config = Pleroma.Config.get(Pleroma.Upload) From 22bde21c4f1a84a1fbe733070e8926366a3c01dc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 15 Apr 2020 15:27:34 +0300 Subject: [PATCH 77/84] remote_ip plug adds remote_ip_found flag --- .../plugs/rate_limiter/rate_limiter.ex | 17 ++-------- lib/pleroma/plugs/remote_ip.ex | 7 ++--- mix.exs | 2 +- mix.lock | 2 +- test/plugs/rate_limiter_test.exs | 31 ++++++++++++------- 5 files changed, 27 insertions(+), 32 deletions(-) diff --git a/lib/pleroma/plugs/rate_limiter/rate_limiter.ex b/lib/pleroma/plugs/rate_limiter/rate_limiter.ex index 1529da717..c51e2c634 100644 --- a/lib/pleroma/plugs/rate_limiter/rate_limiter.ex +++ b/lib/pleroma/plugs/rate_limiter/rate_limiter.ex @@ -110,20 +110,9 @@ defp handle(conn, action_settings) do end def disabled?(conn) do - localhost_or_socket = - case Config.get([Pleroma.Web.Endpoint, :http, :ip]) do - {127, 0, 0, 1} -> true - {0, 0, 0, 0, 0, 0, 0, 1} -> true - {:local, _} -> true - _ -> false - end - - remote_ip_not_found = - if Map.has_key?(conn.assigns, :remote_ip_found), - do: !conn.assigns.remote_ip_found, - else: false - - localhost_or_socket and remote_ip_not_found + if Map.has_key?(conn.assigns, :remote_ip_found), + do: !conn.assigns.remote_ip_found, + else: false end @inspect_bucket_not_found {:error, :not_found} diff --git a/lib/pleroma/plugs/remote_ip.ex b/lib/pleroma/plugs/remote_ip.ex index 0ac9050d0..2eca4f8f6 100644 --- a/lib/pleroma/plugs/remote_ip.ex +++ b/lib/pleroma/plugs/remote_ip.ex @@ -7,8 +7,6 @@ defmodule Pleroma.Plugs.RemoteIp do This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration. """ - import Plug.Conn - @behaviour Plug @headers ~w[ @@ -28,12 +26,11 @@ defmodule Pleroma.Plugs.RemoteIp do def init(_), do: nil - def call(%{remote_ip: original_remote_ip} = conn, _) do + def call(conn, _) do config = Pleroma.Config.get(__MODULE__, []) if Keyword.get(config, :enabled, false) do - %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config)) - assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip) + RemoteIp.call(conn, remote_ip_opts(config)) else conn end diff --git a/mix.exs b/mix.exs index c781995e0..c5e5fd432 100644 --- a/mix.exs +++ b/mix.exs @@ -183,7 +183,7 @@ defp deps do {:flake_id, "~> 0.1.0"}, {:remote_ip, git: "https://git.pleroma.social/pleroma/remote_ip.git", - ref: "825dc00aaba5a1b7c4202a532b696b595dd3bcb3"}, + ref: "b647d0deecaa3acb140854fe4bda5b7e1dc6d1c8"}, {:captcha, git: "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"}, diff --git a/mix.lock b/mix.lock index ba4e3ac44..2b9c54548 100644 --- a/mix.lock +++ b/mix.lock @@ -97,7 +97,7 @@ "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "d736bfa7444112eb840027bb887832a0e403a4a3437f48028c3b29a2dbbd2543"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"}, "recon": {:hex, :recon, "2.5.0", "2f7fcbec2c35034bade2f9717f77059dc54eb4e929a3049ca7ba6775c0bd66cd", [:mix, :rebar3], [], "hexpm", "72f3840fedd94f06315c523f6cecf5b4827233bed7ae3fe135b2a0ebeab5e196"}, - "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "825dc00aaba5a1b7c4202a532b696b595dd3bcb3", [ref: "825dc00aaba5a1b7c4202a532b696b595dd3bcb3"]}, + "remote_ip": {:git, "https://git.pleroma.social/pleroma/remote_ip.git", "b647d0deecaa3acb140854fe4bda5b7e1dc6d1c8", [ref: "b647d0deecaa3acb140854fe4bda5b7e1dc6d1c8"]}, "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm", "13104d7897e38ed7f044c4de953a6c28597d1c952075eb2e328bc6d6f2bfc496"}, "sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm", "2e1ec458f892ffa81f9f8386e3f35a1af6db7a7a37748a64478f13163a1f3573"}, diff --git a/test/plugs/rate_limiter_test.exs b/test/plugs/rate_limiter_test.exs index 0ce9f3a0a..4d3d694f4 100644 --- a/test/plugs/rate_limiter_test.exs +++ b/test/plugs/rate_limiter_test.exs @@ -5,8 +5,10 @@ defmodule Pleroma.Plugs.RateLimiterTest do use Pleroma.Web.ConnCase + alias Phoenix.ConnTest alias Pleroma.Config alias Pleroma.Plugs.RateLimiter + alias Plug.Conn import Pleroma.Factory import Pleroma.Tests.Helpers, only: [clear_config: 1, clear_config: 2] @@ -36,8 +38,15 @@ test "config is required for plug to work" do end test "it is disabled if it remote ip plug is enabled but no remote ip is found" do - Config.put([Pleroma.Web.Endpoint, :http, :ip], {127, 0, 0, 1}) - assert RateLimiter.disabled?(Plug.Conn.assign(build_conn(), :remote_ip_found, false)) + assert RateLimiter.disabled?(Conn.assign(build_conn(), :remote_ip_found, false)) + end + + test "it is enabled if remote ip found" do + refute RateLimiter.disabled?(Conn.assign(build_conn(), :remote_ip_found, true)) + end + + test "it is enabled if remote_ip_found flag doesn't exist" do + refute RateLimiter.disabled?(build_conn()) end test "it restricts based on config values" do @@ -58,7 +67,7 @@ test "it restricts based on config values" do end conn = RateLimiter.call(conn, plug_opts) - assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert %{"error" => "Throttled"} = ConnTest.json_response(conn, :too_many_requests) assert conn.halted Process.sleep(50) @@ -68,7 +77,7 @@ test "it restricts based on config values" do conn = RateLimiter.call(conn, plug_opts) assert {1, 4} = RateLimiter.inspect_bucket(conn, limiter_name, plug_opts) - refute conn.status == Plug.Conn.Status.code(:too_many_requests) + refute conn.status == Conn.Status.code(:too_many_requests) refute conn.resp_body refute conn.halted end @@ -98,7 +107,7 @@ test "`params` option allows different queries to be tracked independently" do plug_opts = RateLimiter.init(name: limiter_name, params: ["id"]) conn = build_conn(:get, "/?id=1") - conn = Plug.Conn.fetch_query_params(conn) + conn = Conn.fetch_query_params(conn) conn_2 = build_conn(:get, "/?id=2") RateLimiter.call(conn, plug_opts) @@ -119,7 +128,7 @@ test "it supports combination of options modifying bucket name" do id = "100" conn = build_conn(:get, "/?id=#{id}") - conn = Plug.Conn.fetch_query_params(conn) + conn = Conn.fetch_query_params(conn) conn_2 = build_conn(:get, "/?id=#{101}") RateLimiter.call(conn, plug_opts) @@ -147,13 +156,13 @@ test "are restricted based on remote IP" do conn = RateLimiter.call(conn, plug_opts) - assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert %{"error" => "Throttled"} = ConnTest.json_response(conn, :too_many_requests) assert conn.halted conn_2 = RateLimiter.call(conn_2, plug_opts) assert {1, 4} = RateLimiter.inspect_bucket(conn_2, limiter_name, plug_opts) - refute conn_2.status == Plug.Conn.Status.code(:too_many_requests) + refute conn_2.status == Conn.Status.code(:too_many_requests) refute conn_2.resp_body refute conn_2.halted end @@ -187,7 +196,7 @@ test "can have limits separate from unauthenticated connections" do conn = RateLimiter.call(conn, plug_opts) - assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert %{"error" => "Throttled"} = ConnTest.json_response(conn, :too_many_requests) assert conn.halted end @@ -210,12 +219,12 @@ test "different users are counted independently" do end conn = RateLimiter.call(conn, plug_opts) - assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert %{"error" => "Throttled"} = ConnTest.json_response(conn, :too_many_requests) assert conn.halted conn_2 = RateLimiter.call(conn_2, plug_opts) assert {1, 4} = RateLimiter.inspect_bucket(conn_2, limiter_name, plug_opts) - refute conn_2.status == Plug.Conn.Status.code(:too_many_requests) + refute conn_2.status == Conn.Status.code(:too_many_requests) refute conn_2.resp_body refute conn_2.halted end From aa0a4a1e78655024e992f9c677efed45593ab7b8 Mon Sep 17 00:00:00 2001 From: Ilja Date: Wed, 15 Apr 2020 19:03:27 +0200 Subject: [PATCH 78/84] small fix in the rewrite_policy example --- docs/configuration/mrf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/mrf.md b/docs/configuration/mrf.md index c3957c255..287416b2a 100644 --- a/docs/configuration/mrf.md +++ b/docs/configuration/mrf.md @@ -113,7 +113,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.RewritePolicy do @impl true def describe do - {:ok, %{mrf_sample: %{content: "new message content"}}}` + {:ok, %{mrf_sample: %{content: "new message content"}}} end end ``` From bde1189c349dc114aca2e9310dda840a1007825f Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 15 Apr 2020 21:19:16 +0300 Subject: [PATCH 79/84] [#2349] Made :skip_plug/2 prevent plug from being executed even if explicitly called. Refactoring. Tests. --- lib/pleroma/plugs/auth_expected_plug.ex | 4 ++ lib/pleroma/plugs/oauth_scopes_plug.ex | 6 +-- lib/pleroma/tests/oauth_test_controller.ex | 31 +++++++++++++ lib/pleroma/web/router.ex | 11 +++++ lib/pleroma/web/web.ex | 32 +++++++++++-- test/plugs/oauth_scopes_plug_test.exs | 13 ++++++ test/web/auth/oauth_test_controller_test.exs | 49 ++++++++++++++++++++ 7 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 lib/pleroma/tests/oauth_test_controller.ex create mode 100644 test/web/auth/oauth_test_controller_test.exs diff --git a/lib/pleroma/plugs/auth_expected_plug.ex b/lib/pleroma/plugs/auth_expected_plug.ex index 9e4a4bec8..f79597dc3 100644 --- a/lib/pleroma/plugs/auth_expected_plug.ex +++ b/lib/pleroma/plugs/auth_expected_plug.ex @@ -10,4 +10,8 @@ def init(options), do: options def call(conn, _) do put_private(conn, :auth_expected, true) end + + def auth_expected?(conn) do + conn.private[:auth_expected] + end end diff --git a/lib/pleroma/plugs/oauth_scopes_plug.ex b/lib/pleroma/plugs/oauth_scopes_plug.ex index b09e1bb4d..66f48c28c 100644 --- a/lib/pleroma/plugs/oauth_scopes_plug.ex +++ b/lib/pleroma/plugs/oauth_scopes_plug.ex @@ -10,13 +10,13 @@ defmodule Pleroma.Plugs.OAuthScopesPlug do alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug alias Pleroma.Plugs.PlugHelper + use Pleroma.Web, :plug + @behaviour Plug def init(%{scopes: _} = options), do: options - def call(%Plug.Conn{assigns: assigns} = conn, %{scopes: scopes} = options) do - conn = PlugHelper.append_to_called_plugs(conn, __MODULE__) - + def perform(%Plug.Conn{assigns: assigns} = conn, %{scopes: scopes} = options) do op = options[:op] || :| token = assigns[:token] diff --git a/lib/pleroma/tests/oauth_test_controller.ex b/lib/pleroma/tests/oauth_test_controller.ex new file mode 100644 index 000000000..58d517f78 --- /dev/null +++ b/lib/pleroma/tests/oauth_test_controller.ex @@ -0,0 +1,31 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +# A test controller reachable only in :test env. +# Serves to test OAuth scopes check skipping / enforcement. +defmodule Pleroma.Tests.OAuthTestController do + @moduledoc false + + use Pleroma.Web, :controller + + alias Pleroma.Plugs.OAuthScopesPlug + + plug(:skip_plug, OAuthScopesPlug when action == :skipped_oauth) + + plug(OAuthScopesPlug, %{scopes: ["read"]} when action != :missed_oauth) + + def skipped_oauth(conn, _params) do + noop(conn) + end + + def performed_oauth(conn, _params) do + noop(conn) + end + + def missed_oauth(conn, _params) do + noop(conn) + end + + defp noop(conn), do: json(conn, %{}) +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 8d13cd6c9..c85ad9f8b 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -672,6 +672,17 @@ defmodule Pleroma.Web.Router do end end + # Test-only routes needed to test action dispatching and plug chain execution + if Pleroma.Config.get(:env) == :test do + scope "/test/authenticated_api", Pleroma.Tests do + pipe_through(:authenticated_api) + + for action <- [:skipped_oauth, :performed_oauth, :missed_oauth] do + get("/#{action}", OAuthTestController, action) + end + end + end + scope "/", Pleroma.Web.MongooseIM do get("/user_exists", MongooseIMController, :user_exists) get("/check_password", MongooseIMController, :check_password) diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex index 1af29ce78..ae7c94640 100644 --- a/lib/pleroma/web/web.ex +++ b/lib/pleroma/web/web.ex @@ -37,15 +37,21 @@ defp set_put_layout(conn, _) do put_layout(conn, Pleroma.Config.get(:app_layout, "app.html")) end - # Marks a plug as intentionally skipped - # (states that the plug is not called for a good reason, not by a mistake) + # Marks a plug intentionally skipped and blocks its execution if it's present in plugs chain defp skip_plug(conn, plug_module) do + try do + plug_module.ensure_skippable() + rescue + UndefinedFunctionError -> + raise "#{plug_module} is not skippable. Append `use Pleroma.Web, :plug` to its code." + end + PlugHelper.append_to_skipped_plugs(conn, plug_module) end # Here we can apply before-action hooks (e.g. verify whether auth checks were preformed) defp action(conn, params) do - if conn.private[:auth_expected] && + if Pleroma.Plugs.AuthExpectedPlug.auth_expected?(conn) && not PlugHelper.plug_called_or_skipped?(conn, Pleroma.Plugs.OAuthScopesPlug) do conn |> render_error( @@ -119,6 +125,26 @@ def channel do end end + def plug do + quote do + alias Pleroma.Plugs.PlugHelper + + def ensure_skippable, do: :noop + + @impl Plug + @doc "If marked as skipped, returns `conn`, and calls `perform/2` otherwise." + def call(%Plug.Conn{} = conn, options) do + if PlugHelper.plug_skipped?(conn, __MODULE__) do + conn + else + conn + |> PlugHelper.append_to_called_plugs(__MODULE__) + |> perform(options) + end + end + end + end + @doc """ When used, dispatch to the appropriate controller/view/etc. """ diff --git a/test/plugs/oauth_scopes_plug_test.exs b/test/plugs/oauth_scopes_plug_test.exs index e79ecf263..abab7abb0 100644 --- a/test/plugs/oauth_scopes_plug_test.exs +++ b/test/plugs/oauth_scopes_plug_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Plugs.OAuthScopesPlugTest do alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.Plugs.PlugHelper alias Pleroma.Repo import Mock @@ -16,6 +17,18 @@ defmodule Pleroma.Plugs.OAuthScopesPlugTest do :ok end + test "is not performed if marked as skipped", %{conn: conn} do + with_mock OAuthScopesPlug, [:passthrough], perform: &passthrough([&1, &2]) do + conn = + conn + |> PlugHelper.append_to_skipped_plugs(OAuthScopesPlug) + |> OAuthScopesPlug.call(%{scopes: ["random_scope"]}) + + refute called(OAuthScopesPlug.perform(:_, :_)) + refute conn.halted + end + end + test "if `token.scopes` fulfills specified 'any of' conditions, " <> "proceeds with no op", %{conn: conn} do diff --git a/test/web/auth/oauth_test_controller_test.exs b/test/web/auth/oauth_test_controller_test.exs new file mode 100644 index 000000000..a2f6009ac --- /dev/null +++ b/test/web/auth/oauth_test_controller_test.exs @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Tests.OAuthTestControllerTest do + use Pleroma.Web.ConnCase + + import Pleroma.Factory + + setup %{conn: conn} do + user = insert(:user) + conn = assign(conn, :user, user) + %{conn: conn, user: user} + end + + test "missed_oauth", %{conn: conn} do + res = + conn + |> get("/test/authenticated_api/missed_oauth") + |> json_response(403) + + assert res == + %{ + "error" => + "Security violation: OAuth scopes check was neither handled nor explicitly skipped." + } + end + + test "skipped_oauth", %{conn: conn} do + conn + |> assign(:token, nil) + |> get("/test/authenticated_api/skipped_oauth") + |> json_response(200) + end + + test "performed_oauth", %{user: user} do + %{conn: good_token_conn} = oauth_access(["read"], user: user) + + good_token_conn + |> get("/test/authenticated_api/performed_oauth") + |> json_response(200) + + %{conn: bad_token_conn} = oauth_access(["follow"], user: user) + + bad_token_conn + |> get("/test/authenticated_api/performed_oauth") + |> json_response(403) + end +end From 72ef6cc4f2f601e26ba84c16ad2c91bd72867629 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 13 Apr 2020 14:07:23 +0300 Subject: [PATCH 80/84] added need_reboot endpoint to admin api --- CHANGELOG.md | 6 ++++ docs/API/admin_api.md | 21 ++++++++--- .../web/admin_api/admin_api_controller.ex | 35 +++++++------------ lib/pleroma/web/router.ex | 1 + .../admin_api/admin_api_controller_test.exs | 18 ++++++++-- 5 files changed, 52 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56b235f6d..804d3aa91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Logger configuration through AdminFE +### Added +
+ API Changes +- Admin API: `GET /api/pleroma/admin/need_reboot`. +
+ ## [2.0.2] - 2020-04-08 ### Added - Support for Funkwhale's `Audio` activity diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 57fb6bc6a..0ba88470a 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -786,6 +786,8 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret ### Restarts pleroma application +**Only works when configuration from database is enabled.** + - Params: none - Response: - On failure: @@ -795,11 +797,24 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret {} ``` +## `GET /api/pleroma/admin/need_reboot` + +### Returns the flag whether the pleroma should be restarted + +- Params: none +- Response: + - `need_reboot` - boolean +```json +{ + "need_reboot": false +} +``` + ## `GET /api/pleroma/admin/config` ### Get list of merged default settings with saved in database. -*If `need_reboot` flag exists in response, instance must be restarted, so reboot time settings can take effect.* +*If `need_reboot` is `true`, instance must be restarted, so reboot time settings can take effect.* **Only works when configuration from database is enabled.** @@ -821,13 +836,12 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret "need_reboot": true } ``` - need_reboot - *optional*, if were changed reboot time settings. ## `POST /api/pleroma/admin/config` ### Update config settings -*If `need_reboot` flag exists in response, instance must be restarted, so reboot time settings can take effect.* +*If `need_reboot` is `true`, instance must be restarted, so reboot time settings can take effect.* **Only works when configuration from database is enabled.** @@ -971,7 +985,6 @@ config :quack, "need_reboot": true } ``` -need_reboot - *optional*, if were changed reboot time settings. ## ` GET /api/pleroma/admin/config/descriptions` diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 831c3bd02..8de7d70a3 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -914,16 +914,7 @@ def config_show(conn, _params) do end) |> List.flatten() - response = %{configs: merged} - - response = - if Restarter.Pleroma.need_reboot?() do - Map.put(response, :need_reboot, true) - else - response - end - - json(conn, response) + json(conn, %{configs: merged, need_reboot: Restarter.Pleroma.need_reboot?()}) end end @@ -950,28 +941,22 @@ def config_update(conn, %{"configs" => configs}) do Config.TransferTask.load_and_update_env(deleted, false) - need_reboot? = - Restarter.Pleroma.need_reboot?() || - Enum.any?(updated, fn config -> + if !Restarter.Pleroma.need_reboot?() do + changed_reboot_settings? = + (updated ++ deleted) + |> Enum.any?(fn config -> group = ConfigDB.from_string(config.group) key = ConfigDB.from_string(config.key) value = ConfigDB.from_binary(config.value) Config.TransferTask.pleroma_need_restart?(group, key, value) end) - response = %{configs: updated} - - response = - if need_reboot? do - Restarter.Pleroma.need_reboot() - Map.put(response, :need_reboot, need_reboot?) - else - response - end + if changed_reboot_settings?, do: Restarter.Pleroma.need_reboot() + end conn |> put_view(ConfigView) - |> render("index.json", response) + |> render("index.json", %{configs: updated, need_reboot: Restarter.Pleroma.need_reboot?()}) end end @@ -983,6 +968,10 @@ def restart(conn, _params) do end end + def need_reboot(conn, _params) do + json(conn, %{need_reboot: Restarter.Pleroma.need_reboot?()}) + end + defp configurable_from_database(conn) do if Config.get(:configurable_from_database) do :ok diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 5f5ec1c81..fd94913a1 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -203,6 +203,7 @@ defmodule Pleroma.Web.Router do get("/config", AdminAPIController, :config_show) post("/config", AdminAPIController, :config_update) get("/config/descriptions", AdminAPIController, :config_descriptions) + get("/need_reboot", AdminAPIController, :need_reboot) get("/restart", AdminAPIController, :restart) get("/moderation_log", AdminAPIController, :list_log) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 60ec895f5..158966365 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -2110,7 +2110,7 @@ test "saving config which need pleroma reboot", %{conn: conn} do |> get("/api/pleroma/admin/config") |> json_response(200) - refute Map.has_key?(configs, "need_reboot") + assert configs["need_reboot"] == false end test "update setting which need reboot, don't change reboot flag until reboot", %{conn: conn} do @@ -2166,7 +2166,7 @@ test "update setting which need reboot, don't change reboot flag until reboot", |> get("/api/pleroma/admin/config") |> json_response(200) - refute Map.has_key?(configs, "need_reboot") + assert configs["need_reboot"] == false end test "saving config with nested merge", %{conn: conn} do @@ -2861,6 +2861,20 @@ test "pleroma restarts", %{conn: conn} do end end + test "need_reboot flag", %{conn: conn} do + assert conn + |> get("/api/pleroma/admin/need_reboot") + |> json_response(200) == %{"need_reboot" => false} + + Restarter.Pleroma.need_reboot() + + assert conn + |> get("/api/pleroma/admin/need_reboot") + |> json_response(200) == %{"need_reboot" => true} + + on_exit(fn -> Restarter.Pleroma.refresh() end) + end + describe "GET /api/pleroma/admin/statuses" do test "returns all public and unlisted statuses", %{conn: conn, admin: admin} do blocked = insert(:user) From 77ee64b9930bf6b439f87112fa35e302f5125aa2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 16 Apr 2020 17:54:57 +0300 Subject: [PATCH 81/84] user: remove blank? --- lib/pleroma/user.ex | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index fab405233..753b0c686 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -343,9 +343,15 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) + name = + case params[:name] do + name when is_binary(name) and byte_size(name) > 0 -> name + _ -> params[:nickname] + end + params = params - |> Map.put(:name, blank?(params[:name]) || params[:nickname]) + |> Map.put(:name, name) |> Map.put_new(:last_refreshed_at, NaiveDateTime.utc_now()) |> truncate_if_exists(:name, name_limit) |> truncate_if_exists(:bio, bio_limit) @@ -1599,9 +1605,6 @@ def get_public_key_for_ap_id(ap_id) do end end - defp blank?(""), do: nil - defp blank?(n), do: n - def ap_enabled?(%User{local: true}), do: true def ap_enabled?(%User{ap_enabled: ap_enabled}), do: ap_enabled def ap_enabled?(_), do: false From 4d330d9df13b7ff5d24fdd8b4eec1e111fa51297 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 16 Apr 2020 18:05:36 +0300 Subject: [PATCH 82/84] fix for use of published from different entities --- lib/pleroma/web/feed/feed_view.ex | 9 +++------ .../web/templates/feed/feed/_activity.atom.eex | 8 ++++---- .../web/templates/feed/feed/_activity.rss.eex | 8 ++++---- .../templates/feed/feed/_tag_activity.atom.eex | 16 ++++++++-------- .../templates/feed/feed/_tag_activity.xml.eex | 15 +++++++-------- test/web/feed/tag_controller_test.exs | 4 ++-- 6 files changed, 28 insertions(+), 32 deletions(-) diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index e18adaea8..1ae03e7e2 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -23,7 +23,7 @@ def pub_date(date) when is_binary(date) do def pub_date(%DateTime{} = date), do: Timex.format!(date, "{RFC822}") def prepare_activity(activity, opts \\ []) do - object = activity_object(activity) + object = Object.normalize(activity) actor = if opts[:actor] do @@ -33,7 +33,6 @@ def prepare_activity(activity, opts \\ []) do %{ activity: activity, data: Map.get(object, :data), - object: object, actor: actor } end @@ -68,9 +67,7 @@ def logo(user) do def last_activity(activities), do: List.last(activities) - def activity_object(activity), do: Object.normalize(activity) - - def activity_title(%{data: %{"content" => content}}, opts \\ %{}) do + def activity_title(%{"content" => content}, opts \\ %{}) do content |> Pleroma.Web.Metadata.Utils.scrub_html() |> Pleroma.Emoji.Formatter.demojify() @@ -78,7 +75,7 @@ def activity_title(%{data: %{"content" => content}}, opts \\ %{}) do |> escape() end - def activity_content(%{data: %{"content" => content}}) do + def activity_content(%{"content" => content}) do content |> String.replace(~r/[\n\r]/, "") |> escape() diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex index ac8a75009..78350f2aa 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -2,10 +2,10 @@ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post <%= @data["id"] %> - <%= activity_title(@object, Keyword.get(@feed_config, :post_title, %{})) %> - <%= activity_content(@object) %> - <%= @data["published"] %> - <%= @data["published"] %> + <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> + <%= activity_content(@data) %> + <%= @activity.data["published"] %> + <%= @activity.data["published"] %> <%= activity_context(@activity) %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex index a4dbed638..a304a16af 100644 --- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -2,10 +2,10 @@ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post <%= @data["id"] %> - <%= activity_title(@object, Keyword.get(@feed_config, :post_title, %{})) %> - <%= activity_content(@object) %> - <%= @data["published"] %> - <%= @data["published"] %> + <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> + <%= activity_content(@data) %> + <%= @activity.data["published"] %> + <%= @activity.data["published"] %> <%= activity_context(@activity) %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex index da4fa6d6c..cf5874a91 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -1,12 +1,12 @@ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - + <%= render @view_module, "_tag_author.atom", assigns %> - + <%= @data["id"] %> - <%= activity_title(@object, Keyword.get(@feed_config, :post_title, %{})) %> - <%= activity_content(@object) %> + <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> + <%= activity_content(@data) %> <%= if @activity.local do %> @@ -15,8 +15,8 @@ <% end %> - <%= @data["published"] %> - <%= @data["published"] %> + <%= @activity.data["published"] %> + <%= @activity.data["published"] %> <%= activity_context(@activity) %> @@ -26,7 +26,7 @@ <%= if @data["summary"] do %> <%= @data["summary"] %> <% end %> - + <%= for id <- @activity.recipients do %> <%= if id == Pleroma.Constants.as_public() do %> <% end %> <% end %> - + <%= for tag <- @data["tag"] || [] do %> <% end %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex index 295574df1..2334e24a2 100644 --- a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex +++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex @@ -1,15 +1,14 @@ - <%= activity_title(@object, Keyword.get(@feed_config, :post_title, %{})) %> - - + <%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %> + + <%= activity_context(@activity) %> <%= activity_context(@activity) %> - <%= pub_date(@data["published"]) %> - - <%= activity_content(@object) %> + <%= pub_date(@activity.data["published"]) %> + + <%= activity_content(@data) %> <%= for attachment <- @data["attachment"] || [] do %> <% end %> - - + diff --git a/test/web/feed/tag_controller_test.exs b/test/web/feed/tag_controller_test.exs index e863df86b..d95aac108 100644 --- a/test/web/feed/tag_controller_test.exs +++ b/test/web/feed/tag_controller_test.exs @@ -150,8 +150,8 @@ test "gets a feed (RSS)", %{conn: conn} do obj2 = Object.normalize(activity2) assert xpath(xml, ~x"//channel/item/description/text()"sl) == [ - HtmlEntities.decode(FeedView.activity_content(obj2)), - HtmlEntities.decode(FeedView.activity_content(obj1)) + HtmlEntities.decode(FeedView.activity_content(obj2.data)), + HtmlEntities.decode(FeedView.activity_content(obj1.data)) ] response = From 304ea09f4c9902a1f96f30541e6c5d253527dd47 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 17 Apr 2020 08:42:48 +0300 Subject: [PATCH 83/84] fix for logger configuration --- lib/pleroma/config/transfer_task.ex | 9 ++++++-- test/config/transfer_task_test.exs | 32 +++++++++++------------------ 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 3871e1cbb..f4722f99d 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -122,7 +122,7 @@ defp configure({_, :backends, _, merged}) do :ok = update_env(:logger, :backends, merged) end - defp configure({group, key, _, merged}) do + defp configure({_, key, _, merged}) when key in [:console, :ex_syslogger] do merged = if key == :console do put_in(merged[:format], merged[:format] <> "\n") @@ -136,7 +136,12 @@ defp configure({group, key, _, merged}) do else: key Logger.configure_backend(backend, merged) - :ok = update_env(:logger, group, merged) + :ok = update_env(:logger, key, merged) + end + + defp configure({_, key, _, merged}) do + Logger.configure([{key, merged}]) + :ok = update_env(:logger, key, merged) end defp update({group, key, value, merged}) do diff --git a/test/config/transfer_task_test.exs b/test/config/transfer_task_test.exs index 0265a6156..00db0b686 100644 --- a/test/config/transfer_task_test.exs +++ b/test/config/transfer_task_test.exs @@ -16,6 +16,7 @@ test "transfer config values from db to env" do refute Application.get_env(:pleroma, :test_key) refute Application.get_env(:idna, :test_key) refute Application.get_env(:quack, :test_key) + initial = Application.get_env(:logger, :level) ConfigDB.create(%{ group: ":pleroma", @@ -35,16 +36,20 @@ test "transfer config values from db to env" do value: [:test_value1, :test_value2] }) + ConfigDB.create(%{group: ":logger", key: ":level", value: :debug}) + TransferTask.start_link([]) assert Application.get_env(:pleroma, :test_key) == [live: 2, com: 3] assert Application.get_env(:idna, :test_key) == [live: 15, com: 35] assert Application.get_env(:quack, :test_key) == [:test_value1, :test_value2] + assert Application.get_env(:logger, :level) == :debug on_exit(fn -> Application.delete_env(:pleroma, :test_key) Application.delete_env(:idna, :test_key) Application.delete_env(:quack, :test_key) + Application.put_env(:logger, :level, initial) end) end @@ -78,8 +83,8 @@ test "transfer config values for 1 group and some keys" do end test "transfer config values with full subkey update" do - emoji = Application.get_env(:pleroma, :emoji) - assets = Application.get_env(:pleroma, :assets) + clear_config(:emoji) + clear_config(:assets) ConfigDB.create(%{ group: ":pleroma", @@ -99,11 +104,6 @@ test "transfer config values with full subkey update" do assert emoji_env[:groups] == [a: 1, b: 2] assets_env = Application.get_env(:pleroma, :assets) assert assets_env[:mascots] == [a: 1, b: 2] - - on_exit(fn -> - Application.put_env(:pleroma, :emoji, emoji) - Application.put_env(:pleroma, :assets, assets) - end) end describe "pleroma restart" do @@ -112,8 +112,7 @@ test "transfer config values with full subkey update" do end test "don't restart if no reboot time settings were changed" do - emoji = Application.get_env(:pleroma, :emoji) - on_exit(fn -> Application.put_env(:pleroma, :emoji, emoji) end) + clear_config(:emoji) ConfigDB.create(%{ group: ":pleroma", @@ -128,8 +127,7 @@ test "don't restart if no reboot time settings were changed" do end test "on reboot time key" do - chat = Application.get_env(:pleroma, :chat) - on_exit(fn -> Application.put_env(:pleroma, :chat, chat) end) + clear_config(:chat) ConfigDB.create(%{ group: ":pleroma", @@ -141,8 +139,7 @@ test "on reboot time key" do end test "on reboot time subkey" do - captcha = Application.get_env(:pleroma, Pleroma.Captcha) - on_exit(fn -> Application.put_env(:pleroma, Pleroma.Captcha, captcha) end) + clear_config(Pleroma.Captcha) ConfigDB.create(%{ group: ":pleroma", @@ -154,13 +151,8 @@ test "on reboot time subkey" do end test "don't restart pleroma on reboot time key and subkey if there is false flag" do - chat = Application.get_env(:pleroma, :chat) - captcha = Application.get_env(:pleroma, Pleroma.Captcha) - - on_exit(fn -> - Application.put_env(:pleroma, :chat, chat) - Application.put_env(:pleroma, Pleroma.Captcha, captcha) - end) + clear_config(:chat) + clear_config(Pleroma.Captcha) ConfigDB.create(%{ group: ":pleroma", From 4d22b100b777b59e79180d5d3ea8615db940b1fc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 17 Apr 2020 12:33:11 +0300 Subject: [PATCH 84/84] move changelogs entries to unreleased section --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce6737408..2239a5288 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - NodeInfo: `pleroma_emoji_reactions` to the `features` list. - Configuration: `:restrict_unauthenticated` setting, restrict access for unauthenticated users to timelines (public and federate), user profiles and statuses. - New HTTP adapter [gun](https://github.com/ninenines/gun). Gun adapter requires minimum OTP version of 22.2 otherwise Pleroma won’t start. For hackney OTP update is not required. +- Mix task to create trusted OAuth App.
API Changes - Mastodon API: Support for `include_types` in `/api/v1/notifications`. - Mastodon API: Added `/api/v1/notifications/:id/dismiss` endpoint. +- Admin API: endpoints for create/update/delete OAuth Apps.
### Fixed @@ -155,7 +157,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Add an option `authorized_fetch_mode` to require HTTP signatures for AP fetches. - ActivityPub: support for `replies` collection (output for outgoing federation & fetching on incoming federation). - Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`) -- Mix task to create trusted OAuth App.
API Changes @@ -202,7 +203,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - ActivityPub: `[:activitypub, :note_replies_output_limit]` setting sets the number of note self-replies to output on outgoing federation. - Admin API: `GET /api/pleroma/admin/stats` to get status count by visibility scope - Admin API: `GET /api/pleroma/admin/statuses` - list all statuses (accepts `godmode` and `local_only`) -- Admin API: endpoints for create/update/delete OAuth Apps.
### Fixed