moon
/
honkoma
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'feature/hide-follows-remote' into 'develop' 

Refactor Follows/Followers counter syncronization and set hide_followers/hide_follows for remote users

See merge request pleroma/pleroma!1411
This commit is contained in:
lain 2019-08-02 11:23:07 +00:00
parent 1fe092e03c 301ea0dc04
commit 5ff8f07ca9
12 changed files with 327 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
config/test.exs
View File

@ -29,7 +29,8 @@
email: "admin@example.com",
notify_email: "noreply@example.com",
skip_thread_containment: false,
federating: false
federating: false,
external_user_synchronization: false
config :pleroma, :activitypub, sign_object_fetches: false

7
lib/pleroma/object/fetcher.ex
View File

@ -114,7 +114,7 @@ defp maybe_date_fetch(headers, date) do
end
end
def fetch_and_contain_remote_object_from_id(id) do
def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
Logger.info("Fetching object #{id} via AP")
date =
@ -141,4 +141,9 @@ def fetch_and_contain_remote_object_from_id(id) do
{:error, e}
end
end
def fetch_and_contain_remote_object_from_id(%{"id" => id}),
do: fetch_and_contain_remote_object_from_id(id)
def fetch_and_contain_remote_object_from_id(_id), do: {:error, "id must be a string"}
end

93
lib/pleroma/user.ex
View File

@ -114,7 +114,9 @@ def ap_following(%User{} = user), do: "#{ap_id(user)}/following"
def user_info(%User{} = user, args \\ %{}) do
following_count =
if args[:following_count], do: args[:following_count], else: following_count(user)
if args[:following_count],
do: args[:following_count],
else: user.info.following_count || following_count(user)
follower_count =
if args[:follower_count], do: args[:follower_count], else: user.info.follower_count
@ -406,6 +408,8 @@ def follow(%User{} = follower, %User{info: info} = followed) do
{1, [follower]} = Repo.update_all(q, [])
follower = maybe_update_following_count(follower)
{:ok, _} = update_follower_count(followed)
set_cache(follower)
@ -425,6 +429,8 @@ def unfollow(%User{} = follower, %User{} = followed) do
{1, [follower]} = Repo.update_all(q, [])
follower = maybe_update_following_count(follower)
{:ok, followed} = update_follower_count(followed)
set_cache(follower)
@ -709,32 +715,73 @@ def update_note_count(%User{} = user) do
|> update_and_set_cache()
end
def update_follower_count(%User{} = user) do
follower_count_query =
User.Query.build(%{followers: user, deactivated: false})
|> select([u], %{count: count(u.id)})
def maybe_fetch_follow_information(user) do
with {:ok, user} <- fetch_follow_information(user) do
user
else
e ->
Logger.error("Follower/Following counter update for #{user.ap_id} failed.\n#{inspect(e)}")
User
|> where(id: ^user.id)
|> join(:inner, [u], s in subquery(follower_count_query))
|> update([u, s],
set: [
info:
fragment(
"jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)",
u.info,
s.count
)
]
)
|> select([u], u)
|> Repo.update_all([])
|> case do
{1, [user]} -> set_cache(user)
_ -> {:error, user}
user
end
end
def fetch_follow_information(user) do
with {:ok, info} <- ActivityPub.fetch_follow_information_for_user(user) do
info_cng = User.Info.follow_information_update(user.info, info)
changeset =
user
|> change()
|> put_embed(:info, info_cng)
update_and_set_cache(changeset)
else
{:error, _} = e -> e
e -> {:error, e}
end
end
def update_follower_count(%User{} = user) do
if user.local or !Pleroma.Config.get([:instance, :external_user_synchronization]) do
follower_count_query =
User.Query.build(%{followers: user, deactivated: false})
|> select([u], %{count: count(u.id)})
User
|> where(id: ^user.id)
|> join(:inner, [u], s in subquery(follower_count_query))
|> update([u, s],
set: [
info:
fragment(
"jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)",
u.info,
s.count
)
]
)
|> select([u], u)
|> Repo.update_all([])
|> case do
{1, [user]} -> set_cache(user)
_ -> {:error, user}
end
else
{:ok, maybe_fetch_follow_information(user)}
end
end
def maybe_update_following_count(%User{local: false} = user) do
if Pleroma.Config.get([:instance, :external_user_synchronization]) do
{:ok, maybe_fetch_follow_information(user)}
else
user
end
end
def maybe_update_following_count(user), do: user
def remove_duplicated_following(%User{following: following} = user) do
uniq_following = Enum.uniq(following)

24
lib/pleroma/user/info.ex
View File

@ -16,6 +16,8 @@ defmodule Pleroma.User.Info do
field(:source_data, :map, default: %{})
field(:note_count, :integer, default: 0)
field(:follower_count, :integer, default: 0)
# Should be filled in only for remote users
field(:following_count, :integer, default: nil)
field(:locked, :boolean, default: false)
field(:confirmation_pending, :boolean, default: false)
field(:confirmation_token, :string, default: nil)
@ -223,7 +225,11 @@ def remote_user_creation(info, params) do
:uri,
:hub,
:topic,
:salmon
:salmon,
:hide_followers,
:hide_follows,
:follower_count,
:following_count
])
end
@ -234,7 +240,11 @@ def user_upgrade(info, params) do
:source_data,
:banner,
:locked,
:magic_key
:magic_key,
:follower_count,
:following_count,
:hide_follows,
:hide_followers
])
end
@ -348,4 +358,14 @@ def remove_reblog_mute(info, ap_id) do
cast(info, params, [:muted_reblogs])
end
def follow_information_update(info, params) do
info
|> cast(params, [
:hide_followers,
:hide_follows,
:follower_count,
:following_count
])
end
end

76
lib/pleroma/web/activity_pub/activity_pub.ex
View File

@ -1009,10 +1009,10 @@ defp object_to_user_data(data) do
user_data = %{
ap_id: data["id"],
info: %{
"ap_enabled" => true,
"source_data" => data,
"banner" => banner,
"locked" => locked
ap_enabled: true,
source_data: data,
banner: banner,
locked: locked
},
avatar: avatar,
name: data["name"],
@ -1036,6 +1036,71 @@ defp object_to_user_data(data) do
{:ok, user_data}
end
def fetch_follow_information_for_user(user) do
with {:ok, following_data} <-
Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
following_count when is_integer(following_count) <- following_data["totalItems"],
{:ok, hide_follows} <- collection_private(following_data),
{:ok, followers_data} <-
Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address),
followers_count when is_integer(followers_count) <- followers_data["totalItems"],
{:ok, hide_followers} <- collection_private(followers_data) do
{:ok,
%{
hide_follows: hide_follows,
follower_count: followers_count,
following_count: following_count,
hide_followers: hide_followers
}}
else
{:error, _} = e ->
e
e ->
{:error, e}
end
end
defp maybe_update_follow_information(data) do
with {:enabled, true} <-
{:enabled, Pleroma.Config.get([:instance, :external_user_synchronization])},
{:ok, info} <- fetch_follow_information_for_user(data) do
info = Map.merge(data.info, info)
Map.put(data, :info, info)
else
{:enabled, false} ->
data
e ->
Logger.error(
"Follower/Following counter update for #{data.ap_id} failed.\n" <> inspect(e)
)
data
end
end
defp collection_private(data) do
if is_map(data["first"]) and
data["first"]["type"] in ["CollectionPage", "OrderedCollectionPage"] do
{:ok, false}
else
with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <-
Fetcher.fetch_and_contain_remote_object_from_id(data["first"]) do
{:ok, false}
else
{:error, {:ok, %{status: code}}} when code in [401, 403] ->
{:ok, true}
{:error, _} = e ->
e
e ->
{:error, e}
end
end
end
def user_data_from_user_object(data) do
with {:ok, data} <- MRF.filter(data),
{:ok, data} <- object_to_user_data(data) do
@ -1047,7 +1112,8 @@ def user_data_from_user_object(data) do
def fetch_and_prepare_user_from_ap_id(ap_id) do
with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
{:ok, data} <- user_data_from_user_object(data) do
{:ok, data} <- user_data_from_user_object(data),
data <- maybe_update_follow_information(data) do
{:ok, data}
else
e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")

33
lib/pleroma/web/activity_pub/transmogrifier.ex
View File

@ -608,13 +608,13 @@ def handle_incoming(
with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do
{:ok, new_user_data} = ActivityPub.user_data_from_user_object(object)
banner = new_user_data[:info]["banner"]
locked = new_user_data[:info]["locked"] || false
banner = new_user_data[:info][:banner]
locked = new_user_data[:info][:locked] || false
update_data =
new_user_data
|> Map.take([:name, :bio, :avatar])
|> Map.put(:info, %{"banner" => banner, "locked" => locked})
|> Map.put(:info, %{banner: banner, locked: locked})
actor
|> User.upgrade_changeset(update_data)
@ -1076,10 +1076,6 @@ def upgrade_user_from_ap_id(ap_id) do
PleromaJobQueue.enqueue(:transmogrifier, __MODULE__, [:user_upgrade, user])
end
if Pleroma.Config.get([:instance, :external_user_synchronization]) do
update_following_followers_counters(user)
end
{:ok, user}
else
%User{} = user -> {:ok, user}
@ -1112,27 +1108,4 @@ def maybe_fix_user_object(data) do
data
|> maybe_fix_user_url
end
def update_following_followers_counters(user) do
info = %{}
following = fetch_counter(user.following_address)
info = if following, do: Map.put(info, :following_count, following), else: info
followers = fetch_counter(user.follower_address)
info = if followers, do: Map.put(info, :follower_count, followers), else: info
User.set_info_cache(user, info)
end
defp fetch_counter(url) do
with {:ok, %{body: body, status: code}} when code in 200..299 <-
Pleroma.HTTP.get(
url,
[{:Accept, "application/activity+json"}]
),
{:ok, data} <- Jason.decode(body) do
data["totalItems"]
end
end
end

1
test/fixtures/users_mock/masto_closed_followers_page.json vendored Normal file
View File

@ -0,0 +1 @@
{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:4001/users/masto_closed/followers?page=1","type":"OrderedCollectionPage","totalItems":437,"next":"http://localhost:4001/users/masto_closed/followers?page=2","partOf":"http://localhost:4001/users/masto_closed/followers","orderedItems":["https://testing.uguu.ltd/users/rin","https://patch.cx/users/rin","https://letsalllovela.in/users/xoxo","https://pleroma.site/users/crushv","https://aria.company/users/boris","https://kawen.space/users/crushv","https://freespeech.host/users/cvcvcv","https://pleroma.site/users/picpub","https://pixelfed.social/users/nosleep","https://boopsnoot.gq/users/5c1896d162f7d337f90492a3","https://pikachu.rocks/users/waifu","https://royal.crablettesare.life/users/crablettes"]}

1
test/fixtures/users_mock/masto_closed_following_page.json vendored Normal file
View File

@ -0,0 +1 @@
{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:4001/users/masto_closed/following?page=1","type":"OrderedCollectionPage","totalItems":152,"next":"http://localhost:4001/users/masto_closed/following?page=2","partOf":"http://localhost:4001/users/masto_closed/following","orderedItems":["https://testing.uguu.ltd/users/rin","https://patch.cx/users/rin","https://letsalllovela.in/users/xoxo","https://pleroma.site/users/crushv","https://aria.company/users/boris","https://kawen.space/users/crushv","https://freespeech.host/users/cvcvcv","https://pleroma.site/users/picpub","https://pixelfed.social/users/nosleep","https://boopsnoot.gq/users/5c1896d162f7d337f90492a3","https://pikachu.rocks/users/waifu","https://royal.crablettesare.life/users/crablettes"]}

16
test/support/http_request_mock.ex
View File

@ -796,6 +796,14 @@ def get("http://localhost:4001/users/masto_closed/followers", _, _, _) do
}}
end
def get("http://localhost:4001/users/masto_closed/followers?page=1", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/users_mock/masto_closed_followers_page.json")
}}
end
def get("http://localhost:4001/users/masto_closed/following", _, _, _) do
{:ok,
%Tesla.Env{
@ -804,6 +812,14 @@ def get("http://localhost:4001/users/masto_closed/following", _, _, _) do
}}
end
def get("http://localhost:4001/users/masto_closed/following?page=1", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/users_mock/masto_closed_following_page.json")
}}
end
def get("http://localhost:4001/users/fuser2/followers", _, _, _) do
{:ok,
%Tesla.Env{

74
test/user_test.exs
View File

@ -1393,4 +1393,78 @@ test "performs update cache if user updated" do
assert %User{bio: "test-bio"} = User.get_cached_by_ap_id(user.ap_id)
end
end
describe "following/followers synchronization" do
setup do
sync = Pleroma.Config.get([:instance, :external_user_synchronization])
on_exit(fn -> Pleroma.Config.put([:instance, :external_user_synchronization], sync) end)
end
test "updates the counters normally on following/getting a follow when disabled" do
Pleroma.Config.put([:instance, :external_user_synchronization], false)
user = insert(:user)
other_user =
insert(:user,
local: false,
follower_address: "http://localhost:4001/users/masto_closed/followers",
following_address: "http://localhost:4001/users/masto_closed/following",
info: %{ap_enabled: true}
)
assert User.user_info(other_user).following_count == 0
assert User.user_info(other_user).follower_count == 0
{:ok, user} = Pleroma.User.follow(user, other_user)
other_user = Pleroma.User.get_by_id(other_user.id)
assert User.user_info(user).following_count == 1
assert User.user_info(other_user).follower_count == 1
end
test "syncronizes the counters with the remote instance for the followed when enabled" do
Pleroma.Config.put([:instance, :external_user_synchronization], false)
user = insert(:user)
other_user =
insert(:user,
local: false,
follower_address: "http://localhost:4001/users/masto_closed/followers",
following_address: "http://localhost:4001/users/masto_closed/following",
info: %{ap_enabled: true}
)
assert User.user_info(other_user).following_count == 0
assert User.user_info(other_user).follower_count == 0
Pleroma.Config.put([:instance, :external_user_synchronization], true)
{:ok, _user} = User.follow(user, other_user)
other_user = User.get_by_id(other_user.id)
assert User.user_info(other_user).follower_count == 437
end
test "syncronizes the counters with the remote instance for the follower when enabled" do
Pleroma.Config.put([:instance, :external_user_synchronization], false)
user = insert(:user)
other_user =
insert(:user,
local: false,
follower_address: "http://localhost:4001/users/masto_closed/followers",
following_address: "http://localhost:4001/users/masto_closed/following",
info: %{ap_enabled: true}
)
assert User.user_info(other_user).following_count == 0
assert User.user_info(other_user).follower_count == 0
Pleroma.Config.put([:instance, :external_user_synchronization], true)
{:ok, other_user} = User.follow(other_user, user)
assert User.user_info(other_user).following_count == 152
end
end
end

61
test/web/activity_pub/activity_pub_test.exs
View File

@ -1128,4 +1128,65 @@ test "fetches only public posts for other users" do
assert result.id == activity.id
end
end
describe "fetch_follow_information_for_user" do
test "syncronizes following/followers counters" do
user =
insert(:user,
local: false,
follower_address: "http://localhost:4001/users/fuser2/followers",
following_address: "http://localhost:4001/users/fuser2/following"
)
{:ok, info} = ActivityPub.fetch_follow_information_for_user(user)
assert info.follower_count == 527
assert info.following_count == 267
end
test "detects hidden followers" do
mock(fn env ->
case env.url do
"http://localhost:4001/users/masto_closed/followers?page=1" ->
%Tesla.Env{status: 403, body: ""}
_ ->
apply(HttpRequestMock, :request, [env])
end
end)
user =
insert(:user,
local: false,
follower_address: "http://localhost:4001/users/masto_closed/followers",
following_address: "http://localhost:4001/users/masto_closed/following"
)
{:ok, info} = ActivityPub.fetch_follow_information_for_user(user)
assert info.hide_followers == true
assert info.hide_follows == false
end
test "detects hidden follows" do
mock(fn env ->
case env.url do
"http://localhost:4001/users/masto_closed/following?page=1" ->
%Tesla.Env{status: 403, body: ""}
_ ->
apply(HttpRequestMock, :request, [env])
end
end)
user =
insert(:user,
local: false,
follower_address: "http://localhost:4001/users/masto_closed/followers",
following_address: "http://localhost:4001/users/masto_closed/following"
)
{:ok, info} = ActivityPub.fetch_follow_information_for_user(user)
assert info.hide_followers == false
assert info.hide_follows == true
end
end
end

28
test/web/activity_pub/transmogrifier_test.exs
View File

@ -1373,32 +1373,4 @@ test "removes recipient's follower collection from cc", %{user: user} do
refute recipient.follower_address in fixed_object["to"]
end
end
test "update_following_followers_counters/1" do
user1 =
insert(:user,
local: false,
follower_address: "http://localhost:4001/users/masto_closed/followers",
following_address: "http://localhost:4001/users/masto_closed/following"
)
user2 =
insert(:user,
local: false,
follower_address: "http://localhost:4001/users/fuser2/followers",
following_address: "http://localhost:4001/users/fuser2/following"
)
Transmogrifier.update_following_followers_counters(user1)
Transmogrifier.update_following_followers_counters(user2)
%{follower_count: followers, following_count: following} = User.get_cached_user_info(user1)
assert followers == 437
assert following == 152
%{follower_count: followers, following_count: following} = User.get_cached_user_info(user2)
assert followers == 527
assert following == 267
end
end