moon
/
spc-pleroma
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'feature/admin-api-status-count-per-instance' into 'develop' 

admin api status count per instance

See merge request pleroma/pleroma!2500
This commit is contained in:
lain 2020-06-24 10:42:07 +00:00
parent 510031ece2 1471b70ef1
commit 88ccade1cb
10 changed files with 324 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
CHANGELOG.md
View File

@ -15,9 +15,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
<details>
<summary>API Changes</summary>
- **Breaking:** Emoji API: changed methods and renamed routes.
</details>
<details>
<summary>Admin API Changes</summary>
- Status visibility stats: now can return stats per instance.
- Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`)
</details>
### Removed
- **Breaking:** removed `with_move` parameter from notifications timeline.
@ -98,6 +107,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
2. Run database migrations (inside Pleroma directory):
- OTP: `./bin/pleroma_ctl migrate`
- From Source: `mix ecto.migrate`
3. Reset status visibility counters (inside Pleroma directory):
- OTP: `./bin/pleroma_ctl refresh_counter_cache`
- From Source: `mix pleroma.refresh_counter_cache`
## [2.0.2] - 2020-04-08

4
docs/API/admin_api.md
View File

@ -1118,6 +1118,10 @@ Loads json generated from `config/descriptions.exs`.
### Stats
- Query Params:
- *optional* `instance`: **string** instance hostname (without protocol) to get stats for
- Example: `https://mypleroma.org/api/pleroma/admin/stats?instance=lain.com`
- Response:
```json

49
lib/mix/tasks/pleroma/refresh_counter_cache.ex
View File

@ -17,30 +17,53 @@ defmodule Mix.Tasks.Pleroma.RefreshCounterCache do
def run([]) do
Mix.Pleroma.start_pleroma()
["public", "unlisted", "private", "direct"]
|> Enum.each(fn visibility ->
count = status_visibility_count_query(visibility)
name = "status_visibility_#{visibility}"
CounterCache.set(name, count)
Mix.Pleroma.shell_info("Set #{name} to #{count}")
instances =
Activity
|> distinct([a], true)
|> select([a], fragment("split_part(?, '/', 3)", a.actor))
|> Repo.all()
instances
|> Enum.with_index(1)
|> Enum.each(fn {instance, i} ->
counters = instance_counters(instance)
CounterCache.set(instance, counters)
Mix.Pleroma.shell_info(
"[#{i}/#{length(instances)}] Setting #{instance} counters: #{inspect(counters)}"
)
end)
Mix.Pleroma.shell_info("Done")
end
defp status_visibility_count_query(visibility) do
defp instance_counters(instance) do
counters = %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0}
Activity
|> where(
|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
|> where([a], fragment("split_part(?, '/', 3) = ?", a.actor, ^instance))
|> select(
[a],
{fragment(
"activity_visibility(?, ?, ?)",
a.actor,
a.recipients,
a.data
), count(a.id)}
)
|> group_by(
[a],
fragment(
"activity_visibility(?, ?, ?) = ?",
"activity_visibility(?, ?, ?)",
a.actor,
a.recipients,
a.data,
^visibility
a.data
)
)
|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
|> Repo.aggregate(:count, :id, timeout: :timer.minutes(30))
|> Repo.all(timeout: :timer.minutes(30))
|> Enum.reduce(counters, fn {visibility, count}, acc ->
Map.put(acc, visibility, count)
end)
end
end

66
lib/pleroma/counter_cache.ex
View File

@ -10,32 +10,70 @@ defmodule Pleroma.CounterCache do
import Ecto.Query
schema "counter_cache" do
field(:name, :string)
field(:count, :integer)
field(:instance, :string)
field(:public, :integer)
field(:unlisted, :integer)
field(:private, :integer)
field(:direct, :integer)
end
def changeset(struct, params) do
struct
|> cast(params, [:name, :count])
|> validate_required([:name])
|> unique_constraint(:name)
|> cast(params, [:instance, :public, :unlisted, :private, :direct])
|> validate_required([:instance])
|> unique_constraint(:instance)
end
def get_as_map(names) when is_list(names) do
def get_by_instance(instance) do
CounterCache
|> where([cc], cc.name in ^names)
|> Repo.all()
|> Enum.group_by(& &1.name, & &1.count)
|> Map.new(fn {k, v} -> {k, hd(v)} end)
|> select([c], %{
"public" => c.public,
"unlisted" => c.unlisted,
"private" => c.private,
"direct" => c.direct
})
|> where([c], c.instance == ^instance)
|> Repo.one()
|> case do
nil -> %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0}
val -> val
end
end
def set(name, count) do
def get_sum do
CounterCache
|> select([c], %{
"public" => type(sum(c.public), :integer),
"unlisted" => type(sum(c.unlisted), :integer),
"private" => type(sum(c.private), :integer),
"direct" => type(sum(c.direct), :integer)
})
|> Repo.one()
end
def set(instance, values) do
params =
Enum.reduce(
["public", "private", "unlisted", "direct"],
%{"instance" => instance},
fn param, acc ->
Map.put_new(acc, param, Map.get(values, param, 0))
end
)
%CounterCache{}
|> changeset(%{"name" => name, "count" => count})
|> changeset(params)
|> Repo.insert(
on_conflict: [set: [count: count]],
on_conflict: [
set: [
public: params["public"],
private: params["private"],
unlisted: params["unlisted"],
direct: params["direct"]
]
],
returning: true,
conflict_target: :name
conflict_target: :instance
)
end
end

21
lib/pleroma/stats.ex
View File

@ -97,20 +97,11 @@ def calculate_stat_data do
}
end
def get_status_visibility_count do
counter_cache =
CounterCache.get_as_map([
"status_visibility_public",
"status_visibility_private",
"status_visibility_unlisted",
"status_visibility_direct"
])
%{
public: counter_cache["status_visibility_public"] || 0,
unlisted: counter_cache["status_visibility_unlisted"] || 0,
private: counter_cache["status_visibility_private"] || 0,
direct: counter_cache["status_visibility_direct"] || 0
}
def get_status_visibility_count(instance \\ nil) do
if is_nil(instance) do
CounterCache.get_sum()
else
CounterCache.get_by_instance(instance)
end
end
end

6
lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
View File

@ -643,10 +643,10 @@ def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" =
json(conn, "")
end
def stats(conn, _) do
count = Stats.get_status_visibility_count()
def stats(conn, params) do
counters = Stats.get_status_visibility_count(params["instance"])
json(conn, %{"status_visibility" => count})
json(conn, %{"status_visibility" => counters})
end
defp page_params(params) do

143
priv/repo/migrations/20200508092434_update_counter_cache_table.exs Normal file
View File

@ -0,0 +1,143 @@
defmodule Pleroma.Repo.Migrations.UpdateCounterCacheTable do
use Ecto.Migration
@function_name "update_status_visibility_counter_cache"
@trigger_name "status_visibility_counter_cache_trigger"
def up do
execute("drop trigger if exists #{@trigger_name} on activities")
execute("drop function if exists #{@function_name}()")
drop_if_exists(unique_index(:counter_cache, [:name]))
drop_if_exists(table(:counter_cache))
create_if_not_exists table(:counter_cache) do
add(:instance, :string, null: false)
add(:direct, :bigint, null: false, default: 0)
add(:private, :bigint, null: false, default: 0)
add(:unlisted, :bigint, null: false, default: 0)
add(:public, :bigint, null: false, default: 0)
end
create_if_not_exists(unique_index(:counter_cache, [:instance]))
"""
CREATE OR REPLACE FUNCTION #{@function_name}()
RETURNS TRIGGER AS
$$
DECLARE
hostname character varying(255);
visibility_new character varying(64);
visibility_old character varying(64);
actor character varying(255);
BEGIN
IF TG_OP = 'DELETE' THEN
actor := OLD.actor;
ELSE
actor := NEW.actor;
END IF;
hostname := split_part(actor, '/', 3);
IF TG_OP = 'INSERT' THEN
visibility_new := activity_visibility(NEW.actor, NEW.recipients, NEW.data);
IF NEW.data->>'type' = 'Create'
AND visibility_new IN ('public', 'unlisted', 'private', 'direct') THEN
EXECUTE format('INSERT INTO "counter_cache" ("instance", %1$I) VALUES ($1, 1)
ON CONFLICT ("instance") DO
UPDATE SET %1$I = "counter_cache".%1$I + 1', visibility_new)
USING hostname;
END IF;
RETURN NEW;
ELSIF TG_OP = 'UPDATE' THEN
visibility_new := activity_visibility(NEW.actor, NEW.recipients, NEW.data);
visibility_old := activity_visibility(OLD.actor, OLD.recipients, OLD.data);
IF (NEW.data->>'type' = 'Create')
AND (OLD.data->>'type' = 'Create')
AND visibility_new != visibility_old
AND visibility_new IN ('public', 'unlisted', 'private', 'direct') THEN
EXECUTE format('UPDATE "counter_cache" SET
%1$I = greatest("counter_cache".%1$I - 1, 0),
%2$I = "counter_cache".%2$I + 1
WHERE "instance" = $1', visibility_old, visibility_new)
USING hostname;
END IF;
RETURN NEW;
ELSIF TG_OP = 'DELETE' THEN
IF OLD.data->>'type' = 'Create' THEN
visibility_old := activity_visibility(OLD.actor, OLD.recipients, OLD.data);
EXECUTE format('UPDATE "counter_cache" SET
%1$I = greatest("counter_cache".%1$I - 1, 0)
WHERE "instance" = $1', visibility_old)
USING hostname;
END IF;
RETURN OLD;
END IF;
END;
$$
LANGUAGE 'plpgsql';
"""
|> execute()
execute("DROP TRIGGER IF EXISTS #{@trigger_name} ON activities")
"""
CREATE TRIGGER #{@trigger_name}
BEFORE
INSERT
OR UPDATE of recipients, data
OR DELETE
ON activities
FOR EACH ROW
EXECUTE PROCEDURE #{@function_name}();
"""
|> execute()
end
def down do
execute("DROP TRIGGER IF EXISTS #{@trigger_name} ON activities")
execute("DROP FUNCTION IF EXISTS #{@function_name}()")
drop_if_exists(unique_index(:counter_cache, [:instance]))
drop_if_exists(table(:counter_cache))
create_if_not_exists table(:counter_cache) do
add(:name, :string, null: false)
add(:count, :bigint, null: false, default: 0)
end
create_if_not_exists(unique_index(:counter_cache, [:name]))
"""
CREATE OR REPLACE FUNCTION #{@function_name}()
RETURNS TRIGGER AS
$$
DECLARE
BEGIN
IF TG_OP = 'INSERT' THEN
IF NEW.data->>'type' = 'Create' THEN
EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1';
END IF;
RETURN NEW;
ELSIF TG_OP = 'UPDATE' THEN
IF (NEW.data->>'type' = 'Create') and (OLD.data->>'type' = 'Create') and activity_visibility(NEW.actor, NEW.recipients, NEW.data) != activity_visibility(OLD.actor, OLD.recipients, OLD.data) THEN
EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1';
EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';';
END IF;
RETURN NEW;
ELSIF TG_OP = 'DELETE' THEN
IF OLD.data->>'type' = 'Create' THEN
EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';';
END IF;
RETURN OLD;
END IF;
END;
$$
LANGUAGE 'plpgsql';
"""
|> execute()
"""
CREATE TRIGGER #{@trigger_name} BEFORE INSERT OR UPDATE of recipients, data OR DELETE ON activities
FOR EACH ROW
EXECUTE PROCEDURE #{@function_name}();
"""
|> execute()
end
end

55
test/stats_test.exs
View File

@ -17,10 +17,11 @@ test "it ignores internal users" do
end
end
describe "status visibility count" do
describe "status visibility sum count" do
test "on new status" do
instance2 = "instance2.tld"
user = insert(:user)
other_user = insert(:user)
other_user = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
CommonAPI.post(user, %{visibility: "public", status: "hey"})
@ -45,24 +46,24 @@ test "on new status" do
})
end)
assert %{direct: 3, private: 4, public: 1, unlisted: 2} =
assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} =
Pleroma.Stats.get_status_visibility_count()
end
test "on status delete" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
assert %{public: 1} = Pleroma.Stats.get_status_visibility_count()
assert %{"public" => 1} = Pleroma.Stats.get_status_visibility_count()
CommonAPI.delete(activity.id, user)
assert %{public: 0} = Pleroma.Stats.get_status_visibility_count()
assert %{"public" => 0} = Pleroma.Stats.get_status_visibility_count()
end
test "on status visibility update" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
assert %{public: 1, private: 0} = Pleroma.Stats.get_status_visibility_count()
assert %{"public" => 1, "private" => 0} = Pleroma.Stats.get_status_visibility_count()
{:ok, _} = CommonAPI.update_activity_scope(activity.id, %{visibility: "private"})
assert %{public: 0, private: 1} = Pleroma.Stats.get_status_visibility_count()
assert %{"public" => 0, "private" => 1} = Pleroma.Stats.get_status_visibility_count()
end
test "doesn't count unrelated activities" do
@ -73,8 +74,46 @@ test "doesn't count unrelated activities" do
CommonAPI.favorite(other_user, activity.id)
CommonAPI.repeat(activity.id, other_user)
assert %{direct: 0, private: 0, public: 1, unlisted: 0} =
assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 0} =
Pleroma.Stats.get_status_visibility_count()
end
end
describe "status visibility by instance count" do
test "single instance" do
local_instance = Pleroma.Web.Endpoint.url() |> String.split("//") |> Enum.at(1)
instance2 = "instance2.tld"
user1 = insert(:user)
user2 = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
CommonAPI.post(user1, %{visibility: "public", status: "hey"})
Enum.each(1..5, fn _ ->
CommonAPI.post(user1, %{
visibility: "unlisted",
status: "hey"
})
end)
Enum.each(1..10, fn _ ->
CommonAPI.post(user1, %{
visibility: "direct",
status: "hey @#{user2.nickname}"
})
end)
Enum.each(1..20, fn _ ->
CommonAPI.post(user2, %{
visibility: "private",
status: "hey"
})
end)
assert %{"direct" => 10, "private" => 0, "public" => 1, "unlisted" => 5} =
Pleroma.Stats.get_status_visibility_count(local_instance)
assert %{"direct" => 0, "private" => 20, "public" => 0, "unlisted" => 0} =
Pleroma.Stats.get_status_visibility_count(instance2)
end
end
end

2
test/tasks/refresh_counter_cache_test.exs
View File

@ -37,7 +37,7 @@ test "counts statuses" do
assert capture_io(fn -> Mix.Tasks.Pleroma.RefreshCounterCache.run([]) end) =~ "Done\n"
assert %{direct: 3, private: 4, public: 1, unlisted: 2} =
assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} =
Pleroma.Stats.get_status_visibility_count()
end
end

20
test/web/admin_api/controllers/admin_api_controller_test.exs
View File

@ -1732,6 +1732,26 @@ test "status visibility count", %{conn: conn} do
assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 2} =
response["status_visibility"]
end
test "by instance", %{conn: conn} do
admin = insert(:user, is_admin: true)
user1 = insert(:user)
instance2 = "instance2.tld"
user2 = insert(:user, %{ap_id: "https://#{instance2}/@actor"})
CommonAPI.post(user1, %{visibility: "public", status: "hey"})
CommonAPI.post(user2, %{visibility: "unlisted", status: "hey"})
CommonAPI.post(user2, %{visibility: "private", status: "hey"})
response =
conn
|> assign(:user, admin)
|> get("/api/pleroma/admin/stats", instance: instance2)
|> json_response(200)
assert %{"direct" => 0, "private" => 1, "public" => 0, "unlisted" => 1} =
response["status_visibility"]
end
end
end