From 48ae3c4347f68e20db7e3e67da32be2e70599fb3 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 5 Dec 2019 20:18:25 +0700 Subject: [PATCH 01/32] Add support for custom modules --- CHANGELOG.md | 1 + config/config.exs | 1 + docs/configuration/cheatsheet.md | 2 ++ lib/pleroma/application.ex | 24 ++++++++++++++++++++++++ 4 files changed, 28 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a06ea211e..6564cf40a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mix task to list all users (`mix pleroma.user list`) - Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache). - MRF: New module which handles incoming posts based on their age. By default, all incoming posts that are older than 2 days will be unlisted and not shown to their followers. +- Support for custom Elixir modules (such as MRF policies)
API Changes diff --git a/config/config.exs b/config/config.exs index b60ffef7d..e1358eda0 100644 --- a/config/config.exs +++ b/config/config.exs @@ -249,6 +249,7 @@ quarantined_instances: [], managed_config: true, static_dir: "instance/static/", + custom_modules_dir: "instance/modules/", allowed_post_formats: [ "text/plain", "text/html", diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index dc2f55229..f73d368c1 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -68,6 +68,8 @@ You shouldn't edit the base config directly to avoid breakages and merge conflic * `account_field_name_length`: An account field name maximum length (default: `512`). * `account_field_value_length`: An account field value maximum length (default: `2048`). * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. +* `custom_modules_dir`: A path to custom Elixir modules (such as MRF policies). + !!! danger This is a Work In Progress, not usable just yet diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9dbd1e26b..5b6e233a6 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -32,6 +32,7 @@ def user_agent do def start(_type, _args) do Pleroma.Config.DeprecationWarnings.warn() setup_instrumenters() + load_custom_modules() # Define workers and child supervisors to be supervised children = @@ -67,6 +68,29 @@ def start(_type, _args) do Supervisor.start_link(children, opts) end + def load_custom_modules() do + dir = Pleroma.Config.get([:instance, :custom_modules_dir]) + + if dir && File.exists?(dir) do + dir + |> File.ls!() + |> Enum.map(&Path.join(dir, &1)) + |> Kernel.ParallelCompiler.compile() + |> case do + {:error, _errors, _warnings} -> + raise "Invalid custom modules" + + {:ok, modules, _warnings} -> + Enum.each(modules, fn mod -> + name = mod |> Atom.to_string() |> String.trim_leading("Elixir.") + IO.puts("Custom module loaded: #{name}") + end) + + :ok + end + end + end + defp setup_instrumenters do require Prometheus.Registry From 1216b546c6bc0540e266fe0f05829f4f683b1ce9 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 5 Dec 2019 20:29:17 +0700 Subject: [PATCH 02/32] Fix credo warning --- lib/pleroma/application.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 5b6e233a6..73364f141 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -68,7 +68,7 @@ def start(_type, _args) do Supervisor.start_link(children, opts) end - def load_custom_modules() do + def load_custom_modules do dir = Pleroma.Config.get([:instance, :custom_modules_dir]) if dir && File.exists?(dir) do From 157bceeda9124cea7ba69eaf6639ca52b3fac7c6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 6 Dec 2019 15:04:46 +0700 Subject: [PATCH 03/32] Move runtime configuration from `:instance` to `:modules` --- config/config.exs | 3 ++- config/releases.exs | 1 + docs/configuration/cheatsheet.md | 12 ++++++++++-- lib/pleroma/application.ex | 2 +- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/config/config.exs b/config/config.exs index e1358eda0..64e33c82f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -249,7 +249,6 @@ quarantined_instances: [], managed_config: true, static_dir: "instance/static/", - custom_modules_dir: "instance/modules/", allowed_post_formats: [ "text/plain", "text/html", @@ -618,6 +617,8 @@ activity_pub: nil, activity_pub_question: 30_000 +config :pleroma, :modules, runtime_dir: "instance/modules" + config :swarm, node_blacklist: [~r/myhtml_.*$/] # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/config/releases.exs b/config/releases.exs index 98c5ceccd..b224960db 100644 --- a/config/releases.exs +++ b/config/releases.exs @@ -2,6 +2,7 @@ config :pleroma, :instance, static_dir: "/var/lib/pleroma/static" config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads" +config :pleroma, :modules, runtime_dir: "/var/lib/pleroma/modules" config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index f73d368c1..413a668c6 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -68,8 +68,6 @@ You shouldn't edit the base config directly to avoid breakages and merge conflic * `account_field_name_length`: An account field name maximum length (default: `512`). * `account_field_value_length`: An account field value maximum length (default: `2048`). * `external_user_synchronization`: Enabling following/followers counters synchronization for external users. -* `custom_modules_dir`: A path to custom Elixir modules (such as MRF policies). - !!! danger This is a Work In Progress, not usable just yet @@ -831,3 +829,13 @@ config :auto_linker, rel: "ugc" ] ``` + +## Custom Runtime Modules (`:modules`) + +* `runtime_dir`: A path to custom Elixir modules (such as MRF policies). + +Example: + +```elixir +config :pleroma, :modules, runtime_dir: "/var/lib/pleroma/modules" +``` diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 73364f141..9d2f3f320 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -69,7 +69,7 @@ def start(_type, _args) do end def load_custom_modules do - dir = Pleroma.Config.get([:instance, :custom_modules_dir]) + dir = Pleroma.Config.get([:modules, :runtime_dir]) if dir && File.exists?(dir) do dir From e4292cbfad47e59c76461fa201bab3e5f791962b Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 6 Dec 2019 15:16:39 +0700 Subject: [PATCH 04/32] Use Kernel.inspect/2 to print loaded custom modules --- lib/pleroma/application.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9d2f3f320..17f6b9c80 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -82,8 +82,7 @@ def load_custom_modules do {:ok, modules, _warnings} -> Enum.each(modules, fn mod -> - name = mod |> Atom.to_string() |> String.trim_leading("Elixir.") - IO.puts("Custom module loaded: #{name}") + IO.puts("Custom module loaded: #{inspect(mod)}") end) :ok From a75d4a41e03979b4d1b9af5205e457d714ff76df Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 6 Dec 2019 17:05:09 +0700 Subject: [PATCH 05/32] Add a test for custom runtime modules --- config/test.exs | 2 ++ lib/pleroma/application.ex | 8 +++++--- test/fixtures/modules/runtime_module.ex | 9 +++++++++ test/runtime_test.exs | 11 +++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 test/fixtures/modules/runtime_module.ex create mode 100644 test/runtime_test.exs diff --git a/config/test.exs b/config/test.exs index 9b737d4d7..8b9bf5c77 100644 --- a/config/test.exs +++ b/config/test.exs @@ -93,6 +93,8 @@ config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock +config :pleroma, :modules, runtime_dir: "test/fixtures/modules" + if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 17f6b9c80..82a005700 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -81,9 +81,11 @@ def load_custom_modules do raise "Invalid custom modules" {:ok, modules, _warnings} -> - Enum.each(modules, fn mod -> - IO.puts("Custom module loaded: #{inspect(mod)}") - end) + if @env != :test do + Enum.each(modules, fn mod -> + IO.puts("Custom module loaded: #{inspect(mod)}") + end) + end :ok end diff --git a/test/fixtures/modules/runtime_module.ex b/test/fixtures/modules/runtime_module.ex new file mode 100644 index 000000000..4711c3532 --- /dev/null +++ b/test/fixtures/modules/runtime_module.ex @@ -0,0 +1,9 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule RuntimeModule do + @moduledoc """ + This is a dummy module to test custom runtime modules. + """ +end diff --git a/test/runtime_test.exs b/test/runtime_test.exs new file mode 100644 index 000000000..f7b6f23d4 --- /dev/null +++ b/test/runtime_test.exs @@ -0,0 +1,11 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.RuntimeTest do + use ExUnit.Case, async: true + + test "it loads custom runtime modules" do + assert Code.ensure_compiled?(RuntimeModule) + end +end From 84f891ea3e31c936bc990a3c2310d539df62fc44 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 9 Dec 2019 18:23:07 +0700 Subject: [PATCH 06/32] Add Pleroma.Utils.compile_dir/1 --- lib/pleroma/application.ex | 4 +--- lib/pleroma/utils.ex | 12 ++++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 lib/pleroma/utils.ex diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 82a005700..104620b37 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -73,9 +73,7 @@ def load_custom_modules do if dir && File.exists?(dir) do dir - |> File.ls!() - |> Enum.map(&Path.join(dir, &1)) - |> Kernel.ParallelCompiler.compile() + |> Pleroma.Utils.compile_dir() |> case do {:error, _errors, _warnings} -> raise "Invalid custom modules" diff --git a/lib/pleroma/utils.ex b/lib/pleroma/utils.ex new file mode 100644 index 000000000..8d36a0001 --- /dev/null +++ b/lib/pleroma/utils.ex @@ -0,0 +1,12 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Utils do + def compile_dir(dir) when is_binary(dir) do + dir + |> File.ls!() + |> Enum.map(&Path.join(dir, &1)) + |> Kernel.ParallelCompiler.compile() + end +end From ed92784e7cfe60756733f518efce14253b1c78d6 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 9 Dec 2019 19:11:54 +0700 Subject: [PATCH 07/32] Set Logger level to :info in prod --- config/prod.exs | 4 ++-- lib/pleroma/application.ex | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/prod.exs b/config/prod.exs index 25873f360..adbce5606 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -20,8 +20,8 @@ config :phoenix, serve_endpoints: true # Do not print debug messages in production -config :logger, :console, level: :warn -config :logger, :ex_syslogger, level: :warn +config :logger, :console, level: :info +config :logger, :ex_syslogger, level: :info # ## SSL Support # diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 104620b37..f47cb0ce9 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Application do import Cachex.Spec use Application + require Logger @name Mix.Project.config()[:name] @version Mix.Project.config()[:version] @@ -81,7 +82,7 @@ def load_custom_modules do {:ok, modules, _warnings} -> if @env != :test do Enum.each(modules, fn mod -> - IO.puts("Custom module loaded: #{inspect(mod)}") + Logger.info("Custom module loaded: #{inspect(mod)}") end) end From 78299ab18205b0bbaf521640e188a862ca27aa61 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 9 Dec 2019 19:12:24 +0700 Subject: [PATCH 08/32] Set Plug.Logger to log at `:debug` level --- lib/pleroma/web/endpoint.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 49735b5c2..5fcce7ca2 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Web.Endpoint do plug(Pleroma.Plugs.TrailingFormatPlug) plug(Plug.RequestId) - plug(Plug.Logger) + plug(Plug.Logger, log: :debug) plug( Plug.Parsers, From b7a57d8e388a03d7d92248aa8c583365bde9d0b1 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 10 Dec 2019 00:38:01 +0700 Subject: [PATCH 09/32] Use Pleroma.Utils.compile_dir/1 in Pleroma.HTML.compile_scrubbers/0 --- lib/pleroma/html.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 2cae29f35..11513106e 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -10,9 +10,7 @@ def compile_scrubbers do dir = Path.join(:code.priv_dir(:pleroma), "scrubbers") dir - |> File.ls!() - |> Enum.map(&Path.join(dir, &1)) - |> Kernel.ParallelCompiler.compile() + |> Pleroma.Utils.compile_dir() |> case do {:error, _errors, _warnings} -> raise "Compiling scrubbers failed" From a37bd5c25587528b9f7a8ac1d148f6a4eb171769 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 10 Dec 2019 15:08:57 +0700 Subject: [PATCH 10/32] Change log level --- lib/pleroma/object/fetcher.ex | 2 +- lib/pleroma/web/activity_pub/publisher.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 4d71c91a8..a1bde90f1 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -154,7 +154,7 @@ defp maybe_date_fetch(headers, date) do end def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do - Logger.info("Fetching object #{id} via AP") + Logger.debug("Fetching object #{id} via AP") date = Pleroma.Signature.signed_date() diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 4ea37fc7b..e834f43ad 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -47,7 +47,7 @@ def is_representable?(%Activity{} = activity) do * `id`: the ActivityStreams URI of the message """ def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do - Logger.info("Federating #{id} to #{inbox}") + Logger.debug("Federating #{id} to #{inbox}") %{host: host, path: path} = URI.parse(inbox) digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64()) From ee6805850c8a86105b7f16d0510cf8465ba24452 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 11 Dec 2019 17:46:07 +0700 Subject: [PATCH 11/32] Set log level to debug for not important messages --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 6 +++--- lib/pleroma/web/activity_pub/mrf/drop_policy.ex | 2 +- .../web/activity_pub/mrf/mediaproxy_warming_policy.ex | 2 +- lib/pleroma/web/activity_pub/publisher.ex | 2 +- lib/pleroma/web/federator/federator.ex | 8 ++++---- lib/pleroma/web/federator/publisher.ex | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index dec5da0d3..5059e3984 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -257,7 +257,7 @@ def inbox(%{assigns: %{valid_signature: true}} = conn, params) do # only accept relayed Creates def inbox(conn, %{"type" => "Create"} = params) do - Logger.info( + Logger.debug( "Signature missing or not from author, relayed Create message, fetching object from source" ) @@ -270,11 +270,11 @@ def inbox(conn, params) do headers = Enum.into(conn.req_headers, %{}) if String.contains?(headers["signature"], params["actor"]) do - Logger.info( + Logger.debug( "Signature validation error for: #{params["actor"]}, make sure you are forwarding the HTTP Host header!" ) - Logger.info(inspect(conn.req_headers)) + Logger.debug(inspect(conn.req_headers)) end json(conn, dgettext("errors", "error")) diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index f7831bc3e..4a5709974 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do @impl true def filter(object) do - Logger.info("REJECTING #{inspect(object)}") + Logger.debug("REJECTING #{inspect(object)}") {:reject, object} end diff --git a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex index 26b8539fe..df774b0f7 100644 --- a/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mediaproxy_warming_policy.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do ] def perform(:prefetch, url) do - Logger.info("Prefetching #{inspect(url)}") + Logger.debug("Prefetching #{inspect(url)}") url |> MediaProxy.url() diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index e834f43ad..aeaddff64 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -223,7 +223,7 @@ def publish(%User{} = actor, %Activity{} = activity) do public = is_public?(activity) if public && Config.get([:instance, :allow_relay]) do - Logger.info(fn -> "Relaying #{activity.data["id"]} out" end) + Logger.debug(fn -> "Relaying #{activity.data["id"]} out" end) Relay.publish(activity) end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index e8a56ebd7..f506a7d24 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -58,7 +58,7 @@ def perform(:publish, activity) do end def perform(:incoming_ap_doc, params) do - Logger.info("Handling incoming AP activity") + Logger.debug("Handling incoming AP activity") params = Utils.normalize_params(params) @@ -71,13 +71,13 @@ def perform(:incoming_ap_doc, params) do {:ok, activity} else %Activity{} -> - Logger.info("Already had #{params["id"]}") + Logger.debug("Already had #{params["id"]}") :error _e -> # Just drop those for now - Logger.info("Unhandled activity") - Logger.info(Jason.encode!(params, pretty: true)) + Logger.debug("Unhandled activity") + Logger.debug(Jason.encode!(params, pretty: true)) :error end end diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index fb9b26649..1d045c644 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -47,7 +47,7 @@ def publish(%User{} = user, %Activity{} = activity) do Config.get([:instance, :federation_publisher_modules]) |> Enum.each(fn module -> if module.is_representable?(activity) do - Logger.info("Publishing #{activity.data["id"]} using #{inspect(module)}") + Logger.debug("Publishing #{activity.data["id"]} using #{inspect(module)}") module.publish(user, activity) end end) From 1a6e30d32ef17e5791de404e5cebb37844264dcb Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 11 Dec 2019 22:32:53 +0700 Subject: [PATCH 12/32] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20b8de887..e2249f897 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Deprecated `User.Info` embedded schema (fields moved to `User`) - Store status data inside Flag activity - Deprecated (reorganized as `UserRelationship` entity) User fields with user AP IDs (`blocks`, `mutes`, `muted_reblogs`, `muted_notifications`, `subscribers`). +- Logger: default log level changed from `warn` to `info`.
API Changes From c6f2735ffa1db7871bcb56c00b6d19e4de346d18 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 12 Dec 2019 14:37:57 +0700 Subject: [PATCH 13/32] Remove runtime modules config example --- docs/configuration/cheatsheet.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 743c188bb..b3a13833c 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -834,9 +834,3 @@ config :auto_linker, ## Custom Runtime Modules (`:modules`) * `runtime_dir`: A path to custom Elixir modules (such as MRF policies). - -Example: - -```elixir -config :pleroma, :modules, runtime_dir: "/var/lib/pleroma/modules" -``` From c8046e1080e613766812e89b1ed28af45580cc33 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 16 Dec 2019 00:48:49 +0300 Subject: [PATCH 14/32] tests: Replace 2 second sleep with offseting updated_at --- test/conversation/participation_test.exs | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 9b2c97963..88dc32198 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -5,7 +5,9 @@ defmodule Pleroma.Conversation.ParticipationTest do use Pleroma.DataCase import Pleroma.Factory + alias Pleroma.Conversation alias Pleroma.Conversation.Participation + alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -97,8 +99,10 @@ test "it creates a participation for a conversation and a user" do assert participation.user_id == user.id assert participation.conversation_id == conversation.id - + + # Needed because updated_at is accurate down to a second :timer.sleep(1000) + # Creating again returns the same participation {:ok, %Participation{} = participation_two} = Participation.create_for_user_and_conversation(user, conversation) @@ -150,9 +154,7 @@ test "it marks all the user's participations as read" do test "gets all the participations for a user, ordered by updated at descending" do user = insert(:user) {:ok, activity_one} = CommonAPI.post(user, %{"status" => "x", "visibility" => "direct"}) - :timer.sleep(1000) {:ok, activity_two} = CommonAPI.post(user, %{"status" => "x", "visibility" => "direct"}) - :timer.sleep(1000) {:ok, activity_three} = CommonAPI.post(user, %{ @@ -161,6 +163,17 @@ test "gets all the participations for a user, ordered by updated at descending" "in_reply_to_status_id" => activity_one.id }) + # Offset participations because the accuracy of updated_at is down to a second + + for {activity, offset} <- [{activity_two, 1}, {activity_three, 2}] do + conversation = Conversation.get_for_ap_id(activity.data["context"]) + participation = Participation.for_user_and_conversation(user, conversation) + updated_at = NaiveDateTime.add(Map.get(participation, :updated_at), offset) + + Ecto.Changeset.change(participation, %{updated_at: updated_at}) + |> Repo.update!() + end + assert [participation_one, participation_two] = Participation.for_user(user) object2 = Pleroma.Object.normalize(activity_two) From 814a99c80f11835ce3fd5d75c38f418662430b04 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 16 Dec 2019 00:50:39 +0300 Subject: [PATCH 15/32] tests: remove a useless 3 second sleep the ids favs use are accurate down to a microsecond, there is no need for it --- test/web/activity_pub/activity_pub_test.exs | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index ad1fb6d02..1520c8a9b 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -1639,13 +1639,10 @@ test "returns a favourite activities sorted by adds to favorite" do {:ok, _, _} = CommonAPI.favorite(a4.id, user) {:ok, _, _} = CommonAPI.favorite(a3.id, other_user) - Process.sleep(1000) {:ok, _, _} = CommonAPI.favorite(a3.id, user) {:ok, _, _} = CommonAPI.favorite(a5.id, other_user) - Process.sleep(1000) {:ok, _, _} = CommonAPI.favorite(a5.id, user) {:ok, _, _} = CommonAPI.favorite(a4.id, other_user) - Process.sleep(1000) {:ok, _, _} = CommonAPI.favorite(a1.id, user) {:ok, _, _} = CommonAPI.favorite(a1.id, other_user) result = ActivityPub.fetch_favourites(user) From 137a3d220bba01e9cacd8007fa030c69830cc9aa Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 16 Dec 2019 00:51:59 +0300 Subject: [PATCH 16/32] tests: remove a useless 1 second sleep there is no need to wait for the task to finish, it will be deleted when the transaction is aborted anyway. --- test/web/twitter_api/util_controller_test.exs | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 734cd2211..43299e147 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -898,8 +898,6 @@ test "with credentials and valid password", %{conn: conn, user: current_user} do |> post("/api/pleroma/delete_account", %{"password" => "test"}) assert json_response(conn, 200) == %{"status" => "success"} - # Wait a second for the started task to end - :timer.sleep(1000) end end end From 54029fe21271e7d4396699ee297f2910cd6fbbdc Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 16 Dec 2019 01:03:13 +0300 Subject: [PATCH 17/32] tests: remove a useless sleep in rate limiter tests It was used to check that authenticated and unauthenticated users have different limits. Instead of sleeping a super low limit for unauthenticated users was set, preventing them from doing 5 requests in the first place. --- test/conversation/participation_test.exs | 2 +- test/plugs/rate_limiter_test.exs | 14 ++------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 88dc32198..ba81c0d4b 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -99,7 +99,7 @@ test "it creates a participation for a conversation and a user" do assert participation.user_id == user.id assert participation.conversation_id == conversation.id - + # Needed because updated_at is accurate down to a second :timer.sleep(1000) diff --git a/test/plugs/rate_limiter_test.exs b/test/plugs/rate_limiter_test.exs index 49f63c424..78f1ea9e4 100644 --- a/test/plugs/rate_limiter_test.exs +++ b/test/plugs/rate_limiter_test.exs @@ -145,9 +145,9 @@ test "are restricted based on remote IP" do test "can have limits seperate from unauthenticated connections" do limiter_name = :test_authenticated - scale = 1000 + scale = 50 limit = 5 - Pleroma.Config.put([:rate_limit, limiter_name], [{1, 10}, {scale, limit}]) + Pleroma.Config.put([:rate_limit, limiter_name], [{1000, 1}, {scale, limit}]) opts = RateLimiter.init(name: limiter_name) @@ -164,16 +164,6 @@ test "can have limits seperate from unauthenticated connections" do assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) assert conn.halted - - Process.sleep(1550) - - conn = conn(:get, "/") |> assign(:user, user) - conn = RateLimiter.call(conn, opts) - assert {1, 4} = RateLimiter.inspect_bucket(conn, limiter_name, opts) - - refute conn.status == Plug.Conn.Status.code(:too_many_requests) - refute conn.resp_body - refute conn.halted end test "diffrerent users are counted independently" do From 6ffbfdeeb2fc5ce7c2bf3a58950f5bf1f9a31e6f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 16 Dec 2019 01:22:01 +0300 Subject: [PATCH 18/32] tests: significantly reduce streamer timeout there is no reason IPC between two processes on the same node should take 4 seconds --- test/web/streamer/streamer_test.exs | 42 ++++++++++++++++------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs index 8911c46b1..b07a7373f 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -16,6 +16,10 @@ defmodule Pleroma.Web.StreamerTest do alias Pleroma.Web.Streamer.Worker @moduletag needs_streamer: true, capture_log: true + + @streamer_timeout 150 + @streamer_start_wait 10 + clear_config_all([:instance, :skip_thread_containment]) describe "user streams" do @@ -28,7 +32,7 @@ defmodule Pleroma.Web.StreamerTest do test "it sends notify to in the 'user' stream", %{user: user, notify: notify} do task = Task.async(fn -> - assert_receive {:text, _}, 4_000 + assert_receive {:text, _}, @streamer_timeout end) Streamer.add_socket( @@ -43,7 +47,7 @@ test "it sends notify to in the 'user' stream", %{user: user, notify: notify} do test "it sends notify to in the 'user:notification' stream", %{user: user, notify: notify} do task = Task.async(fn -> - assert_receive {:text, _}, 4_000 + assert_receive {:text, _}, @streamer_timeout end) Streamer.add_socket( @@ -61,7 +65,7 @@ test "it doesn't send notify to the 'user:notification' stream when a user is bl blocked = insert(:user) {:ok, _user_relationship} = User.block(user, blocked) - task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) + task = Task.async(fn -> refute_receive {:text, _}, @streamer_timeout end) Streamer.add_socket( "user:notification", @@ -79,7 +83,7 @@ test "it doesn't send notify to the 'user:notification' stream when a thread is user: user } do user2 = insert(:user) - task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) + task = Task.async(fn -> refute_receive {:text, _}, @streamer_timeout end) Streamer.add_socket( "user:notification", @@ -97,7 +101,7 @@ test "it doesn't send notify to the 'user:notification' stream' when a domain is user: user } do user2 = insert(:user, %{ap_id: "https://hecking-lewd-place.com/user/meanie"}) - task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) + task = Task.async(fn -> refute_receive {:text, _}, @streamer_timeout end) Streamer.add_socket( "user:notification", @@ -116,7 +120,9 @@ test "it sends follow activities to the 'user:notification' stream", %{ user: user } do user2 = insert(:user) - task = Task.async(fn -> assert_receive {:text, _}, 4_000 end) + task = Task.async(fn -> assert_receive {:text, _}, @streamer_timeout end) + + Process.sleep(@streamer_start_wait) Streamer.add_socket( "user:notification", @@ -137,7 +143,7 @@ test "it sends to public" do task = Task.async(fn -> - assert_receive {:text, _}, 4_000 + assert_receive {:text, _}, @streamer_timeout end) fake_socket = %StreamerSocket{ @@ -164,7 +170,7 @@ test "it sends to public" do } |> Jason.encode!() - assert_receive {:text, received_event}, 4_000 + assert_receive {:text, received_event}, @streamer_timeout assert received_event == expected_event end) @@ -458,9 +464,7 @@ test "it doesn't send posts from muted threads" do {:ok, activity} = CommonAPI.add_mute(user2, activity) - task = Task.async(fn -> refute_receive {:text, _}, 4_000 end) - - Process.sleep(4000) + task = Task.async(fn -> refute_receive {:text, _}, @streamer_timeout end) Streamer.add_socket( "user", @@ -482,7 +486,7 @@ test "it sends conversation update to the 'direct' stream", %{} do task = Task.async(fn -> - assert_receive {:text, received_event}, 4_000 + assert_receive {:text, received_event}, @streamer_timeout assert %{"event" => "conversation", "payload" => received_payload} = Jason.decode!(received_event) @@ -518,13 +522,13 @@ test "it doesn't send conversation update to the 'direct' stream when the last m task = Task.async(fn -> - assert_receive {:text, received_event}, 4_000 + assert_receive {:text, received_event}, @streamer_timeout assert %{"event" => "delete", "payload" => _} = Jason.decode!(received_event) - refute_receive {:text, _}, 4_000 + refute_receive {:text, _}, @streamer_timeout end) - Process.sleep(1000) + Process.sleep(@streamer_start_wait) Streamer.add_socket( "direct", @@ -555,11 +559,11 @@ test "it sends conversation update to the 'direct' stream when a message is dele task = Task.async(fn -> - assert_receive {:text, received_event}, 4_000 + + assert_receive {:text, received_event}, @streamer_timeout assert %{"event" => "delete", "payload" => _} = Jason.decode!(received_event) - assert_receive {:text, received_event}, 4_000 - + assert_receive {:text, received_event}, @streamer_timeout assert %{"event" => "conversation", "payload" => received_payload} = Jason.decode!(received_event) @@ -567,7 +571,7 @@ test "it sends conversation update to the 'direct' stream when a message is dele assert last_status["id"] == to_string(create_activity.id) end) - Process.sleep(1000) + Process.sleep(@streamer_start_wait) Streamer.add_socket( "direct", From eae65e3216cb43f97e58ace7e0a57919332de8b8 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 16 Dec 2019 01:37:33 +0300 Subject: [PATCH 19/32] formatting --- test/web/streamer/streamer_test.exs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/web/streamer/streamer_test.exs b/test/web/streamer/streamer_test.exs index b07a7373f..7166d6f0b 100644 --- a/test/web/streamer/streamer_test.exs +++ b/test/web/streamer/streamer_test.exs @@ -121,7 +121,7 @@ test "it sends follow activities to the 'user:notification' stream", %{ } do user2 = insert(:user) task = Task.async(fn -> assert_receive {:text, _}, @streamer_timeout end) - + Process.sleep(@streamer_start_wait) Streamer.add_socket( @@ -143,7 +143,7 @@ test "it sends to public" do task = Task.async(fn -> - assert_receive {:text, _}, @streamer_timeout + assert_receive {:text, _}, @streamer_timeout end) fake_socket = %StreamerSocket{ @@ -559,11 +559,11 @@ test "it sends conversation update to the 'direct' stream when a message is dele task = Task.async(fn -> - assert_receive {:text, received_event}, @streamer_timeout assert %{"event" => "delete", "payload" => _} = Jason.decode!(received_event) assert_receive {:text, received_event}, @streamer_timeout + assert %{"event" => "conversation", "payload" => received_payload} = Jason.decode!(received_event) From 82c62c5028f517ded121b46b9806053989defdc2 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Dec 2019 15:32:33 +0700 Subject: [PATCH 20/32] Add default endpoint to Pleroma.Captcha.Kocaptcha --- config/test.exs | 2 -- lib/pleroma/captcha/kocaptcha.ex | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/test.exs b/config/test.exs index b48b89c8f..d52ced612 100644 --- a/config/test.exs +++ b/config/test.exs @@ -95,8 +95,6 @@ config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock -config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" - if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else diff --git a/lib/pleroma/captcha/kocaptcha.ex b/lib/pleroma/captcha/kocaptcha.ex index 4e1a07c59..a3128decb 100644 --- a/lib/pleroma/captcha/kocaptcha.ex +++ b/lib/pleroma/captcha/kocaptcha.ex @@ -5,11 +5,13 @@ defmodule Pleroma.Captcha.Kocaptcha do import Pleroma.Web.Gettext alias Pleroma.Captcha.Service + @behaviour Service + @default_endpoint "https://captcha.kotobank.ch" @impl Service def new do - endpoint = Pleroma.Config.get!([__MODULE__, :endpoint]) + endpoint = Pleroma.Config.get([__MODULE__, :endpoint], @default_endpoint) case Tesla.get(endpoint <> "/new") do {:error, _} -> From 8d242f52d4bf44a012603fb2cc6fe4dd9b282ef8 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Dec 2019 19:33:51 +0700 Subject: [PATCH 21/32] Revert "Add default endpoint to Pleroma.Captcha.Kocaptcha" This reverts commit 82c62c5028f517ded121b46b9806053989defdc2. --- config/test.exs | 2 ++ lib/pleroma/captcha/kocaptcha.ex | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/test.exs b/config/test.exs index d52ced612..b48b89c8f 100644 --- a/config/test.exs +++ b/config/test.exs @@ -95,6 +95,8 @@ config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock +config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" + if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else diff --git a/lib/pleroma/captcha/kocaptcha.ex b/lib/pleroma/captcha/kocaptcha.ex index a3128decb..4e1a07c59 100644 --- a/lib/pleroma/captcha/kocaptcha.ex +++ b/lib/pleroma/captcha/kocaptcha.ex @@ -5,13 +5,11 @@ defmodule Pleroma.Captcha.Kocaptcha do import Pleroma.Web.Gettext alias Pleroma.Captcha.Service - @behaviour Service - @default_endpoint "https://captcha.kotobank.ch" @impl Service def new do - endpoint = Pleroma.Config.get([__MODULE__, :endpoint], @default_endpoint) + endpoint = Pleroma.Config.get!([__MODULE__, :endpoint]) case Tesla.get(endpoint <> "/new") do {:error, _} -> From fc484e5ce67117389ce9e2cce22ab1944a759242 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 16 Dec 2019 19:35:10 +0700 Subject: [PATCH 22/32] Add Kocaptcha endpoint to config.exs --- config/config.exs | 2 ++ config/test.exs | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index 370ddd855..47098858b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -70,6 +70,8 @@ seconds_valid: 60, method: Pleroma.Captcha.Native +config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" + config :pleroma, :hackney_pools, federation: [ max_connections: 50, diff --git a/config/test.exs b/config/test.exs index b48b89c8f..d52ced612 100644 --- a/config/test.exs +++ b/config/test.exs @@ -95,8 +95,6 @@ config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock -config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch" - if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" else From 8f79f433bcf6e901d67987a613e909c0b507aa65 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 17 Dec 2019 13:34:07 +0700 Subject: [PATCH 23/32] Hide follower counter when hiding is activated --- lib/pleroma/web/activity_pub/views/user_view.ex | 8 +++++++- test/web/activity_pub/views/user_view_test.exs | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 9059aa634..350c4391d 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -201,7 +201,6 @@ def render("followers.json", %{user: user} = opts) do %{ "id" => "#{user.ap_id}/followers", "type" => "OrderedCollection", - "totalItems" => total, "first" => if showing_items do collection(followers, "#{user.ap_id}/followers", 1, showing_items, total) @@ -209,6 +208,7 @@ def render("followers.json", %{user: user} = opts) do "#{user.ap_id}/followers?page=1" end } + |> maybe_put_total_items(showing_count, total) |> Map.merge(Utils.make_json_ld_header()) end @@ -251,6 +251,12 @@ def render("activity_collection_page.json", %{activities: activities, iri: iri}) |> Map.merge(Utils.make_json_ld_header()) end + defp maybe_put_total_items(map, false, _total), do: map + + defp maybe_put_total_items(map, true, total) do + Map.put(map, "totalItems", total) + end + def collection(collection, iri, page, show_items \\ true, total \\ nil) do offset = (page - 1) * 10 items = Enum.slice(collection, offset, 10) diff --git a/test/web/activity_pub/views/user_view_test.exs b/test/web/activity_pub/views/user_view_test.exs index 3299be2d5..8374b8d23 100644 --- a/test/web/activity_pub/views/user_view_test.exs +++ b/test/web/activity_pub/views/user_view_test.exs @@ -126,7 +126,7 @@ test "sets totalItems to zero when followers are hidden" do {:ok, _other_user, user, _activity} = CommonAPI.follow(other_user, user) assert %{"totalItems" => 1} = UserView.render("followers.json", %{user: user}) user = Map.merge(user, %{hide_followers_count: true, hide_followers: true}) - assert %{"totalItems" => 0} = UserView.render("followers.json", %{user: user}) + refute UserView.render("followers.json", %{user: user}) |> Map.has_key?("totalItems") end test "sets correct totalItems when followers are hidden but the follower counter is not" do From d2f1c4f658a8c995716aee3876cb46a0f48f99cb Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Tue, 17 Dec 2019 16:16:21 +0100 Subject: [PATCH 24/32] Add ActivityPub Object Event type support Adds Event support in the same way Video objects are handled, with the name of the object as message header. Signed-off-by: Thomas Citharel --- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- lib/pleroma/web/activity_pub/utils.ex | 11 ++++++++++- .../web/mastodon_api/views/status_view.ex | 3 ++- .../twitter_api/controllers/util_controller.ex | 3 ++- .../tesla_mock/mobilizon.org-event.json | 1 + .../tesla_mock/mobilizon.org-user.json | 1 + test/object/fetcher_test.exs | 9 +++++++++ test/support/http_request_mock.ex | 18 ++++++++++++++++++ .../mastodon_api/views/status_view_test.exs | 15 +++++++++++++++ 9 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 test/fixtures/tesla_mock/mobilizon.org-event.json create mode 100644 test/fixtures/tesla_mock/mobilizon.org-user.json diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index ecba27bef..3fa789d53 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -397,7 +397,7 @@ def handle_incoming( %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, options ) - when objtype in ["Article", "Note", "Video", "Page", "Question", "Answer"] do + when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer"] do actor = Containment.get_actor(data) data = diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index e87d09134..db7084246 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -22,7 +22,16 @@ defmodule Pleroma.Web.ActivityPub.Utils do require Logger require Pleroma.Constants - @supported_object_types ["Article", "Note", "Video", "Page", "Question", "Answer", "Audio"] + @supported_object_types [ + "Article", + "Note", + "Event", + "Video", + "Page", + "Question", + "Answer", + "Audio" + ] @strip_status_report_states ~w(closed resolved) @supported_report_states ~w(open closed resolved) @valid_visibilities ~w(public unlisted private direct) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index a0257dfa6..e9590224b 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -421,7 +421,8 @@ def get_reply_to(%{data: %{"object" => _object}} = activity, _) do end end - def render_content(%{data: %{"type" => "Video"}} = object) do + def render_content(%{data: %{"type" => object_type}} = object) + when object_type in ["Video", "Event"] do with name when not is_nil(name) and name != "" <- object.data["name"] do "

#{name}

#{object.data["content"]}" else diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 2305bb413..799dd17ae 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -104,7 +104,8 @@ defp follow_template(_), do: "follow_login.html" defp is_status?(acct) do case Pleroma.Object.Fetcher.fetch_and_contain_remote_object_from_id(acct) do - {:ok, %{"type" => type}} when type in ["Article", "Note", "Video", "Page", "Question"] -> + {:ok, %{"type" => type}} + when type in ["Article", "Event", "Note", "Video", "Page", "Question"] -> true _ -> diff --git a/test/fixtures/tesla_mock/mobilizon.org-event.json b/test/fixtures/tesla_mock/mobilizon.org-event.json new file mode 100644 index 000000000..7411cf817 --- /dev/null +++ b/test/fixtures/tesla_mock/mobilizon.org-event.json @@ -0,0 +1 @@ +{"@context":["https://www.w3.org/ns/activitystreams","https://litepub.social/litepub/context.jsonld",{"GeoCoordinates":"sc:GeoCoordinates","Hashtag":"as:Hashtag","Place":"sc:Place","PostalAddress":"sc:PostalAddress","address":{"@id":"sc:address","@type":"sc:PostalAddress"},"addressCountry":"sc:addressCountry","addressLocality":"sc:addressLocality","addressRegion":"sc:addressRegion","category":"sc:category","commentsEnabled":{"@id":"pt:commentsEnabled","@type":"sc:Boolean"},"geo":{"@id":"sc:geo","@type":"sc:GeoCoordinates"},"ical":"http://www.w3.org/2002/12/cal/ical#","joinMode":{"@id":"mz:joinMode","@type":"mz:joinModeType"},"joinModeType":{"@id":"mz:joinModeType","@type":"rdfs:Class"},"location":{"@id":"sc:location","@type":"sc:Place"},"maximumAttendeeCapacity":"sc:maximumAttendeeCapacity","mz":"https://joinmobilizon.org/ns#","postalCode":"sc:postalCode","pt":"https://joinpeertube.org/ns#","repliesModerationOption":{"@id":"mz:repliesModerationOption","@type":"mz:repliesModerationOptionType"},"repliesModerationOptionType":{"@id":"mz:repliesModerationOptionType","@type":"rdfs:Class"},"sc":"http://schema.org#","streetAddress":"sc:streetAddress","uuid":"sc:identifier"}],"actor":"https://mobilizon.org/@tcit","attributedTo":"https://mobilizon.org/@tcit","category":"meeting","cc":[],"commentsEnabled":true,"content":"

Mobilizon is now federated! 🎉

You can view this event from other instances if they are subscribed to mobilizon.org, and soon directly from Mastodon and Pleroma. It is possible that you may see some comments from other instances, including Mastodon ones, just below.

With a Mobilizon account on an instance, you may participate at events from other instances and add comments on events.

Of course, it's still a work in progress: if reports made from an instance on events and comments can be federated, you can't block people right now, and moderators actions are rather limited, but this will definitely get fixed over time until first stable version next year.

Anyway, if you want to come up with some feedback, head over to our forum or - if you feel you have technical skills and are familiar with it - on our Gitlab repository.

Also, to people that want to set Mobilizon themselves even though we really don't advise to do that for now, we have a little documentation but it's quite the early days and you'll probably need some help. No worries, you can chat with us on our Forum or though our Matrix channel.

Check our website for more informations and follow us on Twitter or Mastodon.

","endTime":"2019-12-18T14:00:00Z","ical:status":"CONFIRMED","id":"https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39","joinMode":"free","location":{"address":{"addressCountry":"France","addressLocality":"Nantes","addressRegion":"Pays de la Loire","postalCode":null,"streetAddress":" ","type":"PostalAddress"},"geo":{"latitude":-1.54939699141711,"longitude":47.21617415,"type":"GeoCoordinates"},"id":"https://mobilizon.org/address/1368fdab-1e2c-4de6-bcff-a90c84abdee1","name":"Cour du Château des Ducs de Bretagne","type":"Place"},"maximumAttendeeCapacity":0,"mediaType":"text/html","name":"Mobilizon Launching Party","published":"2019-12-17T11:33:56Z","repliesModerationOption":"allow_all","startTime":"2019-12-18T13:00:00Z","tag":[{"href":"https://mobilizon.org/tags/mobilizon","name":"#Mobilizon","type":"Hashtag"},{"href":"https://mobilizon.org/tags/federation","name":"#Federation","type":"Hashtag"},{"href":"https://mobilizon.org/tags/activitypub","name":"#ActivityPub","type":"Hashtag"},{"href":"https://mobilizon.org/tags/party","name":"#Party","type":"Hashtag"}],"to":["https://www.w3.org/ns/activitystreams#Public"],"type":"Event","updated":"2019-12-17T12:25:01Z","url":"https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39","uuid":"252d5816-00a3-4a89-a66f-15bf65c33e39"} \ No newline at end of file diff --git a/test/fixtures/tesla_mock/mobilizon.org-user.json b/test/fixtures/tesla_mock/mobilizon.org-user.json new file mode 100644 index 000000000..f948ae5f0 --- /dev/null +++ b/test/fixtures/tesla_mock/mobilizon.org-user.json @@ -0,0 +1 @@ +{"@context":["https://www.w3.org/ns/activitystreams","https://litepub.social/litepub/context.jsonld",{"GeoCoordinates":"sc:GeoCoordinates","Hashtag":"as:Hashtag","Place":"sc:Place","PostalAddress":"sc:PostalAddress","address":{"@id":"sc:address","@type":"sc:PostalAddress"},"addressCountry":"sc:addressCountry","addressLocality":"sc:addressLocality","addressRegion":"sc:addressRegion","category":"sc:category","commentsEnabled":{"@id":"pt:commentsEnabled","@type":"sc:Boolean"},"geo":{"@id":"sc:geo","@type":"sc:GeoCoordinates"},"ical":"http://www.w3.org/2002/12/cal/ical#","joinMode":{"@id":"mz:joinMode","@type":"mz:joinModeType"},"joinModeType":{"@id":"mz:joinModeType","@type":"rdfs:Class"},"location":{"@id":"sc:location","@type":"sc:Place"},"maximumAttendeeCapacity":"sc:maximumAttendeeCapacity","mz":"https://joinmobilizon.org/ns#","postalCode":"sc:postalCode","pt":"https://joinpeertube.org/ns#","repliesModerationOption":{"@id":"mz:repliesModerationOption","@type":"mz:repliesModerationOptionType"},"repliesModerationOptionType":{"@id":"mz:repliesModerationOptionType","@type":"rdfs:Class"},"sc":"http://schema.org#","streetAddress":"sc:streetAddress","uuid":"sc:identifier"}],"endpoints":{"sharedInbox":"https://mobilizon.org/inbox"},"followers":"https://mobilizon.org/@tcit/followers","following":"https://mobilizon.org/@tcit/following","icon":{"mediaType":null,"type":"Image","url":"https://mobilizon.org/media/3a5f18c058a8193b1febfaf561f94ae8b91f85ac64c01ddf5ad7b251fb43baf5.jpg?name=profil.jpg"},"id":"https://mobilizon.org/@tcit","inbox":"https://mobilizon.org/@tcit/inbox","manuallyApprovesFollowers":false,"name":"Thomas Citharel","outbox":"https://mobilizon.org/@tcit/outbox","preferredUsername":"tcit","publicKey":{"id":"https://mobilizon.org/@tcit#main-key","owner":"https://mobilizon.org/@tcit","publicKeyPem":"-----BEGIN RSA PUBLIC KEY-----\nMIIBCgKCAQEAtzuZFviv5f12SuA0wZFMuwKS8RIlT3IjPCMLRDhiorZeV3UJ1lik\nDYO6mEh22KDXYgJtNVSYGF0Q5LJivgcvuvU+VQ048iTB1B2x0rHMr47KPByPjfVb\nKDeHt6fkHcLY0JK8UkIxW542wXAg4jX5w3gJi3pgTQrCT8VNyPbH1CaA0uW//9jc\nqzZQVFzpfdJoVOM9E3Urc/u58HC4xOptlM7+B/594ZI9drYwy5m+ZxHwlQUYCva4\n34dvwsfOGxkQyIrzXoep80EnWnFpYCLMcCiz+sEhPYxqLgNE+Cmn/6pv7SIscz6p\neVlQXIchdw+J4yl07paJDkFc6CNTCmaIHQIDAQAB\n-----END RSA PUBLIC KEY-----\n\n"},"summary":"Main profile","type":"Person","url":"https://mobilizon.org/@tcit"} \ No newline at end of file diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs index 9ae6b015d..2aad7a588 100644 --- a/test/object/fetcher_test.exs +++ b/test/object/fetcher_test.exs @@ -77,6 +77,15 @@ test "it can fetch peertube videos" do assert object end + test "it can fetch Mobilizon events" do + {:ok, object} = + Fetcher.fetch_object_from_id( + "https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39" + ) + + assert object + end + test "it can fetch wedistribute articles" do {:ok, object} = Fetcher.fetch_object_from_id("https://wedistribute.org/wp-json/pterotype/v1/object/85810") diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index e3a621f49..f43de700d 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -308,6 +308,24 @@ def get("https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" }} end + def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _, _, + Accept: "application/activity+json" + ) do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/mobilizon.org-event.json") + }} + end + + def get("https://mobilizon.org/@tcit", _, _, Accept: "application/activity+json") do + {:ok, + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/tesla_mock/mobilizon.org-user.json") + }} + end + def get("https://baptiste.gelez.xyz/@/BaptisteGelez", _, _, _) do {:ok, %Tesla.Env{ diff --git a/test/web/mastodon_api/views/status_view_test.exs b/test/web/mastodon_api/views/status_view_test.exs index bdd87a79e..17b6ebcbc 100644 --- a/test/web/mastodon_api/views/status_view_test.exs +++ b/test/web/mastodon_api/views/status_view_test.exs @@ -394,6 +394,21 @@ test "a peertube video" do assert length(represented[:media_attachments]) == 1 end + test "a Mobilizon event" do + user = insert(:user) + + {:ok, object} = + Pleroma.Object.Fetcher.fetch_object_from_id( + "https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39" + ) + + %Activity{} = activity = Activity.get_create_by_object_ap_id(object.data["id"]) + + represented = StatusView.render("show.json", %{for: user, activity: activity}) + + assert represented[:id] == to_string(activity.id) + end + describe "build_tags/1" do test "it returns a a dictionary tags" do object_tags = [ From 407449158fab56943da95af5172db4e464588d0e Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 17 Dec 2019 18:24:16 +0300 Subject: [PATCH 25/32] CI: Add a cache policy for test jobs and fix rum tests recompiling everything --- .gitlab-ci.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 88789035d..dc85eaba2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,13 @@ image: elixir:1.8.1 -variables: +variables: &global_variables POSTGRES_DB: pleroma_test POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres DB_HOST: postgres MIX_ENV: test -cache: +cache: &global_cache_policy key: ${CI_COMMIT_REF_SLUG} paths: - deps @@ -46,6 +46,10 @@ benchmark: unit-testing: stage: test + cache: &testing_cache_policy + <<: *global_cache_policy + policy: pull + services: - name: postgres:9.6 alias: postgres @@ -58,6 +62,7 @@ unit-testing: federated-testing: stage: test + cache: *testing_cache_policy services: - name: minibikini/postgres-with-rum:12 alias: postgres @@ -71,11 +76,13 @@ federated-testing: unit-testing-rum: stage: test + cache: *testing_cache_policy services: - name: minibikini/postgres-with-rum:12 alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] variables: + <<: *global_variables RUM_ENABLED: "true" script: - mix deps.get @@ -86,17 +93,20 @@ unit-testing-rum: lint: stage: test + cache: *testing_cache_policy script: - mix format --check-formatted analysis: stage: test + cache: *testing_cache_policy script: - mix deps.get - mix credo --strict --only=warnings,todo,fixme,consistency,readability docs-deploy: stage: deploy + cache: *testing_cache_policy image: alpine:latest only: - stable@pleroma/pleroma From 404a9ccb9a220f3f52ee03bd69bd3746d95794cc Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Wed, 18 Dec 2019 23:11:42 +0300 Subject: [PATCH 26/32] Stats: return status counts by scope --- CHANGELOG.md | 1 + lib/pleroma/stats.ex | 72 +++++++++++++++++-- test/stats_test.exs | 52 ++++++++++++++ .../controllers/instance_controller_test.exs | 10 ++- 4 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 test/stats_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index c133cd9ec..f6cc193a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking:** Admin API: `PUT /api/pleroma/admin/reports/:id` is now `PATCH /api/pleroma/admin/reports`, see admin_api.md for details - **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string. - **Breaking** replying to reports is now "report notes", enpoint changed from `POST /api/pleroma/admin/reports/:id/respond` to `POST /api/pleroma/admin/reports/:id/notes` +- **Breaking** `/api/v1/stats` now return statuses count by scope (i.e. `all`, `public`, `unlisted`, `direct` and `private`) - Admin API: Return `total` when querying for reports - Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`) - Admin API: Return link alongside with token on password reset diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 8154a09b7..c90e8f409 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -3,11 +3,15 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Stats do + use GenServer + import Ecto.Query + + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User - use GenServer + require Pleroma.Constants @interval 1000 * 60 * 60 @@ -56,7 +60,7 @@ defp initial_data do %{peers: [], stats: %{}} end - defp get_stat_data do + def get_stat_data do peers = from( u in User, @@ -68,13 +72,71 @@ defp get_stat_data do domain_count = Enum.count(peers) - status_count = Repo.aggregate(User.Query.build(%{local: true}), :sum, :note_count) - user_count = Repo.aggregate(User.Query.build(%{local: true, active: true}), :count, :id) %{ peers: peers, - stats: %{domain_count: domain_count, status_count: status_count, user_count: user_count} + stats: %{domain_count: domain_count, status_count: status_count(), user_count: user_count} } end + + defp status_count do + %{ + all: get_all_statuses_count(), + public: public_statuses_query() |> Repo.aggregate(:count, :id), + unlisted: unlisted_statuses_query() |> Repo.aggregate(:count, :id), + direct: direct_statuses_query() |> Repo.aggregate(:count, :id), + private: private_statuses_query() |> Repo.aggregate(:count, :id) + } + end + + defp get_all_statuses_count do + Repo.aggregate(User.Query.build(%{local: true}), :sum, :note_count) + end + + def public_statuses_query do + from(o in Object, + where: fragment("(?)->'to' \\? ?", o.data, ^Pleroma.Constants.as_public()) + ) + end + + def unlisted_statuses_query do + from(o in Object, + where: not fragment("(?)->'to' \\? ?", o.data, ^Pleroma.Constants.as_public()), + where: fragment("(?)->'cc' \\? ?", o.data, ^Pleroma.Constants.as_public()) + ) + end + + def direct_statuses_query do + private_statuses_ids = from(p in private_statuses_query(), select: p.id) |> Repo.all() + + from(o in Object, + where: + fragment( + "? \\? 'directMessage' AND (?->>'directMessage')::boolean = true", + o.data, + o.data + ) or + (not fragment("(?)->'to' \\? ?", o.data, ^Pleroma.Constants.as_public()) and + not fragment("(?)->'cc' \\? ?", o.data, ^Pleroma.Constants.as_public()) and + o.id not in ^private_statuses_ids) + ) + end + + def private_statuses_query do + from(o in subquery(recipients_query()), + where: ilike(o.recipients, "%/followers%") + ) + end + + defp recipients_query do + from(o in Object, + select: %{ + id: o.id, + recipients: fragment("jsonb_array_elements_text((?)->'to')", o.data) + }, + where: not fragment("(?)->'to' \\? ?", o.data, ^Pleroma.Constants.as_public()), + where: not fragment("(?)->'cc' \\? ?", o.data, ^Pleroma.Constants.as_public()) + ) + end end diff --git a/test/stats_test.exs b/test/stats_test.exs new file mode 100644 index 000000000..31c2f8db3 --- /dev/null +++ b/test/stats_test.exs @@ -0,0 +1,52 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.StatsTest do + use Pleroma.DataCase + + import Pleroma.Factory + + alias Pleroma.Web.CommonAPI + + describe "statuses count" do + setup do + user = insert(:user) + other_user = insert(:user) + + CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"}) + + Enum.each(0..1, fn _ -> + CommonAPI.post(user, %{ + "visibility" => "unlisted", + "status" => "hey" + }) + end) + + Enum.each(0..2, fn _ -> + CommonAPI.post(user, %{ + "visibility" => "direct", + "status" => "hey @#{other_user.nickname}" + }) + end) + + Enum.each(0..3, fn _ -> + CommonAPI.post(user, %{ + "visibility" => "private", + "status" => "hey" + }) + end) + + :ok + end + + test "it returns total number of statuses" do + data = Pleroma.Stats.get_stat_data() + + assert data.stats.status_count.public == 1 + assert data.stats.status_count.unlisted == 2 + assert data.stats.status_count.direct == 3 + assert data.stats.status_count.private == 4 + end + end +end diff --git a/test/web/mastodon_api/controllers/instance_controller_test.exs b/test/web/mastodon_api/controllers/instance_controller_test.exs index e00de6b18..7aa7c8648 100644 --- a/test/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/web/mastodon_api/controllers/instance_controller_test.exs @@ -58,7 +58,15 @@ test "get instance stats", %{conn: conn} do assert stats assert stats["user_count"] == 1 - assert stats["status_count"] == 1 + + assert stats["status_count"] == %{ + "all" => 1, + "direct" => 0, + "private" => 0, + "public" => 1, + "unlisted" => 0 + } + assert stats["domain_count"] == 2 end From 432b3067d4c62cd27e35f0b6e7bdc61da63310b9 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 19 Dec 2019 19:25:23 +0700 Subject: [PATCH 27/32] Do not crash when remote user follower and following counters are hidden --- lib/pleroma/web/activity_pub/activity_pub.ex | 40 ++++++++------------ test/web/activity_pub/activity_pub_test.exs | 38 +++++++++++++++++++ 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 16e6b0057..60c9e7e64 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1298,28 +1298,26 @@ defp object_to_user_data(data) do def fetch_follow_information_for_user(user) do with {:ok, following_data} <- Fetcher.fetch_and_contain_remote_object_from_id(user.following_address), - following_count when is_integer(following_count) <- following_data["totalItems"], {:ok, hide_follows} <- collection_private(following_data), {:ok, followers_data} <- Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address), - followers_count when is_integer(followers_count) <- followers_data["totalItems"], {:ok, hide_followers} <- collection_private(followers_data) do {:ok, %{ hide_follows: hide_follows, - follower_count: followers_count, - following_count: following_count, + follower_count: normalize_counter(followers_data["totalItems"]), + following_count: normalize_counter(following_data["totalItems"]), hide_followers: hide_followers }} else - {:error, _} = e -> - e - - e -> - {:error, e} + {:error, _} = e -> e + e -> {:error, e} end end + defp normalize_counter(counter) when is_integer(counter), do: counter + defp normalize_counter(_), do: 0 + defp maybe_update_follow_information(data) do with {:enabled, true} <- {:enabled, Pleroma.Config.get([:instance, :external_user_synchronization])}, @@ -1339,24 +1337,18 @@ defp maybe_update_follow_information(data) do end end + defp collection_private(%{"first" => %{"type" => type}}) + when type in ["CollectionPage", "OrderedCollectionPage"], + do: {:ok, false} + defp collection_private(%{"first" => first}) do - if is_map(first) and - first["type"] in ["CollectionPage", "OrderedCollectionPage"] do + with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <- + Fetcher.fetch_and_contain_remote_object_from_id(first) do {:ok, false} else - with {:ok, %{"type" => type}} when type in ["CollectionPage", "OrderedCollectionPage"] <- - Fetcher.fetch_and_contain_remote_object_from_id(first) do - {:ok, false} - else - {:error, {:ok, %{status: code}}} when code in [401, 403] -> - {:ok, true} - - {:error, _} = e -> - e - - e -> - {:error, e} - end + {:error, {:ok, %{status: code}}} when code in [401, 403] -> {:ok, true} + {:error, _} = e -> e + e -> {:error, e} end end diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 1520c8a9b..ad6b9810c 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -1623,6 +1623,44 @@ test "detects hidden follows/followers for friendica" do assert follow_info.following_count == 32 assert follow_info.hide_follows == true end + + test "doesn't crash when follower and following counters are hidden" do + mock(fn env -> + case env.url do + "http://localhost:4001/users/masto_hidden_counters/following" -> + json(%{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "http://localhost:4001/users/masto_hidden_counters/followers" + }) + + "http://localhost:4001/users/masto_hidden_counters/following?page=1" -> + %Tesla.Env{status: 403, body: ""} + + "http://localhost:4001/users/masto_hidden_counters/followers" -> + json(%{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "http://localhost:4001/users/masto_hidden_counters/following" + }) + + "http://localhost:4001/users/masto_hidden_counters/followers?page=1" -> + %Tesla.Env{status: 403, body: ""} + end + end) + + user = + insert(:user, + local: false, + follower_address: "http://localhost:4001/users/masto_hidden_counters/followers", + following_address: "http://localhost:4001/users/masto_hidden_counters/following" + ) + + {:ok, follow_info} = ActivityPub.fetch_follow_information_for_user(user) + + assert follow_info.hide_followers == true + assert follow_info.follower_count == 0 + assert follow_info.hide_follows == true + assert follow_info.following_count == 0 + end end describe "fetch_favourites/3" do From 4079d66f00adc665e4c70abad03eab254695e793 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 19 Dec 2019 19:47:36 +0300 Subject: [PATCH 28/32] Expose issue via failing test --- test/stats_test.exs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/stats_test.exs b/test/stats_test.exs index 31c2f8db3..02a92dc64 100644 --- a/test/stats_test.exs +++ b/test/stats_test.exs @@ -43,6 +43,7 @@ defmodule Pleroma.StatsTest do test "it returns total number of statuses" do data = Pleroma.Stats.get_stat_data() + assert data.stats.status_count.all == 10 assert data.stats.status_count.public == 1 assert data.stats.status_count.unlisted == 2 assert data.stats.status_count.direct == 3 From 5fc84552d311efd606f66775c55862b3d11ad258 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Thu, 19 Dec 2019 19:52:55 +0300 Subject: [PATCH 29/32] Fix all count --- lib/pleroma/stats.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index c90e8f409..97e8b1990 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -82,7 +82,7 @@ def get_stat_data do defp status_count do %{ - all: get_all_statuses_count(), + all: all_statuses_query() |> Repo.aggregate(:count, :id), public: public_statuses_query() |> Repo.aggregate(:count, :id), unlisted: unlisted_statuses_query() |> Repo.aggregate(:count, :id), direct: direct_statuses_query() |> Repo.aggregate(:count, :id), @@ -90,8 +90,8 @@ defp status_count do } end - defp get_all_statuses_count do - Repo.aggregate(User.Query.build(%{local: true}), :sum, :note_count) + defp all_statuses_query do + from(o in Object, where: fragment("(?)->>'type' = 'Note'", o.data)) end def public_statuses_query do From fc79c691736725e62dad1c7a701d8c5f3435410f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 20 Dec 2019 19:47:44 +0700 Subject: [PATCH 30/32] Remove `/api/account/register` documentation from pleroma_api.md --- docs/API/pleroma_api.md | 53 ----------------------------------------- 1 file changed, 53 deletions(-) diff --git a/docs/API/pleroma_api.md b/docs/API/pleroma_api.md index 7228d805b..689edbcc2 100644 --- a/docs/API/pleroma_api.md +++ b/docs/API/pleroma_api.md @@ -70,59 +70,6 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi * Response: JSON. Returns `{"status": "success"}` if the account was successfully disabled, `{"error": "[error message]"}` otherwise * Example response: `{"error": "Invalid password."}` -## `/api/account/register` -### Register a new user -* Method `POST` -* Authentication: not required -* Params: - * `nickname` - * `fullname` - * `bio` - * `email` - * `password` - * `confirm` - * `captcha_solution`: optional, contains provider-specific captcha solution, - * `captcha_token`: optional, contains provider-specific captcha token - * `token`: invite token required when the registrations aren't public. -* Response: JSON. Returns a user object on success, otherwise returns `{"error": "error_msg"}` -* Example response: -```json -{ - "background_image": null, - "cover_photo": "https://pleroma.soykaf.com/images/banner.png", - "created_at": "Tue Dec 18 16:55:56 +0000 2018", - "default_scope": "public", - "description": "blushy-crushy fediverse idol + pleroma dev\nlet's be friends \nぷれろまの生徒会長。謎の外人。日本語OK. \n公主病.", - "description_html": "blushy-crushy fediverse idol + pleroma dev.
let's be friends
ぷれろまの生徒会長。謎の外人。日本語OK.
公主病.", - "favourites_count": 0, - "fields": [], - "followers_count": 0, - "following": false, - "follows_you": false, - "friends_count": 0, - "id": 6, - "is_local": true, - "locked": false, - "name": "lain", - "name_html": "lain", - "no_rich_text": false, - "pleroma": { - "tags": [] - }, - "profile_image_url": "https://pleroma.soykaf.com/images/avi.png", - "profile_image_url_https": "https://pleroma.soykaf.com/images/avi.png", - "profile_image_url_original": "https://pleroma.soykaf.com/images/avi.png", - "profile_image_url_profile_size": "https://pleroma.soykaf.com/images/avi.png", - "rights": { - "delete_others_notice": false - }, - "screen_name": "lain", - "statuses_count": 0, - "statusnet_blocking": false, - "statusnet_profile_url": "https://pleroma.soykaf.com/users/lain" -} -``` - ## `/api/pleroma/admin/`… See [Admin-API](admin_api.md) From 06ae56a3ae93c494f9c5d15b097d75c6ab7fcc29 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 20 Dec 2019 16:32:04 -0600 Subject: [PATCH 31/32] Posts without media attachments should get the Summary TwitterCard --- lib/pleroma/web/metadata/twitter_card.ex | 2 +- test/web/metadata/twitter_card_test.exs | 29 ++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex index d6a6049b3..67419a666 100644 --- a/lib/pleroma/web/metadata/twitter_card.ex +++ b/lib/pleroma/web/metadata/twitter_card.ex @@ -31,7 +31,7 @@ def build_tags(%{activity_id: id, object: object, user: user}) do if attachments == [] or Metadata.activity_nsfw?(object) do [ image_tag(user), - {:meta, [property: "twitter:card", content: "summary_large_image"], []} + {:meta, [property: "twitter:card", content: "summary"], []} ] else attachments diff --git a/test/web/metadata/twitter_card_test.exs b/test/web/metadata/twitter_card_test.exs index 0814006d2..85a654f52 100644 --- a/test/web/metadata/twitter_card_test.exs +++ b/test/web/metadata/twitter_card_test.exs @@ -26,7 +26,32 @@ test "it renders twitter card for user info" do ] end - test "it does not render attachments if post is nsfw" do + test "it uses summary twittercard if post has no attachment" do + user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994") + {:ok, activity} = CommonAPI.post(user, %{"status" => "HI"}) + + note = + insert(:note, %{ + data: %{ + "actor" => user.ap_id, + "tag" => [], + "id" => "https://pleroma.gov/objects/whatever", + "content" => "pleroma in a nutshell" + } + }) + + result = TwitterCard.build_tags(%{object: note, user: user, activity_id: activity.id}) + + assert [ + {:meta, [property: "twitter:title", content: Utils.user_name_string(user)], []}, + {:meta, [property: "twitter:description", content: "“pleroma in a nutshell”"], []}, + {:meta, [property: "twitter:image", content: "http://localhost:4001/images/avi.png"], + []}, + {:meta, [property: "twitter:card", content: "summary"], []} + ] == result + end + + test "it renders avatar not attachment if post is nsfw and unfurl_nsfw is disabled" do Pleroma.Config.put([Pleroma.Web.Metadata, :unfurl_nsfw], false) user = insert(:user, name: "Jimmy Hendriks", bio: "born 19 March 1994") {:ok, activity} = CommonAPI.post(user, %{"status" => "HI"}) @@ -67,7 +92,7 @@ test "it does not render attachments if post is nsfw" do {:meta, [property: "twitter:description", content: "“pleroma in a nutshell”"], []}, {:meta, [property: "twitter:image", content: "http://localhost:4001/images/avi.png"], []}, - {:meta, [property: "twitter:card", content: "summary_large_image"], []} + {:meta, [property: "twitter:card", content: "summary"], []} ] == result end From e71a13ad57d2604b45c0beb278f47d25c284783a Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sat, 21 Dec 2019 11:41:19 +0000 Subject: [PATCH 32/32] Revert "Merge branch 'feature/status-counts-by-scope' into 'develop'" This reverts merge request !2076 --- CHANGELOG.md | 1 - lib/pleroma/stats.ex | 72 ++----------------- test/stats_test.exs | 53 -------------- .../controllers/instance_controller_test.exs | 10 +-- 4 files changed, 6 insertions(+), 130 deletions(-) delete mode 100644 test/stats_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index f6cc193a2..c133cd9ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking:** Admin API: `PUT /api/pleroma/admin/reports/:id` is now `PATCH /api/pleroma/admin/reports`, see admin_api.md for details - **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string. - **Breaking** replying to reports is now "report notes", enpoint changed from `POST /api/pleroma/admin/reports/:id/respond` to `POST /api/pleroma/admin/reports/:id/notes` -- **Breaking** `/api/v1/stats` now return statuses count by scope (i.e. `all`, `public`, `unlisted`, `direct` and `private`) - Admin API: Return `total` when querying for reports - Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`) - Admin API: Return link alongside with token on password reset diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex index 97e8b1990..8154a09b7 100644 --- a/lib/pleroma/stats.ex +++ b/lib/pleroma/stats.ex @@ -3,15 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Stats do - use GenServer - import Ecto.Query - - alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User - require Pleroma.Constants + use GenServer @interval 1000 * 60 * 60 @@ -60,7 +56,7 @@ defp initial_data do %{peers: [], stats: %{}} end - def get_stat_data do + defp get_stat_data do peers = from( u in User, @@ -72,71 +68,13 @@ def get_stat_data do domain_count = Enum.count(peers) + status_count = Repo.aggregate(User.Query.build(%{local: true}), :sum, :note_count) + user_count = Repo.aggregate(User.Query.build(%{local: true, active: true}), :count, :id) %{ peers: peers, - stats: %{domain_count: domain_count, status_count: status_count(), user_count: user_count} + stats: %{domain_count: domain_count, status_count: status_count, user_count: user_count} } end - - defp status_count do - %{ - all: all_statuses_query() |> Repo.aggregate(:count, :id), - public: public_statuses_query() |> Repo.aggregate(:count, :id), - unlisted: unlisted_statuses_query() |> Repo.aggregate(:count, :id), - direct: direct_statuses_query() |> Repo.aggregate(:count, :id), - private: private_statuses_query() |> Repo.aggregate(:count, :id) - } - end - - defp all_statuses_query do - from(o in Object, where: fragment("(?)->>'type' = 'Note'", o.data)) - end - - def public_statuses_query do - from(o in Object, - where: fragment("(?)->'to' \\? ?", o.data, ^Pleroma.Constants.as_public()) - ) - end - - def unlisted_statuses_query do - from(o in Object, - where: not fragment("(?)->'to' \\? ?", o.data, ^Pleroma.Constants.as_public()), - where: fragment("(?)->'cc' \\? ?", o.data, ^Pleroma.Constants.as_public()) - ) - end - - def direct_statuses_query do - private_statuses_ids = from(p in private_statuses_query(), select: p.id) |> Repo.all() - - from(o in Object, - where: - fragment( - "? \\? 'directMessage' AND (?->>'directMessage')::boolean = true", - o.data, - o.data - ) or - (not fragment("(?)->'to' \\? ?", o.data, ^Pleroma.Constants.as_public()) and - not fragment("(?)->'cc' \\? ?", o.data, ^Pleroma.Constants.as_public()) and - o.id not in ^private_statuses_ids) - ) - end - - def private_statuses_query do - from(o in subquery(recipients_query()), - where: ilike(o.recipients, "%/followers%") - ) - end - - defp recipients_query do - from(o in Object, - select: %{ - id: o.id, - recipients: fragment("jsonb_array_elements_text((?)->'to')", o.data) - }, - where: not fragment("(?)->'to' \\? ?", o.data, ^Pleroma.Constants.as_public()), - where: not fragment("(?)->'cc' \\? ?", o.data, ^Pleroma.Constants.as_public()) - ) - end end diff --git a/test/stats_test.exs b/test/stats_test.exs deleted file mode 100644 index 02a92dc64..000000000 --- a/test/stats_test.exs +++ /dev/null @@ -1,53 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.StatsTest do - use Pleroma.DataCase - - import Pleroma.Factory - - alias Pleroma.Web.CommonAPI - - describe "statuses count" do - setup do - user = insert(:user) - other_user = insert(:user) - - CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"}) - - Enum.each(0..1, fn _ -> - CommonAPI.post(user, %{ - "visibility" => "unlisted", - "status" => "hey" - }) - end) - - Enum.each(0..2, fn _ -> - CommonAPI.post(user, %{ - "visibility" => "direct", - "status" => "hey @#{other_user.nickname}" - }) - end) - - Enum.each(0..3, fn _ -> - CommonAPI.post(user, %{ - "visibility" => "private", - "status" => "hey" - }) - end) - - :ok - end - - test "it returns total number of statuses" do - data = Pleroma.Stats.get_stat_data() - - assert data.stats.status_count.all == 10 - assert data.stats.status_count.public == 1 - assert data.stats.status_count.unlisted == 2 - assert data.stats.status_count.direct == 3 - assert data.stats.status_count.private == 4 - end - end -end diff --git a/test/web/mastodon_api/controllers/instance_controller_test.exs b/test/web/mastodon_api/controllers/instance_controller_test.exs index 7aa7c8648..e00de6b18 100644 --- a/test/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/web/mastodon_api/controllers/instance_controller_test.exs @@ -58,15 +58,7 @@ test "get instance stats", %{conn: conn} do assert stats assert stats["user_count"] == 1 - - assert stats["status_count"] == %{ - "all" => 1, - "direct" => 0, - "private" => 0, - "public" => 1, - "unlisted" => 0 - } - + assert stats["status_count"] == 1 assert stats["domain_count"] == 2 end