Merge branch 'fed-testing-updates' into 'develop'
Add federated test infrastructure See merge request pleroma/pleroma!2014
This commit is contained in:
lain
commit
98990d2d61
|
|
@ -55,6 +55,19 @@ unit-testing:
|
|||
- mix ecto.migrate
|
||||
- mix coveralls --preload-modules
|
||||
|
||||
federated-testing:
|
||||
stage: test
|
||||
services:
|
||||
- name: lainsoykaf/postgres-with-rum
|
||||
alias: postgres
|
||||
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
|
||||
script:
|
||||
- mix deps.get
|
||||
- mix ecto.create
|
||||
- mix ecto.migrate
|
||||
- epmd -daemon
|
||||
- mix test --trace --only federated
|
||||
|
||||
unit-testing-rum:
|
||||
stage: test
|
||||
services:
|
||||
|
|
|
|||
2
mix.exs
2
mix.exs
|
|
@ -102,7 +102,7 @@ defp deps do
|
|||
{:phoenix_ecto, "~> 4.0"},
|
||||
{:ecto_sql, "~> 3.2"},
|
||||
{:postgrex, ">= 0.13.5"},
|
||||
{:oban, "~> 0.8.1"},
|
||||
{:oban, "~> 0.12.0"},
|
||||
{:quantum, "~> 2.3"},
|
||||
{:gettext, "~> 0.15"},
|
||||
{:comeonin, "~> 4.1.1"},
|
||||
|
|
|
|||
8
mix.lock
8
mix.lock
|
|
@ -23,8 +23,8 @@
|
|||
"decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"},
|
||||
"deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
|
||||
"earmark": {:hex, :earmark, "1.4.2", "3aa0bd23bc4c61cf2f1e5d752d1bb470560a6f8539974f767a38923bb20e1d7f", [:mix], [], "hexpm"},
|
||||
"ecto": {:hex, :ecto, "3.2.3", "51274df79862845b388733fddcf6f107d0c8c86e27abe7131fa98f8d30761bda", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
|
||||
"ecto_sql": {:hex, :ecto_sql, "3.2.0", "751cea597e8deb616084894dd75cbabfdbe7255ff01e8c058ca13f0353a3921b", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
|
||||
"ecto": {:hex, :ecto, "3.2.5", "76c864b77948a479e18e69cc1d0f0f4ee7cced1148ffe6a093ff91eba644f0b5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
|
||||
"ecto_sql": {:hex, :ecto_sql, "3.2.2", "d10845bc147b9f61ef485cbf0973c0a337237199bd9bd30dd9542db00aadc26b", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.2.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.2.0 or ~> 0.3.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
|
||||
"esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
|
||||
"eternal": {:hex, :eternal, "1.2.1", "d5b6b2499ba876c57be2581b5b999ee9bdf861c647401066d3eeed111d096bc4", [:mix], [], "hexpm"},
|
||||
"ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"},
|
||||
|
|
@ -67,7 +67,7 @@
|
|||
"myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]},
|
||||
"nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"},
|
||||
"nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
|
||||
"oban": {:hex, :oban, "0.8.1", "4bbf62eb1829f856d69aeb5069ac7036afe07db8221a17de2a9169cc7a58a318", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
|
||||
"oban": {:hex, :oban, "0.12.0", "5477d5ab4a5a201c0b6c89764040ebfc5d2c71c488a36f378016ce5990838f0f", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
|
||||
"parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"},
|
||||
"pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.4", "8dd29ed783f2e12195d7e0a4640effc0a7c37e6537da491f1db01839eee6d053", [:mix], [], "hexpm"},
|
||||
"phoenix": {:hex, :phoenix, "1.4.10", "619e4a545505f562cd294df52294372d012823f4fd9d34a6657a8b242898c255", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
|
||||
|
|
@ -97,7 +97,7 @@
|
|||
"sweet_xml": {:hex, :sweet_xml, "0.6.6", "fc3e91ec5dd7c787b6195757fbcf0abc670cee1e4172687b45183032221b66b8", [:mix], [], "hexpm"},
|
||||
"swoosh": {:hex, :swoosh, "0.23.5", "bfd9404bbf5069b1be2ffd317923ce57e58b332e25dbca2a35dedd7820dfee5a", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.13", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mail, "~> 0.2", [hex: :mail, repo: "hexpm", optional: true]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_cowboy, ">= 1.0.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
|
||||
"syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
|
||||
"telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},
|
||||
"telemetry": {:hex, :telemetry, "0.4.1", "ae2718484892448a24470e6aa341bc847c3277bfb8d4e9289f7474d752c09c7f", [:rebar3], [], "hexpm"},
|
||||
"tesla": {:hex, :tesla, "1.3.0", "f35d72f029e608f9cdc6f6d6fcc7c66cf6d6512a70cfef9206b21b8bd0203a30", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 0.4", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"},
|
||||
"timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
|
||||
"trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,47 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Integration.FederationTest do
|
||||
use Pleroma.DataCase
|
||||
@moduletag :federated
|
||||
import Pleroma.Cluster
|
||||
|
||||
setup_all do
|
||||
Pleroma.Cluster.spawn_default_cluster()
|
||||
:ok
|
||||
end
|
||||
|
||||
@federated1 :"federated1@127.0.0.1"
|
||||
describe "federated cluster primitives" do
|
||||
test "within/2 captures local bindings and executes block on remote node" do
|
||||
captured_binding = :captured
|
||||
|
||||
result =
|
||||
within @federated1 do
|
||||
user = Pleroma.Factory.insert(:user)
|
||||
{captured_binding, node(), user}
|
||||
end
|
||||
|
||||
assert {:captured, @federated1, user} = result
|
||||
refute Pleroma.User.get_by_id(user.id)
|
||||
assert user.id == within(@federated1, do: Pleroma.User.get_by_id(user.id)).id
|
||||
end
|
||||
|
||||
test "runs webserver on customized port" do
|
||||
{nickname, url, url_404} =
|
||||
within @federated1 do
|
||||
import Pleroma.Web.Router.Helpers
|
||||
user = Pleroma.Factory.insert(:user)
|
||||
user_url = account_url(Pleroma.Web.Endpoint, :show, user)
|
||||
url_404 = account_url(Pleroma.Web.Endpoint, :show, "not-exists")
|
||||
|
||||
{user.nickname, user_url, url_404}
|
||||
end
|
||||
|
||||
assert {:ok, {{_, 200, _}, _headers, body}} = :httpc.request(~c"#{url}")
|
||||
assert %{"acct" => ^nickname} = Jason.decode!(body)
|
||||
assert {:ok, {{_, 404, _}, _headers, _body}} = :httpc.request(~c"#{url_404}")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
defmodule Pleroma.Cluster do
|
||||
@moduledoc """
|
||||
Facilities for managing a cluster of slave VM's for federated testing.
|
||||
|
||||
## Spawning the federated cluster
|
||||
|
||||
`spawn_cluster/1` spawns a map of slave nodes that are started
|
||||
within the running VM. During startup, the slave node is sent all configuration
|
||||
from the parent node, as well as all code. After receiving configuration and
|
||||
code, the slave then starts all applications currently running on the parent.
|
||||
The configuration passed to `spawn_cluster/1` overrides any parent application
|
||||
configuration for the provided OTP app and key. This is useful for customizing
|
||||
the Ecto database, Phoenix webserver ports, etc.
|
||||
|
||||
For example, to start a single federated VM named ":federated1", with the
|
||||
Pleroma Endpoint running on port 4123, and with a database named
|
||||
"pleroma_test1", you would run:
|
||||
|
||||
endpoint_conf = Application.fetch_env!(:pleroma, Pleroma.Web.Endpoint)
|
||||
repo_conf = Application.fetch_env!(:pleroma, Pleroma.Repo)
|
||||
|
||||
Pleroma.Cluster.spawn_cluster(%{
|
||||
:"federated1@127.0.0.1" => [
|
||||
{:pleroma, Pleroma.Repo, Keyword.merge(repo_conf, database: "pleroma_test1")},
|
||||
{:pleroma, Pleroma.Web.Endpoint,
|
||||
Keyword.merge(endpoint_conf, http: [port: 4011], url: [port: 4011], server: true)}
|
||||
]
|
||||
})
|
||||
|
||||
*Note*: application configuration for a given key is not merged,
|
||||
so any customization requires first fetching the existing values
|
||||
and merging yourself by providing the merged configuration,
|
||||
such as above with the endpoint config and repo config.
|
||||
|
||||
## Executing code within a remote node
|
||||
|
||||
Use the `within/2` macro to execute code within the context of a remote
|
||||
federated node. The code block captures all local variable bindings from
|
||||
the parent's context and returns the result of the expression after executing
|
||||
it on the remote node. For example:
|
||||
|
||||
import Pleroma.Cluster
|
||||
|
||||
parent_value = 123
|
||||
|
||||
result =
|
||||
within :"federated1@127.0.0.1" do
|
||||
{node(), parent_value}
|
||||
end
|
||||
|
||||
assert result == {:"federated1@127.0.0.1, 123}
|
||||
|
||||
*Note*: while local bindings are captured and available within the block,
|
||||
other parent contexts like required, aliased, or imported modules are not
|
||||
in scope. Those will need to be reimported/aliases/required within the block
|
||||
as `within/2` is a remote procedure call.
|
||||
"""
|
||||
|
||||
@extra_apps Pleroma.Mixfile.application()[:extra_applications]
|
||||
|
||||
@doc """
|
||||
Spawns the default Pleroma federated cluster.
|
||||
|
||||
Values before may be customized as needed for the test suite.
|
||||
"""
|
||||
def spawn_default_cluster do
|
||||
endpoint_conf = Application.fetch_env!(:pleroma, Pleroma.Web.Endpoint)
|
||||
repo_conf = Application.fetch_env!(:pleroma, Pleroma.Repo)
|
||||
|
||||
spawn_cluster(%{
|
||||
:"federated1@127.0.0.1" => [
|
||||
{:pleroma, Pleroma.Repo, Keyword.merge(repo_conf, database: "pleroma_test_federated1")},
|
||||
{:pleroma, Pleroma.Web.Endpoint,
|
||||
Keyword.merge(endpoint_conf, http: [port: 4011], url: [port: 4011], server: true)}
|
||||
],
|
||||
:"federated2@127.0.0.1" => [
|
||||
{:pleroma, Pleroma.Repo, Keyword.merge(repo_conf, database: "pleroma_test_federated2")},
|
||||
{:pleroma, Pleroma.Web.Endpoint,
|
||||
Keyword.merge(endpoint_conf, http: [port: 4012], url: [port: 4012], server: true)}
|
||||
]
|
||||
})
|
||||
end
|
||||
|
||||
@doc """
|
||||
Spawns a configured map of federated nodes.
|
||||
|
||||
See `Pleroma.Cluster` module documentation for details.
|
||||
"""
|
||||
def spawn_cluster(node_configs) do
|
||||
# Turn node into a distributed node with the given long name
|
||||
:net_kernel.start([:"primary@127.0.0.1"])
|
||||
|
||||
# Allow spawned nodes to fetch all code from this node
|
||||
{:ok, _} = :erl_boot_server.start([])
|
||||
allow_boot("127.0.0.1")
|
||||
|
||||
silence_logger_warnings(fn ->
|
||||
node_configs
|
||||
|> Enum.map(&Task.async(fn -> start_slave(&1) end))
|
||||
|> Enum.map(&Task.await(&1, 60_000))
|
||||
end)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Executes block of code again remote node.
|
||||
|
||||
See `Pleroma.Cluster` module documentation for details.
|
||||
"""
|
||||
defmacro within(node, do: block) do
|
||||
quote do
|
||||
rpc(unquote(node), unquote(__MODULE__), :eval_quoted, [
|
||||
unquote(Macro.escape(block)),
|
||||
binding()
|
||||
])
|
||||
end
|
||||
end
|
||||
|
||||
@doc false
|
||||
def eval_quoted(block, binding) do
|
||||
{result, _binding} = Code.eval_quoted(block, binding, __ENV__)
|
||||
result
|
||||
end
|
||||
|
||||
defp start_slave({node_host, override_configs}) do
|
||||
log(node_host, "booting federated VM")
|
||||
{:ok, node} = :slave.start(~c"127.0.0.1", node_name(node_host), vm_args())
|
||||
add_code_paths(node)
|
||||
load_apps_and_transfer_configuration(node, override_configs)
|
||||
ensure_apps_started(node)
|
||||
{:ok, node}
|
||||
end
|
||||
|
||||
def rpc(node, module, function, args) do
|
||||
:rpc.block_call(node, module, function, args)
|
||||
end
|
||||
|
||||
defp vm_args do
|
||||
~c"-loader inet -hosts 127.0.0.1 -setcookie #{:erlang.get_cookie()}"
|
||||
end
|
||||
|
||||
defp allow_boot(host) do
|
||||
{:ok, ipv4} = :inet.parse_ipv4_address(~c"#{host}")
|
||||
:ok = :erl_boot_server.add_slave(ipv4)
|
||||
end
|
||||
|
||||
defp add_code_paths(node) do
|
||||
rpc(node, :code, :add_paths, [:code.get_path()])
|
||||
end
|
||||
|
||||
defp load_apps_and_transfer_configuration(node, override_configs) do
|
||||
Enum.each(Application.loaded_applications(), fn {app_name, _, _} ->
|
||||
app_name
|
||||
|> Application.get_all_env()
|
||||
|> Enum.each(fn {key, primary_config} ->
|
||||
rpc(node, Application, :put_env, [app_name, key, primary_config, [persistent: true]])
|
||||
end)
|
||||
end)
|
||||
|
||||
Enum.each(override_configs, fn {app_name, key, val} ->
|
||||
rpc(node, Application, :put_env, [app_name, key, val, [persistent: true]])
|
||||
end)
|
||||
end
|
||||
|
||||
defp log(node, msg), do: IO.puts("[#{node}] #{msg}")
|
||||
|
||||
defp ensure_apps_started(node) do
|
||||
loaded_names = Enum.map(Application.loaded_applications(), fn {name, _, _} -> name end)
|
||||
app_names = @extra_apps ++ (loaded_names -- @extra_apps)
|
||||
|
||||
rpc(node, Application, :ensure_all_started, [:mix])
|
||||
rpc(node, Mix, :env, [Mix.env()])
|
||||
rpc(node, __MODULE__, :prepare_database, [])
|
||||
|
||||
log(node, "starting application")
|
||||
|
||||
Enum.reduce(app_names, MapSet.new(), fn app, loaded ->
|
||||
if Enum.member?(loaded, app) do
|
||||
loaded
|
||||
else
|
||||
{:ok, started} = rpc(node, Application, :ensure_all_started, [app])
|
||||
MapSet.union(loaded, MapSet.new(started))
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@doc false
|
||||
def prepare_database do
|
||||
log(node(), "preparing database")
|
||||
repo_config = Application.get_env(:pleroma, Pleroma.Repo)
|
||||
repo_config[:adapter].storage_down(repo_config)
|
||||
repo_config[:adapter].storage_up(repo_config)
|
||||
|
||||
{:ok, _, _} =
|
||||
Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
|
||||
Ecto.Migrator.run(repo, :up, log: false, all: true)
|
||||
end)
|
||||
|
||||
Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual)
|
||||
{:ok, _} = Application.ensure_all_started(:ex_machina)
|
||||
end
|
||||
|
||||
defp silence_logger_warnings(func) do
|
||||
prev_level = Logger.level()
|
||||
Logger.configure(level: :error)
|
||||
res = func.()
|
||||
Logger.configure(level: prev_level)
|
||||
|
||||
res
|
||||
end
|
||||
|
||||
defp node_name(node_host) do
|
||||
node_host
|
||||
|> to_string()
|
||||
|> String.split("@")
|
||||
|> Enum.at(0)
|
||||
|> String.to_atom()
|
||||
end
|
||||
end
|
||||
|
|
@ -3,7 +3,8 @@
|
|||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
os_exclude = if :os.type() == {:unix, :darwin}, do: [skip_on_mac: true], else: []
|
||||
ExUnit.start(exclude: os_exclude)
|
||||
ExUnit.start(exclude: [:federated | os_exclude])
|
||||
|
||||
Ecto.Adapters.SQL.Sandbox.mode(Pleroma.Repo, :manual)
|
||||
Mox.defmock(Pleroma.ReverseProxy.ClientMock, for: Pleroma.ReverseProxy.Client)
|
||||
{:ok, _} = Application.ensure_all_started(:ex_machina)
|
||||
|
|
|
|||
Loading…
Reference in New Issue