moon
/
spc-pleroma
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'issue/1798' into 'develop' 

[#1798] fixed migration the settings to DB

See merge request pleroma/pleroma!2655
This commit is contained in:
lain 2020-06-23 10:54:15 +00:00
parent 59e2a1018a 84aa9c78dd
commit b7f297f0bd
9 changed files with 211 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lib/mix/tasks/pleroma/config.ex
View File

@ -52,6 +52,7 @@ def migrate_to_db(file_path \\ nil) do
defp do_migrate_to_db(config_file) do
if File.exists?(config_file) do
shell_info("Migrating settings from file: #{Path.expand(config_file)}")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")

2
lib/pleroma/application.ex
View File

@ -39,7 +39,7 @@ def start(_type, _args) do
Pleroma.HTML.compile_scrubbers()
Config.DeprecationWarnings.warn()
Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
Pleroma.Repo.check_migrations_applied!()
Pleroma.ApplicationRequirements.verify!()
setup_instrumenters()
load_custom_modules()

107
lib/pleroma/application_requirements.ex Normal file
View File

@ -0,0 +1,107 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ApplicationRequirements do
@moduledoc """
The module represents the collection of validations to runs before start server.
"""
defmodule VerifyError, do: defexception([:message])
import Ecto.Query
require Logger
@spec verify!() :: :ok | VerifyError.t()
def verify! do
:ok
|> check_migrations_applied!()
|> check_rum!()
|> handle_result()
end
defp handle_result(:ok), do: :ok
defp handle_result({:error, message}), do: raise(VerifyError, message: message)
# Checks for pending migrations.
#
def check_migrations_applied!(:ok) do
unless Pleroma.Config.get(
[:i_am_aware_this_may_cause_data_loss, :disable_migration_check],
false
) do
{_, res, _} =
Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
down_migrations =
Ecto.Migrator.migrations(repo)
|> Enum.reject(fn
{:up, _, _} -> true
{:down, _, _} -> false
end)
if length(down_migrations) > 0 do
down_migrations_text =
Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end)
Logger.error(
"The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
)
{:error, "Unapplied Migrations detected"}
else
:ok
end
end)
res
else
:ok
end
end
def check_migrations_applied!(result), do: result
# Checks for settings of RUM indexes.
#
defp check_rum!(:ok) do
{_, res, _} =
Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
migrate =
from(o in "columns",
where: o.table_name == "objects",
where: o.column_name == "fts_content"
)
|> repo.exists?(prefix: "information_schema")
setting = Pleroma.Config.get([:database, :rum_enabled], false)
do_check_rum!(setting, migrate)
end)
res
end
defp check_rum!(result), do: result
defp do_check_rum!(setting, migrate) do
case {setting, migrate} do
{true, false} ->
Logger.error(
"Use `RUM` index is enabled, but were not applied migrations for it.\nIf you want to start Pleroma anyway, set\nconfig :pleroma, :database, rum_enabled: false\nOtherwise apply the following migrations:\n`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`"
)
{:error, "Unapplied RUM Migrations detected"}
{false, true} ->
Logger.error(
"Detected applied migrations to use `RUM` index, but `RUM` isn't enable in settings.\nIf you want to use `RUM`, set\nconfig :pleroma, :database, rum_enabled: true\nOtherwise roll `RUM` migrations back.\n`mix ecto.rollback --migrations-path priv/repo/optional_migrations/rum_indexing/`"
)
{:error, "RUM Migrations detected"}
_ ->
:ok
end
end
end

37
lib/pleroma/repo.ex
View File

@ -11,9 +11,7 @@ defmodule Pleroma.Repo do
import Ecto.Query
require Logger
defmodule Instrumenter do
use Prometheus.EctoInstrumenter
end
defmodule Instrumenter, do: use(Prometheus.EctoInstrumenter)
@doc """
Dynamically loads the repository url from the
@ -51,35 +49,6 @@ def get_assoc(resource, association) do
end
end
def check_migrations_applied!() do
unless Pleroma.Config.get(
[:i_am_aware_this_may_cause_data_loss, :disable_migration_check],
false
) do
Ecto.Migrator.with_repo(__MODULE__, fn repo ->
down_migrations =
Ecto.Migrator.migrations(repo)
|> Enum.reject(fn
{:up, _, _} -> true
{:down, _, _} -> false
end)
if length(down_migrations) > 0 do
down_migrations_text =
Enum.map(down_migrations, fn {:down, id, name} -> "- #{name} (#{id})\n" end)
Logger.error(
"The following migrations were not applied:\n#{down_migrations_text}If you want to start Pleroma anyway, set\nconfig :pleroma, :i_am_aware_this_may_cause_data_loss, disable_migration_check: true"
)
raise Pleroma.Repo.UnappliedMigrationsError
end
end)
else
:ok
end
end
def chunk_stream(query, chunk_size) do
# We don't actually need start and end funcitons of resource streaming,
# but it seems to be the only way to not fetch records one-by-one and
@ -107,7 +76,3 @@ def chunk_stream(query, chunk_size) do
)
end
end
defmodule Pleroma.Repo.UnappliedMigrationsError do
defexception message: "Unapplied Migrations detected"
end

4
priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
View File

@ -10,8 +10,8 @@ def up do
execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$
begin
new.fts_content := to_tsvector('english', new.data->>'content');
return new;
new.fts_content := to_tsvector('english', new.data->>'content');
return new;
end
$$ LANGUAGE plpgsql")
execute("create index if not exists objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');")

96
test/application_requirements_test.exs Normal file
View File

@ -0,0 +1,96 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ApplicationRequirementsTest do
use Pleroma.DataCase
import ExUnit.CaptureLog
import Mock
alias Pleroma.Repo
describe "check_rum!" do
setup_with_mocks([
{Pleroma.ApplicationRequirements, [:passthrough],
[check_migrations_applied!: fn _ -> :ok end]}
]) do
:ok
end
setup do: clear_config([:database, :rum_enabled])
test "raises if rum is enabled and detects unapplied rum migrations" do
Pleroma.Config.put([:database, :rum_enabled], true)
with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> false end]}]) do
assert_raise Pleroma.ApplicationRequirements.VerifyError,
"Unapplied RUM Migrations detected",
fn ->
capture_log(&Pleroma.ApplicationRequirements.verify!/0)
end
end
end
test "raises if rum is disabled and detects rum migrations" do
Pleroma.Config.put([:database, :rum_enabled], false)
with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> true end]}]) do
assert_raise Pleroma.ApplicationRequirements.VerifyError,
"RUM Migrations detected",
fn ->
capture_log(&Pleroma.ApplicationRequirements.verify!/0)
end
end
end
test "doesn't do anything if rum enabled and applied migrations" do
Pleroma.Config.put([:database, :rum_enabled], true)
with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> true end]}]) do
assert Pleroma.ApplicationRequirements.verify!() == :ok
end
end
test "doesn't do anything if rum disabled" do
Pleroma.Config.put([:database, :rum_enabled], false)
with_mocks([{Repo, [:passthrough], [exists?: fn _, _ -> false end]}]) do
assert Pleroma.ApplicationRequirements.verify!() == :ok
end
end
end
describe "check_migrations_applied!" do
setup_with_mocks([
{Ecto.Migrator, [],
[
with_repo: fn repo, fun -> passthrough([repo, fun]) end,
migrations: fn Repo ->
[
{:up, 20_191_128_153_944, "fix_missing_following_count"},
{:up, 20_191_203_043_610, "create_report_notes"},
{:down, 20_191_220_174_645, "add_scopes_to_pleroma_feo_auth_records"}
]
end
]}
]) do
:ok
end
setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check])
test "raises if it detects unapplied migrations" do
assert_raise Pleroma.ApplicationRequirements.VerifyError,
"Unapplied Migrations detected",
fn ->
capture_log(&Pleroma.ApplicationRequirements.verify!/0)
end
end
test "doesn't do anything if disabled" do
Pleroma.Config.put([:i_am_aware_this_may_cause_data_loss, :disable_migration_check], true)
assert :ok == Pleroma.ApplicationRequirements.verify!()
end
end
end

2
test/fixtures/config/temp.secret.exs vendored
View File

@ -9,3 +9,5 @@
config :pleroma, Pleroma.Repo, pool: Ecto.Adapters.SQL.Sandbox
config :postgrex, :json_library, Poison
config :pleroma, :database, rum_enabled: true

34
test/repo_test.exs
View File

@ -4,9 +4,7 @@
defmodule Pleroma.RepoTest do
use Pleroma.DataCase
import ExUnit.CaptureLog
import Pleroma.Factory
import Mock
alias Pleroma.User
@ -49,36 +47,4 @@ test "return error if has not assoc " do
assert Repo.get_assoc(token, :user) == {:error, :not_found}
end
end
describe "check_migrations_applied!" do
setup_with_mocks([
{Ecto.Migrator, [],
[
with_repo: fn repo, fun -> passthrough([repo, fun]) end,
migrations: fn Pleroma.Repo ->
[
{:up, 20_191_128_153_944, "fix_missing_following_count"},
{:up, 20_191_203_043_610, "create_report_notes"},
{:down, 20_191_220_174_645, "add_scopes_to_pleroma_feo_auth_records"}
]
end
]}
]) do
:ok
end
setup do: clear_config([:i_am_aware_this_may_cause_data_loss, :disable_migration_check])
test "raises if it detects unapplied migrations" do
assert_raise Pleroma.Repo.UnappliedMigrationsError, fn ->
capture_log(&Repo.check_migrations_applied!/0)
end
end
test "doesn't do anything if disabled" do
Pleroma.Config.put([:i_am_aware_this_may_cause_data_loss, :disable_migration_check], true)
assert :ok == Repo.check_migrations_applied!()
end
end
end

1
test/tasks/config_test.exs
View File

@ -50,6 +50,7 @@ test "filtered settings are migrated to db" do
config3 = ConfigDB.get_by_params(%{group: ":quack", key: ":level"})
refute ConfigDB.get_by_params(%{group: ":pleroma", key: "Pleroma.Repo"})
refute ConfigDB.get_by_params(%{group: ":postgrex", key: ":json_library"})
refute ConfigDB.get_by_params(%{group: ":pleroma", key: ":database"})
assert config1.value == [key: "value", key2: [Repo]]
assert config2.value == [key: "value2", key2: ["Activity"]]