# Pleroma: A lightweight social networking server # Copyright © 2017-2022 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.User.Backup do use Ecto.Schema import Ecto.Changeset import Ecto.Query import Pleroma.Web.Gettext require Pleroma.Constants alias Pleroma.Activity alias Pleroma.Bookmark alias Pleroma.Repo alias Pleroma.User alias Pleroma.User.Backup.State alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.UserView alias Pleroma.Workers.BackupWorker schema "backups" do field(:content_type, :string) field(:file_name, :string) field(:file_size, :integer, default: 0) field(:processed, :boolean, default: false) field(:state, State, default: :invalid) field(:processed_number, :integer, default: 0) belongs_to(:user, User, type: FlakeId.Ecto.CompatType) timestamps() end @report_every 100 def create(user, admin_id \\ nil) do with :ok <- validate_limit(user, admin_id), {:ok, backup} <- user |> new() |> Repo.insert() do BackupWorker.process(backup, admin_id) end end def new(user) do rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip" %__MODULE__{ user_id: user.id, content_type: "application/zip", file_name: name, state: :pending } end def delete(backup) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do Repo.delete(backup) end end defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok defp validate_limit(user, nil) do case get_last(user.id) do %__MODULE__{inserted_at: inserted_at} -> days = Pleroma.Config.get([__MODULE__, :limit_days]) diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) if diff > days do :ok else {:error, dngettext( "errors", "Last export was less than a day ago", "Last export was less than %{days} days ago", days, days: days )} end nil -> :ok end end def get_last(user_id) do __MODULE__ |> where(user_id: ^user_id) |> order_by(desc: :id) |> limit(1) |> Repo.one() end def list(%User{id: user_id}) do __MODULE__ |> where(user_id: ^user_id) |> order_by(desc: :id) |> Repo.all() end def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do __MODULE__ |> where(user_id: ^user_id) |> where([b], b.id != ^latest_id) |> Repo.all() |> Enum.each(&BackupWorker.delete/1) end def get(id), do: Repo.get(__MODULE__, id) defp set_state(backup, state, processed_number \\ nil) do struct = %{state: state} |> Pleroma.Maps.put_if_present(:processed_number, processed_number) backup |> cast(struct, [:state, :processed_number]) |> Repo.update() end def process(%__MODULE__{} = backup) do set_state(backup, :running, 0) current_pid = self() Task.Supervisor.async_nolink( Pleroma.TaskSupervisor, fn -> with {:ok, zip_file} <- export(backup, current_pid), {:ok, %{size: size}} <- File.stat(zip_file), {:ok, _upload} <- upload(backup, zip_file) do backup |> cast( %{ file_size: size, processed: true, state: :complete }, [:file_size, :processed, :state] ) |> Repo.update() send(current_pid, :completed) end end ) wait_backup(backup, backup.processed_number) end defp wait_backup(backup, current_processed) do receive do {:progress, new_processed} -> total_processed = current_processed + new_processed with {:ok, updated_backup} <- set_state(backup, :running, total_processed) do wait_backup(updated_backup, total_processed) else _ -> wait_backup(backup, total_processed) end :completed -> {:ok, get(backup.id)} after 30_000 -> set_state(backup, :failed) end end @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] def export(%__MODULE__{} = backup, caller_pid \\ nil) do backup = Repo.preload(backup, :user) name = String.trim_trailing(backup.file_name, ".zip") dir = dir(name) with :ok <- File.mkdir(dir), :ok <- actor(dir, backup.user, caller_pid), :ok <- statuses(dir, backup.user, caller_pid), :ok <- likes(dir, backup.user, caller_pid), :ok <- bookmarks(dir, backup.user, caller_pid), {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), {:ok, _} <- File.rm_rf(dir) do {:ok, to_string(zip_path)} end end def dir(name) do dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!() Path.join(dir, name) end def upload(%__MODULE__{} = backup, zip_path) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) upload = %Pleroma.Upload{ name: backup.file_name, tempfile: zip_path, content_type: backup.content_type, path: Path.join("backups", backup.file_name) } with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), :ok <- File.rm(zip_path) do {:ok, upload} end end defp actor(dir, user, caller_pid) do with {:ok, json} <- UserView.render("user.json", %{user: user}) |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) |> Jason.encode() do send(caller_pid, {:progress, 1}) File.write(Path.join(dir, "actor.json"), json) end end defp write_header(file, name) do IO.write( file, """ { "@context": "https://www.w3.org/ns/activitystreams", "id": "#{name}.json", "type": "OrderedCollection", "orderedItems": [ """ ) end defp should_report?(num), do: rem(num, @report_every) == 0 defp write(query, dir, name, fun, caller_pid) do path = Path.join(dir, "#{name}.json") with {:ok, file} <- File.open(path, [:write, :utf8]), :ok <- write_header(file, name) do total = query |> Pleroma.Repo.chunk_stream(100) |> Enum.reduce(0, fn i, acc -> with {:ok, data} <- fun.(i), {:ok, str} <- Jason.encode(data), :ok <- IO.write(file, str <> ",\n") do if should_report?(acc + 1) do send(caller_pid, {:progress, @report_every}) end acc + 1 else _ -> acc end end) send(caller_pid, {:progress, rem(total, @report_every)}) with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do File.close(file) end end end defp bookmarks(dir, %{id: user_id} = _user, caller_pid) do Bookmark |> where(user_id: ^user_id) |> join(:inner, [b], activity in assoc(b, :activity)) |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) |> write(dir, "bookmarks", fn a -> {:ok, a.object} end, caller_pid) end defp likes(dir, user, caller_pid) do user.ap_id |> Activity.Queries.by_actor() |> Activity.Queries.by_type("Like") |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) |> write(dir, "likes", fn a -> {:ok, a.object} end, caller_pid) end defp statuses(dir, user, caller_pid) do opts = %{} |> Map.put(:type, ["Create", "Announce"]) |> Map.put(:actor_id, user.ap_id) [ [Pleroma.Constants.as_public(), user.ap_id], User.following(user), Pleroma.List.memberships(user) ] |> Enum.concat() |> ActivityPub.fetch_activities_query(opts) |> write( dir, "outbox", fn a -> with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do {:ok, Map.delete(activity, "@context")} end end, caller_pid ) end end