Merge develop

CHANGELOG.md

@@ -5,10 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
-### Added
-- Experimental websocket-based federation between Pleroma instances.
-- User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
-
 ### Changed
 
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
@@ -19,6 +15,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 - Media preview proxy (requires media proxy be enabled; see `:media_preview_proxy` config for more details).
+- Pleroma API: Importing the mutes users from CSV files.
+- Experimental websocket-based federation between Pleroma instances.
+- Admin API: Importing emoji from a zip file
+- User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
 
 ### Removed
 
@@ -28,6 +28,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Removed `:managed_config` option. In practice, it was accidentally removed with 2.0.0 release when frontends were
 switched to a new configuration mechanism, however it was not officially removed until now.
 
+### Fixed
+- Allow sending out emails again.
+
 ## [2.1.2] - 2020-09-17
 
 ### Security

README.md

@@ -18,15 +18,16 @@ If you are running Linux (glibc or musl) on x86/arm, the recommended way to inst
 ### From Source
 If your platform is not supported, or you just want to be able to edit the source code easily, you may install Pleroma from source.
 
-- [Debian-based](https://docs-develop.pleroma.social/backend/installation/debian_based_en/)
-- [Debian-based (jp)](https://docs-develop.pleroma.social/backend/installation/debian_based_jp/)
 - [Alpine Linux](https://docs-develop.pleroma.social/backend/installation/alpine_linux_en/)
 - [Arch Linux](https://docs-develop.pleroma.social/backend/installation/arch_linux_en/)
+- [CentOS 7](https://docs-develop.pleroma.social/backend/installation/centos7_en/)
+- [Debian-based](https://docs-develop.pleroma.social/backend/installation/debian_based_en/)
+- [Debian-based (jp)](https://docs-develop.pleroma.social/backend/installation/debian_based_jp/)
+- [FreeBSD](https://docs-develop.pleroma.social/backend/installation/freebsd_en/)
 - [Gentoo Linux](https://docs-develop.pleroma.social/backend/installation/gentoo_en/)
 - [NetBSD](https://docs-develop.pleroma.social/backend/installation/netbsd_en/)
 - [OpenBSD](https://docs-develop.pleroma.social/backend/installation/openbsd_en/)
 - [OpenBSD (fi)](https://docs-develop.pleroma.social/backend/installation/openbsd_fi/)
-- [CentOS 7](https://docs-develop.pleroma.social/backend/installation/centos7_en/)
 
 ### OS/Distro packages
 Currently Pleroma is not packaged by any OS/Distros, but if you want to package it for one, we can guide you through the process on our [community channels](#community-channels). If you want to change default options in your Pleroma package, please **discuss it with us first**.

config/description.exs

@@ -2445,7 +2445,7 @@
   %{
     group: :pleroma,
     key: Pleroma.Formatter,
-    label: "Auto Linker",
+    label: "Linkify",
     type: :group,
     description:
       "Configuration for Pleroma's link formatter which parses mentions, hashtags, and URLs.",

docs/API/pleroma_api.md

@@ -44,6 +44,22 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: HTTP 200 on success, 500 on error
 * Note: Users that can't be followed are silently skipped.
 
+## `/api/pleroma/blocks_import`
+### Imports your blocks.
+* Method: `POST`
+* Authentication: required
+* Params:
+    * `list`: STRING or FILE containing a whitespace-separated list of accounts to block
+* Response: HTTP 200 on success, 500 on error
+
+## `/api/pleroma/mutes_import`
+### Imports your mutes.
+* Method: `POST`
+* Authentication: required
+* Params:
+    * `list`: STRING or FILE containing a whitespace-separated list of accounts to mute
+* Response: HTTP 200 on success, 500 on error
+
 ## `/api/pleroma/captcha`
 ### Get a new captcha
 * Method: `GET`

installation/pleroma.nginx

@@ -9,6 +9,12 @@
 proxy_cache_path /tmp/pleroma-media-cache levels=1:2 keys_zone=pleroma_media_cache:10m max_size=10g
                  inactive=720m use_temp_path=off;
 
+# this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+# and `localhost.` resolves to [::0] on some systems: see issue #930
+upstream phoenix {
+    server 127.0.0.1:4000 max_fails=5 fail_timeout=60s;
+}
+
 server {
     server_name    example.tld;
 
@@ -63,19 +69,16 @@ server {
 
     # the nginx default is 1m, not enough for large media uploads
     client_max_body_size 16m;
+    ignore_invalid_headers off;
 
-    location / {
     proxy_http_version 1.1;
     proxy_set_header Upgrade $http_upgrade;
     proxy_set_header Connection "upgrade";
     proxy_set_header Host $http_host;
     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
-        # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+    location / {
-        # and `localhost.` resolves to [::0] on some systems: see issue #930
+        proxy_pass http://phoenix;
-        proxy_pass http://127.0.0.1:4000;
-
-        client_max_body_size 16m;
     }
 
     location ~ ^/(media|proxy) {
@@ -83,12 +86,16 @@ server {
         slice              1m;
         proxy_cache_key    $host$uri$is_args$args$slice_range;
         proxy_set_header   Range $slice_range;
-        proxy_http_version 1.1;
         proxy_cache_valid  200 206 301 304 1h;
         proxy_cache_lock   on;
         proxy_ignore_client_abort on;
         proxy_buffering    on;
         chunked_transfer_encoding on;
-        proxy_pass         http://127.0.0.1:4000;
+        proxy_pass         http://phoenix;
+    }
+
+    location /api/fedsocket/v1 {
+        proxy_request_buffering off;
+        proxy_pass http://phoenix/api/fedsocket/v1;
     }
 }

lib/pleroma/emails/mailer.ex

@@ -35,6 +35,11 @@ def perform(:deliver_async, email, config), do: deliver(email, config)
   def deliver(email, config \\ [])
 
   def deliver(email, config) do
+    # temporary hackney fix until hackney max_connections bug is fixed
+    # https://git.pleroma.social/pleroma/pleroma/-/issues/2101
+    email =
+      Swoosh.Email.put_private(email, :hackney_options, ssl_options: [versions: [:"tlsv1.2"]])
+
     case enabled?() do
       true -> Swoosh.Mailer.deliver(email, parse_config(config))
       false -> {:error, :deliveries_disabled}

lib/pleroma/emoji.ex

@@ -56,6 +56,9 @@ def get(name) do
     end
   end
 
+  @spec exist?(String.t()) :: boolean()
+  def exist?(name), do: not is_nil(get(name))
+
   @doc "Returns all the emojos!!"
   @spec get_all() :: list({String.t(), String.t(), String.t()})
   def get_all do

lib/pleroma/emoji/pack.ex

@@ -17,6 +17,7 @@ defmodule Pleroma.Emoji.Pack do
         }
 
   alias Pleroma.Emoji
+  alias Pleroma.Emoji.Pack
 
   @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
   def create(name) do
@@ -64,24 +65,93 @@ def delete(name) do
     end
   end
 
-  @spec add_file(String.t(), String.t(), Path.t(), Plug.Upload.t() | String.t()) ::
+  @spec unpack_zip_emojies(list(tuple())) :: list(map())
-          {:ok, t()} | {:error, File.posix() | atom()}
+  defp unpack_zip_emojies(zip_files) do
-  def add_file(name, shortcode, filename, file) do
+    Enum.reduce(zip_files, [], fn
-    with :ok <- validate_not_empty([name, shortcode, filename]),
+      {_, path, s, _, _, _}, acc when elem(s, 2) == :regular ->
+        with(
+          filename <- Path.basename(path),
+          shortcode <- Path.basename(filename, Path.extname(filename)),
+          false <- Emoji.exist?(shortcode)
+        ) do
+          [%{path: path, filename: path, shortcode: shortcode} | acc]
+        else
+          _ -> acc
+        end
+
+      _, acc ->
+        acc
+    end)
+  end
+
+  @spec add_file(t(), String.t(), Path.t(), Plug.Upload.t()) ::
+          {:ok, t()}
+          | {:error, File.posix() | atom()}
+  def add_file(%Pack{} = pack, _, _, %Plug.Upload{content_type: "application/zip"} = file) do
+    with {:ok, zip_files} <- :zip.table(to_charlist(file.path)),
+         [_ | _] = emojies <- unpack_zip_emojies(zip_files),
+         {:ok, tmp_dir} <- Pleroma.Utils.tmp_dir("emoji") do
+      try do
+        {:ok, _emoji_files} =
+          :zip.unzip(
+            to_charlist(file.path),
+            [{:file_list, Enum.map(emojies, & &1[:path])}, {:cwd, tmp_dir}]
+          )
+
+        {_, updated_pack} =
+          Enum.map_reduce(emojies, pack, fn item, emoji_pack ->
+            emoji_file = %Plug.Upload{
+              filename: item[:filename],
+              path: Path.join(tmp_dir, item[:path])
+            }
+
+            {:ok, updated_pack} =
+              do_add_file(
+                emoji_pack,
+                item[:shortcode],
+                to_string(item[:filename]),
+                emoji_file
+              )
+
+            {item, updated_pack}
+          end)
+
+        Emoji.reload()
+
+        {:ok, updated_pack}
+      after
+        File.rm_rf(tmp_dir)
+      end
+    else
+      {:error, _} = error ->
+        error
+
+      _ ->
+        {:ok, pack}
+    end
+  end
+
+  def add_file(%Pack{} = pack, shortcode, filename, %Plug.Upload{} = file) do
+    with :ok <- validate_not_empty([shortcode, filename]),
          :ok <- validate_emoji_not_exists(shortcode),
-         {:ok, pack} <- load_pack(name),
+         {:ok, updated_pack} <- do_add_file(pack, shortcode, filename, file) do
-         :ok <- save_file(file, pack, filename),
-         {:ok, updated_pack} <- pack |> put_emoji(shortcode, filename) |> save_pack() do
       Emoji.reload()
       {:ok, updated_pack}
     end
   end
 
-  @spec delete_file(String.t(), String.t()) ::
+  defp do_add_file(pack, shortcode, filename, file) do
+    with :ok <- save_file(file, pack, filename) do
+      pack
+      |> put_emoji(shortcode, filename)
+      |> save_pack()
+    end
+  end
+
+  @spec delete_file(t(), String.t()) ::
           {:ok, t()} | {:error, File.posix() | atom()}
-  def delete_file(name, shortcode) do
+  def delete_file(%Pack{} = pack, shortcode) do
-    with :ok <- validate_not_empty([name, shortcode]),
+    with :ok <- validate_not_empty([shortcode]),
-         {:ok, pack} <- load_pack(name),
          :ok <- remove_file(pack, shortcode),
          {:ok, updated_pack} <- pack |> delete_emoji(shortcode) |> save_pack() do
       Emoji.reload()
@@ -89,11 +159,10 @@ def delete_file(name, shortcode) do
     end
   end
 
-  @spec update_file(String.t(), String.t(), String.t(), String.t(), boolean()) ::
+  @spec update_file(t(), String.t(), String.t(), String.t(), boolean()) ::
           {:ok, t()} | {:error, File.posix() | atom()}
-  def update_file(name, shortcode, new_shortcode, new_filename, force) do
+  def update_file(%Pack{} = pack, shortcode, new_shortcode, new_filename, force) do
-    with :ok <- validate_not_empty([name, shortcode, new_shortcode, new_filename]),
+    with :ok <- validate_not_empty([shortcode, new_shortcode, new_filename]),
-         {:ok, pack} <- load_pack(name),
          {:ok, filename} <- get_filename(pack, shortcode),
          :ok <- validate_emoji_not_exists(new_shortcode, force),
          :ok <- rename_file(pack, filename, new_filename),
@@ -243,9 +312,10 @@ defp validate_emoji_not_exists(shortcode, force \\ false)
   defp validate_emoji_not_exists(_shortcode, true), do: :ok
 
   defp validate_emoji_not_exists(shortcode, _) do
-    case Emoji.get(shortcode) do
+    if Emoji.exist?(shortcode) do
-      nil -> :ok
+      {:error, :already_exists}
-      _ -> {:error, :already_exists}
+    else
+      :ok
     end
   end
 
@@ -386,25 +456,18 @@ defp validate_not_empty(list) do
     end
   end
 
-  defp save_file(file, pack, filename) do
+  defp save_file(%Plug.Upload{path: upload_path}, pack, filename) do
     file_path = Path.join(pack.path, filename)
     create_subdirs(file_path)
 
-    case file do
+    with {:ok, _} <- File.copy(upload_path, file_path) do
-      %Plug.Upload{path: upload_path} ->
+      :ok
-        # Copy the uploaded file from the temporary directory
-        with {:ok, _} <- File.copy(upload_path, file_path), do: :ok
-
-      url when is_binary(url) ->
-        # Download and write the file
-        file_contents = Tesla.get!(url).body
-        File.write(file_path, file_contents)
     end
   end
 
   defp put_emoji(pack, shortcode, filename) do
     files = Map.put(pack.files, shortcode, filename)
-    %{pack | files: files}
+    %{pack | files: files, files_count: length(Map.keys(files))}
   end
 
   defp delete_emoji(pack, shortcode) do

lib/pleroma/plugs/oauth_scopes_plug.ex

@@ -53,7 +53,7 @@ def drop_auth_info(conn) do
     |> assign(:token, nil)
   end
 
-  @doc "Filters descendants of supported scopes"
+  @doc "Keeps those of `scopes` which are descendants of `supported_scopes`"
   def filter_descendants(scopes, supported_scopes) do
     Enum.filter(
       scopes,

lib/pleroma/user.ex

@@ -1719,42 +1719,6 @@ def perform(:delete, %User{} = user) do
 
   def perform(:deactivate_async, user, status), do: deactivate(user, status)
 
-  @spec perform(atom(), User.t(), list()) :: list() | {:error, any()}
-  def perform(:blocks_import, %User{} = blocker, blocked_identifiers)
-      when is_list(blocked_identifiers) do
-    Enum.map(
-      blocked_identifiers,
-      fn blocked_identifier ->
-        with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier),
-             {:ok, _block} <- CommonAPI.block(blocker, blocked) do
-          blocked
-        else
-          err ->
-            Logger.debug("blocks_import failed for #{blocked_identifier} with: #{inspect(err)}")
-            err
-        end
-      end
-    )
-  end
-
-  def perform(:follow_import, %User{} = follower, followed_identifiers)
-      when is_list(followed_identifiers) do
-    Enum.map(
-      followed_identifiers,
-      fn followed_identifier ->
-        with {:ok, %User{} = followed} <- get_or_fetch(followed_identifier),
-             {:ok, follower} <- maybe_direct_follow(follower, followed),
-             {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do
-          followed
-        else
-          err ->
-            Logger.debug("follow_import failed for #{followed_identifier} with: #{inspect(err)}")
-            err
-        end
-      end
-    )
-  end
-
   @spec external_users_query() :: Ecto.Query.t()
   def external_users_query do
     User.Query.build(%{
@@ -1783,21 +1747,6 @@ def external_users(opts \\ []) do
     Repo.all(query)
   end
 
-  def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do
-    BackgroundWorker.enqueue("blocks_import", %{
-      "blocker_id" => blocker.id,
-      "blocked_identifiers" => blocked_identifiers
-    })
-  end
-
-  def follow_import(%User{} = follower, followed_identifiers)
-      when is_list(followed_identifiers) do
-    BackgroundWorker.enqueue("follow_import", %{
-      "follower_id" => follower.id,
-      "followed_identifiers" => followed_identifiers
-    })
-  end
-
   def delete_notifications_from_user_activities(%User{ap_id: ap_id}) do
     Notification
     |> join(:inner, [n], activity in assoc(n, :activity))

lib/pleroma/user/import.ex

@@ -0,0 +1,85 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.Import do
+  use Ecto.Schema
+
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Workers.BackgroundWorker
+
+  require Logger
+
+  @spec perform(atom(), User.t(), list()) :: :ok | list() | {:error, any()}
+  def perform(:mutes_import, %User{} = user, [_ | _] = identifiers) do
+    Enum.map(
+      identifiers,
+      fn identifier ->
+        with {:ok, %User{} = muted_user} <- User.get_or_fetch(identifier),
+             {:ok, _} <- User.mute(user, muted_user) do
+          muted_user
+        else
+          error -> handle_error(:mutes_import, identifier, error)
+        end
+      end
+    )
+  end
+
+  def perform(:blocks_import, %User{} = blocker, [_ | _] = identifiers) do
+    Enum.map(
+      identifiers,
+      fn identifier ->
+        with {:ok, %User{} = blocked} <- User.get_or_fetch(identifier),
+             {:ok, _block} <- CommonAPI.block(blocker, blocked) do
+          blocked
+        else
+          error -> handle_error(:blocks_import, identifier, error)
+        end
+      end
+    )
+  end
+
+  def perform(:follow_import, %User{} = follower, [_ | _] = identifiers) do
+    Enum.map(
+      identifiers,
+      fn identifier ->
+        with {:ok, %User{} = followed} <- User.get_or_fetch(identifier),
+             {:ok, follower} <- User.maybe_direct_follow(follower, followed),
+             {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do
+          followed
+        else
+          error -> handle_error(:follow_import, identifier, error)
+        end
+      end
+    )
+  end
+
+  def perform(_, _, _), do: :ok
+
+  defp handle_error(op, user_id, error) do
+    Logger.debug("#{op} failed for #{user_id} with: #{inspect(error)}")
+    error
+  end
+
+  def blocks_import(%User{} = blocker, [_ | _] = identifiers) do
+    BackgroundWorker.enqueue(
+      "blocks_import",
+      %{"user_id" => blocker.id, "identifiers" => identifiers}
+    )
+  end
+
+  def follow_import(%User{} = follower, [_ | _] = identifiers) do
+    BackgroundWorker.enqueue(
+      "follow_import",
+      %{"user_id" => follower.id, "identifiers" => identifiers}
+    )
+  end
+
+  def mutes_import(%User{} = user, [_ | _] = identifiers) do
+    BackgroundWorker.enqueue(
+      "mutes_import",
+      %{"user_id" => user.id, "identifiers" => identifiers}
+    )
+  end
+end

lib/pleroma/utils.ex

@@ -24,4 +24,24 @@ def compile_dir(dir) when is_binary(dir) do
   def command_available?(command) do
     match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"]))
   end
+
+  @doc "creates the uniq temporary directory"
+  @spec tmp_dir(String.t()) :: {:ok, String.t()} | {:error, :file.posix()}
+  def tmp_dir(prefix \\ "") do
+    sub_dir =
+      [
+        prefix,
+        Timex.to_unix(Timex.now()),
+        :os.getpid(),
+        String.downcase(Integer.to_string(:rand.uniform(0x100000000), 36))
+      ]
+      |> Enum.join("-")
+
+    tmp_dir = Path.join(System.tmp_dir!(), sub_dir)
+
+    case File.mkdir(tmp_dir) do
+      :ok -> {:ok, tmp_dir}
+      error -> error
+    end
+  end
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -841,7 +841,14 @@ defp restrict_blocked(query, %{blocking_user: %User{} = user} = opts) do
     from(
       [activity, object: o] in query,
       where: fragment("not (? = ANY(?))", activity.actor, ^blocked_ap_ids),
-      where: fragment("not (? && ?)", activity.recipients, ^blocked_ap_ids),
+      where:
+        fragment(
+          "((not (? && ?)) or ? = ?)",
+          activity.recipients,
+          ^blocked_ap_ids,
+          activity.actor,
+          ^user.ap_id
+        ),
       where:
         fragment(
           "recipients_contain_blocked_domains(?, ?) = false",

lib/pleroma/web/api_spec/operations/pleroma_emoji_file_operation.ex

@@ -0,0 +1,139 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaEmojiFileOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def create_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Add new file to the pack",
+      operationId: "PleromaAPI.EmojiPackController.add_file",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: request_body("Parameters", create_request(), required: true),
+      parameters: [name_param()],
+      responses: %{
+        200 => Operation.response("Files Object", "application/json", files_object()),
+        422 => Operation.response("Unprocessable Entity", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        409 => Operation.response("Conflict", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp create_request do
+    %Schema{
+      type: :object,
+      required: [:file],
+      properties: %{
+        file: %Schema{
+          description:
+            "File needs to be uploaded with the multipart request or link to remote file",
+          anyOf: [
+            %Schema{type: :string, format: :binary},
+            %Schema{type: :string, format: :uri}
+          ]
+        },
+        shortcode: %Schema{
+          type: :string,
+          description:
+            "Shortcode for new emoji, must be unique for all emoji. If not sended, shortcode will be taken from original filename."
+        },
+        filename: %Schema{
+          type: :string,
+          description:
+            "New emoji file name. If not specified will be taken from original filename."
+        }
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Add new file to the pack",
+      operationId: "PleromaAPI.EmojiPackController.update_file",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: request_body("Parameters", update_request(), required: true),
+      parameters: [name_param()],
+      responses: %{
+        200 => Operation.response("Files Object", "application/json", files_object()),
+        404 => Operation.response("Not Found", "application/json", ApiError),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        409 => Operation.response("Conflict", "application/json", ApiError),
+        422 => Operation.response("Unprocessable Entity", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      type: :object,
+      required: [:shortcode, :new_shortcode, :new_filename],
+      properties: %{
+        shortcode: %Schema{
+          type: :string,
+          description: "Emoji file shortcode"
+        },
+        new_shortcode: %Schema{
+          type: :string,
+          description: "New emoji file shortcode"
+        },
+        new_filename: %Schema{
+          type: :string,
+          description: "New filename for emoji file"
+        },
+        force: %Schema{
+          type: :boolean,
+          description: "With true value to overwrite existing emoji with new shortcode",
+          default: false
+        }
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Emoji Packs"],
+      summary: "Delete emoji file from pack",
+      operationId: "PleromaAPI.EmojiPackController.delete_file",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [
+        name_param(),
+        Operation.parameter(:shortcode, :query, :string, "File shortcode",
+          example: "cofe",
+          required: true
+        )
+      ],
+      responses: %{
+        200 => Operation.response("Files Object", "application/json", files_object()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError),
+        422 => Operation.response("Unprocessable Entity", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp name_param do
+    Operation.parameter(:name, :path, :string, "Pack Name", example: "cofe", required: true)
+  end
+
+  defp files_object do
+    %Schema{
+      type: :object,
+      additionalProperties: %Schema{type: :string},
+      description: "Object with emoji names as keys and filenames as values"
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/pleroma_emoji_pack_operation.ex

@@ -175,111 +175,6 @@ def update_operation do
     }
   end
 
-  def add_file_operation do
-    %Operation{
-      tags: ["Emoji Packs"],
-      summary: "Add new file to the pack",
-      operationId: "PleromaAPI.EmojiPackController.add_file",
-      security: [%{"oAuth" => ["write"]}],
-      requestBody: request_body("Parameters", add_file_request(), required: true),
-      parameters: [name_param()],
-      responses: %{
-        200 => Operation.response("Files Object", "application/json", files_object()),
-        400 => Operation.response("Bad Request", "application/json", ApiError),
-        409 => Operation.response("Conflict", "application/json", ApiError)
-      }
-    }
-  end
-
-  defp add_file_request do
-    %Schema{
-      type: :object,
-      required: [:file],
-      properties: %{
-        file: %Schema{
-          description:
-            "File needs to be uploaded with the multipart request or link to remote file",
-          anyOf: [
-            %Schema{type: :string, format: :binary},
-            %Schema{type: :string, format: :uri}
-          ]
-        },
-        shortcode: %Schema{
-          type: :string,
-          description:
-            "Shortcode for new emoji, must be unique for all emoji. If not sended, shortcode will be taken from original filename."
-        },
-        filename: %Schema{
-          type: :string,
-          description:
-            "New emoji file name. If not specified will be taken from original filename."
-        }
-      }
-    }
-  end
-
-  def update_file_operation do
-    %Operation{
-      tags: ["Emoji Packs"],
-      summary: "Add new file to the pack",
-      operationId: "PleromaAPI.EmojiPackController.update_file",
-      security: [%{"oAuth" => ["write"]}],
-      requestBody: request_body("Parameters", update_file_request(), required: true),
-      parameters: [name_param()],
-      responses: %{
-        200 => Operation.response("Files Object", "application/json", files_object()),
-        400 => Operation.response("Bad Request", "application/json", ApiError),
-        409 => Operation.response("Conflict", "application/json", ApiError)
-      }
-    }
-  end
-
-  defp update_file_request do
-    %Schema{
-      type: :object,
-      required: [:shortcode, :new_shortcode, :new_filename],
-      properties: %{
-        shortcode: %Schema{
-          type: :string,
-          description: "Emoji file shortcode"
-        },
-        new_shortcode: %Schema{
-          type: :string,
-          description: "New emoji file shortcode"
-        },
-        new_filename: %Schema{
-          type: :string,
-          description: "New filename for emoji file"
-        },
-        force: %Schema{
-          type: :boolean,
-          description: "With true value to overwrite existing emoji with new shortcode",
-          default: false
-        }
-      }
-    }
-  end
-
-  def delete_file_operation do
-    %Operation{
-      tags: ["Emoji Packs"],
-      summary: "Delete emoji file from pack",
-      operationId: "PleromaAPI.EmojiPackController.delete_file",
-      security: [%{"oAuth" => ["write"]}],
-      parameters: [
-        name_param(),
-        Operation.parameter(:shortcode, :query, :string, "File shortcode",
-          example: "cofe",
-          required: true
-        )
-      ],
-      responses: %{
-        200 => Operation.response("Files Object", "application/json", files_object()),
-        400 => Operation.response("Bad Request", "application/json", ApiError)
-      }
-    }
-  end
-
   def import_from_filesystem_operation do
     %Operation{
       tags: ["Emoji Packs"],

lib/pleroma/web/api_spec/operations/user_import_operation.ex

@@ -0,0 +1,80 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.UserImportOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  @spec open_api_operation(atom) :: Operation.t()
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def follow_operation do
+    %Operation{
+      tags: ["follow_import"],
+      summary: "Imports your follows.",
+      operationId: "UserImportController.follow",
+      requestBody: request_body("Parameters", import_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:follow"]}]
+    }
+  end
+
+  def blocks_operation do
+    %Operation{
+      tags: ["blocks_import"],
+      summary: "Imports your blocks.",
+      operationId: "UserImportController.blocks",
+      requestBody: request_body("Parameters", import_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:blocks"]}]
+    }
+  end
+
+  def mutes_operation do
+    %Operation{
+      tags: ["mutes_import"],
+      summary: "Imports your mutes.",
+      operationId: "UserImportController.mutes",
+      requestBody: request_body("Parameters", import_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:mutes"]}]
+    }
+  end
+
+  defp import_request do
+    %Schema{
+      type: :object,
+      required: [:list],
+      properties: %{
+        list: %Schema{
+          description:
+            "STRING or FILE containing a whitespace-separated list of accounts to import.",
+          anyOf: [
+            %Schema{type: :string, format: :binary},
+            %Schema{type: :string}
+          ]
+        }
+      }
+    }
+  end
+
+  defp ok_response do
+    Operation.response("Ok", "application/json", %Schema{type: :string, example: "ok"})
+  end
+end

lib/pleroma/web/fed_sockets/outgoing_handler.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.FedSockets.OutgoingHandler do
 
   require Logger
 
+  alias Pleroma.Application
   alias Pleroma.Web.ActivityPub.InternalFetchActor
   alias Pleroma.Web.FedSockets
   alias Pleroma.Web.FedSockets.FedRegistry
@@ -85,9 +86,12 @@ def initiate_connection(uri) do
 
     %{host: host, port: port, path: path} = URI.parse(ws_uri)
 
-    with {:ok, conn_pid} <- :gun.open(to_charlist(host), port),
+    with {:ok, conn_pid} <- :gun.open(to_charlist(host), port, %{protocols: [:http]}),
          {:ok, _} <- :gun.await_up(conn_pid),
-         reference <- :gun.get(conn_pid, to_charlist(path)),
+         reference <-
+           :gun.get(conn_pid, to_charlist(path), [
+             {'user-agent', to_charlist(Application.user_agent())}
+           ]),
          {:response, :fin, 204, _} <- :gun.await(conn_pid, reference),
          headers <- build_headers(uri),
          ref <- :gun.ws_upgrade(conn_pid, to_charlist(path), headers, %{silence_pings: false}) do
@@ -132,7 +136,8 @@ defp build_headers(uri) do
       {'date', date},
       {'digest', to_charlist(digest)},
       {'content-length', to_charlist("#{shake_size}")},
-      {to_charlist("(request-target)"), to_charlist(shake)}
+      {to_charlist("(request-target)"), to_charlist(shake)},
+      {'user-agent', to_charlist(Application.user_agent())}
     ]
   end
 

lib/pleroma/web/federator/federator.ex

@@ -66,14 +66,17 @@ def perform(:publish, activity) do
   def perform(:incoming_ap_doc, params) do
     Logger.debug("Handling incoming AP activity")
 
-    params = Utils.normalize_params(params)
+    actor =
+      params
+      |> Map.get("actor")
+      |> Utils.get_ap_id()
 
     # NOTE: we use the actor ID to do the containment, this is fine because an
     # actor shouldn't be acting on objects outside their own AP server.
-    with {:ok, _user} <- ap_enabled_actor(params["actor"]),
+    with {_, {:ok, _user}} <- {:actor, ap_enabled_actor(actor)},
          nil <- Activity.normalize(params["id"]),
          {_, :ok} <-
-           {:correct_origin?, Containment.contain_origin_from_id(params["actor"], params)},
+           {:correct_origin?, Containment.contain_origin_from_id(actor, params)},
          {:ok, activity} <- Transmogrifier.handle_incoming(params) do
       {:ok, activity}
     else
@@ -85,10 +88,13 @@ def perform(:incoming_ap_doc, params) do
         Logger.debug("Already had #{params["id"]}")
         {:error, :already_present}
 
+      {:actor, e} ->
+        Logger.debug("Unhandled actor #{actor}, #{inspect(e)}")
+        {:error, e}
+
       e ->
         # Just drop those for now
-        Logger.debug("Unhandled activity")
+        Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end)
-        Logger.debug(Jason.encode!(params, pretty: true))
         {:error, e}
     end
   end

lib/pleroma/web/mastodon_api/controllers/auth_controller.ex

@@ -5,6 +5,8 @@
 defmodule Pleroma.Web.MastodonAPI.AuthController do
   use Pleroma.Web, :controller
 
+  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
+
   alias Pleroma.User
   alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Authorization
@@ -61,9 +63,7 @@ def password_reset(conn, params) do
 
     TwitterAPI.password_reset(nickname_or_email)
 
-    conn
+    json_response(conn, :no_content, "")
-    |> put_status(:no_content)
-    |> json("")
   end
 
   defp local_mastodon_root_path(conn) do

lib/pleroma/web/mastodon_api/websocket_handler.ex

@@ -23,8 +23,8 @@ def init(%{qs: qs} = req, state) do
     with params <- Enum.into(:cow_qs.parse_qs(qs), %{}),
          sec_websocket <- :cowboy_req.header("sec-websocket-protocol", req, nil),
          access_token <- Map.get(params, "access_token"),
-         {:ok, user} <- authenticate_request(access_token, sec_websocket),
+         {:ok, user, oauth_token} <- authenticate_request(access_token, sec_websocket),
-         {:ok, topic} <- Streamer.get_topic(Map.get(params, "stream"), user, params) do
+         {:ok, topic} <- Streamer.get_topic(params["stream"], user, oauth_token, params) do
       req =
         if sec_websocket do
           :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req)
@@ -117,7 +117,7 @@ def terminate(reason, _req, state) do
 
   # Public streams without authentication.
   defp authenticate_request(nil, nil) do
-    {:ok, nil}
+    {:ok, nil, nil}
   end
 
   # Authenticated streams.
@@ -125,9 +125,9 @@ defp authenticate_request(access_token, sec_websocket) do
     token = access_token || sec_websocket
 
     with true <- is_bitstring(token),
-         %Token{user_id: user_id} <- Repo.get_by(Token, token: token),
+         oauth_token = %Token{user_id: user_id} <- Repo.get_by(Token, token: token),
          user = %User{} <- User.get_cached_by_id(user_id) do
-      {:ok, user}
+      {:ok, user, oauth_token}
     else
       _ -> {:error, :unauthorized}
     end

lib/pleroma/web/pleroma_api/controllers/emoji_file_controller.ex

@@ -0,0 +1,133 @@
+defmodule Pleroma.Web.PleromaAPI.EmojiFileController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Emoji.Pack
+  alias Pleroma.Web.ApiSpec
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  plug(
+    Pleroma.Plugs.OAuthScopesPlug,
+    %{scopes: ["write"], admin: true}
+    when action in [
+           :create,
+           :update,
+           :delete
+         ]
+  )
+
+  defdelegate open_api_operation(action), to: ApiSpec.PleromaEmojiFileOperation
+
+  def create(%{body_params: params} = conn, %{name: pack_name}) do
+    filename = params[:filename] || get_filename(params[:file])
+    shortcode = params[:shortcode] || Path.basename(filename, Path.extname(filename))
+
+    with {:ok, pack} <- Pack.load_pack(pack_name),
+         {:ok, file} <- get_file(params[:file]),
+         {:ok, pack} <- Pack.add_file(pack, shortcode, filename, file) do
+      json(conn, pack.files)
+    else
+      {:error, :already_exists} ->
+        conn
+        |> put_status(:conflict)
+        |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"})
+
+      {:error, :empty_values} ->
+        conn
+        |> put_status(:unprocessable_entity)
+        |> json(%{error: "pack name, shortcode or filename cannot be empty"})
+
+      {:error, _} = error ->
+        handle_error(conn, error, %{pack_name: pack_name})
+    end
+  end
+
+  def update(%{body_params: %{shortcode: shortcode} = params} = conn, %{name: pack_name}) do
+    new_shortcode = params[:new_shortcode]
+    new_filename = params[:new_filename]
+    force = params[:force]
+
+    with {:ok, pack} <- Pack.load_pack(pack_name),
+         {:ok, pack} <- Pack.update_file(pack, shortcode, new_shortcode, new_filename, force) do
+      json(conn, pack.files)
+    else
+      {:error, :already_exists} ->
+        conn
+        |> put_status(:conflict)
+        |> json(%{
+          error:
+            "New shortcode \"#{new_shortcode}\" is already used. If you want to override emoji use 'force' option"
+        })
+
+      {:error, :empty_values} ->
+        conn
+        |> put_status(:unprocessable_entity)
+        |> json(%{error: "new_shortcode or new_filename cannot be empty"})
+
+      {:error, _} = error ->
+        handle_error(conn, error, %{pack_name: pack_name, code: shortcode})
+    end
+  end
+
+  def delete(conn, %{name: pack_name, shortcode: shortcode}) do
+    with {:ok, pack} <- Pack.load_pack(pack_name),
+         {:ok, pack} <- Pack.delete_file(pack, shortcode) do
+      json(conn, pack.files)
+    else
+      {:error, :empty_values} ->
+        conn
+        |> put_status(:unprocessable_entity)
+        |> json(%{error: "pack name or shortcode cannot be empty"})
+
+      {:error, _} = error ->
+        handle_error(conn, error, %{pack_name: pack_name, code: shortcode})
+    end
+  end
+
+  defp handle_error(conn, {:error, :doesnt_exist}, %{code: emoji_code}) do
+    conn
+    |> put_status(:bad_request)
+    |> json(%{error: "Emoji \"#{emoji_code}\" does not exist"})
+  end
+
+  defp handle_error(conn, {:error, :not_found}, %{pack_name: pack_name}) do
+    conn
+    |> put_status(:not_found)
+    |> json(%{error: "pack \"#{pack_name}\" is not found"})
+  end
+
+  defp handle_error(conn, {:error, _}, _) do
+    render_error(
+      conn,
+      :internal_server_error,
+      "Unexpected error occurred while adding file to pack."
+    )
+  end
+
+  defp get_filename(%Plug.Upload{filename: filename}), do: filename
+  defp get_filename(url) when is_binary(url), do: Path.basename(url)
+
+  def get_file(%Plug.Upload{} = file), do: {:ok, file}
+
+  def get_file(url) when is_binary(url) do
+    with {:ok, %Tesla.Env{body: body, status: code, headers: headers}}
+         when code in 200..299 <- Pleroma.HTTP.get(url) do
+      path = Plug.Upload.random_file!("emoji")
+
+      content_type =
+        case List.keyfind(headers, "content-type", 0) do
+          {"content-type", value} -> value
+          nil -> nil
+        end
+
+      File.write(path, body)
+
+      {:ok,
+       %Plug.Upload{
+         filename: Path.basename(url),
+         path: path,
+         content_type: content_type
+       }}
+    end
+  end
+end

lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex

@@ -14,10 +14,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do
            :download,
            :create,
            :update,
-           :delete,
+           :delete
-           :add_file,
-           :update_file,
-           :delete_file
          ]
   )
 
@@ -184,105 +181,6 @@ def update(%{body_params: %{metadata: metadata}} = conn, %{name: name}) do
     end
   end
 
-  def add_file(%{body_params: params} = conn, %{name: name}) do
-    filename = params[:filename] || get_filename(params[:file])
-    shortcode = params[:shortcode] || Path.basename(filename, Path.extname(filename))
-
-    with {:ok, pack} <- Pack.add_file(name, shortcode, filename, params[:file]) do
-      json(conn, pack.files)
-    else
-      {:error, :already_exists} ->
-        conn
-        |> put_status(:conflict)
-        |> json(%{error: "An emoji with the \"#{shortcode}\" shortcode already exists"})
-
-      {:error, :not_found} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "pack \"#{name}\" is not found"})
-
-      {:error, :empty_values} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "pack name, shortcode or filename cannot be empty"})
-
-      {:error, _} ->
-        render_error(
-          conn,
-          :internal_server_error,
-          "Unexpected error occurred while adding file to pack."
-        )
-    end
-  end
-
-  def update_file(%{body_params: %{shortcode: shortcode} = params} = conn, %{name: name}) do
-    new_shortcode = params[:new_shortcode]
-    new_filename = params[:new_filename]
-    force = params[:force]
-
-    with {:ok, pack} <- Pack.update_file(name, shortcode, new_shortcode, new_filename, force) do
-      json(conn, pack.files)
-    else
-      {:error, :doesnt_exist} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "Emoji \"#{shortcode}\" does not exist"})
-
-      {:error, :already_exists} ->
-        conn
-        |> put_status(:conflict)
-        |> json(%{
-          error:
-            "New shortcode \"#{new_shortcode}\" is already used. If you want to override emoji use 'force' option"
-        })
-
-      {:error, :not_found} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "pack \"#{name}\" is not found"})
-
-      {:error, :empty_values} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "new_shortcode or new_filename cannot be empty"})
-
-      {:error, _} ->
-        render_error(
-          conn,
-          :internal_server_error,
-          "Unexpected error occurred while updating file in pack."
-        )
-    end
-  end
-
-  def delete_file(conn, %{name: name, shortcode: shortcode}) do
-    with {:ok, pack} <- Pack.delete_file(name, shortcode) do
-      json(conn, pack.files)
-    else
-      {:error, :doesnt_exist} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "Emoji \"#{shortcode}\" does not exist"})
-
-      {:error, :not_found} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "pack \"#{name}\" is not found"})
-
-      {:error, :empty_values} ->
-        conn
-        |> put_status(:bad_request)
-        |> json(%{error: "pack name or shortcode cannot be empty"})
-
-      {:error, _} ->
-        render_error(
-          conn,
-          :internal_server_error,
-          "Unexpected error occurred while removing file from pack."
-        )
-    end
-  end
-
   def import_from_filesystem(conn, _params) do
     with {:ok, names} <- Pack.import_from_filesystem() do
       json(conn, names)
@@ -298,7 +196,4 @@ def import_from_filesystem(conn, _params) do
         |> json(%{error: "Error accessing emoji pack directory"})
     end
   end
-
-  defp get_filename(%Plug.Upload{filename: filename}), do: filename
-  defp get_filename(url) when is_binary(url), do: Path.basename(url)
 end

lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex

@@ -0,0 +1,61 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.UserImportController do
+  use Pleroma.Web, :controller
+
+  require Logger
+
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.User
+  alias Pleroma.Web.ApiSpec
+
+  plug(OAuthScopesPlug, %{scopes: ["follow", "write:follows"]} when action == :follow)
+  plug(OAuthScopesPlug, %{scopes: ["follow", "write:blocks"]} when action == :blocks)
+  plug(OAuthScopesPlug, %{scopes: ["follow", "write:mutes"]} when action == :mutes)
+
+  plug(OpenApiSpex.Plug.CastAndValidate)
+  defdelegate open_api_operation(action), to: ApiSpec.UserImportOperation
+
+  def follow(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do
+    follow(%Plug.Conn{conn | body_params: %{list: File.read!(path)}}, %{})
+  end
+
+  def follow(%{assigns: %{user: follower}, body_params: %{list: list}} = conn, _) do
+    identifiers =
+      list
+      |> String.split("\n")
+      |> Enum.map(&(&1 |> String.split(",") |> List.first()))
+      |> List.delete("Account address")
+      |> Enum.map(&(&1 |> String.trim() |> String.trim_leading("@")))
+      |> Enum.reject(&(&1 == ""))
+
+    User.Import.follow_import(follower, identifiers)
+    json(conn, "job started")
+  end
+
+  def blocks(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do
+    blocks(%Plug.Conn{conn | body_params: %{list: File.read!(path)}}, %{})
+  end
+
+  def blocks(%{assigns: %{user: blocker}, body_params: %{list: list}} = conn, _) do
+    User.Import.blocks_import(blocker, prepare_user_identifiers(list))
+    json(conn, "job started")
+  end
+
+  def mutes(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do
+    mutes(%Plug.Conn{conn | body_params: %{list: File.read!(path)}}, %{})
+  end
+
+  def mutes(%{assigns: %{user: user}, body_params: %{list: list}} = conn, _) do
+    User.Import.mutes_import(user, prepare_user_identifiers(list))
+    json(conn, "job started")
+  end
+
+  defp prepare_user_identifiers(list) do
+    list
+    |> String.split()
+    |> Enum.map(&String.trim_leading(&1, "@"))
+  end
+end

lib/pleroma/web/router.ex

@@ -238,9 +238,9 @@ defmodule Pleroma.Web.Router do
       patch("/:name", EmojiPackController, :update)
       delete("/:name", EmojiPackController, :delete)
 
-      post("/:name/files", EmojiPackController, :add_file)
+      post("/:name/files", EmojiFileController, :create)
-      patch("/:name/files", EmojiPackController, :update_file)
+      patch("/:name/files", EmojiFileController, :update)
-      delete("/:name/files", EmojiPackController, :delete_file)
+      delete("/:name/files", EmojiFileController, :delete)
     end
 
     # Pack info / downloading
@@ -269,14 +269,15 @@ defmodule Pleroma.Web.Router do
     post("/delete_account", UtilController, :delete_account)
     put("/notification_settings", UtilController, :update_notificaton_settings)
     post("/disable_account", UtilController, :disable_account)
-
-    post("/blocks_import", UtilController, :blocks_import)
-    post("/follow_import", UtilController, :follow_import)
   end
 
   scope "/api/pleroma", Pleroma.Web.PleromaAPI do
     pipe_through(:authenticated_api)
 
+    post("/mutes_import", UserImportController, :mutes)
+    post("/blocks_import", UserImportController, :blocks)
+    post("/follow_import", UserImportController, :follow)
+
     get("/accounts/mfa", TwoFactorAuthenticationController, :settings)
     get("/accounts/mfa/backup_codes", TwoFactorAuthenticationController, :backup_codes)
     get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup)

lib/pleroma/web/streamer/streamer.ex

@@ -11,10 +11,12 @@ defmodule Pleroma.Web.Streamer do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.StreamerView
 
   @mix_env Mix.env()
@@ -26,53 +28,87 @@ def registry, do: @registry
   @user_streams ["user", "user:notification", "direct", "user:pleroma_chat"]
 
   @doc "Expands and authorizes a stream, and registers the process for streaming."
-  @spec get_topic_and_add_socket(stream :: String.t(), User.t() | nil, Map.t() | nil) ::
+  @spec get_topic_and_add_socket(
+          stream :: String.t(),
+          User.t() | nil,
+          Token.t() | nil,
+          Map.t() | nil
+        ) ::
           {:ok, topic :: String.t()} | {:error, :bad_topic} | {:error, :unauthorized}
-  def get_topic_and_add_socket(stream, user, params \\ %{}) do
+  def get_topic_and_add_socket(stream, user, oauth_token, params \\ %{}) do
-    case get_topic(stream, user, params) do
+    case get_topic(stream, user, oauth_token, params) do
       {:ok, topic} -> add_socket(topic, user)
       error -> error
     end
   end
 
   @doc "Expand and authorizes a stream"
-  @spec get_topic(stream :: String.t(), User.t() | nil, Map.t()) ::
+  @spec get_topic(stream :: String.t(), User.t() | nil, Token.t() | nil, Map.t()) ::
           {:ok, topic :: String.t()} | {:error, :bad_topic}
-  def get_topic(stream, user, params \\ %{})
+  def get_topic(stream, user, oauth_token, params \\ %{})
 
   # Allow all public steams.
-  def get_topic(stream, _, _) when stream in @public_streams do
+  def get_topic(stream, _user, _oauth_token, _params) when stream in @public_streams do
     {:ok, stream}
   end
 
   # Allow all hashtags streams.
-  def get_topic("hashtag", _, %{"tag" => tag}) do
+  def get_topic("hashtag", _user, _oauth_token, %{"tag" => tag} = _params) do
     {:ok, "hashtag:" <> tag}
   end
 
   # Expand user streams.
-  def get_topic(stream, %User{} = user, _) when stream in @user_streams do
+  def get_topic(
-    {:ok, stream <> ":" <> to_string(user.id)}
+        stream,
+        %User{id: user_id} = user,
+        %Token{user_id: token_user_id} = oauth_token,
+        _params
+      )
+      when stream in @user_streams and user_id == token_user_id do
+    # Note: "read" works for all user streams (not mentioning it since it's an ancestor scope)
+    required_scopes =
+      if stream == "user:notification" do
+        ["read:notifications"]
+      else
+        ["read:statuses"]
       end
 
-  def get_topic(stream, _, _) when stream in @user_streams do
+    if OAuthScopesPlug.filter_descendants(required_scopes, oauth_token.scopes) == [] do
+      {:error, :unauthorized}
+    else
+      {:ok, stream <> ":" <> to_string(user.id)}
+    end
+  end
+
+  def get_topic(stream, _user, _oauth_token, _params) when stream in @user_streams do
     {:error, :unauthorized}
   end
 
   # List streams.
-  def get_topic("list", %User{} = user, %{"list" => id}) do
+  def get_topic(
-    if Pleroma.List.get(id, user) do
+        "list",
+        %User{id: user_id} = user,
+        %Token{user_id: token_user_id} = oauth_token,
+        %{"list" => id}
+      )
+      when user_id == token_user_id do
+    cond do
+      OAuthScopesPlug.filter_descendants(["read", "read:lists"], oauth_token.scopes) == [] ->
+        {:error, :unauthorized}
+
+      Pleroma.List.get(id, user) ->
         {:ok, "list:" <> to_string(id)}
-    else
+
+      true ->
         {:error, :bad_topic}
     end
   end
 
-  def get_topic("list", _, _) do
+  def get_topic("list", _user, _oauth_token, _params) do
     {:error, :unauthorized}
   end
 
-  def get_topic(_, _, _) do
+  def get_topic(_stream, _user, _oauth_token, _params) do
     {:error, :bad_topic}
   end
 

lib/pleroma/web/twitter_api/controllers/util_controller.ex

@@ -18,14 +18,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
 
   plug(Pleroma.Web.FederatingPlug when action == :remote_subscribe)
 
-  plug(
-    OAuthScopesPlug,
-    %{scopes: ["follow", "write:follows"]}
-    when action == :follow_import
-  )
-
-  plug(OAuthScopesPlug, %{scopes: ["follow", "write:blocks"]} when action == :blocks_import)
-
   plug(
     OAuthScopesPlug,
     %{scopes: ["write:accounts"]}
@@ -104,33 +96,6 @@ def update_notificaton_settings(%{assigns: %{user: user}} = conn, params) do
     end
   end
 
-  def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do
-    follow_import(conn, %{"list" => File.read!(listfile.path)})
-  end
-
-  def follow_import(%{assigns: %{user: follower}} = conn, %{"list" => list}) do
-    followed_identifiers =
-      list
-      |> String.split("\n")
-      |> Enum.map(&(&1 |> String.split(",") |> List.first()))
-      |> List.delete("Account address")
-      |> Enum.map(&(&1 |> String.trim() |> String.trim_leading("@")))
-      |> Enum.reject(&(&1 == ""))
-
-    User.follow_import(follower, followed_identifiers)
-    json(conn, "job started")
-  end
-
-  def blocks_import(conn, %{"list" => %Plug.Upload{} = listfile}) do
-    blocks_import(conn, %{"list" => File.read!(listfile.path)})
-  end
-
-  def blocks_import(%{assigns: %{user: blocker}} = conn, %{"list" => list}) do
-    blocked_identifiers = list |> String.split() |> Enum.map(&String.trim_leading(&1, "@"))
-    User.blocks_import(blocker, blocked_identifiers)
-    json(conn, "job started")
-  end
-
   def change_password(%{assigns: %{user: user}} = conn, params) do
     case CommonAPI.Utils.confirm_current_password(user, params["password"]) do
       {:ok, user} ->

lib/pleroma/workers/background_worker.ex

@@ -26,26 +26,10 @@ def perform(%Job{args: %{"op" => "force_password_reset", "user_id" => user_id}})
     User.perform(:force_password_reset, user)
   end
 
-  def perform(%Job{
+  def perform(%Job{args: %{"op" => op, "user_id" => user_id, "identifiers" => identifiers}})
-        args: %{
+      when op in ["blocks_import", "follow_import", "mutes_import"] do
-          "op" => "blocks_import",
+    user = User.get_cached_by_id(user_id)
-          "blocker_id" => blocker_id,
+    {:ok, User.Import.perform(String.to_atom(op), user, identifiers)}
-          "blocked_identifiers" => blocked_identifiers
-        }
-      }) do
-    blocker = User.get_cached_by_id(blocker_id)
-    {:ok, User.perform(:blocks_import, blocker, blocked_identifiers)}
-  end
-
-  def perform(%Job{
-        args: %{
-          "op" => "follow_import",
-          "follower_id" => follower_id,
-          "followed_identifiers" => followed_identifiers
-        }
-      }) do
-    follower = User.get_cached_by_id(follower_id)
-    {:ok, User.perform(:follow_import, follower, followed_identifiers)}
   end
 
   def perform(%Job{args: %{"op" => "media_proxy_preload", "message" => message}}) do

test/emoji/pack_test.exs

@@ -0,0 +1,93 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Emoji.PackTest do
+  use ExUnit.Case, async: true
+  alias Pleroma.Emoji.Pack
+
+  @emoji_path Path.join(
+                Pleroma.Config.get!([:instance, :static_dir]),
+                "emoji"
+              )
+
+  setup do
+    pack_path = Path.join(@emoji_path, "dump_pack")
+    File.mkdir(pack_path)
+
+    File.write!(Path.join(pack_path, "pack.json"), """
+    {
+    "files": { },
+    "pack": {
+    "description": "Dump pack", "homepage": "https://pleroma.social",
+    "license": "Test license", "share-files": true
+    }}
+    """)
+
+    {:ok, pack} = Pleroma.Emoji.Pack.load_pack("dump_pack")
+
+    on_exit(fn ->
+      File.rm_rf!(pack_path)
+    end)
+
+    {:ok, pack: pack}
+  end
+
+  describe "add_file/4" do
+    test "add emojies from zip file", %{pack: pack} do
+      file = %Plug.Upload{
+        content_type: "application/zip",
+        filename: "emojis.zip",
+        path: Path.absname("test/fixtures/emojis.zip")
+      }
+
+      {:ok, updated_pack} = Pack.add_file(pack, nil, nil, file)
+
+      assert updated_pack.files == %{
+               "a_trusted_friend-128" => "128px/a_trusted_friend-128.png",
+               "auroraborealis" => "auroraborealis.png",
+               "baby_in_a_box" => "1000px/baby_in_a_box.png",
+               "bear" => "1000px/bear.png",
+               "bear-128" => "128px/bear-128.png"
+             }
+
+      assert updated_pack.files_count == 5
+    end
+  end
+
+  test "returns error when zip file is bad", %{pack: pack} do
+    file = %Plug.Upload{
+      content_type: "application/zip",
+      filename: "emojis.zip",
+      path: Path.absname("test/instance_static/emoji/test_pack/blank.png")
+    }
+
+    assert Pack.add_file(pack, nil, nil, file) == {:error, :einval}
+  end
+
+  test "returns pack when zip file is empty", %{pack: pack} do
+    file = %Plug.Upload{
+      content_type: "application/zip",
+      filename: "emojis.zip",
+      path: Path.absname("test/fixtures/empty.zip")
+    }
+
+    {:ok, updated_pack} = Pack.add_file(pack, nil, nil, file)
+    assert updated_pack == pack
+  end
+
+  test "add emoji file", %{pack: pack} do
+    file = %Plug.Upload{
+      filename: "blank.png",
+      path: "#{@emoji_path}/test_pack/blank.png"
+    }
+
+    {:ok, updated_pack} = Pack.add_file(pack, "test_blank", "test_blank.png", file)
+
+    assert updated_pack.files == %{
+             "test_blank" => "test_blank.png"
+           }
+
+    assert updated_pack.files_count == 1
+  end
+end

test/integration/mastodon_websocket_test.exs

@@ -78,7 +78,7 @@ test "receives well formatted events" do
         Pleroma.Repo.insert(
           OAuth.App.register_changeset(%OAuth.App{}, %{
             client_name: "client",
-            scopes: ["scope"],
+            scopes: ["read"],
             redirect_uris: "url"
           })
         )

test/notification_test.exs

@@ -179,17 +179,19 @@ test "does not create a notification for subscribed users if status is a reply"
   describe "create_notification" do
     @tag needs_streamer: true
     test "it creates a notification for user and send to the 'user' and the 'user:notification' stream" do
-      user = insert(:user)
+      %{user: user, token: oauth_token} = oauth_access(["read"])
 
       task =
         Task.async(fn ->
-          Streamer.get_topic_and_add_socket("user", user)
+          {:ok, _topic} = Streamer.get_topic_and_add_socket("user", user, oauth_token)
           assert_receive {:render_with_user, _, _, _}, 4_000
         end)
 
       task_user_notification =
         Task.async(fn ->
-          Streamer.get_topic_and_add_socket("user:notification", user)
+          {:ok, _topic} =
+            Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
+
           assert_receive {:render_with_user, _, _, _}, 4_000
         end)
 

test/support/data_case.ex

@@ -27,6 +27,21 @@ defmodule Pleroma.DataCase do
       import Ecto.Query
       import Pleroma.DataCase
       use Pleroma.Tests.Helpers
+
+      # Sets up OAuth access with specified scopes
+      defp oauth_access(scopes, opts \\ []) do
+        user =
+          Keyword.get_lazy(opts, :user, fn ->
+            Pleroma.Factory.insert(:user)
+          end)
+
+        token =
+          Keyword.get_lazy(opts, :oauth_token, fn ->
+            Pleroma.Factory.insert(:oauth_token, user: user, scopes: scopes)
+          end)
+
+        %{user: user, token: token}
+      end
     end
   end
 

test/user/import_test.exs

@@ -0,0 +1,76 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.ImportTest do
+  alias Pleroma.Repo
+  alias Pleroma.Tests.ObanHelpers
+  alias Pleroma.User
+
+  use Pleroma.DataCase
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Pleroma.Factory
+
+  setup_all do
+    Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  describe "follow_import" do
+    test "it imports user followings from list" do
+      [user1, user2, user3] = insert_list(3, :user)
+
+      identifiers = [
+        user2.ap_id,
+        user3.nickname
+      ]
+
+      {:ok, job} = User.Import.follow_import(user1, identifiers)
+
+      assert {:ok, result} = ObanHelpers.perform(job)
+      assert is_list(result)
+      assert result == [user2, user3]
+      assert User.following?(user1, user2)
+      assert User.following?(user1, user3)
+    end
+  end
+
+  describe "blocks_import" do
+    test "it imports user blocks from list" do
+      [user1, user2, user3] = insert_list(3, :user)
+
+      identifiers = [
+        user2.ap_id,
+        user3.nickname
+      ]
+
+      {:ok, job} = User.Import.blocks_import(user1, identifiers)
+
+      assert {:ok, result} = ObanHelpers.perform(job)
+      assert is_list(result)
+      assert result == [user2, user3]
+      assert User.blocks?(user1, user2)
+      assert User.blocks?(user1, user3)
+    end
+  end
+
+  describe "mutes_import" do
+    test "it imports user mutes from list" do
+      [user1, user2, user3] = insert_list(3, :user)
+
+      identifiers = [
+        user2.ap_id,
+        user3.nickname
+      ]
+
+      {:ok, job} = User.Import.mutes_import(user1, identifiers)
+
+      assert {:ok, result} = ObanHelpers.perform(job)
+      assert is_list(result)
+      assert result == [user2, user3]
+      assert User.mutes?(user1, user2)
+      assert User.mutes?(user1, user3)
+    end
+  end
+end

test/user_test.exs

@@ -509,7 +509,12 @@ test "it sends a confirm email" do
       cng = User.register_changeset(%User{}, @full_user_data)
       {:ok, registered_user} = User.register(cng)
       ObanHelpers.perform_all()
-      assert_email_sent(Pleroma.Emails.UserEmail.account_confirmation_email(registered_user))
+
+      Pleroma.Emails.UserEmail.account_confirmation_email(registered_user)
+      # temporary hackney fix until hackney max_connections bug is fixed
+      # https://git.pleroma.social/pleroma/pleroma/-/issues/2101
+      |> Swoosh.Email.put_private(:hackney_options, ssl_options: [versions: [:"tlsv1.2"]])
+      |> assert_email_sent()
     end
 
     test "it requires an email, name, nickname and password, bio is optional when account_activation_required is enabled" do
@@ -971,23 +976,6 @@ test "it sets the follower_count property" do
     end
   end
 
-  describe "follow_import" do
-    test "it imports user followings from list" do
-      [user1, user2, user3] = insert_list(3, :user)
-
-      identifiers = [
-        user2.ap_id,
-        user3.nickname
-      ]
-
-      {:ok, job} = User.follow_import(user1, identifiers)
-
-      assert {:ok, result} = ObanHelpers.perform(job)
-      assert is_list(result)
-      assert result == [user2, user3]
-    end
-  end
-
   describe "mutes" do
     test "it mutes people" do
       user = insert(:user)
@@ -1226,23 +1214,6 @@ test "follows take precedence over domain blocks" do
     end
   end
 
-  describe "blocks_import" do
-    test "it imports user blocks from list" do
-      [user1, user2, user3] = insert_list(3, :user)
-
-      identifiers = [
-        user2.ap_id,
-        user3.nickname
-      ]
-
-      {:ok, job} = User.blocks_import(user1, identifiers)
-
-      assert {:ok, result} = ObanHelpers.perform(job)
-      assert is_list(result)
-      assert result == [user2, user3]
-    end
-  end
-
   describe "get_recipients_from_activity" do
     test "works for announces" do
       actor = insert(:user)

test/utils_test.exs

@@ -0,0 +1,15 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.UtilsTest do
+  use ExUnit.Case, async: true
+
+  describe "tmp_dir/1" do
+    test "returns unique temporary directory" do
+      {:ok, path} = Pleroma.Utils.tmp_dir("emoji")
+      assert path =~ ~r/\/tmp\/emoji-(.*)-#{:os.getpid()}-(.*)/
+      File.rm_rf(path)
+    end
+  end
+end

test/web/admin_api/controllers/admin_api_controller_test.exs

@@ -1977,7 +1977,12 @@ test "it resend emails for two users", %{conn: conn, admin: admin} do
                }"
 
       ObanHelpers.perform_all()
-      assert_email_sent(Pleroma.Emails.UserEmail.account_confirmation_email(first_user))
+
+      Pleroma.Emails.UserEmail.account_confirmation_email(first_user)
+      # temporary hackney fix until hackney max_connections bug is fixed
+      # https://git.pleroma.social/pleroma/pleroma/-/issues/2101
+      |> Swoosh.Email.put_private(:hackney_options, ssl_options: [versions: [:"tlsv1.2"]])
+      |> assert_email_sent()
     end
   end
 

test/web/mastodon_api/controllers/auth_controller_test.exs

@@ -61,7 +61,7 @@ test "redirects to the getting-started page when referer is not present", %{conn
     end
 
     test "it returns 204", %{conn: conn} do
-      assert json_response(conn, :no_content)
+      assert empty_json_response(conn)
     end
 
     test "it creates a PasswordResetToken record for user", %{user: user} do
@@ -91,7 +91,7 @@ test "it returns 204", %{conn: conn} do
 
       assert conn
              |> post("/auth/password?nickname=#{user.nickname}")
-             |> json_response(:no_content)
+             |> empty_json_response()
 
       ObanHelpers.perform_all()
       token_record = Repo.get_by(Pleroma.PasswordResetToken, user_id: user.id)
@@ -112,7 +112,7 @@ test "it doesn't fail when a user has no email", %{conn: conn} do
 
       assert conn
              |> post("/auth/password?nickname=#{user.nickname}")
-             |> json_response(:no_content)
+             |> empty_json_response()
     end
   end
 
@@ -125,24 +125,21 @@ test "it doesn't fail when a user has no email", %{conn: conn} do
     test "it returns 204 when user is not found", %{conn: conn, user: user} do
       conn = post(conn, "/auth/password?email=nonexisting_#{user.email}")
 
-      assert conn
+      assert empty_json_response(conn)
-             |> json_response(:no_content)
     end
 
     test "it returns 204 when user is not local", %{conn: conn, user: user} do
       {:ok, user} = Repo.update(Ecto.Changeset.change(user, local: false))
       conn = post(conn, "/auth/password?email=#{user.email}")
 
-      assert conn
+      assert empty_json_response(conn)
-             |> json_response(:no_content)
     end
 
     test "it returns 204 when user is deactivated", %{conn: conn, user: user} do
       {:ok, user} = Repo.update(Ecto.Changeset.change(user, deactivated: true, local: true))
       conn = post(conn, "/auth/password?email=#{user.email}")
 
-      assert conn
+      assert empty_json_response(conn)
-             |> json_response(:no_content)
     end
   end
 

test/web/mastodon_api/controllers/timeline_controller_test.exs

@@ -114,8 +114,16 @@ test "doesn't return replies if follower is posting with blocked user" do
       {:ok, _reply_from_friend} =
         CommonAPI.post(friend, %{status: "status", in_reply_to_status_id: reply_from_blockee})
 
-      res_conn = get(conn, "/api/v1/timelines/public")
+      # Still shows replies from yourself
-      [%{"id" => ^activity_id}] = json_response_and_validate_schema(res_conn, 200)
+      {:ok, %{id: reply_from_me}} =
+        CommonAPI.post(blocker, %{status: "status", in_reply_to_status_id: reply_from_blockee})
+
+      response =
+        get(conn, "/api/v1/timelines/public")
+        |> json_response_and_validate_schema(200)
+
+      assert length(response) == 2
+      [%{"id" => ^reply_from_me}, %{"id" => ^activity_id}] = response
     end
 
     test "doesn't return replies if follow is posting with users from blocked domain" do

test/web/pleroma_api/controllers/emoji_file_controller_test.exs

@@ -0,0 +1,357 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.EmojiFileControllerTest do
+  use Pleroma.Web.ConnCase
+
+  import Tesla.Mock
+  import Pleroma.Factory
+
+  @emoji_path Path.join(
+                Pleroma.Config.get!([:instance, :static_dir]),
+                "emoji"
+              )
+  setup do: clear_config([:auth, :enforce_oauth_admin_scope_usage], false)
+
+  setup do: clear_config([:instance, :public], true)
+
+  setup do
+    admin = insert(:user, is_admin: true)
+    token = insert(:oauth_admin_token, user: admin)
+
+    admin_conn =
+      build_conn()
+      |> assign(:user, admin)
+      |> assign(:token, token)
+
+    Pleroma.Emoji.reload()
+    {:ok, %{admin_conn: admin_conn}}
+  end
+
+  describe "POST/PATCH/DELETE /api/pleroma/emoji/packs/:name/files" do
+    setup do
+      pack_file = "#{@emoji_path}/test_pack/pack.json"
+      original_content = File.read!(pack_file)
+
+      on_exit(fn ->
+        File.write!(pack_file, original_content)
+      end)
+
+      :ok
+    end
+
+    test "upload zip file with emojies", %{admin_conn: admin_conn} do
+      on_exit(fn ->
+        [
+          "128px/a_trusted_friend-128.png",
+          "auroraborealis.png",
+          "1000px/baby_in_a_box.png",
+          "1000px/bear.png",
+          "128px/bear-128.png"
+        ]
+        |> Enum.each(fn path -> File.rm_rf!("#{@emoji_path}/test_pack/#{path}") end)
+      end)
+
+      resp =
+        admin_conn
+        |> put_req_header("content-type", "multipart/form-data")
+        |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+          file: %Plug.Upload{
+            content_type: "application/zip",
+            filename: "emojis.zip",
+            path: Path.absname("test/fixtures/emojis.zip")
+          }
+        })
+        |> json_response_and_validate_schema(200)
+
+      assert resp == %{
+               "a_trusted_friend-128" => "128px/a_trusted_friend-128.png",
+               "auroraborealis" => "auroraborealis.png",
+               "baby_in_a_box" => "1000px/baby_in_a_box.png",
+               "bear" => "1000px/bear.png",
+               "bear-128" => "128px/bear-128.png",
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
+             }
+
+      Enum.each(Map.values(resp), fn path ->
+        assert File.exists?("#{@emoji_path}/test_pack/#{path}")
+      end)
+    end
+
+    test "create shortcode exists", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank",
+               filename: "dir/blank.png",
+               file: %Plug.Upload{
+                 filename: "blank.png",
+                 path: "#{@emoji_path}/test_pack/blank.png"
+               }
+             })
+             |> json_response_and_validate_schema(:conflict) == %{
+               "error" => "An emoji with the \"blank\" shortcode already exists"
+             }
+    end
+
+    test "don't rewrite old emoji", %{admin_conn: admin_conn} do
+      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/dir/") end)
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank3",
+               filename: "dir/blank.png",
+               file: %Plug.Upload{
+                 filename: "blank.png",
+                 path: "#{@emoji_path}/test_pack/blank.png"
+               }
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "blank" => "blank.png",
+               "blank2" => "blank2.png",
+               "blank3" => "dir/blank.png"
+             }
+
+      assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank",
+               new_shortcode: "blank2",
+               new_filename: "dir_2/blank_3.png"
+             })
+             |> json_response_and_validate_schema(:conflict) == %{
+               "error" =>
+                 "New shortcode \"blank2\" is already used. If you want to override emoji use 'force' option"
+             }
+    end
+
+    test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
+      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/dir_2/") end)
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank3",
+               filename: "dir/blank.png",
+               file: %Plug.Upload{
+                 filename: "blank.png",
+                 path: "#{@emoji_path}/test_pack/blank.png"
+               }
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "blank" => "blank.png",
+               "blank2" => "blank2.png",
+               "blank3" => "dir/blank.png"
+             }
+
+      assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank3",
+               new_shortcode: "blank4",
+               new_filename: "dir_2/blank_3.png",
+               force: true
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "blank" => "blank.png",
+               "blank2" => "blank2.png",
+               "blank4" => "dir_2/blank_3.png"
+             }
+
+      assert File.exists?("#{@emoji_path}/test_pack/dir_2/blank_3.png")
+    end
+
+    test "with empty filename", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank2",
+               filename: "",
+               file: %Plug.Upload{
+                 filename: "blank.png",
+                 path: "#{@emoji_path}/test_pack/blank.png"
+               }
+             })
+             |> json_response_and_validate_schema(422) == %{
+               "error" => "pack name, shortcode or filename cannot be empty"
+             }
+    end
+
+    test "add file with not loaded pack", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/not_loaded/files", %{
+               shortcode: "blank3",
+               filename: "dir/blank.png",
+               file: %Plug.Upload{
+                 filename: "blank.png",
+                 path: "#{@emoji_path}/test_pack/blank.png"
+               }
+             })
+             |> json_response_and_validate_schema(:not_found) == %{
+               "error" => "pack \"not_loaded\" is not found"
+             }
+    end
+
+    test "remove file with not loaded pack", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> delete("/api/pleroma/emoji/packs/not_loaded/files?shortcode=blank3")
+             |> json_response_and_validate_schema(:not_found) == %{
+               "error" => "pack \"not_loaded\" is not found"
+             }
+    end
+
+    test "remove file with empty shortcode", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> delete("/api/pleroma/emoji/packs/not_loaded/files?shortcode=")
+             |> json_response_and_validate_schema(:not_found) == %{
+               "error" => "pack \"not_loaded\" is not found"
+             }
+    end
+
+    test "update file with not loaded pack", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> patch("/api/pleroma/emoji/packs/not_loaded/files", %{
+               shortcode: "blank4",
+               new_shortcode: "blank3",
+               new_filename: "dir_2/blank_3.png"
+             })
+             |> json_response_and_validate_schema(:not_found) == %{
+               "error" => "pack \"not_loaded\" is not found"
+             }
+    end
+
+    test "new with shortcode as file with update", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank4",
+               filename: "dir/blank.png",
+               file: %Plug.Upload{
+                 filename: "blank.png",
+                 path: "#{@emoji_path}/test_pack/blank.png"
+               }
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "blank" => "blank.png",
+               "blank4" => "dir/blank.png",
+               "blank2" => "blank2.png"
+             }
+
+      assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank4",
+               new_shortcode: "blank3",
+               new_filename: "dir_2/blank_3.png"
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "blank3" => "dir_2/blank_3.png",
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
+             }
+
+      refute File.exists?("#{@emoji_path}/test_pack/dir/")
+      assert File.exists?("#{@emoji_path}/test_pack/dir_2/blank_3.png")
+
+      assert admin_conn
+             |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
+             |> json_response_and_validate_schema(200) == %{
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
+             }
+
+      refute File.exists?("#{@emoji_path}/test_pack/dir_2/")
+
+      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/dir") end)
+    end
+
+    test "new with shortcode from url", %{admin_conn: admin_conn} do
+      mock(fn
+        %{
+          method: :get,
+          url: "https://test-blank/blank_url.png"
+        } ->
+          text(File.read!("#{@emoji_path}/test_pack/blank.png"))
+      end)
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank_url",
+               file: "https://test-blank/blank_url.png"
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "blank_url" => "blank_url.png",
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
+             }
+
+      assert File.exists?("#{@emoji_path}/test_pack/blank_url.png")
+
+      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/blank_url.png") end)
+    end
+
+    test "new without shortcode", %{admin_conn: admin_conn} do
+      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/shortcode.png") end)
+
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
+               file: %Plug.Upload{
+                 filename: "shortcode.png",
+                 path: "#{Pleroma.Config.get([:instance, :static_dir])}/add/shortcode.png"
+               }
+             })
+             |> json_response_and_validate_schema(200) == %{
+               "shortcode" => "shortcode.png",
+               "blank" => "blank.png",
+               "blank2" => "blank2.png"
+             }
+    end
+
+    test "remove non existing shortcode in pack.json", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
+             |> json_response_and_validate_schema(:bad_request) == %{
+               "error" => "Emoji \"blank3\" does not exist"
+             }
+    end
+
+    test "update non existing emoji", %{admin_conn: admin_conn} do
+      assert admin_conn
+             |> put_req_header("content-type", "multipart/form-data")
+             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
+               shortcode: "blank3",
+               new_shortcode: "blank4",
+               new_filename: "dir_2/blank_3.png"
+             })
+             |> json_response_and_validate_schema(:bad_request) == %{
+               "error" => "Emoji \"blank3\" does not exist"
+             }
+    end
+
+    test "update with empty shortcode", %{admin_conn: admin_conn} do
+      assert %{
+               "error" => "Missing field: new_shortcode."
+             } =
+               admin_conn
+               |> put_req_header("content-type", "multipart/form-data")
+               |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
+                 shortcode: "blank",
+                 new_filename: "dir_2/blank_3.png"
+               })
+               |> json_response_and_validate_schema(:bad_request)
+    end
+  end
+end

test/web/pleroma_api/controllers/emoji_pack_controller_test.exs

@@ -411,293 +411,6 @@ test "when the fallback source doesn't have all the files", ctx do
     end
   end
 
-  describe "POST/PATCH/DELETE /api/pleroma/emoji/packs/:name/files" do
-    setup do
-      pack_file = "#{@emoji_path}/test_pack/pack.json"
-      original_content = File.read!(pack_file)
-
-      on_exit(fn ->
-        File.write!(pack_file, original_content)
-      end)
-
-      :ok
-    end
-
-    test "create shortcode exists", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank",
-               filename: "dir/blank.png",
-               file: %Plug.Upload{
-                 filename: "blank.png",
-                 path: "#{@emoji_path}/test_pack/blank.png"
-               }
-             })
-             |> json_response_and_validate_schema(:conflict) == %{
-               "error" => "An emoji with the \"blank\" shortcode already exists"
-             }
-    end
-
-    test "don't rewrite old emoji", %{admin_conn: admin_conn} do
-      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/dir/") end)
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank3",
-               filename: "dir/blank.png",
-               file: %Plug.Upload{
-                 filename: "blank.png",
-                 path: "#{@emoji_path}/test_pack/blank.png"
-               }
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "blank" => "blank.png",
-               "blank2" => "blank2.png",
-               "blank3" => "dir/blank.png"
-             }
-
-      assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank",
-               new_shortcode: "blank2",
-               new_filename: "dir_2/blank_3.png"
-             })
-             |> json_response_and_validate_schema(:conflict) == %{
-               "error" =>
-                 "New shortcode \"blank2\" is already used. If you want to override emoji use 'force' option"
-             }
-    end
-
-    test "rewrite old emoji with force option", %{admin_conn: admin_conn} do
-      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/dir_2/") end)
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank3",
-               filename: "dir/blank.png",
-               file: %Plug.Upload{
-                 filename: "blank.png",
-                 path: "#{@emoji_path}/test_pack/blank.png"
-               }
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "blank" => "blank.png",
-               "blank2" => "blank2.png",
-               "blank3" => "dir/blank.png"
-             }
-
-      assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank3",
-               new_shortcode: "blank4",
-               new_filename: "dir_2/blank_3.png",
-               force: true
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "blank" => "blank.png",
-               "blank2" => "blank2.png",
-               "blank4" => "dir_2/blank_3.png"
-             }
-
-      assert File.exists?("#{@emoji_path}/test_pack/dir_2/blank_3.png")
-    end
-
-    test "with empty filename", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank2",
-               filename: "",
-               file: %Plug.Upload{
-                 filename: "blank.png",
-                 path: "#{@emoji_path}/test_pack/blank.png"
-               }
-             })
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "pack name, shortcode or filename cannot be empty"
-             }
-    end
-
-    test "add file with not loaded pack", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/not_loaded/files", %{
-               shortcode: "blank3",
-               filename: "dir/blank.png",
-               file: %Plug.Upload{
-                 filename: "blank.png",
-                 path: "#{@emoji_path}/test_pack/blank.png"
-               }
-             })
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "pack \"not_loaded\" is not found"
-             }
-    end
-
-    test "remove file with not loaded pack", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> delete("/api/pleroma/emoji/packs/not_loaded/files?shortcode=blank3")
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "pack \"not_loaded\" is not found"
-             }
-    end
-
-    test "remove file with empty shortcode", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> delete("/api/pleroma/emoji/packs/not_loaded/files?shortcode=")
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "pack name or shortcode cannot be empty"
-             }
-    end
-
-    test "update file with not loaded pack", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> patch("/api/pleroma/emoji/packs/not_loaded/files", %{
-               shortcode: "blank4",
-               new_shortcode: "blank3",
-               new_filename: "dir_2/blank_3.png"
-             })
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "pack \"not_loaded\" is not found"
-             }
-    end
-
-    test "new with shortcode as file with update", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank4",
-               filename: "dir/blank.png",
-               file: %Plug.Upload{
-                 filename: "blank.png",
-                 path: "#{@emoji_path}/test_pack/blank.png"
-               }
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "blank" => "blank.png",
-               "blank4" => "dir/blank.png",
-               "blank2" => "blank2.png"
-             }
-
-      assert File.exists?("#{@emoji_path}/test_pack/dir/blank.png")
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank4",
-               new_shortcode: "blank3",
-               new_filename: "dir_2/blank_3.png"
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "blank3" => "dir_2/blank_3.png",
-               "blank" => "blank.png",
-               "blank2" => "blank2.png"
-             }
-
-      refute File.exists?("#{@emoji_path}/test_pack/dir/")
-      assert File.exists?("#{@emoji_path}/test_pack/dir_2/blank_3.png")
-
-      assert admin_conn
-             |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
-             |> json_response_and_validate_schema(200) == %{
-               "blank" => "blank.png",
-               "blank2" => "blank2.png"
-             }
-
-      refute File.exists?("#{@emoji_path}/test_pack/dir_2/")
-
-      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/dir") end)
-    end
-
-    test "new with shortcode from url", %{admin_conn: admin_conn} do
-      mock(fn
-        %{
-          method: :get,
-          url: "https://test-blank/blank_url.png"
-        } ->
-          text(File.read!("#{@emoji_path}/test_pack/blank.png"))
-      end)
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank_url",
-               file: "https://test-blank/blank_url.png"
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "blank_url" => "blank_url.png",
-               "blank" => "blank.png",
-               "blank2" => "blank2.png"
-             }
-
-      assert File.exists?("#{@emoji_path}/test_pack/blank_url.png")
-
-      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/blank_url.png") end)
-    end
-
-    test "new without shortcode", %{admin_conn: admin_conn} do
-      on_exit(fn -> File.rm_rf!("#{@emoji_path}/test_pack/shortcode.png") end)
-
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> post("/api/pleroma/emoji/packs/test_pack/files", %{
-               file: %Plug.Upload{
-                 filename: "shortcode.png",
-                 path: "#{Pleroma.Config.get([:instance, :static_dir])}/add/shortcode.png"
-               }
-             })
-             |> json_response_and_validate_schema(200) == %{
-               "shortcode" => "shortcode.png",
-               "blank" => "blank.png",
-               "blank2" => "blank2.png"
-             }
-    end
-
-    test "remove non existing shortcode in pack.json", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> delete("/api/pleroma/emoji/packs/test_pack/files?shortcode=blank3")
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "Emoji \"blank3\" does not exist"
-             }
-    end
-
-    test "update non existing emoji", %{admin_conn: admin_conn} do
-      assert admin_conn
-             |> put_req_header("content-type", "multipart/form-data")
-             |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-               shortcode: "blank3",
-               new_shortcode: "blank4",
-               new_filename: "dir_2/blank_3.png"
-             })
-             |> json_response_and_validate_schema(:bad_request) == %{
-               "error" => "Emoji \"blank3\" does not exist"
-             }
-    end
-
-    test "update with empty shortcode", %{admin_conn: admin_conn} do
-      assert %{
-               "error" => "Missing field: new_shortcode."
-             } =
-               admin_conn
-               |> put_req_header("content-type", "multipart/form-data")
-               |> patch("/api/pleroma/emoji/packs/test_pack/files", %{
-                 shortcode: "blank",
-                 new_filename: "dir_2/blank_3.png"
-               })
-               |> json_response_and_validate_schema(:bad_request)
-    end
-  end
-
   describe "POST/DELETE /api/pleroma/emoji/packs/:name" do
     test "creating and deleting a pack", %{admin_conn: admin_conn} do
       assert admin_conn

test/web/pleroma_api/controllers/user_import_controller_test.exs

@@ -0,0 +1,235 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.UserImportControllerTest do
+  use Pleroma.Web.ConnCase
+  use Oban.Testing, repo: Pleroma.Repo
+
+  alias Pleroma.Config
+  alias Pleroma.Tests.ObanHelpers
+
+  import Pleroma.Factory
+  import Mock
+
+  setup do
+    Tesla.Mock.mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  describe "POST /api/pleroma/follow_import" do
+    setup do: oauth_access(["follow"])
+
+    test "it returns HTTP 200", %{conn: conn} do
+      user2 = insert(:user)
+
+      assert "job started" ==
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/follow_import", %{"list" => "#{user2.ap_id}"})
+               |> json_response_and_validate_schema(200)
+    end
+
+    test "it imports follow lists from file", %{conn: conn} do
+      user2 = insert(:user)
+
+      with_mocks([
+        {File, [],
+         read!: fn "follow_list.txt" ->
+           "Account address,Show boosts\n#{user2.ap_id},true"
+         end}
+      ]) do
+        assert "job started" ==
+                 conn
+                 |> put_req_header("content-type", "application/json")
+                 |> post("/api/pleroma/follow_import", %{
+                   "list" => %Plug.Upload{path: "follow_list.txt"}
+                 })
+                 |> json_response_and_validate_schema(200)
+
+        assert [{:ok, job_result}] = ObanHelpers.perform_all()
+        assert job_result == [user2]
+      end
+    end
+
+    test "it imports new-style mastodon follow lists", %{conn: conn} do
+      user2 = insert(:user)
+
+      response =
+        conn
+        |> put_req_header("content-type", "application/json")
+        |> post("/api/pleroma/follow_import", %{
+          "list" => "Account address,Show boosts\n#{user2.ap_id},true"
+        })
+        |> json_response_and_validate_schema(200)
+
+      assert response == "job started"
+    end
+
+    test "requires 'follow' or 'write:follows' permissions" do
+      token1 = insert(:oauth_token, scopes: ["read", "write"])
+      token2 = insert(:oauth_token, scopes: ["follow"])
+      token3 = insert(:oauth_token, scopes: ["something"])
+      another_user = insert(:user)
+
+      for token <- [token1, token2, token3] do
+        conn =
+          build_conn()
+          |> put_req_header("authorization", "Bearer #{token.token}")
+          |> put_req_header("content-type", "application/json")
+          |> post("/api/pleroma/follow_import", %{"list" => "#{another_user.ap_id}"})
+
+        if token == token3 do
+          assert %{"error" => "Insufficient permissions: follow | write:follows."} ==
+                   json_response(conn, 403)
+        else
+          assert json_response(conn, 200)
+        end
+      end
+    end
+
+    test "it imports follows with different nickname variations", %{conn: conn} do
+      users = [user2, user3, user4, user5, user6] = insert_list(5, :user)
+
+      identifiers =
+        [
+          user2.ap_id,
+          user3.nickname,
+          "  ",
+          "@" <> user4.nickname,
+          user5.nickname <> "@localhost",
+          "@" <> user6.nickname <> "@localhost"
+        ]
+        |> Enum.join("\n")
+
+      assert "job started" ==
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/follow_import", %{"list" => identifiers})
+               |> json_response_and_validate_schema(200)
+
+      assert [{:ok, job_result}] = ObanHelpers.perform_all()
+      assert job_result == users
+    end
+  end
+
+  describe "POST /api/pleroma/blocks_import" do
+    # Note: "follow" or "write:blocks" permission is required
+    setup do: oauth_access(["write:blocks"])
+
+    test "it returns HTTP 200", %{conn: conn} do
+      user2 = insert(:user)
+
+      assert "job started" ==
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/blocks_import", %{"list" => "#{user2.ap_id}"})
+               |> json_response_and_validate_schema(200)
+    end
+
+    test "it imports blocks users from file", %{conn: conn} do
+      users = [user2, user3] = insert_list(2, :user)
+
+      with_mocks([
+        {File, [], read!: fn "blocks_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end}
+      ]) do
+        assert "job started" ==
+                 conn
+                 |> put_req_header("content-type", "application/json")
+                 |> post("/api/pleroma/blocks_import", %{
+                   "list" => %Plug.Upload{path: "blocks_list.txt"}
+                 })
+                 |> json_response_and_validate_schema(200)
+
+        assert [{:ok, job_result}] = ObanHelpers.perform_all()
+        assert job_result == users
+      end
+    end
+
+    test "it imports blocks with different nickname variations", %{conn: conn} do
+      users = [user2, user3, user4, user5, user6] = insert_list(5, :user)
+
+      identifiers =
+        [
+          user2.ap_id,
+          user3.nickname,
+          "@" <> user4.nickname,
+          user5.nickname <> "@localhost",
+          "@" <> user6.nickname <> "@localhost"
+        ]
+        |> Enum.join(" ")
+
+      assert "job started" ==
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/blocks_import", %{"list" => identifiers})
+               |> json_response_and_validate_schema(200)
+
+      assert [{:ok, job_result}] = ObanHelpers.perform_all()
+      assert job_result == users
+    end
+  end
+
+  describe "POST /api/pleroma/mutes_import" do
+    # Note: "follow" or "write:mutes" permission is required
+    setup do: oauth_access(["write:mutes"])
+
+    test "it returns HTTP 200", %{user: user, conn: conn} do
+      user2 = insert(:user)
+
+      assert "job started" ==
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/mutes_import", %{"list" => "#{user2.ap_id}"})
+               |> json_response_and_validate_schema(200)
+
+      assert [{:ok, job_result}] = ObanHelpers.perform_all()
+      assert job_result == [user2]
+      assert Pleroma.User.mutes?(user, user2)
+    end
+
+    test "it imports mutes users from file", %{user: user, conn: conn} do
+      users = [user2, user3] = insert_list(2, :user)
+
+      with_mocks([
+        {File, [], read!: fn "mutes_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end}
+      ]) do
+        assert "job started" ==
+                 conn
+                 |> put_req_header("content-type", "application/json")
+                 |> post("/api/pleroma/mutes_import", %{
+                   "list" => %Plug.Upload{path: "mutes_list.txt"}
+                 })
+                 |> json_response_and_validate_schema(200)
+
+        assert [{:ok, job_result}] = ObanHelpers.perform_all()
+        assert job_result == users
+        assert Enum.all?(users, &Pleroma.User.mutes?(user, &1))
+      end
+    end
+
+    test "it imports mutes with different nickname variations", %{user: user, conn: conn} do
+      users = [user2, user3, user4, user5, user6] = insert_list(5, :user)
+
+      identifiers =
+        [
+          user2.ap_id,
+          user3.nickname,
+          "@" <> user4.nickname,
+          user5.nickname <> "@localhost",
+          "@" <> user6.nickname <> "@localhost"
+        ]
+        |> Enum.join(" ")
+
+      assert "job started" ==
+               conn
+               |> put_req_header("content-type", "application/json")
+               |> post("/api/pleroma/mutes_import", %{"list" => identifiers})
+               |> json_response_and_validate_schema(200)
+
+      assert [{:ok, job_result}] = ObanHelpers.perform_all()
+      assert job_result == users
+      assert Enum.all?(users, &Pleroma.User.mutes?(user, &1))
+    end
+  end
+end

test/web/streamer/streamer_test.exs

@@ -21,92 +21,148 @@ defmodule Pleroma.Web.StreamerTest do
 
   setup do: clear_config([:instance, :skip_thread_containment])
 
-  describe "get_topic without an user" do
+  describe "get_topic/_ (unauthenticated)" do
     test "allows public" do
-      assert {:ok, "public"} = Streamer.get_topic("public", nil)
+      assert {:ok, "public"} = Streamer.get_topic("public", nil, nil)
-      assert {:ok, "public:local"} = Streamer.get_topic("public:local", nil)
+      assert {:ok, "public:local"} = Streamer.get_topic("public:local", nil, nil)
-      assert {:ok, "public:media"} = Streamer.get_topic("public:media", nil)
+      assert {:ok, "public:media"} = Streamer.get_topic("public:media", nil, nil)
-      assert {:ok, "public:local:media"} = Streamer.get_topic("public:local:media", nil)
+      assert {:ok, "public:local:media"} = Streamer.get_topic("public:local:media", nil, nil)
     end
 
     test "allows hashtag streams" do
-      assert {:ok, "hashtag:cofe"} = Streamer.get_topic("hashtag", nil, %{"tag" => "cofe"})
+      assert {:ok, "hashtag:cofe"} = Streamer.get_topic("hashtag", nil, nil, %{"tag" => "cofe"})
     end
 
     test "disallows user streams" do
-      assert {:error, _} = Streamer.get_topic("user", nil)
+      assert {:error, _} = Streamer.get_topic("user", nil, nil)
-      assert {:error, _} = Streamer.get_topic("user:notification", nil)
+      assert {:error, _} = Streamer.get_topic("user:notification", nil, nil)
-      assert {:error, _} = Streamer.get_topic("direct", nil)
+      assert {:error, _} = Streamer.get_topic("direct", nil, nil)
     end
 
     test "disallows list streams" do
-      assert {:error, _} = Streamer.get_topic("list", nil, %{"list" => 42})
+      assert {:error, _} = Streamer.get_topic("list", nil, nil, %{"list" => 42})
     end
   end
 
-  describe "get_topic with an user" do
+  describe "get_topic/_ (authenticated)" do
-    setup do
+    setup do: oauth_access(["read"])
-      user = insert(:user)
+
-      {:ok, %{user: user}}
+    test "allows public streams (regardless of OAuth token scopes)", %{
+      user: user,
+      token: read_oauth_token
+    } do
+      with oauth_token <- [nil, read_oauth_token] do
+        assert {:ok, "public"} = Streamer.get_topic("public", user, oauth_token)
+        assert {:ok, "public:local"} = Streamer.get_topic("public:local", user, oauth_token)
+        assert {:ok, "public:media"} = Streamer.get_topic("public:media", user, oauth_token)
+
+        assert {:ok, "public:local:media"} =
+                 Streamer.get_topic("public:local:media", user, oauth_token)
+      end
     end
 
-    test "allows public streams", %{user: user} do
+    test "allows user streams (with proper OAuth token scopes)", %{
-      assert {:ok, "public"} = Streamer.get_topic("public", user)
+      user: user,
-      assert {:ok, "public:local"} = Streamer.get_topic("public:local", user)
+      token: read_oauth_token
-      assert {:ok, "public:media"} = Streamer.get_topic("public:media", user)
+    } do
-      assert {:ok, "public:local:media"} = Streamer.get_topic("public:local:media", user)
+      %{token: read_notifications_token} = oauth_access(["read:notifications"], user: user)
-    end
+      %{token: read_statuses_token} = oauth_access(["read:statuses"], user: user)
+      %{token: badly_scoped_token} = oauth_access(["irrelevant:scope"], user: user)
 
-    test "allows user streams", %{user: user} do
       expected_user_topic = "user:#{user.id}"
-      expected_notif_topic = "user:notification:#{user.id}"
+      expected_notification_topic = "user:notification:#{user.id}"
       expected_direct_topic = "direct:#{user.id}"
-      assert {:ok, ^expected_user_topic} = Streamer.get_topic("user", user)
+      expected_pleroma_chat_topic = "user:pleroma_chat:#{user.id}"
-      assert {:ok, ^expected_notif_topic} = Streamer.get_topic("user:notification", user)
+
-      assert {:ok, ^expected_direct_topic} = Streamer.get_topic("direct", user)
+      for valid_user_token <- [read_oauth_token, read_statuses_token] do
+        assert {:ok, ^expected_user_topic} = Streamer.get_topic("user", user, valid_user_token)
+
+        assert {:ok, ^expected_direct_topic} =
+                 Streamer.get_topic("direct", user, valid_user_token)
+
+        assert {:ok, ^expected_pleroma_chat_topic} =
+                 Streamer.get_topic("user:pleroma_chat", user, valid_user_token)
       end
 
-    test "allows hashtag streams", %{user: user} do
+      for invalid_user_token <- [read_notifications_token, badly_scoped_token],
-      assert {:ok, "hashtag:cofe"} = Streamer.get_topic("hashtag", user, %{"tag" => "cofe"})
+          user_topic <- ["user", "direct", "user:pleroma_chat"] do
+        assert {:error, :unauthorized} = Streamer.get_topic(user_topic, user, invalid_user_token)
       end
 
-    test "disallows registering to an user stream", %{user: user} do
+      for valid_notification_token <- [read_oauth_token, read_notifications_token] do
+        assert {:ok, ^expected_notification_topic} =
+                 Streamer.get_topic("user:notification", user, valid_notification_token)
+      end
+
+      for invalid_notification_token <- [read_statuses_token, badly_scoped_token] do
+        assert {:error, :unauthorized} =
+                 Streamer.get_topic("user:notification", user, invalid_notification_token)
+      end
+    end
+
+    test "allows hashtag streams (regardless of OAuth token scopes)", %{
+      user: user,
+      token: read_oauth_token
+    } do
+      for oauth_token <- [nil, read_oauth_token] do
+        assert {:ok, "hashtag:cofe"} =
+                 Streamer.get_topic("hashtag", user, oauth_token, %{"tag" => "cofe"})
+      end
+    end
+
+    test "disallows registering to another user's stream", %{user: user, token: read_oauth_token} do
       another_user = insert(:user)
-      assert {:error, _} = Streamer.get_topic("user:#{another_user.id}", user)
+      assert {:error, _} = Streamer.get_topic("user:#{another_user.id}", user, read_oauth_token)
-      assert {:error, _} = Streamer.get_topic("user:notification:#{another_user.id}", user)
+
-      assert {:error, _} = Streamer.get_topic("direct:#{another_user.id}", user)
+      assert {:error, _} =
+               Streamer.get_topic("user:notification:#{another_user.id}", user, read_oauth_token)
+
+      assert {:error, _} = Streamer.get_topic("direct:#{another_user.id}", user, read_oauth_token)
     end
 
-    test "allows list stream that are owned by the user", %{user: user} do
+    test "allows list stream that are owned by the user (with `read` or `read:lists` scopes)", %{
+      user: user,
+      token: read_oauth_token
+    } do
+      %{token: read_lists_token} = oauth_access(["read:lists"], user: user)
+      %{token: invalid_token} = oauth_access(["irrelevant:scope"], user: user)
       {:ok, list} = List.create("Test", user)
-      assert {:error, _} = Streamer.get_topic("list:#{list.id}", user)
+
-      assert {:ok, _} = Streamer.get_topic("list", user, %{"list" => list.id})
+      assert {:error, _} = Streamer.get_topic("list:#{list.id}", user, read_oauth_token)
+
+      for valid_token <- [read_oauth_token, read_lists_token] do
+        assert {:ok, _} = Streamer.get_topic("list", user, valid_token, %{"list" => list.id})
       end
 
-    test "disallows list stream that are not owned by the user", %{user: user} do
+      assert {:error, _} = Streamer.get_topic("list", user, invalid_token, %{"list" => list.id})
+    end
+
+    test "disallows list stream that are not owned by the user", %{user: user, token: oauth_token} do
       another_user = insert(:user)
       {:ok, list} = List.create("Test", another_user)
-      assert {:error, _} = Streamer.get_topic("list:#{list.id}", user)
+
-      assert {:error, _} = Streamer.get_topic("list", user, %{"list" => list.id})
+      assert {:error, _} = Streamer.get_topic("list:#{list.id}", user, oauth_token)
+      assert {:error, _} = Streamer.get_topic("list", user, oauth_token, %{"list" => list.id})
     end
   end
 
   describe "user streams" do
     setup do
-      user = insert(:user)
+      %{user: user, token: token} = oauth_access(["read"])
       notify = insert(:notification, user: user, activity: build(:note_activity))
-      {:ok, %{user: user, notify: notify}}
+      {:ok, %{user: user, notify: notify, token: token}}
     end
 
-    test "it streams the user's post in the 'user' stream", %{user: user} do
+    test "it streams the user's post in the 'user' stream", %{user: user, token: oauth_token} do
-      Streamer.get_topic_and_add_socket("user", user)
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
       {:ok, activity} = CommonAPI.post(user, %{status: "hey"})
+
       assert_receive {:render_with_user, _, _, ^activity}
       refute Streamer.filtered_by_user?(user, activity)
     end
 
-    test "it streams boosts of the user in the 'user' stream", %{user: user} do
+    test "it streams boosts of the user in the 'user' stream", %{user: user, token: oauth_token} do
-      Streamer.get_topic_and_add_socket("user", user)
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
 
       other_user = insert(:user)
       {:ok, activity} = CommonAPI.post(other_user, %{status: "hey"})
@@ -117,9 +173,10 @@ test "it streams boosts of the user in the 'user' stream", %{user: user} do
     end
 
     test "it does not stream announces of the user's own posts in the 'user' stream", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
-      Streamer.get_topic_and_add_socket("user", user)
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
 
       other_user = insert(:user)
       {:ok, activity} = CommonAPI.post(user, %{status: "hey"})
@@ -129,9 +186,10 @@ test "it does not stream announces of the user's own posts in the 'user' stream"
     end
 
     test "it does stream notifications announces of the user's own posts in the 'user' stream", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
-      Streamer.get_topic_and_add_socket("user", user)
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
 
       other_user = insert(:user)
       {:ok, activity} = CommonAPI.post(user, %{status: "hey"})
@@ -145,8 +203,11 @@ test "it does stream notifications announces of the user's own posts in the 'use
       refute Streamer.filtered_by_user?(user, notification)
     end
 
-    test "it streams boosts of mastodon user in the 'user' stream", %{user: user} do
+    test "it streams boosts of mastodon user in the 'user' stream", %{
-      Streamer.get_topic_and_add_socket("user", user)
+      user: user,
+      token: oauth_token
+    } do
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
 
       other_user = insert(:user)
       {:ok, activity} = CommonAPI.post(other_user, %{status: "hey"})
@@ -164,21 +225,34 @@ test "it streams boosts of mastodon user in the 'user' stream", %{user: user} do
       refute Streamer.filtered_by_user?(user, announce)
     end
 
-    test "it sends notify to in the 'user' stream", %{user: user, notify: notify} do
+    test "it sends notify to in the 'user' stream", %{
-      Streamer.get_topic_and_add_socket("user", user)
+      user: user,
+      token: oauth_token,
+      notify: notify
+    } do
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
       Streamer.stream("user", notify)
+
       assert_receive {:render_with_user, _, _, ^notify}
       refute Streamer.filtered_by_user?(user, notify)
     end
 
-    test "it sends notify to in the 'user:notification' stream", %{user: user, notify: notify} do
+    test "it sends notify to in the 'user:notification' stream", %{
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      user: user,
+      token: oauth_token,
+      notify: notify
+    } do
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
       Streamer.stream("user:notification", notify)
+
       assert_receive {:render_with_user, _, _, ^notify}
       refute Streamer.filtered_by_user?(user, notify)
     end
 
-    test "it sends chat messages to the 'user:pleroma_chat' stream", %{user: user} do
+    test "it sends chat messages to the 'user:pleroma_chat' stream", %{
+      user: user,
+      token: oauth_token
+    } do
       other_user = insert(:user)
 
       {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno")
@@ -187,7 +261,7 @@ test "it sends chat messages to the 'user:pleroma_chat' stream", %{user: user} d
       cm_ref = MessageReference.for_chat_and_object(chat, object)
       cm_ref = %{cm_ref | chat: chat, object: object}
 
-      Streamer.get_topic_and_add_socket("user:pleroma_chat", user)
+      Streamer.get_topic_and_add_socket("user:pleroma_chat", user, oauth_token)
       Streamer.stream("user:pleroma_chat", {user, cm_ref})
 
       text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref})
@@ -196,7 +270,7 @@ test "it sends chat messages to the 'user:pleroma_chat' stream", %{user: user} d
       assert_receive {:text, ^text}
     end
 
-    test "it sends chat messages to the 'user' stream", %{user: user} do
+    test "it sends chat messages to the 'user' stream", %{user: user, token: oauth_token} do
       other_user = insert(:user)
 
       {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey cirno")
@@ -205,7 +279,7 @@ test "it sends chat messages to the 'user' stream", %{user: user} do
       cm_ref = MessageReference.for_chat_and_object(chat, object)
       cm_ref = %{cm_ref | chat: chat, object: object}
 
-      Streamer.get_topic_and_add_socket("user", user)
+      Streamer.get_topic_and_add_socket("user", user, oauth_token)
       Streamer.stream("user", {user, cm_ref})
 
       text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref})
@@ -214,7 +288,10 @@ test "it sends chat messages to the 'user' stream", %{user: user} do
       assert_receive {:text, ^text}
     end
 
-    test "it sends chat message notifications to the 'user:notification' stream", %{user: user} do
+    test "it sends chat message notifications to the 'user:notification' stream", %{
+      user: user,
+      token: oauth_token
+    } do
       other_user = insert(:user)
 
       {:ok, create_activity} = CommonAPI.post_chat_message(other_user, user, "hey")
@@ -223,19 +300,21 @@ test "it sends chat message notifications to the 'user:notification' stream", %{
         Repo.get_by(Pleroma.Notification, user_id: user.id, activity_id: create_activity.id)
         |> Repo.preload(:activity)
 
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
       Streamer.stream("user:notification", notify)
+
       assert_receive {:render_with_user, _, _, ^notify}
       refute Streamer.filtered_by_user?(user, notify)
     end
 
     test "it doesn't send notify to the 'user:notification' stream when a user is blocked", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
       blocked = insert(:user)
       {:ok, _user_relationship} = User.block(user, blocked)
 
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
 
       {:ok, activity} = CommonAPI.post(user, %{status: ":("})
       {:ok, _} = CommonAPI.favorite(blocked, activity.id)
@@ -244,14 +323,15 @@ test "it doesn't send notify to the 'user:notification' stream when a user is bl
     end
 
     test "it doesn't send notify to the 'user:notification' stream when a thread is muted", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
       user2 = insert(:user)
 
       {:ok, activity} = CommonAPI.post(user, %{status: "super hot take"})
       {:ok, _} = CommonAPI.add_mute(user, activity)
 
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
 
       {:ok, favorite_activity} = CommonAPI.favorite(user2, activity.id)
 
@@ -260,12 +340,13 @@ test "it doesn't send notify to the 'user:notification' stream when a thread is
     end
 
     test "it sends favorite to 'user:notification' stream'", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
       user2 = insert(:user, %{ap_id: "https://hecking-lewd-place.com/user/meanie"})
 
       {:ok, activity} = CommonAPI.post(user, %{status: "super hot take"})
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
       {:ok, favorite_activity} = CommonAPI.favorite(user2, activity.id)
 
       assert_receive {:render_with_user, _, "notification.json", notif}
@@ -274,13 +355,14 @@ test "it sends favorite to 'user:notification' stream'", %{
     end
 
     test "it doesn't send the 'user:notification' stream' when a domain is blocked", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
       user2 = insert(:user, %{ap_id: "https://hecking-lewd-place.com/user/meanie"})
 
       {:ok, user} = User.block_domain(user, "hecking-lewd-place.com")
       {:ok, activity} = CommonAPI.post(user, %{status: "super hot take"})
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
       {:ok, favorite_activity} = CommonAPI.favorite(user2, activity.id)
 
       refute_receive _
@@ -288,7 +370,8 @@ test "it doesn't send the 'user:notification' stream' when a domain is blocked",
     end
 
     test "it sends follow activities to the 'user:notification' stream", %{
-      user: user
+      user: user,
+      token: oauth_token
     } do
       user_url = user.ap_id
       user2 = insert(:user)
@@ -303,7 +386,7 @@ test "it sends follow activities to the 'user:notification' stream", %{
           %Tesla.Env{status: 200, body: body}
       end)
 
-      Streamer.get_topic_and_add_socket("user:notification", user)
+      Streamer.get_topic_and_add_socket("user:notification", user, oauth_token)
       {:ok, _follower, _followed, follow_activity} = CommonAPI.follow(user2, user)
 
       assert_receive {:render_with_user, _, "notification.json", notif}
@@ -312,34 +395,22 @@ test "it sends follow activities to the 'user:notification' stream", %{
     end
   end
 
-  test "it sends to public authenticated" do
+  describe "public streams" do
-    user = insert(:user)
+    test "it sends to public (authenticated)" do
+      %{user: user, token: oauth_token} = oauth_access(["read"])
       other_user = insert(:user)
 
-    Streamer.get_topic_and_add_socket("public", other_user)
+      Streamer.get_topic_and_add_socket("public", user, oauth_token)
 
-    {:ok, activity} = CommonAPI.post(user, %{status: "Test"})
-    assert_receive {:render_with_user, _, _, ^activity}
-    refute Streamer.filtered_by_user?(user, activity)
-  end
-
-  test "works for deletions" do
-    user = insert(:user)
-    other_user = insert(:user)
       {:ok, activity} = CommonAPI.post(other_user, %{status: "Test"})
-
+      assert_receive {:render_with_user, _, _, ^activity}
-    Streamer.get_topic_and_add_socket("public", user)
+      refute Streamer.filtered_by_user?(other_user, activity)
-
-    {:ok, _} = CommonAPI.delete(activity.id, other_user)
-    activity_id = activity.id
-    assert_receive {:text, event}
-    assert %{"event" => "delete", "payload" => ^activity_id} = Jason.decode!(event)
     end
 
-  test "it sends to public unauthenticated" do
+    test "it sends to public (unauthenticated)" do
       user = insert(:user)
 
-    Streamer.get_topic_and_add_socket("public", nil)
+      Streamer.get_topic_and_add_socket("public", nil, nil)
 
       {:ok, activity} = CommonAPI.post(user, %{status: "Test"})
       activity_id = activity.id
@@ -352,11 +423,25 @@ test "it sends to public unauthenticated" do
       assert %{"event" => "delete", "payload" => ^activity_id} = Jason.decode!(event)
     end
 
-  describe "thread_containment" do
+    test "handles deletions" do
+      %{user: user, token: oauth_token} = oauth_access(["read"])
+      other_user = insert(:user)
+      {:ok, activity} = CommonAPI.post(other_user, %{status: "Test"})
+
+      Streamer.get_topic_and_add_socket("public", user, oauth_token)
+
+      {:ok, _} = CommonAPI.delete(activity.id, other_user)
+      activity_id = activity.id
+      assert_receive {:text, event}
+      assert %{"event" => "delete", "payload" => ^activity_id} = Jason.decode!(event)
+    end
+  end
+
+  describe "thread_containment/2" do
     test "it filters to user if recipients invalid and thread containment is enabled" do
       Pleroma.Config.put([:instance, :skip_thread_containment], false)
       author = insert(:user)
-      user = insert(:user)
+      %{user: user, token: oauth_token} = oauth_access(["read"])
       User.follow(user, author, :follow_accept)
 
       activity =
@@ -368,7 +453,7 @@ test "it filters to user if recipients invalid and thread containment is enabled
             )
         )
 
-      Streamer.get_topic_and_add_socket("public", user)
+      Streamer.get_topic_and_add_socket("public", user, oauth_token)
       Streamer.stream("public", activity)
       assert_receive {:render_with_user, _, _, ^activity}
       assert Streamer.filtered_by_user?(user, activity)
@@ -377,7 +462,7 @@ test "it filters to user if recipients invalid and thread containment is enabled
     test "it sends message if recipients invalid and thread containment is disabled" do
       Pleroma.Config.put([:instance, :skip_thread_containment], true)
       author = insert(:user)
-      user = insert(:user)
+      %{user: user, token: oauth_token} = oauth_access(["read"])
       User.follow(user, author, :follow_accept)
 
       activity =
@@ -389,7 +474,7 @@ test "it sends message if recipients invalid and thread containment is disabled"
             )
         )
 
-      Streamer.get_topic_and_add_socket("public", user)
+      Streamer.get_topic_and_add_socket("public", user, oauth_token)
       Streamer.stream("public", activity)
 
       assert_receive {:render_with_user, _, _, ^activity}
@@ -400,6 +485,7 @@ test "it sends message if recipients invalid and thread containment is enabled b
       Pleroma.Config.put([:instance, :skip_thread_containment], false)
       author = insert(:user)
       user = insert(:user, skip_thread_containment: true)
+      %{token: oauth_token} = oauth_access(["read"], user: user)
       User.follow(user, author, :follow_accept)
 
       activity =
@@ -411,7 +497,7 @@ test "it sends message if recipients invalid and thread containment is enabled b
             )
         )
 
-      Streamer.get_topic_and_add_socket("public", user)
+      Streamer.get_topic_and_add_socket("public", user, oauth_token)
       Streamer.stream("public", activity)
 
       assert_receive {:render_with_user, _, _, ^activity}
@@ -420,23 +506,26 @@ test "it sends message if recipients invalid and thread containment is enabled b
   end
 
   describe "blocks" do
-    test "it filters messages involving blocked users" do
+    setup do: oauth_access(["read"])
-      user = insert(:user)
+
+    test "it filters messages involving blocked users", %{user: user, token: oauth_token} do
       blocked_user = insert(:user)
       {:ok, _user_relationship} = User.block(user, blocked_user)
 
-      Streamer.get_topic_and_add_socket("public", user)
+      Streamer.get_topic_and_add_socket("public", user, oauth_token)
       {:ok, activity} = CommonAPI.post(blocked_user, %{status: "Test"})
       assert_receive {:render_with_user, _, _, ^activity}
       assert Streamer.filtered_by_user?(user, activity)
     end
 
-    test "it filters messages transitively involving blocked users" do
+    test "it filters messages transitively involving blocked users", %{
-      blocker = insert(:user)
+      user: blocker,
+      token: blocker_token
+    } do
       blockee = insert(:user)
       friend = insert(:user)
 
-      Streamer.get_topic_and_add_socket("public", blocker)
+      Streamer.get_topic_and_add_socket("public", blocker, blocker_token)
 
       {:ok, _user_relationship} = User.block(blocker, blockee)
 
@@ -458,8 +547,9 @@ test "it filters messages transitively involving blocked users" do
   end
 
   describe "lists" do
-    test "it doesn't send unwanted DMs to list" do
+    setup do: oauth_access(["read"])
-      user_a = insert(:user)
+
+    test "it doesn't send unwanted DMs to list", %{user: user_a, token: user_a_token} do
       user_b = insert(:user)
       user_c = insert(:user)
 
@@ -468,7 +558,7 @@ test "it doesn't send unwanted DMs to list" do
       {:ok, list} = List.create("Test", user_a)
       {:ok, list} = List.follow(list, user_b)
 
-      Streamer.get_topic_and_add_socket("list", user_a, %{"list" => list.id})
+      Streamer.get_topic_and_add_socket("list", user_a, user_a_token, %{"list" => list.id})
 
       {:ok, _activity} =
         CommonAPI.post(user_b, %{
@@ -479,14 +569,13 @@ test "it doesn't send unwanted DMs to list" do
       refute_receive _
     end
 
-    test "it doesn't send unwanted private posts to list" do
+    test "it doesn't send unwanted private posts to list", %{user: user_a, token: user_a_token} do
-      user_a = insert(:user)
       user_b = insert(:user)
 
       {:ok, list} = List.create("Test", user_a)
       {:ok, list} = List.follow(list, user_b)
 
-      Streamer.get_topic_and_add_socket("list", user_a, %{"list" => list.id})
+      Streamer.get_topic_and_add_socket("list", user_a, user_a_token, %{"list" => list.id})
 
       {:ok, _activity} =
         CommonAPI.post(user_b, %{
@@ -497,8 +586,7 @@ test "it doesn't send unwanted private posts to list" do
       refute_receive _
     end
 
-    test "it sends wanted private posts to list" do
+    test "it sends wanted private posts to list", %{user: user_a, token: user_a_token} do
-      user_a = insert(:user)
       user_b = insert(:user)
 
       {:ok, user_a} = User.follow(user_a, user_b)
@@ -506,7 +594,7 @@ test "it sends wanted private posts to list" do
       {:ok, list} = List.create("Test", user_a)
       {:ok, list} = List.follow(list, user_b)
 
-      Streamer.get_topic_and_add_socket("list", user_a, %{"list" => list.id})
+      Streamer.get_topic_and_add_socket("list", user_a, user_a_token, %{"list" => list.id})
 
       {:ok, activity} =
         CommonAPI.post(user_b, %{
@@ -520,8 +608,9 @@ test "it sends wanted private posts to list" do
   end
 
   describe "muted reblogs" do
-    test "it filters muted reblogs" do
+    setup do: oauth_access(["read"])
-      user1 = insert(:user)
+
+    test "it filters muted reblogs", %{user: user1, token: user1_token} do
       user2 = insert(:user)
       user3 = insert(:user)
       CommonAPI.follow(user1, user2)
@@ -529,34 +618,38 @@ test "it filters muted reblogs" do
 
       {:ok, create_activity} = CommonAPI.post(user3, %{status: "I'm kawen"})
 
-      Streamer.get_topic_and_add_socket("user", user1)
+      Streamer.get_topic_and_add_socket("user", user1, user1_token)
       {:ok, announce_activity} = CommonAPI.repeat(create_activity.id, user2)
       assert_receive {:render_with_user, _, _, ^announce_activity}
       assert Streamer.filtered_by_user?(user1, announce_activity)
     end
 
-    test "it filters reblog notification for reblog-muted actors" do
+    test "it filters reblog notification for reblog-muted actors", %{
-      user1 = insert(:user)
+      user: user1,
+      token: user1_token
+    } do
       user2 = insert(:user)
       CommonAPI.follow(user1, user2)
       CommonAPI.hide_reblogs(user1, user2)
 
       {:ok, create_activity} = CommonAPI.post(user1, %{status: "I'm kawen"})
-      Streamer.get_topic_and_add_socket("user", user1)
+      Streamer.get_topic_and_add_socket("user", user1, user1_token)
       {:ok, _announce_activity} = CommonAPI.repeat(create_activity.id, user2)
 
       assert_receive {:render_with_user, _, "notification.json", notif}
       assert Streamer.filtered_by_user?(user1, notif)
     end
 
-    test "it send non-reblog notification for reblog-muted actors" do
+    test "it send non-reblog notification for reblog-muted actors", %{
-      user1 = insert(:user)
+      user: user1,
+      token: user1_token
+    } do
       user2 = insert(:user)
       CommonAPI.follow(user1, user2)
       CommonAPI.hide_reblogs(user1, user2)
 
       {:ok, create_activity} = CommonAPI.post(user1, %{status: "I'm kawen"})
-      Streamer.get_topic_and_add_socket("user", user1)
+      Streamer.get_topic_and_add_socket("user", user1, user1_token)
       {:ok, _favorite_activity} = CommonAPI.favorite(user2, create_activity.id)
 
       assert_receive {:render_with_user, _, "notification.json", notif}
@@ -564,27 +657,28 @@ test "it send non-reblog notification for reblog-muted actors" do
     end
   end
 
+  describe "muted threads" do
     test "it filters posts from muted threads" do
       user = insert(:user)
-    user2 = insert(:user)
+      %{user: user2, token: user2_token} = oauth_access(["read"])
-    Streamer.get_topic_and_add_socket("user", user2)
+      Streamer.get_topic_and_add_socket("user", user2, user2_token)
+
       {:ok, user2, user, _activity} = CommonAPI.follow(user2, user)
       {:ok, activity} = CommonAPI.post(user, %{status: "super hot take"})
       {:ok, _} = CommonAPI.add_mute(user2, activity)
+
       assert_receive {:render_with_user, _, _, ^activity}
       assert Streamer.filtered_by_user?(user2, activity)
     end
-
-  describe "direct streams" do
-    setup do
-      :ok
   end
 
-    test "it sends conversation update to the 'direct' stream", %{} do
+  describe "direct streams" do
-      user = insert(:user)
+    setup do: oauth_access(["read"])
+
+    test "it sends conversation update to the 'direct' stream", %{user: user, token: oauth_token} do
       another_user = insert(:user)
 
-      Streamer.get_topic_and_add_socket("direct", user)
+      Streamer.get_topic_and_add_socket("direct", user, oauth_token)
 
       {:ok, _create_activity} =
         CommonAPI.post(another_user, %{
@@ -602,11 +696,11 @@ test "it sends conversation update to the 'direct' stream", %{} do
       assert last_status["pleroma"]["direct_conversation_id"] == participation.id
     end
 
-    test "it doesn't send conversation update to the 'direct' stream when the last message in the conversation is deleted" do
+    test "it doesn't send conversation update to the 'direct' stream when the last message in the conversation is deleted",
-      user = insert(:user)
+         %{user: user, token: oauth_token} do
       another_user = insert(:user)
 
-      Streamer.get_topic_and_add_socket("direct", user)
+      Streamer.get_topic_and_add_socket("direct", user, oauth_token)
 
       {:ok, create_activity} =
         CommonAPI.post(another_user, %{
@@ -629,10 +723,12 @@ test "it doesn't send conversation update to the 'direct' stream when the last m
       refute_receive _
     end
 
-    test "it sends conversation update to the 'direct' stream when a message is deleted" do
+    test "it sends conversation update to the 'direct' stream when a message is deleted", %{
-      user = insert(:user)
+      user: user,
+      token: oauth_token
+    } do
       another_user = insert(:user)
-      Streamer.get_topic_and_add_socket("direct", user)
+      Streamer.get_topic_and_add_socket("direct", user, oauth_token)
 
       {:ok, create_activity} =
         CommonAPI.post(another_user, %{

test/web/twitter_api/util_controller_test.exs

@@ -21,170 +21,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do
   setup do: clear_config([:instance])
   setup do: clear_config([:frontend_configurations, :pleroma_fe])
 
-  describe "POST /api/pleroma/follow_import" do
-    setup do: oauth_access(["follow"])
-
-    test "it returns HTTP 200", %{conn: conn} do
-      user2 = insert(:user)
-
-      response =
-        conn
-        |> post("/api/pleroma/follow_import", %{"list" => "#{user2.ap_id}"})
-        |> json_response(:ok)
-
-      assert response == "job started"
-    end
-
-    test "it imports follow lists from file", %{user: user1, conn: conn} do
-      user2 = insert(:user)
-
-      with_mocks([
-        {File, [],
-         read!: fn "follow_list.txt" ->
-           "Account address,Show boosts\n#{user2.ap_id},true"
-         end}
-      ]) do
-        response =
-          conn
-          |> post("/api/pleroma/follow_import", %{"list" => %Plug.Upload{path: "follow_list.txt"}})
-          |> json_response(:ok)
-
-        assert response == "job started"
-
-        assert ObanHelpers.member?(
-                 %{
-                   "op" => "follow_import",
-                   "follower_id" => user1.id,
-                   "followed_identifiers" => [user2.ap_id]
-                 },
-                 all_enqueued(worker: Pleroma.Workers.BackgroundWorker)
-               )
-      end
-    end
-
-    test "it imports new-style mastodon follow lists", %{conn: conn} do
-      user2 = insert(:user)
-
-      response =
-        conn
-        |> post("/api/pleroma/follow_import", %{
-          "list" => "Account address,Show boosts\n#{user2.ap_id},true"
-        })
-        |> json_response(:ok)
-
-      assert response == "job started"
-    end
-
-    test "requires 'follow' or 'write:follows' permissions" do
-      token1 = insert(:oauth_token, scopes: ["read", "write"])
-      token2 = insert(:oauth_token, scopes: ["follow"])
-      token3 = insert(:oauth_token, scopes: ["something"])
-      another_user = insert(:user)
-
-      for token <- [token1, token2, token3] do
-        conn =
-          build_conn()
-          |> put_req_header("authorization", "Bearer #{token.token}")
-          |> post("/api/pleroma/follow_import", %{"list" => "#{another_user.ap_id}"})
-
-        if token == token3 do
-          assert %{"error" => "Insufficient permissions: follow | write:follows."} ==
-                   json_response(conn, 403)
-        else
-          assert json_response(conn, 200)
-        end
-      end
-    end
-
-    test "it imports follows with different nickname variations", %{conn: conn} do
-      [user2, user3, user4, user5, user6] = insert_list(5, :user)
-
-      identifiers =
-        [
-          user2.ap_id,
-          user3.nickname,
-          "  ",
-          "@" <> user4.nickname,
-          user5.nickname <> "@localhost",
-          "@" <> user6.nickname <> "@localhost"
-        ]
-        |> Enum.join("\n")
-
-      response =
-        conn
-        |> post("/api/pleroma/follow_import", %{"list" => identifiers})
-        |> json_response(:ok)
-
-      assert response == "job started"
-      assert [{:ok, job_result}] = ObanHelpers.perform_all()
-      assert job_result == [user2, user3, user4, user5, user6]
-    end
-  end
-
-  describe "POST /api/pleroma/blocks_import" do
-    # Note: "follow" or "write:blocks" permission is required
-    setup do: oauth_access(["write:blocks"])
-
-    test "it returns HTTP 200", %{conn: conn} do
-      user2 = insert(:user)
-
-      response =
-        conn
-        |> post("/api/pleroma/blocks_import", %{"list" => "#{user2.ap_id}"})
-        |> json_response(:ok)
-
-      assert response == "job started"
-    end
-
-    test "it imports blocks users from file", %{user: user1, conn: conn} do
-      user2 = insert(:user)
-      user3 = insert(:user)
-
-      with_mocks([
-        {File, [], read!: fn "blocks_list.txt" -> "#{user2.ap_id} #{user3.ap_id}" end}
-      ]) do
-        response =
-          conn
-          |> post("/api/pleroma/blocks_import", %{"list" => %Plug.Upload{path: "blocks_list.txt"}})
-          |> json_response(:ok)
-
-        assert response == "job started"
-
-        assert ObanHelpers.member?(
-                 %{
-                   "op" => "blocks_import",
-                   "blocker_id" => user1.id,
-                   "blocked_identifiers" => [user2.ap_id, user3.ap_id]
-                 },
-                 all_enqueued(worker: Pleroma.Workers.BackgroundWorker)
-               )
-      end
-    end
-
-    test "it imports blocks with different nickname variations", %{conn: conn} do
-      [user2, user3, user4, user5, user6] = insert_list(5, :user)
-
-      identifiers =
-        [
-          user2.ap_id,
-          user3.nickname,
-          "@" <> user4.nickname,
-          user5.nickname <> "@localhost",
-          "@" <> user6.nickname <> "@localhost"
-        ]
-        |> Enum.join(" ")
-
-      response =
-        conn
-        |> post("/api/pleroma/blocks_import", %{"list" => identifiers})
-        |> json_response(:ok)
-
-      assert response == "job started"
-      assert [{:ok, job_result}] = ObanHelpers.perform_all()
-      assert job_result == [user2, user3, user4, user5, user6]
-    end
-  end
-
   describe "PUT /api/pleroma/notification_settings" do
     setup do: oauth_access(["write:accounts"])