Merge remote-tracking branch 'pleroma/develop' into feature/addressable-lists

2019-05-24 21:05:57 +07:00 · 2019-05-24 21:05:57 +07:00 · f333041a0a
parent 3b71612d3d 6499adc6a8
commit f333041a0a
64 changed files with 1191 additions and 212 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -45,7 +45,8 @@ docs-build:
 unit-testing:
  stage: test
  services:
-  - name: postgres:9.6.2
+  - name: lainsoykaf/postgres-with-rum
+    alias: postgres
    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
  script:
    - mix deps.get
@ -54,6 +55,21 @@ unit-testing:
    - mix test --trace --preload-modules
    - mix coveralls

+unit-testing-rum:
+  stage: test
+  services:
+  - name: lainsoykaf/postgres-with-rum
+    alias: postgres
+    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+  variables:
+    RUM_ENABLED: "true"
+  script:
+    - mix deps.get
+    - mix ecto.create
+    - mix ecto.migrate
+    - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
+    - mix test --trace --preload-modules
+
 lint:
  stage: test
  script:
@ -65,7 +81,6 @@ analysis:
    - mix deps.get
    - mix credo --strict --only=warnings,todo,fixme,consistency,readability

-
 docs-deploy:
  stage: deploy
  image: alpine:3.9
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).

 ## [unreleased]
 ### Added
+- Optional SSH access mode. (Needs `erlang-ssh` package on some distributions).
+- [MongooseIM](https://github.com/esl/MongooseIM) http authentication support.
 - LDAP authentication
 - External OAuth provider authentication
 - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc.
@ -23,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `report_uri` option
 - Pleroma API: User subscriptions
 - Pleroma API: Healthcheck endpoint
+- Pleroma API: `/api/v1/pleroma/mascot` per-user frontend mascot configuration endpoints
 - Admin API: Endpoints for listing/revoking invite tokens
 - Admin API: Endpoints for making users follow/unfollow each other
 - Admin API: added filters (role, tags, email, name) for users endpoint
@ -38,6 +41,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Metadata: RelMe provider
 - OAuth: added support for refresh tokens
 - Emoji packs and emoji pack manager
+- Object pruning (`mix pleroma.database prune_objects`)
+- OAuth: added job to clean expired access tokens
+- MRF: Support for rejecting reports from specific instances (`mrf_simple`)
+- MRF: Support for stripping avatars and banner images from specific instances (`mrf_simple`)
 - Addressable lists

 ### Changed
@ -72,6 +79,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Deps: Updated Ecto to 3.0.7
 - Don't ship finmoji by default, they can be installed as an emoji pack
 - Hide deactivated users and their statuses
+- Posts which are marked sensitive or tagged nsfw no longer have link previews.
+- HTTP connection timeout is now set to 10 seconds.
+- Respond with a 404 Not implemented JSON error message when requested API is not implemented

 ### Fixed
 - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended.
@ -103,6 +113,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Correct `reblogged`, `favourited`, and `bookmarked` values in the reblog status JSON
 - Mastodon API: Exposing default scope of the user to anyone
 - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`]
+- User-Agent is now sent correctly for all HTTP requests.

 ## Removed
 - Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations`
--- a/config/config.exs
+++ b/config/config.exs
@ -192,6 +192,7 @@
 # Configures http settings, upstream proxy etc.
 config :pleroma, :http,
  proxy_url: nil,
+  send_user_agent: true,
  adapter: [
    ssl_options: [
      # We don't support TLS v1.3 yet
@ -238,7 +239,8 @@
  welcome_message: nil,
  max_report_comment_size: 1000,
  safe_dm_mentions: false,
-  healthcheck: false
+  healthcheck: false,
+  remote_post_retention_days: 90

 config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800

@ -275,6 +277,19 @@
    showInstanceSpecificPanel: true
  }

+config :pleroma, :assets,
+  mascots: [
+    pleroma_fox_tan: %{
+      url: "/images/pleroma-fox-tan-smol.png",
+      mime_type: "image/png"
+    },
+    pleroma_fox_tan_shy: %{
+      url: "/images/pleroma-fox-tan-shy.png",
+      mime_type: "image/png"
+    }
+  ],
+  default_mascot: :pleroma_fox_tan
+
 config :pleroma, :activitypub,
  accept_blocks: true,
  unfollow_blocked: true,
@ -297,8 +312,11 @@
  media_removal: [],
  media_nsfw: [],
  federated_timeline_removal: [],
+  report_removal: [],
  reject: [],
-  accept: []
+  accept: [],
+  avatar_removal: [],
+  banner_removal: []

 config :pleroma, :mrf_keyword,
  reject: [],
@ -369,6 +387,7 @@
    "activities",
    "api",
    "auth",
+    "check_password",
    "dev",
    "friend-requests",
    "inbox",
@ -389,6 +408,7 @@
    "status",
    "tag",
    "user-search",
+    "user_exists",
    "users",
    "web"
  ]
@ -463,7 +483,11 @@

 config :pleroma, :oauth2,
  token_expires_in: 600,
-  issue_new_refresh_token: true
+  issue_new_refresh_token: true,
+  clean_expired_tokens: false,
+  clean_expired_tokens_interval: 86_400_000
+
+config :pleroma, :database, rum_enabled: false

 config :http_signatures,
  adapter: Pleroma.Signature
--- a/config/test.exs
+++ b/config/test.exs
@ -63,6 +63,12 @@

 config :pleroma, :http_security, report_uri: "https://endpoint.com"

+config :pleroma, :http, send_user_agent: false
+
+rum_enabled = System.get_env("RUM_ENABLED") == "true"
+config :pleroma, :database, rum_enabled: rum_enabled
+IO.puts("RUM enabled: #{rum_enabled}")
+
 try do
  import_config "test.secret.exs"
 rescue
--- a/docs/api/pleroma_api.md
+++ b/docs/api/pleroma_api.md
@ -252,6 +252,45 @@ See [Admin-API](Admin-API.md)
 ]
 ```

+## `/api/v1/pleroma/mascot`
+### Gets user mascot image
+* Method `GET`
+* Authentication: required
+
+* Response: JSON. Returns a mastodon media attachment entity.
+* Example response:
+```json
+{
+    "id": "abcdefg",
+    "url": "https://pleroma.example.org/media/abcdefg.png",
+    "type": "image",
+    "pleroma": {
+        "mime_type": "image/png"
+    }
+}
+```
+
+### Updates user mascot image
+* Method `PUT`
+* Authentication: required
+* Params:
+    * `image`: Multipart image
+* Response: JSON. Returns a mastodon media attachment entity
+  when successful, otherwise returns HTTP 415 `{"error": "error_msg"}`
+* Example response:
+```json
+{
+    "id": "abcdefg",
+    "url": "https://pleroma.example.org/media/abcdefg.png",
+    "type": "image",
+    "pleroma": {
+        "mime_type": "image/png"
+    }
+}
+```
+* Note: Behaves exactly the same as `POST /api/v1/upload`.
+  Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`.
+
 ## `/api/pleroma/notification_settings`
 ### Updates user notification settings
 * Method `PUT`
--- a/docs/config.md
+++ b/docs/config.md
@ -104,6 +104,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`)
 * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`)
 * `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``.
+* `remote_post_retention_days`: the default amount of days to retain remote posts when pruning the database

 ## :app_account_creation
 REST API for creating an account settings
@ -203,12 +204,25 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
 * `hide_post_stats`: Hide notices statistics(repeats, favorites, …)
 * `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …)

+## :assets
+
+This section configures assets to be used with various frontends. Currently the only option
+relates to mascots on the mastodon frontend
+
+* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a
+  `mime_type` key.
+* `default_mascot`: An element from `mascots` - This will be used as the default mascot
+  on MastoFE (default: `:pleroma_fox_tan`)
+
 ## :mrf_simple
 * `media_removal`: List of instances to remove medias from
 * `media_nsfw`: List of instances to put medias as NSFW(sensitive) from
 * `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline
 * `reject`: List of instances to reject any activities from
 * `accept`: List of instances to accept any activities from
+* `report_removal`: List of instances to reject reports from
+* `avatar_removal`: List of instances to strip avatars from
+* `banner_removal`: List of instances to strip banners from

 ## :mrf_rejectnonpublic
 * `allow_followersonly`: whether to allow followers-only posts
@ -467,7 +481,7 @@ config :esshd,
  password_authenticator: "Pleroma.BBS.Authenticator"
 ```

-Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT`
+Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -m PEM -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT`

 ## :auth

@ -539,8 +553,25 @@ Configure OAuth 2 provider capabilities:

 * `token_expires_in` - The lifetime in seconds of the access token.
 * `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token.
+* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`.
+* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours).

 ## :emoji
 * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
 * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]`
 * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays).
+
+## Database options
+
+### RUM indexing for full text search
+* `rum_enabled`: If RUM indexes should be used. Defaults to `false`.
+
+RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum.
+
+Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes.
+
+To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run:
+
+`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`
+
+This will probably take a long time.
--- a/docs/config/howto_mongooseim.md
+++ b/docs/config/howto_mongooseim.md
@ -0,0 +1,10 @@
+# Configuring MongooseIM (XMPP Server) to use Pleroma for authentication
+
+If you want to give your Pleroma users an XMPP (chat) account, you can configure [MongooseIM](https://github.com/esl/MongooseIM) to use your Pleroma server for user authentication, automatically giving every local user an XMPP account.
+
+In general, you just have to follow the configuration described at [https://mongooseim.readthedocs.io/en/latest/authentication-backends/HTTP-authentication-module/](https://mongooseim.readthedocs.io/en/latest/authentication-backends/HTTP-authentication-module/) and do these changes to your mongooseim.cfg.
+
+1. Set the auth_method to `{auth_method, http}`.
+2. Add the http auth pool like this: `{http, global, auth, [{workers, 50}], [{server, "https://yourpleromainstance.com"}]}`
+
+Restart your MongooseIM server, your users should now be able to connect with their Pleroma credentials.
--- a/docs/config/mrf.md
+++ b/docs/config/mrf.md
@ -5,11 +5,12 @@ Possible uses include:

 * marking incoming messages with media from a given account or instance as sensitive
 * rejecting messages from a specific instance
+* rejecting reports (flags) from a specific instance
 * removing/unlisting messages from the public timelines
 * removing media from messages
 * sending only public messages to a specific instance

-The MRF provides user-configurable policies.  The default policy is `NoOpPolicy`, which disables the MRF functionality.  Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.  
+The MRF provides user-configurable policies.  The default policy is `NoOpPolicy`, which disables the MRF functionality.  Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.
 It is possible to use multiple, active MRF policies at the same time.

 ## Quarantine Instances
@ -41,12 +42,13 @@ Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_si
 * `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media.
 * `reject`: Servers in this group will have their messages rejected.
 * `federated_timeline_removal`: Servers in this group will have their messages unlisted from the public timelines by flipping the `to` and `cc` fields.
+* `report_removal`: Servers in this group will have their reports (flags) rejected.

 Servers should be configured as lists.

 ### Example

-This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com` and remove messages from `spam.university` from the federated timeline:
+This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`:

 ```
 config :pleroma, :instance,
@ -56,7 +58,8 @@ config :pleroma, :mrf_simple,
  media_removal: ["illegalporn.biz"],
  media_nsfw: ["porn.biz", "porn.business"],
  reject: ["spam.com"],
-  federated_timeline_removal: ["spam.university"]
+  federated_timeline_removal: ["spam.university"],
+  report_removal: ["whiny.whiner"]

 ```

--- a/installation/caddyfile-pleroma.example
+++ b/installation/caddyfile-pleroma.example
@ -10,7 +10,9 @@ example.tld  {

  gzip

-  proxy / localhost:4000 {
+  # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+  # and `localhost.` resolves to [::0] on some systems: see issue #930
+  proxy / 127.0.0.1:4000 {
    websocket
    transparent
  }
--- a/installation/pleroma-apache.conf
+++ b/installation/pleroma-apache.conf
@ -58,8 +58,10 @@ CustomLog ${APACHE_LOG_DIR}/access.log combined
    RewriteRule /(.*) ws://localhost:4000/$1 [P,L]

    ProxyRequests off
-    ProxyPass / http://localhost:4000/
-    ProxyPassReverse / http://localhost:4000/
+    # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+    # and `localhost.` resolves to [::0] on some systems: see issue #930
+    ProxyPass / http://127.0.0.1:4000/
+    ProxyPassReverse / http://127.0.0.1:4000/

    RequestHeader set Host ${servername}
    ProxyPreserveHost On
--- a/installation/pleroma.nginx
+++ b/installation/pleroma.nginx
@ -69,7 +69,9 @@ server {
        proxy_set_header Connection "upgrade";
        proxy_set_header Host $http_host;

-        proxy_pass http://localhost:4000;
+	# this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
+	# and `localhost.` resolves to [::0] on some systems: see issue #930
+        proxy_pass http://127.0.0.1:4000;

        client_max_body_size 16m;
    }
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@ -5,6 +5,7 @@
 defmodule Mix.Tasks.Pleroma.Database do
  alias Mix.Tasks.Pleroma.Common
  alias Pleroma.Conversation
+  alias Pleroma.Object
  alias Pleroma.Repo
  alias Pleroma.User
  require Logger
@ -23,6 +24,10 @@ defmodule Mix.Tasks.Pleroma.Database do
    Options:
    - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references

+  ## Prune old objects from the database
+
+      mix pleroma.database prune_objects
+
  ## Create a conversation for all existing DMs. Can be safely re-run.

      mix pleroma.database bump_all_conversations
@ -72,4 +77,46 @@ def run(["update_users_following_followers_counts"]) do
    Enum.each(users, &User.remove_duplicated_following/1)
    Enum.each(users, &User.update_follower_count/1)
  end
+
+  def run(["prune_objects" | args]) do
+    import Ecto.Query
+
+    {options, [], []} =
+      OptionParser.parse(
+        args,
+        strict: [
+          vacuum: :boolean
+        ]
+      )
+
+    Common.start_pleroma()
+
+    deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
+
+    Logger.info("Pruning objects older than #{deadline} days")
+
+    time_deadline =
+      NaiveDateTime.utc_now()
+      |> NaiveDateTime.add(-(deadline * 86_400))
+
+    public = "https://www.w3.org/ns/activitystreams#Public"
+
+    from(o in Object,
+      where: fragment("?->'to' \\? ? OR ?->'cc' \\? ?", o.data, ^public, o.data, ^public),
+      where: o.inserted_at < ^time_deadline,
+      where:
+        fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
+    )
+    |> Repo.delete_all(timeout: :infinity)
+
+    if Keyword.get(options, :vacuum) do
+      Logger.info("Runnning VACUUM FULL")
+
+      Repo.query!(
+        "vacuum full;",
+        [],
+        timeout: :infinity
+      )
+    end
+  end
 end
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@ -10,6 +10,7 @@ defmodule Pleroma.Activity do
  alias Pleroma.Notification
  alias Pleroma.Object
  alias Pleroma.Repo
+  alias Pleroma.ThreadMute
  alias Pleroma.User

  import Ecto.Changeset
@ -37,6 +38,7 @@ defmodule Pleroma.Activity do
    field(:local, :boolean, default: true)
    field(:actor, :string)
    field(:recipients, {:array, :string}, default: [])
+    field(:thread_muted?, :boolean, virtual: true)
    # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark
    has_one(:bookmark, Bookmark)
    has_many(:notifications, Notification, on_delete: :delete_all)
@ -90,6 +92,16 @@ def with_preloaded_bookmark(query, %User{} = user) do

  def with_preloaded_bookmark(query, _), do: query

+  def with_set_thread_muted_field(query, %User{} = user) do
+    from([a] in query,
+      left_join: tm in ThreadMute,
+      on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data),
+      select: %Activity{a | thread_muted?: not is_nil(tm.id)}
+    )
+  end
+
+  def with_set_thread_muted_field(query, _), do: query
+
  def get_by_ap_id(ap_id) do
    Repo.one(
      from(
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@ -110,6 +110,7 @@ def start(_type, _args) do
        hackney_pool_children() ++
        [
          worker(Pleroma.Web.Federator.RetryQueue, []),
+          worker(Pleroma.Web.OAuth.Token.CleanWorker, []),
          worker(Pleroma.Stats, []),
          worker(Task, [&Pleroma.Web.Push.init/0], restart: :temporary, id: :web_push_init),
          worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary, id: :federator_init)
@ -131,19 +132,22 @@ def start(_type, _args) do
  defp setup_instrumenters do
    require Prometheus.Registry

-    :ok =
-      :telemetry.attach(
-        "prometheus-ecto",
-        [:pleroma, :repo, :query],
-        &Pleroma.Repo.Instrumenter.handle_event/4,
-        %{}
-      )
+    if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do
+      :ok =
+        :telemetry.attach(
+          "prometheus-ecto",
+          [:pleroma, :repo, :query],
+          &Pleroma.Repo.Instrumenter.handle_event/4,
+          %{}
+        )
+
+      Pleroma.Repo.Instrumenter.setup()
+    end

    Prometheus.Registry.register_collector(:prometheus_process_collector)
    Pleroma.Web.Endpoint.MetricsExporter.setup()
    Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
    Pleroma.Web.Endpoint.Instrumenter.setup()
-    Pleroma.Repo.Instrumenter.setup()
  end

  def enabled_hackney_pools do
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@ -8,7 +8,7 @@ defmodule Pleroma.Formatter do
  alias Pleroma.User
  alias Pleroma.Web.MediaProxy

-  @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/
+  @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/s
  @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
  @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/

--- a/lib/pleroma/http/connection.ex
+++ b/lib/pleroma/http/connection.ex
@ -8,7 +8,7 @@ defmodule Pleroma.HTTP.Connection do
  """

  @hackney_options [
-    connect_timeout: 2_000,
+    connect_timeout: 10_000,
    recv_timeout: 20_000,
    follow_redirect: true,
    pool: :federation
--- a/lib/pleroma/http/request_builder.ex
+++ b/lib/pleroma/http/request_builder.ex
@ -45,8 +45,15 @@ def url(request, u) do
  Add headers to the request
  """
  @spec headers(map(), list(tuple)) :: map()
-  def headers(request, h) do
-    Map.put_new(request, :headers, h)
+  def headers(request, header_list) do
+    header_list =
+      if Pleroma.Config.get([:http, :send_user_agent]) do
+        header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+      else
+        header_list
+      end
+
+    Map.put_new(request, :headers, header_list)
  end

  @doc """
--- a/lib/pleroma/keys.ex
+++ b/lib/pleroma/keys.ex
@ -0,0 +1,44 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Keys do
+  # Native generation of RSA keys is only available since OTP 20+ and in default build conditions
+  # We try at compile time to generate natively an RSA key otherwise we fallback on the old way.
+  try do
+    _ = :public_key.generate_key({:rsa, 2048, 65_537})
+
+    def generate_rsa_pem do
+      key = :public_key.generate_key({:rsa, 2048, 65_537})
+      entry = :public_key.pem_entry_encode(:RSAPrivateKey, key)
+      pem = :public_key.pem_encode([entry]) |> String.trim_trailing()
+      {:ok, pem}
+    end
+  rescue
+    _ ->
+      def generate_rsa_pem do
+        port = Port.open({:spawn, "openssl genrsa"}, [:binary])
+
+        {:ok, pem} =
+          receive do
+            {^port, {:data, pem}} -> {:ok, pem}
+          end
+
+        Port.close(port)
+
+        if Regex.match?(~r/RSA PRIVATE KEY/, pem) do
+          {:ok, pem}
+        else
+          :error
+        end
+      end
+  end
+
+  def keys_from_pem(pem) do
+    [private_key_code] = :public_key.pem_decode(pem)
+    private_key = :public_key.pem_entry_decode(private_key_code)
+    {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key
+    public_key = {:RSAPublicKey, modulus, exponent}
+    {:ok, private_key, public_key}
+  end
+end
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@ -130,6 +130,13 @@ def delete(%Object{data: %{"id" => id}} = object) do
    end
  end

+  def prune(%Object{data: %{"id" => id}} = object) do
+    with {:ok, object} <- Repo.delete(object),
+         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+      {:ok, object}
+    end
+  end
+
  def set_cache(%Object{data: %{"id" => ap_id}} = object) do
    Cachex.put(:object_cache, "object:#{ap_id}", object)
    {:ok, object}
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@ -8,6 +8,19 @@ defmodule Pleroma.Object.Fetcher do

  @httpoison Application.get_env(:pleroma, :httpoison)

+  defp reinject_object(data) do
+    Logger.debug("Reinjecting object #{data["id"]}")
+
+    with data <- Transmogrifier.fix_object(data),
+         {:ok, object} <- Object.create(data) do
+      {:ok, object}
+    else
+      e ->
+        Logger.error("Error while processing object: #{inspect(e)}")
+        {:error, e}
+    end
+  end
+
  # TODO:
  # This will create a Create activity, which we need internally at the moment.
  def fetch_object_from_id(id) do
@ -26,12 +39,17 @@ def fetch_object_from_id(id) do
             "object" => data
           },
           :ok <- Containment.contain_origin(id, params),
-           {:ok, activity} <- Transmogrifier.handle_incoming(params) do
-        {:ok, Object.normalize(activity, false)}
+           {:ok, activity} <- Transmogrifier.handle_incoming(params),
+           {:object, _data, %Object{} = object} <-
+             {:object, data, Object.normalize(activity, false)} do
+        {:ok, object}
      else
        {:error, {:reject, nil}} ->
          {:reject, nil}

+        {:object, data, nil} ->
+          reinject_object(data)
+
        object = %Object{} ->
          {:ok, object}

--- a/lib/pleroma/signature.ex
+++ b/lib/pleroma/signature.ex
@ -5,11 +5,10 @@
 defmodule Pleroma.Signature do
  @behaviour HTTPSignatures.Adapter

+  alias Pleroma.Keys
  alias Pleroma.User
  alias Pleroma.Web.ActivityPub.ActivityPub
  alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.Salmon
-  alias Pleroma.Web.WebFinger

  def fetch_public_key(conn) do
    with actor_id <- Utils.get_ap_id(conn.params["actor"]),
@ -33,8 +32,8 @@ def refetch_public_key(conn) do
  end

  def sign(%User{} = user, headers) do
-    with {:ok, %{info: %{keys: keys}}} <- WebFinger.ensure_keys_present(user),
-         {:ok, private_key, _} <- Salmon.keys_from_pem(keys) do
+    with {:ok, %{info: %{keys: keys}}} <- User.ensure_keys_present(user),
+         {:ok, private_key, _} <- Keys.keys_from_pem(keys) do
      HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers)
    end
  end
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@ -10,6 +10,7 @@ defmodule Pleroma.User do

  alias Comeonin.Pbkdf2
  alias Pleroma.Activity
+  alias Pleroma.Keys
  alias Pleroma.Notification
  alias Pleroma.Object
  alias Pleroma.Registration
@ -1402,4 +1403,44 @@ def toggle_confirmation(%User{} = user) do
    |> put_embed(:info, info_changeset)
    |> update_and_set_cache()
  end
+
+  def get_mascot(%{info: %{mascot: %{} = mascot}}) when not is_nil(mascot) do
+    mascot
+  end
+
+  def get_mascot(%{info: %{mascot: mascot}}) when is_nil(mascot) do
+    # use instance-default
+    config = Pleroma.Config.get([:assets, :mascots])
+    default_mascot = Pleroma.Config.get([:assets, :default_mascot])
+    mascot = Keyword.get(config, default_mascot)
+
+    %{
+      "id" => "default-mascot",
+      "url" => mascot[:url],
+      "preview_url" => mascot[:url],
+      "pleroma" => %{
+        "mime_type" => mascot[:mime_type]
+      }
+    }
+  end
+
+  def ensure_keys_present(user) do
+    info = user.info
+
+    if info.keys do
+      {:ok, user}
+    else
+      {:ok, pem} = Keys.generate_rsa_pem()
+
+      info_cng =
+        info
+        |> User.Info.set_keys(pem)
+
+      cng =
+        Ecto.Changeset.change(user)
+        |> Ecto.Changeset.put_embed(:info, info_cng)
+
+      update_and_set_cache(cng)
+    end
+  end
 end
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@ -43,6 +43,7 @@ defmodule Pleroma.User.Info do
    field(:hide_favorites, :boolean, default: true)
    field(:pinned_activities, {:array, :string}, default: [])
    field(:flavour, :string, default: nil)
+    field(:mascot, :map, default: nil)
    field(:emoji, {:array, :map}, default: [])

    field(:notification_settings, :map,
@ -248,6 +249,14 @@ def mastodon_flavour_update(info, flavour) do
    |> validate_required([:flavour])
  end

+  def mascot_update(info, url) do
+    params = %{mascot: url}
+
+    info
+    |> cast(params, [:mascot])
+    |> validate_required([:mascot])
+  end
+
  def set_source_data(info, source_data) do
    params = %{source_data: source_data}

--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@ -833,6 +833,13 @@ defp maybe_preload_bookmarks(query, opts) do
    |> Activity.with_preloaded_bookmark(opts["user"])
  end

+  defp maybe_set_thread_muted_field(query, %{"skip_preload" => true}), do: query
+
+  defp maybe_set_thread_muted_field(query, opts) do
+    query
+    |> Activity.with_set_thread_muted_field(opts["user"])
+  end
+
  defp maybe_order(query, %{order: :desc}) do
    query
    |> order_by(desc: :id)
@ -849,6 +856,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do
    Activity
    |> maybe_preload_objects(opts)
    |> maybe_preload_bookmarks(opts)
+    |> maybe_set_thread_muted_field(opts)
    |> maybe_order(opts)
    |> restrict_recipients(recipients, opts["user"])
    |> restrict_tag(opts)
@ -918,7 +926,7 @@ def upload(file, opts \\ []) do
    end
  end

-  def user_data_from_user_object(data) do
+  defp object_to_user_data(data) do
    avatar =
      data["icon"]["url"] &&
        %{
@ -965,9 +973,19 @@ def user_data_from_user_object(data) do
    {:ok, user_data}
  end

+  def user_data_from_user_object(data) do
+    with {:ok, data} <- MRF.filter(data),
+         {:ok, data} <- object_to_user_data(data) do
+      {:ok, data}
+    else
+      e -> {:error, e}
+    end
+  end
+
  def fetch_and_prepare_user_from_ap_id(ap_id) do
-    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
-      user_data_from_user_object(data)
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
+         {:ok, data} <- user_data_from_user_object(data) do
+      {:ok, data}
    else
      e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
    end
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@ -39,7 +39,7 @@ def relay_active?(conn, _) do

  def user(conn, %{"nickname" => nickname}) do
    with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
      conn
      |> put_resp_header("content-type", "application/activity+json")
      |> json(UserView.render("user.json", %{user: user}))
@ -106,7 +106,7 @@ def activity(conn, %{"uuid" => uuid}) do

  def following(conn, %{"nickname" => nickname, "page" => page}) do
    with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
      {page, _} = Integer.parse(page)

      conn
@ -117,7 +117,7 @@ def following(conn, %{"nickname" => nickname, "page" => page}) do

  def following(conn, %{"nickname" => nickname}) do
    with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
      conn
      |> put_resp_header("content-type", "application/activity+json")
      |> json(UserView.render("following.json", %{user: user}))
@ -126,7 +126,7 @@ def following(conn, %{"nickname" => nickname}) do

  def followers(conn, %{"nickname" => nickname, "page" => page}) do
    with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
      {page, _} = Integer.parse(page)

      conn
@ -137,7 +137,7 @@ def followers(conn, %{"nickname" => nickname, "page" => page}) do

  def followers(conn, %{"nickname" => nickname}) do
    with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
      conn
      |> put_resp_header("content-type", "application/activity+json")
      |> json(UserView.render("followers.json", %{user: user}))
@ -146,7 +146,7 @@ def followers(conn, %{"nickname" => nickname}) do

  def outbox(conn, %{"nickname" => nickname} = params) do
    with %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
      conn
      |> put_resp_header("content-type", "application/activity+json")
      |> json(UserView.render("outbox.json", %{user: user, max_id: params["max_id"]}))
@ -195,7 +195,7 @@ def inbox(conn, params) do

  def relay(conn, _params) do
    with %User{} = user <- Relay.get_actor(),
-         {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
+         {:ok, user} <- User.ensure_keys_present(user) do
      conn
      |> put_resp_header("content-type", "application/activity+json")
      |> json(UserView.render("user.json", %{user: user}))
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@ -48,10 +48,9 @@ defp check_media_nsfw(
         %{host: actor_host} = _actor_info,
         %{
           "type" => "Create",
-           "object" => %{"attachment" => child_attachment} = child_object
+           "object" => child_object
         } = object
-       )
-       when length(child_attachment) > 0 do
+       ) do
    object =
      if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do
        tags = (child_object["tag"] || []) ++ ["nsfw"]
@ -95,18 +94,63 @@ defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do
    {:ok, object}
  end

+  defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do
+    if actor_host in Pleroma.Config.get([:mrf_simple, :report_removal]) do
+      {:reject, nil}
+    else
+      {:ok, object}
+    end
+  end
+
+  defp check_report_removal(_actor_info, object), do: {:ok, object}
+
+  defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do
+    if actor_host in Pleroma.Config.get([:mrf_simple, :avatar_removal]) do
+      {:ok, Map.delete(object, "icon")}
+    else
+      {:ok, object}
+    end
+  end
+
+  defp check_avatar_removal(_actor_info, object), do: {:ok, object}
+
+  defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do
+    if actor_host in Pleroma.Config.get([:mrf_simple, :banner_removal]) do
+      {:ok, Map.delete(object, "image")}
+    else
+      {:ok, object}
+    end
+  end
+
+  defp check_banner_removal(_actor_info, object), do: {:ok, object}
+
  @impl true
-  def filter(object) do
-    actor_info = URI.parse(object["actor"])
+  def filter(%{"actor" => actor} = object) do
+    actor_info = URI.parse(actor)

    with {:ok, object} <- check_accept(actor_info, object),
         {:ok, object} <- check_reject(actor_info, object),
         {:ok, object} <- check_media_removal(actor_info, object),
         {:ok, object} <- check_media_nsfw(actor_info, object),
-         {:ok, object} <- check_ftl_removal(actor_info, object) do
+         {:ok, object} <- check_ftl_removal(actor_info, object),
+         {:ok, object} <- check_report_removal(actor_info, object) do
      {:ok, object}
    else
      _e -> {:reject, nil}
    end
  end
+
+  def filter(%{"id" => actor, "type" => obj_type} = object)
+      when obj_type in ["Application", "Group", "Organization", "Person", "Service"] do
+    actor_info = URI.parse(actor)
+
+    with {:ok, object} <- check_avatar_removal(actor_info, object),
+         {:ok, object} <- check_banner_removal(actor_info, object) do
+      {:ok, object}
+    else
+      _e -> {:reject, nil}
+    end
+  end
+
+  def filter(object), do: {:ok, object}
 end
--- a/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex
+++ b/lib/pleroma/web/activity_pub/mrf/user_allowlist.ex
@ -19,10 +19,12 @@ defp filter_by_list(%{"actor" => actor} = object, allow_list) do
  end

  @impl true
-  def filter(object) do
-    actor_info = URI.parse(object["actor"])
+  def filter(%{"actor" => actor} = object) do
+    actor_info = URI.parse(actor)
    allow_list = Config.get([:mrf_user_allowlist, String.to_atom(actor_info.host)], [])

    filter_by_list(object, allow_list)
  end
+
+  def filter(object), do: {:ok, object}
 end
--- a/lib/pleroma/web/activity_pub/views/user_view.ex
+++ b/lib/pleroma/web/activity_pub/views/user_view.ex
@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.UserView do
  use Pleroma.Web, :view

+  alias Pleroma.Keys
  alias Pleroma.Repo
  alias Pleroma.User
  alias Pleroma.Web.ActivityPub.ActivityPub
@ -12,8 +13,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do
  alias Pleroma.Web.ActivityPub.Utils
  alias Pleroma.Web.Endpoint
  alias Pleroma.Web.Router.Helpers
-  alias Pleroma.Web.Salmon
-  alias Pleroma.Web.WebFinger

  import Ecto.Query

@ -34,8 +33,8 @@ def render("endpoints.json", _), do: %{}

  # the instance itself is not a Person, but instead an Application
  def render("user.json", %{user: %{nickname: nil} = user}) do
-    {:ok, user} = WebFinger.ensure_keys_present(user)
-    {:ok, _, public_key} = Salmon.keys_from_pem(user.info.keys)
+    {:ok, user} = User.ensure_keys_present(user)
+    {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys)
    public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
    public_key = :public_key.pem_encode([public_key])

@ -62,8 +61,8 @@ def render("user.json", %{user: %{nickname: nil} = user}) do
  end

  def render("user.json", %{user: user}) do
-    {:ok, user} = WebFinger.ensure_keys_present(user)
-    {:ok, _, public_key} = Salmon.keys_from_pem(user.info.keys)
+    {:ok, user} = User.ensure_keys_present(user)
+    {:ok, _, public_key} = Keys.keys_from_pem(user.info.keys)
    public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
    public_key = :public_key.pem_encode([public_key])

--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@ -163,6 +163,7 @@ def post(user, %{"status" => status} = data) do
         bcc <- bcc_for_list(user, visibility),
         context <- make_context(in_reply_to),
         cw <- data["spoiler_text"] || "",
+         sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}),
         full_payload <- String.trim(status <> cw),
         length when length in 1..limit <- String.length(full_payload),
         object <-
@ -175,7 +176,8 @@ def post(user, %{"status" => status} = data) do
             in_reply_to,
             tags,
             cw,
-             cc
+             cc,
+             sensitive
           ),
         object <-
           Map.put(
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@ -232,7 +232,8 @@ def make_note_data(
        in_reply_to,
        tags,
        cw \\ nil,
-        cc \\ []
+        cc \\ [],
+        sensitive \\ false
      ) do
    object = %{
      "type" => "Note",
@ -240,6 +241,7 @@ def make_note_data(
      "cc" => cc,
      "content" => content_html,
      "summary" => cw,
+      "sensitive" => !Enum.member?(["false", "False", "0", false], sensitive),
      "context" => context,
      "attachment" => attachments,
      "actor" => actor,
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@ -11,7 +11,6 @@ defmodule Pleroma.Web.Federator do
  alias Pleroma.Web.ActivityPub.Utils
  alias Pleroma.Web.Federator.Publisher
  alias Pleroma.Web.Federator.RetryQueue
-  alias Pleroma.Web.WebFinger
  alias Pleroma.Web.Websub

  require Logger
@ -77,9 +76,8 @@ def perform(:request_subscription, websub) do
  def perform(:publish, activity) do
    Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end)

-    with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do
-      {:ok, actor} = WebFinger.ensure_keys_present(actor)
-
+    with %User{} = actor <- User.get_cached_by_ap_id(activity.data["actor"]),
+         {:ok, actor} <- User.ensure_keys_present(actor) do
      Publisher.publish(actor, activity)
    end
  end
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@ -707,6 +707,41 @@ def upload(%{assigns: %{user: user}} = conn, %{"file" => file} = data) do
    end
  end

+  def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do
+    with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)),
+         %{} = attachment_data <- Map.put(object.data, "id", object.id),
+         %{type: type} = rendered <-
+           StatusView.render("attachment.json", %{attachment: attachment_data}) do
+      # Reject if not an image
+      if type == "image" do
+        # Sure!
+        # Save to the user's info
+        info_changeset = User.Info.mascot_update(user.info, rendered)
+
+        user_changeset =
+          user
+          |> Ecto.Changeset.change()
+          |> Ecto.Changeset.put_embed(:info, info_changeset)
+
+        {:ok, _user} = User.update_and_set_cache(user_changeset)
+
+        conn
+        |> json(rendered)
+      else
+        conn
+        |> put_resp_content_type("application/json")
+        |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"}))
+      end
+    end
+  end
+
+  def get_mascot(%{assigns: %{user: user}} = conn, _params) do
+    mascot = User.get_mascot(user)
+
+    conn
+    |> json(mascot)
+  end
+
  def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
    with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
         %Object{data: %{"likes" => likes}} <- Object.normalize(object) do
@ -1009,6 +1044,30 @@ def unsubscribe(%{assigns: %{user: user}} = conn, %{"id" => id}) do
    end
  end

+  def status_search_query_with_gin(q, query) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
+          o.data,
+          ^query
+        ),
+      order_by: [desc: :id]
+    )
+  end
+
+  def status_search_query_with_rum(q, query) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "? @@ plainto_tsquery('english', ?)",
+          o.fts_content,
+          ^query
+        ),
+      order_by: [fragment("? <=> now()::date", o.inserted_at)]
+    )
+  end
+
  def status_search(user, query) do
    fetched =
      if Regex.match?(~r/https?:/, query) do
@ -1022,20 +1081,19 @@ def status_search(user, query) do
      end || []

    q =
-      from(
-        [a, o] in Activity.with_preloaded_object(Activity),
+      from([a, o] in Activity.with_preloaded_object(Activity),
        where: fragment("?->>'type' = 'Create'", a.data),
        where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
-        where:
-          fragment(
-            "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
-            o.data,
-            ^query
-          ),
-        limit: 20,
-        order_by: [desc: :id]
+        limit: 20
      )

+    q =
+      if Pleroma.Config.get([:database, :rum_enabled]) do
+        status_search_query_with_rum(q, query)
+      else
+        status_search_query_with_gin(q, query)
+      end
+
    Repo.all(q) ++ fetched
  end

@ -1306,7 +1364,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do
            display_sensitive_media: false,
            reduce_motion: false,
            max_toot_chars: limit,
-            mascot: "/images/pleroma-fox-tan-smol.png"
+            mascot: User.get_mascot(user)["url"]
          },
          rights: %{
            delete_others_notice: present?(user.info.is_moderator),
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@ -112,7 +112,7 @@ defp do_render("account.json", %{user: user} = opts) do
      fields: fields,
      bot: bot,
      source: %{
-        note: "",
+        note: HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
        sensitive: false,
        pleroma: %{}
      },
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@ -157,6 +157,12 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity

    bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil

+    thread_muted? =
+      case activity.thread_muted? do
+        thread_muted? when is_boolean(thread_muted?) -> thread_muted?
+        nil -> CommonAPI.thread_muted?(user, activity)
+      end
+
    attachment_data = object.data["attachment"] || []
    attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)

@ -228,7 +234,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity
      reblogged: reblogged?(activity, opts[:for]),
      favourited: present?(favorited),
      bookmarked: present?(bookmarked),
-      muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user),
+      muted: thread_muted? || User.mutes?(opts[:for], user),
      pinned: pinned?(activity, user),
      sensitive: sensitive,
      spoiler_text: summary_html,
--- a/lib/pleroma/web/mongooseim/mongoose_im_controller.ex
+++ b/lib/pleroma/web/mongooseim/mongoose_im_controller.ex
@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MongooseIM.MongooseIMController do
+  use Pleroma.Web, :controller
+  alias Comeonin.Pbkdf2
+  alias Pleroma.Repo
+  alias Pleroma.User
+
+  def user_exists(conn, %{"user" => username}) do
+    with %User{} <- Repo.get_by(User, nickname: username, local: true) do
+      conn
+      |> json(true)
+    else
+      _ ->
+        conn
+        |> put_status(:not_found)
+        |> json(false)
+    end
+  end
+
+  def check_password(conn, %{"user" => username, "pass" => password}) do
+    with %User{password_hash: password_hash} <-
+           Repo.get_by(User, nickname: username, local: true),
+         true <- Pbkdf2.checkpw(password, password_hash) do
+      conn
+      |> json(true)
+    else
+      false ->
+        conn
+        |> put_status(403)
+        |> json(false)
+
+      _ ->
+        conn
+        |> put_status(:not_found)
+        |> json(false)
+    end
+  end
+end
--- a/lib/pleroma/web/oauth/token.ex
+++ b/lib/pleroma/web/oauth/token.ex
@ -5,7 +5,6 @@
 defmodule Pleroma.Web.OAuth.Token do
  use Ecto.Schema

-  import Ecto.Query
  import Ecto.Changeset

  alias Pleroma.Repo
@ -13,6 +12,7 @@ defmodule Pleroma.Web.OAuth.Token do
  alias Pleroma.Web.OAuth.App
  alias Pleroma.Web.OAuth.Authorization
  alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.Token.Query

  @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
  @type t :: %__MODULE__{}
@ -31,17 +31,17 @@ defmodule Pleroma.Web.OAuth.Token do
  @doc "Gets token for app by access token"
  @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
  def get_by_token(%App{id: app_id} = _app, token) do
-    from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token)
+    Query.get_by_app(app_id)
+    |> Query.get_by_token(token)
    |> Repo.find_resource()
  end

  @doc "Gets token for app by refresh token"
  @spec get_by_refresh_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
  def get_by_refresh_token(%App{id: app_id} = _app, token) do
-    from(t in __MODULE__,
-      where: t.app_id == ^app_id and t.refresh_token == ^token,
-      preload: [:user]
-    )
+    Query.get_by_app(app_id)
+    |> Query.get_by_refresh_token(token)
+    |> Query.preload([:user])
    |> Repo.find_resource()
  end

@ -97,29 +97,25 @@ def create_token(%App{} = app, %User{} = user, attrs \\ %{}) do
  end

  def delete_user_tokens(%User{id: user_id}) do
-    from(
-      t in Token,
-      where: t.user_id == ^user_id
-    )
+    Query.get_by_user(user_id)
    |> Repo.delete_all()
  end

  def delete_user_token(%User{id: user_id}, token_id) do
-    from(
-      t in Token,
-      where: t.user_id == ^user_id,
-      where: t.id == ^token_id
-    )
+    Query.get_by_user(user_id)
+    |> Query.get_by_id(token_id)
+    |> Repo.delete_all()
+  end
+
+  def delete_expired_tokens do
+    Query.get_expired_tokens()
    |> Repo.delete_all()
  end

  def get_user_tokens(%User{id: user_id}) do
-    from(
-      t in Token,
-      where: t.user_id == ^user_id
-    )
+    Query.get_by_user(user_id)
+    |> Query.preload([:app])
    |> Repo.all()
-    |> Repo.preload(:app)
  end

  def is_expired?(%__MODULE__{valid_until: valid_until}) do
--- a/lib/pleroma/web/oauth/token/clean_worker.ex
+++ b/lib/pleroma/web/oauth/token/clean_worker.ex
@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.OAuth.Token.CleanWorker do
+  @moduledoc """
+  The module represents functions to clean an expired oauth tokens.
+  """
+
+  # 10 seconds
+  @start_interval 10_000
+  @interval Pleroma.Config.get(
+              # 24 hours
+              [:oauth2, :clean_expired_tokens_interval],
+              86_400_000
+            )
+  @queue :background
+
+  alias Pleroma.Web.OAuth.Token
+
+  def start_link, do: GenServer.start_link(__MODULE__, nil)
+
+  def init(_) do
+    if Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) do
+      Process.send_after(self(), :perform, @start_interval)
+      {:ok, nil}
+    else
+      :ignore
+    end
+  end
+
+  @doc false
+  def handle_info(:perform, state) do
+    Process.send_after(self(), :perform, @interval)
+    PleromaJobQueue.enqueue(@queue, __MODULE__, [:clean])
+    {:noreply, state}
+  end
+
+  # Job Worker Callbacks
+  def perform(:clean), do: Token.delete_expired_tokens()
+end
--- a/lib/pleroma/web/oauth/token/query.ex
+++ b/lib/pleroma/web/oauth/token/query.ex
@ -0,0 +1,55 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.OAuth.Token.Query do
+  @moduledoc """
+  Contains queries for OAuth Token.
+  """
+
+  import Ecto.Query, only: [from: 2]
+
+  @type query :: Ecto.Queryable.t() | Token.t()
+
+  alias Pleroma.Web.OAuth.Token
+
+  @spec get_by_refresh_token(query, String.t()) :: query
+  def get_by_refresh_token(query \\ Token, refresh_token) do
+    from(q in query, where: q.refresh_token == ^refresh_token)
+  end
+
+  @spec get_by_token(query, String.t()) :: query
+  def get_by_token(query \\ Token, token) do
+    from(q in query, where: q.token == ^token)
+  end
+
+  @spec get_by_app(query, String.t()) :: query
+  def get_by_app(query \\ Token, app_id) do
+    from(q in query, where: q.app_id == ^app_id)
+  end
+
+  @spec get_by_id(query, String.t()) :: query
+  def get_by_id(query \\ Token, id) do
+    from(q in query, where: q.id == ^id)
+  end
+
+  @spec get_expired_tokens(query, DateTime.t() | nil) :: query
+  def get_expired_tokens(query \\ Token, date \\ nil) do
+    expired_date = date || Timex.now()
+    from(q in query, where: fragment("?", q.valid_until) < ^expired_date)
+  end
+
+  @spec get_by_user(query, String.t()) :: query
+  def get_by_user(query \\ Token, user_id) do
+    from(q in query, where: q.user_id == ^user_id)
+  end
+
+  @spec preload(query, any) :: query
+  def preload(query \\ Token, assoc_preload \\ [])
+
+  def preload(query, assoc_preload) when is_list(assoc_preload) do
+    from(q in query, preload: ^assoc_preload)
+  end
+
+  def preload(query, _assoc_preload), do: query
+end
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@ -24,6 +24,7 @@ defp validate_page_url(_), do: :error
  def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
    with true <- Pleroma.Config.get([:rich_media, :enabled]),
         %Object{} = object <- Object.normalize(activity),
+         false <- object.data["sensitive"] || false,
         {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),
         :ok <- validate_page_url(page_url),
         {:ok, rich_media} <- Parser.parse(page_url) do
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@ -352,6 +352,9 @@ defmodule Pleroma.Web.Router do

      post("/pleroma/flavour/:flavour", MastodonAPIController, :set_flavour)

+      get("/pleroma/mascot", MastodonAPIController, :get_mascot)
+      put("/pleroma/mascot", MastodonAPIController, :set_mascot)
+
      post("/reports", MastodonAPIController, :reports)
    end

@ -704,9 +707,15 @@ defmodule Pleroma.Web.Router do
    end
  end

+  scope "/", Pleroma.Web.MongooseIM do
+    get("/user_exists", MongooseIMController, :user_exists)
+    get("/check_password", MongooseIMController, :check_password)
+  end
+
  scope "/", Fallback do
    get("/registration/:token", RedirectController, :registration_page)
    get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta)
+    get("/api*path", RedirectController, :api_not_implemented)
    get("/*path", RedirectController, :redirector)

    options("/*path", RedirectController, :empty)
@ -718,6 +727,12 @@ defmodule Fallback.RedirectController do
  alias Pleroma.User
  alias Pleroma.Web.Metadata

+  def api_not_implemented(conn, _params) do
+    conn
+    |> put_status(404)
+    |> json(%{error: "Not implemented"})
+  end
+
  def redirector(conn, _params, code \\ 200) do
    conn
    |> put_resp_content_type("text/html")
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@ -11,6 +11,7 @@ defmodule Pleroma.Web.Salmon do

  alias Pleroma.Activity
  alias Pleroma.Instances
+  alias Pleroma.Keys
  alias Pleroma.User
  alias Pleroma.Web.ActivityPub.Visibility
  alias Pleroma.Web.Federator.Publisher
@ -89,45 +90,6 @@ def encode_key({:RSAPublicKey, modulus, exponent}) do
    "RSA.#{modulus_enc}.#{exponent_enc}"
  end

-  # Native generation of RSA keys is only available since OTP 20+ and in default build conditions
-  # We try at compile time to generate natively an RSA key otherwise we fallback on the old way.
-  try do
-    _ = :public_key.generate_key({:rsa, 2048, 65_537})
-
-    def generate_rsa_pem do
-      key = :public_key.generate_key({:rsa, 2048, 65_537})
-      entry = :public_key.pem_entry_encode(:RSAPrivateKey, key)
-      pem = :public_key.pem_encode([entry]) |> String.trim_trailing()
-      {:ok, pem}
-    end
-  rescue
-    _ ->
-      def generate_rsa_pem do
-        port = Port.open({:spawn, "openssl genrsa"}, [:binary])
-
-        {:ok, pem} =
-          receive do
-            {^port, {:data, pem}} -> {:ok, pem}
-          end
-
-        Port.close(port)
-
-        if Regex.match?(~r/RSA PRIVATE KEY/, pem) do
-          {:ok, pem}
-        else
-          :error
-        end
-      end
-  end
-
-  def keys_from_pem(pem) do
-    [private_key_code] = :public_key.pem_decode(pem)
-    private_key = :public_key.pem_entry_decode(private_key_code)
-    {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key
-    public_key = {:RSAPublicKey, modulus, exponent}
-    {:ok, private_key, public_key}
-  end
-
  def encode(private_key, doc) do
    type = "application/atom+xml"
    encoding = "base64url"
@ -242,7 +204,7 @@ def publish(%{info: %{keys: keys}} = user, %{data: %{"type" => type}} = activity
        |> :xmerl.export_simple(:xmerl_xml)
        |> to_string

-      {:ok, private, _} = keys_from_pem(keys)
+      {:ok, private, _} = Keys.keys_from_pem(keys)
      {:ok, feed} = encode(private, feed)

      remote_users = remote_users(user, activity)
@ -268,7 +230,7 @@ def publish(%{info: %{keys: keys}} = user, %{data: %{"type" => type}} = activity
  def publish(%{id: id}, _), do: Logger.debug(fn -> "Keys missing for user #{id}" end)

  def gather_webfinger_links(%User{} = user) do
-    {:ok, _private, public} = keys_from_pem(user.info.keys)
+    {:ok, _private, public} = Keys.keys_from_pem(user.info.keys)
    magic_key = encode_key(public)

    [
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@ -284,6 +284,12 @@ def render(
        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
      )

+    thread_muted? =
+      case activity.thread_muted? do
+        thread_muted? when is_boolean(thread_muted?) -> thread_muted?
+        nil -> CommonAPI.thread_muted?(user, activity)
+      end
+
    %{
      "id" => activity.id,
      "uri" => object.data["id"],
@ -314,7 +320,7 @@ def render(
      "summary" => summary,
      "summary_html" => summary |> Formatter.emojify(object.data["emoji"]),
      "card" => card,
-      "muted" => CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user)
+      "muted" => thread_muted? || User.mutes?(opts[:for], user)
    }
  end

--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@ -8,7 +8,6 @@ defmodule Pleroma.Web.WebFinger do
  alias Pleroma.User
  alias Pleroma.Web
  alias Pleroma.Web.Federator.Publisher
-  alias Pleroma.Web.Salmon
  alias Pleroma.Web.XML
  alias Pleroma.XmlBuilder
  require Jason
@ -61,7 +60,7 @@ defp gather_links(%User{} = user) do
  end

  def represent_user(user, "JSON") do
-    {:ok, user} = ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)

    %{
      "subject" => "acct:#{user.nickname}@#{Pleroma.Web.Endpoint.host()}",
@ -71,7 +70,7 @@ def represent_user(user, "JSON") do
  end

  def represent_user(user, "XML") do
-    {:ok, user} = ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)

    links =
      gather_links(user)
@ -88,27 +87,6 @@ def represent_user(user, "XML") do
    |> XmlBuilder.to_doc()
  end

-  # This seems a better fit in Salmon
-  def ensure_keys_present(user) do
-    info = user.info
-
-    if info.keys do
-      {:ok, user}
-    else
-      {:ok, pem} = Salmon.generate_rsa_pem()
-
-      info_cng =
-        info
-        |> User.Info.set_keys(pem)
-
-      cng =
-        Ecto.Changeset.change(user)
-        |> Ecto.Changeset.put_embed(:info, info_cng)
-
-      User.update_and_set_cache(cng)
-    end
-  end
-
  defp get_magic_key(magic_key) do
    "data:application/magic-public-key," <> magic_key = magic_key
    {:ok, magic_key}
--- a/mix.exs
+++ b/mix.exs
@ -42,7 +42,7 @@ def project do
  def application do
    [
      mod: {Pleroma.Application, []},
-      extra_applications: [:logger, :runtime_tools, :comeonin, :esshd, :quack],
+      extra_applications: [:logger, :runtime_tools, :comeonin, :quack],
      included_applications: [:ex_syslogger]
    ]
  end
@ -66,10 +66,7 @@ defp deps do
      {:plug_cowboy, "~> 2.0"},
      {:phoenix_pubsub, "~> 1.1"},
      {:phoenix_ecto, "~> 4.0"},
-      {:ecto_sql,
-       git: "https://github.com/elixir-ecto/ecto_sql",
-       ref: "14cb065a74c488d737d973f7a91bc036c6245f78",
-       override: true},
+      {:ecto_sql, "~> 3.1"},
      {:postgrex, ">= 0.13.5"},
      {:gettext, "~> 0.15"},
      {:comeonin, "~> 4.1.1"},
@ -120,7 +117,7 @@ defp deps do
      {:recon, github: "ferd/recon", tag: "2.4.0"},
      {:quack, "~> 0.1.1"},
      {:benchee, "~> 1.0"},
-      {:esshd, "~> 0.1.0"},
+      {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)},
      {:ex_rated, "~> 1.2"},
      {:plug_static_index_html, "~> 1.0.0"},
      {:excoveralls, "~> 0.11.1", only: :test}
--- a/mix.lock
+++ b/mix.lock
@ -21,7 +21,7 @@
  "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
  "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
  "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
-  "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "14cb065a74c488d737d973f7a91bc036c6245f78", [ref: "14cb065a74c488d737d973f7a91bc036c6245f78"]},
+  "ecto_sql": {:hex, :ecto_sql, "3.1.3", "2c536139190492d9de33c5fefac7323c5eaaa82e1b9bf93482a14649042f7cd9", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
  "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
  "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
  "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"},
--- a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
+++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs
@ -0,0 +1,34 @@
+defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
+  use Ecto.Migration
+
+  def up do
+    execute("create extension if not exists rum")
+    drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+    alter table(:objects) do
+      add(:fts_content, :tsvector)
+    end
+
+    execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$
+    begin
+      new.fts_content := to_tsvector('english', new.data->>'content');
+      return new;
+    end
+    $$ LANGUAGE plpgsql")
+    execute("create index objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');")
+
+    execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects
+    FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
+
+    execute("UPDATE objects SET updated_at = NOW()")
+  end
+
+  def down do
+    execute "drop index objects_fts"
+    execute "drop trigger tsvectorupdate on objects"
+    execute "drop function objects_fts_update()"
+    alter table(:objects) do
+      remove(:fts_content, :tsvector)
+    end
+    create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+  end
+end
--- a/test/activity_test.exs
+++ b/test/activity_test.exs
@ -6,6 +6,7 @@ defmodule Pleroma.ActivityTest do
  use Pleroma.DataCase
  alias Pleroma.Activity
  alias Pleroma.Bookmark
+  alias Pleroma.ThreadMute
  import Pleroma.Factory

  test "returns an activity by it's AP id" do
@ -47,6 +48,31 @@ test "preloading a bookmark" do
    assert queried_activity.bookmark == bookmark3
  end

+  test "setting thread_muted?" do
+    activity = insert(:note_activity)
+    user = insert(:user)
+    annoyed_user = insert(:user)
+    {:ok, _} = ThreadMute.add_mute(annoyed_user.id, activity.data["context"])
+
+    activity_with_unset_thread_muted_field =
+      Ecto.Query.from(Activity)
+      |> Repo.one()
+
+    activity_for_user =
+      Ecto.Query.from(Activity)
+      |> Activity.with_set_thread_muted_field(user)
+      |> Repo.one()
+
+    activity_for_annoyed_user =
+      Ecto.Query.from(Activity)
+      |> Activity.with_set_thread_muted_field(annoyed_user)
+      |> Repo.one()
+
+    assert activity_with_unset_thread_muted_field.thread_muted? == nil
+    assert activity_for_user.thread_muted? == false
+    assert activity_for_annoyed_user.thread_muted? == true
+  end
+
  describe "getting a bookmark" do
    test "when association is loaded" do
      user = insert(:user)
--- a/test/fixtures/sound.mp3
+++ b/test/fixtures/sound.mp3
--- a/test/formatter_test.exs
+++ b/test/formatter_test.exs
@ -206,6 +206,15 @@ test "given the 'safe_mention' option, it will still work without any mention" d
      assert mentions == []
      assert expected_text == text
    end
+
+    test "given the 'safe_mention' option, it will keep text after newlines" do
+      user = insert(:user)
+      text = " @#{user.nickname}\n hey dude\n\nhow are you doing?"
+
+      {expected_text, _, _} = Formatter.linkify(text, safe_mention: true)
+
+      assert expected_text =~ "how are you doing?"
+    end
  end

  describe ".parse_tags" do
--- a/test/keys_test.exs
+++ b/test/keys_test.exs
@ -0,0 +1,20 @@
+defmodule Pleroma.KeysTest do
+  use Pleroma.DataCase
+
+  alias Pleroma.Keys
+
+  test "generates an RSA private key pem" do
+    {:ok, key} = Keys.generate_rsa_pem()
+
+    assert is_binary(key)
+    assert Regex.match?(~r/RSA/, key)
+  end
+
+  test "returns a public and private key from a pem" do
+    pem = File.read!("test/fixtures/private_key.pem")
+    {:ok, private, public} = Keys.keys_from_pem(pem)
+
+    assert elem(private, 0) == :RSAPrivateKey
+    assert elem(public, 0) == :RSAPublicKey
+  end
+end
--- a/test/object/fetcher_test.exs
+++ b/test/object/fetcher_test.exs
@ -87,4 +87,23 @@ test "all objects with fake directions are rejected by the object fetcher" do
        )
    end
  end
+
+  describe "pruning" do
+    test "it can refetch pruned objects" do
+      object_id = "http://mastodon.example.org/@admin/99541947525187367"
+
+      {:ok, object} = Fetcher.fetch_object_from_id(object_id)
+
+      assert object
+
+      {:ok, _object} = Object.prune(object)
+
+      refute Object.get_by_ap_id(object_id)
+
+      {:ok, %Object{} = object_two} = Fetcher.fetch_object_from_id(object_id)
+
+      assert object.data["id"] == object_two.data["id"]
+      assert object.id != object_two.id
+    end
+  end
 end
--- a/test/user_test.exs
+++ b/test/user_test.exs
@ -902,7 +902,7 @@ test "hide a user's statuses from timelines and notifications" do

      assert [activity] == ActivityPub.fetch_public_activities(%{}) |> Repo.preload(:bookmark)

-      assert [activity] ==
+      assert [%{activity | thread_muted?: CommonAPI.thread_muted?(user2, activity)}] ==
               ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2})

      {:ok, _user} = User.deactivate(user)
@ -1251,4 +1251,19 @@ test "if user is unconfirmed" do
      refute user.info.confirmation_token
    end
  end
+
+  describe "ensure_keys_present" do
+    test "it creates keys for a user and stores them in info" do
+      user = insert(:user)
+      refute is_binary(user.info.keys)
+      {:ok, user} = User.ensure_keys_present(user)
+      assert is_binary(user.info.keys)
+    end
+
+    test "it doesn't create keys if there already are some" do
+      user = insert(:user, %{info: %{keys: "xxx"}})
+      {:ok, user} = User.ensure_keys_present(user)
+      assert user.info.keys == "xxx"
+    end
+  end
 end
--- a/test/web/activity_pub/activity_pub_test.exs
+++ b/test/web/activity_pub/activity_pub_test.exs
@ -1005,7 +1005,7 @@ test "it filters broken threads" do
  describe "update" do
    test "it creates an update activity with the new user data" do
      user = insert(:user)
-      {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+      {:ok, user} = User.ensure_keys_present(user)
      user_data = Pleroma.Web.ActivityPub.UserView.render("user.json", %{user: user})

      {:ok, update} =
--- a/test/web/activity_pub/mrf/simple_policy_test.exs
+++ b/test/web/activity_pub/mrf/simple_policy_test.exs
@ -15,8 +15,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do
      media_removal: [],
      media_nsfw: [],
      federated_timeline_removal: [],
+      report_removal: [],
      reject: [],
-      accept: []
+      accept: [],
+      avatar_removal: [],
+      banner_removal: []
    )

    on_exit(fn ->
@ -85,6 +88,33 @@ defp build_media_message do
    }
  end

+  describe "when :report_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :report_removal], [])
+      report_message = build_report_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(report_message) == {:ok, report_message}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :report_removal], ["remote.instance"])
+      report_message = build_report_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(report_message) == {:reject, nil}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+  end
+
+  defp build_report_message do
+    %{
+      "actor" => "https://remote.instance/users/bob",
+      "type" => "Flag"
+    }
+  end
+
  describe "when :federated_timeline_removal" do
    test "is empty" do
      Config.put([:mrf_simple, :federated_timeline_removal], [])
@ -178,6 +208,60 @@ test "has a matching host" do
    end
  end

+  describe "when :avatar_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :avatar_removal], [])
+
+      remote_user = build_remote_user()
+
+      assert SimplePolicy.filter(remote_user) == {:ok, remote_user}
+    end
+
+    test "is not empty but it doesn't have a matching host" do
+      Config.put([:mrf_simple, :avatar_removal], ["non.matching.remote"])
+
+      remote_user = build_remote_user()
+
+      assert SimplePolicy.filter(remote_user) == {:ok, remote_user}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :avatar_removal], ["remote.instance"])
+
+      remote_user = build_remote_user()
+      {:ok, filtered} = SimplePolicy.filter(remote_user)
+
+      refute filtered["icon"]
+    end
+  end
+
+  describe "when :banner_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :banner_removal], [])
+
+      remote_user = build_remote_user()
+
+      assert SimplePolicy.filter(remote_user) == {:ok, remote_user}
+    end
+
+    test "is not empty but it doesn't have a matching host" do
+      Config.put([:mrf_simple, :banner_removal], ["non.matching.remote"])
+
+      remote_user = build_remote_user()
+
+      assert SimplePolicy.filter(remote_user) == {:ok, remote_user}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :banner_removal], ["remote.instance"])
+
+      remote_user = build_remote_user()
+      {:ok, filtered} = SimplePolicy.filter(remote_user)
+
+      refute filtered["image"]
+    end
+  end
+
  defp build_local_message do
    %{
      "actor" => "#{Pleroma.Web.base_url()}/users/alice",
@ -189,4 +273,19 @@ defp build_local_message do
  defp build_remote_message do
    %{"actor" => "https://remote.instance/users/bob"}
  end
+
+  defp build_remote_user do
+    %{
+      "id" => "https://remote.instance/users/bob",
+      "icon" => %{
+        "url" => "http://example.com/image.jpg",
+        "type" => "Image"
+      },
+      "image" => %{
+        "url" => "http://example.com/image.jpg",
+        "type" => "Image"
+      },
+      "type" => "Person"
+    }
+  end
 end
--- a/test/web/activity_pub/views/user_view_test.exs
+++ b/test/web/activity_pub/views/user_view_test.exs
@ -2,11 +2,12 @@ defmodule Pleroma.Web.ActivityPub.UserViewTest do
  use Pleroma.DataCase
  import Pleroma.Factory

+  alias Pleroma.User
  alias Pleroma.Web.ActivityPub.UserView

  test "Renders a user, including the public key" do
    user = insert(:user)
-    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)

    result = UserView.render("user.json", %{user: user})

@ -18,7 +19,7 @@ test "Renders a user, including the public key" do

  test "Does not add an avatar image if the user hasn't set one" do
    user = insert(:user)
-    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)

    result = UserView.render("user.json", %{user: user})
    refute result["icon"]
@ -32,7 +33,7 @@ test "Does not add an avatar image if the user hasn't set one" do
        }
      )

-    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)

    result = UserView.render("user.json", %{user: user})
    assert result["icon"]["url"] == "https://someurl"
@ -42,7 +43,7 @@ test "Does not add an avatar image if the user hasn't set one" do
  describe "endpoints" do
    test "local users have a usable endpoints structure" do
      user = insert(:user)
-      {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+      {:ok, user} = User.ensure_keys_present(user)

      result = UserView.render("user.json", %{user: user})

@ -58,7 +59,7 @@ test "local users have a usable endpoints structure" do

    test "remote users have an empty endpoints structure" do
      user = insert(:user, local: false)
-      {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+      {:ok, user} = User.ensure_keys_present(user)

      result = UserView.render("user.json", %{user: user})

@ -68,7 +69,7 @@ test "remote users have an empty endpoints structure" do

    test "instance users do not expose oAuth endpoints" do
      user = insert(:user, nickname: nil, local: true)
-      {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+      {:ok, user} = User.ensure_keys_present(user)

      result = UserView.render("user.json", %{user: user})

--- a/test/web/admin_api/admin_api_controller_test.exs
+++ b/test/web/admin_api/admin_api_controller_test.exs
@ -397,14 +397,14 @@ test "it returns 500 if `registrations_open` is enabled", %{conn: conn, user: us
    end
  end

-  test "/api/pleroma/admin/invite_token" do
+  test "/api/pleroma/admin/users/invite_token" do
    admin = insert(:user, info: %{is_admin: true})

    conn =
      build_conn()
      |> assign(:user, admin)
      |> put_req_header("accept", "application/json")
-      |> get("/api/pleroma/admin/invite_token")
+      |> get("/api/pleroma/admin/users/invite_token")

    assert conn.status == 200
  end
--- a/test/web/fallback_test.exs
+++ b/test/web/fallback_test.exs
@ -0,0 +1,52 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FallbackTest do
+  use Pleroma.Web.ConnCase
+  import Pleroma.Factory
+
+  test "GET /registration/:token", %{conn: conn} do
+    assert conn
+           |> get("/registration/foo")
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+  end
+
+  test "GET /:maybe_nickname_or_id", %{conn: conn} do
+    user = insert(:user)
+
+    assert conn
+           |> get("/foo")
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+
+    refute conn
+           |> get("/" <> user.nickname)
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+  end
+
+  test "GET /api*path", %{conn: conn} do
+    assert conn
+           |> get("/api/foo")
+           |> json_response(404) == %{"error" => "Not implemented"}
+  end
+
+  test "GET /*path", %{conn: conn} do
+    assert conn
+           |> get("/foo")
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+
+    assert conn
+           |> get("/foo/bar")
+           |> html_response(200) =~ "<!--server-generated-meta-->"
+  end
+
+  test "OPTIONS /*path", %{conn: conn} do
+    assert conn
+           |> options("/foo")
+           |> response(204) == ""
+
+    assert conn
+           |> options("/foo/bar")
+           |> response(204) == ""
+  end
+end
--- a/test/web/mastodon_api/account_view_test.exs
+++ b/test/web/mastodon_api/account_view_test.exs
@ -55,7 +55,7 @@ test "Represent a user account" do
      fields: [],
      bot: false,
      source: %{
-        note: "",
+        note: "valid html",
        sensitive: false,
        pleroma: %{}
      },
@ -120,7 +120,7 @@ test "Represent a Service(bot) account" do
      fields: [],
      bot: true,
      source: %{
-        note: "",
+        note: user.bio,
        sensitive: false,
        pleroma: %{}
      },
@ -209,7 +209,7 @@ test "represent an embedded relationship" do
      fields: [],
      bot: true,
      source: %{
-        note: "",
+        note: user.bio,
        sensitive: false,
        pleroma: %{}
      },
--- a/test/web/mastodon_api/mastodon_api_controller_test.exs
+++ b/test/web/mastodon_api/mastodon_api_controller_test.exs
@ -1455,6 +1455,72 @@ test "media upload", %{conn: conn} do
    assert object.data["actor"] == User.ap_id(user)
  end

+  test "mascot upload", %{conn: conn} do
+    user = insert(:user)
+
+    non_image_file = %Plug.Upload{
+      content_type: "audio/mpeg",
+      path: Path.absname("test/fixtures/sound.mp3"),
+      filename: "sound.mp3"
+    }
+
+    conn =
+      conn
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => non_image_file})
+
+    assert json_response(conn, 415)
+
+    file = %Plug.Upload{
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image.jpg"),
+      filename: "an_image.jpg"
+    }
+
+    conn =
+      build_conn()
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => file})
+
+    assert %{"id" => _, "type" => image} = json_response(conn, 200)
+  end
+
+  test "mascot retrieving", %{conn: conn} do
+    user = insert(:user)
+    # When user hasn't set a mascot, we should just get pleroma tan back
+    conn =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/pleroma/mascot")
+
+    assert %{"url" => url} = json_response(conn, 200)
+    assert url =~ "pleroma-fox-tan-smol"
+
+    # When a user sets their mascot, we should get that back
+    file = %Plug.Upload{
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image.jpg"),
+      filename: "an_image.jpg"
+    }
+
+    conn =
+      build_conn()
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => file})
+
+    assert json_response(conn, 200)
+
+    user = User.get_cached_by_id(user.id)
+
+    conn =
+      build_conn()
+      |> assign(:user, user)
+      |> get("/api/v1/pleroma/mascot")
+
+    assert %{"url" => url, "type" => "image"} = json_response(conn, 200)
+    assert url =~ "an_image"
+  end
+
  test "hashtag timeline", %{conn: conn} do
    following = insert(:user)

--- a/test/web/mongooseim/mongoose_im_controller_test.exs
+++ b/test/web/mongooseim/mongoose_im_controller_test.exs
@ -0,0 +1,59 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MongooseIMController do
+  use Pleroma.Web.ConnCase
+  import Pleroma.Factory
+
+  test "/user_exists", %{conn: conn} do
+    _user = insert(:user, nickname: "lain")
+    _remote_user = insert(:user, nickname: "alice", local: false)
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :user_exists), user: "lain")
+      |> json_response(200)
+
+    assert res == true
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :user_exists), user: "alice")
+      |> json_response(404)
+
+    assert res == false
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :user_exists), user: "bob")
+      |> json_response(404)
+
+    assert res == false
+  end
+
+  test "/check_password", %{conn: conn} do
+    user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt("cool"))
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :check_password), user: user.nickname, pass: "cool")
+      |> json_response(200)
+
+    assert res == true
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :check_password), user: user.nickname, pass: "uncool")
+      |> json_response(403)
+
+    assert res == false
+
+    res =
+      conn
+      |> get(mongoose_im_path(conn, :check_password), user: "nobody", pass: "cool")
+      |> json_response(404)
+
+    assert res == false
+  end
+end
--- a/test/web/oauth/token_test.exs
+++ b/test/web/oauth/token_test.exs
@ -69,4 +69,17 @@ test "deletes all tokens of a user" do

    assert tokens == 2
  end
+
+  test "deletes expired tokens" do
+    insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3))
+    insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3))
+    t3 = insert(:oauth_token)
+    t4 = insert(:oauth_token, valid_until: Timex.shift(Timex.now(), minutes: 10))
+    {tokens, _} = Token.delete_expired_tokens()
+    assert tokens == 2
+    available_tokens = Pleroma.Repo.all(Token)
+
+    token_ids = available_tokens |> Enum.map(& &1.id)
+    assert token_ids == [t3.id, t4.id]
+  end
 end
--- a/test/web/rich_media/helpers_test.exs
+++ b/test/web/rich_media/helpers_test.exs
@ -1,6 +1,7 @@
 defmodule Pleroma.Web.RichMedia.HelpersTest do
  use Pleroma.DataCase

+  alias Pleroma.Object
  alias Pleroma.Web.CommonAPI

  import Pleroma.Factory
@ -59,4 +60,43 @@ test "crawls valid, complete URLs" do

    Pleroma.Config.put([:rich_media, :enabled], false)
  end
+
+  test "refuses to crawl URLs from posts marked sensitive" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        "status" => "http://example.com/ogp",
+        "sensitive" => true
+      })
+
+    %Object{} = object = Object.normalize(activity)
+
+    assert object.data["sensitive"]
+
+    Pleroma.Config.put([:rich_media, :enabled], true)
+
+    assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+
+    Pleroma.Config.put([:rich_media, :enabled], false)
+  end
+
+  test "refuses to crawl URLs from posts tagged NSFW" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        "status" => "http://example.com/ogp #nsfw"
+      })
+
+    %Object{} = object = Object.normalize(activity)
+
+    assert object.data["sensitive"]
+
+    Pleroma.Config.put([:rich_media, :enabled], true)
+
+    assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+
+    Pleroma.Config.put([:rich_media, :enabled], false)
+  end
 end
--- a/test/web/salmon/salmon_test.exs
+++ b/test/web/salmon/salmon_test.exs
@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Salmon.SalmonTest do
  use Pleroma.DataCase
  alias Pleroma.Activity
+  alias Pleroma.Keys
  alias Pleroma.Repo
  alias Pleroma.User
  alias Pleroma.Web.Federator.Publisher
@ -34,12 +35,6 @@ test "errors on wrong magic key" do
    assert Salmon.decode_and_validate(@wrong_magickey, salmon) == :error
  end

-  test "generates an RSA private key pem" do
-    {:ok, key} = Salmon.generate_rsa_pem()
-    assert is_binary(key)
-    assert Regex.match?(~r/RSA/, key)
-  end
-
  test "it encodes a magic key from a public key" do
    key = Salmon.decode_key(@magickey)
    magic_key = Salmon.encode_key(key)
@ -51,18 +46,10 @@ test "it decodes a friendica public key" do
    _key = Salmon.decode_key(@magickey_friendica)
  end

-  test "returns a public and private key from a pem" do
-    pem = File.read!("test/fixtures/private_key.pem")
-    {:ok, private, public} = Salmon.keys_from_pem(pem)
-
-    assert elem(private, 0) == :RSAPrivateKey
-    assert elem(public, 0) == :RSAPublicKey
-  end
-
  test "encodes an xml payload with a private key" do
    doc = File.read!("test/fixtures/incoming_note_activity.xml")
    pem = File.read!("test/fixtures/private_key.pem")
-    {:ok, private, public} = Salmon.keys_from_pem(pem)
+    {:ok, private, public} = Keys.keys_from_pem(pem)

    # Let's try a roundtrip.
    {:ok, salmon} = Salmon.encode(private, doc)
@ -105,7 +92,7 @@ test "it gets a magic key" do

    {:ok, activity} = Repo.insert(%Activity{data: activity_data, recipients: activity_data["to"]})
    user = User.get_cached_by_ap_id(activity.data["actor"])
-    {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user)
+    {:ok, user} = User.ensure_keys_present(user)

    Salmon.publish(user, activity)

--- a/test/web/web_finger/web_finger_test.exs
+++ b/test/web/web_finger/web_finger_test.exs
@ -105,19 +105,4 @@ test "it gets the xrd endpoint for statusnet" do
      assert template == "http://status.alpicola.com/main/xrd?uri={uri}"
    end
  end
-
-  describe "ensure_keys_present" do
-    test "it creates keys for a user and stores them in info" do
-      user = insert(:user)
-      refute is_binary(user.info.keys)
-      {:ok, user} = WebFinger.ensure_keys_present(user)
-      assert is_binary(user.info.keys)
-    end
-
-    test "it doesn't create keys if there already are some" do
-      user = insert(:user, %{info: %{keys: "xxx"}})
-      {:ok, user} = WebFinger.ensure_keys_present(user)
-      assert user.info.keys == "xxx"
-    end
-  end
 end