Merge remote-tracking branch 'pleroma/develop' into feature/disable-account

.gitignore

@@ -3,7 +3,6 @@
 /db
 /deps
 /*.ez
-/uploads
 /test/uploads
 /.elixir_ls
 /test/fixtures/test_tmp.txt
@@ -11,6 +10,7 @@
 /test/tmp/
 /doc
 /instance
+/priv/ssh_keys
 
 # Prevent committing custom emojis
 /priv/static/emoji/custom/*

.gitlab-ci.yml

@@ -48,6 +48,7 @@ unit-testing:
   - name: postgres:9.6.2
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
+    - mix deps.get
     - mix ecto.create
     - mix ecto.migrate
     - mix test --trace --preload-modules
@@ -77,4 +78,4 @@ docs-deploy:
     - echo "${SSH_HOST_KEY}" > ~/.ssh/known_hosts
     - eval $(ssh-agent -s)
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
-    - rsync -hrvz --delete -e "ssh -p ${SSH_PORT}" priv/static/doc/ "${SSH_USER_HOST_LOCATION}/${CI_COMMIT_REF_NAME}" 
+    - rsync -hrvz --delete -e "ssh -p ${SSH_PORT}" priv/static/doc/ "${SSH_USER_HOST_LOCATION}/${CI_COMMIT_REF_NAME}"

CHANGELOG.md

@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `link_name` option
 - Configuration: `fetch_initial_posts` option
 - Configuration: `notify_email` option
+- Configuration: Media proxy `whitelist` option
 - Pleroma API: User subscriptions
 - Pleroma API: Healthcheck endpoint
 - Admin API: Endpoints for listing/revoking invite tokens
@@ -26,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: [Reports](https://docs.joinmastodon.org/api/rest/reports/)
 - ActivityPub C2S: OAuth endpoints
 - Metadata RelMe provider
+- OAuth: added support for refresh tokens
 - Emoji packs and emoji pack manager
 
 ### Changed
@@ -40,11 +42,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: Dedupe enabled by default
 - Configuration: Added `extra_cookie_attrs` for setting non-standard cookie attributes. Defaults to ["SameSite=Lax"] so that remote follows work.
 - Pleroma API: Support for emoji tags in `/api/pleroma/emoji` resulting in a breaking API change
+- Timelines: Messages involving people you have blocked will be excluded from the timeline in all cases instead of just repeats.
 - Mastodon API: Support for `exclude_types`, `limit` and `min_id` in `/api/v1/notifications`
 - Mastodon API: Add `languages` and `registrations` to `/api/v1/instance`
 - Mastodon API: Provide plaintext versions of cw/content in the Status entity
 - Mastodon API: Add `pleroma.conversation_id`, `pleroma.in_reply_to_account_acct` fields to the Status entity
-- Mastodon API: Add `pleroma.tags`, `pleroma.relationship{}`, `pleroma.is_moderator`, `pleroma.is_admin`, `pleroma.confirmation_pending` fields to the User entity
+- Mastodon API: Add `pleroma.tags`, `pleroma.relationship{}`, `pleroma.is_moderator`, `pleroma.is_admin`, `pleroma.confirmation_pending`, `pleroma.hide_followers`, `pleroma.hide_follows`, `pleroma.hide_favorites` fields to the User entity
+- Mastodon API: Add `pleroma.show_role`, `pleroma.no_rich_text` fields to the Source subentity
+- Mastodon API: Add support for updating `no_rich_text`, `hide_followers`, `hide_follows`, `hide_favorites`, `show_role` in `PATCH /api/v1/update_credentials`
 - Mastodon API: Add `pleroma.is_seen` to the Notification entity
 - Mastodon API: Add `pleroma.local` to the Status entity
 - Mastodon API: Add `preview` parameter to `POST /api/v1/statuses`
@@ -54,12 +59,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Deps: Updated Cowboy to 2.6
 - Deps: Updated Ecto to 3.0.7
 - Don't ship finmoji by default, they can be installed as an emoji pack
+- Mastodon API: Added support max_id & since_id for bookmark timeline endpoints.
 
 ### Fixed
+- Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended.
 - Followers counter not being updated when a follower is blocked
 - Deactivated users being able to request an access token
 - Limit on request body in rich media/relme parsers being ignored resulting in a possible memory leak
 - proper Twitter Card generation instead of a dummy
+- Deletions failing for users with a large number of posts
 - NodeInfo: Include admins in `staffAccounts`
 - ActivityPub: Crashing when requesting empty local user's outbox
 - Federation: Handling of objects without `summary` property
@@ -68,16 +76,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Federation: Cope with missing or explicitly nulled address lists
 - Federation: Explicitly ensure activities addressed to `as:Public` become addressed to the followers collection
 - Federation: Better cope with actors which do not declare a followers collection and use `as:Public` with these semantics
+- Federation: Follow requests from remote users who have been blocked will be automatically rejected if appropriate
 - MediaProxy: Parse name from content disposition headers even for non-whitelisted types
 - MediaProxy: S3 link encoding
 - Rich Media: Reject any data which cannot be explicitly encoded into JSON
 - Pleroma API: Importing follows from Mastodon 2.8+
+- Twitter API: Exposing default scope, `no_rich_text` of the user to anyone
+- Twitter API: Returning the `role` object in user entity despite `show_role = false`
 - Mastodon API: `/api/v1/favourites` serving only public activities
 - Mastodon API: Reblogs having `in_reply_to_id` - `null` even when they are replies
 - Mastodon API: Streaming API broadcasting wrong activity id
 - Mastodon API: 500 errors when requesting a card for a private conversation
 - Mastodon API: Handling of `reblogs` in `/api/v1/accounts/:id/follow`
 - Mastodon API: Correct `reblogged`, `favourited`, and `bookmarked` values in the reblog status JSON
+- Mastodon API: Exposing default scope of the user to anyone
 
 ## [0.9.9999] - 2019-04-05
 ### Security

config/config.exs

@@ -221,7 +221,8 @@
   allowed_post_formats: [
     "text/plain",
     "text/html",
-    "text/markdown"
+    "text/markdown",
+    "text/bbcode"
   ],
   mrf_transparency: true,
   autofollowed_nicknames: [],
@@ -326,7 +327,8 @@
       follow_redirect: true,
       pool: :media
     ]
-  ]
+  ],
+  whitelist: []
 
 config :pleroma, :chat, enabled: true
 
@@ -415,6 +417,7 @@
   mailer: 10,
   transmogrifier: 20,
   scheduled_activities: 10,
+  background: 5,
   user: 10
 
 config :pleroma, :fetch_initial_posts,
@@ -442,6 +445,9 @@
   base: System.get_env("LDAP_BASE") || "dc=example,dc=com",
   uid: System.get_env("LDAP_UID") || "cn"
 
+config :esshd,
+  enabled: false
+
 oauth_consumer_strategies = String.split(System.get_env("OAUTH_CONSUMER_STRATEGIES") || "")
 
 ueberauth_providers =
@@ -467,6 +473,10 @@
   total_user_limit: 300,
   enabled: true
 
+config :pleroma, :oauth2,
+  token_expires_in: 600,
+  issue_new_refresh_token: true
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

docs/api/differences_in_mastoapi_responses.md

@@ -1,6 +1,6 @@
 # Differences in Mastodon API responses from vanilla Mastodon
 
-A Pleroma instance can be identified by "<Mastodon version> (compatible; Pleroma <version>)" present in `version` field in response from `/api/v1/instance` 
+A Pleroma instance can be identified by "<Mastodon version> (compatible; Pleroma <version>)" present in `version` field in response from `/api/v1/instance`
 
 ## Flake IDs
 
@@ -38,9 +38,18 @@ Has these additional fields under the `pleroma` object:
 
 - `tags`: Lists an array of tags for the user
 - `relationship{}`: Includes fields as documented for Mastodon API https://docs.joinmastodon.org/api/entities/#relationship
-- `is_moderator`: boolean, true if user is a moderator
+- `is_moderator`: boolean, nullable,  true if user is a moderator
-- `is_admin`: boolean, true if user is an admin
+- `is_admin`: boolean, nullable, true if user is an admin
 - `confirmation_pending`: boolean, true if a new user account is waiting on email confirmation to be activated
+- `hide_followers`: boolean, true when the user has follower hiding enabled
+- `hide_follows`: boolean, true when the user has follow hiding enabled
+
+### Source
+
+Has these additional fields under the `pleroma` object:
+
+- `show_role`: boolean, nullable, true when the user wants his role (e.g admin, moderator) to be shown
+- `no_rich_text` - boolean, nullable, true when html tags are stripped from all statuses requested from the API
 
 ## Account Search
 
@@ -60,3 +69,21 @@ Additional parameters can be added to the JSON body/Form data:
 
 - `preview`: boolean, if set to `true` the post won't be actually posted, but the status entitiy would still be rendered back. This could be useful for previewing rich text/custom emoji, for example.
 - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint.
+
+## PATCH `/api/v1/update_credentials`
+
+Additional parameters can be added to the JSON body/Form data:
+
+- `no_rich_text` - if true, html tags are stripped from all statuses requested from the API
+- `hide_followers` - if true, user's followers will be hidden
+- `hide_follows` - if true, user's follows will be hidden
+- `hide_favorites` - if true, user's favorites timeline will be hidden
+- `show_role` - if true, user's role (e.g admin, moderator) will be exposed to anyone in the API
+- `default_scope` - the scope returned under `privacy` key in Source subentity
+
+## Authentication
+
+*Pleroma supports refreshing tokens.
+
+`POST /oauth/token`
+Post here request with grant_type=refresh_token to obtain new access token. Returns an access token.

docs/config.md

@@ -37,7 +37,7 @@ This filter replaces the filename (not the path) of an upload. For complete obfu
 
 An example for Sendgrid adapter:
 
-```exs
+```elixir
 config :pleroma, Pleroma.Emails.Mailer,
   adapter: Swoosh.Adapters.Sendgrid,
   api_key: "YOUR_API_KEY"
@@ -45,7 +45,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 
 An example for SMTP adapter:
 
-```exs
+```elixir
 config :pleroma, Pleroma.Emails.Mailer,
   adapter: Swoosh.Adapters.SMTP,
   relay: "smtp.gmail.com",
@@ -109,7 +109,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack
 
 An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed:
-```
+```elixir
 config :logger,
   backends: [{ExSyslogger, :ex_syslogger}]
 
@@ -118,7 +118,7 @@ config :logger, :ex_syslogger,
 ```
 
 Another example, keeping console output and adding the pid to syslog output:
-```
+```elixir
 config :logger,
   backends: [:console, {ExSyslogger, :ex_syslogger}]
 
@@ -130,7 +130,7 @@ config :logger, :ex_syslogger,
 See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/)
 
 An example of logging info to local syslog, but warn to a Slack channel:
-```
+```elixir
 config :logger,
   backends: [ {ExSyslogger, :ex_syslogger}, Quack.Logger ],
   level: :info
@@ -156,14 +156,30 @@ Frontends can access these settings at `/api/pleroma/frontend_configurations`
 
 To add your own configuration for PleromaFE, use it like this:
 
-`config :pleroma, :frontend_configurations, pleroma_fe: %{redirectRootNoLogin: "/main/all", ...}`
+```elixir
+config :pleroma, :frontend_configurations,
+  pleroma_fe: %{
+    theme: "pleroma-dark",
+    # ... see /priv/static/static/config.json for the available keys.
+},
+  masto_fe: %{
+    showInstanceSpecificPanel: true
+  }
+```
 
-These settings need to be complete, they will override the defaults. See `priv/static/static/config.json` for the available keys.
+These settings **need to be complete**, they will override the defaults.
+
+NOTE: for versions < 1.0, you need to set [`:fe`](#fe) to false, as shown a few lines below.
 
 ## :fe
 __THIS IS DEPRECATED__
 
-If you are using this method, please change it to the `frontend_configurations` method. Please set this option to false in your config like this: `config :pleroma, :fe, false`.
+If you are using this method, please change it to the [`frontend_configurations`](#frontend_configurations) method.
+Please **set this option to false** in your config like this:
+
+```elixir
+config :pleroma, :fe, false
+```
 
 This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false.
 
@@ -205,6 +221,7 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
 * `enabled`: Enables proxying of remote media to the instance’s proxy
 * `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.
 * `proxy_opts`: All options defined in `Pleroma.ReverseProxy` documentation, defaults to `[max_body_length: (25*1_048_576)]`.
+* `whitelist`: List of domains to bypass the mediaproxy
 
 ## :gopher
 * `enabled`: Enables the gopher interface
@@ -273,7 +290,7 @@ their ActivityPub ID.
 
 An example:
 
-```exs
+```elixir
 config :pleroma, :mrf_user_allowlist,
   "example.org": ["https://example.org/users/admin"]
 ```
@@ -302,7 +319,7 @@ the source code is here: https://github.com/koto-bank/kocaptcha. The default end
 
 Allows to set a token that can be used to authenticate with the admin api without using an actual user by giving it as the 'admin_token' parameter. Example:
 
-```exs
+```elixir
 config :pleroma, :admin_token, "somerandomtoken"
 ```
 
@@ -386,7 +403,7 @@ Configuration for the `auto_linker` library:
 
 Example:
 
-```exs
+```elixir
 config :auto_linker,
   opts: [
     scheme: true,
@@ -427,15 +444,36 @@ Pleroma account will be created with the same name as the LDAP user name.
 * `base`: LDAP base, e.g. "dc=example,dc=com"
 * `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base"
 
+## BBS / SSH access
+
+To enable simple command line interface accessible over ssh, add a setting like this to your configuration file:
+
+```exs
+app_dir = File.cwd!
+priv_dir = Path.join([app_dir, "priv/ssh_keys"])
+
+config :esshd,
+  enabled: true,
+  priv_dir: priv_dir,
+  handler: "Pleroma.BBS.Handler",
+  port: 10_022,
+  password_authenticator: "Pleroma.BBS.Authenticator"
+```
+
+Feel free to adjust the priv_dir and port number. Then you will have to create the key for the keys (in the example `priv/ssh_keys`) and create the host keys with `ssh-keygen -N "" -b 2048 -t rsa -f ssh_host_rsa_key`. After restarting, you should be able to connect to your Pleroma instance with `ssh username@server -p $PORT`
+
 ## :auth
 
+* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator
+* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication
+
 Authentication / authorization settings.
 
 * `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`.
 * `oauth_consumer_template`: OAuth consumer mode authentication form template. By default it's `consumer.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex`.
 * `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by OAUTH_CONSUMER_STRATEGIES environment variable.
 
-# OAuth consumer mode
+## OAuth consumer mode
 
 OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.).
 Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies).
@@ -459,7 +497,7 @@ Note: make sure that `"SameSite=Lax"` is set in `extra_cookie_attrs` when you ha
 Once the app is configured on external OAuth provider side, add app's credentials and strategy-specific settings (if any — e.g. see Microsoft below) to `config/prod.secret.exs`,
 per strategy's documentation (e.g. [ueberauth_twitter](https://github.com/ueberauth/ueberauth_twitter)). Example config basing on environment variables:
 
-```
+```elixir
 # Twitter
 config :ueberauth, Ueberauth.Strategy.Twitter.OAuth,
   consumer_key: System.get_env("TWITTER_CONSUMER_KEY"),
@@ -488,6 +526,13 @@ config :ueberauth, Ueberauth,
   ]
 ```
 
+## OAuth 2.0 provider - :oauth2
+
+Configure OAuth 2 provider capabilities:
+
+* `token_expires_in` - The lifetime in seconds of the access token.
+* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token.
+
 ## :emoji
 * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
 * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]`

installation/download-mastofe-build.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+project_id="74"
+project_branch="rebase/glitch-soc"
+static_dir="instance/static"
+# For bundling:
+# project_branch="pleroma"
+# static_dir="priv/static"
+
+if [[ ! -d "${static_dir}" ]]
+then
+	echo "Error: ${static_dir} directory is missing, are you sure you are running this script at the root of pleroma’s repository?"
+	exit 1
+fi
+
+last_modified="$(curl -s -I 'https://git.pleroma.social/api/v4/projects/'${project_id}'/jobs/artifacts/'${project_branch}'/download?job=build' | grep '^Last-Modified:' | cut -d: -f2-)"
+
+echo "branch:${project_branch}"
+echo "Last-Modified:${last_modified}"
+
+artifact="mastofe.zip"
+
+if [[ -e mastofe.timestamp ]] && [[ "${last_modified}" != "" ]]
+then
+	if [[ "$(cat mastofe.timestamp)" == "${last_modified}" ]]
+	then
+		echo "MastoFE is up-to-date, exiting…"
+		exit 0
+	fi
+fi
+
+curl -c - "https://git.pleroma.social/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=build" -o "${artifact}" || exit
+
+# TODO: Update the emoji as well
+rm -fr "${static_dir}/sw.js" "${static_dir}/packs" || exit
+unzip -q "${artifact}" || exit
+
+cp public/assets/sw.js "${static_dir}/sw.js" || exit
+cp -r public/packs "${static_dir}/packs" || exit
+
+echo "${last_modified}" > mastofe.timestamp
+rm -fr public
+rm -i "${artifact}"

lib/mix/tasks/benchmark.ex

@@ -0,0 +1,25 @@
+defmodule Mix.Tasks.Pleroma.Benchmark do
+  use Mix.Task
+  alias Mix.Tasks.Pleroma.Common
+
+  def run(["search"]) do
+    Common.start_pleroma()
+
+    Benchee.run(%{
+      "search" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPIController.status_search(nil, "cofe")
+      end
+    })
+  end
+
+  def run(["tag"]) do
+    Common.start_pleroma()
+
+    Benchee.run(%{
+      "tag" => fn ->
+        %{"type" => "Create", "tag" => "cofe"}
+        |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
+      end
+    })
+  end
+end

lib/mix/tasks/pleroma/user.ex

@@ -126,7 +126,7 @@ def run(["new", nickname, email | rest]) do
 
     proceed? = assume_yes? or Mix.shell().yes?("Continue?")
 
-    unless not proceed? do
+    if proceed? do
       Common.start_pleroma()
 
       params = %{
@@ -163,7 +163,7 @@ def run(["rm", nickname]) do
     Common.start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      User.delete(user)
+      User.perform(:delete, user)
       Mix.shell().info("User #{nickname} deleted.")
     else
       _ ->
@@ -380,7 +380,7 @@ def run(["delete_activities", nickname]) do
     Common.start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      User.delete_user_activities(user)
+      {:ok, _} = User.delete_user_activities(user)
       Mix.shell().info("User #{nickname} statuses deleted.")
     else
       _ ->

lib/pleroma/activity.ex

@@ -14,6 +14,8 @@ defmodule Pleroma.Activity do
   import Ecto.Query
 
   @type t :: %__MODULE__{}
+  @type actor :: String.t()
+
   @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
 
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
@@ -265,6 +267,11 @@ def all_by_actor_and_id(actor, status_ids) do
     |> Repo.all()
   end
 
+  @spec query_by_actor(actor()) :: Ecto.Query.t()
+  def query_by_actor(actor) do
+    from(a in Activity, where: a.actor == ^actor)
+  end
+
   def restrict_deactivated_users(query) do
     from(activity in query,
       where:

lib/pleroma/bbs/authenticator.ex

@@ -0,0 +1,16 @@
+defmodule Pleroma.BBS.Authenticator do
+  use Sshd.PasswordAuthenticator
+  alias Comeonin.Pbkdf2
+  alias Pleroma.User
+
+  def authenticate(username, password) do
+    username = to_string(username)
+    password = to_string(password)
+
+    with %User{} = user <- User.get_by_nickname(username) do
+      Pbkdf2.checkpw(password, user.password_hash)
+    else
+      _e -> false
+    end
+  end
+end

lib/pleroma/bbs/handler.ex

@@ -0,0 +1,147 @@
+defmodule Pleroma.BBS.Handler do
+  use Sshd.ShellHandler
+  alias Pleroma.Activity
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.CommonAPI
+
+  def on_shell(username, _pubkey, _ip, _port) do
+    :ok = IO.puts("Welcome to #{Pleroma.Config.get([:instance, :name])}!")
+    user = Pleroma.User.get_cached_by_nickname(to_string(username))
+    Logger.debug("#{inspect(user)}")
+    loop(run_state(user: user))
+  end
+
+  def on_connect(username, ip, port, method) do
+    Logger.debug(fn ->
+      """
+      Incoming SSH shell #{inspect(self())} requested for #{username} from #{inspect(ip)}:#{
+        inspect(port)
+      } using #{inspect(method)}
+      """
+    end)
+  end
+
+  def on_disconnect(username, ip, port) do
+    Logger.debug(fn ->
+      "Disconnecting SSH shell for #{username} from #{inspect(ip)}:#{inspect(port)}"
+    end)
+  end
+
+  defp loop(state) do
+    self_pid = self()
+    counter = state.counter
+    prefix = state.prefix
+    user = state.user
+
+    input = spawn(fn -> io_get(self_pid, prefix, counter, user.nickname) end)
+    wait_input(state, input)
+  end
+
+  def puts_activity(activity) do
+    status = Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{activity: activity})
+    IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})")
+    IO.puts(HtmlSanitizeEx.strip_tags(status.content))
+    IO.puts("")
+  end
+
+  def handle_command(state, "help") do
+    IO.puts("Available commands:")
+    IO.puts("help - This help")
+    IO.puts("home - Show the home timeline")
+    IO.puts("p <text> - Post the given text")
+    IO.puts("r <id> <text> - Reply to the post with the given id")
+    IO.puts("quit - Quit")
+
+    state
+  end
+
+  def handle_command(%{user: user} = state, "r " <> text) do
+    text = String.trim(text)
+    [activity_id, rest] = String.split(text, " ", parts: 2)
+
+    with %Activity{} <- Activity.get_by_id(activity_id),
+         {:ok, _activity} <-
+           CommonAPI.post(user, %{"status" => rest, "in_reply_to_status_id" => activity_id}) do
+      IO.puts("Replied!")
+    else
+      _e -> IO.puts("Could not reply...")
+    end
+
+    state
+  end
+
+  def handle_command(%{user: user} = state, "p " <> text) do
+    text = String.trim(text)
+
+    with {:ok, _activity} <- CommonAPI.post(user, %{"status" => text}) do
+      IO.puts("Posted!")
+    else
+      _e -> IO.puts("Could not post...")
+    end
+
+    state
+  end
+
+  def handle_command(state, "home") do
+    user = state.user
+
+    params =
+      %{}
+      |> Map.put("type", ["Create"])
+      |> Map.put("blocking_user", user)
+      |> Map.put("muting_user", user)
+      |> Map.put("user", user)
+
+    activities =
+      [user.ap_id | user.following]
+      |> ActivityPub.fetch_activities(params)
+      |> ActivityPub.contain_timeline(user)
+
+    Enum.each(activities, fn activity ->
+      puts_activity(activity)
+    end)
+
+    state
+  end
+
+  def handle_command(state, command) do
+    IO.puts("Unknown command '#{command}'")
+    state
+  end
+
+  defp wait_input(state, input) do
+    receive do
+      {:input, ^input, "quit\n"} ->
+        IO.puts("Exiting...")
+
+      {:input, ^input, code} when is_binary(code) ->
+        code = String.trim(code)
+
+        state = handle_command(state, code)
+
+        loop(%{state | counter: state.counter + 1})
+
+      {:error, :interrupted} ->
+        IO.puts("Caught Ctrl+C...")
+        loop(%{state | counter: state.counter + 1})
+
+      {:input, ^input, msg} ->
+        :ok = Logger.warn("received unknown message: #{inspect(msg)}")
+        loop(%{state | counter: state.counter + 1})
+    end
+  end
+
+  defp run_state(opts) do
+    %{prefix: "pleroma", counter: 1, user: opts[:user]}
+  end
+
+  defp io_get(pid, prefix, counter, username) do
+    prompt = prompt(prefix, counter, username)
+    send(pid, {:input, self(), IO.gets(:stdio, prompt)})
+  end
+
+  defp prompt(prefix, counter, username) do
+    prompt = "#{username}@#{prefix}:#{counter}>"
+    prompt <> " "
+  end
+end

lib/pleroma/bookmark.ex

@@ -0,0 +1,60 @@
+defmodule Pleroma.Bookmark do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+
+  alias Pleroma.Activity
+  alias Pleroma.Bookmark
+  alias Pleroma.FlakeId
+  alias Pleroma.Repo
+  alias Pleroma.User
+
+  @type t :: %__MODULE__{}
+
+  schema "bookmarks" do
+    belongs_to(:user, User, type: FlakeId)
+    belongs_to(:activity, Activity, type: FlakeId)
+
+    timestamps()
+  end
+
+  @spec create(FlakeId.t(), FlakeId.t()) :: {:ok, Bookmark.t()} | {:error, Changeset.t()}
+  def create(user_id, activity_id) do
+    attrs = %{
+      user_id: user_id,
+      activity_id: activity_id
+    }
+
+    %Bookmark{}
+    |> cast(attrs, [:user_id, :activity_id])
+    |> validate_required([:user_id, :activity_id])
+    |> unique_constraint(:activity_id, name: :bookmarks_user_id_activity_id_index)
+    |> Repo.insert()
+  end
+
+  @spec for_user_query(FlakeId.t()) :: Ecto.Query.t()
+  def for_user_query(user_id) do
+    Bookmark
+    |> where(user_id: ^user_id)
+    |> join(:inner, [b], activity in assoc(b, :activity))
+    |> preload([b, a], activity: a)
+  end
+
+  def get(user_id, activity_id) do
+    Bookmark
+    |> where(user_id: ^user_id)
+    |> where(activity_id: ^activity_id)
+    |> Repo.one()
+  end
+
+  @spec destroy(FlakeId.t(), FlakeId.t()) :: {:ok, Bookmark.t()} | {:error, Changeset.t()}
+  def destroy(user_id, activity_id) do
+    from(b in Bookmark,
+      where: b.user_id == ^user_id,
+      where: b.activity_id == ^activity_id
+    )
+    |> Repo.one()
+    |> Repo.delete()
+  end
+end

lib/pleroma/formatter.ex

@@ -113,9 +113,7 @@ def emojify(text, emoji, strip \\ false) do
 
       html =
         if not strip do
-          "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
+          "<img class='emoji' alt='#{emoji}' title='#{emoji}' src='#{MediaProxy.url(file)}' />"
-            MediaProxy.url(file)
-          }' />"
         else
           ""
         end
@@ -130,12 +128,23 @@ def demojify(text) do
 
   def demojify(text, nil), do: text
 
+  @doc "Outputs a list of the emoji-shortcodes in a text"
   def get_emoji(text) when is_binary(text) do
     Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end)
   end
 
   def get_emoji(_), do: []
 
+  @doc "Outputs a list of the emoji-Maps in a text"
+  def get_emoji_map(text) when is_binary(text) do
+    get_emoji(text)
+    |> Enum.reduce(%{}, fn {name, file, _group}, acc ->
+      Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
+    end)
+  end
+
+  def get_emoji_map(_), do: []
+
   def html_escape({text, mentions, hashtags}, type) do
     {html_escape(text, type), mentions, hashtags}
   end

lib/pleroma/html.ex

@@ -28,12 +28,18 @@ def filter_tags(html, scrubber), do: Scrubber.scrub(html, scrubber)
   def filter_tags(html), do: filter_tags(html, nil)
   def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags)
 
-  def get_cached_scrubbed_html_for_activity(content, scrubbers, activity, key \\ "") do
+  def get_cached_scrubbed_html_for_activity(
+        content,
+        scrubbers,
+        activity,
+        key \\ "",
+        callback \\ fn x -> x end
+      ) do
     key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
 
     Cachex.fetch!(:scrubber_cache, key, fn _key ->
       object = Pleroma.Object.normalize(activity)
-      ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false)
+      ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
     end)
   end
 
@@ -42,24 +48,27 @@ def get_cached_stripped_html_for_activity(content, activity, key) do
       content,
       HtmlSanitizeEx.Scrubber.StripTags,
       activity,
-      key
+      key,
+      &HtmlEntities.decode/1
     )
   end
 
   def ensure_scrubbed_html(
         content,
         scrubbers,
-        false = _fake
+        fake,
+        callback
       ) do
-    {:commit, filter_tags(content, scrubbers)}
+    content =
-  end
+      content
+      |> filter_tags(scrubbers)
+      |> callback.()
 
-  def ensure_scrubbed_html(
+    if fake do
-        content,
+      {:ignore, content}
-        scrubbers,
+    else
-        true = _fake
+      {:commit, content}
-      ) do
+    end
-    {:ignore, filter_tags(content, scrubbers)}
   end
 
   defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
@@ -106,7 +115,14 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
 
   # links
   Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
-  Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"])
+
+  Meta.allow_tag_with_this_attribute_values("a", "class", [
+    "hashtag",
+    "u-url",
+    "mention",
+    "u-url mention",
+    "mention u-url"
+  ])
 
   Meta.allow_tag_with_this_attribute_values("a", "rel", [
     "tag",
@@ -115,12 +131,15 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
     "noreferrer"
   ])
 
+  Meta.allow_tag_with_these_attributes("a", ["name", "title"])
+
   # paragraphs and linebreaks
   Meta.allow_tag_with_these_attributes("br", [])
   Meta.allow_tag_with_these_attributes("p", [])
 
   # microformats
-  Meta.allow_tag_with_these_attributes("span", ["class"])
+  Meta.allow_tag_with_this_attribute_values("span", "class", ["h-card"])
+  Meta.allow_tag_with_these_attributes("span", [])
 
   # allow inline images for custom emoji
   @allow_inline_images Keyword.get(@markup, :allow_inline_images)
@@ -132,6 +151,7 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
     Meta.allow_tag_with_these_attributes("img", [
       "width",
       "height",
+      "class",
       "title",
       "alt"
     ])
@@ -155,7 +175,14 @@ defmodule Pleroma.HTML.Scrubber.Default do
   Meta.strip_comments()
 
   Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
-  Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"])
+
+  Meta.allow_tag_with_this_attribute_values("a", "class", [
+    "hashtag",
+    "u-url",
+    "mention",
+    "u-url mention",
+    "mention u-url"
+  ])
 
   Meta.allow_tag_with_this_attribute_values("a", "rel", [
     "tag",
@@ -164,6 +191,8 @@ defmodule Pleroma.HTML.Scrubber.Default do
     "noreferrer"
   ])
 
+  Meta.allow_tag_with_these_attributes("a", ["name", "title"])
+
   Meta.allow_tag_with_these_attributes("abbr", ["title"])
 
   Meta.allow_tag_with_these_attributes("b", [])
@@ -177,11 +206,13 @@ defmodule Pleroma.HTML.Scrubber.Default do
   Meta.allow_tag_with_these_attributes("ol", [])
   Meta.allow_tag_with_these_attributes("p", [])
   Meta.allow_tag_with_these_attributes("pre", [])
-  Meta.allow_tag_with_these_attributes("span", ["class"])
   Meta.allow_tag_with_these_attributes("strong", [])
   Meta.allow_tag_with_these_attributes("u", [])
   Meta.allow_tag_with_these_attributes("ul", [])
 
+  Meta.allow_tag_with_this_attribute_values("span", "class", ["h-card"])
+  Meta.allow_tag_with_these_attributes("span", [])
+
   @allow_inline_images Keyword.get(@markup, :allow_inline_images)
 
   if @allow_inline_images do
@@ -191,6 +222,7 @@ defmodule Pleroma.HTML.Scrubber.Default do
     Meta.allow_tag_with_these_attributes("img", [
       "width",
       "height",
+      "class",
       "title",
       "alt"
     ])

lib/pleroma/plugs/http_security_plug.ex

@@ -35,7 +35,7 @@ defp headers do
   defp csp_string do
     scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
     static_url = Pleroma.Web.Endpoint.static_url()
-    websocket_url = String.replace(static_url, "http", "ws")
+    websocket_url = Pleroma.Web.Endpoint.websocket_url()
 
     connect_src = "connect-src 'self' #{static_url} #{websocket_url}"
 

lib/pleroma/plugs/oauth_plug.ex

@@ -16,6 +16,16 @@ def init(options), do: options
 
   def call(%{assigns: %{user: %User{}}} = conn, _), do: conn
 
+  def call(%{params: %{"access_token" => access_token}} = conn, _) do
+    with {:ok, user, token_record} <- fetch_user_and_token(access_token) do
+      conn
+      |> assign(:token, token_record)
+      |> assign(:user, user)
+    else
+      _ -> conn
+    end
+  end
+
   def call(conn, _) do
     with {:ok, token_str} <- fetch_token_str(conn),
          {:ok, user, token_record} <- fetch_user_and_token(token_str) do

lib/pleroma/repo.ex

@@ -19,4 +19,32 @@ defmodule Instrumenter do
   def init(_, opts) do
     {:ok, Keyword.put(opts, :url, System.get_env("DATABASE_URL"))}
   end
+
+  @doc "find resource based on prepared query"
+  @spec find_resource(Ecto.Query.t()) :: {:ok, struct()} | {:error, :not_found}
+  def find_resource(%Ecto.Query{} = query) do
+    case __MODULE__.one(query) do
+      nil -> {:error, :not_found}
+      resource -> {:ok, resource}
+    end
+  end
+
+  def find_resource(_query), do: {:error, :not_found}
+
+  @doc """
+  Gets association from cache or loads if need
+
+  ## Examples
+
+    iex> Repo.get_assoc(token, :user)
+    %User{}
+
+  """
+  @spec get_assoc(struct(), atom()) :: {:ok, struct()} | {:error, :not_found}
+  def get_assoc(resource, association) do
+    case __MODULE__.preload(resource, association) do
+      %{^association => assoc} when not is_nil(assoc) -> {:ok, assoc}
+      _ -> {:error, :not_found}
+    end
+  end
 end

lib/pleroma/user.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.User do
 
   alias Comeonin.Pbkdf2
   alias Pleroma.Activity
-  alias Pleroma.Formatter
+  alias Pleroma.Bookmark
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Registration
@@ -53,8 +53,8 @@ defmodule Pleroma.User do
     field(:search_rank, :float, virtual: true)
     field(:search_type, :integer, virtual: true)
     field(:tags, {:array, :string}, default: [])
-    field(:bookmarks, {:array, :string}, default: [])
     field(:last_refreshed_at, :naive_datetime_usec)
+    has_many(:bookmarks, Bookmark)
     has_many(:notifications, Notification)
     has_many(:registrations, Registration)
     embeds_one(:info, Pleroma.User.Info)
@@ -437,7 +437,7 @@ def follow_import(%User{} = follower, followed_identifiers)
     Enum.map(
       followed_identifiers,
       fn followed_identifier ->
-        with %User{} = followed <- get_or_fetch(followed_identifier),
+        with {:ok, %User{} = followed} <- get_or_fetch(followed_identifier),
              {:ok, follower} <- maybe_direct_follow(follower, followed),
              {:ok, _} <- ActivityPub.follow(follower, followed) do
           followed
@@ -521,7 +521,15 @@ def get_cached_by_id(id) do
 
   def get_cached_by_nickname(nickname) do
     key = "nickname:#{nickname}"
-    Cachex.fetch!(:user_cache, key, fn _ -> get_or_fetch_by_nickname(nickname) end)
+
+    Cachex.fetch!(:user_cache, key, fn ->
+      user_result = get_or_fetch_by_nickname(nickname)
+
+      case user_result do
+        {:ok, user} -> {:commit, user}
+        {:error, _error} -> {:ignore, nil}
+      end
+    end)
   end
 
   def get_cached_by_nickname_or_id(nickname_or_id) do
@@ -557,7 +565,7 @@ def fetch_by_nickname(nickname) do
 
   def get_or_fetch_by_nickname(nickname) do
     with %User{} = user <- get_by_nickname(nickname) do
-      user
+      {:ok, user}
     else
       _e ->
         with [_nick, _domain] <- String.split(nickname, "@"),
@@ -567,9 +575,9 @@ def get_or_fetch_by_nickname(nickname) do
             {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
           end
 
-          user
+          {:ok, user}
         else
-          _e -> nil
+          _e -> {:error, "not found " <> nickname}
         end
     end
   end
@@ -920,7 +928,7 @@ def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_i
     Enum.map(
       blocked_identifiers,
       fn blocked_identifier ->
-        with %User{} = blocked <- get_or_fetch(blocked_identifier),
+        with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier),
              {:ok, blocker} <- block(blocker, blocked),
              {:ok, _} <- ActivityPub.block(blocker, blocked) do
           blocked
@@ -1188,7 +1196,12 @@ def update_notification_settings(%User{} = user, settings \\ %{}) do
     |> update_and_set_cache()
   end
 
-  def delete(%User{} = user) do
+  @spec delete(User.t()) :: :ok
+  def delete(%User{} = user),
+    do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user])
+
+  @spec perform(atom(), User.t()) :: {:ok, User.t()}
+  def perform(:delete, %User{} = user) do
     {:ok, user} = User.deactivate(user)
 
     # Remove all relationships
@@ -1204,22 +1217,23 @@ def delete(%User{} = user) do
   end
 
   def delete_user_activities(%User{ap_id: ap_id} = user) do
-    Activity
+    stream =
-    |> where(actor: ^ap_id)
+      ap_id
-    |> Activity.with_preloaded_object()
+      |> Activity.query_by_actor()
-    |> Repo.all()
+      |> Activity.with_preloaded_object()
-    |> Enum.each(fn
+      |> Repo.stream()
-      %{data: %{"type" => "Create"}} = activity ->
-        activity |> Object.normalize() |> ActivityPub.delete()
 
-      # TODO: Do something with likes, follows, repeats.
+    Repo.transaction(fn -> Enum.each(stream, &delete_activity(&1)) end, timeout: :infinity)
-      _ ->
-        "Doing nothing"
-    end)
 
     {:ok, user}
   end
 
+  defp delete_activity(%{data: %{"type" => "Create"}} = activity) do
+    Object.normalize(activity) |> ActivityPub.delete()
+  end
+
+  defp delete_activity(_activity), do: "Doing nothing"
+
   def html_filter_policy(%User{info: %{no_rich_text: true}}) do
     Pleroma.HTML.Scrubber.TwitterText
   end
@@ -1233,11 +1247,11 @@ def fetch_by_ap_id(ap_id) do
 
     case ap_try do
       {:ok, user} ->
-        user
+        {:ok, user}
 
       _ ->
         case OStatus.make_user(ap_id) do
-          {:ok, user} -> user
+          {:ok, user} -> {:ok, user}
           _ -> {:error, "Could not fetch by AP id"}
         end
     end
@@ -1247,20 +1261,20 @@ def get_or_fetch_by_ap_id(ap_id) do
     user = get_cached_by_ap_id(ap_id)
 
     if !is_nil(user) and !User.needs_update?(user) do
-      user
+      {:ok, user}
     else
       # Whether to fetch initial posts for the user (if it's a new user & the fetching is enabled)
       should_fetch_initial = is_nil(user) and Pleroma.Config.get([:fetch_initial_posts, :enabled])
 
-      user = fetch_by_ap_id(ap_id)
+      resp = fetch_by_ap_id(ap_id)
 
       if should_fetch_initial do
-        with %User{} = user do
+        with {:ok, %User{} = user} = resp do
           {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
         end
       end
 
-      user
+      resp
     end
   end
 
@@ -1302,7 +1316,7 @@ def public_key_from_info(%{magic_key: magic_key}) do
   end
 
   def get_public_key_for_ap_id(ap_id) do
-    with %User{} = user <- get_or_fetch_by_ap_id(ap_id),
+    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id),
          {:ok, public_key} <- public_key_from_info(user.info) do
       {:ok, public_key}
     else
@@ -1354,18 +1368,15 @@ def wait_and_refresh(timeout, %User{} = a, %User{} = b) do
     end
   end
 
-  def parse_bio(bio, user \\ %User{info: %{source_data: %{}}})
+  def parse_bio(bio) when is_binary(bio) and bio != "" do
-  def parse_bio(nil, _user), do: ""
+    bio
-  def parse_bio(bio, _user) when bio == "", do: bio
+    |> CommonUtils.format_input("text/plain", mentions_format: :full)
+    |> elem(0)
+  end
 
-  def parse_bio(bio, user) do
+  def parse_bio(_), do: ""
-    emoji =
-      (user.info.source_data["tag"] || [])
-      |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
-      |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} ->
-        {String.trim(name, ":"), url}
-      end)
 
+  def parse_bio(bio, user) when is_binary(bio) and bio != "" do
     # TODO: get profile URLs other than user.ap_id
     profile_urls = [user.ap_id]
 
@@ -1375,9 +1386,10 @@ def parse_bio(bio, user) do
       rel: &RelMe.maybe_put_rel_me(&1, profile_urls)
     )
     |> elem(0)
-    |> Formatter.emojify(emoji)
   end
 
+  def parse_bio(_, _), do: ""
+
   def tag(user_identifiers, tags) when is_list(user_identifiers) do
     Repo.transaction(fn ->
       for user_identifier <- user_identifiers, do: tag(user_identifier, tags)
@@ -1411,22 +1423,6 @@ defp update_tags(%User{} = user, new_tags) do
     updated_user
   end
 
-  def bookmark(%User{} = user, status_id) do
-    bookmarks = Enum.uniq(user.bookmarks ++ [status_id])
-    update_bookmarks(user, bookmarks)
-  end
-
-  def unbookmark(%User{} = user, status_id) do
-    bookmarks = Enum.uniq(user.bookmarks -- [status_id])
-    update_bookmarks(user, bookmarks)
-  end
-
-  def update_bookmarks(%User{} = user, bookmarks) do
-    user
-    |> change(%{bookmarks: bookmarks})
-    |> update_and_set_cache
-  end
-
   defp normalize_tags(tags) do
     [tags]
     |> List.flatten()

lib/pleroma/user/info.ex

@@ -41,6 +41,7 @@ defmodule Pleroma.User.Info do
     field(:hide_favorites, :boolean, default: true)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:flavour, :string, default: nil)
+    field(:emoji, {:array, :map}, default: [])
 
     field(:notification_settings, :map,
       default: %{"remote" => true, "local" => true, "followers" => true, "follows" => true}
@@ -227,14 +228,6 @@ def confirmation_changeset(info, params) do
     cast(info, params, [:confirmation_pending, :confirmation_token])
   end
 
-  def mastodon_profile_update(info, params) do
-    info
-    |> cast(params, [
-      :locked,
-      :banner
-    ])
-  end
-
   def mastodon_settings_update(info, settings) do
     params = %{settings: settings}
 

lib/pleroma/user_invite_token.ex

@@ -24,7 +24,7 @@ defmodule Pleroma.UserInviteToken do
     timestamps()
   end
 
-  @spec create_invite(map()) :: UserInviteToken.t()
+  @spec create_invite(map()) :: {:ok, UserInviteToken.t()}
   def create_invite(params \\ %{}) do
     %UserInviteToken{}
     |> cast(params, [:max_use, :expires_at])

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -168,7 +168,6 @@ def stream_out(activity) do
     public = "https://www.w3.org/ns/activitystreams#Public"
 
     if activity.data["type"] in ["Create", "Announce", "Delete"] do
-      object = Object.normalize(activity)
       Pleroma.Web.Streamer.stream("user", activity)
       Pleroma.Web.Streamer.stream("list", activity)
 
@@ -180,6 +179,8 @@ def stream_out(activity) do
         end
 
         if activity.data["type"] in ["Create"] do
+          object = Object.normalize(activity)
+
           object.data
           |> Map.get("tag", [])
           |> Enum.filter(fn tag -> is_bitstring(tag) end)

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -155,7 +155,7 @@ def outbox(conn, %{"nickname" => nickname} = params) do
 
   def inbox(%{assigns: %{valid_signature: true}} = conn, %{"nickname" => nickname} = params) do
     with %User{} = recipient <- User.get_cached_by_nickname(nickname),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(params["actor"]),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]),
          true <- Utils.recipient_in_message(recipient, actor, params),
          params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do
       Federator.incoming_ap_doc(params)

lib/pleroma/web/activity_pub/relay.ex

@@ -15,7 +15,7 @@ def get_actor do
 
   def follow(target_instance) do
     with %User{} = local_user <- get_actor(),
-         %User{} = target_user <- User.get_or_fetch_by_ap_id(target_instance),
+         {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance),
          {:ok, activity} <- ActivityPub.follow(local_user, target_user) do
       Logger.info("relay: followed instance: #{target_instance}; id=#{activity.data["id"]}")
       {:ok, activity}
@@ -28,7 +28,7 @@ def follow(target_instance) do
 
   def unfollow(target_instance) do
     with %User{} = local_user <- get_actor(),
-         %User{} = target_user <- User.get_or_fetch_by_ap_id(target_instance),
+         {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance),
          {:ok, activity} <- ActivityPub.unfollow(local_user, target_user) do
       Logger.info("relay: unfollowed instance: #{target_instance}: id=#{activity.data["id"]}")
       {:ok, activity}

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -126,7 +126,7 @@ def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collec
   def fix_implicit_addressing(object, _), do: object
 
   def fix_addressing(object) do
-    %User{} = user = User.get_or_fetch_by_ap_id(object["actor"])
+    {:ok, %User{} = user} = User.get_or_fetch_by_ap_id(object["actor"])
     followers_collection = User.ap_followers(user)
 
     object
@@ -407,7 +407,7 @@ def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = obj
       |> fix_addressing
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
-         %User{} = user <- User.get_or_fetch_by_ap_id(data["actor"]) do
+         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do
       object = fix_object(data["object"])
 
       params = %{
@@ -436,22 +436,48 @@ def handle_incoming(
         %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data
       ) do
     with %User{local: true} = followed <- User.get_cached_by_ap_id(followed),
-         %User{} = follower <- User.get_or_fetch_by_ap_id(follower),
+         {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower),
          {:ok, activity} <- ActivityPub.follow(follower, followed, id, false) do
-      if not User.locked?(followed) do
+      with deny_follow_blocked <- Pleroma.Config.get([:user, :deny_follow_blocked]),
+           {:user_blocked, false} <-
+             {:user_blocked, User.blocks?(followed, follower) && deny_follow_blocked},
+           {:user_locked, false} <- {:user_locked, User.locked?(followed)},
+           {:follow, {:ok, follower}} <- {:follow, User.follow(follower, followed)} do
         ActivityPub.accept(%{
           to: [follower.ap_id],
           actor: followed,
           object: data,
           local: true
         })
+      else
+        {:user_blocked, true} ->
+          {:ok, _} = Utils.update_follow_state(activity, "reject")
 
-        User.follow(follower, followed)
+          ActivityPub.reject(%{
+            to: [follower.ap_id],
+            actor: followed,
+            object: data,
+            local: true
+          })
+
+        {:follow, {:error, _}} ->
+          {:ok, _} = Utils.update_follow_state(activity, "reject")
+
+          ActivityPub.reject(%{
+            to: [follower.ap_id],
+            actor: followed,
+            object: data,
+            local: true
+          })
+
+        {:user_locked, true} ->
+          :noop
       end
 
       {:ok, activity}
     else
-      _e -> :error
+      _e ->
+        :error
     end
   end
 
@@ -459,7 +485,7 @@ def handle_incoming(
         %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
@@ -485,7 +511,7 @@ def handle_incoming(
         %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
@@ -509,7 +535,7 @@ def handle_incoming(
         %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do
       {:ok, activity}
@@ -522,7 +548,7 @@ def handle_incoming(
         %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          public <- Visibility.is_public?(data),
          {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do
@@ -577,7 +603,7 @@ def handle_incoming(
     object_id = Utils.get_ap_id(object_id)
 
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          :ok <- Containment.contain_origin(actor.ap_id, object.data),
          {:ok, activity} <- ActivityPub.delete(object, false) do
@@ -596,7 +622,7 @@ def handle_incoming(
         } = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do
       {:ok, activity}
@@ -614,7 +640,7 @@ def handle_incoming(
         } = _data
       ) do
     with %User{local: true} = followed <- User.get_cached_by_ap_id(followed),
-         %User{} = follower <- User.get_or_fetch_by_ap_id(follower),
+         {:ok, %User{} = follower} <- User.get_or_fetch_by_ap_id(follower),
          {:ok, activity} <- ActivityPub.unfollow(follower, followed, id, false) do
       User.unfollow(follower, followed)
       {:ok, activity}
@@ -633,7 +659,7 @@ def handle_incoming(
       ) do
     with true <- Pleroma.Config.get([:activitypub, :accept_blocks]),
          %User{local: true} = blocked <- User.get_cached_by_ap_id(blocked),
-         %User{} = blocker <- User.get_or_fetch_by_ap_id(blocker),
+         {:ok, %User{} = blocker} <- User.get_or_fetch_by_ap_id(blocker),
          {:ok, activity} <- ActivityPub.unblock(blocker, blocked, id, false) do
       User.unblock(blocker, blocked)
       {:ok, activity}
@@ -647,7 +673,7 @@ def handle_incoming(
       ) do
     with true <- Pleroma.Config.get([:activitypub, :accept_blocks]),
          %User{local: true} = blocked = User.get_cached_by_ap_id(blocked),
-         %User{} = blocker = User.get_or_fetch_by_ap_id(blocker),
+         {:ok, %User{} = blocker} = User.get_or_fetch_by_ap_id(blocker),
          {:ok, activity} <- ActivityPub.block(blocker, blocked, id, false) do
       User.unfollow(blocker, blocked)
       User.block(blocker, blocked)
@@ -666,7 +692,7 @@ def handle_incoming(
         } = data
       ) do
     with actor <- Containment.get_actor(data),
-         %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do
       {:ok, activity}
@@ -830,10 +856,16 @@ def add_mention_tags(object) do
     |> Map.put("tag", tags ++ mentions)
   end
 
+  def add_emoji_tags(%User{info: %{"emoji" => _emoji} = user_info} = object) do
+    user_info = add_emoji_tags(user_info)
+
+    object
+    |> Map.put(:info, user_info)
+  end
+
   # TODO: we should probably send mtime instead of unix epoch time for updated
-  def add_emoji_tags(object) do
+  def add_emoji_tags(%{"emoji" => emoji} = object) do
     tags = object["tag"] || []
-    emoji = object["emoji"] || []
 
     out =
       emoji
@@ -851,6 +883,10 @@ def add_emoji_tags(object) do
     |> Map.put("tag", tags ++ out)
   end
 
+  def add_emoji_tags(object) do
+    object
+  end
+
   def set_conversation(object) do
     Map.put(object, "conversation", object["context"])
   end

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -69,6 +69,11 @@ def render("user.json", %{user: user}) do
 
     endpoints = render("endpoints.json", %{user: user})
 
+    user_tags =
+      user
+      |> Transmogrifier.add_emoji_tags()
+      |> Map.get("tag", [])
+
     %{
       "id" => user.ap_id,
       "type" => "Person",
@@ -87,7 +92,7 @@ def render("user.json", %{user: user}) do
         "publicKeyPem" => public_key
       },
       "endpoints" => endpoints,
-      "tag" => user.info.source_data["tag"] || []
+      "tag" => (user.info.source_data["tag"] || []) ++ user_tags
     }
     |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
     |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user))

lib/pleroma/web/auth/authenticator.ex

@@ -42,4 +42,30 @@ def oauth_consumer_template do
     implementation().oauth_consumer_template() ||
       Pleroma.Config.get([:auth, :oauth_consumer_template], "consumer.html")
   end
+
+  @doc "Gets user by nickname or email for auth."
+  @spec fetch_user(String.t()) :: User.t() | nil
+  def fetch_user(name) do
+    User.get_by_nickname_or_email(name)
+  end
+
+  # Gets name and password from conn
+  #
+  @spec fetch_credentials(Plug.Conn.t() | map()) ::
+          {:ok, {name :: any, password :: any}} | {:error, :invalid_credentials}
+  def fetch_credentials(%Plug.Conn{params: params} = _),
+    do: fetch_credentials(params)
+
+  def fetch_credentials(params) do
+    case params do
+      %{"authorization" => %{"name" => name, "password" => password}} ->
+        {:ok, {name, password}}
+
+      %{"grant_type" => "password", "username" => name, "password" => password} ->
+        {:ok, {name, password}}
+
+      _ ->
+        {:error, :invalid_credentials}
+    end
+  end
 end

lib/pleroma/web/auth/ldap_authenticator.ex

@@ -7,6 +7,9 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
 
   require Logger
 
+  import Pleroma.Web.Auth.Authenticator,
+    only: [fetch_credentials: 1, fetch_user: 1]
+
   @behaviour Pleroma.Web.Auth.Authenticator
   @base Pleroma.Web.Auth.PleromaAuthenticator
 
@@ -20,30 +23,20 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
   defdelegate oauth_consumer_template, to: @base
 
   def get_user(%Plug.Conn{} = conn) do
-    if Pleroma.Config.get([:ldap, :enabled]) do
+    with {:ldap, true} <- {:ldap, Pleroma.Config.get([:ldap, :enabled])},
-      {name, password} =
+         {:ok, {name, password}} <- fetch_credentials(conn),
-        case conn.params do
+         %User{} = user <- ldap_user(name, password) do
-          %{"authorization" => %{"name" => name, "password" => password}} ->
+      {:ok, user}
-            {name, password}
-
-          %{"grant_type" => "password", "username" => name, "password" => password} ->
-            {name, password}
-        end
-
-      case ldap_user(name, password) do
-        %User{} = user ->
-          {:ok, user}
-
-        {:error, {:ldap_connection_error, _}} ->
-          # When LDAP is unavailable, try default authenticator
-          @base.get_user(conn)
-
-        error ->
-          error
-      end
     else
-      # Fall back to default authenticator
+      {:error, {:ldap_connection_error, _}} ->
-      @base.get_user(conn)
+        # When LDAP is unavailable, try default authenticator
+        @base.get_user(conn)
+
+      {:ldap, _} ->
+        @base.get_user(conn)
+
+      error ->
+        error
     end
   end
 
@@ -94,7 +87,7 @@ defp bind_user(connection, ldap, name, password) do
 
     case :eldap.simple_bind(connection, "#{uid}=#{name},#{base}", password) do
       :ok ->
-        case User.get_by_nickname_or_email(name) do
+        case fetch_user(name) do
           %User{} = user ->
             user
 

lib/pleroma/web/auth/pleroma_authenticator.ex

@@ -8,19 +8,14 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do
   alias Pleroma.Repo
   alias Pleroma.User
 
+  import Pleroma.Web.Auth.Authenticator,
+    only: [fetch_credentials: 1, fetch_user: 1]
+
   @behaviour Pleroma.Web.Auth.Authenticator
 
   def get_user(%Plug.Conn{} = conn) do
-    {name, password} =
+    with {:ok, {name, password}} <- fetch_credentials(conn),
-      case conn.params do
+         {_, %User{} = user} <- {:user, fetch_user(name)},
-        %{"authorization" => %{"name" => name, "password" => password}} ->
-          {name, password}
-
-        %{"grant_type" => "password", "username" => name, "password" => password} ->
-          {name, password}
-      end
-
-    with {_, %User{} = user} <- {:user, User.get_by_nickname_or_email(name)},
          {_, true} <- {:checkpw, Pbkdf2.checkpw(password, user.password_hash)} do
       {:ok, user}
     else

lib/pleroma/web/common_api/common_api.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Activity
+  alias Pleroma.Bookmark
   alias Pleroma.Formatter
   alias Pleroma.Object
   alias Pleroma.ThreadMute
@@ -150,8 +151,8 @@ def post(user, %{"status" => status} = data) do
            ),
          {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility),
          context <- make_context(in_reply_to),
-         cw <- data["spoiler_text"],
+         cw <- data["spoiler_text"] || "",
-         full_payload <- String.trim(status <> (data["spoiler_text"] || "")),
+         full_payload <- String.trim(status <> cw),
          length when length in 1..limit <- String.length(full_payload),
          object <-
            make_note_data(
@@ -169,10 +170,7 @@ def post(user, %{"status" => status} = data) do
            Map.put(
              object,
              "emoji",
-             (Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"]))
+             Formatter.get_emoji_map(full_payload)
-             |> Enum.reduce(%{}, fn {name, file, _}, acc ->
-               Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
-             end)
            ) do
       res =
         ActivityPub.create(
@@ -282,6 +280,15 @@ def thread_muted?(user, activity) do
     end
   end
 
+  def bookmarked?(user, activity) do
+    with %Bookmark{} <- Bookmark.get(user.id, activity.id) do
+      true
+    else
+      _ ->
+        false
+    end
+  end
+
   def report(user, data) do
     with {:account_id, %{"account_id" => account_id}} <- {:account_id, data},
          {:account, %User{} = account} <- {:account, User.get_cached_by_id(account_id)},

lib/pleroma/web/common_api/utils.ex

@@ -182,6 +182,18 @@ def format_input(text, "text/plain", options) do
         end).()
   end
 
+  @doc """
+  Formatting text as BBCode.
+  """
+  def format_input(text, "text/bbcode", options) do
+    text
+    |> String.replace(~r/\r/, "")
+    |> Formatter.html_escape("text/plain")
+    |> BBCode.to_html()
+    |> (fn {:ok, html} -> html end).()
+    |> Formatter.linkify(options)
+  end
+
   @doc """
   Formatting text to html.
   """

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -6,8 +6,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   use Pleroma.Web, :controller
   alias Ecto.Changeset
   alias Pleroma.Activity
+  alias Pleroma.Bookmark
   alias Pleroma.Config
   alias Pleroma.Filter
+  alias Pleroma.Formatter
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
@@ -35,7 +37,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
 
-  import Pleroma.Web.ControllerHelper, only: [oauth_scopes: 2]
+  alias Pleroma.Web.ControllerHelper
   import Ecto.Query
 
   require Logger
@@ -46,7 +48,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   action_fallback(:errors)
 
   def create_app(conn, params) do
-    scopes = oauth_scopes(params, ["read"])
+    scopes = ControllerHelper.oauth_scopes(params, ["read"])
 
     app_attrs =
       params
@@ -85,7 +87,7 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do
     user_params =
       %{}
       |> add_if_present(params, "display_name", :name)
-      |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value)} end)
+      |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value, user)} end)
       |> add_if_present(params, "avatar", :avatar, fn value ->
         with %Plug.Upload{} <- value,
              {:ok, object} <- ActivityPub.upload(value, type: :avatar) do
@@ -95,9 +97,20 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do
         end
       end)
 
+    emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "")
+
+    user_info_emojis =
+      ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text))
+      |> Enum.dedup()
+
     info_params =
-      %{}
+      [:no_rich_text, :locked, :hide_followers, :hide_follows, :hide_favorites, :show_role]
-      |> add_if_present(params, "locked", :locked, fn value -> {:ok, value == "true"} end)
+      |> Enum.reduce(%{}, fn key, acc ->
+        add_if_present(acc, params, to_string(key), key, fn value ->
+          {:ok, ControllerHelper.truthy_param?(value)}
+        end)
+      end)
+      |> add_if_present(params, "default_scope", :default_scope)
       |> add_if_present(params, "header", :banner, fn value ->
         with %Plug.Upload{} <- value,
              {:ok, object} <- ActivityPub.upload(value, type: :banner) do
@@ -106,8 +119,9 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do
           _ -> :error
         end
       end)
+      |> Map.put(:emoji, user_info_emojis)
 
-    info_cng = User.Info.mastodon_profile_update(user.info, info_params)
+    info_cng = User.Info.profile_update(user.info, info_params)
 
     with changeset <- User.update_changeset(user, user_params),
          changeset <- Ecto.Changeset.put_embed(changeset, :info, info_cng),
@@ -279,6 +293,8 @@ def home_timeline(%{assigns: %{user: user}} = conn, params) do
       |> ActivityPub.contain_timeline(user)
       |> Enum.reverse()
 
+    user = Repo.preload(user, bookmarks: :activity)
+
     conn
     |> add_link_headers(:home_timeline, activities)
     |> put_view(StatusView)
@@ -297,6 +313,8 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do
       |> ActivityPub.fetch_public_activities()
       |> Enum.reverse()
 
+    user = Repo.preload(user, bookmarks: :activity)
+
     conn
     |> add_link_headers(:public_timeline, activities, false, %{"local" => local_only})
     |> put_view(StatusView)
@@ -304,7 +322,8 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do
   end
 
   def user_statuses(%{assigns: %{user: reading_user}} = conn, params) do
-    with %User{} = user <- User.get_cached_by_id(params["id"]) do
+    with %User{} = user <- User.get_cached_by_id(params["id"]),
+         reading_user <- Repo.preload(reading_user, :bookmarks) do
       activities = ActivityPub.fetch_user_activities(user, reading_user, params)
 
       conn
@@ -331,6 +350,8 @@ def dm_timeline(%{assigns: %{user: user}} = conn, params) do
       |> ActivityPub.fetch_activities_query(params)
       |> Pagination.fetch_paginated(params)
 
+    user = Repo.preload(user, bookmarks: :activity)
+
     conn
     |> add_link_headers(:dm_timeline, activities)
     |> put_view(StatusView)
@@ -340,6 +361,8 @@ def dm_timeline(%{assigns: %{user: user}} = conn, params) do
   def get_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{} = activity <- Activity.get_by_id_with_object(id),
          true <- Visibility.visible_for_user?(activity, user) do
+      user = Repo.preload(user, bookmarks: :activity)
+
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user})
@@ -489,6 +512,8 @@ def delete_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
   def reblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, announce, _activity} <- CommonAPI.repeat(ap_id_or_id, user),
          %Activity{} = announce <- Activity.normalize(announce.data) do
+      user = Repo.preload(user, bookmarks: :activity)
+
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: announce, for: user, as: :activity})
@@ -498,6 +523,8 @@ def reblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
   def unreblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, _unannounce, %{data: %{"id" => id}}} <- CommonAPI.unrepeat(ap_id_or_id, user),
          %Activity{} = activity <- Activity.get_create_by_object_ap_id_with_object(id) do
+      user = Repo.preload(user, bookmarks: :activity)
+
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -545,10 +572,11 @@ def unpin_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
 
   def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{} = activity <- Activity.get_by_id_with_object(id),
-         %Object{} = object <- Object.normalize(activity),
          %User{} = user <- User.get_cached_by_nickname(user.nickname),
          true <- Visibility.visible_for_user?(activity, user),
-         {:ok, user} <- User.bookmark(user, object.data["id"]) do
+         {:ok, _bookmark} <- Bookmark.create(user.id, activity.id) do
+      user = Repo.preload(user, bookmarks: :activity)
+
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -557,10 +585,11 @@ def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
 
   def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{} = activity <- Activity.get_by_id_with_object(id),
-         %Object{} = object <- Object.normalize(activity),
          %User{} = user <- User.get_cached_by_nickname(user.nickname),
          true <- Visibility.visible_for_user?(activity, user),
-         {:ok, user} <- User.unbookmark(user, object.data["id"]) do
+         {:ok, _bookmark} <- Bookmark.destroy(user.id, activity.id) do
+      user = Repo.preload(user, bookmarks: :activity)
+
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -683,7 +712,7 @@ def upload(%{assigns: %{user: user}} = conn, %{"file" => file} = data) do
     end
   end
 
-  def favourited_by(conn, %{"id" => id}) do
+  def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
          %Object{data: %{"likes" => likes}} <- Object.normalize(object) do
       q = from(u in User, where: u.ap_id in ^likes)
@@ -691,13 +720,13 @@ def favourited_by(conn, %{"id" => id}) do
 
       conn
       |> put_view(AccountView)
-      |> render(AccountView, "accounts.json", %{users: users, as: :user})
+      |> render(AccountView, "accounts.json", %{for: user, users: users, as: :user})
     else
       _ -> json(conn, [])
     end
   end
 
-  def reblogged_by(conn, %{"id" => id}) do
+  def reblogged_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
          %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do
       q = from(u in User, where: u.ap_id in ^announces)
@@ -705,7 +734,7 @@ def reblogged_by(conn, %{"id" => id}) do
 
       conn
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: users, as: :user})
+      |> render("accounts.json", %{for: user, users: users, as: :user})
     else
       _ -> json(conn, [])
     end
@@ -762,7 +791,7 @@ def followers(%{assigns: %{user: for_user}} = conn, %{"id" => id} = params) do
       conn
       |> add_link_headers(:followers, followers, user)
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: followers, as: :user})
+      |> render("accounts.json", %{for: for_user, users: followers, as: :user})
     end
   end
 
@@ -779,7 +808,7 @@ def following(%{assigns: %{user: for_user}} = conn, %{"id" => id} = params) do
       conn
       |> add_link_headers(:following, followers, user)
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: followers, as: :user})
+      |> render("accounts.json", %{for: for_user, users: followers, as: :user})
     end
   end
 
@@ -787,7 +816,7 @@ def follow_requests(%{assigns: %{user: followed}} = conn, _params) do
     with {:ok, follow_requests} <- User.get_follow_requests(followed) do
       conn
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: follow_requests, as: :user})
+      |> render("accounts.json", %{for: followed, users: follow_requests, as: :user})
     end
   end
 
@@ -1081,6 +1110,8 @@ def favourites(%{assigns: %{user: user}} = conn, params) do
       ActivityPub.fetch_activities([], params)
       |> Enum.reverse()
 
+    user = Repo.preload(user, bookmarks: :activity)
+
     conn
     |> add_link_headers(:favourites, activities)
     |> put_view(StatusView)
@@ -1124,15 +1155,20 @@ def user_favourites(%{assigns: %{user: for_user}} = conn, %{"id" => id} = params
     end
   end
 
-  def bookmarks(%{assigns: %{user: user}} = conn, _) do
+  def bookmarks(%{assigns: %{user: user}} = conn, params) do
     user = User.get_cached_by_id(user.id)
+    user = Repo.preload(user, bookmarks: :activity)
+
+    bookmarks =
+      Bookmark.for_user_query(user.id)
+      |> Pagination.fetch_paginated(params)
 
     activities =
-      user.bookmarks
+      bookmarks
-      |> Enum.map(fn id -> Activity.get_create_by_object_ap_id(id) end)
+      |> Enum.map(fn b -> b.activity end)
-      |> Enum.reverse()
 
     conn
+    |> add_link_headers(:bookmarks, bookmarks)
     |> put_view(StatusView)
     |> render("index.json", %{activities: activities, for: user, as: :activity})
   end
@@ -1207,7 +1243,7 @@ def list_accounts(%{assigns: %{user: user}} = conn, %{"id" => id}) do
          {:ok, users} = Pleroma.List.get_following(list) do
       conn
       |> put_view(AccountView)
-      |> render("accounts.json", %{users: users, as: :user})
+      |> render("accounts.json", %{for: user, users: users, as: :user})
     end
   end
 
@@ -1238,6 +1274,8 @@ def list_timeline(%{assigns: %{user: user}} = conn, %{"list_id" => id} = params)
         |> ActivityPub.fetch_activities_bounded(following, params)
         |> Enum.reverse()
 
+      user = Repo.preload(user, bookmarks: :activity)
+
       conn
       |> put_view(StatusView)
       |> render("index.json", %{activities: activities, for: user, as: :activity})
@@ -1265,8 +1303,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do
       initial_state =
         %{
           meta: %{
-            streaming_api_base_url:
+            streaming_api_base_url: Pleroma.Web.Endpoint.websocket_url(),
-              String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"),
             access_token: token,
             locale: "en",
             domain: Pleroma.Web.Endpoint.host(),
@@ -1623,7 +1660,7 @@ def suggestions(%{assigns: %{user: user}} = conn, _) do
               x,
               "id",
               case User.get_or_fetch(x["acct"]) do
-                %{id: id} -> id
+                {:ok, %User{id: id}} -> id
                 _ -> 0
               end
             )

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -113,21 +113,23 @@ defp do_render("account.json", %{user: user} = opts) do
       bot: bot,
       source: %{
         note: "",
-        privacy: user_info.default_scope,
+        sensitive: false,
-        sensitive: false
+        pleroma: %{}
       },
 
       # Pleroma extension
-      pleroma:
+      pleroma: %{
-        %{
+        confirmation_pending: user_info.confirmation_pending,
-          confirmation_pending: user_info.confirmation_pending,
+        tags: user.tags,
-          tags: user.tags,
+        hide_followers: user.info.hide_followers,
-          is_moderator: user.info.is_moderator,
+        hide_follows: user.info.hide_follows,
-          is_admin: user.info.is_admin,
+        hide_favorites: user.info.hide_favorites,
-          relationship: relationship
+        relationship: relationship
-        }
+      }
-        |> with_notification_settings(user, opts[:for])
     }
+    |> maybe_put_role(user, opts[:for])
+    |> maybe_put_settings(user, opts[:for], user_info)
+    |> maybe_put_notification_settings(user, opts[:for])
   end
 
   defp username_from_nickname(string) when is_binary(string) do
@@ -136,9 +138,37 @@ defp username_from_nickname(string) when is_binary(string) do
 
   defp username_from_nickname(_), do: nil
 
-  defp with_notification_settings(data, %User{id: user_id} = user, %User{id: user_id}) do
+  defp maybe_put_settings(
-    Map.put(data, :notification_settings, user.info.notification_settings)
+         data,
+         %User{id: user_id} = user,
+         %User{id: user_id},
+         user_info
+       ) do
+    data
+    |> Kernel.put_in([:source, :privacy], user_info.default_scope)
+    |> Kernel.put_in([:source, :pleroma, :show_role], user.info.show_role)
+    |> Kernel.put_in([:source, :pleroma, :no_rich_text], user.info.no_rich_text)
   end
 
-  defp with_notification_settings(data, _, _), do: data
+  defp maybe_put_settings(data, _, _, _), do: data
+
+  defp maybe_put_role(data, %User{info: %{show_role: true}} = user, _) do
+    data
+    |> Kernel.put_in([:pleroma, :is_admin], user.info.is_admin)
+    |> Kernel.put_in([:pleroma, :is_moderator], user.info.is_moderator)
+  end
+
+  defp maybe_put_role(data, %User{id: user_id} = user, %User{id: user_id}) do
+    data
+    |> Kernel.put_in([:pleroma, :is_admin], user.info.is_admin)
+    |> Kernel.put_in([:pleroma, :is_moderator], user.info.is_moderator)
+  end
+
+  defp maybe_put_role(data, _, _), do: data
+
+  defp maybe_put_notification_settings(data, %User{id: user_id} = user, %User{id: user_id}) do
+    Kernel.put_in(data, [:pleroma, :notification_settings], user.info.notification_settings)
+  end
+
+  defp maybe_put_notification_settings(data, _, _), do: data
 end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -85,7 +85,8 @@ def render(
 
     activity_object = Object.normalize(activity)
     favorited = opts[:for] && opts[:for].ap_id in (activity_object.data["likes"] || [])
-    bookmarked = opts[:for] && activity_object.data["id"] in opts[:for].bookmarks
+
+    bookmarked = opts[:for] && CommonAPI.bookmarked?(opts[:for], reblogged_activity)
 
     mentions =
       activity.recipients
@@ -148,7 +149,7 @@ def render("status.json", %{activity: %{data: %{"object" => _object}} = activity
 
     favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || [])
 
-    bookmarked = opts[:for] && object.data["id"] in opts[:for].bookmarks
+    bookmarked = opts[:for] && CommonAPI.bookmarked?(opts[:for], activity)
 
     attachment_data = object.data["attachment"] || []
     attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)

lib/pleroma/web/media_proxy/media_proxy.ex

@@ -13,32 +13,44 @@ def url("/" <> _ = url), do: url
 
   def url(url) do
     config = Application.get_env(:pleroma, :media_proxy, [])
+    domain = URI.parse(url).host
 
-    if !Keyword.get(config, :enabled, false) or String.starts_with?(url, Pleroma.Web.base_url()) do
+    cond do
-      url
+      !Keyword.get(config, :enabled, false) or String.starts_with?(url, Pleroma.Web.base_url()) ->
-    else
-      secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
-
-      # Must preserve `%2F` for compatibility with S3
-      # https://git.pleroma.social/pleroma/pleroma/issues/580
-      replacement = get_replacement(url, ":2F:")
-
-      # The URL is url-decoded and encoded again to ensure it is correctly encoded and not twice.
-      base64 =
         url
-        |> String.replace("%2F", replacement)
-        |> URI.decode()
-        |> URI.encode()
-        |> String.replace(replacement, "%2F")
-        |> Base.url_encode64(@base64_opts)
 
-      sig = :crypto.hmac(:sha, secret, base64)
+      Enum.any?(Pleroma.Config.get([:media_proxy, :whitelist]), fn pattern ->
-      sig64 = sig |> Base.url_encode64(@base64_opts)
+        String.equivalent?(domain, pattern)
+      end) ->
+        url
 
-      build_url(sig64, base64, filename(url))
+      true ->
+        encode_url(url)
     end
   end
 
+  def encode_url(url) do
+    secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
+
+    # Must preserve `%2F` for compatibility with S3
+    # https://git.pleroma.social/pleroma/pleroma/issues/580
+    replacement = get_replacement(url, ":2F:")
+
+    # The URL is url-decoded and encoded again to ensure it is correctly encoded and not twice.
+    base64 =
+      url
+      |> String.replace("%2F", replacement)
+      |> URI.decode()
+      |> URI.encode()
+      |> String.replace(replacement, "%2F")
+      |> Base.url_encode64(@base64_opts)
+
+    sig = :crypto.hmac(:sha, secret, base64)
+    sig64 = sig |> Base.url_encode64(@base64_opts)
+
+    build_url(sig64, base64, filename(url))
+  end
+
   def decode_url(sig, url) do
     secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
     sig = Base.url_decode64!(sig, @base64_opts)

lib/pleroma/web/oauth/app.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.App do
   use Ecto.Schema
   import Ecto.Changeset
 
+  @type t :: %__MODULE__{}
   schema "apps" do
     field(:client_name, :string)
     field(:redirect_uris, :string)

lib/pleroma/web/oauth/authorization.ex

@@ -13,6 +13,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
   import Ecto.Changeset
   import Ecto.Query
 
+  @type t :: %__MODULE__{}
   schema "oauth_authorizations" do
     field(:token, :string)
     field(:scopes, {:array, :string}, default: [])
@@ -63,4 +64,11 @@ def delete_user_authorizations(%User{id: user_id}) do
     )
     |> Repo.delete_all()
   end
+
+  @doc "gets auth for app by token"
+  @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
+  def get_by_token(%App{id: app_id} = _app, token) do
+    from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token)
+    |> Repo.find_resource()
+  end
 end

lib/pleroma/web/oauth/oauth_controller.ex

@@ -13,11 +13,15 @@ defmodule Pleroma.Web.OAuth.OAuthController do
   alias Pleroma.Web.OAuth.App
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.Token.Strategy.RefreshToken
+  alias Pleroma.Web.OAuth.Token.Strategy.Revoke, as: RevokeToken
 
   import Pleroma.Web.ControllerHelper, only: [oauth_scopes: 2]
 
   if Pleroma.Config.oauth_consumer_enabled?(), do: plug(Ueberauth)
 
+  @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
+
   plug(:fetch_session)
   plug(:fetch_flash)
 
@@ -138,25 +142,33 @@ defp handle_create_authorization_error(conn, error, %{"authorization" => _}) do
     Authenticator.handle_error(conn, error)
   end
 
+  @doc "Renew access_token with refresh_token"
+  def token_exchange(
+        conn,
+        %{"grant_type" => "refresh_token", "refresh_token" => token} = params
+      ) do
+    with %App{} = app <- get_app_from_request(conn, params),
+         {:ok, %{user: user} = token} <- Token.get_by_refresh_token(app, token),
+         {:ok, token} <- RefreshToken.grant(token) do
+      response_attrs = %{created_at: Token.Utils.format_created_at(token)}
+
+      json(conn, response_token(user, token, response_attrs))
+    else
+      _error ->
+        put_status(conn, 400)
+        |> json(%{error: "Invalid credentials"})
+    end
+  end
+
   def token_exchange(conn, %{"grant_type" => "authorization_code"} = params) do
     with %App{} = app <- get_app_from_request(conn, params),
-         fixed_token = fix_padding(params["code"]),
+         fixed_token = Token.Utils.fix_padding(params["code"]),
-         %Authorization{} = auth <-
+         {:ok, auth} <- Authorization.get_by_token(app, fixed_token),
-           Repo.get_by(Authorization, token: fixed_token, app_id: app.id),
          %User{} = user <- User.get_cached_by_id(auth.user_id),
-         {:ok, token} <- Token.exchange_token(app, auth),
+         {:ok, token} <- Token.exchange_token(app, auth) do
-         {:ok, inserted_at} <- DateTime.from_naive(token.inserted_at, "Etc/UTC") do
+      response_attrs = %{created_at: Token.Utils.format_created_at(token)}
-      response = %{
-        token_type: "Bearer",
-        access_token: token.token,
-        refresh_token: token.refresh_token,
-        created_at: DateTime.to_unix(inserted_at),
-        expires_in: 60 * 10,
-        scope: Enum.join(token.scopes, " "),
-        me: user.ap_id
-      }
 
-      json(conn, response)
+      json(conn, response_token(user, token, response_attrs))
     else
       _error ->
         put_status(conn, 400)
@@ -177,16 +189,7 @@ def token_exchange(
          true <- Enum.any?(scopes),
          {:ok, auth} <- Authorization.create_authorization(app, user, scopes),
          {:ok, token} <- Token.exchange_token(app, auth) do
-      response = %{
+      json(conn, response_token(user, token))
-        token_type: "Bearer",
-        access_token: token.token,
-        refresh_token: token.refresh_token,
-        expires_in: 60 * 10,
-        scope: Enum.join(token.scopes, " "),
-        me: user.ap_id
-      }
-
-      json(conn, response)
     else
       {:auth_active, false} ->
         # Per https://github.com/tootsuite/mastodon/blob/
@@ -218,10 +221,12 @@ def token_exchange(
     token_exchange(conn, params)
   end
 
-  def token_revoke(conn, %{"token" => token} = params) do
+  # Bad request
+  def token_exchange(conn, params), do: bad_request(conn, params)
+
+  def token_revoke(conn, %{"token" => _token} = params) do
     with %App{} = app <- get_app_from_request(conn, params),
-         %Token{} = token <- Repo.get_by(Token, token: token, app_id: app.id),
+         {:ok, _token} <- RevokeToken.revoke(app, params) do
-         {:ok, %Token{}} <- Repo.delete(token) do
       json(conn, %{})
     else
       _error ->
@@ -230,6 +235,15 @@ def token_revoke(conn, %{"token" => token} = params) do
     end
   end
 
+  def token_revoke(conn, params), do: bad_request(conn, params)
+
+  # Response for bad request
+  defp bad_request(conn, _) do
+    conn
+    |> put_status(500)
+    |> json(%{error: "Bad request"})
+  end
+
   @doc "Prepares OAuth request to provider for Ueberauth"
   def prepare_request(conn, %{"provider" => provider, "authorization" => auth_attrs}) do
     scope =
@@ -278,25 +292,22 @@ def callback(conn, params) do
     params = callback_params(params)
 
     with {:ok, registration} <- Authenticator.get_registration(conn) do
-      user = Repo.preload(registration, :user).user
       auth_attrs = Map.take(params, ~w(client_id redirect_uri scope scopes state))
 
-      if user do
+      case Repo.get_assoc(registration, :user) do
-        create_authorization(
+        {:ok, user} ->
-          conn,
+          create_authorization(conn, %{"authorization" => auth_attrs}, user: user)
-          %{"authorization" => auth_attrs},
-          user: user
-        )
-      else
-        registration_params =
-          Map.merge(auth_attrs, %{
-            "nickname" => Registration.nickname(registration),
-            "email" => Registration.email(registration)
-          })
 
-        conn
+        _ ->
-        |> put_session(:registration_id, registration.id)
+          registration_params =
-        |> registration_details(%{"authorization" => registration_params})
+            Map.merge(auth_attrs, %{
+              "nickname" => Registration.nickname(registration),
+              "email" => Registration.email(registration)
+            })
+
+          conn
+          |> put_session(:registration_id, registration.id)
+          |> registration_details(%{"authorization" => registration_params})
       end
     else
       _ ->
@@ -399,36 +410,30 @@ defp do_create_authorization(
     end
   end
 
-  # XXX - for whatever reason our token arrives urlencoded, but Plug.Conn should be
+  defp get_app_from_request(conn, params) do
-  # decoding it.  Investigate sometime.
+    conn
-  defp fix_padding(token) do
+    |> fetch_client_credentials(params)
-    token
+    |> fetch_client
-    |> URI.decode()
-    |> Base.url_decode64!(padding: false)
-    |> Base.url_encode64(padding: false)
   end
 
-  defp get_app_from_request(conn, params) do
+  defp fetch_client({id, secret}) when is_binary(id) and is_binary(secret) do
-    # Per RFC 6749, HTTP Basic is preferred to body params
+    Repo.get_by(App, client_id: id, client_secret: secret)
-    {client_id, client_secret} =
+  end
-      with ["Basic " <> encoded] <- get_req_header(conn, "authorization"),
-           {:ok, decoded} <- Base.decode64(encoded),
-           [id, secret] <-
-             String.split(decoded, ":")
-             |> Enum.map(fn s -> URI.decode_www_form(s) end) do
-        {id, secret}
-      else
-        _ -> {params["client_id"], params["client_secret"]}
-      end
 
-    if client_id && client_secret do
+  defp fetch_client({_id, _secret}), do: nil
-      Repo.get_by(
+
-        App,
+  defp fetch_client_credentials(conn, params) do
-        client_id: client_id,
+    # Per RFC 6749, HTTP Basic is preferred to body params
-        client_secret: client_secret
+    with ["Basic " <> encoded] <- get_req_header(conn, "authorization"),
-      )
+         {:ok, decoded} <- Base.decode64(encoded),
+         [id, secret] <-
+           Enum.map(
+             String.split(decoded, ":"),
+             fn s -> URI.decode_www_form(s) end
+           ) do
+      {id, secret}
     else
-      nil
+      _ -> {params["client_id"], params["client_secret"]}
     end
   end
 
@@ -441,4 +446,16 @@ defp get_session_registration_id(conn), do: get_session(conn, :registration_id)
 
   defp put_session_registration_id(conn, registration_id),
     do: put_session(conn, :registration_id, registration_id)
+
+  defp response_token(%User{} = user, token, opts \\ %{}) do
+    %{
+      token_type: "Bearer",
+      access_token: token.token,
+      refresh_token: token.refresh_token,
+      expires_in: @expires_in,
+      scope: Enum.join(token.scopes, " "),
+      me: user.ap_id
+    }
+    |> Map.merge(opts)
+  end
 end

lib/pleroma/web/oauth/token.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.OAuth.Token do
   use Ecto.Schema
 
   import Ecto.Query
+  import Ecto.Changeset
 
   alias Pleroma.Repo
   alias Pleroma.User
@@ -13,6 +14,9 @@ defmodule Pleroma.Web.OAuth.Token do
   alias Pleroma.Web.OAuth.Authorization
   alias Pleroma.Web.OAuth.Token
 
+  @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
+  @type t :: %__MODULE__{}
+
   schema "oauth_tokens" do
     field(:token, :string)
     field(:refresh_token, :string)
@@ -24,28 +28,67 @@ defmodule Pleroma.Web.OAuth.Token do
     timestamps()
   end
 
+  @doc "Gets token for app by access token"
+  @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
+  def get_by_token(%App{id: app_id} = _app, token) do
+    from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token)
+    |> Repo.find_resource()
+  end
+
+  @doc "Gets token for app by refresh token"
+  @spec get_by_refresh_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
+  def get_by_refresh_token(%App{id: app_id} = _app, token) do
+    from(t in __MODULE__,
+      where: t.app_id == ^app_id and t.refresh_token == ^token,
+      preload: [:user]
+    )
+    |> Repo.find_resource()
+  end
+
   def exchange_token(app, auth) do
     with {:ok, auth} <- Authorization.use_token(auth),
          true <- auth.app_id == app.id do
-      create_token(app, User.get_cached_by_id(auth.user_id), auth.scopes)
+      create_token(
+        app,
+        User.get_cached_by_id(auth.user_id),
+        %{scopes: auth.scopes}
+      )
     end
   end
 
-  def create_token(%App{} = app, %User{} = user, scopes \\ nil) do
+  defp put_token(changeset) do
-    scopes = scopes || app.scopes
+    changeset
-    token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+    |> change(%{token: Token.Utils.generate_token()})
-    refresh_token = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+    |> validate_required([:token])
+    |> unique_constraint(:token)
+  end
 
-    token = %Token{
+  defp put_refresh_token(changeset, attrs) do
-      token: token,
+    refresh_token = Map.get(attrs, :refresh_token, Token.Utils.generate_token())
-      refresh_token: refresh_token,
-      scopes: scopes,
-      user_id: user.id,
-      app_id: app.id,
-      valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10)
-    }
 
-    Repo.insert(token)
+    changeset
+    |> change(%{refresh_token: refresh_token})
+    |> validate_required([:refresh_token])
+    |> unique_constraint(:refresh_token)
+  end
+
+  defp put_valid_until(changeset, attrs) do
+    expires_in =
+      Map.get(attrs, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), @expires_in))
+
+    changeset
+    |> change(%{valid_until: expires_in})
+    |> validate_required([:valid_until])
+  end
+
+  def create_token(%App{} = app, %User{} = user, attrs \\ %{}) do
+    %__MODULE__{user_id: user.id, app_id: app.id}
+    |> cast(%{scopes: attrs[:scopes] || app.scopes}, [:scopes])
+    |> validate_required([:scopes, :user_id, :app_id])
+    |> put_valid_until(attrs)
+    |> put_token
+    |> put_refresh_token(attrs)
+    |> Repo.insert()
   end
 
   def delete_user_tokens(%User{id: user_id}) do
@@ -73,4 +116,10 @@ def get_user_tokens(%User{id: user_id}) do
     |> Repo.all()
     |> Repo.preload(:app)
   end
+
+  def is_expired?(%__MODULE__{valid_until: valid_until}) do
+    NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0
+  end
+
+  def is_expired?(_), do: false
 end

lib/pleroma/web/oauth/token/strategy/refresh_token.ex

@@ -0,0 +1,54 @@
+defmodule Pleroma.Web.OAuth.Token.Strategy.RefreshToken do
+  @moduledoc """
+  Functions for dealing with refresh token strategy.
+  """
+
+  alias Pleroma.Config
+  alias Pleroma.Repo
+  alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.Token.Strategy.Revoke
+
+  @doc """
+  Will grant access token by refresh token.
+  """
+  @spec grant(Token.t()) :: {:ok, Token.t()} | {:error, any()}
+  def grant(token) do
+    access_token = Repo.preload(token, [:user, :app])
+
+    result =
+      Repo.transaction(fn ->
+        token_params = %{
+          app: access_token.app,
+          user: access_token.user,
+          scopes: access_token.scopes
+        }
+
+        access_token
+        |> revoke_access_token()
+        |> create_access_token(token_params)
+      end)
+
+    case result do
+      {:ok, {:error, reason}} -> {:error, reason}
+      {:ok, {:ok, token}} -> {:ok, token}
+      {:error, reason} -> {:error, reason}
+    end
+  end
+
+  defp revoke_access_token(token) do
+    Revoke.revoke(token)
+  end
+
+  defp create_access_token({:error, error}, _), do: {:error, error}
+
+  defp create_access_token({:ok, token}, %{app: app, user: user} = token_params) do
+    Token.create_token(app, user, add_refresh_token(token_params, token.refresh_token))
+  end
+
+  defp add_refresh_token(params, token) do
+    case Config.get([:oauth2, :issue_new_refresh_token], false) do
+      true -> Map.put(params, :refresh_token, token)
+      false -> params
+    end
+  end
+end

lib/pleroma/web/oauth/token/strategy/revoke.ex

@@ -0,0 +1,22 @@
+defmodule Pleroma.Web.OAuth.Token.Strategy.Revoke do
+  @moduledoc """
+  Functions for dealing with revocation.
+  """
+
+  alias Pleroma.Repo
+  alias Pleroma.Web.OAuth.App
+  alias Pleroma.Web.OAuth.Token
+
+  @doc "Finds and revokes access token for app and by token"
+  @spec revoke(App.t(), map()) :: {:ok, Token.t()} | {:error, :not_found | Ecto.Changeset.t()}
+  def revoke(%App{} = app, %{"token" => token} = _attrs) do
+    with {:ok, token} <- Token.get_by_token(app, token),
+         do: revoke(token)
+  end
+
+  @doc "Revokes access token"
+  @spec revoke(Token.t()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()}
+  def revoke(%Token{} = token) do
+    Repo.delete(token)
+  end
+end

lib/pleroma/web/oauth/token/utils.ex

@@ -0,0 +1,30 @@
+defmodule Pleroma.Web.OAuth.Token.Utils do
+  @moduledoc """
+  Auxiliary functions for dealing with tokens.
+  """
+
+  @doc "convert token inserted_at to unix timestamp"
+  def format_created_at(%{inserted_at: inserted_at} = _token) do
+    inserted_at
+    |> DateTime.from_naive!("Etc/UTC")
+    |> DateTime.to_unix()
+  end
+
+  @doc false
+  @spec generate_token(keyword()) :: binary()
+  def generate_token(opts \\ []) do
+    opts
+    |> Keyword.get(:size, 32)
+    |> :crypto.strong_rand_bytes()
+    |> Base.url_encode64(padding: false)
+  end
+
+  # XXX - for whatever reason our token arrives urlencoded, but Plug.Conn should be
+  # decoding it.  Investigate sometime.
+  def fix_padding(token) do
+    token
+    |> URI.decode()
+    |> Base.url_decode64!(padding: false)
+    |> Base.url_encode64(padding: false)
+  end
+end

lib/pleroma/web/push/impl.ex

@@ -21,8 +21,10 @@ defmodule Pleroma.Web.Push.Impl do
   @doc "Performs sending notifications for user subscriptions"
   @spec perform(Notification.t()) :: list(any) | :error
   def perform(
-        %{activity: %{data: %{"type" => activity_type}, id: activity_id}, user_id: user_id} =
+        %{
-          notif
+          activity: %{data: %{"type" => activity_type}, id: activity_id} = activity,
+          user_id: user_id
+        } = notif
       )
       when activity_type in @types do
     actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
@@ -30,13 +32,14 @@ def perform(
     type = Activity.mastodon_notification_type(notif.activity)
     gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
     avatar_url = User.avatar_url(actor)
+    object = Object.normalize(activity)
 
     for subscription <- fetch_subsriptions(user_id),
         get_in(subscription.data, ["alerts", type]) do
       %{
         title: format_title(notif),
         access_token: subscription.token.token,
-        body: format_body(notif, actor),
+        body: format_body(notif, actor, object),
         notification_id: notif.id,
         notification_type: type,
         icon: avatar_url,
@@ -95,25 +98,25 @@ def build_sub(subscription) do
   end
 
   def format_body(
-        %{activity: %{data: %{"type" => "Create", "object" => %{"content" => content}}}},
+        %{activity: %{data: %{"type" => "Create"}}},
-        actor
+        actor,
+        %{data: %{"content" => content}}
       ) do
     "@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
   end
 
   def format_body(
-        %{activity: %{data: %{"type" => "Announce", "object" => activity_id}}},
+        %{activity: %{data: %{"type" => "Announce"}}},
-        actor
+        actor,
+        %{data: %{"content" => content}}
       ) do
-    %Activity{data: %{"object" => %{"id" => object_id}}} = Activity.get_by_ap_id(activity_id)
-    %Object{data: %{"content" => content}} = Object.get_by_ap_id(object_id)
-
     "@#{actor.nickname} repeated: #{Utils.scrub_html_and_truncate(content, 80)}"
   end
 
   def format_body(
         %{activity: %{data: %{"type" => type}}},
-        actor
+        actor,
+        _object
       )
       when type in ["Follow", "Like"] do
     case type do

lib/pleroma/web/twitter_api/controllers/util_controller.ex

@@ -352,7 +352,7 @@ def change_password(%{assigns: %{user: user}} = conn, params) do
   def delete_account(%{assigns: %{user: user}} = conn, params) do
     case CommonAPI.Utils.confirm_current_password(user, params["password"]) do
       {:ok, user} ->
-        Task.start(fn -> User.delete(user) end)
+        User.delete(user)
         json(conn, %{status: "success"})
 
       {:error, msg} ->

lib/pleroma/web/twitter_api/twitter_api.ex

@@ -296,7 +296,7 @@ def search(_user, %{"q" => query} = params) do
   end
 
   def get_external_profile(for_user, uri) do
-    with %User{} = user <- User.get_or_fetch(uri) do
+    with {:ok, %User{} = user} <- User.get_or_fetch(uri) do
       {:ok, UserView.render("show.json", %{user: user, for: for_user})}
     else
       _e ->

lib/pleroma/web/twitter_api/twitter_api_controller.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
   alias Ecto.Changeset
   alias Pleroma.Activity
+  alias Pleroma.Formatter
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -653,7 +654,22 @@ defp build_info_cng(user, params) do
 
   defp parse_profile_bio(user, params) do
     if bio = params["description"] do
-      Map.put(params, "bio", User.parse_bio(bio, user))
+      emojis_text = (params["description"] || "") <> " " <> (params["name"] || "")
+
+      emojis =
+        ((user.info.emoji || []) ++ Formatter.get_emoji_map(emojis_text))
+        |> Enum.dedup()
+
+      user_info =
+        user.info
+        |> Map.put(
+          "emoji",
+          emojis
+        )
+
+      params
+      |> Map.put("bio", User.parse_bio(bio, user))
+      |> Map.put("info", user_info)
     else
       params
     end

lib/pleroma/web/twitter_api/views/user_view.ex

@@ -67,6 +67,13 @@ defp do_render("user.json", %{user: user = %User{}} = assigns) do
         {String.trim(name, ":"), url}
       end)
 
+    emoji = Enum.dedup(emoji ++ user.info.emoji)
+
+    description_html =
+      (user.bio || "")
+      |> HTML.filter_tags(User.html_filter_policy(for_user))
+      |> Formatter.emojify(emoji)
+
     # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``.
     # For example: [{"name": "Pronoun", "value": "she/her"}, …]
     fields =
@@ -74,58 +81,49 @@ defp do_render("user.json", %{user: user = %User{}} = assigns) do
       |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end)
       |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end)
 
-    data = %{
-      "created_at" => user.inserted_at |> Utils.format_naive_asctime(),
-      "description" => HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
-      "description_html" => HTML.filter_tags(user.bio, User.html_filter_policy(for_user)),
-      "favourites_count" => 0,
-      "followers_count" => user_info[:follower_count],
-      "following" => following,
-      "follows_you" => follows_you,
-      "statusnet_blocking" => statusnet_blocking,
-      "friends_count" => user_info[:following_count],
-      "id" => user.id,
-      "name" => user.name || user.nickname,
-      "name_html" =>
-        if(user.name,
-          do: HTML.strip_tags(user.name) |> Formatter.emojify(emoji),
-          else: user.nickname
-        ),
-      "profile_image_url" => image,
-      "profile_image_url_https" => image,
-      "profile_image_url_profile_size" => image,
-      "profile_image_url_original" => image,
-      "rights" => %{
-        "delete_others_notice" => !!user.info.is_moderator,
-        "admin" => !!user.info.is_admin
-      },
-      "screen_name" => user.nickname,
-      "statuses_count" => user_info[:note_count],
-      "statusnet_profile_url" => user.ap_id,
-      "cover_photo" => User.banner_url(user) |> MediaProxy.url(),
-      "background_image" => image_url(user.info.background) |> MediaProxy.url(),
-      "is_local" => user.local,
-      "locked" => user.info.locked,
-      "default_scope" => user.info.default_scope,
-      "no_rich_text" => user.info.no_rich_text,
-      "hide_followers" => user.info.hide_followers,
-      "hide_follows" => user.info.hide_follows,
-      "fields" => fields,
-
-      # Pleroma extension
-      "pleroma" =>
-        %{
-          "confirmation_pending" => user_info.confirmation_pending,
-          "tags" => user.tags
-        }
-        |> maybe_with_activation_status(user, for_user)
-    }
-
     data =
-      if(user.info.is_admin || user.info.is_moderator,
+      %{
-        do: maybe_with_role(data, user, for_user),
+        "created_at" => user.inserted_at |> Utils.format_naive_asctime(),
-        else: data
+        "description" => HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
-      )
+        "description_html" => description_html,
+        "favourites_count" => 0,
+        "followers_count" => user_info[:follower_count],
+        "following" => following,
+        "follows_you" => follows_you,
+        "statusnet_blocking" => statusnet_blocking,
+        "friends_count" => user_info[:following_count],
+        "id" => user.id,
+        "name" => user.name || user.nickname,
+        "name_html" =>
+          if(user.name,
+            do: HTML.strip_tags(user.name) |> Formatter.emojify(emoji),
+            else: user.nickname
+          ),
+        "profile_image_url" => image,
+        "profile_image_url_https" => image,
+        "profile_image_url_profile_size" => image,
+        "profile_image_url_original" => image,
+        "screen_name" => user.nickname,
+        "statuses_count" => user_info[:note_count],
+        "statusnet_profile_url" => user.ap_id,
+        "cover_photo" => User.banner_url(user) |> MediaProxy.url(),
+        "background_image" => image_url(user.info.background) |> MediaProxy.url(),
+        "is_local" => user.local,
+        "locked" => user.info.locked,
+        "hide_followers" => user.info.hide_followers,
+        "hide_follows" => user.info.hide_follows,
+        "fields" => fields,
+
+        # Pleroma extension
+        "pleroma" =>
+          %{
+            "confirmation_pending" => user_info.confirmation_pending,
+            "tags" => user.tags
+          }
+          |> maybe_with_activation_status(user, for_user)
+      }
+      |> maybe_with_user_settings(user, for_user)
+      |> maybe_with_role(user, for_user)
 
     if assigns[:token] do
       Map.put(data, "token", token_string(assigns[:token]))
@@ -141,15 +139,35 @@ defp maybe_with_activation_status(data, user, %User{info: %{is_admin: true}}) do
   defp maybe_with_activation_status(data, _, _), do: data
 
   defp maybe_with_role(data, %User{id: id} = user, %User{id: id}) do
-    Map.merge(data, %{"role" => role(user), "show_role" => user.info.show_role})
+    Map.merge(data, %{
+      "role" => role(user),
+      "show_role" => user.info.show_role,
+      "rights" => %{
+        "delete_others_notice" => !!user.info.is_moderator,
+        "admin" => !!user.info.is_admin
+      }
+    })
   end
 
   defp maybe_with_role(data, %User{info: %{show_role: true}} = user, _user) do
-    Map.merge(data, %{"role" => role(user)})
+    Map.merge(data, %{
+      "role" => role(user),
+      "rights" => %{
+        "delete_others_notice" => !!user.info.is_moderator,
+        "admin" => !!user.info.is_admin
+      }
+    })
   end
 
   defp maybe_with_role(data, _, _), do: data
 
+  defp maybe_with_user_settings(data, %User{info: info, id: id} = _user, %User{id: id}) do
+    data
+    |> Kernel.put_in(["default_scope"], info.default_scope)
+    |> Kernel.put_in(["no_rich_text"], info.no_rich_text)
+  end
+
+  defp maybe_with_user_settings(data, _, _), do: data
   defp role(%User{info: %{:is_admin => true}}), do: "admin"
   defp role(%User{info: %{:is_moderator => true}}), do: "moderator"
   defp role(_), do: "member"

mix.exs

@@ -16,11 +16,11 @@ def project do
 
       # Docs
       name: "Pleroma",
-      source_url: "https://git.pleroma.social/pleroma/pleroma",
-      source_url_pattern:
-        "https://git.pleroma.social/pleroma/pleroma/blob/develop/%{path}#L%{line}",
       homepage_url: "https://pleroma.social/",
+      source_url: "https://git.pleroma.social/pleroma/pleroma",
       docs: [
+        source_url_pattern:
+          "https://git.pleroma.social/pleroma/pleroma/blob/develop/%{path}#L%{line}",
         logo: "priv/static/static/logo.png",
         extras: ["README.md", "CHANGELOG.md"] ++ Path.wildcard("docs/**/*.md"),
         groups_for_extras: [
@@ -41,7 +41,7 @@ def project do
   def application do
     [
       mod: {Pleroma.Application, []},
-      extra_applications: [:logger, :runtime_tools, :comeonin, :quack],
+      extra_applications: [:logger, :runtime_tools, :comeonin, :esshd, :quack],
       included_applications: [:ex_syslogger]
     ]
   end
@@ -84,6 +84,7 @@ defp deps do
       {:ex_aws, "~> 2.0"},
       {:ex_aws_s3, "~> 2.0"},
       {:earmark, "~> 1.3"},
+      {:bbcode, "~> 0.1"},
       {:ex_machina, "~> 2.3", only: :test},
       {:credo, "~> 0.9.3", only: [:dev, :test]},
       {:mock, "~> 0.3.1", only: :test},
@@ -101,7 +102,7 @@ defp deps do
       {:ueberauth, "~> 0.4"},
       {:auto_linker,
        git: "https://git.pleroma.social/pleroma/auto_linker.git",
-       ref: "90613b4bae875a3610c275b7056b61ffdd53210d"},
+       ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"},
       {:pleroma_job_queue, "~> 0.2.0"},
       {:telemetry, "~> 0.3"},
       {:prometheus_ex, "~> 3.0"},
@@ -110,7 +111,9 @@ defp deps do
       {:prometheus_ecto, "~> 1.4"},
       {:prometheus_process_collector, "~> 1.4"},
       {:recon, github: "ferd/recon", tag: "2.4.0"},
-      {:quack, "~> 0.1.1"}
+      {:quack, "~> 0.1.1"},
+      {:benchee, "~> 1.0"},
+      {:esshd, "~> 0.1.0"}
     ] ++ oauth_deps
   end
 

mix.lock

@@ -1,7 +1,9 @@
 %{
   "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"},
-  "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "90613b4bae875a3610c275b7056b61ffdd53210d", [ref: "90613b4bae875a3610c275b7056b61ffdd53210d"]},
+  "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "c00c4e75b35367fa42c95ffd9b8c455bf9995829", [ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"]},
   "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"},
+  "bbcode": {:hex, :bbcode, "0.1.0", "400e618b640b635261611d7fb7f79d104917fc5b084aae371ab6b08477cb035b", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
+  "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm"},
   "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},
   "cachex": {:hex, :cachex, "3.0.2", "1351caa4e26e29f7d7ec1d29b53d6013f0447630bbf382b4fb5d5bad0209f203", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"},
   "calendar": {:hex, :calendar, "0.17.4", "22c5e8d98a4db9494396e5727108dffb820ee0d18fed4b0aa8ab76e4f5bc32f1", [:mix], [{:tzdata, "~> 0.5.8 or ~> 0.1.201603", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
@@ -16,9 +18,11 @@
   "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]},
   "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
   "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"},
+  "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
   "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
   "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
   "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
+  "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
   "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
   "ex_aws": {:hex, :ex_aws, "2.1.0", "b92651527d6c09c479f9013caa9c7331f19cba38a650590d82ebf2c6c16a1d8a", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, repo: "hexpm", optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, repo: "hexpm", optional: true]}], "hexpm"},
   "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"},

priv/repo/migrations/20190413082658_create_bookmarks.exs

@@ -0,0 +1,14 @@
+defmodule Pleroma.Repo.Migrations.CreateBookmarks do
+  use Ecto.Migration
+
+  def change do
+    create table(:bookmarks) do
+      add(:user_id, references(:users, type: :uuid, on_delete: :delete_all))
+      add(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all))
+
+      timestamps()
+    end
+
+    create(unique_index(:bookmarks, [:user_id, :activity_id]))
+  end
+end

priv/repo/migrations/20190414125034_migrate_old_bookmarks.exs

@@ -0,0 +1,29 @@
+defmodule Pleroma.Repo.Migrations.MigrateOldBookmarks do
+  use Ecto.Migration
+  import Ecto.Query
+  alias Pleroma.Activity
+  alias Pleroma.Bookmark
+  alias Pleroma.User
+  alias Pleroma.Repo
+
+  def change do
+    query =
+      from(u in User,
+        where: u.local == true,
+        where: fragment("array_length(bookmarks, 1)") > 0,
+        select: %{id: u.id, bookmarks: fragment("bookmarks")}
+      )
+
+    Repo.stream(query)
+    |> Enum.each(fn %{id: user_id, bookmarks: bookmarks} ->
+      Enum.each(bookmarks, fn ap_id ->
+        activity = Activity.get_create_by_object_ap_id(ap_id)
+	unless is_nil(activity), do: {:ok, _} = Bookmark.create(user_id, activity.id)
+      end)
+    end)
+
+    alter table(:users) do
+      remove(:bookmarks)
+    end
+  end
+end

priv/repo/migrations/20190501125843_add_fts_index_to_objects.exs

@@ -0,0 +1,8 @@
+defmodule Pleroma.Repo.Migrations.AddFTSIndexToObjects do
+  use Ecto.Migration
+
+  def change do
+    drop_if_exists index(:activities, ["(to_tsvector('english', data->'object'->>'content'))"], using: :gin, name: :activities_fts)
+    create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+  end
+end

priv/repo/migrations/20190501133552_add_refresh_token_index_to_token.exs

@@ -0,0 +1,7 @@
+defmodule Pleroma.Repo.Migrations.AddRefreshTokenIndexToToken do
+  use Ecto.Migration
+
+  def change do
+    create(unique_index(:oauth_tokens, [:refresh_token]))
+  end
+end

priv/static/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>Pleroma</title><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link rel=stylesheet href=/static/font/css/fontello.css><link rel=stylesheet href=/static/font/css/animation.css><link href=/static/css/app.ea66966b753e709d7ce58f910a2c003e.css rel=stylesheet></head><body style="display: none"><div id=app></div><script type=text/javascript src=/static/js/manifest.0b2f423dda42f0dbbf65.js></script><script type=text/javascript src=/static/js/vendor.e4475fde034685231799.js></script><script type=text/javascript src=/static/js/app.77434de4e756a5d79672.js></script></body></html>
+<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>Pleroma</title><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link rel=stylesheet href=/static/font/css/fontello.css><link rel=stylesheet href=/static/font/css/animation.css><link href=/static/css/app.a81578273cb4c57163939ab70c80eb06.css rel=stylesheet></head><body style="display: none"><div id=app></div><script type=text/javascript src=/static/js/manifest.bf15f24d205c8cf4ee4a.js></script><script type=text/javascript src=/static/js/vendor.0d1eeaf25aa1d2fc51b0.js></script><script type=text/javascript src=/static/js/app.c914d9a57d5da7aa5553.js></script></body></html>

priv/static/static/config.json

@@ -8,7 +8,6 @@
   "redirectRootLogin": "/main/friends",
   "chatDisabled": false,
   "showInstanceSpecificPanel": false,
-  "scopeOptionsEnabled": false,
   "formattingOptionsEnabled": false,
   "collapseMessageWithSubject": false,
   "scopeCopy": true,
@@ -21,5 +20,6 @@
   "webPushNotifications": false,
   "noAttachmentLinks": false,
   "nsfwCensorImage": "",
-  "showFeaturesPanel": true
+  "showFeaturesPanel": true,
+  "minimalScopesMode": false
 }

priv/static/static/font/config.json

@@ -239,6 +239,18 @@
       "css": "pencil",
       "code": 59416,
       "src": "fontawesome"
+    },
+    {
+      "uid": "671f29fa10dda08074a4c6a341bb4f39",
+      "css": "bell-alt",
+      "code": 61683,
+      "src": "fontawesome"
+    },
+    {
+      "uid": "5bb103cd29de77e0e06a52638527b575",
+      "css": "wrench",
+      "code": 59418,
+      "src": "fontawesome"
     }
   ]
 }

priv/static/static/font/demo.html

@@ -229,11 +229,11 @@ body {
 }
 @font-face {
       font-family: 'fontello';
-      src: url('./font/fontello.eot?50378338');
+      src: url('./font/fontello.eot?60799712');
-      src: url('./font/fontello.eot?50378338#iefix') format('embedded-opentype'),
+      src: url('./font/fontello.eot?60799712#iefix') format('embedded-opentype'),
-           url('./font/fontello.woff?50378338') format('woff'),
+           url('./font/fontello.woff?60799712') format('woff'),
-           url('./font/fontello.ttf?50378338') format('truetype'),
+           url('./font/fontello.ttf?60799712') format('truetype'),
-           url('./font/fontello.svg?50378338#fontello') format('svg');
+           url('./font/fontello.svg?60799712#fontello') format('svg');
       font-weight: normal;
       font-style: normal;
     }
@@ -335,24 +335,29 @@ body {
       </div>
       <div class="row">
         <div class="the-icons span3" title="Code: 0xe818"><i class="demo-icon icon-pencil">&#xe818;</i> <span class="i-name">icon-pencil</span><span class="i-code">0xe818</span></div>
+        <div class="the-icons span3" title="Code: 0xe819"><i class="demo-icon icon-verified">&#xe819;</i> <span class="i-name">icon-verified</span><span class="i-code">0xe819</span></div>
+        <div class="the-icons span3" title="Code: 0xe81a"><i class="demo-icon icon-wrench">&#xe81a;</i> <span class="i-name">icon-wrench</span><span class="i-code">0xe81a</span></div>
         <div class="the-icons span3" title="Code: 0xe832"><i class="demo-icon icon-spin3 animate-spin">&#xe832;</i> <span class="i-name">icon-spin3</span><span class="i-code">0xe832</span></div>
+      </div>
+      <div class="row">
         <div class="the-icons span3" title="Code: 0xe834"><i class="demo-icon icon-spin4 animate-spin">&#xe834;</i> <span class="i-name">icon-spin4</span><span class="i-code">0xe834</span></div>
         <div class="the-icons span3" title="Code: 0xf08e"><i class="demo-icon icon-link-ext">&#xf08e;</i> <span class="i-name">icon-link-ext</span><span class="i-code">0xf08e</span></div>
-      </div>
-      <div class="row">
         <div class="the-icons span3" title="Code: 0xf08f"><i class="demo-icon icon-link-ext-alt">&#xf08f;</i> <span class="i-name">icon-link-ext-alt</span><span class="i-code">0xf08f</span></div>
         <div class="the-icons span3" title="Code: 0xf0c9"><i class="demo-icon icon-menu">&#xf0c9;</i> <span class="i-name">icon-menu</span><span class="i-code">0xf0c9</span></div>
-        <div class="the-icons span3" title="Code: 0xf0e0"><i class="demo-icon icon-mail-alt">&#xf0e0;</i> <span class="i-name">icon-mail-alt</span><span class="i-code">0xf0e0</span></div>
-        <div class="the-icons span3" title="Code: 0xf0e5"><i class="demo-icon icon-comment-empty">&#xf0e5;</i> <span class="i-name">icon-comment-empty</span><span class="i-code">0xf0e5</span></div>
       </div>
       <div class="row">
+        <div class="the-icons span3" title="Code: 0xf0e0"><i class="demo-icon icon-mail-alt">&#xf0e0;</i> <span class="i-name">icon-mail-alt</span><span class="i-code">0xf0e0</span></div>
+        <div class="the-icons span3" title="Code: 0xf0e5"><i class="demo-icon icon-comment-empty">&#xf0e5;</i> <span class="i-name">icon-comment-empty</span><span class="i-code">0xf0e5</span></div>
+        <div class="the-icons span3" title="Code: 0xf0f3"><i class="demo-icon icon-bell-alt">&#xf0f3;</i> <span class="i-name">icon-bell-alt</span><span class="i-code">0xf0f3</span></div>
         <div class="the-icons span3" title="Code: 0xf0fe"><i class="demo-icon icon-plus-squared">&#xf0fe;</i> <span class="i-name">icon-plus-squared</span><span class="i-code">0xf0fe</span></div>
+      </div>
+      <div class="row">
         <div class="the-icons span3" title="Code: 0xf112"><i class="demo-icon icon-reply">&#xf112;</i> <span class="i-name">icon-reply</span><span class="i-code">0xf112</span></div>
         <div class="the-icons span3" title="Code: 0xf13e"><i class="demo-icon icon-lock-open-alt">&#xf13e;</i> <span class="i-name">icon-lock-open-alt</span><span class="i-code">0xf13e</span></div>
         <div class="the-icons span3" title="Code: 0xf144"><i class="demo-icon icon-play-circled">&#xf144;</i> <span class="i-name">icon-play-circled</span><span class="i-code">0xf144</span></div>
+        <div class="the-icons span3" title="Code: 0xf164"><i class="demo-icon icon-thumbs-up-alt">&#xf164;</i> <span class="i-name">icon-thumbs-up-alt</span><span class="i-code">0xf164</span></div>
       </div>
       <div class="row">
-        <div class="the-icons span3" title="Code: 0xf164"><i class="demo-icon icon-thumbs-up-alt">&#xf164;</i> <span class="i-name">icon-thumbs-up-alt</span><span class="i-code">0xf164</span></div>
         <div class="the-icons span3" title="Code: 0xf1e5"><i class="demo-icon icon-binoculars">&#xf1e5;</i> <span class="i-name">icon-binoculars</span><span class="i-code">0xf1e5</span></div>
         <div class="the-icons span3" title="Code: 0xf234"><i class="demo-icon icon-user-plus">&#xf234;</i> <span class="i-name">icon-user-plus</span><span class="i-code">0xf234</span></div>
       </div>

priv/static/static/font/font/fontello.svg

@@ -56,6 +56,10 @@
 
 <glyph glyph-name="pencil" unicode="&#xe818;" d="M203 0l50 51-131 131-51-51v-60h72v-71h60z m291 518q0 12-12 12-5 0-9-4l-303-302q-4-4-4-10 0-12 13-12 5 0 9 4l303 302q3 4 3 10z m-30 107l232-232-464-465h-232v233z m381-54q0-29-20-50l-93-93-232 233 93 92q20 21 50 21 29 0 51-21l131-131q20-22 20-51z" horiz-adv-x="857.1" />
 
+<glyph glyph-name="verified" unicode="&#xe819;" d="M926 453l-19 13c-21 14-30 41-23 65l6 22c10 34-13 69-48 75l-23 4c-25 4-45 23-49 48l-4 23c-6 35-41 57-75 47l-22-7c-24-7-51 2-65 22l-14 20c-21 29-62 33-88 9l-17-16c-19-17-46-21-69-8l-20 11c-31 17-70 3-84-30l-9-22c-9-24-33-39-58-37l-23 1c-36 2-65-28-62-63l2-23c2-25-13-49-36-59l-21-9c-33-14-46-53-29-84l12-20c13-22 10-50-7-69l-15-17c-24-27-19-68 11-88l19-13c21-14 30-41 23-65l-9-23c-10-34 13-69 48-75l23-4c25-4 45-23 49-48l4-23c6-35 41-57 75-47l22 7c24 7 51-2 65-22l14-19c21-29 62-33 88-9l17 16c19 17 46 21 69 8l20-11c31-17 70-3 84 30l9 22c9 24 33 39 58 37l23-1c36-2 65 28 62 63l-1 23c-2 25 13 49 36 59l21 9c33 14 46 53 29 84l-12 20c-13 22-10 50 7 69l15 17c25 26 20 68-9 88z m-399-189l-82-81-81 82-78 79 82 81 78-79 187 186 81-82-187-186z" horiz-adv-x="1000" />
+
+<glyph glyph-name="wrench" unicode="&#xe81a;" d="M214 36q0 14-10 25t-25 10-25-10-11-25 11-25 25-11 25 11 10 25z m360 234l-381-381q-21-20-50-20-29 0-51 20l-59 61q-21 20-21 50 0 29 21 51l380 380q22-55 64-97t97-64z m354 243q0-22-13-59-27-75-92-122t-144-46q-104 0-177 73t-73 177 73 176 177 74q32 0 67-10t60-26q9-6 9-15t-9-16l-163-94v-125l108-60q2 2 44 27t75 45 40 20q8 0 13-5t5-14z" horiz-adv-x="928.6" />
+
 <glyph glyph-name="spin3" unicode="&#xe832;" d="M494 857c-266 0-483-210-494-472-1-19 13-20 13-20l84 0c16 0 19 10 19 18 10 199 176 358 378 358 107 0 205-45 273-118l-58-57c-11-12-11-27 5-31l247-50c21-5 46 11 37 44l-58 227c-2 9-16 22-29 13l-65-60c-89 91-214 148-352 148z m409-508c-16 0-19-10-19-18-10-199-176-358-377-358-108 0-205 45-274 118l59 57c10 12 10 27-5 31l-248 50c-21 5-46-11-37-44l58-227c2-9 16-22 30-13l64 60c89-91 214-148 353-148 265 0 482 210 493 473 1 18-13 19-13 19l-84 0z" horiz-adv-x="1000" />
 
 <glyph glyph-name="spin4" unicode="&#xe834;" d="M498 857c-114 0-228-39-320-116l0 0c173 140 428 130 588-31 134-134 164-332 89-495-10-29-5-50 12-68 21-20 61-23 84 0 3 3 12 15 15 24 71 180 33 393-112 539-99 98-228 147-356 147z m-409-274c-14 0-29-5-39-16-3-3-13-15-15-24-71-180-34-393 112-539 185-185 479-195 676-31l0 0c-173-140-428-130-589 31-134 134-163 333-89 495 11 29 6 50-12 68-11 11-27 17-44 16z" horiz-adv-x="1001" />
@@ -70,6 +74,8 @@
 
 <glyph glyph-name="comment-empty" unicode="&#xf0e5;" d="M500 643q-114 0-213-39t-157-105-59-142q0-62 40-119t113-98l48-28-15-53q-13-51-39-97 85 36 154 96l24 21 32-3q38-5 72-5 114 0 213 39t157 105 59 142-59 142-157 105-213 39z m500-286q0-97-67-179t-182-130-251-48q-39 0-81 4-110-97-257-135-27-8-63-12h-3q-8 0-15 6t-9 15v1q-2 2 0 6t1 6 2 5l4 5t4 5 4 5q4 5 17 19t20 22 17 22 18 28 15 33 15 42q-88 50-138 123t-51 157q0 97 67 179t182 130 251 48 251-48 182-130 67-179z" horiz-adv-x="1000" />
 
+<glyph glyph-name="bell-alt" unicode="&#xf0f3;" d="M509-89q0 8-9 8-33 0-57 24t-23 57q0 9-9 9t-9-9q0-41 29-70t69-28q9 0 9 9z m455 160q0-29-21-50t-50-21h-250q0-59-42-101t-101-42-101 42-42 101h-250q-29 0-50 21t-21 50q28 24 51 49t47 67 42 89 27 115 11 145q0 84 66 157t171 89q-5 10-5 21 0 23 16 38t38 16 38-16 16-38q0-11-5-21 106-16 171-89t66-157q0-78 11-145t28-115 41-89 48-67 50-49z" horiz-adv-x="1000" />
+
 <glyph glyph-name="plus-squared" unicode="&#xf0fe;" d="M714 321v72q0 14-10 25t-25 10h-179v179q0 15-11 25t-25 11h-71q-15 0-25-11t-11-25v-179h-178q-15 0-25-10t-11-25v-72q0-14 11-25t25-10h178v-179q0-14 11-25t25-11h71q15 0 25 11t11 25v179h179q14 0 25 10t10 25z m143 304v-536q0-66-47-113t-114-48h-535q-67 0-114 48t-47 113v536q0 66 47 113t114 48h535q67 0 114-48t47-113z" horiz-adv-x="857.1" />
 
 <glyph glyph-name="reply" unicode="&#xf112;" d="M1000 232q0-93-71-252-1-4-6-13t-7-17-7-12q-7-10-16-10-8 0-13 6t-5 14q0 5 1 15t2 13q3 38 3 69 0 56-10 101t-27 77-45 56-59 39-74 24-86 12-98 3h-125v-143q0-14-10-25t-26-11-25 11l-285 286q-11 10-11 25t11 25l285 286q11 10 25 10t26-10 10-25v-143h125q398 0 488-225 30-75 30-186z" horiz-adv-x="1000" />

test/bbs/handler_test.exs

@@ -0,0 +1,83 @@
+defmodule Pleroma.BBS.HandlerTest do
+  use Pleroma.DataCase
+  alias Pleroma.Activity
+  alias Pleroma.BBS.Handler
+  alias Pleroma.Object
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+
+  import ExUnit.CaptureIO
+  import Pleroma.Factory
+  import Ecto.Query
+
+  test "getting the home timeline" do
+    user = insert(:user)
+    followed = insert(:user)
+
+    {:ok, user} = User.follow(user, followed)
+
+    {:ok, _first} = CommonAPI.post(user, %{"status" => "hey"})
+    {:ok, _second} = CommonAPI.post(followed, %{"status" => "hello"})
+
+    output =
+      capture_io(fn ->
+        Handler.handle_command(%{user: user}, "home")
+      end)
+
+    assert output =~ user.nickname
+    assert output =~ followed.nickname
+
+    assert output =~ "hey"
+    assert output =~ "hello"
+  end
+
+  test "posting" do
+    user = insert(:user)
+
+    output =
+      capture_io(fn ->
+        Handler.handle_command(%{user: user}, "p this is a test post")
+      end)
+
+    assert output =~ "Posted"
+
+    activity =
+      Repo.one(
+        from(a in Activity,
+          where: fragment("?->>'type' = ?", a.data, "Create")
+        )
+      )
+
+    assert activity.actor == user.ap_id
+    object = Object.normalize(activity)
+    assert object.data["content"] == "this is a test post"
+  end
+
+  test "replying" do
+    user = insert(:user)
+    another_user = insert(:user)
+
+    {:ok, activity} = CommonAPI.post(another_user, %{"status" => "this is a test post"})
+
+    output =
+      capture_io(fn ->
+        Handler.handle_command(%{user: user}, "r #{activity.id} this is a reply")
+      end)
+
+    assert output =~ "Replied"
+
+    reply =
+      Repo.one(
+        from(a in Activity,
+          where: fragment("?->>'type' = ?", a.data, "Create"),
+          where: a.actor == ^user.ap_id
+        )
+      )
+
+    assert reply.actor == user.ap_id
+    object = Object.normalize(reply)
+    assert object.data["content"] == "this is a reply"
+    assert object.data["inReplyTo"] == activity.data["object"]
+  end
+end

test/bookmark_test.exs

@@ -0,0 +1,52 @@
+defmodule Pleroma.BookmarkTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+  alias Pleroma.Bookmark
+  alias Pleroma.Web.CommonAPI
+
+  describe "create/2" do
+    test "with valid params" do
+      user = insert(:user)
+      {:ok, activity} = CommonAPI.post(user, %{"status" => "Some cool information"})
+      {:ok, bookmark} = Bookmark.create(user.id, activity.id)
+      assert bookmark.user_id == user.id
+      assert bookmark.activity_id == activity.id
+    end
+
+    test "with invalid params" do
+      {:error, changeset} = Bookmark.create(nil, "")
+      refute changeset.valid?
+
+      assert changeset.errors == [
+               user_id: {"can't be blank", [validation: :required]},
+               activity_id: {"can't be blank", [validation: :required]}
+             ]
+    end
+  end
+
+  describe "destroy/2" do
+    test "with valid params" do
+      user = insert(:user)
+
+      {:ok, activity} = CommonAPI.post(user, %{"status" => "Some cool information"})
+      {:ok, _bookmark} = Bookmark.create(user.id, activity.id)
+
+      {:ok, _deleted_bookmark} = Bookmark.destroy(user.id, activity.id)
+    end
+  end
+
+  describe "get/2" do
+    test "gets a bookmark" do
+      user = insert(:user)
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{
+          "status" =>
+            "Scientists Discover The Secret Behind Tenshi Eating A Corndog Being So Cute – Science Daily"
+        })
+
+      {:ok, bookmark} = Bookmark.create(user.id, activity.id)
+      assert bookmark == Bookmark.get(user.id, activity.id)
+    end
+  end
+end

test/formatter_test.exs

@@ -147,7 +147,7 @@ test "gives a replacement for user links, using local nicknames in user links te
     end
 
     test "gives a replacement for user links when the user is using Osada" do
-      mike = User.get_or_fetch("mike@osada.macgirvin.com")
+      {:ok, mike} = User.get_or_fetch("mike@osada.macgirvin.com")
 
       text = "@mike@osada.macgirvin.com test"
 
@@ -248,7 +248,7 @@ test "it adds cool emoji" do
     text = "I love :firefox:"
 
     expected_result =
-      "I love <img height=\"32px\" width=\"32px\" alt=\"firefox\" title=\"firefox\" src=\"/emoji/Firefox.gif\" />"
+      "I love <img class=\"emoji\" alt=\"firefox\" title=\"firefox\" src=\"/emoji/Firefox.gif\" />"
 
     assert Formatter.emojify(text) == expected_result
   end
@@ -263,7 +263,7 @@ test "it does not add XSS emoji" do
     }
 
     expected_result =
-      "I love <img height=\"32px\" width=\"32px\" alt=\"\" title=\"\" src=\"https://placehold.it/1x1\" />"
+      "I love <img class=\"emoji\" alt=\"\" title=\"\" src=\"https://placehold.it/1x1\" />"
 
     assert Formatter.emojify(text, custom_emoji) == expected_result
   end

test/html_test.exs

@@ -20,6 +20,18 @@ defmodule Pleroma.HTMLTest do
     <img src="http://example.com/image.jpg" onerror="alert('hacked')">
   """
 
+  @html_span_class_sample """
+    <span class="animate-spin">hi</span>
+  """
+
+  @html_span_microformats_sample """
+    <span class="h-card"><a class="u-url mention">@<span>foo</span></a></span>
+  """
+
+  @html_span_invalid_microformats_sample """
+    <span class="h-card"><a class="u-url mention animate-spin">@<span>foo</span></a></span>
+  """
+
   describe "StripTags scrubber" do
     test "works as expected" do
       expected = """
@@ -64,6 +76,36 @@ test "does not allow attribute-based XSS" do
 
       assert expected == HTML.filter_tags(@html_onerror_sample, Pleroma.HTML.Scrubber.TwitterText)
     end
+
+    test "does not allow spans with invalid classes" do
+      expected = """
+      <span>hi</span>
+      """
+
+      assert expected ==
+               HTML.filter_tags(@html_span_class_sample, Pleroma.HTML.Scrubber.TwitterText)
+    end
+
+    test "does allow microformats" do
+      expected = """
+      <span class="h-card"><a class="u-url mention">@<span>foo</span></a></span>
+      """
+
+      assert expected ==
+               HTML.filter_tags(@html_span_microformats_sample, Pleroma.HTML.Scrubber.TwitterText)
+    end
+
+    test "filters invalid microformats markup" do
+      expected = """
+      <span class="h-card"><a>@<span>foo</span></a></span>
+      """
+
+      assert expected ==
+               HTML.filter_tags(
+                 @html_span_invalid_microformats_sample,
+                 Pleroma.HTML.Scrubber.TwitterText
+               )
+    end
   end
 
   describe "default scrubber" do
@@ -88,5 +130,34 @@ test "does not allow attribute-based XSS" do
 
       assert expected == HTML.filter_tags(@html_onerror_sample, Pleroma.HTML.Scrubber.Default)
     end
+
+    test "does not allow spans with invalid classes" do
+      expected = """
+      <span>hi</span>
+      """
+
+      assert expected == HTML.filter_tags(@html_span_class_sample, Pleroma.HTML.Scrubber.Default)
+    end
+
+    test "does allow microformats" do
+      expected = """
+      <span class="h-card"><a class="u-url mention">@<span>foo</span></a></span>
+      """
+
+      assert expected ==
+               HTML.filter_tags(@html_span_microformats_sample, Pleroma.HTML.Scrubber.Default)
+    end
+
+    test "filters invalid microformats markup" do
+      expected = """
+      <span class="h-card"><a>@<span>foo</span></a></span>
+      """
+
+      assert expected ==
+               HTML.filter_tags(
+                 @html_span_invalid_microformats_sample,
+                 Pleroma.HTML.Scrubber.Default
+               )
+    end
   end
 end

test/media_proxy_test.exs

@@ -7,15 +7,15 @@ defmodule Pleroma.MediaProxyTest do
   import Pleroma.Web.MediaProxy
   alias Pleroma.Web.MediaProxy.MediaProxyController
 
+  setup do
+    enabled = Pleroma.Config.get([:media_proxy, :enabled])
+    on_exit(fn -> Pleroma.Config.put([:media_proxy, :enabled], enabled) end)
+    :ok
+  end
+
   describe "when enabled" do
     setup do
-      enabled = Pleroma.Config.get([:media_proxy, :enabled])
+      Pleroma.Config.put([:media_proxy, :enabled], true)
-
-      unless enabled do
-        Pleroma.Config.put([:media_proxy, :enabled], true)
-        on_exit(fn -> Pleroma.Config.put([:media_proxy, :enabled], enabled) end)
-      end
-
       :ok
     end
 
@@ -177,4 +177,13 @@ defp decode_result(encoded) do
     {:ok, decoded} = decode_url(sig, base64)
     decoded
   end
+
+  test "mediaproxy whitelist" do
+    Pleroma.Config.put([:media_proxy, :enabled], true)
+    Pleroma.Config.put([:media_proxy, :whitelist], ["google.com", "feld.me"])
+    url = "https://feld.me/foo.png"
+
+    unencoded = url(url)
+    assert unencoded == url
+  end
 end

test/plugs/oauth_plug_test.exs

@@ -38,6 +38,26 @@ test "with valid token(downcase), it assigns the user", %{conn: conn} = opts do
     assert conn.assigns[:user] == opts[:user]
   end
 
+  test "with valid token(downcase) in url parameters, it assings the user", opts do
+    conn =
+      :get
+      |> build_conn("/?access_token=#{opts[:token]}")
+      |> put_req_header("content-type", "application/json")
+      |> fetch_query_params()
+      |> OAuthPlug.call(%{})
+
+    assert conn.assigns[:user] == opts[:user]
+  end
+
+  test "with valid token(downcase) in body parameters, it assigns the user", opts do
+    conn =
+      :post
+      |> build_conn("/api/v1/statuses", access_token: opts[:token], status: "test")
+      |> OAuthPlug.call(%{})
+
+    assert conn.assigns[:user] == opts[:user]
+  end
+
   test "with invalid token, it not assigns the user", %{conn: conn} do
     conn =
       conn

test/repo_test.exs

@@ -0,0 +1,44 @@
+defmodule Pleroma.RepoTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+
+  describe "find_resource/1" do
+    test "returns user" do
+      user = insert(:user)
+      query = from(t in Pleroma.User, where: t.id == ^user.id)
+      assert Repo.find_resource(query) == {:ok, user}
+    end
+
+    test "returns not_found" do
+      query = from(t in Pleroma.User, where: t.id == ^"9gBuXNpD2NyDmmxxdw")
+      assert Repo.find_resource(query) == {:error, :not_found}
+    end
+  end
+
+  describe "get_assoc/2" do
+    test "get assoc from preloaded data" do
+      user = %Pleroma.User{name: "Agent Smith"}
+      token = %Pleroma.Web.OAuth.Token{insert(:oauth_token) | user: user}
+      assert Repo.get_assoc(token, :user) == {:ok, user}
+    end
+
+    test "get one-to-one assoc from repo" do
+      user = insert(:user, name: "Jimi Hendrix")
+      token = refresh_record(insert(:oauth_token, user: user))
+
+      assert Repo.get_assoc(token, :user) == {:ok, user}
+    end
+
+    test "get one-to-many assoc from repo" do
+      user = insert(:user)
+      notification = refresh_record(insert(:notification, user: user))
+
+      assert Repo.get_assoc(user, :notifications) == {:ok, [notification]}
+    end
+
+    test "return error if has not assoc " do
+      token = insert(:oauth_token, user: nil)
+      assert Repo.get_assoc(token, :user) == {:error, :not_found}
+    end
+  end
+end

test/user_test.exs

@@ -363,7 +363,7 @@ test "it creates confirmed user if :confirmed option is given" do
   describe "get_or_fetch/1" do
     test "gets an existing user by nickname" do
       user = insert(:user)
-      fetched_user = User.get_or_fetch(user.nickname)
+      {:ok, fetched_user} = User.get_or_fetch(user.nickname)
 
       assert user == fetched_user
     end
@@ -380,7 +380,7 @@ test "gets an existing user by ap_id" do
           info: %{}
         )
 
-      fetched_user = User.get_or_fetch(ap_id)
+      {:ok, fetched_user} = User.get_or_fetch(ap_id)
       freshed_user = refresh_record(user)
       assert freshed_user == fetched_user
     end
@@ -389,14 +389,14 @@ test "gets an existing user by ap_id" do
   describe "fetching a user from nickname or trying to build one" do
     test "gets an existing user" do
       user = insert(:user)
-      fetched_user = User.get_or_fetch_by_nickname(user.nickname)
+      {:ok, fetched_user} = User.get_or_fetch_by_nickname(user.nickname)
 
       assert user == fetched_user
     end
 
     test "gets an existing user, case insensitive" do
       user = insert(:user, nickname: "nick")
-      fetched_user = User.get_or_fetch_by_nickname("NICK")
+      {:ok, fetched_user} = User.get_or_fetch_by_nickname("NICK")
 
       assert user == fetched_user
     end
@@ -404,7 +404,7 @@ test "gets an existing user, case insensitive" do
     test "gets an existing user by fully qualified nickname" do
       user = insert(:user)
 
-      fetched_user =
+      {:ok, fetched_user} =
         User.get_or_fetch_by_nickname(user.nickname <> "@" <> Pleroma.Web.Endpoint.host())
 
       assert user == fetched_user
@@ -414,24 +414,24 @@ test "gets an existing user by fully qualified nickname, case insensitive" do
       user = insert(:user, nickname: "nick")
       casing_altered_fqn = String.upcase(user.nickname <> "@" <> Pleroma.Web.Endpoint.host())
 
-      fetched_user = User.get_or_fetch_by_nickname(casing_altered_fqn)
+      {:ok, fetched_user} = User.get_or_fetch_by_nickname(casing_altered_fqn)
 
       assert user == fetched_user
     end
 
     test "fetches an external user via ostatus if no user exists" do
-      fetched_user = User.get_or_fetch_by_nickname("shp@social.heldscal.la")
+      {:ok, fetched_user} = User.get_or_fetch_by_nickname("shp@social.heldscal.la")
       assert fetched_user.nickname == "shp@social.heldscal.la"
     end
 
     test "returns nil if no user could be fetched" do
-      fetched_user = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la")
+      {:error, fetched_user} = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la")
-      assert fetched_user == nil
+      assert fetched_user == "not found nonexistant@social.heldscal.la"
     end
 
     test "returns nil for nonexistant local user" do
-      fetched_user = User.get_or_fetch_by_nickname("nonexistant")
+      {:error, fetched_user} = User.get_or_fetch_by_nickname("nonexistant")
-      assert fetched_user == nil
+      assert fetched_user == "not found nonexistant"
     end
 
     test "updates an existing user, if stale" do
@@ -449,7 +449,7 @@ test "updates an existing user, if stale" do
 
       assert orig_user.last_refreshed_at == a_week_ago
 
-      user = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/admin")
+      {:ok, user} = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/admin")
       assert user.info.source_data["endpoints"]
 
       refute user.last_refreshed_at == orig_user.last_refreshed_at
@@ -888,10 +888,12 @@ test ".delete_user_activities deletes all create activities" do
     user = insert(:user)
 
     {:ok, activity} = CommonAPI.post(user, %{"status" => "2hu"})
-    {:ok, _} = User.delete_user_activities(user)
 
-    # TODO: Remove favorites, repeats, delete activities.
+    Ecto.Adapters.SQL.Sandbox.unboxed_run(Repo, fn ->
-    refute Activity.get_by_id(activity.id)
+      {:ok, _} = User.delete_user_activities(user)
+      # TODO: Remove favorites, repeats, delete activities.
+      refute Activity.get_by_id(activity.id)
+    end)
   end
 
   test ".delete deactivates a user, all follow relationships and all create activities" do
@@ -1162,7 +1164,7 @@ test "preserves hosts in user links text" do
       expected_text =
         "A.k.a. <span class='h-card'><a data-user='#{remote_user.id}' class='u-url mention' href='#{
           remote_user.ap_id
-        }'>" <> "@<span>nick@domain.com</span></a></span>"
+        }'>@<span>nick@domain.com</span></a></span>"
 
       assert expected_text == User.parse_bio(bio, user)
     end
@@ -1184,33 +1186,6 @@ test "Adds rel=me on linkbacked urls" do
     end
   end
 
-  test "bookmarks" do
-    user = insert(:user)
-
-    {:ok, activity1} =
-      CommonAPI.post(user, %{
-        "status" => "heweoo!"
-      })
-
-    id1 = Object.normalize(activity1).data["id"]
-
-    {:ok, activity2} =
-      CommonAPI.post(user, %{
-        "status" => "heweoo!"
-      })
-
-    id2 = Object.normalize(activity2).data["id"]
-
-    assert {:ok, user_state1} = User.bookmark(user, id1)
-    assert user_state1.bookmarks == [id1]
-
-    assert {:ok, user_state2} = User.unbookmark(user, id1)
-    assert user_state2.bookmarks == []
-
-    assert {:ok, user_state3} = User.bookmark(user, id2)
-    assert user_state3.bookmarks == [id2]
-  end
-
   test "follower count is updated when a follower is blocked" do
     user = insert(:user)
     follower = insert(:user)

test/web/activity_pub/transmogrifier_test.exs

@@ -215,6 +215,26 @@ test "it works for incoming follow requests" do
       assert User.following?(User.get_cached_by_ap_id(data["actor"]), user)
     end
 
+    test "it rejects incoming follow requests from blocked users when deny_follow_blocked is enabled" do
+      Pleroma.Config.put([:user, :deny_follow_blocked], true)
+
+      user = insert(:user)
+      {:ok, target} = User.get_or_fetch("http://mastodon.example.org/users/admin")
+
+      {:ok, user} = User.block(user, target)
+
+      data =
+        File.read!("test/fixtures/mastodon-follow-activity.json")
+        |> Poison.decode!()
+        |> Map.put("object", user.ap_id)
+
+      {:ok, %Activity{data: %{"id" => id}}} = Transmogrifier.handle_incoming(data)
+
+      %Activity{} = activity = Activity.get_by_ap_id(id)
+
+      assert activity.data["state"] == "reject"
+    end
+
     test "it works for incoming follow requests from hubzilla" do
       user = insert(:user)
 

test/web/auth/authenticator_test.exs

@@ -0,0 +1,42 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Auth.AuthenticatorTest do
+  use Pleroma.Web.ConnCase
+
+  alias Pleroma.Web.Auth.Authenticator
+  import Pleroma.Factory
+
+  describe "fetch_user/1" do
+    test "returns user by name" do
+      user = insert(:user)
+      assert Authenticator.fetch_user(user.nickname) == user
+    end
+
+    test "returns user by email" do
+      user = insert(:user)
+      assert Authenticator.fetch_user(user.email) == user
+    end
+
+    test "returns nil" do
+      assert Authenticator.fetch_user("email") == nil
+    end
+  end
+
+  describe "fetch_credentials/1" do
+    test "returns name and password from authorization params" do
+      params = %{"authorization" => %{"name" => "test", "password" => "test-pass"}}
+      assert Authenticator.fetch_credentials(params) == {:ok, {"test", "test-pass"}}
+    end
+
+    test "returns name and password with grant_type 'password'" do
+      params = %{"grant_type" => "password", "username" => "test", "password" => "test-pass"}
+      assert Authenticator.fetch_credentials(params) == {:ok, {"test", "test-pass"}}
+    end
+
+    test "returns error" do
+      assert Authenticator.fetch_credentials(%{}) == {:error, :invalid_credentials}
+    end
+  end
+end

test/web/common_api/common_api_utils_test.exs

@@ -119,6 +119,31 @@ test "works for bare text/markdown" do
       assert output == expected
     end
 
+    test "works for bare text/bbcode" do
+      text = "[b]hello world[/b]"
+      expected = "<strong>hello world</strong>"
+
+      {output, [], []} = Utils.format_input(text, "text/bbcode")
+
+      assert output == expected
+
+      text = "[b]hello world![/b]\n\nsecond paragraph!"
+      expected = "<strong>hello world!</strong><br>\n<br>\nsecond paragraph!"
+
+      {output, [], []} = Utils.format_input(text, "text/bbcode")
+
+      assert output == expected
+
+      text = "[b]hello world![/b]\n\n<strong>second paragraph!</strong>"
+
+      expected =
+        "<strong>hello world!</strong><br>\n<br>\n&lt;strong&gt;second paragraph!&lt;/strong&gt;"
+
+      {output, [], []} = Utils.format_input(text, "text/bbcode")
+
+      assert output == expected
+    end
+
     test "works for text/markdown with mentions" do
       {:ok, user} =
         UserBuilder.insert(%{nickname: "user__test", ap_id: "http://foo.com/user__test"})

test/web/mastodon_api/account_view_test.exs

@@ -56,14 +56,17 @@ test "Represent a user account" do
       bot: false,
       source: %{
         note: "",
-        privacy: "public",
+        sensitive: false,
-        sensitive: false
+        pleroma: %{}
       },
       pleroma: %{
         confirmation_pending: false,
         tags: [],
         is_admin: false,
         is_moderator: false,
+        hide_favorites: true,
+        hide_followers: false,
+        hide_follows: false,
         relationship: %{}
       }
     }
@@ -81,8 +84,12 @@ test "Represent the user account for the account owner" do
       "follows" => true
     }
 
-    assert %{pleroma: %{notification_settings: ^notification_settings}} =
+    privacy = user.info.default_scope
-             AccountView.render("account.json", %{user: user, for: user})
+
+    assert %{
+             pleroma: %{notification_settings: ^notification_settings},
+             source: %{privacy: ^privacy}
+           } = AccountView.render("account.json", %{user: user, for: user})
   end
 
   test "Represent a Service(bot) account" do
@@ -114,14 +121,17 @@ test "Represent a Service(bot) account" do
       bot: true,
       source: %{
         note: "",
-        privacy: "public",
+        sensitive: false,
-        sensitive: false
+        pleroma: %{}
       },
       pleroma: %{
         confirmation_pending: false,
         tags: [],
         is_admin: false,
         is_moderator: false,
+        hide_favorites: true,
+        hide_followers: false,
+        hide_follows: false,
         relationship: %{}
       }
     }
@@ -200,14 +210,17 @@ test "represent an embedded relationship" do
       bot: true,
       source: %{
         note: "",
-        privacy: "public",
+        sensitive: false,
-        sensitive: false
+        pleroma: %{}
       },
       pleroma: %{
         confirmation_pending: false,
         tags: [],
         is_admin: false,
         is_moderator: false,
+        hide_favorites: true,
+        hide_followers: false,
+        hide_follows: false,
         relationship: %{
           id: to_string(user.id),
           following: false,

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -1022,7 +1022,7 @@ test "reblogged status for another user", %{conn: conn} do
       user2 = insert(:user)
       user3 = insert(:user)
       CommonAPI.favorite(activity.id, user2)
-      {:ok, user2} = User.bookmark(user2, activity.data["object"]["id"])
+      {:ok, _bookmark} = Pleroma.Bookmark.create(user2.id, activity.id)
       {:ok, reblog_activity1, _object} = CommonAPI.repeat(activity.id, user1)
       {:ok, _, _object} = CommonAPI.repeat(activity.id, user2)
 
@@ -2214,6 +2214,78 @@ test "updates the user's locking status", %{conn: conn} do
       assert user["locked"] == true
     end
 
+    test "updates the user's default scope", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{default_scope: "cofe"})
+
+      assert user = json_response(conn, 200)
+      assert user["source"]["privacy"] == "cofe"
+    end
+
+    test "updates the user's hide_followers status", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{hide_followers: "true"})
+
+      assert user = json_response(conn, 200)
+      assert user["pleroma"]["hide_followers"] == true
+    end
+
+    test "updates the user's hide_follows status", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{hide_follows: "true"})
+
+      assert user = json_response(conn, 200)
+      assert user["pleroma"]["hide_follows"] == true
+    end
+
+    test "updates the user's hide_favorites status", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{hide_favorites: "true"})
+
+      assert user = json_response(conn, 200)
+      assert user["pleroma"]["hide_favorites"] == true
+    end
+
+    test "updates the user's show_role status", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{show_role: "false"})
+
+      assert user = json_response(conn, 200)
+      assert user["source"]["pleroma"]["show_role"] == false
+    end
+
+    test "updates the user's no_rich_text status", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{no_rich_text: "true"})
+
+      assert user = json_response(conn, 200)
+      assert user["source"]["pleroma"]["no_rich_text"] == true
+    end
+
     test "updates the user's name", %{conn: conn} do
       user = insert(:user)
 
@@ -2279,6 +2351,33 @@ test "requires 'write' permission", %{conn: conn} do
         end
       end
     end
+
+    test "updates profile emojos", %{conn: conn} do
+      user = insert(:user)
+
+      note = "*sips :blank:*"
+      name = "I am :firefox:"
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{
+          "note" => note,
+          "display_name" => name
+        })
+
+      assert json_response(conn, 200)
+
+      conn =
+        conn
+        |> get("/api/v1/accounts/#{user.id}")
+
+      assert user = json_response(conn, 200)
+
+      assert user["note"] == note
+      assert user["display_name"] == name
+      assert [%{"shortcode" => "blank"}, %{"shortcode" => "firefox"}] = user["emojis"]
+    end
   end
 
   test "get instance information", %{conn: conn} do