Merge remote-tracking branch 'upstream/develop' into patch-image-description

.credo.exs

@@ -19,7 +19,7 @@
         #
         # You can give explicit globs or simply directories.
         # In the latter case `**/*.{ex,exs}` will be used.
-        included: ["lib/", "src/", "web/", "apps/"],
+        included: ["lib/", "src/", "web/", "apps/", "test/"],
         excluded: [~r"/_build/", ~r"/deps/"]
       },
       #
@@ -57,7 +57,7 @@
 
         # For some checks, like AliasUsage, you can only customize the priority
         # Priority values are: `low, normal, high, higher`
-        {Credo.Check.Design.AliasUsage, priority: :low},
+        {Credo.Check.Design.AliasUsage, priority: :low, if_called_more_often_than: 3},
 
         # For others you can set parameters
 
@@ -104,7 +104,8 @@
         {Credo.Check.Warning.BoolOperationOnSameValues},
         {Credo.Check.Warning.IExPry},
         {Credo.Check.Warning.IoInspect},
-        {Credo.Check.Warning.LazyLogging},
+        # Got too much of them, not sure if relevant
+        {Credo.Check.Warning.LazyLogging, false},
         {Credo.Check.Warning.OperationOnSameValues},
         {Credo.Check.Warning.OperationWithConstantResult},
         {Credo.Check.Warning.UnusedEnumOperation},

.gitignore

@@ -25,6 +25,7 @@ erl_crash.dump
 # secrets files as long as you replace their contents by environment
 # variables.
 /config/*.secret.exs
+/config/generated_config.exs
 
 # Database setup file, some may forget to delete it
 /config/setup_db.psql

README.md

@@ -8,76 +8,68 @@ Pleroma is written in Elixir, high-performance and can run on small devices like
 
 For clients it supports both the [GNU Social API with Qvitter extensions](https://twitter-api.readthedocs.io/en/latest/index.html) and the [Mastodon client API](https://github.com/tootsuite/documentation/blob/master/Using-the-API/API.md).
 
-Client applications that are known to work well:
+- [Client Applications for Pleroma](docs/Clients.md)
 
-* Twidere
+No release has been made yet, but several servers have been online for months already. If you want to run your own server, feel free to contact us at @lain@pleroma.soykaf.com or in our dev chat at #pleroma on freenode or via matrix at <https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org>.
-* Tusky
-* Pawoo (Android + iOS)
-* Subway Tooter
-* Amaroq (iOS)
-* Tootdon (Android + iOS)
-* Tootle (iOS)
-* Whalebird (Windows + Mac + Linux)
-
-No release has been made yet, but several servers have been online for months already. If you want to run your own server, feel free to contact us at @lain@pleroma.soykaf.com or in our dev chat at #pleroma on freenode or via matrix at https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org.
 
 ## Installation
 
 ### Docker
 
-While we don't provide docker files, other people have written very good ones. Take a look at https://github.com/angristan/docker-pleroma or https://github.com/sn0w/pleroma-docker.
+While we don’t provide docker files, other people have written very good ones. Take a look at <https://github.com/angristan/docker-pleroma> or <https://github.com/sn0w/pleroma-docker>.
 
 ### Dependencies
 
 * Postgresql version 9.6 or newer
-* Elixir version 1.7 or newer. If your distribution only has an old version available, check [Elixir's install page](https://elixir-lang.org/install.html) or use a tool like [asdf](https://github.com/asdf-vm/asdf).
+* Elixir version 1.7 or newer. If your distribution only has an old version available, check [Elixir’s install page](https://elixir-lang.org/install.html) or use a tool like [asdf](https://github.com/asdf-vm/asdf).
 * Build-essential tools
 
 ### Configuration
 
-  * Run `mix deps.get` to install elixir dependencies.
+* Run `mix deps.get` to install elixir dependencies.
-
+* Run `mix pleroma.instance gen`. This will ask you questions about your instance and generate a configuration file in `config/generated_config.exs`. Check that and copy it to either `config/dev.secret.exs` or `config/prod.secret.exs`. It will also create a `config/setup_db.psql`, which you should run as the PostgreSQL superuser (i.e., `sudo -u postgres psql -f config/setup_db.psql`). It will create the database, user, and password you gave `mix pleroma.gen.instance` earlier, as well as set up the necessary extensions in the database. PostgreSQL superuser privileges are only needed for this step.
-  * Run `mix pleroma.instance gen`. This will ask you questions about your instance and generate a configuration file in `config/generated_config.exs`. Check that and copy it to either `config/dev.secret.exs` or `config/prod.secret.exs`. It will also create a `config/setup_db.psql`, which you should run as the PostgreSQL superuser (i.e., `sudo -u postgres psql -f config/setup_db.psql`). It will create the database, user, and password you gave `mix pleroma.gen.instance` earlier, as well as set up the necessary extensions in the database. PostgreSQL superuser privileges are only needed for this step.
+* For these next steps, the default will be to run pleroma using the dev configuration file, `config/dev.secret.exs`. To run them using the prod config file, prefix each command at the shell with `MIX_ENV=prod`. For example: `MIX_ENV=prod mix phx.server`. Documentation for the config can be found at [`docs/config.md`](docs/config.md)
-
+* Run `mix ecto.migrate` to run the database migrations. You will have to do this again after certain updates.
-  * For these next steps, the default will be to run pleroma using the dev configuration file, `config/dev.secret.exs`. To run them using the prod config file, prefix each command at the shell with `MIX_ENV=prod`. For example: `MIX_ENV=prod mix phx.server`. Documentation for the config can be found at [``docs/config.md``](docs/config.md)
+* You can check if your instance is configured correctly by running it with `mix phx.server` and checking the instance info endpoint at `/api/v1/instance`. If it shows your uri, name and email correctly, you are configured correctly. If it shows something like `localhost:4000`, your configuration is probably wrong, unless you are running a local development setup.
-
+* The common and convenient way for adding HTTPS is by using Nginx as a reverse proxy. You can look at example Nginx configuration in `installation/pleroma.nginx`. If you need TLS/SSL certificates for HTTPS, you can look get some for free with letsencrypt: <https://letsencrypt.org/>. The simplest way to obtain and install a certificate is to use [Certbot.](https://certbot.eff.org) Depending on your specific setup, certbot may be able to get a certificate and configure your web server automatically.
-  * Run `mix ecto.migrate` to run the database migrations. You will have to do this again after certain updates.
-
-  * You can check if your instance is configured correctly by running it with `mix phx.server` and checking the instance info endpoint at `/api/v1/instance`. If it shows your uri, name and email correctly, you are configured correctly. If it shows something like `localhost:4000`, your configuration is probably wrong, unless you are running a local development setup.
-
-  * The common and convenient way for adding HTTPS is by using Nginx as a reverse proxy. You can look at example Nginx configuration in `installation/pleroma.nginx`. If you need TLS/SSL certificates for HTTPS, you can look get some for free with letsencrypt: <https://letsencrypt.org/>. The simplest way to obtain and install a certificate is to use [Certbot.](https://certbot.eff.org) Depending on your specific setup, certbot may be able to get a certificate and configure your web server automatically.
 
 ## Running
 
-* By default, it listens on port 4000 (TCP), so you can access it on http://localhost:4000/ (if you are on the same machine). In case of an error it will restart automatically.
+* By default, it listens on port 4000 (TCP), so you can access it on <http://localhost:4000/> (if you are on the same machine). In case of an error it will restart automatically.
 
 ### Frontends
+
 Pleroma comes with two frontends. The first one, Pleroma FE, can be reached by normally visiting the site. The other one, based on the Mastodon project, can be found by visiting the /web path of your site.
 
 ### As systemd service (with provided .service file)
-Example .service file can be found in `installation/pleroma.service`. Copy this to `/etc/systemd/system/`.
+
-Running `systemctl enable --now pleroma.service` will run Pleroma and enable startup on boot.
+Example .service file can be found in `installation/pleroma.service`. Copy this to `/etc/systemd/system/`. Running `systemctl enable --now pleroma.service` will run Pleroma and enable startup on boot. Logs can be watched by using `journalctl -fu pleroma.service`.
-Logs can be watched by using `journalctl -fu pleroma.service`.
 
 ### As OpenRC service (with provided RC file)
-Copy ``installation/init.d/pleroma`` to ``/etc/init.d/pleroma``.
+
-You can add it to the services ran by default with:
+Copy `installation/init.d/pleroma` to `/etc/init.d/pleroma`. You can add it to the services ran by default with: `rc-update add pleroma`
-``rc-update add pleroma``
 
 ### Standalone/run by other means
-Run `mix phx.server` in repository's root, it will output log into stdout/stderr.
+
+Run `mix phx.server` in repository’s root, it will output log into stdout/stderr.
 
 ### Using an upstream proxy for federation
 
-Add the following to your `dev.secret.exs` or `prod.secret.exs` if you want to proxify all http requests that pleroma makes to an upstream proxy server:
+Add the following to your `dev.secret.exs` or `prod.secret.exs` if you want to proxify all http requests that Pleroma makes to an upstream proxy server:
 
-    config :pleroma, :http,
+```elixir
-      proxy_url: "127.0.0.1:8123"
+config :pleroma, :http,
+  proxy_url: "127.0.0.1:8123"
+```
 
-This is useful for running pleroma inside Tor or i2p.
+This is useful for running Pleroma inside Tor or I2P.
+
+## Customization and contribution
+
+The [Pleroma Wiki](https://git.pleroma.social/pleroma/pleroma/wikis/home) offers manuals and guides on how to further customize your instance to your liking and how you can contribute to the project.
 
 ## Troubleshooting
 
 ### No incoming federation
 
-Check that you correctly forward the "host" header to backend. It is needed to validate signatures.
+Check that you correctly forward the `host` header to the backend. It is needed to validate signatures.

config/config.exs

@@ -15,6 +15,20 @@
   seconds_valid: 60,
   method: Pleroma.Captcha.Kocaptcha
 
+config :pleroma, :hackney_pools,
+  federation: [
+    max_connections: 50,
+    timeout: 150_000
+  ],
+  media: [
+    max_connections: 50,
+    timeout: 150_000
+  ],
+  upload: [
+    max_connections: 25,
+    timeout: 300_000
+  ]
+
 config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
 
 # Upload configuration
@@ -22,7 +36,14 @@
   uploader: Pleroma.Uploaders.Local,
   filters: [],
   proxy_remote: false,
-  proxy_opts: []
+  proxy_opts: [
+    redirect_on_failure: false,
+    max_body_length: 25 * 1_048_576,
+    http: [
+      follow_redirect: true,
+      pool: :upload
+    ]
+  ]
 
 config :pleroma, Pleroma.Uploaders.Local, uploads: "uploads"
 
@@ -91,6 +112,12 @@
   format: "$time $metadata[$level] $message\n",
   metadata: [:request_id]
 
+config :logger, :ex_syslogger,
+  level: :debug,
+  ident: "Pleroma",
+  format: "$metadata[$level] $message",
+  metadata: [:request_id]
+
 config :mime, :types, %{
   "application/xml" => ["xml"],
   "application/xrd+xml" => ["xrd+xml"],
@@ -119,6 +146,7 @@
   banner_upload_limit: 4_000_000,
   registrations_open: true,
   federating: true,
+  federation_reachability_timeout_days: 7,
   allow_relay: true,
   rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,
   public: true,
@@ -131,7 +159,10 @@
     "text/markdown"
   ],
   finmoji_enabled: true,
-  mrf_transparency: true
+  mrf_transparency: true,
+  autofollowed_nicknames: [],
+  max_pinned_statuses: 1,
+  no_attachment_links: false
 
 config :pleroma, :markup,
   # XXX - unfortunately, inline images must be enabled by default right now, because
@@ -145,6 +176,7 @@
     Pleroma.HTML.Scrubber.Default
   ]
 
+# Deprecated, will be gone in 1.0
 config :pleroma, :fe,
   theme: "pleroma-dark",
   logo: "/static/logo.png",
@@ -163,6 +195,24 @@
   subject_line_behavior: "email",
   always_show_subject_input: true
 
+config :pleroma, :frontend_configurations,
+  pleroma_fe: %{
+    theme: "pleroma-dark",
+    logo: "/static/logo.png",
+    background: "/images/city.jpg",
+    redirectRootNoLogin: "/main/all",
+    redirectRootLogin: "/main/friends",
+    showInstanceSpecificPanel: true,
+    scopeOptionsEnabled: false,
+    formattingOptionsEnabled: false,
+    collapseMessageWithSubject: false,
+    hidePostStats: false,
+    hideUserStats: false,
+    scopeCopy: true,
+    subjectLineBehavior: "email",
+    alwaysShowSubjectInput: true
+  }
+
 config :pleroma, :activitypub,
   accept_blocks: true,
   unfollow_blocked: true,
@@ -177,7 +227,9 @@
   allow_followersonly: false,
   allow_direct: false
 
-config :pleroma, :mrf_hellthread, threshold: 10
+config :pleroma, :mrf_hellthread,
+  delist_threshold: 5,
+  reject_threshold: 10
 
 config :pleroma, :mrf_simple,
   media_removal: [],
@@ -186,7 +238,23 @@
   reject: [],
   accept: []
 
-config :pleroma, :media_proxy, enabled: false
+config :pleroma, :mrf_keyword,
+  reject: [],
+  federated_timeline_removal: [],
+  replace: []
+
+config :pleroma, :rich_media, enabled: true
+
+config :pleroma, :media_proxy,
+  enabled: false,
+  proxy_opts: [
+    redirect_on_failure: false,
+    max_body_length: 25 * 1_048_576,
+    http: [
+      follow_redirect: true,
+      pool: :media
+    ]
+  ]
 
 config :pleroma, :chat, enabled: true
 
@@ -199,6 +267,8 @@
   ip: {0, 0, 0, 0},
   port: 9999
 
+config :pleroma, Pleroma.Web.Metadata, providers: [], unfurl_nsfw: false
+
 config :pleroma, :suggestions,
   enabled: false,
   third_party_engine:
@@ -230,34 +300,34 @@
 
 config :pleroma, Pleroma.User,
   restricted_nicknames: [
-    "about",
-    "~",
-    "main",
-    "users",
-    "settings",
-    "objects",
-    "activities",
-    "web",
-    "registration",
-    "friend-requests",
-    "pleroma",
-    "api",
-    "tag",
-    "notice",
-    "status",
-    "user-search",
-    "ostatus_subscribe",
-    "oauth",
-    "push",
-    "relay",
-    "inbox",
     ".well-known",
-    "nodeinfo",
+    "~",
+    "about",
+    "activities",
+    "api",
     "auth",
-    "proxy",
     "dev",
+    "friend-requests",
+    "inbox",
     "internal",
-    "media"
+    "main",
+    "media",
+    "nodeinfo",
+    "notice",
+    "oauth",
+    "objects",
+    "ostatus_subscribe",
+    "pleroma",
+    "proxy",
+    "push",
+    "registration",
+    "relay",
+    "settings",
+    "status",
+    "tag",
+    "user-search",
+    "users",
+    "web"
   ]
 
 config :pleroma, Pleroma.Web.Federator, max_jobs: 50

config/dev.exs

@@ -16,7 +16,8 @@
   debug_errors: true,
   code_reloader: true,
   check_origin: false,
-  watchers: []
+  watchers: [],
+  secure_cookie_flag: false
 
 config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Local
 

config/test.exs

@@ -36,6 +36,7 @@
 config :pleroma, :websub, Pleroma.Web.WebsubMock
 config :pleroma, :ostatus, Pleroma.Web.OStatusMock
 config :tesla, adapter: Tesla.Mock
+config :pleroma, :rich_media, enabled: false
 
 config :web_push_encryption, :vapid_details,
   subject: "mailto:administrator@example.com",

docs/Clients.md

@@ -0,0 +1,100 @@
+# Pleroma Clients
+Note: Additionnal clients may be working but theses are officially supporting Pleroma.
+Feel free to contact us to be added to this list!
+
+## Desktop
+### Roma for Desktop
+- Homepage: <http://www.pleroma.com/desktop-app/>
+- Source Code: ???
+- Platforms: Windows, Mac, (Linux?)
+- Features: Streaming Ready
+
+### Social
+- Source Code: <https://gitlab.gnome.org/BrainBlasted/Social>
+- Contact: [@brainblasted@social.libre.fi](https://social.libre.fi/users/brainblasted)
+- Platforms: Linux (GNOME)
+- Note(2019-01-28): Not at a pre-alpha stage yet
+
+### Whalebird
+- Homepage: <https://whalebird.org/>
+- Source Code: <https://github.com/h3poteto/whalebird-desktop>
+- Contact: [@h3poteto@pleroma.io](https://pleroma.io/users/h3poteto)
+- Platforms: Windows, Mac, Linux
+- Features: Streaming Ready
+
+## Handheld
+### Amaroq
+- Homepage: <https://itunes.apple.com/us/app/amaroq-for-mastodon/id1214116200>
+- Source Code: <https://github.com/ReticentJohn/Amaroq>
+- Contact: [@eurasierboy@mastodon.social](https://mastodon.social/users/eurasierboy)
+- Platforms: iOS
+- Features: No Streaming
+
+### Nekonium
+- Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/)
+- Source: <https://git.gdgd.jp.net/lin/nekonium/>
+- Contact: [@lin@pleroma.gdgd.jp.net](https://pleroma.gdgd.jp.net/users/lin)
+- Platforms: Android
+- Features: Streaming Ready
+
+### Mastalab
+- Source Code: <https://gitlab.com/tom79/mastalab/>
+- Contact: [@tom79@mastodon.social](https://mastodon.social/users/tom79)
+- Platforms: Android
+- Features: Streaming Ready
+
+### Roma
+- Homepage: <http://www.pleroma.com/>
+- Source Code: ???
+- Platforms: iOS, Android
+- Features: No Streaming
+
+### Tootdon
+- Homepage: <http://tootdon.club/>, <http://blog.mastodon-tootdon.com/>
+- Source Code: ???
+- Contact: [@tootdon@mstdn.jp](https://mstdn.jp/users/tootdon)
+- Platforms: Android, iOS
+- Features: No Streaming
+
+### Tusky
+- Homepage: <https://tuskyapp.github.io/>
+- Source Code: <https://github.com/tuskyapp/Tusky>
+- Contact: [@ConnyDuck@mastodon.social](https://mastodon.social/users/ConnyDuck)
+- Platforms: Android
+- Features: No Streaming
+
+### Twidere
+- Homepage: <https://twidere.mariotaku.org/>
+- Source Code: <https://github.com/TwidereProject/Twidere-Android/>, <https://github.com/TwidereProject/Twidere-iOS/>
+- Contact: <me@mariotaku.org>
+- Platform: Android, iOS
+- Features: No Streaming
+
+## Alternative Web Interfaces
+### Brutaldon
+- Homepage: <https://jfm.carcosa.net/projects/software/brutaldon/>
+- Source Code: <https://github.com/jfmcbrayer/brutaldon>
+- Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc)
+- Features: No Streaming
+
+### Feather
+- Source Code: <https://github.com/kaniini/feather>
+- Contact: [@kaniini@pleroma.site](https://pleroma.site/kaniini)
+- Features: No Streaming
+
+### Halcyon
+- Source Code: <https://notabug.org/halcyon-suite/halcyon>
+- Contact: [@halcyon@social.csswg.org](https://social.csswg.org/users/halcyon)
+- Features: Streaming Ready
+
+### Pinafore
+- Homepage: <https://pinafore.social/>
+- Source Code: <https://github.com/nolanlawson/pinafore>
+- Contact: [@pinafore@mastodon.technology](https://mastodon.technology/users/pinafore)
+- Note: Pleroma support is a secondary goal
+- Features: No Streaming
+
+### Sengi
+- Source Code: <https://github.com/NicolasConstant/sengi>
+- Contact: [@sengi_app@mastodon.social](https://mastodon.social/users/sengi_app)
+- Note(2019-01-28): The development is currently in a early stage.

docs/Pleroma-API.md

@@ -1,13 +1,9 @@
-# Authentication
+# Pleroma API
 
 Requests that require it can be authenticated with [an OAuth token](https://tools.ietf.org/html/rfc6749), the `_pleroma_key` cookie, or [HTTP Basic Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization).
 
-# Request parameters
-
 Request parameters can be passed via [query strings](https://en.wikipedia.org/wiki/Query_string) or as [form data](https://www.w3.org/TR/html401/interact/forms.html). Files must be uploaded as `multipart/form-data`.
 
-# Endpoints
-
 ## `/api/pleroma/emoji`
 ### Lists the custom emoji on that server.
 * Method: `GET`
@@ -15,6 +11,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Params: none
 * Response: JSON
 * Example response: `{"kalsarikannit_f":"/finmoji/128px/kalsarikannit_f-128.png","perkele":"/finmoji/128px/perkele-128.png","blobdab":"/emoji/blobdab.png","happiness":"/finmoji/128px/happiness-128.png"}`
+* Note: Same data as Mastodon API’s `/api/v1/custom_emojis` but in a different format
 
 ## `/api/pleroma/follow_import`
 ### Imports your follows, for example from a Mastodon CSV file.
@@ -55,6 +52,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
     * `confirm`
     * `captcha_solution`: optional, contains provider-specific captcha solution,
     * `captcha_token`: optional, contains provider-specific captcha token
+    * `token`: invite token required when the registerations aren't public.
 * Response: JSON. Returns a user object on success, otherwise returns `{"error": "error_msg"}`
 * Example response:
 ```

docs/config.md

@@ -17,7 +17,7 @@ Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`.
 
 ## Pleroma.Upload.Filter.Mogrify
 
-* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", {"impode", "1"}]`.
+* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"impode", "1"}]`.
 
 ## Pleroma.Upload.Filter.Dedupe
 
@@ -32,7 +32,7 @@ This filter replaces the filename (not the path) of an upload. For complete obfu
 
 ## Pleroma.Mailer
 * `adapter`: one of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters), or `Swoosh.Adapters.Local` for in-memory mailbox.
-* `api_key` / `password` and / or other adapter-specific settings, per the above documentation. 
+* `api_key` / `password` and / or other adapter-specific settings, per the above documentation.
 
 An example for Sendgrid adapter:
 
@@ -72,6 +72,7 @@ config :pleroma, Pleroma.Mailer,
 * `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`).
 * `account_activation_required`: Require users to confirm their emails before signing in.
 * `federating`: Enable federation with other instances
+* `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it.
 * `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance
 * `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default:
   * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default)
@@ -93,13 +94,57 @@ config :pleroma, Pleroma.Mailer,
 * `always_show_subject_input`: When set to false, auto-hide the subject field when it's empty.
 * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with
     older software for theses nicknames.
+* `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature.
+* `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow.
+* `no_attachment_links`: Set to true to disable automatically adding attachment link text to statuses
+
+## :logger
+* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog
+
+An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed:
+```
+config :logger, 
+  backends: [{ExSyslogger, :ex_syslogger}]
+
+config :logger, :ex_syslogger,
+  level: :warn
+```
+
+Another example, keeping console output and adding the pid to syslog output:
+```
+config :logger,
+  backends: [:console, {ExSyslogger, :ex_syslogger}]
+
+config :logger, :ex_syslogger,
+  level: :warn,
+  option: [:pid, :ndelay]
+```
+
+See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/)
+
+
+## :frontend_configurations
+
+This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` are configured.
+
+Frontends can access these settings at `/api/pleroma/frontend_configurations`
+
+To add your own configuration for PleromaFE, use it like this:
+
+`config :pleroma, :frontend_configurations, pleroma_fe: %{redirectRootNoLogin: "/main/all", ...}`
+
+These settings need to be complete, they will override the defaults. See `priv/static/static/config.json` for the available keys.
 
 ## :fe
+__THIS IS DEPRECATED__
+
+If you are using this method, please change it to the `frontend_configurations` method. Please set this option to false in your config like this: `config :pleroma, :fe, false`.
+
 This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false.
 
 * `theme`: Which theme to use, they are defined in ``styles.json``
 * `logo`: URL of the logo, defaults to Pleroma’s logo
-* `logo_mask`: Whenether to mask the logo
+* `logo_mask`: Whether to use only the logo's shape as a mask (true) or as a regular image (false)
 * `logo_margin`: What margin to use around the logo
 * `background`: URL of the background, unless viewing a user profile with a background that is set
 * `redirect_root_no_login`: relative URL which indicates where to redirect when a user isn’t logged in.
@@ -123,7 +168,13 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
 * `allow_direct`: whether to allow direct messages
 
 ## :mrf_hellthread
-* `threshold`: Number of mentioned users after which the message gets discarded as spam
+* `delist_threshold`: Number of mentioned users after which the message gets delisted (the message can still be seen, but it will not show up in public timelines and mentioned users won't get notifications about it). Set to 0 to disable.
+* `reject_threshold`: Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.
+
+## :mrf_keyword
+* `reject`: A list of patterns which result in message being rejected, each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html)
+* `federated_timeline_removal`: A list of patterns which result in message being removed from federated timelines (a.k.a unlisted), each pattern can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html)
+* `replace`: A list of tuples containing `{pattern, replacement}`, `pattern` can be a string or a [regular expression](https://hexdocs.pm/elixir/Regex.html)
 
 ## :media_proxy
 * `enabled`: Enables proxying of remote media to the instance’s proxy
@@ -201,6 +252,32 @@ curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerando
 ## Pleroma.Web.Federator.RetryQueue
 
 * `enabled`: If set to `true`, failed federation jobs will be retried
-* `max_jobs`: The maximum amount of parallel federation jbos running at the same time.
+* `max_jobs`: The maximum amount of parallel federation jobs running at the same time.
 * `initial_timeout`: The initial timeout in seconds
 * `max_retries`: The maximum number of times a federation job is retried
+
+## Pleroma.Web.Metadata
+* `providers`: a list of metadata providers to enable. Providers availible:
+  * Pleroma.Web.Metadata.Providers.OpenGraph
+  * Pleroma.Web.Metadata.Providers.TwitterCard
+* `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews
+
+## :rich_media
+* `enabled`: if enabled the instance will parse metadata from attached links to generate link previews
+
+## :hackney_pools
+
+Advanced. Tweaks Hackney (http client) connections pools.
+
+There's three pools used:
+
+* `:federation` for the federation jobs.
+  You may want this pool max_connections to be at least equal to the number of federator jobs + retry queue jobs.
+* `:media` for rich media, media proxy
+* `:upload` for uploaded media (if using a remote uploader and `proxy_remote: true`)
+
+For each pool, the options are:
+
+* `max_connections` - how much connections a pool can hold
+* `timeout` - retention duration for connections
+

installation/caddyfile-pleroma.example

@@ -23,6 +23,11 @@ example.tld  {
 
   # If you do not want to use the mediaproxy function, remove these lines.
   # To use this directive, you need the http.cache plugin for Caddy.
+  cache {
+    match_path /media
+    default_max_age 720m
+  }
+
   cache {
     match_path /proxy
     default_max_age 720m

installation/init.d/pleroma

@@ -1,7 +1,7 @@
 #!/sbin/openrc-run
 
 # Requires OpenRC >= 0.35
-directory=~pleroma/pleroma
+directory=/opt/pleroma
 
 command=/usr/bin/mix
 command_args="phx.server"
@@ -12,10 +12,10 @@ export PORT=4000
 export MIX_ENV=prod
 
 # Ask process to terminate within 30 seconds, otherwise kill it
-retry="SIGTERM/30 SIGKILL/5"
+retry="SIGTERM/30/SIGKILL/5"
 
 pidfile="/var/run/pleroma.pid"
 
 depend() {
     need nginx postgresql
-}
+}

installation/pleroma.nginx

@@ -15,12 +15,13 @@ server {
     return         301 https://$server_name$request_uri;
 
     # Uncomment this if you need to use the 'webroot' method with certbot. Make sure
-    # that you also create the .well-known/acme-challenge directory structure in pleroma/priv/static and
+    # that the directory exists and that it is accessible by the webserver. If you followed
-    # that is is accessible by the webserver. You may need to load this file with the ssl
+    # the guide, you already ran 'sudo mkdir -p /var/lib/letsencrypt' to create the folder.
-    # server block commented out, run certbot to get the certificate, and then uncomment it.
+    # You may need to load this file with the ssl server block commented out, run certbot
+    # to get the certificate, and then uncomment it.
     #
     # location ~ /\.well-known/acme-challenge {
-    #     root <path to install>/pleroma/priv/static/;
+    #     root /var/lib/letsencrypt/.well-known/acme-challenge;
     # }
 }
 
@@ -79,8 +80,10 @@ server {
         proxy_cache_valid  200 206 301 304 1h;
         proxy_cache_lock on;
         proxy_ignore_client_abort on;
-        proxy_buffering off;
+        proxy_buffering on;
         chunked_transfer_encoding on;
+        proxy_ignore_headers Cache-Control;
+        proxy_hide_header Cache-Control;
         proxy_pass http://localhost:4000;
     }
 }

installation/pleroma.service

@@ -3,18 +3,28 @@ Description=Pleroma social network
 After=network.target postgresql.service
 
 [Service]
-User=pleroma
-WorkingDirectory=/home/pleroma/pleroma
-Environment="HOME=/home/pleroma"
-Environment="MIX_ENV=prod"
-ExecStart=/usr/local/bin/mix phx.server
 ExecReload=/bin/kill $MAINPID
 KillMode=process
 Restart=on-failure
 
+; Name of the user that runs the Pleroma service.
+User=pleroma
+; Declares that Pleroma runs in production mode.
+Environment="MIX_ENV=prod"
+
+; Make sure that all paths fit your installation.
+; Path to the home directory of the user running the Pleroma service.
+Environment="HOME=/var/lib/pleroma"
+; Path to the folder containing the Pleroma installation.
+WorkingDirectory=/opt/pleroma
+; Path to the Mix binary.
+ExecStart=/usr/bin/mix phx.server
+
 ; Some security directives.
 ; Use private /tmp and /var/tmp folders inside a new file system namespace, which are discarded after the process stops.
 PrivateTmp=true
+; The /home, /root, and /run/user folders can not be accessed by this service anymore. If your Pleroma user has its home folder in one of the restricted places, or use one of these folders as its working directory, you have to set this to false.
+ProtectHome=true
 ; Mount /usr, /boot, and /etc as read-only for processes invoked by this service.
 ProtectSystem=full
 ; Sets up a new /dev mount for the process and only adds API pseudo devices like /dev/null, /dev/zero or /dev/random but not physical devices. Disabled by default because it may not work on devices like the Raspberry Pi.

installation/pleroma.vcl

@@ -14,43 +14,45 @@ acl purge {
 sub vcl_recv {
     # Redirect HTTP to HTTPS
     if (std.port(server.ip) != 443) {
-        set req.http.x-redir = "https://" + req.http.host + req.url;
+      set req.http.x-redir = "https://" + req.http.host + req.url;
-        return (synth(750, ""));
+      return (synth(750, ""));
+    }
+
+    # CHUNKED SUPPORT
+    if (req.http.Range ~ "bytes=") {
+      set req.http.x-range = req.http.Range;
     }
 
     # Pipe if WebSockets request is coming through
     if (req.http.upgrade ~ "(?i)websocket") {
-        return (pipe);
+      return (pipe);
     }
 
     # Allow purging of the cache
     if (req.method == "PURGE") {
-        if (!client.ip ~ purge) {
+      if (!client.ip ~ purge) {
-          return(synth(405,"Not allowed."));
+        return(synth(405,"Not allowed."));
-        }
+      }
-        return(purge);
+      return(purge);
     }
 
     # Pleroma MediaProxy - strip headers that will affect caching
     if (req.url ~ "^/proxy/") {
-        unset req.http.Cookie;
+      unset req.http.Cookie;
-        unset req.http.Authorization;
+      unset req.http.Authorization;
-        unset req.http.Accept;
+      unset req.http.Accept;
-        return (hash);
+      return (hash);
     }
 
     # Strip headers that will affect caching from all other static content
     # This also permits caching of individual toots and AP Activities
     if ((req.url ~ "^/(media|static)/") ||
-    (req.url ~ "(?i)\.(html|js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|ttf|pdf|woff|woff2)$"))
+    (req.url ~ "(?i)\.(html|js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|mp4|ogg|webm|svg|swf|ttf|pdf|woff|woff2)$"))
     {
       unset req.http.Cookie;
       unset req.http.Authorization;
       return (hash);
     }
-
-    # Everything else should just be piped to Pleroma
-    return (pipe);
 }
 
 sub vcl_backend_response {
@@ -59,8 +61,11 @@ sub vcl_backend_response {
       set beresp.do_gzip = true;
     }
 
-    # etags are bad
+    # CHUNKED SUPPORT
-    unset beresp.http.etag;
+    if (bereq.http.x-range ~ "bytes=" && beresp.status == 206) {
+      set beresp.ttl = 10m;
+      set beresp.http.CR = beresp.http.content-range;
+    }
 
     # Don't cache objects that require authentication
     if (beresp.http.Authorization && !beresp.http.Cache-Control ~ "public") {
@@ -81,9 +86,9 @@ sub vcl_backend_response {
 
     # Do not cache redirects and errors
     if ((beresp.status >= 300) && (beresp.status < 500)) {
-        set beresp.uncacheable = true;
+      set beresp.uncacheable = true;
-        set beresp.ttl = 30s;
+      set beresp.ttl = 30s;
-        return (deliver);
+      return (deliver);
     }
 
     # Pleroma MediaProxy internally sets headers properly
@@ -92,14 +97,12 @@ sub vcl_backend_response {
     }
 
     # Strip cache-restricting headers from Pleroma on static content that we want to cache
-    # Also enable streaming of cached content to clients (no waiting for Varnish to complete backend fetch)
+    if (bereq.url ~ "(?i)\.(js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|mp4|ogg|webm|svg|swf|ttf|pdf|woff|woff2)$")
-    if (bereq.url ~ "(?i)\.(js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|ttf|pdf|woff|woff2)$")
     {
       unset beresp.http.set-cookie;
       unset beresp.http.Cache-Control;
       unset beresp.http.x-request-id;
       set beresp.http.Cache-Control = "public, max-age=86400";
-      set beresp.do_stream = true;
     }
 }
 
@@ -115,7 +118,30 @@ sub vcl_synth {
 # Ensure WebSockets through the pipe do not close prematurely
 sub vcl_pipe {
     if (req.http.upgrade) {
-        set bereq.http.upgrade = req.http.upgrade;
+      set bereq.http.upgrade = req.http.upgrade;
-        set bereq.http.connection = req.http.connection;
+      set bereq.http.connection = req.http.connection;
+    }
+}
+
+sub vcl_hash {
+    # CHUNKED SUPPORT
+    if (req.http.x-range ~ "bytes=") {
+      hash_data(req.http.x-range);
+      unset req.http.Range;
+    }
+}
+
+sub vcl_backend_fetch {
+    # CHUNKED SUPPORT
+    if (bereq.http.x-range) {
+      set bereq.http.Range = bereq.http.x-range;
+    }
+}
+
+sub vcl_deliver {
+    # CHUNKED SUPPORT
+    if (resp.http.CR) {
+      set resp.http.Content-Range = resp.http.CR;
+      unset resp.http.CR;
     }
 }

lib/mix/tasks/pleroma/instance.ex

@@ -105,6 +105,7 @@ def run(["gen" | rest]) do
         )
 
       secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
+      signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8)
       {web_push_public_key, web_push_private_key} = :crypto.generate_key(:ecdh, :prime256v1)
 
       result_config =
@@ -120,6 +121,7 @@ def run(["gen" | rest]) do
           dbpass: dbpass,
           version: Pleroma.Mixfile.project() |> Keyword.get(:version),
           secret: secret,
+          signing_salt: signing_salt,
           web_push_public_key: Base.url_encode64(web_push_public_key, padding: false),
           web_push_private_key: Base.url_encode64(web_push_private_key, padding: false)
         )

lib/mix/tasks/pleroma/sample_config.eex

@@ -7,7 +7,8 @@ use Mix.Config
 
 config :pleroma, Pleroma.Web.Endpoint,
    url: [host: "<%= domain %>", scheme: "https", port: <%= port %>],
-   secret_key_base: "<%= secret %>"
+   secret_key_base: "<%= secret %>",
+   signing_salt: "<%= signing_salt %>"
 
 config :pleroma, :instance,
   name: "<%= name %>",

lib/mix/tasks/pleroma/uploads.ex

@@ -4,7 +4,8 @@
 
 defmodule Mix.Tasks.Pleroma.Uploads do
   use Mix.Task
-  alias Pleroma.{Upload, Uploaders.Local}
+  alias Pleroma.Upload
+  alias Pleroma.Uploaders.Local
   alias Mix.Tasks.Pleroma.Common
   require Logger
 
@@ -20,7 +21,7 @@ defmodule Mix.Tasks.Pleroma.Uploads do
    - `--delete` - delete local uploads after migrating them to the target uploader
 
 
-   A list of avalible uploaders can be seen in config.exs
+   A list of available uploaders can be seen in config.exs
   """
   def run(["migrate_local", target_uploader | args]) do
     delete? = Enum.member?(args, "--delete")
@@ -96,6 +97,7 @@ def run(["migrate_local", target_uploader | args]) do
       timeout: 150_000
     )
     |> Stream.chunk_every(@log_every)
+    # credo:disable-for-next-line Credo.Check.Warning.UnusedEnumOperation
     |> Enum.reduce(0, fn done, count ->
       count = count + length(done)
       Mix.shell().info("Uploaded #{count}/#{total_count} files")

lib/mix/tasks/pleroma/user.ex

@@ -5,7 +5,8 @@
 defmodule Mix.Tasks.Pleroma.User do
   use Mix.Task
   import Ecto.Changeset
-  alias Pleroma.{Repo, User}
+  alias Pleroma.Repo
+  alias Pleroma.User
   alias Mix.Tasks.Pleroma.Common
 
   @shortdoc "Manages Pleroma users"
@@ -22,6 +23,7 @@ defmodule Mix.Tasks.Pleroma.User do
   - `--password PASSWORD` - the user's password
   - `--moderator`/`--no-moderator` - whether the user is a moderator
   - `--admin`/`--no-admin` - whether the user is an admin
+  - `-y`, `--assume-yes`/`--no-assume-yes` - whether to assume yes to all questions 
 
   ## Generate an invite link.
 
@@ -51,6 +53,14 @@ defmodule Mix.Tasks.Pleroma.User do
   - `--locked`/`--no-locked` - whether the user's account is locked
   - `--moderator`/`--no-moderator` - whether the user is a moderator
   - `--admin`/`--no-admin` - whether the user is an admin
+
+  ## Add tags to a user.
+
+      mix pleroma.user tag NICKNAME TAGS
+
+  ## Delete tags from a user.
+
+      mix pleroma.user untag NICKNAME TAGS
   """
   def run(["new", nickname, email | rest]) do
     {options, [], []} =
@@ -61,7 +71,11 @@ def run(["new", nickname, email | rest]) do
           bio: :string,
           password: :string,
           moderator: :boolean,
-          admin: :boolean
+          admin: :boolean,
+          assume_yes: :boolean
+        ],
+        aliases: [
+          y: :assume_yes
         ]
       )
 
@@ -79,6 +93,7 @@ def run(["new", nickname, email | rest]) do
 
     moderator? = Keyword.get(options, :moderator, false)
     admin? = Keyword.get(options, :admin, false)
+    assume_yes? = Keyword.get(options, :assume_yes, false)
 
     Mix.shell().info("""
     A user will be created with the following information:
@@ -93,7 +108,7 @@ def run(["new", nickname, email | rest]) do
       - admin: #{if(admin?, do: "true", else: "false")}
     """)
 
-    proceed? = Mix.shell().yes?("Continue?")
+    proceed? = assume_yes? or Mix.shell().yes?("Continue?")
 
     unless not proceed? do
       Common.start_pleroma()
@@ -197,7 +212,7 @@ def run(["unsubscribe", nickname]) do
 
       user = Repo.get(User, user.id)
 
-      if length(user.following) == 0 do
+      if Enum.empty?(user.following) do
         Mix.shell().info("Successfully unsubscribed all followers from #{user.nickname}")
       end
     else
@@ -243,6 +258,32 @@ def run(["set", nickname | rest]) do
     end
   end
 
+  def run(["tag", nickname | tags]) do
+    Common.start_pleroma()
+
+    with %User{} = user <- User.get_by_nickname(nickname) do
+      user = user |> User.tag(tags)
+
+      Mix.shell().info("Tags of #{user.nickname}: #{inspect(tags)}")
+    else
+      _ ->
+        Mix.shell().error("Could not change user tags for #{nickname}")
+    end
+  end
+
+  def run(["untag", nickname | tags]) do
+    Common.start_pleroma()
+
+    with %User{} = user <- User.get_by_nickname(nickname) do
+      user = user |> User.untag(tags)
+
+      Mix.shell().info("Tags of #{user.nickname}: #{inspect(tags)}")
+    else
+      _ ->
+        Mix.shell().error("Could not change user tags for #{nickname}")
+    end
+  end
+
   def run(["invite"]) do
     Common.start_pleroma()
 

lib/pleroma/PasswordResetToken.ex

@@ -7,10 +7,12 @@ defmodule Pleroma.PasswordResetToken do
 
   import Ecto.Changeset
 
-  alias Pleroma.{User, PasswordResetToken, Repo}
+  alias Pleroma.User
+  alias Pleroma.Repo
+  alias Pleroma.PasswordResetToken
 
   schema "password_reset_tokens" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:token, :string)
     field(:used, :boolean, default: false)
 

lib/pleroma/activity.ex

@@ -4,10 +4,15 @@
 
 defmodule Pleroma.Activity do
   use Ecto.Schema
-  alias Pleroma.{Repo, Activity, Notification}
+
+  alias Pleroma.Repo
+  alias Pleroma.Activity
+  alias Pleroma.Notification
+
   import Ecto.Query
 
   @type t :: %__MODULE__{}
+  @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
 
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
   @mastodon_notification_types %{
@@ -36,25 +41,11 @@ def get_by_ap_id(ap_id) do
     )
   end
 
-  # TODO:
+  def get_by_id(id) do
-  # Go through these and fix them everywhere.
+    Repo.get(Activity, id)
-  # Wrong name, only returns create activities
-  def all_by_object_ap_id_q(ap_id) do
-    from(
-      activity in Activity,
-      where:
-        fragment(
-          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
-          activity.data,
-          activity.data,
-          ^to_string(ap_id)
-        ),
-      where: fragment("(?)->>'type' = 'Create'", activity.data)
-    )
   end
 
-  # Wrong name, returns all.
+  def by_object_ap_id(ap_id) do
-  def all_non_create_by_object_ap_id_q(ap_id) do
     from(
       activity in Activity,
       where:
@@ -67,12 +58,7 @@ def all_non_create_by_object_ap_id_q(ap_id) do
     )
   end
 
-  # Wrong name plz fix thx
+  def create_by_object_ap_id(ap_ids) when is_list(ap_ids) do
-  def all_by_object_ap_id(ap_id) do
-    Repo.all(all_by_object_ap_id_q(ap_id))
-  end
-
-  def create_activity_by_object_id_query(ap_ids) do
     from(
       activity in Activity,
       where:
@@ -86,19 +72,37 @@ def create_activity_by_object_id_query(ap_ids) do
     )
   end
 
-  def get_create_activity_by_object_ap_id(ap_id) when is_binary(ap_id) do
+  def create_by_object_ap_id(ap_id) do
-    create_activity_by_object_id_query([ap_id])
+    from(
+      activity in Activity,
+      where:
+        fragment(
+          "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
+          activity.data,
+          activity.data,
+          ^to_string(ap_id)
+        ),
+      where: fragment("(?)->>'type' = 'Create'", activity.data)
+    )
+  end
+
+  def get_all_create_by_object_ap_id(ap_id) do
+    Repo.all(create_by_object_ap_id(ap_id))
+  end
+
+  def get_create_by_object_ap_id(ap_id) when is_binary(ap_id) do
+    create_by_object_ap_id(ap_id)
     |> Repo.one()
   end
 
-  def get_create_activity_by_object_ap_id(_), do: nil
+  def get_create_by_object_ap_id(_), do: nil
 
   def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"])
   def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id)
   def normalize(_), do: nil
 
   def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do
-    get_create_activity_by_object_ap_id(ap_id)
+    get_create_by_object_ap_id(ap_id)
   end
 
   def get_in_reply_to_activity(_), do: nil

lib/pleroma/application.ex

@@ -6,11 +6,13 @@ defmodule Pleroma.Application do
   use Application
   import Supervisor.Spec
 
-  @name "Pleroma"
+  @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
+  @repository Mix.Project.config()[:source_url]
   def name, do: @name
   def version, do: @version
   def named_version(), do: @name <> " " <> @version
+  def repository, do: @repository
 
   def user_agent() do
     info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
@@ -22,6 +24,8 @@ def user_agent() do
   def start(_type, _args) do
     import Cachex.Spec
 
+    Pleroma.Config.DeprecationWarnings.warn()
+
     # Define workers and child supervisors to be supervised
     children =
       [
@@ -63,6 +67,17 @@ def start(_type, _args) do
           ],
           id: :cachex_object
         ),
+        worker(
+          Cachex,
+          [
+            :rich_media_cache,
+            [
+              default_ttl: :timer.minutes(120),
+              limit: 5000
+            ]
+          ],
+          id: :cachex_rich_media
+        ),
         worker(
           Cachex,
           [
@@ -88,11 +103,15 @@ def start(_type, _args) do
           ],
           id: :cachex_idem
         ),
-        worker(Pleroma.Web.Federator.RetryQueue, []),
+        worker(Pleroma.FlakeId, [])
-        worker(Pleroma.Web.Federator, []),
-        worker(Pleroma.Stats, []),
-        worker(Pleroma.Web.Push, [])
       ] ++
+        hackney_pool_children() ++
+        [
+          worker(Pleroma.Web.Federator.RetryQueue, []),
+          worker(Pleroma.Web.Federator, []),
+          worker(Pleroma.Stats, []),
+          worker(Pleroma.Web.Push, [])
+        ] ++
         streamer_child() ++
         chat_child() ++
         [
@@ -107,6 +126,20 @@ def start(_type, _args) do
     Supervisor.start_link(children, opts)
   end
 
+  def enabled_hackney_pools() do
+    [:media] ++
+      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
+        [:federation]
+      else
+        []
+      end ++
+      if Pleroma.Config.get([Pleroma.Uploader, :proxy_remote]) do
+        [:upload]
+      else
+        []
+      end
+  end
+
   if Mix.env() == :test do
     defp streamer_child(), do: []
     defp chat_child(), do: []
@@ -123,4 +156,11 @@ defp chat_child() do
       end
     end
   end
+
+  defp hackney_pool_children() do
+    for pool <- enabled_hackney_pools() do
+      options = Pleroma.Config.get([:hackney_pools, pool])
+      :hackney_pool.child_spec(pool, options)
+    end
+  end
 end

lib/pleroma/captcha/captcha.ex

@@ -3,9 +3,9 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha do
+  alias Calendar.DateTime
   alias Plug.Crypto.KeyGenerator
   alias Plug.Crypto.MessageEncryptor
-  alias Calendar.DateTime
 
   use GenServer
 

lib/pleroma/clippy.ex

@@ -0,0 +1,155 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Clippy do
+  @moduledoc false
+  # No software is complete until they have a Clippy implementation.
+  # A ballmer peak _may_ be required to change this module.
+
+  def tip() do
+    tips()
+    |> Enum.random()
+    |> puts()
+  end
+
+  def tips() do
+    host = Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
+
+    [
+      "“πλήρωμα” is “pleroma” in greek",
+      "For an extended Pleroma Clippy Experience, use the “Redmond” themes in Pleroma FE settings",
+      "Staff accounts and MRF policies of Pleroma instances are disclosed on the NodeInfo endpoints for easy transparency!\n
+- https://catgirl.science/misc/nodeinfo.lua?#{host}
+- https://fediverse.network/#{host}/federation",
+      "Pleroma can federate to the Dark Web!\n
+- Tor: https://git.pleroma.social/pleroma/pleroma/wikis/Easy%20Onion%20Federation%20(Tor)
+- i2p: https://git.pleroma.social/pleroma/pleroma/wikis/I2p%20federation",
+      "Lists of Pleroma instances:\n\n- http://distsn.org/pleroma-instances.html\n- https://fediverse.network/pleroma\n- https://the-federation.info/pleroma",
+      "Pleroma uses the LitePub protocol - https://litepub.social",
+      "To receive more federated posts, subscribe to relays!\n
+- How-to: https://git.pleroma.social/pleroma/pleroma/wikis/Admin%20tasks#relay-managment
+- Relays: https://fediverse.network/activityrelay"
+    ]
+  end
+
+  @spec puts(String.t() | [[IO.ANSI.ansicode() | String.t(), ...], ...]) :: nil
+  def puts(text_or_lines) do
+    import IO.ANSI
+
+    lines =
+      if is_binary(text_or_lines) do
+        String.split(text_or_lines, ~r/\n/)
+      else
+        text_or_lines
+      end
+
+    longest_line_size =
+      lines
+      |> Enum.map(&charlist_count_text/1)
+      |> Enum.sort(&>=/2)
+      |> List.first()
+
+    pad_text = longest_line_size
+
+    pad =
+      for(_ <- 1..pad_text, do: "_")
+      |> Enum.join("")
+
+    pad_spaces =
+      for(_ <- 1..pad_text, do: " ")
+      |> Enum.join("")
+
+    spaces = "      "
+
+    pre_lines = [
+      "  /  \\#{spaces}  _#{pad}___",
+      "  |  |#{spaces} / #{pad_spaces}   \\"
+    ]
+
+    for l <- pre_lines do
+      IO.puts(l)
+    end
+
+    clippy_lines = [
+      "  #{bright()}@  @#{reset()}#{spaces} ",
+      "  || ||#{spaces}",
+      "  || ||   <--",
+      "  |\\_/|      ",
+      "  \\___/      "
+    ]
+
+    noclippy_line = "             "
+
+    env = %{
+      max_size: pad_text,
+      pad: pad,
+      pad_spaces: pad_spaces,
+      spaces: spaces,
+      pre_lines: pre_lines,
+      noclippy_line: noclippy_line
+    }
+
+    # surrond one/five line clippy with blank lines around to not fuck up the layout
+    #
+    # yes this fix sucks but it's good enough, have you ever seen a release of windows wihtout some butched
+    # features anyway?
+    lines =
+      if length(lines) == 1 or length(lines) == 5 do
+        [""] ++ lines ++ [""]
+      else
+        lines
+      end
+
+    clippy_line(lines, clippy_lines, env)
+  rescue
+    e ->
+      IO.puts("(Clippy crashed, sorry: #{inspect(e)})")
+      IO.puts(text_or_lines)
+  end
+
+  defp clippy_line([line | lines], [prefix | clippy_lines], env) do
+    IO.puts([prefix <> "| ", rpad_line(line, env.max_size)])
+    clippy_line(lines, clippy_lines, env)
+  end
+
+  # more text lines but clippy's complete
+  defp clippy_line([line | lines], [], env) do
+    IO.puts([env.noclippy_line, "| ", rpad_line(line, env.max_size)])
+
+    if lines == [] do
+      IO.puts(env.noclippy_line <> "\\_#{env.pad}___/")
+    end
+
+    clippy_line(lines, [], env)
+  end
+
+  # no more text lines but clippy's not complete
+  defp clippy_line([], [clippy | clippy_lines], env) do
+    if env.pad do
+      IO.puts(clippy <> "\\_#{env.pad}___/")
+      clippy_line([], clippy_lines, %{env | pad: nil})
+    else
+      IO.puts(clippy)
+      clippy_line([], clippy_lines, env)
+    end
+  end
+
+  defp clippy_line(_, _, _) do
+  end
+
+  defp rpad_line(line, max) do
+    pad = max - (charlist_count_text(line) - 2)
+    pads = Enum.join(for(_ <- 1..pad, do: " "))
+    [IO.ANSI.format(line), pads <> " |"]
+  end
+
+  defp charlist_count_text(line) do
+    if is_list(line) do
+      text = Enum.join(Enum.filter(line, &is_binary/1))
+      String.length(text)
+    else
+      String.length(line)
+    end
+  end
+end

lib/pleroma/config/deprecation_warnings.ex

@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Config.DeprecationWarnings do
+  require Logger
+
+  def check_frontend_config_mechanism() do
+    if Pleroma.Config.get(:fe) do
+      Logger.warn("""
+      !!!DEPRECATION WARNING!!!
+      You are using the old configuration mechanism for the frontend. Please check config.md.
+      """)
+    end
+  end
+
+  def check_hellthread_threshold do
+    if Pleroma.Config.get([:mrf_hellthread, :threshold]) do
+      Logger.warn("""
+      !!!DEPRECATION WARNING!!!
+      You are using the old configuration mechanism for the hellthread filter. Please check config.md.
+      """)
+    end
+  end
+
+  def warn do
+    check_frontend_config_mechanism()
+    check_hellthread_threshold()
+  end
+end

lib/pleroma/emails/user_email.ex

@@ -7,7 +7,8 @@ defmodule Pleroma.UserEmail do
 
   import Swoosh.Email
 
-  alias Pleroma.Web.{Endpoint, Router}
+  alias Pleroma.Web.Endpoint
+  alias Pleroma.Web.Router
 
   defp instance_config, do: Pleroma.Config.get(:instance)
 

lib/pleroma/filter.ex

@@ -4,11 +4,15 @@
 
 defmodule Pleroma.Filter do
   use Ecto.Schema
-  import Ecto.{Changeset, Query}
+
-  alias Pleroma.{User, Repo}
+  import Ecto.Changeset
+  import Ecto.Query
+
+  alias Pleroma.User
+  alias Pleroma.Repo
 
   schema "filters" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:filter_id, :integer)
     field(:hide, :boolean, default: false)
     field(:whole_word, :boolean, default: true)

lib/pleroma/flake_id.ex

@@ -0,0 +1,172 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.FlakeId do
+  @moduledoc """
+  Flake is a decentralized, k-ordered id generation service.
+
+  Adapted from:
+
+  * [flaky](https://github.com/nirvana/flaky), released under the terms of the Truly Free License,
+  * [Flake](https://github.com/boundary/flake), Copyright 2012, Boundary, Apache License, Version 2.0
+  """
+
+  @type t :: binary
+
+  @behaviour Ecto.Type
+  use GenServer
+  require Logger
+  alias __MODULE__
+  import Kernel, except: [to_string: 1]
+
+  defstruct node: nil, time: 0, sq: 0
+
+  @doc "Converts a binary Flake to a String"
+  def to_string(<<0::integer-size(64), id::integer-size(64)>>) do
+    Kernel.to_string(id)
+  end
+
+  def to_string(<<_::integer-size(64), _::integer-size(48), _::integer-size(16)>> = flake) do
+    encode_base62(flake)
+  end
+
+  def to_string(s), do: s
+
+  def from_string(int) when is_integer(int) do
+    from_string(Kernel.to_string(int))
+  end
+
+  for i <- [-1, 0] do
+    def from_string(unquote(i)), do: <<0::integer-size(128)>>
+    def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>>
+  end
+
+  def from_string(<<_::integer-size(128)>> = flake), do: flake
+
+  def from_string(string) when is_binary(string) and byte_size(string) < 18 do
+    case Integer.parse(string) do
+      {id, _} -> <<0::integer-size(64), id::integer-size(64)>>
+      _ -> nil
+    end
+  end
+
+  def from_string(string) do
+    string |> decode_base62 |> from_integer
+  end
+
+  def to_integer(<<integer::integer-size(128)>>), do: integer
+
+  def from_integer(integer) do
+    <<_time::integer-size(64), _node::integer-size(48), _seq::integer-size(16)>> =
+      <<integer::integer-size(128)>>
+  end
+
+  @doc "Generates a Flake"
+  @spec get :: binary
+  def get, do: to_string(:gen_server.call(:flake, :get))
+
+  # -- Ecto.Type API
+  @impl Ecto.Type
+  def type, do: :uuid
+
+  @impl Ecto.Type
+  def cast(value) do
+    {:ok, FlakeId.to_string(value)}
+  end
+
+  @impl Ecto.Type
+  def load(value) do
+    {:ok, FlakeId.to_string(value)}
+  end
+
+  @impl Ecto.Type
+  def dump(value) do
+    {:ok, FlakeId.from_string(value)}
+  end
+
+  def autogenerate(), do: get()
+
+  # -- GenServer API
+  def start_link do
+    :gen_server.start_link({:local, :flake}, __MODULE__, [], [])
+  end
+
+  @impl GenServer
+  def init([]) do
+    {:ok, %FlakeId{node: worker_id(), time: time()}}
+  end
+
+  @impl GenServer
+  def handle_call(:get, _from, state) do
+    {flake, new_state} = get(time(), state)
+    {:reply, flake, new_state}
+  end
+
+  # Matches when the calling time is the same as the state time. Incr. sq
+  defp get(time, %FlakeId{time: time, node: node, sq: seq}) do
+    new_state = %FlakeId{time: time, node: node, sq: seq + 1}
+    {gen_flake(new_state), new_state}
+  end
+
+  # Matches when the times are different, reset sq
+  defp get(newtime, %FlakeId{time: time, node: node}) when newtime > time do
+    new_state = %FlakeId{time: newtime, node: node, sq: 0}
+    {gen_flake(new_state), new_state}
+  end
+
+  # Error when clock is running backwards
+  defp get(newtime, %FlakeId{time: time}) when newtime < time do
+    {:error, :clock_running_backwards}
+  end
+
+  defp gen_flake(%FlakeId{time: time, node: node, sq: seq}) do
+    <<time::integer-size(64), node::integer-size(48), seq::integer-size(16)>>
+  end
+
+  defp nthchar_base62(n) when n <= 9, do: ?0 + n
+  defp nthchar_base62(n) when n <= 35, do: ?A + n - 10
+  defp nthchar_base62(n), do: ?a + n - 36
+
+  defp encode_base62(<<integer::integer-size(128)>>) do
+    integer
+    |> encode_base62([])
+    |> List.to_string()
+  end
+
+  defp encode_base62(int, acc) when int < 0, do: encode_base62(-int, acc)
+  defp encode_base62(int, []) when int == 0, do: '0'
+  defp encode_base62(int, acc) when int == 0, do: acc
+
+  defp encode_base62(int, acc) do
+    r = rem(int, 62)
+    id = div(int, 62)
+    acc = [nthchar_base62(r) | acc]
+    encode_base62(id, acc)
+  end
+
+  defp decode_base62(s) do
+    decode_base62(String.to_charlist(s), 0)
+  end
+
+  defp decode_base62([c | cs], acc) when c >= ?0 and c <= ?9,
+    do: decode_base62(cs, 62 * acc + (c - ?0))
+
+  defp decode_base62([c | cs], acc) when c >= ?A and c <= ?Z,
+    do: decode_base62(cs, 62 * acc + (c - ?A + 10))
+
+  defp decode_base62([c | cs], acc) when c >= ?a and c <= ?z,
+    do: decode_base62(cs, 62 * acc + (c - ?a + 36))
+
+  defp decode_base62([], acc), do: acc
+
+  defp time do
+    {mega_seconds, seconds, micro_seconds} = :erlang.timestamp()
+    1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000)
+  end
+
+  defp worker_id() do
+    <<worker::integer-size(48)>> = :crypto.strong_rand_bytes(6)
+    worker
+  end
+end

lib/pleroma/formatter.ex

@@ -3,10 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Formatter do
+  alias Pleroma.Emoji
+  alias Pleroma.HTML
   alias Pleroma.User
   alias Pleroma.Web.MediaProxy
-  alias Pleroma.HTML
-  alias Pleroma.Emoji
 
   @tag_regex ~r/((?<=[^&])|\A)(\#)(\w+)/u
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
@@ -43,7 +43,7 @@ def emojify(text) do
 
   def emojify(text, nil), do: text
 
-  def emojify(text, emoji) do
+  def emojify(text, emoji, strip \\ false) do
     Enum.reduce(emoji, text, fn {emoji, file}, text ->
       emoji = HTML.strip_tags(emoji)
       file = HTML.strip_tags(file)
@@ -51,14 +51,24 @@ def emojify(text, emoji) do
       String.replace(
         text,
         ":#{emoji}:",
-        "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
+        if not strip do
-          MediaProxy.url(file)
+          "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
-        }' />"
+            MediaProxy.url(file)
+          }' />"
+        else
+          ""
+        end
       )
       |> HTML.filter_tags()
     end)
   end
 
+  def demojify(text) do
+    emojify(text, Emoji.get_all(), true)
+  end
+
+  def demojify(text, nil), do: text
+
   def get_emoji(text) when is_binary(text) do
     Enum.filter(Emoji.get_all(), fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end)
   end
@@ -120,7 +130,7 @@ def add_links({subs, text}) do
   end
 
   @doc "Adds the links to mentioned users"
-  def add_user_links({subs, text}, mentions) do
+  def add_user_links({subs, text}, mentions, options \\ []) do
     mentions =
       mentions
       |> Enum.sort_by(fn {name, _} -> -String.length(name) end)
@@ -142,10 +152,16 @@ def add_user_links({subs, text}, mentions) do
               ap_id
             end
 
-          short_match = String.split(match, "@") |> tl() |> hd()
+          nickname =
+            if options[:format] == :full do
+              User.full_nickname(match)
+            else
+              User.local_nickname(match)
+            end
 
           {uuid,
-           "<span><a data-user='#{id}' class='mention' href='#{ap_id}'>@<span>#{short_match}</span></a></span>"}
+           "<span class='h-card'><a data-user='#{id}' class='u-url mention' href='#{ap_id}'>" <>
+             "@<span>#{nickname}</span></a></span>"}
         end)
 
     {subs, uuid_text}
@@ -168,7 +184,7 @@ def add_hashtag_links({subs, text}, tags) do
       subs ++
         Enum.map(tags, fn {tag_text, tag, uuid} ->
           url =
-            "<a data-tag='#{tag}' href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{
+            "<a class='hashtag' data-tag='#{tag}' href='#{Pleroma.Web.base_url()}/tag/#{tag}' rel='tag'>#{
               tag_text
             }</a>"
 
@@ -183,4 +199,16 @@ def finalize({subs, text}) do
       String.replace(result_text, uuid, replacement)
     end)
   end
+
+  def truncate(text, max_length \\ 200, omission \\ "...") do
+    # Remove trailing whitespace
+    text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}")
+
+    if String.length(text) < max_length do
+      text
+    else
+      length_with_omission = max_length - String.length(omission)
+      String.slice(text, 0, length_with_omission) <> omission
+    end
+  end
 end

lib/pleroma/gopher/server.ex

@@ -37,17 +37,17 @@ def init([ip, port]) do
 
 defmodule Pleroma.Gopher.Server.ProtocolHandler do
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.User
   alias Pleroma.Activity
-  alias Pleroma.Repo
   alias Pleroma.HTML
+  alias Pleroma.User
+  alias Pleroma.Repo
 
   def start_link(ref, socket, transport, opts) do
     pid = spawn_link(__MODULE__, :init, [ref, socket, transport, opts])
     {:ok, pid}
   end
 
-  def init(ref, socket, transport, _Opts = []) do
+  def init(ref, socket, transport, [] = _Opts) do
     :ok = :ranch.accept_ack(ref)
     loop(socket, transport)
   end

lib/pleroma/html.ex

@@ -28,13 +28,18 @@ def filter_tags(html, scrubber), do: Scrubber.scrub(html, scrubber)
   def filter_tags(html), do: filter_tags(html, nil)
   def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags)
 
-  def get_cached_scrubbed_html_for_object(content, scrubbers, object) do
+  def get_cached_scrubbed_html_for_object(content, scrubbers, object, module) do
-    key = "#{generate_scrubber_signature(scrubbers)}|#{object.id}"
+    key = "#{module}#{generate_scrubber_signature(scrubbers)}|#{object.id}"
     Cachex.fetch!(:scrubber_cache, key, fn _key -> ensure_scrubbed_html(content, scrubbers) end)
   end
 
-  def get_cached_stripped_html_for_object(content, object) do
+  def get_cached_stripped_html_for_object(content, object, module) do
-    get_cached_scrubbed_html_for_object(content, HtmlSanitizeEx.Scrubber.StripTags, object)
+    get_cached_scrubbed_html_for_object(
+      content,
+      HtmlSanitizeEx.Scrubber.StripTags,
+      object,
+      module
+    )
   end
 
   def ensure_scrubbed_html(
@@ -50,15 +55,23 @@ defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
 
   defp generate_scrubber_signature(scrubbers) do
     Enum.reduce(scrubbers, "", fn scrubber, signature ->
-      # If a scrubber does not have a version(e.g HtmlSanitizeEx.Scrubber.StripTags) it is assumed it is always 0)
+      "#{signature}#{to_string(scrubber)}"
-      version =
+    end)
-        if Kernel.function_exported?(scrubber, :version, 0) do
+  end
-          scrubber.version
-        else
-          0
-        end
 
-      "#{signature}#{to_string(scrubber)}#{version}"
+  def extract_first_external_url(_, nil), do: {:error, "No content"}
+
+  def extract_first_external_url(object, content) do
+    key = "URL|#{object.id}"
+
+    Cachex.fetch!(:scrubber_cache, key, fn _key ->
+      result =
+        content
+        |> Floki.filter_out("a.mention")
+        |> Floki.attribute("a", "href")
+        |> Enum.at(0)
+
+      {:commit, {:ok, result}}
     end)
   end
 end
@@ -70,29 +83,24 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do
   """
 
   @markup Application.get_env(:pleroma, :markup)
-  @uri_schemes Application.get_env(:pleroma, :uri_schemes, [])
+  @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], [])
-  @valid_schemes Keyword.get(@uri_schemes, :valid_schemes, [])
 
   require HtmlSanitizeEx.Scrubber.Meta
   alias HtmlSanitizeEx.Scrubber.Meta
 
-  def version do
-    0
-  end
-
   Meta.remove_cdata_sections_before_scrub()
   Meta.strip_comments()
 
   # links
   Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
-  Meta.allow_tag_with_these_attributes("a", ["name", "title"])
+  Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"])
 
   # paragraphs and linebreaks
   Meta.allow_tag_with_these_attributes("br", [])
   Meta.allow_tag_with_these_attributes("p", [])
 
   # microformats
-  Meta.allow_tag_with_these_attributes("span", [])
+  Meta.allow_tag_with_these_attributes("span", ["class"])
 
   # allow inline images for custom emoji
   @allow_inline_images Keyword.get(@markup, :allow_inline_images)
@@ -117,20 +125,17 @@ defmodule Pleroma.HTML.Scrubber.Default do
 
   require HtmlSanitizeEx.Scrubber.Meta
   alias HtmlSanitizeEx.Scrubber.Meta
-
+  # credo:disable-for-previous-line
-  def version do
+  # No idea how to fix this one…
-    0
-  end
 
   @markup Application.get_env(:pleroma, :markup)
-  @uri_schemes Application.get_env(:pleroma, :uri_schemes, [])
+  @valid_schemes Pleroma.Config.get([:uri_schemes, :valid_schemes], [])
-  @valid_schemes Keyword.get(@uri_schemes, :valid_schemes, [])
 
   Meta.remove_cdata_sections_before_scrub()
   Meta.strip_comments()
 
   Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes)
-  Meta.allow_tag_with_these_attributes("a", ["name", "title"])
+  Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"])
 
   Meta.allow_tag_with_these_attributes("abbr", ["title"])
 
@@ -145,7 +150,7 @@ def version do
   Meta.allow_tag_with_these_attributes("ol", [])
   Meta.allow_tag_with_these_attributes("p", [])
   Meta.allow_tag_with_these_attributes("pre", [])
-  Meta.allow_tag_with_these_attributes("span", [])
+  Meta.allow_tag_with_these_attributes("span", ["class"])
   Meta.allow_tag_with_these_attributes("strong", [])
   Meta.allow_tag_with_these_attributes("u", [])
   Meta.allow_tag_with_these_attributes("ul", [])
@@ -199,10 +204,6 @@ defmodule Pleroma.HTML.Transform.MediaProxy do
 
   alias Pleroma.Web.MediaProxy
 
-  def version do
-    0
-  end
-
   def before_scrub(html), do: html
 
   def scrub_attribute("img", {"src", "http" <> target}) do

lib/pleroma/http/connection.ex

@@ -10,7 +10,8 @@ defmodule Pleroma.HTTP.Connection do
   @hackney_options [
     timeout: 10000,
     recv_timeout: 20000,
-    follow_redirect: true
+    follow_redirect: true,
+    pool: :federation
   ]
   @adapter Application.get_env(:tesla, :adapter)
 

lib/pleroma/http/http.ex

@@ -31,12 +31,15 @@ def request(method, url, body \\ "", headers \\ [], options \\ []) do
       process_request_options(options)
       |> process_sni_options(url)
 
+    params = Keyword.get(options, :params, [])
+
     %{}
     |> Builder.method(method)
     |> Builder.headers(headers)
     |> Builder.opts(options)
     |> Builder.url(url)
     |> Builder.add_param(:body, :body, body)
+    |> Builder.add_param(:query, :query, params)
     |> Enum.into([])
     |> (&Tesla.request(Connection.new(), &1)).()
   end

lib/pleroma/http/request_builder.ex

@@ -100,6 +100,8 @@ def add_optional_params(request, definitions, [{key, value} | tail]) do
   Map
   """
   @spec add_param(map(), atom, atom, any()) :: map()
+  def add_param(request, :query, :query, values), do: Map.put(request, :query, values)
+
   def add_param(request, :body, :body, value), do: Map.put(request, :body, value)
 
   def add_param(request, :body, key, value) do
@@ -107,7 +109,10 @@ def add_param(request, :body, key, value) do
     |> Map.put_new_lazy(:body, &Tesla.Multipart.new/0)
     |> Map.update!(
       :body,
-      &Tesla.Multipart.add_field(&1, key, Poison.encode!(value),
+      &Tesla.Multipart.add_field(
+        &1,
+        key,
+        Jason.encode!(value),
         headers: [{:"Content-Type", "application/json"}]
       )
     )

lib/pleroma/instances.ex

@@ -0,0 +1,36 @@
+defmodule Pleroma.Instances do
+  @moduledoc "Instances context."
+
+  @adapter Pleroma.Instances.Instance
+
+  defdelegate filter_reachable(urls_or_hosts), to: @adapter
+  defdelegate reachable?(url_or_host), to: @adapter
+  defdelegate set_reachable(url_or_host), to: @adapter
+  defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
+
+  def set_consistently_unreachable(url_or_host),
+    do: set_unreachable(url_or_host, reachability_datetime_threshold())
+
+  def reachability_datetime_threshold do
+    federation_reachability_timeout_days =
+      Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 0
+
+    if federation_reachability_timeout_days > 0 do
+      NaiveDateTime.add(
+        NaiveDateTime.utc_now(),
+        -federation_reachability_timeout_days * 24 * 3600,
+        :second
+      )
+    else
+      ~N[0000-01-01 00:00:00]
+    end
+  end
+
+  def host(url_or_host) when is_binary(url_or_host) do
+    if url_or_host =~ ~r/^http/i do
+      URI.parse(url_or_host).host
+    else
+      url_or_host
+    end
+  end
+end

lib/pleroma/instances/instance.ex

@@ -0,0 +1,113 @@
+defmodule Pleroma.Instances.Instance do
+  @moduledoc "Instance."
+
+  alias Pleroma.Instances
+  alias Pleroma.Repo
+  alias Pleroma.Instances.Instance
+
+  use Ecto.Schema
+
+  import Ecto.Query
+  import Ecto.Changeset
+
+  schema "instances" do
+    field(:host, :string)
+    field(:unreachable_since, :naive_datetime)
+
+    timestamps()
+  end
+
+  defdelegate host(url_or_host), to: Instances
+
+  def changeset(struct, params \\ %{}) do
+    struct
+    |> cast(params, [:host, :unreachable_since])
+    |> validate_required([:host])
+    |> unique_constraint(:host)
+  end
+
+  def filter_reachable([]), do: %{}
+
+  def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
+    hosts =
+      urls_or_hosts
+      |> Enum.map(&(&1 && host(&1)))
+      |> Enum.filter(&(to_string(&1) != ""))
+
+    unreachable_since_by_host =
+      Repo.all(
+        from(i in Instance,
+          where: i.host in ^hosts,
+          select: {i.host, i.unreachable_since}
+        )
+      )
+      |> Map.new(& &1)
+
+    reachability_datetime_threshold = Instances.reachability_datetime_threshold()
+
+    for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
+      host = host(entry)
+      unreachable_since = unreachable_since_by_host[host]
+
+      if !unreachable_since ||
+           NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do
+        {entry, unreachable_since}
+      end
+    end
+    |> Enum.filter(& &1)
+    |> Map.new(& &1)
+  end
+
+  def reachable?(url_or_host) when is_binary(url_or_host) do
+    !Repo.one(
+      from(i in Instance,
+        where:
+          i.host == ^host(url_or_host) and
+            i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
+        select: true
+      )
+    )
+  end
+
+  def reachable?(_), do: true
+
+  def set_reachable(url_or_host) when is_binary(url_or_host) do
+    with host <- host(url_or_host),
+         %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
+      {:ok, _instance} =
+        existing_record
+        |> changeset(%{unreachable_since: nil})
+        |> Repo.update()
+    end
+  end
+
+  def set_reachable(_), do: {:error, nil}
+
+  def set_unreachable(url_or_host, unreachable_since \\ nil)
+
+  def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do
+    unreachable_since = unreachable_since || DateTime.utc_now()
+    host = host(url_or_host)
+    existing_record = Repo.get_by(Instance, %{host: host})
+
+    changes = %{unreachable_since: unreachable_since}
+
+    cond do
+      is_nil(existing_record) ->
+        %Instance{}
+        |> changeset(Map.put(changes, :host, host))
+        |> Repo.insert()
+
+      existing_record.unreachable_since &&
+          NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
+        {:ok, existing_record}
+
+      true ->
+        existing_record
+        |> changeset(changes)
+        |> Repo.update()
+    end
+  end
+
+  def set_unreachable(_, _), do: {:error, nil}
+end

lib/pleroma/list.ex

@@ -4,11 +4,16 @@
 
 defmodule Pleroma.List do
   use Ecto.Schema
-  import Ecto.{Changeset, Query}
+
-  alias Pleroma.{User, Repo, Activity}
+  import Ecto.Query
+  import Ecto.Changeset
+
+  alias Pleroma.Activity
+  alias Pleroma.Repo
+  alias Pleroma.User
 
   schema "lists" do
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:title, :string)
     field(:following, {:array, :string}, default: [])
 

lib/pleroma/mime.ex

@@ -102,10 +102,18 @@ defp check_mime_type(<<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00, _::binary
     "audio/ogg"
   end
 
-  defp check_mime_type(<<0x52, 0x49, 0x46, 0x46, _::binary>>) do
+  defp check_mime_type(<<"RIFF", _::binary-size(4), "WAVE", _::binary>>) do
     "audio/wav"
   end
 
+  defp check_mime_type(<<"RIFF", _::binary-size(4), "WEBP", _::binary>>) do
+    "image/webp"
+  end
+
+  defp check_mime_type(<<"RIFF", _::binary-size(4), "AVI.", _::binary>>) do
+    "video/avi"
+  end
+
   defp check_mime_type(_) do
     @default
   end

lib/pleroma/notification.ex

@@ -4,13 +4,20 @@
 
 defmodule Pleroma.Notification do
   use Ecto.Schema
-  alias Pleroma.{User, Activity, Notification, Repo, Object}
+
+  alias Pleroma.User
+  alias Pleroma.Activity
+  alias Pleroma.Notification
+  alias Pleroma.Repo
+  alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.Web.CommonAPI
+
   import Ecto.Query
 
   schema "notifications" do
     field(:seen, :boolean, default: false)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
-    belongs_to(:activity, Pleroma.Activity)
+    belongs_to(:activity, Activity, type: Pleroma.FlakeId)
 
     timestamps()
   end
@@ -34,7 +41,8 @@ def for_user(user, opts \\ %{}) do
         n in Notification,
         where: n.user_id == ^user.id,
         order_by: [desc: n.id],
-        preload: [:activity],
+        join: activity in assoc(n, :activity),
+        preload: [activity: activity],
         limit: 20
       )
 
@@ -65,7 +73,8 @@ def get(%{id: user_id} = _user, id) do
       from(
         n in Notification,
         where: n.id == ^id,
-        preload: [:activity]
+        join: activity in assoc(n, :activity),
+        preload: [activity: activity]
       )
 
     notification = Repo.one(query)
@@ -96,7 +105,7 @@ def dismiss(%{id: user_id} = _user, id) do
     end
   end
 
-  def create_notifications(%Activity{id: _, data: %{"to" => _, "type" => type}} = activity)
+  def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity)
       when type in ["Create", "Like", "Announce", "Follow"] do
     users = get_notified_from_activity(activity)
 
@@ -109,7 +118,12 @@ def create_notifications(_), do: {:ok, []}
   # TODO move to sql, too.
   def create_notification(%Activity{} = activity, %User{} = user) do
     unless User.blocks?(user, %{ap_id: activity.data["actor"]}) or
-             user.ap_id == activity.data["actor"] do
+             CommonAPI.thread_muted?(user, activity) or user.ap_id == activity.data["actor"] or
+             (activity.data["type"] == "Follow" and
+                Enum.any?(Notification.for_user(user), fn notif ->
+                  notif.activity.data["type"] == "Follow" and
+                    notif.activity.data["actor"] == activity.data["actor"]
+                end)) do
       notification = %Notification{user_id: user.id, activity: activity}
       {:ok, notification} = Repo.insert(notification)
       Pleroma.Web.Streamer.stream("user", notification)
@@ -127,54 +141,12 @@ def get_notified_from_activity(
       when type in ["Create", "Like", "Announce", "Follow"] do
     recipients =
       []
-      |> maybe_notify_to_recipients(activity)
+      |> Utils.maybe_notify_to_recipients(activity)
-      |> maybe_notify_mentioned_recipients(activity)
+      |> Utils.maybe_notify_mentioned_recipients(activity)
       |> Enum.uniq()
 
     User.get_users_from_set(recipients, local_only)
   end
 
   def get_notified_from_activity(_, _local_only), do: []
-
-  defp maybe_notify_to_recipients(
-         recipients,
-         %Activity{data: %{"to" => to, "type" => _type}} = _activity
-       ) do
-    recipients ++ to
-  end
-
-  defp maybe_notify_mentioned_recipients(
-         recipients,
-         %Activity{data: %{"to" => _to, "type" => type} = data} = _activity
-       )
-       when type == "Create" do
-    object = Object.normalize(data["object"])
-
-    object_data =
-      cond do
-        !is_nil(object) ->
-          object.data
-
-        is_map(data["object"]) ->
-          data["object"]
-
-        true ->
-          %{}
-      end
-
-    tagged_mentions = maybe_extract_mentions(object_data)
-
-    recipients ++ tagged_mentions
-  end
-
-  defp maybe_notify_mentioned_recipients(recipients, _), do: recipients
-
-  defp maybe_extract_mentions(%{"tag" => tag}) do
-    tag
-    |> Enum.filter(fn x -> is_map(x) end)
-    |> Enum.filter(fn x -> x["type"] == "Mention" end)
-    |> Enum.map(fn x -> x["href"] end)
-  end
-
-  defp maybe_extract_mentions(_), do: []
 end

lib/pleroma/object.ex

@@ -4,8 +4,15 @@
 
 defmodule Pleroma.Object do
   use Ecto.Schema
-  alias Pleroma.{Repo, Object, User, Activity, ObjectTombstone}
+
-  import Ecto.{Query, Changeset}
+  alias Pleroma.Repo
+  alias Pleroma.Object
+  alias Pleroma.User
+  alias Pleroma.Activity
+  alias Pleroma.ObjectTombstone
+
+  import Ecto.Query
+  import Ecto.Changeset
 
   schema "objects" do
     field(:data, :map)
@@ -31,8 +38,8 @@ def get_by_ap_id(ap_id) do
     Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
   end
 
-  def normalize(obj) when is_map(obj), do: Object.get_by_ap_id(obj["id"])
+  def normalize(%{"id" => ap_id}), do: normalize(ap_id)
-  def normalize(ap_id) when is_binary(ap_id), do: Object.get_by_ap_id(ap_id)
+  def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
   def normalize(_), do: nil
 
   # Owned objects can only be mutated by their owner
@@ -42,24 +49,18 @@ def authorize_mutation(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}),
   # Legacy objects can be mutated by anybody
   def authorize_mutation(%Object{}, %User{}), do: true
 
-  if Mix.env() == :test do
+  def get_cached_by_ap_id(ap_id) do
-    def get_cached_by_ap_id(ap_id) do
+    key = "object:#{ap_id}"
-      get_by_ap_id(ap_id)
-    end
-  else
-    def get_cached_by_ap_id(ap_id) do
-      key = "object:#{ap_id}"
 
-      Cachex.fetch!(:object_cache, key, fn _ ->
+    Cachex.fetch!(:object_cache, key, fn _ ->
-        object = get_by_ap_id(ap_id)
+      object = get_by_ap_id(ap_id)
 
-        if object do
+      if object do
-          {:commit, object}
+        {:commit, object}
-        else
+      else
-          {:ignore, object}
+        {:ignore, object}
-        end
+      end
-      end)
+    end)
-    end
   end
 
   def context_mapping(context) do
@@ -85,9 +86,22 @@ def swap_object_with_tombstone(object) do
 
   def delete(%Object{data: %{"id" => id}} = object) do
     with {:ok, _obj} = swap_object_with_tombstone(object),
-         Repo.delete_all(Activity.all_non_create_by_object_ap_id_q(id)),
+         Repo.delete_all(Activity.by_object_ap_id(id)),
          {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
       {:ok, object}
     end
   end
+
+  def set_cache(%Object{data: %{"id" => ap_id}} = object) do
+    Cachex.put(:object_cache, "object:#{ap_id}", object)
+    {:ok, object}
+  end
+
+  def update_and_set_cache(changeset) do
+    with {:ok, object} <- Repo.update(changeset) do
+      set_cache(object)
+    else
+      e -> e
+    end
+  end
 end

lib/pleroma/plugs/http_security_plug.ex

@@ -33,7 +33,22 @@ defp headers do
   end
 
   defp csp_string do
-    protocol = Config.get([Pleroma.Web.Endpoint, :protocol])
+    scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
+    websocket_url = String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws")
+
+    connect_src =
+      if Mix.env() == :dev do
+        "connect-src 'self' http://localhost:3035/ " <> websocket_url
+      else
+        "connect-src 'self' " <> websocket_url
+      end
+
+    script_src =
+      if Mix.env() == :dev do
+        "script-src 'self' 'unsafe-eval'"
+      else
+        "script-src 'self'"
+      end
 
     [
       "default-src 'none'",
@@ -43,10 +58,10 @@ defp csp_string do
       "media-src 'self' https:",
       "style-src 'self' 'unsafe-inline'",
       "font-src 'self'",
-      "script-src 'self'",
-      "connect-src 'self' " <> String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws"),
       "manifest-src 'self'",
-      if protocol == "https" do
+      connect_src,
+      script_src,
+      if scheme == "https" do
         "upgrade-insecure-requests"
       end
     ]

lib/pleroma/plugs/instance_static.ex

@@ -21,7 +21,7 @@ def file_path(path) do
     end
   end
 
-  @only ~w(index.html static emoji packs sounds images instance favicon.png)
+  @only ~w(index.html static emoji packs sounds images instance favicon.png sw.js sw-pleroma.js)
 
   def init(opts) do
     opts
@@ -33,7 +33,7 @@ def init(opts) do
   for only <- @only do
     at = Plug.Router.Utils.split("/")
 
-    def call(conn = %{request_path: "/" <> unquote(only) <> _}, opts) do
+    def call(%{request_path: "/" <> unquote(only) <> _} = conn, opts) do
       call_static(
         conn,
         opts,

lib/pleroma/plugs/oauth_plug.ex

@@ -6,11 +6,9 @@ defmodule Pleroma.Plugs.OAuthPlug do
   import Plug.Conn
   import Ecto.Query
 
-  alias Pleroma.{
+  alias Pleroma.User
-    User,
+  alias Pleroma.Repo
-    Repo,
+  alias Pleroma.Web.OAuth.Token
-    Web.OAuth.Token
-  }
 
   @realm_reg Regex.compile!("Bearer\:?\s+(.*)$", "i")
 
@@ -33,7 +31,12 @@ def call(conn, _) do
   #
   @spec fetch_user_and_token(String.t()) :: {:ok, User.t(), Token.t()} | nil
   defp fetch_user_and_token(token) do
-    query = from(q in Token, where: q.token == ^token, preload: [:user])
+    query =
+      from(t in Token,
+        where: t.token == ^token,
+        join: user in assoc(t, :user),
+        preload: [user: user]
+      )
 
     with %Token{user: %{info: %{deactivated: false} = _} = user} = token_record <- Repo.one(query) do
       {:ok, user, token_record}

lib/pleroma/plugs/uploaded_media.ex

@@ -23,7 +23,7 @@ def init(_opts) do
     %{static_plug_opts: static_plug_opts}
   end
 
-  def call(conn = %{request_path: <<"/", @path, "/", file::binary>>}, opts) do
+  def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
     config = Pleroma.Config.get([Pleroma.Upload])
 
     with uploader <- Keyword.fetch!(config, :uploader),

lib/pleroma/plugs/user_fetcher_plug.ex

@@ -3,9 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.UserFetcherPlug do
-  import Plug.Conn
-  alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Repo
+
+  import Plug.Conn
 
   def init(options) do
     options

lib/pleroma/reverse_proxy.ex

@@ -275,11 +275,24 @@ defp build_resp_headers(headers, opts) do
 
   defp build_resp_cache_headers(headers, _opts) do
     has_cache? = Enum.any?(headers, fn {k, _} -> k in @resp_cache_headers end)
+    has_cache_control? = List.keymember?(headers, "cache-control", 0)
 
-    if has_cache? do
+    cond do
-      headers
+      has_cache? && has_cache_control? ->
-    else
+        headers
-      List.keystore(headers, "cache-control", 0, {"cache-control", @default_cache_control_header})
+
+      has_cache? ->
+        # There's caching header present but no cache-control -- we need to explicitely override it to public
+        # as Plug defaults to "max-age=0, private, must-revalidate"
+        List.keystore(headers, "cache-control", 0, {"cache-control", "public"})
+
+      true ->
+        List.keystore(
+          headers,
+          "cache-control",
+          0,
+          {"cache-control", @default_cache_control_header}
+        )
     end
   end
 

lib/pleroma/stats.ex

@@ -4,7 +4,8 @@
 
 defmodule Pleroma.Stats do
   import Ecto.Query
-  alias Pleroma.{User, Repo}
+  alias Pleroma.User
+  alias Pleroma.Repo
 
   def start_link do
     agent = Agent.start_link(fn -> {[], %{}} end, name: __MODULE__)
@@ -23,7 +24,7 @@ def get_peers do
   def schedule_update do
     spawn(fn ->
       # 1 hour
-      Process.sleep(1000 * 60 * 60 * 1)
+      Process.sleep(1000 * 60 * 60)
       schedule_update()
     end)
 
@@ -34,10 +35,11 @@ def update_stats do
     peers =
       from(
         u in Pleroma.User,
-        select: fragment("distinct ?->'host'", u.info),
+        select: fragment("distinct split_part(?, '@', 2)", u.nickname),
         where: u.local != ^true
       )
       |> Repo.all()
+      |> Enum.filter(& &1)
 
     domain_count = Enum.count(peers)
 
@@ -45,7 +47,7 @@ def update_stats do
       from(u in User.local_user_query(), select: fragment("sum((?->>'note_count')::int)", u.info))
 
     status_count = Repo.one(status_query)
-    user_count = Repo.aggregate(User.local_user_query(), :count, :id)
+    user_count = Repo.aggregate(User.active_local_user_query(), :count, :id)
 
     Agent.update(__MODULE__, fn _ ->
       {peers, %{domain_count: domain_count, status_count: status_count, user_count: user_count}}

lib/pleroma/thread_mute.ex

@@ -0,0 +1,45 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ThreadMute do
+  use Ecto.Schema
+  alias Pleroma.{Repo, User, ThreadMute}
+  require Ecto.Query
+
+  schema "thread_mutes" do
+    belongs_to(:user, User, type: Pleroma.FlakeId)
+    field(:context, :string)
+  end
+
+  def changeset(mute, params \\ %{}) do
+    mute
+    |> Ecto.Changeset.cast(params, [:user_id, :context])
+    |> Ecto.Changeset.foreign_key_constraint(:user_id)
+    |> Ecto.Changeset.unique_constraint(:user_id, name: :unique_index)
+  end
+
+  def query(user_id, context) do
+    user_id = Pleroma.FlakeId.from_string(user_id)
+
+    ThreadMute
+    |> Ecto.Query.where(user_id: ^user_id)
+    |> Ecto.Query.where(context: ^context)
+  end
+
+  def add_mute(user_id, context) do
+    %ThreadMute{}
+    |> changeset(%{user_id: user_id, context: context})
+    |> Repo.insert()
+  end
+
+  def remove_mute(user_id, context) do
+    query(user_id, context)
+    |> Repo.delete_all()
+  end
+
+  def check_muted(user_id, context) do
+    query(user_id, context)
+    |> Repo.all()
+  end
+end

lib/pleroma/upload.ex

@@ -34,8 +34,9 @@ defmodule Pleroma.Upload do
   require Logger
 
   @type source ::
-          Plug.Upload.t() | data_uri_string ::
+          Plug.Upload.t()
-          String.t() | {:from_local, name :: String.t(), id :: String.t(), path :: String.t()}
+          | (data_uri_string :: String.t())
+          | {:from_local, name :: String.t(), id :: String.t(), path :: String.t()}
 
   @type option ::
           {:type, :avatar | :banner | :background}
@@ -123,10 +124,10 @@ defp get_opts(opts) do
 
           :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Mogrify]]
 
-          :pleroma, Pleroma.Upload.Filter.Mogrify, args: "strip"
+          :pleroma, Pleroma.Upload.Filter.Mogrify, args: ["strip", "auto-orient"]
         """)
 
-        Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: "strip")
+        Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: ["strip", "auto-orient"])
         Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Mogrify])
       else
         opts
@@ -179,7 +180,7 @@ defp prepare_upload(%{"img" => "data:image/" <> image_data}, opts) do
   end
 
   # For Mix.Tasks.MigrateLocalUploads
-  defp prepare_upload(upload = %__MODULE__{tempfile: path}, _opts) do
+  defp prepare_upload(%__MODULE__{tempfile: path} = upload, _opts) do
     with {:ok, content_type} <- Pleroma.MIME.file_mime_type(path) do
       {:ok, %__MODULE__{upload | content_type: content_type}}
     end
@@ -215,6 +216,12 @@ defp tempfile_for_image(data) do
   end
 
   defp url_from_spec(base_url, {:file, path}) do
+    path =
+      path
+      |> URI.encode()
+      |> String.replace("?", "%3F")
+      |> String.replace(":", "%3A")
+
     [base_url, "media", path]
     |> Path.join()
   end

lib/pleroma/upload/filter/dedupe.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.Upload.Filter.Dedupe do
   @behaviour Pleroma.Upload.Filter
   alias Pleroma.Upload
 
-  def filter(upload = %Upload{name: name}) do
+  def filter(%Upload{name: name} = upload) do
     extension = String.split(name, ".") |> List.last()
     shasum = :crypto.hash(:sha256, File.read!(upload.tempfile)) |> Base.encode16(case: :lower)
     filename = shasum <> "." <> extension

lib/pleroma/uploaders/mdii.ex

@@ -24,7 +24,8 @@ def put_file(upload) do
     extension = String.split(upload.name, ".") |> List.last()
     query = "#{cgi}?#{extension}"
 
-    with {:ok, %{status: 200, body: body}} <- @httpoison.post(query, file_data) do
+    with {:ok, %{status: 200, body: body}} <-
+           @httpoison.post(query, file_data, [], adapter: [pool: :default]) do
       remote_file_name = String.split(body) |> List.first()
       public_url = "#{files}/#{remote_file_name}.#{extension}"
       {:ok, {:url, public_url}}

lib/pleroma/uploaders/s3.ex

@@ -9,17 +9,25 @@ defmodule Pleroma.Uploaders.S3 do
   # The file name is re-encoded with S3's constraints here to comply with previous links with less strict filenames
   def get_file(file) do
     config = Pleroma.Config.get([__MODULE__])
+    bucket = Keyword.fetch!(config, :bucket)
+
+    bucket_with_namespace =
+      if namespace = Keyword.get(config, :bucket_namespace) do
+        namespace <> ":" <> bucket
+      else
+        bucket
+      end
 
     {:ok,
      {:url,
       Path.join([
         Keyword.fetch!(config, :public_endpoint),
-        Keyword.fetch!(config, :bucket),
+        bucket_with_namespace,
         strict_encode(URI.decode(file))
       ])}}
   end
 
-  def put_file(upload = %Pleroma.Upload{}) do
+  def put_file(%Pleroma.Upload{} = upload) do
     config = Pleroma.Config.get([__MODULE__])
     bucket = Keyword.get(config, :bucket)
 

lib/pleroma/uploaders/uploader.ex

@@ -27,18 +27,47 @@ defmodule Pleroma.Uploaders.Uploader do
     This allows to correctly proxy or redirect requests to the backend, while allowing to migrate backends without breaking any URL.
   * `{url, url :: String.t}` to bypass `get_file/2` and use the `url` directly in the activity.
   * `{:error, String.t}` error information if the file failed to be saved to the backend.
+  * `:wait_callback` will wait for an http post request at `/api/pleroma/upload_callback/:upload_path` and call the uploader's `http_callback/3` method.
 
 
   """
+  @type file_spec :: {:file | :url, String.t()}
   @callback put_file(Pleroma.Upload.t()) ::
-              :ok | {:ok, {:file | :url, String.t()}} | {:error, String.t()}
+              :ok | {:ok, file_spec()} | {:error, String.t()} | :wait_callback
+
+  @callback http_callback(Plug.Conn.t(), Map.t()) ::
+              {:ok, Plug.Conn.t()}
+              | {:ok, Plug.Conn.t(), file_spec()}
+              | {:error, Plug.Conn.t(), String.t()}
+  @optional_callbacks http_callback: 2
+
+  @spec put_file(module(), Pleroma.Upload.t()) :: {:ok, file_spec()} | {:error, String.t()}
 
-  @spec put_file(module(), Pleroma.Upload.t()) ::
-          {:ok, {:file | :url, String.t()}} | {:error, String.t()}
   def put_file(uploader, upload) do
     case uploader.put_file(upload) do
       :ok -> {:ok, {:file, upload.path}}
-      other -> other
+      :wait_callback -> handle_callback(uploader, upload)
+      {:ok, _} = ok -> ok
+      {:error, _} = error -> error
+    end
+  end
+
+  defp handle_callback(uploader, upload) do
+    :global.register_name({__MODULE__, upload.path}, self())
+
+    receive do
+      {__MODULE__, pid, conn, params} ->
+        case uploader.http_callback(conn, params) do
+          {:ok, conn, ok} ->
+            send(pid, {__MODULE__, conn})
+            {:ok, ok}
+
+          {:error, conn, error} ->
+            send(pid, {__MODULE__, conn})
+            {:error, error}
+        end
+    after
+      30_000 -> {:error, "Uploader callback timeout"}
     end
   end
 end

lib/pleroma/user.ex

@@ -5,18 +5,30 @@
 defmodule Pleroma.User do
   use Ecto.Schema
 
-  import Ecto.{Changeset, Query}
+  import Ecto.Changeset
-  alias Pleroma.{Repo, User, Object, Web, Activity, Notification}
+  import Ecto.Query
+
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Object
+  alias Pleroma.Web
+  alias Pleroma.Activity
+  alias Pleroma.Notification
   alias Comeonin.Pbkdf2
   alias Pleroma.Formatter
   alias Pleroma.Web.CommonAPI.Utils, as: CommonUtils
-  alias Pleroma.Web.{OStatus, Websub, OAuth}
+  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.ActivityPub.{Utils, ActivityPub}
+  alias Pleroma.Web.Websub
+  alias Pleroma.Web.OAuth
+  alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.ActivityPub
 
   require Logger
 
   @type t :: %__MODULE__{}
 
+  @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
+
   @email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/
 
   @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/
@@ -35,8 +47,9 @@ defmodule Pleroma.User do
     field(:avatar, :map)
     field(:local, :boolean, default: true)
     field(:follower_address, :string)
-    field(:search_distance, :float, virtual: true)
+    field(:search_rank, :float, virtual: true)
     field(:tags, {:array, :string}, default: [])
+    field(:bookmarks, {:array, :string}, default: [])
     field(:last_refreshed_at, :naive_datetime)
     has_many(:notifications, Notification)
     embeds_one(:info, Pleroma.User.Info)
@@ -44,20 +57,28 @@ defmodule Pleroma.User do
     timestamps()
   end
 
-  def auth_active?(%User{} = user) do
+  def auth_active?(%User{local: false}), do: true
-    (user.info && !user.info.confirmation_pending) ||
+
-      !Pleroma.Config.get([:instance, :account_activation_required])
+  def auth_active?(%User{info: %User.Info{confirmation_pending: false}}), do: true
+
+  def auth_active?(%User{info: %User.Info{confirmation_pending: true}}),
+    do: !Pleroma.Config.get([:instance, :account_activation_required])
+
+  def auth_active?(_), do: false
+
+  def visible_for?(user, for_user \\ nil)
+
+  def visible_for?(%User{id: user_id}, %User{id: for_id}) when user_id == for_id, do: true
+
+  def visible_for?(%User{} = user, for_user) do
+    auth_active?(user) || superuser?(for_user)
   end
 
-  def remote_or_auth_active?(%User{} = user), do: !user.local || auth_active?(user)
+  def visible_for?(_, _), do: false
 
-  def visible_for?(%User{} = user, for_user \\ nil) do
+  def superuser?(%User{local: true, info: %User.Info{is_admin: true}}), do: true
-    User.remote_or_auth_active?(user) || (for_user && for_user.id == user.id) ||
+  def superuser?(%User{local: true, info: %User.Info{is_moderator: true}}), do: true
-      User.superuser?(for_user)
+  def superuser?(_), do: false
-  end
-
-  def superuser?(nil), do: false
-  def superuser?(%User{} = user), do: user.info && User.Info.superuser?(user.info)
 
   def avatar_url(user) do
     case user.avatar do
@@ -85,12 +106,6 @@ def ap_followers(%User{} = user) do
     "#{ap_id(user)}/followers"
   end
 
-  def follow_changeset(struct, params \\ %{}) do
-    struct
-    |> cast(params, [:following])
-    |> validate_required([:following])
-  end
-
   def user_info(%User{} = user) do
     oneself = if user.local, do: 1, else: 0
 
@@ -229,10 +244,24 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do
     end
   end
 
+  defp autofollow_users(user) do
+    candidates = Pleroma.Config.get([:instance, :autofollowed_nicknames])
+
+    autofollowed_users =
+      from(u in User,
+        where: u.local == true,
+        where: u.nickname in ^candidates
+      )
+      |> Repo.all()
+
+    follow_all(user, autofollowed_users)
+  end
+
   @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)"
   def register(%Ecto.Changeset{} = changeset) do
     with {:ok, user} <- Repo.insert(changeset),
-         {:ok, _} = try_send_confirmation_email(user) do
+         {:ok, user} <- autofollow_users(user),
+         {:ok, _} <- try_send_confirmation_email(user) do
       {:ok, user}
     end
   end
@@ -282,6 +311,38 @@ def maybe_follow(%User{} = follower, %User{info: _info} = followed) do
     end
   end
 
+  @doc "A mass follow for local users. Respects blocks in both directions but does not create activities."
+  @spec follow_all(User.t(), list(User.t())) :: {atom(), User.t()}
+  def follow_all(follower, followeds) do
+    followed_addresses =
+      followeds
+      |> Enum.reject(fn followed -> blocks?(follower, followed) || blocks?(followed, follower) end)
+      |> Enum.map(fn %{follower_address: fa} -> fa end)
+
+    q =
+      from(u in User,
+        where: u.id == ^follower.id,
+        update: [
+          set: [
+            following:
+              fragment(
+                "array(select distinct unnest (array_cat(?, ?)))",
+                u.following,
+                ^followed_addresses
+              )
+          ]
+        ]
+      )
+
+    {1, [follower]} = Repo.update_all(q, [], returning: true)
+
+    Enum.each(followeds, fn followed ->
+      update_follower_count(followed)
+    end)
+
+    set_cache(follower)
+  end
+
   def follow(%User{} = follower, %User{info: info} = followed) do
     user_config = Application.get_env(:pleroma, :user)
     deny_follow_blocked = Keyword.get(user_config, :deny_follow_blocked)
@@ -300,18 +361,17 @@ def follow(%User{} = follower, %User{info: info} = followed) do
           Websub.subscribe(follower, followed)
         end
 
-        following =
+        q =
-          [ap_followers | follower.following]
+          from(u in User,
-          |> Enum.uniq()
+            where: u.id == ^follower.id,
+            update: [push: [following: ^ap_followers]]
+          )
 
-        follower =
+        {1, [follower]} = Repo.update_all(q, [], returning: true)
-          follower
-          |> follow_changeset(%{following: following})
-          |> update_and_set_cache
 
         {:ok, _} = update_follower_count(followed)
 
-        follower
+        set_cache(follower)
     end
   end
 
@@ -319,17 +379,18 @@ def unfollow(%User{} = follower, %User{} = followed) do
     ap_followers = followed.follower_address
 
     if following?(follower, followed) and follower.ap_id != followed.ap_id do
-      following =
+      q =
-        follower.following
+        from(u in User,
-        |> List.delete(ap_followers)
+          where: u.id == ^follower.id,
+          update: [pull: [following: ^ap_followers]]
+        )
 
-      {:ok, follower} =
+      {1, [follower]} = Repo.update_all(q, [], returning: true)
-        follower
-        |> follow_changeset(%{following: following})
-        |> update_and_set_cache
 
       {:ok, followed} = update_follower_count(followed)
 
+      set_cache(follower)
+
       {:ok, follower, Utils.fetch_latest_follow(follower, followed)}
     else
       {:error, "Not subscribed!"}
@@ -363,16 +424,33 @@ def locked?(%User{} = user) do
     user.info.locked || false
   end
 
+  def get_by_id(id) do
+    Repo.get_by(User, id: id)
+  end
+
   def get_by_ap_id(ap_id) do
     Repo.get_by(User, ap_id: ap_id)
   end
 
+  # This is mostly an SPC migration fix. This guesses the user nickname (by taking the last part of the ap_id and the domain) and tries to get that user
+  def get_by_guessed_nickname(ap_id) do
+    domain = URI.parse(ap_id).host
+    name = List.last(String.split(ap_id, "/"))
+    nickname = "#{name}@#{domain}"
+
+    get_by_nickname(nickname)
+  end
+
+  def set_cache(user) do
+    Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
+    Cachex.put(:user_cache, "nickname:#{user.nickname}", user)
+    Cachex.put(:user_cache, "user_info:#{user.id}", user_info(user))
+    {:ok, user}
+  end
+
   def update_and_set_cache(changeset) do
     with {:ok, user} <- Repo.update(changeset) do
-      Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
+      set_cache(user)
-      Cachex.put(:user_cache, "nickname:#{user.nickname}", user)
-      Cachex.put(:user_cache, "user_info:#{user.id}", user_info(user))
-      {:ok, user}
     else
       e -> e
     end
@@ -389,16 +467,37 @@ def get_cached_by_ap_id(ap_id) do
     Cachex.fetch!(:user_cache, key, fn _ -> get_by_ap_id(ap_id) end)
   end
 
+  def get_cached_by_id(id) do
+    key = "id:#{id}"
+
+    ap_id =
+      Cachex.fetch!(:user_cache, key, fn _ ->
+        user = get_by_id(id)
+
+        if user do
+          Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
+          {:commit, user.ap_id}
+        else
+          {:ignore, ""}
+        end
+      end)
+
+    get_cached_by_ap_id(ap_id)
+  end
+
   def get_cached_by_nickname(nickname) do
     key = "nickname:#{nickname}"
     Cachex.fetch!(:user_cache, key, fn _ -> get_or_fetch_by_nickname(nickname) end)
   end
 
+  def get_cached_by_nickname_or_id(nickname_or_id) do
+    get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id)
+  end
+
   def get_by_nickname(nickname) do
     Repo.get_by(User, nickname: nickname) ||
       if Regex.match?(~r(@#{Pleroma.Web.Endpoint.host()})i, nickname) do
-        [local_nickname, _] = String.split(nickname, "@")
+        Repo.get_by(User, nickname: local_nickname(nickname))
-        Repo.get_by(User, nickname: local_nickname)
       end
   end
 
@@ -437,7 +536,7 @@ def get_or_fetch_by_nickname(nickname) do
     end
   end
 
-  def get_followers_query(%User{id: id, follower_address: follower_address}) do
+  def get_followers_query(%User{id: id, follower_address: follower_address}, nil) do
     from(
       u in User,
       where: fragment("? <@ ?", ^[follower_address], u.following),
@@ -445,13 +544,29 @@ def get_followers_query(%User{id: id, follower_address: follower_address}) do
     )
   end
 
-  def get_followers(user) do
+  def get_followers_query(user, page) do
-    q = get_followers_query(user)
+    from(
+      u in get_followers_query(user, nil),
+      limit: 20,
+      offset: ^((page - 1) * 20)
+    )
+  end
+
+  def get_followers_query(user), do: get_followers_query(user, nil)
+
+  def get_followers(user, page \\ nil) do
+    q = get_followers_query(user, page)
 
     {:ok, Repo.all(q)}
   end
 
-  def get_friends_query(%User{id: id, following: following}) do
+  def get_followers_ids(user, page \\ nil) do
+    q = get_followers_query(user, page)
+
+    Repo.all(from(u in q, select: u.id))
+  end
+
+  def get_friends_query(%User{id: id, following: following}, nil) do
     from(
       u in User,
       where: u.follower_address in ^following,
@@ -459,12 +574,28 @@ def get_friends_query(%User{id: id, following: following}) do
     )
   end
 
-  def get_friends(user) do
+  def get_friends_query(user, page) do
-    q = get_friends_query(user)
+    from(
+      u in get_friends_query(user, nil),
+      limit: 20,
+      offset: ^((page - 1) * 20)
+    )
+  end
+
+  def get_friends_query(user), do: get_friends_query(user, nil)
+
+  def get_friends(user, page \\ nil) do
+    q = get_friends_query(user, page)
 
     {:ok, Repo.all(q)}
   end
 
+  def get_friends_ids(user, page \\ nil) do
+    q = get_friends_query(user, page)
+
+    Repo.all(from(u in q, select: u.id))
+  end
+
   def get_follow_requests_query(%User{} = user) do
     from(
       a in Activity,
@@ -596,37 +727,120 @@ def get_recipients_from_activity(%Activity{recipients: to}) do
     Repo.all(query)
   end
 
-  def search(query, resolve \\ false) do
+  def search(query, resolve \\ false, for_user \\ nil) do
-    # strip the beginning @ off if there is a query
+    # Strip the beginning @ off if there is a query
     query = String.trim_leading(query, "@")
 
-    if resolve do
+    if resolve, do: User.get_or_fetch_by_nickname(query)
-      User.get_or_fetch_by_nickname(query)
-    end
 
-    inner =
+    fts_results = do_search(fts_search_subquery(query), for_user)
-      from(
-        u in User,
-        select_merge: %{
-          search_distance:
-            fragment(
-              "? <-> (? || ?)",
-              ^query,
-              u.nickname,
-              u.name
-            )
-        },
-        where: not is_nil(u.nickname)
-      )
 
+    {:ok, trigram_results} =
+      Repo.transaction(fn ->
+        Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
+        do_search(trigram_search_subquery(query), for_user)
+      end)
+
+    Enum.uniq_by(fts_results ++ trigram_results, & &1.id)
+  end
+
+  defp do_search(subquery, for_user, options \\ []) do
     q =
       from(
-        s in subquery(inner),
+        s in subquery(subquery),
-        order_by: s.search_distance,
+        order_by: [desc: s.search_rank],
-        limit: 20
+        limit: ^(options[:limit] || 20)
       )
 
-    Repo.all(q)
+    results =
+      q
+      |> Repo.all()
+      |> Enum.filter(&(&1.search_rank > 0))
+
+    boost_search_results(results, for_user)
+  end
+
+  defp fts_search_subquery(query) do
+    processed_query =
+      query
+      |> String.replace(~r/\W+/, " ")
+      |> String.trim()
+      |> String.split()
+      |> Enum.map(&(&1 <> ":*"))
+      |> Enum.join(" | ")
+
+    from(
+      u in User,
+      select_merge: %{
+        search_rank:
+          fragment(
+            """
+            ts_rank_cd(
+              setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
+              setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'),
+              to_tsquery('simple', ?),
+              32
+            )
+            """,
+            u.nickname,
+            u.name,
+            ^processed_query
+          )
+      },
+      where:
+        fragment(
+          """
+            (setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
+            setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?)
+          """,
+          u.nickname,
+          u.name,
+          ^processed_query
+        )
+    )
+  end
+
+  defp trigram_search_subquery(query) do
+    from(
+      u in User,
+      select_merge: %{
+        search_rank:
+          fragment(
+            "similarity(?, trim(? || ' ' || coalesce(?, '')))",
+            ^query,
+            u.nickname,
+            u.name
+          )
+      },
+      where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^query)
+    )
+  end
+
+  defp boost_search_results(results, nil), do: results
+
+  defp boost_search_results(results, for_user) do
+    friends_ids = get_friends_ids(for_user)
+    followers_ids = get_followers_ids(for_user)
+
+    Enum.map(
+      results,
+      fn u ->
+        search_rank_coef =
+          cond do
+            u.id in friends_ids ->
+              1.2
+
+            u.id in followers_ids ->
+              1.1
+
+            true ->
+              1
+          end
+
+        Map.put(u, :search_rank, u.search_rank * search_rank_coef)
+      end
+    )
+    |> Enum.sort_by(&(-&1.search_rank))
   end
 
   def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do
@@ -726,7 +940,7 @@ def unblock_domain(user, domain) do
     update_and_set_cache(cng)
   end
 
-  def local_user_query() do
+  def local_user_query do
     from(
       u in User,
       where: u.local == true,
@@ -734,7 +948,14 @@ def local_user_query() do
     )
   end
 
-  def moderator_user_query() do
+  def active_local_user_query do
+    from(
+      u in local_user_query(),
+      where: fragment("not (?->'deactivated' @> 'true')", u.info)
+    )
+  end
+
+  def moderator_user_query do
     from(
       u in User,
       where: u.local == true,
@@ -920,7 +1141,7 @@ def parse_bio(bio, user) do
       end)
 
     bio
-    |> CommonUtils.format_input(mentions, tags, "text/plain")
+    |> CommonUtils.format_input(mentions, tags, "text/plain", user_links: [format: :full])
     |> Formatter.emojify(emoji)
   end
 
@@ -957,6 +1178,22 @@ defp update_tags(%User{} = user, new_tags) do
     updated_user
   end
 
+  def bookmark(%User{} = user, status_id) do
+    bookmarks = Enum.uniq(user.bookmarks ++ [status_id])
+    update_bookmarks(user, bookmarks)
+  end
+
+  def unbookmark(%User{} = user, status_id) do
+    bookmarks = Enum.uniq(user.bookmarks -- [status_id])
+    update_bookmarks(user, bookmarks)
+  end
+
+  def update_bookmarks(%User{} = user, bookmarks) do
+    user
+    |> change(%{bookmarks: bookmarks})
+    |> update_and_set_cache
+  end
+
   defp normalize_tags(tags) do
     [tags]
     |> List.flatten()
@@ -970,4 +1207,24 @@ defp local_nickname_regex() do
       @strict_local_nickname_regex
     end
   end
+
+  def local_nickname(nickname_or_mention) do
+    nickname_or_mention
+    |> full_nickname()
+    |> String.split("@")
+    |> hd()
+  end
+
+  def full_nickname(nickname_or_mention),
+    do: String.trim_leading(nickname_or_mention, "@")
+
+  def error_user(ap_id) do
+    %User{
+      name: ap_id,
+      ap_id: ap_id,
+      info: %User.Info{},
+      nickname: "erroruser@example.com",
+      inserted_at: NaiveDateTime.utc_now()
+    }
+  end
 end

lib/pleroma/user/info.ex

@@ -23,6 +23,7 @@ defmodule Pleroma.User.Info do
     field(:ap_enabled, :boolean, default: false)
     field(:is_moderator, :boolean, default: false)
     field(:is_admin, :boolean, default: false)
+    field(:show_role, :boolean, default: true)
     field(:keys, :string, default: nil)
     field(:settings, :map, default: nil)
     field(:magic_key, :string, default: nil)
@@ -30,7 +31,9 @@ defmodule Pleroma.User.Info do
     field(:topic, :string, default: nil)
     field(:hub, :string, default: nil)
     field(:salmon, :string, default: nil)
-    field(:hide_network, :boolean, default: false)
+    field(:hide_followers, :boolean, default: false)
+    field(:hide_follows, :boolean, default: false)
+    field(:pinned_activities, {:array, :string}, default: [])
 
     # Found in the wild
     # ap_id -> Where is this used?
@@ -41,8 +44,6 @@ defmodule Pleroma.User.Info do
     # subject _> Where is this used?
   end
 
-  def superuser?(info), do: info.is_admin || info.is_moderator
-
   def set_activation_status(info, deactivated) do
     params = %{deactivated: deactivated}
 
@@ -144,8 +145,10 @@ def profile_update(info, params) do
       :no_rich_text,
       :default_scope,
       :banner,
-      :hide_network,
+      :hide_follows,
-      :background
+      :hide_followers,
+      :background,
+      :show_role
     ])
   end
 
@@ -195,7 +198,30 @@ def admin_api_update(info, params) do
     info
     |> cast(params, [
       :is_moderator,
-      :is_admin
+      :is_admin,
+      :show_role
     ])
   end
+
+  def add_pinnned_activity(info, %Pleroma.Activity{id: id}) do
+    if id not in info.pinned_activities do
+      max_pinned_statuses = Pleroma.Config.get([:instance, :max_pinned_statuses], 0)
+      params = %{pinned_activities: info.pinned_activities ++ [id]}
+
+      info
+      |> cast(params, [:pinned_activities])
+      |> validate_length(:pinned_activities,
+        max: max_pinned_statuses,
+        message: "You have already pinned the maximum number of statuses"
+      )
+    else
+      change(info)
+    end
+  end
+
+  def remove_pinnned_activity(info, %Pleroma.Activity{id: id}) do
+    params = %{pinned_activities: List.delete(info.pinned_activities, id)}
+
+    cast(info, params, [:pinned_activities])
+  end
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -3,13 +3,22 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
-  alias Pleroma.{Activity, Repo, Object, Upload, User, Notification}
+  alias Pleroma.Activity
-  alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF}
+  alias Pleroma.Repo
+  alias Pleroma.Object
+  alias Pleroma.Upload
+  alias Pleroma.User
+  alias Pleroma.Notification
+  alias Pleroma.Instances
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Federator
   alias Pleroma.Web.OStatus
+
   import Ecto.Query
   import Pleroma.Web.ActivityPub.Utils
+
   require Logger
 
   @httpoison Application.get_env(:pleroma, :httpoison)
@@ -19,23 +28,31 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp get_recipients(%{"type" => "Announce"} = data) do
     to = data["to"] || []
     cc = data["cc"] || []
-    recipients = to ++ cc
     actor = User.get_cached_by_ap_id(data["actor"])
 
-    recipients
+    recipients =
-    |> Enum.filter(fn recipient ->
+      (to ++ cc)
-      case User.get_cached_by_ap_id(recipient) do
+      |> Enum.filter(fn recipient ->
-        nil ->
+        case User.get_cached_by_ap_id(recipient) do
-          true
+          nil ->
+            true
 
-        user ->
+          user ->
-          User.following?(user, actor)
+            User.following?(user, actor)
-      end
+        end
-    end)
+      end)
 
     {recipients, to, cc}
   end
 
+  defp get_recipients(%{"type" => "Create"} = data) do
+    to = data["to"] || []
+    cc = data["cc"] || []
+    actor = data["actor"] || []
+    recipients = (to ++ cc ++ [actor]) |> Enum.uniq()
+    {recipients, to, cc}
+  end
+
   defp get_recipients(data) do
     to = data["to"] || []
     cc = data["cc"] || []
@@ -56,7 +73,7 @@ defp check_actor_is_active(actor) do
     end
   end
 
-  defp check_remote_limit(%{"object" => %{"content" => content}}) do
+  defp check_remote_limit(%{"object" => %{"content" => content}}) when not is_nil(content) do
     limit = Pleroma.Config.get([:instance, :remote_limit])
     String.length(content) <= limit
   end
@@ -80,6 +97,10 @@ def insert(map, local \\ true) when is_map(map) do
           recipients: recipients
         })
 
+      Task.start(fn ->
+        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+      end)
+
       Notification.create_notifications(activity)
       stream_out(activity)
       {:ok, activity}
@@ -92,7 +113,7 @@ def insert(map, local \\ true) when is_map(map) do
   def stream_out(activity) do
     public = "https://www.w3.org/ns/activitystreams#Public"
 
-    if activity.data["type"] in ["Create", "Announce"] do
+    if activity.data["type"] in ["Create", "Announce", "Delete"] do
       Pleroma.Web.Streamer.stream("user", activity)
       Pleroma.Web.Streamer.stream("list", activity)
 
@@ -103,16 +124,18 @@ def stream_out(activity) do
           Pleroma.Web.Streamer.stream("public:local", activity)
         end
 
-        activity.data["object"]
+        if activity.data["type"] in ["Create"] do
-        |> Map.get("tag", [])
+          activity.data["object"]
-        |> Enum.filter(fn tag -> is_bitstring(tag) end)
+          |> Map.get("tag", [])
-        |> Enum.map(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end)
+          |> Enum.filter(fn tag -> is_bitstring(tag) end)
+          |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end)
 
-        if activity.data["object"]["attachment"] != [] do
+          if activity.data["object"]["attachment"] != [] do
-          Pleroma.Web.Streamer.stream("public:media", activity)
+            Pleroma.Web.Streamer.stream("public:media", activity)
 
-          if activity.local do
+            if activity.local do
-            Pleroma.Web.Streamer.stream("public:local:media", activity)
+              Pleroma.Web.Streamer.stream("public:local:media", activity)
+            end
           end
         end
       else
@@ -138,8 +161,9 @@ def create(%{to: to, actor: actor, context: context, object: object} = params) d
              additional
            ),
          {:ok, activity} <- insert(create_data, local),
-         :ok <- maybe_federate(activity),
+         # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
-         {:ok, _actor} <- User.increase_note_count(actor) do
+         {:ok, _actor} <- User.increase_note_count(actor),
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
@@ -224,10 +248,11 @@ def announce(
         %User{ap_id: _} = user,
         %Object{data: %{"id" => _}} = object,
         activity_id \\ nil,
-        local \\ true
+        local \\ true,
+        public \\ true
       ) do
     with true <- is_public?(object),
-         announce_data <- make_announce_data(user, object, activity_id),
+         announce_data <- make_announce_data(user, object, activity_id, public),
          {:ok, activity} <- insert(announce_data, local),
          {:ok, object} <- add_announce_to_object(activity, object),
          :ok <- maybe_federate(activity) do
@@ -285,8 +310,9 @@ def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ tru
 
     with {:ok, _} <- Object.delete(object),
          {:ok, activity} <- insert(data, local),
-         :ok <- maybe_federate(activity),
+         # Changing note count prior to enqueuing federation task in order to avoid race conditions on updating user.info
-         {:ok, _actor} <- User.decrease_note_count(user) do
+         {:ok, _actor} <- User.decrease_note_count(user),
+         :ok <- maybe_federate(activity) do
       {:ok, activity}
     end
   end
@@ -364,21 +390,18 @@ def fetch_public_activities(opts \\ %{}) do
 
   @valid_visibilities ~w[direct unlisted public private]
 
-  defp restrict_visibility(query, %{visibility: "direct"}) do
+  defp restrict_visibility(query, %{visibility: visibility})
-    public = "https://www.w3.org/ns/activitystreams#Public"
+       when visibility in @valid_visibilities do
+    query =
+      from(
+        a in query,
+        where:
+          fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility)
+      )
 
-    from(
+    Ecto.Adapters.SQL.to_sql(:all, Repo, query)
-      activity in query,
+
-      join: sender in User,
+    query
-      on: sender.ap_id == activity.actor,
-      # Are non-direct statuses with no to/cc possible?
-      where:
-        fragment(
-          "not (? && ?)",
-          [^public, sender.follower_address],
-          activity.recipients
-        )
-    )
   end
 
   defp restrict_visibility(_query, %{visibility: visibility})
@@ -394,6 +417,7 @@ def fetch_user_activities(user, reading_user, params \\ %{}) do
       |> Map.put("type", ["Create", "Announce"])
       |> Map.put("actor_id", user.ap_id)
       |> Map.put("whole_db", true)
+      |> Map.put("pinned_activity_ids", user.info.pinned_activities)
 
     recipients =
       if reading_user do
@@ -407,13 +431,42 @@ def fetch_user_activities(user, reading_user, params \\ %{}) do
     |> Enum.reverse()
   end
 
+  defp restrict_since(query, %{"since_id" => ""}), do: query
+
   defp restrict_since(query, %{"since_id" => since_id}) do
     from(activity in query, where: activity.id > ^since_id)
   end
 
   defp restrict_since(query, _), do: query
 
-  defp restrict_tag(query, %{"tag" => tag}) do
+  defp restrict_tag_reject(query, %{"tag_reject" => tag_reject})
+       when is_list(tag_reject) and tag_reject != [] do
+    from(
+      activity in query,
+      where: fragment("(not (? #> '{\"object\",\"tag\"}') \\?| ?)", activity.data, ^tag_reject)
+    )
+  end
+
+  defp restrict_tag_reject(query, _), do: query
+
+  defp restrict_tag_all(query, %{"tag_all" => tag_all})
+       when is_list(tag_all) and tag_all != [] do
+    from(
+      activity in query,
+      where: fragment("(? #> '{\"object\",\"tag\"}') \\?& ?", activity.data, ^tag_all)
+    )
+  end
+
+  defp restrict_tag_all(query, _), do: query
+
+  defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do
+    from(
+      activity in query,
+      where: fragment("(? #> '{\"object\",\"tag\"}') \\?| ?", activity.data, ^tag)
+    )
+  end
+
+  defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do
     from(
       activity in query,
       where: fragment("? <@ (? #> '{\"object\",\"tag\"}')", ^tag, activity.data)
@@ -462,6 +515,8 @@ defp restrict_local(query, %{"local_only" => true}) do
 
   defp restrict_local(query, _), do: query
 
+  defp restrict_max(query, %{"max_id" => ""}), do: query
+
   defp restrict_max(query, %{"max_id" => max_id}) do
     from(activity in query, where: activity.id < ^max_id)
   end
@@ -475,7 +530,7 @@ defp restrict_actor(query, %{"actor_id" => actor_id}) do
   defp restrict_actor(query, _), do: query
 
   defp restrict_type(query, %{"type" => type}) when is_binary(type) do
-    restrict_type(query, %{"type" => [type]})
+    from(activity in query, where: fragment("?->>'type' = ?", activity.data, ^type))
   end
 
   defp restrict_type(query, %{"type" => type}) do
@@ -517,15 +572,6 @@ defp restrict_reblogs(query, %{"exclude_reblogs" => val}) when val == "true" or
 
   defp restrict_reblogs(query, _), do: query
 
-  # Only search through last 100_000 activities by default
-  defp restrict_recent(query, %{"whole_db" => true}), do: query
-
-  defp restrict_recent(query, _) do
-    since = (Repo.aggregate(Activity, :max, :id) || 0) - 100_000
-
-    from(activity in query, where: activity.id > ^since)
-  end
-
   defp restrict_blocked(query, %{"blocking_user" => %User{info: info}}) do
     blocks = info.blocks || []
     domain_blocks = info.domain_blocks || []
@@ -552,6 +598,12 @@ defp restrict_unlisted(query) do
     )
   end
 
+  defp restrict_pinned(query, %{"pinned" => "true", "pinned_activity_ids" => ids}) do
+    from(activity in query, where: activity.id in ^ids)
+  end
+
+  defp restrict_pinned(query, _), do: query
+
   def fetch_activities_query(recipients, opts \\ %{}) do
     base_query =
       from(
@@ -563,6 +615,8 @@ def fetch_activities_query(recipients, opts \\ %{}) do
     base_query
     |> restrict_recipients(recipients, opts["user"])
     |> restrict_tag(opts)
+    |> restrict_tag_reject(opts)
+    |> restrict_tag_all(opts)
     |> restrict_since(opts)
     |> restrict_local(opts)
     |> restrict_limit(opts)
@@ -570,12 +624,12 @@ def fetch_activities_query(recipients, opts \\ %{}) do
     |> restrict_actor(opts)
     |> restrict_type(opts)
     |> restrict_favorited_by(opts)
-    |> restrict_recent(opts)
     |> restrict_blocked(opts)
     |> restrict_media(opts)
     |> restrict_visibility(opts)
     |> restrict_replies(opts)
     |> restrict_reblogs(opts)
+    |> restrict_pinned(opts)
   end
 
   def fetch_activities(recipients, opts \\ %{}) do
@@ -689,7 +743,7 @@ def should_federate?(inbox, public) do
   end
 
   def publish(actor, activity) do
-    followers =
+    remote_followers =
       if actor.follower_address in activity.recipients do
         {:ok, followers} = User.get_followers(actor)
         followers |> Enum.filter(&(!&1.local))
@@ -699,29 +753,31 @@ def publish(actor, activity) do
 
     public = is_public?(activity)
 
-    remote_inboxes =
+    reachable_inboxes_metadata =
-      (Pleroma.Web.Salmon.remote_users(activity) ++ followers)
+      (Pleroma.Web.Salmon.remote_users(activity) ++ remote_followers)
       |> Enum.filter(fn user -> User.ap_enabled?(user) end)
       |> Enum.map(fn %{info: %{source_data: data}} ->
         (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"]
       end)
       |> Enum.uniq()
       |> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
+      |> Instances.filter_reachable()
 
     {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
     json = Jason.encode!(data)
 
-    Enum.each(remote_inboxes, fn inbox ->
+    Enum.each(reachable_inboxes_metadata, fn {inbox, unreachable_since} ->
       Federator.enqueue(:publish_single_ap, %{
         inbox: inbox,
         json: json,
         actor: actor,
-        id: activity.data["id"]
+        id: activity.data["id"],
+        unreachable_since: unreachable_since
       })
     end)
   end
 
-  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
+  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id} = params) do
     Logger.info("Federating #{id} to #{inbox}")
     host = URI.parse(inbox).host
 
@@ -734,15 +790,26 @@ def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
         digest: digest
       })
 
-    @httpoison.post(
+    with {:ok, %{status: code}} when code in 200..299 <-
-      inbox,
+           result =
-      json,
+             @httpoison.post(
-      [
+               inbox,
-        {"Content-Type", "application/activity+json"},
+               json,
-        {"signature", signature},
+               [
-        {"digest", digest}
+                 {"Content-Type", "application/activity+json"},
-      ]
+                 {"signature", signature},
-    )
+                 {"digest", digest}
+               ]
+             ) do
+      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
+        do: Instances.set_reachable(inbox)
+
+      result
+    else
+      {_post_result, response} ->
+        unless params[:unreachable_since], do: Instances.set_unreachable(inbox)
+        {:error, response}
+    end
   end
 
   # TODO:
@@ -801,9 +868,24 @@ def fetch_and_contain_remote_object_from_id(id) do
     end
   end
 
-  def is_public?(activity) do
+  def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false
-    "https://www.w3.org/ns/activitystreams#Public" in (activity.data["to"] ++
+  def is_public?(%Object{data: data}), do: is_public?(data)
-                                                         (activity.data["cc"] || []))
+  def is_public?(%Activity{data: data}), do: is_public?(data)
+  def is_public?(%{"directMessage" => true}), do: false
+
+  def is_public?(data) do
+    "https://www.w3.org/ns/activitystreams#Public" in (data["to"] ++ (data["cc"] || []))
+  end
+
+  def is_private?(activity) do
+    !is_public?(activity) && Enum.any?(activity.data["to"], &String.contains?(&1, "/followers"))
+  end
+
+  def is_direct?(%Activity{data: %{"directMessage" => true}}), do: true
+  def is_direct?(%Object{data: %{"directMessage" => true}}), do: true
+
+  def is_direct?(activity) do
+    !is_public?(activity) && !is_private?(activity)
   end
 
   def visible_for_user?(activity, nil) do

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -4,12 +4,16 @@
 
 defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   use Pleroma.Web, :controller
-  alias Pleroma.{Activity, User, Object}
+
-  alias Pleroma.Web.ActivityPub.{ObjectView, UserView}
+  alias Pleroma.Activity
+  alias Pleroma.User
+  alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.ObjectView
+  alias Pleroma.Web.ActivityPub.UserView
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
-  alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Federator
 
   require Logger
@@ -17,6 +21,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   action_fallback(:errors)
 
   plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
+  plug(:set_requester_reachable when action in [:inbox])
   plug(:relay_active? when action in [:relay])
 
   def relay_active?(conn, _) do
@@ -54,6 +59,49 @@ def object(conn, %{"uuid" => uuid}) do
     end
   end
 
+  def object_likes(conn, %{"uuid" => uuid, "page" => page}) do
+    with ap_id <- o_status_url(conn, :object, uuid),
+         %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
+         {_, true} <- {:public?, ActivityPub.is_public?(object)},
+         likes <- Utils.get_object_likes(object) do
+      {page, _} = Integer.parse(page)
+
+      conn
+      |> put_resp_header("content-type", "application/activity+json")
+      |> json(ObjectView.render("likes.json", ap_id, likes, page))
+    else
+      {:public?, false} ->
+        {:error, :not_found}
+    end
+  end
+
+  def object_likes(conn, %{"uuid" => uuid}) do
+    with ap_id <- o_status_url(conn, :object, uuid),
+         %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
+         {_, true} <- {:public?, ActivityPub.is_public?(object)},
+         likes <- Utils.get_object_likes(object) do
+      conn
+      |> put_resp_header("content-type", "application/activity+json")
+      |> json(ObjectView.render("likes.json", ap_id, likes))
+    else
+      {:public?, false} ->
+        {:error, :not_found}
+    end
+  end
+
+  def activity(conn, %{"uuid" => uuid}) do
+    with ap_id <- o_status_url(conn, :activity, uuid),
+         %Activity{} = activity <- Activity.normalize(ap_id),
+         {_, true} <- {:public?, ActivityPub.is_public?(activity)} do
+      conn
+      |> put_resp_header("content-type", "application/activity+json")
+      |> json(ObjectView.render("object.json", %{object: activity}))
+    else
+      {:public?, false} ->
+        {:error, :not_found}
+    end
+  end
+
   def following(conn, %{"nickname" => nickname, "page" => page}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
          {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
@@ -153,6 +201,14 @@ def relay(conn, _params) do
     end
   end
 
+  def whoami(%{assigns: %{user: %User{} = user}} = conn, _params) do
+    conn
+    |> put_resp_header("content-type", "application/activity+json")
+    |> json(UserView.render("user.json", %{user: user}))
+  end
+
+  def whoami(_conn, _params), do: {:error, :not_found}
+
   def read_inbox(%{assigns: %{user: user}} = conn, %{"nickname" => nickname} = params) do
     if nickname == user.nickname do
       conn
@@ -165,9 +221,48 @@ def read_inbox(%{assigns: %{user: user}} = conn, %{"nickname" => nickname} = par
     end
   end
 
+  def handle_user_activity(user, %{"type" => "Create"} = params) do
+    object =
+      params["object"]
+      |> Map.merge(Map.take(params, ["to", "cc"]))
+      |> Map.put("attributedTo", user.ap_id())
+      |> Transmogrifier.fix_object()
+
+    ActivityPub.create(%{
+      to: params["to"],
+      actor: user,
+      context: object["context"],
+      object: object,
+      additional: Map.take(params, ["cc"])
+    })
+  end
+
+  def handle_user_activity(user, %{"type" => "Delete"} = params) do
+    with %Object{} = object <- Object.normalize(params["object"]),
+         true <- user.info.is_moderator || user.ap_id == object.data["actor"],
+         {:ok, delete} <- ActivityPub.delete(object) do
+      {:ok, delete}
+    else
+      _ -> {:error, "Can't delete object"}
+    end
+  end
+
+  def handle_user_activity(user, %{"type" => "Like"} = params) do
+    with %Object{} = object <- Object.normalize(params["object"]),
+         {:ok, activity, _object} <- ActivityPub.like(user, object) do
+      {:ok, activity}
+    else
+      _ -> {:error, "Can't like object"}
+    end
+  end
+
+  def handle_user_activity(_, _) do
+    {:error, "Unhandled activity type"}
+  end
+
   def update_outbox(
         %{assigns: %{user: user}} = conn,
-        %{"nickname" => nickname, "type" => "Create"} = params
+        %{"nickname" => nickname} = params
       ) do
     if nickname == user.nickname do
       actor = user.ap_id()
@@ -178,24 +273,16 @@ def update_outbox(
         |> Map.put("actor", actor)
         |> Transmogrifier.fix_addressing()
 
-      object =
+      with {:ok, %Activity{} = activity} <- handle_user_activity(user, params) do
-        params["object"]
-        |> Map.merge(Map.take(params, ["to", "cc"]))
-        |> Map.put("attributedTo", actor)
-        |> Transmogrifier.fix_object()
-
-      with {:ok, %Activity{} = activity} <-
-             ActivityPub.create(%{
-               to: params["to"],
-               actor: user,
-               context: object["context"],
-               object: object,
-               additional: Map.take(params, ["cc"])
-             }) do
         conn
         |> put_status(:created)
         |> put_resp_header("location", activity.data["id"])
         |> json(activity.data)
+      else
+        {:error, message} ->
+          conn
+          |> put_status(:bad_request)
+          |> json(message)
       end
     else
       conn
@@ -215,4 +302,13 @@ def errors(conn, _e) do
     |> put_status(500)
     |> json("error")
   end
+
+  defp set_requester_reachable(%Plug.Conn{} = conn, _) do
+    with actor <- conn.params["actor"],
+         true <- is_binary(actor) do
+      Pleroma.Instances.set_reachable(actor)
+    end
+
+    conn
+  end
 end

lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex

@@ -0,0 +1,57 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
+  alias Pleroma.User
+
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  # XXX: this should become User.normalize_by_ap_id() or similar, really.
+  defp normalize_by_ap_id(%{"id" => id}), do: User.get_cached_by_ap_id(id)
+  defp normalize_by_ap_id(uri) when is_binary(uri), do: User.get_cached_by_ap_id(uri)
+  defp normalize_by_ap_id(_), do: nil
+
+  defp score_nickname("followbot@" <> _), do: 1.0
+  defp score_nickname("federationbot@" <> _), do: 1.0
+  defp score_nickname("federation_bot@" <> _), do: 1.0
+  defp score_nickname(_), do: 0.0
+
+  defp score_displayname("federation bot"), do: 1.0
+  defp score_displayname("federationbot"), do: 1.0
+  defp score_displayname("fedibot"), do: 1.0
+  defp score_displayname(_), do: 0.0
+
+  defp determine_if_followbot(%User{nickname: nickname, name: displayname}) do
+    nick_score =
+      nickname
+      |> String.downcase()
+      |> score_nickname()
+
+    name_score =
+      displayname
+      |> String.downcase()
+      |> score_displayname()
+
+    nick_score + name_score
+  end
+
+  defp determine_if_followbot(_), do: 0.0
+
+  @impl true
+  def filter(%{"type" => "Follow", "actor" => actor_id} = message) do
+    %User{} = actor = normalize_by_ap_id(actor_id)
+
+    score = determine_if_followbot(actor)
+
+    # TODO: scan biography data for keywords and score it somehow.
+    if score < 0.8 do
+      {:ok, message}
+    else
+      {:reject, nil}
+    end
+  end
+
+  @impl true
+  def filter(message), do: {:ok, message}
+end

lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex

@@ -3,20 +3,46 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
+  alias Pleroma.User
   @behaviour Pleroma.Web.ActivityPub.MRF
 
-  @impl true
+  defp delist_message(message) do
-  def filter(%{"type" => "Create"} = object) do
+    follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address
-    threshold = Pleroma.Config.get([:mrf_hellthread, :threshold])
-    recipients = (object["to"] || []) ++ (object["cc"] || [])
 
-    if length(recipients) > threshold do
+    message
-      {:reject, nil}
+    |> Map.put("to", [follower_collection])
-    else
+    |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"])
-      {:ok, object}
+  end
+
+  @impl true
+  def filter(%{"type" => "Create"} = message) do
+    delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold])
+
+    reject_threshold =
+      Pleroma.Config.get(
+        [:mrf_hellthread, :reject_threshold],
+        Pleroma.Config.get([:mrf_hellthread, :threshold])
+      )
+
+    recipients = (message["to"] || []) ++ (message["cc"] || [])
+
+    cond do
+      length(recipients) > reject_threshold and reject_threshold > 0 ->
+        {:reject, nil}
+
+      length(recipients) > delist_threshold and delist_threshold > 0 ->
+        if Enum.member?(message["to"], "https://www.w3.org/ns/activitystreams#Public") or
+             Enum.member?(message["cc"], "https://www.w3.org/ns/activitystreams#Public") do
+          {:ok, delist_message(message)}
+        else
+          {:ok, message}
+        end
+
+      true ->
+        {:ok, message}
     end
   end
 
   @impl true
-  def filter(object), do: {:ok, object}
+  def filter(message), do: {:ok, message}
 end

lib/pleroma/web/activity_pub/mrf/keyword_policy.ex

@@ -0,0 +1,81 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
+  @behaviour Pleroma.Web.ActivityPub.MRF
+  defp string_matches?(string, pattern) when is_binary(pattern) do
+    String.contains?(string, pattern)
+  end
+
+  defp string_matches?(string, pattern) do
+    String.match?(string, pattern)
+  end
+
+  defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} = message) do
+    if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
+         string_matches?(content, pattern) or string_matches?(summary, pattern)
+       end) do
+      {:reject, nil}
+    else
+      {:ok, message}
+    end
+  end
+
+  defp check_ftl_removal(
+         %{"to" => to, "object" => %{"content" => content, "summary" => summary}} = message
+       ) do
+    if "https://www.w3.org/ns/activitystreams#Public" in to and
+         Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
+           string_matches?(content, pattern) or string_matches?(summary, pattern)
+         end) do
+      to = List.delete(to, "https://www.w3.org/ns/activitystreams#Public")
+      cc = ["https://www.w3.org/ns/activitystreams#Public" | message["cc"] || []]
+
+      message =
+        message
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+
+      {:ok, message}
+    else
+      {:ok, message}
+    end
+  end
+
+  defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do
+    {content, summary} =
+      Enum.reduce(Pleroma.Config.get([:mrf_keyword, :replace]), {content, summary}, fn {pattern,
+                                                                                        replacement},
+                                                                                       {content_acc,
+                                                                                        summary_acc} ->
+        {String.replace(content_acc, pattern, replacement),
+         String.replace(summary_acc, pattern, replacement)}
+      end)
+
+    {:ok,
+     message
+     |> put_in(["object", "content"], content)
+     |> put_in(["object", "summary"], summary)}
+  end
+
+  @impl true
+  def filter(%{"object" => %{"content" => nil}} = message) do
+    {:ok, message}
+  end
+
+  @impl true
+  def filter(%{"type" => "Create", "object" => %{"content" => _content}} = message) do
+    with {:ok, message} <- check_reject(message),
+         {:ok, message} <- check_ftl_removal(message),
+         {:ok, message} <- check_replace(message) do
+      {:ok, message}
+    else
+      _e ->
+        {:reject, nil}
+    end
+  end
+
+  @impl true
+  def filter(message), do: {:ok, message}
+end

lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex

@@ -0,0 +1,29 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  @impl true
+  def filter(
+        %{
+          "type" => "Create",
+          "object" => %{"content" => content, "attachment" => _attachment} = child_object
+        } = object
+      )
+      when content in [".", "<p>.</p>"] do
+    child_object =
+      child_object
+      |> Map.put("content", "")
+
+    object =
+      object
+      |> Map.put("object", child_object)
+
+    {:ok, object}
+  end
+
+  @impl true
+  def filter(object), do: {:ok, object}
+end

lib/pleroma/web/activity_pub/mrf/tag_policy.ex

@@ -0,0 +1,139 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
+  alias Pleroma.User
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  defp get_tags(%User{tags: tags}) when is_list(tags), do: tags
+  defp get_tags(_), do: []
+
+  defp process_tag(
+         "mrf_tag:media-force-nsfw",
+         %{"type" => "Create", "object" => %{"attachment" => child_attachment} = object} = message
+       )
+       when length(child_attachment) > 0 do
+    tags = (object["tag"] || []) ++ ["nsfw"]
+
+    object =
+      object
+      |> Map.put("tags", tags)
+      |> Map.put("sensitive", true)
+
+    message = Map.put(message, "object", object)
+
+    {:ok, message}
+  end
+
+  defp process_tag(
+         "mrf_tag:media-strip",
+         %{"type" => "Create", "object" => %{"attachment" => child_attachment} = object} = message
+       )
+       when length(child_attachment) > 0 do
+    object = Map.delete(object, "attachment")
+    message = Map.put(message, "object", object)
+
+    {:ok, message}
+  end
+
+  defp process_tag(
+         "mrf_tag:force-unlisted",
+         %{"type" => "Create", "to" => to, "cc" => cc, "actor" => actor} = message
+       ) do
+    user = User.get_cached_by_ap_id(actor)
+
+    if Enum.member?(to, "https://www.w3.org/ns/activitystreams#Public") do
+      to =
+        List.delete(to, "https://www.w3.org/ns/activitystreams#Public") ++ [user.follower_address]
+
+      cc =
+        List.delete(cc, user.follower_address) ++ ["https://www.w3.org/ns/activitystreams#Public"]
+
+      object =
+        message["object"]
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+
+      message =
+        message
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+        |> Map.put("object", object)
+
+      {:ok, message}
+    else
+      {:ok, message}
+    end
+  end
+
+  defp process_tag(
+         "mrf_tag:sandbox",
+         %{"type" => "Create", "to" => to, "cc" => cc, "actor" => actor} = message
+       ) do
+    user = User.get_cached_by_ap_id(actor)
+
+    if Enum.member?(to, "https://www.w3.org/ns/activitystreams#Public") or
+         Enum.member?(cc, "https://www.w3.org/ns/activitystreams#Public") do
+      to =
+        List.delete(to, "https://www.w3.org/ns/activitystreams#Public") ++ [user.follower_address]
+
+      cc = List.delete(cc, "https://www.w3.org/ns/activitystreams#Public")
+
+      object =
+        message["object"]
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+
+      message =
+        message
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+        |> Map.put("object", object)
+
+      {:ok, message}
+    else
+      {:ok, message}
+    end
+  end
+
+  defp process_tag(
+         "mrf_tag:disable-remote-subscription",
+         %{"type" => "Follow", "actor" => actor} = message
+       ) do
+    user = User.get_cached_by_ap_id(actor)
+
+    if user.local == true do
+      {:ok, message}
+    else
+      {:reject, nil}
+    end
+  end
+
+  defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow"}), do: {:reject, nil}
+
+  defp process_tag(_, message), do: {:ok, message}
+
+  def filter_message(actor, message) do
+    User.get_cached_by_ap_id(actor)
+    |> get_tags()
+    |> Enum.reduce({:ok, message}, fn
+      tag, {:ok, message} ->
+        process_tag(tag, message)
+
+      _, error ->
+        error
+    end)
+  end
+
+  @impl true
+  def filter(%{"object" => target_actor, "type" => "Follow"} = message),
+    do: filter_message(target_actor, message)
+
+  @impl true
+  def filter(%{"actor" => actor, "type" => "Create"} = message),
+    do: filter_message(actor, message)
+
+  @impl true
+  def filter(message), do: {:ok, message}
+end

lib/pleroma/web/activity_pub/relay.ex

@@ -3,7 +3,9 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.Relay do
-  alias Pleroma.{User, Object, Activity}
+  alias Pleroma.User
+  alias Pleroma.Object
+  alias Pleroma.Activity
   alias Pleroma.Web.ActivityPub.ActivityPub
   require Logger
 
@@ -40,7 +42,7 @@ def unfollow(target_instance) do
   def publish(%Activity{data: %{"type" => "Create"}} = activity) do
     with %User{} = user <- get_actor(),
          %Object{} = object <- Object.normalize(activity.data["object"]["id"]) do
-      ActivityPub.announce(user, object)
+      ActivityPub.announce(user, object, nil, true, false)
     else
       e -> Logger.error("error: #{inspect(e)}")
     end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -6,9 +6,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   @moduledoc """
   A module to handle coding from internal to wire ActivityPub and back.
   """
+  alias Pleroma.Activity
   alias Pleroma.User
   alias Pleroma.Object
-  alias Pleroma.Activity
   alias Pleroma.Repo
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
@@ -93,12 +93,47 @@ def fix_addressing_list(map, field) do
     end
   end
 
-  def fix_addressing(map) do
+  def fix_explicit_addressing(%{"to" => to, "cc" => cc} = object, explicit_mentions) do
-    map
+    explicit_to =
+      to
+      |> Enum.filter(fn x -> x in explicit_mentions end)
+
+    explicit_cc =
+      to
+      |> Enum.filter(fn x -> x not in explicit_mentions end)
+
+    final_cc =
+      (cc ++ explicit_cc)
+      |> Enum.uniq()
+
+    object
+    |> Map.put("to", explicit_to)
+    |> Map.put("cc", final_cc)
+  end
+
+  def fix_explicit_addressing(object, _explicit_mentions), do: object
+
+  # if directMessage flag is set to true, leave the addressing alone
+  def fix_explicit_addressing(%{"directMessage" => true} = object), do: object
+
+  def fix_explicit_addressing(object) do
+    explicit_mentions =
+      object
+      |> Utils.determine_explicit_mentions()
+
+    explicit_mentions = explicit_mentions ++ ["https://www.w3.org/ns/activitystreams#Public"]
+
+    object
+    |> fix_explicit_addressing(explicit_mentions)
+  end
+
+  def fix_addressing(object) do
+    object
     |> fix_addressing_list("to")
     |> fix_addressing_list("cc")
     |> fix_addressing_list("bto")
     |> fix_addressing_list("bcc")
+    |> fix_explicit_addressing
   end
 
   def fix_actor(%{"attributedTo" => actor} = object) do
@@ -106,11 +141,11 @@ def fix_actor(%{"attributedTo" => actor} = object) do
     |> Map.put("actor", get_actor(%{"actor" => actor}))
   end
 
-  def fix_likes(%{"likes" => likes} = object)
+  # Check for standardisation
-      when is_bitstring(likes) do
+  # This is what Peertube does
-    # Check for standardisation
+  # curl -H 'Accept: application/activity+json' $likes | jq .totalItems
-    # This is what Peertube does
+  # Prismo returns only an integer (count) as "likes"
-    # curl -H 'Accept: application/activity+json' $likes | jq .totalItems
+  def fix_likes(%{"likes" => likes} = object) when not is_map(likes) do
     object
     |> Map.put("likes", [])
     |> Map.put("like_count", 0)
@@ -141,7 +176,7 @@ def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object)
     case fetch_obj_helper(in_reply_to_id) do
       {:ok, replied_object} ->
         with %Activity{} = activity <-
-               Activity.get_create_activity_by_object_ap_id(replied_object.data["id"]) do
+               Activity.get_create_by_object_ap_id(replied_object.data["id"]) do
           object
           |> Map.put("inReplyTo", replied_object.data["id"])
           |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id)
@@ -278,6 +313,8 @@ def fix_tag(%{"tag" => %{"type" => "Hashtag", "name" => hashtag} = tag} = object
     |> Map.put("tag", combined)
   end
 
+  def fix_tag(%{"tag" => %{} = tag} = object), do: Map.put(object, "tag", [tag])
+
   def fix_tag(object), do: object
 
   # content map usually only has one language so this will do for now.
@@ -334,7 +371,7 @@ def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = obj
       Map.put(data, "actor", actor)
       |> fix_addressing
 
-    with nil <- Activity.get_create_activity_by_object_ap_id(object["id"]),
+    with nil <- Activity.get_create_by_object_ap_id(object["id"]),
          %User{} = user <- User.get_or_fetch_by_ap_id(data["actor"]) do
       object = fix_object(data["object"])
 
@@ -348,6 +385,7 @@ def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = obj
         additional:
           Map.take(data, [
             "cc",
+            "directMessage",
             "id"
           ])
       }
@@ -417,9 +455,9 @@ def handle_incoming(
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
          {:ok, activity} <-
-           ActivityPub.accept(%{
+           ActivityPub.reject(%{
              to: follow_activity.data["to"],
-             type: "Accept",
+             type: "Reject",
              actor: followed.ap_id,
              object: follow_activity.data["id"],
              local: false
@@ -451,7 +489,8 @@ def handle_incoming(
     with actor <- get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
-         {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false) do
+         public <- ActivityPub.is_public?(data),
+         {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do
       {:ok, activity}
     else
       _e -> :error
@@ -629,6 +668,7 @@ def prepare_object(object) do
     |> add_mention_tags
     |> add_emoji_tags
     |> add_attributed_to
+    |> add_likes
     |> prepare_attachments
     |> set_conversation
     |> set_reply_to_uri
@@ -641,7 +681,7 @@ def prepare_object(object) do
   #  internal -> Mastodon
   #  """
 
-  def prepare_outgoing(%{"type" => "Create", "object" => %{"type" => "Note"} = object} = data) do
+  def prepare_outgoing(%{"type" => "Create", "object" => object} = data) do
     object =
       object
       |> prepare_object
@@ -788,6 +828,22 @@ def add_attributed_to(object) do
     |> Map.put("attributedTo", attributedTo)
   end
 
+  def add_likes(%{"id" => id, "like_count" => likes} = object) do
+    likes = %{
+      "id" => "#{id}/likes",
+      "first" => "#{id}/likes?page=1",
+      "type" => "OrderedCollection",
+      "totalItems" => likes
+    }
+
+    object
+    |> Map.put("likes", likes)
+  end
+
+  def add_likes(object) do
+    object
+  end
+
   def prepare_attachments(object) do
     attachments =
       (object["attachment"] || [])
@@ -803,7 +859,6 @@ def prepare_attachments(object) do
   defp strip_internal_fields(object) do
     object
     |> Map.drop([
-      "likes",
       "like_count",
       "announcements",
       "announcement_count",
@@ -847,15 +902,10 @@ defp user_upgrade_task(user) do
 
     maybe_retire_websub(user.ap_id)
 
-    # Only do this for recent activties, don't go through the whole db.
-    # Only look at the last 1000 activities.
-    since = (Repo.aggregate(Activity, :max, :id) || 0) - 1_000
-
     q =
       from(
         a in Activity,
         where: ^old_follower_address in a.recipients,
-        where: a.id > ^since,
         update: [
           set: [
             recipients:

lib/pleroma/web/activity_pub/utils.ex

@@ -3,11 +3,19 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.Utils do
-  alias Pleroma.{Repo, Web, Object, Activity, User, Notification}
+  alias Pleroma.Repo
+  alias Pleroma.Web
+  alias Pleroma.Object
+  alias Pleroma.Activity
+  alias Pleroma.User
+  alias Pleroma.Notification
   alias Pleroma.Web.Router.Helpers
   alias Pleroma.Web.Endpoint
-  alias Ecto.{Changeset, UUID}
+  alias Ecto.Changeset
+  alias Ecto.UUID
+
   import Ecto.Query
+
   require Logger
 
   @supported_object_types ["Article", "Note", "Video", "Page"]
@@ -25,6 +33,20 @@ def normalize_params(params) do
     Map.put(params, "actor", get_ap_id(params["actor"]))
   end
 
+  def determine_explicit_mentions(%{"tag" => tag} = _object) when is_list(tag) do
+    tag
+    |> Enum.filter(fn x -> is_map(x) end)
+    |> Enum.filter(fn x -> x["type"] == "Mention" end)
+    |> Enum.map(fn x -> x["href"] end)
+  end
+
+  def determine_explicit_mentions(%{"tag" => tag} = object) when is_map(tag) do
+    Map.put(object, "tag", [tag])
+    |> determine_explicit_mentions()
+  end
+
+  def determine_explicit_mentions(_), do: []
+
   defp recipient_in_collection(ap_id, coll) when is_binary(coll), do: ap_id == coll
   defp recipient_in_collection(ap_id, coll) when is_list(coll), do: ap_id in coll
   defp recipient_in_collection(_, _), do: false
@@ -198,7 +220,7 @@ def update_object_in_activities(%{data: %{"id" => id}} = object) do
     # Update activities that already had this. Could be done in a seperate process.
     # Alternatively, just don't do this and fetch the current object each time. Most
     # could probably be taken from cache.
-    relevant_activities = Activity.all_by_object_ap_id(id)
+    relevant_activities = Activity.get_all_create_by_object_ap_id(id)
 
     Enum.map(relevant_activities, fn activity ->
       new_activity_data = activity.data |> Map.put("object", object.data)
@@ -231,6 +253,27 @@ def get_existing_like(actor, %{data: %{"id" => id}}) do
     Repo.one(query)
   end
 
+  @doc """
+  Returns like activities targeting an object
+  """
+  def get_object_likes(%{data: %{"id" => id}}) do
+    query =
+      from(
+        activity in Activity,
+        # this is to use the index
+        where:
+          fragment(
+            "coalesce((?)->'object'->>'id', (?)->>'object') = ?",
+            activity.data,
+            activity.data,
+            ^id
+          ),
+        where: fragment("(?)->>'type' = 'Like'", activity.data)
+      )
+
+    Repo.all(query)
+  end
+
   def make_like_data(%User{ap_id: ap_id} = actor, %{data: %{"id" => id}} = object, activity_id) do
     data = %{
       "type" => "Like",
@@ -250,7 +293,7 @@ def update_element_in_object(property, element, object) do
            |> Map.put("#{property}_count", length(element))
            |> Map.put("#{property}s", element),
          changeset <- Changeset.change(object, data: new_data),
-         {:ok, object} <- Repo.update(changeset),
+         {:ok, object} <- Object.update_and_set_cache(changeset),
          _ <- update_object_in_activities(object) do
       {:ok, object}
     end
@@ -281,6 +324,25 @@ def remove_like_from_object(%Activity{data: %{"actor" => actor}}, object) do
   @doc """
   Updates a follow activity's state (for locked accounts).
   """
+  def update_follow_state(
+        %Activity{data: %{"actor" => actor, "object" => object, "state" => "pending"}} = activity,
+        state
+      ) do
+    try do
+      Ecto.Adapters.SQL.query!(
+        Repo,
+        "UPDATE activities SET data = jsonb_set(data, '{state}', $1) WHERE data->>'type' = 'Follow' AND data->>'actor' = $2 AND data->>'object' = $3 AND data->>'state' = 'pending'",
+        [state, actor, object]
+      )
+
+      activity = Repo.get(Activity, activity.id)
+      {:ok, activity}
+    rescue
+      e ->
+        {:error, e}
+    end
+  end
+
   def update_follow_state(%Activity{} = activity, state) do
     with new_data <-
            activity.data
@@ -365,9 +427,10 @@ def get_existing_announce(actor, %{data: %{"id" => id}}) do
   """
   # for relayed messages, we only want to send to subscribers
   def make_announce_data(
-        %User{ap_id: ap_id, nickname: nil} = user,
+        %User{ap_id: ap_id} = user,
         %Object{data: %{"id" => id}} = object,
-        activity_id
+        activity_id,
+        false
       ) do
     data = %{
       "type" => "Announce",
@@ -384,7 +447,8 @@ def make_announce_data(
   def make_announce_data(
         %User{ap_id: ap_id} = user,
         %Object{data: %{"id" => id}} = object,
-        activity_id
+        activity_id,
+        true
       ) do
     data = %{
       "type" => "Announce",

lib/pleroma/web/activity_pub/views/object_view.ex

@@ -4,7 +4,8 @@
 
 defmodule Pleroma.Web.ActivityPub.ObjectView do
   use Pleroma.Web, :view
-  alias Pleroma.{Object, Activity}
+  alias Pleroma.Activity
+  alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.Transmogrifier
 
   def render("object.json", %{object: %Object{} = object}) do
@@ -35,4 +36,38 @@ def render("object.json", %{object: %Activity{} = activity}) do
 
     Map.merge(base, additional)
   end
+
+  def render("likes.json", ap_id, likes, page) do
+    collection(likes, "#{ap_id}/likes", page)
+    |> Map.merge(Pleroma.Web.ActivityPub.Utils.make_json_ld_header())
+  end
+
+  def render("likes.json", ap_id, likes) do
+    %{
+      "id" => "#{ap_id}/likes",
+      "type" => "OrderedCollection",
+      "totalItems" => length(likes),
+      "first" => collection(likes, "#{ap_id}/likes", 1)
+    }
+    |> Map.merge(Pleroma.Web.ActivityPub.Utils.make_json_ld_header())
+  end
+
+  def collection(collection, iri, page) do
+    offset = (page - 1) * 10
+    items = Enum.slice(collection, offset, 10)
+    items = Enum.map(items, fn object -> Transmogrifier.prepare_object(object.data) end)
+    total = length(collection)
+
+    map = %{
+      "id" => "#{iri}?page=#{page}",
+      "type" => "OrderedCollectionPage",
+      "partOf" => iri,
+      "totalItems" => total,
+      "orderedItems" => items
+    }
+
+    if offset < total do
+      Map.put(map, "next", "#{iri}?page=#{page + 1}")
+    end
+  end
 end

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -4,15 +4,34 @@
 
 defmodule Pleroma.Web.ActivityPub.UserView do
   use Pleroma.Web, :view
-  alias Pleroma.Web.Salmon
+
   alias Pleroma.Web.WebFinger
+  alias Pleroma.Web.Salmon
   alias Pleroma.User
   alias Pleroma.Repo
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.Router.Helpers
+  alias Pleroma.Web.Endpoint
+
   import Ecto.Query
 
+  def render("endpoints.json", %{user: %User{nickname: nil, local: true} = _user}) do
+    %{"sharedInbox" => Helpers.activity_pub_url(Endpoint, :inbox)}
+  end
+
+  def render("endpoints.json", %{user: %User{local: true} = _user}) do
+    %{
+      "oauthAuthorizationEndpoint" => Helpers.o_auth_url(Endpoint, :authorize),
+      "oauthRegistrationEndpoint" => Helpers.mastodon_api_url(Endpoint, :create_app),
+      "oauthTokenEndpoint" => Helpers.o_auth_url(Endpoint, :token_exchange),
+      "sharedInbox" => Helpers.activity_pub_url(Endpoint, :inbox)
+    }
+  end
+
+  def render("endpoints.json", _), do: %{}
+
   # the instance itself is not a Person, but instead an Application
   def render("user.json", %{user: %{nickname: nil} = user}) do
     {:ok, user} = WebFinger.ensure_keys_present(user)
@@ -20,6 +39,8 @@ def render("user.json", %{user: %{nickname: nil} = user}) do
     public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
     public_key = :public_key.pem_encode([public_key])
 
+    endpoints = render("endpoints.json", %{user: user})
+
     %{
       "id" => user.ap_id,
       "type" => "Application",
@@ -35,9 +56,7 @@ def render("user.json", %{user: %{nickname: nil} = user}) do
         "owner" => user.ap_id,
         "publicKeyPem" => public_key
       },
-      "endpoints" => %{
+      "endpoints" => endpoints
-        "sharedInbox" => "#{Pleroma.Web.Endpoint.url()}/inbox"
-      }
     }
     |> Map.merge(Utils.make_json_ld_header())
   end
@@ -48,6 +67,8 @@ def render("user.json", %{user: user}) do
     public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
     public_key = :public_key.pem_encode([public_key])
 
+    endpoints = render("endpoints.json", %{user: user})
+
     %{
       "id" => user.ap_id,
       "type" => "Person",
@@ -65,9 +86,7 @@ def render("user.json", %{user: user}) do
         "owner" => user.ap_id,
         "publicKeyPem" => public_key
       },
-      "endpoints" => %{
+      "endpoints" => endpoints,
-        "sharedInbox" => "#{Pleroma.Web.Endpoint.url()}/inbox"
-      },
       "icon" => %{
         "type" => "Image",
         "url" => User.avatar_url(user)
@@ -86,7 +105,14 @@ def render("following.json", %{user: user, page: page}) do
     query = from(user in query, select: [:ap_id])
     following = Repo.all(query)
 
-    collection(following, "#{user.ap_id}/following", page, !user.info.hide_network)
+    total =
+      if !user.info.hide_follows do
+        length(following)
+      else
+        0
+      end
+
+    collection(following, "#{user.ap_id}/following", page, !user.info.hide_follows, total)
     |> Map.merge(Utils.make_json_ld_header())
   end
 
@@ -95,11 +121,18 @@ def render("following.json", %{user: user}) do
     query = from(user in query, select: [:ap_id])
     following = Repo.all(query)
 
+    total =
+      if !user.info.hide_follows do
+        length(following)
+      else
+        0
+      end
+
     %{
       "id" => "#{user.ap_id}/following",
       "type" => "OrderedCollection",
-      "totalItems" => length(following),
+      "totalItems" => total,
-      "first" => collection(following, "#{user.ap_id}/following", 1, !user.info.hide_network)
+      "first" => collection(following, "#{user.ap_id}/following", 1, !user.info.hide_follows)
     }
     |> Map.merge(Utils.make_json_ld_header())
   end
@@ -109,7 +142,14 @@ def render("followers.json", %{user: user, page: page}) do
     query = from(user in query, select: [:ap_id])
     followers = Repo.all(query)
 
-    collection(followers, "#{user.ap_id}/followers", page, !user.info.hide_network)
+    total =
+      if !user.info.hide_followers do
+        length(followers)
+      else
+        0
+      end
+
+    collection(followers, "#{user.ap_id}/followers", page, !user.info.hide_followers, total)
     |> Map.merge(Utils.make_json_ld_header())
   end
 
@@ -118,19 +158,24 @@ def render("followers.json", %{user: user}) do
     query = from(user in query, select: [:ap_id])
     followers = Repo.all(query)
 
+    total =
+      if !user.info.hide_followers do
+        length(followers)
+      else
+        0
+      end
+
     %{
       "id" => "#{user.ap_id}/followers",
       "type" => "OrderedCollection",
-      "totalItems" => length(followers),
+      "totalItems" => total,
-      "first" => collection(followers, "#{user.ap_id}/followers", 1, !user.info.hide_network)
+      "first" =>
+        collection(followers, "#{user.ap_id}/followers", 1, !user.info.hide_followers, total)
     }
     |> Map.merge(Utils.make_json_ld_header())
   end
 
   def render("outbox.json", %{user: user, max_id: max_qid}) do
-    # XXX: technically note_count is wrong for this, but it's better than nothing
-    info = User.user_info(user)
-
     params = %{
       "limit" => "10"
     }
@@ -158,16 +203,14 @@ def render("outbox.json", %{user: user, max_id: max_qid}) do
       "id" => "#{iri}?max_id=#{max_id}",
       "type" => "OrderedCollectionPage",
       "partOf" => iri,
-      "totalItems" => info.note_count,
       "orderedItems" => collection,
-      "next" => "#{iri}?max_id=#{min_id - 1}"
+      "next" => "#{iri}?max_id=#{min_id}"
     }
 
     if max_qid == nil do
       %{
         "id" => iri,
         "type" => "OrderedCollection",
-        "totalItems" => info.note_count,
         "first" => page
       }
       |> Map.merge(Utils.make_json_ld_header())
@@ -205,16 +248,14 @@ def render("inbox.json", %{user: user, max_id: max_qid}) do
       "id" => "#{iri}?max_id=#{max_id}",
       "type" => "OrderedCollectionPage",
       "partOf" => iri,
-      "totalItems" => -1,
       "orderedItems" => collection,
-      "next" => "#{iri}?max_id=#{min_id - 1}"
+      "next" => "#{iri}?max_id=#{min_id}"
     }
 
     if max_qid == nil do
       %{
         "id" => iri,
         "type" => "OrderedCollection",
-        "totalItems" => -1,
         "first" => page
       }
       |> Map.merge(Utils.make_json_ld_header())
@@ -239,6 +280,8 @@ def collection(collection, iri, page, show_items \\ true, total \\ nil) do
 
     if offset < total do
       Map.put(map, "next", "#{iri}?page=#{page + 1}")
+    else
+      map
     end
   end
 end

lib/pleroma/web/common_api/common_api.ex

@@ -3,7 +3,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.CommonAPI do
-  alias Pleroma.{User, Repo, Activity, Object}
+  alias Pleroma.User
+  alias Pleroma.Repo
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.ThreadMute
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Formatter
@@ -14,6 +18,7 @@ def delete(activity_id, user) do
     with %Activity{data: %{"object" => %{"id" => object_id}}} <- Repo.get(Activity, activity_id),
          %Object{} = object <- Object.normalize(object_id),
          true <- user.info.is_moderator || user.ap_id == object.data["actor"],
+         {:ok, _} <- unpin(activity_id, user),
          {:ok, delete} <- ActivityPub.delete(object) do
       {:ok, delete}
     end
@@ -102,7 +107,14 @@ def post(user, %{"status" => status} = data) do
              attachments,
              tags,
              get_content_type(data["content_type"]),
-             Enum.member?([true, "true"], data["no_attachment_links"])
+             Enum.member?(
+               [true, "true"],
+               Map.get(
+                 data,
+                 "no_attachment_links",
+                 Pleroma.Config.get([:instance, :no_attachment_links], false)
+               )
+             )
            ),
          context <- make_context(inReplyTo),
          cw <- data["spoiler_text"],
@@ -124,7 +136,7 @@ def post(user, %{"status" => status} = data) do
            Map.put(
              object,
              "emoji",
-             Formatter.get_emoji(status)
+             (Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"]))
              |> Enum.reduce(%{}, fn {name, file}, acc ->
                Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
              end)
@@ -135,7 +147,7 @@ def post(user, %{"status" => status} = data) do
           actor: user,
           context: context,
           object: object,
-          additional: %{"cc" => cc}
+          additional: %{"cc" => cc, "directMessage" => visibility == "direct"}
         })
 
       res
@@ -164,4 +176,71 @@ def update(user) do
       object: Pleroma.Web.ActivityPub.UserView.render("user.json", %{user: user})
     })
   end
+
+  def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do
+    with %Activity{
+           actor: ^user_ap_id,
+           data: %{
+             "type" => "Create",
+             "object" => %{
+               "to" => object_to,
+               "type" => "Note"
+             }
+           }
+         } = activity <- get_by_id_or_ap_id(id_or_ap_id),
+         true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"),
+         %{valid?: true} = info_changeset <-
+           Pleroma.User.Info.add_pinnned_activity(user.info, activity),
+         changeset <-
+           Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset),
+         {:ok, _user} <- User.update_and_set_cache(changeset) do
+      {:ok, activity}
+    else
+      %{errors: [pinned_activities: {err, _}]} ->
+        {:error, err}
+
+      _ ->
+        {:error, "Could not pin"}
+    end
+  end
+
+  def unpin(id_or_ap_id, user) do
+    with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
+         %{valid?: true} = info_changeset <-
+           Pleroma.User.Info.remove_pinnned_activity(user.info, activity),
+         changeset <-
+           Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset),
+         {:ok, _user} <- User.update_and_set_cache(changeset) do
+      {:ok, activity}
+    else
+      %{errors: [pinned_activities: {err, _}]} ->
+        {:error, err}
+
+      _ ->
+        {:error, "Could not unpin"}
+    end
+  end
+
+  def add_mute(user, activity) do
+    with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]) do
+      {:ok, activity}
+    else
+      {:error, _} -> {:error, "conversation is already muted"}
+    end
+  end
+
+  def remove_mute(user, activity) do
+    ThreadMute.remove_mute(user.id, activity.data["context"])
+    {:ok, activity}
+  end
+
+  def thread_muted?(%{id: nil} = _user, _activity), do: false
+
+  def thread_muted?(user, activity) do
+    with [] <- ThreadMute.check_muted(user.id, activity.data["context"]) do
+      false
+    else
+      _ -> true
+    end
+  end
 end

lib/pleroma/web/common_api/utils.ex

@@ -5,22 +5,25 @@
 defmodule Pleroma.Web.CommonAPI.Utils do
   alias Calendar.Strftime
   alias Comeonin.Pbkdf2
-  alias Pleroma.{Activity, Formatter, Object, Repo}
+  alias Pleroma.Activity
+  alias Pleroma.Formatter
+  alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web
-  alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Endpoint
   alias Pleroma.Web.MediaProxy
+  alias Pleroma.Web.ActivityPub.Utils
 
   # This is a hack for twidere.
   def get_by_id_or_ap_id(id) do
-    activity = Repo.get(Activity, id) || Activity.get_create_activity_by_object_ap_id(id)
+    activity = Repo.get(Activity, id) || Activity.get_create_by_object_ap_id(id)
 
     activity &&
       if activity.data["type"] == "Create" do
         activity
       else
-        Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+        Activity.get_create_by_object_ap_id(activity.data["object"])
       end
   end
 
@@ -111,7 +114,7 @@ def make_content_html(
   def make_context(%Activity{data: %{"context" => context}}), do: context
   def make_context(_), do: Utils.generate_context_id()
 
-  def maybe_add_attachments(text, _attachments, _no_links = true), do: text
+  def maybe_add_attachments(text, _attachments, true = _no_links), do: text
 
   def maybe_add_attachments(text, attachments, _no_links) do
     add_attachments(text, attachments)
@@ -132,16 +135,18 @@ def add_attachments(text, attachments) do
     Enum.join([text | attachment_text], "<br>")
   end
 
+  def format_input(text, mentions, tags, format, options \\ [])
+
   @doc """
   Formatting text to plain text.
   """
-  def format_input(text, mentions, tags, "text/plain") do
+  def format_input(text, mentions, tags, "text/plain", options) do
     text
     |> Formatter.html_escape("text/plain")
     |> String.replace(~r/\r?\n/, "<br>")
     |> (&{[], &1}).()
     |> Formatter.add_links()
-    |> Formatter.add_user_links(mentions)
+    |> Formatter.add_user_links(mentions, options[:user_links] || [])
     |> Formatter.add_hashtag_links(tags)
     |> Formatter.finalize()
   end
@@ -149,26 +154,24 @@ def format_input(text, mentions, tags, "text/plain") do
   @doc """
   Formatting text to html.
   """
-  def format_input(text, mentions, _tags, "text/html") do
+  def format_input(text, mentions, _tags, "text/html", options) do
     text
     |> Formatter.html_escape("text/html")
-    |> String.replace(~r/\r?\n/, "<br>")
     |> (&{[], &1}).()
-    |> Formatter.add_user_links(mentions)
+    |> Formatter.add_user_links(mentions, options[:user_links] || [])
     |> Formatter.finalize()
   end
 
   @doc """
   Formatting text to markdown.
   """
-  def format_input(text, mentions, tags, "text/markdown") do
+  def format_input(text, mentions, tags, "text/markdown", options) do
     text
     |> Formatter.mentions_escape(mentions)
     |> Earmark.as_html!()
     |> Formatter.html_escape("text/html")
-    |> String.replace(~r/\r?\n/, "")
     |> (&{[], &1}).()
-    |> Formatter.add_user_links(mentions)
+    |> Formatter.add_user_links(mentions, options[:user_links] || [])
     |> Formatter.add_hashtag_links(tags)
     |> Formatter.finalize()
   end
@@ -277,4 +280,46 @@ def emoji_from_profile(%{info: _info} = user) do
       }
     end)
   end
+
+  def maybe_notify_to_recipients(
+        recipients,
+        %Activity{data: %{"to" => to, "type" => _type}} = _activity
+      ) do
+    recipients ++ to
+  end
+
+  def maybe_notify_mentioned_recipients(
+        recipients,
+        %Activity{data: %{"to" => _to, "type" => type} = data} = _activity
+      )
+      when type == "Create" do
+    object = Object.normalize(data["object"])
+
+    object_data =
+      cond do
+        !is_nil(object) ->
+          object.data
+
+        is_map(data["object"]) ->
+          data["object"]
+
+        true ->
+          %{}
+      end
+
+    tagged_mentions = maybe_extract_mentions(object_data)
+
+    recipients ++ tagged_mentions
+  end
+
+  def maybe_notify_mentioned_recipients(recipients, _), do: recipients
+
+  def maybe_extract_mentions(%{"tag" => tag}) do
+    tag
+    |> Enum.filter(fn x -> is_map(x) end)
+    |> Enum.filter(fn x -> x["type"] == "Mention" end)
+    |> Enum.map(fn x -> x["href"] end)
+  end
+
+  def maybe_extract_mentions(_), do: []
 end

lib/pleroma/web/endpoint.ex

@@ -25,7 +25,7 @@ defmodule Pleroma.Web.Endpoint do
     at: "/",
     from: :pleroma,
     only:
-      ~w(index.html static finmoji emoji packs sounds images instance sw.js favicon.png schemas doc)
+      ~w(index.html static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc)
   )
 
   # Code reloading can be explicitly enabled under the
@@ -82,4 +82,8 @@ def load_from_system_env(config) do
     port = System.get_env("PORT") || raise "expected the PORT environment variable to be set"
     {:ok, Keyword.put(config, :http, [:inet6, port: port])}
   end
+
+  def websocket_url do
+    String.replace_leading(url(), "http", "ws")
+  end
 end

lib/pleroma/web/federator/federator.ex

@@ -4,15 +4,19 @@
 
 defmodule Pleroma.Web.Federator do
   use GenServer
-  alias Pleroma.User
+
   alias Pleroma.Activity
-  alias Pleroma.Web.{WebFinger, Websub}
+  alias Pleroma.User
-  alias Pleroma.Web.Federator.RetryQueue
+  alias Pleroma.Web.WebFinger
+  alias Pleroma.Web.Websub
+  alias Pleroma.Web.Salmon
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.Federator.RetryQueue
   alias Pleroma.Web.OStatus
+
   require Logger
 
   @websub Application.get_env(:pleroma, :websub)
@@ -25,7 +29,7 @@ def init(args) do
   def start_link do
     spawn(fn ->
       # 1 minute
-      Process.sleep(1000 * 60 * 1)
+      Process.sleep(1000 * 60)
       enqueue(:refresh_subscriptions, nil)
     end)
 
@@ -124,6 +128,10 @@ def handle(:incoming_ap_doc, params) do
     end
   end
 
+  def handle(:publish_single_salmon, params) do
+    Salmon.send_to_user(params)
+  end
+
   def handle(:publish_single_ap, params) do
     case ActivityPub.publish_one(params) do
       {:ok, _} ->
@@ -192,8 +200,7 @@ def handle_cast({:enqueue, type, payload, _priority}, state) do
     {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
   end
 
-  def handle_cast(m, state) do
+  def handle_cast(_, state) do
-    IO.inspect("Unknown: #{inspect(m)}, #{inspect(state)}")
     {:noreply, state}
   end
 

lib/pleroma/web/http_signatures/http_signatures.ex

@@ -5,8 +5,9 @@
 # https://tools.ietf.org/html/draft-cavage-http-signatures-08
 defmodule Pleroma.Web.HTTPSignatures do
   alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Utils
+
   require Logger
 
   def split_signature(sig) do

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -4,34 +4,44 @@
 
 defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   use Pleroma.Web, :controller
-  alias Pleroma.{Repo, Object, Activity, User, Notification, Stats}
+  alias Pleroma.Activity
+  alias Pleroma.Config
+  alias Pleroma.Filter
+  alias Pleroma.Notification
+  alias Pleroma.Object
+  alias Pleroma.Repo
+  alias Pleroma.Stats
+  alias Pleroma.User
   alias Pleroma.Web
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.MediaProxy
+  alias Pleroma.Web.Push
+  alias Push.Subscription
 
-  alias Pleroma.Web.MastodonAPI.{
+  alias Pleroma.Web.MastodonAPI.AccountView
-    StatusView,
+  alias Pleroma.Web.MastodonAPI.FilterView
-    AccountView,
+  alias Pleroma.Web.MastodonAPI.ListView
-    MastodonView,
+  alias Pleroma.Web.MastodonAPI.MastodonView
-    ListView,
+  alias Pleroma.Web.MastodonAPI.PushSubscriptionView
-    FilterView,
+  alias Pleroma.Web.MastodonAPI.StatusView
-    PushSubscriptionView
-  }
-
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.OAuth.App
-  alias Pleroma.Web.OAuth.{Authorization, Token, App}
+  alias Pleroma.Web.OAuth.Authorization
-  alias Pleroma.Web.MediaProxy
+  alias Pleroma.Web.OAuth.Token
 
   import Ecto.Query
   require Logger
 
   @httpoison Application.get_env(:pleroma, :httpoison)
+  @local_mastodon_name "Mastodon-Local"
 
   action_fallback(:errors)
 
   def create_app(conn, params) do
-    with cs <- App.register_changeset(%App{}, params) |> IO.inspect(),
+    with cs <- App.register_changeset(%App{}, params),
-         {:ok, app} <- Repo.insert(cs) |> IO.inspect() do
+         false <- cs.changes[:client_name] == @local_mastodon_name,
+         {:ok, app} <- Repo.insert(cs) do
       res = %{
         id: app.id |> to_string,
         name: app.client_name,
@@ -129,7 +139,7 @@ def user(%{assigns: %{user: for_user}} = conn, %{"id" => id}) do
   @mastodon_api_level "2.5.0"
 
   def masto_instance(conn, _params) do
-    instance = Pleroma.Config.get(:instance)
+    instance = Config.get(:instance)
 
     response = %{
       uri: Web.base_url(),
@@ -138,7 +148,7 @@ def masto_instance(conn, _params) do
       version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})",
       email: Keyword.get(instance, :email),
       urls: %{
-        streaming_api: String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws")
+        streaming_api: Pleroma.Web.Endpoint.websocket_url()
       },
       stats: Stats.get_stats(),
       thumbnail: Web.base_url() <> "/instance/thumbnail.jpeg",
@@ -225,7 +235,8 @@ def home_timeline(%{assigns: %{user: user}} = conn, params) do
       |> Map.put("user", user)
 
     activities =
-      ActivityPub.fetch_activities([user.ap_id | user.following], params)
+      [user.ap_id | user.following]
+      |> ActivityPub.fetch_activities(params)
       |> ActivityPub.contain_timeline(user)
       |> Enum.reverse()
 
@@ -238,14 +249,12 @@ def home_timeline(%{assigns: %{user: user}} = conn, params) do
   def public_timeline(%{assigns: %{user: user}} = conn, params) do
     local_only = params["local"] in [true, "True", "true", "1"]
 
-    params =
+    activities =
       params
       |> Map.put("type", ["Create", "Announce"])
       |> Map.put("local_only", local_only)
       |> Map.put("blocking_user", user)
-
+      |> ActivityPub.fetch_public_activities()
-    activities =
-      ActivityPub.fetch_public_activities(params)
       |> Enum.reverse()
 
     conn
@@ -256,13 +265,7 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do
 
   def user_statuses(%{assigns: %{user: reading_user}} = conn, params) do
     with %User{} = user <- Repo.get(User, params["id"]) do
-      # Since Pleroma has no "pinned" posts feature, we'll just set an empty list here
+      activities = ActivityPub.fetch_user_activities(user, reading_user, params)
-      activities =
-        if params["pinned"] == "true" do
-          []
-        else
-          ActivityPub.fetch_user_activities(user, reading_user, params)
-        end
 
       conn
       |> add_link_headers(:user_statuses, activities, params["id"])
@@ -320,6 +323,7 @@ def get_context(%{assigns: %{user: user}} = conn, %{"id" => id}) do
             as: :activity
           )
           |> Enum.reverse(),
+        # credo:disable-for-previous-line Credo.Check.Refactor.PipeChainStart
         descendants:
           StatusView.render(
             "index.json",
@@ -328,6 +332,7 @@ def get_context(%{assigns: %{user: user}} = conn, %{"id" => id}) do
             as: :activity
           )
           |> Enum.reverse()
+        # credo:disable-for-previous-line Credo.Check.Refactor.PipeChainStart
       }
 
       json(conn, result)
@@ -347,7 +352,6 @@ def post_status(%{assigns: %{user: user}} = conn, %{"status" => _} = params) do
     params =
       params
       |> Map.put("in_reply_to_status_id", params["in_reply_to_id"])
-      |> Map.put("no_attachment_links", true)
 
     idempotency_key =
       case get_req_header(conn, "idempotency-key") do
@@ -384,7 +388,7 @@ def reblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
 
   def unreblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, _unannounce, %{data: %{"id" => id}}} <- CommonAPI.unrepeat(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -393,7 +397,7 @@ def unreblog_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
 
   def fav_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, _fav, %{data: %{"id" => id}}} <- CommonAPI.favorite(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -402,7 +406,75 @@ def fav_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
 
   def unfav_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     with {:ok, _, _, %{data: %{"id" => id}}} <- CommonAPI.unfavorite(ap_id_or_id, user),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
+      conn
+      |> put_view(StatusView)
+      |> try_render("status.json", %{activity: activity, for: user, as: :activity})
+    end
+  end
+
+  def pin_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
+    with {:ok, activity} <- CommonAPI.pin(ap_id_or_id, user) do
+      conn
+      |> put_view(StatusView)
+      |> try_render("status.json", %{activity: activity, for: user, as: :activity})
+    else
+      {:error, reason} ->
+        conn
+        |> put_resp_content_type("application/json")
+        |> send_resp(:bad_request, Jason.encode!(%{"error" => reason}))
+    end
+  end
+
+  def unpin_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
+    with {:ok, activity} <- CommonAPI.unpin(ap_id_or_id, user) do
+      conn
+      |> put_view(StatusView)
+      |> try_render("status.json", %{activity: activity, for: user, as: :activity})
+    end
+  end
+
+  def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+    with %Activity{} = activity <- Repo.get(Activity, id),
+         %User{} = user <- User.get_by_nickname(user.nickname),
+         true <- ActivityPub.visible_for_user?(activity, user),
+         {:ok, user} <- User.bookmark(user, activity.data["object"]["id"]) do
+      conn
+      |> put_view(StatusView)
+      |> try_render("status.json", %{activity: activity, for: user, as: :activity})
+    end
+  end
+
+  def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+    with %Activity{} = activity <- Repo.get(Activity, id),
+         %User{} = user <- User.get_by_nickname(user.nickname),
+         true <- ActivityPub.visible_for_user?(activity, user),
+         {:ok, user} <- User.unbookmark(user, activity.data["object"]["id"]) do
+      conn
+      |> put_view(StatusView)
+      |> try_render("status.json", %{activity: activity, for: user, as: :activity})
+    end
+  end
+
+  def mute_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+    activity = Activity.get_by_id(id)
+
+    with {:ok, activity} <- CommonAPI.add_mute(user, activity) do
+      conn
+      |> put_view(StatusView)
+      |> try_render("status.json", %{activity: activity, for: user, as: :activity})
+    else
+      {:error, reason} ->
+        conn
+        |> put_resp_content_type("application/json")
+        |> send_resp(:bad_request, Jason.encode!(%{"error" => reason}))
+    end
+  end
+
+  def unmute_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+    activity = Activity.get_by_id(id)
+
+    with {:ok, activity} <- CommonAPI.remove_mute(user, activity) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
@@ -413,9 +485,8 @@ def notifications(%{assigns: %{user: user}} = conn, params) do
     notifications = Notification.for_user(user, params)
 
     result =
-      Enum.map(notifications, fn x ->
+      notifications
-        render_notification(user, x)
+      |> Enum.map(fn x -> render_notification(user, x) end)
-      end)
       |> Enum.filter(& &1)
 
     conn
@@ -485,7 +556,8 @@ def update_media(%{assigns: %{user: user}} = conn, data) do
 
   def upload(%{assigns: %{user: user}} = conn, %{"file" => file} = data) do
     with {:ok, object} <-
-           ActivityPub.upload(file,
+           ActivityPub.upload(
+             file,
              actor: User.ap_id(user),
              description: Map.get(data, "description")
            ) do
@@ -526,15 +598,32 @@ def reblogged_by(conn, %{"id" => id}) do
   def hashtag_timeline(%{assigns: %{user: user}} = conn, params) do
     local_only = params["local"] in [true, "True", "true", "1"]
 
-    params =
+    tags =
+      [params["tag"], params["any"]]
+      |> List.flatten()
+      |> Enum.uniq()
+      |> Enum.filter(& &1)
+      |> Enum.map(&String.downcase(&1))
+
+    tag_all =
+      params["all"] ||
+        []
+        |> Enum.map(&String.downcase(&1))
+
+    tag_reject =
+      params["none"] ||
+        []
+        |> Enum.map(&String.downcase(&1))
+
+    activities =
       params
       |> Map.put("type", "Create")
       |> Map.put("local_only", local_only)
       |> Map.put("blocking_user", user)
-      |> Map.put("tag", String.downcase(params["tag"]))
+      |> Map.put("tag", tags)
-
+      |> Map.put("tag_all", tag_all)
-    activities =
+      |> Map.put("tag_reject", tag_reject)
-      ActivityPub.fetch_public_activities(params)
+      |> ActivityPub.fetch_public_activities()
       |> Enum.reverse()
 
     conn
@@ -549,7 +638,7 @@ def followers(%{assigns: %{user: for_user}} = conn, %{"id" => id}) do
       followers =
         cond do
           for_user && user.id == for_user.id -> followers
-          user.info.hide_network -> []
+          user.info.hide_followers -> []
           true -> followers
         end
 
@@ -565,7 +654,7 @@ def following(%{assigns: %{user: for_user}} = conn, %{"id" => id}) do
       followers =
         cond do
           for_user && user.id == for_user.id -> followers
-          user.info.hide_network -> []
+          user.info.hide_follows -> []
           true -> followers
         end
 
@@ -634,7 +723,7 @@ def follow(%{assigns: %{user: follower}} = conn, %{"id" => id}) do
          {:ok, _activity} <- ActivityPub.follow(follower, followed),
          {:ok, follower, followed} <-
            User.wait_and_refresh(
-             Pleroma.Config.get([:activitypub, :follow_handshake_timeout]),
+             Config.get([:activitypub, :follow_handshake_timeout]),
              follower,
              followed
            ) do
@@ -725,11 +814,13 @@ def unblock_domain(%{assigns: %{user: blocker}} = conn, %{"domain" => domain}) d
     json(conn, %{})
   end
 
-  def status_search(query) do
+  def status_search(user, query) do
     fetched =
       if Regex.match?(~r/https?:/, query) do
-        with {:ok, object} <- ActivityPub.fetch_object_from_id(query) do
+        with {:ok, object} <- ActivityPub.fetch_object_from_id(query),
-          [Activity.get_create_activity_by_object_ap_id(object.data["id"])]
+             %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
+             true <- ActivityPub.visible_for_user?(activity, user) do
+          [activity]
         else
           _e -> []
         end
@@ -754,14 +845,15 @@ def status_search(query) do
   end
 
   def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, params["resolve"] == "true")
+    accounts = User.search(query, params["resolve"] == "true", user)
 
-    statuses = status_search(query)
+    statuses = status_search(user, query)
 
     tags_path = Web.base_url() <> "/tag/"
 
     tags =
-      String.split(query)
+      query
+      |> String.split()
       |> Enum.uniq()
       |> Enum.filter(fn tag -> String.starts_with?(tag, "#") end)
       |> Enum.map(fn tag -> String.slice(tag, 1..-1) end)
@@ -778,12 +870,13 @@ def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
   end
 
   def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, params["resolve"] == "true")
+    accounts = User.search(query, params["resolve"] == "true", user)
 
-    statuses = status_search(query)
+    statuses = status_search(user, query)
 
     tags =
-      String.split(query)
+      query
+      |> String.split()
       |> Enum.uniq()
       |> Enum.filter(fn tag -> String.starts_with?(tag, "#") end)
       |> Enum.map(fn tag -> String.slice(tag, 1..-1) end)
@@ -799,22 +892,34 @@ def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
   end
 
   def account_search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, params["resolve"] == "true")
+    accounts = User.search(query, params["resolve"] == "true", user)
 
     res = AccountView.render("accounts.json", users: accounts, for: user, as: :user)
 
     json(conn, res)
   end
 
-  def favourites(%{assigns: %{user: user}} = conn, _) do
+  def favourites(%{assigns: %{user: user}} = conn, params) do
-    params =
+    activities =
-      %{}
+      params
       |> Map.put("type", "Create")
       |> Map.put("favorited_by", user.ap_id)
       |> Map.put("blocking_user", user)
+      |> ActivityPub.fetch_public_activities()
+      |> Enum.reverse()
+
+    conn
+    |> add_link_headers(:favourites, activities)
+    |> put_view(StatusView)
+    |> render("index.json", %{activities: activities, for: user, as: :activity})
+  end
+
+  def bookmarks(%{assigns: %{user: user}} = conn, _) do
+    user = Repo.get(User, user.id)
 
     activities =
-      ActivityPub.fetch_public_activities(params)
+      user.bookmarks
+      |> Enum.map(fn id -> Activity.get_create_by_object_ap_id(id) end)
       |> Enum.reverse()
 
     conn
@@ -833,7 +938,10 @@ def get_list(%{assigns: %{user: user}} = conn, %{"id" => id}) do
       res = ListView.render("list.json", list: list)
       json(conn, res)
     else
-      _e -> json(conn, "error")
+      _e ->
+        conn
+        |> put_status(404)
+        |> json(%{error: "Record not found"})
     end
   end
 
@@ -913,12 +1021,10 @@ def list_timeline(%{assigns: %{user: user}} = conn, %{"list_id" => id} = params)
 
       # we must filter the following list for the user to avoid leaking statuses the user
       # does not actually have permission to see (for more info, peruse security issue #270).
-      following_to =
+      activities =
         following
         |> Enum.filter(fn x -> x in user.following end)
-
+        |> ActivityPub.fetch_activities_bounded(following, params)
-      activities =
-        ActivityPub.fetch_activities_bounded(following_to, following, params)
         |> Enum.reverse()
 
       conn
@@ -940,7 +1046,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do
     if user && token do
       mastodon_emoji = mastodonized_emoji()
 
-      limit = Pleroma.Config.get([:instance, :limit])
+      limit = Config.get([:instance, :limit])
 
       accounts =
         Map.put(%{}, user.id, AccountView.render("account.json", %{user: user, for: user}))
@@ -964,8 +1070,8 @@ def index(%{assigns: %{user: user}} = conn, _params) do
             max_toot_chars: limit
           },
           rights: %{
-            delete_others_notice: !!user.info.is_moderator,
+            delete_others_notice: present?(user.info.is_moderator),
-            admin: !!user.info.is_admin
+            admin: present?(user.info.is_admin)
           },
           compose: %{
             me: "#{user.id}",
@@ -1064,7 +1170,9 @@ def login(conn, %{"code" => code}) do
   def login(conn, _) do
     with {:ok, app} <- get_or_make_app() do
       path =
-        o_auth_path(conn, :authorize,
+        o_auth_path(
+          conn,
+          :authorize,
           response_type: "code",
           client_id: app.client_id,
           redirect_uri: ".",
@@ -1077,16 +1185,13 @@ def login(conn, _) do
   end
 
   defp get_or_make_app() do
-    with %App{} = app <- Repo.get_by(App, client_name: "Mastodon-Local") do
+    find_attrs = %{client_name: @local_mastodon_name, redirect_uris: "."}
+
+    with %App{} = app <- Repo.get_by(App, find_attrs) do
       {:ok, app}
     else
       _e ->
-        cs =
+        cs = App.register_changeset(%App{}, Map.put(find_attrs, :scopes, "read,write,follow"))
-          App.register_changeset(%App{}, %{
-            client_name: "Mastodon-Local",
-            redirect_uris: ".",
-            scopes: "read,write,follow"
-          })
 
         Repo.insert(cs)
     end
@@ -1120,7 +1225,7 @@ def empty_object(conn, _) do
 
   def render_notification(user, %{id: id, activity: activity, inserted_at: created_at} = _params) do
     actor = User.get_cached_by_ap_id(activity.data["actor"])
-    parent_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+    parent_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
     mastodon_type = Activity.mastodon_notification_type(activity)
 
     response = %{
@@ -1158,7 +1263,7 @@ def render_notification(user, %{id: id, activity: activity, inserted_at: created
   end
 
   def get_filters(%{assigns: %{user: user}} = conn, _) do
-    filters = Pleroma.Filter.get_filters(user)
+    filters = Filter.get_filters(user)
     res = FilterView.render("filters.json", filters: filters)
     json(conn, res)
   end
@@ -1167,7 +1272,7 @@ def create_filter(
         %{assigns: %{user: user}} = conn,
         %{"phrase" => phrase, "context" => context} = params
       ) do
-    query = %Pleroma.Filter{
+    query = %Filter{
       user_id: user.id,
       phrase: phrase,
       context: context,
@@ -1176,13 +1281,13 @@ def create_filter(
       # expires_at
     }
 
-    {:ok, response} = Pleroma.Filter.create(query)
+    {:ok, response} = Filter.create(query)
     res = FilterView.render("filter.json", filter: response)
     json(conn, res)
   end
 
   def get_filter(%{assigns: %{user: user}} = conn, %{"id" => filter_id}) do
-    filter = Pleroma.Filter.get(filter_id, user)
+    filter = Filter.get(filter_id, user)
     res = FilterView.render("filter.json", filter: filter)
     json(conn, res)
   end
@@ -1191,7 +1296,7 @@ def update_filter(
         %{assigns: %{user: user}} = conn,
         %{"phrase" => phrase, "context" => context, "id" => filter_id} = params
       ) do
-    query = %Pleroma.Filter{
+    query = %Filter{
       user_id: user.id,
       filter_id: filter_id,
       phrase: phrase,
@@ -1201,32 +1306,32 @@ def update_filter(
       # expires_at
     }
 
-    {:ok, response} = Pleroma.Filter.update(query)
+    {:ok, response} = Filter.update(query)
     res = FilterView.render("filter.json", filter: response)
     json(conn, res)
   end
 
   def delete_filter(%{assigns: %{user: user}} = conn, %{"id" => filter_id}) do
-    query = %Pleroma.Filter{
+    query = %Filter{
       user_id: user.id,
       filter_id: filter_id
     }
 
-    {:ok, _} = Pleroma.Filter.delete(query)
+    {:ok, _} = Filter.delete(query)
     json(conn, %{})
   end
 
   def create_push_subscription(%{assigns: %{user: user, token: token}} = conn, params) do
-    true = Pleroma.Web.Push.enabled()
+    true = Push.enabled()
-    Pleroma.Web.Push.Subscription.delete_if_exists(user, token)
+    Subscription.delete_if_exists(user, token)
-    {:ok, subscription} = Pleroma.Web.Push.Subscription.create(user, token, params)
+    {:ok, subscription} = Subscription.create(user, token, params)
     view = PushSubscriptionView.render("push_subscription.json", subscription: subscription)
     json(conn, view)
   end
 
   def get_push_subscription(%{assigns: %{user: user, token: token}} = conn, _params) do
-    true = Pleroma.Web.Push.enabled()
+    true = Push.enabled()
-    subscription = Pleroma.Web.Push.Subscription.get(user, token)
+    subscription = Subscription.get(user, token)
     view = PushSubscriptionView.render("push_subscription.json", subscription: subscription)
     json(conn, view)
   end
@@ -1235,15 +1340,15 @@ def update_push_subscription(
         %{assigns: %{user: user, token: token}} = conn,
         params
       ) do
-    true = Pleroma.Web.Push.enabled()
+    true = Push.enabled()
-    {:ok, subscription} = Pleroma.Web.Push.Subscription.update(user, token, params)
+    {:ok, subscription} = Subscription.update(user, token, params)
     view = PushSubscriptionView.render("push_subscription.json", subscription: subscription)
     json(conn, view)
   end
 
   def delete_push_subscription(%{assigns: %{user: user, token: token}} = conn, _params) do
-    true = Pleroma.Web.Push.enabled()
+    true = Push.enabled()
-    {:ok, _response} = Pleroma.Web.Push.Subscription.delete(user, token)
+    {:ok, _response} = Subscription.delete(user, token)
     json(conn, %{})
   end
 
@@ -1254,17 +1359,21 @@ def errors(conn, _) do
   end
 
   def suggestions(%{assigns: %{user: user}} = conn, _) do
-    suggestions = Pleroma.Config.get(:suggestions)
+    suggestions = Config.get(:suggestions)
 
     if Keyword.get(suggestions, :enabled, false) do
       api = Keyword.get(suggestions, :third_party_engine, "")
       timeout = Keyword.get(suggestions, :timeout, 5000)
       limit = Keyword.get(suggestions, :limit, 23)
 
-      host = Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
+      host = Config.get([Pleroma.Web.Endpoint, :url, :host])
 
       user = user.nickname
-      url = String.replace(api, "{{host}}", host) |> String.replace("{{user}}", user)
+
+      url =
+        api
+        |> String.replace("{{host}}", host)
+        |> String.replace("{{user}}", user)
 
       with {:ok, %{status: 200, body: body}} <-
              @httpoison.get(
@@ -1272,12 +1381,14 @@ def suggestions(%{assigns: %{user: user}} = conn, _) do
                [],
                adapter: [
                  timeout: timeout,
-                 recv_timeout: timeout
+                 recv_timeout: timeout,
+                 pool: :default
                ]
              ),
            {:ok, data} <- Jason.decode(body) do
-        data2 =
+        data =
-          Enum.slice(data, 0, limit)
+          data
+          |> Enum.slice(0, limit)
           |> Enum.map(fn x ->
             Map.put(
               x,
@@ -1296,7 +1407,7 @@ def suggestions(%{assigns: %{user: user}} = conn, _) do
           end)
 
         conn
-        |> json(data2)
+        |> json(data)
       else
         e -> Logger.error("Could not retrieve suggestions at fetch #{url}, #{inspect(e)}")
       end
@@ -1305,6 +1416,22 @@ def suggestions(%{assigns: %{user: user}} = conn, _) do
     end
   end
 
+  def status_card(conn, %{"id" => status_id}) do
+    with %Activity{} = activity <- Repo.get(Activity, status_id),
+         true <- ActivityPub.is_public?(activity) do
+      data =
+        StatusView.render(
+          "card.json",
+          Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+        )
+
+      json(conn, data)
+    else
+      _e ->
+        %{}
+    end
+  end
+
   def try_render(conn, target, params)
       when is_binary(target) do
     res = render(conn, target, params)
@@ -1323,4 +1450,8 @@ def try_render(conn, _, _) do
     |> put_status(501)
     |> json(%{error: "Can't display this activity"})
   end
+
+  defp present?(nil), do: false
+  defp present?(false), do: false
+  defp present?(_), do: true
 end

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -4,11 +4,12 @@
 
 defmodule Pleroma.Web.MastodonAPI.AccountView do
   use Pleroma.Web, :view
-  alias Pleroma.User
+
-  alias Pleroma.Web.MastodonAPI.AccountView
-  alias Pleroma.Web.CommonAPI.Utils
-  alias Pleroma.Web.MediaProxy
   alias Pleroma.HTML
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.MediaProxy
 
   def render("accounts.json", %{users: users} = opts) do
     users
@@ -112,7 +113,9 @@ defp do_render("account.json", %{user: user} = opts) do
       # Pleroma extension
       pleroma: %{
         confirmation_pending: user_info.confirmation_pending,
-        tags: user.tags
+        tags: user.tags,
+        is_moderator: user.info.is_moderator,
+        is_admin: user.info.is_admin
       }
     }
   end

lib/pleroma/web/mastodon_api/views/filter_view.ex

@@ -4,8 +4,8 @@
 
 defmodule Pleroma.Web.MastodonAPI.FilterView do
   use Pleroma.Web, :view
-  alias Pleroma.Web.MastodonAPI.FilterView
   alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.Web.MastodonAPI.FilterView
 
   def render("filters.json", %{filters: filters} = opts) do
     render_many(filters, FilterView, "filter.json", opts)

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -9,10 +9,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   alias Pleroma.HTML
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
-  alias Pleroma.Web.MediaProxy
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.StatusView
+  alias Pleroma.Web.MediaProxy
 
   # TODO: Add cached version.
   defp get_replied_to_activities(activities) do
@@ -25,33 +26,45 @@ defp get_replied_to_activities(activities) do
         nil
     end)
     |> Enum.filter(& &1)
-    |> Activity.create_activity_by_object_id_query()
+    |> Activity.create_by_object_ap_id()
     |> Repo.all()
     |> Enum.reduce(%{}, fn activity, acc ->
       Map.put(acc, activity.data["object"]["id"], activity)
     end)
   end
 
+  defp get_user(ap_id) do
+    cond do
+      user = User.get_cached_by_ap_id(ap_id) ->
+        user
+
+      user = User.get_by_guessed_nickname(ap_id) ->
+        user
+
+      true ->
+        User.error_user(ap_id)
+    end
+  end
+
   def render("index.json", opts) do
     replied_to_activities = get_replied_to_activities(opts.activities)
 
     opts.activities
-    |> render_many(
+    |> safe_render_many(
       StatusView,
       "status.json",
       Map.put(opts, :replied_to_activities, replied_to_activities)
     )
-    |> Enum.filter(fn x -> not is_nil(x) end)
   end
 
   def render(
         "status.json",
         %{activity: %{data: %{"type" => "Announce", "object" => object}} = activity} = opts
       ) do
-    user = User.get_cached_by_ap_id(activity.data["actor"])
+    user = get_user(activity.data["actor"])
     created_at = Utils.to_masto_date(activity.data["published"])
 
-    reblogged = Activity.get_create_activity_by_object_ap_id(object)
+    reblogged = Activity.get_create_by_object_ap_id(object)
     reblogged = render("status.json", Map.put(opts, :activity, reblogged))
 
     mentions =
@@ -75,7 +88,9 @@ def render(
       favourites_count: 0,
       reblogged: false,
       favourited: false,
+      bookmarked: false,
       muted: false,
+      pinned: pinned?(activity, user),
       sensitive: false,
       spoiler_text: "",
       visibility: "public",
@@ -92,7 +107,7 @@ def render(
   end
 
   def render("status.json", %{activity: %{data: %{"object" => object}} = activity} = opts) do
-    user = User.get_cached_by_ap_id(activity.data["actor"])
+    user = get_user(activity.data["actor"])
 
     like_count = object["like_count"] || 0
     announcement_count = object["announcement_count"] || 0
@@ -108,6 +123,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
 
     repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || [])
     favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
+    bookmarked = opts[:for] && object["id"] in opts[:for].bookmarks
 
     attachment_data = object["attachment"] || []
     attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)
@@ -115,12 +131,18 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
     created_at = Utils.to_masto_date(object["published"])
 
     reply_to = get_reply_to(activity, opts)
-    reply_to_user = reply_to && User.get_cached_by_ap_id(reply_to.data["actor"])
+    reply_to_user = reply_to && get_user(reply_to.data["actor"])
 
     content =
       object
       |> render_content()
-      |> HTML.get_cached_scrubbed_html_for_object(User.html_filter_policy(opts[:for]), activity)
+      |> HTML.get_cached_scrubbed_html_for_object(
+        User.html_filter_policy(opts[:for]),
+        activity,
+        __MODULE__
+      )
+
+    card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity))
 
     %{
       id: to_string(activity.id),
@@ -130,6 +152,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
       in_reply_to_id: reply_to && to_string(reply_to.id),
       in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id),
       reblog: nil,
+      card: card,
       content: content,
       created_at: created_at,
       reblogs_count: announcement_count,
@@ -137,7 +160,9 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
       favourites_count: like_count,
       reblogged: present?(repeated),
       favourited: present?(favorited),
-      muted: false,
+      bookmarked: present?(bookmarked),
+      muted: CommonAPI.thread_muted?(user, activity),
+      pinned: pinned?(activity, user),
       sensitive: sensitive,
       spoiler_text: object["summary"] || "",
       visibility: get_visibility(object),
@@ -157,6 +182,46 @@ def render("status.json", _) do
     nil
   end
 
+  def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
+    page_url_data = URI.parse(page_url)
+
+    page_url_data =
+      if rich_media[:url] != nil do
+        URI.merge(page_url_data, URI.parse(rich_media[:url]))
+      else
+        page_url_data
+      end
+
+    page_url = page_url_data |> to_string
+
+    image_url =
+      if rich_media[:image] != nil do
+        URI.merge(page_url_data, URI.parse(rich_media[:image]))
+        |> to_string
+      else
+        nil
+      end
+
+    site_name = rich_media[:site_name] || page_url_data.host
+
+    %{
+      type: "link",
+      provider_name: site_name,
+      provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
+      url: page_url,
+      image: image_url |> MediaProxy.url(),
+      title: rich_media[:title],
+      description: rich_media[:description],
+      pleroma: %{
+        opengraph: rich_media
+      }
+    }
+  end
+
+  def render("card.json", _) do
+    nil
+  end
+
   def render("attachment.json", %{attachment: attachment}) do
     [attachment_url | _] = attachment["url"]
     media_type = attachment_url["mediaType"] || attachment_url["mimeType"] || "image"
@@ -190,7 +255,7 @@ def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do
 
   def get_reply_to(%{data: %{"object" => object}}, _) do
     if object["inReplyTo"] && object["inReplyTo"] != "" do
-      Activity.get_create_activity_by_object_ap_id(object["inReplyTo"])
+      Activity.get_create_by_object_ap_id(object["inReplyTo"])
     else
       nil
     end
@@ -212,6 +277,9 @@ def get_visibility(object) do
       Enum.any?(to, &String.contains?(&1, "/followers")) ->
         "private"
 
+      length(cc) > 0 ->
+        "private"
+
       true ->
         "direct"
     end
@@ -291,4 +359,7 @@ def build_emojis(emojis) do
   defp present?(nil), do: false
   defp present?(false), do: false
   defp present?(_), do: true
+
+  defp pinned?(%Activity{id: id}, %User{info: %{pinned_activities: pinned_activities}}),
+    do: id in pinned_activities
 end

lib/pleroma/web/mastodon_api/websocket_handler.ex

@@ -6,7 +6,8 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
   require Logger
 
   alias Pleroma.Web.OAuth.Token
-  alias Pleroma.{User, Repo}
+  alias Pleroma.Repo
+  alias Pleroma.User
 
   @behaviour :cowboy_websocket_handler
 

lib/pleroma/web/media_proxy/controller.ex

@@ -4,11 +4,12 @@
 
 defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   use Pleroma.Web, :controller
-  alias Pleroma.{Web.MediaProxy, ReverseProxy}
+  alias Pleroma.ReverseProxy
+  alias Pleroma.Web.MediaProxy
 
   @default_proxy_opts [max_body_length: 25 * 1_048_576, http: [follow_redirect: true]]
 
-  def remote(conn, params = %{"sig" => sig64, "url" => url64}) do
+  def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
     with config <- Pleroma.Config.get([:media_proxy], []),
          true <- Keyword.get(config, :enabled, false),
          {:ok, url} <- MediaProxy.decode_url(sig64, url64),

lib/pleroma/web/media_proxy/media_proxy.ex

@@ -9,7 +9,7 @@ def url(nil), do: nil
 
   def url(""), do: nil
 
-  def url(url = "/" <> _), do: url
+  def url("/" <> _ = url), do: url
 
   def url(url) do
     config = Application.get_env(:pleroma, :media_proxy, [])
@@ -19,11 +19,16 @@ def url(url) do
     else
       secret = Application.get_env(:pleroma, Pleroma.Web.Endpoint)[:secret_key_base]
 
+      # Must preserve `%2F` for compatibility with S3 (https://git.pleroma.social/pleroma/pleroma/issues/580)
+      replacement = get_replacement(url, ":2F:")
+
       # The URL is url-decoded and encoded again to ensure it is correctly encoded and not twice.
       base64 =
         url
+        |> String.replace("%2F", replacement)
         |> URI.decode()
         |> URI.encode()
+        |> String.replace(replacement, "%2F")
         |> Base.url_encode64(@base64_opts)
 
       sig = :crypto.hmac(:sha, secret, base64)
@@ -60,4 +65,12 @@ def build_url(sig_base64, url_base64, filename \\ nil) do
     |> Enum.filter(fn value -> value end)
     |> Path.join()
   end
+
+  defp get_replacement(url, replacement) do
+    if String.contains?(url, replacement) do
+      get_replacement(url, replacement <> replacement)
+    else
+      replacement
+    end
+  end
 end

lib/pleroma/web/metadata.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata do
+  alias Phoenix.HTML
+
+  def build_tags(params) do
+    Enum.reduce(Pleroma.Config.get([__MODULE__, :providers], []), "", fn parser, acc ->
+      rendered_html =
+        params
+        |> parser.build_tags()
+        |> Enum.map(&to_tag/1)
+        |> Enum.map(&HTML.safe_to_string/1)
+        |> Enum.join()
+
+      acc <> rendered_html
+    end)
+  end
+
+  def to_tag(data) do
+    with {name, attrs, _content = []} <- data do
+      HTML.Tag.tag(name, attrs)
+    else
+      {name, attrs, content} ->
+        HTML.Tag.content_tag(name, content, attrs)
+
+      _ ->
+        raise ArgumentError, message: "make_tag invalid args"
+    end
+  end
+
+  def activity_nsfw?(%{data: %{"sensitive" => sensitive}}) do
+    Pleroma.Config.get([__MODULE__, :unfurl_nsfw], false) == false and sensitive
+  end
+
+  def activity_nsfw?(_) do
+    false
+  end
+end

lib/pleroma/web/metadata/opengraph.ex

@@ -0,0 +1,156 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata.Providers.OpenGraph do
+  alias Pleroma.HTML
+  alias Pleroma.Formatter
+  alias Pleroma.User
+  alias Pleroma.Web.Metadata
+  alias Pleroma.Web.MediaProxy
+  alias Pleroma.Web.Metadata.Providers.Provider
+
+  @behaviour Provider
+
+  @impl Provider
+  def build_tags(%{
+        object: object,
+        url: url,
+        user: user
+      }) do
+    attachments = build_attachments(object)
+    scrubbed_content = scrub_html_and_truncate(object)
+    # Zero width space
+    content =
+      if scrubbed_content != "" and scrubbed_content != "\u200B" do
+        ": “" <> scrubbed_content <> "”"
+      else
+        ""
+      end
+
+    # Most previews only show og:title which is inconvenient. Instagram
+    # hacks this by putting the description in the title and making the
+    # description longer prefixed by how many likes and shares the post
+    # has. Here we use the descriptive nickname in the title, and expand
+    # the full account & nickname in the description. We also use the cute^Wevil
+    # smart quotes around the status text like Instagram, too.
+    [
+      {:meta,
+       [
+         property: "og:title",
+         content: "#{user.name}" <> content
+       ], []},
+      {:meta, [property: "og:url", content: url], []},
+      {:meta,
+       [
+         property: "og:description",
+         content: "#{user_name_string(user)}" <> content
+       ], []},
+      {:meta, [property: "og:type", content: "website"], []}
+    ] ++
+      if attachments == [] or Metadata.activity_nsfw?(object) do
+        [
+          {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []},
+          {:meta, [property: "og:image:width", content: 150], []},
+          {:meta, [property: "og:image:height", content: 150], []}
+        ]
+      else
+        attachments
+      end
+  end
+
+  @impl Provider
+  def build_tags(%{user: user}) do
+    with truncated_bio = scrub_html_and_truncate(user.bio || "") do
+      [
+        {:meta,
+         [
+           property: "og:title",
+           content: user_name_string(user)
+         ], []},
+        {:meta, [property: "og:url", content: User.profile_url(user)], []},
+        {:meta, [property: "og:description", content: truncated_bio], []},
+        {:meta, [property: "og:type", content: "website"], []},
+        {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []},
+        {:meta, [property: "og:image:width", content: 150], []},
+        {:meta, [property: "og:image:height", content: 150], []}
+      ]
+    end
+  end
+
+  defp build_attachments(%{data: %{"attachment" => attachments}}) do
+    Enum.reduce(attachments, [], fn attachment, acc ->
+      rendered_tags =
+        Enum.reduce(attachment["url"], [], fn url, acc ->
+          media_type =
+            Enum.find(["image", "audio", "video"], fn media_type ->
+              String.starts_with?(url["mediaType"], media_type)
+            end)
+
+          # TODO: Add additional properties to objects when we have the data available.
+          # Also, Whatsapp only wants JPEG or PNGs. It seems that if we add a second og:image
+          # object when a Video or GIF is attached it will display that in the Whatsapp Rich Preview.
+          case media_type do
+            "audio" ->
+              [
+                {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], []}
+                | acc
+              ]
+
+            "image" ->
+              [
+                {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])],
+                 []},
+                {:meta, [property: "og:image:width", content: 150], []},
+                {:meta, [property: "og:image:height", content: 150], []}
+                | acc
+              ]
+
+            "video" ->
+              [
+                {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], []}
+                | acc
+              ]
+
+            _ ->
+              acc
+          end
+        end)
+
+      acc ++ rendered_tags
+    end)
+  end
+
+  defp scrub_html_and_truncate(%{data: %{"content" => content}} = object) do
+    content
+    # html content comes from DB already encoded, decode first and scrub after
+    |> HtmlEntities.decode()
+    |> String.replace(~r/<br\s?\/?>/, " ")
+    |> HTML.get_cached_stripped_html_for_object(object, __MODULE__)
+    |> Formatter.demojify()
+    |> Formatter.truncate()
+  end
+
+  defp scrub_html_and_truncate(content) when is_binary(content) do
+    content
+    # html content comes from DB already encoded, decode first and scrub after
+    |> HtmlEntities.decode()
+    |> String.replace(~r/<br\s?\/?>/, " ")
+    |> HTML.strip_tags()
+    |> Formatter.demojify()
+    |> Formatter.truncate()
+  end
+
+  defp attachment_url(url) do
+    MediaProxy.url(url)
+  end
+
+  defp user_name_string(user) do
+    "#{user.name} " <>
+      if user.local do
+        "(@#{user.nickname}@#{Pleroma.Web.Endpoint.host()})"
+      else
+        "(@#{user.nickname})"
+      end
+  end
+end

lib/pleroma/web/metadata/provider.ex

@@ -0,0 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata.Providers.Provider do
+  @callback build_tags(map()) :: list()
+end

lib/pleroma/web/metadata/twitter_card.ex

@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata.Providers.TwitterCard do
+  alias Pleroma.Web.Metadata.Providers.Provider
+  alias Pleroma.Web.Metadata
+
+  @behaviour Provider
+
+  @impl Provider
+  def build_tags(%{object: object}) do
+    if Metadata.activity_nsfw?(object) or object.data["attachment"] == [] do
+      build_tags(nil)
+    else
+      case find_first_acceptable_media_type(object) do
+        "image" ->
+          [{:meta, [property: "twitter:card", content: "summary_large_image"], []}]
+
+        "audio" ->
+          [{:meta, [property: "twitter:card", content: "player"], []}]
+
+        "video" ->
+          [{:meta, [property: "twitter:card", content: "player"], []}]
+
+        _ ->
+          build_tags(nil)
+      end
+    end
+  end
+
+  @impl Provider
+  def build_tags(_) do
+    [{:meta, [property: "twitter:card", content: "summary"], []}]
+  end
+
+  def find_first_acceptable_media_type(%{data: %{"attachment" => attachment}}) do
+    Enum.find_value(attachment, fn attachment ->
+      Enum.find_value(attachment["url"], fn url ->
+        Enum.find(["image", "audio", "video"], fn media_type ->
+          String.starts_with?(url["mediaType"], media_type)
+        end)
+      end)
+    end)
+  end
+end

lib/pleroma/web/nodeinfo/nodeinfo_controller.ex

@@ -5,10 +5,11 @@
 defmodule Pleroma.Web.Nodeinfo.NodeinfoController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.Stats
-  alias Pleroma.Web
-  alias Pleroma.{User, Repo}
   alias Pleroma.Config
+  alias Pleroma.Repo
+  alias Pleroma.Stats
+  alias Pleroma.User
+  alias Pleroma.Web
   alias Pleroma.Web.ActivityPub.MRF
 
   plug(Pleroma.Web.FederatingPlug)
@@ -19,6 +20,10 @@ def schemas(conn, _params) do
         %{
           rel: "http://nodeinfo.diaspora.software/ns/schema/2.0",
           href: Web.base_url() <> "/nodeinfo/2.0.json"
+        },
+        %{
+          rel: "http://nodeinfo.diaspora.software/ns/schema/2.1",
+          href: Web.base_url() <> "/nodeinfo/2.1.json"
         }
       ]
     }
@@ -26,8 +31,9 @@ def schemas(conn, _params) do
     json(conn, response)
   end
 
-  # Schema definition: https://github.com/jhass/nodeinfo/blob/master/schemas/2.0/schema.json
+  # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field
-  def nodeinfo(conn, %{"version" => "2.0"}) do
+  # under software.
+  def raw_nodeinfo do
     instance = Application.get_env(:pleroma, :instance)
     media_proxy = Application.get_env(:pleroma, :media_proxy)
     suggestions = Application.get_env(:pleroma, :suggestions)
@@ -39,6 +45,33 @@ def nodeinfo(conn, %{"version" => "2.0"}) do
       Application.get_env(:pleroma, :mrf_simple)
       |> Enum.into(%{})
 
+    # This horror is needed to convert regex sigils to strings
+    mrf_keyword =
+      Application.get_env(:pleroma, :mrf_keyword, [])
+      |> Enum.map(fn {key, value} ->
+        {key,
+         Enum.map(value, fn
+           {pattern, replacement} ->
+             %{
+               "pattern" =>
+                 if not is_binary(pattern) do
+                   inspect(pattern)
+                 else
+                   pattern
+                 end,
+               "replacement" => replacement
+             }
+
+           pattern ->
+             if not is_binary(pattern) do
+               inspect(pattern)
+             else
+               pattern
+             end
+         end)}
+      end)
+      |> Enum.into(%{})
+
     mrf_policies =
       MRF.get_policies()
       |> Enum.map(fn policy -> to_string(policy) |> String.split(".") |> List.last() end)
@@ -61,13 +94,12 @@ def nodeinfo(conn, %{"version" => "2.0"}) do
       Config.get([:mrf_user_allowlist], [])
       |> Enum.into(%{}, fn {k, v} -> {k, length(v)} end)
 
-    mrf_transparency = Keyword.get(instance, :mrf_transparency)
-
     federation_response =
-      if mrf_transparency do
+      if Keyword.get(instance, :mrf_transparency) do
         %{
           mrf_policies: mrf_policies,
           mrf_simple: mrf_simple,
+          mrf_keyword: mrf_keyword,
           mrf_user_allowlist: mrf_user_allowlist,
           quarantined_instances: quarantined
         }
@@ -98,10 +130,10 @@ def nodeinfo(conn, %{"version" => "2.0"}) do
       ]
       |> Enum.filter(& &1)
 
-    response = %{
+    %{
       version: "2.0",
       software: %{
-        name: Pleroma.Application.name(),
+        name: Pleroma.Application.name() |> String.downcase(),
         version: Pleroma.Application.version()
       },
       protocols: ["ostatus", "activitypub"],
@@ -142,12 +174,37 @@ def nodeinfo(conn, %{"version" => "2.0"}) do
         restrictedNicknames: Pleroma.Config.get([Pleroma.User, :restricted_nicknames])
       }
     }
+  end
 
+  # Schema definition: https://github.com/jhass/nodeinfo/blob/master/schemas/2.0/schema.json
+  # and https://github.com/jhass/nodeinfo/blob/master/schemas/2.1/schema.json
+  def nodeinfo(conn, %{"version" => "2.0"}) do
     conn
     |> put_resp_header(
       "content-type",
       "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#; charset=utf-8"
     )
+    |> json(raw_nodeinfo())
+  end
+
+  def nodeinfo(conn, %{"version" => "2.1"}) do
+    raw_response = raw_nodeinfo()
+
+    updated_software =
+      raw_response
+      |> Map.get(:software)
+      |> Map.put(:repository, Pleroma.Application.repository())
+
+    response =
+      raw_response
+      |> Map.put(:software, updated_software)
+      |> Map.put(:version, "2.1")
+
+    conn
+    |> put_resp_header(
+      "content-type",
+      "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.1#; charset=utf-8"
+    )
     |> json(response)
   end
 

lib/pleroma/web/oauth/app.ex

@@ -4,7 +4,7 @@
 
 defmodule Pleroma.Web.OAuth.App do
   use Ecto.Schema
-  import Ecto.{Changeset}
+  import Ecto.Changeset
 
   schema "apps" do
     field(:client_name, :string)

lib/pleroma/web/oauth/authorization.ex

@@ -5,16 +5,19 @@
 defmodule Pleroma.Web.OAuth.Authorization do
   use Ecto.Schema
 
-  alias Pleroma.{User, Repo}
+  alias Pleroma.User
-  alias Pleroma.Web.OAuth.{Authorization, App}
+  alias Pleroma.Repo
+  alias Pleroma.Web.OAuth.Authorization
+  alias Pleroma.Web.OAuth.App
 
-  import Ecto.{Changeset, Query}
+  import Ecto.Changeset
+  import Ecto.Query
 
   schema "oauth_authorizations" do
     field(:token, :string)
     field(:valid_until, :naive_datetime)
     field(:used, :boolean, default: false)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()

lib/pleroma/web/oauth/fallback_controller.ex

@@ -9,7 +9,8 @@ defmodule Pleroma.Web.OAuth.FallbackController do
   # No user/password
   def call(conn, _) do
     conn
+    |> put_status(:unauthorized)
     |> put_flash(:error, "Invalid Username/Password")
-    |> OAuthController.authorize(conn.params)
+    |> OAuthController.authorize(conn.params["authorization"])
   end
 end

lib/pleroma/web/oauth/oauth_controller.ex

@@ -5,8 +5,11 @@
 defmodule Pleroma.Web.OAuth.OAuthController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.Web.OAuth.{Authorization, Token, App}
+  alias Pleroma.Web.OAuth.Authorization
-  alias Pleroma.{Repo, User}
+  alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.App
+  alias Pleroma.Repo
+  alias Pleroma.User
   alias Comeonin.Pbkdf2
 
   plug(:fetch_session)
@@ -37,6 +40,7 @@ def create_authorization(conn, %{
          true <- Pbkdf2.checkpw(password, user.password_hash),
          {:auth_active, true} <- {:auth_active, User.auth_active?(user)},
          %App{} = app <- Repo.get_by(App, client_id: client_id),
+         true <- redirect_uri in String.split(app.redirect_uris),
          {:ok, auth} <- Authorization.create_authorization(app, user) do
       # Special case: Local MastodonFE.
       redirect_uri =

lib/pleroma/web/oauth/token.ex

@@ -7,14 +7,17 @@ defmodule Pleroma.Web.OAuth.Token do
 
   import Ecto.Query
 
-  alias Pleroma.{User, Repo}
+  alias Pleroma.User
-  alias Pleroma.Web.OAuth.{Token, App, Authorization}
+  alias Pleroma.Repo
+  alias Pleroma.Web.OAuth.Token
+  alias Pleroma.Web.OAuth.App
+  alias Pleroma.Web.OAuth.Authorization
 
   schema "oauth_tokens" do
     field(:token, :string)
     field(:refresh_token, :string)
     field(:valid_until, :naive_datetime)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()

lib/pleroma/web/ostatus/activity_representer.ex

@@ -3,8 +3,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.OStatus.ActivityRepresenter do
-  alias Pleroma.{Activity, User, Object}
+  alias Pleroma.Activity
+  alias Pleroma.User
+  alias Pleroma.Object
   alias Pleroma.Web.OStatus.UserRepresenter
+
   require Logger
 
   defp get_href(id) do
@@ -183,7 +186,7 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho
     _in_reply_to = get_in_reply_to(activity.data)
     author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
 
-    retweeted_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"])
+    retweeted_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
     retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"])
 
     retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true)

lib/pleroma/web/ostatus/feed_representer.ex

@@ -3,10 +3,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.OStatus.FeedRepresenter do
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.OStatus.{UserRepresenter, ActivityRepresenter}
   alias Pleroma.User
+  alias Pleroma.Web.OStatus
   alias Pleroma.Web.MediaProxy
+  alias Pleroma.Web.OStatus.ActivityRepresenter
+  alias Pleroma.Web.OStatus.UserRepresenter
 
   def to_simple_form(user, activities, _users) do
     most_recent_update =

lib/pleroma/web/ostatus/handlers/follow_handler.ex

@@ -3,7 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.OStatus.FollowHandler do
-  alias Pleroma.Web.{XML, OStatus}
+  alias Pleroma.Web.XML
+  alias Pleroma.Web.OStatus
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.User
 

lib/pleroma/web/ostatus/handlers/note_handler.ex

@@ -4,8 +4,10 @@
 
 defmodule Pleroma.Web.OStatus.NoteHandler do
   require Logger
-  alias Pleroma.Web.{XML, OStatus}
+  alias Pleroma.Web.OStatus
-  alias Pleroma.{Object, Activity}
+  alias Pleroma.Web.XML
+  alias Pleroma.Activity
+  alias Pleroma.Object
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.CommonAPI
@@ -86,7 +88,7 @@ def add_external_url(note, entry) do
   end
 
   def fetch_replied_to_activity(entry, inReplyTo) do
-    with %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(inReplyTo) do
+    with %Activity{} = activity <- Activity.get_create_by_object_ap_id(inReplyTo) do
       activity
     else
       _e ->
@@ -103,7 +105,7 @@ def fetch_replied_to_activity(entry, inReplyTo) do
   # TODO: Clean this up a bit.
   def handle_note(entry, doc \\ nil) do
     with id <- XML.string_from_xpath("//id", entry),
-         activity when is_nil(activity) <- Activity.get_create_activity_by_object_ap_id(id),
+         activity when is_nil(activity) <- Activity.get_create_by_object_ap_id(id),
          [author] <- :xmerl_xpath.string('//author[1]', doc),
          {:ok, actor} <- OStatus.find_make_or_update_user(author),
          content_html <- OStatus.get_content(entry),

lib/pleroma/web/ostatus/handlers/unfollow_handler.ex

@@ -3,7 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.OStatus.UnfollowHandler do
-  alias Pleroma.Web.{XML, OStatus}
+  alias Pleroma.Web.XML
+  alias Pleroma.Web.OStatus
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.User
 

lib/pleroma/web/ostatus/ostatus.ex

@@ -9,11 +9,19 @@ defmodule Pleroma.Web.OStatus do
   import Pleroma.Web.XML
   require Logger
 
-  alias Pleroma.{Repo, User, Web, Object, Activity}
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web
+  alias Pleroma.Object
+  alias Pleroma.Activity
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.{WebFinger, Websub}
-  alias Pleroma.Web.OStatus.{FollowHandler, UnfollowHandler, NoteHandler, DeleteHandler}
   alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.WebFinger
+  alias Pleroma.Web.Websub
+  alias Pleroma.Web.OStatus.FollowHandler
+  alias Pleroma.Web.OStatus.UnfollowHandler
+  alias Pleroma.Web.OStatus.NoteHandler
+  alias Pleroma.Web.OStatus.DeleteHandler
 
   def is_representable?(%Activity{data: data}) do
     object = Object.normalize(data["object"])
@@ -48,6 +56,9 @@ def remote_follow_path do
 
   def handle_incoming(xml_string) do
     with doc when doc != :error <- parse_document(xml_string) do
+      with {:ok, actor_user} <- find_make_or_update_user(doc),
+           do: Pleroma.Instances.set_reachable(actor_user.ap_id)
+
       entries = :xmerl_xpath.string('//entry', doc)
 
       activities =
@@ -148,7 +159,7 @@ def get_or_try_fetching(entry) do
     Logger.debug("Trying to get entry from db")
 
     with id when not is_nil(id) <- string_from_xpath("//activity:object[1]/id", entry),
-         %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do
       {:ok, activity}
     else
       _ ->

lib/pleroma/web/ostatus/ostatus_controller.ex

@@ -5,22 +5,30 @@
 defmodule Pleroma.Web.OStatus.OStatusController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.{User, Activity, Object}
+  alias Pleroma.Activity
-  alias Pleroma.Web.OStatus.{FeedRepresenter, ActivityRepresenter}
+  alias Pleroma.Object
-  alias Pleroma.Repo
+  alias Pleroma.User
-  alias Pleroma.Web.{OStatus, Federator}
-  alias Pleroma.Web.XML
-  alias Pleroma.Web.ActivityPub.ObjectView
-  alias Pleroma.Web.ActivityPub.ActivityPubController
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.ActivityPubController
+  alias Pleroma.Web.ActivityPub.ObjectView
+  alias Pleroma.Web.OStatus.ActivityRepresenter
+  alias Pleroma.Web.OStatus.FeedRepresenter
+  alias Pleroma.Web.Federator
+  alias Pleroma.Web.OStatus
+  alias Pleroma.Web.XML
 
   plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
+
   action_fallback(:errors)
 
   def feed_redirect(conn, %{"nickname" => nickname}) do
     case get_format(conn) do
       "html" ->
-        Fallback.RedirectController.redirector(conn, nil)
+        with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do
+          Fallback.RedirectController.redirector_with_meta(conn, %{user: user})
+        else
+          nil -> {:error, :not_found}
+        end
 
       "activity+json" ->
         ActivityPubController.call(conn, :user)
@@ -90,8 +98,7 @@ def object(conn, %{"uuid" => uuid}) do
       ActivityPubController.call(conn, :object)
     else
       with id <- o_status_url(conn, :object, uuid),
-           {_, %Activity{} = activity} <-
+           {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id(id)},
-             {:activity, Activity.get_create_activity_by_object_ap_id(id)},
            {_, true} <- {:public?, ActivityPub.is_public?(activity)},
            %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
         case get_format(conn) do
@@ -112,45 +119,65 @@ def object(conn, %{"uuid" => uuid}) do
   end
 
   def activity(conn, %{"uuid" => uuid}) do
-    with id <- o_status_url(conn, :activity, uuid),
+    if get_format(conn) == "activity+json" do
-         {_, %Activity{} = activity} <- {:activity, Activity.normalize(id)},
+      ActivityPubController.call(conn, :activity)
-         {_, true} <- {:public?, ActivityPub.is_public?(activity)},
-         %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
-      case format = get_format(conn) do
-        "html" -> redirect(conn, to: "/notice/#{activity.id}")
-        _ -> represent_activity(conn, format, activity, user)
-      end
     else
-      {:public?, false} ->
+      with id <- o_status_url(conn, :activity, uuid),
-        {:error, :not_found}
+           {_, %Activity{} = activity} <- {:activity, Activity.normalize(id)},
+           {_, true} <- {:public?, ActivityPub.is_public?(activity)},
+           %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
+        case format = get_format(conn) do
+          "html" -> redirect(conn, to: "/notice/#{activity.id}")
+          _ -> represent_activity(conn, format, activity, user)
+        end
+      else
+        {:public?, false} ->
+          {:error, :not_found}
 
-      {:activity, nil} ->
+        {:activity, nil} ->
-        {:error, :not_found}
+          {:error, :not_found}
 
-      e ->
+        e ->
-        e
+          e
+      end
     end
   end
 
   def notice(conn, %{"id" => id}) do
-    with {_, %Activity{} = activity} <- {:activity, Repo.get(Activity, id)},
+    with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(id)},
          {_, true} <- {:public?, ActivityPub.is_public?(activity)},
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       case format = get_format(conn) do
         "html" ->
-          conn
+          if activity.data["type"] == "Create" do
-          |> put_resp_content_type("text/html")
+            %Object{} = object = Object.normalize(activity.data["object"])
-          |> send_file(200, Pleroma.Plugs.InstanceStatic.file_path("index.html"))
+
+            Fallback.RedirectController.redirector_with_meta(conn, %{
+              object: object,
+              url:
+                Pleroma.Web.Router.Helpers.o_status_url(
+                  Pleroma.Web.Endpoint,
+                  :notice,
+                  activity.id
+                ),
+              user: user
+            })
+          else
+            Fallback.RedirectController.redirector(conn, nil)
+          end
 
         _ ->
           represent_activity(conn, format, activity, user)
       end
     else
       {:public?, false} ->
-        {:error, :not_found}
+        conn
+        |> put_status(404)
+        |> Fallback.RedirectController.redirector(nil, 404)
 
       {:activity, nil} ->
-        {:error, :not_found}
+        conn
+        |> Fallback.RedirectController.redirector(nil, 404)
 
       e ->
         e

lib/pleroma/web/push/push.ex

@@ -5,7 +5,8 @@
 defmodule Pleroma.Web.Push do
   use GenServer
 
-  alias Pleroma.{Repo, User}
+  alias Pleroma.Repo
+  alias Pleroma.User
   alias Pleroma.Web.Push.Subscription
 
   require Logger

lib/pleroma/web/push/subscription.ex

@@ -4,13 +4,16 @@
 
 defmodule Pleroma.Web.Push.Subscription do
   use Ecto.Schema
+
   import Ecto.Changeset
-  alias Pleroma.{Repo, User}
+
+  alias Pleroma.Repo
+  alias Pleroma.User
   alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.Push.Subscription
 
   schema "push_subscriptions" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     belongs_to(:token, Token)
     field(:endpoint, :string)
     field(:key_p256dh, :string)

lib/pleroma/web/rich_media/helpers.ex

@@ -0,0 +1,21 @@
+# Pleroma: A lightweight social networking server
+# Copyright _ 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Helpers do
+  alias Pleroma.Activity
+  alias Pleroma.Object
+  alias Pleroma.HTML
+  alias Pleroma.Web.RichMedia.Parser
+
+  def fetch_data_for_activity(%Activity{} = activity) do
+    with true <- Pleroma.Config.get([:rich_media, :enabled]),
+         %Object{} = object <- Object.normalize(activity.data["object"]),
+         {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),
+         {:ok, rich_media} <- Parser.parse(page_url) do
+      %{page_url: page_url, rich_media: rich_media}
+    else
+      _ -> %{}
+    end
+  end
+end

lib/pleroma/web/rich_media/parser.ex

@@ -0,0 +1,75 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Parser do
+  @parsers [
+    Pleroma.Web.RichMedia.Parsers.OGP,
+    Pleroma.Web.RichMedia.Parsers.TwitterCard,
+    Pleroma.Web.RichMedia.Parsers.OEmbed
+  ]
+
+  @hackney_options [
+    pool: :media,
+    timeout: 2_000,
+    recv_timeout: 2_000,
+    max_body: 2_000_000
+  ]
+
+  def parse(nil), do: {:error, "No URL provided"}
+
+  if Mix.env() == :test do
+    def parse(url), do: parse_url(url)
+  else
+    def parse(url) do
+      try do
+        Cachex.fetch!(:rich_media_cache, url, fn _ ->
+          {:commit, parse_url(url)}
+        end)
+      rescue
+        e ->
+          {:error, "Cachex error: #{inspect(e)}"}
+      end
+    end
+  end
+
+  defp parse_url(url) do
+    try do
+      {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: @hackney_options)
+
+      html |> maybe_parse() |> clean_parsed_data() |> check_parsed_data()
+    rescue
+      e ->
+        {:error, "Parsing error: #{inspect(e)}"}
+    end
+  end
+
+  defp maybe_parse(html) do
+    Enum.reduce_while(@parsers, %{}, fn parser, acc ->
+      case parser.parse(html, acc) do
+        {:ok, data} -> {:halt, data}
+        {:error, _msg} -> {:cont, acc}
+      end
+    end)
+  end
+
+  defp check_parsed_data(%{title: title} = data) when is_binary(title) and byte_size(title) > 0 do
+    {:ok, data}
+  end
+
+  defp check_parsed_data(data) do
+    {:error, "Found metadata was invalid or incomplete: #{inspect(data)}"}
+  end
+
+  defp clean_parsed_data(data) do
+    data
+    |> Enum.reject(fn {key, val} ->
+      with {:ok, _} <- Jason.encode(%{key => val}) do
+        false
+      else
+        _ -> true
+      end
+    end)
+    |> Map.new()
+  end
+end

lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex

@@ -0,0 +1,30 @@
+defmodule Pleroma.Web.RichMedia.Parsers.MetaTagsParser do
+  def parse(html, data, prefix, error_message, key_name, value_name \\ "content") do
+    with elements = [_ | _] <- get_elements(html, key_name, prefix),
+         meta_data =
+           Enum.reduce(elements, data, fn el, acc ->
+             attributes = normalize_attributes(el, prefix, key_name, value_name)
+
+             Map.merge(acc, attributes)
+           end) do
+      {:ok, meta_data}
+    else
+      _e -> {:error, error_message}
+    end
+  end
+
+  defp get_elements(html, key_name, prefix) do
+    html |> Floki.find("meta[#{key_name}^='#{prefix}:']")
+  end
+
+  defp normalize_attributes(html_node, prefix, key_name, value_name) do
+    {_tag, attributes, _children} = html_node
+
+    data =
+      Enum.into(attributes, %{}, fn {name, value} ->
+        {name, String.trim_leading(value, "#{prefix}:")}
+      end)
+
+    %{String.to_atom(data[key_name]) => data[value_name]}
+  end
+end