Resolve merge conflict

.gitlab-ci.yml

@@ -8,6 +8,7 @@ variables:
   POSTGRES_USER: postgres
   POSTGRES_PASSWORD: postgres
   DB_HOST: postgres
+  MIX_ENV: test
 
 cache:
   key: ${CI_COMMIT_REF_SLUG}
@@ -23,15 +24,15 @@ before_script:
   - mix local.rebar --force
   - mix deps.get
   - mix compile --force
-  - MIX_ENV=test mix ecto.create
+  - mix ecto.create
-  - MIX_ENV=test mix ecto.migrate
+  - mix ecto.migrate
 
 lint:
   stage: lint
   script:
-    - MIX_ENV=test mix format --check-formatted
+    - mix format --check-formatted
 
 unit-testing:
   stage: test
   script:
-    - MIX_ENV=test mix test --trace
+    - mix test --trace

README.md

@@ -30,7 +30,7 @@ While we don't provide docker files, other people have written very good ones. T
 ### Dependencies
 
 * Postgresql version 9.6 or newer
-* Elixir version 1.5 or newer. If your distribution only has an old version available, check [Elixir's install page](https://elixir-lang.org/install.html) or use a tool like [asdf](https://github.com/asdf-vm/asdf).
+* Elixir version 1.7 or newer. If your distribution only has an old version available, check [Elixir's install page](https://elixir-lang.org/install.html) or use a tool like [asdf](https://github.com/asdf-vm/asdf).
 * Build-essential tools
 
 ### Configuration
@@ -39,7 +39,7 @@ While we don't provide docker files, other people have written very good ones. T
 
   * Run `mix pleroma.instance gen`. This will ask you questions about your instance and generate a configuration file in `config/generated_config.exs`. Check that and copy it to either `config/dev.secret.exs` or `config/prod.secret.exs`. It will also create a `config/setup_db.psql`, which you should run as the PostgreSQL superuser (i.e., `sudo -u postgres psql -f config/setup_db.psql`). It will create the database, user, and password you gave `mix pleroma.gen.instance` earlier, as well as set up the necessary extensions in the database. PostgreSQL superuser privileges are only needed for this step.
 
-  * For these next steps, the default will be to run pleroma using the dev configuration file, `config/dev.secret.exs`. To run them using the prod config file, prefix each command at the shell with `MIX_ENV=prod`. For example: `MIX_ENV=prod mix phx.server`. Documentation for the config can be found at [``config/config.md``](config/config.md)
+  * For these next steps, the default will be to run pleroma using the dev configuration file, `config/dev.secret.exs`. To run them using the prod config file, prefix each command at the shell with `MIX_ENV=prod`. For example: `MIX_ENV=prod mix phx.server`. Documentation for the config can be found at [``docs/config.md``](docs/config.md)
 
   * Run `mix ecto.migrate` to run the database migrations. You will have to do this again after certain updates.
 
@@ -55,9 +55,9 @@ While we don't provide docker files, other people have written very good ones. T
 Pleroma comes with two frontends. The first one, Pleroma FE, can be reached by normally visiting the site. The other one, based on the Mastodon project, can be found by visiting the /web path of your site.
 
 ### As systemd service (with provided .service file)
-Example .service file can be found in `installation/pleroma.service` you can put it in `/etc/systemd/system/`.
+Example .service file can be found in `installation/pleroma.service`. Copy this to `/etc/systemd/system/`.
-Running `service pleroma start`
+Running `systemctl enable --now pleroma.service` will run Pleroma and enable startup on boot.
-Logs can be watched by using `journalctl -fu pleroma.service`
+Logs can be watched by using `journalctl -fu pleroma.service`.
 
 ### As OpenRC service (with provided RC file)
 Copy ``installation/init.d/pleroma`` to ``/etc/init.d/pleroma``.
@@ -65,7 +65,7 @@ You can add it to the services ran by default with:
 ``rc-update add pleroma``
 
 ### Standalone/run by other means
-Run `mix phx.server` in repository's root, it will output log into stdout/stderr
+Run `mix phx.server` in repository's root, it will output log into stdout/stderr.
 
 ### Using an upstream proxy for federation
 

config/config.exs

@@ -12,7 +12,7 @@
 
 config :pleroma, Pleroma.Captcha,
   enabled: false,
-  seconds_retained: 180,
+  seconds_valid: 60,
   method: Pleroma.Captcha.Kocaptcha
 
 config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"
@@ -54,6 +54,17 @@
     "xmpp"
   ]
 
+websocket_config = [
+  path: "/websocket",
+  serializer: [
+    {Phoenix.Socket.V1.JSONSerializer, "~> 1.0.0"},
+    {Phoenix.Socket.V2.JSONSerializer, "~> 2.0.0"}
+  ],
+  timeout: 60_000,
+  transport_log: false,
+  compress: false
+]
+
 # Configures the endpoint
 config :pleroma, Pleroma.Web.Endpoint,
   url: [host: "localhost"],
@@ -62,6 +73,8 @@
       {:_,
        [
          {"/api/v1/streaming", Elixir.Pleroma.Web.MastodonAPI.WebsocketHandler, []},
+         {"/socket/websocket", Phoenix.Endpoint.CowboyWebSocket,
+          {nil, {Pleroma.Web.Endpoint, Pleroma.Web.UserSocket, websocket_config}}},
          {:_, Plug.Adapters.Cowboy.Handler, {Pleroma.Web.Endpoint, []}}
        ]}
     ]
@@ -78,6 +91,12 @@
   format: "$time $metadata[$level] $message\n",
   metadata: [:request_id]
 
+config :logger, :ex_syslogger,
+  level: :debug,
+  ident: "Pleroma",
+  format: "$date $time $metadata[$level] $message",
+  metadata: [:request_id]
+
 config :mime, :types, %{
   "application/xml" => ["xml"],
   "application/xrd+xml" => ["xrd+xml"],
@@ -98,7 +117,8 @@
   name: "Pleroma",
   email: "example@example.com",
   description: "A Pleroma instance, an alternative fediverse server",
-  limit: 5000,
+  limit: 5_000,
+  remote_limit: 100_000,
   upload_limit: 16_000_000,
   avatar_upload_limit: 2_000_000,
   background_upload_limit: 4_000_000,
@@ -117,7 +137,9 @@
     "text/markdown"
   ],
   finmoji_enabled: true,
-  mrf_transparency: true
+  mrf_transparency: true,
+  autofollowed_nicknames: [],
+  max_pinned_statuses: 1
 
 config :pleroma, :markup,
   # XXX - unfortunately, inline images must be enabled by default right now, because
@@ -137,8 +159,8 @@
   logo_mask: true,
   logo_margin: "0.1em",
   background: "/static/aurora_borealis.jpg",
-  redirect_root_no_login: "/~/main/all",
+  redirect_root_no_login: "/main/all",
-  redirect_root_login: "/~/main/friends",
+  redirect_root_login: "/main/friends",
   show_instance_panel: true,
   scope_options_enabled: false,
   formatting_options_enabled: false,
@@ -163,6 +185,8 @@
   allow_followersonly: false,
   allow_direct: false
 
+config :pleroma, :mrf_hellthread, threshold: 10
+
 config :pleroma, :mrf_simple,
   media_removal: [],
   media_nsfw: [],
@@ -170,13 +194,7 @@
   reject: [],
   accept: []
 
-config :pleroma, :media_proxy,
+config :pleroma, :media_proxy, enabled: false
-  enabled: false,
-  # base_url: "https://cache.pleroma.social",
-  proxy_opts: [
-    # inline_content_types: [] | false | true,
-    # http: [:insecure]
-  ]
 
 config :pleroma, :chat, enabled: true
 
@@ -220,6 +238,46 @@
   credentials: true,
   headers: ["Authorization", "Content-Type", "Idempotency-Key"]
 
+config :pleroma, Pleroma.User,
+  restricted_nicknames: [
+    ".well-known",
+    "~",
+    "about",
+    "activities",
+    "api",
+    "auth",
+    "dev",
+    "friend-requests",
+    "inbox",
+    "internal",
+    "main",
+    "media",
+    "nodeinfo",
+    "notice",
+    "oauth",
+    "objects",
+    "ostatus_subscribe",
+    "pleroma",
+    "proxy",
+    "push",
+    "registration",
+    "relay",
+    "settings",
+    "status",
+    "tag",
+    "user-search",
+    "users",
+    "web"
+  ]
+
+config :pleroma, Pleroma.Web.Federator, max_jobs: 50
+
+config :pleroma, Pleroma.Web.Federator.RetryQueue,
+  enabled: false,
+  max_jobs: 20,
+  initial_timeout: 30,
+  max_retries: 5
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/test.exs

@@ -9,7 +9,8 @@
 
 # Disable captha for tests
 config :pleroma, Pleroma.Captcha,
-  enabled: true,
+  # It should not be enabled for automatic tests
+  enabled: false,
   # A fake captcha service for tests
   method: Pleroma.Captcha.Mock
 

docs/Admin-API.md

@@ -0,0 +1,100 @@
+# Admin API
+Authentication is required and the user must be an admin.
+
+## `/api/pleroma/admin/user`
+### Remove a user
+* Method `DELETE`
+* Params:
+    * `nickname`
+* Response: User’s nickname
+### Create a user
+* Method: `POST`
+* Params:
+    * `nickname`
+    * `email`
+    * `password`
+* Response: User’s nickname
+
+## `/api/pleroma/admin/users/tag`
+### Tag a list of users
+* Method: `PUT`
+* Params:
+    * `nickname`
+    * `tags`
+### Untag a list of users
+* Method: `DELETE`
+* Params:
+    * `nickname`
+    * `tags`
+
+## `/api/pleroma/admin/permission_group/:nickname`
+### Get user user permission groups membership
+* Method: `GET`
+* Params: none
+* Response:
+```JSON
+{
+	"is_moderator": bool,
+	"is_admin": bool
+}
+```
+
+## `/api/pleroma/admin/permission_group/:nickname/:permission_group`
+Note: Available `:permission_group` is currently moderator and admin. 404 is returned when the permission group doesn’t exist.
+
+### Get user user permission groups membership
+* Method: `GET`
+* Params: none
+* Response:
+```JSON
+{
+	"is_moderator": bool,
+	"is_admin": bool
+}
+```
+### Add user in permission group
+* Method: `POST`
+* Params: none
+* Response:
+    * On failure: ``{"error": "…"}``
+    * On success: JSON of the ``user.info``
+### Remove user from permission group
+* Method: `DELETE`
+* Params: none
+* Response:
+    * On failure: ``{"error": "…"}``
+    * On success: JSON of the ``user.info``
+* Note: An admin cannot revoke their own admin status.
+
+## `/api/pleroma/admin/relay`
+### Follow a Relay
+* Methods: `POST`
+* Params:
+    * `relay_url`
+* Response:
+    * On success: URL of the followed relay
+### Unfollow a Relay
+* Methods: `DELETE`
+* Params:
+    * `relay_url`
+* Response:
+    * On success: URL of the unfollowed relay
+
+## `/api/pleroma/admin/invite_token`
+### Get a account registeration invite token
+* Methods: `GET`
+* Params: none
+* Response: invite token (base64 string)
+
+## `/api/pleroma/admin/email_invite`
+### Sends registration invite via email
+* Methods: `POST`
+* Params:
+    * `email`
+    * `name`, optionnal
+
+## `/api/pleroma/admin/password_reset`
+### Get a password reset token for a given nickname
+* Methods: `GET`
+* Params: none
+* Response: password reset token (base64 string)

docs/Pleroma-API.md

@@ -0,0 +1,98 @@
+# Authentication
+
+Requests that require it can be authenticated with [an OAuth token](https://tools.ietf.org/html/rfc6749), the `_pleroma_key` cookie, or [HTTP Basic Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization).
+
+# Request parameters
+
+Request parameters can be passed via [query strings](https://en.wikipedia.org/wiki/Query_string) or as [form data](https://www.w3.org/TR/html401/interact/forms.html). Files must be uploaded as `multipart/form-data`.
+
+# Endpoints
+
+## `/api/pleroma/emoji`
+### Lists the custom emoji on that server.
+* Method: `GET`
+* Authentication: not required
+* Params: none
+* Response: JSON
+* Example response: `{"kalsarikannit_f":"/finmoji/128px/kalsarikannit_f-128.png","perkele":"/finmoji/128px/perkele-128.png","blobdab":"/emoji/blobdab.png","happiness":"/finmoji/128px/happiness-128.png"}`
+
+## `/api/pleroma/follow_import`
+### Imports your follows, for example from a Mastodon CSV file.
+* Method: `POST`
+* Authentication: required
+* Params:
+    * `list`: STRING or FILE containing a whitespace-separated list of accounts to follow
+* Response: HTTP 200 on success, 500 on error
+* Note: Users that can't be followed are silently skipped.
+
+## `/api/pleroma/captcha`
+### Get a new captcha
+* Method: `GET`
+* Authentication: not required
+* Params: none
+* Response: Provider specific JSON, the only guaranteed parameter is `type` 
+* Example response: `{"type": "kocaptcha", "token": "whatever", "url": "https://captcha.kotobank.ch/endpoint"}`
+
+## `/api/pleroma/delete_account`
+### Delete an account
+* Method `POST`
+* Authentication: required
+* Params: 
+    * `password`: user's password
+* Response: JSON. Returns `{"status": "success"}` if the deletion was successful, `{"error": "[error message]"}` otherwise
+* Example response: `{"error": "Invalid password."}`
+
+## `/api/account/register`
+### Register a new user
+* Method `POST`
+* Authentication: not required
+* Params:
+    * `nickname`
+    * `fullname`
+    * `bio`
+    * `email`
+    * `password`
+    * `confirm`
+    * `captcha_solution`: optional, contains provider-specific captcha solution,
+    * `captcha_token`: optional, contains provider-specific captcha token
+* Response: JSON. Returns a user object on success, otherwise returns `{"error": "error_msg"}`
+* Example response:
+```
+{
+	"background_image": null,
+	"cover_photo": "https://pleroma.soykaf.com/images/banner.png",
+	"created_at": "Tue Dec 18 16:55:56 +0000 2018",
+	"default_scope": "public",
+	"description": "blushy-crushy fediverse idol + pleroma dev\nlet's be friends \nぷれろまの生徒会長。謎の外人。日本語OK. \n公主病.",
+	"description_html": "blushy-crushy fediverse idol + pleroma dev.<br />let's be friends <br />ぷれろまの生徒会長。謎の外人。日本語OK. <br />公主病.",
+	"favourites_count": 0,
+	"fields": [],
+	"followers_count": 0,
+	"following": false,
+	"follows_you": false,
+	"friends_count": 0,
+	"id": 6,
+	"is_local": true,
+	"locked": false,
+	"name": "lain",
+	"name_html": "lain",
+	"no_rich_text": false,
+	"pleroma": {
+		"tags": []
+	},
+	"profile_image_url": "https://pleroma.soykaf.com/images/avi.png",
+	"profile_image_url_https": "https://pleroma.soykaf.com/images/avi.png",
+	"profile_image_url_original": "https://pleroma.soykaf.com/images/avi.png",
+	"profile_image_url_profile_size": "https://pleroma.soykaf.com/images/avi.png",
+	"rights": {
+		"delete_others_notice": false
+	},
+	"screen_name": "lain",
+	"statuses_count": 0,
+	"statusnet_blocking": false,
+	"statusnet_profile_url": "https://pleroma.soykaf.com/users/lain"
+}
+```
+
+## `/api/pleroma/admin/`…
+See [Admin-API](Admin-API.md)

docs/config.md

@@ -63,12 +63,14 @@ config :pleroma, Pleroma.Mailer,
 * `email`: Email used to reach an Administrator/Moderator of the instance
 * `description`: The instance’s description, can be seen in nodeinfo and ``/api/v1/instance``
 * `limit`: Posts character limit (CW/Subject included in the counter)
+* `remote_limit`: Hard character limit beyond which remote posts will be dropped.
 * `upload_limit`: File size limit of uploads (except for avatar, background, banner)
 * `avatar_upload_limit`: File size limit of user’s profile avatars
 * `background_upload_limit`: File size limit of user’s profile backgrounds
 * `banner_upload_limit`: File size limit of user’s profile banners
 * `registrations_open`: Enable registrations for anyone, invitations can be enabled when false.
 * `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`).
+* `account_activation_required`: Require users to confirm their emails before signing in.
 * `federating`: Enable federation with other instances
 * `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance
 * `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default:
@@ -91,6 +93,12 @@ config :pleroma, Pleroma.Mailer,
 * `always_show_subject_input`: When set to false, auto-hide the subject field when it's empty.
 * `extended_nickname_format`: Set to `true` to use extended local nicknames format (allows underscores/dashes). This will break federation with
     older software for theses nicknames.
+* `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature.
+* `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow.
+
+## :logger
+* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog
+See: [logger’s documentation](https://hexdocs.pm/logger/Logger.html) and [ex_syslogger’s documentation](https://hexdocs.pm/ex_syslogger/)
 
 ## :fe
 This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:instance`` is set to false.
@@ -120,6 +128,9 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
 * `allow_followersonly`: whether to allow followers-only posts
 * `allow_direct`: whether to allow direct messages
 
+## :mrf_hellthread
+* `threshold`: Number of mentioned users after which the message gets discarded as spam
+
 ## :media_proxy
 * `enabled`: Enables proxying of remote media to the instance’s proxy
 * `base_url`: The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.
@@ -167,7 +178,7 @@ Web Push Notifications configuration. You can use the mix task `mix web_push.gen
 ## Pleroma.Captcha
 * `enabled`: Whether the captcha should be shown on registration
 * `method`: The method/service to use for captcha
-* `seconds_retained`: The time in seconds for which the captcha is valid (stored in the cache)
+* `seconds_valid`: The time in seconds for which the captcha is valid
 
 ### Pleroma.Captcha.Kocaptcha
 Kocaptcha is a very simple captcha service with a single API endpoint,
@@ -188,3 +199,14 @@ You can then do
 ```
 curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerandomtoken"
 ```
+
+## Pleroma.Web.Federator
+
+* `max_jobs`: The maximum amount of parallel federation jobs running at the same time.
+
+## Pleroma.Web.Federator.RetryQueue
+
+* `enabled`: If set to `true`, failed federation jobs will be retried
+* `max_jobs`: The maximum amount of parallel federation jobs running at the same time.
+* `initial_timeout`: The initial timeout in seconds
+* `max_retries`: The maximum number of times a federation job is retried

installation/pleroma.service

@@ -21,6 +21,8 @@ ProtectSystem=full
 PrivateDevices=false
 ; Ensures that the service process and all its children can never gain new privileges through execve().
 NoNewPrivileges=true
+; Drops the sysadmin capability from the daemon.
+CapabilityBoundingSet=~CAP_SYS_ADMIN
 
 [Install]
 WantedBy=multi-user.target

installation/pleroma.vcl

@@ -14,43 +14,45 @@ acl purge {
 sub vcl_recv {
     # Redirect HTTP to HTTPS
     if (std.port(server.ip) != 443) {
-        set req.http.x-redir = "https://" + req.http.host + req.url;
+      set req.http.x-redir = "https://" + req.http.host + req.url;
-        return (synth(750, ""));
+      return (synth(750, ""));
+    }
+
+    # CHUNKED SUPPORT
+    if (req.http.Range ~ "bytes=") {
+      set req.http.x-range = req.http.Range;
     }
 
     # Pipe if WebSockets request is coming through
     if (req.http.upgrade ~ "(?i)websocket") {
-        return (pipe);
+      return (pipe);
     }
 
     # Allow purging of the cache
     if (req.method == "PURGE") {
-        if (!client.ip ~ purge) {
+      if (!client.ip ~ purge) {
-          return(synth(405,"Not allowed."));
+        return(synth(405,"Not allowed."));
-        }
+      }
-        return(purge);
+      return(purge);
     }
 
     # Pleroma MediaProxy - strip headers that will affect caching
     if (req.url ~ "^/proxy/") {
-        unset req.http.Cookie;
+      unset req.http.Cookie;
-        unset req.http.Authorization;
+      unset req.http.Authorization;
-        unset req.http.Accept;
+      unset req.http.Accept;
-        return (hash);
+      return (hash);
     }
 
     # Strip headers that will affect caching from all other static content
     # This also permits caching of individual toots and AP Activities
     if ((req.url ~ "^/(media|static)/") ||
-    (req.url ~ "(?i)\.(html|js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|ttf|pdf|woff|woff2)$"))
+    (req.url ~ "(?i)\.(html|js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|mp4|ogg|webm|svg|swf|ttf|pdf|woff|woff2)$"))
     {
       unset req.http.Cookie;
       unset req.http.Authorization;
       return (hash);
     }
-
-    # Everything else should just be piped to Pleroma
-    return (pipe);
 }
 
 sub vcl_backend_response {
@@ -59,8 +61,11 @@ sub vcl_backend_response {
       set beresp.do_gzip = true;
     }
 
-    # etags are bad
+    # CHUNKED SUPPORT
-    unset beresp.http.etag;
+    if (bereq.http.x-range ~ "bytes=" && beresp.status == 206) {
+      set beresp.ttl = 10m;
+      set beresp.http.CR = beresp.http.content-range;
+    }
 
     # Don't cache objects that require authentication
     if (beresp.http.Authorization && !beresp.http.Cache-Control ~ "public") {
@@ -81,9 +86,9 @@ sub vcl_backend_response {
 
     # Do not cache redirects and errors
     if ((beresp.status >= 300) && (beresp.status < 500)) {
-        set beresp.uncacheable = true;
+      set beresp.uncacheable = true;
-        set beresp.ttl = 30s;
+      set beresp.ttl = 30s;
-        return (deliver);
+      return (deliver);
     }
 
     # Pleroma MediaProxy internally sets headers properly
@@ -92,14 +97,12 @@ sub vcl_backend_response {
     }
 
     # Strip cache-restricting headers from Pleroma on static content that we want to cache
-    # Also enable streaming of cached content to clients (no waiting for Varnish to complete backend fetch)
+    if (bereq.url ~ "(?i)\.(js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|mp4|ogg|webm|svg|swf|ttf|pdf|woff|woff2)$")
-    if (bereq.url ~ "(?i)\.(js|css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|svg|swf|ttf|pdf|woff|woff2)$")
     {
       unset beresp.http.set-cookie;
       unset beresp.http.Cache-Control;
       unset beresp.http.x-request-id;
       set beresp.http.Cache-Control = "public, max-age=86400";
-      set beresp.do_stream = true;
     }
 }
 
@@ -115,7 +118,30 @@ sub vcl_synth {
 # Ensure WebSockets through the pipe do not close prematurely
 sub vcl_pipe {
     if (req.http.upgrade) {
-        set bereq.http.upgrade = req.http.upgrade;
+      set bereq.http.upgrade = req.http.upgrade;
-        set bereq.http.connection = req.http.connection;
+      set bereq.http.connection = req.http.connection;
+    }
+}
+
+sub vcl_hash {
+    # CHUNKED SUPPORT
+    if (req.http.x-range ~ "bytes=") {
+      hash_data(req.http.x-range);
+      unset req.http.Range;
+    }
+}
+
+sub vcl_backend_fetch {
+    # CHUNKED SUPPORT
+    if (bereq.http.x-range) {
+      set bereq.http.Range = bereq.http.x-range;
+    }
+}
+
+sub vcl_deliver {
+    # CHUNKED SUPPORT
+    if (resp.http.CR) {
+      set resp.http.Content-Range = resp.http.CR;
+      unset resp.http.CR;
     }
 }

lib/mix/tasks/pleroma/common.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Common do
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do

lib/mix/tasks/pleroma/instance.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Instance do
   use Mix.Task
   alias Mix.Tasks.Pleroma.Common
@@ -71,7 +75,7 @@ def run(["gen" | rest]) do
       name =
         Common.get_option(
           options,
-          :name,
+          :instance_name,
           "What is the name of your instance? (e.g. Pleroma/Soykaf)"
         )
 

lib/mix/tasks/pleroma/relay.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Relay do
   use Mix.Task
   alias Pleroma.Web.ActivityPub.Relay

lib/mix/tasks/pleroma/uploads.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Uploads do
   use Mix.Task
   alias Pleroma.{Upload, Uploaders.Local}

lib/mix/tasks/pleroma/user.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.User do
   use Mix.Task
   import Ecto.Changeset
@@ -18,6 +22,7 @@ defmodule Mix.Tasks.Pleroma.User do
   - `--password PASSWORD` - the user's password
   - `--moderator`/`--no-moderator` - whether the user is a moderator
   - `--admin`/`--no-admin` - whether the user is an admin
+  - `-y`, `--assume-yes`/`--no-assume-yes` - whether to assume yes to all questions 
 
   ## Generate an invite link.
 
@@ -57,7 +62,11 @@ def run(["new", nickname, email | rest]) do
           bio: :string,
           password: :string,
           moderator: :boolean,
-          admin: :boolean
+          admin: :boolean,
+          assume_yes: :boolean
+        ],
+        aliases: [
+          y: :assume_yes
         ]
       )
 
@@ -75,6 +84,7 @@ def run(["new", nickname, email | rest]) do
 
     moderator? = Keyword.get(options, :moderator, false)
     admin? = Keyword.get(options, :admin, false)
+    assume_yes? = Keyword.get(options, :assume_yes, false)
 
     Mix.shell().info("""
     A user will be created with the following information:
@@ -89,7 +99,7 @@ def run(["new", nickname, email | rest]) do
       - admin: #{if(admin?, do: "true", else: "false")}
     """)
 
-    proceed? = Mix.shell().yes?("Continue?")
+    proceed? = assume_yes? or Mix.shell().yes?("Continue?")
 
     unless not proceed? do
       Common.start_pleroma()
@@ -103,8 +113,8 @@ def run(["new", nickname, email | rest]) do
         bio: bio
       }
 
-      user = User.register_changeset(%User{}, params)
+      changeset = User.register_changeset(%User{}, params, confirmed: true)
-      Repo.insert!(user)
+      {:ok, _user} = User.register(changeset)
 
       Mix.shell().info("User #{nickname} created")
 

lib/pleroma/PasswordResetToken.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.PasswordResetToken do
   use Ecto.Schema
 

lib/pleroma/activity.ex

@@ -1,8 +1,14 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Activity do
   use Ecto.Schema
   alias Pleroma.{Repo, Activity, Notification}
   import Ecto.Query
 
+  @type t :: %__MODULE__{}
+
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
   @mastodon_notification_types %{
     "Create" => "mention",

lib/pleroma/application.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Application do
   use Application
   import Supervisor.Spec
@@ -25,6 +29,16 @@ def start(_type, _args) do
         supervisor(Pleroma.Repo, []),
         worker(Pleroma.Emoji, []),
         worker(Pleroma.Captcha, []),
+        worker(
+          Cachex,
+          [
+            :used_captcha_cache,
+            [
+              ttl_interval: :timer.seconds(Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid]))
+            ]
+          ],
+          id: :cachex_used_captcha_cache
+        ),
         worker(
           Cachex,
           [
@@ -49,6 +63,27 @@ def start(_type, _args) do
           ],
           id: :cachex_object
         ),
+        worker(
+          Cachex,
+          [
+            :rich_media_cache,
+            [
+              default_ttl: :timer.minutes(120),
+              limit: 5000
+            ]
+          ],
+          id: :cachex_rich_media
+        ),
+        worker(
+          Cachex,
+          [
+            :scrubber_cache,
+            [
+              limit: 2500
+            ]
+          ],
+          id: :cachex_scrubber
+        ),
         worker(
           Cachex,
           [

lib/pleroma/captcha/captcha.ex

@@ -1,7 +1,13 @@
-defmodule Pleroma.Captcha do
+# Pleroma: A lightweight social networking server
-  use GenServer
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
 
-  @ets_options [:ordered_set, :private, :named_table, {:read_concurrency, true}]
+defmodule Pleroma.Captcha do
+  alias Plug.Crypto.KeyGenerator
+  alias Plug.Crypto.MessageEncryptor
+  alias Calendar.DateTime
+
+  use GenServer
 
   @doc false
   def start_link() do
@@ -10,14 +16,6 @@ def start_link() do
 
   @doc false
   def init(_) do
-    # Create a ETS table to store captchas
-    ets_name = Module.concat(method(), Ets)
-    ^ets_name = :ets.new(Module.concat(method(), Ets), @ets_options)
-
-    # Clean up old captchas every few minutes
-    seconds_retained = Pleroma.Config.get!([__MODULE__, :seconds_retained])
-    Process.send_after(self(), :cleanup, 1000 * seconds_retained)
-
     {:ok, nil}
   end
 
@@ -31,8 +29,8 @@ def new() do
   @doc """
   Ask the configured captcha service to validate the captcha
   """
-  def validate(token, captcha) do
+  def validate(token, captcha, answer_data) do
-    GenServer.call(__MODULE__, {:validate, token, captcha})
+    GenServer.call(__MODULE__, {:validate, token, captcha, answer_data})
   end
 
   @doc false
@@ -42,24 +40,71 @@ def handle_call(:new, _from, state) do
     if !enabled do
       {:reply, %{type: :none}, state}
     else
-      {:reply, method().new(), state}
+      new_captcha = method().new()
+
+      secret_key_base = Pleroma.Config.get!([Pleroma.Web.Endpoint, :secret_key_base])
+
+      # This make salt a little different for two keys
+      token = new_captcha[:token]
+      secret = KeyGenerator.generate(secret_key_base, token <> "_encrypt")
+      sign_secret = KeyGenerator.generate(secret_key_base, token <> "_sign")
+      # Basicallty copy what Phoenix.Token does here, add the time to
+      # the actual data and make it a binary to then encrypt it
+      encrypted_captcha_answer =
+        %{
+          at: DateTime.now_utc(),
+          answer_data: new_captcha[:answer_data]
+        }
+        |> :erlang.term_to_binary()
+        |> MessageEncryptor.encrypt(secret, sign_secret)
+
+      {
+        :reply,
+        # Repalce the answer with the encrypted answer
+        %{new_captcha | answer_data: encrypted_captcha_answer},
+        state
+      }
     end
   end
 
   @doc false
-  def handle_call({:validate, token, captcha}, _from, state) do
+  def handle_call({:validate, token, captcha, answer_data}, _from, state) do
-    {:reply, method().validate(token, captcha), state}
+    secret_key_base = Pleroma.Config.get!([Pleroma.Web.Endpoint, :secret_key_base])
-  end
+    secret = KeyGenerator.generate(secret_key_base, token <> "_encrypt")
+    sign_secret = KeyGenerator.generate(secret_key_base, token <> "_sign")
 
-  @doc false
+    # If the time found is less than (current_time - seconds_valid), then the time has already passed.
-  def handle_info(:cleanup, state) do
+    # Later we check that the time found is more than the presumed invalidatation time, that means
-    :ok = method().cleanup()
+    # that the data is still valid and the captcha can be checked
+    seconds_valid = Pleroma.Config.get!([Pleroma.Captcha, :seconds_valid])
+    valid_if_after = DateTime.subtract!(DateTime.now_utc(), seconds_valid)
 
-    seconds_retained = Pleroma.Config.get!([__MODULE__, :seconds_retained])
+    result =
-    # Schedule the next clenup
+      with {:ok, data} <- MessageEncryptor.decrypt(answer_data, secret, sign_secret),
-    Process.send_after(self(), :cleanup, 1000 * seconds_retained)
+           %{at: at, answer_data: answer_md5} <- :erlang.binary_to_term(data) do
+        try do
+          if DateTime.before?(at, valid_if_after), do: throw({:error, "CAPTCHA expired"})
 
-    {:noreply, state}
+          if not is_nil(Cachex.get!(:used_captcha_cache, token)),
+            do: throw({:error, "CAPTCHA already used"})
+
+          res = method().validate(token, captcha, answer_md5)
+          # Throw if an error occurs
+          if res != :ok, do: throw(res)
+
+          # Mark this captcha as used
+          {:ok, _} =
+            Cachex.put(:used_captcha_cache, token, true, ttl: :timer.seconds(seconds_valid))
+
+          :ok
+        catch
+          :throw, e -> e
+        end
+      else
+        _ -> {:error, "Invalid answer data"}
+      end
+
+    {:reply, result, state}
   end
 
   defp method, do: Pleroma.Config.get!([__MODULE__, :method])

lib/pleroma/captcha/captcha_service.ex

@@ -1,12 +1,21 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Captcha.Service do
   @doc """
   Request new captcha from a captcha service.
 
   Returns:
 
-  Service-specific data for using the newly created captcha
+  Type/Name of the service, the token to identify the captcha,
+  the data of the answer and service-specific data to use the newly created captcha
   """
-  @callback new() :: map
+  @callback new() :: %{
+              type: atom(),
+              token: String.t(),
+              answer_data: any()
+            }
 
   @doc """
   Validated the provided captcha solution.
@@ -14,15 +23,15 @@ defmodule Pleroma.Captcha.Service do
   Arguments:
   * `token` the captcha is associated with
   * `captcha` solution of the captcha to validate
+  * `answer_data` is the data needed to validate the answer (presumably encrypted)
 
   Returns:
 
   `true` if captcha is valid, `false` if not
   """
-  @callback validate(token :: String.t(), captcha :: String.t()) :: boolean
+  @callback validate(
-
+              token :: String.t(),
-  @doc """
+              captcha :: String.t(),
-  This function is called periodically to clean up old captchas
+              answer_data :: any()
-  """
+            ) :: :ok | {:error, String.t()}
-  @callback cleanup() :: :ok
 end

lib/pleroma/captcha/kocaptcha.ex

@@ -1,11 +1,11 @@
-defmodule Pleroma.Captcha.Kocaptcha do
+# Pleroma: A lightweight social networking server
-  alias Calendar.DateTime
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
 
+defmodule Pleroma.Captcha.Kocaptcha do
   alias Pleroma.Captcha.Service
   @behaviour Service
 
-  @ets __MODULE__.Ets
-
   @impl Service
   def new() do
     endpoint = Pleroma.Config.get!([__MODULE__, :endpoint])
@@ -17,51 +17,21 @@ def new() do
       {:ok, res} ->
         json_resp = Poison.decode!(res.body)
 
-        token = json_resp["token"]
+        %{
-
+          type: :kocaptcha,
-        true =
+          token: json_resp["token"],
-          :ets.insert(
+          url: endpoint <> json_resp["url"],
-            @ets,
+          answer_data: json_resp["md5"]
-            {token, json_resp["md5"], DateTime.now_utc() |> DateTime.Format.unix()}
-          )
-
-        %{type: :kocaptcha, token: token, url: endpoint <> json_resp["url"]}
-    end
-  end
-
-  @impl Service
-  def validate(token, captcha) do
-    with false <- is_nil(captcha),
-         [{^token, saved_md5, _}] <- :ets.lookup(@ets, token),
-         true <- :crypto.hash(:md5, captcha) |> Base.encode16() == String.upcase(saved_md5) do
-      # Clear the saved value
-      :ets.delete(@ets, token)
-
-      true
-    else
-      _ -> false
-    end
-  end
-
-  @impl Service
-  def cleanup() do
-    seconds_retained = Pleroma.Config.get!([Pleroma.Captcha, :seconds_retained])
-    # If the time in ETS is less than current_time - seconds_retained, then the time has
-    # already passed
-    delete_after =
-      DateTime.subtract!(DateTime.now_utc(), seconds_retained) |> DateTime.Format.unix()
-
-    :ets.select_delete(
-      @ets,
-      [
-        {
-          {:_, :_, :"$1"},
-          [{:<, :"$1", {:const, delete_after}}],
-          [true]
         }
-      ]
+    end
-    )
+  end
 
-    :ok
+  @impl Service
+  def validate(_token, captcha, answer_data) do
+    # Here the token is unsed, because the unencrypted captcha answer is just passed to method
+    if not is_nil(captcha) and
+         :crypto.hash(:md5, captcha) |> Base.encode16() == String.upcase(answer_data),
+       do: :ok,
+       else: {:error, "Invalid CAPTCHA"}
   end
 end

lib/pleroma/config.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Config do
   defmodule Error do
     defexception [:message]

lib/pleroma/emails/mailer.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Mailer do
   use Swoosh.Mailer, otp_app: :pleroma
 end

lib/pleroma/emails/user_email.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.UserEmail do
   @moduledoc "User emails"
 
@@ -15,6 +19,7 @@ defp sender do
 
   defp recipient(email, nil), do: email
   defp recipient(email, name), do: {name, email}
+  defp recipient(%Pleroma.User{} = user), do: recipient(user.email, user.name)
 
   def password_reset_email(user, password_reset_token) when is_binary(password_reset_token) do
     password_reset_url =
@@ -32,7 +37,7 @@ def password_reset_email(user, password_reset_token) when is_binary(password_res
     """
 
     new()
-    |> to(recipient(user.email, user.name))
+    |> to(recipient(user))
     |> from(sender())
     |> subject("Password reset")
     |> html_body(html_body)
@@ -63,4 +68,26 @@ def user_invitation_email(
     |> subject("Invitation to #{instance_name()}")
     |> html_body(html_body)
   end
+
+  def account_confirmation_email(user) do
+    confirmation_url =
+      Router.Helpers.confirm_email_url(
+        Endpoint,
+        :confirm_email,
+        user.id,
+        to_string(user.info.confirmation_token)
+      )
+
+    html_body = """
+    <h3>Welcome to #{instance_name()}!</h3>
+    <p>Email confirmation is required to activate the account.</p>
+    <p>Click the following link to proceed: <a href="#{confirmation_url}">activate your account</a>.</p>
+    """
+
+    new()
+    |> to(recipient(user))
+    |> from(sender())
+    |> subject("#{instance_name()} account confirmation")
+    |> html_body(html_body)
+  end
 end

lib/pleroma/emoji.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Emoji do
   @moduledoc """
   The emojis are loaded from:

lib/pleroma/filter.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Filter do
   use Ecto.Schema
   import Ecto.{Changeset, Query}

lib/pleroma/formatter.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Formatter do
   alias Pleroma.User
   alias Pleroma.Web.MediaProxy
@@ -7,6 +11,9 @@ defmodule Pleroma.Formatter do
   @tag_regex ~r/((?<=[^&])|\A)(\#)(\w+)/u
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
 
+  # Modified from https://www.w3.org/TR/html5/forms.html#valid-e-mail-address
+  @mentions_regex ~r/@[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]*@?[a-zA-Z0-9_-](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/u
+
   def parse_tags(text, data \\ %{}) do
     Regex.scan(@tag_regex, text)
     |> Enum.map(fn ["#" <> tag = full_tag | _] -> {full_tag, String.downcase(tag)} end)
@@ -17,16 +24,15 @@ def parse_tags(text, data \\ %{}) do
         end).()
   end
 
+  @doc "Parses mentions text and returns list {nickname, user}."
+  @spec parse_mentions(binary()) :: list({binary(), User.t()})
   def parse_mentions(text) do
-    # Modified from https://www.w3.org/TR/html5/forms.html#valid-e-mail-address
+    Regex.scan(@mentions_regex, text)
-    regex =
-      ~r/@[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]*@?[a-zA-Z0-9_-](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/u
-
-    Regex.scan(regex, text)
     |> List.flatten()
     |> Enum.uniq()
-    |> Enum.map(fn "@" <> match = full_match ->
+    |> Enum.map(fn nickname ->
-      {full_match, User.get_cached_by_nickname(match)}
+      with nickname <- String.trim_leading(nickname, "@"),
+           do: {"@" <> nickname, User.get_cached_by_nickname(nickname)}
     end)
     |> Enum.filter(fn {_match, user} -> user end)
   end

lib/pleroma/gopher/server.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Gopher.Server do
   use GenServer
   require Logger

lib/pleroma/html.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.HTML do
   alias HtmlSanitizeEx.Scrubber
 
@@ -11,8 +15,11 @@ def get_scrubbers() do
   end
 
   def filter_tags(html, nil) do
-    get_scrubbers()
+    filter_tags(html, get_scrubbers())
-    |> Enum.reduce(html, fn scrubber, html ->
+  end
+
+  def filter_tags(html, scrubbers) when is_list(scrubbers) do
+    Enum.reduce(scrubbers, html, fn scrubber, html ->
       filter_tags(html, scrubber)
     end)
   end
@@ -20,6 +27,37 @@ def filter_tags(html, nil) do
   def filter_tags(html, scrubber), do: Scrubber.scrub(html, scrubber)
   def filter_tags(html), do: filter_tags(html, nil)
   def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags)
+
+  def get_cached_scrubbed_html_for_object(content, scrubbers, object, module) do
+    key = "#{module}#{generate_scrubber_signature(scrubbers)}|#{object.id}"
+    Cachex.fetch!(:scrubber_cache, key, fn _key -> ensure_scrubbed_html(content, scrubbers) end)
+  end
+
+  def get_cached_stripped_html_for_object(content, object, module) do
+    get_cached_scrubbed_html_for_object(
+      content,
+      HtmlSanitizeEx.Scrubber.StripTags,
+      object,
+      module
+    )
+  end
+
+  def ensure_scrubbed_html(
+        content,
+        scrubbers
+      ) do
+    {:commit, filter_tags(content, scrubbers)}
+  end
+
+  defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
+    generate_scrubber_signature([scrubber])
+  end
+
+  defp generate_scrubber_signature(scrubbers) do
+    Enum.reduce(scrubbers, "", fn scrubber, signature ->
+      "#{signature}#{to_string(scrubber)}"
+    end)
+  end
 end
 
 defmodule Pleroma.HTML.Scrubber.TwitterText do

lib/pleroma/http/connection.ex

@@ -1,10 +1,13 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.HTTP.Connection do
   @moduledoc """
   Connection for http-requests.
   """
 
   @hackney_options [
-    pool: :default,
     timeout: 10000,
     recv_timeout: 20000,
     follow_redirect: true

lib/pleroma/http/http.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.HTTP do
   @moduledoc """
 
@@ -6,6 +10,8 @@ defmodule Pleroma.HTTP do
   alias Pleroma.HTTP.Connection
   alias Pleroma.HTTP.RequestBuilder, as: Builder
 
+  @type t :: __MODULE__
+
   @doc """
   Builds and perform http request.
 
@@ -50,7 +56,6 @@ defp process_sni_options(options, url) do
   def process_request_options(options) do
     config = Application.get_env(:pleroma, :http, [])
     proxy = Keyword.get(config, :proxy_url, nil)
-    options = options ++ [adapter: [pool: :default]]
 
     case proxy do
       nil -> options

lib/pleroma/http/request_builder.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.HTTP.RequestBuilder do
   @moduledoc """
   Helper functions for building Tesla requests

lib/pleroma/list.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.List do
   use Ecto.Schema
   import Ecto.{Changeset, Query}

lib/pleroma/mime.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.MIME do
   @moduledoc """
   Returns the mime-type of a binary and optionally a normalized file-name.

lib/pleroma/notification.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Notification do
   use Ecto.Schema
   alias Pleroma.{User, Activity, Notification, Repo, Object}
@@ -76,9 +80,8 @@ def get(%{id: user_id} = _user, id) do
   end
 
   def clear(user) do
-    query = from(n in Notification, where: n.user_id == ^user.id)
+    from(n in Notification, where: n.user_id == ^user.id)
-
+    |> Repo.delete_all()
-    Repo.delete_all(query)
   end
 
   def dismiss(%{id: user_id} = _user, id) do
@@ -106,7 +109,12 @@ def create_notifications(_), do: {:ok, []}
   # TODO move to sql, too.
   def create_notification(%Activity{} = activity, %User{} = user) do
     unless User.blocks?(user, %{ap_id: activity.data["actor"]}) or
-             user.ap_id == activity.data["actor"] do
+             user.ap_id == activity.data["actor"] or
+             (activity.data["type"] == "Follow" and
+                Enum.any?(Notification.for_user(user), fn notif ->
+                  notif.activity.data["type"] == "Follow" and
+                    notif.activity.data["actor"] == activity.data["actor"]
+                end)) do
       notification = %Notification{user_id: user.id, activity: activity}
       {:ok, notification} = Repo.insert(notification)
       Pleroma.Web.Streamer.stream("user", notification)

lib/pleroma/object.ex

@@ -1,6 +1,10 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Object do
   use Ecto.Schema
-  alias Pleroma.{Repo, Object, User, Activity}
+  alias Pleroma.{Repo, Object, User, Activity, ObjectTombstone}
   import Ecto.{Query, Changeset}
 
   schema "objects" do
@@ -62,8 +66,25 @@ def context_mapping(context) do
     Object.change(%Object{}, %{data: %{"id" => context}})
   end
 
+  def make_tombstone(%Object{data: %{"id" => id, "type" => type}}, deleted \\ DateTime.utc_now()) do
+    %ObjectTombstone{
+      id: id,
+      formerType: type,
+      deleted: deleted
+    }
+    |> Map.from_struct()
+  end
+
+  def swap_object_with_tombstone(object) do
+    tombstone = make_tombstone(object)
+
+    object
+    |> Object.change(%{data: tombstone})
+    |> Repo.update()
+  end
+
   def delete(%Object{data: %{"id" => id}} = object) do
-    with Repo.delete(object),
+    with {:ok, _obj} = swap_object_with_tombstone(object),
          Repo.delete_all(Activity.all_non_create_by_object_ap_id_q(id)),
          {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
       {:ok, object}

lib/pleroma/object_tombstone.ex

@@ -0,0 +1,4 @@
+defmodule Pleroma.ObjectTombstone do
+  @enforce_keys [:id, :formerType, :deleted]
+  defstruct [:id, :formerType, :deleted, type: "Tombstone"]
+end

lib/pleroma/plugs/admin_secret_authentication_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do
   import Plug.Conn
   alias Pleroma.User

lib/pleroma/plugs/authentication_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.AuthenticationPlug do
   alias Comeonin.Pbkdf2
   import Plug.Conn

lib/pleroma/plugs/basic_auth_decoder_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.BasicAuthDecoderPlug do
   import Plug.Conn
 

lib/pleroma/plugs/digest.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.Plugs.DigestPlug do
   alias Plug.Conn
   require Logger

lib/pleroma/plugs/ensure_authenticated_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.EnsureAuthenticatedPlug do
   import Plug.Conn
   alias Pleroma.User

lib/pleroma/plugs/ensure_user_key_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.EnsureUserKeyPlug do
   import Plug.Conn
 

lib/pleroma/plugs/federating_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.FederatingPlug do
   import Plug.Conn
 

lib/pleroma/plugs/http_security_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.HTTPSecurityPlug do
   alias Pleroma.Config
   import Plug.Conn

lib/pleroma/plugs/http_signature.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do
   alias Pleroma.Web.HTTPSignatures
   alias Pleroma.Web.ActivityPub.Utils

lib/pleroma/plugs/instance_static.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.InstanceStatic do
   @moduledoc """
   This is a shim to call `Plug.Static` but with runtime `from` configuration.

lib/pleroma/plugs/legacy_authentication_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.LegacyAuthenticationPlug do
   import Plug.Conn
   alias Pleroma.User

lib/pleroma/plugs/oauth_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.OAuthPlug do
   import Plug.Conn
   import Ecto.Query

lib/pleroma/plugs/session_authentication_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.SessionAuthenticationPlug do
   import Plug.Conn
 

lib/pleroma/plugs/set_user_session_id_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.SetUserSessionIdPlug do
   import Plug.Conn
   alias Pleroma.User

lib/pleroma/plugs/uploaded_media.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.UploadedMedia do
   @moduledoc """
   """

lib/pleroma/plugs/user_enabled_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.UserEnabledPlug do
   import Plug.Conn
   alias Pleroma.User

lib/pleroma/plugs/user_fetcher_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.UserFetcherPlug do
   import Plug.Conn
   alias Pleroma.Repo

lib/pleroma/plugs/user_is_admin_plug.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Plugs.UserIsAdminPlug do
   import Plug.Conn
   alias Pleroma.User

lib/pleroma/repo.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Repo do
   use Ecto.Repo, otp_app: :pleroma
 

lib/pleroma/reverse_proxy.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.ReverseProxy do
   @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since if-none-match if-range range)
   @resp_cache_headers ~w(etag date last-modified cache-control)

lib/pleroma/stats.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Stats do
   import Ecto.Query
   alias Pleroma.{User, Repo}

lib/pleroma/upload.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Upload do
   @moduledoc """
   # Upload

lib/pleroma/upload/filter.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Upload.Filter do
   @moduledoc """
   Upload Filter behaviour

lib/pleroma/upload/filter/anonymize_filename.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Upload.Filter.AnonymizeFilename do
   @moduledoc """
   Replaces the original filename with a pre-defined text or randomly generated string.

lib/pleroma/upload/filter/dedupe.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Upload.Filter.Dedupe do
   @behaviour Pleroma.Upload.Filter
   alias Pleroma.Upload

lib/pleroma/upload/filter/mogrifun.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Upload.Filter.Mogrifun do
   @behaviour Pleroma.Upload.Filter
 

lib/pleroma/upload/filter/mogrify.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Upload.Filter.Mogrify do
   @behaviour Pleroma.Upload.Filter
 

lib/pleroma/uploaders/local.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Uploaders.Local do
   @behaviour Pleroma.Uploaders.Uploader
 

lib/pleroma/uploaders/mdii.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Uploaders.MDII do
   alias Pleroma.Config
 

lib/pleroma/uploaders/s3.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Uploaders.S3 do
   @behaviour Pleroma.Uploaders.Uploader
   require Logger

lib/pleroma/uploaders/swift/keystone.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Uploaders.Swift.Keystone do
   use HTTPoison.Base
 

lib/pleroma/uploaders/swift/swift.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Uploaders.Swift.Client do
   use HTTPoison.Base
 

lib/pleroma/uploaders/swift/uploader.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Uploaders.Swift do
   @behaviour Pleroma.Uploaders.Uploader
 

lib/pleroma/uploaders/uploader.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Uploaders.Uploader do
   @moduledoc """
   Defines the contract to put and get an uploaded file to any backend.

lib/pleroma/user.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.User do
   use Ecto.Schema
 
@@ -9,6 +13,8 @@ defmodule Pleroma.User do
   alias Pleroma.Web.{OStatus, Websub, OAuth}
   alias Pleroma.Web.ActivityPub.{Utils, ActivityPub}
 
+  require Logger
+
   @type t :: %__MODULE__{}
 
   @email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/
@@ -38,6 +44,29 @@ defmodule Pleroma.User do
     timestamps()
   end
 
+  def auth_active?(%User{local: false}), do: true
+
+  def auth_active?(%User{info: %User.Info{confirmation_pending: false}}), do: true
+
+  def auth_active?(%User{info: %User.Info{confirmation_pending: true}}),
+    do: !Pleroma.Config.get([:instance, :account_activation_required])
+
+  def auth_active?(_), do: false
+
+  def visible_for?(user, for_user \\ nil)
+
+  def visible_for?(%User{id: user_id}, %User{id: for_id}) when user_id == for_id, do: true
+
+  def visible_for?(%User{} = user, for_user) do
+    auth_active?(user) || superuser?(for_user)
+  end
+
+  def visible_for?(_, _), do: false
+
+  def superuser?(%User{local: true, info: %User.Info{is_admin: true}}), do: true
+  def superuser?(%User{local: true, info: %User.Info{is_moderator: true}}), do: true
+  def superuser?(_), do: false
+
   def avatar_url(user) do
     case user.avatar do
       %{"url" => [%{"href" => href} | _]} -> href
@@ -78,6 +107,7 @@ def user_info(%User{} = user) do
       note_count: user.info.note_count,
       follower_count: user.info.follower_count,
       locked: user.info.locked,
+      confirmation_pending: user.info.confirmation_pending,
       default_scope: user.info.default_scope
     }
   end
@@ -168,7 +198,16 @@ def reset_password(user, data) do
     update_and_set_cache(password_update_changeset(user, data))
   end
 
-  def register_changeset(struct, params \\ %{}) do
+  def register_changeset(struct, params \\ %{}, opts \\ []) do
+    confirmation_status =
+      if opts[:confirmed] || !Pleroma.Config.get([:instance, :account_activation_required]) do
+        :confirmed
+      else
+        :unconfirmed
+      end
+
+    info_change = User.Info.confirmation_changeset(%User.Info{}, confirmation_status)
+
     changeset =
       struct
       |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation])
@@ -176,11 +215,12 @@ def register_changeset(struct, params \\ %{}) do
       |> validate_confirmation(:password)
       |> unique_constraint(:email)
       |> unique_constraint(:nickname)
+      |> validate_exclusion(:nickname, Pleroma.Config.get([Pleroma.User, :restricted_nicknames]))
       |> validate_format(:nickname, local_nickname_regex())
       |> validate_format(:email, @email_regex)
       |> validate_length(:bio, max: 1000)
       |> validate_length(:name, min: 1, max: 100)
-      |> put_change(:info, %Pleroma.User.Info{})
+      |> put_change(:info, info_change)
 
     if changeset.valid? do
       hashed = Pbkdf2.hashpwsalt(changeset.changes[:password])
@@ -197,6 +237,39 @@ def register_changeset(struct, params \\ %{}) do
     end
   end
 
+  defp autofollow_users(user) do
+    candidates = Pleroma.Config.get([:instance, :autofollowed_nicknames])
+
+    autofollowed_users =
+      from(u in User,
+        where: u.local == true,
+        where: u.nickname in ^candidates
+      )
+      |> Repo.all()
+
+    follow_all(user, autofollowed_users)
+  end
+
+  @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)"
+  def register(%Ecto.Changeset{} = changeset) do
+    with {:ok, user} <- Repo.insert(changeset),
+         {:ok, _} <- try_send_confirmation_email(user),
+         {:ok, user} <- autofollow_users(user) do
+      {:ok, user}
+    end
+  end
+
+  def try_send_confirmation_email(%User{} = user) do
+    if user.info.confirmation_pending &&
+         Pleroma.Config.get([:instance, :account_activation_required]) do
+      user
+      |> Pleroma.UserEmail.account_confirmation_email()
+      |> Pleroma.Mailer.deliver()
+    else
+      {:ok, :noop}
+    end
+  end
+
   def needs_update?(%User{local: true}), do: false
 
   def needs_update?(%User{local: false, last_refreshed_at: nil}), do: true
@@ -231,6 +304,25 @@ def maybe_follow(%User{} = follower, %User{info: _info} = followed) do
     end
   end
 
+  @doc "A mass follow for local users. Ignores blocks and has no side effects"
+  @spec follow_all(User.t(), list(User.t())) :: {atom(), User.t()}
+  def follow_all(follower, followeds) do
+    following =
+      (follower.following ++ Enum.map(followeds, fn %{follower_address: fa} -> fa end))
+      |> Enum.uniq()
+
+    {:ok, follower} =
+      follower
+      |> follow_changeset(%{following: following})
+      |> update_and_set_cache
+
+    Enum.each(followeds, fn followed ->
+      update_follower_count(followed)
+    end)
+
+    {:ok, follower}
+  end
+
   def follow(%User{} = follower, %User{info: info} = followed) do
     user_config = Application.get_env(:pleroma, :user)
     deny_follow_blocked = Keyword.get(user_config, :deny_follow_blocked)
@@ -290,6 +382,24 @@ def following?(%User{} = follower, %User{} = followed) do
     Enum.member?(follower.following, followed.follower_address)
   end
 
+  def follow_import(%User{} = follower, followed_identifiers)
+      when is_list(followed_identifiers) do
+    Enum.map(
+      followed_identifiers,
+      fn followed_identifier ->
+        with %User{} = followed <- get_or_fetch(followed_identifier),
+             {:ok, follower} <- maybe_direct_follow(follower, followed),
+             {:ok, _} <- ActivityPub.follow(follower, followed) do
+          followed
+        else
+          err ->
+            Logger.debug("follow_import failed for #{followed_identifier} with: #{inspect(err)}")
+            err
+        end
+      end
+    )
+  end
+
   def locked?(%User{} = user) do
     user.info.locked || false
   end
@@ -302,6 +412,15 @@ def get_by_ap_id(ap_id) do
     Repo.get_by(User, ap_id: ap_id)
   end
 
+  # This is mostly an SPC migration fix. This guesses the user nickname (by taking the last part of the ap_id and the domain) and tries to get that user
+  def get_by_guessed_nickname(ap_id) do
+    domain = URI.parse(ap_id).host
+    name = List.last(String.split(ap_id, "/"))
+    nickname = "#{name}@#{domain}"
+
+    get_by_nickname(nickname)
+  end
+
   def update_and_set_cache(changeset) do
     with {:ok, user} <- Repo.update(changeset) do
       Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
@@ -339,7 +458,11 @@ def get_cached_by_nickname_or_id(nickname_or_id) do
   end
 
   def get_by_nickname(nickname) do
-    Repo.get_by(User, nickname: nickname)
+    Repo.get_by(User, nickname: nickname) ||
+      if Regex.match?(~r(@#{Pleroma.Web.Endpoint.host()})i, nickname) do
+        [local_nickname, _] = String.split(nickname, "@")
+        Repo.get_by(User, nickname: local_nickname)
+      end
   end
 
   def get_by_nickname_or_email(nickname_or_email) do
@@ -377,7 +500,7 @@ def get_or_fetch_by_nickname(nickname) do
     end
   end
 
-  def get_followers_query(%User{id: id, follower_address: follower_address}) do
+  def get_followers_query(%User{id: id, follower_address: follower_address}, nil) do
     from(
       u in User,
       where: fragment("? <@ ?", ^[follower_address], u.following),
@@ -385,13 +508,23 @@ def get_followers_query(%User{id: id, follower_address: follower_address}) do
     )
   end
 
-  def get_followers(user) do
+  def get_followers_query(user, page) do
-    q = get_followers_query(user)
+    from(
+      u in get_followers_query(user, nil),
+      limit: 20,
+      offset: ^((page - 1) * 20)
+    )
+  end
+
+  def get_followers_query(user), do: get_followers_query(user, nil)
+
+  def get_followers(user, page \\ nil) do
+    q = get_followers_query(user, page)
 
     {:ok, Repo.all(q)}
   end
 
-  def get_friends_query(%User{id: id, following: following}) do
+  def get_friends_query(%User{id: id, following: following}, nil) do
     from(
       u in User,
       where: u.follower_address in ^following,
@@ -399,8 +532,18 @@ def get_friends_query(%User{id: id, following: following}) do
     )
   end
 
-  def get_friends(user) do
+  def get_friends_query(user, page) do
-    q = get_friends_query(user)
+    from(
+      u in get_friends_query(user, nil),
+      limit: 20,
+      offset: ^((page - 1) * 20)
+    )
+  end
+
+  def get_friends_query(user), do: get_friends_query(user, nil)
+
+  def get_friends(user, page \\ nil) do
+    q = get_friends_query(user, page)
 
     {:ok, Repo.all(q)}
   end
@@ -435,6 +578,7 @@ def get_follow_requests(%User{} = user) do
       Enum.map(reqs, fn req -> req.actor end)
       |> Enum.uniq()
       |> Enum.map(fn ap_id -> get_by_ap_id(ap_id) end)
+      |> Enum.filter(fn u -> !is_nil(u) end)
       |> Enum.filter(fn u -> !following?(u, user) end)
 
     {:ok, users}
@@ -549,7 +693,7 @@ def search(query, resolve \\ false) do
         select_merge: %{
           search_distance:
             fragment(
-              "? <-> (? || ?)",
+              "? <-> (? || coalesce(?, ''))",
               ^query,
               u.nickname,
               u.name
@@ -568,6 +712,23 @@ def search(query, resolve \\ false) do
     Repo.all(q)
   end
 
+  def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do
+    Enum.map(
+      blocked_identifiers,
+      fn blocked_identifier ->
+        with %User{} = blocked <- get_or_fetch(blocked_identifier),
+             {:ok, blocker} <- block(blocker, blocked),
+             {:ok, _} <- ActivityPub.block(blocker, blocked) do
+          blocked
+        else
+          err ->
+            Logger.debug("blocks_import failed for #{blocked_identifier} with: #{inspect(err)}")
+            err
+        end
+      end
+    )
+  end
+
   def block(blocker, %User{ap_id: ap_id} = blocked) do
     # sever any follow relationships to prevent leaks per activitypub (Pleroma issue #213)
     blocker =
@@ -621,6 +782,9 @@ def blocks?(user, %{ap_id: ap_id}) do
       end)
   end
 
+  def blocked_users(user),
+    do: Repo.all(from(u in User, where: u.ap_id in ^user.info.blocks))
+
   def block_domain(user, domain) do
     info_cng =
       user.info
@@ -706,7 +870,9 @@ def html_filter_policy(%User{info: %{no_rich_text: true}}) do
     Pleroma.HTML.Scrubber.TwitterText
   end
 
-  def html_filter_policy(_), do: nil
+  @default_scrubbers Pleroma.Config.get([:markup, :scrub_policy])
+
+  def html_filter_policy(_), do: @default_scrubbers
 
   def get_or_fetch_by_ap_id(ap_id) do
     user = get_by_ap_id(ap_id)

lib/pleroma/user/info.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.User.Info do
   use Ecto.Schema
   import Ecto.Changeset
@@ -9,6 +13,8 @@ defmodule Pleroma.User.Info do
     field(:note_count, :integer, default: 0)
     field(:follower_count, :integer, default: 0)
     field(:locked, :boolean, default: false)
+    field(:confirmation_pending, :boolean, default: false)
+    field(:confirmation_token, :string, default: nil)
     field(:default_scope, :string, default: "public")
     field(:blocks, {:array, :string}, default: [])
     field(:domain_blocks, {:array, :string}, default: [])
@@ -25,6 +31,7 @@ defmodule Pleroma.User.Info do
     field(:hub, :string, default: nil)
     field(:salmon, :string, default: nil)
     field(:hide_network, :boolean, default: false)
+    field(:pinned_activities, {:array, :integer}, default: [])
 
     # Found in the wild
     # ap_id -> Where is this used?
@@ -141,6 +148,24 @@ def profile_update(info, params) do
     ])
   end
 
+  def confirmation_changeset(info, :confirmed) do
+    confirmation_changeset(info, %{
+      confirmation_pending: false,
+      confirmation_token: nil
+    })
+  end
+
+  def confirmation_changeset(info, :unconfirmed) do
+    confirmation_changeset(info, %{
+      confirmation_pending: true,
+      confirmation_token: :crypto.strong_rand_bytes(32) |> Base.url_encode64()
+    })
+  end
+
+  def confirmation_changeset(info, params) do
+    cast(info, params, [:confirmation_pending, :confirmation_token])
+  end
+
   def mastodon_profile_update(info, params) do
     info
     |> cast(params, [
@@ -172,4 +197,26 @@ def admin_api_update(info, params) do
       :is_admin
     ])
   end
+
+  def add_pinnned_activity(info, %Pleroma.Activity{id: id}) do
+    if id not in info.pinned_activities do
+      max_pinned_statuses = Pleroma.Config.get([:instance, :max_pinned_statuses], 0)
+      params = %{pinned_activities: info.pinned_activities ++ [id]}
+
+      info
+      |> cast(params, [:pinned_activities])
+      |> validate_length(:pinned_activities,
+        max: max_pinned_statuses,
+        message: "You have already pinned the maximum number of statuses"
+      )
+    else
+      change(info)
+    end
+  end
+
+  def remove_pinnned_activity(info, %Pleroma.Activity{id: id}) do
+    params = %{pinned_activities: List.delete(info.pinned_activities, id)}
+
+    cast(info, params, [:pinned_activities])
+  end
 end

lib/pleroma/user_invite_token.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.UserInviteToken do
   use Ecto.Schema
 

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.{Activity, Repo, Object, Upload, User, Notification}
   alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF}
@@ -52,10 +56,18 @@ defp check_actor_is_active(actor) do
     end
   end
 
+  defp check_remote_limit(%{"object" => %{"content" => content}}) do
+    limit = Pleroma.Config.get([:instance, :remote_limit])
+    String.length(content) <= limit
+  end
+
+  defp check_remote_limit(_), do: true
+
   def insert(map, local \\ true) when is_map(map) do
     with nil <- Activity.normalize(map),
          map <- lazy_put_activity_defaults(map),
          :ok <- check_actor_is_active(map["actor"]),
+         {_, true} <- {:remote_limit_error, check_remote_limit(map)},
          {:ok, map} <- MRF.filter(map),
          :ok <- insert_full_object(map) do
       {recipients, _, _} = get_recipients(map)
@@ -352,21 +364,18 @@ def fetch_public_activities(opts \\ %{}) do
 
   @valid_visibilities ~w[direct unlisted public private]
 
-  defp restrict_visibility(query, %{visibility: "direct"}) do
+  defp restrict_visibility(query, %{visibility: visibility})
-    public = "https://www.w3.org/ns/activitystreams#Public"
+       when visibility in @valid_visibilities do
+    query =
+      from(
+        a in query,
+        where:
+          fragment("activity_visibility(?, ?, ?) = ?", a.actor, a.recipients, a.data, ^visibility)
+      )
 
-    from(
+    Ecto.Adapters.SQL.to_sql(:all, Repo, query)
-      activity in query,
+
-      join: sender in User,
+    query
-      on: sender.ap_id == activity.actor,
-      # Are non-direct statuses with no to/cc possible?
-      where:
-        fragment(
-          "not (? && ?)",
-          [^public, sender.follower_address],
-          activity.recipients
-        )
-    )
   end
 
   defp restrict_visibility(_query, %{visibility: visibility})
@@ -382,6 +391,7 @@ def fetch_user_activities(user, reading_user, params \\ %{}) do
       |> Map.put("type", ["Create", "Announce"])
       |> Map.put("actor_id", user.ap_id)
       |> Map.put("whole_db", true)
+      |> Map.put("pinned_activity_ids", user.info.pinned_activities)
 
     recipients =
       if reading_user do
@@ -499,6 +509,12 @@ defp restrict_replies(query, %{"exclude_replies" => val}) when val == "true" or
 
   defp restrict_replies(query, _), do: query
 
+  defp restrict_reblogs(query, %{"exclude_reblogs" => val}) when val == "true" or val == "1" do
+    from(activity in query, where: fragment("?->>'type' != 'Announce'", activity.data))
+  end
+
+  defp restrict_reblogs(query, _), do: query
+
   # Only search through last 100_000 activities by default
   defp restrict_recent(query, %{"whole_db" => true}), do: query
 
@@ -534,6 +550,12 @@ defp restrict_unlisted(query) do
     )
   end
 
+  defp restrict_pinned(query, %{"pinned" => "true", "pinned_activity_ids" => ids}) do
+    from(activity in query, where: activity.id in ^ids)
+  end
+
+  defp restrict_pinned(query, _), do: query
+
   def fetch_activities_query(recipients, opts \\ %{}) do
     base_query =
       from(
@@ -557,6 +579,8 @@ def fetch_activities_query(recipients, opts \\ %{}) do
     |> restrict_media(opts)
     |> restrict_visibility(opts)
     |> restrict_replies(opts)
+    |> restrict_reblogs(opts)
+    |> restrict_pinned(opts)
   end
 
   def fetch_activities(recipients, opts \\ %{}) do
@@ -722,8 +746,7 @@ def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
         {"Content-Type", "application/activity+json"},
         {"signature", signature},
         {"digest", digest}
-      ],
+      ]
-      hackney: [pool: :default]
     )
   end
 
@@ -783,6 +806,10 @@ def fetch_and_contain_remote_object_from_id(id) do
     end
   end
 
+  def is_public?(%Object{data: %{"type" => "Tombstone"}}) do
+    false
+  end
+
   def is_public?(activity) do
     "https://www.w3.org/ns/activitystreams#Public" in (activity.data["to"] ++
                                                          (activity.data["cc"] || []))

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -1,10 +1,15 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   use Pleroma.Web, :controller
-  alias Pleroma.{User, Object}
+  alias Pleroma.{Activity, User, Object}
   alias Pleroma.Web.ActivityPub.{ObjectView, UserView}
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
 
   require Logger
@@ -49,6 +54,19 @@ def object(conn, %{"uuid" => uuid}) do
     end
   end
 
+  def activity(conn, %{"uuid" => uuid}) do
+    with ap_id <- o_status_url(conn, :activity, uuid),
+         %Activity{} = activity <- Activity.normalize(ap_id),
+         {_, true} <- {:public?, ActivityPub.is_public?(activity)} do
+      conn
+      |> put_resp_header("content-type", "application/activity+json")
+      |> json(ObjectView.render("object.json", %{object: activity}))
+    else
+      {:public?, false} ->
+        {:error, :not_found}
+    end
+  end
+
   def following(conn, %{"nickname" => nickname, "page" => page}) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
          {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
@@ -89,19 +107,15 @@ def followers(conn, %{"nickname" => nickname}) do
     end
   end
 
-  def outbox(conn, %{"nickname" => nickname, "max_id" => max_id}) do
+  def outbox(conn, %{"nickname" => nickname} = params) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
          {:ok, user} <- Pleroma.Web.WebFinger.ensure_keys_present(user) do
       conn
       |> put_resp_header("content-type", "application/activity+json")
-      |> json(UserView.render("outbox.json", %{user: user, max_id: max_id}))
+      |> json(UserView.render("outbox.json", %{user: user, max_id: params["max_id"]}))
     end
   end
 
-  def outbox(conn, %{"nickname" => nickname}) do
-    outbox(conn, %{"nickname" => nickname, "max_id" => nil})
-  end
-
   def inbox(%{assigns: %{valid_signature: true}} = conn, %{"nickname" => nickname} = params) do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
          true <- Utils.recipient_in_message(user.ap_id, params),
@@ -152,6 +166,79 @@ def relay(conn, _params) do
     end
   end
 
+  def read_inbox(%{assigns: %{user: user}} = conn, %{"nickname" => nickname} = params) do
+    if nickname == user.nickname do
+      conn
+      |> put_resp_header("content-type", "application/activity+json")
+      |> json(UserView.render("inbox.json", %{user: user, max_id: params["max_id"]}))
+    else
+      conn
+      |> put_status(:forbidden)
+      |> json("can't read inbox of #{nickname} as #{user.nickname}")
+    end
+  end
+
+  def handle_user_activity(user, %{"type" => "Create"} = params) do
+    object =
+      params["object"]
+      |> Map.merge(Map.take(params, ["to", "cc"]))
+      |> Map.put("attributedTo", user.ap_id())
+      |> Transmogrifier.fix_object()
+
+    ActivityPub.create(%{
+      to: params["to"],
+      actor: user,
+      context: object["context"],
+      object: object,
+      additional: Map.take(params, ["cc"])
+    })
+  end
+
+  def handle_user_activity(user, %{"type" => "Delete"} = params) do
+    with %Object{} = object <- Object.normalize(params["object"]),
+         true <- user.info.is_moderator || user.ap_id == object.data["actor"],
+         {:ok, delete} <- ActivityPub.delete(object) do
+      {:ok, delete}
+    else
+      _ -> {:error, "Can't delete object"}
+    end
+  end
+
+  def handle_user_activity(_, _) do
+    {:error, "Unhandled activity type"}
+  end
+
+  def update_outbox(
+        %{assigns: %{user: user}} = conn,
+        %{"nickname" => nickname} = params
+      ) do
+    if nickname == user.nickname do
+      actor = user.ap_id()
+
+      params =
+        params
+        |> Map.drop(["id"])
+        |> Map.put("actor", actor)
+        |> Transmogrifier.fix_addressing()
+
+      with {:ok, %Activity{} = activity} <- handle_user_activity(user, params) do
+        conn
+        |> put_status(:created)
+        |> put_resp_header("location", activity.data["id"])
+        |> json(activity.data)
+      else
+        {:error, message} ->
+          conn
+          |> put_status(:bad_request)
+          |> json(message)
+      end
+    else
+      conn
+      |> put_status(:forbidden)
+      |> json("can't update outbox of #{nickname} as #{user.nickname}")
+    end
+  end
+
   def errors(conn, {:error, :not_found}) do
     conn
     |> put_status(404)

lib/pleroma/web/activity_pub/mrf.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.MRF do
   @callback filter(Map.t()) :: {:ok | :reject, Map.t()}
 

lib/pleroma/web/activity_pub/mrf/drop_policy.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do
   require Logger
   @behaviour Pleroma.Web.ActivityPub.MRF

lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
   alias Pleroma.Object
 

lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex

@@ -0,0 +1,22 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  @impl true
+  def filter(%{"type" => "Create"} = object) do
+    threshold = Pleroma.Config.get([:mrf_hellthread, :threshold])
+    recipients = (object["to"] || []) ++ (object["cc"] || [])
+
+    if length(recipients) > threshold do
+      {:reject, nil}
+    else
+      {:ok, object}
+    end
+  end
+
+  @impl true
+  def filter(object), do: {:ok, object}
+end

lib/pleroma/web/activity_pub/mrf/noop_policy.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do
   @behaviour Pleroma.Web.ActivityPub.MRF
 

lib/pleroma/web/activity_pub/mrf/normalize_markup.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do
   alias Pleroma.HTML
 

lib/pleroma/web/activity_pub/mrf/reject_non_public.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do
   alias Pleroma.User
   @behaviour Pleroma.Web.ActivityPub.MRF

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
   alias Pleroma.User
   @behaviour Pleroma.Web.ActivityPub.MRF

lib/pleroma/web/activity_pub/mrf/user_allowlist.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do
   alias Pleroma.Config
 

lib/pleroma/web/activity_pub/relay.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.Relay do
   alias Pleroma.{User, Object, Activity}
   alias Pleroma.Web.ActivityPub.ActivityPub

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   @moduledoc """
   A module to handle coding from internal to wire ActivityPub and back.
@@ -69,8 +73,8 @@ def contain_origin_from_id(id, %{"id" => other_id} = _params) do
   def fix_object(object) do
     object
     |> fix_actor
-    |> fix_attachments
     |> fix_url
+    |> fix_attachments
     |> fix_context
     |> fix_in_reply_to
     |> fix_emoji
@@ -170,8 +174,14 @@ def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachm
     attachments =
       attachment
       |> Enum.map(fn data ->
-        url = [%{"type" => "Link", "mediaType" => data["mediaType"], "href" => data["url"]}]
+        media_type = data["mediaType"] || data["mimeType"]
-        Map.put(data, "url", url)
+        href = data["url"] || data["href"]
+
+        url = [%{"type" => "Link", "mediaType" => media_type, "href" => href}]
+
+        data
+        |> Map.put("mediaType", media_type)
+        |> Map.put("url", url)
       end)
 
     object
@@ -190,7 +200,22 @@ def fix_url(%{"url" => url} = object) when is_map(url) do
     |> Map.put("url", url["href"])
   end
 
-  def fix_url(%{"url" => url} = object) when is_list(url) do
+  def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do
+    first_element = Enum.at(url, 0)
+
+    link_element =
+      url
+      |> Enum.filter(fn x -> is_map(x) end)
+      |> Enum.filter(fn x -> x["mimeType"] == "text/html" end)
+      |> Enum.at(0)
+
+    object
+    |> Map.put("attachment", [first_element])
+    |> Map.put("url", link_element["href"])
+  end
+
+  def fix_url(%{"type" => object_type, "url" => url} = object)
+      when object_type != "Video" and is_list(url) do
     first_element = Enum.at(url, 0)
 
     url_string =

lib/pleroma/web/activity_pub/utils.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.Utils do
   alias Pleroma.{Repo, Web, Object, Activity, User, Notification}
   alias Pleroma.Web.Router.Helpers

lib/pleroma/web/activity_pub/views/object_view.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.ObjectView do
   use Pleroma.Web, :view
   alias Pleroma.{Object, Activity}

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.UserView do
   use Pleroma.Web, :view
   alias Pleroma.Web.Salmon
@@ -172,6 +176,53 @@ def render("outbox.json", %{user: user, max_id: max_qid}) do
     end
   end
 
+  def render("inbox.json", %{user: user, max_id: max_qid}) do
+    params = %{
+      "limit" => "10"
+    }
+
+    params =
+      if max_qid != nil do
+        Map.put(params, "max_id", max_qid)
+      else
+        params
+      end
+
+    activities = ActivityPub.fetch_activities([user.ap_id | user.following], params)
+
+    min_id = Enum.at(Enum.reverse(activities), 0).id
+    max_id = Enum.at(activities, 0).id
+
+    collection =
+      Enum.map(activities, fn act ->
+        {:ok, data} = Transmogrifier.prepare_outgoing(act.data)
+        data
+      end)
+
+    iri = "#{user.ap_id}/inbox"
+
+    page = %{
+      "id" => "#{iri}?max_id=#{max_id}",
+      "type" => "OrderedCollectionPage",
+      "partOf" => iri,
+      "totalItems" => -1,
+      "orderedItems" => collection,
+      "next" => "#{iri}?max_id=#{min_id - 1}"
+    }
+
+    if max_qid == nil do
+      %{
+        "id" => iri,
+        "type" => "OrderedCollection",
+        "totalItems" => -1,
+        "first" => page
+      }
+      |> Map.merge(Utils.make_json_ld_header())
+    else
+      page |> Map.merge(Utils.make_json_ld_header())
+    end
+  end
+
   def collection(collection, iri, page, show_items \\ true, total \\ nil) do
     offset = (page - 1) * 10
     items = Enum.slice(collection, offset, 10)

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -1,6 +1,10 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   use Pleroma.Web, :controller
-  alias Pleroma.{User, Repo}
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
 
   import Pleroma.Web.ControllerHelper, only: [json_response: 3]
@@ -10,13 +14,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   action_fallback(:errors)
 
   def user_delete(conn, %{"nickname" => nickname}) do
-    user = User.get_by_nickname(nickname)
+    User.get_by_nickname(nickname)
-
+    |> User.delete()
-    if user.local == true do
-      User.delete(user)
-    else
-      User.delete(user)
-    end
 
     conn
     |> json(nickname)
@@ -26,7 +25,7 @@ def user_create(
         conn,
         %{"nickname" => nickname, "email" => email, "password" => password}
       ) do
-    new_user = %{
+    user_data = %{
       nickname: nickname,
       name: nickname,
       email: email,
@@ -35,11 +34,11 @@ def user_create(
       bio: "."
     }
 
-    User.register_changeset(%User{}, new_user)
+    changeset = User.register_changeset(%User{}, user_data, confirmed: true)
-    |> Repo.insert!()
+    {:ok, user} = User.register(changeset)
 
     conn
-    |> json(new_user.nickname)
+    |> json(user.nickname)
   end
 
   def tag_users(conn, %{"nicknames" => nicknames, "tags" => tags}) do

lib/pleroma/web/channels/user_socket.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.UserSocket do
   use Phoenix.Socket
   alias Pleroma.User

lib/pleroma/web/chat_channel.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ChatChannel do
   use Phoenix.Channel
   alias Pleroma.Web.ChatChannel.ChatChannelState

lib/pleroma/web/common_api/common_api.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.{User, Repo, Activity, Object}
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -10,6 +14,7 @@ def delete(activity_id, user) do
     with %Activity{data: %{"object" => %{"id" => object_id}}} <- Repo.get(Activity, activity_id),
          %Object{} = object <- Object.normalize(object_id),
          true <- user.info.is_moderator || user.ap_id == object.data["actor"],
+         {:ok, _} <- unpin(activity_id, user),
          {:ok, delete} <- ActivityPub.delete(object) do
       {:ok, delete}
     end
@@ -98,7 +103,7 @@ def post(user, %{"status" => status} = data) do
              attachments,
              tags,
              get_content_type(data["content_type"]),
-             data["no_attachment_links"]
+             Enum.member?([true, "true"], data["no_attachment_links"])
            ),
          context <- make_context(inReplyTo),
          cw <- data["spoiler_text"],
@@ -120,7 +125,7 @@ def post(user, %{"status" => status} = data) do
            Map.put(
              object,
              "emoji",
-             Formatter.get_emoji(status)
+             (Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"]))
              |> Enum.reduce(%{}, fn {name, file}, acc ->
                Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
              end)
@@ -160,4 +165,48 @@ def update(user) do
       object: Pleroma.Web.ActivityPub.UserView.render("user.json", %{user: user})
     })
   end
+
+  def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do
+    with %Activity{
+           actor: ^user_ap_id,
+           data: %{
+             "type" => "Create",
+             "object" => %{
+               "to" => object_to,
+               "type" => "Note"
+             }
+           }
+         } = activity <- get_by_id_or_ap_id(id_or_ap_id),
+         true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"),
+         %{valid?: true} = info_changeset <-
+           Pleroma.User.Info.add_pinnned_activity(user.info, activity),
+         changeset <-
+           Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset),
+         {:ok, _user} <- User.update_and_set_cache(changeset) do
+      {:ok, activity}
+    else
+      %{errors: [pinned_activities: {err, _}]} ->
+        {:error, err}
+
+      _ ->
+        {:error, "Could not pin"}
+    end
+  end
+
+  def unpin(id_or_ap_id, user) do
+    with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
+         %{valid?: true} = info_changeset <-
+           Pleroma.User.Info.remove_pinnned_activity(user.info, activity),
+         changeset <-
+           Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset),
+         {:ok, _user} <- User.update_and_set_cache(changeset) do
+      {:ok, activity}
+    else
+      %{errors: [pinned_activities: {err, _}]} ->
+        {:error, err}
+
+      _ ->
+        {:error, "Could not unpin"}
+    end
+  end
 end

lib/pleroma/web/common_api/utils.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.CommonAPI.Utils do
   alias Calendar.Strftime
   alias Comeonin.Pbkdf2
@@ -132,7 +136,6 @@ def format_input(text, mentions, tags, "text/plain") do
   def format_input(text, mentions, _tags, "text/html") do
     text
     |> Formatter.html_escape("text/html")
-    |> String.replace(~r/\r?\n/, "<br>")
     |> (&{[], &1}).()
     |> Formatter.add_user_links(mentions)
     |> Formatter.finalize()
@@ -146,7 +149,6 @@ def format_input(text, mentions, tags, "text/markdown") do
     |> Formatter.mentions_escape(mentions)
     |> Earmark.as_html!()
     |> Formatter.html_escape("text/html")
-    |> String.replace(~r/\r?\n/, "")
     |> (&{[], &1}).()
     |> Formatter.add_user_links(mentions)
     |> Formatter.add_hashtag_links(tags)

lib/pleroma/web/controller_helper.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ControllerHelper do
   use Pleroma.Web, :controller
 

lib/pleroma/web/endpoint.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.Endpoint do
   use Phoenix.Endpoint, otp_app: :pleroma
 
@@ -21,7 +25,7 @@ defmodule Pleroma.Web.Endpoint do
     at: "/",
     from: :pleroma,
     only:
-      ~w(index.html static finmoji emoji packs sounds images instance sw.js favicon.png schemas)
+      ~w(index.html static finmoji emoji packs sounds images instance sw.js favicon.png schemas doc)
   )
 
   # Code reloading can be explicitly enabled under the

lib/pleroma/web/federator/federator.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.Federator do
   use GenServer
   alias Pleroma.User
@@ -13,7 +17,6 @@ defmodule Pleroma.Web.Federator do
 
   @websub Application.get_env(:pleroma, :websub)
   @ostatus Application.get_env(:pleroma, :ostatus)
-  @max_jobs 20
 
   def init(args) do
     {:ok, args}
@@ -164,7 +167,7 @@ def enqueue(type, payload, priority \\ 1) do
   end
 
   def maybe_start_job(running_jobs, queue) do
-    if :sets.size(running_jobs) < @max_jobs && queue != [] do
+    if :sets.size(running_jobs) < Pleroma.Config.get([__MODULE__, :max_jobs]) && queue != [] do
       {{type, payload}, queue} = queue_pop(queue)
       {:ok, pid} = Task.start(fn -> handle(type, payload) end)
       mref = Process.monitor(pid)

lib/pleroma/web/federator/retry_queue.ex

@@ -1,22 +1,34 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.Federator.RetryQueue do
   use GenServer
 
   require Logger
 
-  # initial timeout, 5 min
-  @initial_timeout 30_000
-  @max_retries 5
-
   def init(args) do
-    {:ok, args}
+    queue_table = :ets.new(:pleroma_retry_queue, [:bag, :protected])
+
+    {:ok, %{args | queue_table: queue_table, running_jobs: :sets.new()}}
   end
 
   def start_link() do
-    enabled = Pleroma.Config.get([:retry_queue, :enabled], false)
+    enabled =
+      if Mix.env() == :test, do: true, else: Pleroma.Config.get([__MODULE__, :enabled], false)
 
     if enabled do
       Logger.info("Starting retry queue")
-      GenServer.start_link(__MODULE__, %{delivered: 0, dropped: 0}, name: __MODULE__)
+
+      linkres =
+        GenServer.start_link(
+          __MODULE__,
+          %{delivered: 0, dropped: 0, queue_table: nil, running_jobs: nil},
+          name: __MODULE__
+        )
+
+      maybe_kickoff_timer()
+      linkres
     else
       Logger.info("Retry queue disabled")
       :ignore
@@ -27,24 +39,133 @@ def enqueue(data, transport, retries \\ 0) do
     GenServer.cast(__MODULE__, {:maybe_enqueue, data, transport, retries + 1})
   end
 
+  def get_stats() do
+    GenServer.call(__MODULE__, :get_stats)
+  end
+
+  def reset_stats() do
+    GenServer.call(__MODULE__, :reset_stats)
+  end
+
   def get_retry_params(retries) do
-    if retries > @max_retries do
+    if retries > Pleroma.Config.get([__MODULE__, :max_retries]) do
       {:drop, "Max retries reached"}
     else
       {:retry, growth_function(retries)}
     end
   end
 
-  def handle_cast({:maybe_enqueue, data, transport, retries}, %{dropped: drop_count} = state) do
+  def get_retry_timer_interval() do
+    Pleroma.Config.get([:retry_queue, :interval], 1000)
+  end
+
+  defp ets_count_expires(table, current_time) do
+    :ets.select_count(
+      table,
+      [
+        {
+          {:"$1", :"$2"},
+          [{:"=<", :"$1", {:const, current_time}}],
+          [true]
+        }
+      ]
+    )
+  end
+
+  defp ets_pop_n_expired(table, current_time, desired) do
+    {popped, _continuation} =
+      :ets.select(
+        table,
+        [
+          {
+            {:"$1", :"$2"},
+            [{:"=<", :"$1", {:const, current_time}}],
+            [:"$_"]
+          }
+        ],
+        desired
+      )
+
+    popped
+    |> Enum.each(fn e ->
+      :ets.delete_object(table, e)
+    end)
+
+    popped
+  end
+
+  def maybe_start_job(running_jobs, queue_table) do
+    # we don't want to hit the ets or the DateTime more times than we have to
+    # could optimize slightly further by not using the count, and instead grabbing
+    # up to N objects early...
+    current_time = DateTime.to_unix(DateTime.utc_now())
+    n_running_jobs = :sets.size(running_jobs)
+
+    if n_running_jobs < Pleroma.Config.get([__MODULE__, :max_jobs]) do
+      n_ready_jobs = ets_count_expires(queue_table, current_time)
+
+      if n_ready_jobs > 0 do
+        # figure out how many we could start
+        available_job_slots = Pleroma.Config.get([__MODULE__, :max_jobs]) - n_running_jobs
+        start_n_jobs(running_jobs, queue_table, current_time, available_job_slots)
+      else
+        running_jobs
+      end
+    else
+      running_jobs
+    end
+  end
+
+  defp start_n_jobs(running_jobs, _queue_table, _current_time, 0) do
+    running_jobs
+  end
+
+  defp start_n_jobs(running_jobs, queue_table, current_time, available_job_slots)
+       when available_job_slots > 0 do
+    candidates = ets_pop_n_expired(queue_table, current_time, available_job_slots)
+
+    candidates
+    |> List.foldl(running_jobs, fn {_, e}, rj ->
+      {:ok, pid} = Task.start(fn -> worker(e) end)
+      mref = Process.monitor(pid)
+      :sets.add_element(mref, rj)
+    end)
+  end
+
+  def worker({:send, data, transport, retries}) do
+    case transport.publish_one(data) do
+      {:ok, _} ->
+        GenServer.cast(__MODULE__, :inc_delivered)
+        :delivered
+
+      {:error, _reason} ->
+        enqueue(data, transport, retries)
+        :retry
+    end
+  end
+
+  def handle_call(:get_stats, _from, %{delivered: delivery_count, dropped: drop_count} = state) do
+    {:reply, %{delivered: delivery_count, dropped: drop_count}, state}
+  end
+
+  def handle_call(:reset_stats, _from, %{delivered: delivery_count, dropped: drop_count} = state) do
+    {:reply, %{delivered: delivery_count, dropped: drop_count},
+     %{state | delivered: 0, dropped: 0}}
+  end
+
+  def handle_cast(:reset_stats, state) do
+    {:noreply, %{state | delivered: 0, dropped: 0}}
+  end
+
+  def handle_cast(
+        {:maybe_enqueue, data, transport, retries},
+        %{dropped: drop_count, queue_table: queue_table, running_jobs: running_jobs} = state
+      ) do
     case get_retry_params(retries) do
       {:retry, timeout} ->
-        Process.send_after(
+        :ets.insert(queue_table, {timeout, {:send, data, transport, retries}})
-          __MODULE__,
+        running_jobs = maybe_start_job(running_jobs, queue_table)
-          {:send, data, transport, retries},
+        {:noreply, %{state | running_jobs: running_jobs}}
-          timeout
-        )
-
-        {:noreply, state}
 
       {:drop, message} ->
         Logger.debug(message)
@@ -52,6 +173,20 @@ def handle_cast({:maybe_enqueue, data, transport, retries}, %{dropped: drop_coun
     end
   end
 
+  def handle_cast(:kickoff_timer, state) do
+    retry_interval = get_retry_timer_interval()
+    Process.send_after(__MODULE__, :retry_timer_run, retry_interval)
+    {:noreply, state}
+  end
+
+  def handle_cast(:inc_delivered, %{delivered: delivery_count} = state) do
+    {:noreply, %{state | delivered: delivery_count + 1}}
+  end
+
+  def handle_cast(:inc_dropped, %{dropped: drop_count} = state) do
+    {:noreply, %{state | dropped: drop_count + 1}}
+  end
+
   def handle_info({:send, data, transport, retries}, %{delivered: delivery_count} = state) do
     case transport.publish_one(data) do
       {:ok, _} ->
@@ -63,12 +198,40 @@ def handle_info({:send, data, transport, retries}, %{delivered: delivery_count}
     end
   end
 
+  def handle_info(
+        :retry_timer_run,
+        %{queue_table: queue_table, running_jobs: running_jobs} = state
+      ) do
+    maybe_kickoff_timer()
+    running_jobs = maybe_start_job(running_jobs, queue_table)
+    {:noreply, %{state | running_jobs: running_jobs}}
+  end
+
+  def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
+    %{running_jobs: running_jobs, queue_table: queue_table} = state
+    running_jobs = :sets.del_element(ref, running_jobs)
+    running_jobs = maybe_start_job(running_jobs, queue_table)
+    {:noreply, %{state | running_jobs: running_jobs}}
+  end
+
   def handle_info(unknown, state) do
     Logger.debug("RetryQueue: don't know what to do with #{inspect(unknown)}, ignoring")
     {:noreply, state}
   end
 
-  defp growth_function(retries) do
+  if Mix.env() == :test do
-    round(@initial_timeout * :math.pow(retries, 3))
+    defp growth_function(_retries) do
+      _shutit = Pleroma.Config.get([__MODULE__, :initial_timeout])
+      DateTime.to_unix(DateTime.utc_now()) - 1
+    end
+  else
+    defp growth_function(retries) do
+      round(Pleroma.Config.get([__MODULE__, :initial_timeout]) * :math.pow(retries, 3)) +
+        DateTime.to_unix(DateTime.utc_now())
+    end
+  end
+
+  defp maybe_kickoff_timer() do
+    GenServer.cast(__MODULE__, :kickoff_timer)
   end
 end

lib/pleroma/web/gettext.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.Gettext do
   @moduledoc """
   A module providing Internationalization with a gettext-based API.

lib/pleroma/web/http_signatures/http_signatures.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 # https://tools.ietf.org/html/draft-cavage-http-signatures-08
 defmodule Pleroma.Web.HTTPSignatures do
   alias Pleroma.User