Merge branch 'develop' into feature/store-statuses-data-inside-flag

.gitlab-ci.yml

@@ -15,6 +15,7 @@ cache:
 stages:
   - build
   - test
+  - benchmark
   - deploy
   - release
 
@@ -28,6 +29,36 @@ build:
   - mix deps.get
   - mix compile --force
 
+docs-build:
+  stage: build
+  only:
+  - master@pleroma/pleroma
+  - develop@pleroma/pleroma
+  variables:
+    MIX_ENV: dev
+    PLEROMA_BUILD_ENV: prod
+  script:
+    - mix deps.get
+    - mix compile
+    - mix docs
+  artifacts:
+    paths:
+      - priv/static/doc
+
+benchmark:
+  stage: benchmark
+  variables:
+    MIX_ENV: benchmark
+  services:
+  - name: lainsoykaf/postgres-with-rum
+    alias: postgres
+    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+  script:
+    - mix deps.get
+    - mix ecto.create
+    - mix ecto.migrate
+    - mix pleroma.load_testing
+
 unit-testing:
   stage: test
   services:

CHANGELOG.md

@@ -4,8 +4,39 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
+### Removed
+- **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media`
+- **Breaking**: OStatus protocol support
+
+### Changed
+- **Breaking:** Elixir >=1.8 is now required (was >= 1.7)
+- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings)
+- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler
+- Enabled `:instance, extended_nickname_format` in the default config
+- Add `rel="ugc"` to all links in statuses, to prevent SEO spam
+- Extract RSS functionality from OStatus
+- MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities
+<details>
+  <summary>API Changes</summary>
+
+- **Breaking:** Admin API: Return link alongside with token on password reset
+- **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string.
+- Admin API: Return `total` when querying for reports
+- Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`)
+- Admin API: Return link alongside with token on password reset
+- Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`)
+- Mastodon API: `pleroma.thread_muted` to the Status entity
+- Mastodon API: Mark the direct conversation as read for the author when they send a new direct message
+</details>
+
 ### Added
 - Refreshing poll results for remote polls
+- Authentication: Added rate limit for password-authorized actions / login existence checks
+- Mix task to re-count statuses for all users (`mix pleroma.count_statuses`)
+- Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache).
+<details>
+  <summary>API Changes</summary>
+
 - Job queue stats to the healthcheck page
 - Admin API: Add ability to require password reset
 - Mastodon API: Account entities now include `follow_requests_count` (planned Mastodon 3.x addition)
@@ -14,10 +45,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Add `upload_limit`, `avatar_upload_limit`, `background_upload_limit`, and `banner_upload_limit` to `/api/v1/instance`
 - Mastodon API: Add `pleroma.unread_conversation_count` to the Account entity
 - OAuth: support for hierarchical permissions / [Mastodon 2.4.3 OAuth permissions](https://docs.joinmastodon.org/api/permissions/)
-- Authentication: Added rate limit for password-authorized actions / login existence checks
 - Metadata Link: Atom syndication Feed
-- Mix task to re-count statuses for all users (`mix pleroma.count_statuses`)
 - Mastodon API: Add `exclude_visibilities` parameter to the timeline and notification endpoints
+- Admin API: `/users/:nickname/toggle_activation` endpoint is now deprecated in favor of: `/users/activate`, `/users/deactivate`, both accept `nicknames` array
+- Admin API: `POST/DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` are deprecated in favor of: `POST/DELETE /api/pleroma/admin/users/permission_group/:permission_group` (both accept `nicknames` array), `DELETE /api/pleroma/admin/users` (`nickname` query param or `nickname` sent in JSON body) is deprecated in favor of: `DELETE /api/pleroma/admin/users` (`nicknames` query array param or `nicknames` sent in JSON body).
+- Admin API: Add `GET /api/pleroma/admin/relay` endpoint - lists all followed relays
+- Pleroma API: `POST /api/v1/pleroma/conversations/read` to mark all conversations as read
+- Mastodon API: Add `/api/v1/markers` for managing timeline read markers
 
 ### Changed
 - **Breaking:** Elixir >=1.8 is now required (was >= 1.7)
@@ -30,16 +64,39 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities
 - OStatus: Extract RSS functionality
 - Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`)
+- Mastodon API: Mark the direct conversation as read for the author when they send a new direct message
+- Deprecated `User.Info` embedded schema (fields moved to `User`)
 - Store status data inside Flag activity
+</details>
 
 ### Fixed
+- Report emails now include functional links to profiles of remote user accounts
+<details>
+  <summary>API Changes</summary>
+
 - Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`)
 - Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname`
-- Added `:instance, extended_nickname_format` setting to the default config
+</details>
-- Report emails now include functional links to profiles of remote user accounts
 
-## [1.1.0] - 2019-??-??
+## [1.1.2] - 2019-10-18
-**Breaking:** The stable branch has been changed from `master` to `stable`, `master` now points to `release/1.0`
+### Fixed
+- `pleroma_ctl` trying to connect to a running instance when generating the config, which of course doesn't exist.
+
+## [1.1.1] - 2019-10-18
+### Fixed
+- One of the migrations between 1.0.0 and 1.1.0 wiping user info of the relay user because of unexpected behavior of postgresql's `jsonb_set`, resulting in inability to post in the default configuration. If you were affected, please run the following query in postgres console, the relay user will be recreated automatically:
+```
+delete from users where ap_id = 'https://your.instance.hostname/relay';
+```
+- Bad user search matches
+
+## [1.1.0] - 2019-10-14
+**Breaking:** The stable branch has been changed from `master` to `stable`. If you want to keep using 1.0, the `release/1.0` branch will receive security updates for 6 months after 1.1 release.
+
+**OTP Note:** `pleroma_ctl` in 1.0 defaults to `master` and doesn't support specifying arbitrary branches, making `./pleroma_ctl update` fail. To fix this, fetch a version of `pleroma_ctl` from 1.1 using the command below and proceed with the update normally:
+```
+curl -Lo ./bin/pleroma_ctl 'https://git.pleroma.social/pleroma/pleroma/raw/develop/rel/files/bin/pleroma_ctl'
+```
 ### Security
 - Mastodon API: respect post privacy in `/api/v1/statuses/:id/{favourited,reblogged}_by`
 
@@ -47,16 +104,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - **Breaking:** GNU Social API with Qvitter extensions support
 - Emoji: Remove longfox emojis.
 - Remove `Reply-To` header from report emails for admins.
+- ActivityPub: The `/objects/:uuid/likes` endpoint.
 
 ### Changed
 - **Breaking:** Configuration: A setting to explicitly disable the mailer was added, defaulting to true, if you are using a mailer add `config :pleroma, Pleroma.Emails.Mailer, enabled: true` to your config
 - **Breaking:** Configuration: `/media/` is now removed when `base_url` is configured, append `/media/` to your `base_url` config to keep the old behaviour if desired
 - **Breaking:** `/api/pleroma/notifications/read` is moved to `/api/v1/pleroma/notifications/read` and now supports `max_id` and responds with Mastodon API entities.
-- **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string.
 - Configuration: added `config/description.exs`, from which `docs/config.md` is generated
 - Configuration: OpenGraph and TwitterCard providers enabled by default
 - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text
-- Mastodon API: `pleroma.thread_muted` key in the Status entity
 - Federation: Return 403 errors when trying to request pages from a user's follower/following collections if they have `hide_followers`/`hide_follows` set
 - NodeInfo: Return `skipThreadContainment` in `metadata` for the `skip_thread_containment` option
 - NodeInfo: Return `mailerEnabled` in `metadata`
@@ -65,7 +121,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - AdminAPI: Add "godmode" while fetching user statuses (i.e. admin can see private statuses)
 - Improve digest email template
 – Pagination: (optional) return `total` alongside with `items` when paginating
-- Add `rel="ugc"` to all links in statuses, to prevent SEO spam
+- The `Pleroma.FlakeId` module has been replaced with the `flake_id` library.
 
 ### Fixed
 - Following from Osada
@@ -76,21 +132,28 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Misskey's endless polls being unable to render
 - Mastodon API: Embedded relationships not being properly rendered in the Account entity of Status entity
 - Mastodon API: Notifications endpoint crashing if one notification failed to render
+- Mastodon API: `exclude_replies` is correctly handled again.
 - Mastodon API: Add `account_id`, `type`, `offset`, and `limit` to search API (`/api/v1/search` and `/api/v2/search`)
 - Mastodon API, streaming: Fix filtering of notifications based on blocks/mutes/thread mutes
-- ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set
+- Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`)
-- Existing user id not being preserved on insert conflict
+- Mastodon API: Ensure the `account` field is not empty when rendering Notification entities.
+- Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname`
+- Mastodon API: Blocks are now treated consistently between the Streaming API and the Timeline APIs
 - Rich Media: Parser failing when no TTL can be found by image TTL setters
 - Rich Media: The crawled URL is now spliced into the rich media data.
 - ActivityPub S2S: sharedInbox usage has been mostly aligned with the rules in the AP specification.
-- Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected.
+- ActivityPub C2S: follower/following collection pages being inaccessible even when authentifucated if `hide_followers`/ `hide_follows` was set
-- Report email not being sent to admins when the reporter is a remote user
-- Reverse Proxy limiting `max_body_length` was incorrectly defined and only checked `Content-Length` headers which may not be sufficient in some circumstances
 - ActivityPub: Deactivated user deletion
 - ActivityPub: Fix `/users/:nickname/inbox` crashing without an authenticated user
 - MRF: fix ability to follow a relay when AntiFollowbotPolicy was enabled
-- Mastodon API: Blocks are now treated consistently between the Streaming API and the Timeline APIs
+- ActivityPub: Correct addressing of Undo.
-- Mastodon API: `exclude_replies` is correctly handled again.
+- ActivityPub: Correct addressing of profile update activities.
+- ActivityPub: Polls are now refreshed when necessary.
+- Report emails now include functional links to profiles of remote user accounts
+- Existing user id not being preserved on insert conflict
+- Pleroma.Upload base_url was not automatically whitelisted by MediaProxy. Now your custom CDN or file hosting will be accessed directly as expected.
+- Report email not being sent to admins when the reporter is a remote user
+- Reverse Proxy limiting `max_body_length` was incorrectly defined and only checked `Content-Length` headers which may not be sufficient in some circumstances
 
 ### Added
 - Expiring/ephemeral activites. All activities can have expires_at value set, which controls when they should be deleted automatically.
@@ -104,6 +167,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses)
 - Mastodon API, streaming: Add support for passing the token in the `Sec-WebSocket-Protocol` header
 - Mastodon API, extension: Ability to reset avatar, profile banner, and background
+- Mastodon API: Add support for `fields_attributes` API parameter (setting custom fields)
 - Mastodon API: Add support for categories for custom emojis by reusing the group feature. <https://github.com/tootsuite/mastodon/pull/11196>
 - Mastodon API: Add support for muting/unmuting notifications
 - Mastodon API: Add support for the `blocked_by` attribute in the relationship API (`GET /api/v1/accounts/relationships`). <https://github.com/tootsuite/mastodon/pull/10373>
@@ -112,7 +176,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: added `/auth/password` endpoint for password reset with rate limit.
 - Mastodon API: /api/v1/accounts/:id/statuses now supports nicknames or user id
 - Mastodon API: Improve support for the user profile custom fields
-- Mastodon API: follower/following counters are nullified when `hide_follows`/`hide_followers` and `hide_follows_count`/`hide_followers_count` are set
+- Mastodon API: Add support for `fields_attributes` API parameter (setting custom fields)
+- Mastodon API: Added an endpoint to get multiple statuses by IDs (`GET /api/v1/statuses/?ids[]=1&ids[]=2`)
 - Admin API: Return users' tags when querying reports
 - Admin API: Return avatar and display name when querying users
 - Admin API: Allow querying user by ID
@@ -130,11 +195,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Pleroma API: Add `/api/v1/pleroma/accounts/confirmation_resend?email=<email>` for resending account confirmation.
 - Pleroma API: Email change endpoint.
 - Admin API: Added moderation log
-- Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache).
 - Web response cache (currently, enabled for ActivityPub)
-- Mastodon API: Added an endpoint to get multiple statuses by IDs (`GET /api/v1/statuses/?ids[]=1&ids[]=2`)
-- ActivityPub: Add ActivityPub actor's `discoverable` parameter.
-- Admin API: Added moderation log filters (user/start date/end date/search/pagination)
 - Reverse Proxy: Do not retry failed requests to limit pressure on the peer
 
 ### Changed

README.md

@@ -25,7 +25,7 @@ While we don’t provide docker files, other people have written very good ones.
 
 ### Dependencies
 
-* Postgresql version 9.6 or newer
+* Postgresql version 9.6 or newer, including the contrib modules
 * Elixir version 1.7 or newer. If your distribution only has an old version available, check [Elixir’s install page](https://elixir-lang.org/install.html) or use a tool like [asdf](https://github.com/asdf-vm/asdf).
 * Build-essential tools
 
@@ -71,7 +71,7 @@ This is useful for running Pleroma inside Tor or I2P.
 
 ## Customization and contribution
 
-The [Pleroma Documentation](https://docs-develop.pleroma.social/readme.html) offers manuals and guides on how to further customize your instance to your liking and how you can contribute to the project.
+The [Pleroma Documentation](https://docs-develop.pleroma.social) offers manuals and guides on how to further customize your instance to your liking and how you can contribute to the project.
 
 ## Troubleshooting
 

benchmarks/load_testing/fetcher.ex

@@ -0,0 +1,229 @@
+defmodule Pleroma.LoadTesting.Fetcher do
+  use Pleroma.LoadTesting.Helper
+
+  def fetch_user(user) do
+    Benchee.run(%{
+      "By id" => fn -> Repo.get_by(User, id: user.id) end,
+      "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end,
+      "By email" => fn -> Repo.get_by(User, email: user.email) end,
+      "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end
+    })
+  end
+
+  def query_timelines(user) do
+    home_timeline_params = %{
+      "count" => 20,
+      "with_muted" => true,
+      "type" => ["Create", "Announce"],
+      "blocking_user" => user,
+      "muting_user" => user,
+      "user" => user
+    }
+
+    mastodon_public_timeline_params = %{
+      "count" => 20,
+      "local_only" => true,
+      "only_media" => "false",
+      "type" => ["Create", "Announce"],
+      "with_muted" => "true",
+      "blocking_user" => user,
+      "muting_user" => user
+    }
+
+    mastodon_federated_timeline_params = %{
+      "count" => 20,
+      "only_media" => "false",
+      "type" => ["Create", "Announce"],
+      "with_muted" => "true",
+      "blocking_user" => user,
+      "muting_user" => user
+    }
+
+    Benchee.run(%{
+      "User home timeline" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
+          [user.ap_id | user.following],
+          home_timeline_params
+        )
+      end,
+      "User mastodon public timeline" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+          mastodon_public_timeline_params
+        )
+      end,
+      "User mastodon federated public timeline" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+          mastodon_federated_timeline_params
+        )
+      end
+    })
+
+    home_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
+        [user.ap_id | user.following],
+        home_timeline_params
+      )
+
+    public_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
+
+    public_federated_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+        mastodon_federated_timeline_params
+      )
+
+    Benchee.run(%{
+      "Rendering home timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: home_activities,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering public timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: public_activities,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering public federated timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: public_federated_activities,
+          for: user,
+          as: :activity
+        })
+      end
+    })
+  end
+
+  def query_notifications(user) do
+    without_muted_params = %{"count" => "20", "with_muted" => "false"}
+    with_muted_params = %{"count" => "20", "with_muted" => "true"}
+
+    Benchee.run(%{
+      "Notifications without muted" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
+      end,
+      "Notifications with muted" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
+      end
+    })
+
+    without_muted_notifications =
+      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
+
+    with_muted_notifications =
+      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
+
+    Benchee.run(%{
+      "Render notifications without muted" => fn ->
+        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
+          notifications: without_muted_notifications,
+          for: user
+        })
+      end,
+      "Render notifications with muted" => fn ->
+        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
+          notifications: with_muted_notifications,
+          for: user
+        })
+      end
+    })
+  end
+
+  def query_dms(user) do
+    params = %{
+      "count" => "20",
+      "with_muted" => "true",
+      "type" => "Create",
+      "blocking_user" => user,
+      "user" => user,
+      visibility: "direct"
+    }
+
+    Benchee.run(%{
+      "Direct messages with muted" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+        |> Pleroma.Pagination.fetch_paginated(params)
+      end,
+      "Direct messages without muted" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+        |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
+      end
+    })
+
+    dms_with_muted =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+      |> Pleroma.Pagination.fetch_paginated(params)
+
+    dms_without_muted =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+      |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
+
+    Benchee.run(%{
+      "Rendering dms with muted" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: dms_with_muted,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering dms without muted" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: dms_without_muted,
+          for: user,
+          as: :activity
+        })
+      end
+    })
+  end
+
+  def query_long_thread(user, activity) do
+    Benchee.run(%{
+      "Fetch main post" => fn ->
+        Pleroma.Activity.get_by_id_with_object(activity.id)
+      end,
+      "Fetch context of main post" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
+          activity.data["context"],
+          %{
+            "blocking_user" => user,
+            "user" => user,
+            "exclude_id" => activity.id
+          }
+        )
+      end
+    })
+
+    activity = Pleroma.Activity.get_by_id_with_object(activity.id)
+
+    context =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
+        activity.data["context"],
+        %{
+          "blocking_user" => user,
+          "user" => user,
+          "exclude_id" => activity.id
+        }
+      )
+
+    Benchee.run(%{
+      "Render status" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{
+          activity: activity,
+          for: user
+        })
+      end,
+      "Render context" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render(
+          "index.json",
+          for: user,
+          activities: context,
+          as: :activity
+        )
+        |> Enum.reverse()
+      end
+    })
+  end
+end

benchmarks/load_testing/generator.ex

@@ -0,0 +1,352 @@
+defmodule Pleroma.LoadTesting.Generator do
+  use Pleroma.LoadTesting.Helper
+  alias Pleroma.Web.CommonAPI
+
+  def generate_users(opts) do
+    IO.puts("Starting generating #{opts[:users_max]} users...")
+    {time, _} = :timer.tc(fn -> do_generate_users(opts) end)
+
+    IO.puts("Inserting users take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_users(opts) do
+    max = Keyword.get(opts, :users_max)
+
+    Task.async_stream(
+      1..max,
+      &generate_user_data(&1),
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Enum.to_list()
+  end
+
+  defp generate_user_data(i) do
+    remote = Enum.random([true, false])
+
+    user = %User{
+      name: "Test テスト User #{i}",
+      email: "user#{i}@example.com",
+      nickname: "nick#{i}",
+      password_hash:
+        "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg",
+      bio: "Tester Number #{i}",
+      info: %{},
+      local: remote
+    }
+
+    user_urls =
+      if remote do
+        base_url =
+          Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"])
+
+        ap_id = "#{base_url}/users/#{user.nickname}"
+
+        %{
+          ap_id: ap_id,
+          follower_address: ap_id <> "/followers",
+          following_address: ap_id <> "/following",
+          following: [ap_id]
+        }
+      else
+        %{
+          ap_id: User.ap_id(user),
+          follower_address: User.ap_followers(user),
+          following_address: User.ap_following(user),
+          following: [User.ap_id(user)]
+        }
+      end
+
+    user = Map.merge(user, user_urls)
+
+    Repo.insert!(user)
+  end
+
+  def generate_activities(user, users) do
+    do_generate_activities(user, users)
+  end
+
+  defp do_generate_activities(user, users) do
+    IO.puts("Starting generating 20000 common activities...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..20_000,
+          fn _ ->
+            do_generate_activity([user | users])
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting common activities take #{to_sec(time)} sec.\n")
+
+    IO.puts("Starting generating 20000 activities with mentions...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..20_000,
+          fn _ ->
+            do_generate_activity_with_mention(user, users)
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n")
+
+    IO.puts("Starting generating 10000 activities with threads...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..10_000,
+          fn _ ->
+            do_generate_threads([user | users])
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_activity(users) do
+    post = %{
+      "status" => "Some status without mention with random user"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
+  end
+
+  defp do_generate_activity_with_mention(user, users) do
+    mentions_cnt = Enum.random([2, 3, 4, 5])
+    with_user = Enum.random([true, false])
+    users = Enum.shuffle(users)
+    mentions_users = Enum.take(users, mentions_cnt)
+    mentions_users = if with_user, do: [user | mentions_users], else: mentions_users
+
+    mentions_str =
+      Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ")
+
+    post = %{
+      "status" => mentions_str <> "some status with mentions random users"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
+  end
+
+  defp do_generate_threads(users) do
+    thread_length = Enum.random([2, 3, 4, 5])
+    actor = Enum.random(users)
+
+    post = %{
+      "status" => "Start of the thread"
+    }
+
+    {:ok, activity} = CommonAPI.post(actor, post)
+
+    Enum.each(1..thread_length, fn _ ->
+      user = Enum.random(users)
+
+      post = %{
+        "status" => "@#{actor.nickname} reply to thread",
+        "in_reply_to_status_id" => activity.id
+      }
+
+      CommonAPI.post(user, post)
+    end)
+  end
+
+  def generate_remote_activities(user, users) do
+    do_generate_remote_activities(user, users)
+  end
+
+  defp do_generate_remote_activities(user, users) do
+    IO.puts("Starting generating 10000 remote activities...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..10_000,
+          fn i ->
+            do_generate_remote_activity(i, user, users)
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_remote_activity(i, user, users) do
+    actor = Enum.random(users)
+    %{host: host} = URI.parse(actor.ap_id)
+    date = Date.utc_today()
+    datetime = DateTime.utc_now()
+
+    map = %{
+      "actor" => actor.ap_id,
+      "cc" => [actor.follower_address, user.ap_id],
+      "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+      "id" => actor.ap_id <> "/statuses/#{i}/activity",
+      "object" => %{
+        "actor" => actor.ap_id,
+        "atomUri" => actor.ap_id <> "/statuses/#{i}",
+        "attachment" => [],
+        "attributedTo" => actor.ap_id,
+        "bcc" => [],
+        "bto" => [],
+        "cc" => [actor.follower_address, user.ap_id],
+        "content" =>
+          "<p><span class=\"h-card\"><a href=\"" <>
+            user.ap_id <>
+            "\" class=\"u-url mention\">@<span>" <> user.nickname <> "</span></a></span></p>",
+        "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+        "conversation" =>
+          "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+        "emoji" => %{},
+        "id" => actor.ap_id <> "/statuses/#{i}",
+        "inReplyTo" => nil,
+        "inReplyToAtomUri" => nil,
+        "published" => datetime,
+        "sensitive" => true,
+        "summary" => "cw",
+        "tag" => [
+          %{
+            "href" => user.ap_id,
+            "name" => "@#{user.nickname}@#{host}",
+            "type" => "Mention"
+          }
+        ],
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "type" => "Note",
+        "url" => "http://#{host}/@#{actor.nickname}/#{i}"
+      },
+      "published" => datetime,
+      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+      "type" => "Create"
+    }
+
+    Pleroma.Web.ActivityPub.ActivityPub.insert(map, false)
+  end
+
+  def generate_dms(user, users, opts) do
+    IO.puts("Starting generating #{opts[:dms_max]} DMs")
+    {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
+    IO.puts("Inserting dms take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_dms(user, users, opts) do
+    Task.async_stream(
+      1..opts[:dms_max],
+      fn _ ->
+        do_generate_dm(user, users)
+      end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+  end
+
+  defp do_generate_dm(user, users) do
+    post = %{
+      "status" => "@#{user.nickname} some direct message",
+      "visibility" => "direct"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
+  end
+
+  def generate_long_thread(user, users, opts) do
+    IO.puts("Starting generating long thread with #{opts[:thread_length]} replies")
+    {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
+    IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
+    {:ok, activity}
+  end
+
+  defp do_generate_long_thread(user, users, opts) do
+    {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"})
+
+    Task.async_stream(
+      1..opts[:thread_length],
+      fn _ -> do_generate_thread(users, id) end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+
+    activity
+  end
+
+  defp do_generate_thread(users, activity_id) do
+    CommonAPI.post(Enum.random(users), %{
+      "status" => "reply to main post",
+      "in_reply_to_status_id" => activity_id
+    })
+  end
+
+  def generate_non_visible_message(user, users) do
+    IO.puts("Starting generating 1000 non visible posts")
+
+    {time, _} =
+      :timer.tc(fn ->
+        do_generate_non_visible_posts(user, users)
+      end)
+
+    IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_non_visible_posts(user, users) do
+    [not_friend | users] = users
+
+    make_friends(user, users)
+
+    Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+  end
+
+  defp make_friends(_user, []), do: nil
+
+  defp make_friends(user, [friend | users]) do
+    {:ok, _} = User.follow(user, friend)
+    {:ok, _} = User.follow(friend, user)
+    make_friends(user, users)
+  end
+
+  defp do_generate_non_visible_post(not_friend, users) do
+    post = %{
+      "status" => "some non visible post",
+      "visibility" => "private"
+    }
+
+    {:ok, activity} = CommonAPI.post(not_friend, post)
+
+    thread_length = Enum.random([2, 3, 4, 5])
+
+    Enum.each(1..thread_length, fn _ ->
+      user = Enum.random(users)
+
+      post = %{
+        "status" => "@#{not_friend.nickname} reply to non visible post",
+        "in_reply_to_status_id" => activity.id,
+        "visibility" => "private"
+      }
+
+      CommonAPI.post(user, post)
+    end)
+  end
+end

benchmarks/load_testing/helper.ex

@@ -0,0 +1,11 @@
+defmodule Pleroma.LoadTesting.Helper do
+  defmacro __using__(_) do
+    quote do
+      import Ecto.Query
+      alias Pleroma.Repo
+      alias Pleroma.User
+
+      defp to_sec(microseconds), do: microseconds / 1_000_000
+    end
+  end
+end

benchmarks/mix/tasks/pleroma/load_testing.ex

@@ -0,0 +1,134 @@
+defmodule Mix.Tasks.Pleroma.LoadTesting do
+  use Mix.Task
+  use Pleroma.LoadTesting.Helper
+  import Mix.Pleroma
+  import Pleroma.LoadTesting.Generator
+  import Pleroma.LoadTesting.Fetcher
+
+  @shortdoc "Factory for generation data"
+  @moduledoc """
+  Generates data like:
+  - local/remote users
+  - local/remote activities with notifications
+  - direct messages
+  - long thread
+  - non visible posts
+
+  ## Generate data
+      MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000
+      MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000
+
+  Options:
+  - `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u`
+  - `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d`
+  - `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t`
+  """
+
+  @aliases [u: :users, d: :dms, t: :thread_length]
+  @switches [
+    users: :integer,
+    dms: :integer,
+    thread_length: :integer
+  ]
+  @users_default 20_000
+  @dms_default 1_000
+  @thread_length_default 2_000
+
+  def run(args) do
+    start_pleroma()
+    Pleroma.Config.put([:instance, :skip_thread_containment], true)
+    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
+
+    users_max = Keyword.get(opts, :users, @users_default)
+    dms_max = Keyword.get(opts, :dms, @dms_default)
+    thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
+
+    clean_tables()
+
+    opts =
+      Keyword.put(opts, :users_max, users_max)
+      |> Keyword.put(:dms_max, dms_max)
+      |> Keyword.put(:thread_length, thread_length)
+
+    generate_users(opts)
+
+    # main user for queries
+    IO.puts("Fetching local main user...")
+
+    {time, user} =
+      :timer.tc(fn ->
+        Repo.one(
+          from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)
+        )
+      end)
+
+    IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
+
+    IO.puts("Fetching local users...")
+
+    {time, users} =
+      :timer.tc(fn ->
+        Repo.all(
+          from(u in User,
+            where: u.id != ^user.id,
+            where: u.local == true,
+            order_by: fragment("RANDOM()"),
+            limit: 10
+          )
+        )
+      end)
+
+    IO.puts("Fetching local users take #{to_sec(time)} sec.\n")
+
+    IO.puts("Fetching remote users...")
+
+    {time, remote_users} =
+      :timer.tc(fn ->
+        Repo.all(
+          from(u in User,
+            where: u.id != ^user.id,
+            where: u.local == false,
+            order_by: fragment("RANDOM()"),
+            limit: 10
+          )
+        )
+      end)
+
+    IO.puts("Fetching remote users take #{to_sec(time)} sec.\n")
+
+    generate_activities(user, users)
+
+    generate_remote_activities(user, remote_users)
+
+    generate_dms(user, users, opts)
+
+    {:ok, activity} = generate_long_thread(user, users, opts)
+
+    generate_non_visible_message(user, users)
+
+    IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
+
+    IO.puts("Activities in DB: #{Repo.aggregate(from(a in Pleroma.Activity), :count, :id)}")
+
+    IO.puts("Objects in DB: #{Repo.aggregate(from(o in Pleroma.Object), :count, :id)}")
+
+    IO.puts(
+      "Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}"
+    )
+
+    fetch_user(user)
+    query_timelines(user)
+    query_notifications(user)
+    query_dms(user)
+    query_long_thread(user, activity)
+    Pleroma.Config.put([:instance, :skip_thread_containment], false)
+    query_timelines(user)
+  end
+
+  defp clean_tables do
+    IO.puts("Deleting old data...\n")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
+  end
+end

config/benchmark.exs

@@ -0,0 +1,84 @@
+use Mix.Config
+
+# We don't run a server during test. If one is required,
+# you can enable the server option below.
+config :pleroma, Pleroma.Web.Endpoint,
+  http: [port: 4001],
+  url: [port: 4001],
+  server: true
+
+# Disable captha for tests
+config :pleroma, Pleroma.Captcha,
+  # It should not be enabled for automatic tests
+  enabled: false,
+  # A fake captcha service for tests
+  method: Pleroma.Captcha.Mock
+
+# Print only warnings and errors during test
+config :logger, level: :warn
+
+config :pleroma, :auth, oauth_consumer_strategies: []
+
+config :pleroma, Pleroma.Upload, filters: [], link_name: false
+
+config :pleroma, Pleroma.Uploaders.Local, uploads: "test/uploads"
+
+config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Test, enabled: true
+
+config :pleroma, :instance,
+  email: "admin@example.com",
+  notify_email: "noreply@example.com",
+  skip_thread_containment: false,
+  federating: false,
+  external_user_synchronization: false
+
+config :pleroma, :activitypub, sign_object_fetches: false
+
+# Configure your database
+config :pleroma, Pleroma.Repo,
+  adapter: Ecto.Adapters.Postgres,
+  username: "postgres",
+  password: "postgres",
+  database: "pleroma_test",
+  hostname: System.get_env("DB_HOST") || "localhost",
+  pool_size: 10
+
+# Reduce hash rounds for testing
+config :pbkdf2_elixir, rounds: 1
+
+config :tesla, adapter: Tesla.Mock
+
+config :pleroma, :rich_media,
+  enabled: false,
+  ignore_hosts: [],
+  ignore_tld: ["local", "localdomain", "lan"]
+
+config :web_push_encryption, :vapid_details,
+  subject: "mailto:administrator@example.com",
+  public_key:
+    "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4",
+  private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA"
+
+config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock
+
+config :pleroma_job_queue, disabled: true
+
+config :pleroma, Pleroma.ScheduledActivity,
+  daily_user_limit: 2,
+  total_user_limit: 3,
+  enabled: false
+
+config :pleroma, :rate_limit,
+  search: [{1000, 30}, {1000, 30}],
+  app_account_creation: {10_000, 5},
+  password_reset: {1000, 30}
+
+config :pleroma, :http_security, report_uri: "https://endpoint.com"
+
+config :pleroma, :http, send_user_agent: false
+
+rum_enabled = System.get_env("RUM_ENABLED") == "true"
+config :pleroma, :database, rum_enabled: rum_enabled
+IO.puts("RUM enabled: #{rum_enabled}")
+
+config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock

config/config.exs

@@ -59,10 +59,6 @@
     _ -> []
   end
 
-scheduled_jobs =
-  scheduled_jobs ++
-    [{"0 */6 * * * *", {Pleroma.Web.Websub, :refresh_subscriptions, []}}]
-
 config :pleroma, Pleroma.Scheduler,
   global: true,
   overlap: true,
@@ -243,9 +239,7 @@
   federation_incoming_replies_max_depth: 100,
   federation_reachability_timeout_days: 7,
   federation_publisher_modules: [
-    Pleroma.Web.ActivityPub.Publisher,
+    Pleroma.Web.ActivityPub.Publisher
-    Pleroma.Web.Websub,
-    Pleroma.Web.Salmon
   ],
   allow_relay: true,
   rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,
@@ -328,6 +322,16 @@
   ],
   default_mascot: :pleroma_fox_tan
 
+config :pleroma, :manifest,
+  icons: [
+    %{
+      src: "/static/logo.png",
+      type: "image/png"
+    }
+  ],
+  theme_color: "#282c37",
+  background_color: "#191b22"
+
 config :pleroma, :activitypub,
   unfollow_blocked: true,
   outgoing_blocks: true,

config/description.exs

@@ -581,9 +581,7 @@
         type: [:list, :module],
         description: "List of modules for federation publishing",
         suggestions: [
-          Pleroma.Web.ActivityPub.Publisher,
+          Pleroma.Web.ActivityPub.Publisher
-          Pleroma.Web.Websub,
-          Pleroma.Web.Salmo
         ]
       },
       %{
@@ -1100,6 +1098,45 @@
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :manifest,
+    type: :group,
+    description:
+      "This section describe PWA manifest instance-specific values. Currently this option relate only for MastoFE",
+    children: [
+      %{
+        key: :icons,
+        type: {:list, :map},
+        description: "Describe the icons of the app",
+        suggestion: [
+          %{
+            src: "/static/logo.png"
+          },
+          %{
+            src: "/static/icon.png",
+            type: "image/png"
+          },
+          %{
+            src: "/static/icon.ico",
+            sizes: "72x72 96x96 128x128 256x256"
+          }
+        ]
+      },
+      %{
+        key: :theme_color,
+        type: :string,
+        description: "Describe the theme color of the app",
+        suggestions: ["#282c37", "mediumpurple"]
+      },
+      %{
+        key: :background_color,
+        type: :string,
+        description: "Describe the background color of the app",
+        suggestions: ["#191b22", "aliceblue"]
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: :mrf_simple,

config/releases.exs

@@ -1,6 +1,6 @@
 import Config
 
-config :pleroma, :instance, static_dir: "/var/lib/pleroma/static"
+config :pleroma, :instance, static: "/var/lib/pleroma/static"
 config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads"
 
 config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"

docs/API/admin_api.md

@@ -47,7 +47,7 @@ Authentication is required and the user must be an admin.
 }
 ```
 
-## `/api/pleroma/admin/users`
+## DEPRECATED `DELETE /api/pleroma/admin/users`
 
 ### Remove a user
 
@@ -56,6 +56,15 @@ Authentication is required and the user must be an admin.
   - `nickname`
 - Response: User’s nickname
 
+## `DELETE /api/pleroma/admin/users`
+
+### Remove a user
+
+- Method `DELETE`
+- Params:
+  - `nicknames`
+- Response: Array of user nicknames
+
 ### Create a user
 
 - Method: `POST`
@@ -154,28 +163,86 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 }
 ```
 
-### Add user in permission group
+## DEPRECATED `POST /api/pleroma/admin/users/:nickname/permission_group/:permission_group`
+
+### Add user to permission group
 
-- Method: `POST`
 - Params: none
 - Response:
   - On failure: `{"error": "…"}`
-  - On success: JSON of the `user.info`
+  - On success: JSON of the user
+
+## `POST /api/pleroma/admin/users/permission_group/:permission_group`
+
+### Add users to permission group
+
+- Params:
+  - `nicknames`: nicknames array
+- Response:
+  - On failure: `{"error": "…"}`
+  - On success: JSON of the user
+
+## DEPRECATED `DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group`
 
 ### Remove user from permission group
 
-- Method: `DELETE`
 - Params: none
 - Response:
   - On failure: `{"error": "…"}`
-  - On success: JSON of the `user.info`
+  - On success: JSON of the user
 - Note: An admin cannot revoke their own admin status.
 
-## `/api/pleroma/admin/users/:nickname/activation_status`
+## `DELETE /api/pleroma/admin/users/permission_group/:permission_group`
+
+### Remove users from permission group
+
+- Params:
+  - `nicknames`: nicknames array
+- Response:
+  - On failure: `{"error": "…"}`
+  - On success: JSON of the user
+- Note: An admin cannot revoke their own admin status.
+
+## `PATCH /api/pleroma/admin/users/activate`
+
+### Activate user
+
+- Params:
+  - `nicknames`: nicknames array
+- Response:
+
+```json
+{
+  users: [
+    {
+      // user object
+    }
+  ]
+}
+```
+
+## `PATCH /api/pleroma/admin/users/deactivate`
+
+### Deactivate user
+
+- Params:
+  - `nicknames`: nicknames array
+- Response:
+
+```json
+{
+  users: [
+    {
+      // user object
+    }
+  ]
+}
+```
+
+## DEPRECATED `PATCH /api/pleroma/admin/users/:nickname/activation_status`
 
 ### Active or deactivate a user
 
-- Method: `PUT`
 - Params:
   - `nickname`
   - `status` BOOLEAN field, false value means deactivation.
@@ -222,6 +289,14 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 - Response:
   - On success: URL of the unfollowed relay
 
+## `GET /api/pleroma/admin/relay`
+
+### List Relays
+
+- Params: none
+- Response:
+  - On success: JSON array of relays
+
 ## `/api/pleroma/admin/users/invite_token`
 
 ### Create an account registration invite token

docs/API/pleroma_api.md

@@ -367,6 +367,13 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
     * `recipients`: A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though.
 * Response: JSON, statuses (200 - healthy, 503 unhealthy)
 
+## `GET /api/v1/pleroma/conversations/read`
+### Marks all user's conversations as read.
+* Method `POST`
+* Authentication: required
+* Params: None
+* Response: JSON, returns a list of Mastodon Conversation entities that were marked as read (200 - healthy, 503 unhealthy).
+
 ## `GET /api/pleroma/emoji/packs`
 ### Lists the custom emoji packs on the server
 * Method `GET`

docs/configuration/cheatsheet.md

@@ -247,6 +247,35 @@ relates to mascots on the mastodon frontend
 * `default_mascot`: An element from `mascots` - This will be used as the default mascot
   on MastoFE (default: `:pleroma_fox_tan`)
 
+## :manifest
+
+This section describe PWA manifest instance-specific values. Currently this option relate only for MastoFE.
+
+* `icons`: Describe the icons of the app, this a list of maps describing icons in the same way as the
+  [spec](https://www.w3.org/TR/appmanifest/#imageresource-and-its-members) describes it.
+
+  Example:
+
+  ```elixir
+  config :pleroma, :manifest,
+    icons: [
+      %{
+        src: "/static/logo.png"
+      },
+      %{
+        src: "/static/icon.png",
+        type: "image/png"
+      },
+      %{
+        src: "/static/icon.ico",
+        sizes: "72x72 96x96 128x128 256x256"
+      }
+    ]
+  ```
+
+* `theme_color`: Describe the theme color of the app. (Example: `"#282c37"`, `"rebeccapurple"`)
+* `background_color`: Describe the background color of the app. (Example: `"#191b22"`, `"aliceblue"`)
+
 ## :mrf_simple
 * `media_removal`: List of instances to remove medias from
 * `media_nsfw`: List of instances to put medias as NSFW(sensitive) from

lib/mix/tasks/pleroma/database.ex

@@ -28,7 +28,7 @@ def run(["remove_embedded_objects" | args]) do
     Logger.info("Removing embedded objects")
 
     Repo.query!(
-      "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
+      "update activities set data = safe_jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
       [],
       timeout: :infinity
     )
@@ -126,7 +126,7 @@ def run(["fix_likes_collections"]) do
         set: [
           data:
             fragment(
-              "jsonb_set(?, '{likes}', '[]'::jsonb, true)",
+              "safe_jsonb_set(?, '{likes}', '[]'::jsonb, true)",
               object.data
             )
         ]

lib/mix/tasks/pleroma/emoji.ex

@@ -111,19 +111,21 @@ def run(["get-packs" | args]) do
             file_list: files_to_unzip
           )
 
-        IO.puts(IO.ANSI.format(["Writing emoji.txt for ", :bright, pack_name]))
+        IO.puts(IO.ANSI.format(["Writing pack.json for ", :bright, pack_name]))
 
-        emoji_txt_str =
+        pack_json = %{
-          Enum.map(
+          pack: %{
-            files,
+            "license" => pack["license"],
-            fn {shortcode, path} ->
+            "homepage" => pack["homepage"],
-              emojo_path = Path.join("/emoji/#{pack_name}", path)
+            "description" => pack["description"],
-              "#{shortcode}, #{emojo_path}"
+            "fallback-src" => pack["src"],
-            end
+            "fallback-src-sha256" => pack["src_sha256"],
-          )
+            "share-files" => true
-          |> Enum.join("\n")
+          },
+          files: files
+        }
 
-        File.write!(Path.join(pack_path, "emoji.txt"), emoji_txt_str)
+        File.write!(Path.join(pack_path, "pack.json"), Jason.encode!(pack_json, pretty: true))
       else
         IO.puts(IO.ANSI.format([:bright, :red, "No pack named \"#{pack_name}\" found"]))
       end

lib/mix/tasks/pleroma/relay.ex

@@ -5,7 +5,6 @@
 defmodule Mix.Tasks.Pleroma.Relay do
   use Mix.Task
   import Mix.Pleroma
-  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
 
   @shortdoc "Manages remote relays"
@@ -36,13 +35,10 @@ def run(["unfollow", target]) do
   def run(["list"]) do
     start_pleroma()
 
-    with %User{following: following} = _user <- Relay.get_actor() do
+    with {:ok, list} <- Relay.list() do
-      following
+      list |> Enum.each(&shell_info(&1))
-      |> Enum.map(fn entry -> URI.parse(entry).host end)
-      |> Enum.uniq()
-      |> Enum.each(&shell_info(&1))
     else
-      e -> shell_error("Error while fetching relay subscription list: #{inspect(e)}")
+      {:error, e} -> shell_error("Error while fetching relay subscription list: #{inspect(e)}")
     end
   end
 end

lib/mix/tasks/pleroma/user.ex

@@ -5,6 +5,7 @@
 defmodule Mix.Tasks.Pleroma.User do
   use Mix.Task
   import Mix.Pleroma
+  alias Ecto.Changeset
   alias Pleroma.User
   alias Pleroma.UserInviteToken
   alias Pleroma.Web.OAuth
@@ -109,10 +110,10 @@ def run(["toggle_activated", nickname]) do
     start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
-      {:ok, user} = User.deactivate(user, !user.info.deactivated)
+      {:ok, user} = User.deactivate(user, !user.deactivated)
 
       shell_info(
-        "Activation status of #{nickname}: #{if(user.info.deactivated, do: "de", else: "")}activated"
+        "Activation status of #{nickname}: #{if(user.deactivated, do: "de", else: "")}activated"
       )
     else
       _ ->
@@ -340,7 +341,7 @@ def run(["toggle_confirmed", nickname]) do
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
       {:ok, user} = User.toggle_confirmation(user)
 
-      message = if user.info.confirmation_pending, do: "needs", else: "doesn't need"
+      message = if user.confirmation_pending, do: "needs", else: "doesn't need"
 
       shell_info("#{nickname} #{message} confirmation.")
     else
@@ -364,23 +365,32 @@ def run(["sign_out", nickname]) do
   end
 
   defp set_moderator(user, value) do
-    {:ok, user} = User.update_info(user, &User.Info.admin_api_update(&1, %{is_moderator: value}))
+    {:ok, user} =
+      user
+      |> Changeset.change(%{is_moderator: value})
+      |> User.update_and_set_cache()
 
-    shell_info("Moderator status of #{user.nickname}: #{user.info.is_moderator}")
+    shell_info("Moderator status of #{user.nickname}: #{user.is_moderator}")
     user
   end
 
   defp set_admin(user, value) do
-    {:ok, user} = User.update_info(user, &User.Info.admin_api_update(&1, %{is_admin: value}))
+    {:ok, user} =
+      user
+      |> Changeset.change(%{is_admin: value})
+      |> User.update_and_set_cache()
 
-    shell_info("Admin status of #{user.nickname}: #{user.info.is_admin}")
+    shell_info("Admin status of #{user.nickname}: #{user.is_admin}")
     user
   end
 
   defp set_locked(user, value) do
-    {:ok, user} = User.update_info(user, &User.Info.user_upgrade(&1, %{locked: value}))
+    {:ok, user} =
+      user
+      |> Changeset.change(%{locked: value})
+      |> User.update_and_set_cache()
 
-    shell_info("Locked status of #{user.nickname}: #{user.info.locked}")
+    shell_info("Locked status of #{user.nickname}: #{user.locked}")
     user
   end
 end

lib/pleroma/application.ex

@@ -161,11 +161,6 @@ defp task_children(:test) do
         id: :web_push_init,
         start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
         restart: :temporary
-      },
-      %{
-        id: :federator_init,
-        start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]},
-        restart: :temporary
       }
     ]
   end
@@ -177,11 +172,6 @@ defp task_children(_) do
         start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
         restart: :temporary
       },
-      %{
-        id: :federator_init,
-        start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]},
-        restart: :temporary
-      },
       %{
         id: :internal_fetch_init,
         start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},

lib/pleroma/conversation.ex

@@ -67,7 +67,13 @@ def create_or_bump_for(activity, opts \\ []) do
 
       participations =
         Enum.map(users, fn user ->
+          invisible_conversation = Enum.any?(users, &User.blocks?(user, &1))
+
+          unless invisible_conversation do
             User.increment_unread_conversation_count(conversation, user)
+          end
+
+          opts = Keyword.put(opts, :invisible_conversation, invisible_conversation)
 
           {:ok, participation} =
             Participation.create_for_user_and_conversation(user, conversation, opts)

lib/pleroma/conversation/participation.ex

@@ -32,11 +32,20 @@ def creation_cng(struct, params) do
 
   def create_for_user_and_conversation(user, conversation, opts \\ []) do
     read = !!opts[:read]
+    invisible_conversation = !!opts[:invisible_conversation]
+
+    update_on_conflict =
+      if(invisible_conversation, do: [], else: [read: read])
+      |> Keyword.put(:updated_at, NaiveDateTime.utc_now())
 
     %__MODULE__{}
-    |> creation_cng(%{user_id: user.id, conversation_id: conversation.id, read: read})
+    |> creation_cng(%{
+      user_id: user.id,
+      conversation_id: conversation.id,
+      read: invisible_conversation || read
+    })
     |> Repo.insert(
-      on_conflict: [set: [read: read, updated_at: NaiveDateTime.utc_now()]],
+      on_conflict: [set: update_on_conflict],
       returning: true,
       conflict_target: [:user_id, :conversation_id]
     )
@@ -48,6 +57,12 @@ def read_cng(struct, params) do
     |> validate_required([:read])
   end
 
+  def mark_as_read(%User{} = user, %Conversation{} = conversation) do
+    with %__MODULE__{} = participation <- for_user_and_conversation(user, conversation) do
+      mark_as_read(participation)
+    end
+  end
+
   def mark_as_read(participation) do
     participation
     |> read_cng(%{read: true})
@@ -63,6 +78,38 @@ def mark_as_read(participation) do
     end
   end
 
+  def mark_all_as_read(%User{local: true} = user, %User{} = target_user) do
+    target_conversation_ids =
+      __MODULE__
+      |> where([p], p.user_id == ^target_user.id)
+      |> select([p], p.conversation_id)
+      |> Repo.all()
+
+    __MODULE__
+    |> where([p], p.user_id == ^user.id)
+    |> where([p], p.conversation_id in ^target_conversation_ids)
+    |> update([p], set: [read: true])
+    |> Repo.update_all([])
+
+    {:ok, user} = User.set_unread_conversation_count(user)
+    {:ok, user, []}
+  end
+
+  def mark_all_as_read(%User{} = user, %User{}), do: {:ok, user, []}
+
+  def mark_all_as_read(%User{} = user) do
+    {_, participations} =
+      __MODULE__
+      |> where([p], p.user_id == ^user.id)
+      |> where([p], not p.read)
+      |> update([p], set: [read: true])
+      |> select([p], p)
+      |> Repo.update_all([])
+
+    {:ok, user} = User.set_unread_conversation_count(user)
+    {:ok, user, participations}
+  end
+
   def mark_as_unread(participation) do
     participation
     |> read_cng(%{read: false})

lib/pleroma/daemons/digest_email_daemon.ex

@@ -17,7 +17,7 @@ def perform do
     now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
 
     from(u in inactive_users_query,
-      where: fragment(~s(? #> '{"email_notifications","digest"}' @> 'true'), u.info),
+      where: fragment(~s(? ->'digest' @> 'true'), u.email_notifications),
       where: u.last_digest_emailed_at < datetime_add(^now, ^negative_interval, "day"),
       select: u
     )

lib/pleroma/emails/user_email.ex

@@ -72,7 +72,7 @@ def account_confirmation_email(user) do
         Endpoint,
         :confirm_email,
         user.id,
-        to_string(user.info.confirmation_token)
+        to_string(user.confirmation_token)
       )
 
     html_body = """

lib/pleroma/formatter.ex

@@ -127,7 +127,7 @@ def truncate(text, max_length \\ 200, omission \\ "...") do
     end
   end
 
-  defp get_ap_id(%User{info: %{source_data: %{"url" => url}}}) when is_binary(url), do: url
+  defp get_ap_id(%User{source_data: %{"url" => url}}) when is_binary(url), do: url
   defp get_ap_id(%User{ap_id: ap_id}), do: ap_id
 
   defp get_nickname_text(nickname, %{mentions_format: :full}), do: User.full_nickname(nickname)

lib/pleroma/marker.ex

@@ -0,0 +1,74 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Marker do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+
+  alias Ecto.Multi
+  alias Pleroma.Repo
+  alias Pleroma.User
+
+  @timelines ["notifications"]
+
+  schema "markers" do
+    field(:last_read_id, :string, default: "")
+    field(:timeline, :string, default: "")
+    field(:lock_version, :integer, default: 0)
+
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+    timestamps()
+  end
+
+  def get_markers(user, timelines \\ []) do
+    Repo.all(get_query(user, timelines))
+  end
+
+  def upsert(%User{} = user, attrs) do
+    attrs
+    |> Map.take(@timelines)
+    |> Enum.reduce(Multi.new(), fn {timeline, timeline_attrs}, multi ->
+      marker =
+        user
+        |> get_marker(timeline)
+        |> changeset(timeline_attrs)
+
+      Multi.insert(multi, timeline, marker,
+        returning: true,
+        on_conflict: {:replace, [:last_read_id]},
+        conflict_target: [:user_id, :timeline]
+      )
+    end)
+    |> Repo.transaction()
+  end
+
+  defp get_marker(user, timeline) do
+    case Repo.find_resource(get_query(user, timeline)) do
+      {:ok, marker} -> %__MODULE__{marker | user: user}
+      _ -> %__MODULE__{timeline: timeline, user_id: user.id}
+    end
+  end
+
+  @doc false
+  defp changeset(marker, attrs) do
+    marker
+    |> cast(attrs, [:last_read_id])
+    |> validate_required([:user_id, :timeline, :last_read_id])
+    |> validate_inclusion(:timeline, @timelines)
+  end
+
+  defp by_timeline(query, timeline) do
+    from(m in query, where: m.timeline in ^List.wrap(timeline))
+  end
+
+  defp by_user_id(query, id), do: from(m in query, where: m.user_id == ^id)
+
+  defp get_query(user, timelines) do
+    __MODULE__
+    |> by_user_id(user.id)
+    |> by_timeline(timelines)
+  end
+end

lib/pleroma/moderation_log.ex

@@ -86,18 +86,18 @@ defp parse_datetime(datetime) do
     parsed_datetime
   end
 
-  @spec insert_log(%{actor: User, subject: User, action: String.t(), permission: String.t()}) ::
+  @spec insert_log(%{actor: User, subject: [User], action: String.t(), permission: String.t()}) ::
           {:ok, ModerationLog} | {:error, any}
   def insert_log(%{
         actor: %User{} = actor,
-        subject: %User{} = subject,
+        subject: subjects,
         action: action,
         permission: permission
       }) do
     %ModerationLog{
       data: %{
         "actor" => user_to_map(actor),
-        "subject" => user_to_map(subject),
+        "subject" => user_to_map(subjects),
         "action" => action,
         "permission" => permission,
         "message" => ""
@@ -303,13 +303,16 @@ def insert_log(%{
   end
 
   @spec insert_log_entry_with_message(ModerationLog) :: {:ok, ModerationLog} | {:error, any}
-
   defp insert_log_entry_with_message(entry) do
     entry.data["message"]
     |> put_in(get_log_entry_message(entry))
     |> Repo.insert()
   end
 
+  defp user_to_map(users) when is_list(users) do
+    users |> Enum.map(&user_to_map/1)
+  end
+
   defp user_to_map(%User{} = user) do
     user
     |> Map.from_struct()
@@ -349,10 +352,10 @@ def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "delete",
-          "subject" => %{"nickname" => subject_nickname, "type" => "user"}
+          "subject" => subjects
         }
       }) do
-    "@#{actor_nickname} deleted user @#{subject_nickname}"
+    "@#{actor_nickname} deleted users: #{users_to_nicknames_string(subjects)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -363,12 +366,7 @@ def get_log_entry_message(%ModerationLog{
           "subjects" => subjects
         }
       }) do
-    nicknames =
+    "@#{actor_nickname} created users: #{users_to_nicknames_string(subjects)}"
-      subjects
-      |> Enum.map(&"@#{&1["nickname"]}")
-      |> Enum.join(", ")
-
-    "@#{actor_nickname} created users: #{nicknames}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -376,10 +374,10 @@ def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "activate",
-          "subject" => %{"nickname" => subject_nickname, "type" => "user"}
+          "subject" => users
         }
       }) do
-    "@#{actor_nickname} activated user @#{subject_nickname}"
+    "@#{actor_nickname} activated users: #{users_to_nicknames_string(users)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -387,10 +385,10 @@ def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "deactivate",
-          "subject" => %{"nickname" => subject_nickname, "type" => "user"}
+          "subject" => users
         }
       }) do
-    "@#{actor_nickname} deactivated user @#{subject_nickname}"
+    "@#{actor_nickname} deactivated users: #{users_to_nicknames_string(users)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -402,14 +400,9 @@ def get_log_entry_message(%ModerationLog{
           "action" => "tag"
         }
       }) do
-    nicknames_string =
-      nicknames
-      |> Enum.map(&"@#{&1}")
-      |> Enum.join(", ")
-
     tags_string = tags |> Enum.join(", ")
 
-    "@#{actor_nickname} added tags: #{tags_string} to users: #{nicknames_string}"
+    "@#{actor_nickname} added tags: #{tags_string} to users: #{nicknames_to_string(nicknames)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -421,14 +414,9 @@ def get_log_entry_message(%ModerationLog{
           "action" => "untag"
         }
       }) do
-    nicknames_string =
-      nicknames
-      |> Enum.map(&"@#{&1}")
-      |> Enum.join(", ")
-
     tags_string = tags |> Enum.join(", ")
 
-    "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_string}"
+    "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_to_string(nicknames)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -436,11 +424,11 @@ def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "grant",
-          "subject" => %{"nickname" => subject_nickname},
+          "subject" => users,
           "permission" => permission
         }
       }) do
-    "@#{actor_nickname} made @#{subject_nickname} #{permission}"
+    "@#{actor_nickname} made #{users_to_nicknames_string(users)} #{permission}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -448,11 +436,11 @@ def get_log_entry_message(%ModerationLog{
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "revoke",
-          "subject" => %{"nickname" => subject_nickname},
+          "subject" => users,
           "permission" => permission
         }
       }) do
-    "@#{actor_nickname} revoked #{permission} role from @#{subject_nickname}"
+    "@#{actor_nickname} revoked #{permission} role from #{users_to_nicknames_string(users)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -551,4 +539,16 @@ def get_log_entry_message(%ModerationLog{
       }) do
     "@#{actor_nickname} deleted status ##{subject_id}"
   end
+
+  defp nicknames_to_string(nicknames) do
+    nicknames
+    |> Enum.map(&"@#{&1}")
+    |> Enum.join(", ")
+  end
+
+  defp users_to_nicknames_string(users) do
+    users
+    |> Enum.map(&"@#{&1["nickname"]}")
+    |> Enum.join(", ")
+  end
 end

lib/pleroma/notification.ex

@@ -40,7 +40,7 @@ def for_user_query(user, opts \\ []) do
     |> where(
       [n, a],
       fragment(
-        "? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')",
+        "? not in (SELECT ap_id FROM users WHERE deactivated = 'true')",
         a.actor
       )
     )
@@ -55,21 +55,26 @@ def for_user_query(user, opts \\ []) do
     )
     |> preload([n, a, o], activity: {a, object: o})
     |> exclude_muted(user, opts)
+    |> exclude_blocked(user)
     |> exclude_visibility(opts)
   end
 
+  defp exclude_blocked(query, user) do
+    query
+    |> where([n, a], a.actor not in ^user.blocks)
+    |> where(
+      [n, a],
+      fragment("substring(? from '.*://([^/]*)')", a.actor) not in ^user.domain_blocks
+    )
+  end
+
   defp exclude_muted(query, _, %{with_muted: true}) do
     query
   end
 
   defp exclude_muted(query, user, _opts) do
     query
-    |> where([n, a], a.actor not in ^user.info.muted_notifications)
+    |> where([n, a], a.actor not in ^user.muted_notifications)
-    |> where([n, a], a.actor not in ^user.info.blocks)
-    |> where(
-      [n, a],
-      fragment("substring(? from '.*://([^/]*)')", a.actor) not in ^user.info.domain_blocks
-    )
     |> join(:left, [n, a], tm in Pleroma.ThreadMute,
       on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data)
     )
@@ -309,7 +314,7 @@ def skip?(:self, activity, user) do
   def skip?(
         :followers,
         activity,
-        %{info: %{notification_settings: %{"followers" => false}}} = user
+        %{notification_settings: %{"followers" => false}} = user
       ) do
     actor = activity.data["actor"]
     follower = User.get_cached_by_ap_id(actor)
@@ -319,14 +324,14 @@ def skip?(
   def skip?(
         :non_followers,
         activity,
-        %{info: %{notification_settings: %{"non_followers" => false}}} = user
+        %{notification_settings: %{"non_followers" => false}} = user
       ) do
     actor = activity.data["actor"]
     follower = User.get_cached_by_ap_id(actor)
     !User.following?(follower, user)
   end
 
-  def skip?(:follows, activity, %{info: %{notification_settings: %{"follows" => false}}} = user) do
+  def skip?(:follows, activity, %{notification_settings: %{"follows" => false}} = user) do
     actor = activity.data["actor"]
     followed = User.get_cached_by_ap_id(actor)
     User.following?(user, followed)
@@ -335,7 +340,7 @@ def skip?(:follows, activity, %{info: %{notification_settings: %{"follows" => fa
   def skip?(
         :non_follows,
         activity,
-        %{info: %{notification_settings: %{"non_follows" => false}}} = user
+        %{notification_settings: %{"non_follows" => false}} = user
       ) do
     actor = activity.data["actor"]
     followed = User.get_cached_by_ap_id(actor)

lib/pleroma/object.ex

@@ -181,7 +181,7 @@ def increase_replies_count(ap_id) do
         data:
           fragment(
             """
-            jsonb_set(?, '{repliesCount}',
+            safe_jsonb_set(?, '{repliesCount}',
               (coalesce((?->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true)
             """,
             o.data,
@@ -204,7 +204,7 @@ def decrease_replies_count(ap_id) do
         data:
           fragment(
             """
-            jsonb_set(?, '{repliesCount}',
+            safe_jsonb_set(?, '{repliesCount}',
               (greatest(0, (?->>'repliesCount')::int - 1))::varchar::jsonb, true)
             """,
             o.data,

lib/pleroma/object/containment.ex

@@ -32,6 +32,23 @@ def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor)
     get_actor(%{"actor" => actor})
   end
 
+  # TODO: We explicitly allow 'tag' URIs through, due to references to legacy OStatus
+  # objects being present in the test suite environment.  Once these objects are
+  # removed, please also remove this.
+  if Mix.env() == :test do
+    defp compare_uris(_, %URI{scheme: "tag"}), do: :ok
+  end
+
+  defp compare_uris(%URI{} = id_uri, %URI{} = other_uri) do
+    if id_uri.host == other_uri.host do
+      :ok
+    else
+      :error
+    end
+  end
+
+  defp compare_uris(_, _), do: :error
+
   @doc """
   Checks that an imported AP object's actor matches the domain it came from.
   """
@@ -41,11 +58,7 @@ def contain_origin(id, %{"actor" => _actor} = params) do
     id_uri = URI.parse(id)
     actor_uri = URI.parse(get_actor(params))
 
-    if id_uri.host == actor_uri.host do
+    compare_uris(actor_uri, id_uri)
-      :ok
-    else
-      :error
-    end
   end
 
   def contain_origin(id, %{"attributedTo" => actor} = params),
@@ -57,11 +70,7 @@ def contain_origin_from_id(id, %{"id" => other_id} = _params) do
     id_uri = URI.parse(id)
     other_uri = URI.parse(other_id)
 
-    if id_uri.host == other_uri.host do
+    compare_uris(id_uri, other_uri)
-      :ok
-    else
-      :error
-    end
   end
 
   def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}),

lib/pleroma/object/fetcher.ex

@@ -10,7 +10,6 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Signature
   alias Pleroma.Web.ActivityPub.InternalFetchActor
   alias Pleroma.Web.ActivityPub.Transmogrifier
-  alias Pleroma.Web.OStatus
 
   require Logger
   require Pleroma.Constants
@@ -67,7 +66,8 @@ def fetch_object_from_id(id, options \\ []) do
          {:normalize, nil} <- {:normalize, Object.normalize(data, false)},
          params <- prepare_activity_params(data),
          {:containment, :ok} <- {:containment, Containment.contain_origin(id, params)},
-         {:ok, activity} <- Transmogrifier.handle_incoming(params, options),
+         {:transmogrifier, {:ok, activity}} <-
+           {:transmogrifier, Transmogrifier.handle_incoming(params, options)},
          {:object, _data, %Object{} = object} <-
            {:object, data, Object.normalize(activity, false)} do
       {:ok, object}
@@ -75,9 +75,12 @@ def fetch_object_from_id(id, options \\ []) do
       {:containment, _} ->
         {:error, "Object containment failed."}
 
-      {:error, {:reject, nil}} ->
+      {:transmogrifier, {:error, {:reject, nil}}} ->
         {:reject, nil}
 
+      {:transmogrifier, _} ->
+        {:error, "Transmogrifier failure."}
+
       {:object, data, nil} ->
         reinject_object(%Object{}, data)
 
@@ -87,15 +90,11 @@ def fetch_object_from_id(id, options \\ []) do
       {:fetch_object, %Object{} = object} ->
         {:ok, object}
 
-      _e ->
+      {:fetch, {:error, error}} ->
-        # Only fallback when receiving a fetch/normalization error with ActivityPub
+        {:error, error}
-        Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
 
-        # FIXME: OStatus Object Containment?
+      e ->
-        case OStatus.fetch_activity_from_url(id) do
+        e
-          {:ok, [activity | _]} -> {:ok, Object.normalize(activity, false)}
-          e -> e
-        end
     end
   end
 
@@ -114,7 +113,11 @@ def fetch_object_from_id!(id, options \\ []) do
     with {:ok, object} <- fetch_object_from_id(id, options) do
       object
     else
-      _e ->
+      {:error, %Tesla.Mock.Error{}} ->
+        nil
+
+      e ->
+        Logger.error("Error while fetching #{id}: #{inspect(e)}")
         nil
     end
   end
@@ -161,7 +164,7 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
 
     Logger.debug("Fetch headers: #{inspect(headers)}")
 
-    with true <- String.starts_with?(id, "http"),
+    with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
          {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get(id, headers),
          {:ok, data} <- Jason.decode(body),
          :ok <- Containment.contain_origin_from_id(id, data) do
@@ -170,6 +173,12 @@ def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
       {:ok, %{status: code}} when code in [404, 410] ->
         {:error, "Object has been deleted"}
 
+      {:scheme, _} ->
+        {:error, "Unsupported URI scheme"}
+
+      {:error, e} ->
+        {:error, e}
+
       e ->
         {:error, e}
     end

lib/pleroma/plugs/admin_secret_authentication_plug.ex

@@ -19,7 +19,7 @@ def call(%{assigns: %{user: %User{}}} = conn, _), do: conn
   def call(%{params: %{"admin_token" => admin_token}} = conn, _) do
     if secret_token() && admin_token == secret_token() do
       conn
-      |> assign(:user, %User{info: %{is_admin: true}})
+      |> assign(:user, %User{is_admin: true})
     else
       conn
     end

lib/pleroma/plugs/oauth_plug.ex

@@ -71,7 +71,7 @@ defp fetch_user_and_token(token) do
       )
 
     # credo:disable-for-next-line Credo.Check.Readability.MaxLineLength
-    with %Token{user: %{info: %{deactivated: false} = _} = user} = token_record <- Repo.one(query) do
+    with %Token{user: %{deactivated: false} = user} = token_record <- Repo.one(query) do
       {:ok, user, token_record}
     end
   end

lib/pleroma/plugs/user_enabled_plug.ex

@@ -10,7 +10,7 @@ def init(options) do
     options
   end
 
-  def call(%{assigns: %{user: %User{info: %{deactivated: true}}}} = conn, _) do
+  def call(%{assigns: %{user: %User{deactivated: true}}} = conn, _) do
     conn
     |> assign(:user, nil)
   end

lib/pleroma/plugs/user_is_admin_plug.ex

@@ -11,7 +11,7 @@ def init(options) do
     options
   end
 
-  def call(%{assigns: %{user: %User{info: %{is_admin: true}}}} = conn, _) do
+  def call(%{assigns: %{user: %User{is_admin: true}}} = conn, _) do
     conn
   end
 

lib/pleroma/stats.ex

@@ -68,12 +68,7 @@ defp get_stat_data do
 
     domain_count = Enum.count(peers)
 
-    status_query =
+    status_count = Repo.aggregate(User.Query.build(%{local: true}), :sum, :note_count)
-      from(u in User.Query.build(%{local: true}),
-        select: fragment("sum((?->>'note_count')::int)", u.info)
-      )
-
-    status_count = Repo.one(status_query)
 
     user_count = Repo.aggregate(User.Query.build(%{local: true, active: true}), :count, :id)
 

lib/pleroma/upload.ex

@@ -105,7 +105,7 @@ defp get_opts(opts) do
           {Pleroma.Config.get!([:instance, :upload_limit]), "Document"}
       end
 
-    opts = %{
+    %{
       activity_type: Keyword.get(opts, :activity_type, activity_type),
       size_limit: Keyword.get(opts, :size_limit, size_limit),
       uploader: Keyword.get(opts, :uploader, Pleroma.Config.get([__MODULE__, :uploader])),
@@ -118,37 +118,6 @@ defp get_opts(opts) do
           Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url())
         )
     }
-
-    # TODO: 1.0+ : remove old config compatibility
-    opts =
-      if Pleroma.Config.get([__MODULE__, :strip_exif]) == true &&
-           !Enum.member?(opts.filters, Pleroma.Upload.Filter.Mogrify) do
-        Logger.warn("""
-        Pleroma: configuration `:instance, :strip_exif` is deprecated, please instead set:
-
-          :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Mogrify]]
-
-          :pleroma, Pleroma.Upload.Filter.Mogrify, args: ["strip", "auto-orient"]
-        """)
-
-        Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: ["strip", "auto-orient"])
-        Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Mogrify])
-      else
-        opts
-      end
-
-    if Pleroma.Config.get([:instance, :dedupe_media]) == true &&
-         !Enum.member?(opts.filters, Pleroma.Upload.Filter.Dedupe) do
-      Logger.warn("""
-      Pleroma: configuration `:instance, :dedupe_media` is deprecated, please instead set:
-
-      :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Dedupe]]
-      """)
-
-      Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Dedupe])
-    else
-      opts
-    end
   end
 
   defp prepare_upload(%Plug.Upload{} = file, opts) do

lib/pleroma/user/info.ex

@@ -1,478 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.User.Info do
-  use Ecto.Schema
-  import Ecto.Changeset
-
-  alias Pleroma.User.Info
-
-  @type t :: %__MODULE__{}
-
-  embedded_schema do
-    field(:banner, :map, default: %{})
-    field(:background, :map, default: %{})
-    field(:source_data, :map, default: %{})
-    field(:note_count, :integer, default: 0)
-    field(:follower_count, :integer, default: 0)
-    # Should be filled in only for remote users
-    field(:following_count, :integer, default: nil)
-    field(:locked, :boolean, default: false)
-    field(:confirmation_pending, :boolean, default: false)
-    field(:password_reset_pending, :boolean, default: false)
-    field(:confirmation_token, :string, default: nil)
-    field(:default_scope, :string, default: "public")
-    field(:blocks, {:array, :string}, default: [])
-    field(:domain_blocks, {:array, :string}, default: [])
-    field(:mutes, {:array, :string}, default: [])
-    field(:muted_reblogs, {:array, :string}, default: [])
-    field(:muted_notifications, {:array, :string}, default: [])
-    field(:subscribers, {:array, :string}, default: [])
-    field(:deactivated, :boolean, default: false)
-    field(:no_rich_text, :boolean, default: false)
-    field(:ap_enabled, :boolean, default: false)
-    field(:is_moderator, :boolean, default: false)
-    field(:is_admin, :boolean, default: false)
-    field(:show_role, :boolean, default: true)
-    field(:keys, :string, default: nil)
-    field(:settings, :map, default: nil)
-    field(:magic_key, :string, default: nil)
-    field(:uri, :string, default: nil)
-    field(:topic, :string, default: nil)
-    field(:hub, :string, default: nil)
-    field(:salmon, :string, default: nil)
-    field(:hide_followers_count, :boolean, default: false)
-    field(:hide_follows_count, :boolean, default: false)
-    field(:hide_followers, :boolean, default: false)
-    field(:hide_follows, :boolean, default: false)
-    field(:hide_favorites, :boolean, default: true)
-    field(:unread_conversation_count, :integer, default: 0)
-    field(:pinned_activities, {:array, :string}, default: [])
-    field(:email_notifications, :map, default: %{"digest" => false})
-    field(:mascot, :map, default: nil)
-    field(:emoji, {:array, :map}, default: [])
-    field(:pleroma_settings_store, :map, default: %{})
-    field(:fields, {:array, :map}, default: nil)
-    field(:raw_fields, {:array, :map}, default: [])
-    field(:discoverable, :boolean, default: false)
-
-    field(:notification_settings, :map,
-      default: %{
-        "followers" => true,
-        "follows" => true,
-        "non_follows" => true,
-        "non_followers" => true
-      }
-    )
-
-    field(:skip_thread_containment, :boolean, default: false)
-
-    # Found in the wild
-    # ap_id -> Where is this used?
-    # bio -> Where is this used?
-    # avatar -> Where is this used?
-    # fqn -> Where is this used?
-    # host -> Where is this used?
-    # subject _> Where is this used?
-  end
-
-  def set_activation_status(info, deactivated) do
-    params = %{deactivated: deactivated}
-
-    info
-    |> cast(params, [:deactivated])
-    |> validate_required([:deactivated])
-  end
-
-  def set_password_reset_pending(info, pending) do
-    params = %{password_reset_pending: pending}
-
-    info
-    |> cast(params, [:password_reset_pending])
-    |> validate_required([:password_reset_pending])
-  end
-
-  def update_notification_settings(info, settings) do
-    settings =
-      settings
-      |> Enum.map(fn {k, v} -> {k, v in [true, "true", "True", "1"]} end)
-      |> Map.new()
-
-    notification_settings =
-      info.notification_settings
-      |> Map.merge(settings)
-      |> Map.take(["followers", "follows", "non_follows", "non_followers"])
-
-    params = %{notification_settings: notification_settings}
-
-    info
-    |> cast(params, [:notification_settings])
-    |> validate_required([:notification_settings])
-  end
-
-  @doc """
-  Update email notifications in the given User.Info struct.
-
-  Examples:
-
-      iex> update_email_notifications(%Pleroma.User.Info{email_notifications: %{"digest" => false}}, %{"digest" => true})
-      %Pleroma.User.Info{email_notifications: %{"digest" => true}}
-
-  """
-  @spec update_email_notifications(t(), map()) :: Ecto.Changeset.t()
-  def update_email_notifications(info, settings) do
-    email_notifications =
-      info.email_notifications
-      |> Map.merge(settings)
-      |> Map.take(["digest"])
-
-    params = %{email_notifications: email_notifications}
-    fields = [:email_notifications]
-
-    info
-    |> cast(params, fields)
-    |> validate_required(fields)
-  end
-
-  def add_to_note_count(info, number) do
-    set_note_count(info, info.note_count + number)
-  end
-
-  def set_note_count(info, number) do
-    params = %{note_count: Enum.max([0, number])}
-
-    info
-    |> cast(params, [:note_count])
-    |> validate_required([:note_count])
-  end
-
-  def set_follower_count(info, number) do
-    params = %{follower_count: Enum.max([0, number])}
-
-    info
-    |> cast(params, [:follower_count])
-    |> validate_required([:follower_count])
-  end
-
-  def set_mutes(info, mutes) do
-    params = %{mutes: mutes}
-
-    info
-    |> cast(params, [:mutes])
-    |> validate_required([:mutes])
-  end
-
-  @spec set_notification_mutes(Changeset.t(), [String.t()], boolean()) :: Changeset.t()
-  def set_notification_mutes(changeset, muted_notifications, notifications?) do
-    if notifications? do
-      put_change(changeset, :muted_notifications, muted_notifications)
-      |> validate_required([:muted_notifications])
-    else
-      changeset
-    end
-  end
-
-  def set_blocks(info, blocks) do
-    params = %{blocks: blocks}
-
-    info
-    |> cast(params, [:blocks])
-    |> validate_required([:blocks])
-  end
-
-  def set_subscribers(info, subscribers) do
-    params = %{subscribers: subscribers}
-
-    info
-    |> cast(params, [:subscribers])
-    |> validate_required([:subscribers])
-  end
-
-  @spec add_to_mutes(Info.t(), String.t(), boolean()) :: Changeset.t()
-  def add_to_mutes(info, muted, notifications?) do
-    info
-    |> set_mutes(Enum.uniq([muted | info.mutes]))
-    |> set_notification_mutes(
-      Enum.uniq([muted | info.muted_notifications]),
-      notifications?
-    )
-  end
-
-  @spec remove_from_mutes(Info.t(), String.t()) :: Changeset.t()
-  def remove_from_mutes(info, muted) do
-    info
-    |> set_mutes(List.delete(info.mutes, muted))
-    |> set_notification_mutes(List.delete(info.muted_notifications, muted), true)
-  end
-
-  def add_to_block(info, blocked) do
-    set_blocks(info, Enum.uniq([blocked | info.blocks]))
-  end
-
-  def remove_from_block(info, blocked) do
-    set_blocks(info, List.delete(info.blocks, blocked))
-  end
-
-  def add_to_subscribers(info, subscribed) do
-    set_subscribers(info, Enum.uniq([subscribed | info.subscribers]))
-  end
-
-  def remove_from_subscribers(info, subscribed) do
-    set_subscribers(info, List.delete(info.subscribers, subscribed))
-  end
-
-  def set_domain_blocks(info, domain_blocks) do
-    params = %{domain_blocks: domain_blocks}
-
-    info
-    |> cast(params, [:domain_blocks])
-    |> validate_required([:domain_blocks])
-  end
-
-  def add_to_domain_block(info, domain_blocked) do
-    set_domain_blocks(info, Enum.uniq([domain_blocked | info.domain_blocks]))
-  end
-
-  def remove_from_domain_block(info, domain_blocked) do
-    set_domain_blocks(info, List.delete(info.domain_blocks, domain_blocked))
-  end
-
-  def set_keys(info, keys) do
-    params = %{keys: keys}
-
-    info
-    |> cast(params, [:keys])
-    |> validate_required([:keys])
-  end
-
-  def remote_user_creation(info, params) do
-    params =
-      if Map.has_key?(params, :fields) do
-        Map.put(params, :fields, Enum.map(params[:fields], &truncate_field/1))
-      else
-        params
-      end
-
-    info
-    |> cast(params, [
-      :ap_enabled,
-      :source_data,
-      :banner,
-      :locked,
-      :magic_key,
-      :uri,
-      :hub,
-      :topic,
-      :salmon,
-      :hide_followers,
-      :hide_follows,
-      :hide_followers_count,
-      :hide_follows_count,
-      :follower_count,
-      :fields,
-      :following_count,
-      :discoverable
-    ])
-    |> validate_fields(true)
-  end
-
-  def user_upgrade(info, params, remote? \\ false) do
-    info
-    |> cast(params, [
-      :ap_enabled,
-      :source_data,
-      :banner,
-      :locked,
-      :magic_key,
-      :follower_count,
-      :following_count,
-      :hide_follows,
-      :fields,
-      :hide_followers,
-      :discoverable,
-      :hide_followers_count,
-      :hide_follows_count
-    ])
-    |> validate_fields(remote?)
-  end
-
-  def profile_update(info, params) do
-    info
-    |> cast(params, [
-      :locked,
-      :no_rich_text,
-      :default_scope,
-      :banner,
-      :hide_follows,
-      :hide_followers,
-      :hide_followers_count,
-      :hide_follows_count,
-      :hide_favorites,
-      :background,
-      :show_role,
-      :skip_thread_containment,
-      :fields,
-      :raw_fields,
-      :pleroma_settings_store,
-      :discoverable
-    ])
-    |> validate_fields()
-  end
-
-  def validate_fields(changeset, remote? \\ false) do
-    limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields
-    limit = Pleroma.Config.get([:instance, limit_name], 0)
-
-    changeset
-    |> validate_length(:fields, max: limit)
-    |> validate_change(:fields, fn :fields, fields ->
-      if Enum.all?(fields, &valid_field?/1) do
-        []
-      else
-        [fields: "invalid"]
-      end
-    end)
-  end
-
-  defp valid_field?(%{"name" => name, "value" => value}) do
-    name_limit = Pleroma.Config.get([:instance, :account_field_name_length], 255)
-    value_limit = Pleroma.Config.get([:instance, :account_field_value_length], 255)
-
-    is_binary(name) && is_binary(value) && String.length(name) <= name_limit &&
-      String.length(value) <= value_limit
-  end
-
-  defp valid_field?(_), do: false
-
-  defp truncate_field(%{"name" => name, "value" => value}) do
-    {name, _chopped} =
-      String.split_at(name, Pleroma.Config.get([:instance, :account_field_name_length], 255))
-
-    {value, _chopped} =
-      String.split_at(value, Pleroma.Config.get([:instance, :account_field_value_length], 255))
-
-    %{"name" => name, "value" => value}
-  end
-
-  @spec confirmation_changeset(Info.t(), keyword()) :: Changeset.t()
-  def confirmation_changeset(info, opts) do
-    need_confirmation? = Keyword.get(opts, :need_confirmation)
-
-    params =
-      if need_confirmation? do
-        %{
-          confirmation_pending: true,
-          confirmation_token: :crypto.strong_rand_bytes(32) |> Base.url_encode64()
-        }
-      else
-        %{
-          confirmation_pending: false,
-          confirmation_token: nil
-        }
-      end
-
-    cast(info, params, [:confirmation_pending, :confirmation_token])
-  end
-
-  def mastodon_settings_update(info, settings) do
-    params = %{settings: settings}
-
-    info
-    |> cast(params, [:settings])
-    |> validate_required([:settings])
-  end
-
-  def mascot_update(info, url) do
-    params = %{mascot: url}
-
-    info
-    |> cast(params, [:mascot])
-    |> validate_required([:mascot])
-  end
-
-  def set_source_data(info, source_data) do
-    params = %{source_data: source_data}
-
-    info
-    |> cast(params, [:source_data])
-    |> validate_required([:source_data])
-  end
-
-  def admin_api_update(info, params) do
-    info
-    |> cast(params, [
-      :is_moderator,
-      :is_admin,
-      :show_role
-    ])
-  end
-
-  def add_pinnned_activity(info, %Pleroma.Activity{id: id}) do
-    if id not in info.pinned_activities do
-      max_pinned_statuses = Pleroma.Config.get([:instance, :max_pinned_statuses], 0)
-      params = %{pinned_activities: info.pinned_activities ++ [id]}
-
-      info
-      |> cast(params, [:pinned_activities])
-      |> validate_length(:pinned_activities,
-        max: max_pinned_statuses,
-        message: "You have already pinned the maximum number of statuses"
-      )
-    else
-      change(info)
-    end
-  end
-
-  def remove_pinnned_activity(info, %Pleroma.Activity{id: id}) do
-    params = %{pinned_activities: List.delete(info.pinned_activities, id)}
-
-    cast(info, params, [:pinned_activities])
-  end
-
-  def roles(%Info{is_moderator: is_moderator, is_admin: is_admin}) do
-    %{
-      admin: is_admin,
-      moderator: is_moderator
-    }
-  end
-
-  def add_reblog_mute(info, ap_id) do
-    params = %{muted_reblogs: info.muted_reblogs ++ [ap_id]}
-
-    cast(info, params, [:muted_reblogs])
-  end
-
-  def remove_reblog_mute(info, ap_id) do
-    params = %{muted_reblogs: List.delete(info.muted_reblogs, ap_id)}
-
-    cast(info, params, [:muted_reblogs])
-  end
-
-  # ``fields`` is an array of mastodon profile field, containing ``{"name": "…", "value": "…"}``.
-  # For example: [{"name": "Pronoun", "value": "she/her"}, …]
-  def fields(%{fields: nil, source_data: %{"attachment" => attachment}}) do
-    limit = Pleroma.Config.get([:instance, :max_remote_account_fields], 0)
-
-    attachment
-    |> Enum.filter(fn %{"type" => t} -> t == "PropertyValue" end)
-    |> Enum.map(fn fields -> Map.take(fields, ["name", "value"]) end)
-    |> Enum.take(limit)
-  end
-
-  def fields(%{fields: nil}), do: []
-
-  def fields(%{fields: fields}), do: fields
-
-  def follow_information_update(info, params) do
-    info
-    |> cast(params, [
-      :hide_followers,
-      :hide_follows,
-      :follower_count,
-      :following_count,
-      :hide_followers_count,
-      :hide_follows_count
-    ])
-  end
-end

lib/pleroma/user/query.ex

@@ -56,7 +56,6 @@ defmodule Pleroma.User.Query do
 
   @ilike_criteria [:nickname, :name, :query]
   @equal_criteria [:email]
-  @role_criteria [:is_admin, :is_moderator]
   @contains_criteria [:ap_id, :nickname]
 
   @spec build(criteria()) :: Query.t()
@@ -100,15 +99,19 @@ defp compose_query({:tags, tags}, query) when is_list(tags) and length(tags) > 0
     Enum.reduce(tags, query, &prepare_tag_criteria/2)
   end
 
-  defp compose_query({key, _}, query) when key in @role_criteria do
+  defp compose_query({:is_admin, _}, query) do
-    where(query, [u], fragment("(?->? @> 'true')", u.info, ^to_string(key)))
+    where(query, [u], u.is_admin)
+  end
+
+  defp compose_query({:is_moderator, _}, query) do
+    where(query, [u], u.is_moderator)
   end
 
   defp compose_query({:super_users, _}, query) do
     where(
       query,
       [u],
-      fragment("?->'is_admin' @> 'true' OR ?->'is_moderator' @> 'true'", u.info, u.info)
+      u.is_admin or u.is_moderator
     )
   end
 
@@ -117,7 +120,13 @@ defp compose_query({:local, _}, query), do: location_query(query, true)
   defp compose_query({:external, _}, query), do: location_query(query, false)
 
   defp compose_query({:active, _}, query) do
-    where(query, [u], fragment("not (?->'deactivated' @> 'true')", u.info))
+    User.restrict_deactivated(query)
+    |> where([u], not is_nil(u.nickname))
+  end
+
+  defp compose_query({:legacy_active, _}, query) do
+    query
+    |> where([u], fragment("not (?->'deactivated' @> 'true')", u.info))
     |> where([u], not is_nil(u.nickname))
   end
 
@@ -126,7 +135,7 @@ defp compose_query({:deactivated, false}, query) do
   end
 
   defp compose_query({:deactivated, true}, query) do
-    where(query, [u], fragment("?->'deactivated' @> 'true'", u.info))
+    where(query, [u], u.deactivated == ^true)
     |> where([u], not is_nil(u.nickname))
   end
 

lib/pleroma/user/search.ex

@@ -4,11 +4,9 @@
 
 defmodule Pleroma.User.Search do
   alias Pleroma.Pagination
-  alias Pleroma.Repo
   alias Pleroma.User
   import Ecto.Query
 
-  @similarity_threshold 0.25
   @limit 20
 
   def search(query_string, opts \\ []) do
@@ -23,18 +21,10 @@ def search(query_string, opts \\ []) do
 
     maybe_resolve(resolve, for_user, query_string)
 
-    {:ok, results} =
+    results =
-      Repo.transaction(fn ->
-        Ecto.Adapters.SQL.query(
-          Repo,
-          "select set_limit(#{@similarity_threshold})",
-          []
-        )
-
       query_string
       |> search_query(for_user, following)
       |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset)
-      end)
 
     results
   end
@@ -56,26 +46,76 @@ defp search_query(query_string, for_user, following) do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_blocked_domains(for_user)
-    |> search_subqueries(query_string)
+    |> fts_search(query_string)
-    |> union_subqueries
+    |> trigram_rank(query_string)
-    |> distinct_query()
+    |> boost_search_rank(for_user)
-    |> boost_search_rank_query(for_user)
     |> subquery()
     |> order_by(desc: :search_rank)
     |> maybe_restrict_local(for_user)
   end
 
+  @nickname_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~\-@]+$/
+  defp fts_search(query, query_string) do
+    {nickname_weight, name_weight} =
+      if String.match?(query_string, @nickname_regex) do
+        {"A", "B"}
+      else
+        {"B", "A"}
+      end
+
+    query_string = to_tsquery(query_string)
+
+    from(
+      u in query,
+      where:
+        fragment(
+          """
+          (setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)) @@ to_tsquery('simple', ?)
+          """,
+          u.name,
+          ^name_weight,
+          u.nickname,
+          ^nickname_weight,
+          ^query_string
+        )
+    )
+  end
+
+  defp to_tsquery(query_string) do
+    String.trim_trailing(query_string, "@" <> local_domain())
+    |> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ")
+    |> String.trim()
+    |> String.split()
+    |> Enum.map(&(&1 <> ":*"))
+    |> Enum.join(" | ")
+  end
+
+  defp trigram_rank(query, query_string) do
+    from(
+      u in query,
+      select_merge: %{
+        search_rank:
+          fragment(
+            "similarity(?, trim(? || ' ' || coalesce(?, '')))",
+            ^query_string,
+            u.nickname,
+            u.name
+          )
+      }
+    )
+  end
+
   defp base_query(_user, false), do: User
   defp base_query(user, true), do: User.get_followers_query(user)
 
-  defp filter_blocked_user(query, %User{info: %{blocks: blocks}})
+  defp filter_blocked_user(query, %User{blocks: blocks})
        when length(blocks) > 0 do
     from(q in query, where: not (q.ap_id in ^blocks))
   end
 
   defp filter_blocked_user(query, _), do: query
 
-  defp filter_blocked_domains(query, %User{info: %{domain_blocks: domain_blocks}})
+  defp filter_blocked_domains(query, %User{domain_blocks: domain_blocks})
        when length(domain_blocks) > 0 do
     domains = Enum.join(domain_blocks, ",")
 
@@ -87,21 +127,6 @@ defp filter_blocked_domains(query, %User{info: %{domain_blocks: domain_blocks}})
 
   defp filter_blocked_domains(query, _), do: query
 
-  defp union_subqueries({fts_subquery, trigram_subquery}) do
-    from(s in trigram_subquery, union_all: ^fts_subquery)
-  end
-
-  defp search_subqueries(base_query, query_string) do
-    {
-      fts_search_subquery(base_query, query_string),
-      trigram_search_subquery(base_query, query_string)
-    }
-  end
-
-  defp distinct_query(q) do
-    from(s in subquery(q), order_by: s.search_type, distinct: s.id)
-  end
-
   defp maybe_resolve(true, user, query) do
     case {limit(), user} do
       {:all, _} -> :noop
@@ -126,9 +151,9 @@ defp limit, do: Pleroma.Config.get([:instance, :limit_to_local_content], :unauth
 
   defp restrict_local(q), do: where(q, [u], u.local == true)
 
-  defp boost_search_rank_query(query, nil), do: query
+  defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
 
-  defp boost_search_rank_query(query, for_user) do
+  defp boost_search_rank(query, %User{} = for_user) do
     friends_ids = User.get_friends_ids(for_user)
     followers_ids = User.get_followers_ids(for_user)
 
@@ -137,8 +162,8 @@ defp boost_search_rank_query(query, for_user) do
         search_rank:
           fragment(
             """
-             CASE WHEN (?) THEN 0.5 + (?) * 1.3
+             CASE WHEN (?) THEN (?) * 1.5
-             WHEN (?) THEN 0.5 + (?) * 1.2
+             WHEN (?) THEN (?) * 1.3
              WHEN (?) THEN (?) * 1.1
              ELSE (?) END
             """,
@@ -154,70 +179,5 @@ defp boost_search_rank_query(query, for_user) do
     )
   end
 
-  @spec fts_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t()
+  defp boost_search_rank(query, _for_user), do: query
-  defp fts_search_subquery(query, term) do
-    processed_query =
-      String.trim_trailing(term, "@" <> local_domain())
-      |> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ")
-      |> String.trim()
-      |> String.split()
-      |> Enum.map(&(&1 <> ":*"))
-      |> Enum.join(" | ")
-
-    from(
-      u in query,
-      select_merge: %{
-        search_type: ^0,
-        search_rank:
-          fragment(
-            """
-            ts_rank_cd(
-              setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
-              setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'),
-              to_tsquery('simple', ?),
-              32
-            )
-            """,
-            u.nickname,
-            u.name,
-            ^processed_query
-          )
-      },
-      where:
-        fragment(
-          """
-            (setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
-            setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?)
-          """,
-          u.nickname,
-          u.name,
-          ^processed_query
-        )
-    )
-    |> User.restrict_deactivated()
-  end
-
-  @spec trigram_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t()
-  defp trigram_search_subquery(query, term) do
-    term = String.trim_trailing(term, "@" <> local_domain())
-
-    from(
-      u in query,
-      select_merge: %{
-        # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason
-        search_type: fragment("?", 1),
-        search_rank:
-          fragment(
-            "similarity(?, trim(? || ' ' || coalesce(?, '')))",
-            ^term,
-            u.nickname,
-            u.name
-          )
-      },
-      where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term)
-    )
-    |> User.restrict_deactivated()
-  end
-
-  defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Activity.Ir.Topics
   alias Pleroma.Config
   alias Pleroma.Conversation
+  alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Containment
@@ -68,7 +69,7 @@ defp get_recipients(data) do
   defp check_actor_is_active(actor) do
     if not is_nil(actor) do
       with user <- User.get_cached_by_ap_id(actor),
-           false <- user.info.deactivated do
+           false <- user.deactivated do
         true
       else
         _e -> false
@@ -131,7 +132,7 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when
          {:ok, map} <- MRF.filter(map),
          {recipients, _, _} = get_recipients(map),
          {:fake, false, map, recipients} <- {:fake, fake, map, recipients},
-         :ok <- Containment.contain_child(map),
+         {:containment, :ok} <- {:containment, Containment.contain_child(map)},
          {:ok, map, object} <- insert_full_object(map) do
       {:ok, activity} =
         Repo.insert(%Activity{
@@ -153,11 +154,8 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when
 
       Notification.create_notifications(activity)
 
-      participations =
+      conversation = create_or_bump_conversation(activity, map["actor"])
-        activity
+      participations = get_participations(conversation)
-        |> Conversation.create_or_bump_for()
-        |> get_participations()
-
       stream_out(activity)
       stream_out_participations(participations)
       {:ok, activity}
@@ -182,7 +180,20 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when
     end
   end
 
-  defp get_participations({:ok, %{participations: participations}}), do: participations
+  defp create_or_bump_conversation(activity, actor) do
+    with {:ok, conversation} <- Conversation.create_or_bump_for(activity),
+         %User{} = user <- User.get_cached_by_ap_id(actor),
+         Participation.mark_as_read(user, conversation) do
+      {:ok, conversation}
+    end
+  end
+
+  defp get_participations({:ok, conversation}) do
+    conversation
+    |> Repo.preload(:participations, force: true)
+    |> Map.get(:participations)
+  end
+
   defp get_participations(_), do: []
 
   def stream_out_participations(participations) do
@@ -225,6 +236,7 @@ def create(%{to: to, actor: actor, context: context, object: object} = params, f
     # only accept false as false value
     local = !(params[:local] == false)
     published = params[:published]
+    quick_insert? = Pleroma.Config.get([:env]) == :benchmark
 
     with create_data <-
            make_create_data(
@@ -235,12 +247,14 @@ def create(%{to: to, actor: actor, context: context, object: object} = params, f
          {:fake, false, activity} <- {:fake, fake, activity},
          _ <- increase_replies_count_if_reply(create_data),
          _ <- increase_poll_votes_if_vote(create_data),
-         # Changing note count prior to enqueuing federation task in order to avoid
+         {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity},
-         # race conditions on updating user.info
          {:ok, _actor} <- increase_note_count_if_public(actor, activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     else
+      {:quick_insert, true, activity} ->
+        {:ok, activity}
+
       {:fake, true, activity} ->
         {:ok, activity}
 
@@ -429,8 +443,6 @@ def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, options \\ [
          {:ok, activity} <- insert(data, local, false),
          stream_out_participations(object, user),
          _ <- decrease_replies_count_if_reply(object),
-         # Changing note count prior to enqueuing federation task in order to avoid
-         # race conditions on updating user.info
          {:ok, _actor} <- decrease_note_count_if_public(user, object),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
@@ -645,7 +657,7 @@ defp restrict_thread_visibility(query, _, %{skip_thread_containment: true} = _),
 
   defp restrict_thread_visibility(
          query,
-         %{"user" => %User{info: %{skip_thread_containment: true}}},
+         %{"user" => %User{skip_thread_containment: true}},
          _
        ),
        do: query
@@ -683,7 +695,7 @@ def fetch_user_activities(user, reading_user, params \\ %{}) do
       |> Map.put("user", reading_user)
       |> Map.put("actor_id", user.ap_id)
       |> Map.put("whole_db", true)
-      |> Map.put("pinned_activity_ids", user.info.pinned_activities)
+      |> Map.put("pinned_activity_ids", user.pinned_activities)
 
     recipients =
       user_activities_recipients(%{
@@ -844,8 +856,8 @@ defp restrict_reblogs(query, _), do: query
 
   defp restrict_muted(query, %{"with_muted" => val}) when val in [true, "true", "1"], do: query
 
-  defp restrict_muted(query, %{"muting_user" => %User{info: info}} = opts) do
+  defp restrict_muted(query, %{"muting_user" => %User{} = user} = opts) do
-    mutes = info.mutes
+    mutes = user.mutes
 
     query =
       from([activity] in query,
@@ -862,9 +874,9 @@ defp restrict_muted(query, %{"muting_user" => %User{info: info}} = opts) do
 
   defp restrict_muted(query, _), do: query
 
-  defp restrict_blocked(query, %{"blocking_user" => %User{info: info}}) do
+  defp restrict_blocked(query, %{"blocking_user" => %User{} = user}) do
-    blocks = info.blocks || []
+    blocks = user.blocks || []
-    domain_blocks = info.domain_blocks || []
+    domain_blocks = user.domain_blocks || []
 
     query =
       if has_named_binding?(query, :object), do: query, else: Activity.with_joined_object(query)
@@ -905,8 +917,8 @@ defp restrict_pinned(query, %{"pinned" => "true", "pinned_activity_ids" => ids})
 
   defp restrict_pinned(query, _), do: query
 
-  defp restrict_muted_reblogs(query, %{"muting_user" => %User{info: info}}) do
+  defp restrict_muted_reblogs(query, %{"muting_user" => %User{} = user}) do
-    muted_reblogs = info.muted_reblogs || []
+    muted_reblogs = user.muted_reblogs || []
 
     from(
       activity in query,
@@ -1091,17 +1103,17 @@ defp object_to_user_data(data) do
     locked = data["manuallyApprovesFollowers"] || false
     data = Transmogrifier.maybe_fix_user_object(data)
     discoverable = data["discoverable"] || false
+    invisible = data["invisible"] || false
 
     user_data = %{
       ap_id: data["id"],
-      info: %{
       ap_enabled: true,
       source_data: data,
       banner: banner,
       fields: fields,
       locked: locked,
-        discoverable: discoverable
+      discoverable: discoverable,
-      },
+      invisible: invisible,
       avatar: avatar,
       name: data["name"],
       follower_address: data["followers"],
@@ -1153,7 +1165,7 @@ defp maybe_update_follow_information(data) do
     with {:enabled, true} <-
            {:enabled, Pleroma.Config.get([:instance, :external_user_synchronization])},
          {:ok, info} <- fetch_follow_information_for_user(data) do
-      info = Map.merge(data.info, info)
+      info = Map.merge(data[:info] || %{}, info)
       Map.put(data, :info, info)
     else
       {:enabled, false} ->
@@ -1204,7 +1216,9 @@ def fetch_and_prepare_user_from_ap_id(ap_id) do
          data <- maybe_update_follow_information(data) do
       {:ok, data}
     else
-      e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
+      e ->
+        Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
+        {:error, e}
     end
   end
 

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -137,7 +137,7 @@ def following(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname, "p
     with %User{} = user <- User.get_cached_by_nickname(nickname),
          {user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user),
          {:show_follows, true} <-
-           {:show_follows, (for_user && for_user == user) || !user.info.hide_follows} do
+           {:show_follows, (for_user && for_user == user) || !user.hide_follows} do
       {page, _} = Integer.parse(page)
 
       conn
@@ -174,7 +174,7 @@ def followers(%{assigns: %{user: for_user}} = conn, %{"nickname" => nickname, "p
     with %User{} = user <- User.get_cached_by_nickname(nickname),
          {user, for_user} <- ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user),
          {:show_followers, true} <-
-           {:show_followers, (for_user && for_user == user) || !user.info.hide_followers} do
+           {:show_followers, (for_user && for_user == user) || !user.hide_followers} do
       {page, _} = Integer.parse(page)
 
       conn
@@ -387,7 +387,7 @@ def handle_user_activity(user, %{"type" => "Create"} = params) do
 
   def handle_user_activity(user, %{"type" => "Delete"} = params) do
     with %Object{} = object <- Object.normalize(params["object"]),
-         true <- user.info.is_moderator || user.ap_id == object.data["actor"],
+         true <- user.is_moderator || user.ap_id == object.data["actor"],
          {:ok, delete} <- ActivityPub.delete(object) do
       {:ok, delete}
     else

lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex

@@ -11,7 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do
 
   # has the user successfully posted before?
   defp old_user?(%User{} = u) do
-    u.info.note_count > 0 || u.info.follower_count > 0
+    u.note_count > 0 || u.follower_count > 0
   end
 
   # does the post contain links?

lib/pleroma/web/activity_pub/publisher.ex

@@ -129,7 +129,7 @@ defp recipients(actor, activity) do
           []
       end
 
-    Pleroma.Web.Salmon.remote_users(actor, activity) ++ followers ++ fetchers
+    Pleroma.Web.Federator.Publisher.remote_users(actor, activity) ++ followers ++ fetchers
   end
 
   defp get_cc_ap_ids(ap_id, recipients) do
@@ -140,7 +140,7 @@ defp get_cc_ap_ids(ap_id, recipients) do
     |> Enum.map(& &1.ap_id)
   end
 
-  defp maybe_use_sharedinbox(%User{info: %{source_data: data}}),
+  defp maybe_use_sharedinbox(%User{source_data: data}),
     do: (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"]
 
   @doc """
@@ -156,7 +156,7 @@ defp maybe_use_sharedinbox(%User{info: %{source_data: data}}),
   """
   def determine_inbox(
         %Activity{data: activity_data},
-        %User{info: %{source_data: data}} = user
+        %User{source_data: data} = user
       ) do
     to = activity_data["to"] || []
     cc = activity_data["cc"] || []
@@ -190,12 +190,12 @@ def publish(%User{} = actor, %{data: %{"bcc" => bcc}} = activity)
 
     recipients
     |> Enum.filter(&User.ap_enabled?/1)
-    |> Enum.map(fn %{info: %{source_data: data}} -> data["inbox"] end)
+    |> Enum.map(fn %{source_data: data} -> data["inbox"] end)
     |> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
     |> Instances.filter_reachable()
     |> Enum.each(fn {inbox, unreachable_since} ->
       %User{ap_id: ap_id} =
-        Enum.find(recipients, fn %{info: %{source_data: data}} -> data["inbox"] == inbox end)
+        Enum.find(recipients, fn %{source_data: data} -> data["inbox"] == inbox end)
 
       # Get all the recipients on the same host and add them to cc. Otherwise, a remote
       # instance would only accept a first message for the first recipient and ignore the rest.

lib/pleroma/web/activity_pub/relay.ex

@@ -10,8 +10,12 @@ defmodule Pleroma.Web.ActivityPub.Relay do
   require Logger
 
   def get_actor do
+    actor =
       "#{Pleroma.Web.Endpoint.url()}/relay"
       |> User.get_or_create_service_actor_by_ap_id()
+
+    {:ok, actor} = User.set_invisible(actor, true)
+    actor
   end
 
   @spec follow(String.t()) :: {:ok, Activity.t()} | {:error, any()}
@@ -51,6 +55,20 @@ def publish(%Activity{data: %{"type" => "Create"}} = activity) do
 
   def publish(_), do: {:error, "Not implemented"}
 
+  @spec list() :: {:ok, [String.t()]} | {:error, any()}
+  def list do
+    with %User{following: following} = _user <- get_actor() do
+      list =
+        following
+        |> Enum.map(fn entry -> URI.parse(entry).host end)
+        |> Enum.uniq()
+
+      {:ok, list}
+    else
+      error -> format_error(error)
+    end
+  end
+
   defp format_error({:error, error}), do: format_error(error)
 
   defp format_error(error) do

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -596,13 +596,18 @@ def handle_incoming(
           data,
         _options
       )
-      when object_type in ["Person", "Application", "Service", "Organization"] do
+      when object_type in [
+             "Person",
+             "Application",
+             "Service",
+             "Organization"
+           ] do
     with %User{ap_id: ^actor_id} = actor <- User.get_cached_by_ap_id(object["id"]) do
       {:ok, new_user_data} = ActivityPub.user_data_from_user_object(object)
 
-      banner = new_user_data[:info][:banner]
+      locked = new_user_data[:locked] || false
-      locked = new_user_data[:info][:locked] || false
+      attachment = get_in(new_user_data, [:source_data, "attachment"]) || []
-      attachment = get_in(new_user_data, [:info, :source_data, "attachment"]) || []
+      invisible = new_user_data[:invisible] || false
 
       fields =
         attachment
@@ -611,8 +616,10 @@ def handle_incoming(
 
       update_data =
         new_user_data
-        |> Map.take([:name, :bio, :avatar])
+        |> Map.take([:avatar, :banner, :bio, :name])
-        |> Map.put(:info, %{banner: banner, locked: locked, fields: fields})
+        |> Map.put(:fields, fields)
+        |> Map.put(:locked, locked)
+        |> Map.put(:invisible, invisible)
 
       actor
       |> User.upgrade_changeset(update_data, true)
@@ -979,7 +986,7 @@ defp build_mention_tag(%{ap_id: ap_id, nickname: nickname} = _) do
     %{"type" => "Mention", "href" => ap_id, "name" => "@#{nickname}"}
   end
 
-  def take_emoji_tags(%User{info: %{emoji: emoji} = _user_info} = _user) do
+  def take_emoji_tags(%User{emoji: emoji}) do
     emoji
     |> Enum.flat_map(&Map.to_list/1)
     |> Enum.map(&build_emoji_tag/1)
@@ -1073,8 +1080,6 @@ def perform(:user_upgrade, user) do
 
     Repo.update_all(q, [])
 
-    maybe_retire_websub(user.ap_id)
-
     q =
       from(
         a in Activity,
@@ -1117,19 +1122,6 @@ defp upgrade_user(user, data) do
     |> User.update_and_set_cache()
   end
 
-  def maybe_retire_websub(ap_id) do
-    # some sanity checks
-    if is_binary(ap_id) && String.length(ap_id) > 8 do
-      q =
-        from(
-          ws in Pleroma.Web.Websub.WebsubClientSubscription,
-          where: fragment("? like ?", ws.topic, ^"#{ap_id}%")
-        )
-
-      Repo.delete_all(q)
-    end
-  end
-
   def maybe_fix_user_url(%{"url" => url} = data) when is_map(url) do
     Map.put(data, "url", url["href"])
   end

lib/pleroma/web/activity_pub/utils.ex

@@ -51,26 +51,28 @@ def determine_explicit_mentions(%{"tag" => tag} = object) when is_map(tag) do
 
   def determine_explicit_mentions(_), do: []
 
-  @spec recipient_in_collection(any(), any()) :: boolean()
+  @spec label_in_collection?(any(), any()) :: boolean()
-  defp recipient_in_collection(ap_id, coll) when is_binary(coll), do: ap_id == coll
+  defp label_in_collection?(ap_id, coll) when is_binary(coll), do: ap_id == coll
-  defp recipient_in_collection(ap_id, coll) when is_list(coll), do: ap_id in coll
+  defp label_in_collection?(ap_id, coll) when is_list(coll), do: ap_id in coll
-  defp recipient_in_collection(_, _), do: false
+  defp label_in_collection?(_, _), do: false
+
+  @spec label_in_message?(String.t(), map()) :: boolean()
+  def label_in_message?(label, params),
+    do:
+      [params["to"], params["cc"], params["bto"], params["bcc"]]
+      |> Enum.any?(&label_in_collection?(label, &1))
+
+  @spec unaddressed_message?(map()) :: boolean()
+  def unaddressed_message?(params),
+    do:
+      [params["to"], params["cc"], params["bto"], params["bcc"]]
+      |> Enum.all?(&is_nil(&1))
 
   @spec recipient_in_message(User.t(), User.t(), map()) :: boolean()
-  def recipient_in_message(%User{ap_id: ap_id} = recipient, %User{} = actor, params) do
+  def recipient_in_message(%User{ap_id: ap_id} = recipient, %User{} = actor, params),
-    addresses = [params["to"], params["cc"], params["bto"], params["bcc"]]
+    do:
-
+      label_in_message?(ap_id, params) || unaddressed_message?(params) ||
-    cond do
+        User.following?(recipient, actor)
-      Enum.any?(addresses, &recipient_in_collection(ap_id, &1)) -> true
-      # if the message is unaddressed at all, then assume it is directly addressed
-      # to the recipient
-      Enum.all?(addresses, &is_nil(&1)) -> true
-      # if the message is sent from somebody the user is following, then assume it
-      # is addressed to the recipient
-      User.following?(recipient, actor) -> true
-      true -> false
-    end
-  end
 
   defp extract_list(target) when is_binary(target), do: [target]
   defp extract_list(lst) when is_list(lst), do: lst
@@ -78,8 +80,8 @@ defp extract_list(_), do: []
 
   def maybe_splice_recipient(ap_id, params) do
     need_splice? =
-      !recipient_in_collection(ap_id, params["to"]) &&
+      !label_in_collection?(ap_id, params["to"]) &&
-        !recipient_in_collection(ap_id, params["cc"])
+        !label_in_collection?(ap_id, params["cc"])
 
     if need_splice? do
       cc_list = extract_list(params["cc"])
@@ -493,11 +495,15 @@ def add_announce_to_object(
         %Activity{data: %{"actor" => actor}},
         object
       ) do
+    unless actor |> User.get_cached_by_ap_id() |> User.invisible?() do
       announcements = take_announcements(object)
 
       with announcements <- Enum.uniq([actor | announcements]) do
         update_element_in_object("announcement", announcements, object)
       end
+    else
+      {:ok, object}
+    end
   end
 
   def add_announce_to_object(_, object), do: {:ok, object}

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -55,7 +55,8 @@ def render("service.json", %{user: user}) do
         "owner" => user.ap_id,
         "publicKeyPem" => public_key
       },
-      "endpoints" => endpoints
+      "endpoints" => endpoints,
+      "invisible" => User.invisible?(user)
     }
     |> Map.merge(Utils.make_json_ld_header())
   end
@@ -78,8 +79,8 @@ def render("user.json", %{user: user}) do
     emoji_tags = Transmogrifier.take_emoji_tags(user)
 
     fields =
-      user.info
+      user
-      |> User.Info.fields()
+      |> User.fields()
       |> Enum.map(fn %{"name" => name, "value" => value} ->
         %{
           "name" => Pleroma.HTML.strip_tags(name),
@@ -99,7 +100,7 @@ def render("user.json", %{user: user}) do
       "name" => user.name,
       "summary" => user.bio,
       "url" => user.ap_id,
-      "manuallyApprovesFollowers" => user.info.locked,
+      "manuallyApprovesFollowers" => user.locked,
       "publicKey" => %{
         "id" => "#{user.ap_id}#main-key",
         "owner" => user.ap_id,
@@ -107,8 +108,8 @@ def render("user.json", %{user: user}) do
       },
       "endpoints" => endpoints,
       "attachment" => fields,
-      "tag" => (user.info.source_data["tag"] || []) ++ emoji_tags,
+      "tag" => (user.source_data["tag"] || []) ++ emoji_tags,
-      "discoverable" => user.info.discoverable
+      "discoverable" => user.discoverable
     }
     |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))
     |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user))
@@ -116,8 +117,8 @@ def render("user.json", %{user: user}) do
   end
 
   def render("following.json", %{user: user, page: page} = opts) do
-    showing_items = (opts[:for] && opts[:for] == user) || !user.info.hide_follows
+    showing_items = (opts[:for] && opts[:for] == user) || !user.hide_follows
-    showing_count = showing_items || !user.info.hide_follows_count
+    showing_count = showing_items || !user.hide_follows_count
 
     query = User.get_friends_query(user)
     query = from(user in query, select: [:ap_id])
@@ -135,8 +136,8 @@ def render("following.json", %{user: user, page: page} = opts) do
   end
 
   def render("following.json", %{user: user} = opts) do
-    showing_items = (opts[:for] && opts[:for] == user) || !user.info.hide_follows
+    showing_items = (opts[:for] && opts[:for] == user) || !user.hide_follows
-    showing_count = showing_items || !user.info.hide_follows_count
+    showing_count = showing_items || !user.hide_follows_count
 
     query = User.get_friends_query(user)
     query = from(user in query, select: [:ap_id])
@@ -155,7 +156,7 @@ def render("following.json", %{user: user} = opts) do
       "totalItems" => total,
       "first" =>
         if showing_items do
-          collection(following, "#{user.ap_id}/following", 1, !user.info.hide_follows)
+          collection(following, "#{user.ap_id}/following", 1, !user.hide_follows)
         else
           "#{user.ap_id}/following?page=1"
         end
@@ -164,8 +165,8 @@ def render("following.json", %{user: user} = opts) do
   end
 
   def render("followers.json", %{user: user, page: page} = opts) do
-    showing_items = (opts[:for] && opts[:for] == user) || !user.info.hide_followers
+    showing_items = (opts[:for] && opts[:for] == user) || !user.hide_followers
-    showing_count = showing_items || !user.info.hide_followers_count
+    showing_count = showing_items || !user.hide_followers_count
 
     query = User.get_followers_query(user)
     query = from(user in query, select: [:ap_id])
@@ -183,8 +184,8 @@ def render("followers.json", %{user: user, page: page} = opts) do
   end
 
   def render("followers.json", %{user: user} = opts) do
-    showing_items = (opts[:for] && opts[:for] == user) || !user.info.hide_followers
+    showing_items = (opts[:for] && opts[:for] == user) || !user.hide_followers
-    showing_count = showing_items || !user.info.hide_followers_count
+    showing_count = showing_items || !user.hide_followers_count
 
     query = User.get_followers_query(user)
     query = from(user in query, select: [:ap_id])

lib/pleroma/web/activity_pub/visibility.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Visibility do
   alias Pleroma.Object
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.Utils
 
   require Pleroma.Constants
 
@@ -15,7 +16,7 @@ def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false
   def is_public?(%Object{data: data}), do: is_public?(data)
   def is_public?(%Activity{data: data}), do: is_public?(data)
   def is_public?(%{"directMessage" => true}), do: false
-  def is_public?(data), do: Pleroma.Constants.as_public() in (data["to"] ++ (data["cc"] || []))
+  def is_public?(data), do: Utils.label_in_message?(Pleroma.Constants.as_public(), data)
 
   def is_private?(activity) do
     with false <- is_public?(activity),

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -46,11 +46,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
            :user_delete,
            :users_create,
            :user_toggle_activation,
+           :user_activate,
+           :user_deactivate,
            :tag_users,
            :untag_users,
            :right_add,
-           :right_delete,
+           :right_delete
-           :set_activation_status
          ]
   )
 
@@ -98,7 +99,7 @@ def user_delete(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
 
     ModerationLog.insert_log(%{
       actor: admin,
-      subject: user,
+      subject: [user],
       action: "delete"
     })
 
@@ -106,6 +107,20 @@ def user_delete(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
     |> json(nickname)
   end
 
+  def user_delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = nicknames |> Enum.map(&User.get_cached_by_nickname/1)
+    User.delete(users)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "delete"
+    })
+
+    conn
+    |> json(nicknames)
+  end
+
   def user_follow(%{assigns: %{user: admin}} = conn, %{
         "follower" => follower_nick,
         "followed" => followed_nick
@@ -234,13 +249,13 @@ def list_user_statuses(conn, %{"nickname" => nickname} = params) do
   def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
     user = User.get_cached_by_nickname(nickname)
 
-    {:ok, updated_user} = User.deactivate(user, !user.info.deactivated)
+    {:ok, updated_user} = User.deactivate(user, !user.deactivated)
 
-    action = if user.info.deactivated, do: "activate", else: "deactivate"
+    action = if user.deactivated, do: "activate", else: "deactivate"
 
     ModerationLog.insert_log(%{
       actor: admin,
-      subject: user,
+      subject: [user],
       action: action
     })
 
@@ -249,6 +264,36 @@ def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => ni
     |> render("show.json", %{user: updated_user})
   end
 
+  def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.deactivate(users, false)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "activate"
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: Keyword.values(updated_users)})
+  end
+
+  def user_deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.deactivate(users, true)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "deactivate"
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: Keyword.values(updated_users)})
+  end
+
   def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do
     with {:ok, _} <- User.tag(nicknames, tags) do
       ModerationLog.insert_log(%{
@@ -313,26 +358,51 @@ defp maybe_parse_filters(filters) do
     |> Enum.into(%{}, &{&1, true})
   end
 
+  def right_add_multiple(%{assigns: %{user: admin}} = conn, %{
+        "permission_group" => permission_group,
+        "nicknames" => nicknames
+      })
+      when permission_group in ["moderator", "admin"] do
+    update = %{:"is_#{permission_group}" => true}
+
+    users = nicknames |> Enum.map(&User.get_cached_by_nickname/1)
+
+    for u <- users, do: User.admin_api_update(u, update)
+
+    ModerationLog.insert_log(%{
+      action: "grant",
+      actor: admin,
+      subject: users,
+      permission: permission_group
+    })
+
+    json(conn, update)
+  end
+
+  def right_add_multiple(conn, _) do
+    render_error(conn, :not_found, "No such permission_group")
+  end
+
   def right_add(%{assigns: %{user: admin}} = conn, %{
         "permission_group" => permission_group,
         "nickname" => nickname
       })
       when permission_group in ["moderator", "admin"] do
-    info = Map.put(%{}, "is_" <> permission_group, true)
+    fields = %{:"is_#{permission_group}" => true}
 
     {:ok, user} =
       nickname
       |> User.get_cached_by_nickname()
-      |> User.update_info(&User.Info.admin_api_update(&1, info))
+      |> User.admin_api_update(fields)
 
     ModerationLog.insert_log(%{
       action: "grant",
       actor: admin,
-      subject: user,
+      subject: [user],
       permission: permission_group
     })
 
-    json(conn, info)
+    json(conn, fields)
   end
 
   def right_add(conn, _) do
@@ -344,13 +414,41 @@ def right_get(conn, %{"nickname" => nickname}) do
 
     conn
     |> json(%{
-      is_moderator: user.info.is_moderator,
+      is_moderator: user.is_moderator,
-      is_admin: user.info.is_admin
+      is_admin: user.is_admin
     })
   end
 
-  def right_delete(%{assigns: %{user: %{nickname: nickname}}} = conn, %{"nickname" => nickname}) do
+  def right_delete_multiple(
-    render_error(conn, :forbidden, "You can't revoke your own admin status.")
+        %{assigns: %{user: %{nickname: admin_nickname} = admin}} = conn,
+        %{
+          "permission_group" => permission_group,
+          "nicknames" => nicknames
+        }
+      )
+      when permission_group in ["moderator", "admin"] do
+    with false <- Enum.member?(nicknames, admin_nickname) do
+      update = %{:"is_#{permission_group}" => false}
+
+      users = nicknames |> Enum.map(&User.get_cached_by_nickname/1)
+
+      for u <- users, do: User.admin_api_update(u, update)
+
+      ModerationLog.insert_log(%{
+        action: "revoke",
+        actor: admin,
+        subject: users,
+        permission: permission_group
+      })
+
+      json(conn, update)
+    else
+      _ -> render_error(conn, :forbidden, "You can't revoke your own admin/moderator status.")
+    end
+  end
+
+  def right_delete_multiple(conn, _) do
+    render_error(conn, :not_found, "No such permission_group")
   end
 
   def right_delete(
@@ -361,43 +459,34 @@ def right_delete(
         }
       )
       when permission_group in ["moderator", "admin"] do
-    info = Map.put(%{}, "is_" <> permission_group, false)
+    fields = %{:"is_#{permission_group}" => false}
 
     {:ok, user} =
       nickname
       |> User.get_cached_by_nickname()
-      |> User.update_info(&User.Info.admin_api_update(&1, info))
+      |> User.admin_api_update(fields)
 
     ModerationLog.insert_log(%{
       action: "revoke",
       actor: admin,
-      subject: user,
+      subject: [user],
       permission: permission_group
     })
 
-    json(conn, info)
+    json(conn, fields)
   end
 
-  def right_delete(conn, _) do
+  def right_delete(%{assigns: %{user: %{nickname: nickname}}} = conn, %{"nickname" => nickname}) do
-    render_error(conn, :not_found, "No such permission_group")
+    render_error(conn, :forbidden, "You can't revoke your own admin status.")
   end
 
-  def set_activation_status(%{assigns: %{user: admin}} = conn, %{
+  def relay_list(conn, _params) do
-        "nickname" => nickname,
+    with {:ok, list} <- Relay.list() do
-        "status" => status
+      json(conn, %{relays: list})
-      }) do
+    else
-    with {:ok, status} <- Ecto.Type.cast(:boolean, status),
+      _ ->
-         %User{} = user <- User.get_cached_by_nickname(nickname),
+        conn
-         {:ok, _} <- User.deactivate(user, !status) do
+        |> put_status(500)
-      action = if(user.info.deactivated, do: "activate", else: "deactivate")
-
-      ModerationLog.insert_log(%{
-        actor: admin,
-        subject: user,
-        action: action
-      })
-
-      json_response(conn, :no_content, "")
     end
   end
 

lib/pleroma/web/admin_api/views/account_view.ex

@@ -7,7 +7,6 @@ defmodule Pleroma.Web.AdminAPI.AccountView do
 
   alias Pleroma.HTML
   alias Pleroma.User
-  alias Pleroma.User.Info
   alias Pleroma.Web.AdminAPI.AccountView
   alias Pleroma.Web.MediaProxy
 
@@ -19,6 +18,12 @@ def render("index.json", %{users: users, count: count, page_size: page_size}) do
     }
   end
 
+  def render("index.json", %{users: users}) do
+    %{
+      users: render_many(users, AccountView, "show.json", as: :user)
+    }
+  end
+
   def render("show.json", %{user: user}) do
     avatar = User.avatar_url(user) |> MediaProxy.url()
     display_name = HTML.strip_tags(user.name || user.nickname)
@@ -28,9 +33,9 @@ def render("show.json", %{user: user}) do
       "avatar" => avatar,
       "nickname" => user.nickname,
       "display_name" => display_name,
-      "deactivated" => user.info.deactivated,
+      "deactivated" => user.deactivated,
       "local" => user.local,
-      "roles" => Info.roles(user.info),
+      "roles" => User.roles(user),
       "tags" => user.tags || []
     }
   end

lib/pleroma/web/common_api/common_api.ex

@@ -263,10 +263,10 @@ defp maybe_create_activity_expiration(result, _), do: result
   # Updates the emojis for a user based on their profile
   def update(user) do
     emoji = emoji_from_profile(user)
-    source_data = user.info |> Map.get(:source_data, %{}) |> Map.put("tag", emoji)
+    source_data = Map.put(user.source_data, "tag", emoji)
 
     user =
-      case User.update_info(user, &User.Info.set_source_data(&1, source_data)) do
+      case User.update_source_data(user, source_data) do
         {:ok, user} -> user
         _ -> user
       end
@@ -287,20 +287,20 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do
            object: %Object{data: %{"type" => "Note"}}
          } = activity <- get_by_id_or_ap_id(id_or_ap_id),
          true <- Visibility.is_public?(activity),
-         {:ok, _user} <- User.update_info(user, &User.Info.add_pinnned_activity(&1, activity)) do
+         {:ok, _user} <- User.add_pinnned_activity(user, activity) do
       {:ok, activity}
     else
-      {:error, %{changes: %{info: %{errors: [pinned_activities: {err, _}]}}}} -> {:error, err}
+      {:error, %{errors: [pinned_activities: {err, _}]}} -> {:error, err}
       _ -> {:error, dgettext("errors", "Could not pin")}
     end
   end
 
   def unpin(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         {:ok, _user} <- User.update_info(user, &User.Info.remove_pinnned_activity(&1, activity)) do
+         {:ok, _user} <- User.remove_pinnned_activity(user, activity) do
       {:ok, activity}
     else
-      %{errors: [pinned_activities: {err, _}]} -> {:error, err}
+      {:error, %{errors: [pinned_activities: {err, _}]}} -> {:error, err}
       _ -> {:error, dgettext("errors", "Could not unpin")}
     end
   end
@@ -392,14 +392,14 @@ defp set_visibility(activity, %{"visibility" => visibility}) do
   defp set_visibility(activity, _), do: {:ok, activity}
 
   def hide_reblogs(user, %{ap_id: ap_id} = _muted) do
-    if ap_id not in user.info.muted_reblogs do
+    if ap_id not in user.muted_reblogs do
-      User.update_info(user, &User.Info.add_reblog_mute(&1, ap_id))
+      User.add_reblog_mute(user, ap_id)
     end
   end
 
   def show_reblogs(user, %{ap_id: ap_id} = _muted) do
-    if ap_id in user.info.muted_reblogs do
+    if ap_id in user.muted_reblogs do
-      User.update_info(user, &User.Info.remove_reblog_mute(&1, ap_id))
+      User.remove_reblog_mute(user, ap_id)
     end
   end
 end

lib/pleroma/web/federator/federator.ex

@@ -10,19 +10,11 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Federator.Publisher
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.Websub
   alias Pleroma.Workers.PublisherWorker
   alias Pleroma.Workers.ReceiverWorker
-  alias Pleroma.Workers.SubscriberWorker
 
   require Logger
 
-  def init do
-    # To do: consider removing this call in favor of scheduled execution (`quantum`-based)
-    refresh_subscriptions(schedule_in: 60)
-  end
-
   @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)"
   # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
   def allowed_incoming_reply_depth?(depth) do
@@ -37,10 +29,6 @@ def allowed_incoming_reply_depth?(depth) do
 
   # Client API
 
-  def incoming_doc(doc) do
-    ReceiverWorker.enqueue("incoming_doc", %{"body" => doc})
-  end
-
   def incoming_ap_doc(params) do
     ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params})
   end
@@ -53,18 +41,6 @@ def publish(activity) do
     PublisherWorker.enqueue("publish", %{"activity_id" => activity.id})
   end
 
-  def verify_websub(websub) do
-    SubscriberWorker.enqueue("verify_websub", %{"websub_id" => websub.id})
-  end
-
-  def request_subscription(websub) do
-    SubscriberWorker.enqueue("request_subscription", %{"websub_id" => websub.id})
-  end
-
-  def refresh_subscriptions(worker_args \\ []) do
-    SubscriberWorker.enqueue("refresh_subscriptions", %{}, worker_args ++ [max_attempts: 1])
-  end
-
   # Job Worker Callbacks
 
   @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()}
@@ -81,11 +57,6 @@ def perform(:publish, activity) do
     end
   end
 
-  def perform(:incoming_doc, doc) do
-    Logger.info("Got document, trying to parse")
-    OStatus.handle_incoming(doc)
-  end
-
   def perform(:incoming_ap_doc, params) do
     Logger.info("Handling incoming AP activity")
 
@@ -111,29 +82,6 @@ def perform(:incoming_ap_doc, params) do
     end
   end
 
-  def perform(:request_subscription, websub) do
-    Logger.debug("Refreshing #{websub.topic}")
-
-    with {:ok, websub} <- Websub.request_subscription(websub) do
-      Logger.debug("Successfully refreshed #{websub.topic}")
-    else
-      _e -> Logger.debug("Couldn't refresh #{websub.topic}")
-    end
-  end
-
-  def perform(:verify_websub, websub) do
-    Logger.debug(fn ->
-      "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})"
-    end)
-
-    Websub.verify(websub)
-  end
-
-  def perform(:refresh_subscriptions) do
-    Logger.debug("Federator running refresh subscriptions")
-    Websub.refresh_subscriptions()
-  end
-
   def ap_enabled_actor(id) do
     user = User.get_cached_by_ap_id(id)
 

lib/pleroma/web/federator/publisher.ex

@@ -80,4 +80,30 @@ def gather_nodeinfo_protocol_names do
       links ++ module.gather_nodeinfo_protocol_names()
     end)
   end
+
+  @doc """
+  Gathers a set of remote users given an IR envelope.
+  """
+  def remote_users(%User{id: user_id}, %{data: %{"to" => to} = data}) do
+    cc = Map.get(data, "cc", [])
+
+    bcc =
+      data
+      |> Map.get("bcc", [])
+      |> Enum.reduce([], fn ap_id, bcc ->
+        case Pleroma.List.get_by_ap_id(ap_id) do
+          %Pleroma.List{user_id: ^user_id} = list ->
+            {:ok, following} = Pleroma.List.get_following(list)
+            bcc ++ Enum.map(following, & &1.ap_id)
+
+          _ ->
+            bcc
+        end
+      end)
+
+    [to, cc, bcc]
+    |> Enum.concat()
+    |> Enum.map(&User.get_cached_by_ap_id/1)
+    |> Enum.filter(fn user -> user && !user.local end)
+  end
 end

lib/pleroma/web/masto_fe_controller.ex

@@ -34,9 +34,15 @@ def index(%{assigns: %{user: user}} = conn, _params) do
     end
   end
 
+  @doc "GET /web/manifest.json"
+  def manifest(conn, _params) do
+    conn
+    |> render("manifest.json")
+  end
+
   @doc "PUT /api/web/settings"
   def put_settings(%{assigns: %{user: user}} = conn, %{"data" => settings} = _params) do
-    with {:ok, _} <- User.update_info(user, &User.Info.mastodon_settings_update(&1, settings)) do
+    with {:ok, _} <- User.mastodon_settings_update(user, settings) do
       json(conn, %{})
     else
       e ->

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -130,25 +130,6 @@ def verify_credentials(%{assigns: %{user: user}} = conn, _) do
   def update_credentials(%{assigns: %{user: original_user}} = conn, params) do
     user = original_user
 
-    user_params =
-      %{}
-      |> add_if_present(params, "display_name", :name)
-      |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value, user)} end)
-      |> add_if_present(params, "avatar", :avatar, fn value ->
-        with %Plug.Upload{} <- value,
-             {:ok, object} <- ActivityPub.upload(value, type: :avatar) do
-          {:ok, object.data}
-        end
-      end)
-
-    emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "")
-
-    user_info_emojis =
-      user.info
-      |> Map.get(:emoji, [])
-      |> Enum.concat(Emoji.Formatter.get_emoji_map(emojis_text))
-      |> Enum.dedup()
-
     params =
       if Map.has_key?(params, "fields_attributes") do
         Map.update!(params, "fields_attributes", fn fields ->
@@ -160,7 +141,7 @@ def update_credentials(%{assigns: %{user: original_user}} = conn, params) do
         params
       end
 
-    info_params =
+    user_params =
       [
         :no_rich_text,
         :locked,
@@ -176,15 +157,13 @@ def update_credentials(%{assigns: %{user: original_user}} = conn, params) do
       |> Enum.reduce(%{}, fn key, acc ->
         add_if_present(acc, params, to_string(key), key, &{:ok, truthy_param?(&1)})
       end)
-      |> add_if_present(params, "default_scope", :default_scope)
+      |> add_if_present(params, "display_name", :name)
-      |> add_if_present(params, "fields_attributes", :fields, fn fields ->
+      |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value, user)} end)
-        fields = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end)
+      |> add_if_present(params, "avatar", :avatar, fn value ->
-
+        with %Plug.Upload{} <- value,
-        {:ok, fields}
+             {:ok, object} <- ActivityPub.upload(value, type: :avatar) do
-      end)
+          {:ok, object.data}
-      |> add_if_present(params, "fields_attributes", :raw_fields)
+        end
-      |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value ->
-        {:ok, Map.merge(user.info.pleroma_settings_store, value)}
       end)
       |> add_if_present(params, "header", :banner, fn value ->
         with %Plug.Upload{} <- value,
@@ -198,12 +177,27 @@ def update_credentials(%{assigns: %{user: original_user}} = conn, params) do
           {:ok, object.data}
         end
       end)
-      |> Map.put(:emoji, user_info_emojis)
+      |> add_if_present(params, "fields_attributes", :fields, fn fields ->
+        fields = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end)
 
-    changeset =
+        {:ok, fields}
+      end)
+      |> add_if_present(params, "fields_attributes", :raw_fields)
+      |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value ->
+        {:ok, Map.merge(user.pleroma_settings_store, value)}
+      end)
+      |> add_if_present(params, "default_scope", :default_scope)
+
+    emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "")
+
+    user_emojis =
       user
-      |> User.update_changeset(user_params)
+      |> Map.get(:emoji, [])
-      |> User.change_info(&User.Info.profile_update(&1, info_params))
+      |> Enum.concat(Emoji.Formatter.get_emoji_map(emojis_text))
+      |> Enum.dedup()
+
+    user_params = Map.put(user_params, :emoji, user_emojis)
+    changeset = User.update_changeset(user, user_params)
 
     with {:ok, user} <- User.update_and_set_cache(changeset) do
       if original_user != user, do: CommonAPI.update(user)
@@ -269,7 +263,7 @@ def followers(%{assigns: %{user: for_user, account: user}} = conn, params) do
     followers =
       cond do
         for_user && user.id == for_user.id -> MastodonAPI.get_followers(user, params)
-        user.info.hide_followers -> []
+        user.hide_followers -> []
         true -> MastodonAPI.get_followers(user, params)
       end
 
@@ -283,7 +277,7 @@ def following(%{assigns: %{user: for_user, account: user}} = conn, params) do
     followers =
       cond do
         for_user && user.id == for_user.id -> MastodonAPI.get_friends(user, params)
-        user.info.hide_follows -> []
+        user.hide_follows -> []
         true -> MastodonAPI.get_friends(user, params)
       end
 

lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex

@@ -21,8 +21,8 @@ defmodule Pleroma.Web.MastodonAPI.DomainBlockController do
   plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug)
 
   @doc "GET /api/v1/domain_blocks"
-  def index(%{assigns: %{user: %{info: info}}} = conn, _) do
+  def index(%{assigns: %{user: user}} = conn, _) do
-    json(conn, Map.get(info, :domain_blocks, []))
+    json(conn, Map.get(user, :domain_blocks, []))
   end
 
   @doc "POST /api/v1/domain_blocks"

lib/pleroma/web/mastodon_api/controllers/marker_controller.ex

@@ -0,0 +1,32 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MastodonAPI.MarkerController do
+  use Pleroma.Web, :controller
+  alias Pleroma.Plugs.OAuthScopesPlug
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:statuses"]}
+    when action == :index
+  )
+
+  plug(OAuthScopesPlug, %{scopes: ["write:statuses"]} when action == :upsert)
+  plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug)
+  action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
+
+  # GET /api/v1/markers
+  def index(%{assigns: %{user: user}} = conn, params) do
+    markers = Pleroma.Marker.get_markers(user, params["timeline"])
+    render(conn, "markers.json", %{markers: markers})
+  end
+
+  # POST /api/v1/markers
+  def upsert(%{assigns: %{user: user}} = conn, params) do
+    with {:ok, result} <- Pleroma.Marker.upsert(user, params),
+         markers <- Map.values(result) do
+      render(conn, "markers.json", %{markers: markers})
+    end
+  end
+end

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -74,23 +74,23 @@ defp do_render("show.json", %{user: user} = opts) do
     user_info = User.get_cached_user_info(user)
 
     following_count =
-      if !user.info.hide_follows_count or !user.info.hide_follows or opts[:for] == user do
+      if !user.hide_follows_count or !user.hide_follows or opts[:for] == user do
         user_info.following_count
       else
         0
       end
 
     followers_count =
-      if !user.info.hide_followers_count or !user.info.hide_followers or opts[:for] == user do
+      if !user.hide_followers_count or !user.hide_followers or opts[:for] == user do
         user_info.follower_count
       else
         0
       end
 
-    bot = (user.info.source_data["type"] || "Person") in ["Application", "Service"]
+    bot = (user.source_data["type"] || "Person") in ["Application", "Service"]
 
     emojis =
-      (user.info.source_data["tag"] || [])
+      (user.source_data["tag"] || [])
       |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end)
       |> Enum.map(fn %{"icon" => %{"url" => url}, "name" => name} ->
         %{
@@ -102,8 +102,8 @@ defp do_render("show.json", %{user: user} = opts) do
       end)
 
     fields =
-      user.info
+      user
-      |> User.Info.fields()
+      |> User.fields()
       |> Enum.map(fn %{"name" => name, "value" => value} ->
         %{
           "name" => Pleroma.HTML.strip_tags(name),
@@ -111,23 +111,19 @@ defp do_render("show.json", %{user: user} = opts) do
         }
       end)
 
-    raw_fields = Map.get(user.info, :raw_fields, [])
-
     bio = HTML.filter_tags(user.bio, User.html_filter_policy(opts[:for]))
     relationship = render("relationship.json", %{user: opts[:for], target: user})
 
-    discoverable = user.info.discoverable
-
     %{
       id: to_string(user.id),
       username: username_from_nickname(user.nickname),
       acct: user.nickname,
       display_name: display_name,
-      locked: user_info.locked,
+      locked: user.locked,
       created_at: Utils.to_masto_date(user.inserted_at),
       followers_count: followers_count,
       following_count: following_count,
-      statuses_count: user_info.note_count,
+      statuses_count: user.note_count,
       note: bio || "",
       url: User.profile_url(user),
       avatar: image,
@@ -140,9 +136,9 @@ defp do_render("show.json", %{user: user} = opts) do
       source: %{
         note: HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
         sensitive: false,
-        fields: raw_fields,
+        fields: user.raw_fields,
         pleroma: %{
-          discoverable: discoverable
+          discoverable: user.discoverable
         }
       },
 
@@ -150,14 +146,14 @@ defp do_render("show.json", %{user: user} = opts) do
       pleroma: %{
         confirmation_pending: user_info.confirmation_pending,
         tags: user.tags,
-        hide_followers_count: user.info.hide_followers_count,
+        hide_followers_count: user.hide_followers_count,
-        hide_follows_count: user.info.hide_follows_count,
+        hide_follows_count: user.hide_follows_count,
-        hide_followers: user.info.hide_followers,
+        hide_followers: user.hide_followers,
-        hide_follows: user.info.hide_follows,
+        hide_follows: user.hide_follows,
-        hide_favorites: user.info.hide_favorites,
+        hide_favorites: user.hide_favorites,
         relationship: relationship,
-        skip_thread_containment: user.info.skip_thread_containment,
+        skip_thread_containment: user.skip_thread_containment,
-        background_image: image_url(user.info.background) |> MediaProxy.url()
+        background_image: image_url(user.background) |> MediaProxy.url()
       }
     }
     |> maybe_put_role(user, opts[:for])
@@ -195,21 +191,21 @@ defp maybe_put_settings(
          data,
          %User{id: user_id} = user,
          %User{id: user_id},
-         user_info
+         _user_info
        ) do
     data
-    |> Kernel.put_in([:source, :privacy], user_info.default_scope)
+    |> Kernel.put_in([:source, :privacy], user.default_scope)
-    |> Kernel.put_in([:source, :pleroma, :show_role], user.info.show_role)
+    |> Kernel.put_in([:source, :pleroma, :show_role], user.show_role)
-    |> Kernel.put_in([:source, :pleroma, :no_rich_text], user.info.no_rich_text)
+    |> Kernel.put_in([:source, :pleroma, :no_rich_text], user.no_rich_text)
   end
 
   defp maybe_put_settings(data, _, _, _), do: data
 
-  defp maybe_put_settings_store(data, %User{info: info, id: id}, %User{id: id}, %{
+  defp maybe_put_settings_store(data, %User{} = user, %User{}, %{
          with_pleroma_settings: true
        }) do
     data
-    |> Kernel.put_in([:pleroma, :settings_store], info.pleroma_settings_store)
+    |> Kernel.put_in([:pleroma, :settings_store], user.pleroma_settings_store)
   end
 
   defp maybe_put_settings_store(data, _, _, _), do: data
@@ -223,28 +219,28 @@ defp maybe_put_chat_token(data, %User{id: id}, %User{id: id}, %{
 
   defp maybe_put_chat_token(data, _, _, _), do: data
 
-  defp maybe_put_role(data, %User{info: %{show_role: true}} = user, _) do
+  defp maybe_put_role(data, %User{show_role: true} = user, _) do
     data
-    |> Kernel.put_in([:pleroma, :is_admin], user.info.is_admin)
+    |> Kernel.put_in([:pleroma, :is_admin], user.is_admin)
-    |> Kernel.put_in([:pleroma, :is_moderator], user.info.is_moderator)
+    |> Kernel.put_in([:pleroma, :is_moderator], user.is_moderator)
   end
 
   defp maybe_put_role(data, %User{id: user_id} = user, %User{id: user_id}) do
     data
-    |> Kernel.put_in([:pleroma, :is_admin], user.info.is_admin)
+    |> Kernel.put_in([:pleroma, :is_admin], user.is_admin)
-    |> Kernel.put_in([:pleroma, :is_moderator], user.info.is_moderator)
+    |> Kernel.put_in([:pleroma, :is_moderator], user.is_moderator)
   end
 
   defp maybe_put_role(data, _, _), do: data
 
   defp maybe_put_notification_settings(data, %User{id: user_id} = user, %User{id: user_id}) do
-    Kernel.put_in(data, [:pleroma, :notification_settings], user.info.notification_settings)
+    Kernel.put_in(data, [:pleroma, :notification_settings], user.notification_settings)
   end
 
   defp maybe_put_notification_settings(data, _, _), do: data
 
-  defp maybe_put_activation_status(data, user, %User{info: %{is_admin: true}}) do
+  defp maybe_put_activation_status(data, user, %User{is_admin: true}) do
-    Kernel.put_in(data, [:pleroma, :deactivated], user.info.deactivated)
+    Kernel.put_in(data, [:pleroma, :deactivated], user.deactivated)
   end
 
   defp maybe_put_activation_status(data, _, _), do: data
@@ -253,7 +249,7 @@ defp maybe_put_unread_conversation_count(data, %User{id: user_id} = user, %User{
     data
     |> Kernel.put_in(
       [:pleroma, :unread_conversation_count],
-      user.info.unread_conversation_count
+      user.unread_conversation_count
     )
   end
 

lib/pleroma/web/mastodon_api/views/marker_view.ex

@@ -0,0 +1,17 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MastodonAPI.MarkerView do
+  use Pleroma.Web, :view
+
+  def render("markers.json", %{markers: markers}) do
+    Enum.reduce(markers, %{}, fn m, acc ->
+      Map.put_new(acc, m.timeline, %{
+        last_read_id: m.last_read_id,
+        version: m.lock_version,
+        updated_at: NaiveDateTime.to_iso8601(m.updated_at)
+      })
+    end)
+  end
+end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -498,6 +498,6 @@ defp present?(nil), do: false
   defp present?(false), do: false
   defp present?(_), do: true
 
-  defp pinned?(%Activity{id: id}, %User{info: %{pinned_activities: pinned_activities}}),
+  defp pinned?(%Activity{id: id}, %User{pinned_activities: pinned_activities}),
     do: id in pinned_activities
 end

lib/pleroma/web/mastodon_api/websocket_handler.ex

@@ -35,6 +35,13 @@ def init(%{qs: qs} = req, state) do
          {_, stream} <- List.keyfind(params, "stream", 0),
          {:ok, user} <- allow_request(stream, [access_token, sec_websocket]),
          topic when is_binary(topic) <- expand_topic(stream, params) do
+      req =
+        if sec_websocket do
+          :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req)
+        else
+          req
+        end
+
       {:cowboy_websocket, req, %{user: user, topic: topic}, %{idle_timeout: @timeout}}
     else
       {:error, code} ->

lib/pleroma/web/oauth/oauth_controller.ex

@@ -202,9 +202,9 @@ def token_exchange(
     with {:ok, %User{} = user} <- Authenticator.get_user(conn),
          {:ok, app} <- Token.Utils.fetch_app(conn),
          {:auth_active, true} <- {:auth_active, User.auth_active?(user)},
-         {:user_active, true} <- {:user_active, !user.info.deactivated},
+         {:user_active, true} <- {:user_active, !user.deactivated},
          {:password_reset_pending, false} <-
-           {:password_reset_pending, user.info.password_reset_pending},
+           {:password_reset_pending, user.password_reset_pending},
          {:ok, scopes} <- validate_scopes(app, params),
          {:ok, auth} <- Authorization.create_authorization(app, user, scopes),
          {:ok, token} <- Token.exchange_token(app, auth) do

lib/pleroma/web/ostatus/activity_representer.ex

@@ -1,313 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.ActivityRepresenter do
-  alias Pleroma.Activity
-  alias Pleroma.Object
-  alias Pleroma.User
-  alias Pleroma.Web.OStatus.UserRepresenter
-
-  require Logger
-  require Pleroma.Constants
-
-  defp get_href(id) do
-    with %Object{data: %{"external_url" => external_url}} <- Object.get_cached_by_ap_id(id) do
-      external_url
-    else
-      _e -> id
-    end
-  end
-
-  defp get_in_reply_to(activity) do
-    with %Object{data: %{"inReplyTo" => in_reply_to}} <- Object.normalize(activity) do
-      [
-        {:"thr:in-reply-to",
-         [ref: to_charlist(in_reply_to), href: to_charlist(get_href(in_reply_to))], []}
-      ]
-    else
-      _ ->
-        []
-    end
-  end
-
-  defp get_mentions(to) do
-    Enum.map(to, fn id ->
-      cond do
-        # Special handling for the AP/Ostatus public collections
-        Pleroma.Constants.as_public() == id ->
-          {:link,
-           [
-             rel: "mentioned",
-             "ostatus:object-type": "http://activitystrea.ms/schema/1.0/collection",
-             href: "http://activityschema.org/collection/public"
-           ], []}
-
-        # Ostatus doesn't handle follower collections, ignore these.
-        Regex.match?(~r/^#{Pleroma.Web.base_url()}.+followers$/, id) ->
-          []
-
-        true ->
-          {:link,
-           [
-             rel: "mentioned",
-             "ostatus:object-type": "http://activitystrea.ms/schema/1.0/person",
-             href: id
-           ], []}
-      end
-    end)
-  end
-
-  defp get_links(%{local: true}, %{"id" => object_id}) do
-    h = fn str -> [to_charlist(str)] end
-
-    [
-      {:link, [type: ['application/atom+xml'], href: h.(object_id), rel: 'self'], []},
-      {:link, [type: ['text/html'], href: h.(object_id), rel: 'alternate'], []}
-    ]
-  end
-
-  defp get_links(%{local: false}, %{"external_url" => external_url}) do
-    h = fn str -> [to_charlist(str)] end
-
-    [
-      {:link, [type: ['text/html'], href: h.(external_url), rel: 'alternate'], []}
-    ]
-  end
-
-  defp get_links(_activity, _object_data), do: []
-
-  defp get_emoji_links(emojis) do
-    Enum.map(emojis, fn {emoji, file} ->
-      {:link, [name: to_charlist(emoji), rel: 'emoji', href: to_charlist(file)], []}
-    end)
-  end
-
-  def to_simple_form(activity, user, with_author \\ false)
-
-  def to_simple_form(%{data: %{"type" => "Create"}} = activity, user, with_author) do
-    h = fn str -> [to_charlist(str)] end
-
-    object = Object.normalize(activity)
-
-    updated_at = object.data["published"]
-    inserted_at = object.data["published"]
-
-    attachments =
-      Enum.map(object.data["attachment"] || [], fn attachment ->
-        url = hd(attachment["url"])
-
-        {:link,
-         [rel: 'enclosure', href: to_charlist(url["href"]), type: to_charlist(url["mediaType"])],
-         []}
-      end)
-
-    in_reply_to = get_in_reply_to(activity)
-    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
-    mentions = activity.recipients |> get_mentions
-
-    categories =
-      (object.data["tag"] || [])
-      |> Enum.map(fn tag ->
-        if is_binary(tag) do
-          {:category, [term: to_charlist(tag)], []}
-        else
-          nil
-        end
-      end)
-      |> Enum.filter(& &1)
-
-    emoji_links = get_emoji_links(object.data["emoji"] || %{})
-
-    summary =
-      if object.data["summary"] do
-        [{:summary, [], h.(object.data["summary"])}]
-      else
-        []
-      end
-
-    [
-      {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']},
-      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']},
-      # For notes, federate the object id.
-      {:id, h.(object.data["id"])},
-      {:title, ['New note by #{user.nickname}']},
-      {:content, [type: 'html'], h.(object.data["content"] |> String.replace(~r/[\n\r]/, ""))},
-      {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)},
-      {:"ostatus:conversation", [ref: h.(activity.data["context"])],
-       h.(activity.data["context"])},
-      {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}
-    ] ++
-      summary ++
-      get_links(activity, object.data) ++
-      categories ++ attachments ++ in_reply_to ++ author ++ mentions ++ emoji_links
-  end
-
-  def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) do
-    h = fn str -> [to_charlist(str)] end
-
-    updated_at = activity.data["published"]
-    inserted_at = activity.data["published"]
-
-    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
-    mentions = activity.recipients |> get_mentions
-
-    [
-      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/favorite']},
-      {:id, h.(activity.data["id"])},
-      {:title, ['New favorite by #{user.nickname}']},
-      {:content, [type: 'html'], ['#{user.nickname} favorited something']},
-      {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)},
-      {:"activity:object",
-       [
-         {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']},
-         # For notes, federate the object id.
-         {:id, h.(activity.data["object"])}
-       ]},
-      {:"ostatus:conversation", [ref: h.(activity.data["context"])],
-       h.(activity.data["context"])},
-      {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
-      {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []},
-      {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []}
-    ] ++ author ++ mentions
-  end
-
-  def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_author) do
-    h = fn str -> [to_charlist(str)] end
-
-    updated_at = activity.data["published"]
-    inserted_at = activity.data["published"]
-
-    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
-
-    retweeted_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
-    retweeted_object = Object.normalize(retweeted_activity)
-    retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"])
-
-    retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true)
-
-    mentions =
-      ([retweeted_user.ap_id] ++ activity.recipients)
-      |> Enum.uniq()
-      |> get_mentions()
-
-    [
-      {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']},
-      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']},
-      {:id, h.(activity.data["id"])},
-      {:title, ['#{user.nickname} repeated a notice']},
-      {:content, [type: 'html'], ['RT #{retweeted_object.data["content"]}']},
-      {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)},
-      {:"ostatus:conversation", [ref: h.(activity.data["context"])],
-       h.(activity.data["context"])},
-      {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
-      {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []},
-      {:"activity:object", retweeted_xml}
-    ] ++ mentions ++ author
-  end
-
-  def to_simple_form(%{data: %{"type" => "Follow"}} = activity, user, with_author) do
-    h = fn str -> [to_charlist(str)] end
-
-    updated_at = activity.data["published"]
-    inserted_at = activity.data["published"]
-
-    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
-
-    mentions = (activity.recipients || []) |> get_mentions
-
-    [
-      {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']},
-      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/follow']},
-      {:id, h.(activity.data["id"])},
-      {:title, ['#{user.nickname} started following #{activity.data["object"]}']},
-      {:content, [type: 'html'],
-       ['#{user.nickname} started following #{activity.data["object"]}']},
-      {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)},
-      {:"activity:object",
-       [
-         {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/person']},
-         {:id, h.(activity.data["object"])},
-         {:uri, h.(activity.data["object"])}
-       ]},
-      {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}
-    ] ++ mentions ++ author
-  end
-
-  # Only undos of follow for now. Will need to get redone once there are more
-  def to_simple_form(
-        %{data: %{"type" => "Undo", "object" => %{"type" => "Follow"} = follow_activity}} =
-          activity,
-        user,
-        with_author
-      ) do
-    h = fn str -> [to_charlist(str)] end
-
-    updated_at = activity.data["published"]
-    inserted_at = activity.data["published"]
-
-    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
-
-    mentions = (activity.recipients || []) |> get_mentions
-    follow_activity = Activity.normalize(follow_activity)
-
-    [
-      {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']},
-      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/unfollow']},
-      {:id, h.(activity.data["id"])},
-      {:title, ['#{user.nickname} stopped following #{follow_activity.data["object"]}']},
-      {:content, [type: 'html'],
-       ['#{user.nickname} stopped following #{follow_activity.data["object"]}']},
-      {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)},
-      {:"activity:object",
-       [
-         {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/person']},
-         {:id, h.(follow_activity.data["object"])},
-         {:uri, h.(follow_activity.data["object"])}
-       ]},
-      {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}
-    ] ++ mentions ++ author
-  end
-
-  def to_simple_form(%{data: %{"type" => "Delete"}} = activity, user, with_author) do
-    h = fn str -> [to_charlist(str)] end
-
-    updated_at = activity.data["published"]
-    inserted_at = activity.data["published"]
-
-    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
-
-    [
-      {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']},
-      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/delete']},
-      {:id, h.(activity.data["object"])},
-      {:title, ['An object was deleted']},
-      {:content, [type: 'html'], ['An object was deleted']},
-      {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)}
-    ] ++ author
-  end
-
-  def to_simple_form(_, _, _), do: nil
-
-  def wrap_with_entry(simple_form) do
-    [
-      {
-        :entry,
-        [
-          xmlns: 'http://www.w3.org/2005/Atom',
-          "xmlns:thr": 'http://purl.org/syndication/thread/1.0',
-          "xmlns:activity": 'http://activitystrea.ms/spec/1.0/',
-          "xmlns:poco": 'http://portablecontacts.net/spec/1.0',
-          "xmlns:ostatus": 'http://ostatus.org/schema/1.0'
-        ],
-        simple_form
-      }
-    ]
-  end
-end

lib/pleroma/web/ostatus/feed_representer.ex

@@ -1,66 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.FeedRepresenter do
-  alias Pleroma.User
-  alias Pleroma.Web.MediaProxy
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.OStatus.ActivityRepresenter
-  alias Pleroma.Web.OStatus.UserRepresenter
-
-  def to_simple_form(user, activities, _users) do
-    most_recent_update =
-      (List.first(activities) || user).updated_at
-      |> NaiveDateTime.to_iso8601()
-
-    h = fn str -> [to_charlist(str)] end
-
-    last_activity = List.last(activities)
-
-    entries =
-      activities
-      |> Enum.map(fn activity ->
-        {:entry, ActivityRepresenter.to_simple_form(activity, user)}
-      end)
-      |> Enum.filter(fn {_, form} -> form end)
-
-    [
-      {
-        :feed,
-        [
-          xmlns: 'http://www.w3.org/2005/Atom',
-          "xmlns:thr": 'http://purl.org/syndication/thread/1.0',
-          "xmlns:activity": 'http://activitystrea.ms/spec/1.0/',
-          "xmlns:poco": 'http://portablecontacts.net/spec/1.0',
-          "xmlns:ostatus": 'http://ostatus.org/schema/1.0'
-        ],
-        [
-          {:id, h.(OStatus.feed_path(user))},
-          {:title, ['#{user.nickname}\'s timeline']},
-          {:updated, h.(most_recent_update)},
-          {:logo, [to_charlist(User.avatar_url(user) |> MediaProxy.url())]},
-          {:link, [rel: 'hub', href: h.(OStatus.pubsub_path(user))], []},
-          {:link, [rel: 'salmon', href: h.(OStatus.salmon_path(user))], []},
-          {:link, [rel: 'self', href: h.(OStatus.feed_path(user)), type: 'application/atom+xml'],
-           []},
-          {:author, UserRepresenter.to_simple_form(user)}
-        ] ++
-          if last_activity do
-            [
-              {:link,
-               [
-                 rel: 'next',
-                 href:
-                   to_charlist(OStatus.feed_path(user)) ++
-                     '?max_id=' ++ to_charlist(last_activity.id),
-                 type: 'application/atom+xml'
-               ], []}
-            ]
-          else
-            []
-          end ++ entries
-      }
-    ]
-  end
-end

lib/pleroma/web/ostatus/handlers/delete_handler.ex

@@ -1,18 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.DeleteHandler do
-  require Logger
-  alias Pleroma.Object
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.XML
-
-  def handle_delete(entry, _doc \\ nil) do
-    with id <- XML.string_from_xpath("//id", entry),
-         %Object{} = object <- Object.normalize(id),
-         {:ok, delete} <- ActivityPub.delete(object, local: false) do
-      delete
-    end
-  end
-end

lib/pleroma/web/ostatus/handlers/follow_handler.ex

@@ -1,26 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.FollowHandler do
-  alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.XML
-
-  def handle(entry, doc) do
-    with {:ok, actor} <- OStatus.find_make_or_update_actor(doc),
-         id when not is_nil(id) <- XML.string_from_xpath("/entry/id", entry),
-         followed_uri when not is_nil(followed_uri) <-
-           XML.string_from_xpath("/entry/activity:object/id", entry),
-         {:ok, followed} <- OStatus.find_or_make_user(followed_uri),
-         {:locked, false} <- {:locked, followed.info.locked},
-         {:ok, activity} <- ActivityPub.follow(actor, followed, id, false) do
-      User.follow(actor, followed)
-      {:ok, activity}
-    else
-      {:locked, true} ->
-        {:error, "It's not possible to follow locked accounts over OStatus"}
-    end
-  end
-end

lib/pleroma/web/ostatus/handlers/note_handler.ex

@@ -1,168 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.NoteHandler do
-  require Logger
-  require Pleroma.Constants
-
-  alias Pleroma.Activity
-  alias Pleroma.Object
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.Federator
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.XML
-
-  @doc """
-  Get the context for this note. Uses this:
-  1. The context of the parent activity
-  2. The conversation reference in the ostatus xml
-  3. A newly generated context id.
-  """
-  def get_context(entry, in_reply_to) do
-    context =
-      (XML.string_from_xpath("//ostatus:conversation[1]", entry) ||
-         XML.string_from_xpath("//ostatus:conversation[1]/@ref", entry) || "")
-      |> String.trim()
-
-    with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(in_reply_to) do
-      context
-    else
-      _e ->
-        if String.length(context) > 0 do
-          context
-        else
-          Utils.generate_context_id()
-        end
-    end
-  end
-
-  def get_people_mentions(entry) do
-    :xmerl_xpath.string(
-      '//link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]',
-      entry
-    )
-    |> Enum.map(fn person -> XML.string_from_xpath("@href", person) end)
-  end
-
-  def get_collection_mentions(entry) do
-    transmogrify = fn
-      "http://activityschema.org/collection/public" ->
-        Pleroma.Constants.as_public()
-
-      group ->
-        group
-    end
-
-    :xmerl_xpath.string(
-      '//link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/collection"]',
-      entry
-    )
-    |> Enum.map(fn collection -> XML.string_from_xpath("@href", collection) |> transmogrify.() end)
-  end
-
-  def get_mentions(entry) do
-    (get_people_mentions(entry) ++ get_collection_mentions(entry))
-    |> Enum.filter(& &1)
-  end
-
-  def get_emoji(entry) do
-    try do
-      :xmerl_xpath.string('//link[@rel="emoji"]', entry)
-      |> Enum.reduce(%{}, fn emoji, acc ->
-        Map.put(acc, XML.string_from_xpath("@name", emoji), XML.string_from_xpath("@href", emoji))
-      end)
-    rescue
-      _e -> nil
-    end
-  end
-
-  def make_to_list(actor, mentions) do
-    [
-      actor.follower_address
-    ] ++ mentions
-  end
-
-  def add_external_url(note, entry) do
-    url = XML.string_from_xpath("//link[@rel='alternate' and @type='text/html']/@href", entry)
-    Map.put(note, "external_url", url)
-  end
-
-  def fetch_replied_to_activity(entry, in_reply_to, options \\ []) do
-    with %Activity{} = activity <- Activity.get_create_by_object_ap_id(in_reply_to) do
-      activity
-    else
-      _e ->
-        with true <- Federator.allowed_incoming_reply_depth?(options[:depth]),
-             in_reply_to_href when not is_nil(in_reply_to_href) <-
-               XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry),
-             {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href, options) do
-          activity
-        else
-          _e -> nil
-        end
-    end
-  end
-
-  # TODO: Clean this up a bit.
-  def handle_note(entry, doc \\ nil, options \\ []) do
-    with id <- XML.string_from_xpath("//id", entry),
-         activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id),
-         [author] <- :xmerl_xpath.string('//author[1]', doc),
-         {:ok, actor} <- OStatus.find_make_or_update_actor(author),
-         content_html <- OStatus.get_content(entry),
-         cw <- OStatus.get_cw(entry),
-         in_reply_to <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry),
-         options <- Keyword.put(options, :depth, (options[:depth] || 0) + 1),
-         in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to, options),
-         in_reply_to_object <-
-           (in_reply_to_activity && Object.normalize(in_reply_to_activity)) || nil,
-         in_reply_to <- (in_reply_to_object && in_reply_to_object.data["id"]) || in_reply_to,
-         attachments <- OStatus.get_attachments(entry),
-         context <- get_context(entry, in_reply_to),
-         tags <- OStatus.get_tags(entry),
-         mentions <- get_mentions(entry),
-         to <- make_to_list(actor, mentions),
-         date <- XML.string_from_xpath("//published", entry),
-         unlisted <- XML.string_from_xpath("//mastodon:scope", entry) == "unlisted",
-         cc <- if(unlisted, do: [Pleroma.Constants.as_public()], else: []),
-         note <-
-           CommonAPI.Utils.make_note_data(
-             actor.ap_id,
-             to,
-             context,
-             content_html,
-             attachments,
-             in_reply_to_activity,
-             [],
-             cw
-           ),
-         note <- note |> Map.put("id", id) |> Map.put("tag", tags),
-         note <- note |> Map.put("published", date),
-         note <- note |> Map.put("emoji", get_emoji(entry)),
-         note <- add_external_url(note, entry),
-         note <- note |> Map.put("cc", cc),
-         # TODO: Handle this case in make_note_data
-         note <-
-           if(
-             in_reply_to && !in_reply_to_activity,
-             do: note |> Map.put("inReplyTo", in_reply_to),
-             else: note
-           ) do
-      ActivityPub.create(%{
-        to: to,
-        actor: actor,
-        context: context,
-        object: note,
-        published: date,
-        local: false,
-        additional: %{"cc" => cc}
-      })
-    else
-      %Activity{} = activity -> {:ok, activity}
-      e -> {:error, e}
-    end
-  end
-end

lib/pleroma/web/ostatus/handlers/unfollow_handler.ex

@@ -1,22 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.UnfollowHandler do
-  alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.XML
-
-  def handle(entry, doc) do
-    with {:ok, actor} <- OStatus.find_make_or_update_actor(doc),
-         id when not is_nil(id) <- XML.string_from_xpath("/entry/id", entry),
-         followed_uri when not is_nil(followed_uri) <-
-           XML.string_from_xpath("/entry/activity:object/id", entry),
-         {:ok, followed} <- OStatus.find_or_make_user(followed_uri),
-         {:ok, activity} <- ActivityPub.unfollow(actor, followed, id, false) do
-      User.unfollow(actor, followed)
-      {:ok, activity}
-    end
-  end
-end

lib/pleroma/web/ostatus/ostatus.ex

@@ -1,395 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus do
-  import Pleroma.Web.XML
-  require Logger
-
-  alias Pleroma.Activity
-  alias Pleroma.HTTP
-  alias Pleroma.Object
-  alias Pleroma.User
-  alias Pleroma.Web
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Transmogrifier
-  alias Pleroma.Web.ActivityPub.Visibility
-  alias Pleroma.Web.OStatus.DeleteHandler
-  alias Pleroma.Web.OStatus.FollowHandler
-  alias Pleroma.Web.OStatus.NoteHandler
-  alias Pleroma.Web.OStatus.UnfollowHandler
-  alias Pleroma.Web.WebFinger
-  alias Pleroma.Web.Websub
-
-  def is_representable?(%Activity{} = activity) do
-    object = Object.normalize(activity)
-
-    cond do
-      is_nil(object) ->
-        false
-
-      Visibility.is_public?(activity) && object.data["type"] == "Note" ->
-        true
-
-      true ->
-        false
-    end
-  end
-
-  def feed_path(user), do: "#{user.ap_id}/feed.atom"
-
-  def pubsub_path(user), do: "#{Web.base_url()}/push/hub/#{user.nickname}"
-
-  def salmon_path(user), do: "#{user.ap_id}/salmon"
-
-  def remote_follow_path, do: "#{Web.base_url()}/ostatus_subscribe?acct={uri}"
-
-  def handle_incoming(xml_string, options \\ []) do
-    with doc when doc != :error <- parse_document(xml_string) do
-      with {:ok, actor_user} <- find_make_or_update_actor(doc),
-           do: Pleroma.Instances.set_reachable(actor_user.ap_id)
-
-      entries = :xmerl_xpath.string('//entry', doc)
-
-      activities =
-        Enum.map(entries, fn entry ->
-          {:xmlObj, :string, object_type} =
-            :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry)
-
-          {:xmlObj, :string, verb} = :xmerl_xpath.string('string(/entry/activity:verb[1])', entry)
-          Logger.debug("Handling #{verb}")
-
-          try do
-            case verb do
-              'http://activitystrea.ms/schema/1.0/delete' ->
-                with {:ok, activity} <- DeleteHandler.handle_delete(entry, doc), do: activity
-
-              'http://activitystrea.ms/schema/1.0/follow' ->
-                with {:ok, activity} <- FollowHandler.handle(entry, doc), do: activity
-
-              'http://activitystrea.ms/schema/1.0/unfollow' ->
-                with {:ok, activity} <- UnfollowHandler.handle(entry, doc), do: activity
-
-              'http://activitystrea.ms/schema/1.0/share' ->
-                with {:ok, activity, retweeted_activity} <- handle_share(entry, doc),
-                     do: [activity, retweeted_activity]
-
-              'http://activitystrea.ms/schema/1.0/favorite' ->
-                with {:ok, activity, favorited_activity} <- handle_favorite(entry, doc),
-                     do: [activity, favorited_activity]
-
-              _ ->
-                case object_type do
-                  'http://activitystrea.ms/schema/1.0/note' ->
-                    with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options),
-                         do: activity
-
-                  'http://activitystrea.ms/schema/1.0/comment' ->
-                    with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options),
-                         do: activity
-
-                  _ ->
-                    Logger.error("Couldn't parse incoming document")
-                    nil
-                end
-            end
-          rescue
-            e ->
-              Logger.error("Error occured while handling activity")
-              Logger.error(xml_string)
-              Logger.error(inspect(e))
-              nil
-          end
-        end)
-        |> Enum.filter(& &1)
-
-      {:ok, activities}
-    else
-      _e -> {:error, []}
-    end
-  end
-
-  def make_share(entry, doc, retweeted_activity) do
-    with {:ok, actor} <- find_make_or_update_actor(doc),
-         %Object{} = object <- Object.normalize(retweeted_activity),
-         id when not is_nil(id) <- string_from_xpath("/entry/id", entry),
-         {:ok, activity, _object} = ActivityPub.announce(actor, object, id, false) do
-      {:ok, activity}
-    end
-  end
-
-  def handle_share(entry, doc) do
-    with {:ok, retweeted_activity} <- get_or_build_object(entry),
-         {:ok, activity} <- make_share(entry, doc, retweeted_activity) do
-      {:ok, activity, retweeted_activity}
-    else
-      e -> {:error, e}
-    end
-  end
-
-  def make_favorite(entry, doc, favorited_activity) do
-    with {:ok, actor} <- find_make_or_update_actor(doc),
-         %Object{} = object <- Object.normalize(favorited_activity),
-         id when not is_nil(id) <- string_from_xpath("/entry/id", entry),
-         {:ok, activity, _object} = ActivityPub.like(actor, object, id, false) do
-      {:ok, activity}
-    end
-  end
-
-  def get_or_build_object(entry) do
-    with {:ok, activity} <- get_or_try_fetching(entry) do
-      {:ok, activity}
-    else
-      _e ->
-        with [object] <- :xmerl_xpath.string('/entry/activity:object', entry) do
-          NoteHandler.handle_note(object, object)
-        end
-    end
-  end
-
-  def get_or_try_fetching(entry) do
-    Logger.debug("Trying to get entry from db")
-
-    with id when not is_nil(id) <- string_from_xpath("//activity:object[1]/id", entry),
-         %Activity{} = activity <- Activity.get_create_by_object_ap_id_with_object(id) do
-      {:ok, activity}
-    else
-      _ ->
-        Logger.debug("Couldn't get, will try to fetch")
-
-        with href when not is_nil(href) <-
-               string_from_xpath("//activity:object[1]/link[@type=\"text/html\"]/@href", entry),
-             {:ok, [favorited_activity]} <- fetch_activity_from_url(href) do
-          {:ok, favorited_activity}
-        else
-          e -> Logger.debug("Couldn't find href: #{inspect(e)}")
-        end
-    end
-  end
-
-  def handle_favorite(entry, doc) do
-    with {:ok, favorited_activity} <- get_or_try_fetching(entry),
-         {:ok, activity} <- make_favorite(entry, doc, favorited_activity) do
-      {:ok, activity, favorited_activity}
-    else
-      e -> {:error, e}
-    end
-  end
-
-  def get_attachments(entry) do
-    :xmerl_xpath.string('/entry/link[@rel="enclosure"]', entry)
-    |> Enum.map(fn enclosure ->
-      with href when not is_nil(href) <- string_from_xpath("/link/@href", enclosure),
-           type when not is_nil(type) <- string_from_xpath("/link/@type", enclosure) do
-        %{
-          "type" => "Attachment",
-          "url" => [
-            %{
-              "type" => "Link",
-              "mediaType" => type,
-              "href" => href
-            }
-          ]
-        }
-      end
-    end)
-    |> Enum.filter(& &1)
-  end
-
-  @doc """
-    Gets the content from a an entry.
-  """
-  def get_content(entry) do
-    string_from_xpath("//content", entry)
-  end
-
-  @doc """
-    Get the cw that mastodon uses.
-  """
-  def get_cw(entry) do
-    case string_from_xpath("/*/summary", entry) do
-      cw when not is_nil(cw) -> cw
-      _ -> nil
-    end
-  end
-
-  def get_tags(entry) do
-    :xmerl_xpath.string('//category', entry)
-    |> Enum.map(fn category -> string_from_xpath("/category/@term", category) end)
-    |> Enum.filter(& &1)
-    |> Enum.map(&String.downcase/1)
-  end
-
-  def maybe_update(doc, user) do
-    case string_from_xpath("//author[1]/ap_enabled", doc) do
-      "true" ->
-        Transmogrifier.upgrade_user_from_ap_id(user.ap_id)
-
-      _ ->
-        maybe_update_ostatus(doc, user)
-    end
-  end
-
-  def maybe_update_ostatus(doc, user) do
-    old_data = Map.take(user, [:bio, :avatar, :name])
-
-    with false <- user.local,
-         avatar <- make_avatar_object(doc),
-         bio <- string_from_xpath("//author[1]/summary", doc),
-         name <- string_from_xpath("//author[1]/poco:displayName", doc),
-         new_data <- %{
-           avatar: avatar || old_data.avatar,
-           name: name || old_data.name,
-           bio: bio || old_data.bio
-         },
-         false <- new_data == old_data do
-      change = Ecto.Changeset.change(user, new_data)
-      User.update_and_set_cache(change)
-    else
-      _ ->
-        {:ok, user}
-    end
-  end
-
-  def find_make_or_update_actor(doc) do
-    uri = string_from_xpath("//author/uri[1]", doc)
-
-    with {:ok, %User{} = user} <- find_or_make_user(uri),
-         {:ap_enabled, false} <- {:ap_enabled, User.ap_enabled?(user)} do
-      maybe_update(doc, user)
-    else
-      {:ap_enabled, true} ->
-        {:error, :invalid_protocol}
-
-      _ ->
-        {:error, :unknown_user}
-    end
-  end
-
-  @spec find_or_make_user(String.t()) :: {:ok, User.t()}
-  def find_or_make_user(uri) do
-    case User.get_by_ap_id(uri) do
-      %User{} = user -> {:ok, user}
-      _ -> make_user(uri)
-    end
-  end
-
-  @spec make_user(String.t(), boolean()) :: {:ok, User.t()} | {:error, any()}
-  def make_user(uri, update \\ false) do
-    with {:ok, info} <- gather_user_info(uri) do
-      with false <- update,
-           %User{} = user <- User.get_cached_by_ap_id(info["uri"]) do
-        {:ok, user}
-      else
-        _e -> User.insert_or_update_user(build_user_data(info))
-      end
-    end
-  end
-
-  defp build_user_data(info) do
-    %{
-      name: info["name"],
-      nickname: info["nickname"] <> "@" <> info["host"],
-      ap_id: info["uri"],
-      info: info,
-      avatar: info["avatar"],
-      bio: info["bio"]
-    }
-  end
-
-  # TODO: Just takes the first one for now.
-  def make_avatar_object(author_doc, rel \\ "avatar") do
-    href = string_from_xpath("//author[1]/link[@rel=\"#{rel}\"]/@href", author_doc)
-    type = string_from_xpath("//author[1]/link[@rel=\"#{rel}\"]/@type", author_doc)
-
-    if href do
-      %{
-        "type" => "Image",
-        "url" => [%{"type" => "Link", "mediaType" => type, "href" => href}]
-      }
-    else
-      nil
-    end
-  end
-
-  @spec gather_user_info(String.t()) :: {:ok, map()} | {:error, any()}
-  def gather_user_info(username) do
-    with {:ok, webfinger_data} <- WebFinger.finger(username),
-         {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data["topic"]) do
-      data =
-        webfinger_data
-        |> Map.merge(feed_data)
-        |> Map.put("fqn", username)
-
-      {:ok, data}
-    else
-      e ->
-        Logger.debug(fn -> "Couldn't gather info for #{username}" end)
-        {:error, e}
-    end
-  end
-
-  # Regex-based 'parsing' so we don't have to pull in a full html parser
-  # It's a hack anyway. Maybe revisit this in the future
-  @mastodon_regex ~r/<link href='(.*)' rel='alternate' type='application\/atom\+xml'>/
-  @gs_regex ~r/<link title=.* href="(.*)" type="application\/atom\+xml" rel="alternate">/
-  @gs_classic_regex ~r/<link rel="alternate" href="(.*)" type="application\/atom\+xml" title=.*>/
-  def get_atom_url(body) do
-    cond do
-      Regex.match?(@mastodon_regex, body) ->
-        [[_, match]] = Regex.scan(@mastodon_regex, body)
-        {:ok, match}
-
-      Regex.match?(@gs_regex, body) ->
-        [[_, match]] = Regex.scan(@gs_regex, body)
-        {:ok, match}
-
-      Regex.match?(@gs_classic_regex, body) ->
-        [[_, match]] = Regex.scan(@gs_classic_regex, body)
-        {:ok, match}
-
-      true ->
-        Logger.debug(fn -> "Couldn't find Atom link in #{inspect(body)}" end)
-        {:error, "Couldn't find the Atom link"}
-    end
-  end
-
-  def fetch_activity_from_atom_url(url, options \\ []) do
-    with true <- String.starts_with?(url, "http"),
-         {:ok, %{body: body, status: code}} when code in 200..299 <-
-           HTTP.get(url, [{:Accept, "application/atom+xml"}]) do
-      Logger.debug("Got document from #{url}, handling...")
-      handle_incoming(body, options)
-    else
-      e ->
-        Logger.debug("Couldn't get #{url}: #{inspect(e)}")
-        e
-    end
-  end
-
-  def fetch_activity_from_html_url(url, options \\ []) do
-    Logger.debug("Trying to fetch #{url}")
-
-    with true <- String.starts_with?(url, "http"),
-         {:ok, %{body: body}} <- HTTP.get(url, []),
-         {:ok, atom_url} <- get_atom_url(body) do
-      fetch_activity_from_atom_url(atom_url, options)
-    else
-      e ->
-        Logger.debug("Couldn't get #{url}: #{inspect(e)}")
-        e
-    end
-  end
-
-  def fetch_activity_from_url(url, options \\ []) do
-    with {:ok, [_ | _] = activities} <- fetch_activity_from_atom_url(url, options) do
-      {:ok, activities}
-    else
-      _e -> fetch_activity_from_html_url(url, options)
-    end
-  rescue
-    e ->
-      Logger.debug("Couldn't get #{url}: #{inspect(e)}")
-      {:error, "Couldn't get #{url}: #{inspect(e)}"}
-  end
-end

lib/pleroma/web/ostatus/ostatus_controller.ex

@@ -13,19 +13,14 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.ActivityPub.ObjectView
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Endpoint
-  alias Pleroma.Web.Federator
   alias Pleroma.Web.Metadata.PlayerView
-  alias Pleroma.Web.OStatus.ActivityRepresenter
   alias Pleroma.Web.Router
-  alias Pleroma.Web.XML
 
   plug(
     Pleroma.Plugs.RateLimiter,
     {:ap_routes, params: ["uuid"]} when action in [:object, :activity]
   )
 
-  plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
-
   plug(
     Pleroma.Plugs.SetFormatPlug
     when action in [:object, :activity, :notice]
@@ -33,32 +28,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do
 
   action_fallback(:errors)
 
-  defp decode_or_retry(body) do
-    with {:ok, magic_key} <- Pleroma.Web.Salmon.fetch_magic_key(body),
-         {:ok, doc} <- Pleroma.Web.Salmon.decode_and_validate(magic_key, body) do
-      {:ok, doc}
-    else
-      _e ->
-        with [decoded | _] <- Pleroma.Web.Salmon.decode(body),
-             doc <- XML.parse_document(decoded),
-             uri when not is_nil(uri) <- XML.string_from_xpath("/entry/author[1]/uri", doc),
-             {:ok, _} <- Pleroma.Web.OStatus.make_user(uri, true),
-             {:ok, magic_key} <- Pleroma.Web.Salmon.fetch_magic_key(body),
-             {:ok, doc} <- Pleroma.Web.Salmon.decode_and_validate(magic_key, body) do
-          {:ok, doc}
-        end
-    end
-  end
-
-  def salmon_incoming(conn, _) do
-    {:ok, body, _conn} = read_body(conn)
-    {:ok, doc} = decode_or_retry(body)
-
-    Federator.incoming_doc(doc)
-
-    send_resp(conn, 200, "")
-  end
-
   def object(%{assigns: %{format: format}} = conn, %{"uuid" => _uuid})
       when format in ["json", "activity+json"] do
     ActivityPubController.call(conn, :object)
@@ -179,23 +148,10 @@ defp represent_activity(
     |> render("object.json", %{object: object})
   end
 
-  defp represent_activity(_conn, "activity+json", _, _) do
+  defp represent_activity(_conn, _, _, _) do
     {:error, :not_found}
   end
 
-  defp represent_activity(conn, _, activity, user) do
-    response =
-      activity
-      |> ActivityRepresenter.to_simple_form(user, true)
-      |> ActivityRepresenter.wrap_with_entry()
-      |> :xmerl.export_simple(:xmerl_xml)
-      |> to_string
-
-    conn
-    |> put_resp_content_type("application/atom+xml")
-    |> send_resp(200, response)
-  end
-
   def errors(conn, {:error, :not_found}) do
     render_error(conn, :not_found, "Not found")
   end

lib/pleroma/web/ostatus/user_representer.ex

@@ -1,41 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.UserRepresenter do
-  alias Pleroma.User
-
-  def to_simple_form(user) do
-    ap_id = to_charlist(user.ap_id)
-    nickname = to_charlist(user.nickname)
-    name = to_charlist(user.name)
-    bio = to_charlist(user.bio)
-    avatar_url = to_charlist(User.avatar_url(user))
-
-    banner =
-      if banner_url = User.banner_url(user) do
-        [{:link, [rel: 'header', href: banner_url], []}]
-      else
-        []
-      end
-
-    ap_enabled =
-      if user.local do
-        [{:ap_enabled, ['true']}]
-      else
-        []
-      end
-
-    [
-      {:id, [ap_id]},
-      {:"activity:object", ['http://activitystrea.ms/schema/1.0/person']},
-      {:uri, [ap_id]},
-      {:"poco:preferredUsername", [nickname]},
-      {:"poco:displayName", [name]},
-      {:"poco:note", [bio]},
-      {:summary, [bio]},
-      {:name, [nickname]},
-      {:link, [rel: 'avatar', href: avatar_url], []}
-    ] ++ banner ++ ap_enabled
-  end
-end

lib/pleroma/web/pleroma_api/controllers/account_controller.ex

@@ -80,9 +80,7 @@ def update_avatar(%{assigns: %{user: user}} = conn, params) do
 
   @doc "PATCH /api/v1/pleroma/accounts/update_banner"
   def update_banner(%{assigns: %{user: user}} = conn, %{"banner" => ""}) do
-    new_info = %{"banner" => %{}}
+    with {:ok, user} <- User.update_banner(user, %{}) do
-
-    with {:ok, user} <- User.update_info(user, &User.Info.profile_update(&1, new_info)) do
       CommonAPI.update(user)
       json(conn, %{url: nil})
     end
@@ -90,8 +88,7 @@ def update_banner(%{assigns: %{user: user}} = conn, %{"banner" => ""}) do
 
   def update_banner(%{assigns: %{user: user}} = conn, params) do
     with {:ok, object} <- ActivityPub.upload(%{"img" => params["banner"]}, type: :banner),
-         new_info <- %{"banner" => object.data},
+         {:ok, user} <- User.update_banner(user, object.data) do
-         {:ok, user} <- User.update_info(user, &User.Info.profile_update(&1, new_info)) do
       CommonAPI.update(user)
       %{"url" => [%{"href" => href} | _]} = object.data
 
@@ -101,17 +98,14 @@ def update_banner(%{assigns: %{user: user}} = conn, params) do
 
   @doc "PATCH /api/v1/pleroma/accounts/update_background"
   def update_background(%{assigns: %{user: user}} = conn, %{"img" => ""}) do
-    new_info = %{"background" => %{}}
+    with {:ok, _user} <- User.update_background(user, %{}) do
-
-    with {:ok, _user} <- User.update_info(user, &User.Info.profile_update(&1, new_info)) do
       json(conn, %{url: nil})
     end
   end
 
   def update_background(%{assigns: %{user: user}} = conn, params) do
     with {:ok, object} <- ActivityPub.upload(params, type: :background),
-         new_info <- %{"background" => object.data},
+         {:ok, _user} <- User.update_background(user, object.data) do
-         {:ok, _user} <- User.update_info(user, &User.Info.profile_update(&1, new_info)) do
       %{"url" => [%{"href" => href} | _]} = object.data
 
       json(conn, %{url: href})
@@ -119,7 +113,7 @@ def update_background(%{assigns: %{user: user}} = conn, params) do
   end
 
   @doc "GET /api/v1/pleroma/accounts/:id/favourites"
-  def favourites(%{assigns: %{account: %{info: %{hide_favorites: true}}}} = conn, _params) do
+  def favourites(%{assigns: %{account: %{hide_favorites: true}}} = conn, _params) do
     render_error(conn, :forbidden, "Can't get favorites")
   end
 

lib/pleroma/web/pleroma_api/controllers/mascot_controller.ex

@@ -24,9 +24,7 @@ def update(%{assigns: %{user: user}} = conn, %{"file" => file}) do
     with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)),
          # Reject if not an image
          %{type: "image"} = attachment <- render_attachment(object) do
-      # Sure!
+      {:ok, _user} = User.mascot_update(user, attachment)
-      # Save to the user's info
-      {:ok, _user} = User.update_info(user, &User.Info.mascot_update(&1, attachment))
 
       json(conn, attachment)
     else

lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex

@@ -79,6 +79,15 @@ def update_conversation(
     end
   end
 
+  def read_conversations(%{assigns: %{user: user}} = conn, _params) do
+    with {:ok, _, participations} <- Participation.mark_all_as_read(user) do
+      conn
+      |> add_link_headers(participations)
+      |> put_view(ConversationView)
+      |> render("participations.json", participations: participations, for: user)
+    end
+  end
+
   def read_notification(%{assigns: %{user: user}} = conn, %{"id" => notification_id}) do
     with {:ok, notification} <- Notification.read_one(user, notification_id) do
       conn

lib/pleroma/web/push/impl.ex

@@ -125,6 +125,10 @@ def format_body(
     end
   end
 
+  def format_title(%{activity: %{data: %{"directMessage" => true}}}) do
+    "New Direct Message"
+  end
+
   def format_title(%{activity: %{data: %{"type" => type}}}) do
     case type do
       "Create" -> "New Mention"

lib/pleroma/web/router.ex

@@ -137,11 +137,14 @@ defmodule Pleroma.Web.Router do
     delete("/users", AdminAPIController, :user_delete)
     post("/users", AdminAPIController, :users_create)
     patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
+    patch("/users/activate", AdminAPIController, :user_activate)
+    patch("/users/deactivate", AdminAPIController, :user_deactivate)
     put("/users/tag", AdminAPIController, :tag_users)
     delete("/users/tag", AdminAPIController, :untag_users)
 
     get("/users/:nickname/permission_group", AdminAPIController, :right_get)
     get("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_get)
+
     post("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_add)
 
     delete(
@@ -150,8 +153,15 @@ defmodule Pleroma.Web.Router do
       :right_delete
     )
 
-    put("/users/:nickname/activation_status", AdminAPIController, :set_activation_status)
+    post("/users/permission_group/:permission_group", AdminAPIController, :right_add_multiple)
 
+    delete(
+      "/users/permission_group/:permission_group",
+      AdminAPIController,
+      :right_delete_multiple
+    )
+
+    get("/relay", AdminAPIController, :relay_list)
     post("/relay", AdminAPIController, :relay_follow)
     delete("/relay", AdminAPIController, :relay_unfollow)
 
@@ -256,6 +266,7 @@ defmodule Pleroma.Web.Router do
 
       get("/conversations/:id/statuses", PleromaAPIController, :conversation_statuses)
       get("/conversations/:id", PleromaAPIController, :conversation)
+      post("/conversations/read", PleromaAPIController, :read_conversations)
     end
 
     scope [] do
@@ -394,6 +405,9 @@ defmodule Pleroma.Web.Router do
     get("/push/subscription", SubscriptionController, :get)
     put("/push/subscription", SubscriptionController, :update)
     delete("/push/subscription", SubscriptionController, :delete)
+
+    get("/markers", MarkerController, :index)
+    post("/markers", MarkerController, :upsert)
   end
 
   scope "/api/web", Pleroma.Web do
@@ -499,11 +513,6 @@ defmodule Pleroma.Web.Router do
     get("/users/:nickname/feed", Feed.FeedController, :feed)
     get("/users/:nickname", Feed.FeedController, :feed_redirect)
 
-    post("/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming)
-    post("/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request)
-    get("/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation)
-    post("/push/subscriptions/:id", Websub.WebsubController, :websub_incoming)
-
     get("/mailer/unsubscribe/:token", Mailer.SubscriptionController, :unsubscribe)
   end
 
@@ -586,6 +595,12 @@ defmodule Pleroma.Web.Router do
     get("/:version", Nodeinfo.NodeinfoController, :nodeinfo)
   end
 
+  scope "/", Pleroma.Web do
+    pipe_through(:api)
+
+    get("/web/manifest.json", MastoFEController, :manifest)
+  end
+
   scope "/", Pleroma.Web do
     pipe_through(:mastodon_html)
 

lib/pleroma/web/salmon/salmon.ex

@@ -1,254 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.Salmon do
-  @behaviour Pleroma.Web.Federator.Publisher
-
-  use Bitwise
-
-  alias Pleroma.Activity
-  alias Pleroma.HTTP
-  alias Pleroma.Instances
-  alias Pleroma.Keys
-  alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.Visibility
-  alias Pleroma.Web.Federator.Publisher
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.OStatus.ActivityRepresenter
-  alias Pleroma.Web.XML
-
-  require Logger
-
-  def decode(salmon) do
-    doc = XML.parse_document(salmon)
-
-    {:xmlObj, :string, data} = :xmerl_xpath.string('string(//me:data[1])', doc)
-    {:xmlObj, :string, sig} = :xmerl_xpath.string('string(//me:sig[1])', doc)
-    {:xmlObj, :string, alg} = :xmerl_xpath.string('string(//me:alg[1])', doc)
-    {:xmlObj, :string, encoding} = :xmerl_xpath.string('string(//me:encoding[1])', doc)
-    {:xmlObj, :string, type} = :xmerl_xpath.string('string(//me:data[1]/@type)', doc)
-
-    {:ok, data} = Base.url_decode64(to_string(data), ignore: :whitespace)
-    {:ok, sig} = Base.url_decode64(to_string(sig), ignore: :whitespace)
-    alg = to_string(alg)
-    encoding = to_string(encoding)
-    type = to_string(type)
-
-    [data, type, encoding, alg, sig]
-  end
-
-  def fetch_magic_key(salmon) do
-    with [data, _, _, _, _] <- decode(salmon),
-         doc <- XML.parse_document(data),
-         uri when not is_nil(uri) <- XML.string_from_xpath("/entry/author[1]/uri", doc),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(uri),
-         magic_key <- encode_key(public_key) do
-      {:ok, magic_key}
-    end
-  end
-
-  def decode_and_validate(magickey, salmon) do
-    [data, type, encoding, alg, sig] = decode(salmon)
-
-    signed_text =
-      [data, type, encoding, alg]
-      |> Enum.map(&Base.url_encode64/1)
-      |> Enum.join(".")
-
-    key = decode_key(magickey)
-
-    verify = :public_key.verify(signed_text, :sha256, sig, key)
-
-    if verify do
-      {:ok, data}
-    else
-      :error
-    end
-  end
-
-  def decode_key("RSA." <> magickey) do
-    make_integer = fn bin ->
-      list = :erlang.binary_to_list(bin)
-      Enum.reduce(list, 0, fn el, acc -> acc <<< 8 ||| el end)
-    end
-
-    [modulus, exponent] =
-      magickey
-      |> String.split(".")
-      |> Enum.map(fn n -> Base.url_decode64!(n, padding: false) end)
-      |> Enum.map(make_integer)
-
-    {:RSAPublicKey, modulus, exponent}
-  end
-
-  def encode_key({:RSAPublicKey, modulus, exponent}) do
-    modulus_enc = :binary.encode_unsigned(modulus) |> Base.url_encode64()
-    exponent_enc = :binary.encode_unsigned(exponent) |> Base.url_encode64()
-
-    "RSA.#{modulus_enc}.#{exponent_enc}"
-  end
-
-  def encode(private_key, doc) do
-    type = "application/atom+xml"
-    encoding = "base64url"
-    alg = "RSA-SHA256"
-
-    signed_text =
-      [doc, type, encoding, alg]
-      |> Enum.map(&Base.url_encode64/1)
-      |> Enum.join(".")
-
-    signature =
-      signed_text
-      |> :public_key.sign(:sha256, private_key)
-      |> to_string
-      |> Base.url_encode64()
-
-    doc_base64 =
-      doc
-      |> Base.url_encode64()
-
-    # Don't need proper xml building, these strings are safe to leave unescaped
-    salmon = """
-    <?xml version="1.0" encoding="UTF-8"?>
-    <me:env xmlns:me="http://salmon-protocol.org/ns/magic-env">
-      <me:data type="application/atom+xml">#{doc_base64}</me:data>
-      <me:encoding>#{encoding}</me:encoding>
-      <me:alg>#{alg}</me:alg>
-      <me:sig>#{signature}</me:sig>
-    </me:env>
-    """
-
-    {:ok, salmon}
-  end
-
-  def remote_users(%User{id: user_id}, %{data: %{"to" => to} = data}) do
-    cc = Map.get(data, "cc", [])
-
-    bcc =
-      data
-      |> Map.get("bcc", [])
-      |> Enum.reduce([], fn ap_id, bcc ->
-        case Pleroma.List.get_by_ap_id(ap_id) do
-          %Pleroma.List{user_id: ^user_id} = list ->
-            {:ok, following} = Pleroma.List.get_following(list)
-            bcc ++ Enum.map(following, & &1.ap_id)
-
-          _ ->
-            bcc
-        end
-      end)
-
-    [to, cc, bcc]
-    |> Enum.concat()
-    |> Enum.map(&User.get_cached_by_ap_id/1)
-    |> Enum.filter(fn user -> user && !user.local end)
-  end
-
-  @doc "Pushes an activity to remote account."
-  def publish_one(%{recipient: %{info: %{salmon: salmon}}} = params),
-    do: publish_one(Map.put(params, :recipient, salmon))
-
-  def publish_one(%{recipient: url, feed: feed} = params) when is_binary(url) do
-    with {:ok, %{status: code}} when code in 200..299 <-
-           HTTP.post(
-             url,
-             feed,
-             [{"Content-Type", "application/magic-envelope+xml"}]
-           ) do
-      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
-        do: Instances.set_reachable(url)
-
-      Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
-      {:ok, code}
-    else
-      e ->
-        unless params[:unreachable_since], do: Instances.set_reachable(url)
-        Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
-        {:error, "Unreachable instance"}
-    end
-  end
-
-  def publish_one(%{recipient_id: recipient_id} = params) do
-    recipient = User.get_cached_by_id(recipient_id)
-
-    params
-    |> Map.delete(:recipient_id)
-    |> Map.put(:recipient, recipient)
-    |> publish_one()
-  end
-
-  def publish_one(_), do: :noop
-
-  @supported_activities [
-    "Create",
-    "Follow",
-    "Like",
-    "Announce",
-    "Undo",
-    "Delete"
-  ]
-
-  def is_representable?(%Activity{data: %{"type" => type}} = activity)
-      when type in @supported_activities,
-      do: Visibility.is_public?(activity)
-
-  def is_representable?(_), do: false
-
-  @doc """
-  Publishes an activity to remote accounts
-  """
-  @spec publish(User.t(), Pleroma.Activity.t()) :: none
-  def publish(user, activity)
-
-  def publish(%{keys: keys} = user, %{data: %{"type" => type}} = activity)
-      when type in @supported_activities do
-    feed = ActivityRepresenter.to_simple_form(activity, user, true)
-
-    if feed do
-      feed =
-        ActivityRepresenter.wrap_with_entry(feed)
-        |> :xmerl.export_simple(:xmerl_xml)
-        |> to_string
-
-      {:ok, private, _} = Keys.keys_from_pem(keys)
-      {:ok, feed} = encode(private, feed)
-
-      remote_users = remote_users(user, activity)
-
-      salmon_urls = Enum.map(remote_users, & &1.info.salmon)
-      reachable_urls_metadata = Instances.filter_reachable(salmon_urls)
-      reachable_urls = Map.keys(reachable_urls_metadata)
-
-      remote_users
-      |> Enum.filter(&(&1.info.salmon in reachable_urls))
-      |> Enum.each(fn remote_user ->
-        Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
-
-        Publisher.enqueue_one(__MODULE__, %{
-          recipient_id: remote_user.id,
-          feed: feed,
-          unreachable_since: reachable_urls_metadata[remote_user.info.salmon]
-        })
-      end)
-    end
-  end
-
-  def publish(%{id: id}, _), do: Logger.debug(fn -> "Keys missing for user #{id}" end)
-
-  def gather_webfinger_links(%User{} = user) do
-    {:ok, _private, public} = Keys.keys_from_pem(user.keys)
-    magic_key = encode_key(public)
-
-    [
-      %{"rel" => "salmon", "href" => OStatus.salmon_path(user)},
-      %{
-        "rel" => "magic-public-key",
-        "href" => "data:application/magic-public-key,#{magic_key}"
-      }
-    ]
-  end
-
-  def gather_nodeinfo_protocol_names, do: []
-end

lib/pleroma/web/streamer/streamer.ex

@@ -49,7 +49,7 @@ defp handle_should_send(:test) do
     end
   end
 
-  defp handle_should_send(_) do
+  defp handle_should_send(:benchmark), do: false
-    true
+
-  end
+  defp handle_should_send(_), do: true
 end

lib/pleroma/web/streamer/worker.ex

@@ -129,12 +129,12 @@ defp do_stream(%{topic: topic, item: item}) do
   end
 
   defp should_send?(%User{} = user, %Activity{} = item) do
-    blocks = user.info.blocks || []
+    blocks = user.blocks || []
-    mutes = user.info.mutes || []
+    mutes = user.mutes || []
-    reblog_mutes = user.info.muted_reblogs || []
+    reblog_mutes = user.muted_reblogs || []
     recipient_blocks = MapSet.new(blocks ++ mutes)
     recipients = MapSet.new(item.recipients)
-    domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.info.domain_blocks)
+    domain_blocks = Pleroma.Web.ActivityPub.MRF.subdomains_regex(user.domain_blocks)
 
     with parent when not is_nil(parent) <- Object.normalize(item),
          true <- Enum.all?([blocks, mutes, reblog_mutes], &(item.actor not in &1)),
@@ -212,7 +212,7 @@ def push_to_socket(topics, topic, item) do
   end
 
   @spec thread_containment(Activity.t(), User.t()) :: boolean()
-  defp thread_containment(_activity, %User{info: %{skip_thread_containment: true}}), do: true
+  defp thread_containment(_activity, %User{skip_thread_containment: true}), do: true
 
   defp thread_containment(activity, user) do
     if Config.get([:instance, :skip_thread_containment]) do

lib/pleroma/web/templates/feed/feed/feed.xml.eex

@@ -10,8 +10,6 @@
   <title><%= @user.nickname <> "'s timeline" %></title>
   <updated><%= most_recent_update(@activities, @user) %></updated>
   <logo><%= logo(@user) %></logo>
-  <link rel="hub" href="<%= websub_url(@conn, :websub_subscription_request, @user.nickname) %>"/>
-  <link rel="salmon" href="<%= o_status_url(@conn, :salmon_incoming, @user.nickname) %>"/>
   <link rel="self" href="<%= '#{feed_url(@conn, :feed, @user.nickname)}.atom' %>" type="application/atom+xml"/>
 
   <%= render @view_module, "_author.xml", assigns %>

lib/pleroma/web/templates/masto_fe/index.html.eex

@@ -4,9 +4,13 @@
 <meta charset='utf-8'>
 <meta content='width=device-width, initial-scale=1' name='viewport'>
 <title>
-<%= Pleroma.Config.get([:instance, :name]) %>
+<%= Config.get([:instance, :name]) %>
 </title>
 <link rel="icon" type="image/png" href="/favicon.png"/>
+<link rel="manifest" type="applicaton/manifest+json" href="<%= masto_fe_path(Pleroma.Web.Endpoint, :manifest) %>" />
+
+<meta name="theme-color" content="<%= Config.get([:manifest, :theme_color]) %>" />
+
 <script crossorigin='anonymous' src="/packs/locales.js"></script>
 <script crossorigin='anonymous' src="/packs/locales/glitch/en.js"></script>
 

lib/pleroma/web/twitter_api/twitter_api_controller.ex

@@ -20,11 +20,12 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
   action_fallback(:errors)
 
   def confirm_email(conn, %{"user_id" => uid, "token" => token}) do
-    new_info = [need_confirmation: false]
+    with %User{} = user <- User.get_cached_by_id(uid),
-
+         true <- user.local and user.confirmation_pending and user.confirmation_token == token,
-    with %User{info: info} = user <- User.get_cached_by_id(uid),
+         {:ok, _} <-
-         true <- user.local and info.confirmation_pending and info.confirmation_token == token,
+           user
-         {:ok, _} <- User.update_info(user, &User.Info.confirmation_changeset(&1, new_info)) do
+           |> User.confirmation_changeset(need_confirmation: false)
+           |> User.update_and_set_cache() do
       redirect(conn, to: "/")
     end
   end

lib/pleroma/web/views/masto_fe_view.ex

@@ -61,12 +61,12 @@ def initial_state(token, user, custom_emojis) do
       },
       poll_limits: Config.get([:instance, :poll_limits]),
       rights: %{
-        delete_others_notice: present?(user.info.is_moderator),
+        delete_others_notice: present?(user.is_moderator),
-        admin: present?(user.info.is_admin)
+        admin: present?(user.is_admin)
       },
       compose: %{
         me: "#{user.id}",
-        default_privacy: user.info.default_scope,
+        default_privacy: user.default_scope,
         default_sensitive: false,
         allow_content_types: Config.get([:instance, :allowed_post_formats])
       },
@@ -86,7 +86,7 @@ def initial_state(token, user, custom_emojis) do
           "video\/mp4"
         ]
       },
-      settings: user.info.settings || @default_settings,
+      settings: user.settings || @default_settings,
       push_subscription: nil,
       accounts: %{user.id => render(AccountView, "show.json", user: user, for: user)},
       custom_emojis: render(CustomEmojiView, "index.json", custom_emojis: custom_emojis),
@@ -99,4 +99,23 @@ def initial_state(token, user, custom_emojis) do
   defp present?(nil), do: false
   defp present?(false), do: false
   defp present?(_), do: true
+
+  def render("manifest.json", _params) do
+    %{
+      name: Config.get([:instance, :name]),
+      description: Config.get([:instance, :description]),
+      icons: Config.get([:manifest, :icons]),
+      theme_color: Config.get([:manifest, :theme_color]),
+      background_color: Config.get([:manifest, :background_color]),
+      display: "standalone",
+      scope: Pleroma.Web.base_url(),
+      start_url: masto_fe_path(Pleroma.Web.Endpoint, :index, ["getting-started"]),
+      categories: [
+        "social"
+      ],
+      serviceworker: %{
+        src: "/sw.js"
+      }
+    }
+  end
 end

lib/pleroma/web/web_finger/web_finger.ex

@@ -108,7 +108,6 @@ defp webfinger_from_xml(doc) do
              doc
            ),
          subject <- XML.string_from_xpath("//Subject", doc),
-         salmon <- XML.string_from_xpath(~s{//Link[@rel="salmon"]/@href}, doc),
          subscribe_address <-
            XML.string_from_xpath(
              ~s{//Link[@rel="http://ostatus.org/schema/1.0/subscribe"]/@template},
@@ -123,7 +122,6 @@ defp webfinger_from_xml(doc) do
         "magic_key" => magic_key,
         "topic" => topic,
         "subject" => subject,
-        "salmon" => salmon,
         "subscribe_address" => subscribe_address,
         "ap_id" => ap_id
       }
@@ -148,16 +146,6 @@ defp webfinger_from_json(doc) do
           {"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", "self"} ->
             Map.put(data, "ap_id", link["href"])
 
-          {_, "magic-public-key"} ->
-            "data:application/magic-public-key," <> magic_key = link["href"]
-            Map.put(data, "magic_key", magic_key)
-
-          {"application/atom+xml", "http://schemas.google.com/g/2010#updates-from"} ->
-            Map.put(data, "topic", link["href"])
-
-          {_, "salmon"} ->
-            Map.put(data, "salmon", link["href"])
-
           {_, "http://ostatus.org/schema/1.0/subscribe"} ->
             Map.put(data, "subscribe_address", link["template"])
 

lib/pleroma/web/websub/websub.ex

@@ -1,332 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.Websub do
-  alias Ecto.Changeset
-  alias Pleroma.Activity
-  alias Pleroma.HTTP
-  alias Pleroma.Instances
-  alias Pleroma.Repo
-  alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.Visibility
-  alias Pleroma.Web.Endpoint
-  alias Pleroma.Web.Federator
-  alias Pleroma.Web.Federator.Publisher
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.OStatus.FeedRepresenter
-  alias Pleroma.Web.Router.Helpers
-  alias Pleroma.Web.Websub.WebsubClientSubscription
-  alias Pleroma.Web.Websub.WebsubServerSubscription
-  alias Pleroma.Web.XML
-  require Logger
-
-  import Ecto.Query
-
-  @behaviour Pleroma.Web.Federator.Publisher
-
-  def verify(subscription, getter \\ &HTTP.get/3) do
-    challenge = Base.encode16(:crypto.strong_rand_bytes(8))
-    lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at)
-    lease_seconds = lease_seconds |> to_string
-
-    params = %{
-      "hub.challenge": challenge,
-      "hub.lease_seconds": lease_seconds,
-      "hub.topic": subscription.topic,
-      "hub.mode": "subscribe"
-    }
-
-    url = hd(String.split(subscription.callback, "?"))
-    query = URI.parse(subscription.callback).query || ""
-    params = Map.merge(params, URI.decode_query(query))
-
-    with {:ok, response} <- getter.(url, [], params: params),
-         ^challenge <- response.body do
-      changeset = Changeset.change(subscription, %{state: "active"})
-      Repo.update(changeset)
-    else
-      e ->
-        Logger.debug("Couldn't verify subscription")
-        Logger.debug(inspect(e))
-        {:error, subscription}
-    end
-  end
-
-  @supported_activities [
-    "Create",
-    "Follow",
-    "Like",
-    "Announce",
-    "Undo",
-    "Delete"
-  ]
-
-  def is_representable?(%Activity{data: %{"type" => type}} = activity)
-      when type in @supported_activities,
-      do: Visibility.is_public?(activity)
-
-  def is_representable?(_), do: false
-
-  def publish(topic, user, %{data: %{"type" => type}} = activity)
-      when type in @supported_activities do
-    response =
-      user
-      |> FeedRepresenter.to_simple_form([activity], [user])
-      |> :xmerl.export_simple(:xmerl_xml)
-      |> to_string
-
-    query =
-      from(
-        sub in WebsubServerSubscription,
-        where: sub.topic == ^topic and sub.state == "active",
-        where: fragment("? > (NOW() at time zone 'UTC')", sub.valid_until)
-      )
-
-    subscriptions = Repo.all(query)
-
-    callbacks = Enum.map(subscriptions, & &1.callback)
-    reachable_callbacks_metadata = Instances.filter_reachable(callbacks)
-    reachable_callbacks = Map.keys(reachable_callbacks_metadata)
-
-    subscriptions
-    |> Enum.filter(&(&1.callback in reachable_callbacks))
-    |> Enum.each(fn sub ->
-      data = %{
-        xml: response,
-        topic: topic,
-        callback: sub.callback,
-        secret: sub.secret,
-        unreachable_since: reachable_callbacks_metadata[sub.callback]
-      }
-
-      Publisher.enqueue_one(__MODULE__, data)
-    end)
-  end
-
-  def publish(_, _, _), do: ""
-
-  def publish(actor, activity), do: publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity)
-
-  def sign(secret, doc) do
-    :crypto.hmac(:sha, secret, to_string(doc)) |> Base.encode16() |> String.downcase()
-  end
-
-  def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) do
-    with {:ok, topic} <- valid_topic(params, user),
-         {:ok, lease_time} <- lease_time(params),
-         secret <- params["hub.secret"],
-         callback <- params["hub.callback"] do
-      subscription = get_subscription(topic, callback)
-
-      data = %{
-        state: subscription.state || "requested",
-        topic: topic,
-        secret: secret,
-        callback: callback
-      }
-
-      change = Changeset.change(subscription, data)
-      websub = Repo.insert_or_update!(change)
-
-      change =
-        Changeset.change(websub, %{valid_until: NaiveDateTime.add(websub.updated_at, lease_time)})
-
-      websub = Repo.update!(change)
-
-      Federator.verify_websub(websub)
-
-      {:ok, websub}
-    else
-      {:error, reason} ->
-        Logger.debug("Couldn't create subscription")
-        Logger.debug(inspect(reason))
-
-        {:error, reason}
-    end
-  end
-
-  def incoming_subscription_request(user, params) do
-    Logger.info("Unhandled WebSub request for #{user.nickname}: #{inspect(params)}")
-
-    {:error, "Invalid WebSub request"}
-  end
-
-  defp get_subscription(topic, callback) do
-    Repo.get_by(WebsubServerSubscription, topic: topic, callback: callback) ||
-      %WebsubServerSubscription{}
-  end
-
-  # Temp hack for mastodon.
-  defp lease_time(%{"hub.lease_seconds" => ""}) do
-    # three days
-    {:ok, 60 * 60 * 24 * 3}
-  end
-
-  defp lease_time(%{"hub.lease_seconds" => lease_seconds}) do
-    {:ok, String.to_integer(lease_seconds)}
-  end
-
-  defp lease_time(_) do
-    # three days
-    {:ok, 60 * 60 * 24 * 3}
-  end
-
-  defp valid_topic(%{"hub.topic" => topic}, user) do
-    if topic == OStatus.feed_path(user) do
-      {:ok, OStatus.feed_path(user)}
-    else
-      {:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"}
-    end
-  end
-
-  def subscribe(subscriber, subscribed, requester \\ &request_subscription/1) do
-    topic = subscribed.info.topic
-    # FIXME: Race condition, use transactions
-    {:ok, subscription} =
-      with subscription when not is_nil(subscription) <-
-             Repo.get_by(WebsubClientSubscription, topic: topic) do
-        subscribers = [subscriber.ap_id | subscription.subscribers] |> Enum.uniq()
-        change = Ecto.Changeset.change(subscription, %{subscribers: subscribers})
-        Repo.update(change)
-      else
-        _e ->
-          subscription = %WebsubClientSubscription{
-            topic: topic,
-            hub: subscribed.info.hub,
-            subscribers: [subscriber.ap_id],
-            state: "requested",
-            secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64(),
-            user: subscribed
-          }
-
-          Repo.insert(subscription)
-      end
-
-    requester.(subscription)
-  end
-
-  def gather_feed_data(topic, getter \\ &HTTP.get/1) do
-    with {:ok, response} <- getter.(topic),
-         status when status in 200..299 <- response.status,
-         body <- response.body,
-         doc <- XML.parse_document(body),
-         uri when not is_nil(uri) <- XML.string_from_xpath("/feed/author[1]/uri", doc),
-         hub when not is_nil(hub) <- XML.string_from_xpath(~S{/feed/link[@rel="hub"]/@href}, doc) do
-      name = XML.string_from_xpath("/feed/author[1]/name", doc)
-      preferred_username = XML.string_from_xpath("/feed/author[1]/poco:preferredUsername", doc)
-      display_name = XML.string_from_xpath("/feed/author[1]/poco:displayName", doc)
-      avatar = OStatus.make_avatar_object(doc)
-      bio = XML.string_from_xpath("/feed/author[1]/summary", doc)
-
-      {:ok,
-       %{
-         "uri" => uri,
-         "hub" => hub,
-         "nickname" => preferred_username || name,
-         "name" => display_name || name,
-         "host" => URI.parse(uri).host,
-         "avatar" => avatar,
-         "bio" => bio
-       }}
-    else
-      e ->
-        {:error, e}
-    end
-  end
-
-  def request_subscription(websub, poster \\ &HTTP.post/3, timeout \\ 10_000) do
-    data = [
-      "hub.mode": "subscribe",
-      "hub.topic": websub.topic,
-      "hub.secret": websub.secret,
-      "hub.callback": Helpers.websub_url(Endpoint, :websub_subscription_confirmation, websub.id)
-    ]
-
-    # This checks once a second if we are confirmed yet
-    websub_checker = fn ->
-      helper = fn helper ->
-        :timer.sleep(1000)
-        websub = Repo.get_by(WebsubClientSubscription, id: websub.id, state: "accepted")
-        if websub, do: websub, else: helper.(helper)
-      end
-
-      helper.(helper)
-    end
-
-    task = Task.async(websub_checker)
-
-    with {:ok, %{status: 202}} <-
-           poster.(websub.hub, {:form, data}, "Content-type": "application/x-www-form-urlencoded"),
-         {:ok, websub} <- Task.yield(task, timeout) do
-      {:ok, websub}
-    else
-      e ->
-        Task.shutdown(task)
-
-        change = Ecto.Changeset.change(websub, %{state: "rejected"})
-        {:ok, websub} = Repo.update(change)
-
-        Logger.debug(fn -> "Couldn't confirm subscription: #{inspect(websub)}" end)
-        Logger.debug(fn -> "error: #{inspect(e)}" end)
-
-        {:error, websub}
-    end
-  end
-
-  def refresh_subscriptions(delta \\ 60 * 60 * 24) do
-    Logger.debug("Refreshing subscriptions")
-
-    cut_off = NaiveDateTime.add(NaiveDateTime.utc_now(), delta)
-
-    query = from(sub in WebsubClientSubscription, where: sub.valid_until < ^cut_off)
-
-    subs = Repo.all(query)
-
-    Enum.each(subs, fn sub ->
-      Federator.request_subscription(sub)
-    end)
-  end
-
-  def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret} = params) do
-    signature = sign(secret || "", xml)
-    Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
-
-    with {:ok, %{status: code}} when code in 200..299 <-
-           HTTP.post(
-             callback,
-             xml,
-             [
-               {"Content-Type", "application/atom+xml"},
-               {"X-Hub-Signature", "sha1=#{signature}"}
-             ]
-           ) do
-      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
-        do: Instances.set_reachable(callback)
-
-      Logger.info(fn -> "Pushed to #{callback}, code #{code}" end)
-      {:ok, code}
-    else
-      {_post_result, response} ->
-        unless params[:unreachable_since], do: Instances.set_reachable(callback)
-        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(response)}" end)
-        {:error, response}
-    end
-  end
-
-  def gather_webfinger_links(%User{} = user) do
-    [
-      %{
-        "rel" => "http://schemas.google.com/g/2010#updates-from",
-        "type" => "application/atom+xml",
-        "href" => OStatus.feed_path(user)
-      },
-      %{
-        "rel" => "http://ostatus.org/schema/1.0/subscribe",
-        "template" => OStatus.remote_follow_path()
-      }
-    ]
-  end
-
-  def gather_nodeinfo_protocol_names, do: ["ostatus"]
-end

lib/pleroma/web/websub/websub_client_subscription.ex

@@ -1,20 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.Websub.WebsubClientSubscription do
-  use Ecto.Schema
-  alias Pleroma.User
-
-  schema "websub_client_subscriptions" do
-    field(:topic, :string)
-    field(:secret, :string)
-    field(:valid_until, :naive_datetime_usec)
-    field(:state, :string)
-    field(:subscribers, {:array, :string}, default: [])
-    field(:hub, :string)
-    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
-
-    timestamps()
-  end
-end

lib/pleroma/web/websub/websub_controller.ex

@@ -1,99 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.Websub.WebsubController do
-  use Pleroma.Web, :controller
-
-  alias Pleroma.Repo
-  alias Pleroma.User
-  alias Pleroma.Web.Federator
-  alias Pleroma.Web.Websub
-  alias Pleroma.Web.Websub.WebsubClientSubscription
-
-  require Logger
-
-  plug(
-    Pleroma.Web.FederatingPlug
-    when action in [
-           :websub_subscription_request,
-           :websub_subscription_confirmation,
-           :websub_incoming
-         ]
-  )
-
-  def websub_subscription_request(conn, %{"nickname" => nickname} = params) do
-    user = User.get_cached_by_nickname(nickname)
-
-    with {:ok, _websub} <- Websub.incoming_subscription_request(user, params) do
-      conn
-      |> send_resp(202, "Accepted")
-    else
-      {:error, reason} ->
-        conn
-        |> send_resp(500, reason)
-    end
-  end
-
-  # TODO: Extract this into the Websub module
-  def websub_subscription_confirmation(
-        conn,
-        %{
-          "id" => id,
-          "hub.mode" => "subscribe",
-          "hub.challenge" => challenge,
-          "hub.topic" => topic
-        } = params
-      ) do
-    Logger.debug("Got WebSub confirmation")
-    Logger.debug(inspect(params))
-
-    lease_seconds =
-      if params["hub.lease_seconds"] do
-        String.to_integer(params["hub.lease_seconds"])
-      else
-        # Guess 3 days
-        60 * 60 * 24 * 3
-      end
-
-    with %WebsubClientSubscription{} = websub <-
-           Repo.get_by(WebsubClientSubscription, id: id, topic: topic) do
-      valid_until = NaiveDateTime.add(NaiveDateTime.utc_now(), lease_seconds)
-      change = Ecto.Changeset.change(websub, %{state: "accepted", valid_until: valid_until})
-      {:ok, _websub} = Repo.update(change)
-
-      conn
-      |> send_resp(200, challenge)
-    else
-      _e ->
-        conn
-        |> send_resp(500, "Error")
-    end
-  end
-
-  def websub_subscription_confirmation(conn, params) do
-    Logger.info("Invalid WebSub confirmation request: #{inspect(params)}")
-
-    conn
-    |> send_resp(500, "Invalid parameters")
-  end
-
-  def websub_incoming(conn, %{"id" => id}) do
-    with "sha1=" <> signature <- hd(get_req_header(conn, "x-hub-signature")),
-         signature <- String.downcase(signature),
-         %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id),
-         {:ok, body, _conn} = read_body(conn),
-         ^signature <- Websub.sign(websub.secret, body) do
-      Federator.incoming_doc(body)
-
-      conn
-      |> send_resp(200, "OK")
-    else
-      _e ->
-        Logger.debug("Can't handle incoming subscription post")
-
-        conn
-        |> send_resp(500, "Error")
-    end
-  end
-end

lib/pleroma/web/websub/websub_server_subscription.ex

@@ -1,17 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.Websub.WebsubServerSubscription do
-  use Ecto.Schema
-
-  schema "websub_server_subscriptions" do
-    field(:topic, :string)
-    field(:callback, :string)
-    field(:secret, :string)
-    field(:valid_until, :naive_datetime)
-    field(:state, :string)
-
-    timestamps()
-  end
-end

lib/pleroma/workers/receiver_worker.ex

@@ -8,10 +8,6 @@ defmodule Pleroma.Workers.ReceiverWorker do
   use Pleroma.Workers.WorkerHelper, queue: "federator_incoming"
 
   @impl Oban.Worker
-  def perform(%{"op" => "incoming_doc", "body" => doc}, _job) do
-    Federator.perform(:incoming_doc, doc)
-  end
-
   def perform(%{"op" => "incoming_ap_doc", "params" => params}, _job) do
     Federator.perform(:incoming_ap_doc, params)
   end

lib/pleroma/workers/subscriber_worker.ex

@@ -1,26 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Workers.SubscriberWorker do
-  alias Pleroma.Repo
-  alias Pleroma.Web.Federator
-  alias Pleroma.Web.Websub
-
-  use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing"
-
-  @impl Oban.Worker
-  def perform(%{"op" => "refresh_subscriptions"}, _job) do
-    Federator.perform(:refresh_subscriptions)
-  end
-
-  def perform(%{"op" => "request_subscription", "websub_id" => websub_id}, _job) do
-    websub = Repo.get(Websub.WebsubClientSubscription, websub_id)
-    Federator.perform(:request_subscription, websub)
-  end
-
-  def perform(%{"op" => "verify_websub", "websub_id" => websub_id}, _job) do
-    websub = Repo.get(Websub.WebsubServerSubscription, websub_id)
-    Federator.perform(:verify_websub, websub)
-  end
-end

mix.exs

@@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do
   def project do
     [
       app: :pleroma,
-      version: version("1.0.0"),
+      version: version("1.1.50"),
       elixir: "~> 1.8",
       elixirc_paths: elixirc_paths(Mix.env()),
       compilers: [:phoenix, :gettext] ++ Mix.compilers(),
@@ -69,6 +69,7 @@ def application do
   end
 
   # Specifies which paths to compile per environment.
+  defp elixirc_paths(:benchmark), do: ["lib", "benchmarks"]
   defp elixirc_paths(:test), do: ["lib", "test/support"]
   defp elixirc_paths(_), do: ["lib"]
 

priv/repo/migrations/20190710125158_add_following_address_from_source_data.exs

@@ -5,7 +5,11 @@ defmodule Pleroma.Repo.Migrations.AddFollowingAddressFromSourceData do
 
   def change do
     query =
-      User.external_users_query()
+      User.Query.build(%{
+        external: true,
+        legacy_active: true,
+        order_by: :id
+      })
       |> select([u], struct(u, [:id, :ap_id, :info]))
 
     Pleroma.Repo.stream(query)

priv/repo/migrations/20190711042021_create_safe_jsonb_set.exs

@@ -0,0 +1,22 @@
+defmodule Pleroma.Repo.Migrations.CreateSafeJsonbSet do
+  use Ecto.Migration
+  alias Pleroma.User
+
+  def change do
+    execute("""
+    create or replace function safe_jsonb_set(target jsonb, path text[], new_value jsonb, create_missing boolean default true) returns jsonb as $$
+    declare
+      result jsonb;
+    begin
+      result := jsonb_set(target, path, coalesce(new_value, 'null'::jsonb), create_missing);
+      if result is NULL then
+        raise 'jsonb_set tried to wipe the object, please report this incindent to Pleroma bug tracker. https://git.pleroma.social/pleroma/pleroma/issues/new';
+        return target;
+      else
+        return result;
+      end if;
+    end;
+    $$ language plpgsql;
+    """)
+  end
+end

priv/repo/migrations/20190711042024_copy_muted_to_muted_notifications.exs

@@ -1,10 +1,9 @@
 defmodule Pleroma.Repo.Migrations.CopyMutedToMutedNotifications do
   use Ecto.Migration
-  alias Pleroma.User
 
   def change do
     execute(
-      "update users set info = jsonb_set(info, '{muted_notifications}', info->'mutes', true) where local = true"
+      "update users set info = safe_jsonb_set(info, '{muted_notifications}', info->'mutes', true) where local = true"
     )
   end
 end

priv/repo/migrations/20191009154606_add_user_info_columns.exs

@@ -0,0 +1,53 @@
+defmodule Pleroma.Repo.Migrations.AddUsersInfoColumns do
+  use Ecto.Migration
+
+  @jsonb_array_default "'[]'::jsonb"
+
+  def change do
+    alter table(:users) do
+      add_if_not_exists(:banner, :map, default: %{})
+      add_if_not_exists(:background, :map, default: %{})
+      add_if_not_exists(:source_data, :map, default: %{})
+      add_if_not_exists(:note_count, :integer, default: 0)
+      add_if_not_exists(:follower_count, :integer, default: 0)
+      add_if_not_exists(:following_count, :integer, default: nil)
+      add_if_not_exists(:locked, :boolean, default: false, null: false)
+      add_if_not_exists(:confirmation_pending, :boolean, default: false, null: false)
+      add_if_not_exists(:password_reset_pending, :boolean, default: false, null: false)
+      add_if_not_exists(:confirmation_token, :text, default: nil)
+      add_if_not_exists(:default_scope, :string, default: "public")
+      add_if_not_exists(:blocks, {:array, :text}, default: [])
+      add_if_not_exists(:domain_blocks, {:array, :text}, default: [])
+      add_if_not_exists(:mutes, {:array, :text}, default: [])
+      add_if_not_exists(:muted_reblogs, {:array, :text}, default: [])
+      add_if_not_exists(:muted_notifications, {:array, :text}, default: [])
+      add_if_not_exists(:subscribers, {:array, :text}, default: [])
+      add_if_not_exists(:deactivated, :boolean, default: false, null: false)
+      add_if_not_exists(:no_rich_text, :boolean, default: false, null: false)
+      add_if_not_exists(:ap_enabled, :boolean, default: false, null: false)
+      add_if_not_exists(:is_moderator, :boolean, default: false, null: false)
+      add_if_not_exists(:is_admin, :boolean, default: false, null: false)
+      add_if_not_exists(:show_role, :boolean, default: true, null: false)
+      add_if_not_exists(:settings, :map, default: nil)
+      add_if_not_exists(:magic_key, :text, default: nil)
+      add_if_not_exists(:uri, :text, default: nil)
+      add_if_not_exists(:hide_followers_count, :boolean, default: false, null: false)
+      add_if_not_exists(:hide_follows_count, :boolean, default: false, null: false)
+      add_if_not_exists(:hide_followers, :boolean, default: false, null: false)
+      add_if_not_exists(:hide_follows, :boolean, default: false, null: false)
+      add_if_not_exists(:hide_favorites, :boolean, default: true, null: false)
+      add_if_not_exists(:unread_conversation_count, :integer, default: 0)
+      add_if_not_exists(:pinned_activities, {:array, :text}, default: [])
+      add_if_not_exists(:email_notifications, :map, default: %{"digest" => false})
+      add_if_not_exists(:mascot, :map, default: nil)
+      add_if_not_exists(:emoji, :map, default: fragment(@jsonb_array_default))
+      add_if_not_exists(:pleroma_settings_store, :map, default: %{})
+      add_if_not_exists(:fields, :map, default: fragment(@jsonb_array_default))
+      add_if_not_exists(:raw_fields, :map, default: fragment(@jsonb_array_default))
+      add_if_not_exists(:discoverable, :boolean, default: false, null: false)
+      add_if_not_exists(:invisible, :boolean, default: false, null: false)
+      add_if_not_exists(:notification_settings, :map, default: %{})
+      add_if_not_exists(:skip_thread_containment, :boolean, default: false, null: false)
+    end
+  end
+end

priv/repo/migrations/20191009154608_copy_users_info_fields_to_users.exs

@@ -0,0 +1,145 @@
+defmodule Pleroma.Repo.Migrations.CopyUsersInfoFieldsToUsers do
+  use Ecto.Migration
+
+  @jsonb_array_default "'[]'::jsonb"
+
+  @info_fields [
+    :banner,
+    :background,
+    :source_data,
+    :note_count,
+    :follower_count,
+    :following_count,
+    :locked,
+    :confirmation_pending,
+    :password_reset_pending,
+    :confirmation_token,
+    :default_scope,
+    :blocks,
+    :domain_blocks,
+    :mutes,
+    :muted_reblogs,
+    :muted_notifications,
+    :subscribers,
+    :deactivated,
+    :no_rich_text,
+    :ap_enabled,
+    :is_moderator,
+    :is_admin,
+    :show_role,
+    :settings,
+    :magic_key,
+    :uri,
+    :hide_followers_count,
+    :hide_follows_count,
+    :hide_followers,
+    :hide_follows,
+    :hide_favorites,
+    :unread_conversation_count,
+    :pinned_activities,
+    :email_notifications,
+    :mascot,
+    :emoji,
+    :pleroma_settings_store,
+    :fields,
+    :raw_fields,
+    :discoverable,
+    :invisible,
+    :skip_thread_containment,
+    :notification_settings
+  ]
+
+  @jsonb_fields [
+    :banner,
+    :background,
+    :source_data,
+    :settings,
+    :email_notifications,
+    :mascot,
+    :pleroma_settings_store,
+    :notification_settings
+  ]
+
+  @array_jsonb_fields [:emoji, :fields, :raw_fields]
+
+  @int_fields [:note_count, :follower_count, :following_count, :unread_conversation_count]
+
+  @boolean_fields [
+    :locked,
+    :confirmation_pending,
+    :password_reset_pending,
+    :deactivated,
+    :no_rich_text,
+    :ap_enabled,
+    :is_moderator,
+    :is_admin,
+    :show_role,
+    :hide_followers_count,
+    :hide_follows_count,
+    :hide_followers,
+    :hide_follows,
+    :hide_favorites,
+    :discoverable,
+    :invisible,
+    :skip_thread_containment
+  ]
+
+  @array_text_fields [
+    :blocks,
+    :domain_blocks,
+    :mutes,
+    :muted_reblogs,
+    :muted_notifications,
+    :subscribers,
+    :pinned_activities
+  ]
+
+  def change do
+    if direction() == :up do
+      sets =
+        for f <- @info_fields do
+          set_field = "#{f} ="
+
+          # Coercion of null::jsonb to NULL
+          jsonb = "case when info->>'#{f}' IS NULL then null else info->'#{f}' end"
+
+          cond do
+            f in @jsonb_fields ->
+              "#{set_field} #{jsonb}"
+
+            f in @array_jsonb_fields ->
+              "#{set_field} coalesce(#{jsonb}, #{@jsonb_array_default})"
+
+            f in @int_fields ->
+              "#{set_field} (info->>'#{f}')::int"
+
+            f in @boolean_fields ->
+              "#{set_field} coalesce((info->>'#{f}')::boolean, false)"
+
+            f in @array_text_fields ->
+              "#{set_field} ARRAY(SELECT jsonb_array_elements_text(#{jsonb}))"
+
+            true ->
+              "#{set_field} info->>'#{f}'"
+          end
+        end
+        |> Enum.join(", ")
+
+      execute("update users set " <> sets)
+
+      for index_name <- [
+            :users_deactivated_index,
+            :users_is_moderator_index,
+            :users_is_admin_index,
+            :users_subscribers_index
+          ] do
+        drop_if_exists(index(:users, [], name: index_name))
+      end
+    end
+
+    create_if_not_exists(index(:users, [:deactivated]))
+    create_if_not_exists(index(:users, [:is_moderator]))
+    create_if_not_exists(index(:users, [:is_admin]))
+    create_if_not_exists(index(:users, [:subscribers]))
+  end
+end

priv/repo/migrations/20191014181019_create_markers.exs

@@ -0,0 +1,15 @@
+defmodule Pleroma.Repo.Migrations.CreateMarkers do
+  use Ecto.Migration
+
+  def change do
+    create_if_not_exists table(:markers) do
+      add(:user_id, references(:users, type: :uuid, on_delete: :delete_all))
+      add(:timeline, :string, default: "", null: false)
+      add(:last_read_id, :string, default: "", null: false)
+      add(:lock_version, :integer, default: 0, null: false)
+      timestamps()
+    end
+
+    create_if_not_exists(unique_index(:markers, [:user_id, :timeline]))
+  end
+end

priv/repo/migrations/20191017225002_drop_websub_tables.exs

@@ -0,0 +1,10 @@
+defmodule Pleroma.Repo.Migrations.DropWebsubTables do
+  use Ecto.Migration
+
+  def up do
+    drop_if_exists(table(:websub_client_subscriptions))
+    drop_if_exists(table(:websub_server_subscriptions))
+  end
+
+  def down, do: :noop
+end

priv/repo/migrations/20191026190317_set_not_null_for_activities.exs

@@ -0,0 +1,17 @@
+defmodule Pleroma.Repo.Migrations.SetNotNullForActivities do
+  use Ecto.Migration
+
+  # modify/3 function will require index recreation, so using execute/1 instead
+
+  def up do
+    execute("ALTER TABLE activities
+    ALTER COLUMN data SET NOT NULL,
+    ALTER COLUMN local SET NOT NULL")
+  end
+
+  def down do
+    execute("ALTER TABLE activities
+    ALTER COLUMN data DROP NOT NULL,
+    ALTER COLUMN local DROP NOT NULL")
+  end
+end