Merge remote-tracking branch 'remotes/origin/develop' into ostatus-controller-no-auth-check-on-non-federating-instances

CHANGELOG.md

@@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ### Changed
 
 - **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
+- **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
 - Search: Users are now findable by their urls.
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.

config/config.exs

@@ -677,7 +677,18 @@
 
 config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600
 
-config :pleroma, Pleroma.Plugs.RemoteIp, enabled: true
+config :pleroma, Pleroma.Plugs.RemoteIp,
+  enabled: true,
+  headers: ["x-forwarded-for"],
+  proxies: [],
+  reserved: [
+    "127.0.0.0/8",
+    "::1/128",
+    "fc00::/7",
+    "10.0.0.0/8",
+    "172.16.0.0/12",
+    "192.168.0.0/16"
+  ]
 
 config :pleroma, :static_fe, enabled: false
 

config/description.exs

@@ -44,11 +44,13 @@
   },
   %{
     key: "git",
+    label: "Git Repository URL",
     type: :string,
     description: "URL of the git repository of the frontend"
   },
   %{
     key: "build_url",
+    label: "Build URL",
     type: :string,
     description:
       "Either an url to a zip file containing the frontend or a template to build it by inserting the `ref`. The string `${ref}` will be replaced by the configured `ref`.",
@@ -56,6 +58,7 @@
   },
   %{
     key: "build_dir",
+    label: "Build directory",
     type: :string,
     description: "The directory inside the zip file "
   }
@@ -3262,20 +3265,22 @@
       %{
         key: :headers,
         type: {:list, :string},
-        description:
+        description: """
-          "A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Default: `~w[forwarded x-forwarded-for x-client-ip x-real-ip]`."
+          A list of strings naming the HTTP headers to use when deriving the true client IP. Default: `["x-forwarded-for"]`.
+        """
       },
       %{
         key: :proxies,
         type: {:list, :string},
         description:
-          "A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Default: `[]`."
+          "A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128."
       },
       %{
         key: :reserved,
         type: {:list, :string},
-        description:
+        description: """
-          "Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network)."
+          A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`
+        """
       }
     ]
   },
@@ -3681,9 +3686,7 @@
         type: :map,
         description:
           "A map containing available frontends and parameters for their installation.",
-        children: [
+        children: frontend_options
-          frontend_options
-        ]
       }
     ]
   },

docs/administration/backup.md

@@ -5,20 +5,24 @@
 1. Stop the Pleroma service.
 2. Go to the working directory of Pleroma (default is `/opt/pleroma`)
 3. Run `sudo -Hu postgres pg_dump -d <pleroma_db> --format=custom -f </path/to/backup_location/pleroma.pgdump>` (make sure the postgres user has write access to the destination file)
-4. Copy `pleroma.pgdump`, `config/prod.secret.exs` and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
+4. Copy `pleroma.pgdump`, `config/prod.secret.exs`, `config/setup_db.psql` (if still available) and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
 5. Restart the Pleroma service.
 
 ## Restore/Move
 
-1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). Try to use the same database name.
+1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers).
 2. Stop the Pleroma service.
 3. Go to the working directory of Pleroma (default is `/opt/pleroma`)
 4. Copy the above mentioned files back to their original position.
-5. Drop the existing database and recreate an empty one `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'CREATE DATABASE <pleroma_db>;';`
+5. Drop the existing database if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;'` 
-6. Run `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
+6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`. 
-7. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
+
-8. Restart the Pleroma service.
+  Alternatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the database user, name, and password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed.
-9. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.
+
+7. Now restore the Pleroma instance's data into the empty database schema: `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
+8. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
+9. Restart the Pleroma service.
+10. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.
 
 [^1]: Prefix with `MIX_ENV=prod` to run it using the production config file.
 

docs/configuration/cheatsheet.md

@@ -426,9 +426,9 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
 Available options:
 
 * `enabled` - Enable/disable the plug. Defaults to `false`.
-* `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Defaults to `["x-forwarded-for"]`.
+* `headers` - A list of strings naming the HTTP headers to use when deriving the true client IP address. Defaults to `["x-forwarded-for"]`.
-* `proxies` - A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Defaults to `[]`.
+* `proxies` - A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128.
-* `reserved` - Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network).
+* `reserved` - A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`.
 
 
 ### :rate_limit

lib/pleroma/emails/admin_email.ex

@@ -88,7 +88,7 @@ def new_unapproved_registration(to, account) do
     html_body = """
     <p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p>
     <blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote>
-    <a href="#{Pleroma.Web.base_url()}/pleroma/admin">Visit AdminFE</a>
+    <a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
     """
 
     new()

lib/pleroma/plugs/remote_ip.ex

@@ -7,48 +7,42 @@ defmodule Pleroma.Plugs.RemoteIp do
   This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
   """
 
+  alias Pleroma.Config
   import Plug.Conn
 
   @behaviour Plug
 
-  @headers ~w[
-    x-forwarded-for
-  ]
-
-  # https://en.wikipedia.org/wiki/Localhost
-  # https://en.wikipedia.org/wiki/Private_network
-  @reserved ~w[
-    127.0.0.0/8
-    ::1/128
-    fc00::/7
-    10.0.0.0/8
-    172.16.0.0/12
-    192.168.0.0/16
-  ]
-
   def init(_), do: nil
 
   def call(%{remote_ip: original_remote_ip} = conn, _) do
-    config = Pleroma.Config.get(__MODULE__, [])
+    if Config.get([__MODULE__, :enabled]) do
-
+      %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts())
-    if Keyword.get(config, :enabled, false) do
-      %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config))
       assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip)
     else
       conn
     end
   end
 
-  defp remote_ip_opts(config) do
+  defp remote_ip_opts do
-    headers = config |> Keyword.get(:headers, @headers) |> MapSet.new()
+    headers = Config.get([__MODULE__, :headers], []) |> MapSet.new()
-    reserved = Keyword.get(config, :reserved, @reserved)
+    reserved = Config.get([__MODULE__, :reserved], [])
 
     proxies =
-      config
+      Config.get([__MODULE__, :proxies], [])
-      |> Keyword.get(:proxies, [])
       |> Enum.concat(reserved)
-      |> Enum.map(&InetCidr.parse/1)
+      |> Enum.map(&maybe_add_cidr/1)
 
     {headers, proxies}
   end
+
+  defp maybe_add_cidr(proxy) when is_binary(proxy) do
+    proxy =
+      cond do
+        "/" in String.codepoints(proxy) -> proxy
+        InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
+        InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
+      end
+
+    InetCidr.parse(proxy, true)
+  end
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -790,7 +790,17 @@ defp restrict_replies(query, %{
       [activity, object] in query,
       where:
         fragment(
-          "?->>'inReplyTo' is null OR ? && array_remove(?, ?) OR ? = ?",
+          """
+          ?->>'type' != 'Create'     -- This isn't a Create      
+          OR ?->>'inReplyTo' is null -- this isn't a reply
+          OR ? && array_remove(?, ?) -- The recipient is us or one of our friends, 
+                                     -- unless they are the author (because authors 
+                                     -- are also part of the recipients). This leads
+                                     -- to a bug that self-replies by friends won't
+                                     -- show up.
+          OR ? = ?                   -- The actor is us
+          """,
+          activity.data,
           object.data,
           ^[user.ap_id | User.get_cached_user_friends_ap_ids(user)],
           activity.recipients,

lib/pleroma/web/rich_media/helpers.ex

@@ -57,7 +57,6 @@ defp get_tld(host) do
 
   def fetch_data_for_object(object) do
     with true <- Config.get([:rich_media, :enabled]),
-         false <- object.data["sensitive"] || false,
          {:ok, page_url} <-
            HTML.extract_first_external_url_from_object(object),
          :ok <- validate_page_url(page_url),

test/emails/admin_email_test.exs

@@ -63,7 +63,7 @@ test "new unapproved registration email" do
     assert res.html_body == """
            <p>New account for review: <a href="#{account_url}">@#{account.nickname}</a></p>
            <blockquote>Plz let me in</blockquote>
-           <a href="http://localhost:4001/pleroma/admin">Visit AdminFE</a>
+           <a href="http://localhost:4001/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
            """
   end
 end

test/plugs/remote_ip_test.exs

@@ -3,13 +3,27 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.RemoteIpTest do
-  use ExUnit.Case, async: true
+  use ExUnit.Case
   use Plug.Test
 
   alias Pleroma.Plugs.RemoteIp
 
-  import Pleroma.Tests.Helpers, only: [clear_config: 1, clear_config: 2]
+  import Pleroma.Tests.Helpers, only: [clear_config: 2]
-  setup do: clear_config(RemoteIp)
+
+  setup do:
+          clear_config(RemoteIp,
+            enabled: true,
+            headers: ["x-forwarded-for"],
+            proxies: [],
+            reserved: [
+              "127.0.0.0/8",
+              "::1/128",
+              "fc00::/7",
+              "10.0.0.0/8",
+              "172.16.0.0/12",
+              "192.168.0.0/16"
+            ]
+          )
 
   test "disabled" do
     Pleroma.Config.put(RemoteIp, enabled: false)
@@ -25,8 +39,6 @@ test "disabled" do
   end
 
   test "enabled" do
-    Pleroma.Config.put(RemoteIp, enabled: true)
-
     conn =
       conn(:get, "/")
       |> put_req_header("x-forwarded-for", "1.1.1.1")
@@ -54,8 +66,6 @@ test "custom headers" do
   end
 
   test "custom proxies" do
-    Pleroma.Config.put(RemoteIp, enabled: true)
-
     conn =
       conn(:get, "/")
       |> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1, 173.245.48.2")
@@ -72,4 +82,27 @@ test "custom proxies" do
 
     assert conn.remote_ip == {1, 1, 1, 1}
   end
+
+  test "proxies set without CIDR format" do
+    Pleroma.Config.put([RemoteIp, :proxies], ["173.245.48.1"])
+
+    conn =
+      conn(:get, "/")
+      |> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1")
+      |> RemoteIp.call(nil)
+
+    assert conn.remote_ip == {1, 1, 1, 1}
+  end
+
+  test "proxies set `nonsensical` CIDR" do
+    Pleroma.Config.put([RemoteIp, :reserved], ["127.0.0.0/8"])
+    Pleroma.Config.put([RemoteIp, :proxies], ["10.0.0.3/24"])
+
+    conn =
+      conn(:get, "/")
+      |> put_req_header("x-forwarded-for", "10.0.0.3, 1.1.1.1")
+      |> RemoteIp.call(nil)
+
+    assert conn.remote_ip == {1, 1, 1, 1}
+  end
 end

test/web/activity_pub/activity_pub_test.exs

@@ -2177,4 +2177,84 @@ test "does nothing with a clashing nickname and the same ap id" do
       assert user.nickname == orig_user.nickname
     end
   end
+
+  describe "reply filtering" do
+    test "`following` still contains announcements by friends" do
+      user = insert(:user)
+      followed = insert(:user)
+      not_followed = insert(:user)
+
+      User.follow(user, followed)
+
+      {:ok, followed_post} = CommonAPI.post(followed, %{status: "Hello"})
+
+      {:ok, not_followed_to_followed} =
+        CommonAPI.post(not_followed, %{
+          status: "Also hello",
+          in_reply_to_status_id: followed_post.id
+        })
+
+      {:ok, retoot} = CommonAPI.repeat(not_followed_to_followed.id, followed)
+
+      params =
+        %{}
+        |> Map.put(:type, ["Create", "Announce"])
+        |> Map.put(:blocking_user, user)
+        |> Map.put(:muting_user, user)
+        |> Map.put(:reply_filtering_user, user)
+        |> Map.put(:reply_visibility, "following")
+        |> Map.put(:announce_filtering_user, user)
+        |> Map.put(:user, user)
+
+      activities =
+        [user.ap_id | User.following(user)]
+        |> ActivityPub.fetch_activities(params)
+
+      followed_post_id = followed_post.id
+      retoot_id = retoot.id
+
+      assert [%{id: ^followed_post_id}, %{id: ^retoot_id}] = activities
+
+      assert length(activities) == 2
+    end
+
+    # This test is skipped because, while this is the desired behavior,
+    # there seems to be no good way to achieve it with the method that
+    # we currently use for detecting to who a reply is directed.
+    # This is a TODO and should be fixed by a later rewrite of the code
+    # in question.
+    @tag skip: true
+    test "`following` still contains self-replies by friends" do
+      user = insert(:user)
+      followed = insert(:user)
+      not_followed = insert(:user)
+
+      User.follow(user, followed)
+
+      {:ok, followed_post} = CommonAPI.post(followed, %{status: "Hello"})
+      {:ok, not_followed_post} = CommonAPI.post(not_followed, %{status: "Also hello"})
+
+      {:ok, _followed_to_not_followed} =
+        CommonAPI.post(followed, %{status: "sup", in_reply_to_status_id: not_followed_post.id})
+
+      {:ok, _followed_self_reply} =
+        CommonAPI.post(followed, %{status: "Also cofe", in_reply_to_status_id: followed_post.id})
+
+      params =
+        %{}
+        |> Map.put(:type, ["Create", "Announce"])
+        |> Map.put(:blocking_user, user)
+        |> Map.put(:muting_user, user)
+        |> Map.put(:reply_filtering_user, user)
+        |> Map.put(:reply_visibility, "following")
+        |> Map.put(:announce_filtering_user, user)
+        |> Map.put(:user, user)
+
+      activities =
+        [user.ap_id | User.following(user)]
+        |> ActivityPub.fetch_activities(params)
+
+      assert length(activities) == 2
+    end
+  end
 end

test/web/rich_media/helpers_test.exs

@@ -64,41 +64,6 @@ test "crawls valid, complete URLs" do
              Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
   end
 
-  test "refuses to crawl URLs from posts marked sensitive" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{
-        status: "http://example.com/ogp",
-        sensitive: true
-      })
-
-    %Object{} = object = Object.normalize(activity)
-
-    assert object.data["sensitive"]
-
-    Config.put([:rich_media, :enabled], true)
-
-    assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-  end
-
-  test "refuses to crawl URLs from posts tagged NSFW" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{
-        status: "http://example.com/ogp #nsfw"
-      })
-
-    %Object{} = object = Object.normalize(activity)
-
-    assert object.data["sensitive"]
-
-    Config.put([:rich_media, :enabled], true)
-
-    assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-  end
-
   test "refuses to crawl URLs of private network from posts" do
     user = insert(:user)