moon
/
honkoma
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'remotes/origin/develop' into ostatus-controller-no-auth-check-on-non-federating-instances

This commit is contained in:
Ivan Tashkinov 2020-10-11 22:34:48 +03:00
parent 89c595b772 bc3cf0fee0
commit 2498e569f1
13 changed files with 190 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
### Changed
- **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
- **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
- Search: Users are now findable by their urls.
- Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
- Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.

13
config/config.exs
View File

@ -677,7 +677,18 @@
config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600
config :pleroma, Pleroma.Plugs.RemoteIp, enabled: true
config :pleroma, Pleroma.Plugs.RemoteIp,
enabled: true,
headers: ["x-forwarded-for"],
proxies: [],
reserved: [
"127.0.0.0/8",
"::1/128",
"fc00::/7",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16"
]
config :pleroma, :static_fe, enabled: false

19
config/description.exs
View File

@ -44,11 +44,13 @@
},
%{
key: "git",
label: "Git Repository URL",
type: :string,
description: "URL of the git repository of the frontend"
},
%{
key: "build_url",
label: "Build URL",
type: :string,
description:
"Either an url to a zip file containing the frontend or a template to build it by inserting the `ref`. The string `${ref}` will be replaced by the configured `ref`.",
@ -56,6 +58,7 @@
},
%{
key: "build_dir",
label: "Build directory",
type: :string,
description: "The directory inside the zip file "
}
@ -3262,20 +3265,22 @@
%{
key: :headers,
type: {:list, :string},
description:
"A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Default: `~w[forwarded x-forwarded-for x-client-ip x-real-ip]`."
description: """
A list of strings naming the HTTP headers to use when deriving the true client IP. Default: `["x-forwarded-for"]`.
"""
},
%{
key: :proxies,
type: {:list, :string},
description:
"A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Default: `[]`."
"A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128."
},
%{
key: :reserved,
type: {:list, :string},
description:
"Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network)."
description: """
A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`
"""
}
]
},
@ -3681,9 +3686,7 @@
type: :map,
description:
"A map containing available frontends and parameters for their installation.",
children: [
frontend_options
]
children: frontend_options
}
]
},

18
docs/administration/backup.md
View File

@ -5,20 +5,24 @@
1. Stop the Pleroma service.
2. Go to the working directory of Pleroma (default is `/opt/pleroma`)
3. Run `sudo -Hu postgres pg_dump -d <pleroma_db> --format=custom -f </path/to/backup_location/pleroma.pgdump>` (make sure the postgres user has write access to the destination file)
4. Copy `pleroma.pgdump`, `config/prod.secret.exs` and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
4. Copy `pleroma.pgdump`, `config/prod.secret.exs`, `config/setup_db.psql` (if still available) and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
5. Restart the Pleroma service.
## Restore/Move
1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). Try to use the same database name.
1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers).
2. Stop the Pleroma service.
3. Go to the working directory of Pleroma (default is `/opt/pleroma`)
4. Copy the above mentioned files back to their original position.
5. Drop the existing database and recreate an empty one `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'CREATE DATABASE <pleroma_db>;';`
6. Run `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
7. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
8. Restart the Pleroma service.
9. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.
5. Drop the existing database if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;'`
6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`.
Alternatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the database user, name, and password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed.
7. Now restore the Pleroma instance's data into the empty database schema: `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
8. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
9. Restart the Pleroma service.
10. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.
[^1]: Prefix with `MIX_ENV=prod` to run it using the production config file.

6
docs/configuration/cheatsheet.md
View File

@ -426,9 +426,9 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
Available options:
* `enabled` - Enable/disable the plug. Defaults to `false`.
* `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Defaults to `["x-forwarded-for"]`.
* `proxies` - A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Defaults to `[]`.
* `reserved` - Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network).
* `headers` - A list of strings naming the HTTP headers to use when deriving the true client IP address. Defaults to `["x-forwarded-for"]`.
* `proxies` - A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128.
* `reserved` - A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`.
### :rate_limit

2
lib/pleroma/emails/admin_email.ex
View File

@ -88,7 +88,7 @@ def new_unapproved_registration(to, account) do
html_body = """
<p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p>
<blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote>
<a href="#{Pleroma.Web.base_url()}/pleroma/admin">Visit AdminFE</a>
<a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
"""
new()

44
lib/pleroma/plugs/remote_ip.ex
View File

@ -7,48 +7,42 @@ defmodule Pleroma.Plugs.RemoteIp do
This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
"""
alias Pleroma.Config
import Plug.Conn
@behaviour Plug
@headers ~w[
x-forwarded-for
]
# https://en.wikipedia.org/wiki/Localhost
# https://en.wikipedia.org/wiki/Private_network
@reserved ~w[
127.0.0.0/8
::1/128
fc00::/7
10.0.0.0/8
172.16.0.0/12
192.168.0.0/16
]
def init(_), do: nil
def call(%{remote_ip: original_remote_ip} = conn, _) do
config = Pleroma.Config.get(__MODULE__, [])
if Keyword.get(config, :enabled, false) do
%{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config))
if Config.get([__MODULE__, :enabled]) do
%{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts())
assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip)
else
conn
end
end
defp remote_ip_opts(config) do
headers = config |> Keyword.get(:headers, @headers) |> MapSet.new()
reserved = Keyword.get(config, :reserved, @reserved)
defp remote_ip_opts do
headers = Config.get([__MODULE__, :headers], []) |> MapSet.new()
reserved = Config.get([__MODULE__, :reserved], [])
proxies =
config
|> Keyword.get(:proxies, [])
Config.get([__MODULE__, :proxies], [])
|> Enum.concat(reserved)
|> Enum.map(&InetCidr.parse/1)
|> Enum.map(&maybe_add_cidr/1)
{headers, proxies}
end
defp maybe_add_cidr(proxy) when is_binary(proxy) do
proxy =
cond do
"/" in String.codepoints(proxy) -> proxy
InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32"
InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128"
end
InetCidr.parse(proxy, true)
end
end

12
lib/pleroma/web/activity_pub/activity_pub.ex
View File

@ -790,7 +790,17 @@ defp restrict_replies(query, %{
[activity, object] in query,
where:
fragment(
"?->>'inReplyTo' is null OR ? && array_remove(?, ?) OR ? = ?",
"""
?->>'type' != 'Create' -- This isn't a Create
OR ?->>'inReplyTo' is null -- this isn't a reply
OR ? && array_remove(?, ?) -- The recipient is us or one of our friends,
-- unless they are the author (because authors
-- are also part of the recipients). This leads
-- to a bug that self-replies by friends won't
-- show up.
OR ? = ? -- The actor is us
""",
activity.data,
object.data,
^[user.ap_id | User.get_cached_user_friends_ap_ids(user)],
activity.recipients,

1
lib/pleroma/web/rich_media/helpers.ex
View File

@ -57,7 +57,6 @@ defp get_tld(host) do
def fetch_data_for_object(object) do
with true <- Config.get([:rich_media, :enabled]),
false <- object.data["sensitive"] || false,
{:ok, page_url} <-
HTML.extract_first_external_url_from_object(object),
:ok <- validate_page_url(page_url),

2
test/emails/admin_email_test.exs
View File

@ -63,7 +63,7 @@ test "new unapproved registration email" do
assert res.html_body == """
<p>New account for review: <a href="#{account_url}">@#{account.nickname}</a></p>
<blockquote>Plz let me in</blockquote>
<a href="http://localhost:4001/pleroma/admin">Visit AdminFE</a>
<a href="http://localhost:4001/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
"""
end
end

47
test/plugs/remote_ip_test.exs
View File

@ -3,13 +3,27 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Plugs.RemoteIpTest do
use ExUnit.Case, async: true
use ExUnit.Case
use Plug.Test
alias Pleroma.Plugs.RemoteIp
import Pleroma.Tests.Helpers, only: [clear_config: 1, clear_config: 2]
setup do: clear_config(RemoteIp)
import Pleroma.Tests.Helpers, only: [clear_config: 2]
setup do:
clear_config(RemoteIp,
enabled: true,
headers: ["x-forwarded-for"],
proxies: [],
reserved: [
"127.0.0.0/8",
"::1/128",
"fc00::/7",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16"
]
)
test "disabled" do
Pleroma.Config.put(RemoteIp, enabled: false)
@ -25,8 +39,6 @@ test "disabled" do
end
test "enabled" do
Pleroma.Config.put(RemoteIp, enabled: true)
conn =
conn(:get, "/")
|> put_req_header("x-forwarded-for", "1.1.1.1")
@ -54,8 +66,6 @@ test "custom headers" do
end
test "custom proxies" do
Pleroma.Config.put(RemoteIp, enabled: true)
conn =
conn(:get, "/")
|> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1, 173.245.48.2")
@ -72,4 +82,27 @@ test "custom proxies" do
assert conn.remote_ip == {1, 1, 1, 1}
end
test "proxies set without CIDR format" do
Pleroma.Config.put([RemoteIp, :proxies], ["173.245.48.1"])
conn =
conn(:get, "/")
|> put_req_header("x-forwarded-for", "173.245.48.1, 1.1.1.1")
|> RemoteIp.call(nil)
assert conn.remote_ip == {1, 1, 1, 1}
end
test "proxies set `nonsensical` CIDR" do
Pleroma.Config.put([RemoteIp, :reserved], ["127.0.0.0/8"])
Pleroma.Config.put([RemoteIp, :proxies], ["10.0.0.3/24"])
conn =
conn(:get, "/")
|> put_req_header("x-forwarded-for", "10.0.0.3, 1.1.1.1")
|> RemoteIp.call(nil)
assert conn.remote_ip == {1, 1, 1, 1}
end
end

80
test/web/activity_pub/activity_pub_test.exs
View File

@ -2177,4 +2177,84 @@ test "does nothing with a clashing nickname and the same ap id" do
assert user.nickname == orig_user.nickname
end
end
describe "reply filtering" do
test "`following` still contains announcements by friends" do
user = insert(:user)
followed = insert(:user)
not_followed = insert(:user)
User.follow(user, followed)
{:ok, followed_post} = CommonAPI.post(followed, %{status: "Hello"})
{:ok, not_followed_to_followed} =
CommonAPI.post(not_followed, %{
status: "Also hello",
in_reply_to_status_id: followed_post.id
})
{:ok, retoot} = CommonAPI.repeat(not_followed_to_followed.id, followed)
params =
%{}
|> Map.put(:type, ["Create", "Announce"])
|> Map.put(:blocking_user, user)
|> Map.put(:muting_user, user)
|> Map.put(:reply_filtering_user, user)
|> Map.put(:reply_visibility, "following")
|> Map.put(:announce_filtering_user, user)
|> Map.put(:user, user)
activities =
[user.ap_id | User.following(user)]
|> ActivityPub.fetch_activities(params)
followed_post_id = followed_post.id
retoot_id = retoot.id
assert [%{id: ^followed_post_id}, %{id: ^retoot_id}] = activities
assert length(activities) == 2
end
# This test is skipped because, while this is the desired behavior,
# there seems to be no good way to achieve it with the method that
# we currently use for detecting to who a reply is directed.
# This is a TODO and should be fixed by a later rewrite of the code
# in question.
@tag skip: true
test "`following` still contains self-replies by friends" do
user = insert(:user)
followed = insert(:user)
not_followed = insert(:user)
User.follow(user, followed)
{:ok, followed_post} = CommonAPI.post(followed, %{status: "Hello"})
{:ok, not_followed_post} = CommonAPI.post(not_followed, %{status: "Also hello"})
{:ok, _followed_to_not_followed} =
CommonAPI.post(followed, %{status: "sup", in_reply_to_status_id: not_followed_post.id})
{:ok, _followed_self_reply} =
CommonAPI.post(followed, %{status: "Also cofe", in_reply_to_status_id: followed_post.id})
params =
%{}
|> Map.put(:type, ["Create", "Announce"])
|> Map.put(:blocking_user, user)
|> Map.put(:muting_user, user)
|> Map.put(:reply_filtering_user, user)
|> Map.put(:reply_visibility, "following")
|> Map.put(:announce_filtering_user, user)
|> Map.put(:user, user)
activities =
[user.ap_id | User.following(user)]
|> ActivityPub.fetch_activities(params)
assert length(activities) == 2
end
end
end

35
test/web/rich_media/helpers_test.exs
View File

@ -64,41 +64,6 @@ test "crawls valid, complete URLs" do
Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
end
test "refuses to crawl URLs from posts marked sensitive" do
user = insert(:user)
{:ok, activity} =
CommonAPI.post(user, %{
status: "http://example.com/ogp",
sensitive: true
})
%Object{} = object = Object.normalize(activity)
assert object.data["sensitive"]
Config.put([:rich_media, :enabled], true)
assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
end
test "refuses to crawl URLs from posts tagged NSFW" do
user = insert(:user)
{:ok, activity} =
CommonAPI.post(user, %{
status: "http://example.com/ogp #nsfw"
})
%Object{} = object = Object.normalize(activity)
assert object.data["sensitive"]
Config.put([:rich_media, :enabled], true)
assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
end
test "refuses to crawl URLs of private network from posts" do
user = insert(:user)