moon
/
honkoma
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'issues/1016' into 'develop' 

[#1016] add clear formatted text/ removes links with private ip

See merge request pleroma/pleroma!1335
This commit is contained in:
rinpatch 2019-06-26 03:55:20 +00:00
parent d80859731e 437fd6046d
commit e6f036b9ad
8 changed files with 102 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CHANGELOG.md
View File

@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## [unreleased]
### Security
- Mastodon API: Fix display names not being sanitized
- Rich media: Do not crawl private IP ranges
### Added
- Add a generic settings store for frontends / clients to use.
- Explicit addressing option for posting.
@ -63,6 +65,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Configuration: `skip_thread_containment` option
- Configuration: `rate_limit` option. See `Pleroma.Plugs.RateLimiter` documentation for details.
- MRF: Support for filtering out likely spam messages by rejecting posts from new users that contain links.
- Configuration: `ignore_hosts` option
- Configuration: `ignore_tld` option
### Changed
- **Breaking:** bind to 127.0.0.1 instead of 0.0.0.0 by default
@ -102,6 +106,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Posts which are marked sensitive or tagged nsfw no longer have link previews.
- HTTP connection timeout is now set to 10 seconds.
- Respond with a 404 Not implemented JSON error message when requested API is not implemented
- Rich Media: crawl only https URLs.
### Fixed
- Follow requests don't get 'stuck' anymore.

5
config/config.exs
View File

@ -330,7 +330,10 @@
config :pleroma, :mrf_subchain, match_actor: %{}
config :pleroma, :rich_media, enabled: true
config :pleroma, :rich_media,
enabled: true,
ignore_hosts: [],
ignore_tld: ["local", "localdomain", "lan"]
config :pleroma, :media_proxy,
enabled: false,

6
config/test.exs
View File

@ -43,7 +43,11 @@
config :pbkdf2_elixir, rounds: 1
config :tesla, adapter: Tesla.Mock
config :pleroma, :rich_media, enabled: false
config :pleroma, :rich_media,
enabled: false,
ignore_hosts: [],
ignore_tld: ["local", "localdomain", "lan"]
config :web_push_encryption, :vapid_details,
subject: "mailto:administrator@example.com",

2
docs/config.md
View File

@ -417,6 +417,8 @@ This config contains two queues: `federator_incoming` and `federator_outgoing`.
## :rich_media
* `enabled`: if enabled the instance will parse metadata from attached links to generate link previews
* `ignore_hosts`: list of hosts which will be ignored by the metadata parser. For example `["accounts.google.com", "xss.website"]`, defaults to `[]`.
* `ignore_tld`: list TLDs (top-level domains) which will ignore for parse metadata. default is ["local", "localdomain", "lan"]
## :fetch_initial_posts
* `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts

40
lib/pleroma/web/rich_media/helpers.ex
View File

@ -4,27 +4,53 @@
defmodule Pleroma.Web.RichMedia.Helpers do
alias Pleroma.Activity
alias Pleroma.Config
alias Pleroma.HTML
alias Pleroma.Object
alias Pleroma.Web.RichMedia.Parser
@spec validate_page_url(any()) :: :ok | :error
defp validate_page_url(page_url) when is_binary(page_url) do
validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld]
if AutoLinker.Parser.url?(page_url, scheme: true, validate_tld: validate_tld) do
URI.parse(page_url) |> validate_page_url
else
page_url
|> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld)
|> parse_uri(page_url)
end
defp validate_page_url(%URI{host: host, scheme: scheme, authority: authority})
when scheme == "https" and not is_nil(authority) do
cond do
host in Config.get([:rich_media, :ignore_hosts], []) ->
:error
get_tld(host) in Config.get([:rich_media, :ignore_tld], []) ->
:error
true ->
:ok
end
end
defp validate_page_url(%URI{authority: nil}), do: :error
defp validate_page_url(%URI{scheme: nil}), do: :error
defp validate_page_url(%URI{}), do: :ok
defp validate_page_url(_), do: :error
defp parse_uri(true, url) do
url
|> URI.parse()
|> validate_page_url
end
defp parse_uri(_, _), do: :error
defp get_tld(host) do
host
|> String.split(".")
|> Enum.reverse()
|> hd
end
def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
with true <- Pleroma.Config.get([:rich_media, :enabled]),
with true <- Config.get([:rich_media, :enabled]),
%Object{} = object <- Object.normalize(activity),
false <- object.data["sensitive"] || false,
{:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),

16
test/support/http_request_mock.ex
View File

@ -757,6 +757,14 @@ def get("http://example.com/ogp", _, _, _) do
{:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}}
end
def get("https://example.com/ogp", _, _, _) do
{:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}}
end
def get("https://pleroma.local/notice/9kCP7V", _, _, _) do
{:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}}
end
def get("http://example.com/ogp-missing-data", _, _, _) do
{:ok,
%Tesla.Env{
@ -765,6 +773,14 @@ def get("http://example.com/ogp-missing-data", _, _, _) do
}}
end
def get("https://example.com/ogp-missing-data", _, _, _) do
{:ok,
%Tesla.Env{
status: 200,
body: File.read!("test/fixtures/rich_media/ogp-missing-data.html")
}}
end
def get("http://example.com/malformed", _, _, _) do
{:ok,
%Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/malformed-data.html")}}

9
test/web/mastodon_api/mastodon_api_controller_test.exs
View File

@ -312,7 +312,7 @@ test "posting a status with OGP link preview", %{conn: conn} do
conn
|> assign(:user, user)
|> post("/api/v1/statuses", %{
"status" => "http://example.com/ogp"
"status" => "https://example.com/ogp"
})
assert %{"id" => id, "card" => %{"title" => "The Rock"}} = json_response(conn, 200)
@ -2557,7 +2557,7 @@ test "max pinned statuses", %{conn: conn, user: user, activity: activity_one} do
end
test "returns rich-media card", %{conn: conn, user: user} do
{:ok, activity} = CommonAPI.post(user, %{"status" => "http://example.com/ogp"})
{:ok, activity} = CommonAPI.post(user, %{"status" => "https://example.com/ogp"})
card_data = %{
"image" => "http://ia.media-imdb.com/images/rock.jpg",
@ -2589,7 +2589,7 @@ test "returns rich-media card", %{conn: conn, user: user} do
# works with private posts
{:ok, activity} =
CommonAPI.post(user, %{"status" => "http://example.com/ogp", "visibility" => "direct"})
CommonAPI.post(user, %{"status" => "https://example.com/ogp", "visibility" => "direct"})
response_two =
conn
@ -2601,7 +2601,8 @@ test "returns rich-media card", %{conn: conn, user: user} do
end
test "replaces missing description with an empty string", %{conn: conn, user: user} do
{:ok, activity} = CommonAPI.post(user, %{"status" => "http://example.com/ogp-missing-data"})
{:ok, activity} =
CommonAPI.post(user, %{"status" => "https://example.com/ogp-missing-data"})
response =
conn

47
test/web/rich_media/helpers_test.exs
View File

@ -1,14 +1,19 @@
defmodule Pleroma.Web.RichMedia.HelpersTest do
use Pleroma.DataCase
alias Pleroma.Config
alias Pleroma.Object
alias Pleroma.Web.CommonAPI
alias Pleroma.Web.RichMedia.Helpers
import Pleroma.Factory
import Tesla.Mock
setup do
mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
rich_media = Config.get([:rich_media, :enabled])
on_exit(fn -> Config.put([:rich_media, :enabled], rich_media) end)
:ok
end
@ -21,11 +26,9 @@ test "refuses to crawl incomplete URLs" do
"content_type" => "text/markdown"
})
Pleroma.Config.put([:rich_media, :enabled], true)
Config.put([:rich_media, :enabled], true)
assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
Pleroma.Config.put([:rich_media, :enabled], false)
end
test "refuses to crawl malformed URLs" do
@ -37,11 +40,9 @@ test "refuses to crawl malformed URLs" do
"content_type" => "text/markdown"
})
Pleroma.Config.put([:rich_media, :enabled], true)
Config.put([:rich_media, :enabled], true)
assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
Pleroma.Config.put([:rich_media, :enabled], false)
end
test "crawls valid, complete URLs" do
@ -49,16 +50,14 @@ test "crawls valid, complete URLs" do
{:ok, activity} =
CommonAPI.post(user, %{
"status" => "[test](http://example.com/ogp)",
"status" => "[test](https://example.com/ogp)",
"content_type" => "text/markdown"
})
Pleroma.Config.put([:rich_media, :enabled], true)
Config.put([:rich_media, :enabled], true)
assert %{page_url: "http://example.com/ogp", rich_media: _} =
assert %{page_url: "https://example.com/ogp", rich_media: _} =
Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
Pleroma.Config.put([:rich_media, :enabled], false)
end
test "refuses to crawl URLs from posts marked sensitive" do
@ -74,11 +73,9 @@ test "refuses to crawl URLs from posts marked sensitive" do
assert object.data["sensitive"]
Pleroma.Config.put([:rich_media, :enabled], true)
Config.put([:rich_media, :enabled], true)
assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
Pleroma.Config.put([:rich_media, :enabled], false)
end
test "refuses to crawl URLs from posts tagged NSFW" do
@ -93,10 +90,28 @@ test "refuses to crawl URLs from posts tagged NSFW" do
assert object.data["sensitive"]
Pleroma.Config.put([:rich_media, :enabled], true)
Config.put([:rich_media, :enabled], true)
assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
end
Pleroma.Config.put([:rich_media, :enabled], false)
test "refuses to crawl URLs of private network from posts" do
user = insert(:user)
{:ok, activity} =
CommonAPI.post(user, %{"status" => "http://127.0.0.1:4000/notice/9kCP7VNyPJXFOXDrgO"})
{:ok, activity2} = CommonAPI.post(user, %{"status" => "https://10.111.10.1/notice/9kCP7V"})
{:ok, activity3} = CommonAPI.post(user, %{"status" => "https://172.16.32.40/notice/9kCP7V"})
{:ok, activity4} = CommonAPI.post(user, %{"status" => "https://192.168.10.40/notice/9kCP7V"})
{:ok, activity5} = CommonAPI.post(user, %{"status" => "https://pleroma.local/notice/9kCP7V"})
Config.put([:rich_media, :enabled], true)
assert %{} = Helpers.fetch_data_for_activity(activity)
assert %{} = Helpers.fetch_data_for_activity(activity2)
assert %{} = Helpers.fetch_data_for_activity(activity3)
assert %{} = Helpers.fetch_data_for_activity(activity4)
assert %{} = Helpers.fetch_data_for_activity(activity5)
end
end