Merge branch 'develop' into feature/digest-email

This commit is contained in:
Roman Chvanikov 2019-06-30 21:23:35 +03:00
commit d2cb18b2a3
16 changed files with 297 additions and 39 deletions

View File

@ -3,6 +3,17 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## [Unreleased]
### Added
- MRF: Support for priming the mediaproxy cache (`Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`)
- Mastodon API: Support for the [`tagged` filter](https://github.com/tootsuite/mastodon/pull/9755) in [`GET /api/v1/accounts/:id/statuses`](https://docs.joinmastodon.org/api/rest/accounts/#get-api-v1-accounts-id-statuses)
### Fixed
- Not being able to pin unlisted posts
### Changed
- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text
## [1.0.0] - 2019-06-29
### Security
- Mastodon API: Fix display names not being sanitized

View File

@ -36,7 +36,7 @@ No specific configuration.
This filter replaces the filename (not the path) of an upload. For complete obfuscation, add
`Pleroma.Upload.Filter.Dedupe` before AnonymizeFilename.
* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used.
* `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used. You can get the original filename extension by using `{extension}`, for example `custom-file-name.{extension}`.
## Pleroma.Emails.Mailer
* `adapter`: one of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters), or `Swoosh.Adapters.Local` for in-memory mailbox.
@ -98,6 +98,7 @@ config :pleroma, Pleroma.Emails.Mailer,
* `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section)
* `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:.
* `Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy`: Rejects posts from likely spambots by rejecting posts from new users that contain links.
* `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy`: Crawls attachments using their MediaProxy URLs so that the MediaProxy cache is primed.
* `public`: Makes the client API in authentificated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network.
* `quarantined_instances`: List of ActivityPub instances where private(DMs, followers-only) activities will not be send.
* `managed_config`: Whenether the config for pleroma-fe is configured in this config or in ``static/config.json``
@ -279,7 +280,7 @@ config :pleroma, :mrf_subchain,
## Pleroma.Web.Endpoint
`Phoenix` endpoint configuration, all configuration options can be viewed [here](https://hexdocs.pm/phoenix/Phoenix.Endpoint.html#module-dynamic-configuration), only common options are listed here
* `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here
* `http` - a list containing http protocol configuration, all configuration options can be viewed [here](https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html#module-options), only common options are listed here. For deployment using docker, you need to set this to `[ip: {0,0,0,0}, port: 4000]` to make pleroma accessible from other containers (such as your nginx server).
- `ip` - a tuple consisting of 4 integers
- `port`
* `url` - a list containing the configuration for generating urls, accepts

View File

@ -49,7 +49,7 @@ mkdir -p /var/lib/pleroma/static
chown -R pleroma /var/lib/pleroma
# If you use the local uploader with default settings your uploads should be located in `~pleroma/uploads`
mv ~pleroma/uploads /var/lib/pleroma/uploads
mv ~pleroma/uploads/* /var/lib/pleroma/uploads
# If you have created the custom public files directory with default settings it should be located in `~pleroma/instance/static`
mv ~pleroma/instance/static /var/lib/pleroma/static
@ -122,13 +122,15 @@ su pleroma -s $SHELL -lc "./bin/pleroma stop"
## Setting up a system service
OTP releases have different service files than from-source installs so they need to be copied over again.
**Warning:** The service files assume pleroma user's home directory is `/opt/pleroma`, please make sure all paths fit your installation.
Debian/Ubuntu:
```sh
# Copy the service into a proper directory
cp ~pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
# Reload service files
systemctl reload-daemon
systemctl daemon-reload
# Reenable pleroma to start on boot
systemctl reenable pleroma

View File

@ -38,7 +38,7 @@ def put([key], value), do: put(key, value)
def put([parent_key | keys], value) do
parent =
Application.get_env(:pleroma, parent_key)
Application.get_env(:pleroma, parent_key, [])
|> put_in(keys, value)
Application.put_env(:pleroma, parent_key, parent)

View File

@ -10,10 +10,19 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do
"""
@behaviour Pleroma.Upload.Filter
def filter(upload) do
extension = List.last(String.split(upload.name, "."))
name = Pleroma.Config.get([__MODULE__, :text], random(extension))
{:ok, %Pleroma.Upload{upload | name: name}}
alias Pleroma.Config
alias Pleroma.Upload
def filter(%Upload{name: name} = upload) do
extension = List.last(String.split(name, "."))
name = predefined_name(extension) || random(extension)
{:ok, %Upload{upload | name: name}}
end
@spec predefined_name(String.t()) :: String.t() | nil
defp predefined_name(extension) do
with name when not is_nil(name) <- Config.get([__MODULE__, :text]),
do: String.replace(name, "{extension}", extension)
end
defp random(extension) do

View File

@ -0,0 +1,56 @@
# Pleroma: A lightweight social networking server
# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
@moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them"
@behaviour Pleroma.Web.ActivityPub.MRF
alias Pleroma.HTTP
alias Pleroma.Web.MediaProxy
require Logger
@hackney_options [
pool: :media,
recv_timeout: 10_000
]
def perform(:prefetch, url) do
Logger.info("Prefetching #{inspect(url)}")
url
|> MediaProxy.url()
|> HTTP.get([], adapter: @hackney_options)
end
def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do
Enum.each(attachments, fn
%{"url" => url} when is_list(url) ->
url
|> Enum.each(fn
%{"href" => href} ->
PleromaJobQueue.enqueue(:background, __MODULE__, [:prefetch, href])
x ->
Logger.debug("Unhandled attachment URL object #{inspect(x)}")
end)
x ->
Logger.debug("Unhandled attachment #{inspect(x)}")
end)
end
@impl true
def filter(
%{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message
)
when is_list(attachments) and length(attachments) > 0 do
PleromaJobQueue.enqueue(:background, __MODULE__, [:preload, message])
{:ok, message}
end
@impl true
def filter(message), do: {:ok, message}
end

View File

@ -5,8 +5,11 @@
defmodule Pleroma.Web.AdminAPI.AccountView do
use Pleroma.Web, :view
alias Pleroma.HTML
alias Pleroma.User
alias Pleroma.User.Info
alias Pleroma.Web.AdminAPI.AccountView
alias Pleroma.Web.MediaProxy
def render("index.json", %{users: users, count: count, page_size: page_size}) do
%{
@ -17,9 +20,14 @@ def render("index.json", %{users: users, count: count, page_size: page_size}) do
end
def render("show.json", %{user: user}) do
avatar = User.avatar_url(user) |> MediaProxy.url()
display_name = HTML.strip_tags(user.name || user.nickname)
%{
"id" => user.id,
"avatar" => avatar,
"nickname" => user.nickname,
"display_name" => display_name,
"deactivated" => user.info.deactivated,
"local" => user.local,
"roles" => Info.roles(user.info),

View File

@ -8,7 +8,6 @@ defmodule Pleroma.Web.AdminAPI.ReportView do
alias Pleroma.HTML
alias Pleroma.User
alias Pleroma.Web.CommonAPI.Utils
alias Pleroma.Web.MastodonAPI.AccountView
alias Pleroma.Web.MastodonAPI.StatusView
def render("index.json", %{reports: reports}) do
@ -38,12 +37,17 @@ def render("show.json", %{report: report}) do
%{
id: report.id,
account: AccountView.render("account.json", %{user: account}),
actor: AccountView.render("account.json", %{user: user}),
account: merge_account_views(account),
actor: merge_account_views(user),
content: content,
created_at: created_at,
statuses: StatusView.render("index.json", %{activities: statuses, as: :activity}),
state: report.data["state"]
}
end
defp merge_account_views(user) do
Pleroma.Web.MastodonAPI.AccountView.render("account.json", %{user: user})
|> Map.merge(Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user}))
end
end

View File

@ -11,6 +11,7 @@ defmodule Pleroma.Web.CommonAPI do
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.ActivityPub.Visibility
import Pleroma.Web.CommonAPI.Utils
@ -284,12 +285,11 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do
},
object: %Object{
data: %{
"to" => object_to,
"type" => "Note"
}
}
} = activity <- get_by_id_or_ap_id(id_or_ap_id),
true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"),
true <- Visibility.is_public?(activity),
%{valid?: true} = info_changeset <-
User.Info.add_pinnned_activity(user.info, activity),
changeset <-

View File

@ -356,6 +356,10 @@ def public_timeline(%{assigns: %{user: user}} = conn, params) do
def user_statuses(%{assigns: %{user: reading_user}} = conn, params) do
with %User{} = user <- User.get_cached_by_id(params["id"]) do
params =
params
|> Map.put("tag", params["tagged"])
activities = ActivityPub.fetch_user_activities(user, reading_user, params)
conn

View File

@ -0,0 +1,40 @@
defmodule Pleroma.Upload.Filter.AnonymizeFilenameTest do
use Pleroma.DataCase
alias Pleroma.Config
alias Pleroma.Upload
setup do
custom_filename = Config.get([Upload.Filter.AnonymizeFilename, :text])
on_exit(fn ->
Config.put([Upload.Filter.AnonymizeFilename, :text], custom_filename)
end)
upload_file = %Upload{
name: "an… image.jpg",
content_type: "image/jpg",
path: Path.absname("test/fixtures/image_tmp.jpg")
}
%{upload_file: upload_file}
end
test "it replaces filename on pre-defined text", %{upload_file: upload_file} do
Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.png")
{:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
assert name == "custom-file.png"
end
test "it replaces filename on pre-defined text expression", %{upload_file: upload_file} do
Config.put([Upload.Filter.AnonymizeFilename, :text], "custom-file.{extension}")
{:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
assert name == "custom-file.jpg"
end
test "it replaces filename on random text", %{upload_file: upload_file} do
{:ok, %Upload{name: name}} = Upload.Filter.AnonymizeFilename.filter(upload_file)
assert <<_::bytes-size(14)>> <> ".jpg" = name
refute name == "an… image.jpg"
end
end

View File

@ -0,0 +1,45 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do
use Pleroma.DataCase
alias Pleroma.HTTP
alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy
import Mock
@message %{
"type" => "Create",
"object" => %{
"type" => "Note",
"content" => "content",
"attachment" => [
%{"url" => [%{"href" => "http://example.com/image.jpg"}]}
]
}
}
test "it prefetches media proxy URIs" do
with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do
MediaProxyWarmingPolicy.filter(@message)
assert called(HTTP.get(:_, :_, :_))
end
end
test "it does nothing when no attachments are present" do
object =
@message["object"]
|> Map.delete("attachment")
message =
@message
|> Map.put("object", object)
with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do
MediaProxyWarmingPolicy.filter(message)
refute called(HTTP.get(:_, :_, :_))
end
end
end

View File

@ -6,9 +6,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
use Pleroma.Web.ConnCase
alias Pleroma.Activity
alias Pleroma.HTML
alias Pleroma.User
alias Pleroma.UserInviteToken
alias Pleroma.Web.CommonAPI
alias Pleroma.Web.MediaProxy
import Pleroma.Factory
describe "/api/pleroma/admin/users" do
@ -58,7 +60,9 @@ test "Show", %{conn: conn} do
"local" => true,
"nickname" => user.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(user) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user.name || user.nickname)
}
assert expected == json_response(conn, 200)
@ -445,7 +449,9 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do
"nickname" => admin.nickname,
"roles" => %{"admin" => true, "moderator" => false},
"local" => true,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(admin) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(admin.name || admin.nickname)
},
%{
"deactivated" => user.info.deactivated,
@ -453,7 +459,9 @@ test "renders users array for the first page", %{conn: conn, admin: admin} do
"nickname" => user.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => false,
"tags" => ["foo", "bar"]
"tags" => ["foo", "bar"],
"avatar" => User.avatar_url(user) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user.name || user.nickname)
}
]
|> Enum.sort_by(& &1["nickname"])
@ -492,7 +500,9 @@ test "regular search", %{conn: conn} do
"nickname" => user.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => true,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(user) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user.name || user.nickname)
}
]
}
@ -514,7 +524,9 @@ test "search by domain", %{conn: conn} do
"nickname" => user.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => true,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(user) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user.name || user.nickname)
}
]
}
@ -536,7 +548,9 @@ test "search by full nickname", %{conn: conn} do
"nickname" => user.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => true,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(user) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user.name || user.nickname)
}
]
}
@ -558,7 +572,9 @@ test "search by display name", %{conn: conn} do
"nickname" => user.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => true,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(user) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user.name || user.nickname)
}
]
}
@ -580,7 +596,9 @@ test "search by email", %{conn: conn} do
"nickname" => user.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => true,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(user) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user.name || user.nickname)
}
]
}
@ -602,7 +620,9 @@ test "regular search with page size", %{conn: conn} do
"nickname" => user.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => true,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(user) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user.name || user.nickname)
}
]
}
@ -619,7 +639,9 @@ test "regular search with page size", %{conn: conn} do
"nickname" => user2.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => true,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(user2) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user2.name || user2.nickname)
}
]
}
@ -646,7 +668,9 @@ test "only local users" do
"nickname" => user.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => true,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(user) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user.name || user.nickname)
}
]
}
@ -671,7 +695,9 @@ test "only local users with no query", %{admin: old_admin} do
"nickname" => user.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => true,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(user) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user.name || user.nickname)
},
%{
"deactivated" => admin.info.deactivated,
@ -679,7 +705,9 @@ test "only local users with no query", %{admin: old_admin} do
"nickname" => admin.nickname,
"roles" => %{"admin" => true, "moderator" => false},
"local" => true,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(admin) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(admin.name || admin.nickname)
},
%{
"deactivated" => false,
@ -687,7 +715,9 @@ test "only local users with no query", %{admin: old_admin} do
"local" => true,
"nickname" => old_admin.nickname,
"roles" => %{"admin" => true, "moderator" => false},
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(old_admin) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(old_admin.name || old_admin.nickname)
}
]
|> Enum.sort_by(& &1["nickname"])
@ -714,7 +744,9 @@ test "load only admins", %{conn: conn, admin: admin} do
"nickname" => admin.nickname,
"roles" => %{"admin" => true, "moderator" => false},
"local" => admin.local,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(admin) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(admin.name || admin.nickname)
},
%{
"deactivated" => false,
@ -722,7 +754,9 @@ test "load only admins", %{conn: conn, admin: admin} do
"nickname" => second_admin.nickname,
"roles" => %{"admin" => true, "moderator" => false},
"local" => second_admin.local,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(second_admin) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(second_admin.name || second_admin.nickname)
}
]
|> Enum.sort_by(& &1["nickname"])
@ -751,7 +785,9 @@ test "load only moderators", %{conn: conn} do
"nickname" => moderator.nickname,
"roles" => %{"admin" => false, "moderator" => true},
"local" => moderator.local,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(moderator) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(moderator.name || moderator.nickname)
}
]
}
@ -773,7 +809,9 @@ test "load users with tags list", %{conn: conn} do
"nickname" => user1.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => user1.local,
"tags" => ["first"]
"tags" => ["first"],
"avatar" => User.avatar_url(user1) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user1.name || user1.nickname)
},
%{
"deactivated" => false,
@ -781,7 +819,9 @@ test "load users with tags list", %{conn: conn} do
"nickname" => user2.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => user2.local,
"tags" => ["second"]
"tags" => ["second"],
"avatar" => User.avatar_url(user2) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user2.name || user2.nickname)
}
]
|> Enum.sort_by(& &1["nickname"])
@ -815,7 +855,9 @@ test "it works with multiple filters" do
"nickname" => user.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => user.local,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(user) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user.name || user.nickname)
}
]
}
@ -838,7 +880,9 @@ test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation" do
"nickname" => user.nickname,
"roles" => %{"admin" => false, "moderator" => false},
"local" => true,
"tags" => []
"tags" => [],
"avatar" => User.avatar_url(user) |> MediaProxy.url(),
"display_name" => HTML.strip_tags(user.name || user.nickname)
}
end

View File

@ -18,8 +18,16 @@ test "renders a report" do
expected = %{
content: nil,
actor: AccountView.render("account.json", %{user: user}),
account: AccountView.render("account.json", %{user: other_user}),
actor:
Map.merge(
AccountView.render("account.json", %{user: user}),
Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user})
),
account:
Map.merge(
AccountView.render("account.json", %{user: other_user}),
Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: other_user})
),
statuses: [],
state: "open",
id: activity.id
@ -42,8 +50,16 @@ test "includes reported statuses" do
expected = %{
content: nil,
actor: AccountView.render("account.json", %{user: user}),
account: AccountView.render("account.json", %{user: other_user}),
actor:
Map.merge(
AccountView.render("account.json", %{user: user}),
Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: user})
),
account:
Map.merge(
AccountView.render("account.json", %{user: other_user}),
Pleroma.Web.AdminAPI.AccountView.render("show.json", %{user: other_user})
),
statuses: [StatusView.render("status.json", %{activity: activity})],
state: "open",
id: report_activity.id

View File

@ -188,6 +188,11 @@ test "pin status", %{user: user, activity: activity} do
assert %User{info: %{pinned_activities: [^id]}} = user
end
test "unlisted statuses can be pinned", %{user: user} do
{:ok, activity} = CommonAPI.post(user, %{"status" => "HI!!!", "visibility" => "unlisted"})
assert {:ok, ^activity} = CommonAPI.pin(activity.id, user)
end
test "only self-authored can be pinned", %{activity: activity} do
user = insert(:user)

View File

@ -1408,6 +1408,19 @@ test "gets a user's statuses without reblogs", %{conn: conn} do
assert [%{"id" => id}] = json_response(conn, 200)
assert id == to_string(post.id)
end
test "filters user's statuses by a hashtag", %{conn: conn} do
user = insert(:user)
{:ok, post} = CommonAPI.post(user, %{"status" => "#hashtag"})
{:ok, _post} = CommonAPI.post(user, %{"status" => "hashtag"})
conn =
conn
|> get("/api/v1/accounts/#{user.id}/statuses", %{"tagged" => "hashtag"})
assert [%{"id" => id}] = json_response(conn, 200)
assert id == to_string(post.id)
end
end
describe "user relationships" do