From 3078e62488ad1d94d1d3b83faf9f2b070e4aff06 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 21 Jan 2021 12:25:18 -0600 Subject: [PATCH 01/10] Update Apache configuration. This has been tested. --- installation/pleroma-apache.conf | 91 ++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 40 deletions(-) diff --git a/installation/pleroma-apache.conf b/installation/pleroma-apache.conf index 0d627f2d7..139abe9e1 100644 --- a/installation/pleroma-apache.conf +++ b/installation/pleroma-apache.conf @@ -1,73 +1,84 @@ -# default Apache site config for Pleroma -# -# needed modules: define headers proxy proxy_http proxy_wstunnel rewrite ssl -# optional modules: cache cache_disk +# Sample Apache config for Pleroma # # Simple installation instructions: -# 1. Install your TLS certificate, possibly using Let's Encrypt. -# 2. Replace 'example.tld' with your instance's domain wherever it appears. -# 3. This assumes a Debian style Apache config. Copy this file to -# /etc/apache2/sites-available/ and then add a symlink to it in -# /etc/apache2/sites-enabled/ by running 'a2ensite pleroma-apache.conf', then restart Apache. +# 1. Install your TLS certificate. We recommend using Let's Encrypt via Certbot +# 2. Replace 'example.tld' with your instance's domain. +# 3. This assumes a Debian-style Apache config. Copy this file to +# /etc/apache2/sites-available/ and then activate the site by running +# 'a2ensite pleroma-apache.conf', then restart Apache. # # Optional: enable disk-based caching for the media proxy # For details, see https://git.pleroma.social/pleroma/pleroma/wikis/How%20to%20activate%20mediaproxy # -# 1. Create the directory listed below as the CacheRoot, and make sure +# 1. Create a directory as shown below for the CacheRoot and make sure # the Apache user can write to it. # 2. Configure Apache's htcacheclean to clean the directory periodically. -# 3. Run 'a2enmod cache cache_disk' and restart Apache. +# Your OS may provide a service you can enable to do this automatically. Define servername example.tld + + LoadModule proxy_module libexec/apache24/mod_proxy.so + + + LoadModule proxy_http_module libexec/apache24/mod_proxy_http.so + + + LoadModule proxy_wstunnel_module libexec/apache24/mod_proxy_wstunnel.so + + + LoadModule rewrite_module libexec/apache24/mod_rewrite.so + + + LoadModule ssl_module libexec/apache24/mod_ssl.so + + + LoadModule cache_module libexec/apache24/mod_cache.so + + + LoadModule cache_disk_module libexec/apache24/mod_cache_disk.so + + ServerName ${servername} ServerTokens Prod -ErrorLog ${APACHE_LOG_DIR}/error.log -CustomLog ${APACHE_LOG_DIR}/access.log combined +# If you want Pleroma-specific logs +#ErrorLog /var/log/httpd-pleroma-error.log +#CustomLog /var/log/httpd-pleroma-access.log combined - Redirect permanent / https://${servername} + RewriteEngine on + RewriteCond %{SERVER_NAME} =${servername} + RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [END,NE,R=permanent] SSLEngine on SSLCertificateFile /etc/letsencrypt/live/${servername}/fullchain.pem SSLCertificateKeyFile /etc/letsencrypt/live/${servername}/privkey.pem + # Make sure you have the certbot-apache module installed + Include /etc/letsencrypt/options-ssl-apache.conf - # Mozilla modern configuration, tweak to your needs - SSLProtocol all -SSLv3 -TLSv1 -TLSv1.1 - SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256 - SSLHonorCipherOrder on - SSLCompression off - SSLSessionTickets off - - # uncomment the following to enable mediaproxy caching on disk - # - # CacheRoot /var/cache/apache2/mod_cache_disk - # CacheDirLevels 1 - # CacheDirLength 2 - # CacheEnable disk /proxy - # CacheLock on - # + # Uncomment the following to enable MediaProxy caching on disk + #CacheRoot /tmp/pleroma-media-cache/ + #CacheDirLevels 1 + #CacheDirLength 2 + #CacheEnable disk /proxy + #CacheLock on + #CacheHeader on + #CacheDetailHeader on + ## 16MB max filesize for caching, configure as desired + #CacheMaxFileSize 16000000 + #CacheDefaultExpire 86400 RewriteEngine On RewriteCond %{HTTP:Connection} Upgrade [NC] RewriteCond %{HTTP:Upgrade} websocket [NC] - RewriteRule /(.*) ws://localhost:4000/$1 [P,L] + RewriteRule /(.*) ws://127.0.0.1:4000/$1 [P,L] + #ProxyRequests must be off or you open your server to abuse as an open proxy ProxyRequests off - # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only - # and `localhost.` resolves to [::0] on some systems: see issue #930 ProxyPass / http://127.0.0.1:4000/ ProxyPassReverse / http://127.0.0.1:4000/ - - RequestHeader set Host ${servername} ProxyPreserveHost On - -# OCSP Stapling, only in httpd 2.3.3 and later -SSLUseStapling on -SSLStaplingResponderTimeout 5 -SSLStaplingReturnResponderErrors off -SSLStaplingCache shmcb:/var/run/ocsp(128000) From 003402df401f2bbf46e47017e3b7a2ec27615ea2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 21 Jan 2021 14:20:13 -0600 Subject: [PATCH 02/10] Add ability to invalidate cache entries for Apache --- config/config.exs | 4 ++- docs/configuration/cheatsheet.md | 5 ++-- installation/apache-cache-purge.sh.example | 25 +++++++++++++++++++ .../web/media_proxy/invalidation/script.ex | 19 ++++++++++++++ 4 files changed, 50 insertions(+), 3 deletions(-) create mode 100755 installation/apache-cache-purge.sh.example diff --git a/config/config.exs b/config/config.exs index c4a690799..5eca250bb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -438,7 +438,9 @@ headers: [], options: [] -config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil +config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, + script_path: nil, + url_format: nil # Note: media preview proxy depends on media proxy to be enabled config :pleroma, :media_preview_proxy, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 5c0fd6487..9d4b07bf4 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -321,9 +321,10 @@ This section describe PWA manifest instance-specific values. Currently this opti #### Pleroma.Web.MediaProxy.Invalidation.Script This strategy allow perform external shell script to purge cache. -Urls of attachments pass to script as arguments. +Urls of attachments are passed to the script as arguments. -* `script_path`: path to external script. +* `script_path`: Path to the external script. +* `url_format`: Set to `:htcacheclean` if using Apache's htcacheclean utility. Example: diff --git a/installation/apache-cache-purge.sh.example b/installation/apache-cache-purge.sh.example new file mode 100755 index 000000000..be1d36841 --- /dev/null +++ b/installation/apache-cache-purge.sh.example @@ -0,0 +1,25 @@ +#!/bin/sh + +# A simple shell script to delete a media from Apache's mod_disk_cache. + +SCRIPTNAME=${0##*/} + +# mod_disk_cache directory +CACHE_DIRECTORY="/tmp/pleroma-media-cache" + +## Removes an item via the htcacheclean utility +## $1 - the filename, can be a pattern . +## $2 - the cache directory. +purge_item() { + htcacheclean -p "${2}" "${1}" +} # purge_item + +purge() { + for url in "$@" + do + echo "$SCRIPTNAME delete \`$url\` from cache ($CACHE_DIRECTORY)" + purge_item "$url" $CACHE_DIRECTORY + done +} + +purge "$@" diff --git a/lib/pleroma/web/media_proxy/invalidation/script.ex b/lib/pleroma/web/media_proxy/invalidation/script.ex index 0f66c2fe3..c447614fa 100644 --- a/lib/pleroma/web/media_proxy/invalidation/script.ex +++ b/lib/pleroma/web/media_proxy/invalidation/script.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do def purge(urls, opts \\ []) do args = urls + |> format_urls(Keyword.get(opts, :url_format)) |> List.wrap() |> Enum.uniq() |> Enum.join(" ") @@ -40,4 +41,22 @@ defp handle_result(error, _) do Logger.error("Error while cache purge: #{inspect(error)}") {:error, inspect(error)} end + + def format_urls(urls, :htcacheclean) do + urls + |> Enum.map(fn url -> + uri = URI.parse(url) + + query = + if !is_nil(uri.query) do + "?" <> uri.query + else + "?" + end + + uri.scheme <> "://" <> uri.host <> ":#{inspect(uri.port)}" <> uri.path <> query + end) + end + + def format_urls(urls, _), do: urls end From e5b32aab92444ea1b4c5ec9e5e78cfcc909aaa73 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 21 Jan 2021 14:41:28 -0600 Subject: [PATCH 03/10] rename function --- lib/pleroma/web/media_proxy/invalidation/script.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/media_proxy/invalidation/script.ex b/lib/pleroma/web/media_proxy/invalidation/script.ex index c447614fa..87a21166c 100644 --- a/lib/pleroma/web/media_proxy/invalidation/script.ex +++ b/lib/pleroma/web/media_proxy/invalidation/script.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do def purge(urls, opts \\ []) do args = urls - |> format_urls(Keyword.get(opts, :url_format)) + |> maybe_format_urls(Keyword.get(opts, :url_format)) |> List.wrap() |> Enum.uniq() |> Enum.join(" ") @@ -42,7 +42,7 @@ defp handle_result(error, _) do {:error, inspect(error)} end - def format_urls(urls, :htcacheclean) do + def maybe_format_urls(urls, :htcacheclean) do urls |> Enum.map(fn url -> uri = URI.parse(url) @@ -58,5 +58,5 @@ def format_urls(urls, :htcacheclean) do end) end - def format_urls(urls, _), do: urls + def maybe_format_urls(urls, _), do: urls end From 0c485d555583971153fd44ec6aa9256a8503b150 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 21 Jan 2021 14:42:08 -0600 Subject: [PATCH 04/10] Improve description --- test/pleroma/web/media_proxy/invalidation/script_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/web/media_proxy/invalidation/script_test.exs b/test/pleroma/web/media_proxy/invalidation/script_test.exs index bcb6ab73c..854de8a3b 100644 --- a/test/pleroma/web/media_proxy/invalidation/script_test.exs +++ b/test/pleroma/web/media_proxy/invalidation/script_test.exs @@ -8,7 +8,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do import ExUnit.CaptureLog - test "it logger error when script not found" do + test "it logs error when script is not found" do assert capture_log(fn -> assert Invalidation.Script.purge( ["http://example.com/media/example.jpg"], From 42e49529c24090b0cb6f92f2bba154de5feb6855 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 21 Jan 2021 14:42:16 -0600 Subject: [PATCH 05/10] Test URL formatting --- .../media_proxy/invalidation/script_test.exs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/pleroma/web/media_proxy/invalidation/script_test.exs b/test/pleroma/web/media_proxy/invalidation/script_test.exs index 854de8a3b..e9629b72b 100644 --- a/test/pleroma/web/media_proxy/invalidation/script_test.exs +++ b/test/pleroma/web/media_proxy/invalidation/script_test.exs @@ -23,4 +23,30 @@ test "it logs error when script is not found" do ) == {:error, "\"not found script path\""} end) end + + describe "url formatting" do + setup do + urls = [ + "https://bikeshed.party/media/foo.png", + "http://safe.millennial.space/proxy/wheeeee.gif", + "https://lain.com/proxy/mediafile.mp4?foo&bar=true", + "http://localhost:4000/media/upload.jpeg" + ] + + [urls: urls] + end + + test "with invalid formatter", %{urls: urls} do + assert urls == Invalidation.Script.maybe_format_urls(urls, nil) + end + + test "with :htcacheclean formatter", %{urls: urls} do + assert [ + "https://bikeshed.party:443/media/foo.png?", + "http://safe.millennial.space:80/proxy/wheeeee.gif?", + "https://lain.com:443/proxy/mediafile.mp4?foo&bar=true", + "http://localhost:4000/media/upload.jpeg?" + ] == Invalidation.Script.maybe_format_urls(urls, :htcacheclean) + end + end end From e709dec2eb7b588e242f3b776761ba89484f446a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 21 Jan 2021 14:52:02 -0600 Subject: [PATCH 06/10] Add Invalidation Script url_format setting --- config/description.exs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/config/description.exs b/config/description.exs index 715a0d0c3..d7dc264ee 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1627,13 +1627,20 @@ group: :pleroma, key: Pleroma.Web.MediaProxy.Invalidation.Script, type: :group, - description: "Script invalidate settings", + description: "Invalidation script settings", children: [ %{ key: :script_path, type: :string, - description: "Path to shell script. Which will run purge cache.", + description: "Path to executable script which will purge cached items.", suggestions: ["./installation/nginx-cache-purge.sh.example"] + }, + %{ + key: :url_format, + type: :string, + description: + "Optional URL format preprocessing. Only required for Apache's htcacheclean.", + suggestions: [":htcacheclean"] } ] }, From c29cf65ec7f681ffde70b44e745cf8bf660fc6c0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 21 Jan 2021 14:53:38 -0600 Subject: [PATCH 07/10] Document improved Apache support --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1dfeae01..fea8b92de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Emoji: Support the full Unicode 13.1 set of Emoji for reactions, plus regional indicators. - Admin API: Reports now ordered by newest - Deprecated `Pleroma.Uploaders.S3, :public_endpoint`. Now `Pleroma.Upload, :base_url` is the standard configuration key for all uploaders. +- Improved Apache webserver support: updated sample configuration, MediaProxy cache invalidation now supported ### Added From 8373cb645b7f357eedbc3a45a2e75a81376e6ef8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 25 Jan 2021 18:15:04 -0600 Subject: [PATCH 08/10] Add sudo rule, remove quoting that breaks the for loop --- installation/apache-cache-purge.sh.example | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/installation/apache-cache-purge.sh.example b/installation/apache-cache-purge.sh.example index be1d36841..62997038d 100755 --- a/installation/apache-cache-purge.sh.example +++ b/installation/apache-cache-purge.sh.example @@ -1,6 +1,10 @@ #!/bin/sh # A simple shell script to delete a media from Apache's mod_disk_cache. +# You will likely need to setup a sudo rule like the following: +# +# Cmnd_Alias HTCACHECLEAN = /usr/local/sbin/htcacheclean +# pleroma ALL=HTCACHECLEAN, NOPASSWD: HTCACHECLEAN SCRIPTNAME=${0##*/} @@ -11,15 +15,15 @@ CACHE_DIRECTORY="/tmp/pleroma-media-cache" ## $1 - the filename, can be a pattern . ## $2 - the cache directory. purge_item() { - htcacheclean -p "${2}" "${1}" + sudo htcacheclean -v -p "${2}" "${1}" } # purge_item purge() { - for url in "$@" + for url in $@ do echo "$SCRIPTNAME delete \`$url\` from cache ($CACHE_DIRECTORY)" purge_item "$url" $CACHE_DIRECTORY done } -purge "$@" +purge $@ From c6ef87d585b63e9e26b16176b65a67d10e4b706b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 25 Jan 2021 18:20:07 -0600 Subject: [PATCH 09/10] Note the requirement for the url_format parameter --- installation/apache-cache-purge.sh.example | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/installation/apache-cache-purge.sh.example b/installation/apache-cache-purge.sh.example index 62997038d..7b4262875 100755 --- a/installation/apache-cache-purge.sh.example +++ b/installation/apache-cache-purge.sh.example @@ -5,6 +5,13 @@ # # Cmnd_Alias HTCACHECLEAN = /usr/local/sbin/htcacheclean # pleroma ALL=HTCACHECLEAN, NOPASSWD: HTCACHECLEAN +# +# Please also ensure you have enabled: +# +# config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, url_format: :htcacheclean +# +# which will correctly format the URLs passed to this script for the htcacheclean utility. +# SCRIPTNAME=${0##*/} From 01fc7d809d33ea546abded0d1540e3b41dbb0e6a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 25 Jan 2021 18:23:05 -0600 Subject: [PATCH 10/10] Clarify the state of mediaproxy cache invalidation for Apache --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6522fbdcd..66427764c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Emoji: Support the full Unicode 13.1 set of Emoji for reactions, plus regional indicators. - Admin API: Reports now ordered by newest - Deprecated `Pleroma.Uploaders.S3, :public_endpoint`. Now `Pleroma.Upload, :base_url` is the standard configuration key for all uploaders. -- Improved Apache webserver support: updated sample configuration, MediaProxy cache invalidation now supported +- Improved Apache webserver support: updated sample configuration, MediaProxy cache invalidation verified with the included sample script ### Added