Merge branch 'cache-plug' into 'develop'

Add Pleroma.Plugs.Cache

Closes #1174

See merge request pleroma/pleroma!1612

CHANGELOG.md

@@ -109,6 +109,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix Tasks: `mix pleroma.database fix_likes_collections`
 - Federation: Remove `likes` from objects.
 - Admin API: Added moderation log
+- Web response cache (currently, enabled for ActivityPub)
 
 ### Changed
 - Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text

config/config.exs

@@ -560,6 +560,10 @@
 
 config :pleroma, Pleroma.ActivityExpiration, enabled: true
 
+config :pleroma, :web_cache_ttl,
+  activity_pub: nil,
+  activity_pub_question: 30_000
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

docs/config.md

@@ -690,3 +690,12 @@ Supported rate limiters:
 * `:relation_id_action` for actions on relation with a specific user (follow, unfollow)
 * `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses
 * `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user
+
+## :web_cache_ttl
+
+The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration.
+
+Available caches:
+
+* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration).
+* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds).

lib/pleroma/activity.ex

@@ -308,10 +308,19 @@ def delete_by_ap_id(id) when is_binary(id) do
       %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id
       _ -> nil
     end)
+    |> purge_web_resp_cache()
   end
 
   def delete_by_ap_id(_), do: nil
 
+  defp purge_web_resp_cache(%Activity{} = activity) do
+    %{path: path} = URI.parse(activity.data["id"])
+    Cachex.del(:web_resp_cache, path)
+    activity
+  end
+
+  defp purge_web_resp_cache(nil), do: nil
+
   for {ap_type, type} <- @mastodon_notification_types do
     def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}),
       do: unquote(type)

lib/pleroma/application.ex

@@ -116,7 +116,8 @@ defp cachex_children do
       build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
       build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
       build_cachex("scrubber", limit: 2500),
-      build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500)
+      build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
+      build_cachex("web_resp", limit: 2500)
     ]
   end
 

lib/pleroma/object.ex

@@ -130,14 +130,16 @@ def swap_object_with_tombstone(object) do
   def delete(%Object{data: %{"id" => id}} = object) do
     with {:ok, _obj} = swap_object_with_tombstone(object),
          deleted_activity = Activity.delete_by_ap_id(id),
-         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
+         {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
       {:ok, object, deleted_activity}
     end
   end
 
   def prune(%Object{data: %{"id" => id}} = object) do
     with {:ok, object} <- Repo.delete(object),
-         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+         {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
+         {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
       {:ok, object}
     end
   end

lib/pleroma/plugs/cache.ex

@@ -0,0 +1,122 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.Cache do
+  @moduledoc """
+  Caches successful GET responses.
+
+  To enable the cache add the plug to a router pipeline or controller:
+
+      plug(Pleroma.Plugs.Cache)
+
+  ## Configuration
+
+  To configure the plug you need to pass settings as the second argument to the `plug/2` macro:
+
+      plug(Pleroma.Plugs.Cache, [ttl: nil, query_params: true])
+
+  Available options:
+
+  - `ttl`:  An expiration time (time-to-live). This value should be in milliseconds or `nil` to disable expiration. Defaults to `nil`.
+  - `query_params`: Take URL query string into account (`true`), ignore it (`false`) or limit to specific params only (list). Defaults to `true`.
+
+  Additionally, you can overwrite the TTL inside a controller action by assigning `cache_ttl` to the connection struct:
+
+      def index(conn, _params) do
+        ttl = 60_000 # one minute
+
+        conn
+        |> assign(:cache_ttl, ttl)
+        |> render("index.html")
+      end
+
+  """
+
+  import Phoenix.Controller, only: [current_path: 1, json: 2]
+  import Plug.Conn
+
+  @behaviour Plug
+
+  @defaults %{ttl: nil, query_params: true}
+
+  @impl true
+  def init([]), do: @defaults
+
+  def init(opts) do
+    opts = Map.new(opts)
+    Map.merge(@defaults, opts)
+  end
+
+  @impl true
+  def call(%{method: "GET"} = conn, opts) do
+    key = cache_key(conn, opts)
+
+    case Cachex.get(:web_resp_cache, key) do
+      {:ok, nil} ->
+        cache_resp(conn, opts)
+
+      {:ok, record} ->
+        send_cached(conn, record)
+
+      {atom, message} when atom in [:ignore, :error] ->
+        render_error(conn, message)
+    end
+  end
+
+  def call(conn, _), do: conn
+
+  # full path including query params
+  defp cache_key(conn, %{query_params: true}), do: current_path(conn)
+
+  # request path without query params
+  defp cache_key(conn, %{query_params: false}), do: conn.request_path
+
+  # request path with specific query params
+  defp cache_key(conn, %{query_params: query_params}) when is_list(query_params) do
+    query_string =
+      conn.params
+      |> Map.take(query_params)
+      |> URI.encode_query()
+
+    conn.request_path <> "?" <> query_string
+  end
+
+  defp cache_resp(conn, opts) do
+    register_before_send(conn, fn
+      %{status: 200, resp_body: body} = conn ->
+        ttl = Map.get(conn.assigns, :cache_ttl, opts.ttl)
+        key = cache_key(conn, opts)
+        content_type = content_type(conn)
+        record = {content_type, body}
+
+        Cachex.put(:web_resp_cache, key, record, ttl: ttl)
+
+        put_resp_header(conn, "x-cache", "MISS from Pleroma")
+
+      conn ->
+        conn
+    end)
+  end
+
+  defp content_type(conn) do
+    conn
+    |> Plug.Conn.get_resp_header("content-type")
+    |> hd()
+  end
+
+  defp send_cached(conn, {content_type, body}) do
+    conn
+    |> put_resp_content_type(content_type, nil)
+    |> put_resp_header("x-cache", "HIT from Pleroma")
+    |> send_resp(:ok, body)
+    |> halt()
+  end
+
+  defp render_error(conn, message) do
+    conn
+    |> put_status(:internal_server_error)
+    |> json(%{error: message})
+    |> halt()
+  end
+end

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -23,6 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   action_fallback(:errors)
 
+  plug(Pleroma.Plugs.Cache, [query_params: false] when action in [:activity, :object])
   plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
   plug(:set_requester_reachable when action in [:inbox])
   plug(:relay_active? when action in [:relay])
@@ -53,8 +54,10 @@ def object(conn, %{"uuid" => uuid}) do
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
          {_, true} <- {:public?, Visibility.is_public?(object)} do
       conn
+      |> set_cache_ttl_for(object)
       |> put_resp_content_type("application/activity+json")
-      |> json(ObjectView.render("object.json", %{object: object}))
+      |> put_view(ObjectView)
+      |> render("object.json", object: object)
     else
       {:public?, false} ->
         {:error, :not_found}
@@ -96,14 +99,36 @@ def activity(conn, %{"uuid" => uuid}) do
          %Activity{} = activity <- Activity.normalize(ap_id),
          {_, true} <- {:public?, Visibility.is_public?(activity)} do
       conn
+      |> set_cache_ttl_for(activity)
       |> put_resp_content_type("application/activity+json")
-      |> json(ObjectView.render("object.json", %{object: activity}))
+      |> put_view(ObjectView)
+      |> render("object.json", object: activity)
     else
-      {:public?, false} ->
+      {:public?, false} -> {:error, :not_found}
-        {:error, :not_found}
+      nil -> {:error, :not_found}
     end
   end
 
+  defp set_cache_ttl_for(conn, %Activity{object: object}) do
+    set_cache_ttl_for(conn, object)
+  end
+
+  defp set_cache_ttl_for(conn, entity) do
+    ttl =
+      case entity do
+        %Object{data: %{"type" => "Question"}} ->
+          Pleroma.Config.get([:web_cache_ttl, :activity_pub_question])
+
+        %Object{} ->
+          Pleroma.Config.get([:web_cache_ttl, :activity_pub])
+
+        _ ->
+          nil
+      end
+
+    assign(conn, :cache_ttl, ttl)
+  end
+
   # GET /relay/following
   def following(%{assigns: %{relay: true}} = conn, _params) do
     conn

test/object_test.exs

@@ -53,9 +53,12 @@ test "ensures cache is cleared for the object" do
 
       assert object == cached_object
 
+      Cachex.put(:web_resp_cache, URI.parse(object.data["id"]).path, "cofe")
+
       Object.delete(cached_object)
 
       {:ok, nil} = Cachex.get(:object_cache, "object:#{object.data["id"]}")
+      {:ok, nil} = Cachex.get(:web_resp_cache, URI.parse(object.data["id"]).path)
 
       cached_object = Object.get_cached_by_ap_id(object.data["id"])
 

test/plugs/cache_test.exs

@@ -0,0 +1,186 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.CacheTest do
+  use ExUnit.Case, async: true
+  use Plug.Test
+
+  alias Pleroma.Plugs.Cache
+
+  @miss_resp {200,
+              [
+                {"cache-control", "max-age=0, private, must-revalidate"},
+                {"content-type", "cofe/hot; charset=utf-8"},
+                {"x-cache", "MISS from Pleroma"}
+              ], "cofe"}
+
+  @hit_resp {200,
+             [
+               {"cache-control", "max-age=0, private, must-revalidate"},
+               {"content-type", "cofe/hot; charset=utf-8"},
+               {"x-cache", "HIT from Pleroma"}
+             ], "cofe"}
+
+  @ttl 5
+
+  setup do
+    Cachex.clear(:web_resp_cache)
+    :ok
+  end
+
+  test "caches a response" do
+    assert @miss_resp ==
+             conn(:get, "/")
+             |> Cache.call(%{query_params: false, ttl: nil})
+             |> put_resp_content_type("cofe/hot")
+             |> send_resp(:ok, "cofe")
+             |> sent_resp()
+
+    assert_raise(Plug.Conn.AlreadySentError, fn ->
+      conn(:get, "/")
+      |> Cache.call(%{query_params: false, ttl: nil})
+      |> put_resp_content_type("cofe/hot")
+      |> send_resp(:ok, "cofe")
+      |> sent_resp()
+    end)
+
+    assert @hit_resp ==
+             conn(:get, "/")
+             |> Cache.call(%{query_params: false, ttl: nil})
+             |> sent_resp()
+  end
+
+  test "ttl is set" do
+    assert @miss_resp ==
+             conn(:get, "/")
+             |> Cache.call(%{query_params: false, ttl: @ttl})
+             |> put_resp_content_type("cofe/hot")
+             |> send_resp(:ok, "cofe")
+             |> sent_resp()
+
+    assert @hit_resp ==
+             conn(:get, "/")
+             |> Cache.call(%{query_params: false, ttl: @ttl})
+             |> sent_resp()
+
+    :timer.sleep(@ttl + 1)
+
+    assert @miss_resp ==
+             conn(:get, "/")
+             |> Cache.call(%{query_params: false, ttl: @ttl})
+             |> put_resp_content_type("cofe/hot")
+             |> send_resp(:ok, "cofe")
+             |> sent_resp()
+  end
+
+  test "set ttl via conn.assigns" do
+    assert @miss_resp ==
+             conn(:get, "/")
+             |> Cache.call(%{query_params: false, ttl: nil})
+             |> put_resp_content_type("cofe/hot")
+             |> assign(:cache_ttl, @ttl)
+             |> send_resp(:ok, "cofe")
+             |> sent_resp()
+
+    assert @hit_resp ==
+             conn(:get, "/")
+             |> Cache.call(%{query_params: false, ttl: nil})
+             |> sent_resp()
+
+    :timer.sleep(@ttl + 1)
+
+    assert @miss_resp ==
+             conn(:get, "/")
+             |> Cache.call(%{query_params: false, ttl: nil})
+             |> put_resp_content_type("cofe/hot")
+             |> send_resp(:ok, "cofe")
+             |> sent_resp()
+  end
+
+  test "ignore query string when `query_params` is false" do
+    assert @miss_resp ==
+             conn(:get, "/?cofe")
+             |> Cache.call(%{query_params: false, ttl: nil})
+             |> put_resp_content_type("cofe/hot")
+             |> send_resp(:ok, "cofe")
+             |> sent_resp()
+
+    assert @hit_resp ==
+             conn(:get, "/?cofefe")
+             |> Cache.call(%{query_params: false, ttl: nil})
+             |> sent_resp()
+  end
+
+  test "take query string into account when `query_params` is true" do
+    assert @miss_resp ==
+             conn(:get, "/?cofe")
+             |> Cache.call(%{query_params: true, ttl: nil})
+             |> put_resp_content_type("cofe/hot")
+             |> send_resp(:ok, "cofe")
+             |> sent_resp()
+
+    assert @miss_resp ==
+             conn(:get, "/?cofefe")
+             |> Cache.call(%{query_params: true, ttl: nil})
+             |> put_resp_content_type("cofe/hot")
+             |> send_resp(:ok, "cofe")
+             |> sent_resp()
+  end
+
+  test "take specific query params into account when `query_params` is list" do
+    assert @miss_resp ==
+             conn(:get, "/?a=1&b=2&c=3&foo=bar")
+             |> fetch_query_params()
+             |> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil})
+             |> put_resp_content_type("cofe/hot")
+             |> send_resp(:ok, "cofe")
+             |> sent_resp()
+
+    assert @hit_resp ==
+             conn(:get, "/?bar=foo&c=3&b=2&a=1")
+             |> fetch_query_params()
+             |> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil})
+             |> sent_resp()
+
+    assert @miss_resp ==
+             conn(:get, "/?bar=foo&c=3&b=2&a=2")
+             |> fetch_query_params()
+             |> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil})
+             |> put_resp_content_type("cofe/hot")
+             |> send_resp(:ok, "cofe")
+             |> sent_resp()
+  end
+
+  test "ignore not GET requests" do
+    expected =
+      {200,
+       [
+         {"cache-control", "max-age=0, private, must-revalidate"},
+         {"content-type", "cofe/hot; charset=utf-8"}
+       ], "cofe"}
+
+    assert expected ==
+             conn(:post, "/")
+             |> Cache.call(%{query_params: true, ttl: nil})
+             |> put_resp_content_type("cofe/hot")
+             |> send_resp(:ok, "cofe")
+             |> sent_resp()
+  end
+
+  test "ignore non-successful responses" do
+    expected =
+      {418,
+       [
+         {"cache-control", "max-age=0, private, must-revalidate"},
+         {"content-type", "tea/iced; charset=utf-8"}
+       ], "🥤"}
+
+    assert expected ==
+             conn(:get, "/cofe")
+             |> Cache.call(%{query_params: true, ttl: nil})
+             |> put_resp_content_type("tea/iced")
+             |> send_resp(:im_a_teapot, "🥤")
+             |> sent_resp()
+  end
+end

test/web/activity_pub/activity_pub_controller_test.exs

@@ -175,6 +175,49 @@ test "it returns 404 for tombstone objects", %{conn: conn} do
 
       assert json_response(conn, 404)
     end
+
+    test "it caches a response", %{conn: conn} do
+      note = insert(:note)
+      uuid = String.split(note.data["id"], "/") |> List.last()
+
+      conn1 =
+        conn
+        |> put_req_header("accept", "application/activity+json")
+        |> get("/objects/#{uuid}")
+
+      assert json_response(conn1, :ok)
+      assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"}))
+
+      conn2 =
+        conn
+        |> put_req_header("accept", "application/activity+json")
+        |> get("/objects/#{uuid}")
+
+      assert json_response(conn1, :ok) == json_response(conn2, :ok)
+      assert Enum.any?(conn2.resp_headers, &(&1 == {"x-cache", "HIT from Pleroma"}))
+    end
+
+    test "cached purged after object deletion", %{conn: conn} do
+      note = insert(:note)
+      uuid = String.split(note.data["id"], "/") |> List.last()
+
+      conn1 =
+        conn
+        |> put_req_header("accept", "application/activity+json")
+        |> get("/objects/#{uuid}")
+
+      assert json_response(conn1, :ok)
+      assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"}))
+
+      Object.delete(note)
+
+      conn2 =
+        conn
+        |> put_req_header("accept", "application/activity+json")
+        |> get("/objects/#{uuid}")
+
+      assert "Not found" == json_response(conn2, :not_found)
+    end
   end
 
   describe "/object/:uuid/likes" do
@@ -264,6 +307,51 @@ test "it returns 404 for non-public activities", %{conn: conn} do
 
       assert json_response(conn, 404)
     end
+
+    test "it caches a response", %{conn: conn} do
+      activity = insert(:note_activity)
+      uuid = String.split(activity.data["id"], "/") |> List.last()
+
+      conn1 =
+        conn
+        |> put_req_header("accept", "application/activity+json")
+        |> get("/activities/#{uuid}")
+
+      assert json_response(conn1, :ok)
+      assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"}))
+
+      conn2 =
+        conn
+        |> put_req_header("accept", "application/activity+json")
+        |> get("/activities/#{uuid}")
+
+      assert json_response(conn1, :ok) == json_response(conn2, :ok)
+      assert Enum.any?(conn2.resp_headers, &(&1 == {"x-cache", "HIT from Pleroma"}))
+    end
+
+    test "cached purged after activity deletion", %{conn: conn} do
+      user = insert(:user)
+      {:ok, activity} = CommonAPI.post(user, %{"status" => "cofe"})
+
+      uuid = String.split(activity.data["id"], "/") |> List.last()
+
+      conn1 =
+        conn
+        |> put_req_header("accept", "application/activity+json")
+        |> get("/activities/#{uuid}")
+
+      assert json_response(conn1, :ok)
+      assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"}))
+
+      Activity.delete_by_ap_id(activity.object.data["id"])
+
+      conn2 =
+        conn
+        |> put_req_header("accept", "application/activity+json")
+        |> get("/activities/#{uuid}")
+
+      assert "Not found" == json_response(conn2, :not_found)
+    end
   end
 
   describe "/inbox" do