Merge branch 'backport/feature-poll-refresh' into 'maint/1.1'

backport: poll refreshing

See merge request pleroma/pleroma!1804

CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Blocks are now treated consistently between the Streaming API and the Timeline APIs
 - ActivityPub: Correct addressing of Undo.
 - ActivityPub: Correct addressing of profile update activities.
+- ActivityPub: Polls are now refreshed when necessary.
 - Mastodon API: Ensure the `account` field is not empty when rendering Notification entities.
 - Report emails now include functional links to profiles of remote user accounts
 

lib/pleroma/constants.ex

@@ -6,4 +6,16 @@ defmodule Pleroma.Constants do
   use Const
 
   const(as_public, do: "https://www.w3.org/ns/activitystreams#Public")
+
+  const(object_internal_fields,
+    do: [
+      "likes",
+      "like_count",
+      "announcements",
+      "announcement_count",
+      "emoji",
+      "context_id",
+      "deleted_activity_id"
+    ]
+  )
 end

lib/pleroma/object.ex

@@ -38,6 +38,24 @@ def change(struct, params \\ %{}) do
   def get_by_id(nil), do: nil
   def get_by_id(id), do: Repo.get(Object, id)
 
+  def get_by_id_and_maybe_refetch(id, opts \\ []) do
+    %{updated_at: updated_at} = object = get_by_id(id)
+
+    if opts[:interval] &&
+         NaiveDateTime.diff(NaiveDateTime.utc_now(), updated_at) > opts[:interval] do
+      case Fetcher.refetch_object(object) do
+        {:ok, %Object{} = object} ->
+          object
+
+        e ->
+          Logger.error("Couldn't refresh #{object.data["id"]}:\n#{inspect(e)}")
+          object
+      end
+    else
+      object
+    end
+  end
+
   def get_by_ap_id(nil), do: nil
 
   def get_by_ap_id(ap_id) do

lib/pleroma/object/fetcher.ex

@@ -6,18 +6,39 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.HTTP
   alias Pleroma.Object
   alias Pleroma.Object.Containment
+  alias Pleroma.Repo
   alias Pleroma.Signature
   alias Pleroma.Web.ActivityPub.InternalFetchActor
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.OStatus
 
   require Logger
+  require Pleroma.Constants
 
-  defp reinject_object(data) do
+  defp touch_changeset(changeset) do
+    updated_at =
+      NaiveDateTime.utc_now()
+      |> NaiveDateTime.truncate(:second)
+
+    Ecto.Changeset.put_change(changeset, :updated_at, updated_at)
+  end
+
+  defp maybe_reinject_internal_fields(data, %{data: %{} = old_data}) do
+    internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields())
+
+    Map.merge(data, internal_fields)
+  end
+
+  defp maybe_reinject_internal_fields(data, _), do: data
+
+  defp reinject_object(struct, data) do
     Logger.debug("Reinjecting object #{data["id"]}")
 
     with data <- Transmogrifier.fix_object(data),
-         {:ok, object} <- Object.create(data) do
+         data <- maybe_reinject_internal_fields(data, struct),
+         changeset <- Object.change(struct, %{data: data}),
+         changeset <- touch_changeset(changeset),
+         {:ok, object} <- Repo.insert_or_update(changeset) do
       {:ok, object}
     else
       e ->
@@ -26,6 +47,17 @@ defp reinject_object(data) do
     end
   end
 
+  def refetch_object(%Object{data: %{"id" => id}} = object) do
+    with {:local, false} <- {:local, String.starts_with?(id, Pleroma.Web.base_url() <> "/")},
+         {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
+         {:ok, object} <- reinject_object(object, data) do
+      {:ok, object}
+    else
+      {:local, true} -> object
+      e -> {:error, e}
+    end
+  end
+
   # TODO:
   # This will create a Create activity, which we need internally at the moment.
   def fetch_object_from_id(id, options \\ []) do
@@ -57,7 +89,7 @@ def fetch_object_from_id(id, options \\ []) do
           {:reject, nil}
 
         {:object, data, nil} ->
-          reinject_object(data)
+          reinject_object(%Object{}, data)
 
         {:normalize, object = %Object{}} ->
           {:ok, object}

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -997,15 +997,7 @@ def prepare_attachments(object) do
 
   defp strip_internal_fields(object) do
     object
-    |> Map.drop([
-      "likes",
-      "like_count",
-      "announcements",
-      "announcement_count",
-      "emoji",
-      "context_id",
-      "deleted_activity_id"
-    ])
+    |> Map.drop(Pleroma.Constants.object_internal_fields())
   end
 
   defp strip_internal_tags(%{"tag" => tags} = object) do

lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex

@@ -485,7 +485,7 @@ def get_context(%{assigns: %{user: user}} = conn, %{"id" => id}) do
   end
 
   def get_poll(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with %Object{} = object <- Object.get_by_id(id),
+    with %Object{} = object <- Object.get_by_id_and_maybe_refetch(id, interval: 60),
          %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
          true <- Visibility.visible_for_user?(activity, user) do
       conn

test/fixtures/tesla_mock/poll_modified.json

@@ -0,0 +1 @@
+{"@context":["https://www.w3.org/ns/activitystreams","https://patch.cx/schemas/litepub-0.1.jsonld",{"@language":"und"}],"actor":"https://patch.cx/users/rin","attachment":[],"attributedTo":"https://patch.cx/users/rin","cc":["https://patch.cx/users/rin/followers"],"closed":"2019-09-19T00:32:36.785333","content":"can you vote on this poll?","context":"https://patch.cx/contexts/626ecafd-3377-46c4-b908-3721a4d4373c","conversation":"https://patch.cx/contexts/626ecafd-3377-46c4-b908-3721a4d4373c","id":"https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d","oneOf":[{"name":"yes","replies":{"totalItems":8,"type":"Collection"},"type":"Note"},{"name":"no","replies":{"totalItems":3,"type":"Collection"},"type":"Note"}],"published":"2019-09-18T14:32:36.802152Z","sensitive":false,"summary":"","tag":[],"to":["https://www.w3.org/ns/activitystreams#Public"],"type":"Question"}

test/fixtures/tesla_mock/poll_original.json

@@ -0,0 +1 @@
+{"@context":["https://www.w3.org/ns/activitystreams","https://patch.cx/schemas/litepub-0.1.jsonld",{"@language":"und"}],"actor":"https://patch.cx/users/rin","attachment":[],"attributedTo":"https://patch.cx/users/rin","cc":["https://patch.cx/users/rin/followers"],"closed":"2019-09-19T00:32:36.785333","content":"can you vote on this poll?","context":"https://patch.cx/contexts/626ecafd-3377-46c4-b908-3721a4d4373c","conversation":"https://patch.cx/contexts/626ecafd-3377-46c4-b908-3721a4d4373c","id":"https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d","oneOf":[{"name":"yes","replies":{"totalItems":4,"type":"Collection"},"type":"Note"},{"name":"no","replies":{"totalItems":0,"type":"Collection"},"type":"Note"}],"published":"2019-09-18T14:32:36.802152Z","sensitive":false,"summary":"","tag":[],"to":["https://www.w3.org/ns/activitystreams#Public"],"type":"Question"}

test/fixtures/tesla_mock/rin.json

@@ -0,0 +1 @@
+{"@context":["https://www.w3.org/ns/activitystreams","https://patch.cx/schemas/litepub-0.1.jsonld",{"@language":"und"}],"attachment":[],"endpoints":{"oauthAuthorizationEndpoint":"https://patch.cx/oauth/authorize","oauthRegistrationEndpoint":"https://patch.cx/api/v1/apps","oauthTokenEndpoint":"https://patch.cx/oauth/token","sharedInbox":"https://patch.cx/inbox"},"followers":"https://patch.cx/users/rin/followers","following":"https://patch.cx/users/rin/following","icon":{"type":"Image","url":"https://patch.cx/media/4e914f5b84e4a259a3f6c2d2edc9ab642f2ab05f3e3d9c52c81fc2d984b3d51e.jpg"},"id":"https://patch.cx/users/rin","image":{"type":"Image","url":"https://patch.cx/media/f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg?name=f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg"},"inbox":"https://patch.cx/users/rin/inbox","manuallyApprovesFollowers":false,"name":"rinpatch","outbox":"https://patch.cx/users/rin/outbox","preferredUsername":"rin","publicKey":{"id":"https://patch.cx/users/rin#main-key","owner":"https://patch.cx/users/rin","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5DLtwGXNZElJyxFGfcVc\nXANhaMadj/iYYQwZjOJTV9QsbtiNBeIK54PJrYuU0/0YIdrvS1iqheX5IwXRhcwa\nhm3ZyLz7XeN9st7FBni4BmZMBtMpxAuYuu5p/jbWy13qAiYOhPreCx0wrWgm/lBD\n9mkgaxIxPooBE0S4ZWEJIDIV1Vft3AWcRUyWW1vIBK0uZzs6GYshbQZB952S0yo4\nFzI1hABGHncH8UvuFauh4EZ8tY7/X5I0pGRnDOcRN1dAht5w5yTA+6r5kebiFQjP\nIzN/eCO/a9Flrj9YGW7HDNtjSOH0A31PLRGlJtJO3yK57dnf5ppyCZGfL4emShQo\ncQIDAQAB\n-----END PUBLIC KEY-----\n\n"},"summary":"your friendly neighborhood pleroma developer<br>I like cute things and distributed systems, and really hate delete and redrafts","tag":[],"type":"Person","url":"https://patch.cx/users/rin"}

test/object_test.exs

@@ -4,10 +4,13 @@
 
 defmodule Pleroma.ObjectTest do
   use Pleroma.DataCase
+  import ExUnit.CaptureLog
   import Pleroma.Factory
   import Tesla.Mock
+  alias Pleroma.Activity
   alias Pleroma.Object
   alias Pleroma.Repo
+  alias Pleroma.Web.CommonAPI
 
   setup do
     mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
@@ -89,4 +92,110 @@ test "does not fetch unknown objects when fetch_remote is false" do
              )
     end
   end
+
+  describe "get_by_id_and_maybe_refetch" do
+    setup do
+      mock(fn
+        %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} ->
+          %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_original.json")}
+
+        env ->
+          apply(HttpRequestMock, :request, [env])
+      end)
+
+      mock_modified = fn resp ->
+        mock(fn
+          %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} ->
+            resp
+
+          env ->
+            apply(HttpRequestMock, :request, [env])
+        end)
+      end
+
+      on_exit(fn -> mock(fn env -> apply(HttpRequestMock, :request, [env]) end) end)
+
+      [mock_modified: mock_modified]
+    end
+
+    test "refetches if the time since the last refetch is greater than the interval", %{
+      mock_modified: mock_modified
+    } do
+      %Object{} =
+        object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
+
+      assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
+      assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
+
+      mock_modified.(%Tesla.Env{
+        status: 200,
+        body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
+      })
+
+      updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
+      assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8
+      assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3
+    end
+
+    test "returns the old object if refetch fails", %{mock_modified: mock_modified} do
+      %Object{} =
+        object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
+
+      assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
+      assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
+
+      assert capture_log(fn ->
+               mock_modified.(%Tesla.Env{status: 404, body: ""})
+
+               updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
+               assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4
+               assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0
+             end) =~
+               "[error] Couldn't refresh https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"
+    end
+
+    test "does not refetch if the time since the last refetch is greater than the interval", %{
+      mock_modified: mock_modified
+    } do
+      %Object{} =
+        object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
+
+      assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
+      assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
+
+      mock_modified.(%Tesla.Env{
+        status: 200,
+        body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
+      })
+
+      updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: 100)
+      assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4
+      assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0
+    end
+
+    test "preserves internal fields on refetch", %{mock_modified: mock_modified} do
+      %Object{} =
+        object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
+
+      assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
+      assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
+
+      user = insert(:user)
+      activity = Activity.get_create_by_object_ap_id(object.data["id"])
+      {:ok, _activity, object} = CommonAPI.favorite(activity.id, user)
+
+      assert object.data["like_count"] == 1
+
+      mock_modified.(%Tesla.Env{
+        status: 200,
+        body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
+      })
+
+      updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
+      assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8
+      assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3
+
+      assert updated_object.data["like_count"] == 1
+    end
+  end
 end

test/support/http_request_mock.ex

@@ -1004,6 +1004,10 @@ def get("https://skippers-bin.com/users/7v1w1r8ce6", _, _, _) do
     {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sjw.json")}}
   end
 
+  def get("https://patch.cx/users/rin", _, _, _) do
+    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/rin.json")}}
+  end
+
   def get(url, query, body, headers) do
     {:error,
      "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{