Merge branch 'fix/federation-context-issues' into 'develop'

Fix reply context fixing (Pleroma replies to Misskey threads) and removal of context objects

See merge request pleroma/pleroma!3717

config/config.exs

@@ -673,6 +673,8 @@
 
 config :pleroma, :populate_hashtags_table, fault_rate_allowance: 0.01
 
+config :pleroma, :delete_context_objects, fault_rate_allowance: 0.01
+
 config :pleroma, :env, Mix.env()
 
 config :http_signatures,

config/description.exs

@@ -495,6 +495,27 @@
       }
     ]
   },
+  %{
+    group: :pleroma,
+    key: :delete_context_objects,
+    type: :group,
+    description: "`delete_context_objects` background migration settings",
+    children: [
+      %{
+        key: :fault_rate_allowance,
+        type: :float,
+        description:
+          "Max accepted rate of objects that failed in the migration. Any value from 0.0 which tolerates no errors to 1.0 which will enable the feature even if context object deletion failed for all records.",
+        suggestions: [0.01]
+      },
+      %{
+        key: :sleep_interval_ms,
+        type: :integer,
+        description:
+          "Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances)."
+      }
+    ]
+  },
   %{
     group: :pleroma,
     key: :instance,

lib/pleroma/application.ex

@@ -249,7 +249,8 @@ defp dont_run_in_test(_) do
 
   defp background_migrators do
     [
-      Pleroma.Migrators.HashtagsTableMigrator
+      Pleroma.Migrators.HashtagsTableMigrator,
+      Pleroma.Migrators.ContextObjectsDeletionMigrator
     ]
   end
 

lib/pleroma/data_migration.ex

@@ -42,4 +42,5 @@ def get_by_name(name) do
   end
 
   def populate_hashtags_table, do: get_by_name("populate_hashtags_table")
+  def delete_context_objects, do: get_by_name("delete_context_objects")
 end

lib/pleroma/migrators/context_objects_deletion_migrator.ex

@@ -0,0 +1,139 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Migrators.ContextObjectsDeletionMigrator do
+  defmodule State do
+    use Pleroma.Migrators.Support.BaseMigratorState
+
+    @impl Pleroma.Migrators.Support.BaseMigratorState
+    defdelegate data_migration(), to: Pleroma.DataMigration, as: :delete_context_objects
+  end
+
+  use Pleroma.Migrators.Support.BaseMigrator
+
+  alias Pleroma.Migrators.Support.BaseMigrator
+  alias Pleroma.Object
+
+  @doc "This migration removes objects created exclusively for contexts, containing only an `id` field."
+
+  @impl BaseMigrator
+  def feature_config_path, do: [:features, :delete_context_objects]
+
+  @impl BaseMigrator
+  def fault_rate_allowance, do: Config.get([:delete_context_objects, :fault_rate_allowance], 0)
+
+  @impl BaseMigrator
+  def perform do
+    data_migration_id = data_migration_id()
+    max_processed_id = get_stat(:max_processed_id, 0)
+
+    Logger.info("Deleting context objects from `objects` (from oid: #{max_processed_id})...")
+
+    query()
+    |> where([object], object.id > ^max_processed_id)
+    |> Repo.chunk_stream(100, :batches, timeout: :infinity)
+    |> Stream.each(fn objects ->
+      object_ids = Enum.map(objects, & &1.id)
+
+      results = Enum.map(object_ids, &delete_context_object(&1))
+
+      failed_ids =
+        results
+        |> Enum.filter(&(elem(&1, 0) == :error))
+        |> Enum.map(&elem(&1, 1))
+
+      chunk_affected_count =
+        results
+        |> Enum.filter(&(elem(&1, 0) == :ok))
+        |> length()
+
+      for failed_id <- failed_ids do
+        _ =
+          Repo.query(
+            "INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <>
+              "VALUES ($1, $2) ON CONFLICT DO NOTHING;",
+            [data_migration_id, failed_id]
+          )
+      end
+
+      _ =
+        Repo.query(
+          "DELETE FROM data_migration_failed_ids " <>
+            "WHERE data_migration_id = $1 AND record_id = ANY($2)",
+          [data_migration_id, object_ids -- failed_ids]
+        )
+
+      max_object_id = Enum.at(object_ids, -1)
+
+      put_stat(:max_processed_id, max_object_id)
+      increment_stat(:iteration_processed_count, length(object_ids))
+      increment_stat(:processed_count, length(object_ids))
+      increment_stat(:failed_count, length(failed_ids))
+      increment_stat(:affected_count, chunk_affected_count)
+      put_stat(:records_per_second, records_per_second())
+      persist_state()
+
+      # A quick and dirty approach to controlling the load this background migration imposes
+      sleep_interval = Config.get([:delete_context_objects, :sleep_interval_ms], 0)
+      Process.sleep(sleep_interval)
+    end)
+    |> Stream.run()
+  end
+
+  @impl BaseMigrator
+  def query do
+    # Context objects have no activity type, and only one field, `id`.
+    # Only those context objects are without types.
+    from(
+      object in Object,
+      where: fragment("(?)->'type' IS NULL", object.data),
+      select: %{
+        id: object.id
+      }
+    )
+  end
+
+  @spec delete_context_object(integer()) :: {:ok | :error, integer()}
+  defp delete_context_object(id) do
+    result =
+      %Object{id: id}
+      |> Repo.delete()
+      |> elem(0)
+
+    {result, id}
+  end
+
+  @impl BaseMigrator
+  def retry_failed do
+    data_migration_id = data_migration_id()
+
+    failed_objects_query()
+    |> Repo.chunk_stream(100, :one)
+    |> Stream.each(fn object ->
+      with {res, _} when res != :error <- delete_context_object(object.id) do
+        _ =
+          Repo.query(
+            "DELETE FROM data_migration_failed_ids " <>
+              "WHERE data_migration_id = $1 AND record_id = $2",
+            [data_migration_id, object.id]
+          )
+      end
+    end)
+    |> Stream.run()
+
+    put_stat(:failed_count, failures_count())
+    persist_state()
+
+    force_continue()
+  end
+
+  defp failed_objects_query do
+    from(o in Object)
+    |> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"),
+      on: dmf.record_id == o.id
+    )
+    |> where([_o, dmf], dmf.data_migration_id == ^data_migration_id())
+    |> order_by([o], asc: o.id)
+  end
+end

lib/pleroma/object.ex

@@ -207,10 +207,6 @@ def get_cached_by_ap_id(ap_id) do
     end
   end
 
-  def context_mapping(context) do
-    Object.change(%Object{}, %{data: %{"id" => context}})
-  end
-
   def make_tombstone(%Object{data: %{"id" => id, "type" => type}}, deleted \\ DateTime.utc_now()) do
     %ObjectTombstone{
       id: id,

lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex

@@ -97,7 +97,7 @@ def changeset(struct, data) do
   defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Article", "Note", "Page"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_presence()

lib/pleroma/web/activity_pub/object_validators/common_fields.ex

@@ -52,8 +52,6 @@ defmacro status_object_fields do
       field(:summary, :string)
 
       field(:context, :string)
-      # short identifier for PleromaFE to group statuses by context
-      field(:context_id, :integer)
 
       field(:sensitive, :boolean, default: false)
       field(:replies_count, :integer, default: 0)

lib/pleroma/web/activity_pub/object_validators/common_fixes.ex

@@ -22,14 +22,15 @@ def cast_and_filter_recipients(message, field, follower_collection, field_fallba
   end
 
   def fix_object_defaults(data) do
-    %{data: %{"id" => context}, id: context_id} =
+    context =
-      Utils.create_context(data["context"] || data["conversation"])
+      Utils.maybe_create_context(
+        data["context"] || data["conversation"] || data["inReplyTo"] || data["id"]
+      )
 
     %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"])
 
     data
     |> Map.put("context", context)
-    |> Map.put("context_id", context_id)
     |> cast_and_filter_recipients("to", follower_collection)
     |> cast_and_filter_recipients("cc", follower_collection)
     |> cast_and_filter_recipients("bto", follower_collection)

lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex

@@ -75,7 +75,7 @@ def fix(data, meta) do
 
     data
     |> CommonFixes.fix_actor()
-    |> Map.put_new("context", object["context"])
+    |> Map.put("context", object["context"])
     |> fix_addressing(object)
   end
 

lib/pleroma/web/activity_pub/object_validators/event_validator.ex

@@ -62,7 +62,7 @@ def changeset(struct, data) do
   defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Event"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_presence()

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -80,7 +80,7 @@ def changeset(struct, data) do
   defp validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Question"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_presence()

lib/pleroma/web/activity_pub/utils.ex

@@ -154,22 +154,7 @@ def get_notified_from_object(object) do
     Notification.get_notified_from_activity(%Activity{data: object}, false)
   end
 
-  def create_context(context) do
+  def maybe_create_context(context), do: context || generate_id("contexts")
-    context = context || generate_id("contexts")
-
-    # Ecto has problems accessing the constraint inside the jsonb,
-    # so we explicitly check for the existed object before insert
-    object = Object.get_cached_by_ap_id(context)
-
-    with true <- is_nil(object),
-         changeset <- Object.context_mapping(context),
-         {:ok, inserted_object} <- Repo.insert(changeset) do
-      inserted_object
-    else
-      _ ->
-        object
-    end
-  end
 
   @doc """
   Enqueues an activity for federation if it's local
@@ -201,18 +186,16 @@ def lazy_put_activity_defaults(map, true) do
     |> Map.put_new("id", "pleroma:fakeid")
     |> Map.put_new_lazy("published", &make_date/0)
     |> Map.put_new("context", "pleroma:fakecontext")
-    |> Map.put_new("context_id", -1)
     |> lazy_put_object_defaults(true)
   end
 
   def lazy_put_activity_defaults(map, _fake?) do
-    %{data: %{"id" => context}, id: context_id} = create_context(map["context"])
+    context = maybe_create_context(map["context"])
 
     map
     |> Map.put_new_lazy("id", &generate_activity_id/0)
     |> Map.put_new_lazy("published", &make_date/0)
     |> Map.put_new("context", context)
-    |> Map.put_new("context_id", context_id)
     |> lazy_put_object_defaults(false)
   end
 
@@ -226,7 +209,6 @@ defp lazy_put_object_defaults(%{"object" => map} = activity, true)
       |> Map.put_new("id", "pleroma:fake_object_id")
       |> Map.put_new_lazy("published", &make_date/0)
       |> Map.put_new("context", activity["context"])
-      |> Map.put_new("context_id", activity["context_id"])
       |> Map.put_new("fake", true)
 
     %{activity | "object" => object}
@@ -239,7 +221,6 @@ defp lazy_put_object_defaults(%{"object" => map} = activity, _)
       |> Map.put_new_lazy("id", &generate_object_id/0)
       |> Map.put_new_lazy("published", &make_date/0)
       |> Map.put_new("context", activity["context"])
-      |> Map.put_new("context_id", activity["context_id"])
 
     %{activity | "object" => object}
   end

lib/pleroma/web/api_spec/schemas/status.ex

@@ -148,9 +148,15 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
             description:
               "A map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`"
           },
+          context: %Schema{
+            type: :string,
+            description: "The thread identifier the status is associated with"
+          },
           conversation_id: %Schema{
             type: :integer,
-            description: "The ID of the AP context the status is associated with (if any)"
+            deprecated: true,
+            description:
+              "The ID of the AP context the status is associated with (if any); deprecated, please use `context` instead"
           },
           direct_conversation_id: %Schema{
             type: :integer,
@@ -325,6 +331,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do
       "pinned" => false,
       "pleroma" => %{
         "content" => %{"text/plain" => "foobar"},
+        "context" => "http://localhost:4001/objects/8b4c0c80-6a37-4d2a-b1b9-05a19e3875aa",
         "conversation_id" => 345_972,
         "direct_conversation_id" => nil,
         "emoji_reactions" => [],

lib/pleroma/web/common_api/utils.ex

@@ -453,35 +453,6 @@ def get_report_statuses(%User{ap_id: actor}, %{status_ids: status_ids})
 
   def get_report_statuses(_, _), do: {:ok, nil}
 
-  # DEPRECATED mostly, context objects are now created at insertion time.
-  def context_to_conversation_id(context) do
-    with %Object{id: id} <- Object.get_cached_by_ap_id(context) do
-      id
-    else
-      _e ->
-        changeset = Object.context_mapping(context)
-
-        case Repo.insert(changeset) do
-          {:ok, %{id: id}} ->
-            id
-
-          # This should be solved by an upsert, but it seems ecto
-          # has problems accessing the constraint inside the jsonb.
-          {:error, _} ->
-            Object.get_cached_by_ap_id(context).id
-        end
-    end
-  end
-
-  def conversation_id_to_context(id) do
-    with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do
-      context
-    else
-      _e ->
-        {:error, dgettext("errors", "No such conversation")}
-    end
-  end
-
   def validate_character_limit("" = _full_payload, [] = _attachments) do
     {:error, dgettext("errors", "Cannot post an empty status without attachments")}
   end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -57,11 +57,19 @@ defp get_replied_to_activities(activities) do
     end)
   end
 
-  defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id),
+  # DEPRECATED This field seems to be a left-over from the StatusNet era.
-    do: context_id
+  # If your application uses `pleroma.conversation_id`: this field is deprecated.
+  # It is currently stubbed instead by doing a CRC32 of the context, and
+  # clearing the MSB to avoid overflow exceptions with signed integers on the
+  # different clients using this field (Java/Kotlin code, mostly; see Husky.)
+  # This should be removed in a future version of Pleroma. Pleroma-FE currently
+  # depends on this field, as well.
+  defp get_context_id(%{data: %{"context" => context}}) when is_binary(context) do
+    use Bitwise
 
-  defp get_context_id(%{data: %{"context" => context}}) when is_binary(context),
+    :erlang.crc32(context)
-    do: Utils.context_to_conversation_id(context)
+    |> band(bnot(0x8000_0000))
+  end
 
   defp get_context_id(_), do: nil
 
@@ -388,6 +396,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
       pleroma: %{
         local: activity.local,
         conversation_id: get_context_id(activity),
+        context: object.data["context"],
         in_reply_to_account_acct: reply_to_user && reply_to_user.nickname,
         content: %{"text/plain" => content_plaintext},
         spoiler_text: %{"text/plain" => summary},

priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs

@@ -0,0 +1,18 @@
+defmodule Pleroma.Repo.Migrations.DataMigrationDeleteContextObjects do
+  use Ecto.Migration
+
+  require Logger
+
+  def up do
+    dt = NaiveDateTime.utc_now()
+
+    execute(
+      "INSERT INTO data_migrations(name, inserted_at, updated_at) " <>
+        "VALUES ('delete_context_objects', '#{dt}', '#{dt}') ON CONFLICT DO NOTHING;"
+    )
+  end
+
+  def down do
+    execute("DELETE FROM data_migrations WHERE name = 'delete_context_objects';")
+  end
+end

test/fixtures/create-pleroma-reply-to-misskey-thread.json

@@ -0,0 +1,61 @@
+{
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://p.helene.moe/schemas/litepub-0.1.jsonld",
+        {
+            "@language": "und"
+        }
+    ],
+    "actor": "https://p.helene.moe/users/helene",
+    "attachment": [],
+    "attributedTo": "https://p.helene.moe/users/helene",
+    "cc": [
+        "https://p.helene.moe/users/helene/followers"
+    ],
+    "context": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
+    "conversation": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
+    "directMessage": false,
+    "id": "https://p.helene.moe/activities/5f80db86-a9bb-4883-9845-fbdbd1478f3a",
+    "object": {
+        "actor": "https://p.helene.moe/users/helene",
+        "attachment": [],
+        "attributedTo": "https://p.helene.moe/users/helene",
+        "cc": [
+            "https://p.helene.moe/users/helene/followers"
+        ],
+        "content": "<span class=\"h-card\"><a class=\"u-url mention\" data-user=\"AHntpQ4T3J4OSnpgMC\" href=\"https://mk.absturztau.be/@mametsuko\" rel=\"ugc\">@<span>mametsuko</span></a></span> meow",
+        "context": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
+        "conversation": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
+        "id": "https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4",
+        "inReplyTo": "https://mk.absturztau.be/notes/93e7nm8wqg",
+        "published": "2022-08-02T13:46:58.403996Z",
+        "sensitive": null,
+        "source": "@mametsuko@mk.absturztau.be meow",
+        "summary": "",
+        "tag": [
+            {
+                "href": "https://mk.absturztau.be/users/8ozbzjs3o8",
+                "name": "@mametsuko@mk.absturztau.be",
+                "type": "Mention"
+            }
+        ],
+        "to": [
+            "https://mk.absturztau.be/users/8ozbzjs3o8",
+            "https://www.w3.org/ns/activitystreams#Public"
+        ],
+        "type": "Note"
+    },
+    "published": "2022-08-02T13:46:58.403883Z",
+    "tag": [
+        {
+            "href": "https://mk.absturztau.be/users/8ozbzjs3o8",
+            "name": "@mametsuko@mk.absturztau.be",
+            "type": "Mention"
+        }
+    ],
+    "to": [
+        "https://mk.absturztau.be/users/8ozbzjs3o8",
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "type": "Create"
+}

test/fixtures/tesla_mock/helene@p.helene.moe.json

@@ -0,0 +1,50 @@
+{
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://p.helene.moe/schemas/litepub-0.1.jsonld",
+        {
+            "@language": "und"
+        }
+    ],
+    "alsoKnownAs": [],
+    "attachment": [
+        {
+            "name": "Timezone",
+            "type": "PropertyValue",
+            "value": "UTC+2 (Paris/Berlin)"
+        }
+    ],
+    "capabilities": {
+        "acceptsChatMessages": true
+    },
+    "discoverable": true,
+    "endpoints": {
+        "oauthAuthorizationEndpoint": "https://p.helene.moe/oauth/authorize",
+        "oauthRegistrationEndpoint": "https://p.helene.moe/api/v1/apps",
+        "oauthTokenEndpoint": "https://p.helene.moe/oauth/token",
+        "sharedInbox": "https://p.helene.moe/inbox",
+        "uploadMedia": "https://p.helene.moe/api/ap/upload_media"
+    },
+    "featured": "https://p.helene.moe/users/helene/collections/featured",
+    "followers": "https://p.helene.moe/users/helene/followers",
+    "following": "https://p.helene.moe/users/helene/following",
+    "icon": {
+        "type": "Image",
+        "url": "https://p.helene.moe/media/9a39209daa5a66b7ebb0547b08bf8360aa9d8d65a4ffba2603c6ffbe6aecb432.jpg"
+    },
+    "id": "https://p.helene.moe/users/helene",
+    "inbox": "https://p.helene.moe/users/helene/inbox",
+    "manuallyApprovesFollowers": false,
+    "name": "Hélène",
+    "outbox": "https://p.helene.moe/users/helene/outbox",
+    "preferredUsername": "helene",
+    "publicKey": {
+        "id": "https://p.helene.moe/users/helene#main-key",
+        "owner": "https://p.helene.moe/users/helene",
+        "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtoSBPU/VS2Kx3f6ap3zv\nZVacJsgUfaoFb3c2ii/FRh9RmRVlarq8sJXcjsQt1e0oxWaWJaIDDwyKZPt6hXae\nrY/AiGGeNu+NA+BtY7l7+9Yu67HUyT62+1qAwYHKBXX3fLOPs/YmQI0Tt0c4wKAG\nKEkiYsRizghgpzUC6jqdKV71DJkUZ8yhckCGb2fLko1ajbWEssdaP51aLsyRMyC2\nuzeWrxtD4O/HG0ea4S6y5X6hnsAHIK4Y3nnyIQ6pn4tOsl3HgqkjXE9MmZSvMCFx\nBq89TfZrVXNa2gSZdZLdbbJstzEScQWNt1p6tA6rM+e4JXYGr+rMdF3G+jV7afI2\nFQIDAQAB\n-----END PUBLIC KEY-----\n\n"
+    },
+    "summary": "I can speak: Français, English, Deutsch (nicht sehr gut), 日本語 (not very well)",
+    "tag": [],
+    "type": "Person",
+    "url": "https://p.helene.moe/users/helene"
+}

test/fixtures/tesla_mock/mametsuko@mk.absturztau.be.json

@@ -0,0 +1,65 @@
+{
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+        {
+            "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+            "sensitive": "as:sensitive",
+            "Hashtag": "as:Hashtag",
+            "quoteUrl": "as:quoteUrl",
+            "toot": "http://joinmastodon.org/ns#",
+            "Emoji": "toot:Emoji",
+            "featured": "toot:featured",
+            "discoverable": "toot:discoverable",
+            "schema": "http://schema.org#",
+            "PropertyValue": "schema:PropertyValue",
+            "value": "schema:value",
+            "misskey": "https://misskey-hub.net/ns#",
+            "_misskey_content": "misskey:_misskey_content",
+            "_misskey_quote": "misskey:_misskey_quote",
+            "_misskey_reaction": "misskey:_misskey_reaction",
+            "_misskey_votes": "misskey:_misskey_votes",
+            "_misskey_talk": "misskey:_misskey_talk",
+            "isCat": "misskey:isCat",
+            "vcard": "http://www.w3.org/2006/vcard/ns#"
+        }
+    ],
+    "type": "Person",
+    "id": "https://mk.absturztau.be/users/8ozbzjs3o8",
+    "inbox": "https://mk.absturztau.be/users/8ozbzjs3o8/inbox",
+    "outbox": "https://mk.absturztau.be/users/8ozbzjs3o8/outbox",
+    "followers": "https://mk.absturztau.be/users/8ozbzjs3o8/followers",
+    "following": "https://mk.absturztau.be/users/8ozbzjs3o8/following",
+    "featured": "https://mk.absturztau.be/users/8ozbzjs3o8/collections/featured",
+    "sharedInbox": "https://mk.absturztau.be/inbox",
+    "endpoints": {
+        "sharedInbox": "https://mk.absturztau.be/inbox"
+    },
+    "url": "https://mk.absturztau.be/@mametsuko",
+    "preferredUsername": "mametsuko",
+    "name": "mametschko",
+    "summary": "<p><span>nya, ich bin eine Brotperson</span></p>",
+    "icon": {
+        "type": "Image",
+        "url": "https://mk.absturztau.be/files/webpublic-3b5594f4-fa52-4548-b4e3-c379ae2143ed",
+        "sensitive": false,
+        "name": null
+    },
+    "image": {
+        "type": "Image",
+        "url": "https://mk.absturztau.be/files/webpublic-0d03b03d-b14b-4916-ac3d-8a137118ec84",
+        "sensitive": false,
+        "name": null
+    },
+    "tag": [],
+    "manuallyApprovesFollowers": true,
+    "discoverable": false,
+    "publicKey": {
+        "id": "https://mk.absturztau.be/users/8ozbzjs3o8#main-key",
+        "type": "Key",
+        "owner": "https://mk.absturztau.be/users/8ozbzjs3o8",
+        "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAuN/S1spBGmh8FXI1Bt16\nXB7Cc0QutBp7UPgmDNHjOfsq0zrF4g3L1UBxvrpU0XX77XPMCd9yPvGwAYURH2mv\ntIcYuE+R90VLDmBu5MTVthcG2D874eCZ2rD2YsEYmN5AjTX7QBIqCck+qDhVWkkM\nEZ6S5Ht6IJ5Of74eKffXElQI/C6QB+9uEDOmPk0jCzgI5gw7xvJqFj/DIF4kUUAu\nA89JqaFZzZlkrSrj4cr48bLN/YOmpdaHu0BKHaDSHct4+MqlixqovgdB6RboCEDw\ne4Aeav7+Q0Y9oGIvuggg0Q+nCubnVNnaPyzd817tpPVzyZmTts+DKyDuv90SX3nR\nsPaNa5Ty60eqplUk4b7X1gSvuzBJUFBxTVV84WnjwoeoydaS6rSyjCDPGLBjaByc\nFyWMMEb/zlQyhLZfBlvT7k96wRSsMszh2hDALWmgYIhq/jNwINvALJ1GKLNHHKZ4\nyz2LnxVpRm2rWrZzbvtcnSQOt3LaPSZn8Wgwv4buyHF02iuVuIamZVtKexsE1Ixl\nIi9qa3AKEc5gOzYXhRhvHaruzoCehUbb/UHC5c8Tto8L5G1xYzjLP3qj3PT9w/wM\n+k1Ra/4JhuAnVFROOoOmx9rIELLHH7juY2nhM7plGhyt1M5gysgqEloij8QzyQU2\nZK1YlAERG2XFO6br8omhcmECAwEAAQ==\n-----END PUBLIC KEY-----\n"
+    },
+    "isCat": true,
+    "vcard:Address": "Vienna, Austria"
+}

test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json

@@ -0,0 +1 @@
+{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","Hashtag":"as:Hashtag","quoteUrl":"as:quoteUrl","toot":"http://joinmastodon.org/ns#","Emoji":"toot:Emoji","featured":"toot:featured","discoverable":"toot:discoverable","schema":"http://schema.org#","PropertyValue":"schema:PropertyValue","value":"schema:value","misskey":"https://misskey-hub.net/ns#","_misskey_content":"misskey:_misskey_content","_misskey_quote":"misskey:_misskey_quote","_misskey_reaction":"misskey:_misskey_reaction","_misskey_votes":"misskey:_misskey_votes","_misskey_talk":"misskey:_misskey_talk","isCat":"misskey:isCat","vcard":"http://www.w3.org/2006/vcard/ns#"}],"id":"https://mk.absturztau.be/notes/93e7nm8wqg/activity","actor":"https://mk.absturztau.be/users/8ozbzjs3o8","type":"Create","published":"2022-08-01T11:06:49.568Z","object":{"id":"https://mk.absturztau.be/notes/93e7nm8wqg","type":"Note","attributedTo":"https://mk.absturztau.be/users/8ozbzjs3o8","summary":null,"content":"<p><span>meow</span></p>","_misskey_content":"meow","published":"2022-08-01T11:06:49.568Z","to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://mk.absturztau.be/users/8ozbzjs3o8/followers"],"inReplyTo":null,"attachment":[],"sensitive":false,"tag":[]},"to":["https://www.w3.org/ns/activitystreams#Public"],"cc":["https://mk.absturztau.be/users/8ozbzjs3o8/followers"]}

test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg.json

@@ -0,0 +1,44 @@
+{
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+        {
+            "manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
+            "sensitive": "as:sensitive",
+            "Hashtag": "as:Hashtag",
+            "quoteUrl": "as:quoteUrl",
+            "toot": "http://joinmastodon.org/ns#",
+            "Emoji": "toot:Emoji",
+            "featured": "toot:featured",
+            "discoverable": "toot:discoverable",
+            "schema": "http://schema.org#",
+            "PropertyValue": "schema:PropertyValue",
+            "value": "schema:value",
+            "misskey": "https://misskey-hub.net/ns#",
+            "_misskey_content": "misskey:_misskey_content",
+            "_misskey_quote": "misskey:_misskey_quote",
+            "_misskey_reaction": "misskey:_misskey_reaction",
+            "_misskey_votes": "misskey:_misskey_votes",
+            "_misskey_talk": "misskey:_misskey_talk",
+            "isCat": "misskey:isCat",
+            "vcard": "http://www.w3.org/2006/vcard/ns#"
+        }
+    ],
+    "id": "https://mk.absturztau.be/notes/93e7nm8wqg",
+    "type": "Note",
+    "attributedTo": "https://mk.absturztau.be/users/8ozbzjs3o8",
+    "summary": null,
+    "content": "<p><span>meow</span></p>",
+    "_misskey_content": "meow",
+    "published": "2022-08-01T11:06:49.568Z",
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "cc": [
+        "https://mk.absturztau.be/users/8ozbzjs3o8/followers"
+    ],
+    "inReplyTo": null,
+    "attachment": [],
+    "sensitive": false,
+    "tag": []
+}

test/fixtures/tesla_mock/p.helene.moe-AM7S6vZQmL6pI9TgPY.json

@@ -0,0 +1,36 @@
+{
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://p.helene.moe/schemas/litepub-0.1.jsonld",
+        {
+            "@language": "und"
+        }
+    ],
+    "actor": "https://p.helene.moe/users/helene",
+    "attachment": [],
+    "attributedTo": "https://p.helene.moe/users/helene",
+    "cc": [
+        "https://p.helene.moe/users/helene/followers"
+    ],
+    "content": "<span class=\"h-card\"><a class=\"u-url mention\" data-user=\"AHntpQ4T3J4OSnpgMC\" href=\"https://mk.absturztau.be/@mametsuko\" rel=\"ugc\">@<span>mametsuko</span></a></span> meow",
+    "context": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
+    "conversation": "https://p.helene.moe/contexts/cc324643-5583-4c3f-91d2-c6ed37db159d",
+    "id": "https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4",
+    "inReplyTo": "https://mk.absturztau.be/notes/93e7nm8wqg",
+    "published": "2022-08-02T13:46:58.403996Z",
+    "sensitive": null,
+    "source": "@mametsuko@mk.absturztau.be meow",
+    "summary": "",
+    "tag": [
+        {
+            "href": "https://mk.absturztau.be/users/8ozbzjs3o8",
+            "name": "@mametsuko@mk.absturztau.be",
+            "type": "Mention"
+        }
+    ],
+    "to": [
+        "https://mk.absturztau.be/users/8ozbzjs3o8",
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "type": "Note"
+}

test/pleroma/web/activity_pub/activity_pub_test.exs

@@ -554,7 +554,6 @@ test "inserts a given map into the activity database, giving it an id if it has
       assert activity.data["ok"] == data["ok"]
       assert activity.data["id"] == given_id
       assert activity.data["context"] == "blabla"
-      assert activity.data["context_id"]
     end
 
     test "adds a context when none is there" do
@@ -576,8 +575,6 @@ test "adds a context when none is there" do
 
       assert is_binary(activity.data["context"])
       assert is_binary(object.data["context"])
-      assert activity.data["context_id"]
-      assert object.data["context_id"]
     end
 
     test "adds an id to a given object if it lacks one and is a note and inserts it to the object database" do
@@ -1612,7 +1609,7 @@ test "it can create a Flag activity",
                })
 
       assert Repo.aggregate(Activity, :count, :id) == 1
-      assert Repo.aggregate(Object, :count, :id) == 2
+      assert Repo.aggregate(Object, :count, :id) == 1
       assert Repo.aggregate(Notification, :count, :id) == 0
     end
   end

test/pleroma/web/activity_pub/object_validators/create_generic_validator_test.exs

@@ -23,10 +23,10 @@ test "a Create/Note from Roadhouse validates" do
     {:ok, object_data} = ObjectValidator.cast_and_apply(note_activity["object"])
     meta = [object_data: ObjectValidator.stringify_keys(object_data)]
 
-    %{valid?: true} = CreateGenericValidator.cast_and_validate(note_activity, meta)
+    assert %{valid?: true} = CreateGenericValidator.cast_and_validate(note_activity, meta)
   end
 
-  test "a Create/Note with mismatched context is invalid" do
+  test "a Create/Note with mismatched context uses the Note's context" do
     user = insert(:user)
 
     note = %{
@@ -54,6 +54,9 @@ test "a Create/Note with mismatched context is invalid" do
     {:ok, object_data} = ObjectValidator.cast_and_apply(note_activity["object"])
     meta = [object_data: ObjectValidator.stringify_keys(object_data)]
 
-    %{valid?: false} = CreateGenericValidator.cast_and_validate(note_activity, meta)
+    validated = CreateGenericValidator.cast_and_validate(note_activity, meta)
+
+    assert validated.valid?
+    assert {:context, note["context"]} in validated.changes
   end
 end

test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs

@@ -707,4 +707,42 @@ test "take_emoji_tags/1" do
              }
            ]
   end
+
+  test "the standalone note uses its own ID when context is missing" do
+    insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8")
+
+    activity =
+      "test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json"
+      |> File.read!()
+      |> Jason.decode!()
+
+    {:ok, %Activity{} = modified} = Transmogrifier.handle_incoming(activity)
+    object = Object.normalize(modified, fetch: false)
+
+    assert object.data["context"] == object.data["id"]
+    assert modified.data["context"] == object.data["id"]
+  end
+
+  test "the reply note uses its parent's ID when context is missing and reply is unreachable" do
+    insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8")
+
+    activity =
+      "test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json"
+      |> File.read!()
+      |> Jason.decode!()
+
+    object =
+      activity["object"]
+      |> Map.put("inReplyTo", "https://404.site/object/went-to-buy-milk")
+
+    activity =
+      activity
+      |> Map.put("object", object)
+
+    {:ok, %Activity{} = modified} = Transmogrifier.handle_incoming(activity)
+    object = Object.normalize(modified, fetch: false)
+
+    assert object.data["context"] == object.data["inReplyTo"]
+    assert modified.data["context"] == object.data["inReplyTo"]
+  end
 end

test/pleroma/web/activity_pub/transmogrifier/question_handling_test.exs

@@ -33,8 +33,6 @@ test "Mastodon Question activity" do
     assert object.data["context"] ==
              "tag:mastodon.sdf.org,2019-05-10:objectId=15095122:objectType=Conversation"
 
-    assert object.data["context_id"]
-
     assert object.data["anyOf"] == []
 
     assert Enum.sort(object.data["oneOf"]) ==
@@ -68,7 +66,6 @@ test "Mastodon Question activity" do
     reply_object = Object.normalize(reply_activity, fetch: false)
 
     assert reply_object.data["context"] == object.data["context"]
-    assert reply_object.data["context_id"] == object.data["context_id"]
   end
 
   test "Mastodon Question activity with HTML tags in plaintext" do

test/pleroma/web/activity_pub/transmogrifier_test.exs

@@ -108,15 +108,20 @@ test "it accepts Move activities" do
       assert activity.data["type"] == "Move"
     end
 
-    test "a reply with mismatched context is rejected" do
+    test "it fixes both the Create and object contexts in a reply" do
-      insert(:user, ap_id: "https://macgirvin.com/channel/mike")
+      insert(:user, ap_id: "https://mk.absturztau.be/users/8ozbzjs3o8")
+      insert(:user, ap_id: "https://p.helene.moe/users/helene")
 
-      note_activity =
+      create_activity =
-        "test/fixtures/roadhouse-create-activity.json"
+        "test/fixtures/create-pleroma-reply-to-misskey-thread.json"
         |> File.read!()
         |> Jason.decode!()
 
-      assert {:error, _} = Transmogrifier.handle_incoming(note_activity)
+      assert {:ok, %Activity{} = activity} = Transmogrifier.handle_incoming(create_activity)
+
+      object = Object.normalize(activity, fetch: false)
+
+      assert activity.data["context"] == object.data["context"]
     end
   end
 
@@ -227,7 +232,6 @@ test "it strips internal fields" do
       assert is_nil(modified["object"]["like_count"])
       assert is_nil(modified["object"]["announcements"])
       assert is_nil(modified["object"]["announcement_count"])
-      assert is_nil(modified["object"]["context_id"])
       assert is_nil(modified["object"]["generator"])
     end
 
@@ -242,7 +246,6 @@ test "it strips internal fields of article" do
       assert is_nil(modified["object"]["like_count"])
       assert is_nil(modified["object"]["announcements"])
       assert is_nil(modified["object"]["announcement_count"])
-      assert is_nil(modified["object"]["context_id"])
       assert is_nil(modified["object"]["likes"])
     end
 

test/pleroma/web/activity_pub/utils_test.exs

@@ -429,7 +429,6 @@ test "returns map with id and published data" do
       object = Object.normalize(note_activity, fetch: false)
       res = Utils.lazy_put_activity_defaults(%{"context" => object.data["id"]})
       assert res["context"] == object.data["id"]
-      assert res["context_id"] == object.id
       assert res["id"]
       assert res["published"]
     end
@@ -437,7 +436,6 @@ test "returns map with id and published data" do
     test "returns map with fake id and published data" do
       assert %{
                "context" => "pleroma:fakecontext",
-               "context_id" => -1,
                "id" => "pleroma:fakeid",
                "published" => _
              } = Utils.lazy_put_activity_defaults(%{}, true)
@@ -454,13 +452,11 @@ test "returns activity data with object" do
         })
 
       assert res["context"] == object.data["id"]
-      assert res["context_id"] == object.id
       assert res["id"]
       assert res["published"]
       assert res["object"]["id"]
       assert res["object"]["published"]
       assert res["object"]["context"] == object.data["id"]
-      assert res["object"]["context_id"] == object.id
     end
   end
 

test/pleroma/web/common_api/utils_test.exs

@@ -4,7 +4,6 @@
 
 defmodule Pleroma.Web.CommonAPI.UtilsTest do
   alias Pleroma.Builders.UserBuilder
-  alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.ActivityDraft
   alias Pleroma.Web.CommonAPI.Utils
@@ -273,22 +272,6 @@ test "delegated renderers" do
     end
   end
 
-  describe "context_to_conversation_id" do
-    test "creates a mapping object" do
-      conversation_id = Utils.context_to_conversation_id("random context")
-      object = Object.get_by_ap_id("random context")
-
-      assert conversation_id == object.id
-    end
-
-    test "returns an existing mapping for an existing object" do
-      {:ok, object} = Object.context_mapping("random context") |> Repo.insert()
-      conversation_id = Utils.context_to_conversation_id("random context")
-
-      assert conversation_id == object.id
-    end
-  end
-
   describe "formats date to asctime" do
     test "when date is in ISO 8601 format" do
       date = DateTime.utc_now() |> DateTime.to_iso8601()
@@ -517,17 +500,6 @@ test "returns empty string when date invalid" do
     end
   end
 
-  describe "conversation_id_to_context/1" do
-    test "returns id" do
-      object = insert(:note)
-      assert Utils.conversation_id_to_context(object.id) == object.data["id"]
-    end
-
-    test "returns error if object not found" do
-      assert Utils.conversation_id_to_context("123") == {:error, "No such conversation"}
-    end
-  end
-
   describe "maybe_notify_mentioned_recipients/2" do
     test "returns recipients when activity is not `Create`" do
       activity = insert(:like_activity)

test/pleroma/web/mastodon_api/controllers/status_controller_test.exs

@@ -262,6 +262,7 @@ test "posting a fake status", %{conn: conn} do
         |> Map.put("url", nil)
         |> Map.put("uri", nil)
         |> Map.put("created_at", nil)
+        |> Kernel.put_in(["pleroma", "context"], nil)
         |> Kernel.put_in(["pleroma", "conversation_id"], nil)
 
       fake_conn =
@@ -285,6 +286,7 @@ test "posting a fake status", %{conn: conn} do
         |> Map.put("url", nil)
         |> Map.put("uri", nil)
         |> Map.put("created_at", nil)
+        |> Kernel.put_in(["pleroma", "context"], nil)
         |> Kernel.put_in(["pleroma", "conversation_id"], nil)
 
       assert real_status == fake_status

test/pleroma/web/mastodon_api/views/status_view_test.exs

@@ -14,10 +14,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
   alias Pleroma.User
   alias Pleroma.UserRelationship
   alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.StatusView
 
+  require Bitwise
+
   import Pleroma.Factory
   import Tesla.Mock
   import OpenApiSpex.TestAssertions
@@ -226,7 +227,7 @@ test "a note activity" do
     object_data = Object.normalize(note, fetch: false).data
     user = User.get_cached_by_ap_id(note.data["actor"])
 
-    convo_id = Utils.context_to_conversation_id(object_data["context"])
+    convo_id = :erlang.crc32(object_data["context"]) |> Bitwise.band(Bitwise.bnot(0x8000_0000))
 
     status = StatusView.render("show.json", %{activity: note})
 
@@ -280,6 +281,7 @@ test "a note activity" do
       pleroma: %{
         local: true,
         conversation_id: convo_id,
+        context: object_data["context"],
         in_reply_to_account_acct: nil,
         content: %{"text/plain" => HTML.strip_tags(object_data["content"])},
         spoiler_text: %{"text/plain" => HTML.strip_tags(object_data["summary"])},

test/support/http_request_mock.ex

@@ -1084,6 +1084,14 @@ def get("http://404.site" <> _, _, _, _) do
      }}
   end
 
+  def get("https://404.site" <> _, _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 404,
+       body: ""
+     }}
+  end
+
   def get(
         "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=acct:lain@zetsubou.xn--q9jyb4c",
         _,
@@ -1401,6 +1409,51 @@ def get("https://gleasonator.com/users/macgirvin/collections/featured", _, _, _)
      }}
   end
 
+  def get("https://mk.absturztau.be/users/8ozbzjs3o8", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/mametsuko@mk.absturztau.be.json"),
+       headers: activitypub_object_headers()
+     }}
+  end
+
+  def get("https://p.helene.moe/users/helene", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/helene@p.helene.moe.json"),
+       headers: activitypub_object_headers()
+     }}
+  end
+
+  def get("https://mk.absturztau.be/notes/93e7nm8wqg", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg.json"),
+       headers: activitypub_object_headers()
+     }}
+  end
+
+  def get("https://mk.absturztau.be/notes/93e7nm8wqg/activity", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/mk.absturztau.be-93e7nm8wqg-activity.json"),
+       headers: activitypub_object_headers()
+     }}
+  end
+
+  def get("https://p.helene.moe/objects/fd5910ac-d9dc-412e-8d1d-914b203296c4", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/p.helene.moe-AM7S6vZQmL6pI9TgPY.json"),
+       headers: activitypub_object_headers()
+     }}
+  end
+
   def get(url, query, body, headers) do
     {:error,
      "Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"}