Merge branch 'issue/1577' into 'develop'

[#1577] fix mediaType of object

See merge request pleroma/pleroma!2372

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   A module to handle coding from internal to wire ActivityPub and back.
   """
   alias Pleroma.Activity
+  alias Pleroma.EarmarkRenderer
   alias Pleroma.FollowingRelationship
   alias Pleroma.Object
   alias Pleroma.Object.Containment
@@ -43,6 +44,7 @@ def fix_object(object, options \\ []) do
     |> fix_addressing
     |> fix_summary
     |> fix_type(options)
+    |> fix_content
   end
 
   def fix_summary(%{"summary" => nil} = object) do
@@ -357,6 +359,18 @@ def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options)
 
   def fix_type(object, _), do: object
 
+  defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = object)
+       when is_binary(content) do
+    html_content =
+      content
+      |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
+      |> Pleroma.HTML.filter_tags()
+
+    Map.merge(object, %{"content" => html_content, "mediaType" => "text/html"})
+  end
+
+  defp fix_content(object), do: object
+
   defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do
     with true <- id =~ "follows",
          %User{local: true} = follower <- User.get_cached_by_ap_id(follower_id),
@@ -1207,18 +1221,24 @@ def add_attributed_to(object) do
 
   def prepare_attachments(object) do
     attachments =
-      (object["attachment"] || [])
+      object
+      |> Map.get("attachment", [])
       |> Enum.map(fn data ->
         [%{"mediaType" => media_type, "href" => href} | _] = data["url"]
-        %{"url" => href, "mediaType" => media_type, "name" => data["name"], "type" => "Document"}
+
+        %{
+          "url" => href,
+          "mediaType" => media_type,
+          "name" => data["name"],
+          "type" => "Document"
+        }
       end)
 
     Map.put(object, "attachment", attachments)
   end
 
   def strip_internal_fields(object) do
-    object
+    Map.drop(object, Pleroma.Constants.object_internal_fields())
-    |> Map.drop(Pleroma.Constants.object_internal_fields())
   end
 
   defp strip_internal_tags(%{"tag" => tags} = object) do

test/fixtures/tesla_mock/craigmaloney.json

@@ -0,0 +1,112 @@
+{
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+        {
+            "CacheFile": "pt:CacheFile",
+            "Hashtag": "as:Hashtag",
+            "Infohash": "pt:Infohash",
+            "RsaSignature2017": "https://w3id.org/security#RsaSignature2017",
+            "category": "sc:category",
+            "commentsEnabled": {
+                "@id": "pt:commentsEnabled",
+                "@type": "sc:Boolean"
+            },
+            "downloadEnabled": {
+                "@id": "pt:downloadEnabled",
+                "@type": "sc:Boolean"
+            },
+            "expires": "sc:expires",
+            "fps": {
+                "@id": "pt:fps",
+                "@type": "sc:Number"
+            },
+            "language": "sc:inLanguage",
+            "licence": "sc:license",
+            "originallyPublishedAt": "sc:datePublished",
+            "position": {
+                "@id": "pt:position",
+                "@type": "sc:Number"
+            },
+            "pt": "https://joinpeertube.org/ns#",
+            "sc": "http://schema.org#",
+            "sensitive": "as:sensitive",
+            "size": {
+                "@id": "pt:size",
+                "@type": "sc:Number"
+            },
+            "startTimestamp": {
+                "@id": "pt:startTimestamp",
+                "@type": "sc:Number"
+            },
+            "state": {
+                "@id": "pt:state",
+                "@type": "sc:Number"
+            },
+            "stopTimestamp": {
+                "@id": "pt:stopTimestamp",
+                "@type": "sc:Number"
+            },
+            "subtitleLanguage": "sc:subtitleLanguage",
+            "support": {
+                "@id": "pt:support",
+                "@type": "sc:Text"
+            },
+            "uuid": "sc:identifier",
+            "views": {
+                "@id": "pt:views",
+                "@type": "sc:Number"
+            },
+            "waitTranscoding": {
+                "@id": "pt:waitTranscoding",
+                "@type": "sc:Boolean"
+            }
+        },
+        {
+            "comments": {
+                "@id": "as:comments",
+                "@type": "@id"
+            },
+            "dislikes": {
+                "@id": "as:dislikes",
+                "@type": "@id"
+            },
+            "likes": {
+                "@id": "as:likes",
+                "@type": "@id"
+            },
+            "playlists": {
+                "@id": "pt:playlists",
+                "@type": "@id"
+            },
+            "shares": {
+                "@id": "as:shares",
+                "@type": "@id"
+            }
+        }
+    ],
+    "endpoints": {
+        "sharedInbox": "https://peertube.social/inbox"
+    },
+    "followers": "https://peertube.social/accounts/craigmaloney/followers",
+    "following": "https://peertube.social/accounts/craigmaloney/following",
+    "icon": {
+        "mediaType": "image/png",
+        "type": "Image",
+        "url": "https://peertube.social/lazy-static/avatars/87bd694b-95bc-4066-83f4-bddfcd2b9caa.png"
+    },
+    "id": "https://peertube.social/accounts/craigmaloney",
+    "inbox": "https://peertube.social/accounts/craigmaloney/inbox",
+    "name": "Craig Maloney",
+    "outbox": "https://peertube.social/accounts/craigmaloney/outbox",
+    "playlists": "https://peertube.social/accounts/craigmaloney/playlists",
+    "preferredUsername": "craigmaloney",
+    "publicKey": {
+        "id": "https://peertube.social/accounts/craigmaloney#main-key",
+        "owner": "https://peertube.social/accounts/craigmaloney",
+        "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA9qvGIYUW01yc8CCsrwxK\n5OXlV5s7EbNWY8tJr/p1oGuELZwAnG2XKxtdbvgcCT+YxL5uRXIdCFIIIKrzRFr/\nHfS0mOgNT9u3gu+SstCNgtatciT0RVP77yiC3b2NHq1NRRvvVhzQb4cpIWObIxqh\nb2ypDClTc7XaKtgmQCbwZlGyZMT+EKz/vustD6BlpGsglRkm7iES6s1PPGb1BU+n\nS94KhbS2DOFiLcXCVWt0QarokIIuKznp4+xP1axKyP+SkT5AHx08Nd5TYFb2C1Jl\nz0WD/1q0mAN62m7QrA3SQPUgB+wWD+S3Nzf7FwNPiP4srbBgxVEUnji/r9mQ6BXC\nrQIDAQAB\n-----END PUBLIC KEY-----"
+    },
+    "summary": null,
+    "type": "Person",
+    "url": "https://peertube.social/accounts/craigmaloney"
+}

test/fixtures/tesla_mock/peertube-social.json

@@ -0,0 +1,234 @@
+{
+    "@context": [
+        "https://www.w3.org/ns/activitystreams",
+        "https://w3id.org/security/v1",
+        {
+            "CacheFile": "pt:CacheFile",
+            "Hashtag": "as:Hashtag",
+            "Infohash": "pt:Infohash",
+            "RsaSignature2017": "https://w3id.org/security#RsaSignature2017",
+            "category": "sc:category",
+            "commentsEnabled": {
+                "@id": "pt:commentsEnabled",
+                "@type": "sc:Boolean"
+            },
+            "downloadEnabled": {
+                "@id": "pt:downloadEnabled",
+                "@type": "sc:Boolean"
+            },
+            "expires": "sc:expires",
+            "fps": {
+                "@id": "pt:fps",
+                "@type": "sc:Number"
+            },
+            "language": "sc:inLanguage",
+            "licence": "sc:license",
+            "originallyPublishedAt": "sc:datePublished",
+            "position": {
+                "@id": "pt:position",
+                "@type": "sc:Number"
+            },
+            "pt": "https://joinpeertube.org/ns#",
+            "sc": "http://schema.org#",
+            "sensitive": "as:sensitive",
+            "size": {
+                "@id": "pt:size",
+                "@type": "sc:Number"
+            },
+            "startTimestamp": {
+                "@id": "pt:startTimestamp",
+                "@type": "sc:Number"
+            },
+            "state": {
+                "@id": "pt:state",
+                "@type": "sc:Number"
+            },
+            "stopTimestamp": {
+                "@id": "pt:stopTimestamp",
+                "@type": "sc:Number"
+            },
+            "subtitleLanguage": "sc:subtitleLanguage",
+            "support": {
+                "@id": "pt:support",
+                "@type": "sc:Text"
+            },
+            "uuid": "sc:identifier",
+            "views": {
+                "@id": "pt:views",
+                "@type": "sc:Number"
+            },
+            "waitTranscoding": {
+                "@id": "pt:waitTranscoding",
+                "@type": "sc:Boolean"
+            }
+        },
+        {
+            "comments": {
+                "@id": "as:comments",
+                "@type": "@id"
+            },
+            "dislikes": {
+                "@id": "as:dislikes",
+                "@type": "@id"
+            },
+            "likes": {
+                "@id": "as:likes",
+                "@type": "@id"
+            },
+            "playlists": {
+                "@id": "pt:playlists",
+                "@type": "@id"
+            },
+            "shares": {
+                "@id": "as:shares",
+                "@type": "@id"
+            }
+        }
+    ],
+    "attributedTo": [
+        {
+            "id": "https://peertube.social/accounts/craigmaloney",
+            "type": "Person"
+        },
+        {
+            "id": "https://peertube.social/video-channels/9909c7d9-6b5b-4aae-9164-c1af7229c91c",
+            "type": "Group"
+        }
+    ],
+    "category": {
+        "identifier": "15",
+        "name": "Science & Technology"
+    },
+    "cc": [
+        "https://peertube.social/accounts/craigmaloney/followers"
+    ],
+    "comments": "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe/comments",
+    "commentsEnabled": true,
+    "content": "Support this and our other Michigan!/usr/group videos and meetings. Learn more at http://mug.org/membership\n\nTwenty Years in Jail: FreeBSD's Jails, Then and Now\n\nJails started as a limited virtualization system, but over the last two years they've...",
+    "dislikes": "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe/dislikes",
+    "downloadEnabled": true,
+    "duration": "PT5151S",
+    "icon": {
+        "height": 122,
+        "mediaType": "image/jpeg",
+        "type": "Image",
+        "url": "https://peertube.social/static/thumbnails/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe.jpg",
+        "width": 223
+    },
+    "id": "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe",
+    "language": {
+        "identifier": "en",
+        "name": "English"
+    },
+    "licence": {
+        "identifier": "1",
+        "name": "Attribution"
+    },
+    "likes": "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe/likes",
+    "mediaType": "text/markdown",
+    "name": "Twenty Years in Jail: FreeBSD's Jails, Then and Now",
+    "originallyPublishedAt": "2019-08-13T00:00:00.000Z",
+    "published": "2020-02-12T01:06:08.054Z",
+    "sensitive": false,
+    "shares": "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe/announces",
+    "state": 1,
+    "subtitleLanguage": [],
+    "support": "Learn more at http://mug.org",
+    "tag": [
+        {
+            "name": "linux",
+            "type": "Hashtag"
+        },
+        {
+            "name": "mug.org",
+            "type": "Hashtag"
+        },
+        {
+            "name": "open",
+            "type": "Hashtag"
+        },
+        {
+            "name": "oss",
+            "type": "Hashtag"
+        },
+        {
+            "name": "source",
+            "type": "Hashtag"
+        }
+    ],
+    "to": [
+        "https://www.w3.org/ns/activitystreams#Public"
+    ],
+    "type": "Video",
+    "updated": "2020-02-15T15:01:09.474Z",
+    "url": [
+        {
+            "href": "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe",
+            "mediaType": "text/html",
+            "type": "Link"
+        },
+        {
+            "fps": 30,
+            "height": 240,
+            "href": "https://peertube.social/static/webseed/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-240.mp4",
+            "mediaType": "video/mp4",
+            "size": 119465800,
+            "type": "Link"
+        },
+        {
+            "height": 240,
+            "href": "https://peertube.social/static/torrents/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-240.torrent",
+            "mediaType": "application/x-bittorrent",
+            "type": "Link"
+        },
+        {
+            "height": 240,
+            "href": "magnet:?xs=https%3A%2F%2Fpeertube.social%2Fstatic%2Ftorrents%2F278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-240.torrent&xt=urn:btih:b3365331a8543bf48d09add56d7fe4b1cbbb5659&dn=Twenty+Years+in+Jail%3A+FreeBSD's+Jails%2C+Then+and+Now&tr=wss%3A%2F%2Fpeertube.social%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fpeertube.social%2Ftracker%2Fannounce&ws=https%3A%2F%2Fpeertube.social%2Fstatic%2Fwebseed%2F278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-240.mp4",
+            "mediaType": "application/x-bittorrent;x-scheme-handler/magnet",
+            "type": "Link"
+        },
+        {
+            "fps": 30,
+            "height": 360,
+            "href": "https://peertube.social/static/webseed/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-360.mp4",
+            "mediaType": "video/mp4",
+            "size": 143930318,
+            "type": "Link"
+        },
+        {
+            "height": 360,
+            "href": "https://peertube.social/static/torrents/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-360.torrent",
+            "mediaType": "application/x-bittorrent",
+            "type": "Link"
+        },
+        {
+            "height": 360,
+            "href": "magnet:?xs=https%3A%2F%2Fpeertube.social%2Fstatic%2Ftorrents%2F278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-360.torrent&xt=urn:btih:0d37b23c98cb0d89e28b5dc8f49b3c97a041e569&dn=Twenty+Years+in+Jail%3A+FreeBSD's+Jails%2C+Then+and+Now&tr=wss%3A%2F%2Fpeertube.social%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fpeertube.social%2Ftracker%2Fannounce&ws=https%3A%2F%2Fpeertube.social%2Fstatic%2Fwebseed%2F278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-360.mp4",
+            "mediaType": "application/x-bittorrent;x-scheme-handler/magnet",
+            "type": "Link"
+        },
+        {
+            "fps": 30,
+            "height": 480,
+            "href": "https://peertube.social/static/webseed/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-480.mp4",
+            "mediaType": "video/mp4",
+            "size": 130530754,
+            "type": "Link"
+        },
+        {
+            "height": 480,
+            "href": "https://peertube.social/static/torrents/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-480.torrent",
+            "mediaType": "application/x-bittorrent",
+            "type": "Link"
+        },
+        {
+            "height": 480,
+            "href": "magnet:?xs=https%3A%2F%2Fpeertube.social%2Fstatic%2Ftorrents%2F278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-480.torrent&xt=urn:btih:3a13ff822ad9494165eff6167183ddaaabc1372a&dn=Twenty+Years+in+Jail%3A+FreeBSD's+Jails%2C+Then+and+Now&tr=wss%3A%2F%2Fpeertube.social%3A443%2Ftracker%2Fsocket&tr=https%3A%2F%2Fpeertube.social%2Ftracker%2Fannounce&ws=https%3A%2F%2Fpeertube.social%2Fstatic%2Fwebseed%2F278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe-480.mp4",
+            "mediaType": "application/x-bittorrent;x-scheme-handler/magnet",
+            "type": "Link"
+        }
+    ],
+    "uuid": "278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe",
+    "views": 2,
+    "waitTranscoding": false
+}

test/support/http_request_mock.ex

@@ -308,6 +308,22 @@ def get("https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3"
      }}
   end
 
+  def get("https://peertube.social/accounts/craigmaloney", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/craigmaloney.json")
+     }}
+  end
+
+  def get("https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/tesla_mock/peertube-social.json")
+     }}
+  end
+
   def get("https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39", _, _, [
         {"accept", "application/activity+json"}
       ]) do

test/web/activity_pub/transmogrifier_test.exs

@@ -1221,6 +1221,35 @@ test "it rejects activities without a valid ID" do
       :error = Transmogrifier.handle_incoming(data)
     end
 
+    test "skip converting the content when it is nil" do
+      object_id = "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe"
+
+      {:ok, object} = Fetcher.fetch_and_contain_remote_object_from_id(object_id)
+
+      result =
+        Pleroma.Web.ActivityPub.Transmogrifier.fix_object(Map.merge(object, %{"content" => nil}))
+
+      assert result["content"] == nil
+    end
+
+    test "it converts content of object to html" do
+      object_id = "https://peertube.social/videos/watch/278d2b7c-0f38-4aaa-afe6-9ecc0c4a34fe"
+
+      {:ok, %{"content" => content_markdown}} =
+        Fetcher.fetch_and_contain_remote_object_from_id(object_id)
+
+      {:ok, %Pleroma.Object{data: %{"content" => content}} = object} =
+        Fetcher.fetch_object_from_id(object_id)
+
+      assert content_markdown ==
+               "Support this and our other Michigan!/usr/group videos and meetings. Learn more at http://mug.org/membership\n\nTwenty Years in Jail: FreeBSD's Jails, Then and Now\n\nJails started as a limited virtualization system, but over the last two years they've..."
+
+      assert content ==
+               "<p>Support this and our other Michigan!/usr/group videos and meetings. Learn more at <a href=\"http://mug.org/membership\">http://mug.org/membership</a></p><p>Twenty Years in Jail: FreeBSD’s Jails, Then and Now</p><p>Jails started as a limited virtualization system, but over the last two years they’ve…</p>"
+
+      assert object.data["mediaType"] == "text/html"
+    end
+
     test "it remaps video URLs as attachments if necessary" do
       {:ok, object} =
         Fetcher.fetch_object_from_id(