Merge branch 'fix/1005-idna-domains' into 'develop'

support for idna domains

Closes #1005

See merge request pleroma/pleroma!1330

lib/pleroma/user/search.ex

@@ -18,8 +18,7 @@ def search(query_string, opts \\ []) do
 
     for_user = Keyword.get(opts, :for_user)
 
-    # Strip the beginning @ off if there is a query
-    query_string = String.trim_leading(query_string, "@")
+    query_string = format_query(query_string)
 
     maybe_resolve(resolve, for_user, query_string)
 
@@ -40,6 +39,18 @@ def search(query_string, opts \\ []) do
     results
   end
 
+  defp format_query(query_string) do
+    # Strip the beginning @ off if there is a query
+    query_string = String.trim_leading(query_string, "@")
+
+    with [name, domain] <- String.split(query_string, "@"),
+         formatted_domain <- String.replace(domain, ~r/[!-\-|@|[-`|{-~|\/|:]+/, "") do
+      name <> "@" <> to_string(:idna.encode(formatted_domain))
+    else
+      _ -> query_string
+    end
+  end
+
   defp search_query(query_string, for_user, following) do
     for_user
     |> base_query(following)
@@ -151,7 +162,7 @@ defp boost_search_rank_query(query, for_user) do
   defp fts_search_subquery(query, term) do
     processed_query =
       String.trim_trailing(term, "@" <> local_domain())
-      |> String.replace(~r/\W+/, " ")
+      |> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ")
       |> String.trim()
       |> String.split()
       |> Enum.map(&(&1 <> ":*"))

test/fixtures/host-meta-zetsubou.xn--q9jyb4c.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<XRD
+  xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0">
+  <Link rel="lrdd" template="https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource={uri}" type="application/xrd+xml" />
+</XRD>

test/fixtures/lain.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<XRD
+  xmlns="http://docs.oasis-open.org/ns/xri/xrd-1.0">
+  <Subject>acct:lain@zetsubou.xn--q9jyb4c</Subject>
+  <Alias>https://zetsubou.xn--q9jyb4c/users/lain</Alias>
+  <Link href="https://zetsubou.xn--q9jyb4c/users/lain/feed.atom" rel="http://schemas.google.com/g/2010#updates-from" type="application/atom+xml" />
+  <Link href="https://zetsubou.xn--q9jyb4c/users/lain" rel="http://webfinger.net/rel/profile-page" type="text/html" />
+  <Link href="https://zetsubou.xn--q9jyb4c/users/lain/salmon" rel="salmon" />
+  <Link href="data:application/magic-public-key,RSA.7yTJNuPH7wSsg6sMH4XLi-OL6JL8idyRMwNsWy2xzKWPJRWVK5hxG1kMGQ4qC_9ksqIaT7c7DIQFJYYbhRTnXYdac1UxaWivzl5l2HYPOOF1_-gbE6TCaI4ItTQo5eB4yyy3zozrIuv_GY8W0Ww58Re8Z_G4DFFmnipgiBKNaHthxNQqtxcK-o4rUv3xdyr_M9KYi3QISCGiaV_t8xkdVREixzNmWpsqM5YZ46xXT0SiGSHDubLE_OGhyvWqf_WkJrnDBETL3WjXU4QsPmBbVBgLvLcHei_uAD-9d3QImSuWwBXXQZIzY7Diro6u8dZuPIoLmnbUp1-mViBwCUMWSQ==.AQAB" rel="magic-public-key" />
+  <Link href="https://zetsubou.xn--q9jyb4c/users/lain" rel="self" type="application/activity+json" />
+  <Link rel="http://ostatus.org/schema/1.0/subscribe" template="https://zetsubou.xn--q9jyb4c/ostatus_subscribe?acct={uri}" />
+</XRD>

test/support/http_request_mock.ex

@@ -840,6 +840,45 @@ def get("http://404.site" <> _, _, _, _) do
      }}
   end
 
+  def get(
+        "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=lain@zetsubou.xn--q9jyb4c",
+        _,
+        _,
+        Accept: "application/xrd+xml,application/jrd+json"
+      ) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/lain.xml")
+     }}
+  end
+
+  def get(
+        "https://zetsubou.xn--q9jyb4c/.well-known/webfinger?resource=https://zetsubou.xn--q9jyb4c/users/lain",
+        _,
+        _,
+        Accept: "application/xrd+xml,application/jrd+json"
+      ) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/lain.xml")
+     }}
+  end
+
+  def get(
+        "https://zetsubou.xn--q9jyb4c/.well-known/host-meta",
+        _,
+        _,
+        _
+      ) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       body: File.read!("test/fixtures/host-meta-zetsubou.xn--q9jyb4c.xml")
+     }}
+  end
+
   def get(url, query, body, headers) do
     {:error,
      "Not implemented the mock response for get #{inspect(url)}, #{query}, #{inspect(body)}, #{

test/user_search_test.exs

@@ -248,5 +248,57 @@ test "local user search with users" do
       [result] = User.search("lain@localhost", resolve: true, for_user: user)
       assert Map.put(result, :search_rank, nil) |> Map.put(:search_type, nil) == local_user
     end
+
+    test "works with idna domains" do
+      user = insert(:user, nickname: "lain@" <> to_string(:idna.encode("zetsubou.みんな")))
+
+      results = User.search("lain@zetsubou.みんな", resolve: false, for_user: user)
+
+      result = List.first(results)
+
+      assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
+    end
+
+    test "works with idna domains converted input" do
+      user = insert(:user, nickname: "lain@" <> to_string(:idna.encode("zetsubou.みんな")))
+
+      results =
+        User.search("lain@zetsubou." <> to_string(:idna.encode("zetsubou.みんな")),
+          resolve: false,
+          for_user: user
+        )
+
+      result = List.first(results)
+
+      assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
+    end
+
+    test "works with idna domains and bad chars in domain" do
+      user = insert(:user, nickname: "lain@" <> to_string(:idna.encode("zetsubou.みんな")))
+
+      results =
+        User.search("lain@zetsubou!@#$%^&*()+,-/:;<=>?[]'_{}|~`.みんな",
+          resolve: false,
+          for_user: user
+        )
+
+      result = List.first(results)
+
+      assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
+    end
+
+    test "works with idna domains and query as link" do
+      user = insert(:user, nickname: "lain@" <> to_string(:idna.encode("zetsubou.みんな")))
+
+      results =
+        User.search("https://zetsubou.みんな/users/lain",
+          resolve: false,
+          for_user: user
+        )
+
+      result = List.first(results)
+
+      assert user == result |> Map.put(:search_rank, nil) |> Map.put(:search_type, nil)
+    end
   end
 end

test/web/web_finger/web_finger_test.exs

@@ -104,5 +104,16 @@ test "it gets the xrd endpoint for statusnet" do
 
       assert template == "http://status.alpicola.com/main/xrd?uri={uri}"
     end
+
+    test "it works with idna domains as nickname" do
+      nickname = "lain@" <> to_string(:idna.encode("zetsubou.みんな"))
+
+      {:ok, _data} = WebFinger.finger(nickname)
+    end
+
+    test "it works with idna domains as link" do
+      ap_id = "https://" <> to_string(:idna.encode("zetsubou.みんな")) <> "/users/lain"
+      {:ok, _data} = WebFinger.finger(ap_id)
+    end
   end
 end