Merge branch 'linkify' into 'develop'

Fix Linkify

See merge request pleroma/pleroma!2792

CHANGELOG.md

@@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 - **Breaking:** Elixir >=1.9 is now required (was >= 1.8)
+- **Breaking:** Configuration: `:auto_linker, :opts` moved to `:pleroma, Pleroma.Formatter`. Old config namespace is deprecated.
 - In Conversations, return only direct messages as `last_status`
 - Using the `only_media` filter on timelines will now exclude reblog media
 - MFR policy to set global expiration for all local Create activities

config/config.exs

@@ -527,16 +527,14 @@
     federator_outgoing: 5
   ]
 
-config :auto_linker,
+config :pleroma, Pleroma.Formatter,
-  opts: [
-    extra: true,
-    # TODO: Set to :no_scheme when it works properly
-    validate_tld: true,
   class: false,
-    strip_prefix: false,
+  rel: "ugc",
   new_window: false,
-    rel: "ugc"
+  truncate: false,
-  ]
+  strip_prefix: false,
+  extra: true,
+  validate_tld: :no_scheme
 
 config :pleroma, :ldap,
   enabled: System.get_env("LDAP_ENABLED") == "true",

config/description.exs

@@ -2216,11 +2216,12 @@
     ]
   },
   %{
-    group: :auto_linker,
+    group: :pleroma,
-    key: :opts,
+    key: Pleroma.Formatter,
     label: "Auto Linker",
     type: :group,
-    description: "Configuration for the auto_linker library",
+    description:
+      "Configuration for Pleroma's link formatter which parses mentions, hashtags, and URLs.",
     children: [
       %{
         key: :class,
@@ -2237,24 +2238,31 @@
       %{
         key: :new_window,
         type: :boolean,
-        description: "Link URLs will open in new window/tab"
+        description: "Link URLs will open in a new window/tab."
       },
       %{
         key: :truncate,
         type: [:integer, false],
         description:
-          "Set to a number to truncate URLs longer then the number. Truncated URLs will end in `..`",
+          "Set to a number to truncate URLs longer than the number. Truncated URLs will end in `...`",
         suggestions: [15, false]
       },
       %{
         key: :strip_prefix,
         type: :boolean,
-        description: "Strip the scheme prefix"
+        description: "Strip the scheme prefix."
       },
       %{
         key: :extra,
         type: :boolean,
         description: "Link URLs with rarely used schemes (magnet, ipfs, irc, etc.)"
+      },
+      %{
+        key: :validate_tld,
+        type: [:atom, :boolean],
+        description:
+          "Set to false to disable TLD validation for URLs/emails. Can be set to :no_scheme to validate TLDs only for URLs without a scheme (e.g `example.com` will be validated, but `http://example.loki` won't)",
+        suggestions: [:no_scheme, true]
       }
     ]
   },

docs/configuration/cheatsheet.md

@@ -934,30 +934,29 @@ Configure OAuth 2 provider capabilities:
 ### :uri_schemes
 * `valid_schemes`: List of the scheme part that is considered valid to be an URL.
 
-### :auto_linker
+### Pleroma.Formatter
 
-Configuration for the `auto_linker` library:
+Configuration for Pleroma's link formatter which parses mentions, hashtags, and URLs.
 
-* `class: "auto-linker"` - specify the class to be added to the generated link. false to clear.
+* `class` - specify the class to be added to the generated link (default: `false`)
-* `rel: "noopener noreferrer"` - override the rel attribute. false to clear.
+* `rel` - specify the rel attribute (default: `ugc`)
-* `new_window: true` - set to false to remove `target='_blank'` attribute.
+* `new_window` - adds `target="_blank"` attribute (default: `false`)
-* `scheme: false` - Set to true to link urls with schema `http://google.com`.
+* `truncate` - Set to a number to truncate URLs longer then the number. Truncated URLs will end in `...` (default: `false`)
-* `truncate: false` - Set to a number to truncate urls longer then the number. Truncated urls will end in `..`.
+* `strip_prefix` - Strip the scheme prefix (default: `false`)
-* `strip_prefix: true` - Strip the scheme prefix.
+* `extra` - link URLs with rarely used schemes (magnet, ipfs, irc, etc.) (default: `true`)
-* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.).
+* `validate_tld` - Set to false to disable TLD validation for URLs/emails. Can be set to :no_scheme to validate TLDs only for urls without a scheme (e.g `example.com` will be validated, but `http://example.loki` won't) (default: `:no_scheme`)
 
 Example:
 
 ```elixir
-config :auto_linker,
+config :pleroma, Pleroma.Formatter,
-  opts: [
-    scheme: true,
-    extra: true,
   class: false,
-    strip_prefix: false,
+  rel: "ugc",
   new_window: false,
-    rel: "ugc"
+  truncate: false,
-  ]
+  strip_prefix: false,
+  extra: true,
+  validate_tld: :no_scheme
 ```
 
 ## Custom Runtime Modules (`:modules`)

lib/pleroma/config/config_db.ex

@@ -156,7 +156,6 @@ defp only_full_update?(%ConfigDB{group: group, key: key}) do
       {:quack, :meta},
       {:mime, :types},
       {:cors_plug, [:max_age, :methods, :expose, :headers]},
-      {:auto_linker, :opts},
       {:swarm, :node_blacklist},
       {:logger, :backends}
     ]

lib/pleroma/formatter.ex

@@ -10,11 +10,15 @@ defmodule Pleroma.Formatter do
   @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
 
-  @auto_linker_config hashtag: true,
+  defp linkify_opts do
+    Pleroma.Config.get(Pleroma.Formatter) ++
+      [
+        hashtag: true,
         hashtag_handler: &Pleroma.Formatter.hashtag_handler/4,
         mention: true,
-                      mention_handler: &Pleroma.Formatter.mention_handler/4,
+        mention_handler: &Pleroma.Formatter.mention_handler/4
-                      scheme: true
+      ]
+  end
 
   def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do
     case User.get_cached_by_nickname(nickname) do
@@ -80,19 +84,19 @@ def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do
   @spec linkify(String.t(), keyword()) ::
           {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]}
   def linkify(text, options \\ []) do
-    options = options ++ @auto_linker_config
+    options = linkify_opts() ++ options
 
     if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do
       %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text)
       acc = %{mentions: MapSet.new(), tags: MapSet.new()}
 
-      {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options)
+      {text_mentions, %{mentions: mentions}} = Linkify.link_map(mentions, acc, options)
-      {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options)
+      {text_rest, %{tags: tags}} = Linkify.link_map(rest, acc, options)
 
       {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)}
     else
       acc = %{mentions: MapSet.new(), tags: MapSet.new()}
-      {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options)
+      {text, %{mentions: mentions, tags: tags}} = Linkify.link_map(text, acc, options)
 
       {text, MapSet.to_list(mentions), MapSet.to_list(tags)}
     end
@@ -111,9 +115,9 @@ def mentions_escape(text, options \\ []) do
 
     if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do
       %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text)
-      AutoLinker.link(mentions, options) <> AutoLinker.link(rest, options)
+      Linkify.link(mentions, options) <> Linkify.link(rest, options)
     else
-      AutoLinker.link(text, options)
+      Linkify.link(text, options)
     end
   end
 

lib/pleroma/web/rich_media/helpers.ex

@@ -11,10 +11,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do
 
   @spec validate_page_url(URI.t() | binary()) :: :ok | :error
   defp validate_page_url(page_url) when is_binary(page_url) do
-    validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld]
+    validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld])
 
     page_url
-    |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld)
+    |> Linkify.Parser.url?(validate_tld: validate_tld)
     |> parse_uri(page_url)
   end
 

mix.exs

@@ -167,9 +167,7 @@ defp deps do
       {:floki, "~> 0.25"},
       {:timex, "~> 3.5"},
       {:ueberauth, "~> 0.4"},
-      {:auto_linker,
+      {:linkify, "~> 0.2.0"},
-       git: "https://git.pleroma.social/pleroma/auto_linker.git",
-       ref: "95e8188490e97505c56636c1379ffdf036c1fdde"},
       {:http_signatures,
        git: "https://git.pleroma.social/pleroma/http_signatures.git",
        ref: "293d77bb6f4a67ac8bde1428735c3b42f22cbb30"},

mix.lock

@@ -1,6 +1,5 @@
 %{
   "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm", "11b18c220bcc2eab63b5470c038ef10eb6783bcb1fcdb11aa4137defa5ac1bb8"},
-  "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "95e8188490e97505c56636c1379ffdf036c1fdde", [ref: "95e8188490e97505c56636c1379ffdf036c1fdde"]},
   "base62": {:hex, :base62, "1.2.1", "4866763e08555a7b3917064e9eef9194c41667276c51b59de2bc42c6ea65f806", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "3b29948de2013d3f93aa898c884a9dff847e7aec75d9d6d8c1dc4c61c2716c42"},
   "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"},
   "bbcode": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/bbcode.git", "f2d267675e9a7e1ad1ea9beb4cc23382762b66c2", [ref: "v0.2.0"]},
@@ -63,6 +62,7 @@
   "jose": {:hex, :jose, "1.10.1", "16d8e460dae7203c6d1efa3f277e25b5af8b659febfc2f2eb4bacf87f128b80a", [:mix, :rebar3], [], "hexpm", "3c7ddc8a9394b92891db7c2771da94bf819834a1a4c92e30857b7d582e2f8257"},
   "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"},
   "libring": {:hex, :libring, "1.4.0", "41246ba2f3fbc76b3971f6bce83119dfec1eee17e977a48d8a9cfaaf58c2a8d6", [:mix], [], "hexpm"},
+  "linkify": {:hex, :linkify, "0.2.0", "2518bbbea21d2caa9d372424e1ad845b640c6630e2d016f1bd1f518f9ebcca28", [:mix], [], "hexpm", "b8ca8a68b79e30b7938d6c996085f3db14939f29538a59ca5101988bb7f917f6"},
   "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"},
   "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm", "d34f013c156db51ad57cc556891b9720e6a1c1df5fe2e15af999c84d6cebeb1a"},

priv/repo/migrations/20200716195806_autolinker_to_linkify.exs

@@ -0,0 +1,36 @@
+defmodule Pleroma.Repo.Migrations.AutolinkerToLinkify do
+  use Ecto.Migration
+  alias Pleroma.ConfigDB
+
+  @autolinker_path %{group: :auto_linker, key: :opts}
+  @linkify_path %{group: :pleroma, key: Pleroma.Formatter}
+
+  @compat_opts [:class, :rel, :new_window, :truncate, :strip_prefix, :extra]
+
+  def change do
+    with {:ok, {old, new}} <- maybe_get_params() do
+      move_config(old, new)
+    end
+  end
+
+  defp move_config(%{} = old, %{} = new) do
+    {:ok, _} = ConfigDB.update_or_create(new)
+    {:ok, _} = ConfigDB.delete(old)
+    :ok
+  end
+
+  defp maybe_get_params() do
+    with %ConfigDB{value: opts} <- ConfigDB.get_by_params(@autolinker_path),
+         opts <- transform_opts(opts),
+         %{} = linkify_params <- Map.put(@linkify_path, :value, opts) do
+      {:ok, {@autolinker_path, linkify_params}}
+    end
+  end
+
+  def transform_opts(opts) when is_list(opts) do
+    opts
+    |> Enum.into(%{})
+    |> Map.take(@compat_opts)
+    |> Map.to_list()
+  end
+end

priv/repo/migrations/20200722185515_fix_malformed_formatter_config.exs

@@ -0,0 +1,26 @@
+defmodule Pleroma.Repo.Migrations.FixMalformedFormatterConfig do
+  use Ecto.Migration
+  alias Pleroma.ConfigDB
+
+  @config_path %{group: :pleroma, key: Pleroma.Formatter}
+
+  def change do
+    with %ConfigDB{value: %{} = opts} <- ConfigDB.get_by_params(@config_path),
+         fixed_opts <- Map.to_list(opts) do
+      fix_config(fixed_opts)
+    else
+      _ -> :skipped
+    end
+  end
+
+  defp fix_config(fixed_opts) when is_list(fixed_opts) do
+    {:ok, _} =
+      ConfigDB.update_or_create(%{
+        group: :pleroma,
+        key: Pleroma.Formatter,
+        value: fixed_opts
+      })
+
+    :ok
+  end
+end

test/formatter_test.exs

@@ -10,6 +10,7 @@ defmodule Pleroma.FormatterTest do
   import Pleroma.Factory
 
   setup_all do
+    clear_config(Pleroma.Formatter)
     Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
     :ok
   end
@@ -255,6 +256,36 @@ test "it can parse mentions and return the relevant users" do
 
       assert {_text, ^expected_mentions, []} = Formatter.linkify(text)
     end
+
+    test "it parses URL containing local mention" do
+      _user = insert(:user, %{nickname: "lain"})
+
+      text = "https://example.com/@lain"
+
+      expected = ~S(<a href="https://example.com/@lain" rel="ugc">https://example.com/@lain</a>)
+
+      assert {^expected, [], []} = Formatter.linkify(text)
+    end
+
+    test "it correctly parses angry face D:< with mention" do
+      lain =
+        insert(:user, %{
+          nickname: "lain@lain.com",
+          ap_id: "https://lain.com/users/lain",
+          id: "9qrWmR0cKniB0YU0TA"
+        })
+
+      text = "@lain@lain.com D:<"
+
+      expected_text =
+        ~S(<span class="h-card"><a class="u-url mention" data-user="9qrWmR0cKniB0YU0TA" href="https://lain.com/users/lain" rel="ugc">@<span>lain</span></a></span> D:<)
+
+      expected_mentions = [
+        {"@lain@lain.com", lain}
+      ]
+
+      assert {^expected_text, ^expected_mentions, []} = Formatter.linkify(text)
+    end
   end
 
   describe ".parse_tags" do

test/migrations/20200716195806_autolinker_to_linkify_test.exs

@@ -0,0 +1,68 @@
+defmodule Pleroma.Repo.Migrations.AutolinkerToLinkifyTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+  import Pleroma.Tests.Helpers
+  alias Pleroma.ConfigDB
+
+  setup do: clear_config(Pleroma.Formatter)
+  setup_all do: require_migration("20200716195806_autolinker_to_linkify")
+
+  test "change/0 converts auto_linker opts for Pleroma.Formatter", %{migration: migration} do
+    autolinker_opts = [
+      extra: true,
+      validate_tld: true,
+      class: false,
+      strip_prefix: false,
+      new_window: false,
+      rel: "testing"
+    ]
+
+    insert(:config, group: :auto_linker, key: :opts, value: autolinker_opts)
+
+    migration.change()
+
+    assert nil == ConfigDB.get_by_params(%{group: :auto_linker, key: :opts})
+
+    %{value: new_opts} = ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Formatter})
+
+    assert new_opts == [
+             class: false,
+             extra: true,
+             new_window: false,
+             rel: "testing",
+             strip_prefix: false
+           ]
+
+    Pleroma.Config.put(Pleroma.Formatter, new_opts)
+    assert new_opts == Pleroma.Config.get(Pleroma.Formatter)
+
+    {text, _mentions, []} =
+      Pleroma.Formatter.linkify(
+        "https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\n\nOmg will COVID finally end Black Friday???"
+      )
+
+    assert text ==
+             "<a href=\"https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\" rel=\"testing\">https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7</a>\n\nOmg will COVID finally end Black Friday???"
+  end
+
+  test "transform_opts/1 returns a list of compatible opts", %{migration: migration} do
+    old_opts = [
+      extra: true,
+      validate_tld: true,
+      class: false,
+      strip_prefix: false,
+      new_window: false,
+      rel: "qqq"
+    ]
+
+    expected_opts = [
+      class: false,
+      extra: true,
+      new_window: false,
+      rel: "qqq",
+      strip_prefix: false
+    ]
+
+    assert migration.transform_opts(old_opts) == expected_opts
+  end
+end

test/migrations/20200722185515_fix_malformed_formatter_config_test.exs

@@ -0,0 +1,66 @@
+defmodule Pleroma.Repo.Migrations.FixMalformedFormatterConfigTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+  import Pleroma.Tests.Helpers
+  alias Pleroma.ConfigDB
+
+  setup do: clear_config(Pleroma.Formatter)
+  setup_all do: require_migration("20200722185515_fix_malformed_formatter_config")
+
+  test "change/0 converts a map into a list", %{migration: migration} do
+    incorrect_opts = %{
+      class: false,
+      extra: true,
+      new_window: false,
+      rel: "F",
+      strip_prefix: false
+    }
+
+    insert(:config, group: :pleroma, key: Pleroma.Formatter, value: incorrect_opts)
+
+    assert :ok == migration.change()
+
+    %{value: new_opts} = ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Formatter})
+
+    assert new_opts == [
+             class: false,
+             extra: true,
+             new_window: false,
+             rel: "F",
+             strip_prefix: false
+           ]
+
+    Pleroma.Config.put(Pleroma.Formatter, new_opts)
+    assert new_opts == Pleroma.Config.get(Pleroma.Formatter)
+
+    {text, _mentions, []} =
+      Pleroma.Formatter.linkify(
+        "https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\n\nOmg will COVID finally end Black Friday???"
+      )
+
+    assert text ==
+             "<a href=\"https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7\" rel=\"F\">https://www.businessinsider.com/walmart-will-close-stores-on-thanksgiving-ending-black-friday-tradition-2020-7</a>\n\nOmg will COVID finally end Black Friday???"
+  end
+
+  test "change/0 skips if Pleroma.Formatter config is already a list", %{migration: migration} do
+    opts = [
+      class: false,
+      extra: true,
+      new_window: false,
+      rel: "ugc",
+      strip_prefix: false
+    ]
+
+    insert(:config, group: :pleroma, key: Pleroma.Formatter, value: opts)
+
+    assert :skipped == migration.change()
+
+    %{value: new_opts} = ConfigDB.get_by_params(%{group: :pleroma, key: Pleroma.Formatter})
+
+    assert new_opts == opts
+  end
+
+  test "change/0 skips if Pleroma.Formatter is empty", %{migration: migration} do
+    assert :skipped == migration.change()
+  end
+end

test/support/helpers.ex

@@ -32,6 +32,11 @@ defmacro clear_config(config_path, temp_setting) do
     end
   end
 
+  def require_migration(migration_name) do
+    [{module, _}] = Code.require_file("#{migration_name}.exs", "priv/repo/migrations")
+    {:ok, %{migration: module}}
+  end
+
   defmacro __using__(_opts) do
     quote do
       import Pleroma.Tests.Helpers,