From c3ac9424d2affe87df82c14dc243f507fa639343 Mon Sep 17 00:00:00 2001 From: Egor Date: Tue, 26 Feb 2019 23:32:26 +0000 Subject: [PATCH] AutoLinker --- config/config.exs | 10 + docs/config.md | 27 ++- lib/pleroma/formatter.ex | 194 +++++------------- lib/pleroma/user.ex | 6 +- lib/pleroma/web/common_api/common_api.ex | 28 +-- lib/pleroma/web/common_api/utils.ex | 86 ++++---- mix.exs | 5 +- mix.lock | 1 + test/formatter_test.exs | 112 ++++------ test/web/common_api/common_api_utils_test.exs | 32 ++- 10 files changed, 211 insertions(+), 290 deletions(-) diff --git a/config/config.exs b/config/config.exs index 6119aaea1..7e4ac1100 100644 --- a/config/config.exs +++ b/config/config.exs @@ -344,6 +344,16 @@ federator_outgoing: [max_jobs: 50], mailer: [max_jobs: 10] +config :auto_linker, + opts: [ + scheme: true, + extra: true, + class: false, + strip_prefix: false, + new_window: false, + rel: false + ] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/docs/config.md b/docs/config.md index 14723b727..d1bf2a6f4 100644 --- a/docs/config.md +++ b/docs/config.md @@ -107,7 +107,7 @@ config :pleroma, Pleroma.Mailer, An example to enable ONLY ExSyslogger (f/ex in ``prod.secret.exs``) with info and debug suppressed: ``` -config :logger, +config :logger, backends: [{ExSyslogger, :ex_syslogger}] config :logger, :ex_syslogger, @@ -301,3 +301,28 @@ For each pool, the options are: * `max_connections` - how much connections a pool can hold * `timeout` - retention duration for connections +## :auto_linker + +Configuration for the `auto_linker` library: + +* `class: "auto-linker"` - specify the class to be added to the generated link. false to clear +* `rel: "noopener noreferrer"` - override the rel attribute. false to clear +* `new_window: true` - set to false to remove `target='_blank'` attribute +* `scheme: false` - Set to true to link urls with schema `http://google.com` +* `truncate: false` - Set to a number to truncate urls longer then the number. Truncated urls will end in `..` +* `strip_prefix: true` - Strip the scheme prefix +* `extra: false` - link urls with rarely used schemes (magnet, ipfs, irc, etc.) + +Example: + +```exs +config :auto_linker, + opts: [ + scheme: true, + extra: true, + class: false, + strip_prefix: false, + new_window: false, + rel: false + ] +``` diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index f31aafa0d..51d08c5ee 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -8,33 +8,51 @@ defmodule Pleroma.Formatter do alias Pleroma.User alias Pleroma.Web.MediaProxy - @tag_regex ~r/((?<=[^&])|\A)(\#)(\w+)/u @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ + @link_regex ~r{((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+}ui - # Modified from https://www.w3.org/TR/html5/forms.html#valid-e-mail-address - @mentions_regex ~r/@[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]*@?[a-zA-Z0-9_-](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/u + @auto_linker_config hashtag: true, + hashtag_handler: &Pleroma.Formatter.hashtag_handler/4, + mention: true, + mention_handler: &Pleroma.Formatter.mention_handler/4 - def parse_tags(text, data \\ %{}) do - Regex.scan(@tag_regex, text) - |> Enum.map(fn ["#" <> tag = full_tag | _] -> {full_tag, String.downcase(tag)} end) - |> (fn map -> - if data["sensitive"] in [true, "True", "true", "1"], - do: [{"#nsfw", "nsfw"}] ++ map, - else: map - end).() + def mention_handler("@" <> nickname, buffer, opts, acc) do + case User.get_cached_by_nickname(nickname) do + %User{id: id} = user -> + ap_id = get_ap_id(user) + nickname_text = get_nickname_text(nickname, opts) |> maybe_escape(opts) + + link = + "@#{ + nickname_text + }" + + {link, %{acc | mentions: MapSet.put(acc.mentions, {"@" <> nickname, user})}} + + _ -> + {buffer, acc} + end end - @doc "Parses mentions text and returns list {nickname, user}." - @spec parse_mentions(binary()) :: list({binary(), User.t()}) - def parse_mentions(text) do - Regex.scan(@mentions_regex, text) - |> List.flatten() - |> Enum.uniq() - |> Enum.map(fn nickname -> - with nickname <- String.trim_leading(nickname, "@"), - do: {"@" <> nickname, User.get_cached_by_nickname(nickname)} - end) - |> Enum.filter(fn {_match, user} -> user end) + def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do + tag = String.downcase(tag) + url = "#{Pleroma.Web.base_url()}/tag/#{tag}" + link = "" + + {link, %{acc | tags: MapSet.put(acc.tags, {tag_text, tag})}} + end + + @doc """ + Parses a text and replace plain text links with HTML. Returns a tuple with a result text, mentions, and hashtags. + """ + @spec linkify(String.t(), keyword()) :: + {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]} + def linkify(text, options \\ []) do + options = options ++ @auto_linker_config + acc = %{mentions: MapSet.new(), tags: MapSet.new()} + {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options) + + {text, MapSet.to_list(mentions), MapSet.to_list(tags)} end def emojify(text) do @@ -48,9 +66,7 @@ def emojify(text, emoji, strip \\ false) do emoji = HTML.strip_tags(emoji) file = HTML.strip_tags(file) - String.replace( - text, - ":#{emoji}:", + html = if not strip do "#{emoji} Enum.join("") end - @doc """ - Escapes a special characters in mention names. - """ - @spec mentions_escape(String.t(), list({String.t(), any()})) :: String.t() - def mentions_escape(text, mentions) do - mentions - |> Enum.reduce(text, fn {name, _}, acc -> - escape_name = String.replace(name, @markdown_characters_regex, "\\\\\\1") - String.replace(acc, name, escape_name) - end) - end - - @doc "changes scheme:... urls to html links" - def add_links({subs, text}) do - links = - text - |> String.split([" ", "\t", "
"]) - |> Enum.filter(fn word -> String.starts_with?(word, @valid_schemes) end) - |> Enum.filter(fn word -> Regex.match?(@link_regex, word) end) - |> Enum.map(fn url -> {Ecto.UUID.generate(), url} end) - |> Enum.sort_by(fn {_, url} -> -String.length(url) end) - - uuid_text = - links - |> Enum.reduce(text, fn {uuid, url}, acc -> String.replace(acc, url, uuid) end) - - subs = - subs ++ - Enum.map(links, fn {uuid, url} -> - {uuid, "#{url}"} - end) - - {subs, uuid_text} - end - - @doc "Adds the links to mentioned users" - def add_user_links({subs, text}, mentions, options \\ []) do - mentions = - mentions - |> Enum.sort_by(fn {name, _} -> -String.length(name) end) - |> Enum.map(fn {name, user} -> {name, user, Ecto.UUID.generate()} end) - - uuid_text = - mentions - |> Enum.reduce(text, fn {match, _user, uuid}, text -> - String.replace(text, match, uuid) - end) - - subs = - subs ++ - Enum.map(mentions, fn {match, %User{id: id, ap_id: ap_id, info: info}, uuid} -> - ap_id = - if is_binary(info.source_data["url"]) do - info.source_data["url"] - else - ap_id - end - - nickname = - if options[:format] == :full do - User.full_nickname(match) - else - User.local_nickname(match) - end - - {uuid, - "" <> - "@#{nickname}"} - end) - - {subs, uuid_text} - end - - @doc "Adds the hashtag links" - def add_hashtag_links({subs, text}, tags) do - tags = - tags - |> Enum.sort_by(fn {name, _} -> -String.length(name) end) - |> Enum.map(fn {name, short} -> {name, short, Ecto.UUID.generate()} end) - - uuid_text = - tags - |> Enum.reduce(text, fn {match, _short, uuid}, text -> - String.replace(text, ~r/((?<=[^&])|(\A))#{match}/, uuid) - end) - - subs = - subs ++ - Enum.map(tags, fn {tag_text, tag, uuid} -> - url = - "" - - {uuid, url} - end) - - {subs, uuid_text} - end - - def finalize({subs, text}) do - Enum.reduce(subs, text, fn {uuid, replacement}, result_text -> - String.replace(result_text, uuid, replacement) - end) - end - def truncate(text, max_length \\ 200, omission \\ "...") do # Remove trailing whitespace text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}") @@ -211,4 +115,16 @@ def truncate(text, max_length \\ 200, omission \\ "...") do String.slice(text, 0, length_with_omission) <> omission end end + + defp get_ap_id(%User{info: %{source_data: %{"url" => url}}}) when is_binary(url), do: url + defp get_ap_id(%User{ap_id: ap_id}), do: ap_id + + defp get_nickname_text(nickname, %{mentions_format: :full}), do: User.full_nickname(nickname) + defp get_nickname_text(nickname, _), do: User.local_nickname(nickname) + + defp maybe_escape(str, %{mentions_escape: true}) do + String.replace(str, @markdown_characters_regex, "\\\\\\1") + end + + defp maybe_escape(str, _), do: str end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 12e0e818e..01d532ab3 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1193,9 +1193,6 @@ def parse_bio(nil, _user), do: "" def parse_bio(bio, _user) when bio == "", do: bio def parse_bio(bio, user) do - mentions = Formatter.parse_mentions(bio) - tags = Formatter.parse_tags(bio) - emoji = (user.info.source_data["tag"] || []) |> Enum.filter(fn %{"type" => t} -> t == "Emoji" end) @@ -1204,7 +1201,8 @@ def parse_bio(bio, user) do end) bio - |> CommonUtils.format_input(mentions, tags, "text/plain", user_links: [format: :full]) + |> CommonUtils.format_input("text/plain", mentions_format: :full) + |> elem(0) |> Formatter.emojify(emoji) end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index e788337cc..7114d6de6 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -82,40 +82,20 @@ def get_visibility(%{"in_reply_to_status_id" => status_id}) when not is_nil(stat def get_visibility(_), do: "public" - defp get_content_type(content_type) do - if Enum.member?(Pleroma.Config.get([:instance, :allowed_post_formats]), content_type) do - content_type - else - "text/plain" - end - end - def post(user, %{"status" => status} = data) do visibility = get_visibility(data) limit = Pleroma.Config.get([:instance, :limit]) with status <- String.trim(status), attachments <- attachments_from_ids(data), - mentions <- Formatter.parse_mentions(status), inReplyTo <- get_replied_to_activity(data["in_reply_to_status_id"]), - {to, cc} <- to_for_user_and_mentions(user, mentions, inReplyTo, visibility), - tags <- Formatter.parse_tags(status, data), - content_html <- + {content_html, mentions, tags} <- make_content_html( status, - mentions, attachments, - tags, - get_content_type(data["content_type"]), - Enum.member?( - [true, "true"], - Map.get( - data, - "no_attachment_links", - Pleroma.Config.get([:instance, :no_attachment_links], false) - ) - ) + data ), + {to, cc} <- to_for_user_and_mentions(user, mentions, inReplyTo, visibility), context <- make_context(inReplyTo), cw <- data["spoiler_text"], full_payload <- String.trim(status <> (data["spoiler_text"] || "")), @@ -247,7 +227,7 @@ def thread_muted?(user, activity) do def report(user, data) do with {:account_id, %{"account_id" => account_id}} <- {:account_id, data}, {:account, %User{} = account} <- {:account, User.get_by_id(account_id)}, - {:ok, content_html} <- make_report_content_html(data["comment"]), + {:ok, {content_html, _, _}} <- make_report_content_html(data["comment"]), {:ok, statuses} <- get_report_statuses(account, data), {:ok, activity} <- ActivityPub.flag(%{ diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 1d3a314ce..20123854d 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Web + alias Pleroma.Config alias Pleroma.Web.Endpoint alias Pleroma.Web.MediaProxy alias Pleroma.Web.ActivityPub.Utils @@ -100,24 +100,45 @@ def to_for_user_and_mentions(_user, mentions, inReplyTo, "direct") do def make_content_html( status, - mentions, attachments, - tags, - content_type, - no_attachment_links \\ false + data ) do + no_attachment_links = + data + |> Map.get("no_attachment_links", Config.get([:instance, :no_attachment_links])) + |> Kernel.in([true, "true"]) + + content_type = get_content_type(data["content_type"]) + status - |> format_input(mentions, tags, content_type) + |> format_input(content_type) |> maybe_add_attachments(attachments, no_attachment_links) + |> maybe_add_nsfw_tag(data) end + defp get_content_type(content_type) do + if Enum.member?(Config.get([:instance, :allowed_post_formats]), content_type) do + content_type + else + "text/plain" + end + end + + defp maybe_add_nsfw_tag({text, mentions, tags}, %{"sensitive" => sensitive}) + when sensitive in [true, "True", "true", "1"] do + {text, mentions, [{"#nsfw", "nsfw"} | tags]} + end + + defp maybe_add_nsfw_tag(data, _), do: data + def make_context(%Activity{data: %{"context" => context}}), do: context def make_context(_), do: Utils.generate_context_id() - def maybe_add_attachments(text, _attachments, true = _no_links), do: text + def maybe_add_attachments(parsed, _attachments, true = _no_links), do: parsed - def maybe_add_attachments(text, attachments, _no_links) do - add_attachments(text, attachments) + def maybe_add_attachments({text, mentions, tags}, attachments, _no_links) do + text = add_attachments(text, attachments) + {text, mentions, tags} end def add_attachments(text, attachments) do @@ -135,56 +156,39 @@ def add_attachments(text, attachments) do Enum.join([text | attachment_text], "
") end - def format_input(text, mentions, tags, format, options \\ []) + def format_input(text, format, options \\ []) @doc """ Formatting text to plain text. """ - def format_input(text, mentions, tags, "text/plain", options) do + def format_input(text, "text/plain", options) do text |> Formatter.html_escape("text/plain") - |> String.replace(~r/\r?\n/, "
") - |> (&{[], &1}).() - |> Formatter.add_links() - |> Formatter.add_user_links(mentions, options[:user_links] || []) - |> Formatter.add_hashtag_links(tags) - |> Formatter.finalize() + |> Formatter.linkify(options) + |> (fn {text, mentions, tags} -> + {String.replace(text, ~r/\r?\n/, "
"), mentions, tags} + end).() end @doc """ Formatting text to html. """ - def format_input(text, mentions, _tags, "text/html", options) do + def format_input(text, "text/html", options) do text |> Formatter.html_escape("text/html") - |> (&{[], &1}).() - |> Formatter.add_user_links(mentions, options[:user_links] || []) - |> Formatter.finalize() + |> Formatter.linkify(options) end @doc """ Formatting text to markdown. """ - def format_input(text, mentions, tags, "text/markdown", options) do + def format_input(text, "text/markdown", options) do + options = Keyword.put(options, :mentions_escape, true) + text - |> Formatter.mentions_escape(mentions) - |> Earmark.as_html!() |> Formatter.html_escape("text/html") - |> (&{[], &1}).() - |> Formatter.add_user_links(mentions, options[:user_links] || []) - |> Formatter.add_hashtag_links(tags) - |> Formatter.finalize() - end - - def add_tag_links(text, tags) do - tags = - tags - |> Enum.sort_by(fn {tag, _} -> -String.length(tag) end) - - Enum.reduce(tags, text, fn {full, tag}, text -> - url = "" - String.replace(text, full, url) - end) + |> Formatter.linkify(options) + |> (fn {text, mentions, tags} -> {Earmark.as_html!(text), mentions, tags} end).() end def make_note_data( @@ -323,13 +327,13 @@ def maybe_extract_mentions(%{"tag" => tag}) do def maybe_extract_mentions(_), do: [] - def make_report_content_html(nil), do: {:ok, nil} + def make_report_content_html(nil), do: {:ok, {nil, [], []}} def make_report_content_html(comment) do max_size = Pleroma.Config.get([:instance, :max_report_comment_size], 1000) if String.length(comment) <= max_size do - {:ok, format_input(comment, [], [], "text/plain")} + {:ok, format_input(comment, "text/plain")} else {:error, "Comment must be up to #{max_size} characters"} end diff --git a/mix.exs b/mix.exs index d78825769..5392d94d1 100644 --- a/mix.exs +++ b/mix.exs @@ -90,7 +90,10 @@ defp deps do {:websocket_client, git: "https://github.com/jeremyong/websocket_client.git", only: :test}, {:floki, "~> 0.20.0"}, {:ex_syslogger, github: "slashmili/ex_syslogger", tag: "1.4.0"}, - {:timex, "~> 3.5"} + {:timex, "~> 3.5"}, + {:auto_linker, + git: "https://git.pleroma.social/pleroma/auto_linker.git", + ref: "94193ca5f97c1f9fdf3d1469653e2d46fac34bcd"} ] end diff --git a/mix.lock b/mix.lock index 5ffaedd16..918702444 100644 --- a/mix.lock +++ b/mix.lock @@ -1,4 +1,5 @@ %{ + "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "94193ca5f97c1f9fdf3d1469653e2d46fac34bcd", [ref: "94193ca5f97c1f9fdf3d1469653e2d46fac34bcd"]}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"}, "cachex": {:hex, :cachex, "3.0.2", "1351caa4e26e29f7d7ec1d29b53d6013f0447630bbf382b4fb5d5bad0209f203", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/test/formatter_test.exs b/test/formatter_test.exs index f14077d25..7d8864bf4 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -21,22 +21,16 @@ test "turns hashtags into links" do expected_text = "I love and " - tags = Formatter.parse_tags(text) - - assert expected_text == - Formatter.add_hashtag_links({[], text}, tags) |> Formatter.finalize() + assert {^expected_text, [], _tags} = Formatter.linkify(text) end test "does not turn html characters to tags" do - text = "Fact #3: pleroma does what mastodon't" + text = "#fact_3: pleroma does what mastodon't" expected_text = - "Fact : pleroma does what mastodon't" + ": pleroma does what mastodon't" - tags = Formatter.parse_tags(text) - - assert expected_text == - Formatter.add_hashtag_links({[], text}, tags) |> Formatter.finalize() + assert {^expected_text, [], _tags} = Formatter.linkify(text) end end @@ -47,79 +41,79 @@ test "turning urls into links" do expected = "Hey, check out https://www.youtube.com/watch?v=8Zg1-TufF%20zY?x=1&y=2#blabla ." - assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected + assert {^expected, [], []} = Formatter.linkify(text) text = "https://mastodon.social/@lambadalambda" expected = "https://mastodon.social/@lambadalambda" - assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected + assert {^expected, [], []} = Formatter.linkify(text) text = "https://mastodon.social:4000/@lambadalambda" expected = "https://mastodon.social:4000/@lambadalambda" - assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected + assert {^expected, [], []} = Formatter.linkify(text) text = "@lambadalambda" expected = "@lambadalambda" - assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected + assert {^expected, [], []} = Formatter.linkify(text) text = "http://www.cs.vu.nl/~ast/intel/" expected = "http://www.cs.vu.nl/~ast/intel/" - assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected + assert {^expected, [], []} = Formatter.linkify(text) text = "https://forum.zdoom.org/viewtopic.php?f=44&t=57087" expected = "https://forum.zdoom.org/viewtopic.php?f=44&t=57087" - assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected + assert {^expected, [], []} = Formatter.linkify(text) text = "https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul" expected = "https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul" - assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected + assert {^expected, [], []} = Formatter.linkify(text) text = "https://www.google.co.jp/search?q=Nasim+Aghdam" expected = "https://www.google.co.jp/search?q=Nasim+Aghdam" - assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected + assert {^expected, [], []} = Formatter.linkify(text) text = "https://en.wikipedia.org/wiki/Duff's_device" expected = "https://en.wikipedia.org/wiki/Duff's_device" - assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected + assert {^expected, [], []} = Formatter.linkify(text) text = "https://pleroma.com https://pleroma.com/sucks" expected = "https://pleroma.com https://pleroma.com/sucks" - assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected + assert {^expected, [], []} = Formatter.linkify(text) text = "xmpp:contact@hacktivis.me" expected = "xmpp:contact@hacktivis.me" - assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected + assert {^expected, [], []} = Formatter.linkify(text) text = "magnet:?xt=urn:btih:7ec9d298e91d6e4394d1379caf073c77ff3e3136&tr=udp%3A%2F%2Fopentor.org%3A2710&tr=udp%3A%2F%2Ftracker.blackunicorn.xyz%3A6969&tr=udp%3A%2F%2Ftracker.ccc.de%3A80&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.openbittorrent.com%3A80&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com" expected = "#{text}" - assert Formatter.add_links({[], text}) |> Formatter.finalize() == expected + assert {^expected, [], []} = Formatter.linkify(text) end end @@ -136,12 +130,9 @@ test "gives a replacement for user links, using local nicknames in user links te archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"}) - mentions = Pleroma.Formatter.parse_mentions(text) + {text, mentions, []} = Formatter.linkify(text) - {subs, text} = Formatter.add_user_links({[], text}, mentions) - - assert length(subs) == 3 - Enum.each(subs, fn {uuid, _} -> assert String.contains?(text, uuid) end) + assert length(mentions) == 3 expected_text = "@archaeme" - assert expected_text == Formatter.finalize({subs, text}) + assert expected_text == text end test "gives a replacement for user links when the user is using Osada" do @@ -160,48 +151,35 @@ test "gives a replacement for user links when the user is using Osada" do text = "@mike@osada.macgirvin.com test" - mentions = Formatter.parse_mentions(text) + {text, mentions, []} = Formatter.linkify(text) - {subs, text} = Formatter.add_user_links({[], text}, mentions) - - assert length(subs) == 1 - Enum.each(subs, fn {uuid, _} -> assert String.contains?(text, uuid) end) + assert length(mentions) == 1 expected_text = "@mike test" - assert expected_text == Formatter.finalize({subs, text}) + assert expected_text == text end test "gives a replacement for single-character local nicknames" do text = "@o hi" o = insert(:user, %{nickname: "o"}) - mentions = Formatter.parse_mentions(text) + {text, mentions, []} = Formatter.linkify(text) - {subs, text} = Formatter.add_user_links({[], text}, mentions) - - assert length(subs) == 1 - Enum.each(subs, fn {uuid, _} -> assert String.contains?(text, uuid) end) + assert length(mentions) == 1 expected_text = "@o hi" - assert expected_text == Formatter.finalize({subs, text}) + assert expected_text == text end test "does not give a replacement for single-character local nicknames who don't exist" do text = "@a hi" - mentions = Formatter.parse_mentions(text) - - {subs, text} = Formatter.add_user_links({[], text}, mentions) - - assert Enum.empty?(subs) - Enum.each(subs, fn {uuid, _} -> assert String.contains?(text, uuid) end) - expected_text = "@a hi" - assert expected_text == Formatter.finalize({subs, text}) + assert {^expected_text, [] = _mentions, [] = _tags} = Formatter.linkify(text) end end @@ -209,14 +187,14 @@ test "does not give a replacement for single-character local nicknames who don't test "parses tags in the text" do text = "Here's a #Test. Maybe these are #working or not. What about #漢字? And #は｡" - expected = [ + expected_tags = [ {"#Test", "test"}, {"#working", "working"}, - {"#漢字", "漢字"}, - {"#は", "は"} + {"#は", "は"}, + {"#漢字", "漢字"} ] - assert Formatter.parse_tags(text) == expected + assert {_text, [], ^expected_tags} = Formatter.linkify(text) end end @@ -230,15 +208,15 @@ test "it can parse mentions and return the relevant users" do archaeme = insert(:user, %{nickname: "archaeme"}) archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"}) - expected_result = [ - {"@gsimg", gsimg}, + expected_mentions = [ {"@archaeme", archaeme}, {"@archaeme@archae.me", archaeme_remote}, - {"@o", o}, - {"@jimm", jimm} + {"@gsimg", gsimg}, + {"@jimm", jimm}, + {"@o", o} ] - assert Formatter.parse_mentions(text) == expected_result + assert {_text, ^expected_mentions, []} = Formatter.linkify(text) end test "it adds cool emoji" do @@ -281,22 +259,10 @@ test "it doesn't die when text is absent" do assert Formatter.get_emoji(text) == [] end - describe "/mentions_escape" do - test "it returns text with escaped mention names" do - text = """ - @a_breakin_glass@cybre.space - (also, little voice inside my head thinking "maybe this will encourage people - pronouncing it properly instead of saying _raKEWdo_ ") - """ + test "it escapes HTML in plain text" do + text = "hello & world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1" + expected = "hello & world google.com/?a=b&c=d \n http://test.com/?a=b&c=d 1" - escape_text = """ - @a\\_breakin\\_glass@cybre\\.space - (also, little voice inside my head thinking \"maybe this will encourage people - pronouncing it properly instead of saying _raKEWdo_ \") - """ - - mentions = [{"@a_breakin_glass@cybre.space", %{}}] - assert Formatter.mentions_escape(text, mentions) == escape_text - end + assert Formatter.html_escape(text, "text/plain") == expected end end diff --git a/test/web/common_api/common_api_utils_test.exs b/test/web/common_api/common_api_utils_test.exs index faed6b685..dc7b4c229 100644 --- a/test/web/common_api/common_api_utils_test.exs +++ b/test/web/common_api/common_api_utils_test.exs @@ -57,19 +57,19 @@ test "parses emoji from name and bio" do assert expected == Utils.emoji_from_profile(user) end - describe "format_input/4" do + describe "format_input/3" do test "works for bare text/plain" do text = "hello world!" expected = "hello world!" - output = Utils.format_input(text, [], [], "text/plain") + {output, [], []} = Utils.format_input(text, "text/plain") assert output == expected text = "hello world!\n\nsecond paragraph!" expected = "hello world!

second paragraph!" - output = Utils.format_input(text, [], [], "text/plain") + {output, [], []} = Utils.format_input(text, "text/plain") assert output == expected end @@ -78,14 +78,14 @@ test "works for bare text/html" do text = "

hello world!

" expected = "

hello world!

" - output = Utils.format_input(text, [], [], "text/html") + {output, [], []} = Utils.format_input(text, "text/html") assert output == expected text = "

hello world!

\n\n

second paragraph

" expected = "

hello world!

\n\n

second paragraph

" - output = Utils.format_input(text, [], [], "text/html") + {output, [], []} = Utils.format_input(text, "text/html") assert output == expected end @@ -94,14 +94,32 @@ test "works for bare text/markdown" do text = "**hello world**" expected = "

hello world

\n" - output = Utils.format_input(text, [], [], "text/markdown") + {output, [], []} = Utils.format_input(text, "text/markdown") assert output == expected text = "**hello world**\n\n*another paragraph*" expected = "

hello world

\n

another paragraph

\n" - output = Utils.format_input(text, [], [], "text/markdown") + {output, [], []} = Utils.format_input(text, "text/markdown") + + assert output == expected + end + + test "works for text/markdown with mentions" do + {:ok, user} = + UserBuilder.insert(%{nickname: "user__test", ap_id: "http://foo.com/user__test"}) + + text = "**hello world**\n\n*another @user__test and @user__test google.com paragraph*" + + expected = + "

hello world

\n

another @user__test and @user__test google.com paragraph

\n" + + {output, _, _} = Utils.format_input(text, "text/markdown") assert output == expected end