Merge branch 'feature/770-add-emoji-tags' into 'develop'

Feature/770 add emoji tags

See merge request pleroma/pleroma!998

.gitignore

@@ -35,3 +35,6 @@ erl_crash.dump
 
 # Editor config
 /.vscode/
+
+# Prevent committing docs files
+/priv/static/doc/*

config/config.exs

@@ -58,7 +58,13 @@
   cgi: "https://mdii.sakura.ne.jp/mdii-post.cgi",
   files: "https://mdii.sakura.ne.jp"
 
-config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"]
+config :pleroma, :emoji,
+  shortcode_globs: ["/emoji/custom/**/*.png"],
+  groups: [
+    # Put groups that have higher priority than defaults here. Example in `docs/config/custom_emoji.md`
+    Finmoji: "/finmoji/128px/*-128.png",
+    Custom: ["/emoji/*.png", "/emoji/custom/*.png"]
+  ]
 
 config :pleroma, :uri_schemes,
   valid_schemes: [

config/emoji.txt

@@ -1,5 +1,5 @@
-firefox, /emoji/Firefox.gif
-blank, /emoji/blank.png
+firefox, /emoji/Firefox.gif, Gif,Fun
+blank, /emoji/blank.png, Fun
 f_00b, /emoji/f_00b.png
 f_00b11b, /emoji/f_00b11b.png
 f_00b33b, /emoji/f_00b33b.png
@@ -28,4 +28,3 @@ f_33b00b, /emoji/f_33b00b.png
 f_33b22b, /emoji/f_33b22b.png
 f_33h, /emoji/f_33h.png
 f_33t, /emoji/f_33t.png
-

docs/api/pleroma_api.md

@@ -10,7 +10,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Authentication: not required
 * Params: none
 * Response: JSON
-* Example response: `{"kalsarikannit_f":"/finmoji/128px/kalsarikannit_f-128.png","perkele":"/finmoji/128px/perkele-128.png","blobdab":"/emoji/blobdab.png","happiness":"/finmoji/128px/happiness-128.png"}`
+* Example response: `[{"kalsarikannit_f":{"tags":["Finmoji"],"image_url":"/finmoji/128px/kalsarikannit_f-128.png"}},{"perkele":{"tags":["Finmoji"],"image_url":"/finmoji/128px/perkele-128.png"}},{"blobdab":{"tags":["SomeTag"],"image_url":"/emoji/blobdab.png"}},"happiness":{"tags":["Finmoji"],"image_url":"/finmoji/128px/happiness-128.png"}}]`
 * Note: Same data as Mastodon API’s `/api/v1/custom_emojis` but in a different format
 
 ## `/api/pleroma/follow_import`
@@ -27,14 +27,14 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Method: `GET`
 * Authentication: not required
 * Params: none
-* Response: Provider specific JSON, the only guaranteed parameter is `type` 
+* Response: Provider specific JSON, the only guaranteed parameter is `type`
 * Example response: `{"type": "kocaptcha", "token": "whatever", "url": "https://captcha.kotobank.ch/endpoint"}`
 
 ## `/api/pleroma/delete_account`
 ### Delete an account
 * Method `POST`
 * Authentication: required
-* Params: 
+* Params:
     * `password`: user's password
 * Response: JSON. Returns `{"status": "success"}` if the deletion was successful, `{"error": "[error message]"}` otherwise
 * Example response: `{"error": "Invalid password."}`

docs/config/custom_emoji.md

@@ -11,8 +11,43 @@ image files (in `/priv/static/emoji/custom`): `happy.png` and `sad.png`
 
 content of `config/custom_emoji.txt`:
 ```
-happy, /emoji/custom/happy.png
-sad, /emoji/custom/sad.png
+happy, /emoji/custom/happy.png, Tag1,Tag2
+sad, /emoji/custom/sad.png, Tag1
+foo, /emoji/custom/foo.png
 ```
 
 The files should be PNG (APNG is okay with `.png` for `image/png` Content-type) and under 50kb for compatibility with mastodon.
+
+## Emoji tags (groups)
+
+Default tags are set in `config.exs`.
+```elixir
+config :pleroma, :emoji,
+  shortcode_globs: ["/emoji/custom/**/*.png"],
+  groups: [
+    Finmoji: "/finmoji/128px/*-128.png",
+    Custom: ["/emoji/*.png", "/emoji/custom/*.png"]
+  ]
+```
+
+Order of the `groups` matters, so to override default tags just put your group on top of the list. E.g:
+```elixir
+config :pleroma, :emoji,
+  shortcode_globs: ["/emoji/custom/**/*.png"],
+  groups: [
+    "Finmoji special": "/finmoji/128px/a_trusted_friend-128.png", # special file
+    "Cirno": "/emoji/custom/cirno*.png", # png files in /emoji/custom/ which start with `cirno`
+    "Special group": "/emoji/custom/special_folder/*.png", # png files in /emoji/custom/special_folder/
+    "Another group": "/emoji/custom/special_folder/*/.png", # png files in /emoji/custom/special_folder/ subfolders
+    Finmoji: "/finmoji/128px/*-128.png",
+    Custom: ["/emoji/*.png", "/emoji/custom/*.png"]
+  ]
+```
+
+Priority of tags assigns in emoji.txt and custom.txt:
+
+`tag in file > special group setting in config.exs > default setting in config.exs`
+
+Priority for globs:
+
+`special group setting in config.exs > default setting in config.exs`

lib/pleroma/emoji.ex

@@ -8,13 +8,19 @@ defmodule Pleroma.Emoji do
 
     * the built-in Finmojis (if enabled in configuration),
     * the files: `config/emoji.txt` and `config/custom_emoji.txt`
-    * glob paths
+    * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder
 
   This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime.
   """
   use GenServer
+
+  @type pattern :: Regex.t() | module() | String.t()
+  @type patterns :: pattern() | [pattern()]
+  @type group_patterns :: keyword(patterns())
+
   @ets __MODULE__.Ets
   @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}]
+  @groups Application.get_env(:pleroma, :emoji)[:groups]
 
   @doc false
   def start_link do
@@ -73,13 +79,14 @@ def code_change(_old_vsn, state, _extra) do
   end
 
   defp load do
+    finmoji_enabled = Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled)
+    shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || []
+
     emojis =
-      (load_finmoji(Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled)) ++
+      (load_finmoji(finmoji_enabled) ++
          load_from_file("config/emoji.txt") ++
          load_from_file("config/custom_emoji.txt") ++
-         load_from_globs(
-           Keyword.get(Application.get_env(:pleroma, :emoji, []), :shortcode_globs, [])
-         ))
+         load_from_globs(shortcode_globs))
       |> Enum.reject(fn value -> value == nil end)
 
     true = :ets.insert(@ets, emojis)
@@ -151,9 +158,12 @@ defp load do
     "white_nights",
     "woollysocks"
   ]
+
   defp load_finmoji(true) do
     Enum.map(@finmoji, fn finmoji ->
-      {finmoji, "/finmoji/128px/#{finmoji}-128.png"}
+      file_name = "/finmoji/128px/#{finmoji}-128.png"
+      group = match_extra(@groups, file_name)
+      {finmoji, file_name, to_string(group)}
     end)
   end
 
@@ -172,8 +182,14 @@ defp load_from_file_stream(stream) do
     |> Stream.map(&String.trim/1)
     |> Stream.map(fn line ->
       case String.split(line, ~r/,\s*/) do
-        [name, file] -> {name, file}
-        _ -> nil
+        [name, file, tags] ->
+          {name, file, tags}
+
+        [name, file] ->
+          {name, file, to_string(match_extra(@groups, file))}
+
+        _ ->
+          nil
       end
     end)
     |> Enum.to_list()
@@ -190,9 +206,40 @@ defp load_from_globs(globs) do
       |> Enum.concat()
 
     Enum.map(paths, fn path ->
+      tag = match_extra(@groups, Path.join("/", Path.relative_to(path, static_path)))
       shortcode = Path.basename(path, Path.extname(path))
       external_path = Path.join("/", Path.relative_to(path, static_path))
-      {shortcode, external_path}
+      {shortcode, external_path, to_string(tag)}
+    end)
+  end
+
+  @doc """
+  Finds a matching group for the given emoji filename
+  """
+  @spec match_extra(group_patterns(), String.t()) :: atom() | nil
+  def match_extra(group_patterns, filename) do
+    match_group_patterns(group_patterns, fn pattern ->
+      case pattern do
+        %Regex{} = regex -> Regex.match?(regex, filename)
+        string when is_binary(string) -> filename == string
+      end
+    end)
+  end
+
+  defp match_group_patterns(group_patterns, matcher) do
+    Enum.find_value(group_patterns, fn {group, patterns} ->
+      patterns =
+        patterns
+        |> List.wrap()
+        |> Enum.map(fn pattern ->
+          if String.contains?(pattern, "*") do
+            ~r(#{String.replace(pattern, "*", ".*")})
+          else
+            pattern
+          end
+        end)
+
+      Enum.any?(patterns, matcher) && group
     end)
   end
 end

lib/pleroma/formatter.ex

@@ -77,9 +77,9 @@ def emojify(text) do
   def emojify(text, nil), do: text
 
   def emojify(text, emoji, strip \\ false) do
-    Enum.reduce(emoji, text, fn {emoji, file}, text ->
-      emoji = HTML.strip_tags(emoji)
-      file = HTML.strip_tags(file)
+    Enum.reduce(emoji, text, fn emoji_data, text ->
+      emoji = HTML.strip_tags(elem(emoji_data, 0))
+      file = HTML.strip_tags(elem(emoji_data, 1))
 
       html =
         if not strip do
@@ -101,7 +101,7 @@ def demojify(text) do
   def demojify(text, nil), do: text
 
   def get_emoji(text) when is_binary(text) do
-    Enum.filter(Emoji.get_all(), fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end)
+    Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end)
   end
 
   def get_emoji(_), do: []

lib/pleroma/web/common_api/common_api.ex

@@ -167,7 +167,7 @@ def post(user, %{"status" => status} = data) do
              object,
              "emoji",
              (Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"]))
-             |> Enum.reduce(%{}, fn {name, file}, acc ->
+             |> Enum.reduce(%{}, fn {name, file, _}, acc ->
                Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}")
              end)
            ) do

lib/pleroma/web/common_api/utils.ex

@@ -293,7 +293,7 @@ def confirm_current_password(user, password) do
 
   def emoji_from_profile(%{info: _info} = user) do
     (Formatter.get_emoji(user.bio) ++ Formatter.get_emoji(user.name))
-    |> Enum.map(fn {shortcode, url} ->
+    |> Enum.map(fn {shortcode, url, _} ->
       %{
         "type" => "Emoji",
         "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{url}"},

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -181,14 +181,15 @@ def peers(conn, _params) do
 
   defp mastodonized_emoji do
     Pleroma.Emoji.get_all()
-    |> Enum.map(fn {shortcode, relative_url} ->
+    |> Enum.map(fn {shortcode, relative_url, tags} ->
       url = to_string(URI.merge(Web.base_url(), relative_url))
 
       %{
         "shortcode" => shortcode,
         "static_url" => url,
         "visible_in_picker" => true,
-        "url" => url
+        "url" => url,
+        "tags" => String.split(tags, ",")
       }
     end)
   end

lib/pleroma/web/twitter_api/controllers/util_controller.ex

@@ -283,7 +283,13 @@ def version(conn, _params) do
   end
 
   def emoji(conn, _params) do
-    json(conn, Enum.into(Emoji.get_all(), %{}))
+    emoji =
+      Emoji.get_all()
+      |> Enum.map(fn {short_code, path, tags} ->
+        %{short_code => %{image_url: path, tags: String.split(tags, ",")}}
+      end)
+
+    json(conn, emoji)
   end
 
   def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do

test/emoji_test.exs

@@ -0,0 +1,106 @@
+defmodule Pleroma.EmojiTest do
+  use ExUnit.Case, async: true
+  alias Pleroma.Emoji
+
+  describe "get_all/0" do
+    setup do
+      emoji_list = Emoji.get_all()
+      {:ok, emoji_list: emoji_list}
+    end
+
+    test "first emoji", %{emoji_list: emoji_list} do
+      [emoji | _others] = emoji_list
+      {code, path, tags} = emoji
+
+      assert tuple_size(emoji) == 3
+      assert is_binary(code)
+      assert is_binary(path)
+      assert is_binary(tags)
+    end
+
+    test "random emoji", %{emoji_list: emoji_list} do
+      emoji = Enum.random(emoji_list)
+      {code, path, tags} = emoji
+
+      assert tuple_size(emoji) == 3
+      assert is_binary(code)
+      assert is_binary(path)
+      assert is_binary(tags)
+    end
+  end
+
+  describe "match_extra/2" do
+    setup do
+      groups = [
+        "list of files": ["/emoji/custom/first_file.png", "/emoji/custom/second_file.png"],
+        "wildcard folder": "/emoji/custom/*/file.png",
+        "wildcard files": "/emoji/custom/folder/*.png",
+        "special file": "/emoji/custom/special.png"
+      ]
+
+      {:ok, groups: groups}
+    end
+
+    test "config for list of files", %{groups: groups} do
+      group =
+        groups
+        |> Emoji.match_extra("/emoji/custom/first_file.png")
+        |> to_string()
+
+      assert group == "list of files"
+    end
+
+    test "config with wildcard folder", %{groups: groups} do
+      group =
+        groups
+        |> Emoji.match_extra("/emoji/custom/some_folder/file.png")
+        |> to_string()
+
+      assert group == "wildcard folder"
+    end
+
+    test "config with wildcard folder and subfolders", %{groups: groups} do
+      group =
+        groups
+        |> Emoji.match_extra("/emoji/custom/some_folder/another_folder/file.png")
+        |> to_string()
+
+      assert group == "wildcard folder"
+    end
+
+    test "config with wildcard files", %{groups: groups} do
+      group =
+        groups
+        |> Emoji.match_extra("/emoji/custom/folder/some_file.png")
+        |> to_string()
+
+      assert group == "wildcard files"
+    end
+
+    test "config with wildcard files and subfolders", %{groups: groups} do
+      group =
+        groups
+        |> Emoji.match_extra("/emoji/custom/folder/another_folder/some_file.png")
+        |> to_string()
+
+      assert group == "wildcard files"
+    end
+
+    test "config for special file", %{groups: groups} do
+      group =
+        groups
+        |> Emoji.match_extra("/emoji/custom/special.png")
+        |> to_string()
+
+      assert group == "special file"
+    end
+
+    test "no mathing returns nil", %{groups: groups} do
+      group =
+        groups
+        |> Emoji.match_extra("/emoji/some_undefined.png")
+
+      refute group
+    end
+  end
+end

test/formatter_test.exs

@@ -271,7 +271,9 @@ test "it does not add XSS emoji" do
   test "it returns the emoji used in the text" do
     text = "I love :moominmamma:"
 
-    assert Formatter.get_emoji(text) == [{"moominmamma", "/finmoji/128px/moominmamma-128.png"}]
+    assert Formatter.get_emoji(text) == [
+             {"moominmamma", "/finmoji/128px/moominmamma-128.png", "Finmoji"}
+           ]
   end
 
   test "it returns a nice empty result when no emojis are present" do

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -2342,6 +2342,22 @@ test "accounts fetches correct account for nicknames beginning with numbers", %{
     assert acc_two == acc_three
   end
 
+  describe "custom emoji" do
+    test "with tags", %{conn: conn} do
+      [emoji | _body] =
+        conn
+        |> get("/api/v1/custom_emojis")
+        |> json_response(200)
+
+      assert Map.has_key?(emoji, "shortcode")
+      assert Map.has_key?(emoji, "static_url")
+      assert Map.has_key?(emoji, "tags")
+      assert is_list(emoji["tags"])
+      assert Map.has_key?(emoji, "url")
+      assert Map.has_key?(emoji, "visible_in_picker")
+    end
+  end
+
   describe "index/2 redirections" do
     setup %{conn: conn} do
       session_opts = [

test/web/twitter_api/util_controller_test.exs

@@ -170,6 +170,27 @@ test "returns everything in :pleroma, :frontend_configurations", %{conn: conn} d
     end
   end
 
+  describe "/api/pleroma/emoji" do
+    test "returns json with custom emoji with tags", %{conn: conn} do
+      [emoji | _body] =
+        conn
+        |> get("/api/pleroma/emoji")
+        |> json_response(200)
+
+      [key] = Map.keys(emoji)
+
+      %{
+        ^key => %{
+          "image_url" => url,
+          "tags" => tags
+        }
+      } = emoji
+
+      assert is_binary(url)
+      assert is_list(tags)
+    end
+  end
+
   describe "GET /ostatus_subscribe?acct=...." do
     test "adds status to pleroma instance if the `acct` is a status", %{conn: conn} do
       conn =