[#534] Merged `upstream/develop`.

README.md

@@ -12,6 +12,7 @@ Client applications that are known to work well:
 
 * Twidere
 * Tusky
+* Mastalab
 * Pawoo (Android + iOS)
 * Subway Tooter
 * Amaroq (iOS)

config/config.exs

@@ -209,6 +209,8 @@
   ip: {0, 0, 0, 0},
   port: 9999
 
+config :pleroma, Pleroma.Web.Metadata, providers: [], unfurl_nsfw: false
+
 config :pleroma, :suggestions,
   enabled: false,
   third_party_engine:

docs/Pleroma-API.md

@@ -15,6 +15,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Params: none
 * Response: JSON
 * Example response: `{"kalsarikannit_f":"/finmoji/128px/kalsarikannit_f-128.png","perkele":"/finmoji/128px/perkele-128.png","blobdab":"/emoji/blobdab.png","happiness":"/finmoji/128px/happiness-128.png"}`
+* Note: Same data as Mastodon API’s `/api/v1/custom_emojis` but in a different format
 
 ## `/api/pleroma/follow_import`
 ### Imports your follows, for example from a Mastodon CSV file.

docs/config.md

@@ -212,3 +212,9 @@ curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerando
 * `max_jobs`: The maximum amount of parallel federation jobs running at the same time.
 * `initial_timeout`: The initial timeout in seconds
 * `max_retries`: The maximum number of times a federation job is retried
+
+## Pleroma.Web.Metadata
+* `providers`: a list of metadata providers to enable. Providers availible:
+  * Pleroma.Web.Metadata.Providers.OpenGraph
+  * Pleroma.Web.Metadata.Providers.TwitterCard
+* `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews

installation/init.d/pleroma

@@ -12,7 +12,7 @@ export PORT=4000
 export MIX_ENV=prod
 
 # Ask process to terminate within 30 seconds, otherwise kill it
-retry="SIGTERM/30 SIGKILL/5"
+retry="SIGTERM/30/SIGKILL/5"
 
 pidfile="/var/run/pleroma.pid"
 

lib/pleroma/PasswordResetToken.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.PasswordResetToken do
   alias Pleroma.{User, PasswordResetToken, Repo}
 
   schema "password_reset_tokens" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:token, :string)
     field(:used, :boolean, default: false)
 

lib/pleroma/activity.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Activity do
   import Ecto.Query
 
   @type t :: %__MODULE__{}
+  @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
 
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
   @mastodon_notification_types %{

lib/pleroma/application.ex

@@ -99,6 +99,7 @@ def start(_type, _args) do
           ],
           id: :cachex_idem
         ),
+        worker(Pleroma.FlakeId, []),
         worker(Pleroma.Web.Federator.RetryQueue, []),
         worker(Pleroma.Web.Federator, []),
         worker(Pleroma.Stats, []),

lib/pleroma/clippy.ex

@@ -0,0 +1,155 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Clippy do
+  @moduledoc false
+  # No software is complete until they have a Clippy implementation.
+  # A ballmer peak _may_ be required to change this module.
+
+  def tip() do
+    tips()
+    |> Enum.random()
+    |> puts()
+  end
+
+  def tips() do
+    host = Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
+
+    [
+      "“πλήρωμα” is “pleroma” in greek",
+      "For an extended Pleroma Clippy Experience, use the “Redmond” themes in Pleroma FE settings",
+      "Staff accounts and MRF policies of Pleroma instances are disclosed on the NodeInfo endpoints for easy transparency!\n
+- https://catgirl.science/misc/nodeinfo.lua?#{host}
+- https://fediverse.network/#{host}/federation",
+      "Pleroma can federate to the Dark Web!\n
+- Tor: https://git.pleroma.social/pleroma/pleroma/wikis/Easy%20Onion%20Federation%20(Tor)
+- i2p: https://git.pleroma.social/pleroma/pleroma/wikis/I2p%20federation",
+      "Lists of Pleroma instances:\n\n- http://distsn.org/pleroma-instances.html\n- https://fediverse.network/pleroma\n- https://the-federation.info/pleroma",
+      "Pleroma uses the LitePub protocol - https://litepub.social",
+      "To receive more federated posts, subscribe to relays!\n
+- How-to: https://git.pleroma.social/pleroma/pleroma/wikis/Admin%20tasks#relay-managment
+- Relays: https://fediverse.network/activityrelay"
+    ]
+  end
+
+  @spec puts(String.t() | [[IO.ANSI.ansicode() | String.t(), ...], ...]) :: nil
+  def puts(text_or_lines) do
+    import IO.ANSI
+
+    lines =
+      if is_binary(text_or_lines) do
+        String.split(text_or_lines, ~r/\n/)
+      else
+        text_or_lines
+      end
+
+    longest_line_size =
+      lines
+      |> Enum.map(&charlist_count_text/1)
+      |> Enum.sort(&>=/2)
+      |> List.first()
+
+    pad_text = longest_line_size
+
+    pad =
+      for(_ <- 1..pad_text, do: "_")
+      |> Enum.join("")
+
+    pad_spaces =
+      for(_ <- 1..pad_text, do: " ")
+      |> Enum.join("")
+
+    spaces = "      "
+
+    pre_lines = [
+      "  /  \\#{spaces}  _#{pad}___",
+      "  |  |#{spaces} / #{pad_spaces}   \\"
+    ]
+
+    for l <- pre_lines do
+      IO.puts(l)
+    end
+
+    clippy_lines = [
+      "  #{bright()}@  @#{reset()}#{spaces} ",
+      "  || ||#{spaces}",
+      "  || ||   <--",
+      "  |\\_/|      ",
+      "  \\___/      "
+    ]
+
+    noclippy_line = "             "
+
+    env = %{
+      max_size: pad_text,
+      pad: pad,
+      pad_spaces: pad_spaces,
+      spaces: spaces,
+      pre_lines: pre_lines,
+      noclippy_line: noclippy_line
+    }
+
+    # surrond one/five line clippy with blank lines around to not fuck up the layout
+    #
+    # yes this fix sucks but it's good enough, have you ever seen a release of windows wihtout some butched
+    # features anyway?
+    lines =
+      if length(lines) == 1 or length(lines) == 5 do
+        [""] ++ lines ++ [""]
+      else
+        lines
+      end
+
+    clippy_line(lines, clippy_lines, env)
+  rescue
+    e ->
+      IO.puts("(Clippy crashed, sorry: #{inspect(e)})")
+      IO.puts(text_or_lines)
+  end
+
+  defp clippy_line([line | lines], [prefix | clippy_lines], env) do
+    IO.puts([prefix <> "| ", rpad_line(line, env.max_size)])
+    clippy_line(lines, clippy_lines, env)
+  end
+
+  # more text lines but clippy's complete
+  defp clippy_line([line | lines], [], env) do
+    IO.puts([env.noclippy_line, "| ", rpad_line(line, env.max_size)])
+
+    if lines == [] do
+      IO.puts(env.noclippy_line <> "\\_#{env.pad}___/")
+    end
+
+    clippy_line(lines, [], env)
+  end
+
+  # no more text lines but clippy's not complete
+  defp clippy_line([], [clippy | clippy_lines], env) do
+    if env.pad do
+      IO.puts(clippy <> "\\_#{env.pad}___/")
+      clippy_line([], clippy_lines, %{env | pad: nil})
+    else
+      IO.puts(clippy)
+      clippy_line([], clippy_lines, env)
+    end
+  end
+
+  defp clippy_line(_, _, _) do
+  end
+
+  defp rpad_line(line, max) do
+    pad = max - (charlist_count_text(line) - 2)
+    pads = Enum.join(for(_ <- 1..pad, do: " "))
+    [IO.ANSI.format(line), pads <> " |"]
+  end
+
+  defp charlist_count_text(line) do
+    if is_list(line) do
+      text = Enum.join(Enum.filter(line, &is_binary/1))
+      String.length(text)
+    else
+      String.length(line)
+    end
+  end
+end

lib/pleroma/filter.ex

@@ -8,7 +8,7 @@ defmodule Pleroma.Filter do
   alias Pleroma.{User, Repo}
 
   schema "filters" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:filter_id, :integer)
     field(:hide, :boolean, default: false)
     field(:whole_word, :boolean, default: true)

lib/pleroma/flake_id.ex

@@ -0,0 +1,172 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.FlakeId do
+  @moduledoc """
+  Flake is a decentralized, k-ordered id generation service.
+
+  Adapted from:
+
+  * [flaky](https://github.com/nirvana/flaky), released under the terms of the Truly Free License,
+  * [Flake](https://github.com/boundary/flake), Copyright 2012, Boundary, Apache License, Version 2.0
+  """
+
+  @type t :: binary
+
+  @behaviour Ecto.Type
+  use GenServer
+  require Logger
+  alias __MODULE__
+  import Kernel, except: [to_string: 1]
+
+  defstruct node: nil, time: 0, sq: 0
+
+  @doc "Converts a binary Flake to a String"
+  def to_string(<<0::integer-size(64), id::integer-size(64)>>) do
+    Kernel.to_string(id)
+  end
+
+  def to_string(flake = <<_::integer-size(64), _::integer-size(48), _::integer-size(16)>>) do
+    encode_base62(flake)
+  end
+
+  def to_string(s), do: s
+
+  def from_string(int) when is_integer(int) do
+    from_string(Kernel.to_string(int))
+  end
+
+  for i <- [-1, 0] do
+    def from_string(unquote(i)), do: <<0::integer-size(128)>>
+    def from_string(unquote(Kernel.to_string(i))), do: <<0::integer-size(128)>>
+  end
+
+  def from_string(flake = <<_::integer-size(128)>>), do: flake
+
+  def from_string(string) when is_binary(string) and byte_size(string) < 18 do
+    case Integer.parse(string) do
+      {id, _} -> <<0::integer-size(64), id::integer-size(64)>>
+      _ -> nil
+    end
+  end
+
+  def from_string(string) do
+    string |> decode_base62 |> from_integer
+  end
+
+  def to_integer(<<integer::integer-size(128)>>), do: integer
+
+  def from_integer(integer) do
+    <<_time::integer-size(64), _node::integer-size(48), _seq::integer-size(16)>> =
+      <<integer::integer-size(128)>>
+  end
+
+  @doc "Generates a Flake"
+  @spec get :: binary
+  def get, do: to_string(:gen_server.call(:flake, :get))
+
+  # -- Ecto.Type API
+  @impl Ecto.Type
+  def type, do: :uuid
+
+  @impl Ecto.Type
+  def cast(value) do
+    {:ok, FlakeId.to_string(value)}
+  end
+
+  @impl Ecto.Type
+  def load(value) do
+    {:ok, FlakeId.to_string(value)}
+  end
+
+  @impl Ecto.Type
+  def dump(value) do
+    {:ok, FlakeId.from_string(value)}
+  end
+
+  def autogenerate(), do: get()
+
+  # -- GenServer API
+  def start_link do
+    :gen_server.start_link({:local, :flake}, __MODULE__, [], [])
+  end
+
+  @impl GenServer
+  def init([]) do
+    {:ok, %FlakeId{node: worker_id(), time: time()}}
+  end
+
+  @impl GenServer
+  def handle_call(:get, _from, state) do
+    {flake, new_state} = get(time(), state)
+    {:reply, flake, new_state}
+  end
+
+  # Matches when the calling time is the same as the state time. Incr. sq
+  defp get(time, %FlakeId{time: time, node: node, sq: seq}) do
+    new_state = %FlakeId{time: time, node: node, sq: seq + 1}
+    {gen_flake(new_state), new_state}
+  end
+
+  # Matches when the times are different, reset sq
+  defp get(newtime, %FlakeId{time: time, node: node}) when newtime > time do
+    new_state = %FlakeId{time: newtime, node: node, sq: 0}
+    {gen_flake(new_state), new_state}
+  end
+
+  # Error when clock is running backwards
+  defp get(newtime, %FlakeId{time: time}) when newtime < time do
+    {:error, :clock_running_backwards}
+  end
+
+  defp gen_flake(%FlakeId{time: time, node: node, sq: seq}) do
+    <<time::integer-size(64), node::integer-size(48), seq::integer-size(16)>>
+  end
+
+  defp nthchar_base62(n) when n <= 9, do: ?0 + n
+  defp nthchar_base62(n) when n <= 35, do: ?A + n - 10
+  defp nthchar_base62(n), do: ?a + n - 36
+
+  defp encode_base62(<<integer::integer-size(128)>>) do
+    integer
+    |> encode_base62([])
+    |> List.to_string()
+  end
+
+  defp encode_base62(int, acc) when int < 0, do: encode_base62(-int, acc)
+  defp encode_base62(int, []) when int == 0, do: '0'
+  defp encode_base62(int, acc) when int == 0, do: acc
+
+  defp encode_base62(int, acc) do
+    r = rem(int, 62)
+    id = div(int, 62)
+    acc = [nthchar_base62(r) | acc]
+    encode_base62(id, acc)
+  end
+
+  defp decode_base62(s) do
+    decode_base62(String.to_charlist(s), 0)
+  end
+
+  defp decode_base62([c | cs], acc) when c >= ?0 and c <= ?9,
+    do: decode_base62(cs, 62 * acc + (c - ?0))
+
+  defp decode_base62([c | cs], acc) when c >= ?A and c <= ?Z,
+    do: decode_base62(cs, 62 * acc + (c - ?A + 10))
+
+  defp decode_base62([c | cs], acc) when c >= ?a and c <= ?z,
+    do: decode_base62(cs, 62 * acc + (c - ?a + 36))
+
+  defp decode_base62([], acc), do: acc
+
+  defp time do
+    {mega_seconds, seconds, micro_seconds} = :erlang.timestamp()
+    1_000_000_000 * mega_seconds + seconds * 1000 + :erlang.trunc(micro_seconds / 1000)
+  end
+
+  defp worker_id() do
+    <<worker::integer-size(48)>> = :crypto.strong_rand_bytes(6)
+    worker
+  end
+end

lib/pleroma/formatter.ex

@@ -43,7 +43,7 @@ def emojify(text) do
 
   def emojify(text, nil), do: text
 
-  def emojify(text, emoji) do
+  def emojify(text, emoji, strip \\ false) do
     Enum.reduce(emoji, text, fn {emoji, file}, text ->
       emoji = HTML.strip_tags(emoji)
       file = HTML.strip_tags(file)
@@ -51,14 +51,24 @@ def emojify(text, emoji) do
       String.replace(
         text,
         ":#{emoji}:",
-        "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
+        if not strip do
-          MediaProxy.url(file)
+          "<img height='32px' width='32px' alt='#{emoji}' title='#{emoji}' src='#{
-        }' />"
+            MediaProxy.url(file)
+          }' />"
+        else
+          ""
+        end
       )
       |> HTML.filter_tags()
     end)
   end
 
+  def demojify(text) do
+    emojify(text, Emoji.get_all(), true)
+  end
+
+  def demojify(text, nil), do: text
+
   def get_emoji(text) when is_binary(text) do
     Enum.filter(Emoji.get_all(), fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end)
   end
@@ -189,4 +199,16 @@ def finalize({subs, text}) do
       String.replace(result_text, uuid, replacement)
     end)
   end
+
+  def truncate(text, max_length \\ 200, omission \\ "...") do
+    # Remove trailing whitespace
+    text = Regex.replace(~r/([^ \t\r\n])([ \t]+$)/u, text, "\\g{1}")
+
+    if String.length(text) < max_length do
+      text
+    else
+      length_with_omission = max_length - String.length(omission)
+      String.slice(text, 0, length_with_omission) <> omission
+    end
+  end
 end

lib/pleroma/html.ex

@@ -58,6 +58,20 @@ defp generate_scrubber_signature(scrubbers) do
       "#{signature}#{to_string(scrubber)}"
     end)
   end
+
+  def extract_first_external_url(object, content) do
+    key = "URL|#{object.id}"
+
+    Cachex.fetch!(:scrubber_cache, key, fn _key ->
+      result =
+        content
+        |> Floki.filter_out("a.mention")
+        |> Floki.attribute("a", "href")
+        |> Enum.at(0)
+
+      {:commit, result}
+    end)
+  end
 end
 
 defmodule Pleroma.HTML.Scrubber.TwitterText do

lib/pleroma/list.ex

@@ -8,7 +8,7 @@ defmodule Pleroma.List do
   alias Pleroma.{User, Repo, Activity}
 
   schema "lists" do
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     field(:title, :string)
     field(:following, {:array, :string}, default: [])
 

lib/pleroma/notification.ex

@@ -4,13 +4,14 @@
 
 defmodule Pleroma.Notification do
   use Ecto.Schema
-  alias Pleroma.{User, Activity, Notification, Repo, Object}
+  alias Pleroma.{User, Activity, Notification, Repo}
+  alias Pleroma.Web.CommonAPI.Utils
   import Ecto.Query
 
   schema "notifications" do
     field(:seen, :boolean, default: false)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
-    belongs_to(:activity, Pleroma.Activity)
+    belongs_to(:activity, Activity, type: Pleroma.FlakeId)
 
     timestamps()
   end
@@ -34,7 +35,8 @@ def for_user(user, opts \\ %{}) do
         n in Notification,
         where: n.user_id == ^user.id,
         order_by: [desc: n.id],
-        preload: [:activity],
+        join: activity in assoc(n, :activity),
+        preload: [activity: activity],
         limit: 20
       )
 
@@ -65,7 +67,8 @@ def get(%{id: user_id} = _user, id) do
       from(
         n in Notification,
         where: n.id == ^id,
-        preload: [:activity]
+        join: activity in assoc(n, :activity),
+        preload: [activity: activity]
       )
 
     notification = Repo.one(query)
@@ -96,7 +99,7 @@ def dismiss(%{id: user_id} = _user, id) do
     end
   end
 
-  def create_notifications(%Activity{id: _, data: %{"to" => _, "type" => type}} = activity)
+  def create_notifications(%Activity{data: %{"to" => _, "type" => type}} = activity)
       when type in ["Create", "Like", "Announce", "Follow"] do
     users = get_notified_from_activity(activity)
 
@@ -132,54 +135,12 @@ def get_notified_from_activity(
       when type in ["Create", "Like", "Announce", "Follow"] do
     recipients =
       []
-      |> maybe_notify_to_recipients(activity)
+      |> Utils.maybe_notify_to_recipients(activity)
-      |> maybe_notify_mentioned_recipients(activity)
+      |> Utils.maybe_notify_mentioned_recipients(activity)
       |> Enum.uniq()
 
     User.get_users_from_set(recipients, local_only)
   end
 
   def get_notified_from_activity(_, _local_only), do: []
-
-  defp maybe_notify_to_recipients(
-         recipients,
-         %Activity{data: %{"to" => to, "type" => _type}} = _activity
-       ) do
-    recipients ++ to
-  end
-
-  defp maybe_notify_mentioned_recipients(
-         recipients,
-         %Activity{data: %{"to" => _to, "type" => type} = data} = _activity
-       )
-       when type == "Create" do
-    object = Object.normalize(data["object"])
-
-    object_data =
-      cond do
-        !is_nil(object) ->
-          object.data
-
-        is_map(data["object"]) ->
-          data["object"]
-
-        true ->
-          %{}
-      end
-
-    tagged_mentions = maybe_extract_mentions(object_data)
-
-    recipients ++ tagged_mentions
-  end
-
-  defp maybe_notify_mentioned_recipients(recipients, _), do: recipients
-
-  defp maybe_extract_mentions(%{"tag" => tag}) do
-    tag
-    |> Enum.filter(fn x -> is_map(x) end)
-    |> Enum.filter(fn x -> x["type"] == "Mention" end)
-    |> Enum.map(fn x -> x["href"] end)
-  end
-
-  defp maybe_extract_mentions(_), do: []
 end

lib/pleroma/plugs/oauth_plug.ex

@@ -33,7 +33,12 @@ def call(conn, _) do
   #
   @spec fetch_user_and_token(String.t()) :: {:ok, User.t(), Token.t()} | nil
   defp fetch_user_and_token(token) do
-    query = from(q in Token, where: q.token == ^token, preload: [:user])
+    query =
+      from(t in Token,
+        where: t.token == ^token,
+        join: user in assoc(t, :user),
+        preload: [user: user]
+      )
 
     with %Token{user: %{info: %{deactivated: false} = _} = user} = token_record <- Repo.one(query) do
       {:ok, user, token_record}

lib/pleroma/user.ex

@@ -17,6 +17,8 @@ defmodule Pleroma.User do
 
   @type t :: %__MODULE__{}
 
+  @primary_key {:id, Pleroma.FlakeId, autogenerate: true}
+
   @email_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$/
 
   @strict_local_nickname_regex ~r/^[a-zA-Z\d]+$/
@@ -404,6 +406,10 @@ def locked?(%User{} = user) do
     user.info.locked || false
   end
 
+  def get_by_id(id) do
+    Repo.get_by(User, id: id)
+  end
+
   def get_by_ap_id(ap_id) do
     Repo.get_by(User, ap_id: ap_id)
   end
@@ -439,11 +445,33 @@ def get_cached_by_ap_id(ap_id) do
     Cachex.fetch!(:user_cache, key, fn _ -> get_by_ap_id(ap_id) end)
   end
 
+  def get_cached_by_id(id) do
+    key = "id:#{id}"
+
+    ap_id =
+      Cachex.fetch!(:user_cache, key, fn _ ->
+        user = get_by_id(id)
+
+        if user do
+          Cachex.put(:user_cache, "ap_id:#{user.ap_id}", user)
+          {:commit, user.ap_id}
+        else
+          {:ignore, ""}
+        end
+      end)
+
+    get_cached_by_ap_id(ap_id)
+  end
+
   def get_cached_by_nickname(nickname) do
     key = "nickname:#{nickname}"
     Cachex.fetch!(:user_cache, key, fn _ -> get_or_fetch_by_nickname(nickname) end)
   end
 
+  def get_cached_by_nickname_or_id(nickname_or_id) do
+    get_cached_by_id(nickname_or_id) || get_cached_by_nickname(nickname_or_id)
+  end
+
   def get_by_nickname(nickname) do
     Repo.get_by(User, nickname: nickname) ||
       if Regex.match?(~r(@#{Pleroma.Web.Endpoint.host()})i, nickname) do

lib/pleroma/user/info.ex

@@ -31,7 +31,7 @@ defmodule Pleroma.User.Info do
     field(:hub, :string, default: nil)
     field(:salmon, :string, default: nil)
     field(:hide_network, :boolean, default: false)
-    field(:pinned_activities, {:array, :integer}, default: [])
+    field(:pinned_activities, {:array, :string}, default: [])
 
     # Found in the wild
     # ap_id -> Where is this used?

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -36,6 +36,14 @@ defp get_recipients(%{"type" => "Announce"} = data) do
     {recipients, to, cc}
   end
 
+  defp get_recipients(%{"type" => "Create"} = data) do
+    to = data["to"] || []
+    cc = data["cc"] || []
+    actor = data["actor"] || []
+    recipients = (to ++ cc ++ [actor]) |> Enum.uniq()
+    {recipients, to, cc}
+  end
+
   defp get_recipients(data) do
     to = data["to"] || []
     cc = data["cc"] || []
@@ -56,7 +64,7 @@ defp check_actor_is_active(actor) do
     end
   end
 
-  defp check_remote_limit(%{"object" => %{"content" => content}}) do
+  defp check_remote_limit(%{"object" => %{"content" => content}}) when not is_nil(content) do
     limit = Pleroma.Config.get([:instance, :remote_limit])
     String.length(content) <= limit
   end
@@ -410,13 +418,42 @@ def fetch_user_activities(user, reading_user, params \\ %{}) do
     |> Enum.reverse()
   end
 
+  defp restrict_since(query, %{"since_id" => ""}), do: query
+
   defp restrict_since(query, %{"since_id" => since_id}) do
     from(activity in query, where: activity.id > ^since_id)
   end
 
   defp restrict_since(query, _), do: query
 
-  defp restrict_tag(query, %{"tag" => tag}) do
+  defp restrict_tag_reject(query, %{"tag_reject" => tag_reject})
+       when is_list(tag_reject) and tag_reject != [] do
+    from(
+      activity in query,
+      where: fragment("(not (? #> '{\"object\",\"tag\"}') \\?| ?)", activity.data, ^tag_reject)
+    )
+  end
+
+  defp restrict_tag_reject(query, _), do: query
+
+  defp restrict_tag_all(query, %{"tag_all" => tag_all})
+       when is_list(tag_all) and tag_all != [] do
+    from(
+      activity in query,
+      where: fragment("(? #> '{\"object\",\"tag\"}') \\?& ?", activity.data, ^tag_all)
+    )
+  end
+
+  defp restrict_tag_all(query, _), do: query
+
+  defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do
+    from(
+      activity in query,
+      where: fragment("(? #> '{\"object\",\"tag\"}') \\?| ?", activity.data, ^tag)
+    )
+  end
+
+  defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do
     from(
       activity in query,
       where: fragment("? <@ (? #> '{\"object\",\"tag\"}')", ^tag, activity.data)
@@ -465,6 +502,8 @@ defp restrict_local(query, %{"local_only" => true}) do
 
   defp restrict_local(query, _), do: query
 
+  defp restrict_max(query, %{"max_id" => ""}), do: query
+
   defp restrict_max(query, %{"max_id" => max_id}) do
     from(activity in query, where: activity.id < ^max_id)
   end
@@ -563,6 +602,8 @@ def fetch_activities_query(recipients, opts \\ %{}) do
     base_query
     |> restrict_recipients(recipients, opts["user"])
     |> restrict_tag(opts)
+    |> restrict_tag_reject(opts)
+    |> restrict_tag_all(opts)
     |> restrict_since(opts)
     |> restrict_local(opts)
     |> restrict_limit(opts)

lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex

@@ -0,0 +1,57 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
+  alias Pleroma.User
+
+  @behaviour Pleroma.Web.ActivityPub.MRF
+
+  # XXX: this should become User.normalize_by_ap_id() or similar, really.
+  defp normalize_by_ap_id(%{"id" => id}), do: User.get_cached_by_ap_id(id)
+  defp normalize_by_ap_id(uri) when is_binary(uri), do: User.get_cached_by_ap_id(uri)
+  defp normalize_by_ap_id(_), do: nil
+
+  defp score_nickname("followbot@" <> _), do: 1.0
+  defp score_nickname("federationbot@" <> _), do: 1.0
+  defp score_nickname("federation_bot@" <> _), do: 1.0
+  defp score_nickname(_), do: 0.0
+
+  defp score_displayname("federation bot"), do: 1.0
+  defp score_displayname("federationbot"), do: 1.0
+  defp score_displayname("fedibot"), do: 1.0
+  defp score_displayname(_), do: 0.0
+
+  defp determine_if_followbot(%User{nickname: nickname, name: displayname}) do
+    nick_score =
+      nickname
+      |> String.downcase()
+      |> score_nickname()
+
+    name_score =
+      displayname
+      |> String.downcase()
+      |> score_displayname()
+
+    nick_score + name_score
+  end
+
+  defp determine_if_followbot(_), do: 0.0
+
+  @impl true
+  def filter(%{"type" => "Follow", "actor" => actor_id} = message) do
+    %User{} = actor = normalize_by_ap_id(actor_id)
+
+    score = determine_if_followbot(actor)
+
+    # TODO: scan biography data for keywords and score it somehow.
+    if score < 0.8 do
+      {:ok, message}
+    else
+      {:reject, nil}
+    end
+  end
+
+  @impl true
+  def filter(message), do: {:ok, message}
+end

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -141,11 +141,11 @@ def fix_actor(%{"attributedTo" => actor} = object) do
     |> Map.put("actor", get_actor(%{"actor" => actor}))
   end
 
-  def fix_likes(%{"likes" => likes} = object)
+  # Check for standardisation
-      when is_bitstring(likes) do
+  # This is what Peertube does
-    # Check for standardisation
+  # curl -H 'Accept: application/activity+json' $likes | jq .totalItems
-    # This is what Peertube does
+  # Prismo returns only an integer (count) as "likes"
-    # curl -H 'Accept: application/activity+json' $likes | jq .totalItems
+  def fix_likes(%{"likes" => likes} = object) when not is_map(likes) do
     object
     |> Map.put("likes", [])
     |> Map.put("like_count", 0)
@@ -900,15 +900,10 @@ defp user_upgrade_task(user) do
 
     maybe_retire_websub(user.ap_id)
 
-    # Only do this for recent activties, don't go through the whole db.
-    # Only look at the last 1000 activities.
-    since = (Repo.aggregate(Activity, :max, :id) || 0) - 1_000
-
     q =
       from(
         a in Activity,
         where: ^old_follower_address in a.recipients,
-        where: a.id > ^since,
         update: [
           set: [
             recipients:

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -160,7 +160,7 @@ def render("outbox.json", %{user: user, max_id: max_qid}) do
       "partOf" => iri,
       "totalItems" => info.note_count,
       "orderedItems" => collection,
-      "next" => "#{iri}?max_id=#{min_id - 1}"
+      "next" => "#{iri}?max_id=#{min_id}"
     }
 
     if max_qid == nil do
@@ -207,7 +207,7 @@ def render("inbox.json", %{user: user, max_id: max_qid}) do
       "partOf" => iri,
       "totalItems" => -1,
       "orderedItems" => collection,
-      "next" => "#{iri}?max_id=#{min_id - 1}"
+      "next" => "#{iri}?max_id=#{min_id}"
     }
 
     if max_qid == nil do

lib/pleroma/web/common_api/utils.ex

@@ -261,4 +261,46 @@ def emoji_from_profile(%{info: _info} = user) do
       }
     end)
   end
+
+  def maybe_notify_to_recipients(
+        recipients,
+        %Activity{data: %{"to" => to, "type" => _type}} = _activity
+      ) do
+    recipients ++ to
+  end
+
+  def maybe_notify_mentioned_recipients(
+        recipients,
+        %Activity{data: %{"to" => _to, "type" => type} = data} = _activity
+      )
+      when type == "Create" do
+    object = Object.normalize(data["object"])
+
+    object_data =
+      cond do
+        !is_nil(object) ->
+          object.data
+
+        is_map(data["object"]) ->
+          data["object"]
+
+        true ->
+          %{}
+      end
+
+    tagged_mentions = maybe_extract_mentions(object_data)
+
+    recipients ++ tagged_mentions
+  end
+
+  def maybe_notify_mentioned_recipients(recipients, _), do: recipients
+
+  def maybe_extract_mentions(%{"tag" => tag}) do
+    tag
+    |> Enum.filter(fn x -> is_map(x) end)
+    |> Enum.filter(fn x -> x["type"] == "Mention" end)
+    |> Enum.map(fn x -> x["href"] end)
+  end
+
+  def maybe_extract_mentions(_), do: []
 end

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   use Pleroma.Web, :controller
   alias Pleroma.{Repo, Object, Activity, User, Notification, Stats}
   alias Pleroma.Web
+  alias Pleroma.HTML
 
   alias Pleroma.Web.MastodonAPI.{
     StatusView,
@@ -540,15 +541,34 @@ def reblogged_by(conn, %{"id" => id}) do
   def hashtag_timeline(%{assigns: %{user: user}} = conn, params) do
     local_only = params["local"] in [true, "True", "true", "1"]
 
-    params =
+    tags =
+      [params["tag"], params["any"]]
+      |> List.flatten()
+      |> Enum.uniq()
+      |> Enum.filter(& &1)
+      |> Enum.map(&String.downcase(&1))
+
+    tag_all =
+      params["all"] ||
+        []
+        |> Enum.map(&String.downcase(&1))
+
+    tag_reject =
+      params["none"] ||
+        []
+        |> Enum.map(&String.downcase(&1))
+
+    query_params =
       params
       |> Map.put("type", "Create")
       |> Map.put("local_only", local_only)
       |> Map.put("blocking_user", user)
-      |> Map.put("tag", String.downcase(params["tag"]))
+      |> Map.put("tag", tags)
+      |> Map.put("tag_all", tag_all)
+      |> Map.put("tag_reject", tag_reject)
 
     activities =
-      ActivityPub.fetch_public_activities(params)
+      ActivityPub.fetch_public_activities(query_params)
       |> Enum.reverse()
 
     conn
@@ -1322,6 +1342,29 @@ def suggestions(%{assigns: %{user: user}} = conn, _) do
     end
   end
 
+  def get_status_card(status_id) do
+    with %Activity{} = activity <- Repo.get(Activity, status_id),
+         true <- ActivityPub.is_public?(activity),
+         %Object{} = object <- Object.normalize(activity.data["object"]),
+         page_url <- HTML.extract_first_external_url(object, object.data["content"]),
+         {:ok, rich_media} <- Pleroma.Web.RichMedia.Parser.parse(page_url) do
+      page_url = rich_media[:url] || page_url
+      site_name = rich_media[:site_name] || URI.parse(page_url).host
+
+      rich_media
+      |> Map.take([:image, :title, :description])
+      |> Map.put(:type, "link")
+      |> Map.put(:provider_name, site_name)
+      |> Map.put(:url, page_url)
+    else
+      _ -> %{}
+    end
+  end
+
+  def status_card(conn, %{"id" => status_id}) do
+    json(conn, get_status_card(status_id))
+  end
+
   def try_render(conn, target, params)
       when is_binary(target) do
     res = render(conn, target, params)

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -112,7 +112,9 @@ defp do_render("account.json", %{user: user} = opts) do
       # Pleroma extension
       pleroma: %{
         confirmation_pending: user_info.confirmation_pending,
-        tags: user.tags
+        tags: user.tags,
+        is_moderator: user.info.is_moderator,
+        is_admin: user.info.is_admin
       }
     }
   end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -49,12 +49,11 @@ def render("index.json", opts) do
     replied_to_activities = get_replied_to_activities(opts.activities)
 
     opts.activities
-    |> render_many(
+    |> safe_render_many(
       StatusView,
       "status.json",
       Map.put(opts, :replied_to_activities, replied_to_activities)
     )
-    |> Enum.filter(fn x -> not is_nil(x) end)
   end
 
   def render(

lib/pleroma/web/metadata.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata do
+  alias Phoenix.HTML
+
+  def build_tags(params) do
+    Enum.reduce(Pleroma.Config.get([__MODULE__, :providers], []), "", fn parser, acc ->
+      rendered_html =
+        params
+        |> parser.build_tags()
+        |> Enum.map(&to_tag/1)
+        |> Enum.map(&HTML.safe_to_string/1)
+        |> Enum.join()
+
+      acc <> rendered_html
+    end)
+  end
+
+  def to_tag(data) do
+    with {name, attrs, _content = []} <- data do
+      HTML.Tag.tag(name, attrs)
+    else
+      {name, attrs, content} ->
+        HTML.Tag.content_tag(name, content, attrs)
+
+      _ ->
+        raise ArgumentError, message: "make_tag invalid args"
+    end
+  end
+
+  def activity_nsfw?(%{data: %{"sensitive" => sensitive}}) do
+    Pleroma.Config.get([__MODULE__, :unfurl_nsfw], false) == false and sensitive
+  end
+
+  def activity_nsfw?(_) do
+    false
+  end
+end

lib/pleroma/web/metadata/opengraph.ex

@@ -0,0 +1,154 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata.Providers.OpenGraph do
+  alias Pleroma.Web.Metadata.Providers.Provider
+  alias Pleroma.Web.Metadata
+  alias Pleroma.{HTML, Formatter, User}
+  alias Pleroma.Web.MediaProxy
+
+  @behaviour Provider
+
+  @impl Provider
+  def build_tags(%{
+        object: object,
+        url: url,
+        user: user
+      }) do
+    attachments = build_attachments(object)
+    scrubbed_content = scrub_html_and_truncate(object)
+    # Zero width space
+    content =
+      if scrubbed_content != "" and scrubbed_content != "\u200B" do
+        ": “" <> scrubbed_content <> "”"
+      else
+        ""
+      end
+
+    # Most previews only show og:title which is inconvenient. Instagram
+    # hacks this by putting the description in the title and making the
+    # description longer prefixed by how many likes and shares the post
+    # has. Here we use the descriptive nickname in the title, and expand
+    # the full account & nickname in the description. We also use the cute^Wevil
+    # smart quotes around the status text like Instagram, too.
+    [
+      {:meta,
+       [
+         property: "og:title",
+         content: "#{user.name}" <> content
+       ], []},
+      {:meta, [property: "og:url", content: url], []},
+      {:meta,
+       [
+         property: "og:description",
+         content: "#{user_name_string(user)}" <> content
+       ], []},
+      {:meta, [property: "og:type", content: "website"], []}
+    ] ++
+      if attachments == [] or Metadata.activity_nsfw?(object) do
+        [
+          {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []},
+          {:meta, [property: "og:image:width", content: 150], []},
+          {:meta, [property: "og:image:height", content: 150], []}
+        ]
+      else
+        attachments
+      end
+  end
+
+  @impl Provider
+  def build_tags(%{user: user}) do
+    with truncated_bio = scrub_html_and_truncate(user.bio || "") do
+      [
+        {:meta,
+         [
+           property: "og:title",
+           content: user_name_string(user)
+         ], []},
+        {:meta, [property: "og:url", content: User.profile_url(user)], []},
+        {:meta, [property: "og:description", content: truncated_bio], []},
+        {:meta, [property: "og:type", content: "website"], []},
+        {:meta, [property: "og:image", content: attachment_url(User.avatar_url(user))], []},
+        {:meta, [property: "og:image:width", content: 150], []},
+        {:meta, [property: "og:image:height", content: 150], []}
+      ]
+    end
+  end
+
+  defp build_attachments(%{data: %{"attachment" => attachments}}) do
+    Enum.reduce(attachments, [], fn attachment, acc ->
+      rendered_tags =
+        Enum.reduce(attachment["url"], [], fn url, acc ->
+          media_type =
+            Enum.find(["image", "audio", "video"], fn media_type ->
+              String.starts_with?(url["mediaType"], media_type)
+            end)
+
+          # TODO: Add additional properties to objects when we have the data available.
+          # Also, Whatsapp only wants JPEG or PNGs. It seems that if we add a second og:image
+          # object when a Video or GIF is attached it will display that in the Whatsapp Rich Preview.
+          case media_type do
+            "audio" ->
+              [
+                {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], []}
+                | acc
+              ]
+
+            "image" ->
+              [
+                {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])],
+                 []},
+                {:meta, [property: "og:image:width", content: 150], []},
+                {:meta, [property: "og:image:height", content: 150], []}
+                | acc
+              ]
+
+            "video" ->
+              [
+                {:meta, [property: "og:" <> media_type, content: attachment_url(url["href"])], []}
+                | acc
+              ]
+
+            _ ->
+              acc
+          end
+        end)
+
+      acc ++ rendered_tags
+    end)
+  end
+
+  defp scrub_html_and_truncate(%{data: %{"content" => content}} = object) do
+    content
+    # html content comes from DB already encoded, decode first and scrub after
+    |> HtmlEntities.decode()
+    |> String.replace(~r/<br\s?\/?>/, " ")
+    |> HTML.get_cached_stripped_html_for_object(object, __MODULE__)
+    |> Formatter.demojify()
+    |> Formatter.truncate()
+  end
+
+  defp scrub_html_and_truncate(content) when is_binary(content) do
+    content
+    # html content comes from DB already encoded, decode first and scrub after
+    |> HtmlEntities.decode()
+    |> String.replace(~r/<br\s?\/?>/, " ")
+    |> HTML.strip_tags()
+    |> Formatter.demojify()
+    |> Formatter.truncate()
+  end
+
+  defp attachment_url(url) do
+    MediaProxy.url(url)
+  end
+
+  defp user_name_string(user) do
+    "#{user.name} " <>
+      if user.local do
+        "(@#{user.nickname}@#{Pleroma.Web.Endpoint.host()})"
+      else
+        "(@#{user.nickname})"
+      end
+  end
+end

lib/pleroma/web/metadata/provider.ex

@@ -0,0 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata.Providers.Provider do
+  @callback build_tags(map()) :: list()
+end

lib/pleroma/web/metadata/twitter_card.ex

@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata.Providers.TwitterCard do
+  alias Pleroma.Web.Metadata.Providers.Provider
+  alias Pleroma.Web.Metadata
+
+  @behaviour Provider
+
+  @impl Provider
+  def build_tags(%{object: object}) do
+    if Metadata.activity_nsfw?(object) or object.data["attachment"] == [] do
+      build_tags(nil)
+    else
+      case find_first_acceptable_media_type(object) do
+        "image" ->
+          [{:meta, [property: "twitter:card", content: "summary_large_image"], []}]
+
+        "audio" ->
+          [{:meta, [property: "twitter:card", content: "player"], []}]
+
+        "video" ->
+          [{:meta, [property: "twitter:card", content: "player"], []}]
+
+        _ ->
+          build_tags(nil)
+      end
+    end
+  end
+
+  @impl Provider
+  def build_tags(_) do
+    [{:meta, [property: "twitter:card", content: "summary"], []}]
+  end
+
+  def find_first_acceptable_media_type(%{data: %{"attachment" => attachment}}) do
+    Enum.find_value(attachment, fn attachment ->
+      Enum.find_value(attachment["url"], fn url ->
+        Enum.find(["image", "audio", "video"], fn media_type ->
+          String.starts_with?(url["mediaType"], media_type)
+        end)
+      end)
+    end)
+  end
+end

lib/pleroma/web/oauth/authorization.ex

@@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
     field(:token, :string)
     field(:valid_until, :naive_datetime)
     field(:used, :boolean, default: false)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()

lib/pleroma/web/oauth/fallback_controller.ex

@@ -9,7 +9,8 @@ defmodule Pleroma.Web.OAuth.FallbackController do
   # No user/password
   def call(conn, _) do
     conn
+    |> put_status(:unauthorized)
     |> put_flash(:error, "Invalid Username/Password")
-    |> OAuthController.authorize(conn.params)
+    |> OAuthController.authorize(conn.params["authorization"])
   end
 end

lib/pleroma/web/oauth/token.ex

@@ -14,7 +14,7 @@ defmodule Pleroma.Web.OAuth.Token do
     field(:token, :string)
     field(:refresh_token, :string)
     field(:valid_until, :naive_datetime)
-    belongs_to(:user, Pleroma.User)
+    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()

lib/pleroma/web/ostatus/ostatus_controller.ex

@@ -7,7 +7,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do
 
   alias Pleroma.{User, Activity, Object}
   alias Pleroma.Web.OStatus.{FeedRepresenter, ActivityRepresenter}
-  alias Pleroma.Repo
   alias Pleroma.Web.{OStatus, Federator}
   alias Pleroma.Web.XML
   alias Pleroma.Web.ActivityPub.ObjectView
@@ -22,7 +21,11 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   def feed_redirect(conn, %{"nickname" => nickname}) do
     case get_format(conn) do
       "html" ->
-        Fallback.RedirectController.redirector(conn, nil)
+        with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do
+          Fallback.RedirectController.redirector_with_meta(conn, %{user: user})
+        else
+          nil -> {:error, :not_found}
+        end
 
       "activity+json" ->
         ActivityPubController.call(conn, :user)
@@ -138,24 +141,40 @@ def activity(conn, %{"uuid" => uuid}) do
   end
 
   def notice(conn, %{"id" => id}) do
-    with {_, %Activity{} = activity} <- {:activity, Repo.get(Activity, id)},
+    with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(id)},
          {_, true} <- {:public?, ActivityPub.is_public?(activity)},
          %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       case format = get_format(conn) do
         "html" ->
-          conn
+          if activity.data["type"] == "Create" do
-          |> put_resp_content_type("text/html")
+            %Object{} = object = Object.normalize(activity.data["object"])
-          |> send_file(200, Pleroma.Plugs.InstanceStatic.file_path("index.html"))
+
+            Fallback.RedirectController.redirector_with_meta(conn, %{
+              object: object,
+              url:
+                Pleroma.Web.Router.Helpers.o_status_url(
+                  Pleroma.Web.Endpoint,
+                  :notice,
+                  activity.id
+                ),
+              user: user
+            })
+          else
+            Fallback.RedirectController.redirector(conn, nil)
+          end
 
         _ ->
           represent_activity(conn, format, activity, user)
       end
     else
       {:public?, false} ->
-        {:error, :not_found}
+        conn
+        |> put_status(404)
+        |> Fallback.RedirectController.redirector(nil, 404)
 
       {:activity, nil} ->
-        {:error, :not_found}
+        conn
+        |> Fallback.RedirectController.redirector(nil, 404)
 
       e ->
         e

lib/pleroma/web/push/subscription.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.Web.Push.Subscription do
   alias Pleroma.Web.Push.Subscription
 
   schema "push_subscriptions" do
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     belongs_to(:token, Token)
     field(:endpoint, :string)
     field(:key_p256dh, :string)

lib/pleroma/web/rich_media/parser.ex

@@ -5,11 +5,19 @@ defmodule Pleroma.Web.RichMedia.Parser do
     Pleroma.Web.RichMedia.Parsers.OEmbed
   ]
 
+  def parse(nil), do: {:error, "No URL provided"}
+
   if Mix.env() == :test do
     def parse(url), do: parse_url(url)
   else
-    def parse(url),
+    def parse(url) do
-      do: Cachex.fetch!(:rich_media_cache, url, fn _ -> parse_url(url) end)
+      with {:ok, data} <- Cachex.fetch(:rich_media_cache, url, fn _ -> parse_url(url) end) do
+        data
+      else
+        _e ->
+          {:error, "Parsing error"}
+      end
+    end
   end
 
   defp parse_url(url) do

lib/pleroma/web/router.ex

@@ -258,7 +258,7 @@ defmodule Pleroma.Web.Router do
 
     get("/statuses/:id", MastodonAPIController, :get_status)
     get("/statuses/:id/context", MastodonAPIController, :get_context)
-    get("/statuses/:id/card", MastodonAPIController, :empty_object)
+    get("/statuses/:id/card", MastodonAPIController, :status_card)
     get("/statuses/:id/favourited_by", MastodonAPIController, :favourited_by)
     get("/statuses/:id/reblogged_by", MastodonAPIController, :reblogged_by)
 
@@ -396,7 +396,11 @@ defmodule Pleroma.Web.Router do
   end
 
   pipeline :ostatus do
-    plug(:accepts, ["xml", "atom", "html", "activity+json"])
+    plug(:accepts, ["html", "xml", "atom", "activity+json"])
+  end
+
+  pipeline :oembed do
+    plug(:accepts, ["json", "xml"])
   end
 
   scope "/", Pleroma.Web do
@@ -414,6 +418,12 @@ defmodule Pleroma.Web.Router do
     post("/push/subscriptions/:id", Websub.WebsubController, :websub_incoming)
   end
 
+  scope "/", Pleroma.Web do
+    pipe_through(:oembed)
+
+    get("/oembed", OEmbed.OEmbedController, :url)
+  end
+
   pipeline :activitypub do
     plug(:accepts, ["activity+json"])
     plug(Pleroma.Web.Plugs.HTTPSignaturePlug)
@@ -501,6 +511,7 @@ defmodule Pleroma.Web.Router do
 
   scope "/", Fallback do
     get("/registration/:token", RedirectController, :registration_page)
+    get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta)
     get("/*path", RedirectController, :redirector)
 
     options("/*path", RedirectController, :empty)
@@ -509,11 +520,36 @@ defmodule Pleroma.Web.Router do
 
 defmodule Fallback.RedirectController do
   use Pleroma.Web, :controller
+  alias Pleroma.Web.Metadata
+  alias Pleroma.User
 
-  def redirector(conn, _params) do
+  def redirector(conn, _params, code \\ 200) do
     conn
     |> put_resp_content_type("text/html")
-    |> send_file(200, Pleroma.Plugs.InstanceStatic.file_path("index.html"))
+    |> send_file(code, index_file_path())
+  end
+
+  def redirector_with_meta(conn, %{"maybe_nickname_or_id" => maybe_nickname_or_id} = params) do
+    with %User{} = user <- User.get_cached_by_nickname_or_id(maybe_nickname_or_id) do
+      redirector_with_meta(conn, %{user: user})
+    else
+      nil ->
+        redirector(conn, params)
+    end
+  end
+
+  def redirector_with_meta(conn, params) do
+    {:ok, index_content} = File.read(index_file_path())
+    tags = Metadata.build_tags(params)
+    response = String.replace(index_content, "<!--server-generated-meta-->", tags)
+
+    conn
+    |> put_resp_content_type("text/html")
+    |> send_resp(200, response)
+  end
+
+  def index_file_path do
+    Pleroma.Plugs.InstanceStatic.file_path("index.html")
   end
 
   def registration_page(conn, params) do

lib/pleroma/web/twitter_api/representers/activity_representer.ex

@@ -158,7 +158,9 @@ def to_map(
     mentions = opts[:mentioned] || []
 
     attentions =
-      activity.recipients
+      []
+      |> Utils.maybe_notify_to_recipients(activity)
+      |> Utils.maybe_notify_mentioned_recipients(activity)
       |> Enum.map(fn ap_id -> Enum.find(mentions, fn user -> ap_id == user.ap_id end) end)
       |> Enum.filter(& &1)
       |> Enum.map(fn user -> UserView.render("show.json", %{user: user, for: opts[:for]}) end)

lib/pleroma/web/twitter_api/twitter_api_controller.ex

@@ -265,8 +265,6 @@ def fetch_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
   end
 
   def fetch_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    id = String.to_integer(id)
-
     with context when is_binary(context) <- TwitterAPI.conversation_id_to_context(id),
          activities <-
            ActivityPub.fetch_activities_for_context(context, %{
@@ -340,44 +338,47 @@ def get_by_id_or_ap_id(id) do
   end
 
   def favorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
+    with {:ok, activity} <- TwitterAPI.fav(user, id) do
-         {:ok, activity} <- TwitterAPI.fav(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def unfavorite(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
+    with {:ok, activity} <- TwitterAPI.unfav(user, id) do
-         {:ok, activity} <- TwitterAPI.unfav(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def retweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
+    with {:ok, activity} <- TwitterAPI.repeat(user, id) do
-         {:ok, activity} <- TwitterAPI.repeat(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def unretweet(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
+    with {:ok, activity} <- TwitterAPI.unrepeat(user, id) do
-         {:ok, activity} <- TwitterAPI.unrepeat(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
+    else
+      _ -> json_reply(conn, 400, Jason.encode!(%{}))
     end
   end
 
   def pin(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
+    with {:ok, activity} <- TwitterAPI.pin(user, id) do
-         {:ok, activity} <- TwitterAPI.pin(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
@@ -388,8 +389,7 @@ def pin(%{assigns: %{user: user}} = conn, %{"id" => id}) do
   end
 
   def unpin(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with {_, {:ok, id}} <- {:param_cast, Ecto.Type.cast(:integer, id)},
+    with {:ok, activity} <- TwitterAPI.unpin(user, id) do
-         {:ok, activity} <- TwitterAPI.unpin(user, id) do
       conn
       |> put_view(ActivityView)
       |> render("activity.json", %{activity: activity, for: user})
@@ -556,7 +556,6 @@ def friend_requests(conn, params) do
 
   def approve_friend_request(conn, %{"user_id" => uid} = _params) do
     with followed <- conn.assigns[:user],
-         uid when is_number(uid) <- String.to_integer(uid),
          %User{} = follower <- Repo.get(User, uid),
          {:ok, follower} <- User.maybe_follow(follower, followed),
          %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
@@ -578,7 +577,6 @@ def approve_friend_request(conn, %{"user_id" => uid} = _params) do
 
   def deny_friend_request(conn, %{"user_id" => uid} = _params) do
     with followed <- conn.assigns[:user],
-         uid when is_number(uid) <- String.to_integer(uid),
          %User{} = follower <- Repo.get(User, uid),
          %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),

lib/pleroma/web/twitter_api/views/activity_view.ex

@@ -114,7 +114,7 @@ def render("index.json", opts) do
       |> Map.put(:context_ids, context_ids)
       |> Map.put(:users, users)
 
-    render_many(
+    safe_render_many(
       opts.activities,
       ActivityView,
       "activity.json",
@@ -236,7 +236,9 @@ def render(
     pinned = activity.id in user.info.pinned_activities
 
     attentions =
-      activity.recipients
+      []
+      |> Utils.maybe_notify_to_recipients(activity)
+      |> Utils.maybe_notify_mentioned_recipients(activity)
       |> Enum.map(fn ap_id -> get_user(ap_id, opts) end)
       |> Enum.filter(& &1)
       |> Enum.map(fn user -> UserView.render("show.json", %{user: user, for: opts[:for]}) end)

lib/pleroma/web/twitter_api/views/user_view.ex

@@ -108,6 +108,7 @@ defp do_render("user.json", %{user: user = %User{}} = assigns) do
       "locked" => user.info.locked,
       "default_scope" => user.info.default_scope,
       "no_rich_text" => user.info.no_rich_text,
+      "hide_network" => user.info.hide_network,
       "fields" => fields,
 
       # Pleroma extension

lib/pleroma/web/web.ex

@@ -38,6 +38,33 @@ def view do
       import Phoenix.Controller, only: [get_csrf_token: 0, get_flash: 2, view_module: 1]
 
       import Pleroma.Web.{ErrorHelpers, Gettext, Router.Helpers}
+
+      require Logger
+
+      @doc "Same as `render/3` but wrapped in a rescue block"
+      def safe_render(view, template, assigns \\ %{}) do
+        Phoenix.View.render(view, template, assigns)
+      rescue
+        error ->
+          Logger.error(
+            "#{__MODULE__} failed to render #{inspect({view, template})}: #{inspect(error)}"
+          )
+
+          Logger.error(inspect(__STACKTRACE__))
+          nil
+      end
+
+      @doc """
+      Same as `render_many/4` but wrapped in rescue block.
+      """
+      def safe_render_many(collection, view, template, assigns \\ %{}) do
+        Enum.map(collection, fn resource ->
+          as = Map.get(assigns, :as) || view.__resource__
+          assigns = Map.put(assigns, as, resource)
+          safe_render(view, template, assigns)
+        end)
+        |> Enum.filter(& &1)
+      end
     end
   end
 

lib/pleroma/web/websub/websub_client_subscription.ex

@@ -13,7 +13,7 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do
     field(:state, :string)
     field(:subscribers, {:array, :string}, default: [])
     field(:hub, :string)
-    belongs_to(:user, User)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
 
     timestamps()
   end

mix.exs

@@ -59,6 +59,7 @@ defp deps do
       {:pbkdf2_elixir, "~> 0.12.3"},
       {:trailing_format_plug, "~> 0.0.7"},
       {:html_sanitize_ex, "~> 1.3.0"},
+      {:html_entities, "~> 0.4"},
       {:phoenix_html, "~> 2.10"},
       {:calendar, "~> 0.17.4"},
       {:cachex, "~> 3.0.2"},

priv/repo/migrations/20181218172826_users_and_activities_flake_id.exs

@@ -0,0 +1,125 @@
+defmodule Pleroma.Repo.Migrations.UsersAndActivitiesFlakeId do
+  use Ecto.Migration
+  alias Pleroma.Clippy
+  require Integer
+  import Ecto.Query
+  alias Pleroma.Repo
+
+  # This migrates from int serial IDs to custom Flake:
+  #   1- create a temporary uuid column
+  #   2- fill this column with compatibility ids (see below)
+  #   3- remove pkeys constraints
+  #   4- update relation pkeys with the new ids
+  #   5- rename the temporary column to id
+  #   6- re-create the constraints
+  def change do
+    # Old serial int ids are transformed to 128bits with extra padding.
+    # The application (in `Pleroma.FlakeId`) handles theses IDs properly as integers; to keep compatibility
+    # with previously issued ids.
+    #execute "update activities set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);"
+    #execute "update users set external_id = CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid);"
+
+    clippy = start_clippy_heartbeats()
+
+    # Lock both tables to avoid a running server to meddling with our transaction
+    execute "LOCK TABLE activities;"
+    execute "LOCK TABLE users;"
+
+    execute """
+      ALTER TABLE activities
+      DROP CONSTRAINT activities_pkey CASCADE,
+      ALTER COLUMN id DROP default,
+      ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid),
+      ADD PRIMARY KEY (id);
+    """
+
+    execute """
+    ALTER TABLE users
+    DROP CONSTRAINT users_pkey CASCADE,
+    ALTER COLUMN id DROP default,
+    ALTER COLUMN id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(id), 32, '0' ) AS uuid),
+    ADD PRIMARY KEY (id);
+    """
+
+    execute "UPDATE users SET info = jsonb_set(info, '{pinned_activities}', array_to_json(ARRAY(select jsonb_array_elements_text(info->'pinned_activities')))::jsonb);"
+
+    # Fkeys:
+    # Activities - Referenced by:
+    #   TABLE "notifications" CONSTRAINT "notifications_activity_id_fkey" FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE
+    # Users - Referenced by:
+    #  TABLE "filters" CONSTRAINT "filters_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    #  TABLE "lists" CONSTRAINT "lists_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    #  TABLE "notifications" CONSTRAINT "notifications_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    #  TABLE "oauth_authorizations" CONSTRAINT "oauth_authorizations_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
+    #  TABLE "oauth_tokens" CONSTRAINT "oauth_tokens_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
+    #  TABLE "password_reset_tokens" CONSTRAINT "password_reset_tokens_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
+    #  TABLE "push_subscriptions" CONSTRAINT "push_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
+    #  TABLE "websub_client_subscriptions" CONSTRAINT "websub_client_subscriptions_user_id_fkey" FOREIGN KEY (user_id) REFERENCES users(id)
+
+    execute """
+    ALTER TABLE notifications
+    ALTER COLUMN activity_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(activity_id), 32, '0' ) AS uuid),
+    ADD CONSTRAINT notifications_activity_id_fkey FOREIGN KEY (activity_id) REFERENCES activities(id) ON DELETE CASCADE;
+    """
+
+    for table <- ~w(notifications filters lists oauth_authorizations oauth_tokens password_reset_tokens push_subscriptions websub_client_subscriptions) do
+      execute """
+      ALTER TABLE #{table}
+      ALTER COLUMN user_id SET DATA TYPE uuid USING CAST( LPAD( TO_HEX(user_id), 32, '0' ) AS uuid),
+      ADD CONSTRAINT #{table}_user_id_fkey FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;
+      """
+    end
+
+    flush()
+
+    stop_clippy_heartbeats(clippy)
+  end
+
+  defp start_clippy_heartbeats() do
+    count = from(a in "activities", select: count(a.id)) |> Repo.one!
+
+    if count > 5000 do
+      heartbeat_interval = :timer.minutes(2) + :timer.seconds(30)
+      all_tips = Clippy.tips() ++ [
+        "The migration is still running, maybe it's time for another “tea”?",
+        "Happy rabbits practice a cute behavior known as a\n“binky:” they jump up in the air\nand twist\nand spin around!",
+        "Nothing and everything.\n\nI still work.",
+        "Pleroma runs on a Raspberry Pi!\n\n  … but this migration will take forever if you\nactually run on a raspberry pi",
+        "Status? Stati? Post? Note? Toot?\nRepeat? Reboost? Boost? Retweet? Retoot??\n\nI-I'm confused.",
+      ]
+
+      heartbeat = fn(heartbeat, runs, all_tips, tips) ->
+        tips = if Integer.is_even(runs) do
+          tips = if tips == [], do: all_tips, else: tips
+          [tip | tips] = Enum.shuffle(tips)
+          Clippy.puts(tip)
+          tips
+        else
+          IO.puts "\n -- #{DateTime.to_string(DateTime.utc_now())} Migration still running, please wait…\n"
+          tips
+        end
+        :timer.sleep(heartbeat_interval)
+        heartbeat.(heartbeat, runs + 1, all_tips, tips)
+      end
+
+      Clippy.puts [
+        [:red, :bright, "It looks like you are running an older instance!"],
+        [""],
+        [:bright, "This migration may take a long time", :reset, " -- so you probably should"],
+        ["go drink a cofe, or a tea, or a beer, a whiskey, a vodka,"],
+        ["while it runs to deal with your temporary fediverse pause!"]
+      ]
+      :timer.sleep(heartbeat_interval)
+      spawn_link(fn() -> heartbeat.(heartbeat, 1, all_tips, []) end)
+    end
+  end
+
+  defp stop_clippy_heartbeats(pid) do
+    if pid do
+      Process.unlink(pid)
+      Process.exit(pid, :kill)
+      Clippy.puts [[:green, :bright, "Hurray!!", "", "", "Migration completed!"]]
+    end
+  end
+
+end

priv/repo/migrations/20190124131141_update_activity_visibility_again.exs

@@ -0,0 +1,37 @@
+defmodule Pleroma.Repo.Migrations.UpdateActivityVisibilityAgain do
+  use Ecto.Migration
+  @disable_ddl_transaction true
+
+  def up do
+    definition = """
+    create or replace function activity_visibility(actor varchar, recipients varchar[], data jsonb) returns varchar as $$
+    DECLARE
+      fa varchar;
+      public varchar := 'https://www.w3.org/ns/activitystreams#Public';
+    BEGIN
+      SELECT COALESCE(users.follower_address, '') into fa from public.users where users.ap_id = actor;
+
+      IF data->'to' ? public THEN
+        RETURN 'public';
+      ELSIF data->'cc' ? public THEN
+        RETURN 'unlisted';
+      ELSIF ARRAY[fa] && recipients THEN
+        RETURN 'private';
+      ELSIF not(ARRAY[fa, public] && recipients) THEN
+        RETURN 'direct';
+      ELSE
+        RETURN 'unknown';
+      END IF;
+    END;
+    $$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE SECURITY DEFINER;
+    """
+
+    execute(definition)
+
+  end
+
+  def down do
+
+  end
+
+end

priv/repo/migrations/20190126160540_change_push_subscriptions_varchar.exs

@@ -0,0 +1,9 @@
+defmodule Pleroma.Repo.Migrations.ChangePushSubscriptionsVarchar do
+  use Ecto.Migration
+
+  def change do
+    alter table(:push_subscriptions) do
+      modify(:endpoint, :varchar)
+    end
+  end
+end

priv/repo/migrations/20190127151220_add_activities_likes_index.exs

@@ -0,0 +1,8 @@
+defmodule Pleroma.Repo.Migrations.AddActivitiesLikesIndex do
+  use Ecto.Migration
+  @disable_ddl_transaction true
+
+  def change do
+    create index(:activities, ["((data #> '{\"object\",\"likes\"}'))"], concurrently: true, name: :activities_likes, using: :gin)
+  end
+end

priv/static/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>Pleroma</title><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link rel=stylesheet href=/static/font/css/fontello.css><link rel=stylesheet href=/static/font/css/animation.css><link href=/static/css/app.3d3e30a9afb8c41739656f496e8c79e6.css rel=stylesheet></head><body style="display: none"><div id=app></div><script type=text/javascript src=/static/js/manifest.e58590e04ca06ebbea1e.js></script><script type=text/javascript src=/static/js/vendor.61fac267296f19262d14.js></script><script type=text/javascript src=/static/js/app.76e23c93f1de5902c4d7.js></script></body></html>
+<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>Pleroma</title><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link rel=stylesheet href=/static/font/css/fontello.css><link rel=stylesheet href=/static/font/css/animation.css><link href=/static/css/app.3d3e30a9afb8c41739656f496e8c79e6.css rel=stylesheet></head><body style="display: none"><div id=app></div><script type=text/javascript src=/static/js/manifest.8dc8d7a1dc85bfdf2b14.js></script><script type=text/javascript src=/static/js/vendor.61fd03d8471aaadcf63c.js></script><script type=text/javascript src=/static/js/app.ddbd2a89e264d04e0d6d.js></script></body></html>

priv/static/static/config.json

@@ -18,5 +18,6 @@
   "hideUserStats": false,
   "loginMethod": "password",
   "webPushNotifications": false,
-  "noAttachmentLinks": false
+  "noAttachmentLinks": false,
+  "nsfwCensorImage": ""
 }

test/flake_id_test.exs

@@ -0,0 +1,42 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.FlakeIdTest do
+  use Pleroma.DataCase
+  import Kernel, except: [to_string: 1]
+  import Pleroma.FlakeId
+
+  describe "fake flakes (compatibility with older serial integers)" do
+    test "from_string/1" do
+      fake_flake = <<0::integer-size(64), 42::integer-size(64)>>
+      assert from_string("42") == fake_flake
+      assert from_string(42) == fake_flake
+    end
+
+    test "zero or -1 is a null flake" do
+      fake_flake = <<0::integer-size(128)>>
+      assert from_string("0") == fake_flake
+      assert from_string("-1") == fake_flake
+    end
+
+    test "to_string/1" do
+      fake_flake = <<0::integer-size(64), 42::integer-size(64)>>
+      assert to_string(fake_flake) == "42"
+    end
+  end
+
+  test "ecto type behaviour" do
+    flake = <<0, 0, 1, 104, 80, 229, 2, 235, 140, 22, 69, 201, 53, 210, 0, 0>>
+    flake_s = "9eoozpwTul5mjSEDRI"
+
+    assert cast(flake) == {:ok, flake_s}
+    assert cast(flake_s) == {:ok, flake_s}
+
+    assert load(flake) == {:ok, flake_s}
+    assert load(flake_s) == {:ok, flake_s}
+
+    assert dump(flake_s) == {:ok, flake}
+    assert dump(flake) == {:ok, flake}
+  end
+end

test/support/http_request_mock.ex

@@ -653,6 +653,14 @@ def get("https://social.heldscal.la/user/23211", _, _, Accept: "application/acti
     {:ok, Tesla.Mock.json(%{"id" => "https://social.heldscal.la/user/23211"}, status: 200)}
   end
 
+  def get("http://example.com/ogp", _, _, _) do
+    {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}}
+  end
+
+  def get("http://example.com/empty", _, _, _) do
+    {:ok, %Tesla.Env{status: 200, body: "hello"}}
+  end
+
   def get("http://404.site" <> _, _, _, _) do
     {:ok,
      %Tesla.Env{

test/user_test.exs

@@ -672,12 +672,13 @@ test "get recipients from activity" do
         "status" => "hey @#{addressed.nickname} @#{addressed_remote.nickname}"
       })
 
-    assert [addressed] == User.get_recipients_from_activity(activity)
+    assert Enum.map([actor, addressed], & &1.ap_id) --
+             Enum.map(User.get_recipients_from_activity(activity), & &1.ap_id) == []
 
     {:ok, user} = User.follow(user, actor)
     {:ok, _user_two} = User.follow(user_two, actor)
     recipients = User.get_recipients_from_activity(activity)
-    assert length(recipients) == 2
+    assert length(recipients) == 3
     assert user in recipients
     assert addressed in recipients
   end

test/web/activity_pub/activity_pub_test.exs

@@ -65,6 +65,34 @@ test "it returns a user" do
       assert user.info.ap_enabled
       assert user.follower_address == "http://mastodon.example.org/users/admin/followers"
     end
+
+    test "it fetches the appropriate tag-restricted posts" do
+      user = insert(:user)
+
+      {:ok, status_one} = CommonAPI.post(user, %{"status" => ". #test"})
+      {:ok, status_two} = CommonAPI.post(user, %{"status" => ". #essais"})
+      {:ok, status_three} = CommonAPI.post(user, %{"status" => ". #test #reject"})
+
+      fetch_one = ActivityPub.fetch_activities([], %{"tag" => "test"})
+      fetch_two = ActivityPub.fetch_activities([], %{"tag" => ["test", "essais"]})
+
+      fetch_three =
+        ActivityPub.fetch_activities([], %{
+          "tag" => ["test", "essais"],
+          "tag_reject" => ["reject"]
+        })
+
+      fetch_four =
+        ActivityPub.fetch_activities([], %{
+          "tag" => ["test"],
+          "tag_all" => ["test", "reject"]
+        })
+
+      assert fetch_one == [status_one, status_three]
+      assert fetch_two == [status_one, status_two, status_three]
+      assert fetch_three == [status_one, status_two]
+      assert fetch_four == [status_three]
+    end
   end
 
   describe "insertion" do
@@ -86,6 +114,17 @@ test "drops activities beyond a certain limit" do
       assert {:error, {:remote_limit_error, _}} = ActivityPub.insert(data)
     end
 
+    test "doesn't drop activities with content being null" do
+      data = %{
+        "ok" => true,
+        "object" => %{
+          "content" => nil
+        }
+      }
+
+      assert {:ok, _} = ActivityPub.insert(data)
+    end
+
     test "returns the activity if one with the same id is already in" do
       activity = insert(:note_activity)
       {:ok, new_activity} = ActivityPub.insert(activity.data)
@@ -161,7 +200,7 @@ test "removes doubled 'to' recipients" do
 
       assert activity.data["to"] == ["user1", "user2"]
       assert activity.actor == user.ap_id
-      assert activity.recipients == ["user1", "user2"]
+      assert activity.recipients == ["user1", "user2", user.ap_id]
     end
   end
 

test/web/activity_pub/mrf/anti_followbot_policy_test.exs

@@ -0,0 +1,57 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicyTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+
+  alias Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy
+
+  describe "blocking based on attributes" do
+    test "matches followbots by nickname" do
+      actor = insert(:user, %{nickname: "followbot@example.com"})
+      target = insert(:user)
+
+      message = %{
+        "@context" => "https://www.w3.org/ns/activitystreams",
+        "type" => "Follow",
+        "actor" => actor.ap_id,
+        "object" => target.ap_id,
+        "id" => "https://example.com/activities/1234"
+      }
+
+      {:reject, nil} = AntiFollowbotPolicy.filter(message)
+    end
+
+    test "matches followbots by display name" do
+      actor = insert(:user, %{name: "Federation Bot"})
+      target = insert(:user)
+
+      message = %{
+        "@context" => "https://www.w3.org/ns/activitystreams",
+        "type" => "Follow",
+        "actor" => actor.ap_id,
+        "object" => target.ap_id,
+        "id" => "https://example.com/activities/1234"
+      }
+
+      {:reject, nil} = AntiFollowbotPolicy.filter(message)
+    end
+  end
+
+  test "it allows non-followbots" do
+    actor = insert(:user)
+    target = insert(:user)
+
+    message = %{
+      "@context" => "https://www.w3.org/ns/activitystreams",
+      "type" => "Follow",
+      "actor" => actor.ap_id,
+      "object" => target.ap_id,
+      "id" => "https://example.com/activities/1234"
+    }
+
+    {:ok, _} = AntiFollowbotPolicy.filter(message)
+  end
+end

test/web/mastodon_api/account_view_test.exs

@@ -61,7 +61,9 @@ test "Represent a user account" do
       },
       pleroma: %{
         confirmation_pending: false,
-        tags: []
+        tags: [],
+        is_admin: false,
+        is_moderator: false
       }
     }
 
@@ -102,7 +104,9 @@ test "Represent a Service(bot) account" do
       },
       pleroma: %{
         confirmation_pending: false,
-        tags: []
+        tags: [],
+        is_admin: false,
+        is_moderator: false
       }
     }
 

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -148,7 +148,7 @@ test "posting a direct status", %{conn: conn} do
 
     assert %{"id" => id, "visibility" => "direct"} = json_response(conn, 200)
     assert activity = Repo.get(Activity, id)
-    assert activity.recipients == [user2.ap_id]
+    assert activity.recipients == [user2.ap_id, user1.ap_id]
     assert activity.data["to"] == [user2.ap_id]
     assert activity.data["cc"] == []
   end
@@ -182,6 +182,16 @@ test "direct timeline", %{conn: conn} do
     assert %{"visibility" => "direct"} = status
     assert status["url"] != direct.data["id"]
 
+    # User should be able to see his own direct message
+    res_conn =
+      build_conn()
+      |> assign(:user, user_one)
+      |> get("api/v1/timelines/direct")
+
+    [status] = json_response(res_conn, 200)
+
+    assert %{"visibility" => "direct"} = status
+
     # Both should be visible here
     res_conn =
       conn
@@ -1034,6 +1044,34 @@ test "hashtag timeline", %{conn: conn} do
     end)
   end
 
+  test "multi-hashtag timeline", %{conn: conn} do
+    user = insert(:user)
+
+    {:ok, activity_test} = CommonAPI.post(user, %{"status" => "#test"})
+    {:ok, activity_test1} = CommonAPI.post(user, %{"status" => "#test #test1"})
+    {:ok, activity_none} = CommonAPI.post(user, %{"status" => "#test #none"})
+
+    any_test =
+      conn
+      |> get("/api/v1/timelines/tag/test", %{"any" => ["test1"]})
+
+    [status_none, status_test1, status_test] = json_response(any_test, 200)
+
+    assert to_string(activity_test.id) == status_test["id"]
+    assert to_string(activity_test1.id) == status_test1["id"]
+    assert to_string(activity_none.id) == status_none["id"]
+
+    restricted_test =
+      conn
+      |> get("/api/v1/timelines/tag/test", %{"all" => ["test1"], "none" => ["none"]})
+
+    assert [status_test1] == json_response(restricted_test, 200)
+
+    all_test = conn |> get("/api/v1/timelines/tag/test", %{"all" => ["none"]})
+
+    assert [status_none] == json_response(all_test, 200)
+  end
+
   test "getting followers", %{conn: conn} do
     user = insert(:user)
     other_user = insert(:user)
@@ -1613,5 +1651,22 @@ test "max pinned statuses", %{conn: conn, user: user, activity: activity_one} do
                |> post("/api/v1/statuses/#{activity_two.id}/pin")
                |> json_response(400)
     end
+
+    test "Status rich-media Card", %{conn: conn, user: user} do
+      {:ok, activity} = CommonAPI.post(user, %{"status" => "http://example.com/ogp"})
+
+      response =
+        conn
+        |> get("/api/v1/statuses/#{activity.id}/card")
+        |> json_response(200)
+
+      assert response == %{
+               "image" => "http://ia.media-imdb.com/images/rock.jpg",
+               "provider_name" => "www.imdb.com",
+               "title" => "The Rock",
+               "type" => "link",
+               "url" => "http://www.imdb.com/title/tt0117500/"
+             }
+    end
   end
 end

test/web/mastodon_api/status_view_test.exs

@@ -149,7 +149,10 @@ test "contains mentions" do
 
     status = StatusView.render("status.json", %{activity: activity})
 
-    assert status.mentions == [AccountView.render("mention.json", %{user: user})]
+    actor = Repo.get_by(User, ap_id: activity.actor)
+
+    assert status.mentions ==
+             Enum.map([user, actor], fn u -> AccountView.render("mention.json", %{user: u}) end)
   end
 
   test "attachments" do

test/web/metadata/opengraph_test.exs

@@ -0,0 +1,94 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Metadata.Providers.OpenGraphTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+  alias Pleroma.Web.Metadata.Providers.OpenGraph
+
+  test "it renders all supported types of attachments and skips unknown types" do
+    user = insert(:user)
+
+    note =
+      insert(:note, %{
+        data: %{
+          "actor" => user.ap_id,
+          "tag" => [],
+          "id" => "https://pleroma.gov/objects/whatever",
+          "content" => "pleroma in a nutshell",
+          "attachment" => [
+            %{
+              "url" => [
+                %{"mediaType" => "image/png", "href" => "https://pleroma.gov/tenshi.png"}
+              ]
+            },
+            %{
+              "url" => [
+                %{
+                  "mediaType" => "application/octet-stream",
+                  "href" => "https://pleroma.gov/fqa/badapple.sfc"
+                }
+              ]
+            },
+            %{
+              "url" => [
+                %{"mediaType" => "video/webm", "href" => "https://pleroma.gov/about/juche.webm"}
+              ]
+            },
+            %{
+              "url" => [
+                %{
+                  "mediaType" => "audio/basic",
+                  "href" => "http://www.gnu.org/music/free-software-song.au"
+                }
+              ]
+            }
+          ]
+        }
+      })
+
+    result = OpenGraph.build_tags(%{object: note, url: note.data["id"], user: user})
+
+    assert Enum.all?(
+             [
+               {:meta, [property: "og:image", content: "https://pleroma.gov/tenshi.png"], []},
+               {:meta,
+                [property: "og:audio", content: "http://www.gnu.org/music/free-software-song.au"],
+                []},
+               {:meta, [property: "og:video", content: "https://pleroma.gov/about/juche.webm"],
+                []}
+             ],
+             fn element -> element in result end
+           )
+  end
+
+  test "it does not render attachments if post is nsfw" do
+    Pleroma.Config.put([Pleroma.Web.Metadata, :unfurl_nsfw], false)
+    user = insert(:user, avatar: %{"url" => [%{"href" => "https://pleroma.gov/tenshi.png"}]})
+
+    note =
+      insert(:note, %{
+        data: %{
+          "actor" => user.ap_id,
+          "id" => "https://pleroma.gov/objects/whatever",
+          "content" => "#cuteposting #nsfw #hambaga",
+          "tag" => ["cuteposting", "nsfw", "hambaga"],
+          "sensitive" => true,
+          "attachment" => [
+            %{
+              "url" => [
+                %{"mediaType" => "image/png", "href" => "https://misskey.microsoft/corndog.png"}
+              ]
+            }
+          ]
+        }
+      })
+
+    result = OpenGraph.build_tags(%{object: note, url: note.data["id"], user: user})
+
+    assert {:meta, [property: "og:image", content: "https://pleroma.gov/tenshi.png"], []} in result
+
+    refute {:meta, [property: "og:image", content: "https://misskey.microsoft/corndog.png"], []} in result
+  end
+end

test/web/oauth/oauth_controller_test.exs

@@ -34,6 +34,31 @@ test "redirects with oauth authorization" do
     assert Repo.get_by(Authorization, token: code)
   end
 
+  test "correctly handles wrong credentials", %{conn: conn} do
+    user = insert(:user)
+    app = insert(:oauth_app)
+
+    result =
+      conn
+      |> post("/oauth/authorize", %{
+        "authorization" => %{
+          "name" => user.nickname,
+          "password" => "wrong",
+          "client_id" => app.client_id,
+          "redirect_uri" => app.redirect_uris,
+          "state" => "statepassed"
+        }
+      })
+      |> html_response(:unauthorized)
+
+    # Keep the details
+    assert result =~ app.client_id
+    assert result =~ app.redirect_uris
+
+    # Error message
+    assert result =~ "Invalid"
+  end
+
   test "issues a token for an all-body request" do
     user = insert(:user)
     app = insert(:oauth_app)

test/web/ostatus/ostatus_controller_test.exs

@@ -108,6 +108,7 @@ test "gets an object", %{conn: conn} do
 
     conn =
       conn
+      |> put_req_header("accept", "application/xml")
       |> get(url)
 
     expected =
@@ -134,31 +135,34 @@ test "404s on nonexisting objects", %{conn: conn} do
     |> response(404)
   end
 
+  test "gets an activity in xml format", %{conn: conn} do
+    note_activity = insert(:note_activity)
+    [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["id"]))
+
+    conn
+    |> put_req_header("accept", "application/xml")
+    |> get("/activities/#{uuid}")
+    |> response(200)
+  end
+
   test "404s on deleted objects", %{conn: conn} do
     note_activity = insert(:note_activity)
     [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["object"]["id"]))
     object = Object.get_by_ap_id(note_activity.data["object"]["id"])
 
     conn
+    |> put_req_header("accept", "application/xml")
     |> get("/objects/#{uuid}")
     |> response(200)
 
     Object.delete(object)
 
     conn
+    |> put_req_header("accept", "application/xml")
     |> get("/objects/#{uuid}")
     |> response(404)
   end
 
-  test "gets an activity", %{conn: conn} do
-    note_activity = insert(:note_activity)
-    [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["id"]))
-
-    conn
-    |> get("/activities/#{uuid}")
-    |> response(200)
-  end
-
   test "404s on private activities", %{conn: conn} do
     note_activity = insert(:direct_note_activity)
     [_, uuid] = hd(Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["id"]))
@@ -174,7 +178,7 @@ test "404s on nonexistent activities", %{conn: conn} do
     |> response(404)
   end
 
-  test "gets a notice", %{conn: conn} do
+  test "gets a notice in xml format", %{conn: conn} do
     note_activity = insert(:note_activity)
 
     conn

test/web/rich_media/controllers/rich_media_controller_test.exs

@@ -1,19 +1,14 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.RichMedia.RichMediaControllerTest do
   use Pleroma.Web.ConnCase
   import Pleroma.Factory
+  import Tesla.Mock
 
   setup do
-    Tesla.Mock.mock(fn
+    mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
-      %{
-        method: :get,
-        url: "http://example.com/ogp"
-      } ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}
-
-      %{method: :get, url: "http://example.com/empty"} ->
-        %Tesla.Env{status: 200, body: "hello"}
-    end)
-
     :ok
   end
 

test/web/twitter_api/twitter_api_controller_test.exs

@@ -797,7 +797,7 @@ test "with credentials, invalid activity", %{conn: conn, user: current_user} do
         |> with_credentials(current_user.nickname, "test")
         |> post("/api/favorites/create/1.json")
 
-      assert json_response(conn, 500)
+      assert json_response(conn, 400)
     end
   end
 
@@ -1621,7 +1621,7 @@ test "it approves a friend request" do
       conn =
         build_conn()
         |> assign(:user, user)
-        |> post("/api/pleroma/friendships/approve", %{"user_id" => to_string(other_user.id)})
+        |> post("/api/pleroma/friendships/approve", %{"user_id" => other_user.id})
 
       assert relationship = json_response(conn, 200)
       assert other_user.id == relationship["id"]
@@ -1644,7 +1644,7 @@ test "it denies a friend request" do
       conn =
         build_conn()
         |> assign(:user, user)
-        |> post("/api/pleroma/friendships/deny", %{"user_id" => to_string(other_user.id)})
+        |> post("/api/pleroma/friendships/deny", %{"user_id" => other_user.id})
 
       assert relationship = json_response(conn, 200)
       assert other_user.id == relationship["id"]

test/web/twitter_api/views/user_view_test.exs

@@ -100,6 +100,7 @@ test "A user" do
       "locked" => false,
       "default_scope" => "public",
       "no_rich_text" => false,
+      "hide_network" => false,
       "fields" => [],
       "pleroma" => %{
         "confirmation_pending" => false,
@@ -146,6 +147,7 @@ test "A user for a given other follower", %{user: user} do
       "locked" => false,
       "default_scope" => "public",
       "no_rich_text" => false,
+      "hide_network" => false,
       "fields" => [],
       "pleroma" => %{
         "confirmation_pending" => false,
@@ -193,6 +195,7 @@ test "A user that follows you", %{user: user} do
       "locked" => false,
       "default_scope" => "public",
       "no_rich_text" => false,
+      "hide_network" => false,
       "fields" => [],
       "pleroma" => %{
         "confirmation_pending" => false,
@@ -254,6 +257,7 @@ test "A blocked user for the blocker" do
       "locked" => false,
       "default_scope" => "public",
       "no_rich_text" => false,
+      "hide_network" => false,
       "fields" => [],
       "pleroma" => %{
         "confirmation_pending" => false,