Merge remote-tracking branch 'upstream/develop' into media-proxy

README.md

@@ -22,16 +22,14 @@ No release has been made yet, but several servers have been online for months al
 ### Dependencies
 
 * Postgresql version 9.6 or newer
-* Elixir version 1.4 or newer
+* Elixir version 1.4 or newer (you will also need erlang-dev, erlang-parsetools, erlang-xmerl packages)
 * Build-essential tools
 
 ### Configuration
 
   * Run `mix deps.get` to install elixir dependencies.
 
-  * Run `mix generate_config`. This will ask you a few questions about your instance and generate a configuration file in `config/generated_config.exs`. Check that and copy it to either `config/dev.secret.exs` or `config/prod.secret.exs`.
-
-  * Configure your database settings in `{dev,prod}.secret.exs` and either create the database with psql or run `mix ecto.create`.
+  * Run `mix generate_config`. This will ask you a few questions about your instance and generate a configuration file in `config/generated_config.exs`. Check that and copy it to either `config/dev.secret.exs` or `config/prod.secret.exs`. It will also create a `config/setup_db.psql`, which you need to run as PostgreSQL superuser (i.e. `sudo su postgres -c "psql -f config/setup_db.psql"`). It will setup a pleroma db user, database and will setup needed extensions that need to be set up once as superuser.
 
   * Run `mix ecto.migrate` to run the database migrations. You will have to do this again after certain updates.
 

installation/pleroma.nginx

@@ -22,6 +22,9 @@ server {
     server_name example.tld;
 
     location / {
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
         proxy_pass http://localhost:4000;
     }
 

lib/mix/tasks/generate_config.ex

@@ -9,9 +9,14 @@ def run(_) do
     name = IO.gets("What is the name of your instance? (e.g. Pleroma/Soykaf): ") |> String.trim
     email = IO.gets("What's your admin email address: ") |> String.trim
     secret =  :crypto.strong_rand_bytes(64) |> Base.encode64 |> binary_part(0, 64)
+    dbpass =  :crypto.strong_rand_bytes(64) |> Base.encode64 |> binary_part(0, 64)
+
+    resultSql = EEx.eval_file("lib/mix/tasks/sample_psql.eex", [dbpass: dbpass])
+    result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret, dbpass: dbpass])
 
-    result = EEx.eval_file("lib/mix/tasks/sample_config.eex", [domain: domain, email: email, name: name, secret: secret])
     IO.puts("\nWriting config to config/generated_config.exs.\n\nCheck it and configure your database, then copy it to either config/dev.secret.exs or config/prod.secret.exs")
     File.write("config/generated_config.exs", result)
+    IO.puts("\nWriting setup_db.psql, please run it as postgre superuser, i.e.: sudo su postgres -c 'psql -f config/setup_db.psql'")
+    File.write("config/setup_db.psql", resultSql)
   end
 end

lib/mix/tasks/sample_config.eex

@@ -13,8 +13,8 @@ config :pleroma, :instance,
 # Configure your database
 config :pleroma, Pleroma.Repo,
   adapter: Ecto.Adapters.Postgres,
-  username: "postgres",
-  password: "postgres",
+  username: "pleroma",
+  password: "<%= dbpass %>",
   database: "pleroma_dev",
   hostname: "localhost",
   pool_size: 10

lib/mix/tasks/sample_psql.eex

@@ -0,0 +1,8 @@
+CREATE USER pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB;
+-- in case someone runs this second time accidentally
+ALTER USER pleroma WITH ENCRYPTED PASSWORD '<%= dbpass %>' CREATEDB;
+CREATE DATABASE pleroma_dev;
+ALTER DATABASE pleroma_dev OWNER TO pleroma;
+\c pleroma_dev;
+--Extensions made by ecto.migrate that need superuser access
+CREATE EXTENSION IF NOT EXISTS citext;

lib/pleroma/activity.ex

@@ -7,7 +7,7 @@ defmodule Pleroma.Activity do
     field :data, :map
     field :local, :boolean, default: true
     field :actor, :string
-    has_many :notifications, Notification
+    has_many :notifications, Notification, on_delete: :delete_all
 
     timestamps()
   end

lib/pleroma/formatter.ex

@@ -1,7 +1,7 @@
 defmodule Pleroma.Formatter do
   alias Pleroma.User
 
-  @link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~]+[\w\/]/u
+  @link_regex ~r/https?:\/\/[\w\.\/?=\-#%&@~\(\)]+[\w\/]/u
   def linkify(text) do
     Regex.replace(@link_regex, text, "<a href='\\0'>\\0</a>")
   end
@@ -24,6 +24,15 @@ def parse_mentions(text) do
     |> Enum.filter(fn ({_match, user}) -> user end)
   end
 
+  def html_escape(text) do
+    Regex.split(@link_regex, text, include_captures: true)
+    |> Enum.map_every(2, fn chunk ->
+      {:safe, part} = Phoenix.HTML.html_escape(chunk)
+      part
+    end)
+    |> Enum.join("")
+  end
+
   @finmoji [
     "a_trusted_friend",
     "alandislands",

lib/pleroma/plugs/authentication_plug.ex

@@ -12,6 +12,7 @@ def call(%{assigns: %{user: %User{}}} = conn, _), do: conn
   def call(conn, opts) do
     with {:ok, username, password} <- decode_header(conn),
          {:ok, user} <- opts[:fetcher].(username),
+         false <- !!user.info["deactivated"],
          saved_user_id <- get_session(conn, :user_id),
          {:ok, verified_user} <- verify(user, password, saved_user_id)
     do

lib/pleroma/plugs/oauth_plug.ex

@@ -16,7 +16,8 @@ def call(conn, _) do
             end
     with token when not is_nil(token) <- token,
          %Token{user_id: user_id} <- Repo.get_by(Token, token: token),
-         %User{} = user <- Repo.get(User, user_id) do
+         %User{} = user <- Repo.get(User, user_id),
+         false <- !!user.info["deactivated"] do
       conn
       |> assign(:user, user)
     else

lib/pleroma/user.ex

@@ -5,7 +5,7 @@ defmodule Pleroma.User do
   alias Pleroma.{Repo, User, Object, Web, Activity, Notification}
   alias Comeonin.Pbkdf2
   alias Pleroma.Web.{OStatus, Websub}
-  alias Pleroma.Web.ActivityPub.Utils
+  alias Pleroma.Web.ActivityPub.{Utils, ActivityPub}
 
   schema "users" do
     field :bio, :string
@@ -113,7 +113,7 @@ def password_update_changeset(struct, params) do
   end
 
   def reset_password(user, data) do
-    Repo.update(password_update_changeset(user, data))
+    update_and_set_cache(password_update_changeset(user, data))
   end
 
   def register_changeset(struct, params \\ %{}) do
@@ -142,9 +142,9 @@ def register_changeset(struct, params \\ %{}) do
     end
   end
 
-  def follow(%User{} = follower, %User{} = followed) do
+  def follow(%User{} = follower, %User{info: info} = followed) do
     ap_followers = followed.follower_address
-    if following?(follower, followed) do
+    if following?(follower, followed) or info["deactivated"] do
       {:error,
        "Could not follow user: #{followed.nickname} is already on your list."}
     else
@@ -157,7 +157,7 @@ def follow(%User{} = follower, %User{} = followed) do
 
       follower = follower
       |> follow_changeset(%{following: following})
-      |> Repo.update
+      |> update_and_set_cache
 
       {:ok, _} = update_follower_count(followed)
 
@@ -173,7 +173,7 @@ def unfollow(%User{} = follower, %User{} = followed) do
 
       { :ok, follower } = follower
       |> follow_changeset(%{following: following})
-      |> Repo.update
+      |> update_and_set_cache
 
       {:ok, followed} = update_follower_count(followed)
 
@@ -191,6 +191,17 @@ def get_by_ap_id(ap_id) do
     Repo.get_by(User, ap_id: ap_id)
   end
 
+  def update_and_set_cache(changeset) do
+    with {:ok, user} <- Repo.update(changeset) do
+      Cachex.set(:user_cache, "ap_id:#{user.ap_id}", user)
+      Cachex.set(:user_cache, "nickname:#{user.nickname}", user)
+      Cachex.set(:user_cache, "user_info:#{user.id}", user_info(user))
+      {:ok, user}
+    else
+      e -> e
+    end
+  end
+
   def get_cached_by_ap_id(ap_id) do
     key = "ap_id:#{ap_id}"
     Cachex.get!(:user_cache, key, fallback: fn(_) -> get_by_ap_id(ap_id) end)
@@ -245,7 +256,7 @@ def increase_note_count(%User{} = user) do
 
     cs = info_changeset(user, %{info: new_info})
 
-    Repo.update(cs)
+    update_and_set_cache(cs)
   end
 
   def update_note_count(%User{} = user) do
@@ -259,7 +270,7 @@ def update_note_count(%User{} = user) do
 
     cs = info_changeset(user, %{info: new_info})
 
-    Repo.update(cs)
+    update_and_set_cache(cs)
   end
 
   def update_follower_count(%User{} = user) do
@@ -274,7 +285,7 @@ def update_follower_count(%User{} = user) do
 
     cs = info_changeset(user, %{info: new_info})
 
-    Repo.update(cs)
+    update_and_set_cache(cs)
   end
 
   def get_notified_from_activity(%Activity{data: %{"to" => to}}) do
@@ -312,7 +323,7 @@ def block(user, %{ap_id: ap_id}) do
     new_info = Map.put(user.info, "blocks", new_blocks)
 
     cs = User.info_changeset(user, %{info: new_info})
-    Repo.update(cs)
+    update_and_set_cache(cs)
   end
 
   def unblock(user, %{ap_id: ap_id}) do
@@ -321,7 +332,7 @@ def unblock(user, %{ap_id: ap_id}) do
     new_info = Map.put(user.info, "blocks", new_blocks)
 
     cs = User.info_changeset(user, %{info: new_info})
-    Repo.update(cs)
+    update_and_set_cache(cs)
   end
 
   def blocks?(user, %{ap_id: ap_id}) do
@@ -334,4 +345,35 @@ def local_user_query() do
       where: u.local == true
   end
 
+  def deactivate (%User{} = user) do
+    new_info = Map.put(user.info, "deactivated", true)
+    cs = User.info_changeset(user, %{info: new_info})
+    update_and_set_cache(cs)
+  end
+
+  def delete (%User{} = user) do
+    {:ok, user} = User.deactivate(user)
+
+    # Remove all relationships
+    {:ok, followers } = User.get_followers(user)
+    followers
+    |> Enum.each(fn (follower) -> User.unfollow(follower, user) end)
+
+    {:ok, friends} = User.get_friends(user)
+    friends
+    |> Enum.each(fn (followed) -> User.unfollow(user, followed) end)
+
+    query = from a in Activity,
+      where: a.actor == ^user.ap_id
+
+    Repo.all(query)
+    |> Enum.each(fn (activity) ->
+      case activity.data["type"] do
+        "Create" -> ActivityPub.delete(Object.get_by_ap_id(activity.data["object"]["id"]))
+        _ -> "Doing nothing" # TODO: Do something with likes, follows, repeats.
+      end
+    end)
+
+    :ok
+  end
 end

lib/pleroma/web/activity_pub/utils.ex

@@ -29,7 +29,12 @@ def generate_id(type) do
   Enqueues an activity for federation if it's local
   """
   def maybe_federate(%Activity{local: true} = activity) do
-    Pleroma.Web.Federator.enqueue(:publish, activity)
+    priority = case activity.data["type"] do
+                 "Delete" -> 10
+                 "Create" -> 1
+                 _ -> 5
+               end
+    Pleroma.Web.Federator.enqueue(:publish, activity, priority)
     :ok
   end
   def maybe_federate(_), do: :ok

lib/pleroma/web/common_api/common_api.ex

@@ -56,7 +56,7 @@ def post(user, %{"status" => status} = data) do
          inReplyTo <- get_replied_to_activity(data["in_reply_to_status_id"]),
          to <- to_for_user_and_mentions(user, mentions, inReplyTo),
          tags <- Formatter.parse_tags(status, data),
-         content_html <- make_content_html(status, mentions, attachments, tags),
+         content_html <- make_content_html(status, mentions, attachments, tags, data["no_attachment_links"]),
          context <- make_context(inReplyTo),
          cw <- data["spoiler_text"],
          object <- make_note_data(user.ap_id, to, context, content_html, attachments, inReplyTo, tags, cw),

lib/pleroma/web/common_api/utils.ex

@@ -38,15 +38,19 @@ def to_for_user_and_mentions(user, mentions, inReplyTo) do
     end
   end
 
-  def make_content_html(status, mentions, attachments, tags) do
+  def make_content_html(status, mentions, attachments, tags, no_attachment_links \\ false) do
     status
     |> format_input(mentions, tags)
-    |> add_attachments(attachments)
+    |> maybe_add_attachments(attachments, no_attachment_links)
   end
 
   def make_context(%Activity{data: %{"context" => context}}), do: context
   def make_context(_), do: Utils.generate_context_id
 
+  def maybe_add_attachments(text, attachments, _no_links = true), do: text
+  def maybe_add_attachments(text, attachments, _no_links) do
+    add_attachments(text, attachments)
+  end
   def add_attachments(text, attachments) do
     attachment_text = Enum.map(attachments, fn
       (%{"url" => [%{"href" => href} | _]}) ->
@@ -58,8 +62,8 @@ def add_attachments(text, attachments) do
   end
 
   def format_input(text, mentions, _tags) do
-    Phoenix.HTML.html_escape(text)
-    |> elem(1)
+    text
+    |> Formatter.html_escape
     |> Formatter.linkify
     |> String.replace("\n", "<br>")
     |> add_user_links(mentions)

lib/pleroma/web/federator/federator.ex

@@ -15,8 +15,8 @@ def start_link do
       enqueue(:refresh_subscriptions, nil)
     end)
     GenServer.start_link(__MODULE__, %{
-          in: {:sets.new(), :queue.new()},
-          out: {:sets.new(), :queue.new()}
+          in: {:sets.new(), []},
+          out: {:sets.new(), []}
                          }, name: __MODULE__)
   end
 
@@ -79,17 +79,17 @@ def handle(type, _) do
     {:error, "Don't know what do do with this"}
   end
 
-  def enqueue(type, payload) do
+  def enqueue(type, payload, priority \\ 1) do
     if Mix.env == :test do
       handle(type, payload)
     else
-      GenServer.cast(__MODULE__, {:enqueue, type, payload})
+      GenServer.cast(__MODULE__, {:enqueue, type, payload, priority})
     end
   end
 
   def maybe_start_job(running_jobs, queue) do
-    if (:sets.size(running_jobs) < @max_jobs) && !:queue.is_empty(queue) do
-      {{:value, {type, payload}}, queue} = :queue.out(queue)
+    if (:sets.size(running_jobs) < @max_jobs) && queue != [] do
+      {{type, payload}, queue} = queue_pop(queue)
       {:ok, pid} = Task.start(fn -> handle(type, payload) end)
       mref = Process.monitor(pid)
       {:sets.add_element(mref, running_jobs), queue}
@@ -98,16 +98,16 @@ def maybe_start_job(running_jobs, queue) do
     end
   end
 
-  def handle_cast({:enqueue, type, payload}, state) when type in [:incoming_doc] do
+  def handle_cast({:enqueue, type, payload, priority}, state) when type in [:incoming_doc] do
     %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
-    i_queue = :queue.in({type, payload}, i_queue)
+    i_queue = enqueue_sorted(i_queue, {type, payload}, 1)
     {i_running_jobs, i_queue} = maybe_start_job(i_running_jobs, i_queue)
     {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
   end
 
-  def handle_cast({:enqueue, type, payload}, state) do
+  def handle_cast({:enqueue, type, payload, priority}, state) do
     %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}} = state
-    o_queue = :queue.in({type, payload}, o_queue)
+    o_queue = enqueue_sorted(o_queue, {type, payload}, 1)
     {o_running_jobs, o_queue} = maybe_start_job(o_running_jobs, o_queue)
     {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
   end
@@ -126,4 +126,13 @@ def handle_info({:DOWN, ref, :process, _pid, _reason}, state) do
 
     {:noreply, %{in: {i_running_jobs, i_queue}, out: {o_running_jobs, o_queue}}}
   end
+
+  def enqueue_sorted(queue, element, priority) do
+    [%{item: element, priority: priority} | queue]
+    |> Enum.sort_by(fn (%{priority: priority}) -> priority end)
+  end
+
+  def queue_pop([%{item: element} | queue]) do
+    {element, queue}
+  end
 end

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -212,6 +212,7 @@ def get_context(%{assigns: %{user: user}} = conn, %{"id" => id}) do
   def post_status(%{assigns: %{user: user}} = conn, %{"status" => _} = params) do
     params = params
     |> Map.put("in_reply_to_status_id", params["in_reply_to_id"])
+    |> Map.put("no_attachment_links", true)
 
     {:ok, activity} = CommonAPI.post(user, params)
     render conn, StatusView, "status.json", %{activity: activity, for: user, as: :activity}

test/formatter_test.exs

@@ -25,6 +25,16 @@ test "turning urls into links" do
       expected = "<a href='http://www.cs.vu.nl/~ast/intel/'>http://www.cs.vu.nl/~ast/intel/</a>"
 
       assert Formatter.linkify(text) == expected
+
+      text = "https://forum.zdoom.org/viewtopic.php?f=44&t=57087"
+      expected = "<a href='https://forum.zdoom.org/viewtopic.php?f=44&t=57087'>https://forum.zdoom.org/viewtopic.php?f=44&t=57087</a>"
+
+      assert Formatter.linkify(text) == expected
+
+      text = "https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul"
+      expected = "<a href='https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul'>https://en.wikipedia.org/wiki/Sophia_(Gnosticism)#Mythos_of_the_soul</a>"
+
+      assert Formatter.linkify(text) == expected
     end
   end
 

test/plugs/authentication_plug_test.exs

@@ -14,6 +14,13 @@ defp fetch_nil(_name) do
     password_hash: Comeonin.Pbkdf2.hashpwsalt("guy")
   }
 
+  @deactivated %User{
+    id: 1,
+    name: "dude",
+    password_hash: Comeonin.Pbkdf2.hashpwsalt("guy"),
+    info: %{"deactivated" => true}
+  }
+
   @session_opts [
     store: :cookie,
     key: "_test",
@@ -131,6 +138,26 @@ test "it assigns the user", %{conn: conn} do
     end
   end
 
+  describe "with a correct authorization header for an deactiviated user" do
+    test "it halts the appication", %{conn: conn} do
+      opts = %{
+        optional: false,
+        fetcher: fn _ -> @deactivated end
+      }
+
+      header = basic_auth_enc("dude", "guy")
+
+      conn = conn
+        |> Plug.Session.call(Plug.Session.init(@session_opts))
+        |> fetch_session
+        |> put_req_header("authorization", header)
+        |> AuthenticationPlug.call(opts)
+
+      assert conn.status == 403
+      assert conn.halted == true
+    end
+  end
+
   describe "with a user_id in the session for an existing user" do
     test "it assigns the user", %{conn: conn} do
       opts = %{

test/user_test.exs

@@ -1,6 +1,6 @@
 defmodule Pleroma.UserTest do
   alias Pleroma.Builders.UserBuilder
-  alias Pleroma.{User, Repo}
+  alias Pleroma.{User, Repo, Activity}
   alias Pleroma.Web.OStatus
   alias Pleroma.Web.Websub.WebsubClientSubscription
   alias Pleroma.Web.CommonAPI
@@ -39,6 +39,13 @@ test "follow takes a user and another user" do
     assert User.ap_followers(followed) in user.following
   end
 
+  test "can't follow a deactivated users" do
+    user = insert(:user)
+    followed = insert(:user, info: %{"deactivated" => true})
+
+    {:error, _} = User.follow(user, followed)
+  end
+
   test "following a remote user will ensure a websub subscription is present" do
     user = insert(:user)
     {:ok, followed} = OStatus.make_user("shp@social.heldscal.la")
@@ -325,5 +332,42 @@ test "get recipients from activity" do
     assert user in recipients
     assert addressed in recipients
   end
+
+  test ".deactivate deactivates a user" do
+    user = insert(:user)
+    assert false == !!user.info["deactivated"]
+    {:ok, user} = User.deactivate(user)
+    assert true == user.info["deactivated"]
   end
 
+  test ".delete deactivates a user, all follow relationships and all create activities" do
+    user = insert(:user)
+    followed = insert(:user)
+    follower = insert(:user)
+
+    {:ok, user} = User.follow(user, followed)
+    {:ok, follower} = User.follow(follower, user)
+
+    {:ok, activity} = CommonAPI.post(user, %{"status" => "2hu"})
+    {:ok, activity_two} = CommonAPI.post(follower, %{"status" => "3hu"})
+
+    {:ok, _, _} = CommonAPI.favorite(activity_two.id, user)
+    {:ok, _, _} = CommonAPI.favorite(activity.id, follower)
+    {:ok, _, _} = CommonAPI.repeat(activity.id, follower)
+
+    :ok = User.delete(user)
+
+    followed = Repo.get(User, followed.id)
+    follower = Repo.get(User, follower.id)
+    user = Repo.get(User, user.id)
+
+    assert user.info["deactivated"]
+
+    refute User.following?(user, followed)
+    refute User.following?(followed, follower)
+
+    # TODO: Remove favorites, repeats, delete activities.
+
+    refute Repo.get(Activity, activity.id)
+  end
+end

test/web/federator_test.exs

@@ -0,0 +1,20 @@
+defmodule Pleroma.Web.FederatorTest do
+  alias Pleroma.Web.Federator
+  use Pleroma.DataCase
+
+  test "enqueues an element according to priority" do
+    queue = [%{item: 1, priority: 2}]
+
+    new_queue = Federator.enqueue_sorted(queue, 2, 1)
+    assert new_queue == [%{item: 2, priority: 1}, %{item: 1, priority: 2}]
+
+    new_queue = Federator.enqueue_sorted(queue, 2, 3)
+    assert new_queue == [%{item: 1, priority: 2}, %{item: 2, priority: 3}]
+  end
+
+  test "pop first item" do
+    queue = [%{item: 2, priority: 1}, %{item: 1, priority: 2}]
+
+    assert {2, [%{item: 1, priority: 2}]} = Federator.queue_pop(queue)
+  end
+end