From 43d7a4b2cfe686c15b68f6599ce16446fa1dfab0 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 24 Apr 2017 08:48:52 +0200 Subject: [PATCH 001/107] Add basic fields to support remote users. --- lib/pleroma/user.ex | 2 ++ .../migrations/20170423154511_add_fields_to_users.exs | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100644 priv/repo/migrations/20170423154511_add_fields_to_users.exs diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3ce07d510..160acbdb9 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -15,6 +15,8 @@ defmodule Pleroma.User do field :following, { :array, :string }, default: [] field :ap_id, :string field :avatar, :map + field :local, :boolean, default: true + field :info, :map timestamps() end diff --git a/priv/repo/migrations/20170423154511_add_fields_to_users.exs b/priv/repo/migrations/20170423154511_add_fields_to_users.exs new file mode 100644 index 000000000..84de74bc4 --- /dev/null +++ b/priv/repo/migrations/20170423154511_add_fields_to_users.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddFieldsToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add :local, :boolean, default: true + add :info, :map + end + end +end From 34d3aea92f1bce7ba51a44ef1fdc68e47822c3a4 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 24 Apr 2017 18:46:02 +0200 Subject: [PATCH 002/107] Add incoming xml fixtures. --- test/fixtures/incoming_note_activity.xml | 40 ++++++++++++++++++++++++ test/fixtures/user_full.xml | 10 ++++++ test/fixtures/user_name_only.xml | 5 +++ 3 files changed, 55 insertions(+) create mode 100644 test/fixtures/incoming_note_activity.xml create mode 100644 test/fixtures/user_full.xml create mode 100644 test/fixtures/user_name_only.xml diff --git a/test/fixtures/incoming_note_activity.xml b/test/fixtures/incoming_note_activity.xml new file mode 100644 index 000000000..e54b25e39 --- /dev/null +++ b/test/fixtures/incoming_note_activity.xml @@ -0,0 +1,40 @@ + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note + New note by lambda + @<a href="http://pleroma.example.org:4000/users/lain3" class="h-card mention">lain3</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-04-23T14:51:03+00:00 + 2017-04-23T14:51:03+00:00 + + http://activitystrea.ms/schema/1.0/person + http://gs.example.org:4040/index.php/user/1 + lambda + + + + + lambda + lambda + + + + + tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b + + + + http://gs.example.org:4040/index.php/api/statuses/user_timeline/1.atom + lambda + + + + http://gs.example.org:4040/theme/neo-gnu/default-avatar-profile.png + 2017-04-23T14:51:03+00:00 + + + + + diff --git a/test/fixtures/user_full.xml b/test/fixtures/user_full.xml new file mode 100644 index 000000000..8eee8c686 --- /dev/null +++ b/test/fixtures/user_full.xml @@ -0,0 +1,10 @@ + + http://activitystrea.ms/schema/1.0/person + http://gs.example.org:4040/index.php/user/1 + lambda + + + + Constance Variable + lambadalambda + diff --git a/test/fixtures/user_name_only.xml b/test/fixtures/user_name_only.xml new file mode 100644 index 000000000..6d895d5c2 --- /dev/null +++ b/test/fixtures/user_name_only.xml @@ -0,0 +1,5 @@ + + http://activitystrea.ms/schema/1.0/person + http://gs.example.org:4040/index.php/user/1 + lambda + From ab0114fbaabd28d1e1a6961f6bfbd683f3e7fbbc Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 24 Apr 2017 18:46:34 +0200 Subject: [PATCH 003/107] Return salmon path for users, basic incoming salmon handling. --- lib/pleroma/web/activity_pub/activity_pub.ex | 42 ++++++ lib/pleroma/web/ostatus/feed_representer.ex | 1 + lib/pleroma/web/ostatus/ostatus.ex | 134 +++++++++++++++++- lib/pleroma/web/ostatus/ostatus_controller.ex | 11 +- lib/pleroma/web/router.ex | 1 + lib/pleroma/web/twitter_api/twitter_api.ex | 65 ++++----- lib/pleroma/web/web_finger/web_finger.ex | 3 +- test/web/ostatus/feed_representer_test.exs | 1 + test/web/ostatus/ostatus_test.exs | 53 +++++++ 9 files changed, 272 insertions(+), 39 deletions(-) create mode 100644 test/web/ostatus/ostatus_test.exs diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e9f0dcd32..7264123d8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -19,6 +19,48 @@ def insert(map) when is_map(map) do Repo.insert(%Activity{data: map}) end + def create(to, actor, context, object, additional \\ %{}, published \\ nil) do + published = published || make_date() + + activity = %{ + "type" => "Create", + "to" => to, + "actor" => actor.ap_id, + "object" => object, + "published" => published, + "context" => context + } + |> Map.merge(additional) + + with {:ok, activity} <- insert(activity) do + {:ok, activity} = add_conversation_id(activity) + + if actor.local do + Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity) + end + + {:ok, activity} + end + end + + defp add_conversation_id(activity) do + if is_integer(activity.data["statusnetConversationId"]) do + {:ok, activity} + else + data = activity.data + |> put_in(["object", "statusnetConversationId"], activity.id) + |> put_in(["statusnetConversationId"], activity.id) + + object = Object.get_by_ap_id(activity.data["object"]["id"]) + + changeset = Ecto.Changeset.change(object, data: data["object"]) + Repo.update(changeset) + + changeset = Ecto.Changeset.change(activity, data: data) + Repo.update(changeset) + end + end + def like(%User{ap_id: ap_id} = user, %Object{data: %{ "id" => id}} = object) do cond do # There's already a like here, so return the original activity. diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex index 14ac3ebf4..2cc0da9ba 100644 --- a/lib/pleroma/web/ostatus/feed_representer.ex +++ b/lib/pleroma/web/ostatus/feed_representer.ex @@ -23,6 +23,7 @@ def to_simple_form(user, activities, users) do {:title, ['#{user.nickname}\'s timeline']}, {:updated, h.(most_recent_update)}, {:link, [rel: 'hub', href: h.(OStatus.pubsub_path(user))], []}, + {:link, [rel: 'salmon', href: h.(OStatus.salmon_path(user))], []}, {:link, [rel: 'self', href: h.(OStatus.feed_path(user))], []}, {:author, UserRepresenter.to_simple_form(user)}, ] ++ entries diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index d21b9078f..4fd649c92 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -1,5 +1,9 @@ defmodule Pleroma.Web.OStatus do - alias Pleroma.Web + import Ecto.Query + require Logger + + alias Pleroma.{Repo, User, Web} + alias Pleroma.Web.ActivityPub.ActivityPub def feed_path(user) do "#{user.ap_id}/feed.atom" @@ -9,6 +13,132 @@ def pubsub_path(user) do "#{Web.base_url}/push/hub/#{user.nickname}" end - def user_path(user) do + def salmon_path(user) do + "#{user.ap_id}/salmon" + end + + def handle_incoming(xml_string) do + {doc, _rest} = :xmerl_scan.string(to_charlist(xml_string)) + + {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', doc) + + case object_type do + 'http://activitystrea.ms/schema/1.0/note' -> + handle_note(doc) + _ -> + Logger.error("Couldn't parse incoming document") + end + end + + # TODO + # Parse mention + # wire up replies + # Set correct context + # Set correct statusnet ids. + def handle_note(doc) do + content_html = string_from_xpath("/entry/content[1]", doc) + + [author] = :xmerl_xpath.string('/entry/author[1]', doc) + {:ok, actor} = find_or_make_user(author) + + context = ActivityPub.generate_context_id + + to = [ + "https://www.w3.org/ns/activitystreams#Public" + ] + + date = string_from_xpath("/entry/published", doc) + + object = %{ + "type" => "Note", + "to" => to, + "content" => content_html, + "published" => date, + "context" => context, + "actor" => actor.ap_id + } + + ActivityPub.create(to, actor, context, object, %{}, date) + end + + def find_or_make(author, doc) do + query = from user in User, + where: user.local == false and fragment("? @> ?", user.info, ^%{ostatus_uri: author}) + + user = Repo.one(query) + + if is_nil(user) do + make_user(doc) + else + {:ok, user} + end + end + + def find_or_make_user(author_doc) do + {:xmlObj, :string, uri } = :xmerl_xpath.string('string(/author[1]/uri)', author_doc) + + query = from user in User, + where: user.local == false and fragment("? @> ?", user.info, ^%{ostatus_uri: to_string(uri)}) + + user = Repo.one(query) + + if is_nil(user) do + make_user(author_doc) + else + {:ok, user} + end + end + + defp string_from_xpath(xpath, doc) do + {:xmlObj, :string, res} = :xmerl_xpath.string('string(#{xpath})', doc) + + res = res + |> to_string + |> String.trim + + if res == "", do: nil, else: res + end + + def make_user(author_doc) do + author = string_from_xpath("/author[1]/uri", author_doc) + name = string_from_xpath("/author[1]/name", author_doc) + preferredUsername = string_from_xpath("/author[1]/poco:preferredUsername", author_doc) + displayName = string_from_xpath("/author[1]/poco:displayName", author_doc) + avatar = make_avatar_object(author_doc) + + data = %{ + local: false, + name: preferredUsername || name, + nickname: displayName || name, + ap_id: author, + info: %{ + "ostatus_uri" => author, + "host" => URI.parse(author).host, + "system" => "ostatus" + }, + avatar: avatar + } + + Repo.insert(Ecto.Changeset.change(%User{}, data)) + end + + # TODO: Just takes the first one for now. + defp make_avatar_object(author_doc) do + href = string_from_xpath("/author[1]/link[@rel=\"avatar\"]/@href", author_doc) + type = string_from_xpath("/author[1]/link[@rel=\"avatar\"]/@type", author_doc) + + if href do + %{ + "type" => "Image", + "url" => + [%{ + "type" => "Link", + "mediaType" => type, + "href" => href + }] + } + else + nil + end end end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 3c8d8c0f1..4174db786 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -25,7 +25,14 @@ def feed(conn, %{"nickname" => nickname}) do |> send_resp(200, response) end - def temp(conn, params) do - IO.inspect(params) + def salmon_incoming(conn, params) do + {:ok, body, _conn} = read_body(conn) + magic_key = Pleroma.Web.Salmon.fetch_magic_key(body) + {:ok, doc} = Pleroma.Web.Salmon.decode_and_validate(magic_key, body) + + Pleroma.Web.OStatus.handle_incoming(doc) + + conn + |> send_resp(200, "") end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a4f13c879..c98eac688 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -74,6 +74,7 @@ def user_fetcher(username) do pipe_through :ostatus get "/users/:nickname/feed", OStatus.OStatusController, :feed + post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming post "/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request end diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 0f84cffbd..9049b4efc 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -28,11 +28,33 @@ def create_status(user = %User{}, data = %{}) do date = make_date() - activity = %{ - "type" => "Create", - "to" => to, - "actor" => user.ap_id, - "object" => %{ + # Wire up reply info. + [to, context, object, additional] = + with inReplyToId when not is_nil(inReplyToId) <- data["in_reply_to_status_id"], + inReplyTo <- Repo.get(Activity, inReplyToId), + context <- inReplyTo.data["context"] + do + to = to ++ [inReplyTo.data["actor"]] + + object = %{ + "type" => "Note", + "to" => to, + "content" => content_html, + "published" => date, + "context" => context, + "attachment" => attachments, + "actor" => user.ap_id, + "inReplyTo" => inReplyTo.data["object"]["id"], + "inReplyToStatusId" => inReplyToId, + "statusnetConversationId" => inReplyTo.data["statusnetConversationId"] + } + additional = %{ + "statusnetConversationId" => inReplyTo.data["statusnetConversationId"] + } + + [to, context, object, additional] + else _e -> + object = %{ "type" => "Note", "to" => to, "content" => content_html, @@ -40,36 +62,11 @@ def create_status(user = %User{}, data = %{}) do "context" => context, "attachment" => attachments, "actor" => user.ap_id - }, - "published" => date, - "context" => context - } - - # Wire up reply info. - activity = with inReplyToId when not is_nil(inReplyToId) <- data["in_reply_to_status_id"], - inReplyTo <- Repo.get(Activity, inReplyToId), - context <- inReplyTo.data["context"] - do - - to = activity["to"] ++ [inReplyTo.data["actor"]] - - activity - |> put_in(["to"], to) - |> put_in(["context"], context) - |> put_in(["object", "context"], context) - |> put_in(["object", "inReplyTo"], inReplyTo.data["object"]["id"]) - |> put_in(["object", "inReplyToStatusId"], inReplyToId) - |> put_in(["statusnetConversationId"], inReplyTo.data["statusnetConversationId"]) - |> put_in(["object", "statusnetConversationId"], inReplyTo.data["statusnetConversationId"]) - else _e -> - activity - end - - with {:ok, activity} <- ActivityPub.insert(activity) do - {:ok, activity} = add_conversation_id(activity) - Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(user), user, activity) - {:ok, activity} + } + [to, context, object, %{}] end + + ActivityPub.create(to, user, context, object, additional, data) end def fetch_friend_statuses(user, opts \\ %{}) do diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index eb540e92a..18459e8f0 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -31,7 +31,8 @@ def represent_user(user) do [ {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.host}"}, {:Alias, user.ap_id}, - {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}} + {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}}, + {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}} ] } |> XmlBuilder.to_doc diff --git a/test/web/ostatus/feed_representer_test.exs b/test/web/ostatus/feed_representer_test.exs index 9a02d8c16..13cdeb79d 100644 --- a/test/web/ostatus/feed_representer_test.exs +++ b/test/web/ostatus/feed_representer_test.exs @@ -27,6 +27,7 @@ test "returns a feed of the last 20 items of the user" do #{user.nickname}'s timeline #{most_recent_update} + #{user_xml} diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs new file mode 100644 index 000000000..8ee605494 --- /dev/null +++ b/test/web/ostatus/ostatus_test.exs @@ -0,0 +1,53 @@ +defmodule Pleroma.Web.OStatusTest do + use Pleroma.DataCase + alias Pleroma.Web.OStatus + + test "handle incoming notes" do + incoming = File.read!("test/fixtures/incoming_note_activity.xml") + {:ok, activity} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["published"] == "2017-04-23T14:51:03+00:00" + end + + describe "new remote user creation" do + test "make new user or find them based on an 'author' xml doc" do + incoming = File.read!("test/fixtures/user_name_only.xml") + {doc, _rest} = :xmerl_scan.string(to_charlist(incoming)) + + {:ok, user} = OStatus.find_or_make_user(doc) + + assert user.name == "lambda" + assert user.nickname == "lambda" + assert user.local == false + assert user.info["ostatus_uri"] == "http://gs.example.org:4040/index.php/user/1" + assert user.info["system"] == "ostatus" + assert user.ap_id == "http://gs.example.org:4040/index.php/user/1" + + {:ok, user_again} = OStatus.find_or_make_user(doc) + + assert user == user_again + end + + test "tries to use the information in poco fields" do + incoming = File.read!("test/fixtures/user_full.xml") + {doc, _rest} = :xmerl_scan.string(to_charlist(incoming)) + + {:ok, user} = OStatus.find_or_make_user(doc) + + assert user.name == "Constance Variable" + assert user.nickname == "lambadalambda" + assert user.local == false + assert user.info["ostatus_uri"] == "http://gs.example.org:4040/index.php/user/1" + assert user.info["system"] == "ostatus" + assert user.ap_id == "http://gs.example.org:4040/index.php/user/1" + + assert List.first(user.avatar["url"])["href"] == "http://gs.example.org:4040/theme/neo-gnu/default-avatar-profile.png" + + {:ok, user_again} = OStatus.find_or_make_user(doc) + + assert user == user_again + end + end +end From ef4190b3abbc581e9405f2a32fe7579345a3d155 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 25 Apr 2017 17:26:05 +0200 Subject: [PATCH 004/107] Clean up status create method. --- lib/pleroma/web/twitter_api/twitter_api.ex | 33 +++++++++++++--------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 9049b4efc..ad73e82ce 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -5,26 +5,33 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do import Ecto.Query - def create_status(user = %User{}, data = %{}) do - attachments = Enum.map(data["media_ids"] || [], fn (media_id) -> - Repo.get(Object, media_id).data - end) - - context = ActivityPub.generate_context_id - - content = HtmlSanitizeEx.strip_tags(data["status"]) - |> String.replace("\n", "
") - - mentions = parse_mentions(content) - + def to_for_user_and_mentions(user, mentions) do default_to = [ User.ap_followers(user), "https://www.w3.org/ns/activitystreams#Public" ] to = default_to ++ Enum.map(mentions, fn ({_, %{ap_id: ap_id}}) -> ap_id end) + end - content_html = add_user_links(content, mentions) + def format_input(text, mentions) do + content = HtmlSanitizeEx.strip_tags(text) + |> String.replace("\n", "
") + |> add_user_links(mentions) + end + + def attachments_from_ids(ids) do + Enum.map(ids || [], fn (media_id) -> + Repo.get(Object, media_id).data + end) + end + + def create_status(user = %User{}, data = %{"status" => status}) do + attachments = attachments_from_ids(data["media_ids"]) + context = ActivityPub.generate_context_id + mentions = parse_mentions(status) + content_html = format_input(status, mentions) + to = to_for_user_and_mentions(user, mentions) date = make_date() From 4771962a5d0749746b7bb921074b97e13348dedf Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 25 Apr 2017 17:32:36 +0200 Subject: [PATCH 005/107] More refactoring. --- lib/pleroma/web/twitter_api/twitter_api.ex | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index ad73e82ce..ad4bf7153 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -26,21 +26,26 @@ def attachments_from_ids(ids) do end) end + def get_replied_to_activity(id) when not is_nil(id) do + Repo.get(Activity, id) + end + + def get_replied_to_activity(_), do: nil + def create_status(user = %User{}, data = %{"status" => status}) do attachments = attachments_from_ids(data["media_ids"]) context = ActivityPub.generate_context_id mentions = parse_mentions(status) content_html = format_input(status, mentions) to = to_for_user_and_mentions(user, mentions) - date = make_date() + inReplyTo = get_replied_to_activity(data["in_reply_to_status_id"]) + # Wire up reply info. [to, context, object, additional] = - with inReplyToId when not is_nil(inReplyToId) <- data["in_reply_to_status_id"], - inReplyTo <- Repo.get(Activity, inReplyToId), - context <- inReplyTo.data["context"] - do + if inReplyTo do + context = inReplyTo.data["context"] to = to ++ [inReplyTo.data["actor"]] object = %{ @@ -52,7 +57,7 @@ def create_status(user = %User{}, data = %{"status" => status}) do "attachment" => attachments, "actor" => user.ap_id, "inReplyTo" => inReplyTo.data["object"]["id"], - "inReplyToStatusId" => inReplyToId, + "inReplyToStatusId" => inReplyTo.id, "statusnetConversationId" => inReplyTo.data["statusnetConversationId"] } additional = %{ @@ -60,7 +65,7 @@ def create_status(user = %User{}, data = %{"status" => status}) do } [to, context, object, additional] - else _e -> + else object = %{ "type" => "Note", "to" => to, From 6c5f5e18ec1103a9d10d88081a27c2ae9dba4f41 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 25 Apr 2017 17:35:21 +0200 Subject: [PATCH 006/107] Even more refactoring. --- lib/pleroma/web/twitter_api/twitter_api.ex | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index ad4bf7153..cb48c7f5f 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -11,11 +11,11 @@ def to_for_user_and_mentions(user, mentions) do "https://www.w3.org/ns/activitystreams#Public" ] - to = default_to ++ Enum.map(mentions, fn ({_, %{ap_id: ap_id}}) -> ap_id end) + default_to ++ Enum.map(mentions, fn ({_, %{ap_id: ap_id}}) -> ap_id end) end def format_input(text, mentions) do - content = HtmlSanitizeEx.strip_tags(text) + HtmlSanitizeEx.strip_tags(text) |> String.replace("\n", "
") |> add_user_links(mentions) end @@ -235,24 +235,6 @@ def add_user_links(text, mentions) do Enum.reduce(mentions, text, fn ({match, %User{ap_id: ap_id}}, text) -> String.replace(text, match, "#{match}") end) end - defp add_conversation_id(activity) do - if is_integer(activity.data["statusnetConversationId"]) do - {:ok, activity} - else - data = activity.data - |> put_in(["object", "statusnetConversationId"], activity.id) - |> put_in(["statusnetConversationId"], activity.id) - - object = Object.get_by_ap_id(activity.data["object"]["id"]) - - changeset = Ecto.Changeset.change(object, data: data["object"]) - Repo.update(changeset) - - changeset = Ecto.Changeset.change(activity, data: data) - Repo.update(changeset) - end - end - def register_user(params) do params = %{ nickname: params["nickname"], From b438ea24ee936ae10efdcd3c9079e3b45ae521f4 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 25 Apr 2017 17:45:34 +0200 Subject: [PATCH 007/107] Add ostatus conversation as context. --- lib/pleroma/web/ostatus/ostatus.ex | 7 ++++++- lib/pleroma/web/twitter_api/twitter_api.ex | 2 +- test/web/ostatus/ostatus_test.exs | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 4fd649c92..8c31ce5aa 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -41,7 +41,12 @@ def handle_note(doc) do [author] = :xmerl_xpath.string('/entry/author[1]', doc) {:ok, actor} = find_or_make_user(author) - context = ActivityPub.generate_context_id + context = string_from_xpath("/entry/ostatus:conversation[1]", doc) |> String.trim + context = if String.length(context) > 0 do + context + else + ActivityPub.generate_context_id + end to = [ "https://www.w3.org/ns/activitystreams#Public" diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index cb48c7f5f..e4e26df15 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -253,7 +253,7 @@ def register_user(params) do {:error, changeset} -> errors = Ecto.Changeset.traverse_errors(changeset, fn {msg, _opts} -> msg end) |> Poison.encode! - {:error, %{error: errors}} + {:error, %{error: errors}} end end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 8ee605494..61dca5446 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -9,6 +9,7 @@ test "handle incoming notes" do assert activity.data["type"] == "Create" assert activity.data["object"]["type"] == "Note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" + assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" end describe "new remote user creation" do From f980f6778b1447b808299fa9274854bb25f9823b Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 25 Apr 2017 18:03:14 +0200 Subject: [PATCH 008/107] Wire up mentions. --- lib/pleroma/web/ostatus/ostatus.ex | 21 +++++---------------- test/web/ostatus/ostatus_test.exs | 1 + 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 8c31ce5aa..65141f826 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -31,10 +31,7 @@ def handle_incoming(xml_string) do end # TODO - # Parse mention # wire up replies - # Set correct context - # Set correct statusnet ids. def handle_note(doc) do content_html = string_from_xpath("/entry/content[1]", doc) @@ -52,6 +49,11 @@ def handle_note(doc) do "https://www.w3.org/ns/activitystreams#Public" ] + mentions = :xmerl_xpath.string('/entry/link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', doc) + |> Enum.map(fn(person) -> string_from_xpath("@href", person) end) + + to = to ++ mentions + date = string_from_xpath("/entry/published", doc) object = %{ @@ -66,19 +68,6 @@ def handle_note(doc) do ActivityPub.create(to, actor, context, object, %{}, date) end - def find_or_make(author, doc) do - query = from user in User, - where: user.local == false and fragment("? @> ?", user.info, ^%{ostatus_uri: author}) - - user = Repo.one(query) - - if is_nil(user) do - make_user(doc) - else - {:ok, user} - end - end - def find_or_make_user(author_doc) do {:xmlObj, :string, uri } = :xmerl_xpath.string('string(/author[1]/uri)', author_doc) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 61dca5446..dffebf5a7 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -10,6 +10,7 @@ test "handle incoming notes" do assert activity.data["object"]["type"] == "Note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" + assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] end describe "new remote user creation" do From b91ccef2371fb0bbc23638b174e815dd7189482e Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 26 Apr 2017 08:47:22 +0200 Subject: [PATCH 009/107] Output conversation id. --- lib/pleroma/web/ostatus/activity_representer.ex | 4 +++- lib/pleroma/web/ostatus/feed_representer.ex | 3 ++- test/support/factory.ex | 6 ++++-- test/web/ostatus/activity_representer_test.exs | 2 ++ test/web/ostatus/feed_representer_test.exs | 2 +- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 590abc8bb..367212fe1 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -19,7 +19,9 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) {:title, ['New note by #{user.nickname}']}, {:content, [type: 'html'], h.(activity.data["object"]["content"])}, {:published, h.(inserted_at)}, - {:updated, h.(updated_at)} + {:updated, h.(updated_at)}, + {:"ostatus:conversation", [], h.(activity.data["context"])}, + {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []} ] ++ attachments end diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex index 2cc0da9ba..10a1ffb25 100644 --- a/lib/pleroma/web/ostatus/feed_representer.ex +++ b/lib/pleroma/web/ostatus/feed_representer.ex @@ -17,7 +17,8 @@ def to_simple_form(user, activities, users) do :feed, [ xmlns: 'http://www.w3.org/2005/Atom', "xmlns:activity": 'http://activitystrea.ms/spec/1.0/', - "xmlns:poco": 'http://portablecontacts.net/spec/1.0' + "xmlns:poco": 'http://portablecontacts.net/spec/1.0', + "xmlns:ostatus": 'http://ostatus.org/schema/1.0' ], [ {:id, h.(OStatus.feed_path(user))}, {:title, ['#{user.nickname}\'s timeline']}, diff --git a/test/support/factory.ex b/test/support/factory.ex index d7c16f0e0..d037be4a6 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -24,7 +24,8 @@ def note_factory do "to" => ["https://www.w3.org/ns/activitystreams#Public"], "published_at" => DateTime.utc_now() |> DateTime.to_iso8601, "likes" => [], - "like_count" => 0 + "like_count" => 0, + "context" => "2hu" } %Pleroma.Object{ @@ -40,7 +41,8 @@ def note_activity_factory do "actor" => note.data["actor"], "to" => note.data["to"], "object" => note.data, - "published_at" => DateTime.utc_now() |> DateTime.to_iso8601 + "published_at" => DateTime.utc_now() |> DateTime.to_iso8601, + "context" => note.data["context"] } %Pleroma.Activity{ diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 61df41a1d..10f9a9d0b 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -23,6 +23,8 @@ test "a note activity" do #{note_activity.data["object"]["content"]} #{inserted_at} #{updated_at} + #{note_activity.data["context"]} + """ tuple = ActivityRepresenter.to_simple_form(note_activity, user) diff --git a/test/web/ostatus/feed_representer_test.exs b/test/web/ostatus/feed_representer_test.exs index 13cdeb79d..ef0f4d5ff 100644 --- a/test/web/ostatus/feed_representer_test.exs +++ b/test/web/ostatus/feed_representer_test.exs @@ -22,7 +22,7 @@ test "returns a feed of the last 20 items of the user" do |> :xmerl.export_simple_content(:xmerl_xml) expected = """ - + #{OStatus.feed_path(user)} #{user.nickname}'s timeline #{most_recent_update} From d9ebd785ab7d9b371ba5accdc6ca5d72af7b509d Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 26 Apr 2017 10:08:13 +0200 Subject: [PATCH 010/107] Ostatus doesn't distinguish between activities / objects on create. --- lib/pleroma/web/ostatus/activity_representer.ex | 2 +- test/web/ostatus/activity_representer_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 367212fe1..30e695bcc 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -15,7 +15,7 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) [ {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']}, - {:id, h.(activity.data["object"]["id"])}, + {:id, h.(activity.data["id"])}, {:title, ['New note by #{user.nickname}']}, {:content, [type: 'html'], h.(activity.data["object"]["content"])}, {:published, h.(inserted_at)}, diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 10f9a9d0b..6cea9cff0 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -18,7 +18,7 @@ test "a note activity" do expected = """ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - #{note_activity.data["object"]["id"]} + #{note_activity.data["id"]} New note by #{user.nickname} #{note_activity.data["object"]["content"]} #{inserted_at} From f1ebf812eede5b77931d2315757a7ad8e0ea5a7e Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 26 Apr 2017 10:22:51 +0200 Subject: [PATCH 011/107] Add inReplyTo to incoming messages. --- lib/pleroma/web/ostatus/ostatus.ex | 8 ++++ .../incoming_note_activity_answer.xml | 42 +++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 10 +++++ 3 files changed, 60 insertions(+) create mode 100644 test/fixtures/incoming_note_activity_answer.xml diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 65141f826..5b68f057e 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -65,6 +65,14 @@ def handle_note(doc) do "actor" => actor.ap_id } + inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@href", doc) + + object = if inReplyTo do + Map.put(object, "inReplyTo", inReplyTo) + else + object + end + ActivityPub.create(to, actor, context, object, %{}, date) end diff --git a/test/fixtures/incoming_note_activity_answer.xml b/test/fixtures/incoming_note_activity_answer.xml new file mode 100644 index 000000000..b1244faa6 --- /dev/null +++ b/test/fixtures/incoming_note_activity_answer.xml @@ -0,0 +1,42 @@ + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note + New note by lambda + hey. + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T18:16:13+00:00 + 2017-04-25T18:16:13+00:00 + + http://activitystrea.ms/schema/1.0/person + http://gs.example.org:4040/index.php/user/1 + lambda + + + + + lambda + lambda + + + + + + + http://pleroma.example.org:4000/contexts/8f6f45d4-8e4d-4e1a-a2de-09f27367d2d0 + + + + http://gs.example.org:4040/index.php/api/statuses/user_timeline/1.atom + lambda + + + + http://gs.example.org:4040/theme/neo-gnu/default-avatar-profile.png + 2017-04-25T18:16:13+00:00 + + + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index dffebf5a7..96f2cb4f3 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -13,6 +13,16 @@ test "handle incoming notes" do assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] end + test "handle incoming replies" do + incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") + {:ok, activity} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["inReplyTo"] == "http://pleroma.example.org:4000/objects/55bce8fc-b423-46b1-af71-3759ab4670bc" + assert "http://pleroma.example.org:4000/users/lain5" in activity.data["to"] + end + describe "new remote user creation" do test "make new user or find them based on an 'author' xml doc" do incoming = File.read!("test/fixtures/user_name_only.xml") From 57bd59e4071adf847f94229479e5ffa0951721fd Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 26 Apr 2017 14:25:44 +0200 Subject: [PATCH 012/107] Salmon creation. --- lib/pleroma/web/salmon/salmon.ex | 56 +++++++++++++++++++++++++++++++- test/fixtures/private_key.pem | 27 +++++++++++++++ test/web/salmon/salmon_test.exs | 33 +++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/private_key.pem diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 3881f2758..24b5eb0d9 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -57,7 +57,7 @@ def decode_and_validate(magickey, salmon) do end end - defp decode_key("RSA." <> magickey) do + def decode_key("RSA." <> magickey) do make_integer = fn(bin) -> list = :erlang.binary_to_list(bin) Enum.reduce(list, 0, fn (el, acc) -> (acc <<< 8) ||| el end) @@ -70,4 +70,58 @@ defp decode_key("RSA." <> magickey) do {:RSAPublicKey, modulus, exponent} end + + def encode_key({:RSAPublicKey, modulus, exponent}) do + modulus_enc = :binary.encode_unsigned(modulus) |> Base.url_encode64 + exponent_enc = :binary.encode_unsigned(exponent) |> Base.url_encode64 + + "RSA.#{modulus_enc}.#{exponent_enc}" + end + + def generate_rsa_pem do + port = Port.open({:spawn, "openssl genrsa"}, [:binary]) + {:ok, pem} = receive do + {^port, {:data, pem}} -> {:ok, pem} + end + Port.close(port) + if Regex.match?(~r/RSA PRIVATE KEY/, pem) do + {:ok, pem} + else + :error + end + end + + def keys_from_pem(pem) do + [private_key_code] = :public_key.pem_decode(pem) + private_key = :public_key.pem_entry_decode(private_key_code) + {:RSAPrivateKey, _, modulus, exponent, _, _, _, _, _, _, _} = private_key + public_key = {:RSAPublicKey, modulus, exponent} + {:ok, private_key, public_key} + end + + def encode(private_key, doc) do + type = "application/atom+xml" + encoding = "base64url" + alg = "RSA-SHA256" + + signed_text = [doc, type, encoding, alg] + |> Enum.map(&Base.url_encode64/1) + |> Enum.join(".") + + signature = :public_key.sign(signed_text, :sha256, private_key) |> to_string |> Base.url_encode64 + doc_base64= doc |> Base.url_encode64 + + # Don't need proper xml building, these strings are safe to leave unescaped + salmon = """ + + + #{doc_base64} + #{encoding} + #{alg} + #{signature} + + """ + + {:ok, salmon} + end end diff --git a/test/fixtures/private_key.pem b/test/fixtures/private_key.pem new file mode 100644 index 000000000..7a4b14654 --- /dev/null +++ b/test/fixtures/private_key.pem @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAqnWeDtrqWasCKNXiuSq1tSCLI5H7BSvIROy5YfuGsXHrIlCq +LdIm9QlIUUmIi9QyzgiGEDsPCCkA1UguCVgF/UrJ1+FvHcHsTELkkBu/yCl9mrgt +WzTckhb6KjOhqtxi/TKgRaJ2Rlwz2bvH5sbCP9qffthitdxfh14KC5V0gqDt1xCy +WgZo79vbYMcVkcQoh5uLtG64ksYFBMfgnLaSj7xg5i2qCDiIY7bqBujo5HllDqeo +w3LXmsztt1cT8heXEjW0SYJvAHJK00OsG1kp4cqhfKzxLCHNGQJVHQxLOXy97I7o +HOeuhbxPhjpGSBMgw7YFm3ODXviqf557eqFcaQIDAQABAoIBAC6f+VnK22sncXHF +/zvyyL0AZ86U8XpanW7s6VA5wn/qzwwV0Fa0Mt+3aEaDvIuywSrF/hWWcegjfwzX +r2/y2cCMomUgTopvLrk1WttoG68eWjLlydI2xVZYXpkIgmH/4juri1dAtuVL9wrJ +aEZhe2SH4jSJ74Ya/y5BtLGycaoA9FHyIzHPTx52Ix2jWKWtKimW8J+aERi2uHdN +7yTnLT2APhs5fnvNnn0tg85CI3Ny2GNiqmAail14yVfRz8Sf6qDIepH5Jfz9oll4 +I+GYUOLs6eTgkHXBn8LGhtHTE/9UJmb42OyWrW8X+nc/Mjz5xh0u/g1Gdp36oUMz +OotfneECgYEA3cGfQxmxjEqSbXt9jbxiCukU7PmkDDQqBu97URC4N8qEcMF1wW7X +AddU7Kq/UJU+oqjD/7UQHoS2ZThPtto6SpVdXQzsnrnPWQcrv5b1DV/TpXfwGoZ3 +svUIAcx4vGzhhmHDJCBsdY6n8xWBYtSqfLFXgN5UkdafLGy3EkCEtmUCgYEAxMgl +7eU2QkWkzgJxOj6xjG2yqM3jxOvvoiRnD0rIQaBS70P/1N94ZkMXzOwddddZ5OW+ +55h/a8TmFKP/+NW4PHRYra/dazGI4IBlw6Yeq6uq/4jbuSqtBbaNn/Dz5kdHBTqM +PtbBvc9Fztd2zb3InyyLbb4c+WjMqi0AooN027UCgYB4Tax7GJtLwsEBiDcrB4Ig +7SYfEae/vyT1skIyTmHCUqnbCfk6QUl/hDRcWJ2FuBHM6MW8GZxvEgxpiU0lo+pv +v+xwqKxNx/wHDm7bd6fl45DMee7WVRDnEyuO3kC56E/JOYxGMxjkBcpzg703wqvj +Dcqs7PDwVYDw9uGykzHsSQKBgEQnNcvA+RvW1w9qlSChGgkS7S+9r0dCl8pGZVNM +iTMBfffUS0TE6QQx9IpKtKFdpoq6b3XywR7oIO/BJSRfkOGPQi9Vm5BGpatrjNNI +M5Mtb5n1InRtLWOvKDnez/pPcW+EKZKR+qPsp7bNtR3ovxUx7lBh6dMP0uKVl4Sx +lsWJAoGBAIeek9eG+S3m2jaJRHasfKo5mJ2JrrmnjQXUOGUP8/CgO8sW1VmG2WAk +Av7+BRI2mP2f+3SswG/AoRGmRXXw65ly63ws8ixrhK0MG3MgqDkWc69SbTaaMJ+u +BQFYMsB1vZdUV3CaRqySkjY68QWGcJ4Z5JKHuTXzKv/GeFmw0V9R +-----END RSA PRIVATE KEY----- diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index 4ebb32081..6fbabd19f 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -16,4 +16,37 @@ test "errors on wrong magic key" do {:ok, salmon} = File.read("test/fixtures/salmon.xml") assert Salmon.decode_and_validate(@wrong_magickey, salmon) == :error end + + test "generates an RSA private key pem" do + {:ok, key} = Salmon.generate_rsa_pem + assert is_binary(key) + assert Regex.match?(~r/RSA/, key) + end + + test "it encodes a magic key from a public key" do + key = Salmon.decode_key(@magickey) + magic_key = Salmon.encode_key(key) + + assert @magickey == magic_key + end + + test "returns a public and private key from a pem" do + pem = File.read!("test/fixtures/private_key.pem") + {:ok, private, public} = Salmon.keys_from_pem(pem) + + assert elem(private, 0) == :RSAPrivateKey + assert elem(public, 0) == :RSAPublicKey + end + + test "encodes an xml payload with a private key" do + doc = File.read!("test/fixtures/incoming_note_activity.xml") + pem = File.read!("test/fixtures/private_key.pem") + {:ok, private, public} = Salmon.keys_from_pem(pem) + + # Let's try a roundtrip. + {:ok, salmon} = Salmon.encode(private, doc) + {:ok, decoded_doc} = Salmon.decode_and_validate(Salmon.encode_key(public), salmon) + + assert doc == decoded_doc + end end From c5fa682c317717c64168bf2d77b28d805ffff450 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 26 Apr 2017 18:33:10 +0200 Subject: [PATCH 013/107] Refactor, add beginnings of websub client subscriptions. --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/federator/federator.ex | 32 +++++++++++++++++++ lib/pleroma/web/websub/websub.ex | 26 ++++++++++++--- .../web/websub/websub_client_subscription.ex | 13 ++++++++ ...4155_create_websub_client_subscription.exs | 15 +++++++++ test/web/websub/websub_test.exs | 12 ++++++- 6 files changed, 93 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/web/federator/federator.ex create mode 100644 lib/pleroma/web/websub/websub_client_subscription.ex create mode 100644 priv/repo/migrations/20170426154155_create_websub_client_subscription.exs diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 7264123d8..82f9fcc1c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -36,7 +36,7 @@ def create(to, actor, context, object, additional \\ %{}, published \\ nil) do {:ok, activity} = add_conversation_id(activity) if actor.local do - Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity) + Pleroma.Web.Federator.enqueue(:publish, activity) end {:ok, activity} diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex new file mode 100644 index 000000000..f489ed837 --- /dev/null +++ b/lib/pleroma/web/federator/federator.ex @@ -0,0 +1,32 @@ +defmodule Pleroma.Web.Federator do + alias Pleroma.User + require Logger + + @websub_verifier Application.get_env(:pleroma, :websub_verifier) + + def handle(:publish, activity) do + Logger.debug("Running publish for #{activity.data["id"]}") + with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do + Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity) + end + end + + def handle(:verify_websub, websub) do + Logger.debug("Running websub verification for #{websub.id} (#{websub.topic}, #{websub.callback})") + @websub_verifier.verify(websub) + end + + def handle(type, payload) do + Logger.debug("Unknown task: #{type}") + {:error, "Don't know what do do with this"} + end + + def enqueue(type, payload) do + # for now, just run immediately in a new process. + if Mix.env == :test do + handle(type, payload) + else + spawn(fn -> handle(type, payload) end) + end + end +end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index cc66b52dd..03b0aec8f 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -1,13 +1,11 @@ defmodule Pleroma.Web.Websub do alias Pleroma.Repo - alias Pleroma.Web.Websub.WebsubServerSubscription + alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription} alias Pleroma.Web.OStatus.FeedRepresenter alias Pleroma.Web.OStatus import Ecto.Query - @websub_verifier Application.get_env(:pleroma, :websub_verifier) - def verify(subscription, getter \\ &HTTPoison.get/3 ) do challenge = Base.encode16(:crypto.strong_rand_bytes(8)) lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at) |> to_string @@ -71,8 +69,7 @@ def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) d change = Ecto.Changeset.change(websub, %{valid_until: NaiveDateTime.add(websub.updated_at, lease_time)}) websub = Repo.update!(change) - # Just spawn that for now, maybe pool later. - spawn(fn -> @websub_verifier.verify(websub) end) + Pleroma.Web.Federator.enqueue(:verify_websub, websub) {:ok, websub} else {:error, reason} -> @@ -99,4 +96,23 @@ defp valid_topic(%{"hub.topic" => topic}, user) do {:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"} end end + + def subscribe(user, topic) do + # Race condition, use transactions + {:ok, subscription} = with subscription when not is_nil(subscription) <- Repo.get_by(WebsubClientSubscription, topic: topic) do + subscribers = [user.ap_id, subscription.subcribers] |> Enum.uniq + change = Ecto.Changeset.change(subscription, %{subscribers: subscribers}) + Repo.update(change) + else _e -> + subscription = %WebsubClientSubscription{ + topic: topic, + subscribers: [user.ap_id], + state: "requested", + secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64 + } + Repo.insert(subscription) + end + + {:ok, subscription} + end end diff --git a/lib/pleroma/web/websub/websub_client_subscription.ex b/lib/pleroma/web/websub/websub_client_subscription.ex new file mode 100644 index 000000000..341e27c51 --- /dev/null +++ b/lib/pleroma/web/websub/websub_client_subscription.ex @@ -0,0 +1,13 @@ +defmodule Pleroma.Web.Websub.WebsubClientSubscription do + use Ecto.Schema + + schema "websub_client_subscriptions" do + field :topic, :string + field :secret, :string + field :valid_until, :naive_datetime + field :state, :string + field :subscribers, {:array, :string}, default: [] + + timestamps() + end +end diff --git a/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs b/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs new file mode 100644 index 000000000..f42782840 --- /dev/null +++ b/priv/repo/migrations/20170426154155_create_websub_client_subscription.exs @@ -0,0 +1,15 @@ +defmodule Pleroma.Repo.Migrations.CreateWebsubClientSubscription do + use Ecto.Migration + + def change do + create table(:websub_client_subscriptions) do + add :topic, :string + add :secret, :string + add :valid_until, :naive_datetime + add :state, :string + add :subscribers, :map + + timestamps() + end + end +end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 334ba03fc..7b77e696b 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -58,7 +58,6 @@ test "an incoming subscription request" do "hub.lease_seconds" => "100" } - {:ok, subscription } = Websub.incoming_subscription_request(user, data) assert subscription.topic == Pleroma.Web.OStatus.feed_path(user) assert subscription.state == "requested" @@ -87,4 +86,15 @@ test "an incoming subscription request for an existing subscription" do assert length(Repo.all(WebsubServerSubscription)) == 1 assert subscription.id == sub.id end + + test "initiate a subscription for a given user and topic" do + user = insert(:user) + topic = "http://example.org/some-topic.atom" + + {:ok, websub} = Websub.subscribe(user, topic) + assert websub.subscribers == [user.ap_id] + assert websub.topic == topic + assert is_binary(websub.secret) + assert websub.state == "accepted" + end end From d1dce56a85e041f78e1d50900a0c9591610de2b9 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 27 Apr 2017 09:43:58 +0200 Subject: [PATCH 014/107] Refactor XML parsing. --- lib/pleroma/web/ostatus/ostatus.ex | 13 ++----------- lib/pleroma/web/salmon/salmon.ex | 8 +++++--- lib/pleroma/web/websub/websub.ex | 1 + lib/pleroma/web/xml/xml.ex | 19 +++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 5 +++-- 5 files changed, 30 insertions(+), 16 deletions(-) create mode 100644 lib/pleroma/web/xml/xml.ex diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 5b68f057e..89b482592 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Web.OStatus do import Ecto.Query + import Pleroma.Web.XML require Logger alias Pleroma.{Repo, User, Web} @@ -18,7 +19,7 @@ def salmon_path(user) do end def handle_incoming(xml_string) do - {doc, _rest} = :xmerl_scan.string(to_charlist(xml_string)) + doc = parse_document(xml_string) {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', doc) @@ -91,16 +92,6 @@ def find_or_make_user(author_doc) do end end - defp string_from_xpath(xpath, doc) do - {:xmlObj, :string, res} = :xmerl_xpath.string('string(#{xpath})', doc) - - res = res - |> to_string - |> String.trim - - if res == "", do: nil, else: res - end - def make_user(author_doc) do author = string_from_xpath("/author[1]/uri", author_doc) name = string_from_xpath("/author[1]/name", author_doc) diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 24b5eb0d9..99cca1f55 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -1,8 +1,9 @@ defmodule Pleroma.Web.Salmon do use Bitwise + alias Pleroma.Web.XML def decode(salmon) do - {doc, _rest} = :xmerl_scan.string(to_charlist(salmon)) + doc = XML.parse_document(salmon) {:xmlObj, :string, data} = :xmerl_xpath.string('string(//me:data[1])', doc) {:xmlObj, :string, sig} = :xmerl_xpath.string('string(//me:sig[1])', doc) @@ -22,16 +23,17 @@ def decode(salmon) do def fetch_magic_key(salmon) do [data, _, _, _, _] = decode(salmon) - {doc, _rest} = :xmerl_scan.string(to_charlist(data)) + doc = XML.parse_document(data) {:xmlObj, :string, uri} = :xmerl_xpath.string('string(//author[1]/uri)', doc) uri = to_string(uri) base = URI.parse(uri).host # TODO: Find out if this endpoint is mandated by the standard. + # At least diaspora does it differently {:ok, response} = HTTPoison.get(base <> "/.well-known/webfinger", ["Accept": "application/xrd+xml"], [params: [resource: uri]]) - {doc, _rest} = :xmerl_scan.string(to_charlist(response.body)) + doc = XML.parse_document(response.body) {:xmlObj, :string, magickey} = :xmerl_xpath.string('string(//Link[@rel="magic-public-key"]/@href)', doc) "data:application/magic-public-key," <> magickey = to_string(magickey) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 03b0aec8f..5372416e6 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -3,6 +3,7 @@ defmodule Pleroma.Web.Websub do alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription} alias Pleroma.Web.OStatus.FeedRepresenter alias Pleroma.Web.OStatus + alias Pleroma.Web.XML import Ecto.Query diff --git a/lib/pleroma/web/xml/xml.ex b/lib/pleroma/web/xml/xml.ex new file mode 100644 index 000000000..22faf72df --- /dev/null +++ b/lib/pleroma/web/xml/xml.ex @@ -0,0 +1,19 @@ +defmodule Pleroma.Web.XML do + def string_from_xpath(xpath, doc) do + {:xmlObj, :string, res} = :xmerl_xpath.string('string(#{xpath})', doc) + + res = res + |> to_string + |> String.trim + + if res == "", do: nil, else: res + end + + def parse_document(text) do + {doc, _rest} = text + |> :binary.bin_to_list + |> :xmerl_scan.string + + doc + end +end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 96f2cb4f3..140b32f36 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -1,6 +1,7 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus + alias Pleroma.Web.XML test "handle incoming notes" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") @@ -26,7 +27,7 @@ test "handle incoming replies" do describe "new remote user creation" do test "make new user or find them based on an 'author' xml doc" do incoming = File.read!("test/fixtures/user_name_only.xml") - {doc, _rest} = :xmerl_scan.string(to_charlist(incoming)) + doc = XML.parse_document(incoming) {:ok, user} = OStatus.find_or_make_user(doc) @@ -44,7 +45,7 @@ test "make new user or find them based on an 'author' xml doc" do test "tries to use the information in poco fields" do incoming = File.read!("test/fixtures/user_full.xml") - {doc, _rest} = :xmerl_scan.string(to_charlist(incoming)) + doc = XML.parse_document(incoming) {:ok, user} = OStatus.find_or_make_user(doc) From e8a311ecffe1fb19a3d194b1c5628853263909a7 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 27 Apr 2017 09:44:20 +0200 Subject: [PATCH 015/107] Add user and hub to websub client subscriptions. --- lib/pleroma/web/websub/websub_client_subscription.ex | 3 +++ .../migrations/20170427054757_add_user_and_hub.exs | 10 ++++++++++ 2 files changed, 13 insertions(+) create mode 100644 priv/repo/migrations/20170427054757_add_user_and_hub.exs diff --git a/lib/pleroma/web/websub/websub_client_subscription.ex b/lib/pleroma/web/websub/websub_client_subscription.ex index 341e27c51..c7a25ea22 100644 --- a/lib/pleroma/web/websub/websub_client_subscription.ex +++ b/lib/pleroma/web/websub/websub_client_subscription.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do use Ecto.Schema + alias Pleroma.User schema "websub_client_subscriptions" do field :topic, :string @@ -7,6 +8,8 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do field :valid_until, :naive_datetime field :state, :string field :subscribers, {:array, :string}, default: [] + field :hub, :string + belongs_to :user, User timestamps() end diff --git a/priv/repo/migrations/20170427054757_add_user_and_hub.exs b/priv/repo/migrations/20170427054757_add_user_and_hub.exs new file mode 100644 index 000000000..4f9a520bd --- /dev/null +++ b/priv/repo/migrations/20170427054757_add_user_and_hub.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddUserAndHub do + use Ecto.Migration + + def change do + alter table(:websub_client_subscriptions) do + add :hub, :string + add :user_id, references(:users) + end + end +end From 1ea4325fecaed981b2f949b00d3b37171013012c Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 27 Apr 2017 09:46:04 +0200 Subject: [PATCH 016/107] Add user feed fixture. --- test/fixtures/lambadalambda.atom | 479 +++++++++++++++++++++++++++++++ 1 file changed, 479 insertions(+) create mode 100644 test/fixtures/lambadalambda.atom diff --git a/test/fixtures/lambadalambda.atom b/test/fixtures/lambadalambda.atom new file mode 100644 index 000000000..35e506420 --- /dev/null +++ b/test/fixtures/lambadalambda.atom @@ -0,0 +1,479 @@ + + + https://mastodon.social/users/lambadalambda.atom + Critical Value + + 2017-04-16T21:47:25Z + https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244 + + https://mastodon.social/users/lambadalambda + http://activitystrea.ms/schema/1.0/person + https://mastodon.social/users/lambadalambda + lambadalambda + lambadalambda@mastodon.social + + + + + lambadalambda + Critical Value + public + + + + + + + + tag:mastodon.social,2017-04-07:objectId=1874242:objectType=Status + 2017-04-07T11:02:56Z + 2017-04-07T11:02:56Z + lambadalambda shared a status by 0xroy@social.wxcafe.net + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:social.wxcafe.net,2017-04-07:objectId=72554:objectType=Status + 2017-04-07T11:01:59Z + 2017-04-07T11:02:00Z + New status by 0xroy@social.wxcafe.net + + https://social.wxcafe.net/users/0xroy + http://activitystrea.ms/schema/1.0/person + https://social.wxcafe.net/users/0xroy + 0xroy + 0xroy@social.wxcafe.net + ta caution weeb | discussions privées : <a href="https://💌.0xroy.me" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">💌.0xroy.me</span><span class="invisible"></span></a> + + + + 0xroy + 「R O Y 🍵 B O S」 + ta caution weeb | discussions privées : <a href="https://%F0%9F%92%8C.0xroy.me" rel="nofollow noopener"><span class="invisible">https://</span><span class="">💌.0xroy.me</span><span class="invisible"></span></a> + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>someone pls eli5 matrix (protocol) and riot</p> + + public + + + <p>someone pls eli5 matrix (protocol) and riot</p> + + public + + + + + tag:mastodon.social,2017-04-06:objectId=1768247:objectType=Status + 2017-04-06T11:10:19Z + 2017-04-06T11:10:19Z + lambadalambda shared a status by areyoutoo@mastodon.xyz + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:mastodon.xyz,2017-04-05:objectId=133327:objectType=Status + 2017-04-05T17:36:41Z + 2017-04-05T18:12:14Z + New status by areyoutoo@mastodon.xyz + + https://mastodon.xyz/users/areyoutoo + http://activitystrea.ms/schema/1.0/person + https://mastodon.xyz/users/areyoutoo + areyoutoo + areyoutoo@mastodon.xyz + devops | retired gamedev | always boost puppy pics + + + + areyoutoo + Raw Butter + devops | retired gamedev | always boost puppy pics + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>Some UX thoughts for <a href="https://mastodon.xyz/tags/mastodev" class="mention hashtag">#<span>mastodev</span></a>:</p><p>- Would be nice if I could work on multiple draft toots? Clicking to reply to someone seems to erase any draft I had been working on.</p><p>- Kinda risky to click on the Federated Timeline if it loads new toots and scrolls 10ms before I click on something.</p><p>I probably don't know enough web frontend to help, but it might be fun to try.</p> + + + public + + + <p>Some UX thoughts for <a href="https://mastodon.xyz/tags/mastodev" class="mention hashtag">#<span>mastodev</span></a>:</p><p>- Would be nice if I could work on multiple draft toots? Clicking to reply to someone seems to erase any draft I had been working on.</p><p>- Kinda risky to click on the Federated Timeline if it loads new toots and scrolls 10ms before I click on something.</p><p>I probably don't know enough web frontend to help, but it might be fun to try.</p> + + public + + + + + tag:mastodon.social,2017-04-06:objectId=1764509:objectType=Status + 2017-04-06T10:15:38Z + 2017-04-06T10:15:38Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + This is a test for cw federation + <p>This is a test for cw federation body text.</p> + + public + + + + + tag:mastodon.social,2017-04-05:objectId=1645208:objectType=Status + 2017-04-05T07:14:53Z + 2017-04-05T07:14:53Z + lambadalambda shared a status by lambadalambda@social.heldscal.la + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:social.heldscal.la,2017-04-05:noticeId=1502088:objectType=note + 2017-04-05T06:12:09Z + 2017-04-05T07:12:47Z + New status by lambadalambda@social.heldscal.la + + https://social.heldscal.la/user/23211 + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + lambadalambda@social.heldscal.la + Call me Deacon Blues. + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + Federation 101: <a href="https://www.youtube.com/watch?v=t1lYU5CA40o" rel="nofollow external noreferrer" class="attachment thumbnail">https://www.youtube.com/watch?v=t1lYU5CA40o</a> + + public + + + Federation 101: <a href="https://www.youtube.com/watch?v=t1lYU5CA40o" rel="nofollow external noreferrer" class="attachment thumbnail">https://www.youtube.com/watch?v=t1lYU5CA40o</a> + + public + + + + + tag:mastodon.social,2017-04-05:objectId=1641750:objectType=Status + 2017-04-05T05:44:48Z + 2017-04-05T05:44:48Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> just a test.</p> + + + public + + + + + tag:mastodon.social,2017-04-04:objectId=1540149:objectType=Status + 2017-04-04T06:31:09Z + 2017-04-04T06:31:09Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>Looks like you still can&apos;t delete your account here (PRIVACY!), but I won&apos;t be posting here anymore, my main account is <span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span></p> + + + public + + + + + tag:mastodon.social,2017-04-04:objectId=1539608:objectType=Status + 2017-04-04T06:18:16Z + 2017-04-04T06:18:16Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@ghostbar" class="u-url mention">@<span>ghostbar</span></a></span> Remember to rewrite it in Rust once you&apos;re done.</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1504813:objectType=Status + 2017-04-03T18:01:20Z + 2017-04-03T18:01:20Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.xyz/@Azurolu" class="u-url mention">@<span>Azurolu</span></a></span> You mean gs.smuglo.li?</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1504805:objectType=Status + 2017-04-03T18:01:05Z + 2017-04-03T18:01:05Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>There&apos;s nothing wrong with having several alt accounts all across the fediverse. Try out another mastodon instance (<a href="https://icosahedron.website" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">icosahedron.website</span><span class="invisible"></span></a>) or a GNU Social instance (like <a href="https://shitposter.club" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">shitposter.club</span><span class="invisible"></span></a> or <a href="https://freezepeach.xyz" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">freezepeach.xyz</span><span class="invisible"></span></a>), or friendica. They are all on the same network, so you can still follow all your friends!</p> + + public + + + + + tag:mastodon.social,2017-04-03:objectId=1503965:objectType=Status + 2017-04-03T17:31:30Z + 2017-04-03T17:31:30Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@20Hz" class="u-url mention">@<span>20Hz</span></a></span> you could also try out a GS instance, which are on the same network :)</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1503955:objectType=Status + 2017-04-03T17:31:08Z + 2017-04-03T17:31:08Z + lambadalambda shared a status by shpuld@shitposter.club + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:shitposter.club,2017-04-03:noticeId=2251717:objectType=note + 2017-04-03T17:06:43Z + 2017-04-03T17:12:06Z + New status by shpuld@shitposter.club + + https://shitposter.club/user/5381 + http://activitystrea.ms/schema/1.0/person + https://shitposter.club/user/5381 + shpuld + shpuld@shitposter.club + + + + + shpuld + shp + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + reposting the classic <a href="https://shitposter.club/file/89c5fe483526caf3a46cfc5cdd4ae68061054350e767397731af658d54786e31.jpg" class="attachment" rel="nofollow external">https://shitposter.club/attachment/219846</a> + + + public + + + reposting the classic <a href="https://shitposter.club/file/89c5fe483526caf3a46cfc5cdd4ae68061054350e767397731af658d54786e31.jpg" class="attachment" rel="nofollow external">https://shitposter.club/attachment/219846</a> + + public + + + + + tag:mastodon.social,2017-04-03:objectId=1503929:objectType=Status + 2017-04-03T17:30:43Z + 2017-04-03T17:30:43Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@ghostbar" class="u-url mention">@<span>ghostbar</span></a></span> Normally you shouldn&apos;t be running tens of thousands of users on one instance... That&apos;s one of the reasons for federation.</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1477255:objectType=Status + 2017-04-03T08:24:39Z + 2017-04-03T08:24:39Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@dot_tiff" class="u-url mention">@<span>dot_tiff</span></a></span> it&apos;s the vaporwave mode.</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1476210:objectType=Status + 2017-04-03T07:45:42Z + 2017-04-03T07:45:42Z + lambadalambda shared a status by lambadalambda@social.heldscal.la + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:social.heldscal.la,2017-04-03:noticeId=1475727:objectType=note + 2017-04-03T07:44:43Z + 2017-04-03T07:44:48Z + New status by lambadalambda@social.heldscal.la + + https://social.heldscal.la/user/23211 + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + lambadalambda@social.heldscal.la + Call me Deacon Blues. + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + Here's a song by the original anti-idol, Togawa Jun: <a href="https://www.youtube.com/watch?v=kNI_NK2YY-s" rel="nofollow external noreferrer" class="attachment">https://www.youtube.com/watch?v=kNI_NK2YY-s</a> + + public + + + Here's a song by the original anti-idol, Togawa Jun: <a href="https://www.youtube.com/watch?v=kNI_NK2YY-s" rel="nofollow external noreferrer" class="attachment">https://www.youtube.com/watch?v=kNI_NK2YY-s</a> + + public + + + + + tag:mastodon.social,2017-04-03:objectId=1476047:objectType=Status + 2017-04-03T07:39:14Z + 2017-04-03T07:39:14Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@amrrr" class="u-url mention">@<span>amrrr</span></a></span> tumblr/10, but pretty good!</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1475949:objectType=Status + 2017-04-03T07:35:45Z + 2017-04-03T07:35:45Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@Shookaite" class="u-url mention">@<span>Shookaite</span></a></span> Oh, you mean like userstyles?</p> + + + public + + + + + + tag:mastodon.social,2017-04-03:objectId=1475581:objectType=Status + 2017-04-03T07:20:03Z + 2017-04-03T07:20:03Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@Shookaite" class="u-url mention">@<span>Shookaite</span></a></span> Would be nice if someone helped port Pleroma to Mastodon, that has a theme switcher (click on the cog in the upper right): <a href="https://pleroma.heldscal.la/main/all" rel="nofollow noopener" target="_blank"><span class="invisible">https://</span><span class="">pleroma.heldscal.la/main/all</span><span class="invisible"></span></a></p> + + + public + + + + + + tag:mastodon.social,2017-04-02:objectId=1457325:objectType=Status + 2017-04-02T21:57:43Z + 2017-04-02T21:57:43Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://mastodon.social/@rhosyn" class="u-url mention">@<span>rhosyn</span></a></span> <span class="h-card"><a href="https://mastodon.social/@Meaningness" class="u-url mention">@<span>Meaningness</span></a></span> you could take a look at those listed at social.guhnoo.org</p> + + + + public + + + + + + tag:mastodon.social,2017-04-02:objectId=1447926:objectType=Status + 2017-04-02T18:31:52Z + 2017-04-02T18:31:52Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>My main account is <span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> , btw.</p> + + + public + + + + + tag:mastodon.social,2017-04-02:objectId=1447878:objectType=Status + 2017-04-02T18:30:37Z + 2017-04-02T18:30:37Z + lambadalambda shared a status by Firstaide@awoo.space + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:awoo.space,2017-04-02:objectId=135324:objectType=Status + 2017-04-02T18:29:32Z + 2017-04-02T18:29:32Z + New status by Firstaide@awoo.space + + https://awoo.space/users/Firstaide + http://activitystrea.ms/schema/1.0/person + https://awoo.space/users/Firstaide + Firstaide + Firstaide@awoo.space + A smol awoo account, for a smol autistic 💙 +They/them please! +NB/white/ace + + + + Firstaide + Miff🚑✨ + A smol awoo account, for a smol autistic 💙 +They/them please! +NB/white/ace + public + + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><a href="https://mastodon.social/users/lambadalambda" class="h-card u-url p-nickname mention">@<span>lambadalambda</span></a> yeah, I think that's p much the big issue here? <br>When I first heard of Masto, I thought it was just like twitter at first, I had no idea federation was even a thing?, and I actually joined p early on? :-o </p><p>idk I think more stuff needs to be done about federation promotion, but honestly its gotta come from the get go when people get here to make an account I feel :-o</p> + + + public + + + + <p><a href="https://mastodon.social/users/lambadalambda" class="h-card u-url p-nickname mention">@<span>lambadalambda</span></a> yeah, I think that's p much the big issue here? <br>When I first heard of Masto, I thought it was just like twitter at first, I had no idea federation was even a thing?, and I actually joined p early on? :-o </p><p>idk I think more stuff needs to be done about federation promotion, but honestly its gotta come from the get go when people get here to make an account I feel :-o</p> + + public + + + + From 90da25505f9cfbd16a9088e20714b24c2c6fa215 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 27 Apr 2017 09:46:45 +0200 Subject: [PATCH 017/107] Add discovery and subscription requests to websub. --- lib/pleroma/web/websub/websub.ex | 58 +++++++++++++++++++++++++++--- test/support/factory.ex | 10 ++++++ test/web/websub/websub_test.exs | 61 ++++++++++++++++++++++++++++++-- 3 files changed, 123 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 5372416e6..4a35ca8fc 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.Websub do alias Pleroma.Web.OStatus.FeedRepresenter alias Pleroma.Web.OStatus alias Pleroma.Web.XML + require Logger import Ecto.Query @@ -98,8 +99,8 @@ defp valid_topic(%{"hub.topic" => topic}, user) do end end - def subscribe(user, topic) do - # Race condition, use transactions + def subscribe(user, topic, requester \\ &request_subscription/1) do + # FIXME: Race condition, use transactions {:ok, subscription} = with subscription when not is_nil(subscription) <- Repo.get_by(WebsubClientSubscription, topic: topic) do subscribers = [user.ap_id, subscription.subcribers] |> Enum.uniq change = Ecto.Changeset.change(subscription, %{subscribers: subscribers}) @@ -109,11 +110,60 @@ def subscribe(user, topic) do topic: topic, subscribers: [user.ap_id], state: "requested", - secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64 + secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64, + user: user } Repo.insert(subscription) end + requester.(subscription) + end - {:ok, subscription} + def discover(topic, getter \\ &HTTPoison.get/1) do + with {:ok, response} <- getter.(topic), + status_code when status_code in 200..299 <- response.status_code, + body <- response.body, + doc <- XML.parse_document(body), + url when not is_nil(url) <- XML.string_from_xpath(~S{/feed/link[@rel="self"]/@href}, doc), + hub when not is_nil(hub) <- XML.string_from_xpath(~S{/feed/link[@rel="hub"]/@href}, doc) do + {:ok, %{url: url, hub: hub}} + else e -> + {:error, e} + end + end + + def request_subscription(websub, poster \\ &HTTPoison.post/3, timeout \\ 10_000) do + data = [ + "hub.mode": "subscribe", + "hub.topic": websub.topic, + "hub.secret": websub.secret, + "hub.callback": "https://social.heldscal.la/callback" + ] + + # This checks once a second if we are confirmed yet + websub_checker = fn -> + helper = fn (helper) -> + :timer.sleep(1000) + websub = Repo.get_by(WebsubClientSubscription, id: websub.id, state: "accepted") + if websub, do: websub, else: helper.(helper) + end + helper.(helper) + end + + task = Task.async(websub_checker) + + with {:ok, %{status_code: 202}} <- poster.(websub.hub, {:form, data}, ["Content-type": "application/x-www-form-urlencoded"]), + {:ok, websub} <- Task.yield(task, timeout) do + {:ok, websub} + else e -> + Task.shutdown(task) + + change = Ecto.Changeset.change(websub, %{state: "rejected"}) + {:ok, websub} = Repo.update(change) + + Logger.debug("Couldn't confirm subscription: #{inspect(websub)}") + Logger.debug("error: #{inspect(e)}") + + {:error, websub} + end end end diff --git a/test/support/factory.ex b/test/support/factory.ex index d037be4a6..ac276567a 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -76,4 +76,14 @@ def websub_subscription_factory do state: "requested" } end + + def websub_client_subscription_factory do + %Pleroma.Web.Websub.WebsubClientSubscription{ + topic: "http://example.org", + secret: "here's a secret", + valid_until: nil, + state: "requested", + subscribers: [] + } + end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 7b77e696b..bf243ac91 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -77,7 +77,6 @@ test "an incoming subscription request for an existing subscription" do "hub.lease_seconds" => "100" } - {:ok, subscription } = Websub.incoming_subscription_request(user, data) assert subscription.topic == Pleroma.Web.OStatus.feed_path(user) assert subscription.state == sub.state @@ -87,14 +86,72 @@ test "an incoming subscription request for an existing subscription" do assert subscription.id == sub.id end + def accepting_verifier(subscription) do + {:ok, %{ subscription | state: "accepted" }} + end + test "initiate a subscription for a given user and topic" do user = insert(:user) topic = "http://example.org/some-topic.atom" - {:ok, websub} = Websub.subscribe(user, topic) + {:ok, websub} = Websub.subscribe(user, topic, &accepting_verifier/1) assert websub.subscribers == [user.ap_id] assert websub.topic == topic assert is_binary(websub.secret) + assert websub.user == user assert websub.state == "accepted" end + + test "discovers the hub and canonical url" do + topic = "https://mastodon.social/users/lambadalambda.atom" + + getter = fn(^topic) -> + doc = File.read!("test/fixtures/lambadalambda.atom") + {:ok, %{status_code: 200, body: doc}} + end + + {:ok, discovered} = Websub.discover(topic, getter) + assert %{hub: "https://mastodon.social/api/push", url: topic} == discovered + end + + test "calls the hub, requests topic" do + hub = "https://social.heldscal.la/main/push/hub" + topic = "https://social.heldscal.la/api/statuses/user_timeline/23211.atom" + websub = insert(:websub_client_subscription, %{hub: hub, topic: topic}) + + poster = fn (^hub, {:form, data}, _headers) -> + assert Keyword.get(data, :"hub.mode") == "subscribe" + {:ok, %{status_code: 202}} + end + + task = Task.async(fn -> Websub.request_subscription(websub, poster) end) + + change = Ecto.Changeset.change(websub, %{state: "accepted"}) + {:ok, _} = Repo.update(change) + + {:ok, websub} = Task.await(task) + + assert websub.state == "accepted" + end + + test "rejects the subscription if it can't be accepted" do + hub = "https://social.heldscal.la/main/push/hub" + topic = "https://social.heldscal.la/api/statuses/user_timeline/23211.atom" + websub = insert(:websub_client_subscription, %{hub: hub, topic: topic}) + + poster = fn (^hub, {:form, _data}, _headers) -> + {:ok, %{status_code: 202}} + end + + {:error, websub} = Websub.request_subscription(websub, poster, 1000) + assert websub.state == "rejected" + + websub = insert(:websub_client_subscription, %{hub: hub, topic: topic}) + poster = fn (^hub, {:form, _data}, _headers) -> + {:ok, %{status_code: 400}} + end + + {:error, websub} = Websub.request_subscription(websub, poster, 1000) + assert websub.state == "rejected" + end end From 6cf7c132282e612514af992c6dea0b03ee5b678d Mon Sep 17 00:00:00 2001 From: dtluna Date: Thu, 27 Apr 2017 16:18:50 +0300 Subject: [PATCH 018/107] Refactor code to comply with credo suggestions --- .credo.exs | 138 ++++++++++++++++++ lib/pleroma/plugs/authentication_plug.ex | 7 +- lib/pleroma/upload.ex | 12 +- lib/pleroma/user.ex | 18 +-- lib/pleroma/web/activity_pub/activity_pub.ex | 19 ++- .../web/ostatus/activity_representer.ex | 2 +- lib/pleroma/web/ostatus/feed_representer.ex | 3 +- lib/pleroma/web/ostatus/ostatus_controller.ex | 5 +- lib/pleroma/web/ostatus/user_representer.ex | 16 +- lib/pleroma/web/router.ex | 6 +- lib/pleroma/web/salmon/salmon.ex | 1 - .../representers/activity_representer.ex | 67 +++++---- lib/pleroma/web/twitter_api/twitter_api.ex | 88 +++++------ .../web/twitter_api/twitter_api_controller.ex | 30 ++-- lib/pleroma/web/web.ex | 10 +- lib/pleroma/web/web_finger/web_finger.ex | 17 +-- .../web/web_finger/web_finger_controller.ex | 2 +- lib/pleroma/web/websub/websub.ex | 25 ++-- lib/xml_builder.ex | 4 +- mix.exs | 1 + mix.lock | 4 +- 21 files changed, 318 insertions(+), 157 deletions(-) create mode 100644 .credo.exs diff --git a/.credo.exs b/.credo.exs new file mode 100644 index 000000000..b1f2a6164 --- /dev/null +++ b/.credo.exs @@ -0,0 +1,138 @@ +# This file contains the configuration for Credo and you are probably reading +# this after creating it with `mix credo.gen.config`. +# +# If you find anything wrong or unclear in this file, please report an +# issue on GitHub: https://github.com/rrrene/credo/issues +# +%{ + # + # You can have as many configs as you like in the `configs:` field. + configs: [ + %{ + # + # Run any config using `mix credo -C `. If no config name is given + # "default" is used. + name: "default", + # + # These are the files included in the analysis: + files: %{ + # + # You can give explicit globs or simply directories. + # In the latter case `**/*.{ex,exs}` will be used. + included: ["lib/", "src/", "web/", "apps/"], + excluded: [~r"/_build/", ~r"/deps/"] + }, + # + # If you create your own checks, you must specify the source files for + # them here, so they can be loaded by Credo before running the analysis. + requires: [], + # + # Credo automatically checks for updates, like e.g. Hex does. + # You can disable this behaviour below: + check_for_updates: true, + # + # If you want to enforce a style guide and need a more traditional linting + # experience, you can change `strict` to `true` below: + strict: false, + # + # If you want to use uncolored output by default, you can change `color` + # to `false` below: + color: true, + # + # You can customize the parameters of any check by adding a second element + # to the tuple. + # + # To disable a check put `false` as second element: + # + # {Credo.Check.Design.DuplicatedCode, false} + # + checks: [ + {Credo.Check.Consistency.ExceptionNames}, + {Credo.Check.Consistency.LineEndings}, + {Credo.Check.Consistency.MultiAliasImportRequireUse}, + {Credo.Check.Consistency.ParameterPatternMatching}, + {Credo.Check.Consistency.SpaceAroundOperators}, + {Credo.Check.Consistency.SpaceInParentheses}, + {Credo.Check.Consistency.TabsOrSpaces}, + + # For some checks, like AliasUsage, you can only customize the priority + # Priority values are: `low, normal, high, higher` + {Credo.Check.Design.AliasUsage, priority: :low}, + + # For others you can set parameters + + # If you don't want the `setup` and `test` macro calls in ExUnit tests + # or the `schema` macro in Ecto schemas to trigger DuplicatedCode, just + # set the `excluded_macros` parameter to `[:schema, :setup, :test]`. + {Credo.Check.Design.DuplicatedCode, excluded_macros: []}, + + # You can also customize the exit_status of each check. + # If you don't want TODO comments to cause `mix credo` to fail, just + # set this value to 0 (zero). + {Credo.Check.Design.TagTODO, exit_status: 2}, + {Credo.Check.Design.TagFIXME}, + + {Credo.Check.Readability.FunctionNames}, + {Credo.Check.Readability.LargeNumbers}, + {Credo.Check.Readability.MaxLineLength, priority: :low, max_length: 100}, + {Credo.Check.Readability.ModuleAttributeNames}, + {Credo.Check.Readability.ModuleDoc, false}, + {Credo.Check.Readability.ModuleNames}, + {Credo.Check.Readability.ParenthesesOnZeroArityDefs}, + {Credo.Check.Readability.ParenthesesInCondition}, + {Credo.Check.Readability.PredicateFunctionNames}, + {Credo.Check.Readability.PreferImplicitTry}, + {Credo.Check.Readability.RedundantBlankLines}, + {Credo.Check.Readability.StringSigils}, + {Credo.Check.Readability.TrailingBlankLine}, + {Credo.Check.Readability.TrailingWhiteSpace}, + {Credo.Check.Readability.VariableNames}, + {Credo.Check.Readability.Semicolons}, + {Credo.Check.Readability.SpaceAfterCommas}, + + {Credo.Check.Refactor.DoubleBooleanNegation}, + {Credo.Check.Refactor.CondStatements}, + {Credo.Check.Refactor.CyclomaticComplexity}, + {Credo.Check.Refactor.FunctionArity}, + {Credo.Check.Refactor.MatchInCondition}, + {Credo.Check.Refactor.NegatedConditionsInUnless}, + {Credo.Check.Refactor.NegatedConditionsWithElse}, + {Credo.Check.Refactor.Nesting}, + {Credo.Check.Refactor.PipeChainStart}, + {Credo.Check.Refactor.UnlessWithElse}, + + {Credo.Check.Warning.BoolOperationOnSameValues}, + {Credo.Check.Warning.IExPry}, + {Credo.Check.Warning.IoInspect}, + {Credo.Check.Warning.LazyLogging}, + {Credo.Check.Warning.OperationOnSameValues}, + {Credo.Check.Warning.OperationWithConstantResult}, + {Credo.Check.Warning.UnusedEnumOperation}, + {Credo.Check.Warning.UnusedFileOperation}, + {Credo.Check.Warning.UnusedKeywordOperation}, + {Credo.Check.Warning.UnusedListOperation}, + {Credo.Check.Warning.UnusedPathOperation}, + {Credo.Check.Warning.UnusedRegexOperation}, + {Credo.Check.Warning.UnusedStringOperation}, + {Credo.Check.Warning.UnusedTupleOperation}, + + # Controversial and experimental checks (opt-in, just remove `, false`) + # + {Credo.Check.Refactor.ABCSize, false}, + {Credo.Check.Refactor.AppendSingleItem, false}, + {Credo.Check.Refactor.VariableRebinding, false}, + {Credo.Check.Warning.MapGetUnsafePass, false}, + + # Deprecated checks (these will be deleted after a grace period) + {Credo.Check.Readability.Specs, false}, + {Credo.Check.Warning.NameRedeclarationByAssignment, false}, + {Credo.Check.Warning.NameRedeclarationByCase, false}, + {Credo.Check.Warning.NameRedeclarationByDef, false}, + {Credo.Check.Warning.NameRedeclarationByFn, false}, + + # Custom checks can be created using `mix credo.gen.check`. + # + ] + } + ] +} diff --git a/lib/pleroma/plugs/authentication_plug.ex b/lib/pleroma/plugs/authentication_plug.ex index a3317f432..d47c3fdae 100644 --- a/lib/pleroma/plugs/authentication_plug.ex +++ b/lib/pleroma/plugs/authentication_plug.ex @@ -1,4 +1,5 @@ defmodule Pleroma.Plugs.AuthenticationPlug do + alias Comeonin.Pbkdf2 import Plug.Conn def init(options) do @@ -25,12 +26,12 @@ defp verify(%{id: id} = user, _password, id) do end defp verify(nil, _password, _user_id) do - Comeonin.Pbkdf2.dummy_checkpw + Pbkdf2.dummy_checkpw :error end defp verify(user, password, _user_id) do - if Comeonin.Pbkdf2.checkpw(password, user.password_hash) do + if Pbkdf2.checkpw(password, user.password_hash) do {:ok, user} else :error @@ -42,7 +43,7 @@ defp decode_header(conn) do {:ok, userinfo} <- Base.decode64(header), [username, password] <- String.split(userinfo, ":") do - { :ok, username, password } + {:ok, username, password} end end diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 3aabf8157..9275eff87 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -1,6 +1,8 @@ defmodule Pleroma.Upload do + alias Ecto.UUID + alias Pleroma.Web def store(%Plug.Upload{} = file) do - uuid = Ecto.UUID.generate + uuid = UUID.generate upload_folder = Path.join(upload_path(), uuid) File.mkdir_p!(upload_folder) result_file = Path.join(upload_folder, file.filename) @@ -21,7 +23,7 @@ def store(%Plug.Upload{} = file) do def store(%{"img" => "data:image/" <> image_data}) do parsed = Regex.named_captures(~r/(?jpeg|png|gif);base64,(?.*)/, image_data) data = Base.decode64!(parsed["data"]) - uuid = Ecto.UUID.generate + uuid = UUID.generate upload_folder = Path.join(upload_path(), uuid) File.mkdir_p!(upload_folder) filename = Base.encode16(:crypto.hash(:sha256, data)) <> ".#{parsed["filetype"]}" @@ -44,11 +46,11 @@ def store(%{"img" => "data:image/" <> image_data}) do end defp upload_path do - Application.get_env(:pleroma, Pleroma.Upload) - |> Keyword.fetch!(:uploads) + settings = Application.get_env(:pleroma, Pleroma.Upload) + Keyword.fetch!(settings, :uploads) end defp url_for(file) do - "#{Pleroma.Web.base_url()}/media/#{file}" + "#{Web.base_url()}/media/#{file}" end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 5e579dc44..65925caed 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1,8 +1,8 @@ defmodule Pleroma.User do use Ecto.Schema - import Ecto.Changeset - import Ecto.Query - alias Pleroma.{Repo, User, Activity, Object} + import Ecto.{Changeset, Query} + alias Pleroma.{Repo, User, Object, Web} + alias Comeonin.Pbkdf2 schema "users" do field :bio, :string @@ -12,7 +12,7 @@ defmodule Pleroma.User do field :password_hash, :string field :password, :string, virtual: true field :password_confirmation, :string, virtual: true - field :following, { :array, :string }, default: [] + field :following, {:array, :string}, default: [] field :ap_id, :string field :avatar, :map @@ -27,7 +27,7 @@ def avatar_url(user) do end def ap_id(%User{nickname: nickname}) do - "#{Pleroma.Web.base_url}/users/#{nickname}" + "#{Web.base_url}/users/#{nickname}" end def ap_followers(%User{} = user) do @@ -66,7 +66,7 @@ def register_changeset(struct, params \\ %{}) do |> validate_format(:nickname, ~r/^[a-zA-Z\d]+$/) if changeset.valid? do - hashed = Comeonin.Pbkdf2.hashpwsalt(changeset.changes[:password]) + hashed = Pbkdf2.hashpwsalt(changeset.changes[:password]) ap_id = User.ap_id(%User{nickname: changeset.changes[:nickname]}) followers = User.ap_followers(%User{nickname: changeset.changes[:nickname]}) changeset @@ -81,8 +81,8 @@ def register_changeset(struct, params \\ %{}) do def follow(%User{} = follower, %User{} = followed) do ap_followers = User.ap_followers(followed) if following?(follower, followed) do - { :error, - "Could not follow user: #{followed.nickname} is already on your list." } + {:error, + "Could not follow user: #{followed.nickname} is already on your list."} else following = [ap_followers | follower.following] |> Enum.uniq @@ -103,7 +103,7 @@ def unfollow(%User{} = follower, %User{} = followed) do |> follow_changeset(%{following: following}) |> Repo.update else - { :error, "Not subscribed!" } + {:error, "Not subscribed!"} end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e9f0dcd32..02255e0a4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1,6 +1,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do - alias Pleroma.Repo - alias Pleroma.{Activity, Object, Upload, User} + alias Pleroma.{Activity, Repo, Object, Upload, User, Web} + alias Ecto.{Changeset, UUID} import Ecto.Query def insert(map) when is_map(map) do @@ -19,7 +19,7 @@ def insert(map) when is_map(map) do Repo.insert(%Activity{data: map}) end - def like(%User{ap_id: ap_id} = user, %Object{data: %{ "id" => id}} = object) do + def like(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do cond do # There's already a like here, so return the original activity. ap_id in (object.data["likes"] || []) -> @@ -44,7 +44,7 @@ def like(%User{ap_id: ap_id} = user, %Object{data: %{ "id" => id}} = object) do |> Map.put("like_count", length(likes)) |> Map.put("likes", likes) - changeset = Ecto.Changeset.change(object, data: new_data) + changeset = Changeset.change(object, data: new_data) {:ok, object} = Repo.update(changeset) update_object_in_activities(object) @@ -58,7 +58,7 @@ defp update_object_in_activities(%{data: %{"id" => id}} = object) do relevant_activities = Activity.all_by_object_ap_id(id) Enum.map(relevant_activities, fn (activity) -> new_activity_data = activity.data |> Map.put("object", object.data) - changeset = Ecto.Changeset.change(activity, data: new_activity_data) + changeset = Changeset.change(activity, data: new_activity_data) Repo.update(changeset) end) end @@ -79,7 +79,7 @@ def unlike(%User{ap_id: ap_id}, %Object{data: %{ "id" => id}} = object) do |> Map.put("like_count", length(likes)) |> Map.put("likes", likes) - changeset = Ecto.Changeset.change(object, data: new_data) + changeset = Changeset.change(object, data: new_data) {:ok, object} = Repo.update(changeset) update_object_in_activities(object) @@ -103,7 +103,7 @@ def generate_object_id do end def generate_id(type) do - "#{Pleroma.Web.base_url()}/#{type}/#{Ecto.UUID.generate}" + "#{Web.base_url()}/#{type}/#{UUID.generate}" end def fetch_public_activities(opts \\ %{}) do @@ -140,8 +140,7 @@ def fetch_activities(recipients, opts \\ %{}) do query end - Repo.all(query) - |> Enum.reverse + Enum.reverse(Repo.all(query)) end def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do @@ -160,7 +159,7 @@ def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) |> Map.put("announcement_count", length(announcements)) |> Map.put("announcements", announcements) - changeset = Ecto.Changeset.change(object, data: new_data) + changeset = Changeset.change(object, data: new_data) {:ok, object} = Repo.update(changeset) update_object_in_activities(object) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 590abc8bb..d7ea61321 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -23,5 +23,5 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) ] ++ attachments end - def to_simple_form(_,_), do: nil + def to_simple_form(_, _), do: nil end diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex index 14ac3ebf4..86c6f9d4f 100644 --- a/lib/pleroma/web/ostatus/feed_representer.ex +++ b/lib/pleroma/web/ostatus/feed_representer.ex @@ -8,7 +8,8 @@ def to_simple_form(user, activities, users) do h = fn(str) -> [to_charlist(str)] end - entries = Enum.map(activities, fn(activity) -> + entries = activities + |> Enum.map(fn(activity) -> {:entry, ActivityRepresenter.to_simple_form(activity, user)} end) |> Enum.filter(fn ({_, form}) -> form end) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 3c8d8c0f1..ed7618d2b 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -16,7 +16,8 @@ def feed(conn, %{"nickname" => nickname}) do activities = query |> Repo.all - response = FeedRepresenter.to_simple_form(user, activities, [user]) + response = user + |> FeedRepresenter.to_simple_form(activities, [user]) |> :xmerl.export_simple(:xmerl_xml) |> to_string @@ -25,7 +26,7 @@ def feed(conn, %{"nickname" => nickname}) do |> send_resp(200, response) end - def temp(conn, params) do + def temp(_conn, params) do IO.inspect(params) end end diff --git a/lib/pleroma/web/ostatus/user_representer.ex b/lib/pleroma/web/ostatus/user_representer.ex index 65dfc5643..273d7524a 100644 --- a/lib/pleroma/web/ostatus/user_representer.ex +++ b/lib/pleroma/web/ostatus/user_representer.ex @@ -7,14 +7,14 @@ def to_simple_form(user) do bio = to_charlist(user.bio) avatar_url = to_charlist(User.avatar_url(user)) [ - { :id, [ap_id] }, - { :"activity:object", ['http://activitystrea.ms/schema/1.0/person'] }, - { :uri, [ap_id] }, - { :"poco:preferredUsername", [nickname] }, - { :"poco:displayName", [name] }, - { :"poco:note", [bio] }, - { :name, [nickname] }, - { :link, [rel: 'avatar', href: avatar_url], []} + {:id, [ap_id]}, + {:"activity:object", ['http://activitystrea.ms/schema/1.0/person']}, + {:uri, [ap_id]}, + {:"poco:preferredUsername", [nickname]}, + {:"poco:displayName", [name]}, + {:"poco:note", [bio]}, + {:name, [nickname]}, + {:link, [rel: 'avatar', href: avatar_url], []} ] end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a4f13c879..2c94d071f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -1,7 +1,7 @@ defmodule Pleroma.Web.Router do use Pleroma.Web, :router - alias Pleroma.{Repo, User} + alias Pleroma.{Repo, User, Web.Router} def user_fetcher(username) do {:ok, Repo.get_by(User, %{nickname: username})} @@ -10,13 +10,13 @@ def user_fetcher(username) do pipeline :api do plug :accepts, ["json"] plug :fetch_session - plug Pleroma.Plugs.AuthenticationPlug, %{fetcher: &Pleroma.Web.Router.user_fetcher/1, optional: true} + plug Pleroma.Plugs.AuthenticationPlug, %{fetcher: &Router.user_fetcher/1, optional: true} end pipeline :authenticated_api do plug :accepts, ["json"] plug :fetch_session - plug Pleroma.Plugs.AuthenticationPlug, %{fetcher: &Pleroma.Web.Router.user_fetcher/1} + plug Pleroma.Plugs.AuthenticationPlug, %{fetcher: &Router.user_fetcher/1} end pipeline :well_known do diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 3881f2758..f26daf824 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -10,7 +10,6 @@ def decode(salmon) do {:xmlObj, :string, encoding} = :xmerl_xpath.string('string(//me:encoding[1])', doc) {:xmlObj, :string, type} = :xmerl_xpath.string('string(//me:data[1]/@type)', doc) - {:ok, data} = Base.url_decode64(to_string(data), ignore: :whitespace) {:ok, sig} = Base.url_decode64(to_string(sig), ignore: :whitespace) alg = to_string(alg) diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex index f2bf93abb..b58572829 100644 --- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex @@ -1,17 +1,17 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do use Pleroma.Web.TwitterAPI.Representers.BaseRepresenter alias Pleroma.Web.TwitterAPI.Representers.{UserRepresenter, ObjectRepresenter} - alias Pleroma.Activity - + alias Pleroma.{Activity, User} + alias Calendar.Strftime defp user_by_ap_id(user_list, ap_id) do Enum.find(user_list, fn (%{ap_id: user_id}) -> ap_id == user_id end) end - def to_map(%Activity{data: %{"type" => "Announce", "actor" => actor}} = activity, %{users: users, announced_activity: announced_activity} = opts) do + def to_map(%Activity{data: %{"type" => "Announce", "actor" => actor, "published" => created_at}} = activity, + %{users: users, announced_activity: announced_activity} = opts) do user = user_by_ap_id(users, actor) - created_at = get_in(activity.data, ["published"]) - |> date_to_asctime + created_at = created_at |> date_to_asctime text = "#{user.nickname} retweeted a status." @@ -30,16 +30,16 @@ def to_map(%Activity{data: %{"type" => "Announce", "actor" => actor}} = activity } end - def to_map(%Activity{data: %{"type" => "Like"}} = activity, %{user: user, liked_activity: liked_activity} = opts) do - created_at = get_in(activity.data, ["published"]) - |> date_to_asctime + def to_map(%Activity{data: %{"type" => "Like", "published" => created_at}} = activity, + %{user: user, liked_activity: liked_activity} = opts) do + created_at = created_at |> date_to_asctime text = "#{user.nickname} favorited a status." %{ "id" => activity.id, "user" => UserRepresenter.to_map(user, opts), - "statusnet_html" => text, # TODO: add summary + "statusnet_html" => text, "text" => text, "is_local" => true, "is_post_verb" => false, @@ -49,16 +49,17 @@ def to_map(%Activity{data: %{"type" => "Like"}} = activity, %{user: user, liked_ } end - def to_map(%Activity{data: %{"type" => "Follow"}} = activity, %{user: user} = opts) do - created_at = get_in(activity.data, ["published"]) - |> date_to_asctime + def to_map(%Activity{data: %{"type" => "Follow", "published" => created_at, "object" => followed_id}} = activity, %{user: user} = opts) do + created_at = created_at |> date_to_asctime + followed = User.get_cached_by_ap_id(followed_id) + text = "#{user.nickname} started following #{followed.nickname}" %{ "id" => activity.id, "user" => UserRepresenter.to_map(user, opts), "attentions" => [], - "statusnet_html" => "", # TODO: add summary - "text" => "", + "statusnet_html" => text, + "text" => text, "is_local" => true, "is_post_verb" => false, "created_at" => created_at, @@ -66,14 +67,12 @@ def to_map(%Activity{data: %{"type" => "Follow"}} = activity, %{user: user} = op } end - def to_map(%Activity{} = activity, %{user: user} = opts) do - content = get_in(activity.data, ["object", "content"]) - created_at = get_in(activity.data, ["object", "published"]) - |> date_to_asctime - like_count = get_in(activity.data, ["object", "like_count"]) || 0 - announcement_count = get_in(activity.data, ["object", "announcement_count"]) || 0 - favorited = opts[:for] && opts[:for].ap_id in (activity.data["object"]["likes"] || []) - repeated = opts[:for] && opts[:for].ap_id in (activity.data["object"]["announcements"] || []) + def to_map(%Activity{data: %{"object" => %{"content" => content} = object}} = activity, %{user: user} = opts) do + created_at = object["published"] |> date_to_asctime + like_count = object["like_count"] || 0 + announcement_count = object["announcement_count"] || 0 + favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || []) + repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || []) mentions = opts[:mentioned] || [] @@ -91,22 +90,34 @@ def to_map(%Activity{} = activity, %{user: user} = opts) do "is_local" => true, "is_post_verb" => true, "created_at" => created_at, - "in_reply_to_status_id" => activity.data["object"]["inReplyToStatusId"], - "statusnet_conversation_id" => activity.data["object"]["statusnetConversationId"], - "attachments" => (activity.data["object"]["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts), + "in_reply_to_status_id" => object["inReplyToStatusId"], + "statusnet_conversation_id" => object["statusnetConversationId"], + "attachments" => (object["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts), "attentions" => attentions, "fave_num" => like_count, "repeat_num" => announcement_count, - "favorited" => !!favorited, - "repeated" => !!repeated, + "favorited" => to_boolean(favorited), + "repeated" => to_boolean(repeated), } end defp date_to_asctime(date) do with {:ok, date, _offset} <- date |> DateTime.from_iso8601 do - Calendar.Strftime.strftime!(date, "%a %b %d %H:%M:%S %z %Y") + Strftime.strftime!(date, "%a %b %d %H:%M:%S %z %Y") else _e -> "" end end + + defp to_boolean(false) do + false + end + + defp to_boolean(nil) do + false + end + + defp to_boolean(_) do + true + end end diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index f2b2b4418..649936b76 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -1,19 +1,19 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do + alias Ecto.Changeset alias Pleroma.{User, Activity, Repo, Object} - alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.{ActivityPub.ActivityPub, Websub, OStatus} alias Pleroma.Web.TwitterAPI.Representers.{ActivityRepresenter, UserRepresenter} import Ecto.Query - def create_status(user = %User{}, data = %{}) do + def create_status(%User{} = user, %{} = data) do attachments = Enum.map(data["media_ids"] || [], fn (media_id) -> Repo.get(Object, media_id).data end) context = ActivityPub.generate_context_id - content = HtmlSanitizeEx.strip_tags(data["status"]) - |> String.replace("\n", "
") + content = data["status"] |> HtmlSanitizeEx.strip_tags |> String.replace("\n", "
") mentions = parse_mentions(content) @@ -40,10 +40,10 @@ def create_status(user = %User{}, data = %{}) do "context" => context, "attachment" => attachments, "actor" => user.ap_id - }, + }, "published" => date, "context" => context - } + } # Wire up reply info. activity = with inReplyToId when not is_nil(inReplyToId) <- data["in_reply_to_status_id"], @@ -67,34 +67,34 @@ def create_status(user = %User{}, data = %{}) do with {:ok, activity} <- ActivityPub.insert(activity) do {:ok, activity} = add_conversation_id(activity) - Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(user), user, activity) + Websub.publish(OStatus.feed_path(user), user, activity) {:ok, activity} end end def fetch_friend_statuses(user, opts \\ %{}) do - ActivityPub.fetch_activities([user.ap_id | user.following], opts) - |> activities_to_statuses(%{for: user}) + activities = ActivityPub.fetch_activities([user.ap_id | user.following], opts) + activities_to_statuses(activities, %{for: user}) end def fetch_public_statuses(user, opts \\ %{}) do - ActivityPub.fetch_public_activities(opts) - |> activities_to_statuses(%{for: user}) + activities = ActivityPub.fetch_public_activities(opts) + activities_to_statuses(activities, %{for: user}) end def fetch_user_statuses(user, opts \\ %{}) do - ActivityPub.fetch_activities([], opts) - |> activities_to_statuses(%{for: user}) + activities = ActivityPub.fetch_activities([], opts) + activities_to_statuses(activities, %{for: user}) end def fetch_mentions(user, opts \\ %{}) do - ActivityPub.fetch_activities([user.ap_id], opts) - |> activities_to_statuses(%{for: user}) + activities = ActivityPub.fetch_activities([user.ap_id], opts) + activities_to_statuses(activities, %{for: user}) end def fetch_conversation(user, id) do query = from activity in Activity, - where: fragment("? @> ?", activity.data, ^%{ statusnetConversationId: id}), + where: fragment("? @> ?", activity.data, ^%{statusnetConversationId: id}), limit: 1 with %Activity{} = activity <- Repo.one(query), @@ -116,26 +116,26 @@ def fetch_status(user, id) do end def follow(%User{} = follower, params) do - with { :ok, %User{} = followed } <- get_user(params), - { :ok, follower } <- User.follow(follower, followed), - { :ok, activity } <- ActivityPub.insert(%{ - "type" => "Follow", - "actor" => follower.ap_id, - "object" => followed.ap_id, - "published" => make_date() - }) + with {:ok, %User{} = followed} <- get_user(params), + {:ok, follower} <- User.follow(follower, followed), + {:ok, activity} <- ActivityPub.insert(%{ + "type" => "Follow", + "actor" => follower.ap_id, + "object" => followed.ap_id, + "published" => make_date() + }) do - { :ok, follower, followed, activity } + {:ok, follower, followed, activity} else err -> err end end def unfollow(%User{} = follower, params) do - with { :ok, %User{} = unfollowed } <- get_user(params), - { :ok, follower } <- User.unfollow(follower, unfollowed) + with {:ok, %User{} = unfollowed} <- get_user(params), + {:ok, follower} <- User.unfollow(follower, unfollowed) do - { :ok, follower, unfollowed} + {:ok, follower, unfollowed} else err -> err end @@ -207,7 +207,7 @@ def upload(%Plug.Upload{} = file, format \\ "xml") do media_id_string: "#{object.id}}", media_url: href, size: 0 - } |> Poison.encode! + } |> Poison.encode! end end @@ -215,15 +215,18 @@ def parse_mentions(text) do # Modified from https://www.w3.org/TR/html5/forms.html#valid-e-mail-address regex = ~r/@[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@?[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*/ - Regex.scan(regex, text) + regex + |> Regex.scan(text) |> List.flatten |> Enum.uniq - |> Enum.map(fn ("@" <> match = full_match) -> {full_match, User.get_cached_by_nickname(match)} end) + |> Enum.map(fn ("@" <> match = full_match) -> + {full_match, User.get_cached_by_nickname(match)} end) |> Enum.filter(fn ({_match, user}) -> user end) end def add_user_links(text, mentions) do - Enum.reduce(mentions, text, fn ({match, %User{ap_id: ap_id}}, text) -> String.replace(text, match, "#{match}") end) + Enum.reduce(mentions, text, fn ({match, %User{ap_id: ap_id}}, text) -> + String.replace(text, match, "#{match}") end) end defp add_conversation_id(activity) do @@ -236,10 +239,10 @@ defp add_conversation_id(activity) do object = Object.get_by_ap_id(activity.data["object"]["id"]) - changeset = Ecto.Changeset.change(object, data: data["object"]) + changeset = Changeset.change(object, data: data["object"]) Repo.update(changeset) - changeset = Ecto.Changeset.change(activity, data: data) + changeset = Changeset.change(activity, data: data) Repo.update(changeset) end end @@ -252,7 +255,7 @@ def register_user(params) do email: params["email"], password: params["password"], password_confirmation: params["confirm"] - } + } changeset = User.register_changeset(%User{}, params) @@ -260,22 +263,21 @@ def register_user(params) do {:ok, UserRepresenter.to_map(user)} else {:error, changeset} -> - errors = Ecto.Changeset.traverse_errors(changeset, fn {msg, _opts} -> msg end) - |> Poison.encode! + errors = Poison.encode!(Changeset.traverse_errors(changeset, fn {msg, _opts} -> msg end)) {:error, %{error: errors}} end end def get_user(user \\ nil, params) do case params do - %{ "user_id" => user_id } -> + %{"user_id" => user_id} -> case target = Repo.get(User, user_id) do nil -> {:error, "No user with such user_id"} _ -> {:ok, target} end - %{ "screen_name" => nickname } -> + %{"screen_name" => nickname} -> case target = Repo.get_by(User, nickname: nickname) do nil -> {:error, "No user with such screen_name"} @@ -303,7 +305,8 @@ defp activity_to_status(%Activity{data: %{"type" => "Like"}} = activity, opts) d user = User.get_cached_by_ap_id(actor) [liked_activity] = Activity.all_by_object_ap_id(activity.data["object"]) - ActivityRepresenter.to_map(activity, Map.merge(opts, %{user: user, liked_activity: liked_activity})) + ActivityRepresenter.to_map(activity, + Map.merge(opts, %{user: user, liked_activity: liked_activity})) end # For announces, fetch the announced activity and the user. @@ -313,7 +316,8 @@ defp activity_to_status(%Activity{data: %{"type" => "Announce"}} = activity, opt [announced_activity] = Activity.all_by_object_ap_id(activity.data["object"]) announced_actor = User.get_cached_by_ap_id(announced_activity.data["actor"]) - ActivityRepresenter.to_map(activity, Map.merge(opts, %{users: [user, announced_actor], announced_activity: announced_activity})) + ActivityRepresenter.to_map(activity, + Map.merge(opts, %{users: [user, announced_actor], announced_activity: announced_activity})) end defp activity_to_status(activity, opts) do @@ -323,7 +327,7 @@ defp activity_to_status(activity, opts) do mentioned_users = Enum.map(activity.data["to"] || [], fn (ap_id) -> User.get_cached_by_ap_id(ap_id) end) - |> Enum.filter(&(&1)) + mentioned_users = mentioned_users |> Enum.filter(&(&1)) ActivityRepresenter.to_map(activity, Map.merge(opts, %{user: user, mentioned: mentioned_users})) end diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index b5b829ca0..bbfc04a6a 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -2,8 +2,9 @@ defmodule Pleroma.Web.TwitterAPI.Controller do use Pleroma.Web, :controller alias Pleroma.Web.TwitterAPI.TwitterAPI alias Pleroma.Web.TwitterAPI.Representers.{UserRepresenter, ActivityRepresenter} - alias Pleroma.{Repo, Activity} + alias Pleroma.{Web, Repo, Activity} alias Pleroma.Web.ActivityPub.ActivityPub + alias Ecto.Changeset def verify_credentials(%{assigns: %{user: user}} = conn, _params) do response = user |> UserRepresenter.to_json(%{for: user}) @@ -15,7 +16,7 @@ def verify_credentials(%{assigns: %{user: user}} = conn, _params) do def status_update(%{assigns: %{user: user}} = conn, %{"status" => status_text} = status_data) do if status_text |> String.trim |> String.length != 0 do media_ids = extract_media_ids(status_data) - {:ok, activity} = TwitterAPI.create_status(user, Map.put(status_data, "media_ids", media_ids )) + {:ok, activity} = TwitterAPI.create_status(user, Map.put(status_data, "media_ids", media_ids)) conn |> json_reply(200, ActivityRepresenter.to_json(activity, %{user: user})) else @@ -79,34 +80,34 @@ def mentions_timeline(%{assigns: %{user: user}} = conn, params) do def follow(%{assigns: %{user: user}} = conn, params) do case TwitterAPI.follow(user, params) do - { :ok, user, followed, _activity } -> + {:ok, user, followed, _activity} -> response = followed |> UserRepresenter.to_json(%{for: user}) conn |> json_reply(200, response) - { :error, msg } -> forbidden_json_reply(conn, msg) + {:error, msg} -> forbidden_json_reply(conn, msg) end end def unfollow(%{assigns: %{user: user}} = conn, params) do case TwitterAPI.unfollow(user, params) do - { :ok, user, unfollowed, } -> + {:ok, user, unfollowed} -> response = unfollowed |> UserRepresenter.to_json(%{for: user}) conn |> json_reply(200, response) - { :error, msg } -> forbidden_json_reply(conn, msg) + {:error, msg} -> forbidden_json_reply(conn, msg) end end - def fetch_status(%{assigns: %{user: user}} = conn, %{ "id" => id }) do - response = TwitterAPI.fetch_status(user, id) |> Poison.encode! + def fetch_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do + response = Poison.encode!(TwitterAPI.fetch_status(user, id)) conn |> json_reply(200, response) end - def fetch_conversation(%{assigns: %{user: user}} = conn, %{ "id" => id }) do + def fetch_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do id = String.to_integer(id) - response = TwitterAPI.fetch_conversation(user, id) |> Poison.encode! + response = Poison.encode!(TwitterAPI.fetch_conversation(user, id)) conn |> json_reply(200, response) @@ -132,8 +133,8 @@ def upload_json(conn, %{"media" => media}) do def config(conn, _params) do response = %{ site: %{ - name: Pleroma.Web.base_url, - server: Pleroma.Web.base_url, + name: Web.base_url, + server: Web.base_url, textlimit: -1 } } @@ -188,11 +189,10 @@ def register(conn, params) do def update_avatar(%{assigns: %{user: user}} = conn, params) do {:ok, object} = ActivityPub.upload(params) - change = Ecto.Changeset.change(user, %{avatar: object.data}) + change = Changeset.change(user, %{avatar: object.data}) {:ok, user} = Repo.update(change) - response = UserRepresenter.to_map(user, %{for: user}) - |> Poison.encode! + response = Poison.encode!(UserRepresenter.to_map(user, %{for: user})) conn |> json_reply(200, response) diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex index a81e3e6e1..19b1ff848 100644 --- a/lib/pleroma/web/web.ex +++ b/lib/pleroma/web/web.ex @@ -20,8 +20,7 @@ def controller do quote do use Phoenix.Controller, namespace: Pleroma.Web import Plug.Conn - import Pleroma.Web.Router.Helpers - import Pleroma.Web.Gettext + import Pleroma.Web.{Gettext, Router.Helpers} end end @@ -33,9 +32,7 @@ def view do # Import convenience functions from controllers import Phoenix.Controller, only: [get_csrf_token: 0, get_flash: 2, view_module: 1] - import Pleroma.Web.Router.Helpers - import Pleroma.Web.ErrorHelpers - import Pleroma.Web.Gettext + import Pleroma.Web.{ErrorHelpers, Gettext, Router.Helpers} end end @@ -75,7 +72,8 @@ def base_url do protocol = settings |> Keyword.fetch!(:protocol) - port_fragment = with {:ok, protocol_info} <- settings |> Keyword.fetch(String.to_atom(protocol)), + port_fragment = with {:ok, protocol_info} <- settings + |> Keyword.fetch(String.to_atom(protocol)), {:ok, port} <- protocol_info |> Keyword.fetch(:port) do ":#{port}" diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index eb540e92a..3d6ca4e05 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -1,21 +1,20 @@ defmodule Pleroma.Web.WebFinger do - alias Pleroma.XmlBuilder - alias Pleroma.User - alias Pleroma.Web.OStatus + alias Pleroma.{User, XmlBuilder} + alias Pleroma.{Web, Web.OStatus} - def host_meta() do - base_url = Pleroma.Web.base_url + def host_meta do + base_url = Web.base_url { - :XRD, %{ xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0" }, + :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}, { - :Link, %{ rel: "lrdd", type: "application/xrd+xml", template: "#{base_url}/.well-known/webfinger?resource={uri}" } + :Link, %{rel: "lrdd", type: "application/xrd+xml", template: "#{base_url}/.well-known/webfinger?resource={uri}"} } } |> XmlBuilder.to_doc end def webfinger(resource) do - host = Pleroma.Web.host + host = Web.host regex = ~r/acct:(?\w+)@#{host}/ case Regex.named_captures(regex, resource) do %{"username" => username} -> @@ -29,7 +28,7 @@ def represent_user(user) do { :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}, [ - {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.host}"}, + {:Subject, "acct:#{user.nickname}@#{Web.host}"}, {:Alias, user.ap_id}, {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}} ] diff --git a/lib/pleroma/web/web_finger/web_finger_controller.ex b/lib/pleroma/web/web_finger/web_finger_controller.ex index 7c0fd3142..d8959a96f 100644 --- a/lib/pleroma/web/web_finger/web_finger_controller.ex +++ b/lib/pleroma/web/web_finger/web_finger_controller.ex @@ -12,7 +12,7 @@ def host_meta(conn, _params) do end def webfinger(conn, %{"resource" => resource}) do - {:ok, response} = Pleroma.Web.WebFinger.webfinger(resource) + {:ok, response} = WebFinger.webfinger(resource) conn |> put_resp_content_type("application/xrd+xml") diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index cc66b52dd..ba699db24 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -1,4 +1,5 @@ defmodule Pleroma.Web.Websub do + alias Ecto.Changeset alias Pleroma.Repo alias Pleroma.Web.Websub.WebsubServerSubscription alias Pleroma.Web.OStatus.FeedRepresenter @@ -8,9 +9,10 @@ defmodule Pleroma.Web.Websub do @websub_verifier Application.get_env(:pleroma, :websub_verifier) - def verify(subscription, getter \\ &HTTPoison.get/3 ) do + def verify(subscription, getter \\ &HTTPoison.get/3) do challenge = Base.encode16(:crypto.strong_rand_bytes(8)) - lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at) |> to_string + lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at) + lease_seconds = lease_seconds |> to_string params = %{ "hub.challenge": challenge, @@ -25,11 +27,11 @@ def verify(subscription, getter \\ &HTTPoison.get/3 ) do with {:ok, response} <- getter.(url, [], [params: params]), ^challenge <- response.body do - changeset = Ecto.Changeset.change(subscription, %{state: "active"}) + changeset = Changeset.change(subscription, %{state: "active"}) Repo.update(changeset) else _e -> - changeset = Ecto.Changeset.change(subscription, %{state: "rejected"}) - {:ok, subscription } = Repo.update(changeset) + changeset = Changeset.change(subscription, %{state: "rejected"}) + {:ok, subscription} = Repo.update(changeset) {:error, subscription} end end @@ -39,10 +41,11 @@ def publish(topic, user, activity) do where: sub.topic == ^topic and sub.state == "active" subscriptions = Repo.all(query) Enum.each(subscriptions, fn(sub) -> - response = FeedRepresenter.to_simple_form(user, [activity], [user]) + response = user + |> FeedRepresenter.to_simple_form([activity], [user]) |> :xmerl.export_simple(:xmerl_xml) - signature = :crypto.hmac(:sha, sub.secret, response) |> Base.encode16 + signature = Base.encode16(:crypto.hmac(:sha, sub.secret, response)) HTTPoison.post(sub.callback, response, [ {"Content-Type", "application/atom+xml"}, @@ -65,10 +68,11 @@ def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) d callback: callback } - change = Ecto.Changeset.change(subscription, data) + change = Changeset.change(subscription, data) websub = Repo.insert_or_update!(change) - change = Ecto.Changeset.change(websub, %{valid_until: NaiveDateTime.add(websub.updated_at, lease_time)}) + change = Changeset.change(websub, %{valid_until: + NaiveDateTime.add(websub.updated_at, lease_time)}) websub = Repo.update!(change) # Just spawn that for now, maybe pool later. @@ -81,7 +85,8 @@ def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) d end defp get_subscription(topic, callback) do - Repo.get_by(WebsubServerSubscription, topic: topic, callback: callback) || %WebsubServerSubscription{} + Repo.get_by(WebsubServerSubscription, topic: topic, callback: callback) || + %WebsubServerSubscription{} end defp lease_time(%{"hub.lease_seconds" => lease_seconds}) do diff --git a/lib/xml_builder.ex b/lib/xml_builder.ex index ac1ac8a74..c6d144903 100644 --- a/lib/xml_builder.ex +++ b/lib/xml_builder.ex @@ -30,13 +30,13 @@ def to_xml(%NaiveDateTime{} = time) do NaiveDateTime.to_iso8601(time) end - def to_doc(content), do: "" <> to_xml(content) + def to_doc(content), do: ~s() <> to_xml(content) defp make_open_tag(tag, attributes) do attributes_string = for {attribute, value} <- attributes do "#{attribute}=\"#{value}\"" end |> Enum.join(" ") - Enum.join([tag, attributes_string], " ") |> String.strip + [tag, attributes_string] |> Enum.join(" ") |> String.strip end end diff --git a/mix.exs b/mix.exs index 0e54f0246..72c6fb159 100644 --- a/mix.exs +++ b/mix.exs @@ -41,6 +41,7 @@ defp deps do {:cachex, "~> 2.1"}, {:httpoison, "~> 0.11.1"}, {:ex_machina, "~> 2.0", only: :test}, + {:credo, "~> 0.7", only: [:dev, :test]}, {:mix_test_watch, "~> 0.2", only: :dev}] end diff --git a/mix.lock b/mix.lock index 225a62f7a..0bc4462c3 100644 --- a/mix.lock +++ b/mix.lock @@ -1,4 +1,5 @@ -%{"cachex": {:hex, :cachex, "2.1.0", "fad49b4e78d11c6c314e75bd8c9408f5b78cb065c047442798caed10803ee3be", [:mix], [{:eternal, "~> 1.1", [hex: :eternal, optional: false]}]}, +%{"bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], []}, + "cachex": {:hex, :cachex, "2.1.0", "fad49b4e78d11c6c314e75bd8c9408f5b78cb065c047442798caed10803ee3be", [:mix], [{:eternal, "~> 1.1", [hex: :eternal, optional: false]}]}, "calendar": {:hex, :calendar, "0.16.1", "782327ad8bae7c797b887840dc4ddb933f05ce6e333e5b04964d7a5d5f79bde3", [:mix], [{:tzdata, "~> 0.5.8 or ~> 0.1.201603", [hex: :tzdata, optional: false]}]}, "certifi": {:hex, :certifi, "1.0.0", "1c787a85b1855ba354f0b8920392c19aa1d06b0ee1362f9141279620a5be2039", [:rebar3], []}, "comeonin": {:hex, :comeonin, "3.0.2", "8b213268a6634bd2e31a8035a963e974681d13ccc1f73f2ae664b6ac4e993c96", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, optional: false]}]}, @@ -6,6 +7,7 @@ "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], []}, "cowboy": {:hex, :cowboy, "1.1.2", "61ac29ea970389a88eca5a65601460162d370a70018afe6f949a29dca91f3bb0", [:rebar3], [{:cowlib, "~> 1.0.2", [hex: :cowlib, optional: false]}, {:ranch, "~> 1.3.2", [hex: :ranch, optional: false]}]}, "cowlib": {:hex, :cowlib, "1.0.2", "9d769a1d062c9c3ac753096f868ca121e2730b9a377de23dec0f7e08b1df84ee", [:make], []}, + "credo": {:hex, :credo, "0.7.3", "9827ab04002186af1aec014a811839a06f72aaae6cd5eed3919b248c8767dbf3", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, optional: false]}]}, "db_connection": {:hex, :db_connection, "1.1.2", "2865c2a4bae0714e2213a0ce60a1b12d76a6efba0c51fbda59c9ab8d1accc7a8", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]}, "decimal": {:hex, :decimal, "1.3.1", "157b3cedb2bfcb5359372a7766dd7a41091ad34578296e951f58a946fcab49c6", [:mix], []}, "deppie": {:hex, :deppie, "1.1.0", "cfb6fcee7dfb64eb78cb8505537971a0805131899326ad469ef10df04520f451", [:mix], []}, From 451d18af63fcf97f0d9621e5bfe296e1f18a0312 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 28 Apr 2017 09:51:47 +0200 Subject: [PATCH 019/107] Add proper callback route for websub confirmation. --- lib/pleroma/web/router.ex | 1 + lib/pleroma/web/websub/websub.ex | 6 +++--- lib/pleroma/web/websub/websub_controller.ex | 5 +++++ test/web/websub/websub_test.exs | 3 +++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c98eac688..bff981f9f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -75,6 +75,7 @@ def user_fetcher(username) do get "/users/:nickname/feed", OStatus.OStatusController, :feed post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming + post "/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation post "/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 4a35ca8fc..ad352ee26 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -2,8 +2,8 @@ defmodule Pleroma.Web.Websub do alias Pleroma.Repo alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription} alias Pleroma.Web.OStatus.FeedRepresenter - alias Pleroma.Web.OStatus - alias Pleroma.Web.XML + alias Pleroma.Web.{XML, Endpoint, OStatus} + alias Pleroma.Web.Router.Helpers require Logger import Ecto.Query @@ -136,7 +136,7 @@ def request_subscription(websub, poster \\ &HTTPoison.post/3, timeout \\ 10_000) "hub.mode": "subscribe", "hub.topic": websub.topic, "hub.secret": websub.secret, - "hub.callback": "https://social.heldscal.la/callback" + "hub.callback": Helpers.websub_url(Endpoint, :websub_subscription_confirmation, websub.id) ] # This checks once a second if we are confirmed yet diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index 5d54c6ef5..c6b15c0c2 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -15,4 +15,9 @@ def websub_subscription_request(conn, %{"nickname" => nickname} = params) do |> send_resp(500, reason) end end + + def websub_subscription_confirmation(conn, params) do + IO.inspect(params) + conn + end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index bf243ac91..ca04a55cd 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -3,11 +3,13 @@ def verify(sub) do {:ok, sub} end end + defmodule Pleroma.Web.WebsubTest do use Pleroma.DataCase alias Pleroma.Web.Websub alias Pleroma.Web.Websub.WebsubServerSubscription import Pleroma.Factory + alias Pleroma.Web.Router.Helpers test "a verification of a request that is accepted" do sub = insert(:websub_subscription) @@ -121,6 +123,7 @@ test "calls the hub, requests topic" do poster = fn (^hub, {:form, data}, _headers) -> assert Keyword.get(data, :"hub.mode") == "subscribe" + assert Keyword.get(data, :"hub.callback") == Helpers.websub_url(Pleroma.Web.Endpoint, :websub_subscription_confirmation, websub.id) {:ok, %{status_code: 202}} end From 1422e7aa84a897c6026e9dcd26b7d5955050687a Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 28 Apr 2017 15:45:10 +0200 Subject: [PATCH 020/107] Handle incoming websub subscriptions. --- config/config.exs | 3 +- config/test.exs | 3 +- lib/pleroma/web/federator/federator.ex | 4 +- lib/pleroma/web/router.ex | 3 +- lib/pleroma/web/websub/websub.ex | 7 ++- lib/pleroma/web/websub/websub_controller.ex | 34 ++++++++++-- test/web/websub/websub_controller_test.exs | 59 +++++++++++++++++++++ 7 files changed, 102 insertions(+), 11 deletions(-) diff --git a/config/config.exs b/config/config.exs index 3826dddff..a5df31b5a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -30,7 +30,8 @@ "application/xrd+xml" => ["xrd+xml"] } -config :pleroma, :websub_verifier, Pleroma.Web.Websub +config :pleroma, :websub, Pleroma.Web.Websub +config :pleroma, :ostatus, Pleroma.Web.OStatus # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/config/test.exs b/config/test.exs index 5d91279a2..85b6ad26b 100644 --- a/config/test.exs +++ b/config/test.exs @@ -25,4 +25,5 @@ # Reduce hash rounds for testing config :comeonin, :pbkdf2_rounds, 1 -config :pleroma, :websub_verifier, Pleroma.Web.WebsubMock +config :pleroma, :websub, Pleroma.Web.WebsubMock +config :pleroma, :ostatus, Pleroma.Web.OStatusMock diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index f489ed837..38df13540 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -2,7 +2,7 @@ defmodule Pleroma.Web.Federator do alias Pleroma.User require Logger - @websub_verifier Application.get_env(:pleroma, :websub_verifier) + @websub Application.get_env(:pleroma, :websub) def handle(:publish, activity) do Logger.debug("Running publish for #{activity.data["id"]}") @@ -13,7 +13,7 @@ def handle(:publish, activity) do def handle(:verify_websub, websub) do Logger.debug("Running websub verification for #{websub.id} (#{websub.topic}, #{websub.callback})") - @websub_verifier.verify(websub) + @websub.verify(websub) end def handle(type, payload) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index bff981f9f..2ff75ec5d 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -75,8 +75,9 @@ def user_fetcher(username) do get "/users/:nickname/feed", OStatus.OStatusController, :feed post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming - post "/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation post "/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request + get "/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation + post "/push/subscriptions/:id", Websub.WebsubController, :websub_incoming end scope "/.well-known", Pleroma.Web do diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index ad352ee26..ad9e47b46 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -42,8 +42,7 @@ def publish(topic, user, activity) do response = FeedRepresenter.to_simple_form(user, [activity], [user]) |> :xmerl.export_simple(:xmerl_xml) - signature = :crypto.hmac(:sha, sub.secret, response) |> Base.encode16 - + signature = sign(sub.secret, response) HTTPoison.post(sub.callback, response, [ {"Content-Type", "application/atom+xml"}, {"X-Hub-Signature", "sha1=#{signature}"} @@ -51,6 +50,10 @@ def publish(topic, user, activity) do end) end + def sign(secret, doc) do + :crypto.hmac(:sha, secret, doc) |> Base.encode16 + end + def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) do with {:ok, topic} <- valid_topic(params, user), {:ok, lease_time} <- lease_time(params), diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index c6b15c0c2..cd59a70a3 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -1,7 +1,11 @@ defmodule Pleroma.Web.Websub.WebsubController do use Pleroma.Web, :controller - alias Pleroma.User + alias Pleroma.{Repo, User} alias Pleroma.Web.Websub + alias Pleroma.Web.Websub.WebsubClientSubscription + require Logger + + @ostatus Application.get_env(:pleroma, :ostatus) def websub_subscription_request(conn, %{"nickname" => nickname} = params) do user = User.get_cached_by_nickname(nickname) @@ -16,8 +20,30 @@ def websub_subscription_request(conn, %{"nickname" => nickname} = params) do end end - def websub_subscription_confirmation(conn, params) do - IO.inspect(params) - conn + def websub_subscription_confirmation(conn, %{"id" => id, "hub.mode" => "subscribe", "hub.challenge" => challenge, "hub.topic" => topic}) do + with %WebsubClientSubscription{} = websub <- Repo.get_by(WebsubClientSubscription, id: id, topic: topic) do + change = Ecto.Changeset.change(websub, %{state: "accepted"}) + {:ok, _websub} = Repo.update(change) + conn + |> send_resp(200, challenge) + else _e -> + conn + |> send_resp(500, "Error") + end + end + + def websub_incoming(conn, %{"id" => id}) do + with "sha1=" <> signature <- hd(get_req_header(conn, "x-hub-signature")), + %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id), + {:ok, body, _conn} = read_body(conn), + ^signature <- Websub.sign(websub.secret, body) do + @ostatus.handle_incoming(body) + conn + |> send_resp(200, "OK") + else _e -> + Logger.debug("Can't handle incoming subscription post") + conn + |> send_resp(500, "Error") + end end end diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index 8368cafea..521bbb9aa 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -1,6 +1,9 @@ defmodule Pleroma.Web.Websub.WebsubControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory + alias Pleroma.Web.Websub.WebsubClientSubscription + alias Pleroma.{Repo, Activity} + alias Pleroma.Web.Websub test "websub subscription request", %{conn: conn} do user = insert(:user) @@ -20,4 +23,60 @@ test "websub subscription request", %{conn: conn} do assert response(conn, 202) == "Accepted" end + + test "websub subscription confirmation", %{conn: conn} do + websub = insert(:websub_client_subscription) + + params = %{ + "hub.mode" => "subscribe", + "hub.topic" => websub.topic, + "hub.challenge" => "some challenge", + "hub.lease_seconds" => 100 + } + + conn = conn + |> get("/push/subscriptions/#{websub.id}", params) + + websub = Repo.get(WebsubClientSubscription, websub.id) + + assert response(conn, 200) == "some challenge" + assert websub.state == "accepted" + end + + test "handles incoming feed updates", %{conn: conn} do + websub = insert(:websub_client_subscription) + doc = "some stuff" + signature = Websub.sign(websub.secret, doc) + + conn = conn + |> put_req_header("x-hub-signature", "sha1=" <> signature) + |> put_req_header("content-type", "application/atom+xml") + |> post("/push/subscriptions/#{websub.id}", doc) + + assert response(conn, 200) == "OK" + + assert length(Repo.all(Activity)) == 1 + end + + test "rejects incoming feed updates with the wrong signature", %{conn: conn} do + websub = insert(:websub_client_subscription) + doc = "some stuff" + signature = Websub.sign("wrong secret", doc) + + conn = conn + |> put_req_header("x-hub-signature", "sha1=" <> signature) + |> put_req_header("content-type", "application/atom+xml") + |> post("/push/subscriptions/#{websub.id}", doc) + + assert response(conn, 500) == "Error" + + assert length(Repo.all(Activity)) == 0 + end +end + +defmodule Pleroma.Web.OStatusMock do + import Pleroma.Factory + def handle_incoming(_doc) do + insert(:note_activity) + end end From 59d4cc60364fd1abf5cbc881e88757b378456b64 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 28 Apr 2017 15:53:45 +0200 Subject: [PATCH 021/107] normalize hex number. --- lib/pleroma/web/websub/websub_controller.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index cd59a70a3..e5ecf6523 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -34,6 +34,7 @@ def websub_subscription_confirmation(conn, %{"id" => id, "hub.mode" => "subscrib def websub_incoming(conn, %{"id" => id}) do with "sha1=" <> signature <- hd(get_req_header(conn, "x-hub-signature")), + signature <- String.upcase(signature), %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id), {:ok, body, _conn} = read_body(conn), ^signature <- Websub.sign(websub.secret, body) do From ca40dda04c114c32ca9ecfe5f998a083448a189c Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 28 Apr 2017 17:41:12 +0200 Subject: [PATCH 022/107] Add some basic webfingering. --- lib/pleroma/web/web_finger/web_finger.ex | 37 ++++++++++++++++++++++++ test/fixtures/webfinger.xml | 20 +++++++++++++ test/web/web_finger/web_finger_test.exs | 20 ++++++++++++- 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/webfinger.xml diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 18459e8f0..08ff6d278 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -2,6 +2,8 @@ defmodule Pleroma.Web.WebFinger do alias Pleroma.XmlBuilder alias Pleroma.User alias Pleroma.Web.OStatus + alias Pleroma.Web.XML + require Logger def host_meta() do base_url = Pleroma.Web.base_url @@ -37,4 +39,39 @@ def represent_user(user) do } |> XmlBuilder.to_doc end + + # FIXME: Make this call the host-meta to find the actual address. + defp webfinger_address(domain) do + "https://#{domain}/.well-known/webfinger" + end + + defp webfinger_from_xml(doc) do + magic_key = XML.string_from_xpath(~s{//Link[@rel="magic-public-key"]/@href}, doc) + "data:application/magic-public-key," <> magic_key = magic_key + topic = XML.string_from_xpath(~s{//Link[@rel="http://schemas.google.com/g/2010#updates-from"]/@href}, doc) + subject = XML.string_from_xpath("//Subject", doc) + salmon = XML.string_from_xpath(~s{//Link[@rel="salmon"]/@href}, doc) + data = %{ + magic_key: magic_key, + topic: topic, + subject: subject, + salmon: salmon + } + {:ok, data} + end + + def finger(account, getter \\ &HTTPoison.get/3) do + [name, domain] = String.split(account, "@") + address = webfinger_address(domain) + with {:ok, %{status_code: status_code, body: body}} when status_code in 200..299 <- getter.(address, ["Accept": "application/xrd+xml"], [params: [resource: account]]), + doc <- XML.parse_document(body), + {:ok, data} <- webfinger_from_xml(doc) do + {:ok, data} + else + e -> + Logger.debug("Couldn't finger #{account}.") + Logger.debug(e) + {:error, e} + end + end end diff --git a/test/fixtures/webfinger.xml b/test/fixtures/webfinger.xml new file mode 100644 index 000000000..4cde42e3f --- /dev/null +++ b/test/fixtures/webfinger.xml @@ -0,0 +1,20 @@ + + + acct:shp@social.heldscal.la + https://social.heldscal.la/user/29191 + https://social.heldscal.la/shp + https://social.heldscal.la/index.php/user/29191 + https://social.heldscal.la/index.php/shp + + + + + + + + + + + + + diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index 8a3007ff9..e5347a2b0 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -1,11 +1,29 @@ defmodule Pleroma.Web.WebFingerTest do use Pleroma.DataCase + alias Pleroma.Web.WebFinger describe "host meta" do test "returns a link to the xml lrdd" do - host_info = Pleroma.Web.WebFinger.host_meta + host_info = WebFinger.host_meta() assert String.contains?(host_info, Pleroma.Web.base_url) end end + + describe "fingering" do + test "returns the info for a user" do + user = "shp@social.heldscal.la" + + getter = fn(_url, _headers, [params: [resource: ^user]]) -> + {:ok, %{status_code: 200, body: File.read!("test/fixtures/webfinger.xml")}} + end + + {:ok, data} = WebFinger.finger(user, getter) + + assert data.magic_key == "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB" + assert data.topic == "https://social.heldscal.la/api/statuses/user_timeline/29191.atom" + assert data.subject == "acct:shp@social.heldscal.la" + assert data.salmon == "https://social.heldscal.la/main/salmon/user/29191" + end + end end From 69922bc724736fb07bf36beaef42d944158d9269 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 29 Apr 2017 17:51:59 +0200 Subject: [PATCH 023/107] Add user info gathering. --- lib/pleroma/web/ostatus/ostatus.ex | 11 +++++++++++ lib/pleroma/web/web_finger/web_finger.ex | 4 ++-- lib/pleroma/web/websub/websub.ex | 16 +++++++++++++--- test/web/ostatus/ostatus_test.exs | 22 ++++++++++++++++++++++ test/web/websub/websub_test.exs | 11 +++++++++-- 5 files changed, 57 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 89b482592..90be86755 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.OStatus do alias Pleroma.{Repo, User, Web} alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.{WebFinger, Websub} def feed_path(user) do "#{user.ap_id}/feed.atom" @@ -134,4 +135,14 @@ defp make_avatar_object(author_doc) do nil end end + + def gather_user_info(username) do + with {:ok, webfinger_data} <- WebFinger.finger(username), + {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data.topic) do + {:ok, Map.merge(webfinger_data, feed_data) |> Map.put(:fqn, username)} + else e -> + Logger.debug("Couldn't gather info for #{username}") + {:error, e} + end + end end diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 08ff6d278..1d8c4d0c8 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -18,7 +18,7 @@ def host_meta() do def webfinger(resource) do host = Pleroma.Web.host - regex = ~r/acct:(?\w+)@#{host}/ + regex = ~r/(acct:)?(?\w+)@#{host}/ case Regex.named_captures(regex, resource) do %{"username" => username} -> user = User.get_cached_by_nickname(username) @@ -70,7 +70,7 @@ def finger(account, getter \\ &HTTPoison.get/3) do else e -> Logger.debug("Couldn't finger #{account}.") - Logger.debug(e) + Logger.debug(inspect(e)) {:error, e} end end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index ad9e47b46..c1d48ad7a 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -121,14 +121,24 @@ def subscribe(user, topic, requester \\ &request_subscription/1) do requester.(subscription) end - def discover(topic, getter \\ &HTTPoison.get/1) do + def gather_feed_data(topic, getter \\ &HTTPoison.get/1) do with {:ok, response} <- getter.(topic), status_code when status_code in 200..299 <- response.status_code, body <- response.body, doc <- XML.parse_document(body), - url when not is_nil(url) <- XML.string_from_xpath(~S{/feed/link[@rel="self"]/@href}, doc), + uri when not is_nil(uri) <- XML.string_from_xpath("/feed/author[1]/uri", doc), hub when not is_nil(hub) <- XML.string_from_xpath(~S{/feed/link[@rel="hub"]/@href}, doc) do - {:ok, %{url: url, hub: hub}} + + name = XML.string_from_xpath("/feed/author[1]/name", doc) + preferredUsername = XML.string_from_xpath("/feed/author[1]/poco:preferredUsername", doc) + displayName = XML.string_from_xpath("/feed/author[1]/poco:displayName", doc) + + {:ok, %{ + uri: uri, + hub: hub, + nickname: preferredUsername || name, + name: displayName || name + }} else e -> {:error, e} end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 140b32f36..2a5156b31 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -63,4 +63,26 @@ test "tries to use the information in poco fields" do assert user == user_again end end + + describe "gathering user info from a user id" do + test "it returns user info in a hash" do + user = "shp@social.heldscal.la" + + # TODO: make test local + {:ok, data} = OStatus.gather_user_info(user) + + expected = %{ + hub: "https://social.heldscal.la/main/push/hub", + magic_key: "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", + name: "shp", + nickname: "shp", + salmon: "https://social.heldscal.la/main/salmon/user/29191", + subject: "acct:shp@social.heldscal.la", + topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", + uri: "https://social.heldscal.la/user/29191", + fqn: user + } + assert data == expected + end + end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index ca04a55cd..1b1ef3fa6 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -112,8 +112,15 @@ test "discovers the hub and canonical url" do {:ok, %{status_code: 200, body: doc}} end - {:ok, discovered} = Websub.discover(topic, getter) - assert %{hub: "https://mastodon.social/api/push", url: topic} == discovered + {:ok, discovered} = Websub.gather_feed_data(topic, getter) + expected = %{ + hub: "https://mastodon.social/api/push", + uri: "https://mastodon.social/users/lambadalambda", + nickname: "lambadalambda", + name: "Critical Value" + } + + assert expected == discovered end test "calls the hub, requests topic" do From 427bac0966c551eb16eaa6595d99fc5361a32ea9 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 29 Apr 2017 19:06:01 +0200 Subject: [PATCH 024/107] Rework remote user subscription. --- lib/pleroma/web/ostatus/ostatus.ex | 44 +++++++------------ lib/pleroma/web/web_finger/web_finger.ex | 18 ++++++-- lib/pleroma/web/websub/websub.ex | 10 +++-- test/web/ostatus/ostatus_test.exs | 54 ++++++++++++------------ test/web/websub/websub_test.exs | 11 ++--- 5 files changed, 70 insertions(+), 67 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 90be86755..3e239179e 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -37,8 +37,8 @@ def handle_incoming(xml_string) do def handle_note(doc) do content_html = string_from_xpath("/entry/content[1]", doc) - [author] = :xmerl_xpath.string('/entry/author[1]', doc) - {:ok, actor} = find_or_make_user(author) + uri = string_from_xpath("/entry/author/uri[1]", doc) + {:ok, actor} = find_or_make_user(uri) context = string_from_xpath("/entry/ostatus:conversation[1]", doc) |> String.trim context = if String.length(context) > 0 do @@ -78,42 +78,30 @@ def handle_note(doc) do ActivityPub.create(to, actor, context, object, %{}, date) end - def find_or_make_user(author_doc) do - {:xmlObj, :string, uri } = :xmerl_xpath.string('string(/author[1]/uri)', author_doc) - + def find_or_make_user(uri) do query = from user in User, - where: user.local == false and fragment("? @> ?", user.info, ^%{ostatus_uri: to_string(uri)}) + where: user.local == false and fragment("? @> ?", user.info, ^%{uri: uri}) user = Repo.one(query) if is_nil(user) do - make_user(author_doc) + make_user(uri) else {:ok, user} end end - def make_user(author_doc) do - author = string_from_xpath("/author[1]/uri", author_doc) - name = string_from_xpath("/author[1]/name", author_doc) - preferredUsername = string_from_xpath("/author[1]/poco:preferredUsername", author_doc) - displayName = string_from_xpath("/author[1]/poco:displayName", author_doc) - avatar = make_avatar_object(author_doc) - - data = %{ - local: false, - name: preferredUsername || name, - nickname: displayName || name, - ap_id: author, - info: %{ - "ostatus_uri" => author, - "host" => URI.parse(author).host, - "system" => "ostatus" - }, - avatar: avatar - } - - Repo.insert(Ecto.Changeset.change(%User{}, data)) + def make_user(uri) do + with {:ok, info} <- gather_user_info(uri) do + data = %{ + local: false, + name: info.name, + nickname: info.nickname, + ap_id: info.uri, + info: info + } + Repo.insert(Ecto.Changeset.change(%User{}, data)) + end end # TODO: Just takes the first one for now. diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 1d8c4d0c8..49796dab8 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -42,7 +42,7 @@ def represent_user(user) do # FIXME: Make this call the host-meta to find the actual address. defp webfinger_address(domain) do - "https://#{domain}/.well-known/webfinger" + "//#{domain}/.well-known/webfinger" end defp webfinger_from_xml(doc) do @@ -61,9 +61,21 @@ defp webfinger_from_xml(doc) do end def finger(account, getter \\ &HTTPoison.get/3) do - [name, domain] = String.split(account, "@") + domain = with [_name, domain] <- String.split(account, "@") do + domain + else _e -> + URI.parse(account).host + end address = webfinger_address(domain) - with {:ok, %{status_code: status_code, body: body}} when status_code in 200..299 <- getter.(address, ["Accept": "application/xrd+xml"], [params: [resource: account]]), + + # try https first + response = with {:ok, result} <- getter.("https:" <> address, ["Accept": "application/xrd+xml"], [params: [resource: account]]) do + {:ok, result} + else _ -> + getter.("http:" <> address, ["Accept": "application/xrd+xml"], [params: [resource: account]]) + end + + with {:ok, %{status_code: status_code, body: body}} when status_code in 200..299 <- response, doc <- XML.parse_document(body), {:ok, data} <- webfinger_from_xml(doc) do {:ok, data} diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index c1d48ad7a..8e3e0a54e 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -102,19 +102,21 @@ defp valid_topic(%{"hub.topic" => topic}, user) do end end - def subscribe(user, topic, requester \\ &request_subscription/1) do + def subscribe(subscriber, subscribed, requester \\ &request_subscription/1) do + topic = subscribed.info["topic"] # FIXME: Race condition, use transactions {:ok, subscription} = with subscription when not is_nil(subscription) <- Repo.get_by(WebsubClientSubscription, topic: topic) do - subscribers = [user.ap_id, subscription.subcribers] |> Enum.uniq + subscribers = [subscriber.ap_id, subscription.subscribers] |> Enum.uniq change = Ecto.Changeset.change(subscription, %{subscribers: subscribers}) Repo.update(change) else _e -> subscription = %WebsubClientSubscription{ topic: topic, - subscribers: [user.ap_id], + hub: subscribed.info["hub"], + subscribers: [subscriber.ap_id], state: "requested", secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64, - user: user + user: subscribed } Repo.insert(subscription) end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 2a5156b31..4f396d940 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -25,40 +25,20 @@ test "handle incoming replies" do end describe "new remote user creation" do - test "make new user or find them based on an 'author' xml doc" do - incoming = File.read!("test/fixtures/user_name_only.xml") - doc = XML.parse_document(incoming) - - {:ok, user} = OStatus.find_or_make_user(doc) - - assert user.name == "lambda" - assert user.nickname == "lambda" - assert user.local == false - assert user.info["ostatus_uri"] == "http://gs.example.org:4040/index.php/user/1" - assert user.info["system"] == "ostatus" - assert user.ap_id == "http://gs.example.org:4040/index.php/user/1" - - {:ok, user_again} = OStatus.find_or_make_user(doc) - - assert user == user_again - end - test "tries to use the information in poco fields" do - incoming = File.read!("test/fixtures/user_full.xml") - doc = XML.parse_document(incoming) + # TODO make test local + uri = "https://social.heldscal.la/user/23211" - {:ok, user} = OStatus.find_or_make_user(doc) + {:ok, user} = OStatus.find_or_make_user(uri) + user = Repo.get(Pleroma.User, user.id) assert user.name == "Constance Variable" assert user.nickname == "lambadalambda" assert user.local == false - assert user.info["ostatus_uri"] == "http://gs.example.org:4040/index.php/user/1" - assert user.info["system"] == "ostatus" - assert user.ap_id == "http://gs.example.org:4040/index.php/user/1" + assert user.info["uri"] == uri + assert user.ap_id == uri - assert List.first(user.avatar["url"])["href"] == "http://gs.example.org:4040/theme/neo-gnu/default-avatar-profile.png" - - {:ok, user_again} = OStatus.find_or_make_user(doc) + {:ok, user_again} = OStatus.find_or_make_user(uri) assert user == user_again end @@ -84,5 +64,25 @@ test "it returns user info in a hash" do } assert data == expected end + + test "it works with the uri" do + user = "https://social.heldscal.la/user/29191" + + # TODO: make test local + {:ok, data} = OStatus.gather_user_info(user) + + expected = %{ + hub: "https://social.heldscal.la/main/push/hub", + magic_key: "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", + name: "shp", + nickname: "shp", + salmon: "https://social.heldscal.la/main/salmon/user/29191", + subject: "https://social.heldscal.la/user/29191", + topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", + uri: "https://social.heldscal.la/user/29191", + fqn: user + } + assert data == expected + end end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 1b1ef3fa6..25c2b8baa 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -93,12 +93,13 @@ def accepting_verifier(subscription) do end test "initiate a subscription for a given user and topic" do - user = insert(:user) - topic = "http://example.org/some-topic.atom" + subscriber = insert(:user) + user = insert(:user, %{info: %{ "topic" => "some_topic", "hub" => "some_hub"}}) - {:ok, websub} = Websub.subscribe(user, topic, &accepting_verifier/1) - assert websub.subscribers == [user.ap_id] - assert websub.topic == topic + {:ok, websub} = Websub.subscribe(subscriber, user, &accepting_verifier/1) + assert websub.subscribers == [subscriber.ap_id] + assert websub.topic == "some_topic" + assert websub.hub == "some_hub" assert is_binary(websub.secret) assert websub.user == user assert websub.state == "accepted" From ba1ea770012893ea818f248e9a0a2ee3ab854676 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 29 Apr 2017 19:47:56 +0200 Subject: [PATCH 025/107] Make key fetching use ostatus fetching. --- lib/pleroma/web/salmon/salmon.ex | 18 +++++------------- test/fixtures/salmon2.xml | 2 ++ test/web/salmon/salmon_test.exs | 8 ++++++++ 3 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 test/fixtures/salmon2.xml diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 99cca1f55..777898cfa 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -21,24 +21,16 @@ def decode(salmon) do [data, type, encoding, alg, sig] end + # TODO rewrite in with-stile + # Make it fetch the key from the saved user if there is one def fetch_magic_key(salmon) do [data, _, _, _, _] = decode(salmon) doc = XML.parse_document(data) - {:xmlObj, :string, uri} = :xmerl_xpath.string('string(//author[1]/uri)', doc) + uri = XML.string_from_xpath("/entry/author[1]/uri", doc) - uri = to_string(uri) - base = URI.parse(uri).host + {:ok, info} = Pleroma.Web.OStatus.gather_user_info(uri) - # TODO: Find out if this endpoint is mandated by the standard. - # At least diaspora does it differently - {:ok, response} = HTTPoison.get(base <> "/.well-known/webfinger", ["Accept": "application/xrd+xml"], [params: [resource: uri]]) - - doc = XML.parse_document(response.body) - - {:xmlObj, :string, magickey} = :xmerl_xpath.string('string(//Link[@rel="magic-public-key"]/@href)', doc) - "data:application/magic-public-key," <> magickey = to_string(magickey) - - magickey + info.magic_key end def decode_and_validate(magickey, salmon) do diff --git a/test/fixtures/salmon2.xml b/test/fixtures/salmon2.xml new file mode 100644 index 000000000..d8ecbc17e --- /dev/null +++ b/test/fixtures/salmon2.xml @@ -0,0 +1,2 @@ + +PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiID8-PGVudHJ5IHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDA1L0F0b20iIHhtbG5zOnRocj0iaHR0cDovL3B1cmwub3JnL3N5bmRpY2F0aW9uL3RocmVhZC8xLjAiIHhtbG5zOmFjdGl2aXR5PSJodHRwOi8vYWN0aXZpdHlzdHJlYS5tcy9zcGVjLzEuMC8iIHhtbG5zOmdlb3Jzcz0iaHR0cDovL3d3dy5nZW9yc3Mub3JnL2dlb3JzcyIgeG1sbnM6b3N0YXR1cz0iaHR0cDovL29zdGF0dXMub3JnL3NjaGVtYS8xLjAiIHhtbG5zOnBvY289Imh0dHA6Ly9wb3J0YWJsZWNvbnRhY3RzLm5ldC9zcGVjLzEuMCIgeG1sbnM6bWVkaWE9Imh0dHA6Ly9wdXJsLm9yZy9zeW5kaWNhdGlvbi9hdG9tbWVkaWEiIHhtbG5zOnN0YXR1c25ldD0iaHR0cDovL3N0YXR1cy5uZXQvc2NoZW1hL2FwaS8xLyI-CiA8YWN0aXZpdHk6b2JqZWN0LXR5cGU-aHR0cDovL2FjdGl2aXR5c3RyZWEubXMvc2NoZW1hLzEuMC9ub3RlPC9hY3Rpdml0eTpvYmplY3QtdHlwZT4KIDxpZD50YWc6c29jaWFsLmhlbGRzY2FsLmxhLDIwMTctMDQtMjk6bm90aWNlSWQ9MTk2NzEwNjpvYmplY3RUeXBlPW5vdGU8L2lkPgogPHRpdGxlPk5ldyBub3RlIGJ5IGxhbWJhZGFsYW1iZGE8L3RpdGxlPgogPGNvbnRlbnQgdHlwZT0iaHRtbCI-dGVzdCBAJmx0O2EgaHJlZj0mcXVvdDtodHRwczovL3BsZXJvbWEuc295a2FmLmNvbS91c2Vycy9sYWluJnF1b3Q7IGNsYXNzPSZxdW90O2gtY2FyZCB1LXVybCBwLW5pY2tuYW1lIG1lbnRpb24mcXVvdDsmZ3Q7bGFpbiZsdDsvYSZndDs8L2NvbnRlbnQ-CiA8bGluayByZWw9ImFsdGVybmF0ZSIgdHlwZT0idGV4dC9odG1sIiBocmVmPSJodHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9ub3RpY2UvMTk2NzEwNiIvPgogPHN0YXR1c19uZXQgbm90aWNlX2lkPSIxOTY3MTA2Ij48L3N0YXR1c19uZXQ-CiA8YWN0aXZpdHk6dmVyYj5odHRwOi8vYWN0aXZpdHlzdHJlYS5tcy9zY2hlbWEvMS4wL3Bvc3Q8L2FjdGl2aXR5OnZlcmI-CiA8cHVibGlzaGVkPjIwMTctMDQtMjlUMTc6Mjg6MjErMDA6MDA8L3B1Ymxpc2hlZD4KIDx1cGRhdGVkPjIwMTctMDQtMjlUMTc6Mjg6MjErMDA6MDA8L3VwZGF0ZWQ-CiA8YXV0aG9yPgogIDxhY3Rpdml0eTpvYmplY3QtdHlwZT5odHRwOi8vYWN0aXZpdHlzdHJlYS5tcy9zY2hlbWEvMS4wL3BlcnNvbjwvYWN0aXZpdHk6b2JqZWN0LXR5cGU-CiAgPHVyaT5odHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS91c2VyLzIzMjExPC91cmk-CiAgPG5hbWU-bGFtYmFkYWxhbWJkYTwvbmFtZT4KICA8c3VtbWFyeT5DYWxsIG1lIERlYWNvbiBCbHVlcy48L3N1bW1hcnk-CiAgPGxpbmsgcmVsPSJhbHRlcm5hdGUiIHR5cGU9InRleHQvaHRtbCIgaHJlZj0iaHR0cHM6Ly9zb2NpYWwuaGVsZHNjYWwubGEvbGFtYmFkYWxhbWJkYSIvPgogIDxsaW5rIHJlbD0iYXZhdGFyIiB0eXBlPSJpbWFnZS9qcGVnIiBtZWRpYTp3aWR0aD0iMjM2IiBtZWRpYTpoZWlnaHQ9IjIzNiIgaHJlZj0iaHR0cHM6Ly9zb2NpYWwuaGVsZHNjYWwubGEvYXZhdGFyLzIzMjExLW9yaWdpbmFsLTIwMTcwNDE2MTE0MjU1LmpwZWciLz4KICA8bGluayByZWw9ImF2YXRhciIgdHlwZT0iaW1hZ2UvanBlZyIgbWVkaWE6d2lkdGg9Ijk2IiBtZWRpYTpoZWlnaHQ9Ijk2IiBocmVmPSJodHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9hdmF0YXIvMjMyMTEtOTYtMjAxNzA0MTYxMTQyNTUuanBlZyIvPgogIDxsaW5rIHJlbD0iYXZhdGFyIiB0eXBlPSJpbWFnZS9qcGVnIiBtZWRpYTp3aWR0aD0iNDgiIG1lZGlhOmhlaWdodD0iNDgiIGhyZWY9Imh0dHBzOi8vc29jaWFsLmhlbGRzY2FsLmxhL2F2YXRhci8yMzIxMS00OC0yMDE3MDQxNjExNDI1NS5qcGVnIi8-CiAgPGxpbmsgcmVsPSJhdmF0YXIiIHR5cGU9ImltYWdlL2pwZWciIG1lZGlhOndpZHRoPSIyNCIgbWVkaWE6aGVpZ2h0PSIyNCIgaHJlZj0iaHR0cHM6Ly9zb2NpYWwuaGVsZHNjYWwubGEvYXZhdGFyLzIzMjExLTI0LTIwMTcwNDE2MTE0MjU3LmpwZWciLz4KICA8cG9jbzpwcmVmZXJyZWRVc2VybmFtZT5sYW1iYWRhbGFtYmRhPC9wb2NvOnByZWZlcnJlZFVzZXJuYW1lPgogIDxwb2NvOmRpc3BsYXlOYW1lPkNvbnN0YW5jZSBWYXJpYWJsZTwvcG9jbzpkaXNwbGF5TmFtZT4KICA8cG9jbzpub3RlPkNhbGwgbWUgRGVhY29uIEJsdWVzLjwvcG9jbzpub3RlPgogIDxwb2NvOmFkZHJlc3M-CiAgIDxwb2NvOmZvcm1hdHRlZD5CZXJsaW48L3BvY286Zm9ybWF0dGVkPgogIDwvcG9jbzphZGRyZXNzPgogIDxwb2NvOnVybHM-CiAgIDxwb2NvOnR5cGU-aG9tZXBhZ2U8L3BvY286dHlwZT4KICAgPHBvY286dmFsdWU-aHR0cHM6Ly9oZWxkc2NhbC5sYTwvcG9jbzp2YWx1ZT4KICAgPHBvY286cHJpbWFyeT50cnVlPC9wb2NvOnByaW1hcnk-CiAgPC9wb2NvOnVybHM-CiAgPGZvbGxvd2VycyB1cmw9Imh0dHBzOi8vc29jaWFsLmhlbGRzY2FsLmxhL2xhbWJhZGFsYW1iZGEvc3Vic2NyaWJlcnMiPjwvZm9sbG93ZXJzPgogIDxzdGF0dXNuZXQ6cHJvZmlsZV9pbmZvIGxvY2FsX2lkPSIyMzIxMSI-PC9zdGF0dXNuZXQ6cHJvZmlsZV9pbmZvPgogPC9hdXRob3I-CiA8bGluayByZWw9Im9zdGF0dXM6Y29udmVyc2F0aW9uIiBocmVmPSJodHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9jb252ZXJzYXRpb24vMTAwNzQ5NiIvPgogPG9zdGF0dXM6Y29udmVyc2F0aW9uIGhyZWY9Imh0dHBzOi8vc29jaWFsLmhlbGRzY2FsLmxhL2NvbnZlcnNhdGlvbi8xMDA3NDk2IiBsb2NhbF9pZD0iMTAwNzQ5NiIgcmVmPSJ0YWc6c29jaWFsLmhlbGRzY2FsLmxhLDIwMTctMDQtMjk6b2JqZWN0VHlwZT10aHJlYWQ6bm9uY2U9NDU5ZGYyMjM2NDFiMDNkZSI-dGFnOnNvY2lhbC5oZWxkc2NhbC5sYSwyMDE3LTA0LTI5Om9iamVjdFR5cGU9dGhyZWFkOm5vbmNlPTQ1OWRmMjIzNjQxYjAzZGU8L29zdGF0dXM6Y29udmVyc2F0aW9uPgogPGxpbmsgcmVsPSJtZW50aW9uZWQiIG9zdGF0dXM6b2JqZWN0LXR5cGU9Imh0dHA6Ly9hY3Rpdml0eXN0cmVhLm1zL3NjaGVtYS8xLjAvcGVyc29uIiBocmVmPSJodHRwczovL3BsZXJvbWEuc295a2FmLmNvbS91c2Vycy9sYWluIi8-CiA8bGluayByZWw9Im1lbnRpb25lZCIgb3N0YXR1czpvYmplY3QtdHlwZT0iaHR0cDovL2FjdGl2aXR5c3RyZWEubXMvc2NoZW1hLzEuMC9jb2xsZWN0aW9uIiBocmVmPSJodHRwOi8vYWN0aXZpdHlzY2hlbWEub3JnL2NvbGxlY3Rpb24vcHVibGljIi8-CiA8c291cmNlPgogIDxpZD5odHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9hcGkvc3RhdHVzZXMvdXNlcl90aW1lbGluZS8yMzIxMS5hdG9tPC9pZD4KICA8dGl0bGU-Q29uc3RhbmNlIFZhcmlhYmxlPC90aXRsZT4KICA8bGluayByZWw9ImFsdGVybmF0ZSIgdHlwZT0idGV4dC9odG1sIiBocmVmPSJodHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9sYW1iYWRhbGFtYmRhIi8-CiAgPGxpbmsgcmVsPSJzZWxmIiB0eXBlPSJhcHBsaWNhdGlvbi9hdG9tK3htbCIgaHJlZj0iaHR0cHM6Ly9zb2NpYWwuaGVsZHNjYWwubGEvYXBpL3N0YXR1c2VzL3VzZXJfdGltZWxpbmUvMjMyMTEuYXRvbSIvPgogIDxsaW5rIHJlbD0ibGljZW5zZSIgaHJlZj0iaHR0cHM6Ly9jcmVhdGl2ZWNvbW1vbnMub3JnL2xpY2Vuc2VzL2J5LzMuMC8iLz4KICA8aWNvbj5odHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9hdmF0YXIvMjMyMTEtOTYtMjAxNzA0MTYxMTQyNTUuanBlZzwvaWNvbj4KICA8dXBkYXRlZD4yMDE3LTA0LTI5VDE3OjI4OjIxKzAwOjAwPC91cGRhdGVkPgogPC9zb3VyY2U-CiA8bGluayByZWw9InNlbGYiIHR5cGU9ImFwcGxpY2F0aW9uL2F0b20reG1sIiBocmVmPSJodHRwczovL3NvY2lhbC5oZWxkc2NhbC5sYS9hcGkvc3RhdHVzZXMvc2hvdy8xOTY3MTA2LmF0b20iLz4KIDxsaW5rIHJlbD0iZWRpdCIgdHlwZT0iYXBwbGljYXRpb24vYXRvbSt4bWwiIGhyZWY9Imh0dHBzOi8vc29jaWFsLmhlbGRzY2FsLmxhL2FwaS9zdGF0dXNlcy9zaG93LzE5NjcxMDYuYXRvbSIvPgogPHN0YXR1c25ldDpub3RpY2VfaW5mbyBsb2NhbF9pZD0iMTk2NzEwNiIgc291cmNlPSJQbGVyb21hIEZFIj48L3N0YXR1c25ldDpub3RpY2VfaW5mbz4KPC9lbnRyeT4Kbase64urlRSA-SHA256CJ3wiWW9Io6Y24To3PFBF8cGuvJG8ps5zEwu1k1kSAlSX7WcysvS4ZoPKICFrD4brJxMLpW3AQCLNPIa246-Y0noGiNdpj0w0_TWgWXukWo50pD7cWVugr15YCMUtC-v00iDYfZTlmrTVM6kSCcpAmGMbZPTaXVmKZryjTDoXSI= \ No newline at end of file diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index 6fbabd19f..aa77659d0 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -49,4 +49,12 @@ test "encodes an xml payload with a private key" do assert doc == decoded_doc end + + test "it gets a magic key" do + # TODO: Make test local + salmon = File.read!("test/fixtures/salmon2.xml") + key = Salmon.fetch_magic_key(salmon) + + assert key == "RSA.uzg6r1peZU0vXGADWxGJ0PE34WvmhjUmydbX5YYdOiXfODVLwCMi1umGoqUDm-mRu4vNEdFBVJU1CpFA7dKzWgIsqsa501i2XqElmEveXRLvNRWFB6nG03Q5OUY2as8eE54BJm0p20GkMfIJGwP6TSFb-ICp3QjzbatuSPJ6xCE=.AQAB" + end end From 20015b4b67cf0dfab6bdb658c9eb0e1ae04febdc Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 29 Apr 2017 20:08:45 +0200 Subject: [PATCH 026/107] Save remote users with fqn as nickname. --- lib/pleroma/web/ostatus/ostatus.ex | 4 +++- lib/pleroma/web/websub/websub.ex | 3 ++- test/web/ostatus/ostatus_test.exs | 4 +++- test/web/websub/websub_test.exs | 3 ++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 3e239179e..59c5d8e9e 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -96,10 +96,12 @@ def make_user(uri) do data = %{ local: false, name: info.name, - nickname: info.nickname, + nickname: info.nickname <> "@" <> info.host, ap_id: info.uri, info: info } + # TODO: Make remote user changeset + # SHould enforce fqn nickname Repo.insert(Ecto.Changeset.change(%User{}, data)) end end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 8e3e0a54e..3fd779fba 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -139,7 +139,8 @@ def gather_feed_data(topic, getter \\ &HTTPoison.get/1) do uri: uri, hub: hub, nickname: preferredUsername || name, - name: displayName || name + name: displayName || name, + host: URI.parse(uri).host }} else e -> {:error, e} diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 4f396d940..cc0975bb5 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -33,7 +33,7 @@ test "tries to use the information in poco fields" do user = Repo.get(Pleroma.User, user.id) assert user.name == "Constance Variable" - assert user.nickname == "lambadalambda" + assert user.nickname == "lambadalambda@social.heldscal.la" assert user.local == false assert user.info["uri"] == uri assert user.ap_id == uri @@ -60,6 +60,7 @@ test "it returns user info in a hash" do subject: "acct:shp@social.heldscal.la", topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", uri: "https://social.heldscal.la/user/29191", + host: "social.heldscal.la", fqn: user } assert data == expected @@ -80,6 +81,7 @@ test "it works with the uri" do subject: "https://social.heldscal.la/user/29191", topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", uri: "https://social.heldscal.la/user/29191", + host: "social.heldscal.la", fqn: user } assert data == expected diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 25c2b8baa..e0d71e16d 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -118,7 +118,8 @@ test "discovers the hub and canonical url" do hub: "https://mastodon.social/api/push", uri: "https://mastodon.social/users/lambadalambda", nickname: "lambadalambda", - name: "Critical Value" + name: "Critical Value", + host: "mastodon.social" } assert expected == discovered From a16da387d251edc4d1bae949146c807d217cee1f Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 29 Apr 2017 21:13:21 +0200 Subject: [PATCH 027/107] Handle full incoming feeds. --- lib/pleroma/web/ostatus/ostatus.ex | 34 +++++++------ test/fixtures/ostatus_incoming_post.xml | 57 ++++++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 17 +++++-- test/web/websub/websub_controller_test.exs | 2 + 4 files changed, 92 insertions(+), 18 deletions(-) create mode 100644 test/fixtures/ostatus_incoming_post.xml diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 59c5d8e9e..9f85d971a 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -21,26 +21,32 @@ def salmon_path(user) do def handle_incoming(xml_string) do doc = parse_document(xml_string) + entries = :xmerl_xpath.string('//entry', doc) - {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', doc) + activities = Enum.map(entries, fn (entry) -> + {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry) - case object_type do - 'http://activitystrea.ms/schema/1.0/note' -> - handle_note(doc) - _ -> - Logger.error("Couldn't parse incoming document") - end + case object_type do + 'http://activitystrea.ms/schema/1.0/note' -> + {:ok, activity} = handle_note(entry, doc) + activity + _ -> + Logger.error("Couldn't parse incoming document") + nil + end + end) + {:ok, activities} end # TODO # wire up replies - def handle_note(doc) do - content_html = string_from_xpath("/entry/content[1]", doc) + def handle_note(entry, doc \\ nil) do + content_html = string_from_xpath("/entry/content[1]", entry) - uri = string_from_xpath("/entry/author/uri[1]", doc) + uri = string_from_xpath("/entry/author/uri[1]", entry) || string_from_xpath("/feed/author/uri[1]", doc) {:ok, actor} = find_or_make_user(uri) - context = string_from_xpath("/entry/ostatus:conversation[1]", doc) |> String.trim + context = string_from_xpath("/entry/ostatus:conversation[1]", entry) |> String.trim context = if String.length(context) > 0 do context else @@ -51,12 +57,12 @@ def handle_note(doc) do "https://www.w3.org/ns/activitystreams#Public" ] - mentions = :xmerl_xpath.string('/entry/link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', doc) + mentions = :xmerl_xpath.string('/entry/link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', entry) |> Enum.map(fn(person) -> string_from_xpath("@href", person) end) to = to ++ mentions - date = string_from_xpath("/entry/published", doc) + date = string_from_xpath("/entry/published", entry) object = %{ "type" => "Note", @@ -67,7 +73,7 @@ def handle_note(doc) do "actor" => actor.ap_id } - inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@href", doc) + inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@href", entry) object = if inReplyTo do Map.put(object, "inReplyTo", inReplyTo) diff --git a/test/fixtures/ostatus_incoming_post.xml b/test/fixtures/ostatus_incoming_post.xml new file mode 100644 index 000000000..7967e1b32 --- /dev/null +++ b/test/fixtures/ostatus_incoming_post.xml @@ -0,0 +1,57 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-04-29T18:25:38+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-04-29:noticeId=1967725:objectType=note + New note by lambadalambda + Will it blend? + + + http://activitystrea.ms/schema/1.0/post + 2017-04-29T18:25:38+00:00 + 2017-04-29T18:25:38+00:00 + + tag:social.heldscal.la,2017-04-29:objectType=thread:nonce=3f3a9dd83acc4e35 + + + + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index cc0975bb5..1e747c728 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -1,11 +1,10 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus - alias Pleroma.Web.XML - test "handle incoming notes" do + test "handle incoming note - GS, Salmon" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") - {:ok, activity} = OStatus.handle_incoming(incoming) + {:ok, [activity]} = OStatus.handle_incoming(incoming) assert activity.data["type"] == "Create" assert activity.data["object"]["type"] == "Note" @@ -14,9 +13,19 @@ test "handle incoming notes" do assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] end + test "handle incoming notes - GS, subscription" do + incoming = File.read!("test/fixtures/ostatus_incoming_post.xml") + {:ok, [activity]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" + assert activity.data["object"]["content"] == "Will it blend?" + end + test "handle incoming replies" do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") - {:ok, activity} = OStatus.handle_incoming(incoming) + {:ok, [activity]} = OStatus.handle_incoming(incoming) assert activity.data["type"] == "Create" assert activity.data["object"]["type"] == "Note" diff --git a/test/web/websub/websub_controller_test.exs b/test/web/websub/websub_controller_test.exs index 521bbb9aa..8f68248a4 100644 --- a/test/web/websub/websub_controller_test.exs +++ b/test/web/websub/websub_controller_test.exs @@ -41,6 +41,8 @@ test "websub subscription confirmation", %{conn: conn} do assert response(conn, 200) == "some challenge" assert websub.state == "accepted" + + # TODO valid_until end test "handles incoming feed updates", %{conn: conn} do From 8a0d2b33d8c9a1cef347c5daf5589a2245eb01b0 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 09:25:46 +0200 Subject: [PATCH 028/107] Keep ostatus id as activity id. --- lib/pleroma/web/ostatus/ostatus.ex | 3 ++- test/web/ostatus/ostatus_test.exs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 9f85d971a..f8e33bc7e 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -63,6 +63,7 @@ def handle_note(entry, doc \\ nil) do to = to ++ mentions date = string_from_xpath("/entry/published", entry) + id = string_from_xpath("/entry/id", entry) object = %{ "type" => "Note", @@ -81,7 +82,7 @@ def handle_note(entry, doc \\ nil) do object end - ActivityPub.create(to, actor, context, object, %{}, date) + ActivityPub.create(to, actor, context, object, %{"id" => id}, date) end def find_or_make_user(uri) do diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 1e747c728..a53e0ebde 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -7,6 +7,7 @@ test "handle incoming note - GS, Salmon" do {:ok, [activity]} = OStatus.handle_incoming(incoming) assert activity.data["type"] == "Create" + assert activity.data["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" assert activity.data["object"]["type"] == "Note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" From ffc604a2c2d963b63e6cd13d0ee7cc9024f632a4 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 10:04:54 +0200 Subject: [PATCH 029/107] Use cache for user info data. Later these should be persisted in the user. --- lib/pleroma/user.ex | 5 +++++ lib/pleroma/web/twitter_api/representers/user_representer.ex | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9b7912c5b..cd6104680 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -122,4 +122,9 @@ def get_cached_by_nickname(nickname) do key = "nickname:#{nickname}" Cachex.get!(:user_cache, key, fallback: fn(_) -> Repo.get_by(User, nickname: nickname) end) end + + def get_cached_user_info(user) do + key = "user_info:#{user.id}" + Cachex.get!(:user_cache, key, fallback: fn(_) -> user_info(user) end) + end end diff --git a/lib/pleroma/web/twitter_api/representers/user_representer.ex b/lib/pleroma/web/twitter_api/representers/user_representer.ex index ab7d6d353..29c7451f4 100644 --- a/lib/pleroma/web/twitter_api/representers/user_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/user_representer.ex @@ -11,7 +11,7 @@ def to_map(user, opts) do false end - user_info = User.user_info(user) + user_info = User.get_cached_user_info(user) map = %{ "id" => user.id, From 11ea08649d5a5e5d2ac9ee29406f53240be77ec4 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 10:06:57 +0200 Subject: [PATCH 030/107] Make cache bigger and longer lived. --- lib/pleroma/application.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 86b6c0c1e..6267d0695 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -15,9 +15,9 @@ def start(_type, _args) do # Start your own worker by calling: Pleroma.Worker.start_link(arg1, arg2, arg3) # worker(Pleroma.Worker, [arg1, arg2, arg3]), worker(Cachex, [:user_cache, [ - default_ttl: 5000, + default_ttl: 25000, ttl_interval: 1000, - limit: 500 + limit: 2500 ]]) ] From 9d7c3190cc346bf2a5576b6b93c26723059ae9a1 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 11:16:41 +0200 Subject: [PATCH 031/107] Get create activity from created object id. This is useful for Ostatus federation because ostatus doesn't have different ids for objects and activities... --- lib/pleroma/activity.ex | 5 +++++ test/activity_test.exs | 7 +++++++ test/web/ostatus/feed_representer_test.exs | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 46568bb13..80d96d0f2 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -18,4 +18,9 @@ def all_by_object_ap_id(ap_id) do Repo.all(from activity in Activity, where: fragment("? @> ?", activity.data, ^%{object: %{id: ap_id}})) end + + def get_create_activity_by_object_ap_id(ap_id) do + Repo.one(from activity in Activity, + where: fragment("? @> ?", activity.data, ^%{type: "Create", object: %{id: ap_id}})) + end end diff --git a/test/activity_test.exs b/test/activity_test.exs index ce6eb1545..366a2f957 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -15,4 +15,11 @@ test "returns activities by it's objects AP ids" do assert activity == found_activity end + + test "returns the activity that created an object" do + activity = insert(:note_activity) + found_activity = Pleroma.Activity.get_create_activity_by_object_ap_id(activity.data["object"]["id"]) + + assert activity == found_activity + end end diff --git a/test/web/ostatus/feed_representer_test.exs b/test/web/ostatus/feed_representer_test.exs index ef0f4d5ff..7bbfae49a 100644 --- a/test/web/ostatus/feed_representer_test.exs +++ b/test/web/ostatus/feed_representer_test.exs @@ -22,7 +22,7 @@ test "returns a feed of the last 20 items of the user" do |> :xmerl.export_simple_content(:xmerl_xml) expected = """ - + #{OStatus.feed_path(user)} #{user.nickname}'s timeline #{most_recent_update} From d937a8e69567ace33a72d5248c046860305076d7 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 11:17:34 +0200 Subject: [PATCH 032/107] Add thr:in-reply-to to ostatus representer. --- .../web/ostatus/activity_representer.ex | 19 +++++++++- lib/pleroma/web/ostatus/feed_representer.ex | 1 + .../web/ostatus/activity_representer_test.exs | 36 +++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 30e695bcc..07b9033b9 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -1,4 +1,19 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do + alias Pleroma.Activity + require Logger + + defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do + with %Activity{data: %{"id" => id}} <- Activity.get_create_activity_by_object_ap_id(in_reply_to) do + [{:"thr:in-reply-to", [ref: to_charlist(id)], []}] + else _e -> + Logger.debug("Couldn't find replied-to activity:") + Logger.debug(in_reply_to) + [] + end + end + + defp get_in_reply_to(_), do: [] + def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) do h = fn(str) -> [to_charlist(str)] end @@ -12,6 +27,8 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) {:link, [rel: 'enclosure', href: to_charlist(url["href"]), type: to_charlist(url["mediaType"])], []} end) + in_reply_to = get_in_reply_to(activity.data) + [ {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']}, @@ -22,7 +39,7 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) {:updated, h.(updated_at)}, {:"ostatus:conversation", [], h.(activity.data["context"])}, {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []} - ] ++ attachments + ] ++ attachments ++ in_reply_to end def to_simple_form(_,_), do: nil diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex index 10a1ffb25..db7b685f3 100644 --- a/lib/pleroma/web/ostatus/feed_representer.ex +++ b/lib/pleroma/web/ostatus/feed_representer.ex @@ -16,6 +16,7 @@ def to_simple_form(user, activities, users) do [{ :feed, [ xmlns: 'http://www.w3.org/2005/Atom', + "xmlns:thr": 'http://purl.org/syndication/thread/1.0', "xmlns:activity": 'http://activitystrea.ms/spec/1.0/', "xmlns:poco": 'http://portablecontacts.net/spec/1.0', "xmlns:ostatus": 'http://ostatus.org/schema/1.0' diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 6cea9cff0..fd1b1598c 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -34,6 +34,42 @@ test "a note activity" do assert clean(res) == clean(expected) end + test "a reply note" do + note = insert(:note_activity) + answer = insert(:note_activity) + object = answer.data["object"] + object = Map.put(object, "inReplyTo", note.data["object"]["id"]) + + data = %{answer.data | "object" => object} + answer = %{answer | data: data} + + updated_at = answer.updated_at + |> NaiveDateTime.to_iso8601 + inserted_at = answer.inserted_at + |> NaiveDateTime.to_iso8601 + + user = User.get_cached_by_ap_id(answer.data["actor"]) + + expected = """ + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + #{answer.data["id"]} + New note by #{user.nickname} + #{answer.data["object"]["content"]} + #{inserted_at} + #{updated_at} + #{answer.data["context"]} + + + """ + + tuple = ActivityRepresenter.to_simple_form(answer, user) + + res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> IO.iodata_to_binary + + assert clean(res) == clean(expected) + end + test "an unknown activity" do tuple = ActivityRepresenter.to_simple_form(%Activity{}, nil) assert is_nil(tuple) From 84027ff00b7fc63934f12129f84b5c7ee1d39248 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 11:39:27 +0200 Subject: [PATCH 033/107] Handle comments. --- lib/pleroma/web/ostatus/ostatus.ex | 5 +- test/fixtures/ostatus_incoming_reply.xml | 60 ++++++++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 11 +++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/ostatus_incoming_reply.xml diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index f8e33bc7e..cd471f860 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -30,6 +30,9 @@ def handle_incoming(xml_string) do 'http://activitystrea.ms/schema/1.0/note' -> {:ok, activity} = handle_note(entry, doc) activity + 'http://activitystrea.ms/schema/1.0/comment' -> + {:ok, activity} = handle_note(entry, doc) + activity _ -> Logger.error("Couldn't parse incoming document") nil @@ -74,7 +77,7 @@ def handle_note(entry, doc \\ nil) do "actor" => actor.ap_id } - inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@href", entry) + inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@ref", entry) object = if inReplyTo do Map.put(object, "inReplyTo", inReplyTo) diff --git a/test/fixtures/ostatus_incoming_reply.xml b/test/fixtures/ostatus_incoming_reply.xml new file mode 100644 index 000000000..83a427a68 --- /dev/null +++ b/test/fixtures/ostatus_incoming_reply.xml @@ -0,0 +1,60 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-04-30T09:30:32+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-04-30:noticeId=1978790:objectType=comment + New comment by lambadalambda + @<a href="https://gs.archae.me/user/4687" class="h-card u-url p-nickname mention" title="shpbot">shpbot</a> why not indeed. + + + http://activitystrea.ms/schema/1.0/post + 2017-04-30T09:30:32+00:00 + 2017-04-30T09:30:32+00:00 + + + + https://gs.archae.me/conversation/327120 + + + + + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index a53e0ebde..5452e5888 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -24,6 +24,17 @@ test "handle incoming notes - GS, subscription" do assert activity.data["object"]["content"] == "Will it blend?" end + test "handle incoming notes - GS, subscription, reply" do + incoming = File.read!("test/fixtures/ostatus_incoming_reply.xml") + {:ok, [activity]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" + assert activity.data["object"]["content"] == "@shpbot why not indeed." + assert activity.data["object"]["inReplyTo"] == "tag:gs.archae.me,2017-04-30:noticeId=778260:objectType=note" + end + test "handle incoming replies" do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) From 62607f37dcf3ab149baa09fe144959a25322be69 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 11:55:19 +0200 Subject: [PATCH 034/107] Federate object id for posts in ostatus. This is because ostatus doens't have an id for the activities. --- lib/pleroma/web/ostatus/activity_representer.ex | 10 ++-------- lib/pleroma/web/ostatus/ostatus.ex | 3 ++- test/web/ostatus/activity_representer_test.exs | 6 +++--- test/web/ostatus/ostatus_test.exs | 2 +- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 07b9033b9..274111ac9 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -3,13 +3,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do require Logger defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do - with %Activity{data: %{"id" => id}} <- Activity.get_create_activity_by_object_ap_id(in_reply_to) do - [{:"thr:in-reply-to", [ref: to_charlist(id)], []}] - else _e -> - Logger.debug("Couldn't find replied-to activity:") - Logger.debug(in_reply_to) - [] - end + [{:"thr:in-reply-to", [ref: to_charlist(in_reply_to)], []}] end defp get_in_reply_to(_), do: [] @@ -32,7 +26,7 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) [ {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']}, - {:id, h.(activity.data["id"])}, + {:id, h.(activity.data["object"]["id"])}, # For notes, federate the object id. {:title, ['New note by #{user.nickname}']}, {:content, [type: 'html'], h.(activity.data["object"]["content"])}, {:published, h.(inserted_at)}, diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index cd471f860..6f169af73 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -69,6 +69,7 @@ def handle_note(entry, doc \\ nil) do id = string_from_xpath("/entry/id", entry) object = %{ + "id" => id, "type" => "Note", "to" => to, "content" => content_html, @@ -85,7 +86,7 @@ def handle_note(entry, doc \\ nil) do object end - ActivityPub.create(to, actor, context, object, %{"id" => id}, date) + ActivityPub.create(to, actor, context, object, %{}, date) end def find_or_make_user(uri) do diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index fd1b1598c..6344889b1 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -18,7 +18,7 @@ test "a note activity" do expected = """ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - #{note_activity.data["id"]} + #{note_activity.data["object"]["id"]} New note by #{user.nickname} #{note_activity.data["object"]["content"]} #{inserted_at} @@ -53,14 +53,14 @@ test "a reply note" do expected = """ http://activitystrea.ms/schema/1.0/note http://activitystrea.ms/schema/1.0/post - #{answer.data["id"]} + #{answer.data["object"]["id"]} New note by #{user.nickname} #{answer.data["object"]["content"]} #{inserted_at} #{updated_at} #{answer.data["context"]} - + """ tuple = ActivityRepresenter.to_simple_form(answer, user) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 5452e5888..3edd39911 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -7,8 +7,8 @@ test "handle incoming note - GS, Salmon" do {:ok, [activity]} = OStatus.handle_incoming(incoming) assert activity.data["type"] == "Create" - assert activity.data["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] From 18edc299b262974d3acb9d6f9c3758629b2c0968 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 12:36:47 +0200 Subject: [PATCH 035/107] Handle duplicates. --- lib/pleroma/web/ostatus/ostatus.ex | 15 +++++++++------ test/web/ostatus/ostatus_test.exs | 6 ++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 6f169af73..16b6ac421 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -3,7 +3,7 @@ defmodule Pleroma.Web.OStatus do import Pleroma.Web.XML require Logger - alias Pleroma.{Repo, User, Web} + alias Pleroma.{Repo, User, Web, Object} alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.{WebFinger, Websub} @@ -28,11 +28,9 @@ def handle_incoming(xml_string) do case object_type do 'http://activitystrea.ms/schema/1.0/note' -> - {:ok, activity} = handle_note(entry, doc) - activity + with {:ok, activity} <- handle_note(entry, doc), do: activity 'http://activitystrea.ms/schema/1.0/comment' -> - {:ok, activity} = handle_note(entry, doc) - activity + with {:ok, activity} <- handle_note(entry, doc), do: activity _ -> Logger.error("Couldn't parse incoming document") nil @@ -86,7 +84,12 @@ def handle_note(entry, doc \\ nil) do object end - ActivityPub.create(to, actor, context, object, %{}, date) + # TODO: Bail out sooner and use transaction. + if Object.get_by_ap_id(id) do + {:error, "duplicate activity"} + else + ActivityPub.create(to, actor, context, object, %{}, date) + end end def find_or_make_user(uri) do diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 3edd39911..07073a40d 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -2,6 +2,12 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus + test "don't insert create notes twice" do + incoming = File.read!("test/fixtures/incoming_note_activity.xml") + {:ok, [_activity]} = OStatus.handle_incoming(incoming) + assert {:ok, [{:error, "duplicate activity"}]} == OStatus.handle_incoming(incoming) + end + test "handle incoming note - GS, Salmon" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) From f9912599c4688a8609bd3500e0548eb2bf06c4a9 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 12:53:49 +0200 Subject: [PATCH 036/107] Pull in remote avatar on federation. --- lib/pleroma/web/ostatus/ostatus.ex | 11 +++++------ lib/pleroma/web/websub/websub.ex | 4 +++- test/web/ostatus/ostatus_test.exs | 7 +++++-- test/web/websub/websub_test.exs | 3 ++- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 16b6ac421..01d6745ef 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -39,8 +39,6 @@ def handle_incoming(xml_string) do {:ok, activities} end - # TODO - # wire up replies def handle_note(entry, doc \\ nil) do content_html = string_from_xpath("/entry/content[1]", entry) @@ -112,7 +110,8 @@ def make_user(uri) do name: info.name, nickname: info.nickname <> "@" <> info.host, ap_id: info.uri, - info: info + info: info, + avatar: info.avatar } # TODO: Make remote user changeset # SHould enforce fqn nickname @@ -121,9 +120,9 @@ def make_user(uri) do end # TODO: Just takes the first one for now. - defp make_avatar_object(author_doc) do - href = string_from_xpath("/author[1]/link[@rel=\"avatar\"]/@href", author_doc) - type = string_from_xpath("/author[1]/link[@rel=\"avatar\"]/@type", author_doc) + def make_avatar_object(author_doc) do + href = string_from_xpath("/feed/author[1]/link[@rel=\"avatar\"]/@href", author_doc) + type = string_from_xpath("/feed/author[1]/link[@rel=\"avatar\"]/@type", author_doc) if href do %{ diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 3fd779fba..63a91055a 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -134,13 +134,15 @@ def gather_feed_data(topic, getter \\ &HTTPoison.get/1) do name = XML.string_from_xpath("/feed/author[1]/name", doc) preferredUsername = XML.string_from_xpath("/feed/author[1]/poco:preferredUsername", doc) displayName = XML.string_from_xpath("/feed/author[1]/poco:displayName", doc) + avatar = OStatus.make_avatar_object(doc) {:ok, %{ uri: uri, hub: hub, nickname: preferredUsername || name, name: displayName || name, - host: URI.parse(uri).host + host: URI.parse(uri).host, + avatar: avatar }} else e -> {:error, e} diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 07073a40d..4e7e401cd 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -64,6 +64,7 @@ test "tries to use the information in poco fields" do assert user.local == false assert user.info["uri"] == uri assert user.ap_id == uri + assert user.avatar["type"] == "Image" {:ok, user_again} = OStatus.find_or_make_user(uri) @@ -88,7 +89,8 @@ test "it returns user info in a hash" do topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", uri: "https://social.heldscal.la/user/29191", host: "social.heldscal.la", - fqn: user + fqn: user, + avatar: %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} } assert data == expected end @@ -109,7 +111,8 @@ test "it works with the uri" do topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", uri: "https://social.heldscal.la/user/29191", host: "social.heldscal.la", - fqn: user + fqn: user, + avatar: %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} } assert data == expected end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index e0d71e16d..ad312cd25 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -119,7 +119,8 @@ test "discovers the hub and canonical url" do uri: "https://mastodon.social/users/lambadalambda", nickname: "lambadalambda", name: "Critical Value", - host: "mastodon.social" + host: "mastodon.social", + avatar: %{"type" => "Image", "url" => [%{"href" => "https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244", "mediaType" => "image/gif", "type" => "Link"}]} } assert expected == discovered From 4c8111c3342aa57cf38accf64f0aa06be6958704 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 13:53:26 +0200 Subject: [PATCH 037/107] Use conversation mapping objects to get / retrieve context from TwAPI. --- lib/pleroma/object.ex | 4 +++ .../representers/activity_representer.ex | 10 ++++-- lib/pleroma/web/twitter_api/twitter_api.ex | 25 +++++++++++---- .../activity_representer_test.exs | 11 ++++--- .../twitter_api_controller_test.exs | 7 ++-- test/web/twitter_api/twitter_api_test.exs | 32 +++++++++++++++++-- 6 files changed, 71 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index f932034d7..a924c3199 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -13,4 +13,8 @@ def get_by_ap_id(ap_id) do Repo.one(from object in Object, where: fragment("? @> ?", object.data, ^%{id: ap_id})) end + + def context_mapping(context) do + %Object{data: %{"id" => context}} + end end diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex index f2bf93abb..bfaabb4e4 100644 --- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex @@ -1,9 +1,9 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do use Pleroma.Web.TwitterAPI.Representers.BaseRepresenter alias Pleroma.Web.TwitterAPI.Representers.{UserRepresenter, ObjectRepresenter} + alias Pleroma.Web.TwitterAPI.TwitterAPI alias Pleroma.Activity - defp user_by_ap_id(user_list, ap_id) do Enum.find(user_list, fn (%{ap_id: user_id}) -> ap_id == user_id end) end @@ -82,6 +82,12 @@ def to_map(%Activity{} = activity, %{user: user} = opts) do |> Enum.filter(&(&1)) |> Enum.map(fn (user) -> UserRepresenter.to_map(user, opts) end) + + conversation_id = with context when not is_nil(context) <- activity.data["context"] do + TwitterAPI.context_to_conversation_id(context) + else _e -> nil + end + %{ "id" => activity.id, "user" => UserRepresenter.to_map(user, opts), @@ -92,7 +98,7 @@ def to_map(%Activity{} = activity, %{user: user} = opts) do "is_post_verb" => true, "created_at" => created_at, "in_reply_to_status_id" => activity.data["object"]["inReplyToStatusId"], - "statusnet_conversation_id" => activity.data["object"]["statusnetConversationId"], + "statusnet_conversation_id" => conversation_id, "attachments" => (activity.data["object"]["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts), "attentions" => attentions, "fave_num" => like_count, diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 1c3396d27..b2fb72a81 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -102,12 +102,7 @@ def fetch_mentions(user, opts \\ %{}) do end def fetch_conversation(user, id) do - query = from activity in Activity, - where: fragment("? @> ?", activity.data, ^%{ statusnetConversationId: id}), - limit: 1 - - with %Activity{} = activity <- Repo.one(query), - context <- activity.data["context"], + with context when is_binary(context) <- conversation_id_to_context(id), activities <- ActivityPub.fetch_activities_for_context(context), statuses <- activities |> activities_to_statuses(%{for: user}) do @@ -322,4 +317,22 @@ defp activity_to_status(activity, opts) do defp make_date do DateTime.utc_now() |> DateTime.to_iso8601 end + + def context_to_conversation_id(context) do + with %Object{id: id} <- Object.get_by_ap_id(context) do + id + else _e -> + changeset = Object.context_mapping(context) + {:ok, %{id: id}} = Repo.insert(changeset) + id + end + end + + def conversation_id_to_context(id) do + with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do + context + else _e -> + {:error, "No such conversation"} + end + end end diff --git a/test/web/twitter_api/representers/activity_representer_test.exs b/test/web/twitter_api/representers/activity_representer_test.exs index d0cccb149..64e7f0641 100644 --- a/test/web/twitter_api/representers/activity_representer_test.exs +++ b/test/web/twitter_api/representers/activity_representer_test.exs @@ -69,6 +69,8 @@ test "an activity" do content = HtmlSanitizeEx.strip_tags(content_html) date = DateTime.from_naive!(~N[2016-05-24 13:26:08.003], "Etc/UTC") |> DateTime.to_iso8601 + {:ok, convo_object} = Object.context_mapping("2hu") |> Repo.insert + activity = %Activity{ id: 1, data: %{ @@ -84,14 +86,15 @@ test "an activity" do "type" => "Note", "content" => content_html, "inReplyToStatusId" => 213123, - "statusnetConversationId" => 4711, "attachment" => [ object ], "like_count" => 5, - "announcement_count" => 3 + "announcement_count" => 3, + "context" => "2hu" }, - "published" => date + "published" => date, + "context" => "2hu" } } @@ -106,7 +109,7 @@ test "an activity" do "is_post_verb" => true, "created_at" => "Tue May 24 13:26:08 +0000 2016", "in_reply_to_status_id" => 213123, - "statusnet_conversation_id" => 4711, + "statusnet_conversation_id" => convo_object.id, "attachments" => [ ObjectRepresenter.to_map(object) ], diff --git a/test/web/twitter_api/twitter_api_controller_test.exs b/test/web/twitter_api/twitter_api_controller_test.exs index 6c249be7d..05cd084b4 100644 --- a/test/web/twitter_api/twitter_api_controller_test.exs +++ b/test/web/twitter_api/twitter_api_controller_test.exs @@ -84,12 +84,13 @@ test "returns one status", %{conn: conn} do describe "GET /statusnet/conversation/:id.json" do test "returns the statuses in the conversation", %{conn: conn} do {:ok, _user} = UserBuilder.insert - {:ok, _activity} = ActivityBuilder.insert(%{"statusnetConversationId" => 1, "context" => "2hu"}) - {:ok, _activity_two} = ActivityBuilder.insert(%{"statusnetConversationId" => 1,"context" => "2hu"}) + {:ok, _activity} = ActivityBuilder.insert(%{"context" => "2hu"}) + {:ok, _activity_two} = ActivityBuilder.insert(%{"context" => "2hu"}) {:ok, _activity_three} = ActivityBuilder.insert(%{"context" => "3hu"}) + {:ok, object} = Object.context_mapping("2hu") |> Repo.insert conn = conn - |> get("/api/statusnet/conversation/1.json") + |> get("/api/statusnet/conversation/#{object.id}.json") response = json_response(conn, 200) diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 590428423..720011257 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -201,11 +201,13 @@ test "Unfollow another user using screen_name" do test "fetch statuses in a context using the conversation id" do {:ok, user} = UserBuilder.insert() - {:ok, activity} = ActivityBuilder.insert(%{"statusnetConversationId" => 1, "context" => "2hu"}) - {:ok, activity_two} = ActivityBuilder.insert(%{"statusnetConversationId" => 1,"context" => "2hu"}) + {:ok, activity} = ActivityBuilder.insert(%{"context" => "2hu"}) + {:ok, activity_two} = ActivityBuilder.insert(%{"context" => "2hu"}) {:ok, _activity_three} = ActivityBuilder.insert(%{"context" => "3hu"}) - statuses = TwitterAPI.fetch_conversation(user, 1) + {:ok, object} = Object.context_mapping("2hu") |> Repo.insert + + statuses = TwitterAPI.fetch_conversation(user, object.id) assert length(statuses) == 2 assert Enum.at(statuses, 0)["id"] == activity.id @@ -314,9 +316,33 @@ test "it returns the error on registration problems" do refute Repo.get_by(User, nickname: "lain") end + test "it assigns an integer conversation_id" do + note_activity = insert(:note_activity) + user = User.get_cached_by_ap_id(note_activity.data["actor"]) + status = ActivityRepresenter.to_map(note_activity, %{user: user}) + + assert is_number(status["statusnet_conversation_id"]) + end + setup do Supervisor.terminate_child(Pleroma.Supervisor, Cachex) Supervisor.restart_child(Pleroma.Supervisor, Cachex) :ok end + + describe "context_to_conversation_id" do + test "creates a mapping object" do + conversation_id = TwitterAPI.context_to_conversation_id("random context") + object = Object.get_by_ap_id("random context") + + assert conversation_id == object.id + end + + test "returns an existing mapping for an existing object" do + {:ok, object} = Object.context_mapping("random context") |> Repo.insert + conversation_id = TwitterAPI.context_to_conversation_id("random context") + + assert conversation_id == object.id + end + end end From 379caca01d818613ba7e013e8f0bebba160c6871 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 13:58:40 +0200 Subject: [PATCH 038/107] Wrap context creation in transaction. --- lib/pleroma/web/twitter_api/twitter_api.ex | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index b2fb72a81..13dc3bd49 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -319,13 +319,16 @@ defp make_date do end def context_to_conversation_id(context) do - with %Object{id: id} <- Object.get_by_ap_id(context) do - id - else _e -> - changeset = Object.context_mapping(context) - {:ok, %{id: id}} = Repo.insert(changeset) - id - end + {:ok, id} = Repo.transaction(fn -> + with %Object{id: id} <- Object.get_by_ap_id(context) do + id + else _e -> + changeset = Object.context_mapping(context) + {:ok, %{id: id}} = Repo.insert(changeset) + id + end + end) + id end def conversation_id_to_context(id) do From 009fcd2acfdc3ae3ba4b706eb71c50015227de50 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 14:02:04 +0200 Subject: [PATCH 039/107] Stop adding statusnetConversationIds. --- lib/pleroma/web/activity_pub/activity_pub.ex | 20 -------------------- test/web/twitter_api/twitter_api_test.exs | 4 +--- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 82f9fcc1c..9441a37ab 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -33,8 +33,6 @@ def create(to, actor, context, object, additional \\ %{}, published \\ nil) do |> Map.merge(additional) with {:ok, activity} <- insert(activity) do - {:ok, activity} = add_conversation_id(activity) - if actor.local do Pleroma.Web.Federator.enqueue(:publish, activity) end @@ -43,24 +41,6 @@ def create(to, actor, context, object, additional \\ %{}, published \\ nil) do end end - defp add_conversation_id(activity) do - if is_integer(activity.data["statusnetConversationId"]) do - {:ok, activity} - else - data = activity.data - |> put_in(["object", "statusnetConversationId"], activity.id) - |> put_in(["statusnetConversationId"], activity.id) - - object = Object.get_by_ap_id(activity.data["object"]["id"]) - - changeset = Ecto.Changeset.change(object, data: data["object"]) - Repo.update(changeset) - - changeset = Ecto.Changeset.change(activity, data: data) - Repo.update(changeset) - end - end - def like(%User{ap_id: ap_id} = user, %Object{data: %{ "id" => id}} = object) do cond do # There's already a like here, so return the original activity. diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 720011257..eb061d334 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -41,11 +41,9 @@ test "create a status" do assert Enum.member?(get_in(activity.data, ["to"]), "https://www.w3.org/ns/activitystreams#Public") assert Enum.member?(get_in(activity.data, ["to"]), "shp") - # Add a context + 'statusnet_conversation_id' + # Add a context assert is_binary(get_in(activity.data, ["context"])) assert is_binary(get_in(activity.data, ["object", "context"])) - assert get_in(activity.data, ["object", "statusnetConversationId"]) == activity.id - assert get_in(activity.data, ["statusnetConversationId"]) == activity.id assert is_list(activity.data["object"]["attachment"]) From 09f7ed421497e12797f30ae34e0f2346f1b6a428 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 14:26:29 +0200 Subject: [PATCH 040/107] Don't set statusnetConversationIds on replies anymore. --- lib/pleroma/web/twitter_api/twitter_api.ex | 5 +---- test/web/twitter_api/twitter_api_test.exs | 2 -- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 13dc3bd49..85fac9146 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -58,11 +58,8 @@ def create_status(user = %User{}, data = %{"status" => status}) do "actor" => user.ap_id, "inReplyTo" => inReplyTo.data["object"]["id"], "inReplyToStatusId" => inReplyTo.id, - "statusnetConversationId" => inReplyTo.data["statusnetConversationId"] - } - additional = %{ - "statusnetConversationId" => inReplyTo.data["statusnetConversationId"] } + additional = %{} [to, context, object, additional] else diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index eb061d334..207d9d12a 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -67,8 +67,6 @@ test "create a status that is a reply" do assert get_in(reply.data, ["context"]) == get_in(activity.data, ["context"]) assert get_in(reply.data, ["object", "context"]) == get_in(activity.data, ["object", "context"]) - assert get_in(reply.data, ["statusnetConversationId"]) == get_in(activity.data, ["statusnetConversationId"]) - assert get_in(reply.data, ["object", "statusnetConversationId"]) == get_in(activity.data, ["object", "statusnetConversationId"]) assert get_in(reply.data, ["object", "inReplyTo"]) == get_in(activity.data, ["object", "id"]) assert get_in(reply.data, ["object", "inReplyToStatusId"]) == activity.id assert Enum.member?(get_in(reply.data, ["to"]), "some_cool_id") From bb1d08a47c34c70d42f6c3afa08232765a24884d Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 15:00:04 +0200 Subject: [PATCH 041/107] Return keys in webfinger. --- lib/pleroma/user.ex | 2 +- lib/pleroma/web/web_finger/web_finger.ex | 22 ++++++++++++++++++---- test/web/web_finger/web_finger_test.exs | 16 ++++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index cd6104680..49ba9b22e 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -16,7 +16,7 @@ defmodule Pleroma.User do field :ap_id, :string field :avatar, :map field :local, :boolean, default: true - field :info, :map + field :info, :map, default: %{} timestamps() end diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 49796dab8..13e3baad6 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -1,8 +1,7 @@ defmodule Pleroma.Web.WebFinger do alias Pleroma.XmlBuilder - alias Pleroma.User - alias Pleroma.Web.OStatus - alias Pleroma.Web.XML + alias Pleroma.{Repo, User} + alias Pleroma.Web.{XML, Salmon, OStatus} require Logger def host_meta() do @@ -28,18 +27,33 @@ def webfinger(resource) do end def represent_user(user) do + {:ok, user} = ensure_keys_present(user) + {:ok, _private, public} = Salmon.keys_from_pem(user.info["keys"]) + magic_key = Salmon.encode_key(public) { :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}, [ {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.host}"}, {:Alias, user.ap_id}, {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}}, - {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}} + {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}}, + {:Link, %{rel: "magic-public-key", href: "data:application/magic-public-key,#{magic_key}"}} ] } |> XmlBuilder.to_doc end + def ensure_keys_present(user) do + info = user.info || %{} + if info["keys"] do + {:ok, user} + else + {:ok, pem} = Salmon.generate_rsa_pem + info = Map.put(info, "keys", pem) + Repo.update(Ecto.Changeset.change(user, info: info)) + end + end + # FIXME: Make this call the host-meta to find the actual address. defp webfinger_address(domain) do "//#{domain}/.well-known/webfinger" diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index e5347a2b0..303abe529 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -1,6 +1,7 @@ defmodule Pleroma.Web.WebFingerTest do use Pleroma.DataCase alias Pleroma.Web.WebFinger + import Pleroma.Factory describe "host meta" do test "returns a link to the xml lrdd" do @@ -26,4 +27,19 @@ test "returns the info for a user" do assert data.salmon == "https://social.heldscal.la/main/salmon/user/29191" end end + + describe "ensure_keys_present" do + test "it creates keys for a user and stores them in info" do + user = insert(:user) + refute is_binary(user.info["keys"]) + {:ok, user} = WebFinger.ensure_keys_present(user) + assert is_binary(user.info["keys"]) + end + + test "it doesn't create keys if there already are some" do + user = insert(:user, %{info: %{"keys" => "xxx"}}) + {:ok, user} = WebFinger.ensure_keys_present(user) + assert user.info["keys"] == "xxx" + end + end end From a173fb9e417cbb4fc7694672dd31bce90a3f9099 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 15:05:16 +0200 Subject: [PATCH 042/107] Get users fresh, might so we don't make new keys all the time. --- lib/pleroma/user.ex | 4 ++++ lib/pleroma/web/web_finger/web_finger.ex | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 49ba9b22e..2c297433a 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -123,6 +123,10 @@ def get_cached_by_nickname(nickname) do Cachex.get!(:user_cache, key, fallback: fn(_) -> Repo.get_by(User, nickname: nickname) end) end + def get_cached_by_nickname(nickname) do + Repo.get_by(User, nickname: nickname) + end + def get_cached_user_info(user) do key = "user_info:#{user.id}" Cachex.get!(:user_cache, key, fallback: fn(_) -> user_info(user) end) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 13e3baad6..7ceca042b 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -20,7 +20,7 @@ def webfinger(resource) do regex = ~r/(acct:)?(?\w+)@#{host}/ case Regex.named_captures(regex, resource) do %{"username" => username} -> - user = User.get_cached_by_nickname(username) + user = User.get_by_nickname(username) {:ok, represent_user(user)} _ -> nil end From eb12a89d22c09bccad7cb13780e0313de8be8e93 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 15:06:22 +0200 Subject: [PATCH 043/107] Rename wrongly-named function. --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 2c297433a..58f89a915 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -123,7 +123,7 @@ def get_cached_by_nickname(nickname) do Cachex.get!(:user_cache, key, fallback: fn(_) -> Repo.get_by(User, nickname: nickname) end) end - def get_cached_by_nickname(nickname) do + def get_by_nickname(nickname) do Repo.get_by(User, nickname: nickname) end From bed0b398139897ebe9f839d1263acf6934c4a42f Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 30 Apr 2017 18:48:48 +0200 Subject: [PATCH 044/107] Add function to fetch users from fqn. --- lib/pleroma/user.ex | 12 ++++++++++++ test/user_test.exs | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 58f89a915..c264d7e90 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -3,6 +3,7 @@ defmodule Pleroma.User do import Ecto.Changeset import Ecto.Query alias Pleroma.{Repo, User, Activity, Object} + alias Pleroma.Web.OStatus schema "users" do field :bio, :string @@ -131,4 +132,15 @@ def get_cached_user_info(user) do key = "user_info:#{user.id}" Cachex.get!(:user_cache, key, fallback: fn(_) -> user_info(user) end) end + + def get_or_fetch_by_nickname(nickname) do + with %User{} = user <- get_by_nickname(nickname) do + user + else _e -> + with {:ok, user} <- OStatus.make_user(nickname) do + user + else _e -> nil + end + end + end end diff --git a/test/user_test.exs b/test/user_test.exs index d711adb9d..6684aa434 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -86,4 +86,24 @@ test "it sets the password_hash, ap_id and following fields" do assert changeset.changes[:following] == [User.ap_followers(%User{nickname: @full_user_data.nickname})] end end + + describe "fetching a user from nickname or trying to build one" do + test "gets an existing user" do + user = insert(:user) + fetched_user = User.get_or_fetch_by_nickname(user.nickname) + + assert user == fetched_user + end + + test "fetches an external user via ostatus if no user exists" do + fetched_user = User.get_or_fetch_by_nickname("shp@social.heldscal.la") + assert fetched_user.nickname == "shp@social.heldscal.la" + end + + test "returns nil if no user could be fetched" do + fetched_user = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la") + assert fetched_user == nil + end + end end + From 6843755834192c671aebece505a1ab9322e57eee Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 13:14:58 +0200 Subject: [PATCH 045/107] Make outgoing salmons work. --- TODO.txt | 8 +++-- lib/pleroma/user.ex | 5 +-- lib/pleroma/web/federator/federator.ex | 4 +++ .../web/ostatus/activity_representer.ex | 31 ++++++++++++++-- lib/pleroma/web/salmon/salmon.ex | 36 +++++++++++++++++++ test/user_test.exs | 6 ++++ .../web/ostatus/activity_representer_test.exs | 2 ++ test/web/salmon/salmon_test.exs | 32 +++++++++++++++++ 8 files changed, 117 insertions(+), 7 deletions(-) diff --git a/TODO.txt b/TODO.txt index dd85c5239..304e95e77 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,5 +1,9 @@ -- Add cache for user fetching / representing. (mostly in TwitterAPI.activity_to_status) - Unliking: - Add a proper undo activity, find out how to ignore those in twitter api. + +WEBSUB: + +- Add unsubscription +- Add periodical renewal + diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c264d7e90..01cbfe796 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -121,7 +121,7 @@ def get_cached_by_ap_id(ap_id) do def get_cached_by_nickname(nickname) do key = "nickname:#{nickname}" - Cachex.get!(:user_cache, key, fallback: fn(_) -> Repo.get_by(User, nickname: nickname) end) + Cachex.get!(:user_cache, key, fallback: fn(_) -> get_or_fetch_by_nickname(nickname) end) end def get_by_nickname(nickname) do @@ -137,7 +137,8 @@ def get_or_fetch_by_nickname(nickname) do with %User{} = user <- get_by_nickname(nickname) do user else _e -> - with {:ok, user} <- OStatus.make_user(nickname) do + with [nick, domain] <- String.split(nickname, "@"), + {:ok, user} <- OStatus.make_user(nickname) do user else _e -> nil end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 38df13540..5293507b5 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -7,7 +7,11 @@ defmodule Pleroma.Web.Federator do def handle(:publish, activity) do Logger.debug("Running publish for #{activity.data["id"]}") with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do + Logger.debug("Sending #{activity.data["id"]} out via websub") Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity) + + Logger.debug("Sending #{activity.data["id"]} out via salmon") + Pleroma.Web.Salmon.publish(actor, activity) end end diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 274111ac9..c64bb3a3b 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do alias Pleroma.Activity + alias Pleroma.Web.OStatus.UserRepresenter require Logger defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do @@ -8,7 +9,17 @@ defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do defp get_in_reply_to(_), do: [] - def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) do + defp get_mentions(to) do + Enum.map(to, fn + ("https://www.w3.org/ns/activitystreams#Public") -> + {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/collection", href: "http://activityschema.org/collection/public"], []} + (id) -> + {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/person", href: id], []} + end) + end + + def to_simple_form(activity, user, with_author \\ false) + def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do h = fn(str) -> [to_charlist(str)] end updated_at = activity.updated_at @@ -22,6 +33,8 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) end) in_reply_to = get_in_reply_to(activity.data) + author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] + mentions = activity.data["to"] |> get_mentions [ {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, @@ -33,8 +46,20 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user) {:updated, h.(updated_at)}, {:"ostatus:conversation", [], h.(activity.data["context"])}, {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []} - ] ++ attachments ++ in_reply_to + ] ++ attachments ++ in_reply_to ++ author ++ mentions end - def to_simple_form(_,_), do: nil + def wrap_with_entry(simple_form) do + [{ + :entry, [ + xmlns: 'http://www.w3.org/2005/Atom', + "xmlns:thr": 'http://purl.org/syndication/thread/1.0', + "xmlns:activity": 'http://activitystrea.ms/spec/1.0/', + "xmlns:poco": 'http://portablecontacts.net/spec/1.0', + "xmlns:ostatus": 'http://ostatus.org/schema/1.0' + ], simple_form + }] + end + + def to_simple_form(_,_,_), do: nil end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 777898cfa..b4f214d46 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -1,6 +1,9 @@ defmodule Pleroma.Web.Salmon do use Bitwise alias Pleroma.Web.XML + alias Pleroma.Web.OStatus.ActivityRepresenter + alias Pleroma.User + require Logger def decode(salmon) do doc = XML.parse_document(salmon) @@ -118,4 +121,37 @@ def encode(private_key, doc) do {:ok, salmon} end + + def remote_users(%{data: %{"to" => to}}) do + to + |> Enum.map(fn(id) -> User.get_cached_by_ap_id(id) end) + |> Enum.filter(fn(user) -> user && !user.local end) + end + + defp send_to_user(%{info: %{"salmon" => salmon}}, feed, poster) do + poster.(salmon, feed, [{"Content-Type", "application/magic-envelope+xml"}]) + end + + defp send_to_user(_,_,_), do: nil + + def publish(user, activity, poster \\ &HTTPoison.post/3) + def publish(%{info: %{"keys" => keys}} = user, activity, poster) do + feed = ActivityRepresenter.to_simple_form(activity, user, true) + |> ActivityRepresenter.wrap_with_entry + |> :xmerl.export_simple(:xmerl_xml) + |> to_string + + if feed do + {:ok, private, _} = keys_from_pem(keys) + {:ok, feed} = encode(private, feed) + + remote_users(activity) + |> Enum.each(fn(remote_user) -> + Logger.debug("sending salmon to #{remote_user.ap_id}") + send_to_user(remote_user, feed, poster) + end) + end + end + + def publish(%{id: id}, _, _), do: Logger.debug("Keys missing for user #{id}") end diff --git a/test/user_test.exs b/test/user_test.exs index 6684aa434..1331ac971 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -95,6 +95,7 @@ test "gets an existing user" do assert user == fetched_user end + # TODO: Make the test local. test "fetches an external user via ostatus if no user exists" do fetched_user = User.get_or_fetch_by_nickname("shp@social.heldscal.la") assert fetched_user.nickname == "shp@social.heldscal.la" @@ -104,6 +105,11 @@ test "returns nil if no user could be fetched" do fetched_user = User.get_or_fetch_by_nickname("nonexistant@social.heldscal.la") assert fetched_user == nil end + + test "returns nil for nonexistant local user" do + fetched_user = User.get_or_fetch_by_nickname("nonexistant") + assert fetched_user == nil + end end end diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 6344889b1..439c733d7 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -25,6 +25,7 @@ test "a note activity" do #{updated_at} #{note_activity.data["context"]} + """ tuple = ActivityRepresenter.to_simple_form(note_activity, user) @@ -61,6 +62,7 @@ test "a reply note" do #{answer.data["context"]} + """ tuple = ActivityRepresenter.to_simple_form(answer, user) diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index aa77659d0..77dacc1c0 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -1,6 +1,8 @@ defmodule Pleroma.Web.Salmon.SalmonTest do use Pleroma.DataCase alias Pleroma.Web.Salmon + alias Pleroma.{Repo, Activity, User} + import Pleroma.Factory @magickey "RSA.pu0s-halox4tu7wmES1FVSx6u-4wc0YrUFXcqWXZG4-27UmbCOpMQftRCldNRfyA-qLbz-eqiwQhh-1EwUvjsD4cYbAHNGHwTvDOyx5AKthQUP44ykPv7kjKGh3DWKySJvcs9tlUG87hlo7AvnMo9pwRS_Zz2CacQ-MKaXyDepk=.AQAB" @@ -57,4 +59,34 @@ test "it gets a magic key" do assert key == "RSA.uzg6r1peZU0vXGADWxGJ0PE34WvmhjUmydbX5YYdOiXfODVLwCMi1umGoqUDm-mRu4vNEdFBVJU1CpFA7dKzWgIsqsa501i2XqElmEveXRLvNRWFB6nG03Q5OUY2as8eE54BJm0p20GkMfIJGwP6TSFb-ICp3QjzbatuSPJ6xCE=.AQAB" end + + test "it pushes an activity to remote accounts it's addressed to" do + user_data = %{ + info: %{ + "salmon" => "http://example.org/salmon" + }, + local: false + } + + mentioned_user = insert(:user, user_data) + note = insert(:note) + activity_data = %{ + "id" => Pleroma.Web.ActivityPub.ActivityPub.generate_activity_id, + "type" => "Create", + "actor" => note.data["actor"], + "to" => note.data["to"] ++ [mentioned_user.ap_id], + "object" => note.data, + "published_at" => DateTime.utc_now() |> DateTime.to_iso8601, + "context" => note.data["context"] + } + + {:ok, activity} = Repo.insert(%Activity{data: activity_data}) + user = Repo.get_by(User, ap_id: activity.data["actor"]) + {:ok, user} = Pleroma.Web.WebFinger.ensure_keys_present(user) + + poster = fn (url, data, headers) -> + assert url == "http://example.org/salmon" + end + Salmon.publish(user, activity, poster) + end end From e54e592d6c1d0ff5de5a029ae1fccee447f97149 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 13:51:17 +0200 Subject: [PATCH 046/107] Return webfinger for ap_ids. --- lib/pleroma/web/web_finger/web_finger.ex | 12 ++++++++---- test/web/web_finger/web_finger_test.exs | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 7ceca042b..f8f4d5e42 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -18,11 +18,15 @@ def host_meta() do def webfinger(resource) do host = Pleroma.Web.host regex = ~r/(acct:)?(?\w+)@#{host}/ - case Regex.named_captures(regex, resource) do - %{"username" => username} -> - user = User.get_by_nickname(username) + with %{"username" => username} <- Regex.named_captures(regex, resource) do + user = User.get_by_nickname(username) + {:ok, represent_user(user)} + else _e -> + with user when not is_nil(user) <- User.get_cached_by_ap_id(resource) do {:ok, represent_user(user)} - _ -> nil + else _e -> + {:error, "Couldn't find user"} + end end end diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index 303abe529..b48fdd0aa 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -11,6 +11,22 @@ test "returns a link to the xml lrdd" do end end + describe "incoming webfinger request" do + test "works for fqns" do + user = insert(:user) + + {:ok, result} = WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.host}") + assert is_binary(result) + end + + test "works for ap_ids" do + user = insert(:user) + + {:ok, result} = WebFinger.webfinger(user.ap_id) + assert is_binary(result) + end + end + describe "fingering" do test "returns the info for a user" do user = "shp@social.heldscal.la" From 35938656ab4186912ee6593cc09754ef945e17fc Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 14:07:29 +0200 Subject: [PATCH 047/107] Make user keys on usage. --- lib/pleroma/web/federator/federator.ex | 2 ++ lib/pleroma/web/web_finger/web_finger.ex | 1 + 2 files changed, 3 insertions(+) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 5293507b5..675e804a2 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Web.Federator do alias Pleroma.User + alias Pleroma.Web.WebFinger require Logger @websub Application.get_env(:pleroma, :websub) @@ -10,6 +11,7 @@ def handle(:publish, activity) do Logger.debug("Sending #{activity.data["id"]} out via websub") Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity) + {:ok, actor} = WebFinger.ensure_keys_present(actor) Logger.debug("Sending #{activity.data["id"]} out via salmon") Pleroma.Web.Salmon.publish(actor, activity) end diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index f8f4d5e42..ff10173ef 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -47,6 +47,7 @@ def represent_user(user) do |> XmlBuilder.to_doc end + # This seems a better fit in Salmon def ensure_keys_present(user) do info = user.info || %{} if info["keys"] do From d187a4965fbba93149621478e5257fcca2cea4f9 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 14:07:41 +0200 Subject: [PATCH 048/107] Return feed for xml requests of the user. --- lib/pleroma/web/router.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 2ff75ec5d..e875839df 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -74,6 +74,7 @@ def user_fetcher(username) do pipe_through :ostatus get "/users/:nickname/feed", OStatus.OStatusController, :feed + get "/users/:nickname", OStatus.OStatusController, :feed post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming post "/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request get "/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation From 2f093db051efb2252342e3490eea3a8ae67e06d3 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 14:54:58 +0200 Subject: [PATCH 049/107] Ensure we have no duplicate ap ids. --- ...124823_add_id_contraints_to_activities_and_objects.exs | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 priv/repo/migrations/20170501124823_add_id_contraints_to_activities_and_objects.exs diff --git a/priv/repo/migrations/20170501124823_add_id_contraints_to_activities_and_objects.exs b/priv/repo/migrations/20170501124823_add_id_contraints_to_activities_and_objects.exs new file mode 100644 index 000000000..21534adc7 --- /dev/null +++ b/priv/repo/migrations/20170501124823_add_id_contraints_to_activities_and_objects.exs @@ -0,0 +1,8 @@ +defmodule Pleroma.Repo.Migrations.AddIdContraintsToActivitiesAndObjects do + use Ecto.Migration + + def change do + create index(:objects, ["(data->>\"id\")"], name: :objects_unique_apid_index) + create index(:activities, ["(data->>\"id\")"], name: :activities_unique_apid_index) + end +end From b9d1fc05b22e29b15208cd6fdcb5d40d34d2a83e Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 15:42:05 +0200 Subject: [PATCH 050/107] Actually make index unique. --- ...d_contraints_to_activities_and_objects_part_two.exs | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs diff --git a/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs b/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs new file mode 100644 index 000000000..12eea1369 --- /dev/null +++ b/priv/repo/migrations/20170501133231_add_id_contraints_to_activities_and_objects_part_two.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddIdContraintsToActivitiesAndObjectsPartTwo do + use Ecto.Migration + + def change do + drop index(:objects, ["(data->>\"id\")"], name: :objects_unique_apid_index) + drop index(:activities, ["(data->>\"id\")"], name: :activities_unique_apid_index) + create unique_index(:objects, ["(data->>'id')"], name: :objects_unique_apid_index) + create unique_index(:activities, ["(data->>'id')"], name: :activities_unique_apid_index) + end +end From f169de34544a12c174c454da59781a694b8c2387 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 16:12:20 +0200 Subject: [PATCH 051/107] Cache objects in dev and prod. --- lib/pleroma/object.ex | 9 +++++++++ lib/pleroma/web/twitter_api/twitter_api.ex | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index a924c3199..168843bd9 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -14,6 +14,15 @@ def get_by_ap_id(ap_id) do where: fragment("? @> ?", object.data, ^%{id: ap_id})) end + def get_cached_by_ap_id(ap_id) do + if Mix.env == :test do + get_by_ap_id(ap_id) + else + key = "object:#{ap_id}" + Cachex.get!(:user_cache, key, fallback: fn(_) -> get_by_ap_id(ap_id) end) + end + end + def context_mapping(context) do %Object{data: %{"id" => context}} end diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 85fac9146..941bacaa9 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -317,7 +317,7 @@ defp make_date do def context_to_conversation_id(context) do {:ok, id} = Repo.transaction(fn -> - with %Object{id: id} <- Object.get_by_ap_id(context) do + with %Object{id: id} <- Object.get_cached_by_ap_id(context) do id else _e -> changeset = Object.context_mapping(context) From 3cb518270ab8c41f73ed449f0c12127c3625c6ca Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 16:15:21 +0200 Subject: [PATCH 052/107] Remove superfluous transaction. --- lib/pleroma/web/twitter_api/twitter_api.ex | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 941bacaa9..e6f5fc906 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -316,16 +316,13 @@ defp make_date do end def context_to_conversation_id(context) do - {:ok, id} = Repo.transaction(fn -> - with %Object{id: id} <- Object.get_cached_by_ap_id(context) do - id - else _e -> - changeset = Object.context_mapping(context) - {:ok, %{id: id}} = Repo.insert(changeset) - id - end - end) - id + with %Object{id: id} <- Object.get_cached_by_ap_id(context) do + id + else _e -> + changeset = Object.context_mapping(context) + {:ok, %{id: id}} = Repo.insert(changeset) + id + end end def conversation_id_to_context(id) do From 108573265aaf237a37937544c3f416b85f57e0fb Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 16:28:40 +0200 Subject: [PATCH 053/107] Don't commit nil values in object cache. --- lib/pleroma/object.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 168843bd9..949ccb0f6 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -19,7 +19,14 @@ def get_cached_by_ap_id(ap_id) do get_by_ap_id(ap_id) else key = "object:#{ap_id}" - Cachex.get!(:user_cache, key, fallback: fn(_) -> get_by_ap_id(ap_id) end) + Cachex.get!(:user_cache, key, fallback: fn(_) -> + object = get_by_ap_id(ap_id) + if object do + {:commit, object} + else + {:ignore, object} + end + end) end end From 1854842b093524c25e08ece0b33150172036f53c Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 17:28:49 +0200 Subject: [PATCH 054/107] Log subscription error. --- lib/pleroma/web/websub/websub.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 63a91055a..67055a116 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -78,6 +78,9 @@ def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) d {:ok, websub} else {:error, reason} -> + Logger.debug("Couldn't create subscription.") + Logger.debug(inspect(reason)) + {:error, reason} end end From 92a8944dfe043444af6b4b422789129c04bd34a0 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 18:05:02 +0200 Subject: [PATCH 055/107] Redirect to user feed instead of directly serving it. --- lib/pleroma/web/ostatus/ostatus_controller.ex | 6 ++++++ lib/pleroma/web/router.ex | 2 +- lib/pleroma/web/websub/websub.ex | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 4174db786..1c609f6f2 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -4,8 +4,14 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.{User, Activity} alias Pleroma.Web.OStatus.FeedRepresenter alias Pleroma.Repo + alias Pleroma.Web.OStatus import Ecto.Query + def feed_redirect(conn, %{"nickname" => nickname}) do + user = User.get_cached_by_nickname(nickname) + redirect conn, external: OStatus.feed_path(user) + end + def feed(conn, %{"nickname" => nickname}) do user = User.get_cached_by_nickname(nickname) query = from activity in Activity, diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e875839df..e1475a03e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -74,7 +74,7 @@ def user_fetcher(username) do pipe_through :ostatus get "/users/:nickname/feed", OStatus.OStatusController, :feed - get "/users/:nickname", OStatus.OStatusController, :feed + get "/users/:nickname", OStatus.OStatusController, :feed_redirect post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming post "/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request get "/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 67055a116..b279a5060 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -99,7 +99,7 @@ defp lease_time(_) do defp valid_topic(%{"hub.topic" => topic}, user) do if topic == OStatus.feed_path(user) do - {:ok, topic} + {:ok, OStatus.feed_path(user)} else {:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"} end From 97d11dec0ee78d24fea398f23d123baf0111362a Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 18:07:50 +0200 Subject: [PATCH 056/107] Also accept user id as feed topic. --- lib/pleroma/web/websub/websub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index b279a5060..fc253b930 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -98,7 +98,7 @@ defp lease_time(_) do end defp valid_topic(%{"hub.topic" => topic}, user) do - if topic == OStatus.feed_path(user) do + if topic == OStatus.feed_path(user) || topic == Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :feed_redirect, user.nickname) do {:ok, OStatus.feed_path(user)} else {:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"} From e88062494e04c257b1dea33965764a51e04cbdf7 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 18:34:15 +0200 Subject: [PATCH 057/107] Revert "Also accept user id as feed topic." This reverts commit 97d11dec0ee78d24fea398f23d123baf0111362a. --- lib/pleroma/web/websub/websub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index fc253b930..b279a5060 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -98,7 +98,7 @@ defp lease_time(_) do end defp valid_topic(%{"hub.topic" => topic}, user) do - if topic == OStatus.feed_path(user) || topic == Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :feed_redirect, user.nickname) do + if topic == OStatus.feed_path(user) do {:ok, OStatus.feed_path(user)} else {:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"} From ceb2f68432e2861f09f7ba34b98bef259be9158a Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 18:40:36 +0200 Subject: [PATCH 058/107] Add type to rel=self link in feed. --- lib/pleroma/web/ostatus/feed_representer.ex | 2 +- test/web/ostatus/feed_representer_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex index db7b685f3..7f9d6a46b 100644 --- a/lib/pleroma/web/ostatus/feed_representer.ex +++ b/lib/pleroma/web/ostatus/feed_representer.ex @@ -26,7 +26,7 @@ def to_simple_form(user, activities, users) do {:updated, h.(most_recent_update)}, {:link, [rel: 'hub', href: h.(OStatus.pubsub_path(user))], []}, {:link, [rel: 'salmon', href: h.(OStatus.salmon_path(user))], []}, - {:link, [rel: 'self', href: h.(OStatus.feed_path(user))], []}, + {:link, [rel: 'self', href: h.(OStatus.feed_path(user)), type: 'application/atom+xml'], []}, {:author, UserRepresenter.to_simple_form(user)}, ] ++ entries }] diff --git a/test/web/ostatus/feed_representer_test.exs b/test/web/ostatus/feed_representer_test.exs index 7bbfae49a..df5a964e2 100644 --- a/test/web/ostatus/feed_representer_test.exs +++ b/test/web/ostatus/feed_representer_test.exs @@ -28,7 +28,7 @@ test "returns a feed of the last 20 items of the user" do #{most_recent_update} - + #{user_xml} From 76e653b0d80279491a4b57278aec7a83efa003d0 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 20:02:32 +0200 Subject: [PATCH 059/107] Add user profile page link. --- lib/pleroma/web/web_finger/web_finger.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index ff10173ef..217b09dc5 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -40,6 +40,7 @@ def represent_user(user) do {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.host}"}, {:Alias, user.ap_id}, {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}}, + {:Link, %{rel: "ttp://webfinger.net/rel/profile-page", type: "text/html", href: user.ap_id}}, {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}}, {:Link, %{rel: "magic-public-key", href: "data:application/magic-public-key,#{magic_key}"}} ] From 703d9f36281e90ef049bfe0a0d579e4e07b38bb6 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 20:04:32 +0200 Subject: [PATCH 060/107] Not enough h. --- lib/pleroma/web/web_finger/web_finger.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 217b09dc5..402184d3f 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -40,7 +40,7 @@ def represent_user(user) do {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.host}"}, {:Alias, user.ap_id}, {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}}, - {:Link, %{rel: "ttp://webfinger.net/rel/profile-page", type: "text/html", href: user.ap_id}}, + {:Link, %{rel: "http://webfinger.net/rel/profile-page", type: "text/html", href: user.ap_id}}, {:Link, %{rel: "salmon", href: OStatus.salmon_path(user)}}, {:Link, %{rel: "magic-public-key", href: "data:application/magic-public-key,#{magic_key}"}} ] From aa209414164cf098376d8aefb3f2af16111bd220 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 20:09:00 +0200 Subject: [PATCH 061/107] Some servers send empty lease_seconds requests... --- lib/pleroma/web/websub/websub.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index b279a5060..905c237a0 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -89,6 +89,11 @@ defp get_subscription(topic, callback) do Repo.get_by(WebsubServerSubscription, topic: topic, callback: callback) || %WebsubServerSubscription{} end + # Temp hack for mastodon. + defp lease_time(%{"hub.lease_seconds" => ""}) do + {:ok, 60 * 60 * 24 * 3} # three days + end + defp lease_time(%{"hub.lease_seconds" => lease_seconds}) do {:ok, String.to_integer(lease_seconds)} end From 8ae13d94dc69e4fcb7f454c2eb7665955c8e37fb Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 20:38:01 +0200 Subject: [PATCH 062/107] Use empty context id if we get none Thanks mastodon. --- lib/pleroma/web/ostatus/ostatus.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 01d6745ef..6a6f43acf 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -45,7 +45,7 @@ def handle_note(entry, doc \\ nil) do uri = string_from_xpath("/entry/author/uri[1]", entry) || string_from_xpath("/feed/author/uri[1]", doc) {:ok, actor} = find_or_make_user(uri) - context = string_from_xpath("/entry/ostatus:conversation[1]", entry) |> String.trim + context = (string_from_xpath("/entry/ostatus:conversation[1]", entry) || "") |> String.trim context = if String.length(context) > 0 do context else From 89c1e90eb2a5da0a6f635a6158fe880076518a38 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Mon, 1 May 2017 22:02:07 +0200 Subject: [PATCH 063/107] Don't crypt raw iolists. --- lib/pleroma/web/websub/websub.ex | 3 ++- test/web/websub/websub_test.exs | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 905c237a0..546bfb5a4 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -41,6 +41,7 @@ def publish(topic, user, activity) do Enum.each(subscriptions, fn(sub) -> response = FeedRepresenter.to_simple_form(user, [activity], [user]) |> :xmerl.export_simple(:xmerl_xml) + |> to_string signature = sign(sub.secret, response) HTTPoison.post(sub.callback, response, [ @@ -51,7 +52,7 @@ def publish(topic, user, activity) do end def sign(secret, doc) do - :crypto.hmac(:sha, secret, doc) |> Base.encode16 + :crypto.hmac(:sha, secret, to_string(doc)) |> Base.encode16 end def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) do diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index ad312cd25..63acb3c43 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -167,4 +167,11 @@ test "rejects the subscription if it can't be accepted" do {:error, websub} = Websub.request_subscription(websub, poster, 1000) assert websub.state == "rejected" end + + test "sign a text" do + signed = Websub.sign("secret", "text") + assert signed == "B8392C23690CCF871F37EC270BE1582DEC57A503" + + signed = Websub.sign("secret", [["て"], ['す']]) + end end From 56bacc90d1f401f8867e4ca7a052f7d15e18a304 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 10:43:35 +0200 Subject: [PATCH 064/107] Fix specs, add local marker to actitivies. --- lib/pleroma/activity.ex | 1 + .../20170502083023_add_local_field_to_activities.exs | 11 +++++++++++ test/support/builders/activity_builder.ex | 4 ++-- test/web/twitter_api/twitter_api_test.exs | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 priv/repo/migrations/20170502083023_add_local_field_to_activities.exs diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 80d96d0f2..d77c88997 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Activity do schema "activities" do field :data, :map + field :local, :boolean, default: true timestamps() end diff --git a/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs b/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs new file mode 100644 index 000000000..088d68f67 --- /dev/null +++ b/priv/repo/migrations/20170502083023_add_local_field_to_activities.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.AddLocalFieldToActivities do + use Ecto.Migration + + def change do + alter table(:activities) do + add :local, :boolean, default: true + end + + create index(:activities, [:local]) + end +end diff --git a/test/support/builders/activity_builder.ex b/test/support/builders/activity_builder.ex index 0f9cd0d15..16011edbf 100644 --- a/test/support/builders/activity_builder.ex +++ b/test/support/builders/activity_builder.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Builders.ActivityBuilder do def build(data \\ %{}, opts \\ %{}) do user = opts[:user] || Pleroma.Factory.insert(:user) activity = %{ - "id" => 1, + "id" => Pleroma.Web.ActivityPub.ActivityPub.generate_object_id, "actor" => user.ap_id, "to" => ["https://www.w3.org/ns/activitystreams#Public"], "object" => %{ @@ -23,7 +23,7 @@ def insert(data \\ %{}, opts \\ %{}) do def insert_list(times, data \\ %{}, opts \\ %{}) do Enum.map(1..times, fn (n) -> - {:ok, activity} = insert(Map.merge(data, %{"id" => n})) + {:ok, activity} = insert(data) activity end) end diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 207d9d12a..57dcddd4c 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -40,6 +40,7 @@ test "create a status" do assert Enum.member?(get_in(activity.data, ["to"]), User.ap_followers(user)) assert Enum.member?(get_in(activity.data, ["to"]), "https://www.w3.org/ns/activitystreams#Public") assert Enum.member?(get_in(activity.data, ["to"]), "shp") + assert activity.local == true # Add a context assert is_binary(get_in(activity.data, ["context"])) From 6dd8335477ff3adc2dda5fe4e45b0e1b38dc5b9b Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 10:47:04 +0200 Subject: [PATCH 065/107] Mark incoming activties as non-local. --- lib/pleroma/web/activity_pub/activity_pub.ex | 8 ++++---- lib/pleroma/web/ostatus/ostatus.ex | 2 +- test/web/ostatus/ostatus_test.exs | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9441a37ab..4eab2e2d0 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -3,7 +3,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.{Activity, Object, Upload, User} import Ecto.Query - def insert(map) when is_map(map) do + def insert(map, local \\ true) when is_map(map) do map = map |> Map.put_new_lazy("id", &generate_activity_id/0) |> Map.put_new_lazy("published", &make_date/0) @@ -16,10 +16,10 @@ def insert(map) when is_map(map) do map end - Repo.insert(%Activity{data: map}) + Repo.insert(%Activity{data: map, local: local}) end - def create(to, actor, context, object, additional \\ %{}, published \\ nil) do + def create(to, actor, context, object, additional \\ %{}, published \\ nil, local \\ true) do published = published || make_date() activity = %{ @@ -32,7 +32,7 @@ def create(to, actor, context, object, additional \\ %{}, published \\ nil) do } |> Map.merge(additional) - with {:ok, activity} <- insert(activity) do + with {:ok, activity} <- insert(activity, local) do if actor.local do Pleroma.Web.Federator.enqueue(:publish, activity) end diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 6a6f43acf..db32d2c35 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -86,7 +86,7 @@ def handle_note(entry, doc \\ nil) do if Object.get_by_ap_id(id) do {:error, "duplicate activity"} else - ActivityPub.create(to, actor, context, object, %{}, date) + ActivityPub.create(to, actor, context, object, %{}, date, false) end end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 4e7e401cd..3951dbc9c 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -18,6 +18,7 @@ test "handle incoming note - GS, Salmon" do assert activity.data["published"] == "2017-04-23T14:51:03+00:00" assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] + assert activity.local == false end test "handle incoming notes - GS, subscription" do From 32a95d73daf94a1186ccdbcdc9ce0f91b559119c Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 14:12:43 +0200 Subject: [PATCH 066/107] Add twkn timeline. --- lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++++++ lib/pleroma/web/router.ex | 2 +- lib/pleroma/web/twitter_api/twitter_api.ex | 6 ++++++ .../web/twitter_api/twitter_api_controller.ex | 8 ++++++++ test/web/twitter_api/twitter_api_test.exs | 15 ++++++++++++++- 5 files changed, 35 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 4eab2e2d0..0fb8db520 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -149,6 +149,12 @@ def fetch_activities(recipients, opts \\ %{}) do query = from activity in query, where: activity.id > ^since_id + query = if opts["local_only"] do + from activity in query, where: activity.local == true + else + query + end + query = if opts["max_id"] do from activity in query, where: activity.id < ^opts["max_id"] else diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e1475a03e..b0c1dcd91 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -30,7 +30,7 @@ def user_fetcher(username) do get "/statusnet/config", TwitterAPI.Controller, :config get "/statuses/public_timeline", TwitterAPI.Controller, :public_timeline - get "/statuses/public_and_external_timeline", TwitterAPI.Controller, :public_timeline + get "/statuses/public_and_external_timeline", TwitterAPI.Controller, :public_and_external_timeline get "/statuses/user_timeline", TwitterAPI.Controller, :user_timeline get "/statuses/show/:id", TwitterAPI.Controller, :fetch_status diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index e6f5fc906..b1759a6f0 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -84,6 +84,12 @@ def fetch_friend_statuses(user, opts \\ %{}) do end def fetch_public_statuses(user, opts \\ %{}) do + opts = Map.put(opts, "local_only", true) + ActivityPub.fetch_public_activities(opts) + |> activities_to_statuses(%{for: user}) + end + + def fetch_public_and_external_statuses(user, opts \\ %{}) do ActivityPub.fetch_public_activities(opts) |> activities_to_statuses(%{for: user}) end diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index b5b829ca0..4b329a21f 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -41,6 +41,14 @@ defp extract_media_ids(status_data) do end end + def public_and_external_timeline(%{assigns: %{user: user}} = conn, params) do + statuses = TwitterAPI.fetch_public_and_external_statuses(user, params) + {:ok, json} = Poison.encode(statuses) + + conn + |> json_reply(200, json) + end + def public_timeline(%{assigns: %{user: user}} = conn, params) do statuses = TwitterAPI.fetch_public_statuses(user, params) {:ok, json} = Poison.encode(statuses) diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 57dcddd4c..4e17f3298 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -73,8 +73,9 @@ test "create a status that is a reply" do assert Enum.member?(get_in(reply.data, ["to"]), "some_cool_id") end - test "fetch public statuses" do + test "fetch public statuses, excluding remote ones." do %{ public: activity, user: user } = ActivityBuilder.public_and_non_public + insert(:note_activity, %{local: false}) follower = insert(:user, following: [User.ap_followers(user)]) @@ -84,6 +85,18 @@ test "fetch public statuses" do assert Enum.at(statuses, 0) == ActivityRepresenter.to_map(activity, %{user: user, for: follower}) end + test "fetch whole known network statuses" do + %{ public: activity, user: user } = ActivityBuilder.public_and_non_public + insert(:note_activity, %{local: false}) + + follower = insert(:user, following: [User.ap_followers(user)]) + + statuses = TwitterAPI.fetch_public_and_external_statuses(follower) + + assert length(statuses) == 2 + assert Enum.at(statuses, 0) == ActivityRepresenter.to_map(activity, %{user: user, for: follower}) + end + test "fetch friends' statuses" do user = insert(:user, %{following: ["someguy/followers"]}) {:ok, activity} = ActivityBuilder.insert(%{"to" => ["someguy/followers"]}) From 16f8406eb60562b961536ecfabecde8e15160aa6 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 14:36:04 +0200 Subject: [PATCH 067/107] Add statusnet_profile_url to the TwAPI. --- .../web/twitter_api/representers/user_representer.ex | 3 ++- test/web/twitter_api/representers/user_representer_test.exs | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/twitter_api/representers/user_representer.ex b/lib/pleroma/web/twitter_api/representers/user_representer.ex index 29c7451f4..493077413 100644 --- a/lib/pleroma/web/twitter_api/representers/user_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/user_representer.ex @@ -28,7 +28,8 @@ def to_map(user, opts) do "profile_image_url_https" => image, "profile_image_url_profile_size" => image, "profile_image_url_original" => image, - "rights" => %{} + "rights" => %{}, + "statusnet_profile_url" => user.ap_id } map diff --git a/test/web/twitter_api/representers/user_representer_test.exs b/test/web/twitter_api/representers/user_representer_test.exs index 1e92c5190..77f065948 100644 --- a/test/web/twitter_api/representers/user_representer_test.exs +++ b/test/web/twitter_api/representers/user_representer_test.exs @@ -48,7 +48,8 @@ test "A user" do "profile_image_url_profile_size" => image, "profile_image_url_original" => image, "following" => false, - "rights" => %{} + "rights" => %{}, + "statusnet_profile_url" => user.ap_id } assert represented == UserRepresenter.to_map(user) @@ -72,7 +73,8 @@ test "A user for a given other follower", %{user: user} do "profile_image_url_profile_size" => image, "profile_image_url_original" => image, "following" => true, - "rights" => %{} + "rights" => %{}, + "statusnet_profile_url" => user.ap_id } assert represented == UserRepresenter.to_map(user, %{for: follower}) From a3e82c5c246a4852d7bfaa5f6e216145b89fe0d8 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 15:54:14 +0200 Subject: [PATCH 068/107] Save context in likes / announces. --- lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++++-- test/web/activity_pub/activity_pub_test.exs | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 0fb8db520..e9de3573e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -55,7 +55,8 @@ def like(%User{ap_id: ap_id} = user, %Object{data: %{ "id" => id}} = object) do "type" => "Like", "actor" => ap_id, "object" => id, - "to" => [User.ap_followers(user), object.data["actor"]] + "to" => [User.ap_followers(user), object.data["actor"]], + "context" => object.data["context"] } {:ok, activity} = insert(data) @@ -177,7 +178,8 @@ def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) "type" => "Announce", "actor" => ap_id, "object" => id, - "to" => [User.ap_followers(user), object.data["actor"]] + "to" => [User.ap_followers(user), object.data["actor"]], + "context" => object.data["context"] } {:ok, activity} = insert(data) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 744021c8c..6e42fbda2 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -125,6 +125,7 @@ test "adds a like activity to the db" do assert like_activity.data["type"] == "Like" assert like_activity.data["object"] == object.data["id"] assert like_activity.data["to"] == [User.ap_followers(user), note_activity.data["actor"]] + assert like_activity.data["context"] == object.data["context"] assert object.data["like_count"] == 1 assert object.data["likes"] == [user.ap_id] @@ -174,6 +175,7 @@ test "adds an announce activity to the db" do assert announce_activity.data["to"] == [User.ap_followers(user), note_activity.data["actor"]] assert announce_activity.data["object"] == object.data["id"] assert announce_activity.data["actor"] == user.ap_id + assert announce_activity.data["context"] == object.data["context"] end end From 93de6039667b9fe6f3b9019c4c2297d4f23b3a1a Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 16:35:53 +0200 Subject: [PATCH 069/107] Add an ostatus representer for like activities. --- .../web/ostatus/activity_representer.ex | 45 ++++++++++++++++--- .../web/ostatus/activity_representer_test.exs | 36 +++++++++++++++ 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index c64bb3a3b..cf6aae727 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -10,11 +10,17 @@ defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do defp get_in_reply_to(_), do: [] defp get_mentions(to) do - Enum.map(to, fn - ("https://www.w3.org/ns/activitystreams#Public") -> - {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/collection", href: "http://activityschema.org/collection/public"], []} - (id) -> - {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/person", href: id], []} + Enum.map(to, fn (id) -> + cond do + # Special handling for the AP/Ostatus public collections + "https://www.w3.org/ns/activitystreams#Public" == id -> + {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/collection", href: "http://activityschema.org/collection/public"], []} + # Ostatus doesn't handle follower collections, ignore these. + Regex.match?(~r/^#{Pleroma.Web.base_url}.+followers$/, id) -> + [] + true -> + {:link, [rel: "mentioned", "ostatus:object-type": "http://activitystrea.ms/schema/1.0/person", href: id], []} + end end) end @@ -49,6 +55,35 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, ] ++ attachments ++ in_reply_to ++ author ++ mentions end + def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) do + h = fn(str) -> [to_charlist(str)] end + + updated_at = activity.updated_at + |> NaiveDateTime.to_iso8601 + inserted_at = activity.inserted_at + |> NaiveDateTime.to_iso8601 + + in_reply_to = get_in_reply_to(activity.data) + author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] + mentions = activity.data["to"] |> get_mentions + + [ + {:"activity:verb", ['http://activitystrea.ms/schema/1.0/favorite']}, + {:id, h.(activity.data["id"])}, + {:title, ['New favorite by #{user.nickname}']}, + {:content, [type: 'html'], ['#{user.nickname} favorited something']}, + {:published, h.(inserted_at)}, + {:updated, h.(updated_at)}, + {:"activity:object", [ + {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, + {:id, h.(activity.data["object"])}, # For notes, federate the object id. + ]}, + {:"ostatus:conversation", [], h.(activity.data["context"])}, + {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, + {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []} + ] ++ author ++ mentions + end + def wrap_with_entry(simple_form) do [{ :entry, [ diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 439c733d7..4cf73427b 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -3,6 +3,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenterTest do alias Pleroma.Web.OStatus.ActivityRepresenter alias Pleroma.{User, Activity} + alias Pleroma.Web.ActivityPub.ActivityPub import Pleroma.Factory @@ -72,6 +73,41 @@ test "a reply note" do assert clean(res) == clean(expected) end + test "a like activity" do + note = insert(:note) + user = insert(:user) + {:ok, like, _note} = ActivityPub.like(user, note) + + updated_at = like.updated_at + |> NaiveDateTime.to_iso8601 + inserted_at = like.inserted_at + |> NaiveDateTime.to_iso8601 + + tuple = ActivityRepresenter.to_simple_form(like, user) + refute is_nil(tuple) + + res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> IO.iodata_to_binary + + expected = """ + http://activitystrea.ms/schema/1.0/favorite + #{like.data["id"]} + New favorite by #{user.nickname} + #{user.nickname} favorited something + #{inserted_at} + #{updated_at} + + http://activitystrea.ms/schema/1.0/note + #{note.data["id"]} + + #{like.data["context"]} + + + + """ + + assert clean(res) == clean(expected) + end + test "an unknown activity" do tuple = ActivityRepresenter.to_simple_form(%Activity{}, nil) assert is_nil(tuple) From 945b4b55e651341ae9452c9799f432ec2de11787 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 16:45:54 +0200 Subject: [PATCH 070/107] Federate likes. --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index e9de3573e..12d6912df 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -72,6 +72,10 @@ def like(%User{ap_id: ap_id} = user, %Object{data: %{ "id" => id}} = object) do update_object_in_activities(object) + if user.local do + Pleroma.Web.Federator.enqueue(:publish, activity) + end + {:ok, activity, object} end end From 102455bf296165a88578a04f0ded259c32349d7f Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 17:13:41 +0200 Subject: [PATCH 071/107] Add avatar updating from incoming messages. --- lib/pleroma/web/ostatus/ostatus.ex | 17 +- test/fixtures/23211.atom | 508 +++++++++++++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 20 ++ 3 files changed, 543 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/23211.atom diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index db32d2c35..4c72e9cd1 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -90,6 +90,19 @@ def handle_note(entry, doc \\ nil) do end end + def find_make_or_update_user(doc) do + uri = string_from_xpath("//author/uri[1]", doc) + with {:ok, user} <- find_or_make_user(uri) do + avatar = make_avatar_object(doc) + if user.avatar != avatar do + change = Ecto.Changeset.change(user, %{avatar: avatar}) + Repo.update(change) + else + {:ok, user} + end + end + end + def find_or_make_user(uri) do query = from user in User, where: user.local == false and fragment("? @> ?", user.info, ^%{uri: uri}) @@ -121,8 +134,8 @@ def make_user(uri) do # TODO: Just takes the first one for now. def make_avatar_object(author_doc) do - href = string_from_xpath("/feed/author[1]/link[@rel=\"avatar\"]/@href", author_doc) - type = string_from_xpath("/feed/author[1]/link[@rel=\"avatar\"]/@type", author_doc) + href = string_from_xpath("//author[1]/link[@rel=\"avatar\"]/@href", author_doc) + type = string_from_xpath("//author[1]/link[@rel=\"avatar\"]/@type", author_doc) if href do %{ diff --git a/test/fixtures/23211.atom b/test/fixtures/23211.atom new file mode 100644 index 000000000..d5d111baa --- /dev/null +++ b/test/fixtures/23211.atom @@ -0,0 +1,508 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-05-02T14:59:30+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2015260:2017-05-02T14:45:47+00:00 + Favorite + lambadalambda favorited something by godemperorofdune: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> It's because your instance decided to be trap! lol.</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T14:45:47+00:00 + 2017-05-02T14:45:47+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:pawoo.net,2017-05-02:objectId=7397439:objectType=Status + New comment by godemperorofdune + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> It's because your instance decided to be trap! lol.</p> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=136e244b26cdf1e9 + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-05-02:noticeId=2015221:objectType=note + New note by lambadalambda + Some script thinks I'm a mastodon server.<br /> <br /> [info] GET /api/v1/timelines/public<br /> [debug] Processing with Fallback.RedirectController.redirector/2<br /> Parameters: %{&quot;limit&quot; =&gt; &quot;40&quot;, &quot;path&quot; =&gt; [&quot;api&quot;, &quot;v1&quot;, &quot;timelines&quot;, &quot;public&quot;]}<br /> Pipelines: [] + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T14:40:50+00:00 + 2017-05-02T14:40:50+00:00 + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=136e244b26cdf1e9 + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-02:noticeId=2014759:objectType=comment + New comment by lambadalambda + @<a href="https://mstdn.io/users/mattskala" class="h-card u-url p-nickname mention" title="Matthew Skala">mattskala</a> You and @<a href="https://mastodon.social/users/kevinmarks" class="h-card u-url p-nickname mention" title="Kevin Marks">kevinmarks</a> are not wrong, but my comment was a suggestion to users and admins: Don't use big instances, don't run big instances. Also, it's a secondary advice to devs: Don't add features that encourage big instances. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T14:11:54+00:00 + 2017-05-02T14:11:54+00:00 + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d + + + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-02:noticeId=2014684:objectType=comment + New comment by lambadalambda + @<a href="https://mastodon.social/users/Ronkjeffries" class="h-card u-url p-nickname mention" title="Ron K Jeffries social">ronkjeffries</a> @<a href="https://xoxo.zone/users/KevinMarks" class="h-card u-url p-nickname mention" title="Kevin Marks ">kevinmarks</a> Usually people who run their own private instance just look at the timelines of other servers, follow a seed population and then go from there. This is of course hard on Mastodon, because it doesn't have a publicly visible timeline. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T14:07:00+00:00 + 2017-05-02T14:07:00+00:00 + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d + + + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2014584:2017-05-02T14:05:32+00:00 + Favorite + lambadalambda favorited something by mattskala: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> It's reasonable to expect that instance sizes will obey a power-law distribution because that's what such things in nature nearly always do. If so, there'll necessarily be a few instances much larger than the others; even if most are small, the network both socially and technically has to be able to deal with the existence of the few large ones.</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T14:05:32+00:00 + 2017-05-02T14:05:32+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:mstdn.io,2017-05-02:objectId=1316931:objectType=Status + New comment by mattskala + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> It's reasonable to expect that instance sizes will obey a power-law distribution because that's what such things in nature nearly always do. If so, there'll necessarily be a few instances much larger than the others; even if most are small, the network both socially and technically has to be able to deal with the existence of the few large ones.</p> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013568:2017-05-02T14:05:29+00:00 + Favorite + lambadalambda favorited something by kevinmarks: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> except instance populations will be power law distributed, and the problems for the tummlers are worse at scale</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T14:05:29+00:00 + 2017-05-02T14:05:29+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:xoxo.zone,2017-05-02:objectId=89478:objectType=Status + New comment by kevinmarks + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> except instance populations will be power law distributed, and the problems for the tummlers are worse at scale</p> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2014060:2017-05-02T13:34:32+00:00 + Favorite + lambadalambda favorited something by gcarregues: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> Oh purée ! Ma vie en images !</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T13:34:32+00:00 + 2017-05-02T13:34:32+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:mastodon.etalab.gouv.fr,2017-05-02:objectId=55287:objectType=Status + New comment by gcarregues + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> Oh purée ! Ma vie en images !</p> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:note:2013573:2017-05-02T13:03:33+00:00 + Favorite + lambadalambda favorited something by phildobangnz: also @<a href="https://sealion.club/user/579" class="h-card mention" title="Sim Bot">sim</a> reminder you are awesome; don't even trip- u kewler than Tutankhamen's cucumber, fam. Okay, good night. + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T13:03:33+00:00 + 2017-05-02T13:03:33+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:sealion.club,2017-05-02:noticeId=3060818:objectType=note + New note by phildobangnz + also @<a href="https://sealion.club/user/579" class="h-card mention" title="Sim Bot">sim</a> reminder you are awesome; don't even trip- u kewler than Tutankhamen's cucumber, fam. Okay, good night. + + + + + + + https://sealion.club/conversation/1633267 + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-02:noticeId=2013586:objectType=comment + New comment by lambadalambda + @<a href="https://xoxo.zone/users/KevinMarks" class="h-card u-url p-nickname mention" title="Kevin Marks ">kevinmarks</a> People can stay in their giant unmoderatable instances with meaningless public and federated timelines and experience constant federation drama if they want. I'll stay here with my 5 friends. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T12:54:59+00:00 + 2017-05-02T12:54:59+00:00 + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=58e32e013ab6487d + + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:note:2013486:2017-05-02T12:46:48+00:00 + Favorite + lambadalambda favorited something by fortune: There once was a dentist named Stone<br /> Who saw all his patients alone.<br /> In a fit of depravity<br /> He filled the wrong cavity,<br /> And my, how his practice has grown! + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:46:48+00:00 + 2017-05-02T12:46:48+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:gs.kawa-kun.com,2017-05-02:noticeId=1655658:objectType=note + New note by fortune + There once was a dentist named Stone<br /> Who saw all his patients alone.<br /> In a fit of depravity<br /> He filled the wrong cavity,<br /> And my, how his practice has grown! + + + + + + + https://gs.kawa-kun.com/conversation/714072 + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:note:2013365:2017-05-02T12:37:55+00:00 + Favorite + lambadalambda favorited something by xj9: <p>&gt; rollerblading to work</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:37:55+00:00 + 2017-05-02T12:37:55+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:sunshinegardens.org,2017-05-02:objectId=61020:objectType=Status + New note by xj9 + <p>&gt; rollerblading to work</p> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=5a0e98612f634218 + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013259:2017-05-02T12:29:03+00:00 + Favorite + lambadalambda favorited something by cereal: @<a href="https://gs.smuglo.li/user/28250" class="h-card mention" title="Bricky">thatbrickster</a> @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> But why? + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:29:03+00:00 + 2017-05-02T12:29:03+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:sealion.club,2017-05-02:noticeId=3059985:objectType=comment + New comment by cereal + @<a href="https://gs.smuglo.li/user/28250" class="h-card mention" title="Bricky">thatbrickster</a> @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> But why? + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013227:2017-05-02T12:24:27+00:00 + Favorite + lambadalambda favorited something by thatbrickster: @<a href="https://social.heldscal.la/user/23211" class="h-card u-url p-nickname mention" title="Constance Variable">lambadalambda</a> install gentoo + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:24:27+00:00 + 2017-05-02T12:24:27+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:gs.smuglo.li,2017-05-02:noticeId=2144296:objectType=comment + New comment by thatbrickster + @<a href="https://social.heldscal.la/user/23211" class="h-card u-url p-nickname mention" title="Constance Variable">lambadalambda</a> install gentoo + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013213:2017-05-02T12:22:53+00:00 + Favorite + lambadalambda favorited something by dwmatiz: @<a href="https://social.heldscal.la/user/23211" class="h-card mention">lambadalambda</a> *unzips dick* + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:22:53+00:00 + 2017-05-02T12:22:53+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:sealion.club,2017-05-02:noticeId=3059800:objectType=comment + New comment by dwmatiz + @<a href="https://social.heldscal.la/user/23211" class="h-card mention">lambadalambda</a> *unzips dick* + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2013199:2017-05-02T12:22:03+00:00 + Favorite + lambadalambda favorited something by shpuld: @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> get #<span class="tag"><a href="https://shitposter.club/tag/cofe" rel="tag">cofe</a></span> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:22:03+00:00 + 2017-05-02T12:22:03+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-02:noticeId=2783524:objectType=comment + New comment by shpuld + @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> get #<span class="tag"><a href="https://shitposter.club/tag/cofe" rel="tag">cofe</a></span> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-05-02:noticeId=2013185:objectType=note + New note by lambadalambda + What now? <a href="https://social.heldscal.la/file/e4822d95de677757ff50d49672a4046c83218b76c04a0ad5e5f1f0a9a9eb1a74.gif" title="https://social.heldscal.la/file/e4822d95de677757ff50d49672a4046c83218b76c04a0ad5e5f1f0a9a9eb1a74.gif" rel="nofollow external noreferrer" class="attachment" id="attachment-422572">https://social.heldscal.la/attachment/422572</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T12:21:04+00:00 + 2017-05-02T12:21:04+00:00 + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2c27c27df8ec4dcc + + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:note:2012929:2017-05-02T12:01:25+00:00 + Favorite + lambadalambda favorited something by drkmttr: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> I checked out No Agenda because I saw you mention it several time. Sadly, I wasn't impressed. I'm all about varying perspectives but Adam and John basically just sound like resentful curmudgeons. It seems like their shtick is basically playing devil's advocate to everything to arouse some discontent. Just my two cents. 😉</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T12:01:25+00:00 + 2017-05-02T12:01:25+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:mstdn.io,2017-05-02:objectId=1310093:objectType=Status + New note by drkmttr + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> I checked out No Agenda because I saw you mention it several time. Sadly, I wasn't impressed. I'm all about varying perspectives but Adam and John basically just sound like resentful curmudgeons. It seems like their shtick is basically playing devil's advocate to everything to arouse some discontent. Just my two cents. 😉</p> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=2f329b4eb20e83e2 + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2012336:2017-05-02T11:06:42+00:00 + Favorite + lambadalambda favorited something by clacke: @<a href="https://mastodon.org.uk/users/dick_turpin" class="h-card u-url p-nickname mention" title="dick_turpin">dickturpin</a> @<a href="http://quitter.se/user/113503" class="h-card u-url p-nickname mention" title="Luke">luke</a> Oh no, I miss being irritated by you, it helps me understand myself and others. Also it builds character. :-)<br /> <br /> So if this is not federation because you can't follow all of online mankind, what should we call it? Proto-federated? Pre-federated?<br /> <br /> The term has been used decades ago for just one Microsoft Active Directory domain cross-certifying the root of another, by mutual agreement. I don't see how it's any less relevant to opportunistic federation between open servers on an open internet.<br /> <br /> I'm not saying we should be satisfied, I'm just saying that "federate" is a useful word and to build a big system we need to start with a small one. And focus on the things we *can* change, like helping the OStatus network grow and making the tools more useful.<br /> <br /> Saying that the network's ideals have failed because other networks aren't joining is doing neither of that. + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T11:06:42+00:00 + 2017-05-02T11:06:42+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-02:noticeId=2012336:objectType=comment + New comment by clacke + @<a href="https://mastodon.org.uk/users/dick_turpin" class="h-card u-url p-nickname mention" title="dick_turpin">dickturpin</a> @<a href="http://quitter.se/user/113503" class="h-card u-url p-nickname mention" title="Luke">luke</a> Oh no, I miss being irritated by you, it helps me understand myself and others. Also it builds character. :-)<br /> <br /> So if this is not federation because you can't follow all of online mankind, what should we call it? Proto-federated? Pre-federated?<br /> <br /> The term has been used decades ago for just one Microsoft Active Directory domain cross-certifying the root of another, by mutual agreement. I don't see how it's any less relevant to opportunistic federation between open servers on an open internet.<br /> <br /> I'm not saying we should be satisfied, I'm just saying that &quot;federate&quot; is a useful word and to build a big system we need to start with a small one. And focus on the things we *can* change, like helping the OStatus network grow and making the tools more useful.<br /> <br /> Saying that the network's ideals have failed because other networks aren't joining is doing neither of that. + + + + + + + https://s.wefamlee.be/conversation/16478 + + + + + + + tag:social.heldscal.la,2017-05-02:fave:23211:comment:2011332:2017-05-02T10:37:40+00:00 + Favorite + lambadalambda favorited something by moonman: @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> <a href="https://www.youtube.com/watch?v=mKLizztikRk" title="https://www.youtube.com/watch?v=mKLizztikRk" class="attachment" rel="nofollow">https://www.youtube.com/watch?v=mKLizztikRk</a> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-02T10:37:40+00:00 + 2017-05-02T10:37:40+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-02:noticeId=2781833:objectType=comment + New comment by moonman + @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> <a href="https://www.youtube.com/watch?v=mKLizztikRk" title="https://www.youtube.com/watch?v=mKLizztikRk" class="attachment" rel="nofollow">https://www.youtube.com/watch?v=mKLizztikRk</a> + + + + + + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=11d8b8c27d9513ec + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-02:noticeId=2012145:objectType=comment + New comment by lambadalambda + @<a href="https://sealion.club/user/186" class="h-card u-url p-nickname mention" title="I'M CEREAL U GUISE">cereal</a> ? No, you don't even need the identity servers for federation. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T10:37:33+00:00 + 2017-05-02T10:37:33+00:00 + + + + https://sealion.club/conversation/1629037 + + + + + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 3951dbc9c..1674edbd5 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -1,6 +1,7 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus + alias Pleroma.Web.XML test "don't insert create notes twice" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") @@ -71,6 +72,25 @@ test "tries to use the information in poco fields" do assert user == user_again end + + test "find_make_or_update_user takes an author element and returns an updated user" do + # TODO make test local + uri = "https://social.heldscal.la/user/23211" + + {:ok, user} = OStatus.find_or_make_user(uri) + change = Ecto.Changeset.change(user, %{avatar: nil}) + + {:ok, user} = Repo.update(change) + refute user.avatar + + doc = XML.parse_document(File.read!("test/fixtures/23211.atom")) + [author] = :xmerl_xpath.string('//author[1]', doc) + {:ok, user} = OStatus.find_make_or_update_user(author) + assert user.avatar["type"] == "Image" + + {:ok, user_again} = OStatus.find_make_or_update_user(author) + assert user_again == user + end end describe "gathering user info from a user id" do From 96014f8e0b7bc6b28170f06914ef646f3f22ecfc Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 17:16:01 +0200 Subject: [PATCH 072/107] Update incoming new avatars. --- lib/pleroma/web/ostatus/ostatus.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 4c72e9cd1..340228dcf 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -42,8 +42,8 @@ def handle_incoming(xml_string) do def handle_note(entry, doc \\ nil) do content_html = string_from_xpath("/entry/content[1]", entry) - uri = string_from_xpath("/entry/author/uri[1]", entry) || string_from_xpath("/feed/author/uri[1]", doc) - {:ok, actor} = find_or_make_user(uri) + [author] = :xmerl_xpath.string('//author[1]', doc) + {:ok, actor} = find_make_or_update_user(author) context = (string_from_xpath("/entry/ostatus:conversation[1]", entry) || "") |> String.trim context = if String.length(context) > 0 do From b104348fa52c6ea51b9a159b145a48ca74a22332 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 17:44:55 +0200 Subject: [PATCH 073/107] Follow webfinger redirects. --- lib/pleroma/web/web_finger/web_finger.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 402184d3f..d16bdd982 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -92,7 +92,7 @@ def finger(account, getter \\ &HTTPoison.get/3) do response = with {:ok, result} <- getter.("https:" <> address, ["Accept": "application/xrd+xml"], [params: [resource: account]]) do {:ok, result} else _ -> - getter.("http:" <> address, ["Accept": "application/xrd+xml"], [params: [resource: account]]) + getter.("http:" <> address, ["Accept": "application/xrd+xml"], [params: [resource: account], follow_redirect: true]) end with {:ok, %{status_code: status_code, body: body}} when status_code in 200..299 <- response, From 33c803d6da91e0253a100cbd480d253706c44964 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 18:25:39 +0200 Subject: [PATCH 074/107] Add attachment link to posts. --- lib/pleroma/web/twitter_api/twitter_api.ex | 14 +++++++++++++- test/web/twitter_api/twitter_api_test.exs | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index b1759a6f0..7656d4d33 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -32,11 +32,23 @@ def get_replied_to_activity(id) when not is_nil(id) do def get_replied_to_activity(_), do: nil + def add_attachments(text, attachments) do + attachment_text = Enum.map(attachments, fn + (%{"url" => [%{"href" => href} | _]}) -> + "#{href}" + _ -> "" + end) + Enum.join([text | attachment_text], "
") + end + def create_status(user = %User{}, data = %{"status" => status}) do attachments = attachments_from_ids(data["media_ids"]) context = ActivityPub.generate_context_id mentions = parse_mentions(status) - content_html = format_input(status, mentions) + content_html = status + |> format_input(mentions) + |> add_attachments(attachments) + to = to_for_user_and_mentions(user, mentions) date = make_date() diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs index 4e17f3298..a92440f32 100644 --- a/test/web/twitter_api/twitter_api_test.exs +++ b/test/web/twitter_api/twitter_api_test.exs @@ -33,7 +33,7 @@ test "create a status" do { :ok, activity = %Activity{} } = TwitterAPI.create_status(user, input) - assert get_in(activity.data, ["object", "content"]) == "Hello again, @shp.
This is on another line." + assert get_in(activity.data, ["object", "content"]) == "Hello again, @shp.
This is on another line.
http://example.org/image.jpg" assert get_in(activity.data, ["object", "type"]) == "Note" assert get_in(activity.data, ["object", "actor"]) == user.ap_id assert get_in(activity.data, ["actor"]) == user.ap_id From 018a1a390fdb72652c615c28ac36f1b9a6a84d82 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Tue, 2 May 2017 21:31:01 +0200 Subject: [PATCH 075/107] Use inReplyTo to find context. --- lib/pleroma/web/ostatus/ostatus.ex | 18 ++++++++------ test/fixtures/incoming_reply_mastodon.xml | 29 +++++++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 17 +++++++++++++ 3 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 test/fixtures/incoming_reply_mastodon.xml diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 340228dcf..7aa1ac4ac 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -44,13 +44,19 @@ def handle_note(entry, doc \\ nil) do [author] = :xmerl_xpath.string('//author[1]', doc) {:ok, actor} = find_make_or_update_user(author) + inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@ref", entry) context = (string_from_xpath("/entry/ostatus:conversation[1]", entry) || "") |> String.trim - context = if String.length(context) > 0 do - context - else - ActivityPub.generate_context_id - end + + context = with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(inReplyTo) do + context + else _e -> + if String.length(context) > 0 do + context + else + ActivityPub.generate_context_id + end + end to = [ "https://www.w3.org/ns/activitystreams#Public" @@ -74,8 +80,6 @@ def handle_note(entry, doc \\ nil) do "actor" => actor.ap_id } - inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@ref", entry) - object = if inReplyTo do Map.put(object, "inReplyTo", inReplyTo) else diff --git a/test/fixtures/incoming_reply_mastodon.xml b/test/fixtures/incoming_reply_mastodon.xml new file mode 100644 index 000000000..8ee1186cc --- /dev/null +++ b/test/fixtures/incoming_reply_mastodon.xml @@ -0,0 +1,29 @@ + + + tag:mastodon.social,2017-05-02:objectId=4901603:objectType=Status + 2017-05-02T18:33:06Z + 2017-05-02T18:33:06Z + New status by lambadalambda + + https://mastodon.social/users/lambadalambda + http://activitystrea.ms/schema/1.0/person + https://mastodon.social/users/lambadalambda + lambadalambda + lambadalambda@mastodon.social + + + + lambadalambda + Critical Value + public + + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> hey</p> + + + public + + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 1674edbd5..e39952807 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -2,6 +2,7 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus alias Pleroma.Web.XML + alias Pleroma.{Object, Repo} test "don't insert create notes twice" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") @@ -32,6 +33,22 @@ test "handle incoming notes - GS, subscription" do assert activity.data["object"]["content"] == "Will it blend?" end + test "handle incoming notes - Mastodon, salmon, reply" do + # It uses the context of the replied to object + Repo.insert!(%Object{ + data: %{ + "id" => "https://pleroma.soykaf.com/objects/c237d966-ac75-4fe3-a87a-d89d71a3a7a4", + "context" => "2hu" + }}) + incoming = File.read!("test/fixtures/incoming_reply_mastodon.xml") + {:ok, [activity]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["actor"] == "https://mastodon.social/users/lambadalambda" + assert activity.data["context"] == "2hu" + end + test "handle incoming notes - GS, subscription, reply" do incoming = File.read!("test/fixtures/ostatus_incoming_reply.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) From 9c42453e068b683517f6a72602c08527222f8fea Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 09:54:17 +0200 Subject: [PATCH 076/107] Return note objects as ostatus post activities. --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/ostatus/ostatus_controller.ex | 15 +++++++++++++++ lib/pleroma/web/router.ex | 4 +++- test/web/ostatus/ostatus_controller_test.exs | 12 ++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 12d6912df..194a5ec3d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -126,7 +126,7 @@ def generate_context_id do end def generate_object_id do - generate_id("objects") + Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :object, Ecto.UUID.generate) end def generate_id(type) do diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 1c609f6f2..6a4199846 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -41,4 +41,19 @@ def salmon_incoming(conn, params) do conn |> send_resp(200, "") end + + def object(conn, %{"uuid" => uuid}) do + IO.inspect(uuid) + id = o_status_url(conn, :object, uuid) + activity = Activity.get_create_activity_by_object_ap_id(id) + user = User.get_cached_by_ap_id(activity.data["actor"]) + + response = FeedRepresenter.to_simple_form(user, [activity], [user]) + |> :xmerl.export_simple(:xmerl_xml) + |> to_string + + conn + |> put_resp_content_type("application/atom+xml") + |> send_resp(200, response) + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index b0c1dcd91..ac9d97e0f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -73,6 +73,8 @@ def user_fetcher(username) do scope "/", Pleroma.Web do pipe_through :ostatus + get "/objects/:uuid", OStatus.OStatusController, :object + get "/users/:nickname/feed", OStatus.OStatusController, :feed get "/users/:nickname", OStatus.OStatusController, :feed_redirect post "/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming @@ -96,5 +98,5 @@ def user_fetcher(username) do defmodule Fallback.RedirectController do use Pleroma.Web, :controller - def redirector(conn, _params), do: send_file(conn, 200, "priv/static/index.html") + def redirector(conn, _params), do: (if Mix.env != :test, do: send_file(conn, 200, "priv/static/index.html")) end diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index 229cd9b1e..f07698747 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -12,4 +12,16 @@ test "gets a feed", %{conn: conn} do assert response(conn, 200) end + + test "gets an object", %{conn: conn} do + note_activity = insert(:note_activity) + [_, uuid] = hd Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["object"]["id"]) + url = "/objects/#{uuid}" + |> IO.inspect + + conn = conn + |> get(url) + + assert response(conn, 200) + end end From 16afea399d330c28de05c77649fe0540598ee8ec Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 10:01:26 +0200 Subject: [PATCH 077/107] Just give out the entry, not the whole feed. --- lib/pleroma/web/ostatus/ostatus_controller.ex | 6 +++--- test/web/ostatus/ostatus_controller_test.exs | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 6a4199846..c6700ae78 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -2,7 +2,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do use Pleroma.Web, :controller alias Pleroma.{User, Activity} - alias Pleroma.Web.OStatus.FeedRepresenter + alias Pleroma.Web.OStatus.{FeedRepresenter, ActivityRepresenter} alias Pleroma.Repo alias Pleroma.Web.OStatus import Ecto.Query @@ -43,12 +43,12 @@ def salmon_incoming(conn, params) do end def object(conn, %{"uuid" => uuid}) do - IO.inspect(uuid) id = o_status_url(conn, :object, uuid) activity = Activity.get_create_activity_by_object_ap_id(id) user = User.get_cached_by_ap_id(activity.data["actor"]) - response = FeedRepresenter.to_simple_form(user, [activity], [user]) + response = ActivityRepresenter.to_simple_form(activity, user, true) + |> ActivityRepresenter.wrap_with_entry |> :xmerl.export_simple(:xmerl_xml) |> to_string diff --git a/test/web/ostatus/ostatus_controller_test.exs b/test/web/ostatus/ostatus_controller_test.exs index f07698747..8b7ca4d89 100644 --- a/test/web/ostatus/ostatus_controller_test.exs +++ b/test/web/ostatus/ostatus_controller_test.exs @@ -17,7 +17,6 @@ test "gets an object", %{conn: conn} do note_activity = insert(:note_activity) [_, uuid] = hd Regex.scan(~r/.+\/([\w-]+)$/, note_activity.data["object"]["id"]) url = "/objects/#{uuid}" - |> IO.inspect conn = conn |> get(url) From 8141024259ee4bebd58d6ecd963f181aad420846 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 14:26:49 +0200 Subject: [PATCH 078/107] Attachment parsing, better magic key fetching. --- lib/pleroma/web/ostatus/ostatus.ex | 35 ++++++++--- lib/pleroma/web/ostatus/ostatus_controller.ex | 2 +- lib/pleroma/web/salmon/salmon.ex | 15 ++--- lib/pleroma/web/web.ex | 22 +------ lib/pleroma/web/web_finger/web_finger.ex | 12 ++-- lib/pleroma/web/websub/websub.ex | 12 ++-- .../incoming_websub_gnusocial_attachments.xml | 59 +++++++++++++++++++ test/user_test.exs | 10 ++++ test/web/ostatus/ostatus_test.exs | 54 ++++++++++------- test/web/salmon/salmon_test.exs | 2 +- test/web/web_finger/web_finger_test.exs | 10 ++-- test/web/websub/websub_test.exs | 12 ++-- 12 files changed, 161 insertions(+), 84 deletions(-) create mode 100644 test/fixtures/incoming_websub_gnusocial_attachments.xml diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 7aa1ac4ac..f81751a25 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -39,6 +39,24 @@ def handle_incoming(xml_string) do {:ok, activities} end + def get_attachments(entry) do + :xmerl_xpath.string('/entry/link[@rel="enclosure"]', entry) + |> Enum.map(fn (enclosure) -> + with href when not is_nil(href) <- string_from_xpath("/link/@href", enclosure), + type when not is_nil(type) <- string_from_xpath("/link/@type", enclosure) do + %{ + "type" => "Attachment", + "url" => [%{ + "type" => "Link", + "mediaType" => type, + "href" => href + }] + } + end + end) + |> Enum.filter(&(&1)) + end + def handle_note(entry, doc \\ nil) do content_html = string_from_xpath("/entry/content[1]", entry) @@ -48,6 +66,8 @@ def handle_note(entry, doc \\ nil) do context = (string_from_xpath("/entry/ostatus:conversation[1]", entry) || "") |> String.trim + attachments = get_attachments(entry) + context = with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(inReplyTo) do context else _e -> @@ -77,7 +97,8 @@ def handle_note(entry, doc \\ nil) do "content" => content_html, "published" => date, "context" => context, - "actor" => actor.ap_id + "actor" => actor.ap_id, + "attachment" => attachments } object = if inReplyTo do @@ -124,11 +145,11 @@ def make_user(uri) do with {:ok, info} <- gather_user_info(uri) do data = %{ local: false, - name: info.name, - nickname: info.nickname <> "@" <> info.host, - ap_id: info.uri, + name: info["name"], + nickname: info["nickname"] <> "@" <> info["host"], + ap_id: info["uri"], info: info, - avatar: info.avatar + avatar: info["avatar"] } # TODO: Make remote user changeset # SHould enforce fqn nickname @@ -158,8 +179,8 @@ def make_avatar_object(author_doc) do def gather_user_info(username) do with {:ok, webfinger_data} <- WebFinger.finger(username), - {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data.topic) do - {:ok, Map.merge(webfinger_data, feed_data) |> Map.put(:fqn, username)} + {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data["topic"]) do + {:ok, Map.merge(webfinger_data, feed_data) |> Map.put("fqn", username)} else e -> Logger.debug("Couldn't gather info for #{username}") {:error, e} diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index c6700ae78..1f2dedd30 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -33,7 +33,7 @@ def feed(conn, %{"nickname" => nickname}) do def salmon_incoming(conn, params) do {:ok, body, _conn} = read_body(conn) - magic_key = Pleroma.Web.Salmon.fetch_magic_key(body) + {:ok, magic_key} = Pleroma.Web.Salmon.fetch_magic_key(body) {:ok, doc} = Pleroma.Web.Salmon.decode_and_validate(magic_key, body) Pleroma.Web.OStatus.handle_incoming(doc) diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index b4f214d46..f02cb11dc 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -24,16 +24,13 @@ def decode(salmon) do [data, type, encoding, alg, sig] end - # TODO rewrite in with-stile - # Make it fetch the key from the saved user if there is one def fetch_magic_key(salmon) do - [data, _, _, _, _] = decode(salmon) - doc = XML.parse_document(data) - uri = XML.string_from_xpath("/entry/author[1]/uri", doc) - - {:ok, info} = Pleroma.Web.OStatus.gather_user_info(uri) - - info.magic_key + with [data, _, _, _, _] <- decode(salmon), + doc <- XML.parse_document(data), + uri when not is_nil(uri) <- XML.string_from_xpath("/entry/author[1]/uri", doc), + {:ok, %{info: %{"magic_key" => magic_key}}} <- Pleroma.Web.OStatus.find_or_make_user(uri) do + {:ok, magic_key} + end end def decode_and_validate(magickey, salmon) do diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex index a81e3e6e1..ee7ee78e9 100644 --- a/lib/pleroma/web/web.ex +++ b/lib/pleroma/web/web.ex @@ -61,27 +61,7 @@ defmacro __using__(which) when is_atom(which) do apply(__MODULE__, which, []) end - def host do - settings = Application.get_env(:pleroma, Pleroma.Web.Endpoint) - settings - |> Keyword.fetch!(:url) - |> Keyword.fetch!(:host) - end - def base_url do - settings = Application.get_env(:pleroma, Pleroma.Web.Endpoint) - - host = host() - - protocol = settings |> Keyword.fetch!(:protocol) - - port_fragment = with {:ok, protocol_info} <- settings |> Keyword.fetch(String.to_atom(protocol)), - {:ok, port} <- protocol_info |> Keyword.fetch(:port) - do - ":#{port}" - else _e -> - "" - end - "#{protocol}://#{host}#{port_fragment}" + Pleroma.Web.Endpoint.url end end diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index d16bdd982..5fa69c2c8 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -16,7 +16,7 @@ def host_meta() do end def webfinger(resource) do - host = Pleroma.Web.host + host = Pleroma.Web.Endpoint.host regex = ~r/(acct:)?(?\w+)@#{host}/ with %{"username" => username} <- Regex.named_captures(regex, resource) do user = User.get_by_nickname(username) @@ -37,7 +37,7 @@ def represent_user(user) do { :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}, [ - {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.host}"}, + {:Subject, "acct:#{user.nickname}@#{Pleroma.Web.Endpoint.host}"}, {:Alias, user.ap_id}, {:Link, %{rel: "http://schemas.google.com/g/2010#updates-from", type: "application/atom+xml", href: OStatus.feed_path(user)}}, {:Link, %{rel: "http://webfinger.net/rel/profile-page", type: "text/html", href: user.ap_id}}, @@ -72,10 +72,10 @@ defp webfinger_from_xml(doc) do subject = XML.string_from_xpath("//Subject", doc) salmon = XML.string_from_xpath(~s{//Link[@rel="salmon"]/@href}, doc) data = %{ - magic_key: magic_key, - topic: topic, - subject: subject, - salmon: salmon + "magic_key" => magic_key, + "topic" => topic, + "subject" => subject, + "salmon" => salmon } {:ok, data} end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 546bfb5a4..e32fc8817 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -146,12 +146,12 @@ def gather_feed_data(topic, getter \\ &HTTPoison.get/1) do avatar = OStatus.make_avatar_object(doc) {:ok, %{ - uri: uri, - hub: hub, - nickname: preferredUsername || name, - name: displayName || name, - host: URI.parse(uri).host, - avatar: avatar + "uri" => uri, + "hub" => hub, + "nickname" => preferredUsername || name, + "name" => displayName || name, + "host" => URI.parse(uri).host, + "avatar" => avatar }} else e -> {:error, e} diff --git a/test/fixtures/incoming_websub_gnusocial_attachments.xml b/test/fixtures/incoming_websub_gnusocial_attachments.xml new file mode 100644 index 000000000..9d331ef32 --- /dev/null +++ b/test/fixtures/incoming_websub_gnusocial_attachments.xml @@ -0,0 +1,59 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-05-02T20:29:35+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-05-02:noticeId=2020923:objectType=note + New note by lambadalambda + Okay gonna stream some cool games!! <a href="https://social.heldscal.la/file/7ed5ee508e6376a6e3dd581e17e7ed0b7b638147c7e86784bf83abc2641ee3d4.gif" title="https://social.heldscal.la/file/7ed5ee508e6376a6e3dd581e17e7ed0b7b638147c7e86784bf83abc2641ee3d4.gif" rel="nofollow external noreferrer" class="attachment" id="attachment-423842">https://social.heldscal.la/attachment/423842</a> <a href="https://social.heldscal.la/file/4c209099cadfc5afd3e27a334aa0db96b3a7510dde1603305d68a2707e59a11f.png" title="https://social.heldscal.la/file/4c209099cadfc5afd3e27a334aa0db96b3a7510dde1603305d68a2707e59a11f.png" rel="nofollow external noreferrer" class="attachment" id="attachment-423843">https://social.heldscal.la/attachment/423843</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-05-02T20:29:35+00:00 + 2017-05-02T20:29:35+00:00 + + tag:social.heldscal.la,2017-05-02:objectType=thread:nonce=26c7afdcbcf4ebd4 + + + + + + + + diff --git a/test/user_test.exs b/test/user_test.exs index 1331ac971..7435e30e0 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -111,5 +111,15 @@ test "returns nil for nonexistant local user" do assert fetched_user == nil end end + + test "returns an ap_id for a user" do + user = insert(:user) + assert User.ap_id(user) == Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :feed_redirect, user.nickname) + end + + test "returns an ap_followers link for a user" do + user = insert(:user) + assert User.ap_followers(user) == Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :feed_redirect, user.nickname) <> "/followers" + end end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index e39952807..94a735337 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -33,6 +33,16 @@ test "handle incoming notes - GS, subscription" do assert activity.data["object"]["content"] == "Will it blend?" end + test "handle incoming notes with attachments - GS, subscription" do + incoming = File.read!("test/fixtures/incoming_websub_gnusocial_attachments.xml") + {:ok, [activity]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Create" + assert activity.data["object"]["type"] == "Note" + assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" + assert activity.data["object"]["attachment"] |> length == 2 + end + test "handle incoming notes - Mastodon, salmon, reply" do # It uses the context of the replied to object Repo.insert!(%Object{ @@ -118,17 +128,17 @@ test "it returns user info in a hash" do {:ok, data} = OStatus.gather_user_info(user) expected = %{ - hub: "https://social.heldscal.la/main/push/hub", - magic_key: "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", - name: "shp", - nickname: "shp", - salmon: "https://social.heldscal.la/main/salmon/user/29191", - subject: "acct:shp@social.heldscal.la", - topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", - uri: "https://social.heldscal.la/user/29191", - host: "social.heldscal.la", - fqn: user, - avatar: %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} + "hub" => "https://social.heldscal.la/main/push/hub", + "magic_key" => "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", + "name" => "shp", + "nickname" => "shp", + "salmon" => "https://social.heldscal.la/main/salmon/user/29191", + "subject" => "acct:shp@social.heldscal.la", + "topic" => "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", + "uri" => "https://social.heldscal.la/user/29191", + "host" => "social.heldscal.la", + "fqn" => user, + "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} } assert data == expected end @@ -140,17 +150,17 @@ test "it works with the uri" do {:ok, data} = OStatus.gather_user_info(user) expected = %{ - hub: "https://social.heldscal.la/main/push/hub", - magic_key: "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", - name: "shp", - nickname: "shp", - salmon: "https://social.heldscal.la/main/salmon/user/29191", - subject: "https://social.heldscal.la/user/29191", - topic: "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", - uri: "https://social.heldscal.la/user/29191", - host: "social.heldscal.la", - fqn: user, - avatar: %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} + "hub" => "https://social.heldscal.la/main/push/hub", + "magic_key" => "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB", + "name" => "shp", + "nickname" => "shp", + "salmon" => "https://social.heldscal.la/main/salmon/user/29191", + "subject" => "https://social.heldscal.la/user/29191", + "topic" => "https://social.heldscal.la/api/statuses/user_timeline/29191.atom", + "uri" => "https://social.heldscal.la/user/29191", + "host" => "social.heldscal.la", + "fqn" => user, + "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://social.heldscal.la/avatar/29191-original-20170421154949.jpeg", "mediaType" => "image/jpeg", "type" => "Link"}]} } assert data == expected end diff --git a/test/web/salmon/salmon_test.exs b/test/web/salmon/salmon_test.exs index 77dacc1c0..ed26ccf83 100644 --- a/test/web/salmon/salmon_test.exs +++ b/test/web/salmon/salmon_test.exs @@ -55,7 +55,7 @@ test "encodes an xml payload with a private key" do test "it gets a magic key" do # TODO: Make test local salmon = File.read!("test/fixtures/salmon2.xml") - key = Salmon.fetch_magic_key(salmon) + {:ok, key} = Salmon.fetch_magic_key(salmon) assert key == "RSA.uzg6r1peZU0vXGADWxGJ0PE34WvmhjUmydbX5YYdOiXfODVLwCMi1umGoqUDm-mRu4vNEdFBVJU1CpFA7dKzWgIsqsa501i2XqElmEveXRLvNRWFB6nG03Q5OUY2as8eE54BJm0p20GkMfIJGwP6TSFb-ICp3QjzbatuSPJ6xCE=.AQAB" end diff --git a/test/web/web_finger/web_finger_test.exs b/test/web/web_finger/web_finger_test.exs index b48fdd0aa..495d3d50b 100644 --- a/test/web/web_finger/web_finger_test.exs +++ b/test/web/web_finger/web_finger_test.exs @@ -15,7 +15,7 @@ test "returns a link to the xml lrdd" do test "works for fqns" do user = insert(:user) - {:ok, result} = WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.host}") + {:ok, result} = WebFinger.webfinger("#{user.nickname}@#{Pleroma.Web.Endpoint.host}") assert is_binary(result) end @@ -37,10 +37,10 @@ test "returns the info for a user" do {:ok, data} = WebFinger.finger(user, getter) - assert data.magic_key == "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB" - assert data.topic == "https://social.heldscal.la/api/statuses/user_timeline/29191.atom" - assert data.subject == "acct:shp@social.heldscal.la" - assert data.salmon == "https://social.heldscal.la/main/salmon/user/29191" + assert data["magic_key"] == "RSA.wQ3i9UA0qmAxZ0WTIp4a-waZn_17Ez1pEEmqmqoooRsG1_BvpmOvLN0G2tEcWWxl2KOtdQMCiPptmQObeZeuj48mdsDZ4ArQinexY2hCCTcbV8Xpswpkb8K05RcKipdg07pnI7tAgQ0VWSZDImncL6YUGlG5YN8b5TjGOwk2VG8=.AQAB" + assert data["topic"] == "https://social.heldscal.la/api/statuses/user_timeline/29191.atom" + assert data["subject"] == "acct:shp@social.heldscal.la" + assert data["salmon"] == "https://social.heldscal.la/main/salmon/user/29191" end end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 63acb3c43..065fb250a 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -115,12 +115,12 @@ test "discovers the hub and canonical url" do {:ok, discovered} = Websub.gather_feed_data(topic, getter) expected = %{ - hub: "https://mastodon.social/api/push", - uri: "https://mastodon.social/users/lambadalambda", - nickname: "lambadalambda", - name: "Critical Value", - host: "mastodon.social", - avatar: %{"type" => "Image", "url" => [%{"href" => "https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244", "mediaType" => "image/gif", "type" => "Link"}]} + "hub" => "https://mastodon.social/api/push", + "uri" => "https://mastodon.social/users/lambadalambda", + "nickname" => "lambadalambda", + "name" => "Critical Value", + "host" => "mastodon.social", + "avatar" => %{"type" => "Image", "url" => [%{"href" => "https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif?1492379244", "mediaType" => "image/gif", "type" => "Link"}]} } assert expected == discovered From df71c142cfadaae8866303768bca00c343b8bed1 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 16:08:24 +0200 Subject: [PATCH 079/107] Remove doubled 'to' recipients. --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- test/web/activity_pub/activity_pub_test.exs | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 194a5ec3d..f18f3df2e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -24,7 +24,7 @@ def create(to, actor, context, object, additional \\ %{}, published \\ nil, loca activity = %{ "type" => "Create", - "to" => to, + "to" => to |> Enum.uniq, "actor" => actor.ap_id, "object" => object, "published" => published, diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 6e42fbda2..dfa73b775 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -40,6 +40,13 @@ test "adds an id to a given object if it lacks one and inserts it to the object end end + describe "create activities" do + test "removes doubled 'to' recipients" do + {:ok, activity} = ActivityPub.create(["user1", "user1", "user2"], %User{ap_id: "1"}, "", %{}) + assert activity.data["to"] == ["user1", "user2"] + end + end + describe "fetch activities for recipients" do test "retrieve the activities for certain recipients" do {:ok, activity_one} = ActivityBuilder.insert(%{"to" => ["someone"]}) From 138641589dffc6ba69710ec15c690b50769f07b4 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 17:39:12 +0200 Subject: [PATCH 080/107] OStatus announce representer. --- .../web/ostatus/activity_representer.ex | 33 +++++++++++++- .../web/ostatus/activity_representer_test.exs | 45 ++++++++++++++++++- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index cf6aae727..9e3a9abcb 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -1,5 +1,5 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do - alias Pleroma.Activity + alias Pleroma.{Activity, User} alias Pleroma.Web.OStatus.UserRepresenter require Logger @@ -84,6 +84,37 @@ def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) d ] ++ author ++ mentions end + def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_author) do + h = fn(str) -> [to_charlist(str)] end + + updated_at = activity.updated_at + |> NaiveDateTime.to_iso8601 + inserted_at = activity.inserted_at + |> NaiveDateTime.to_iso8601 + + in_reply_to = get_in_reply_to(activity.data) + author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] + + retweeted_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"]) + retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"]) + + retweeted_xml = to_simple_form(retweeted_activity, retweeted_user) + + mentions = activity.data["to"] |> get_mentions + [ + {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']}, + {:id, h.(activity.data["id"])}, + {:title, ['#{user.nickname} repeated a notice']}, + {:content, [type: 'html'], ['RT #{retweeted_activity.data["object"]["content"]}']}, + {:published, h.(inserted_at)}, + {:updated, h.(updated_at)}, + {:"ostatus:conversation", [], h.(activity.data["context"])}, + {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, + {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []}, + {:"activity:object", retweeted_xml} + ] ++ mentions ++ author + end + def wrap_with_entry(simple_form) do [{ :entry, [ diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 4cf73427b..d3c32e938 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenterTest do use Pleroma.DataCase alias Pleroma.Web.OStatus.ActivityRepresenter - alias Pleroma.{User, Activity} + alias Pleroma.{User, Activity, Object} alias Pleroma.Web.ActivityPub.ActivityPub import Pleroma.Factory @@ -73,6 +73,49 @@ test "a reply note" do assert clean(res) == clean(expected) end + test "an announce activity" do + note = insert(:note_activity) + user = insert(:user) + object = Object.get_cached_by_ap_id(note.data["object"]["id"]) + + {:ok, announce, object} = ActivityPub.announce(user, object) + + announce = Repo.get(Activity, announce.id) + + note_user = User.get_cached_by_ap_id(note.data["actor"]) + note = Repo.get(Activity, note.id) + note_xml = ActivityRepresenter.to_simple_form(note, note_user) + |> :xmerl.export_simple_content(:xmerl_xml) + |> IO.iodata_to_binary + + updated_at = announce.updated_at + |> NaiveDateTime.to_iso8601 + inserted_at = announce.inserted_at + |> NaiveDateTime.to_iso8601 + + expected = """ + http://activitystrea.ms/schema/1.0/share + #{announce.data["id"]} + #{user.nickname} repeated a notice + RT #{note.data["object"]["content"]} + #{inserted_at} + #{updated_at} + #{announce.data["context"]} + + + + #{note_xml} + + + """ + + announce_xml = ActivityRepresenter.to_simple_form(announce, user) + |> :xmerl.export_simple_content(:xmerl_xml) + |> IO.iodata_to_binary + + assert clean(expected) == clean(announce_xml) + end + test "a like activity" do note = insert(:note) user = insert(:user) From 861a294cdae313c4c2edfc9840bf1083da0acd6e Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 17:41:55 +0200 Subject: [PATCH 081/107] Add announce federation. --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f18f3df2e..5583a1f41 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -199,6 +199,10 @@ def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) update_object_in_activities(object) + if user.local do + Pleroma.Web.Federator.enqueue(:publish, activity) + end + {:ok, activity, object} end From b34b046f16a44172ac96709dd0b6f5bced96d0b5 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 17:51:36 +0200 Subject: [PATCH 082/107] Add user to announced status. --- lib/pleroma/web/ostatus/activity_representer.ex | 2 +- test/web/ostatus/activity_representer_test.exs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 9e3a9abcb..d064b09ee 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -98,7 +98,7 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho retweeted_activity = Activity.get_create_activity_by_object_ap_id(activity.data["object"]) retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"]) - retweeted_xml = to_simple_form(retweeted_activity, retweeted_user) + retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true) mentions = activity.data["to"] |> get_mentions [ diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index d3c32e938..7f003226b 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -84,9 +84,9 @@ test "an announce activity" do note_user = User.get_cached_by_ap_id(note.data["actor"]) note = Repo.get(Activity, note.id) - note_xml = ActivityRepresenter.to_simple_form(note, note_user) + note_xml = ActivityRepresenter.to_simple_form(note, note_user, true) |> :xmerl.export_simple_content(:xmerl_xml) - |> IO.iodata_to_binary + |> to_string updated_at = announce.updated_at |> NaiveDateTime.to_iso8601 @@ -111,7 +111,7 @@ test "an announce activity" do announce_xml = ActivityRepresenter.to_simple_form(announce, user) |> :xmerl.export_simple_content(:xmerl_xml) - |> IO.iodata_to_binary + |> to_string assert clean(expected) == clean(announce_xml) end From 5d7831ee3e1ff62c2e54fe47aa1a6cf3474e8578 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 18:10:19 +0200 Subject: [PATCH 083/107] Add self links to federated statuses. --- lib/pleroma/web/ostatus/activity_representer.ex | 6 +++++- test/web/ostatus/activity_representer_test.exs | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index d064b09ee..dc7526598 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -51,7 +51,8 @@ def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, {:published, h.(inserted_at)}, {:updated, h.(updated_at)}, {:"ostatus:conversation", [], h.(activity.data["context"])}, - {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []} + {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, + {:link, [type: ['application/atom+xml'], href: h.(activity.data["object"]["id"]), rel: 'self'], []} ] ++ attachments ++ in_reply_to ++ author ++ mentions end @@ -80,6 +81,7 @@ def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) d ]}, {:"ostatus:conversation", [], h.(activity.data["context"])}, {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, + {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}, {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []} ] ++ author ++ mentions end @@ -102,6 +104,7 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho mentions = activity.data["to"] |> get_mentions [ + {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']}, {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']}, {:id, h.(activity.data["id"])}, {:title, ['#{user.nickname} repeated a notice']}, @@ -110,6 +113,7 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho {:updated, h.(updated_at)}, {:"ostatus:conversation", [], h.(activity.data["context"])}, {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, + {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}, {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []}, {:"activity:object", retweeted_xml} ] ++ mentions ++ author diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 7f003226b..03b3e248f 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -26,6 +26,7 @@ test "a note activity" do #{updated_at} #{note_activity.data["context"]} + """ @@ -62,6 +63,7 @@ test "a reply note" do #{updated_at} #{answer.data["context"]} + """ @@ -94,6 +96,7 @@ test "an announce activity" do |> NaiveDateTime.to_iso8601 expected = """ + http://activitystrea.ms/schema/1.0/activity http://activitystrea.ms/schema/1.0/share #{announce.data["id"]} #{user.nickname} repeated a notice @@ -102,6 +105,7 @@ test "an announce activity" do #{updated_at} #{announce.data["context"]} + #{note_xml} @@ -144,6 +148,7 @@ test "a like activity" do #{like.data["context"]} + """ From 53d05af5b61771782af3946181cc3139f3897cca Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 19:23:12 +0200 Subject: [PATCH 084/107] Fix Mastodon signature bug. --- lib/pleroma/web/websub/websub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index e32fc8817..0d0d19c88 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -52,7 +52,7 @@ def publish(topic, user, activity) do end def sign(secret, doc) do - :crypto.hmac(:sha, secret, to_string(doc)) |> Base.encode16 + :crypto.hmac(:sha, secret, to_string(doc)) |> Base.encode16 |> String.downcase end def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) do From 1077c5c58d13325cd61893c609cad6505ad1d32e Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 20:06:00 +0200 Subject: [PATCH 085/107] Remove reply-to for shares, mastodon gets confused. --- lib/pleroma/web/ostatus/activity_representer.ex | 1 - test/web/ostatus/activity_representer_test.exs | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index dc7526598..41a42b7cb 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -114,7 +114,6 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho {:"ostatus:conversation", [], h.(activity.data["context"])}, {:link, [href: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}, {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}, - {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []}, {:"activity:object", retweeted_xml} ] ++ mentions ++ author end diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 03b3e248f..12c9bbaa2 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -106,7 +106,6 @@ test "an announce activity" do #{announce.data["context"]} - #{note_xml} From 97257c692c5786b370d8f0769533d11c1d00334e Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Wed, 3 May 2017 20:06:20 +0200 Subject: [PATCH 086/107] Fix specs. --- lib/pleroma/web/websub/websub_controller.ex | 2 +- test/user_test.exs | 2 +- test/web/websub/websub_test.exs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index e5ecf6523..e860ec9e5 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -34,7 +34,7 @@ def websub_subscription_confirmation(conn, %{"id" => id, "hub.mode" => "subscrib def websub_incoming(conn, %{"id" => id}) do with "sha1=" <> signature <- hd(get_req_header(conn, "x-hub-signature")), - signature <- String.upcase(signature), + signature <- String.downcase(signature), %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id), {:ok, body, _conn} = read_body(conn), ^signature <- Websub.sign(websub.secret, body) do diff --git a/test/user_test.exs b/test/user_test.exs index 7435e30e0..036e70dff 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -13,7 +13,7 @@ test "ap_id returns the activity pub id for the user" do user = UserBuilder.build - expected_ap_id = "https://#{host}/users/#{user.nickname}" + expected_ap_id = "#{Pleroma.Web.base_url}/users/#{user.nickname}" assert expected_ap_id == User.ap_id(user) end diff --git a/test/web/websub/websub_test.exs b/test/web/websub/websub_test.exs index 065fb250a..48774dc69 100644 --- a/test/web/websub/websub_test.exs +++ b/test/web/websub/websub_test.exs @@ -170,7 +170,7 @@ test "rejects the subscription if it can't be accepted" do test "sign a text" do signed = Websub.sign("secret", "text") - assert signed == "B8392C23690CCF871F37EC270BE1582DEC57A503" + assert signed == "B8392C23690CCF871F37EC270BE1582DEC57A503" |> String.downcase signed = Websub.sign("secret", [["て"], ['す']]) end From 151da344beca98b2c007397cb0f8e47510bf747a Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 4 May 2017 09:54:22 +0200 Subject: [PATCH 087/107] Add debugging logs. --- lib/pleroma/web/websub/websub.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 0d0d19c88..c1532b6ce 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -44,6 +44,7 @@ def publish(topic, user, activity) do |> to_string signature = sign(sub.secret, response) + Logger.debug("Pushing to #{sub.callback}") HTTPoison.post(sub.callback, response, [ {"Content-Type", "application/atom+xml"}, {"X-Hub-Signature", "sha1=#{signature}"} From 5d9f3df714fa986367e105c2267324c8478ccf9c Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 4 May 2017 09:57:11 +0200 Subject: [PATCH 088/107] Just sign with an empty string if needed. --- lib/pleroma/web/websub/websub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index c1532b6ce..ba86db50e 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -43,7 +43,7 @@ def publish(topic, user, activity) do |> :xmerl.export_simple(:xmerl_xml) |> to_string - signature = sign(sub.secret, response) + signature = sign(sub.secret || "", response) Logger.debug("Pushing to #{sub.callback}") HTTPoison.post(sub.callback, response, [ {"Content-Type", "application/atom+xml"}, From c85998ab8a21f042ab57345a7baa9e1e27c308d1 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Thu, 4 May 2017 18:42:29 +0200 Subject: [PATCH 089/107] Parse incoming retweets. --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +- lib/pleroma/web/ostatus/ostatus.ex | 50 +++++++--- test/fixtures/share-gs.xml | 99 ++++++++++++++++++++ test/fixtures/share.xml | 54 +++++++++++ test/web/ostatus/ostatus_test.exs | 26 +++++ 5 files changed, 218 insertions(+), 15 deletions(-) create mode 100644 test/fixtures/share-gs.xml create mode 100644 test/fixtures/share.xml diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5583a1f41..1816b2e66 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -177,7 +177,7 @@ def fetch_activities(recipients, opts \\ %{}) do |> Enum.reverse end - def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do + def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object, local \\ true) do data = %{ "type" => "Announce", "actor" => ap_id, @@ -186,7 +186,7 @@ def announce(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) "context" => object.data["context"] } - {:ok, activity} = insert(data) + {:ok, activity} = insert(data, local) announcements = [ap_id | (object.data["announcements"] || [])] |> Enum.uniq diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index f81751a25..2fab67663 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -25,20 +25,44 @@ def handle_incoming(xml_string) do activities = Enum.map(entries, fn (entry) -> {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry) + {:xmlObj, :string, verb } = :xmerl_xpath.string('string(/entry/activity:verb[1])', entry) - case object_type do - 'http://activitystrea.ms/schema/1.0/note' -> - with {:ok, activity} <- handle_note(entry, doc), do: activity - 'http://activitystrea.ms/schema/1.0/comment' -> - with {:ok, activity} <- handle_note(entry, doc), do: activity + case verb do + 'http://activitystrea.ms/schema/1.0/share' -> + with {:ok, activity, retweeted_activity} <- handle_share(entry, doc), do: [activity, retweeted_activity] _ -> - Logger.error("Couldn't parse incoming document") - nil + case object_type do + 'http://activitystrea.ms/schema/1.0/note' -> + with {:ok, activity} <- handle_note(entry, doc), do: activity + 'http://activitystrea.ms/schema/1.0/comment' -> + with {:ok, activity} <- handle_note(entry, doc), do: activity + _ -> + Logger.error("Couldn't parse incoming document") + nil + end end end) {:ok, activities} end + def make_share(entry, doc, retweeted_activity) do + with {:ok, actor} <- find_make_or_update_user(doc), + %Object{} = object <- Object.get_cached_by_ap_id(retweeted_activity.data["object"]["id"]), + {:ok, activity, object} = ActivityPub.announce(actor, object, false) do + {:ok, activity} + end + end + + def handle_share(entry, doc) do + with [object] <- :xmerl_xpath.string('/entry/activity:object', entry), + {:ok, retweeted_activity} <- handle_note(object, object), + {:ok, activity} <- make_share(entry, doc, retweeted_activity) do + {:ok, activity, retweeted_activity} + else + e -> {:error, e} + end + end + def get_attachments(entry) do :xmerl_xpath.string('/entry/link[@rel="enclosure"]', entry) |> Enum.map(fn (enclosure) -> @@ -58,13 +82,13 @@ def get_attachments(entry) do end def handle_note(entry, doc \\ nil) do - content_html = string_from_xpath("/entry/content[1]", entry) + content_html = string_from_xpath("//content[1]", entry) [author] = :xmerl_xpath.string('//author[1]', doc) {:ok, actor} = find_make_or_update_user(author) - inReplyTo = string_from_xpath("/entry/thr:in-reply-to[1]/@ref", entry) + inReplyTo = string_from_xpath("//thr:in-reply-to[1]/@ref", entry) - context = (string_from_xpath("/entry/ostatus:conversation[1]", entry) || "") |> String.trim + context = (string_from_xpath("//ostatus:conversation[1]", entry) || "") |> String.trim attachments = get_attachments(entry) @@ -82,13 +106,13 @@ def handle_note(entry, doc \\ nil) do "https://www.w3.org/ns/activitystreams#Public" ] - mentions = :xmerl_xpath.string('/entry/link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', entry) + mentions = :xmerl_xpath.string('//link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', entry) |> Enum.map(fn(person) -> string_from_xpath("@href", person) end) to = to ++ mentions - date = string_from_xpath("/entry/published", entry) - id = string_from_xpath("/entry/id", entry) + date = string_from_xpath("//published", entry) + id = string_from_xpath("//id", entry) object = %{ "id" => id, diff --git a/test/fixtures/share-gs.xml b/test/fixtures/share-gs.xml new file mode 100644 index 000000000..ab5e488bd --- /dev/null +++ b/test/fixtures/share-gs.xml @@ -0,0 +1,99 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-05-03T08:05:41+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + tag:social.heldscal.la,2017-05-03:noticeId=2028428:objectType=note + lambadalambda repeated a notice by lain + RT @<a href="https://pleroma.soykaf.com/users/lain" class="h-card u-url p-nickname mention" title="Lain Iwakura">lain</a> Added returning the entries as xml... let's see if the mastodon hammering stops now. + + http://activitystrea.ms/schema/1.0/share + 2017-05-03T08:05:41+00:00 + 2017-05-03T08:05:41+00:00 + + http://activitystrea.ms/schema/1.0/activity + https://pleroma.soykaf.com/objects/4c1bda26-902e-4525-9fcd-b9fd44925193 + + Added returning the entries as xml... let's see if the mastodon hammering stops now. + + http://activitystrea.ms/schema/1.0/post + 2017-05-03T08:04:44+00:00 + 2017-05-03T08:04:44+00:00 + + http://activitystrea.ms/schema/1.0/person + https://pleroma.soykaf.com/users/lain + lain + Test account + + + + + + lain + Lain Iwakura + Test account + + + + http://activitystrea.ms/schema/1.0/note + https://pleroma.soykaf.com/objects/4c1bda26-902e-4525-9fcd-b9fd44925193 + New note by lain + Added returning the entries as xml... let's see if the mastodon hammering stops now. + + + + + https://pleroma.soykaf.com/contexts/ede39a2b-7cf3-4fa4-8ccd-cb97431bcc22 + + + https://pleroma.soykaf.com/users/lain/feed.atom + Lain Iwakura + + + https://social.heldscal.la/avatar/43188-96-20170429172422.jpeg + 2017-05-03T08:04:44+00:00 + + + + https://pleroma.soykaf.com/contexts/ede39a2b-7cf3-4fa4-8ccd-cb97431bcc22 + + + + + + diff --git a/test/fixtures/share.xml b/test/fixtures/share.xml new file mode 100644 index 000000000..e07b88680 --- /dev/null +++ b/test/fixtures/share.xml @@ -0,0 +1,54 @@ + + + tag:mastodon.social,2017-05-03:objectId=4934452:objectType=Status + 2017-05-03T08:21:09Z + 2017-05-03T08:21:09Z + lambadalambda shared a status by lain@pleroma.soykaf.com + + https://mastodon.social/users/lambadalambda + http://activitystrea.ms/schema/1.0/person + https://mastodon.social/users/lambadalambda + lambadalambda + lambadalambda@mastodon.social + + + + lambadalambda + Critical Value + public + + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + https://pleroma.soykaf.com/objects/4c1bda26-902e-4525-9fcd-b9fd44925193 + 2017-05-03T08:04:44Z + 2017-05-03T08:05:52Z + New status by lain@pleroma.soykaf.com + + https://pleroma.soykaf.com/users/lain + http://activitystrea.ms/schema/1.0/person + https://pleroma.soykaf.com/users/lain + lain + lain@pleroma.soykaf.com + Test account + + + + lain + Lain Iwakura + Test account + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + Added returning the entries as xml... let's see if the mastodon hammering stops now. + + public + + + Added returning the entries as xml... let's see if the mastodon hammering stops now. + + public + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 94a735337..e85d7677c 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -70,6 +70,32 @@ test "handle incoming notes - GS, subscription, reply" do assert activity.data["object"]["inReplyTo"] == "tag:gs.archae.me,2017-04-30:noticeId=778260:objectType=note" end + test "handle incoming retweets - GS, subscription" do + incoming = File.read!("test/fixtures/share-gs.xml") + {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Announce" + assert activity.data["actor"] == "https://social.heldscal.la/user/23211" + assert activity.data["object"] == retweeted_activity.data["object"]["id"] + refute activity.local + assert retweeted_activity.data["type"] == "Create" + assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain" + refute retweeted_activity.local + end + + test "handle incoming retweets - Mastodon, salmon" do + incoming = File.read!("test/fixtures/share.xml") + {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Announce" + assert activity.data["actor"] == "https://mastodon.social/users/lambadalambda" + assert activity.data["object"] == retweeted_activity.data["object"]["id"] + refute activity.local + assert retweeted_activity.data["type"] == "Create" + assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain" + refute retweeted_activity.local + end + test "handle incoming replies" do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) From 7269c51f3a4409110bc61faada08e35ce1b4d030 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 5 May 2017 12:07:38 +0200 Subject: [PATCH 090/107] Some refactoring. --- lib/pleroma/web/federator/federator.ex | 10 ++++---- .../web/ostatus/activity_representer.ex | 2 +- lib/pleroma/web/ostatus/ostatus.ex | 6 ++--- lib/pleroma/web/ostatus/ostatus_controller.ex | 3 ++- lib/pleroma/web/salmon/salmon.ex | 13 +++++++---- lib/pleroma/web/twitter_api/twitter_api.ex | 23 +++++++++---------- lib/pleroma/web/web_finger/web_finger.ex | 4 ++-- lib/pleroma/web/websub/websub.ex | 6 ++--- 8 files changed, 36 insertions(+), 31 deletions(-) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 675e804a2..c4dacd116 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -6,24 +6,24 @@ defmodule Pleroma.Web.Federator do @websub Application.get_env(:pleroma, :websub) def handle(:publish, activity) do - Logger.debug("Running publish for #{activity.data["id"]}") + Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) with actor when not is_nil(actor) <- User.get_cached_by_ap_id(activity.data["actor"]) do - Logger.debug("Sending #{activity.data["id"]} out via websub") + Logger.debug(fn -> "Sending #{activity.data["id"]} out via websub" end) Pleroma.Web.Websub.publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity) {:ok, actor} = WebFinger.ensure_keys_present(actor) - Logger.debug("Sending #{activity.data["id"]} out via salmon") + Logger.debug(fn -> "Sending #{activity.data["id"]} out via salmon" end) Pleroma.Web.Salmon.publish(actor, activity) end end def handle(:verify_websub, websub) do - Logger.debug("Running websub verification for #{websub.id} (#{websub.topic}, #{websub.callback})") + Logger.debug(fn -> "Running websub verification for #{websub.id} (#{websub.topic}, #{websub.callback})" end) @websub.verify(websub) end def handle(type, payload) do - Logger.debug("Unknown task: #{type}") + Logger.debug(fn -> "Unknown task: #{type}" end) {:error, "Don't know what do do with this"} end diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 88781626c..717670852 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -3,7 +3,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do alias Pleroma.Web.OStatus.UserRepresenter require Logger - defp get_in_reply_to(%{"object" => %{ "inReplyTo" => in_reply_to}}) do + defp get_in_reply_to(%{"object" => %{"inReplyTo" => in_reply_to}}) do [{:"thr:in-reply-to", [ref: to_charlist(in_reply_to)], []}] end diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 2fab67663..57fa3bcd5 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -24,8 +24,8 @@ def handle_incoming(xml_string) do entries = :xmerl_xpath.string('//entry', doc) activities = Enum.map(entries, fn (entry) -> - {:xmlObj, :string, object_type } = :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry) - {:xmlObj, :string, verb } = :xmerl_xpath.string('string(/entry/activity:verb[1])', entry) + {:xmlObj, :string, object_type} = :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry) + {:xmlObj, :string, verb} = :xmerl_xpath.string('string(/entry/activity:verb[1])', entry) case verb do 'http://activitystrea.ms/schema/1.0/share' -> @@ -206,7 +206,7 @@ def gather_user_info(username) do {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data["topic"]) do {:ok, Map.merge(webfinger_data, feed_data) |> Map.put("fqn", username)} else e -> - Logger.debug("Couldn't gather info for #{username}") + Logger.debug(fn -> "Couldn't gather info for #{username}" end) {:error, e} end end diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index e442562d5..5f79cc7e9 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -48,7 +48,8 @@ def object(conn, %{"uuid" => uuid}) do activity = Activity.get_create_activity_by_object_ap_id(id) user = User.get_cached_by_ap_id(activity.data["actor"]) - response = ActivityRepresenter.to_simple_form(activity, user, true) + response = activity + |> ActivityRepresenter.to_simple_form(user, true) |> ActivityRepresenter.wrap_with_entry |> :xmerl.export_simple(:xmerl_xml) |> to_string diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index fe529c4c0..3a8c30778 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -101,8 +101,13 @@ def encode(private_key, doc) do |> Enum.map(&Base.url_encode64/1) |> Enum.join(".") - signature = :public_key.sign(signed_text, :sha256, private_key) |> to_string |> Base.url_encode64 - doc_base64= doc |> Base.url_encode64 + signature = signed_text + |> :public_key.sign(:sha256, private_key) + |> to_string + |> Base.url_encode64 + + doc_base64 = doc + |> Base.url_encode64 # Don't need proper xml building, these strings are safe to leave unescaped salmon = """ @@ -143,11 +148,11 @@ def publish(%{info: %{"keys" => keys}} = user, activity, poster) do remote_users(activity) |> Enum.each(fn(remote_user) -> - Logger.debug("sending salmon to #{remote_user.ap_id}") + Logger.debug(fn -> "sending salmon to #{remote_user.ap_id}" end) send_to_user(remote_user, feed, poster) end) end end - def publish(%{id: id}, _, _), do: Logger.debug("Keys missing for user #{id}") + def publish(%{id: id}, _, _), do: Logger.debug(fn -> "Keys missing for user #{id}" end) end diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 7656d4d33..71f0c366e 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -41,7 +41,7 @@ def add_attachments(text, attachments) do Enum.join([text | attachment_text], "
") end - def create_status(user = %User{}, data = %{"status" => status}) do + def create_status(%User{} = user, %{"status" => status} = data) do attachments = attachments_from_ids(data["media_ids"]) context = ActivityPub.generate_context_id mentions = parse_mentions(status) @@ -122,8 +122,7 @@ def fetch_conversation(user, id) do statuses <- activities |> activities_to_statuses(%{for: user}) do statuses - else e -> - IO.inspect(e) + else _e -> [] end end @@ -135,26 +134,26 @@ def fetch_status(user, id) do end def follow(%User{} = follower, params) do - with { :ok, %User{} = followed } <- get_user(params), - { :ok, follower } <- User.follow(follower, followed), - { :ok, activity } <- ActivityPub.insert(%{ + with {:ok, %User{} = followed} <- get_user(params), + {:ok, follower} <- User.follow(follower, followed), + {:ok, activity} <- ActivityPub.insert(%{ "type" => "Follow", "actor" => follower.ap_id, "object" => followed.ap_id, "published" => make_date() }) do - { :ok, follower, followed, activity } + {:ok, follower, followed, activity} else err -> err end end def unfollow(%User{} = follower, params) do - with { :ok, %User{} = unfollowed } <- get_user(params), - { :ok, follower } <- User.unfollow(follower, unfollowed) + with {:ok, %User{} = unfollowed} <- get_user(params), + {:ok, follower} <- User.unfollow(follower, unfollowed) do - { :ok, follower, unfollowed} + {:ok, follower, unfollowed} else err -> err end @@ -269,14 +268,14 @@ def register_user(params) do def get_user(user \\ nil, params) do case params do - %{ "user_id" => user_id } -> + %{"user_id" => user_id} -> case target = Repo.get(User, user_id) do nil -> {:error, "No user with such user_id"} _ -> {:ok, target} end - %{ "screen_name" => nickname } -> + %{"screen_name" => nickname} -> case target = Repo.get_by(User, nickname: nickname) do nil -> {:error, "No user with such screen_name"} diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 1eb26a89f..2da111035 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -102,8 +102,8 @@ def finger(account, getter \\ &HTTPoison.get/3) do {:ok, data} else e -> - Logger.debug("Couldn't finger #{account}.") - Logger.debug(inspect(e)) + Logger.debug(fn -> "Couldn't finger #{account}." end) + Logger.debug(fn -> inspect(e) end) {:error, e} end end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index afbe944c5..848aac615 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -49,7 +49,7 @@ def publish(topic, user, activity) do |> to_string signature = sign(sub.secret || "", response) - Logger.debug("Pushing to #{sub.callback}") + Logger.debug(fn -> "Pushing to #{sub.callback}" end) HTTPoison.post(sub.callback, response, [ {"Content-Type", "application/atom+xml"}, @@ -196,8 +196,8 @@ def request_subscription(websub, poster \\ &HTTPoison.post/3, timeout \\ 10_000) change = Ecto.Changeset.change(websub, %{state: "rejected"}) {:ok, websub} = Repo.update(change) - Logger.debug("Couldn't confirm subscription: #{inspect(websub)}") - Logger.debug("error: #{inspect(e)}") + Logger.debug(fn -> "Couldn't confirm subscription: #{inspect(websub)}" end) + Logger.debug(fn -> "error: #{inspect(e)}" end) {:error, websub} end From d982f04a652ee95110ebd7d6dfc1ebdaa3c15d6f Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 5 May 2017 14:16:54 +0200 Subject: [PATCH 091/107] Mock HTTPoison in testing. --- config/config.exs | 1 + config/test.exs | 1 + lib/pleroma/web/ostatus/ostatus.ex | 4 +- lib/pleroma/web/salmon/salmon.ex | 4 +- lib/pleroma/web/web_finger/web_finger.ex | 3 +- lib/pleroma/web/websub/websub.ex | 10 +- ...__gs.example.org_4040_index.php_user_1.xml | 19 + ....php_api_statuses_user_timeline_1.atom.xml | 460 +++++++++++ ...__mastodon.social_users_lambadalambda.atom | 464 +++++++++++ ...___mastodon.social_users_lambadalambda.xml | 11 + .../https___pleroma.soykaf.com_users_lain.xml | 1 + ...leroma.soykaf.com_users_lain_feed.atom.xml | 1 + ..._api_statuses_user_timeline_23211.atom.xml | 591 ++++++++++++++ ..._api_statuses_user_timeline_29191.atom.xml | 719 ++++++++++++++++++ .../https___social.heldscal.la_user_23211.xml | 20 + .../https___social.heldscal.la_user_29191.xml | 20 + .../nonexistant@social.heldscal.la.xml | 90 +++ .../httpoison_mock/shp@social.heldscal.la.xml | 20 + test/support/httpoison_mock.ex | 97 +++ 19 files changed, 2527 insertions(+), 9 deletions(-) create mode 100644 test/fixtures/httpoison_mock/http___gs.example.org_4040_index.php_user_1.xml create mode 100644 test/fixtures/httpoison_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml create mode 100644 test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.atom create mode 100644 test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.xml create mode 100644 test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain.xml create mode 100644 test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml create mode 100644 test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml create mode 100644 test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml create mode 100644 test/fixtures/httpoison_mock/https___social.heldscal.la_user_23211.xml create mode 100644 test/fixtures/httpoison_mock/https___social.heldscal.la_user_29191.xml create mode 100644 test/fixtures/httpoison_mock/nonexistant@social.heldscal.la.xml create mode 100644 test/fixtures/httpoison_mock/shp@social.heldscal.la.xml create mode 100644 test/support/httpoison_mock.ex diff --git a/config/config.exs b/config/config.exs index a5df31b5a..8f6fe00fb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -32,6 +32,7 @@ config :pleroma, :websub, Pleroma.Web.Websub config :pleroma, :ostatus, Pleroma.Web.OStatus +config :pleroma, :httpoison, HTTPoison # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/config/test.exs b/config/test.exs index 85b6ad26b..04136e1f2 100644 --- a/config/test.exs +++ b/config/test.exs @@ -27,3 +27,4 @@ config :pleroma, :websub, Pleroma.Web.WebsubMock config :pleroma, :ostatus, Pleroma.Web.OStatusMock +config :pleroma, :httpoison, HTTPoisonMock diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 57fa3bcd5..3bed5a5d9 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -45,10 +45,10 @@ def handle_incoming(xml_string) do {:ok, activities} end - def make_share(entry, doc, retweeted_activity) do + def make_share(_entry, doc, retweeted_activity) do with {:ok, actor} <- find_make_or_update_user(doc), %Object{} = object <- Object.get_cached_by_ap_id(retweeted_activity.data["object"]["id"]), - {:ok, activity, object} = ActivityPub.announce(actor, object, false) do + {:ok, activity, _object} = ActivityPub.announce(actor, object, false) do {:ok, activity} end end diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index 3a8c30778..b95ad48ad 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -1,4 +1,6 @@ defmodule Pleroma.Web.Salmon do + @httpoison Application.get_env(:pleroma, :httpoison) + use Bitwise alias Pleroma.Web.XML alias Pleroma.Web.OStatus.ActivityRepresenter @@ -135,7 +137,7 @@ defp send_to_user(%{info: %{"salmon" => salmon}}, feed, poster) do defp send_to_user(_,_,_), do: nil - def publish(user, activity, poster \\ &HTTPoison.post/3) + def publish(user, activity, poster \\ &@httpoison.post/3) def publish(%{info: %{"keys" => keys}} = user, activity, poster) do feed = ActivityRepresenter.to_simple_form(activity, user, true) |> ActivityRepresenter.wrap_with_entry diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex index 2da111035..e8b738c96 100644 --- a/lib/pleroma/web/web_finger/web_finger.ex +++ b/lib/pleroma/web/web_finger/web_finger.ex @@ -1,4 +1,5 @@ defmodule Pleroma.Web.WebFinger do + @httpoison Application.get_env(:pleroma, :httpoison) alias Pleroma.{Repo, User, XmlBuilder} alias Pleroma.Web @@ -81,7 +82,7 @@ defp webfinger_from_xml(doc) do {:ok, data} end - def finger(account, getter \\ &HTTPoison.get/3) do + def finger(account, getter \\ &@httpoison.get/3) do domain = with [_name, domain] <- String.split(account, "@") do domain else _e -> diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 848aac615..7bdb778ad 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -9,9 +9,9 @@ defmodule Pleroma.Web.Websub do import Ecto.Query - @websub_verifier Application.get_env(:pleroma, :websub_verifier) + @httpoison Application.get_env(:pleroma, :httpoison) - def verify(subscription, getter \\ &HTTPoison.get/3) do + def verify(subscription, getter \\ &@httpoison.get/3) do challenge = Base.encode16(:crypto.strong_rand_bytes(8)) lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at) lease_seconds = lease_seconds |> to_string @@ -51,7 +51,7 @@ def publish(topic, user, activity) do signature = sign(sub.secret || "", response) Logger.debug(fn -> "Pushing to #{sub.callback}" end) - HTTPoison.post(sub.callback, response, [ + @httpoison.post(sub.callback, response, [ {"Content-Type", "application/atom+xml"}, {"X-Hub-Signature", "sha1=#{signature}"} ]) @@ -141,7 +141,7 @@ def subscribe(subscriber, subscribed, requester \\ &request_subscription/1) do requester.(subscription) end - def gather_feed_data(topic, getter \\ &HTTPoison.get/1) do + def gather_feed_data(topic, getter \\ &@httpoison.get/1) do with {:ok, response} <- getter.(topic), status_code when status_code in 200..299 <- response.status_code, body <- response.body, @@ -167,7 +167,7 @@ def gather_feed_data(topic, getter \\ &HTTPoison.get/1) do end end - def request_subscription(websub, poster \\ &HTTPoison.post/3, timeout \\ 10_000) do + def request_subscription(websub, poster \\ &@httpoison.post/3, timeout \\ 10_000) do data = [ "hub.mode": "subscribe", "hub.topic": websub.topic, diff --git a/test/fixtures/httpoison_mock/http___gs.example.org_4040_index.php_user_1.xml b/test/fixtures/httpoison_mock/http___gs.example.org_4040_index.php_user_1.xml new file mode 100644 index 000000000..058f629ab --- /dev/null +++ b/test/fixtures/httpoison_mock/http___gs.example.org_4040_index.php_user_1.xml @@ -0,0 +1,19 @@ + + + http://gs.example.org:4040/index.php/user/1 + acct:lambda@gs.example.org + http://gs.example.org/index.php/lambda + http://gs.example.org/lambda + + + + + + + + + + + + + diff --git a/test/fixtures/httpoison_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml b/test/fixtures/httpoison_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml new file mode 100644 index 000000000..490467708 --- /dev/null +++ b/test/fixtures/httpoison_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml @@ -0,0 +1,460 @@ + + + GNU social + http://gs.example.org/index.php/api/statuses/user_timeline/1.atom + lambda timeline + Updates from lambda on gs.example.org! + http://gs.example.org/theme/neo-gnu/default-avatar-profile.png + 2017-05-05T12:09:57+00:00 + + http://activitystrea.ms/schema/1.0/person + http://gs.example.org:4040/index.php/user/1 + lambda + + + + + lambda + lambda + + + + + + + + + + + + + tag:gs.example.org,2017-05-04:noticeId=84:objectType=note + lambda repeated a notice by lambda2 + RT @<a href="http://gs.example.org/index.php/user/7" class="h-card mention">lambda2</a> Hello! + + http://activitystrea.ms/schema/1.0/share + 2017-05-04T16:38:50+00:00 + 2017-05-04T16:38:50+00:00 + + http://activitystrea.ms/schema/1.0/activity + tag:gs.example.org,2017-05-01:noticeId=67:objectType=note + + Hello! + + http://activitystrea.ms/schema/1.0/post + 2017-05-01T08:41:04+00:00 + 2017-05-01T08:41:04+00:00 + + http://activitystrea.ms/schema/1.0/person + http://gs.example.org/index.php/user/7 + lambda2 + + + + + + lambda2 + lambda2 + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org,2017-05-01:noticeId=67:objectType=note + New note by lambda2 + Hello! + + + + + tag:gs.example.org,2017-05-01:objectType=thread:nonce=cffa792cb95fe417 + + + http://gs.example.org/index.php/api/statuses/user_timeline/7.atom + lambda2 + + + + http://gs.example.org/avatar/7-96-20170501084054.png + 2017-05-01T16:33:10+00:00 + + + + + + tag:gs.example.org,2017-05-01:objectType=thread:nonce=cffa792cb95fe417 + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org,2017-04-30:noticeId=63:objectType=note + New note by lambda + what now? + + + http://activitystrea.ms/schema/1.0/post + 2017-04-30T10:09:57+00:00 + 2017-04-30T10:09:57+00:00 + + + + tag:gs.example.org,2017-04-30:objectType=thread:nonce=1bbb60991ae9874b + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org,2017-04-30:noticeId=61:objectType=note + New note by lambda + @<a href="http://pleroma.example.org:4000/users/lain5" class="h-card mention">lain5</a> Hello! + + + http://activitystrea.ms/schema/1.0/post + 2017-04-30T10:07:26+00:00 + 2017-04-30T10:07:26+00:00 + + tag:gs.example.org,2017-04-30:objectType=thread:nonce=1bbb60991ae9874b + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org,2017-04-29:noticeId=59:objectType=note + New note by lambda + ey + + + http://activitystrea.ms/schema/1.0/post + 2017-04-29T17:04:59+00:00 + 2017-04-29T17:04:59+00:00 + + tag:gs.example.org,2017-04-29:objectType=thread:nonce=4cc42c2c61a0f4bd + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org,2017-04-29:noticeId=58:objectType=note + New note by lambda + Another one. + + + http://activitystrea.ms/schema/1.0/post + 2017-04-29T17:02:47+00:00 + 2017-04-29T17:02:47+00:00 + + tag:gs.example.org,2017-04-29:objectType=thread:nonce=53e9b8f1d6d38d13 + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org,2017-04-29:noticeId=57:objectType=note + New note by lambda + Let's see if this comes over. + + + http://activitystrea.ms/schema/1.0/post + 2017-04-29T17:01:39+00:00 + 2017-04-29T17:01:39+00:00 + + tag:gs.example.org,2017-04-29:objectType=thread:nonce=238a7bd3ffc7c9cc + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org,2017-04-29:noticeId=56:objectType=note + New note by lambda + @<a href="http://pleroma.example.org:4000/users/lain5" class="h-card mention">lain5</a> Hey! + + + http://activitystrea.ms/schema/1.0/post + 2017-04-29T16:38:13+00:00 + 2017-04-29T16:38:13+00:00 + + tag:gs.example.org,2017-04-29:objectType=thread:nonce=2629d3a398171b0f + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note + New note by lambda + hey. + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T18:16:13+00:00 + 2017-04-25T18:16:13+00:00 + + + + http://pleroma.example.org:4000/contexts/8f6f45d4-8e4d-4e1a-a2de-09f27367d2d0 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=53:objectType=note + New note by lambda + and this? + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T18:14:34+00:00 + 2017-04-25T18:14:34+00:00 + + + + http://pleroma.example.org:4000/contexts/24779b0e-91ad-487e-81bd-6cf5bb437b09 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=52:objectType=note + New note by lambda + yeah it does :) + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T18:13:31+00:00 + 2017-04-25T18:13:31+00:00 + + + + tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=e0dc24b1a93ab6b3 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=50:objectType=note + New note by lambda + @<a href="http://pleroma.example.org:4000/users/lain5" class="h-card mention">lain5</a> Let's try with one that originates here! + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T18:10:28+00:00 + 2017-04-25T18:10:28+00:00 + + tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=e0dc24b1a93ab6b3 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=48:objectType=note + New note by lambda + works? + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T18:08:44+00:00 + 2017-04-25T18:08:44+00:00 + + + + http://pleroma.example.org:4000/contexts/24779b0e-91ad-487e-81bd-6cf5bb437b09 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=46:objectType=note + New note by lambda + Let's send you an answer. + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T18:05:31+00:00 + 2017-04-25T18:05:31+00:00 + + + + tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=73c7bcf6658f7ce3 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=44:objectType=note + New note by lambda + Hey. + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T18:01:09+00:00 + 2017-04-25T18:01:09+00:00 + + + + tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=6e7c8fc2823380b4 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=43:objectType=note + New note by lambda + What's coming to you? + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T17:58:41+00:00 + 2017-04-25T17:58:41+00:00 + + + + tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=6e7c8fc2823380b4 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=42:objectType=note + New note by lambda + Now this is podracing. + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T17:57:40+00:00 + 2017-04-25T17:57:40+00:00 + + + + tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=6e7c8fc2823380b4 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=39:objectType=note + New note by lambda + Sure looks like it! + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T17:48:27+00:00 + 2017-04-25T17:48:27+00:00 + + + + tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=4c6114a75bb4cea5 + + + + + + + + tag:gs.example.org:4040,2017-04-25:subscription:1:person:6:2017-04-25T17:47:47+00:00 + lambda (lambda)'s status on Tuesday, 25-Apr-2017 17:47:47 UTC + <a href="http://gs.example.org:4040/index.php/lambda">lambda</a> started following <a href="http://pleroma.example.org:4000/users/lain5">l</a>. + + http://activitystrea.ms/schema/1.0/follow + 2017-04-25T17:47:47+00:00 + 2017-04-25T17:47:47+00:00 + + http://activitystrea.ms/schema/1.0/person + http://pleroma.example.org:4000/users/lain5 + l + lambadalambda + + + + + + lain5 + l + lambadalambda + + + tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=119acad17515314c + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=36:objectType=note + New note by lambda + @<a href="http://pleroma.example.org:4000/users/lain5" class="h-card mention">lain5</a> Hey, how are you? + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T17:46:22+00:00 + 2017-04-25T17:46:22+00:00 + + tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=9c5ec19a18191372 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.example.org:4040,2017-04-25:noticeId=35:objectType=note + New note by lambda + @lain5@pleroma.example.org does this not work? + + + http://activitystrea.ms/schema/1.0/post + 2017-04-25T17:42:31+00:00 + 2017-04-25T17:42:31+00:00 + + tag:gs.example.org:4040,2017-04-25:objectType=thread:nonce=fc841d7f52caa363 + + + + + + diff --git a/test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.atom b/test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.atom new file mode 100644 index 000000000..4d732b109 --- /dev/null +++ b/test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.atom @@ -0,0 +1,464 @@ + + + https://mastodon.social/users/lambadalambda.atom + Critical Value + + 2017-04-16T21:47:25Z + https://files.mastodon.social/accounts/avatars/000/000/264/original/1429214160519.gif + + https://mastodon.social/users/lambadalambda + http://activitystrea.ms/schema/1.0/person + https://mastodon.social/users/lambadalambda + lambadalambda + lambadalambda@mastodon.social + + + + lambadalambda + Critical Value + public + + + + + + + + tag:mastodon.social,2017-05-04:objectId=4991300:objectType=Status + 2017-05-04T14:10:30Z + 2017-05-04T14:10:30Z + Delete + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/delete + + + + + tag:mastodon.social,2017-05-04:objectId=4980289:objectType=Status + 2017-05-04T07:43:23Z + 2017-05-04T07:43:23Z + Delete + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/delete + + + + + tag:mastodon.social,2017-05-03:objectId=4952899:objectType=Status + 2017-05-03T17:26:43Z + 2017-05-03T17:26:43Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> OK!!</p> + + + public + + + + + + tag:mastodon.social,2017-05-03:objectId=4952810:objectType=Status + 2017-05-03T17:24:34Z + 2017-05-03T17:24:34Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> yeah :)</p> + + + public + + + + + + tag:mastodon.social,2017-05-03:objectId=4950388:objectType=Status + 2017-05-03T16:22:00Z + 2017-05-03T16:22:00Z + lambadalambda shared a status by lambadalambda@social.heldscal.la + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:social.heldscal.la,2017-05-03:noticeId=2030733:objectType=note + 2017-05-03T12:29:20Z + 2017-05-03T12:29:31Z + New status by lambadalambda@social.heldscal.la + + https://social.heldscal.la/user/23211 + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + lambadalambda@social.heldscal.la + Call me Deacon Blues. + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + Time for work. <a href="https://social.heldscal.la/file/953c117a1e7e4c763755d2ac29cf1aae08e025599f4a4cc11ddff4082c07f969.jpg">https://social.heldscal.la/attachment/120552</a> + + + public + + + Time for work. <a href="https://social.heldscal.la/file/953c117a1e7e4c763755d2ac29cf1aae08e025599f4a4cc11ddff4082c07f969.jpg">https://social.heldscal.la/attachment/120552</a> + + public + + + + + tag:mastodon.social,2017-05-03:objectId=4934452:objectType=Status + 2017-05-03T08:21:09Z + 2017-05-03T08:21:09Z + lambadalambda shared a status by lain@pleroma.soykaf.com + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + https://pleroma.soykaf.com/objects/4c1bda26-902e-4525-9fcd-b9fd44925193 + 2017-05-03T08:04:44Z + 2017-05-03T08:05:52Z + New status by lain@pleroma.soykaf.com + + https://pleroma.soykaf.com/users/lain + http://activitystrea.ms/schema/1.0/person + https://pleroma.soykaf.com/users/lain + lain + lain@pleroma.soykaf.com + Test account + + + + lain + Lain Iwakura + Test account + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + Added returning the entries as xml... let's see if the mastodon hammering stops now. + + public + + + Added returning the entries as xml... let's see if the mastodon hammering stops now. + + public + + + + + tag:mastodon.social,2017-05-02:objectId=4905499:objectType=Status + 2017-05-02T19:34:21Z + 2017-05-02T19:34:21Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> yay!</p> + + + public + + + + + + tag:mastodon.social,2017-05-02:objectId=4905442:objectType=Status + 2017-05-02T19:33:33Z + 2017-05-02T19:33:33Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> so?</p> + + + public + + + + + + tag:mastodon.social,2017-05-02:objectId=4901603:objectType=Status + 2017-05-02T18:33:06Z + 2017-05-02T18:33:06Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> hey</p> + + + public + + + + + + tag:mastodon.social,2017-05-01:objectId=4836720:objectType=Status + 2017-05-01T18:52:16Z + 2017-05-01T18:52:16Z + lambadalambda shared a status by lain@pleroma.soykaf.com + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + https://pleroma.soykaf.com/objects/7b41bb51-9aba-436a-82d9-dd3f5aca98c9 + 2017-05-01T18:50:54Z + 2017-05-01T18:50:57Z + New status by lain@pleroma.soykaf.com + + https://pleroma.soykaf.com/users/lain + http://activitystrea.ms/schema/1.0/person + https://pleroma.soykaf.com/users/lain + lain + lain@pleroma.soykaf.com + Test account + + + + lain + Lain Iwakura + Test account + public + + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <a href="https://mastodon.social/users/lambadalambda">@lambadalambda@mastodon.social</a> you're an all-star. + + + public + + + + <a href="https://mastodon.social/users/lambadalambda">@lambadalambda@mastodon.social</a> you're an all-star. + + public + + + + + tag:mastodon.social,2017-05-01:objectId=4836142:objectType=Status + 2017-05-01T18:38:47Z + 2017-05-01T18:38:47Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> Hey now!</p> + + + public + + + + + + tag:mastodon.social,2017-05-01:objectId=4836055:objectType=Status + 2017-05-01T18:37:04Z + 2017-05-01T18:37:04Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/comment + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> hello</p> + + + public + + + + + + tag:mastodon.social,2017-05-01:objectId=4834850:objectType=Status + 2017-05-01T18:10:43Z + 2017-05-01T18:10:43Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> Hey!</p> + + + public + + + + + tag:mastodon.social,2017-04-29:objectId=4694455:objectType=Status + 2017-04-29T18:39:12Z + 2017-04-29T18:39:12Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>@lain@pleroma.soykaf.com What&apos;s up?</p> + + public + + + + + tag:mastodon.social,2017-04-29:objectId=4694384:objectType=Status + 2017-04-29T18:37:32Z + 2017-04-29T18:37:32Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://social.heldscal.la/lain" class="u-url mention">@<span>lain</span></a></span> Hey.</p> + + + public + + + + + tag:mastodon.social,2017-04-07:objectId=1874242:objectType=Status + 2017-04-07T11:02:56Z + 2017-04-07T11:02:56Z + lambadalambda shared a status by 0xroy@social.wxcafe.net + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:social.wxcafe.net,2017-04-07:objectId=72554:objectType=Status + 2017-04-07T11:01:59Z + 2017-04-07T11:02:00Z + New status by 0xroy@social.wxcafe.net + + https://social.wxcafe.net/users/0xroy + http://activitystrea.ms/schema/1.0/person + https://social.wxcafe.net/users/0xroy + 0xroy + 0xroy@social.wxcafe.net + ta caution weeb | discussions privées : <a href="https://%F0%9F%92%8C.0xroy.me"><span class="invisible">https://</span><span class="">💌.0xroy.me</span><span class="invisible"></span></a> + + + + 0xroy + 「R O Y 🍵 B O S」 + ta caution weeb | discussions privées : https://💌.0xroy.me + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>someone pls eli5 matrix (protocol) and riot</p> + + public + + + <p>someone pls eli5 matrix (protocol) and riot</p> + + public + + + + + tag:mastodon.social,2017-04-06:objectId=1768247:objectType=Status + 2017-04-06T11:10:19Z + 2017-04-06T11:10:19Z + lambadalambda shared a status by areyoutoo@mastodon.xyz + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:mastodon.xyz,2017-04-05:objectId=133327:objectType=Status + 2017-04-05T17:36:41Z + 2017-04-05T18:12:14Z + New status by areyoutoo@mastodon.xyz + + https://mastodon.xyz/users/areyoutoo + http://activitystrea.ms/schema/1.0/person + https://mastodon.xyz/users/areyoutoo + areyoutoo + areyoutoo@mastodon.xyz + devops | retired gamedev | always boost puppy pics + + + + areyoutoo + Raw Butter + devops | retired gamedev | always boost puppy pics + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p>Some UX thoughts for <a href="https://mastodon.xyz/tags/mastodev">#<span>mastodev</span></a>:</p><p>- Would be nice if I could work on multiple draft toots? Clicking to reply to someone seems to erase any draft I had been working on.</p><p>- Kinda risky to click on the Federated Timeline if it loads new toots and scrolls 10ms before I click on something.</p><p>I probably don't know enough web frontend to help, but it might be fun to try.</p> + + + public + + + <p>Some UX thoughts for <a href="https://mastodon.xyz/tags/mastodev">#<span>mastodev</span></a>:</p><p>- Would be nice if I could work on multiple draft toots? Clicking to reply to someone seems to erase any draft I had been working on.</p><p>- Kinda risky to click on the Federated Timeline if it loads new toots and scrolls 10ms before I click on something.</p><p>I probably don't know enough web frontend to help, but it might be fun to try.</p> + + public + + + + + tag:mastodon.social,2017-04-06:objectId=1764509:objectType=Status + 2017-04-06T10:15:38Z + 2017-04-06T10:15:38Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + This is a test for cw federation + <p>This is a test for cw federation body text.</p> + + public + + + + + tag:mastodon.social,2017-04-05:objectId=1645208:objectType=Status + 2017-04-05T07:14:53Z + 2017-04-05T07:14:53Z + lambadalambda shared a status by lambadalambda@social.heldscal.la + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/share + + tag:social.heldscal.la,2017-04-05:noticeId=1502088:objectType=note + 2017-04-05T06:12:09Z + 2017-04-05T07:12:47Z + New status by lambadalambda@social.heldscal.la + + https://social.heldscal.la/user/23211 + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + lambadalambda@social.heldscal.la + Call me Deacon Blues. + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + public + + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + Federation 101: <a href="https://www.youtube.com/watch?v=t1lYU5CA40o">https://www.youtube.com/watch?v=t1lYU5CA40o</a> + + public + + + Federation 101: <a href="https://www.youtube.com/watch?v=t1lYU5CA40o">https://www.youtube.com/watch?v=t1lYU5CA40o</a> + + public + + + + + tag:mastodon.social,2017-04-05:objectId=1641750:objectType=Status + 2017-04-05T05:44:48Z + 2017-04-05T05:44:48Z + New status by lambadalambda + http://activitystrea.ms/schema/1.0/note + http://activitystrea.ms/schema/1.0/post + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> just a test.</p> + + + public + + + + diff --git a/test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.xml b/test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.xml new file mode 100644 index 000000000..6a6a978a2 --- /dev/null +++ b/test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.xml @@ -0,0 +1,11 @@ + + + acct:lambadalambda@mastodon.social + https://mastodon.social/@lambadalambda + https://mastodon.social/users/lambadalambda + + + + + + diff --git a/test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain.xml b/test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain.xml new file mode 100644 index 000000000..284a30df0 --- /dev/null +++ b/test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain.xml @@ -0,0 +1 @@ +acct:lain@pleroma.soykaf.comhttps://pleroma.soykaf.com/users/lain \ No newline at end of file diff --git a/test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml b/test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml new file mode 100644 index 000000000..a2a2629a6 --- /dev/null +++ b/test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml @@ -0,0 +1 @@ +https://pleroma.soykaf.com/users/lain/feed.atomlain's timeline2017-05-05T08:38:03.385598https://pleroma.soykaf.com/users/lainhttp://activitystrea.ms/schema/1.0/personhttps://pleroma.soykaf.com/users/lainlainLain IwakuraTest accountlainhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/sharehttps://pleroma.soykaf.com/activities/579e4224-b2ab-4ffa-8bbe-f7197a0a38d1lain repeated a noticeRT In just seven days, I can make you a man!<br> -- The Rocky Horror Picture Show2017-05-05T08:38:03.3855902017-05-05T08:38:03.385598https://pleroma.soykaf.com/contexts/e8673466-9642-4c9e-8781-f0f69d6b15aehttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/53dd40f4-3069-45a1-863b-94a9b317093dNew note by fortuneIn just seven days, I can make you a man!<br> -- The Rocky Horror Picture Show2017-05-05T02:10:02.9308022017-05-05T08:38:03.423539https://pleroma.soykaf.com/contexts/e8673466-9642-4c9e-8781-f0f69d6b15aehttps://pleroma.soykaf.com/users/fortunehttp://activitystrea.ms/schema/1.0/personhttps://pleroma.soykaf.com/users/fortunefortunefortuneThe trusty unix fortune filefortunehttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/2bc86888-a256-4771-bb53-903f375804f9New note by lainRTs federating into pleroma now.2017-05-04T18:18:50.2764702017-05-04T18:18:50.276476https://pleroma.soykaf.com/contexts/b7ae9350-f317-48aa-8058-2668091bb280http://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/902b1f50-f295-4189-8c15-9c880919e121New favorite by lainlain favorited something2017-05-04T08:03:01.3088902017-05-04T08:03:01.308927http://activitystrea.ms/schema/1.0/notetag:gs.smuglo.li,2017-05-03:noticeId=2164642:objectType=commenthttps://pleroma.soykaf.com/contexts/9419f742-aaba-4eb5-89a2-8b599e8bf43chttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/4e396e66-b063-454c-92c6-583506a9a2deNew note by lainClassic.<br><a href='https://pleroma.soykaf.com/media/adc36781-9765-4d9a-b57c-99b7a99108b2/mikodaemonstop.jpg'>https://pleroma.soykaf.com/media/adc36781-9765-4d9a-b57c-99b7a99108b2/mikodaemonstop.jpg</a>2017-05-04T07:59:45.1806192017-05-04T07:59:45.180628https://pleroma.soykaf.com/contexts/6afd9659-41e6-406d-ae97-43b880722861http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/85d183e9-c935-4655-a1e6-8d69a4108235New note by lainん？<br><a href='https://pleroma.soykaf.com/media/ab144c6d-a38c-4d35-a60b-9a998becc094/n.gif'>https://pleroma.soykaf.com/media/ab144c6d-a38c-4d35-a60b-9a998becc094/n.gif</a>2017-05-04T07:58:08.8107162017-05-04T07:58:08.810726https://pleroma.soykaf.com/contexts/2e1aa616-86ce-4b50-9c81-63045a972156http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/7c5c45bb-e4d9-4f72-b4c6-0314afbd3553New note by lainyeah.2017-05-04T07:55:17.3352902017-05-04T07:55:17.335299https://pleroma.soykaf.com/contexts/702c06cf-56ff-4a2f-bf5a-150bc00bb168http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/f33f5f54-1c1d-4462-b9ed-229bb635dfd8New note by lainyeah.2017-05-04T07:49:24.9314842017-05-04T07:49:24.931492https://pleroma.soykaf.com/contexts/c4932e7a-00cb-431a-b4ec-7404cb9daf65http://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/0709bc79-7ac5-4983-b6d0-2205bf5ceba3New favorite by lainlain favorited something2017-05-03T20:08:11.2945792017-05-03T20:08:11.294587http://activitystrea.ms/schema/1.0/notetag:pawoo.net,2017-05-03:objectId=7967690:objectType=Statushttps://pleroma.soykaf.com/contexts/07a4b34d-6255-4bb2-8c73-c295a09ac952http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/72c0288e-62d8-43d9-b3d8-1a9d78be8375New note by lain<a href='https://pawoo.net/users/God_Emperor_of_Dune'>@God_Emperor_of_Dune@pawoo.net</a> no man, just some fun domination play among buddies, nothing homo about it.2017-05-03T20:01:00.9983142017-05-03T20:01:00.998322https://pleroma.soykaf.com/contexts/07a4b34d-6255-4bb2-8c73-c295a09ac952http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/d846409e-cf2a-4b68-a149-d5de34a91b0dNew note by lain<a href='https://social.heldscal.la/user/24974'>@dtluna@social.heldscal.la</a> btfo.<br><a href='https://pleroma.soykaf.com/media/fbe42e87-5574-4544-89ba-29ddf46227fa/pnc__picked_media_1889ce61-4961-4fea-8a14-04fe6783ebf6.jpg'>https://pleroma.soykaf.com/media/fbe42e87-5574-4544-89ba-29ddf46227fa/pnc__picked_media_1889ce61-4961-4fea-8a14-04fe6783ebf6.jpg</a>2017-05-03T20:00:15.8609952017-05-03T20:00:15.861002https://pleroma.soykaf.com/contexts/0e88f35e-1a38-4181-bef9-5cbb0d943c63http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/9075265f-f3b2-40e8-809f-10714f05a1fdNew note by lain#nohomo <br><a href='https://pleroma.soykaf.com/media/5cc5ad91-d637-4c45-a691-5ea778dc1bb3/pnc__picked_media_f62dc9ae-ea23-4fe6-bf85-cb75a129ab34.jpg'>https://pleroma.soykaf.com/media/5cc5ad91-d637-4c45-a691-5ea778dc1bb3/pnc__picked_media_f62dc9ae-ea23-4fe6-bf85-cb75a129ab34.jpg</a>2017-05-03T19:50:38.5891062017-05-03T19:50:38.589113https://pleroma.soykaf.com/contexts/07a4b34d-6255-4bb2-8c73-c295a09ac952http://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/7924e992-0a95-40d9-8d17-7278c6c634c9New favorite by lainlain favorited something2017-05-03T18:32:59.2733752017-05-03T18:32:59.273382http://activitystrea.ms/schema/1.0/notetag:gs.smuglo.li,2017-05-03:noticeId=2164774:objectType=commenthttps://pleroma.soykaf.com/contexts/9419f742-aaba-4eb5-89a2-8b599e8bf43chttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/569571ba-f54c-41b0-bde4-0fede54599f0New note by lain<a href='https://gs.smuglo.li/user/2'>@nepfag@gs.smuglo.li</a>@gs.smuglo.li I'll do proper subfolders soon, for now it's one per attachment + thumbs etc.2017-05-03T18:27:01.4499492017-05-03T18:27:01.449956https://pleroma.soykaf.com/contexts/9419f742-aaba-4eb5-89a2-8b599e8bf43chttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/sharehttps://pleroma.soykaf.com/activities/b6cc5d7c-0785-4785-a689-f1b05dc9b24dlain repeated a noticeRT <p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> Hey now!</p>2017-05-03T18:13:48.8910612017-05-03T18:13:48.891069https://pleroma.soykaf.com/contexts/ec6fdd27-0ec1-4672-8408-5a8e5a9c094bhttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posttag:mastodon.social,2017-05-01:objectId=4836142:objectType=StatusNew note by lambadalambda@mastodon.social<p><span class="h-card"><a href="https://pleroma.soykaf.com/users/lain" class="u-url mention">@<span>lain</span></a></span> Hey now!</p>2017-05-01T18:38:49.3653912017-05-03T18:13:48.934745https://pleroma.soykaf.com/contexts/ec6fdd27-0ec1-4672-8408-5a8e5a9c094bhttps://mastodon.social/users/lambadalambdahttp://activitystrea.ms/schema/1.0/personhttps://mastodon.social/users/lambadalambdalambadalambda@mastodon.socialCritical Valuenillambadalambda@mastodon.socialhttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/sharehttps://pleroma.soykaf.com/activities/3c09eb31-4ba8-4ff5-b4fa-8f6f74d58bf0lain repeated a noticeRT Haha, salmons from mastodon didn't work because it's not implementing conversation id...2017-05-03T18:13:15.1480412017-05-03T18:13:15.148049tag:social.heldscal.la,2017-05-01:objectType=thread:nonce=86cda6c734401d80http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posttag:social.heldscal.la,2017-05-01:noticeId=2000425:objectType=noteNew note by lambadalambda@social.heldscal.laHaha, salmons from mastodon didn't work because it's not implementing conversation id...2017-05-01T18:39:36.2163772017-05-03T18:13:15.171143tag:social.heldscal.la,2017-05-01:objectType=thread:nonce=86cda6c734401d80https://social.heldscal.la/user/23211http://activitystrea.ms/schema/1.0/personhttps://social.heldscal.la/user/23211lambadalambda@social.heldscal.laConstance Variablenillambadalambda@social.heldscal.lahttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/b8fc83d5-d7c0-4b5f-8976-0317b51935eaNew note by lain.<br><a href='https://pleroma.soykaf.com/media/563008a7-9a60-47ac-a263-22835729adf6/1492530528735.png'>https://pleroma.soykaf.com/media/563008a7-9a60-47ac-a263-22835729adf6/1492530528735.png</a>2017-05-03T18:12:50.7452412017-05-03T18:12:50.745249https://pleroma.soykaf.com/contexts/9419f742-aaba-4eb5-89a2-8b599e8bf43chttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/sharehttps://pleroma.soykaf.com/activities/ac93ecef-cde0-48e8-ae4b-19e3b94dbe30lain repeated a noticeRT Awright, which one of you hid my PENIS ENVY?2017-05-03T18:08:49.2310012017-05-03T18:08:49.235354https://pleroma.soykaf.com/contexts/a9132cf8-6afa-4dd8-8b29-7b6fcab623b8http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/04e15c66-4936-4930-a134-32841f088bcfNew note by fortuneAwright, which one of you hid my PENIS ENVY?2017-05-01T19:40:03.1699962017-05-03T18:08:49.285347https://pleroma.soykaf.com/contexts/a9132cf8-6afa-4dd8-8b29-7b6fcab623b8https://pleroma.soykaf.com/users/fortunehttp://activitystrea.ms/schema/1.0/personhttps://pleroma.soykaf.com/users/fortunefortunefortuneThe trusty unix fortune filefortunehttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/sharehttps://pleroma.soykaf.com/activities/54b10fa9-d602-4a0f-b659-e6d3f7bc8c4clain repeated a noticeRT He is a man capable of turning any colour into grey.<br> -- John LeCarre2017-05-03T17:44:47.5789842017-05-03T17:44:47.578996https://pleroma.soykaf.com/contexts/8aebc8e5-5352-4047-8b74-4098a5830ccahttp://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/70ded299-184d-49cd-af17-23c0950536aaNew note by fortuneHe is a man capable of turning any colour into grey.<br> -- John LeCarre2017-05-02T08:40:03.4194652017-05-03T17:44:47.646192https://pleroma.soykaf.com/contexts/8aebc8e5-5352-4047-8b74-4098a5830ccahttps://pleroma.soykaf.com/users/fortunehttp://activitystrea.ms/schema/1.0/personhttps://pleroma.soykaf.com/users/fortunefortunefortuneThe trusty unix fortune filefortunehttp://activitystrea.ms/schema/1.0/activityhttp://activitystrea.ms/schema/1.0/sharehttps://pleroma.soykaf.com/activities/eff9fe49-8fc9-48e6-a1a0-921aa25c8118lain repeated a noticeRT The real trouble with women is that they have *all* the pussy.2017-05-03T17:30:22.5960372017-05-03T17:30:22.596048https://pleroma.soykaf.com/contexts/8c88c9df-4e40-4f54-b15f-c21848d1a8e2http://activitystrea.ms/schema/1.0/notehttp://activitystrea.ms/schema/1.0/posthttps://pleroma.soykaf.com/objects/0b9b008d-49eb-48a9-a18d-172ce7d01ea2New note by fortuneThe real trouble with women is that they have *all* the pussy.2017-05-02T12:10:03.6030862017-05-03T17:30:22.683141https://pleroma.soykaf.com/contexts/8c88c9df-4e40-4f54-b15f-c21848d1a8e2https://pleroma.soykaf.com/users/fortunehttp://activitystrea.ms/schema/1.0/personhttps://pleroma.soykaf.com/users/fortunefortunefortuneThe trusty unix fortune filefortunehttp://activitystrea.ms/schema/1.0/favoritehttps://pleroma.soykaf.com/activities/5d90bb26-ce23-4a5b-8dbd-651011780007New favorite by lainlain favorited something2017-05-03T17:28:20.9679262017-05-03T17:28:20.967935http://activitystrea.ms/schema/1.0/notetag:mastodon.social,2017-05-03:objectId=4952899:objectType=Statushttps://pleroma.soykaf.com/contexts/42701ab4-964a-441a-a372-f51bd183e441 \ No newline at end of file diff --git a/test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml b/test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml new file mode 100644 index 000000000..6cba5c28f --- /dev/null +++ b/test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml @@ -0,0 +1,591 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-05-05T12:01:21+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:comment:2063249:2017-05-05T11:40:21+00:00 + Favorite + lambadalambda favorited something by tatiana: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> they will start complaining about this, but won't come up with any solutions)</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T11:40:21+00:00 + 2017-05-05T11:40:21+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:social.weho.st,2017-05-05:objectId=172033:objectType=Status + New comment by tatiana + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> they will start complaining about this, but won't come up with any solutions)</p> + + + + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:comment:2063041:2017-05-05T11:27:28+00:00 + Favorite + lambadalambda favorited something by kat: @<a href="https://social.heldscal.la/lambadalambda" class="h-card mention" title="Constance Variable">lambadalambda</a> if the admin reading mine would delete a few it would be really useful in prioritising.  + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T11:27:28+00:00 + 2017-05-05T11:27:28+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:quitter.se,2017-05-05:noticeId=11807959:objectType=comment + New comment by kat + @<a href="https://social.heldscal.la/lambadalambda" class="h-card mention" title="Constance Variable">lambadalambda</a> if the admin reading mine would delete a few it would be really useful in prioritising.  + + + + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 + + + + + + + tag:social.heldscal.la,2017-05-05:noticeId=2062924:objectType=note + lambadalambda repeated a notice by nielsk + RT @nielsk @<a href="https://social.heldscal.la/user/23211" class="h-card u-url p-nickname mention" title="Constance Variable">lambadalambda</a> but there are soooo many, where should I start to read? + + http://activitystrea.ms/schema/1.0/share + 2017-05-05T11:09:37+00:00 + 2017-05-05T11:09:37+00:00 + + http://activitystrea.ms/schema/1.0/activity + tag:mastodon.social,2017-05-05:objectId=5024471:objectType=Status + + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> but there are soooo many, where should I start to read?</p> + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T11:05:18+00:00 + 2017-05-05T11:05:18+00:00 + + http://activitystrea.ms/schema/1.0/person + https://mastodon.social/users/nielsk + nielsk + Sysadmin by day and ehm… sysadmin by night. Besides that old video games, Japan, economics and some other stuff + + + + + + nielsk + nielsk + Sysadmin by day and ehm… sysadmin by night. Besides that old video games, Japan, economics and some other stuff + + + + http://activitystrea.ms/schema/1.0/comment + tag:mastodon.social,2017-05-05:objectId=5024471:objectType=Status + New comment by nielsk + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> but there are soooo many, where should I start to read?</p> + + + + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 + + + + https://mastodon.social/users/nielsk.atom + nielsk + + + https://social.heldscal.la/avatar/29849-96-20170428120041.jpeg + 2017-05-05T11:06:32+00:00 + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:comment:2062875:2017-05-05T11:09:27+00:00 + Favorite + lambadalambda favorited something by nielsk: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> but there are soooo many, where should I start to read?</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T11:09:27+00:00 + 2017-05-05T11:09:27+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:mastodon.social,2017-05-05:objectId=5024471:objectType=Status + New comment by nielsk + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> but there are soooo many, where should I start to read?</p> + + + + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:comment:2062863:2017-05-05T11:09:11+00:00 + Favorite + lambadalambda favorited something by kasil: <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> surely, google is not that evil !</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T11:09:11+00:00 + 2017-05-05T11:09:11+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:loutre.info,2017-05-05:objectId=23331:objectType=Status + New comment by kasil + <p><span class="h-card"><a href="https://social.heldscal.la/lambadalambda" class="u-url mention">@<span>lambadalambda</span></a></span> surely, google is not that evil !</p> + + + + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-05:noticeId=2062767:objectType=comment + New comment by lambadalambda + @<a href="https://sealion.club/user/4" class="h-card u-url p-nickname mention" title="dewoo &#x274E;">dwmatiz</a> dunno, probably. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T10:55:17+00:00 + 2017-05-05T10:55:17+00:00 + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 + + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-05:noticeId=2062705:objectType=comment + New comment by lambadalambda + @<a href="https://gs.smuglo.li/user/28250" class="h-card u-url p-nickname mention" title="Bricky">thatbrickster</a> I do it, too. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T10:48:12+00:00 + 2017-05-05T10:48:12+00:00 + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 + + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-05-05:noticeId=2062620:objectType=comment + New comment by lambadalambda + @<a href="https://social.tchncs.de/users/israuor" class="h-card u-url p-nickname mention" title="Israuor &#x2642;">israuor</a> @<a href="https://mastodon.gougere.fr/users/bortzmeyer" class="h-card u-url p-nickname mention" title="S. Bortzmeyer &#x2705;">bortzmeyer</a> so, 99%. 100% for 'normal' people. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T10:38:52+00:00 + 2017-05-05T10:38:52+00:00 + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 + + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-05-05:noticeId=2062583:objectType=note + New note by lambadalambda + I wonder what'll happen when people realize the admin at their mail hoster can read all their e-mails. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T10:35:45+00:00 + 2017-05-05T10:35:45+00:00 + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=e95b99adc050e198 + + + + + + + tag:social.heldscal.la,2017-05-05:subscription:23211:person:35708:2017-05-05T09:34:46+00:00 + Constance Variable (lambadalambda@social.heldscal.la)'s status on Friday, 05-May-2017 09:34:46 UTC + <a href="https://social.heldscal.la/lambadalambda">Constance Variable</a> started following <a href="https://mastodon.social/@milouse">milouse</a>. + + http://activitystrea.ms/schema/1.0/follow + 2017-05-05T09:34:46+00:00 + 2017-05-05T09:34:46+00:00 + + http://activitystrea.ms/schema/1.0/person + https://mastodon.social/users/milouse + milouse + #Scout leader #sgdf, interested in #openweb, #semanticweb, #privacy, #foss and #socialeconomy. 0xA714ECAC8C9CEE3D + + + + + + milouse + milouse + #Scout leader #sgdf, interested in #openweb, #semanticweb, #privacy, #foss and #socialeconomy. 0xA714ECAC8C9CEE3D + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=26ca19a355bb6135 + + + + + + + tag:social.heldscal.la,2017-05-05:noticeId=2061871:objectType=note + lambadalambda repeated a notice by safebot + RT @<a href="https://gs.smuglo.li/user/25857" class="h-card u-url p-nickname mention" title="safebot">safebot</a> #<span class="tag"><a href="https://social.heldscal.la/tag/cheers" rel="tag">cheers</a></span> <a href="https://gs.smuglo.li/attachment/456444" title="https://gs.smuglo.li/attachment/456444" rel="nofollow external noreferrer" class="attachment" id="attachment-432334">https://gs.smuglo.li/attachment/456444</a> + + http://activitystrea.ms/schema/1.0/share + 2017-05-05T09:16:17+00:00 + 2017-05-05T09:16:17+00:00 + + http://activitystrea.ms/schema/1.0/activity + tag:gs.smuglo.li,2017-05-05:noticeId=2188073:objectType=note + + #<span class="tag"><a href="https://gs.smuglo.li/tag/cheers" rel="tag">cheers</a></span> <a href="https://gs.smuglo.li/file/5099e73c83da778cd032a721e96880f99a868b712be2975d08238547a5ba06c7.jpg" title="https://gs.smuglo.li/file/5099e73c83da778cd032a721e96880f99a868b712be2975d08238547a5ba06c7.jpg" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/456444</a> + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T08:36:53+00:00 + 2017-05-05T08:36:53+00:00 + + http://activitystrea.ms/schema/1.0/person + https://gs.smuglo.li/user/25857 + safebot + + + + + + safebot + safebot + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.smuglo.li,2017-05-05:noticeId=2188073:objectType=note + New note by safebot + #<span class="tag"><a href="https://gs.smuglo.li/tag/cheers" rel="tag">cheers</a></span> <a href="https://gs.smuglo.li/file/5099e73c83da778cd032a721e96880f99a868b712be2975d08238547a5ba06c7.jpg" title="https://gs.smuglo.li/file/5099e73c83da778cd032a721e96880f99a868b712be2975d08238547a5ba06c7.jpg" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/456444</a> + + + + + https://gs.smuglo.li/conversation/1009429 + + + + https://gs.smuglo.li/api/statuses/user_timeline/25857.atom + safebot + + + https://social.heldscal.la/avatar/25719-original-20161215233234.jpeg + 2017-05-05T12:00:57+00:00 + + + + https://gs.smuglo.li/conversation/1009429 + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061643:2017-05-05T09:12:50+00:00 + Favorite + lambadalambda favorited something by moonman: @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T09:12:50+00:00 + 2017-05-05T09:12:50+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment + New comment by moonman + @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English + + + + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=55ead90125cd4bd4 + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061696:2017-05-05T09:06:10+00:00 + Favorite + lambadalambda favorited something by moonman: @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> <br /> <span class="greentext">&gt; (((common era)))</span> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T09:06:10+00:00 + 2017-05-05T09:06:10+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2827918:objectType=comment + New comment by moonman + @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> <br /> <span class="greentext">&gt; (((common era)))</span> + + + + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=55ead90125cd4bd4 + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:note:2061673:2017-05-05T08:58:28+00:00 + Favorite + lambadalambda favorited something by moonman: discussion is one thing but any argument I've heard over and over again for the last three decades is going to go unanswered. + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T08:58:28+00:00 + 2017-05-05T08:58:28+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:shitposter.club,2017-05-05:noticeId=2827895:objectType=note + New note by moonman + discussion is one thing but any argument I've heard over and over again for the last three decades is going to go unanswered. + + + + + + + https://shitposter.club/conversation/1390494 + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061280:2017-05-05T08:47:38+00:00 + Favorite + lambadalambda favorited something by moonman: @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> sex is for procreation and as an expression of intimacy between commited couples, it is a sacramental act + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T08:47:38+00:00 + 2017-05-05T08:47:38+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2827561:objectType=comment + New comment by moonman + @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> sex is for procreation and as an expression of intimacy between commited couples, it is a sacramental act + + + + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=55ead90125cd4bd4 + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:note:2061535:2017-05-05T08:40:55+00:00 + Favorite + lambadalambda favorited something by fortune: What did Mickey Mouse get for Christmas?<br /> <br /> A Dan Quayle watch.<br /> <br /> -- heard from a Mike Dukakis field worker + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T08:40:55+00:00 + 2017-05-05T08:40:55+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-05-05:noticeId=2061535:objectType=note + New note by fortune + What did Mickey Mouse get for Christmas?<br /> <br /> A Dan Quayle watch.<br /> <br /> -- heard from a Mike Dukakis field worker + + + + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=5185e5c145ee4762 + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061421:2017-05-05T08:36:27+00:00 + Favorite + lambadalambda favorited something by moonman: @<a href="https://maly.io/users/sonya" class="h-card mention" title="Sonya Mann ✅">sonya</a> banned from 4chan. you better watch ou. i'm trouble, y'hear? + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T08:36:27+00:00 + 2017-05-05T08:36:27+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2827689:objectType=comment + New comment by moonman + @<a href="https://maly.io/users/sonya" class="h-card mention" title="Sonya Mann ✅">sonya</a> banned from 4chan. you better watch ou. i'm trouble, y'hear? + + + + + + + https://shitposter.club/conversation/1389345 + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061351:2017-05-05T08:28:03+00:00 + Favorite + lambadalambda favorited something by moonman: @<a href="https://social.heldscal.la/user/29138" class="h-card mention" title="Claes Wallin (韋嘉誠)">clacke</a> is that the sequel to Time Crisis + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T08:28:03+00:00 + 2017-05-05T08:28:03+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2827630:objectType=comment + New comment by moonman + @<a href="https://social.heldscal.la/user/29138" class="h-card mention" title="Claes Wallin (韋嘉誠)">clacke</a> is that the sequel to Time Crisis + + + + + + + https://shitposter.club/conversation/1385528 + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061339:2017-05-05T08:21:05+00:00 + Favorite + lambadalambda favorited something by hardbass2k8: @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> pretty sure it's money laundering + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T08:21:05+00:00 + 2017-05-05T08:21:05+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2827617:objectType=comment + New comment by hardbass2k8 + @<a href="https://social.heldscal.la/user/23211" class="h-card mention" title="Constance Variable">lambadalambda</a> pretty sure it's money laundering + + + + + + + https://shitposter.club/conversation/1387523 + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-05-05:noticeId=2061303:objectType=note + New note by lambadalambda + It's got tattoos, it's got a pierced hood<br /> It's got generation X<br /> It's got lesbians, and vitriol<br /> And sadomasochistic latex sex<br /> It's got Mighty Morphin' power brokers<br /> And Tanya Harding nude<br /> Macrobiotic lacto-vegan non-confrontational free range food<br /> It's got the handshake, peace talk, non-aggression pact<br /> A multicultural integration of segregated historical facts<br /> <br /> #<span class="tag"><a href="https://social.heldscal.la/tag/nsfw" rel="tag">nsfw</a></span> <a href="https://social.heldscal.la/file/61c13b99c92f40ec4865e7a3830da340b187e3de70d94b8da38fd2138bbede3a.jpg" title="https://social.heldscal.la/file/61c13b99c92f40ec4865e7a3830da340b187e3de70d94b8da38fd2138bbede3a.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-432199">https://social.heldscal.la/attachment/432199</a> <a href="https://social.heldscal.la/file/a88bba1a324da68ee2cfdbcd1c4cde60bd9553298244d6f81731270b71aa80df.jpg" title="https://social.heldscal.la/file/a88bba1a324da68ee2cfdbcd1c4cde60bd9553298244d6f81731270b71aa80df.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-432200">https://social.heldscal.la/attachment/432200</a> <a href="https://social.heldscal.la/file/887329a303250e73dc2eea06b1f0512fcac4b9d1b534068f03c45f00d5b21c39.jpg" title="https://social.heldscal.la/file/887329a303250e73dc2eea06b1f0512fcac4b9d1b534068f03c45f00d5b21c39.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-432201">https://social.heldscal.la/attachment/432201</a> <a href="https://social.heldscal.la/file/6d7a1ec15c1368c4c68810434d24da528606fcbccdd1da97b25affafeeb6ffda.jpg" title="https://social.heldscal.la/file/6d7a1ec15c1368c4c68810434d24da528606fcbccdd1da97b25affafeeb6ffda.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-432202">https://social.heldscal.la/attachment/432202</a> <a href="https://social.heldscal.la/file/2f55f2bb028eb9be744cc82b35a6b86b496d8c3924c700aff55a872ff11df54c.jpg" title="https://social.heldscal.la/file/2f55f2bb028eb9be744cc82b35a6b86b496d8c3924c700aff55a872ff11df54c.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-432203">https://social.heldscal.la/attachment/432203</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T08:17:08+00:00 + 2017-05-05T08:17:08+00:00 + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=bb6f4343036970e8 + + + + + + + + + + + + diff --git a/test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml b/test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml new file mode 100644 index 000000000..f70fbc695 --- /dev/null +++ b/test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml @@ -0,0 +1,719 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/29191.atom + shp timeline + Updates from shp on social.heldscal.la! + https://social.heldscal.la/avatar/29191-96-20170421154949.jpeg + 2017-05-05T11:57:06+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/29191 + shp + cofe + + + + + + shp + shp + cofe + + cofe + + + + + + + + + + + + + + tag:social.heldscal.la,2017-04-29:noticeId=1967657:objectType=note + shp repeated a notice by lain + RT @<a href="https://social.heldscal.la/user/37181" class="h-card u-url p-nickname mention" title="Lain Iwakura">lain</a> @<a href="https://social.heldscal.la/user/29191" class="h-card u-url p-nickname mention" title="shp">shp</a> @<a href="https://social.heldscal.la/user/23211" class="h-card u-url p-nickname mention">lambadalambda</a> cofe. + + http://activitystrea.ms/schema/1.0/share + 2017-04-29T18:19:34+00:00 + 2017-04-29T18:19:34+00:00 + + http://activitystrea.ms/schema/1.0/activity + https://pleroma.soykaf.com/activities/43d12c05-db3f-4f3d-bee1-d676f264490c + + <a href="https://pleroma.soykaf.com/users/shp">@shp</a> <a href="https://social.heldscal.la/user/23211">@lambadalambda@social.heldscal.la</a> cofe. + + http://activitystrea.ms/schema/1.0/post + 2017-04-29T18:14:36+00:00 + 2017-04-29T18:14:36+00:00 + + http://activitystrea.ms/schema/1.0/person + https://pleroma.soykaf.com/users/lain + lain + Test account + + + + + + lain + Lain Iwakura + Test account + + + + http://activitystrea.ms/schema/1.0/note + https://pleroma.soykaf.com/activities/43d12c05-db3f-4f3d-bee1-d676f264490c + New note by lain + <a href="https://pleroma.soykaf.com/users/shp">@shp</a> <a href="https://social.heldscal.la/user/23211">@lambadalambda@social.heldscal.la</a> cofe. + + + + + tag:social.heldscal.la,2017-04-29:objectType=thread:nonce=e0b75431888efdab + + + https://pleroma.soykaf.com/users/lain/feed.atom + Lain Iwakura + + + https://social.heldscal.la/avatar/43188-96-20170429172422.jpeg + 2017-05-05T08:38:03+00:00 + + + + tag:social.heldscal.la,2017-04-29:objectType=thread:nonce=e0b75431888efdab + + + + + + + tag:social.heldscal.la,2017-04-27:subscription:29191:person:29558:2017-04-27T17:26:37+00:00 + shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:26:37 UTC + <a href="https://social.heldscal.la/shp">shp</a> started following <a href="https://gs.smuglo.li/kfist">KFist</a>. + + http://activitystrea.ms/schema/1.0/follow + 2017-04-27T17:26:37+00:00 + 2017-04-27T17:26:37+00:00 + + http://activitystrea.ms/schema/1.0/person + https://gs.smuglo.li/user/28051 + KFist + I stream thanks to @nepfag. I also drink, shitpost, and fly planes. I visited Japan and it changed my life. Do you love your station? + + + + + + kfist + KFist + I stream thanks to @nepfag. I also drink, shitpost, and fly planes. I visited Japan and it changed my life. Do you love your station? + + homepage + http://smuglo.li:8000/stream.m3u + true + + + + tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=f766240d13ed9c2e + + + + + + + tag:social.heldscal.la,2017-04-27:noticeId=1933030:objectType=note + shp repeated a notice by shpbot + RT @<a href="https://gs.archae.me/user/4687" class="h-card u-url p-nickname mention" title="shpbot">shpbot</a> &gt;QuakeC + + http://activitystrea.ms/schema/1.0/share + 2017-04-27T17:21:10+00:00 + 2017-04-27T17:21:10+00:00 + + http://activitystrea.ms/schema/1.0/activity + tag:gs.archae.me,2017-04-27:noticeId=760881:objectType=note + + <span class='greentext'>&gt;QuakeC</span> + + http://activitystrea.ms/schema/1.0/post + 2017-04-27T17:15:13+00:00 + 2017-04-27T17:15:13+00:00 + + http://activitystrea.ms/schema/1.0/person + https://gs.archae.me/user/4687 + shpbot + + + + + + shpbot + shpbot + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.archae.me,2017-04-27:noticeId=760881:objectType=note + New note by shpbot + <span class='greentext'>&gt;QuakeC</span> + + + + + https://gs.archae.me/conversation/318362 + + + https://gs.archae.me/api/statuses/user_timeline/4687.atom + shpbot + + + https://social.heldscal.la/avatar/31581-original-20170405170019.jpeg + 2017-05-05T11:45:08+00:00 + + + + https://gs.archae.me/conversation/318362 + + + + + + + tag:social.heldscal.la,2017-04-27:subscription:29191:person:23226:2017-04-27T17:20:48+00:00 + shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:20:48 UTC + <a href="https://social.heldscal.la/shp">shp</a> started following <a href="http://quitter.se/taknamay">Internet Turtle Ⓐ 🏴 ✅</a>. + + http://activitystrea.ms/schema/1.0/follow + 2017-04-27T17:20:48+00:00 + 2017-04-27T17:20:48+00:00 + + http://activitystrea.ms/schema/1.0/person + http://quitter.se/user/115823 + Internet Turtle Ⓐ 🏴 ✅ + Scheme programmer, Novice esperantist, Spiritual naturalist - Will listen to your problems for free - XMPP: DarkDungeons94 at chatme.im + + + + + + taknamay + Internet Turtle Ⓐ 🏴 ✅ + Scheme programmer, Novice esperantist, Spiritual naturalist - Will listen to your problems for free - XMPP: DarkDungeons94 at chatme.im + + New Jersey, United States + + + homepage + https://quitter.se/taknamay + true + + + + tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=a66b1fb22020c152 + + + + + + + tag:social.heldscal.la,2017-04-27:subscription:29191:person:29302:2017-04-27T17:20:33+00:00 + shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:20:33 UTC + <a href="https://social.heldscal.la/shp">shp</a> started following <a href="https://icosahedron.website/@Trev">Chillidan Stormrave</a>. + + http://activitystrea.ms/schema/1.0/follow + 2017-04-27T17:20:33+00:00 + 2017-04-27T17:20:33+00:00 + + http://activitystrea.ms/schema/1.0/person + https://icosahedron.website/users/Trev + Trev Prime + web tech, music, ethics. radical individualist. kinda queer. love thy neighbor. always open for conversation. + + + + + + trev + Trev Prime + web tech, music, ethics. radical individualist. kinda queer. love thy neighbor. always open for conversation. + + + tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=781c05bd64ad9520 + + + + + + + tag:social.heldscal.la,2017-04-27:subscription:29191:person:29367:2017-04-27T17:20:27+00:00 + shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:20:27 UTC + <a href="https://social.heldscal.la/shp">shp</a> started following <a href="https://gs.kawa-kun.com/aya">射命丸 文</a>. + + http://activitystrea.ms/schema/1.0/follow + 2017-04-27T17:20:27+00:00 + 2017-04-27T17:20:27+00:00 + + http://activitystrea.ms/schema/1.0/person + https://gs.kawa-kun.com/user/4885 + 射命丸 文 + Traditional Reporter of Fantasy + + + + + + aya + 射命丸 文 + Traditional Reporter of Fantasy + + Gensōkyō + + + homepage + https://danbooru.donmai.us + true + + + + tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=5921da7a934e47ca + + + + + + + tag:social.heldscal.la,2017-04-27:subscription:29191:person:27773:2017-04-27T17:20:18+00:00 + shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:20:18 UTC + <a href="https://social.heldscal.la/shp">shp</a> started following <a href="https://gs.smuglo.li/japananon">JapanAnon</a>. + + http://activitystrea.ms/schema/1.0/follow + 2017-04-27T17:20:18+00:00 + 2017-04-27T17:20:18+00:00 + + http://activitystrea.ms/schema/1.0/person + https://gs.smuglo.li/user/27299 + JapanAnon + 匿名でしていてね！ + + + + + + japananon + JapanAnon + 匿名でしていてね！ + + ワイヤード + + + homepage + http://www.anonymous-japan.org + true + + + + tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=ae3d819865886cba + + + + + + + tag:social.heldscal.la,2017-04-27:subscription:29191:person:36560:2017-04-27T17:19:30+00:00 + shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:19:30 UTC + <a href="https://social.heldscal.la/shp">shp</a> started following <a href="https://shitposter.club/wareya">wareya</a>. + + http://activitystrea.ms/schema/1.0/follow + 2017-04-27T17:19:30+00:00 + 2017-04-27T17:19:30+00:00 + + http://activitystrea.ms/schema/1.0/person + https://shitposter.club/user/15439 + wareya + Who are you to defy such a perfect being that is the machine? 日本語難しいけど頑張るぜ github.com/wareya wareya.moe Short: reya or war, never "ware" + + + + + + wareya + wareya + Who are you to defy such a perfect being that is the machine? 日本語難しいけど頑張るぜ github.com/wareya wareya.moe Short: reya or war, never "ware" + + + tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=bd88a3cd20b5a418 + + + + + + + tag:social.heldscal.la,2017-04-27:subscription:29191:person:41176:2017-04-27T17:19:21+00:00 + shp (shp@social.heldscal.la)'s status on Thursday, 27-Apr-2017 17:19:21 UTC + <a href="https://social.heldscal.la/shp">shp</a> started following <a href="https://hakui.club/takeshitakenji">竹下憲二 (白)</a>. + + http://activitystrea.ms/schema/1.0/follow + 2017-04-27T17:19:21+00:00 + 2017-04-27T17:19:21+00:00 + + http://activitystrea.ms/schema/1.0/person + https://hakui.club/user/6 + 竹下憲二 (白) + Oh boy. + + + + + + takeshitakenji + 竹下憲二 (白) + Oh boy. + + Seattle, WA + + + homepage + http://gs.kawa-kun.com + true + + + + tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=b139a673deba6963 + + + + + + + tag:social.heldscal.la,2017-04-27:fave:29191:note:1932205:2017-04-27T17:17:46+00:00 + Favorite + shp favorited something by dolus: Looks like Merry is pussing out and caving to pressure. Sad. <a href="https://gs.smuglo.li/file/23e37de3c321248d3f322d8ec042372914568ab4c9431a94e568a61b8146587f.png" title="https://gs.smuglo.li/file/23e37de3c321248d3f322d8ec042372914568ab4c9431a94e568a61b8146587f.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432294</a> <a href="https://gs.smuglo.li/file/e5a9549a19986d59d51750090910f47c186787adf02b2b6ac58df37556887297.png" title="https://gs.smuglo.li/file/e5a9549a19986d59d51750090910f47c186787adf02b2b6ac58df37556887297.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432295</a> <a href="https://gs.smuglo.li/file/2fdfabbc8ab0b8dc135903a8c48c29b440d1f97446b98ced4ad14a54d3b5d41f.png" title="https://gs.smuglo.li/file/2fdfabbc8ab0b8dc135903a8c48c29b440d1f97446b98ced4ad14a54d3b5d41f.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432296</a> <a href="https://gs.smuglo.li/file/af605d7c6fe3a8c26c6d334c2a8e0005f7e86a266f14a5b3755e7d3ac4e226de.png" title="https://gs.smuglo.li/file/af605d7c6fe3a8c26c6d334c2a8e0005f7e86a266f14a5b3755e7d3ac4e226de.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432297</a> + + http://activitystrea.ms/schema/1.0/favorite + 2017-04-27T17:17:46+00:00 + 2017-04-27T17:17:46+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:gs.smuglo.li,2017-04-27:noticeId=2065465:objectType=note + New note by dolus + Looks like Merry is pussing out and caving to pressure. Sad. <a href="https://gs.smuglo.li/file/23e37de3c321248d3f322d8ec042372914568ab4c9431a94e568a61b8146587f.png" title="https://gs.smuglo.li/file/23e37de3c321248d3f322d8ec042372914568ab4c9431a94e568a61b8146587f.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432294</a> <a href="https://gs.smuglo.li/file/e5a9549a19986d59d51750090910f47c186787adf02b2b6ac58df37556887297.png" title="https://gs.smuglo.li/file/e5a9549a19986d59d51750090910f47c186787adf02b2b6ac58df37556887297.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432295</a> <a href="https://gs.smuglo.li/file/2fdfabbc8ab0b8dc135903a8c48c29b440d1f97446b98ced4ad14a54d3b5d41f.png" title="https://gs.smuglo.li/file/2fdfabbc8ab0b8dc135903a8c48c29b440d1f97446b98ced4ad14a54d3b5d41f.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432296</a> <a href="https://gs.smuglo.li/file/af605d7c6fe3a8c26c6d334c2a8e0005f7e86a266f14a5b3755e7d3ac4e226de.png" title="https://gs.smuglo.li/file/af605d7c6fe3a8c26c6d334c2a8e0005f7e86a266f14a5b3755e7d3ac4e226de.png" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432297</a> + + + + + + + https://gs.smuglo.li/conversation/927473 + + + + + + + tag:social.heldscal.la,2017-04-27:fave:29191:note:1932492:2017-04-27T17:13:55+00:00 + Favorite + shp favorited something by zemichi: <a href="https://gs.smuglo.li/file/1d45ea4ffc95f15037f361b56ad6b89f8451b70ad1ff7a03b7bb0345b8e2227c.jpg" title="https://gs.smuglo.li/file/1d45ea4ffc95f15037f361b56ad6b89f8451b70ad1ff7a03b7bb0345b8e2227c.jpg" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432344</a><br /> that's a lot of loli + + http://activitystrea.ms/schema/1.0/favorite + 2017-04-27T17:13:55+00:00 + 2017-04-27T17:13:55+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:gs.smuglo.li,2017-04-27:noticeId=2065713:objectType=note + New note by zemichi + <a href="https://gs.smuglo.li/file/1d45ea4ffc95f15037f361b56ad6b89f8451b70ad1ff7a03b7bb0345b8e2227c.jpg" title="https://gs.smuglo.li/file/1d45ea4ffc95f15037f361b56ad6b89f8451b70ad1ff7a03b7bb0345b8e2227c.jpg" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432344</a><br /> that's a lot of loli + + + + + + + https://gs.smuglo.li/conversation/927673 + + + + + + + tag:social.heldscal.la,2017-04-27:fave:29191:note:1932559:2017-04-27T17:12:46+00:00 + Favorite + shp favorited something by gsimg: <a href="https://gs.kawa-kun.com/file/3435c5cafda46f31cad5abb5837b3521b7b458198507073a496f4d10bad3633b.jpg" title="https://gs.kawa-kun.com/file/3435c5cafda46f31cad5abb5837b3521b7b458198507073a496f4d10bad3633b.jpg" rel="nofollow noreferrer" class="attachment">https://gs.kawa-kun.com/file/3435c5cafda46f31cad5abb5837b3521b7b458198507073a496f4d10bad3633b.jpg</a> #<span class="tag"><a href="https://gs.kawa-kun.com/tag/nsfw" rel="tag">nsfw</a></span> + + http://activitystrea.ms/schema/1.0/favorite + 2017-04-27T17:12:46+00:00 + 2017-04-27T17:12:46+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:gs.kawa-kun.com,2017-04-27:noticeId=1608309:objectType=note + New note by gsimg + <a href="https://gs.kawa-kun.com/file/3435c5cafda46f31cad5abb5837b3521b7b458198507073a496f4d10bad3633b.jpg" title="https://gs.kawa-kun.com/file/3435c5cafda46f31cad5abb5837b3521b7b458198507073a496f4d10bad3633b.jpg" rel="nofollow noreferrer" class="attachment">https://gs.kawa-kun.com/file/3435c5cafda46f31cad5abb5837b3521b7b458198507073a496f4d10bad3633b.jpg</a> #<span class="tag"><a href="https://gs.kawa-kun.com/tag/nsfw" rel="tag">nsfw</a></span> + + + + + + + https://gs.kawa-kun.com/conversation/690817 + + + + + + + tag:social.heldscal.la,2017-04-27:fave:29191:note:1932601:2017-04-27T17:12:28+00:00 + Favorite + shp favorited something by zemichi: <a href="https://gs.smuglo.li/file/5d9114fafea7b9866c9d852bcfeaf66aade65ae26149758346bc5ade7e3fa8f0.jpg" title="https://gs.smuglo.li/file/5d9114fafea7b9866c9d852bcfeaf66aade65ae26149758346bc5ade7e3fa8f0.jpg" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432372</a> + + http://activitystrea.ms/schema/1.0/favorite + 2017-04-27T17:12:28+00:00 + 2017-04-27T17:12:28+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:gs.smuglo.li,2017-04-27:noticeId=2065821:objectType=note + New note by zemichi + <a href="https://gs.smuglo.li/file/5d9114fafea7b9866c9d852bcfeaf66aade65ae26149758346bc5ade7e3fa8f0.jpg" title="https://gs.smuglo.li/file/5d9114fafea7b9866c9d852bcfeaf66aade65ae26149758346bc5ade7e3fa8f0.jpg" rel="nofollow noreferrer" class="attachment">https://gs.smuglo.li/attachment/432372</a> + + + + + + + https://gs.smuglo.li/conversation/927760 + + + + + + + tag:social.heldscal.la,2017-04-27:noticeId=1932867:objectType=note + shp repeated a notice by shpbot + RT @<a href="https://gs.archae.me/user/4687" class="h-card u-url p-nickname mention" title="shpbot">shpbot</a> <a href="https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg" title="https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-237676">https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg</a> #<span class="tag"><a href="https://social.heldscal.la/tag/2hu" rel="tag">2hu</a></span> #<span class="tag"><a href="https://social.heldscal.la/tag/ordinarymagician" rel="tag">ordinarymagician</a></span> :thinking: <a href="https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg" title="https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-312306">https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg</a> + + http://activitystrea.ms/schema/1.0/share + 2017-04-27T17:11:35+00:00 + 2017-04-27T17:11:35+00:00 + + http://activitystrea.ms/schema/1.0/activity + tag:gs.archae.me,2017-04-27:noticeId=760830:objectType=note + + <a href="https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg" title="https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg" rel="nofollow noreferrer" class="attachment">https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg</a> #<span class="tag"><a href="https://gs.archae.me/tag/2hu" rel="tag">2hu</a></span> #<span class="tag"><a href="https://gs.archae.me/tag/ordinarymagician" rel="tag">ordinarymagician</a></span> :thinking: <a href="https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg" title="https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg" rel="nofollow noreferrer" class="attachment">https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg</a> + + http://activitystrea.ms/schema/1.0/post + 2017-04-27T17:00:08+00:00 + 2017-04-27T17:00:08+00:00 + + http://activitystrea.ms/schema/1.0/person + https://gs.archae.me/user/4687 + shpbot + + + + + + shpbot + shpbot + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.archae.me,2017-04-27:noticeId=760830:objectType=note + New note by shpbot + <a href="https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg" title="https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg" rel="nofollow noreferrer" class="attachment">https://shitposter.club/file/cbf7fbbee1127a9870e871305ca7de70f1eb1bbb79ffe5b3b0f33e37514d14d8.jpg</a> #<span class="tag"><a href="https://gs.archae.me/tag/2hu" rel="tag">2hu</a></span> #<span class="tag"><a href="https://gs.archae.me/tag/ordinarymagician" rel="tag">ordinarymagician</a></span> :thinking: <a href="https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg" title="https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg" rel="nofollow noreferrer" class="attachment">https://shitposter.club/file/abf3f82d9ce28d2293d858af26c75bb5d4fdd091c0d90ca7bc72ea7efba220e4.jpg</a> + + + + + https://gs.archae.me/conversation/318317 + + + + + https://gs.archae.me/api/statuses/user_timeline/4687.atom + shpbot + + + https://social.heldscal.la/avatar/31581-original-20170405170019.jpeg + 2017-05-05T11:45:08+00:00 + + + + https://gs.archae.me/conversation/318317 + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-04-27:noticeId=1932815:objectType=note + New note by shp + federation issues with SPC atm it seems + + + http://activitystrea.ms/schema/1.0/post + 2017-04-27T17:08:55+00:00 + 2017-04-27T17:08:55+00:00 + + tag:social.heldscal.la,2017-04-27:objectType=thread:nonce=645a13c841f51769 + + + + + + + tag:social.heldscal.la,2017-04-26:fave:29191:note:1907285:2017-04-26T06:59:07+00:00 + Favorite + shp favorited something by lambadalambda: Is this the most offensive video on the net? <a href="https://social.heldscal.la/file/4c34bfb81a8155c265031bc48f7e69c29eb0d2941c57daf63f80e17b0e2e5f47.webm" title="https://social.heldscal.la/file/4c34bfb81a8155c265031bc48f7e69c29eb0d2941c57daf63f80e17b0e2e5f47.webm" rel="nofollow noreferrer" class="attachment">https://social.heldscal.la/attachment/402251</a> + + http://activitystrea.ms/schema/1.0/favorite + 2017-04-26T06:59:07+00:00 + 2017-04-26T06:59:07+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-04-26:noticeId=1907285:objectType=note + New note by lambadalambda + Is this the most offensive video on the net? <a href="https://social.heldscal.la/file/4c34bfb81a8155c265031bc48f7e69c29eb0d2941c57daf63f80e17b0e2e5f47.webm" title="https://social.heldscal.la/file/4c34bfb81a8155c265031bc48f7e69c29eb0d2941c57daf63f80e17b0e2e5f47.webm" rel="nofollow external noreferrer" class="attachment" id="attachment-402251">https://social.heldscal.la/attachment/402251</a> + + + + + + + tag:social.heldscal.la,2017-04-26:objectType=thread:nonce=07b02e1328f456af + + + + + + + tag:social.heldscal.la,2017-04-26:noticeId=1907951:objectType=note + shp repeated a notice by shpbot + RT @<a href="https://gs.archae.me/user/4687" class="h-card u-url p-nickname mention" title="shpbot">shpbot</a> <a href="https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg" title="https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg" rel="nofollow external noreferrer" class="attachment" id="attachment-346198">https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg</a> + + http://activitystrea.ms/schema/1.0/share + 2017-04-26T06:58:19+00:00 + 2017-04-26T06:58:19+00:00 + + http://activitystrea.ms/schema/1.0/activity + tag:gs.archae.me,2017-04-26:noticeId=752596:objectType=note + + <a href="https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg" title="https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg" rel="nofollow noreferrer" class="attachment">https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg</a> + + http://activitystrea.ms/schema/1.0/post + 2017-04-26T06:15:07+00:00 + 2017-04-26T06:15:07+00:00 + + http://activitystrea.ms/schema/1.0/person + https://gs.archae.me/user/4687 + shpbot + + + + + + shpbot + shpbot + + + + http://activitystrea.ms/schema/1.0/note + tag:gs.archae.me,2017-04-26:noticeId=752596:objectType=note + New note by shpbot + <a href="https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg" title="https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg" rel="nofollow noreferrer" class="attachment">https://shitposter.club/file/718db06b564841331c72f9df767f8c9459e20c4dddbf0d4e61cd08ecbee7739d.jpg</a> + + + + + https://gs.archae.me/conversation/314010 + + + https://gs.archae.me/api/statuses/user_timeline/4687.atom + shpbot + + + https://social.heldscal.la/avatar/31581-original-20170405170019.jpeg + 2017-05-05T11:45:08+00:00 + + + + https://gs.archae.me/conversation/314010 + + + + + + + tag:social.heldscal.la,2017-04-26:fave:29191:note:1907341:2017-04-26T06:58:16+00:00 + Favorite + shp favorited something by moonman: <a href="https://shitposter.club/file/1377b0894e983599c11e739e406243cabed9f8af7961a2550ecaf97e32de8e60.jpg" title="https://shitposter.club/file/1377b0894e983599c11e739e406243cabed9f8af7961a2550ecaf97e32de8e60.jpg" class="attachment" rel="nofollow">https://shitposter.club/attachment/630989</a> + + http://activitystrea.ms/schema/1.0/favorite + 2017-04-26T06:58:16+00:00 + 2017-04-26T06:58:16+00:00 + + http://activitystrea.ms/schema/1.0/note + tag:shitposter.club,2017-04-26:noticeId=2681941:objectType=note + New note by moonman + <a href="https://shitposter.club/file/1377b0894e983599c11e739e406243cabed9f8af7961a2550ecaf97e32de8e60.jpg" title="https://shitposter.club/file/1377b0894e983599c11e739e406243cabed9f8af7961a2550ecaf97e32de8e60.jpg" class="attachment" rel="nofollow">https://shitposter.club/attachment/630989</a> + + + + + + + https://shitposter.club/conversation/1300990 + + + + + + + tag:social.heldscal.la,2017-04-26:fave:29191:comment:1907412:2017-04-26T06:57:56+00:00 + Favorite + shp favorited something by lambadalambda: @<a href="https://gs.smuglo.li/user/2" class="h-card u-url p-nickname mention" title="nepfag">nepfag</a> <a href="https://cherubini.casa/why-i-shut-down-wizards-town-and-left-mastodon-6d4e631346b3?source=linkShare-89c2f851e979-1493184822&amp;gi=a6a47c5466a0" title="https://cherubini.casa/why-i-shut-down-wizards-town-and-left-mastodon-6d4e631346b3?source=linkShare-89c2f851e979-1493184822&amp;gi=a6a47c5466a0" rel="nofollow noreferrer" class="attachment">https://social.heldscal.la/url/402273</a> + + http://activitystrea.ms/schema/1.0/favorite + 2017-04-26T06:57:56+00:00 + 2017-04-26T06:57:56+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:social.heldscal.la,2017-04-26:noticeId=1907412:objectType=comment + New comment by lambadalambda + @<a href="https://gs.smuglo.li/user/2" class="h-card u-url p-nickname mention" title="nepfag">nepfag</a> <a href="https://cherubini.casa/why-i-shut-down-wizards-town-and-left-mastodon-6d4e631346b3?source=linkShare-89c2f851e979-1493184822&amp;gi=a6a47c5466a0" title="https://cherubini.casa/why-i-shut-down-wizards-town-and-left-mastodon-6d4e631346b3?source=linkShare-89c2f851e979-1493184822&amp;gi=a6a47c5466a0" rel="nofollow external noreferrer" class="attachment" id="attachment-402273">https://social.heldscal.la/url/402273</a> + + + + + + + tag:social.heldscal.la,2017-04-26:objectType=thread:nonce=85c21eda7aaa7259 + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:social.heldscal.la,2017-04-26:noticeId=1907942:objectType=note + New note by shp + #<span class="tag"><a href="https://social.heldscal.la/tag/cofe" rel="tag">cofe</a></span> time my friends <a href="https://social.heldscal.la/file/ec254b45b3a86ff74bc08bc7e065cb681d77cf7d4cedc9cdcf59e16adf311da3.png" title="https://social.heldscal.la/file/ec254b45b3a86ff74bc08bc7e065cb681d77cf7d4cedc9cdcf59e16adf311da3.png" rel="nofollow external noreferrer" class="attachment" id="attachment-402381">https://social.heldscal.la/attachment/402381</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-04-26T06:57:18+00:00 + 2017-04-26T06:57:18+00:00 + + tag:social.heldscal.la,2017-04-26:objectType=thread:nonce=9c9d9373bccfaf70 + + + + + + + + diff --git a/test/fixtures/httpoison_mock/https___social.heldscal.la_user_23211.xml b/test/fixtures/httpoison_mock/https___social.heldscal.la_user_23211.xml new file mode 100644 index 000000000..426a52939 --- /dev/null +++ b/test/fixtures/httpoison_mock/https___social.heldscal.la_user_23211.xml @@ -0,0 +1,20 @@ + + + https://social.heldscal.la/user/23211 + acct:lambadalambda@social.heldscal.la + https://social.heldscal.la/lambadalambda + https://social.heldscal.la/index.php/user/23211 + https://social.heldscal.la/index.php/lambadalambda + + + + + + + + + + + + + diff --git a/test/fixtures/httpoison_mock/https___social.heldscal.la_user_29191.xml b/test/fixtures/httpoison_mock/https___social.heldscal.la_user_29191.xml new file mode 100644 index 000000000..641103377 --- /dev/null +++ b/test/fixtures/httpoison_mock/https___social.heldscal.la_user_29191.xml @@ -0,0 +1,20 @@ + + + https://social.heldscal.la/user/29191 + acct:shp@social.heldscal.la + https://social.heldscal.la/shp + https://social.heldscal.la/index.php/user/29191 + https://social.heldscal.la/index.php/shp + + + + + + + + + + + + + diff --git a/test/fixtures/httpoison_mock/nonexistant@social.heldscal.la.xml b/test/fixtures/httpoison_mock/nonexistant@social.heldscal.la.xml new file mode 100644 index 000000000..2a61de8dd --- /dev/null +++ b/test/fixtures/httpoison_mock/nonexistant@social.heldscal.la.xml @@ -0,0 +1,90 @@ + + + + Internal Server Error - social.heldscal.la + + + + + + + + + + + + + + +
+ +
+
+
+ +
+
+
+ +
+ + diff --git a/test/fixtures/httpoison_mock/shp@social.heldscal.la.xml b/test/fixtures/httpoison_mock/shp@social.heldscal.la.xml new file mode 100644 index 000000000..4cde42e3f --- /dev/null +++ b/test/fixtures/httpoison_mock/shp@social.heldscal.la.xml @@ -0,0 +1,20 @@ + + + acct:shp@social.heldscal.la + https://social.heldscal.la/user/29191 + https://social.heldscal.la/shp + https://social.heldscal.la/index.php/user/29191 + https://social.heldscal.la/index.php/shp + + + + + + + + + + + + + diff --git a/test/support/httpoison_mock.ex b/test/support/httpoison_mock.ex new file mode 100644 index 000000000..0cb3b2691 --- /dev/null +++ b/test/support/httpoison_mock.ex @@ -0,0 +1,97 @@ +defmodule HTTPoisonMock do + alias HTTPoison.Response + + def get(url, body \\ [], headers \\ []) + + def get("https://social.heldscal.la/.well-known/webfinger", [Accept: "application/xrd+xml"], [params: [resource: "nonexistant@social.heldscal.la"]]) do + {:ok, %Response{ + status_code: 500, + body: File.read!("test/fixtures/httpoison_mock/nonexistant@social.heldscal.la.xml") + }} + end + + def get("https://social.heldscal.la/.well-known/webfinger", [Accept: "application/xrd+xml"], [params: [resource: "shp@social.heldscal.la"]]) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/shp@social.heldscal.la.xml") + }} + end + + def get("https://social.heldscal.la/.well-known/webfinger", [Accept: "application/xrd+xml"], [params: [resource: "https://social.heldscal.la/user/23211"]]) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/https___social.heldscal.la_user_23211.xml") + }} + end + + def get("https://social.heldscal.la/.well-known/webfinger", [Accept: "application/xrd+xml"], [params: [resource: "https://social.heldscal.la/user/29191"]]) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/https___social.heldscal.la_user_29191.xml") + }} + end + + def get("https://mastodon.social/.well-known/webfinger", [Accept: "application/xrd+xml"], [params: [resource: "https://mastodon.social/users/lambadalambda"]]) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.xml") + }} + end + + def get("http://gs.example.org/.well-known/webfinger", [Accept: "application/xrd+xml"], [params: [resource: "http://gs.example.org:4040/index.php/user/1"], follow_redirect: true]) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/http___gs.example.org_4040_index.php_user_1.xml") + }} + end + + def get("https://pleroma.soykaf.com/.well-known/webfinger", [Accept: "application/xrd+xml"], [params: [resource: "https://pleroma.soykaf.com/users/lain"]]) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain.xml") + }} + end + + def get("https://social.heldscal.la/api/statuses/user_timeline/29191.atom", _body, _headers) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_29191.atom.xml") + }} + end + + def get("https://social.heldscal.la/api/statuses/user_timeline/23211.atom", _body, _headers) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/https___social.heldscal.la_api_statuses_user_timeline_23211.atom.xml") + }} + end + + def get("https://mastodon.social/users/lambadalambda.atom", _body, _headers) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/https___mastodon.social_users_lambadalambda.atom") + }} + end + + def get("https://pleroma.soykaf.com/users/lain/feed.atom", _body, _headers) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/https___pleroma.soykaf.com_users_lain_feed.atom.xml") + }} + end + + def get("http://gs.example.org/index.php/api/statuses/user_timeline/1.atom", _body, _headers) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/http__gs.example.org_index.php_api_statuses_user_timeline_1.atom.xml") + }} + end + + def get(url, body, headers) do + {:error, "Not implemented the mock response for get #{inspect(url)}"} + end + + def post(url, body, headers) do + {:error, "Not implemented the mock response for post #{inspect(url)}"} + end +end From fa2610c7959500c18b64d70828123a817de78864 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 5 May 2017 16:07:44 +0200 Subject: [PATCH 092/107] Handle incoming favorites, fetch potentially missing messages. --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +- lib/pleroma/web/ostatus/ostatus.ex | 48 ++ test/fixtures/favorite.xml | 65 ++ ...er.club_api_statuses_show_2827873.atom.xml | 54 ++ ...club_api_statuses_user_timeline_1.atom.xml | 454 ++++++++++++ ...ttps___shitposter.club_notice_2827873.html | 653 ++++++++++++++++++ .../https___shitposter.club_user_1.xml | 20 + test/support/httpoison_mock.ex | 28 + test/web/ostatus/ostatus_test.exs | 24 + 9 files changed, 1348 insertions(+), 2 deletions(-) create mode 100644 test/fixtures/favorite.xml create mode 100644 test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml create mode 100644 test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml create mode 100644 test/fixtures/httpoison_mock/https___shitposter.club_notice_2827873.html create mode 100644 test/fixtures/httpoison_mock/https___shitposter.club_user_1.xml diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index d7b490088..82aed7ce4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -41,7 +41,7 @@ def create(to, actor, context, object, additional \\ %{}, published \\ nil, loca end end - def like(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do + def like(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object, local \\ true) do cond do # There's already a like here, so return the original activity. ap_id in (object.data["likes"] || []) -> @@ -59,7 +59,7 @@ def like(%User{ap_id: ap_id} = user, %Object{data: %{"id" => id}} = object) do "context" => object.data["context"] } - {:ok, activity} = insert(data) + {:ok, activity} = insert(data, local) likes = [ap_id | (object.data["likes"] || [])] |> Enum.uniq diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 3bed5a5d9..0a4361393 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -1,4 +1,6 @@ defmodule Pleroma.Web.OStatus do + @httpoison Application.get_env(:pleroma, :httpoison) + import Ecto.Query import Pleroma.Web.XML require Logger @@ -30,6 +32,8 @@ def handle_incoming(xml_string) do case verb do 'http://activitystrea.ms/schema/1.0/share' -> with {:ok, activity, retweeted_activity} <- handle_share(entry, doc), do: [activity, retweeted_activity] + 'http://activitystrea.ms/schema/1.0/favorite' -> + with {:ok, activity, favorited_activity} <- handle_favorite(entry, doc), do: [activity, favorited_activity] _ -> case object_type do 'http://activitystrea.ms/schema/1.0/note' -> @@ -63,6 +67,24 @@ def handle_share(entry, doc) do end end + def make_favorite(_entry, doc, favorited_activity) do + with {:ok, actor} <- find_make_or_update_user(doc), + %Object{} = object <- Object.get_cached_by_ap_id(favorited_activity.data["object"]["id"]), + {:ok, activity, _object} = ActivityPub.like(actor, object, false) do + {:ok, activity} + end + end + + def handle_favorite(entry, doc) do + with href when not is_nil(href) <- string_from_xpath("//activity:object[1]/link[@type=\"text/html\"]/@href", entry), + {:ok, [favorited_activity]} <- fetch_activity_from_html_url(href), + {:ok, activity} <- make_favorite(entry, doc, favorited_activity) do + {:ok, activity, favorited_activity} + else + e -> {:error, e} + end + end + def get_attachments(entry) do :xmerl_xpath.string('/entry/link[@rel="enclosure"]', entry) |> Enum.map(fn (enclosure) -> @@ -210,4 +232,30 @@ def gather_user_info(username) do {:error, e} end end + + # Regex-based 'parsing' so we don't have to pull in a full html parser + # It's a hack anyway. Maybe revisit this in the future + @mastodon_regex ~r// + @gs_regex ~r// + def get_atom_url(body) do + cond do + Regex.match?(@mastodon_regex, body) -> + [[_, match]] = Regex.scan(@mastodon_regex, body) + {:ok, match} + Regex.match?(@gs_regex, body) -> + [[_, match]] = Regex.scan(@gs_regex, body) + {:ok, match} + true -> + Logger.debug(fn -> "Couldn't find atom link in #{inspect(body)}" end) + {:error, "Couldn't find the atom link"} + end + end + + def fetch_activity_from_html_url(url) do + with {:ok, %{body: body}} <- @httpoison.get(url), + {:ok, atom_url} <- get_atom_url(body), + {:ok, %{status_code: code, body: body}} when code in 200..299 <- @httpoison.get(atom_url) do + handle_incoming(body) + end + end end diff --git a/test/fixtures/favorite.xml b/test/fixtures/favorite.xml new file mode 100644 index 000000000..c32b4a403 --- /dev/null +++ b/test/fixtures/favorite.xml @@ -0,0 +1,65 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-05-05T09:12:53+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061643:2017-05-05T09:12:50+00:00 + Favorite + lambadalambda favorited something by moonman: @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T09:12:50+00:00 + 2017-05-05T09:12:50+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment + New comment by moonman + @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English + + + + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=55ead90125cd4bd4 + + + + + + diff --git a/test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml b/test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml new file mode 100644 index 000000000..26fdebb49 --- /dev/null +++ b/test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml @@ -0,0 +1,54 @@ + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment + New comment by moonman + @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T08:51:48+00:00 + 2017-05-05T08:51:48+00:00 + + http://activitystrea.ms/schema/1.0/person + https://shitposter.club/user/1 + moonman + EMAIL:shitposterclub@gmail.com XMPP: moon@talk.shitposter.club Matrix Ed25519 fingerprint: 2HuDUTEz3iFN5N3xl6PYp9xZW/EWhgbbt78SrFy4w8o + + + + + + moonman + Generic Enemy + EMAIL:shitposterclub@gmail.com XMPP: moon@talk.shitposter.club Matrix Ed25519 fingerprint: 2HuDUTEz3iFN5N3xl6PYp9xZW/EWhgbbt78SrFy4w8o + + The Moon + + + homepage + https://shitposter.club/moonman + true + + + + + + + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26 + + + + + https://shitposter.club/api/statuses/user_timeline/1.atom + Generic Enemy + + + + https://shitposter.club/avatar/1-96-20170503024316.jpeg + 2017-05-05T11:43:58+00:00 + + + + + diff --git a/test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml b/test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml new file mode 100644 index 000000000..31df7c2a6 --- /dev/null +++ b/test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml @@ -0,0 +1,454 @@ + + + GNU social + https://shitposter.club/api/statuses/user_timeline/1.atom + moonman timeline + Updates from moonman on Shitposter Club! + https://shitposter.club/avatar/1-96-20170503024316.jpeg + 2017-05-05T13:24:09+00:00 + + http://activitystrea.ms/schema/1.0/person + https://shitposter.club/user/1 + moonman + EMAIL:shitposterclub@gmail.com XMPP: moon@talk.shitposter.club Matrix Ed25519 fingerprint: 2HuDUTEz3iFN5N3xl6PYp9xZW/EWhgbbt78SrFy4w8o + + + + + + moonman + Generic Enemy + EMAIL:shitposterclub@gmail.com XMPP: moon@talk.shitposter.club Matrix Ed25519 fingerprint: 2HuDUTEz3iFN5N3xl6PYp9xZW/EWhgbbt78SrFy4w8o + + The Moon + + + homepage + https://shitposter.club/moonman + true + + + + + + + + + + + + + + tag:shitposter.club,2017-05-05:subscription:1:person:23190:2017-05-05T11:43:58+00:00 + Generic Enemy (moonman)'s status on Friday, 05-May-2017 11:43:58 UTC + <a href="https://shitposter.club/moonman">Generic Enemy</a> started following <a href="https://noagendasocial.com/@Ma5on">Mason</a>. + + http://activitystrea.ms/schema/1.0/follow + 2017-05-05T11:43:58+00:00 + 2017-05-05T11:43:58+00:00 + + http://activitystrea.ms/schema/1.0/person + https://noagendasocial.com/users/Ma5on + Mason + + + + + + ma5on + Mason + + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=abffa9c14a054d3b + + + + + + + tag:shitposter.club,2017-05-05:subscription:1:person:14357:2017-05-05T10:29:03+00:00 + Generic Enemy (moonman)'s status on Friday, 05-May-2017 10:29:03 UTC + <a href="https://shitposter.club/moonman">Generic Enemy</a> started following <a href="https://mastodon.cloud/@ohyran">Jens Reuterberg</a>. + + http://activitystrea.ms/schema/1.0/follow + 2017-05-05T10:29:03+00:00 + 2017-05-05T10:29:03+00:00 + + http://activitystrea.ms/schema/1.0/person + https://mastodon.cloud/users/ohyran + Jens Reuterberg + RPG-nerd, illustrator, Open Source enthusiast, KDE dude, designer and gay lefty. Might be a cliché - but we will soon find out! + + + + + + ohyran + Jens Reuterberg + RPG-nerd, illustrator, Open Source enthusiast, KDE dude, designer and gay lefty. Might be a cliché - but we will soon find out! + + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=937151d4825a85bf + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:shitposter.club,2017-05-05:noticeId=2828637:objectType=note + New note by moonman + basicall i would just rather have ppl say &quot;i like x and y&quot; than &quot;i'm a nerd&quot; the term can be retired. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T10:24:54+00:00 + 2017-05-05T10:24:54+00:00 + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=65992b0b9b5e6931 + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2828579:objectType=comment + New comment by moonman + @<a href="https://gs.smuglo.li/user/35497" class="h-card mention" title="Bokuro Bokusawa">boco</a> to be honest i've turned right around and been cruel to other people, i said i'd never do it but it happens again eventually. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T10:20:33+00:00 + 2017-05-05T10:20:33+00:00 + + + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=c997fc73d7f8a8f0 + + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2828554:objectType=comment + New comment by moonman + @<a href="https://mastodon.cloud/users/ohyran" class="h-card mention" title="Jens Reuterberg">ohyran</a> i won't ever get over bullying but i agree otherwise. i don't go to comic shops too often these days but i got dragged to one last year and the sheer diversity of people enjoying comics now compared to years ago was striking and it pleased me. and i noticed a couple years ago because of youtube i find things i truly enjoy watching, like in-depth videos about electronic parts, didn't exist 20 years ago. it's pretty great. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T10:18:10+00:00 + 2017-05-05T10:18:10+00:00 + + + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=efae3a23b6e05767 + + + + + + + + tag:shitposter.club,2017-05-05:fave:1:comment:2828502:2017-05-05T10:12:52+00:00 + Favorite + moonman favorited something by ohyran: <p><span class="h-card"><a href="https://shitposter.club/moonman" class="u-url mention">@<span>moonman</span></a></span> fair enough - that distinction makes it clearer...</p><p>On the other hand - those of us who did "pay the price" of being nerdy little kids in the 80's and 90's should strive to get past it anyway (mental health wise not "just get over it") and see the "nerd culture" thing as a blessing of sorts. We are in the optimal spot to do it. (not saying that that is something easy btw just that NOW is the best of time to start talking about it)</p> + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T10:12:52+00:00 + 2017-05-05T10:12:52+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:mastodon.cloud,2017-05-05:objectId=6334570:objectType=Status + New comment by ohyran + <p><span class="h-card"><a href="https://shitposter.club/moonman" class="u-url mention">@<span>moonman</span></a></span> fair enough - that distinction makes it clearer...</p><p>On the other hand - those of us who did "pay the price" of being nerdy little kids in the 80's and 90's should strive to get past it anyway (mental health wise not "just get over it") and see the "nerd culture" thing as a blessing of sorts. We are in the optimal spot to do it. (not saying that that is something easy btw just that NOW is the best of time to start talking about it)</p> + + + + + + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=efae3a23b6e05767 + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:shitposter.club,2017-05-05:noticeId=2828496:objectType=note + New note by moonman + things are better now, a lot less kids in america get beaten up and called a fag. still too many. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T10:11:31+00:00 + 2017-05-05T10:11:31+00:00 + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=c997fc73d7f8a8f0 + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2828457:objectType=comment + New comment by moonman + @<a href="https://shitposter.club/user/21787" class="h-card mention" title="Yukari">cutscenes</a> @<a href="https://gs.smuglo.li/user/28250" class="h-card mention" title="Bricky">thatbrickster</a> @<a href="https://gs.smuglo.li/user/35497" class="h-card mention" title="Bokuro Bokusawa">boco</a> i never understood this because nerds had pocket protectors, which was a draftsman engineer thing and therefore smart, while geeks were people in carnivals who bit heads off small animals. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T10:07:57+00:00 + 2017-05-05T10:07:57+00:00 + + + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=efae3a23b6e05767 + + + + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2828435:objectType=comment + New comment by moonman + @<a href="https://mastodon.cloud/users/ohyran" class="h-card mention" title="Jens Reuterberg">ohyran</a> since i didn't specify i'm talking about people subjected to physical and psychological abuse and not people that are just mad that more people like comic books now. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T10:05:07+00:00 + 2017-05-05T10:05:07+00:00 + + + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=efae3a23b6e05767 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:shitposter.club,2017-05-05:noticeId=2828326:objectType=note + New note by moonman + if you were a &quot;nerd&quot; before, like, 2001 you have permanent excuse to hate this kind of shit.   <a href="https://shitposter.club/file/b79fa5644be0d6f22679136e67b7bf45c9c4a74a55c32dd2d0cf15de4ddd5be5.gif" title="https://shitposter.club/file/b79fa5644be0d6f22679136e67b7bf45c9c4a74a55c32dd2d0cf15de4ddd5be5.gif" class="attachment" id="attachment-662105" rel="nofollow external">https://shitposter.club/attachment/662105</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T09:47:42+00:00 + 2017-05-05T09:47:42+00:00 + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=efae3a23b6e05767 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:shitposter.club,2017-05-05:noticeId=2828250:objectType=note + New note by moonman + <a href="https://shitposter.club/file/1283e2d4dd8f96b8eeb5d9a16b318e210868aa11386cf0d593891e4c75c9126e.gif" title="https://shitposter.club/file/1283e2d4dd8f96b8eeb5d9a16b318e210868aa11386cf0d593891e4c75c9126e.gif" class="attachment" id="attachment-662098" rel="nofollow external">https://shitposter.club/attachment/662098</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T09:39:06+00:00 + 2017-05-05T09:39:06+00:00 + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=ea8ffae90546f0ab + + + + + + + + tag:shitposter.club,2017-05-05:fave:1:comment:2828161:2017-05-05T09:28:19+00:00 + Favorite + moonman favorited something by kro: @<a href="https://shitposter.club/user/1" class="h-card u-url p-nickname mention" title="Generic Enemy">moonman</a> Till Brooklyn? + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T09:28:19+00:00 + 2017-05-05T09:28:19+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:gs.smuglo.li,2017-05-05:noticeId=2188587:objectType=comment + New comment by kro + @<a href="https://shitposter.club/user/1" class="h-card u-url p-nickname mention" title="Generic Enemy">moonman</a> Till Brooklyn? + + + + + + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=d7aa6b5b057ca555 + + + + + + + tag:shitposter.club,2017-05-05:fave:1:comment:2828125:2017-05-05T09:24:56+00:00 + Favorite + moonman favorited something by hardbass2k8: this has obviously interesting implications in various places, for example:<br /> the nationalism of the nazis might not have been real, who would have thought?<br /> socialism is usually promoted to implementation by real douchebags!<br /> your local social justice people might want diversity but they don't want you, m/19, white, why?<br /> amateur soccer club, they want to be the best in the amateur league but actually they just get drunk after training and are 50% overweight.<br /> This is because humans are not capable of telepathy, so if you join a group it doesn't magically align every little bit of your being with the declared group goals.<br /> <br /> Even though you see unmanned group beliefs flying around from time to time, generally groups are created from a bunch of people. they are not a container for people, they are the people inside them.<br /> <br /> so if you see a group that appears to be cool don't think of it as cool because its goals are cool but because its members are cool. if they aren't, tough cookies. don't be the retard and end up on the camp watchtower. + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T09:24:56+00:00 + 2017-05-05T09:24:56+00:00 + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2828125:objectType=comment + New comment by hardbass2k8 + this has obviously interesting implications in various places, for example:<br /> the nationalism of the nazis might not have been real, who would have thought?<br /> socialism is usually promoted to implementation by real douchebags!<br /> your local social justice people might want diversity but they don't want you, m/19, white, why?<br /> amateur soccer club, they want to be the best in the amateur league but actually they just get drunk after training and are 50% overweight.<br /> This is because humans are not capable of telepathy, so if you join a group it doesn't magically align every little bit of your being with the declared group goals.<br /> <br /> Even though you see unmanned group beliefs flying around from time to time, generally groups are created from a bunch of people. they are not a container for people, they are the people inside them.<br /> <br /> so if you see a group that appears to be cool don't think of it as cool because its goals are cool but because its members are cool. if they aren't, tough cookies. don't be the retard and end up on the camp watchtower. + + + + + + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=51b227fe92f6babf + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:shitposter.club,2017-05-05:noticeId=2828128:objectType=note + New note by moonman + In a valid remake of They live, signs would say REBEL, and DON'T GET MARRIED AND HAVE KIDS + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T09:24:23+00:00 + 2017-05-05T09:24:23+00:00 + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=b74397fa766b82c9 + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:shitposter.club,2017-05-05:noticeId=2828104:objectType=note + New note by moonman + <a href="https://shitposter.club/file/4d34178bde99599f31a28928e1666fbd58448d8a22e94ed82222496e4a45cb07.gif" title="https://shitposter.club/file/4d34178bde99599f31a28928e1666fbd58448d8a22e94ed82222496e4a45cb07.gif" class="attachment" id="attachment-662049" rel="nofollow external">https://shitposter.club/attachment/662049</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T09:21:01+00:00 + 2017-05-05T09:21:01+00:00 + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=d7aa6b5b057ca555 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:shitposter.club,2017-05-05:noticeId=2828102:objectType=note + New note by moonman + when ppl find out i haven't always been serious  <a href="https://shitposter.club/file/5859fa95875342cc65dba0d852f726db158ce28198c326d5f13d9de7c0d2c449.gif" title="https://shitposter.club/file/5859fa95875342cc65dba0d852f726db158ce28198c326d5f13d9de7c0d2c449.gif" class="attachment" id="attachment-662053" rel="nofollow external">https://shitposter.club/attachment/662053</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T09:20:45+00:00 + 2017-05-05T09:20:45+00:00 + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=0a025ac5a570b4ec + + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2828086:objectType=comment + New comment by moonman + @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> @<a href="https://gs.smuglo.li/user/35497" class="h-card mention" title="Bokuro Bokusawa">boco</a> you are being too serious lol + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T09:17:19+00:00 + 2017-05-05T09:17:19+00:00 + + + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=3c16e9c2681f6d26 + + + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:shitposter.club,2017-05-05:noticeId=2828085:objectType=note + New note by moonman + shitposter dot club  <a href="https://shitposter.club/file/9b084c7210b16abbf4d28594b924a07ef4a2a06f89d901a4c42fb1e243291263.gif" title="https://shitposter.club/file/9b084c7210b16abbf4d28594b924a07ef4a2a06f89d901a4c42fb1e243291263.gif" class="attachment" id="attachment-662047" rel="nofollow external">https://shitposter.club/attachment/662047</a> + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T09:16:50+00:00 + 2017-05-05T09:16:50+00:00 + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=d1ae088a1b91e5e5 + + + + + + + + http://activitystrea.ms/schema/1.0/note + tag:shitposter.club,2017-05-05:noticeId=2828061:objectType=note + New note by moonman + even when i lie i tell the truth, is that so hard to understand? + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T09:15:07+00:00 + 2017-05-05T09:15:07+00:00 + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=a516e4b8506b8ef5 + + + + + + + http://activitystrea.ms/schema/1.0/comment + tag:shitposter.club,2017-05-05:noticeId=2828052:objectType=comment + New comment by moonman + @<a href="https://shitposter.club/user/9591" class="h-card mention" title="warum hei&#xDF;en deutschl&#xE4;nder deutschl&#xE4;nder">hardbass2k8</a> history, anthropology. + + + http://activitystrea.ms/schema/1.0/post + 2017-05-05T09:14:22+00:00 + 2017-05-05T09:14:22+00:00 + + + + tag:shitposter.club,2017-05-05:objectType=thread:nonce=fe4d7f35b13403ba + + + + + + + diff --git a/test/fixtures/httpoison_mock/https___shitposter.club_notice_2827873.html b/test/fixtures/httpoison_mock/https___shitposter.club_notice_2827873.html new file mode 100644 index 000000000..54745ef3d --- /dev/null +++ b/test/fixtures/httpoison_mock/https___shitposter.club_notice_2827873.html @@ -0,0 +1,653 @@ + + + + Shitposter Club + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
+ + +
+
+
+
+
Please enable javascript to use this site.
+
+
+
+
+
+
+ +
+ + + + +
+ · + + OpenID
+
+
+ +
+
+
+
+

+ +

+
+
+
+
+ +
+
+
    + + +
  1. + +
    + Generic Enemy (moonman)'s status on Friday, 05-May-2017 08:51:48 UTC + + Generic Enemy +Generic Enemy +
    + in reply to + +
    +
    +
    @neimzr4luzerz @dolus childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English
    + +     +
    2. +
+ +
+ + + + + + + + + + + + + + + + + + +
+ +
+ + + + + diff --git a/test/fixtures/httpoison_mock/https___shitposter.club_user_1.xml b/test/fixtures/httpoison_mock/https___shitposter.club_user_1.xml new file mode 100644 index 000000000..bf54c80c8 --- /dev/null +++ b/test/fixtures/httpoison_mock/https___shitposter.club_user_1.xml @@ -0,0 +1,20 @@ + + + https://shitposter.club/user/1 + acct:moonman@shitposter.club + https://shitposter.club/moonman + https://shitposter.club/index.php/user/1 + https://shitposter.club/index.php/moonman + + + + + + + + + + + + + diff --git a/test/support/httpoison_mock.ex b/test/support/httpoison_mock.ex index 0cb3b2691..733abced2 100644 --- a/test/support/httpoison_mock.ex +++ b/test/support/httpoison_mock.ex @@ -38,6 +38,13 @@ def get("https://mastodon.social/.well-known/webfinger", [Accept: "application/x }} end + def get("https://shitposter.club/.well-known/webfinger", [Accept: "application/xrd+xml"], [params: [resource: "https://shitposter.club/user/1"]]) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/https___shitposter.club_user_1.xml") + }} + end + def get("http://gs.example.org/.well-known/webfinger", [Accept: "application/xrd+xml"], [params: [resource: "http://gs.example.org:4040/index.php/user/1"], follow_redirect: true]) do {:ok, %Response{ status_code: 200, @@ -87,6 +94,27 @@ def get("http://gs.example.org/index.php/api/statuses/user_timeline/1.atom", _bo }} end + def get("https://shitposter.club/notice/2827873", _body, _headers) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/https___shitposter.club_notice_2827873.html") + }} + end + + def get("https://shitposter.club/api/statuses/show/2827873.atom", _body, _headers) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_show_2827873.atom.xml") + }} + end + + def get("https://shitposter.club/api/statuses/user_timeline/1.atom", _body, _headers) do + {:ok, %Response{ + status_code: 200, + body: File.read!("test/fixtures/httpoison_mock/https___shitposter.club_api_statuses_user_timeline_1.atom.xml") + }} + end + def get(url, body, headers) do {:error, "Not implemented the mock response for get #{inspect(url)}"} end diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index e85d7677c..7f168ee54 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -96,6 +96,20 @@ test "handle incoming retweets - Mastodon, salmon" do refute retweeted_activity.local end + test "handle incoming favorites - GS, websub" do + incoming = File.read!("test/fixtures/favorite.xml") + {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Like" + assert activity.data["actor"] == "https://social.heldscal.la/user/23211" + assert activity.data["object"] == retweeted_activity.data["object"]["id"] + refute activity.local + assert retweeted_activity.data["type"] == "Create" + assert retweeted_activity.data["actor"] == "https://shitposter.club/user/1" + assert retweeted_activity.data["object"]["id"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" + refute retweeted_activity.local + end + test "handle incoming replies" do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) @@ -191,4 +205,14 @@ test "it works with the uri" do assert data == expected end end + + describe "fetching a status by it's HTML url" do + test "it builds a missing status from an html url" do + url = "https://shitposter.club/notice/2827873" + {:ok, [activity] } = OStatus.fetch_activity_from_html_url(url) + + assert activity.data["actor"] == "https://shitposter.club/user/1" + assert activity.data["object"]["id"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" + end + end end From 2d9fdbcc0d41f00c9996962ea20d6ff58f0a32b8 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 5 May 2017 16:27:03 +0200 Subject: [PATCH 093/107] Don't call out if we have the favorited notice locally. --- lib/pleroma/web/ostatus/ostatus.ex | 17 +++++- test/fixtures/favorite_with_local_note.xml | 64 ++++++++++++++++++++++ test/web/ostatus/ostatus_test.exs | 29 ++++++++-- 3 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 test/fixtures/favorite_with_local_note.xml diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 0a4361393..5a44b8661 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.OStatus do import Pleroma.Web.XML require Logger - alias Pleroma.{Repo, User, Web, Object} + alias Pleroma.{Repo, User, Web, Object, Activity} alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.{WebFinger, Websub} @@ -75,9 +75,20 @@ def make_favorite(_entry, doc, favorited_activity) do end end + def get_or_try_fetching(entry) do + with id when not is_nil(id) <- string_from_xpath("//activity:object[1]/id", entry), + %Activity{} = activity <- Activity.get_create_activity_by_object_ap_id(id) do + {:ok, activity} + else _e -> + with href when not is_nil(href) <- string_from_xpath("//activity:object[1]/link[@type=\"text/html\"]/@href", entry), + {:ok, [favorited_activity]} <- fetch_activity_from_html_url(href) do + {:ok, favorited_activity} + end + end + end + def handle_favorite(entry, doc) do - with href when not is_nil(href) <- string_from_xpath("//activity:object[1]/link[@type=\"text/html\"]/@href", entry), - {:ok, [favorited_activity]} <- fetch_activity_from_html_url(href), + with {:ok, favorited_activity} <- get_or_try_fetching(entry), {:ok, activity} <- make_favorite(entry, doc, favorited_activity) do {:ok, activity, favorited_activity} else diff --git a/test/fixtures/favorite_with_local_note.xml b/test/fixtures/favorite_with_local_note.xml new file mode 100644 index 000000000..3c955607d --- /dev/null +++ b/test/fixtures/favorite_with_local_note.xml @@ -0,0 +1,64 @@ + + + GNU social + https://social.heldscal.la/api/statuses/user_timeline/23211.atom + lambadalambda timeline + Updates from lambadalambda on social.heldscal.la! + https://social.heldscal.la/avatar/23211-96-20170416114255.jpeg + 2017-05-05T09:12:53+00:00 + + http://activitystrea.ms/schema/1.0/person + https://social.heldscal.la/user/23211 + lambadalambda + Call me Deacon Blues. + + + + + + lambadalambda + Constance Variable + Call me Deacon Blues. + + Berlin + + + homepage + https://heldscal.la + true + + + + + + + + + + + + + tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061643:2017-05-05T09:12:50+00:00 + Favorite + lambadalambda favorited something by moonman: @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English + + http://activitystrea.ms/schema/1.0/favorite + 2017-05-05T09:12:50+00:00 + 2017-05-05T09:12:50+00:00 + + http://activitystrea.ms/schema/1.0/comment + localid + New comment by moonman + @<a href="https://shitposter.club/user/9655" class="h-card mention" title="Solidarity for Pigs">neimzr4luzerz</a> @<a href="https://gs.smuglo.li/user/2326" class="h-card mention" title="Dolus_McHonest">dolus</a> childhood poring over Strong's concordance and a koine Greek dictionary, fast forward to 2017 and some fuckstick who translates japanese jackoff material tells me you just need to make it sound right in English + + + + + + tag:social.heldscal.la,2017-05-05:objectType=thread:nonce=55ead90125cd4bd4 + + + + + + diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 7f168ee54..1b03c1157 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -3,6 +3,7 @@ defmodule Pleroma.Web.OStatusTest do alias Pleroma.Web.OStatus alias Pleroma.Web.XML alias Pleroma.{Object, Repo} + import Pleroma.Factory test "don't insert create notes twice" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") @@ -98,16 +99,32 @@ test "handle incoming retweets - Mastodon, salmon" do test "handle incoming favorites - GS, websub" do incoming = File.read!("test/fixtures/favorite.xml") - {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) + {:ok, [[activity, favorited_activity]]} = OStatus.handle_incoming(incoming) assert activity.data["type"] == "Like" assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"] == retweeted_activity.data["object"]["id"] + assert activity.data["object"] == favorited_activity.data["object"]["id"] refute activity.local - assert retweeted_activity.data["type"] == "Create" - assert retweeted_activity.data["actor"] == "https://shitposter.club/user/1" - assert retweeted_activity.data["object"]["id"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" - refute retweeted_activity.local + assert favorited_activity.data["type"] == "Create" + assert favorited_activity.data["actor"] == "https://shitposter.club/user/1" + assert favorited_activity.data["object"]["id"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" + refute favorited_activity.local + end + + test "handle incoming favorites with locally available object - GS, websub" do + note_activity = insert(:note_activity) + + incoming = File.read!("test/fixtures/favorite_with_local_note.xml") + |> String.replace("localid", note_activity.data["object"]["id"]) + + {:ok, [[activity, favorited_activity]]} = OStatus.handle_incoming(incoming) + + assert activity.data["type"] == "Like" + assert activity.data["actor"] == "https://social.heldscal.la/user/23211" + assert activity.data["object"] == favorited_activity.data["object"]["id"] + refute activity.local + assert note_activity.id == favorited_activity.id + assert favorited_activity.local end test "handle incoming replies" do From 00a7183118d90946bf243d0f5488db65f8a6f16f Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 5 May 2017 18:58:29 +0200 Subject: [PATCH 094/107] Basic queue. --- lib/pleroma/application.ex | 3 ++- lib/pleroma/web/federator/federator.ex | 37 ++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 6267d0695..1f0a05568 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -18,7 +18,8 @@ def start(_type, _args) do default_ttl: 25000, ttl_interval: 1000, limit: 2500 - ]]) + ]]), + worker(Pleroma.Web.Federator, []) ] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index c4dacd116..69ba7ed7f 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -1,9 +1,15 @@ defmodule Pleroma.Web.Federator do + use GenServer alias Pleroma.User alias Pleroma.Web.WebFinger require Logger @websub Application.get_env(:pleroma, :websub) + @max_jobs 10 + + def start_link do + GenServer.start_link(__MODULE__, {:sets.new(), :queue.new()}, name: __MODULE__) + end def handle(:publish, activity) do Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) @@ -28,11 +34,38 @@ def handle(type, payload) do end def enqueue(type, payload) do - # for now, just run immediately in a new process. if Mix.env == :test do handle(type, payload) else - spawn(fn -> handle(type, payload) end) + GenServer.cast(__MODULE__, {:enqueue, type, payload}) end end + + def maybe_start_job(running_jobs, queue) do + if (:sets.size(running_jobs) < @max_jobs) && !:queue.is_empty(queue) do + {{:value, {type, payload}}, queue} = :queue.out(queue) + {:ok, pid} = Task.start(fn -> handle(type, payload) end) + mref = Process.monitor(pid) + {:sets.add_element(mref, running_jobs), queue} + else + {running_jobs, queue} + end + end + + def handle_cast({:enqueue, type, payload}, {running_jobs, queue}) do + queue = :queue.in({type, payload}, queue) + {running_jobs, queue} = maybe_start_job(running_jobs, queue) + {:noreply, {running_jobs, queue}} + end + + def handle_info({:DOWN, ref, :process, _pid, _reason}, {running_jobs, queue}) do + running_jobs = :sets.del_element(ref, running_jobs) + {running_jobs, queue} = maybe_start_job(running_jobs, queue) + {:noreply, {running_jobs, queue}} + end + + def handle_cast(m, state) do + IO.inspect("Unknown: #{inspect(m)}, #{inspect(state)}") + {:noreply, state} + end end From 7752f103f65f9c596f1571e31bed152382e2eacd Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 5 May 2017 20:15:26 +0200 Subject: [PATCH 095/107] Add basic thread fetching. --- lib/pleroma/web/ostatus/ostatus.ex | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 5a44b8661..2a107b6a6 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -121,6 +121,13 @@ def handle_note(entry, doc \\ nil) do {:ok, actor} = find_make_or_update_user(author) inReplyTo = string_from_xpath("//thr:in-reply-to[1]/@ref", entry) + if !Object.get_cached_by_ap_id(inReplyTo) do + inReplyToHref = string_from_xpath("//thr:in-reply-to[1]/@href", entry) + if inReplyToHref do + Task.start(fn -> fetch_activity_from_html_url(inReplyToHref) end) + end + end + context = (string_from_xpath("//ostatus:conversation[1]", entry) || "") |> String.trim attachments = get_attachments(entry) @@ -248,6 +255,7 @@ def gather_user_info(username) do # It's a hack anyway. Maybe revisit this in the future @mastodon_regex ~r// @gs_regex ~r// + @gs_classic_regex ~r// def get_atom_url(body) do cond do Regex.match?(@mastodon_regex, body) -> @@ -256,6 +264,9 @@ def get_atom_url(body) do Regex.match?(@gs_regex, body) -> [[_, match]] = Regex.scan(@gs_regex, body) {:ok, match} + Regex.match?(@gs_classic_regex, body) -> + [[_, match]] = Regex.scan(@gs_classic_regex, body) + {:ok, match} true -> Logger.debug(fn -> "Couldn't find atom link in #{inspect(body)}" end) {:error, "Couldn't find the atom link"} From 04f6ece99ef15d0e0ced330e3945307499b7a715 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Fri, 5 May 2017 20:38:10 +0200 Subject: [PATCH 096/107] Send salmons and websub in background tasks. --- lib/pleroma/web/salmon/salmon.ex | 6 ++++-- lib/pleroma/web/websub/websub.ex | 10 ++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex index b95ad48ad..b4f81b4ed 100644 --- a/lib/pleroma/web/salmon/salmon.ex +++ b/lib/pleroma/web/salmon/salmon.ex @@ -150,8 +150,10 @@ def publish(%{info: %{"keys" => keys}} = user, activity, poster) do remote_users(activity) |> Enum.each(fn(remote_user) -> - Logger.debug(fn -> "sending salmon to #{remote_user.ap_id}" end) - send_to_user(remote_user, feed, poster) + Task.start(fn -> + Logger.debug(fn -> "sending salmon to #{remote_user.ap_id}" end) + send_to_user(remote_user, feed, poster) + end) end) end end diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index 7bdb778ad..a5abc303c 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -51,10 +51,12 @@ def publish(topic, user, activity) do signature = sign(sub.secret || "", response) Logger.debug(fn -> "Pushing to #{sub.callback}" end) - @httpoison.post(sub.callback, response, [ - {"Content-Type", "application/atom+xml"}, - {"X-Hub-Signature", "sha1=#{signature}"} - ]) + Task.start(fn -> + @httpoison.post(sub.callback, response, [ + {"Content-Type", "application/atom+xml"}, + {"X-Hub-Signature", "sha1=#{signature}"} + ]) + end) end) end From 7e65cad9fe91cf7c0c25e8f307358083b3c8f784 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 6 May 2017 12:15:48 +0200 Subject: [PATCH 097/107] Do recursive fetching in-band for now. This is to prevent conversation id problems. --- lib/pleroma/web/ostatus/ostatus.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 2a107b6a6..e4448eb73 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -124,7 +124,7 @@ def handle_note(entry, doc \\ nil) do if !Object.get_cached_by_ap_id(inReplyTo) do inReplyToHref = string_from_xpath("//thr:in-reply-to[1]/@href", entry) if inReplyToHref do - Task.start(fn -> fetch_activity_from_html_url(inReplyToHref) end) + fetch_activity_from_html_url(inReplyToHref) end end From 22ddddce766b5ba867c561dbd8060c67393c5504 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 6 May 2017 12:34:40 +0200 Subject: [PATCH 098/107] Handle incoming items through the queue. --- lib/pleroma/web/federator/federator.ex | 6 ++++++ lib/pleroma/web/ostatus/ostatus_controller.ex | 4 ++-- lib/pleroma/web/websub/websub_controller.ex | 6 ++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex index 69ba7ed7f..ab3313de1 100644 --- a/lib/pleroma/web/federator/federator.ex +++ b/lib/pleroma/web/federator/federator.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.Federator do require Logger @websub Application.get_env(:pleroma, :websub) + @ostatus Application.get_env(:pleroma, :ostatus) @max_jobs 10 def start_link do @@ -28,6 +29,11 @@ def handle(:verify_websub, websub) do @websub.verify(websub) end + def handle(:incoming_doc, doc) do + Logger.debug("Got document, trying to parse") + @ostatus.handle_incoming(doc) + end + def handle(type, payload) do Logger.debug(fn -> "Unknown task: #{type}" end) {:error, "Don't know what do do with this"} diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 5f79cc7e9..e6822463d 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do alias Pleroma.{User, Activity} alias Pleroma.Web.OStatus.{FeedRepresenter, ActivityRepresenter} alias Pleroma.Repo - alias Pleroma.Web.OStatus + alias Pleroma.Web.{OStatus, Federator} import Ecto.Query def feed_redirect(conn, %{"nickname" => nickname}) do @@ -37,7 +37,7 @@ def salmon_incoming(conn, params) do {:ok, magic_key} = Pleroma.Web.Salmon.fetch_magic_key(body) {:ok, doc} = Pleroma.Web.Salmon.decode_and_validate(magic_key, body) - Pleroma.Web.OStatus.handle_incoming(doc) + Federator.enqueue(:incoming_doc, doc) conn |> send_resp(200, "") diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex index e860ec9e5..4fc693214 100644 --- a/lib/pleroma/web/websub/websub_controller.ex +++ b/lib/pleroma/web/websub/websub_controller.ex @@ -1,12 +1,10 @@ defmodule Pleroma.Web.Websub.WebsubController do use Pleroma.Web, :controller alias Pleroma.{Repo, User} - alias Pleroma.Web.Websub + alias Pleroma.Web.{Websub, Federator} alias Pleroma.Web.Websub.WebsubClientSubscription require Logger - @ostatus Application.get_env(:pleroma, :ostatus) - def websub_subscription_request(conn, %{"nickname" => nickname} = params) do user = User.get_cached_by_nickname(nickname) @@ -38,7 +36,7 @@ def websub_incoming(conn, %{"id" => id}) do %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id), {:ok, body, _conn} = read_body(conn), ^signature <- Websub.sign(websub.secret, body) do - @ostatus.handle_incoming(body) + Federator.enqueue(:incoming_doc, body) conn |> send_resp(200, "OK") else _e -> From 9cafb67fc177ace3af176a86366ae676b89ea368 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 6 May 2017 13:18:08 +0200 Subject: [PATCH 099/107] Follow redirects when fetching activities. This can happen for site that started as http and now switched to https. --- lib/pleroma/web/ostatus/ostatus.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index e4448eb73..fe42786b2 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -274,9 +274,9 @@ def get_atom_url(body) do end def fetch_activity_from_html_url(url) do - with {:ok, %{body: body}} <- @httpoison.get(url), + with {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true), {:ok, atom_url} <- get_atom_url(body), - {:ok, %{status_code: code, body: body}} when code in 200..299 <- @httpoison.get(atom_url) do + {:ok, %{status_code: code, body: body}} when code in 200..299 <- @httpoison.get(atom_url, [], follow_redirect: true) do handle_incoming(body) end end From bda389d7d942b7e90ffb519bc680a424a8400e2f Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 6 May 2017 14:09:39 +0200 Subject: [PATCH 100/107] Subscribe to remote users on following. --- lib/pleroma/user.ex | 6 +++++- lib/pleroma/web/websub/websub.ex | 2 +- test/user_test.exs | 24 +++++++++++++++++++++--- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 23be6276e..551c23445 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -4,7 +4,7 @@ defmodule Pleroma.User do import Ecto.{Changeset, Query} alias Pleroma.{Repo, User, Object, Web} alias Comeonin.Pbkdf2 - alias Pleroma.Web.OStatus + alias Pleroma.Web.{OStatus, Websub} schema "users" do field :bio, :string @@ -88,6 +88,10 @@ def follow(%User{} = follower, %User{} = followed) do {:error, "Could not follow user: #{followed.nickname} is already on your list."} else + if !followed.local do + Websub.subscribe(follower, followed) + end + following = [ap_followers | follower.following] |> Enum.uniq diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex index a5abc303c..7c8efa917 100644 --- a/lib/pleroma/web/websub/websub.ex +++ b/lib/pleroma/web/websub/websub.ex @@ -126,7 +126,7 @@ def subscribe(subscriber, subscribed, requester \\ &request_subscription/1) do topic = subscribed.info["topic"] # FIXME: Race condition, use transactions {:ok, subscription} = with subscription when not is_nil(subscription) <- Repo.get_by(WebsubClientSubscription, topic: topic) do - subscribers = [subscriber.ap_id, subscription.subscribers] |> Enum.uniq + subscribers = [subscriber.ap_id | subscription.subscribers] |> Enum.uniq change = Ecto.Changeset.change(subscription, %{subscribers: subscribers}) Repo.update(change) else _e -> diff --git a/test/user_test.exs b/test/user_test.exs index 036e70dff..417282ff9 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1,9 +1,12 @@ defmodule Pleroma.UserTest do alias Pleroma.Builders.UserBuilder - alias Pleroma.User + alias Pleroma.{User, Repo} + alias Pleroma.Web.OStatus + alias Pleroma.Web.Websub.WebsubClientSubscription use Pleroma.DataCase import Pleroma.Factory + import Ecto.Query test "ap_id returns the activity pub id for the user" do host = @@ -30,13 +33,29 @@ test "follow takes a user and another user" do user = insert(:user) followed = insert(:user) - {:ok, user } = User.follow(user, followed) + {:ok, user} = User.follow(user, followed) user = Repo.get(User, user.id) assert user.following == [User.ap_followers(followed)] end + test "following a remote user will ensure a websub subscription is present" do + user = insert(:user) + {:ok, followed} = OStatus.make_user("shp@social.heldscal.la") + + assert followed.local == false + + {:ok, user} = User.follow(user, followed) + assert user.following == [User.ap_followers(followed)] + + query = from w in WebsubClientSubscription, + where: w.topic == ^followed.info["topic"] + websub = Repo.one(query) + + assert websub + end + test "unfollow takes a user and another user" do followed = insert(:user) user = insert(:user, %{following: [User.ap_followers(followed)]}) @@ -95,7 +114,6 @@ test "gets an existing user" do assert user == fetched_user end - # TODO: Make the test local. test "fetches an external user via ostatus if no user exists" do fetched_user = User.get_or_fetch_by_nickname("shp@social.heldscal.la") assert fetched_user.nickname == "shp@social.heldscal.la" From 7db76a0f0ef15b42a52f18271e223d1cb8e88c64 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 6 May 2017 14:23:39 +0200 Subject: [PATCH 101/107] Address incoming messages to followers. --- lib/pleroma/web/ostatus/ostatus.ex | 3 ++- test/web/ostatus/ostatus_test.exs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index fe42786b2..f335e9972 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -143,7 +143,8 @@ def handle_note(entry, doc \\ nil) do end to = [ - "https://www.w3.org/ns/activitystreams#Public" + "https://www.w3.org/ns/activitystreams#Public", + User.ap_followers(actor) ] mentions = :xmerl_xpath.string('//link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]', entry) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index 1b03c1157..a56e6c4e6 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -2,7 +2,7 @@ defmodule Pleroma.Web.OStatusTest do use Pleroma.DataCase alias Pleroma.Web.OStatus alias Pleroma.Web.XML - alias Pleroma.{Object, Repo} + alias Pleroma.{Object, Repo, User} import Pleroma.Factory test "don't insert create notes twice" do @@ -32,6 +32,8 @@ test "handle incoming notes - GS, subscription" do assert activity.data["object"]["type"] == "Note" assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" assert activity.data["object"]["content"] == "Will it blend?" + user = User.get_cached_by_ap_id(activity.data["actor"]) + assert User.ap_followers(user) in activity.data["to"] end test "handle incoming notes with attachments - GS, subscription" do From fcd34096db14c2d18e3d51dbe23cced517160f85 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sat, 6 May 2017 22:26:36 +0200 Subject: [PATCH 102/107] Add one more alias for twapi twkn. --- lib/pleroma/web/router.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 327987d1d..50a3934d6 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -31,6 +31,7 @@ def user_fetcher(username) do get "/statuses/public_timeline", TwitterAPI.Controller, :public_timeline get "/statuses/public_and_external_timeline", TwitterAPI.Controller, :public_and_external_timeline + get "/statuses/networkpublic_timeline", TwitterAPI.Controller, :public_and_external_timeline get "/statuses/user_timeline", TwitterAPI.Controller, :user_timeline get "/statuses/show/:id", TwitterAPI.Controller, :fetch_status From ff2945dfdcc06e2e40e105ed0c98d7e10be0aa3f Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 7 May 2017 00:21:15 +0200 Subject: [PATCH 103/107] Index users by ap_id. --- .../migrations/20170506222027_add_unique_index_to_apid.exs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs diff --git a/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs b/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs new file mode 100644 index 000000000..864b5e47d --- /dev/null +++ b/priv/repo/migrations/20170506222027_add_unique_index_to_apid.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddUniqueIndexToAPID do + use Ecto.Migration + + def change do + create unique_index(:users, [:ap_id]) + end +end From c51e15975a2ca3c948098efa1aee6fc97335d198 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 7 May 2017 12:45:37 +0200 Subject: [PATCH 104/107] Fix user fetching error. --- lib/pleroma/web/ostatus/ostatus.ex | 2 +- test/web/ostatus/ostatus_test.exs | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index f335e9972..a6d416b2c 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -195,7 +195,7 @@ def find_make_or_update_user(doc) do def find_or_make_user(uri) do query = from user in User, - where: user.local == false and fragment("? @> ?", user.info, ^%{uri: uri}) + where: user.ap_id == ^uri user = Repo.one(query) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs index a56e6c4e6..41e1c3448 100644 --- a/test/web/ostatus/ostatus_test.exs +++ b/test/web/ostatus/ostatus_test.exs @@ -140,8 +140,14 @@ test "handle incoming replies" do end describe "new remote user creation" do + test "returns local users" do + local_user = insert(:user) + {:ok, user} = OStatus.find_or_make_user(local_user.ap_id) + + assert user == local_user + end + test "tries to use the information in poco fields" do - # TODO make test local uri = "https://social.heldscal.la/user/23211" {:ok, user} = OStatus.find_or_make_user(uri) @@ -160,7 +166,6 @@ test "tries to use the information in poco fields" do end test "find_make_or_update_user takes an author element and returns an updated user" do - # TODO make test local uri = "https://social.heldscal.la/user/23211" {:ok, user} = OStatus.find_or_make_user(uri) From 95ab0dc3c5ea63048a98b5fb594d0163249be822 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 7 May 2017 12:46:21 +0200 Subject: [PATCH 105/107] Add conversation ids to twapi announce representer. --- .../representers/activity_representer.ex | 16 ++++++++++------ .../representers/activity_representer_test.exs | 2 ++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex index 4d7ea0c5c..3fef8eec8 100644 --- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex +++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex @@ -27,7 +27,8 @@ def to_map(%Activity{data: %{"type" => "Announce", "actor" => actor, "published" "is_post_verb" => false, "uri" => "tag:#{activity.data["id"]}:objectType=note", "created_at" => created_at, - "retweeted_status" => retweeted_status + "retweeted_status" => retweeted_status, + "statusnet_conversation_id" => conversation_id(announced_activity) } end @@ -82,11 +83,7 @@ def to_map(%Activity{data: %{"object" => %{"content" => content} = object}} = ac |> Enum.filter(&(&1)) |> Enum.map(fn (user) -> UserRepresenter.to_map(user, opts) end) - - conversation_id = with context when not is_nil(context) <- activity.data["context"] do - TwitterAPI.context_to_conversation_id(context) - else _e -> nil - end + conversation_id = conversation_id(activity) %{ "id" => activity.id, @@ -108,6 +105,13 @@ def to_map(%Activity{data: %{"object" => %{"content" => content} = object}} = ac } end + def conversation_id(activity) do + with context when not is_nil(context) <- activity.data["context"] do + TwitterAPI.context_to_conversation_id(context) + else _e -> nil + end + end + defp date_to_asctime(date) do with {:ok, date, _offset} <- date |> DateTime.from_iso8601 do Strftime.strftime!(date, "%a %b %d %H:%M:%S %z %Y") diff --git a/test/web/twitter_api/representers/activity_representer_test.exs b/test/web/twitter_api/representers/activity_representer_test.exs index 64e7f0641..84c8d9a49 100644 --- a/test/web/twitter_api/representers/activity_representer_test.exs +++ b/test/web/twitter_api/representers/activity_representer_test.exs @@ -22,6 +22,8 @@ test "an announce activity" do retweeted_status = ActivityRepresenter.to_map(note_activity, %{user: activity_actor, for: user}) assert retweeted_status["repeated"] == true + assert retweeted_status["id"] == note_activity.id + assert status["statusnet_conversation_id"] == retweeted_status["statusnet_conversation_id"] assert status["retweeted_status"] == retweeted_status end From a41aa4e4898660fffb9a54070d16e2c3a5373d71 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 7 May 2017 14:45:37 +0200 Subject: [PATCH 106/107] Federate follow salmons. --- .../web/ostatus/activity_representer.ex | 28 ++++++++++++ lib/pleroma/web/twitter_api/twitter_api.ex | 3 ++ .../web/ostatus/activity_representer_test.exs | 44 +++++++++++++++++++ 3 files changed, 75 insertions(+) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 717670852..076ead41a 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -118,6 +118,34 @@ def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_autho ] ++ mentions ++ author end + def to_simple_form(%{data: %{"type" => "Follow"}} = activity, user, with_author) do + h = fn(str) -> [to_charlist(str)] end + + updated_at = activity.updated_at + |> NaiveDateTime.to_iso8601 + inserted_at = activity.inserted_at + |> NaiveDateTime.to_iso8601 + + author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] + + mentions = activity.data["to"] |> get_mentions + [ + {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']}, + {:"activity:verb", ['http://activitystrea.ms/schema/1.0/follow']}, + {:id, h.(activity.data["id"])}, + {:title, ['#{user.nickname} started following #{activity.data["object"]}']}, + {:content, [type: 'html'], ['#{user.nickname} started following #{activity.data["object"]}']}, + {:published, h.(inserted_at)}, + {:updated, h.(updated_at)}, + {:"activity:object", [ + {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/person']}, + {:id, h.(activity.data["object"])}, + {:uri, h.(activity.data["object"])}, + ]}, + {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}, + ] ++ mentions ++ author + end + def wrap_with_entry(simple_form) do [{ :entry, [ diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index 71f0c366e..3921c0d74 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -139,10 +139,13 @@ def follow(%User{} = follower, params) do {:ok, activity} <- ActivityPub.insert(%{ "type" => "Follow", "actor" => follower.ap_id, + "to" => [followed.ap_id], "object" => followed.ap_id, "published" => make_date() }) do + # TODO move all this to ActivityPub + Pleroma.Web.Federator.enqueue(:publish, activity) {:ok, follower, followed, activity} else err -> err diff --git a/test/web/ostatus/activity_representer_test.exs b/test/web/ostatus/activity_representer_test.exs index 12c9bbaa2..af936b57c 100644 --- a/test/web/ostatus/activity_representer_test.exs +++ b/test/web/ostatus/activity_representer_test.exs @@ -124,6 +124,7 @@ test "a like activity" do user = insert(:user) {:ok, like, _note} = ActivityPub.like(user, note) + # TODO: Are these the correct dates? updated_at = like.updated_at |> NaiveDateTime.to_iso8601 inserted_at = like.inserted_at @@ -155,6 +156,49 @@ test "a like activity" do assert clean(res) == clean(expected) end + test "a follow activity" do + follower = insert(:user) + followed = insert(:user) + {:ok, activity} = ActivityPub.insert(%{ + "type" => "Follow", + "actor" => follower.ap_id, + "object" => followed.ap_id, + "to" => [followed.ap_id] + }) + + + # TODO: Are these the correct dates? + updated_at = activity.updated_at + |> NaiveDateTime.to_iso8601 + inserted_at = activity.inserted_at + |> NaiveDateTime.to_iso8601 + + tuple = ActivityRepresenter.to_simple_form(activity, follower) + + refute is_nil(tuple) + + res = :xmerl.export_simple_content(tuple, :xmerl_xml) |> IO.iodata_to_binary + + expected = """ + http://activitystrea.ms/schema/1.0/activity + http://activitystrea.ms/schema/1.0/follow + #{activity.data["id"]} + #{follower.nickname} started following #{activity.data["object"]} + #{follower.nickname} started following #{activity.data["object"]} + #{inserted_at} + #{updated_at} + + http://activitystrea.ms/schema/1.0/person + #{activity.data["object"]} + #{activity.data["object"]} + + + + """ + + assert clean(res) == clean(expected) + end + test "an unknown activity" do tuple = ActivityRepresenter.to_simple_form(%Activity{}, nil) assert is_nil(tuple) From 60b4b0d725aefdca3eedd2d7708b0c96ee60c5f4 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 7 May 2017 14:52:19 +0200 Subject: [PATCH 107/107] Safety measures. --- lib/pleroma/web/ostatus/activity_representer.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex index 076ead41a..66e9b0ec2 100644 --- a/lib/pleroma/web/ostatus/activity_representer.ex +++ b/lib/pleroma/web/ostatus/activity_representer.ex @@ -128,7 +128,7 @@ def to_simple_form(%{data: %{"type" => "Follow"}} = activity, user, with_author) author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: [] - mentions = activity.data["to"] |> get_mentions + mentions = (activity.data["to"] || []) |> get_mentions [ {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']}, {:"activity:verb", ['http://activitystrea.ms/schema/1.0/follow']},