Also fetch atom links.

This commit is contained in:
Roger Braun 2017-08-04 16:57:38 +02:00
parent 27b8e3affb
commit 763756f879
6 changed files with 92 additions and 42 deletions

View File

@ -10,7 +10,7 @@ def fetch_replied_to_activity(entry, inReplyTo) do
if inReplyTo && !Object.get_cached_by_ap_id(inReplyTo) do if inReplyTo && !Object.get_cached_by_ap_id(inReplyTo) do
inReplyToHref = XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry) inReplyToHref = XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry)
if inReplyToHref do if inReplyToHref do
OStatus.fetch_activity_from_html_url(inReplyToHref) OStatus.fetch_activity_from_url(inReplyToHref)
else else
Logger.debug("Couldn't find a href link to #{inReplyTo}") Logger.debug("Couldn't find a href link to #{inReplyTo}")
end end

View File

@ -24,7 +24,7 @@ def salmon_path(user) do
end end
def handle_incoming(xml_string) do def handle_incoming(xml_string) do
doc = parse_document(xml_string) with doc when doc != :error <- parse_document(xml_string) do
entries = :xmerl_xpath.string('//entry', doc) entries = :xmerl_xpath.string('//entry', doc)
activities = Enum.map(entries, fn (entry) -> activities = Enum.map(entries, fn (entry) ->
@ -63,6 +63,9 @@ def handle_incoming(xml_string) do
|> Enum.filter(&(&1)) |> Enum.filter(&(&1))
{:ok, activities} {:ok, activities}
else
_e -> {:error, []}
end
end end
def make_share(entry, doc, retweeted_activity) do def make_share(entry, doc, retweeted_activity) do
@ -111,7 +114,7 @@ def get_or_try_fetching(entry) do
else e -> else e ->
Logger.debug("Couldn't get, will try to fetch") Logger.debug("Couldn't get, will try to fetch")
with href when not is_nil(href) <- string_from_xpath("//activity:object[1]/link[@type=\"text/html\"]/@href", entry), with href when not is_nil(href) <- string_from_xpath("//activity:object[1]/link[@type=\"text/html\"]/@href", entry),
{:ok, [favorited_activity]} <- fetch_activity_from_html_url(href) do {:ok, [favorited_activity]} <- fetch_activity_from_url(href) do
{:ok, favorited_activity} {:ok, favorited_activity}
else e -> Logger.debug("Couldn't find href: #{inspect(e)}") else e -> Logger.debug("Couldn't find href: #{inspect(e)}")
end end
@ -278,14 +281,30 @@ def get_atom_url(body) do
end end
end end
def fetch_activity_from_html_url(url) do def fetch_activity_from_atom_url(url) do
Logger.debug("Trying to fetch #{url}") with {:ok, %{body: body, status_code: code}} when code in 200..299 <- @httpoison.get(url, [Accept: "application/atom+xml"], follow_redirect: true, timeout: 10000, recv_timeout: 20000) do
with {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000),
{:ok, atom_url} <- get_atom_url(body),
{:ok, %{status_code: code, body: body}} when code in 200..299 <- @httpoison.get(atom_url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000) do
Logger.debug("Got document from #{url}, handling...") Logger.debug("Got document from #{url}, handling...")
handle_incoming(body) handle_incoming(body)
else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}") else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}")
end end
end end
def fetch_activity_from_html_url(url) do
Logger.debug("Trying to fetch #{url}")
with {:ok, %{body: body}} <- @httpoison.get(url, [], follow_redirect: true, timeout: 10000, recv_timeout: 20000),
{:ok, atom_url} <- get_atom_url(body) do
fetch_activity_from_atom_url(atom_url)
else e -> Logger.debug("Couldn't get #{url}: #{inspect(e)}")
end
end
def fetch_activity_from_url(url) do
with {:ok, activities} <- fetch_activity_from_atom_url(url) do
{:ok, activities}
else
_e -> with {:ok, activities} <- fetch_activity_from_html_url(url) do
{:ok, activities}
end
end
end
end end

View File

@ -0,0 +1 @@
<?xml version="1.0"?><entry xmlns="http://www.w3.org/2005/Atom" xmlns:thr="http://purl.org/syndication/thread/1.0" xmlns:activity="http://activitystrea.ms/spec/1.0/" xmlns:poco="http://portablecontacts.net/spec/1.0" xmlns:ostatus="http://ostatus.org/schema/1.0"><activity:object-type>http://activitystrea.ms/schema/1.0/note</activity:object-type><activity:verb>http://activitystrea.ms/schema/1.0/post</activity:verb><id>https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056</id><title>New note by eal</title><content type="html">&lt;a href='https://shitposter.club/user/5381'&gt;@shpuld&lt;/a&gt; &lt;a href='https://pleroma.hjkos.com/users/hj'&gt;@hj&lt;/a&gt; IM NOT GAY DAD</content><published>2017-08-04T12:51:26.130592Z</published><updated>2017-08-04T12:51:26.130592Z</updated><ostatus:conversation>https://pleroma.hjkos.com/contexts/53093c74-2100-4bf4-aac6-66d1973d03ef</ostatus:conversation><link ref="https://pleroma.hjkos.com/contexts/53093c74-2100-4bf4-aac6-66d1973d03ef" rel="ostatus:conversation"/><link type="application/atom+xml" href="https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056" rel="self"/><link type="text/html" href="https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056" rel="alternate"/><thr:in-reply-to ref="tag:shitposter.club,2017-08-04:noticeId=4027863:objectType=comment" href="https://shitposter.club/notice/4027863"/><author><id>https://social.sakamoto.gq/users/eal</id><activity:object>http://activitystrea.ms/schema/1.0/person</activity:object><uri>https://social.sakamoto.gq/users/eal</uri><poco:preferredUsername>eal</poco:preferredUsername><poco:displayName>坂本</poco:displayName><poco:note>(・ヮ・)</poco:note><name>eal</name><link rel="avatar" href="https://social.sakamoto.gq/media/7646c027-3614-4ee1-93f9-eea8f244a0d7/1A2EFE3153B9C9C3826DB511D043A26597C9F7178C8A4899FBBF808972D1659F.png"/></author><link rel="mentioned" ostatus:object-type="http://activitystrea.ms/schema/1.0/person" href="https://shitposter.club/user/5381"/><link rel="mentioned" ostatus:object-type="http://activitystrea.ms/schema/1.0/collection" href="http://activityschema.org/collection/public"/><link rel="mentioned" ostatus:object-type="http://activitystrea.ms/schema/1.0/person" href="https://pleroma.hjkos.com/users/hj"/></entry>

File diff suppressed because one or more lines are too long

View File

@ -87,6 +87,13 @@ def get("https://pleroma.soykaf.com/users/lain/feed.atom", _body, _headers) do
}} }}
end end
def get("https://social.sakamoto.gq/users/eal/feed.atom", _body, _headers) do
{:ok, %Response{
status_code: 200,
body: File.read!("test/fixtures/httpoison_mock/sakamoto_eal_feed.atom")
}}
end
def get("http://gs.example.org/index.php/api/statuses/user_timeline/1.atom", _body, _headers) do def get("http://gs.example.org/index.php/api/statuses/user_timeline/1.atom", _body, _headers) do
{:ok, %Response{ {:ok, %Response{
status_code: 200, status_code: 200,
@ -142,6 +149,13 @@ def get("https://mamot.fr/.well-known/webfinger", [Accept: "application/xrd+xml"
}} }}
end end
def get("https://social.sakamoto.gq/.well-known/webfinger", [Accept: "application/xrd+xml"], [params: [resource: "https://social.sakamoto.gq/users/eal"]]) do
{:ok, %Response{
status_code: 200,
body: File.read!("test/fixtures/httpoison_mock/eal_sakamoto.xml")
}}
end
def get("https://mamot.fr/users/Skruyb.atom", _, _) do def get("https://mamot.fr/users/Skruyb.atom", _, _) do
{:ok, %Response{ {:ok, %Response{
status_code: 200, status_code: 200,
@ -149,6 +163,14 @@ def get("https://mamot.fr/users/Skruyb.atom", _, _) do
}} }}
end end
def get("https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056", [Accept: "application/atom+xml"], _) do
{:ok, %Response{
status_code: 200,
body: File.read!("test/fixtures/httpoison_mock/sakamoto.atom")
}}
end
def get(url, body, headers) do def get(url, body, headers) do
{:error, "Not implemented the mock response for get #{inspect(url)}, #{inspect(body)}, #{inspect(headers)}"} {:error, "Not implemented the mock response for get #{inspect(url)}, #{inspect(body)}, #{inspect(headers)}"}
end end

View File

@ -326,11 +326,18 @@ test "it works with the uri" do
describe "fetching a status by it's HTML url" do describe "fetching a status by it's HTML url" do
test "it builds a missing status from an html url" do test "it builds a missing status from an html url" do
url = "https://shitposter.club/notice/2827873" url = "https://shitposter.club/notice/2827873"
{:ok, [activity] } = OStatus.fetch_activity_from_html_url(url) {:ok, [activity] } = OStatus.fetch_activity_from_url(url)
assert activity.data["actor"] == "https://shitposter.club/user/1" assert activity.data["actor"] == "https://shitposter.club/user/1"
assert activity.data["object"]["id"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" assert activity.data["object"]["id"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment"
end end
test "it works for atom notes, too" do
url = "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056"
{:ok, [activity] } = OStatus.fetch_activity_from_url(url)
assert activity.data["actor"] == "https://social.sakamoto.gq/users/eal"
assert activity.data["object"]["id"] == url
end
end end
test "insert or update a user from given data" do test "insert or update a user from given data" do