From fa2a6d5d6b24657ddbda4ef11d2e6dbcb59545d3 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Thu, 7 Apr 2022 18:25:02 +0200 Subject: [PATCH 1/9] feat: simple, but not stupid, uploader for IPFS fix: format fix with credo --- config/config.exs | 4 +++ config/description.exs | 24 +++++++++++++ config/dev.exs | 4 +++ lib/pleroma/upload.ex | 13 +++++-- lib/pleroma/uploaders/ipfs.ex | 64 +++++++++++++++++++++++++++++++++++ 5 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/uploaders/ipfs.ex diff --git a/config/config.exs b/config/config.exs index 6a5acda09..7efad0061 100644 --- a/config/config.exs +++ b/config/config.exs @@ -82,6 +82,10 @@ # region: "us-east-1", # may be required for Amazon AWS scheme: "https://" +config :pleroma, Pleroma.Uploaders.IPFS, + post_gateway_url: nil, + get_gateway_url: nil + config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"], pack_extensions: [".png", ".gif"], diff --git a/config/description.exs b/config/description.exs index 7caad18b4..d87bbb9b8 100644 --- a/config/description.exs +++ b/config/description.exs @@ -136,6 +136,30 @@ } ] }, + %{ + group: :pleroma, + key: Pleroma.Uploaders.IPFS, + type: :group, + description: "IPFS uploader-related settings", + children: [ + %{ + key: :get_gateway_url, + type: :string, + description: "GET Gateway URL", + suggestions: [ + "get_gateway_url" + ] + }, + %{ + key: :post_gateway_url, + type: :string, + description: "POST Gateway URL", + suggestions: [ + "post_gateway_url" + ] + } + ] + }, %{ group: :pleroma, key: Pleroma.Uploaders.S3, diff --git a/config/dev.exs b/config/dev.exs index ab3e83c12..89a84bf05 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -58,6 +58,10 @@ # https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects config :phoenix, :plug_init_mode, :runtime +config :pleroma, Pleroma.Uploaders.IPFS, + post_gateway_url: nil, + get_gateway_url: nil + if File.exists?("./config/dev.secret.exs") do import_config "dev.secret.exs" else diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index db2909276..de39bcd6c 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -235,8 +235,14 @@ defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do "" end - [base_url, path] - |> Path.join() + uploader = Config.get([Pleroma.Upload, :uploader]) + + if uploader == Pleroma.Uploaders.IPFS && String.contains?(base_url, "{CID}") do + String.replace(base_url, "{CID}", path) + else + [base_url, path] + |> Path.join() + end end defp url_from_spec(_upload, _base_url, {:url, url}), do: url @@ -273,6 +279,9 @@ def base_url do Path.join([upload_base_url, bucket_with_namespace]) end + Pleroma.Uploaders.IPFS -> + Config.get([Pleroma.Uploaders.IPFS, :get_gateway_url]) + _ -> public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" end diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex new file mode 100644 index 000000000..b46e9322e --- /dev/null +++ b/lib/pleroma/uploaders/ipfs.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Uploaders.IPFS do + @behaviour Pleroma.Uploaders.Uploader + require Logger + + alias Pleroma.Config + alias Tesla.Multipart + + @impl true + def get_file(file) do + b_url = Pleroma.Upload.base_url() + + if String.contains?(b_url, "{CID}") do + {:ok, {:url, String.replace(b_url, "{CID}", URI.decode(file))}} + else + {:error, "IPFS Get URL doesn't contain '{CID}' placeholder"} + end + end + + @impl true + def put_file(%Pleroma.Upload{} = upload) do + config = Config.get([__MODULE__]) + post_base_url = Keyword.get(config, :post_gateway_url) + + mp = + Multipart.new() + |> Multipart.add_content_type_param("charset=utf-8") + |> Multipart.add_file(upload.tempfile) + + final_url = Path.join([post_base_url, "/api/v0/add"]) + + case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do + {:ok, ret} -> + case Jason.decode(ret.body) do + {:ok, ret} -> + {:ok, {:file, ret["Hash"]}} + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "JSON decode failed"} + end + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "IPFS Gateway Upload failed"} + end + end + + @impl true + def delete_file(file) do + config = Config.get([__MODULE__]) + post_base_url = Keyword.get(config, :post_gateway_url) + + final_url = Path.join([post_base_url, "/api/v0/files/rm"]) + + case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do + {:ok, %{status_code: 204}} -> :ok + error -> {:error, inspect(error)} + end + end +end From 43dfa58ebda407a0813d398bee8d0ae3e5c9fd5b Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Mon, 11 Apr 2022 15:10:01 +0200 Subject: [PATCH 2/9] added tests for ipfs uploader. adapted changelog.md accordingly. improved ipfs uploader with external suggestions fix lint description.exs --- CHANGELOG.md | 1 + config/description.exs | 5 +- config/dev.exs | 4 -- docs/configuration/cheatsheet.md | 13 ++++ lib/pleroma/upload.ex | 6 +- lib/pleroma/uploaders/ipfs.ex | 14 +++-- test/pleroma/uploaders/ipfs_test.exs | 88 ++++++++++++++++++++++++++++ 7 files changed, 116 insertions(+), 15 deletions(-) create mode 100644 test/pleroma/uploaders/ipfs_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index f1beb0cd0..b2185b1ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - PleromaAPI: Add `GET /api/v1/pleroma/birthdays` API endpoint - Make backend-rendered pages translatable. This includes emails. Pages returned as a HTTP response are translated using the language specified in the `userLanguage` cookie, or the `Accept-Language` header. Emails are translated using the `language` field when registering. This language can be changed by `PATCH /api/v1/accounts/update_credentials` with the `language` field. - Uploadfilter `Pleroma.Upload.Filter.Exiftool.ReadDescription` returns description values to the FE so they can pre fill the image description field +- Uploader: Add support for uploading attachments using IPFS ### Fixed - Subscription(Bell) Notifications: Don't create from Pipeline Ingested replies diff --git a/config/description.exs b/config/description.exs index d87bbb9b8..1fe5f01f0 100644 --- a/config/description.exs +++ b/config/description.exs @@ -147,7 +147,8 @@ type: :string, description: "GET Gateway URL", suggestions: [ - "get_gateway_url" + "https://ipfs.mydomain.com/<%= cid %>", + "https://<%= cid %>.ipfs.mydomain.com/" ] }, %{ @@ -155,7 +156,7 @@ type: :string, description: "POST Gateway URL", suggestions: [ - "post_gateway_url" + "http://localhost:5001/" ] } ] diff --git a/config/dev.exs b/config/dev.exs index 89a84bf05..ab3e83c12 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -58,10 +58,6 @@ # https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects config :phoenix, :plug_init_mode, :runtime -config :pleroma, Pleroma.Uploaders.IPFS, - post_gateway_url: nil, - get_gateway_url: nil - if File.exists?("./config/dev.secret.exs") do import_config "dev.secret.exs" else diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 74642397b..7e1f9c934 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -614,6 +614,19 @@ config :ex_aws, :s3, host: "s3.eu-central-1.amazonaws.com" ``` +#### Pleroma.Uploaders.IPFS + +* `post_gateway_url`: URL with port of POST Gateway (unauthenticated) +* `get_gateway_url`: URL of public GET Gateway + +Example: + +```elixir +config :pleroma, Pleroma.Uploaders.IPFS, + post_gateway_url: "http://localhost:5001", + get_gateway_url: "http://<%= cid %>.ipfs.mydomain.com" +``` + ### Upload filters #### Pleroma.Upload.Filter.AnonymizeFilename diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index de39bcd6c..b51d23f9e 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -235,10 +235,8 @@ defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do "" end - uploader = Config.get([Pleroma.Upload, :uploader]) - - if uploader == Pleroma.Uploaders.IPFS && String.contains?(base_url, "{CID}") do - String.replace(base_url, "{CID}", path) + if String.contains?(base_url, "<%= cid %>") do + EEx.eval_string(base_url, cid: path) else [base_url, path] |> Path.join() diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index b46e9322e..722c68fa1 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -13,10 +13,10 @@ defmodule Pleroma.Uploaders.IPFS do def get_file(file) do b_url = Pleroma.Upload.base_url() - if String.contains?(b_url, "{CID}") do - {:ok, {:url, String.replace(b_url, "{CID}", URI.decode(file))}} + if String.contains?(b_url, "<%= cid %>") do + {:ok, {:url, EEx.eval_string(b_url, cid: URI.decode(file))}} else - {:error, "IPFS Get URL doesn't contain '{CID}' placeholder"} + {:error, "IPFS Get URL doesn't contain 'cid' placeholder"} end end @@ -36,7 +36,11 @@ def put_file(%Pleroma.Upload{} = upload) do {:ok, ret} -> case Jason.decode(ret.body) do {:ok, ret} -> - {:ok, {:file, ret["Hash"]}} + if Map.has_key?(ret, "Hash") do + {:ok, {:file, ret["Hash"]}} + else + {:error, "JSON doesn't contain Hash value"} + end error -> Logger.error("#{__MODULE__}: #{inspect(error)}") @@ -45,7 +49,7 @@ def put_file(%Pleroma.Upload{} = upload) do error -> Logger.error("#{__MODULE__}: #{inspect(error)}") - {:error, "IPFS Gateway Upload failed"} + {:error, "IPFS Gateway upload failed"} end end diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs new file mode 100644 index 000000000..f9ae046cf --- /dev/null +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -0,0 +1,88 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Uploaders.IPFSTest do + use Pleroma.DataCase + + alias Pleroma.Uploaders.IPFS + alias Tesla.Multipart + + import Mock + import ExUnit.CaptureLog + + setup do + clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.IPFS) + clear_config([Pleroma.Uploaders.IPFS]) + + clear_config( + [Pleroma.Uploaders.IPFS, :get_gateway_url], + "https://<%= cid %>.ipfs.mydomain.com" + ) + + clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001") + end + + describe "get_file/1" do + test "it returns path to ipfs file with cid as subdomain" do + assert IPFS.get_file("testcid") == { + :ok, + {:url, "https://testcid.ipfs.mydomain.com"} + } + end + + test "it returns path to ipfs file with cid as path" do + clear_config( + [Pleroma.Uploaders.IPFS, :get_gateway_url], + "https://ipfs.mydomain.com/ipfs/<%= cid %>" + ) + + assert IPFS.get_file("testcid") == { + :ok, + {:url, "https://ipfs.mydomain.com/ipfs/testcid"} + } + end + end + + describe "put_file/1" do + setup do + file_upload = %Pleroma.Upload{ + name: "image-tet.jpg", + content_type: "image/jpeg", + path: "test_folder/image-tet.jpg", + tempfile: Path.absname("test/instance_static/add/shortcode.png") + } + + [file_upload: file_upload] + end + + test "save file", %{file_upload: file_upload} do + with_mock Pleroma.HTTP, + post: fn _, _, _, _ -> + {:ok, + %Tesla.Env{ + status: 200, + body: "{\"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}" + }} + end do + assert IPFS.put_file(file_upload) == + {:ok, {:file, "bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi"}} + end + end + + test "returns error", %{file_upload: file_upload} do + with_mock Pleroma.HTTP, post: fn _, _, _, _ -> {:error, "IPFS Gateway upload failed"} end do + assert capture_log(fn -> + assert IPFS.put_file(file_upload) == {:error, "IPFS Gateway upload failed"} + end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}" + end + end + end + + describe "delete_file/1" do + test_with_mock "deletes file", Pleroma.HTTP, + post: fn _, _, _, _ -> {:ok, %{status_code: 204}} end do + assert :ok = IPFS.delete_file("image.jpg") + end + end +end From 44659ecd65fb2251f9130fcecf1732b8931104c1 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Sat, 16 Apr 2022 09:38:49 +0200 Subject: [PATCH 3/9] ipfs: revert to String.replace for cid placeholder ipfs: fix lint --- config/description.exs | 4 ++-- docs/configuration/cheatsheet.md | 2 +- lib/pleroma/upload.ex | 4 ++-- lib/pleroma/uploaders/ipfs.ex | 7 +++++-- test/pleroma/uploaders/ipfs_test.exs | 4 ++-- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/config/description.exs b/config/description.exs index 1fe5f01f0..b180e7308 100644 --- a/config/description.exs +++ b/config/description.exs @@ -147,8 +147,8 @@ type: :string, description: "GET Gateway URL", suggestions: [ - "https://ipfs.mydomain.com/<%= cid %>", - "https://<%= cid %>.ipfs.mydomain.com/" + "https://ipfs.mydomain.com/{CID}", + "https://{CID}.ipfs.mydomain.com/" ] }, %{ diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 7e1f9c934..d35b33574 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -624,7 +624,7 @@ Example: ```elixir config :pleroma, Pleroma.Uploaders.IPFS, post_gateway_url: "http://localhost:5001", - get_gateway_url: "http://<%= cid %>.ipfs.mydomain.com" + get_gateway_url: "http://{CID}.ipfs.mydomain.com" ``` ### Upload filters diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index b51d23f9e..8a01cf613 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -235,8 +235,8 @@ defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do "" end - if String.contains?(base_url, "<%= cid %>") do - EEx.eval_string(base_url, cid: path) + if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do + String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path) else [base_url, path] |> Path.join() diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index 722c68fa1..dde520d8e 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -9,12 +9,15 @@ defmodule Pleroma.Uploaders.IPFS do alias Pleroma.Config alias Tesla.Multipart + @placeholder "{CID}" + def placeholder, do: @placeholder + @impl true def get_file(file) do b_url = Pleroma.Upload.base_url() - if String.contains?(b_url, "<%= cid %>") do - {:ok, {:url, EEx.eval_string(b_url, cid: URI.decode(file))}} + if String.contains?(b_url, @placeholder) do + {:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}} else {:error, "IPFS Get URL doesn't contain 'cid' placeholder"} end diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index f9ae046cf..fc87fa378 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -17,7 +17,7 @@ defmodule Pleroma.Uploaders.IPFSTest do clear_config( [Pleroma.Uploaders.IPFS, :get_gateway_url], - "https://<%= cid %>.ipfs.mydomain.com" + "https://{CID}.ipfs.mydomain.com" ) clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001") @@ -34,7 +34,7 @@ test "it returns path to ipfs file with cid as subdomain" do test "it returns path to ipfs file with cid as path" do clear_config( [Pleroma.Uploaders.IPFS, :get_gateway_url], - "https://ipfs.mydomain.com/ipfs/<%= cid %>" + "https://ipfs.mydomain.com/ipfs/{CID}" ) assert IPFS.get_file("testcid") == { From 7c1af86f979ecebcd38995e5278fe2d59a36eda5 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Mon, 9 May 2022 12:15:40 +0200 Subject: [PATCH 4/9] ipfs: refactor final_url generation. add tests for final_url fix lint --- lib/pleroma/uploaders/ipfs.ex | 19 ++++++++++--------- test/pleroma/uploaders/ipfs_test.exs | 13 ++++++++++++- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index dde520d8e..7a7481d81 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -12,6 +12,13 @@ defmodule Pleroma.Uploaders.IPFS do @placeholder "{CID}" def placeholder, do: @placeholder + def get_final_url(method) do + config = Config.get([__MODULE__]) + post_base_url = Keyword.get(config, :post_gateway_url) + + Path.join([post_base_url, method]) + end + @impl true def get_file(file) do b_url = Pleroma.Upload.base_url() @@ -25,15 +32,12 @@ def get_file(file) do @impl true def put_file(%Pleroma.Upload{} = upload) do - config = Config.get([__MODULE__]) - post_base_url = Keyword.get(config, :post_gateway_url) - mp = Multipart.new() |> Multipart.add_content_type_param("charset=utf-8") |> Multipart.add_file(upload.tempfile) - final_url = Path.join([post_base_url, "/api/v0/add"]) + final_url = get_final_url("/api/v0/add") case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do {:ok, ret} -> @@ -42,7 +46,7 @@ def put_file(%Pleroma.Upload{} = upload) do if Map.has_key?(ret, "Hash") do {:ok, {:file, ret["Hash"]}} else - {:error, "JSON doesn't contain Hash value"} + {:error, "JSON doesn't contain Hash key"} end error -> @@ -58,10 +62,7 @@ def put_file(%Pleroma.Upload{} = upload) do @impl true def delete_file(file) do - config = Config.get([__MODULE__]) - post_base_url = Keyword.get(config, :post_gateway_url) - - final_url = Path.join([post_base_url, "/api/v0/files/rm"]) + final_url = get_final_url("/api/v0/files/rm") case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do {:ok, %{status_code: 204}} -> :ok diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index fc87fa378..d567272d2 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -23,6 +23,16 @@ defmodule Pleroma.Uploaders.IPFSTest do clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001") end + describe "get_final_url" do + test "it returns the final url for put_file" do + assert IPFS.get_final_url("/api/v0/add") == "http://localhost:5001/api/v0/add" + end + + test "it returns the final url for delete_file" do + assert IPFS.get_final_url("/api/v0/files/rm") == "http://localhost:5001/api/v0/files/rm" + end + end + describe "get_file/1" do test "it returns path to ipfs file with cid as subdomain" do assert IPFS.get_file("testcid") == { @@ -62,7 +72,8 @@ test "save file", %{file_upload: file_upload} do {:ok, %Tesla.Env{ status: 200, - body: "{\"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}" + body: + "{\"Name\":\"image-tet.jpg\",\"Size\":\"5000\", \"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}" }} end do assert IPFS.put_file(file_upload) == From 98f268e5ecc5bab98c98270a582f8b3f0e3be4e8 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Thu, 9 Jun 2022 19:24:13 +0200 Subject: [PATCH 5/9] ipfs: small refactor and more tests --- lib/pleroma/uploaders/ipfs.ex | 24 ++++++++++++++---------- test/pleroma/uploaders/ipfs_test.exs | 24 ++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index 7a7481d81..9f6f26e2e 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -9,16 +9,24 @@ defmodule Pleroma.Uploaders.IPFS do alias Pleroma.Config alias Tesla.Multipart - @placeholder "{CID}" - def placeholder, do: @placeholder - - def get_final_url(method) do + defp get_final_url(method) do config = Config.get([__MODULE__]) post_base_url = Keyword.get(config, :post_gateway_url) Path.join([post_base_url, method]) end + def put_file_endpoint() do + get_final_url("/api/v0/add") + end + + def delete_file_endpoint() do + get_final_url("/api/v0/files/rm") + end + + @placeholder "{CID}" + def placeholder, do: @placeholder + @impl true def get_file(file) do b_url = Pleroma.Upload.base_url() @@ -37,9 +45,7 @@ def put_file(%Pleroma.Upload{} = upload) do |> Multipart.add_content_type_param("charset=utf-8") |> Multipart.add_file(upload.tempfile) - final_url = get_final_url("/api/v0/add") - - case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do + case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do {:ok, ret} -> case Jason.decode(ret.body) do {:ok, ret} -> @@ -62,9 +68,7 @@ def put_file(%Pleroma.Upload{} = upload) do @impl true def delete_file(file) do - final_url = get_final_url("/api/v0/files/rm") - - case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do + case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do {:ok, %{status_code: 204}} -> :ok error -> {:error, inspect(error)} end diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index d567272d2..f2b880d9f 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -25,11 +25,11 @@ defmodule Pleroma.Uploaders.IPFSTest do describe "get_final_url" do test "it returns the final url for put_file" do - assert IPFS.get_final_url("/api/v0/add") == "http://localhost:5001/api/v0/add" + assert IPFS.put_file_endpoint() == "http://localhost:5001/api/v0/add" end test "it returns the final url for delete_file" do - assert IPFS.get_final_url("/api/v0/files/rm") == "http://localhost:5001/api/v0/files/rm" + assert IPFS.delete_file_endpoint() == "http://localhost:5001/api/v0/files/rm" end end @@ -88,6 +88,26 @@ test "returns error", %{file_upload: file_upload} do end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}" end end + + test "returns error if JSON decode fails", %{file_upload: file_upload} do + with_mocks([ + {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]}, + {Jason, [], [decode: fn _ -> {:error, "JSON decode failed"} end]} + ]) do + assert capture_log(fn -> + assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"} + end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"JSON decode failed\"}" + end + end + + test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do + with_mocks([ + {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]}, + {Jason, [], [decode: fn _ -> {:ok, %{}} end]} + ]) do + assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"} + end + end end describe "delete_file/1" do From 254f2ea85400ebd692fc4a45f5ac22fedd49ec09 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Thu, 9 Jun 2022 23:38:50 +0200 Subject: [PATCH 6/9] ipfs: remove unused alias fix analysis job --- lib/pleroma/uploaders/ipfs.ex | 4 ++-- test/pleroma/uploaders/ipfs_test.exs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index 9f6f26e2e..32e06c5cf 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -16,11 +16,11 @@ defp get_final_url(method) do Path.join([post_base_url, method]) end - def put_file_endpoint() do + def put_file_endpoint do get_final_url("/api/v0/add") end - def delete_file_endpoint() do + def delete_file_endpoint do get_final_url("/api/v0/files/rm") end diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index f2b880d9f..5edb6266b 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -6,7 +6,6 @@ defmodule Pleroma.Uploaders.IPFSTest do use Pleroma.DataCase alias Pleroma.Uploaders.IPFS - alias Tesla.Multipart import Mock import ExUnit.CaptureLog From 5e097eb91def0efd3cd0008309fd524fcfd88e15 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Tue, 28 Jun 2022 17:53:44 +0200 Subject: [PATCH 7/9] ipfs: better tests with @ilja suggestions --- test/pleroma/uploaders/ipfs_test.exs | 38 ++++++++++++++++++---------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index 5edb6266b..9df3f239c 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Uploaders.IPFSTest do use Pleroma.DataCase alias Pleroma.Uploaders.IPFS + alias Tesla.Multipart import Mock import ExUnit.CaptureLog @@ -62,12 +63,17 @@ test "it returns path to ipfs file with cid as path" do tempfile: Path.absname("test/instance_static/add/shortcode.png") } - [file_upload: file_upload] + mp = + Multipart.new() + |> Multipart.add_content_type_param("charset=utf-8") + |> Multipart.add_file(file_upload.tempfile) + + [file_upload: file_upload, mp: mp] end test "save file", %{file_upload: file_upload} do with_mock Pleroma.HTTP, - post: fn _, _, _, _ -> + post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> {:ok, %Tesla.Env{ status: 200, @@ -81,7 +87,10 @@ test "save file", %{file_upload: file_upload} do end test "returns error", %{file_upload: file_upload} do - with_mock Pleroma.HTTP, post: fn _, _, _, _ -> {:error, "IPFS Gateway upload failed"} end do + with_mock Pleroma.HTTP, + post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + {:error, "IPFS Gateway upload failed"} + end do assert capture_log(fn -> assert IPFS.put_file(file_upload) == {:error, "IPFS Gateway upload failed"} end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}" @@ -89,21 +98,22 @@ test "returns error", %{file_upload: file_upload} do end test "returns error if JSON decode fails", %{file_upload: file_upload} do - with_mocks([ - {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]}, - {Jason, [], [decode: fn _ -> {:error, "JSON decode failed"} end]} - ]) do + with_mock Pleroma.HTTP, [], + post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + {:ok, %Tesla.Env{status: 200, body: 'invalid'}} + end do assert capture_log(fn -> assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"} - end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"JSON decode failed\"}" + end) =~ + "Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError{data: \"invalid\", position: 0, token: nil}}" end end test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do - with_mocks([ - {Pleroma.HTTP, [], [post: fn _, _, _, _ -> {:ok, %Tesla.Env{status: 200, body: ''}} end]}, - {Jason, [], [decode: fn _ -> {:ok, %{}} end]} - ]) do + with_mock Pleroma.HTTP, [], + post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + {:ok, %Tesla.Env{status: 200, body: '{"key": "value"}'}} + end do assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"} end end @@ -111,7 +121,9 @@ test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_ describe "delete_file/1" do test_with_mock "deletes file", Pleroma.HTTP, - post: fn _, _, _, _ -> {:ok, %{status_code: 204}} end do + post: fn "http://localhost:5001/api/v0/files/rm", "", [], params: [arg: "image.jpg"] -> + {:ok, %{status_code: 204}} + end do assert :ok = IPFS.delete_file("image.jpg") end end From 21d9091f5e422493ff69fe59db9c965e0d511369 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Fri, 8 Jul 2022 10:06:46 +0200 Subject: [PATCH 8/9] ipfs: replacing single quotes with double quotes --- test/pleroma/uploaders/ipfs_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index 9df3f239c..853d185e5 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -100,7 +100,7 @@ test "returns error", %{file_upload: file_upload} do test "returns error if JSON decode fails", %{file_upload: file_upload} do with_mock Pleroma.HTTP, [], post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> - {:ok, %Tesla.Env{status: 200, body: 'invalid'}} + {:ok, %Tesla.Env{status: 200, body: "invalid"}} end do assert capture_log(fn -> assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"} @@ -112,7 +112,7 @@ test "returns error if JSON decode fails", %{file_upload: file_upload} do test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do with_mock Pleroma.HTTP, [], post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> - {:ok, %Tesla.Env{status: 200, body: '{"key": "value"}'}} + {:ok, %Tesla.Env{status: 200, body: "{\"key\": \"value\"}"}} end do assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"} end From 3055c1598b43ee9460b88880e2752c68e9cf6edb Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 May 2024 17:22:18 +0400 Subject: [PATCH 9/9] IPFSTest: Fix configuration mocking --- config/test.exs | 1 + lib/pleroma/upload.ex | 2 +- lib/pleroma/uploaders/ipfs.ex | 7 +-- test/pleroma/uploaders/ipfs_test.exs | 70 +++++++++++++++++++--------- 4 files changed, 55 insertions(+), 25 deletions(-) diff --git a/config/test.exs b/config/test.exs index 9b4113dd5..3345bb3a9 100644 --- a/config/test.exs +++ b/config/test.exs @@ -153,6 +153,7 @@ config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock +config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock peer_module = if String.to_integer(System.otp_release()) >= 25 do diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 2c6b23c39..35c7c02a5 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -282,7 +282,7 @@ def base_url do end Pleroma.Uploaders.IPFS -> - Config.get([Pleroma.Uploaders.IPFS, :get_gateway_url]) + @config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url]) _ -> public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index 32e06c5cf..d171e4652 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -6,11 +6,12 @@ defmodule Pleroma.Uploaders.IPFS do @behaviour Pleroma.Uploaders.Uploader require Logger - alias Pleroma.Config alias Tesla.Multipart + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + defp get_final_url(method) do - config = Config.get([__MODULE__]) + config = @config_impl.get([__MODULE__]) post_base_url = Keyword.get(config, :post_gateway_url) Path.join([post_base_url, method]) @@ -69,7 +70,7 @@ def put_file(%Pleroma.Upload{} = upload) do @impl true def delete_file(file) do case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do - {:ok, %{status_code: 204}} -> :ok + {:ok, %{status: 204}} -> :ok error -> {:error, inspect(error)} end end diff --git a/test/pleroma/uploaders/ipfs_test.exs b/test/pleroma/uploaders/ipfs_test.exs index 853d185e5..cf325b54f 100644 --- a/test/pleroma/uploaders/ipfs_test.exs +++ b/test/pleroma/uploaders/ipfs_test.exs @@ -8,22 +8,22 @@ defmodule Pleroma.Uploaders.IPFSTest do alias Pleroma.Uploaders.IPFS alias Tesla.Multipart - import Mock import ExUnit.CaptureLog + import Mock + import Mox - setup do - clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.IPFS) - clear_config([Pleroma.Uploaders.IPFS]) - - clear_config( - [Pleroma.Uploaders.IPFS, :get_gateway_url], - "https://{CID}.ipfs.mydomain.com" - ) - - clear_config([Pleroma.Uploaders.IPFS, :post_gateway_url], "http://localhost:5001") - end + alias Pleroma.UnstubbedConfigMock, as: Config describe "get_final_url" do + setup do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS] -> + [post_gateway_url: "http://localhost:5001"] + end) + + :ok + end + test "it returns the final url for put_file" do assert IPFS.put_file_endpoint() == "http://localhost:5001/api/v0/add" end @@ -34,7 +34,21 @@ test "it returns the final url for delete_file" do end describe "get_file/1" do + setup do + Config + |> expect(:get, fn [Pleroma.Upload, :uploader] -> Pleroma.Uploaders.IPFS end) + |> expect(:get, fn [Pleroma.Upload, :base_url] -> nil end) + |> expect(:get, fn [Pleroma.Uploaders.IPFS, :public_endpoint] -> nil end) + + :ok + end + test "it returns path to ipfs file with cid as subdomain" do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] -> + "https://{CID}.ipfs.mydomain.com" + end) + assert IPFS.get_file("testcid") == { :ok, {:url, "https://testcid.ipfs.mydomain.com"} @@ -42,10 +56,10 @@ test "it returns path to ipfs file with cid as subdomain" do end test "it returns path to ipfs file with cid as path" do - clear_config( - [Pleroma.Uploaders.IPFS, :get_gateway_url], + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] -> "https://ipfs.mydomain.com/ipfs/{CID}" - ) + end) assert IPFS.get_file("testcid") == { :ok, @@ -56,6 +70,11 @@ test "it returns path to ipfs file with cid as path" do describe "put_file/1" do setup do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS] -> + [post_gateway_url: "http://localhost:5001"] + end) + file_upload = %Pleroma.Upload{ name: "image-tet.jpg", content_type: "image/jpeg", @@ -73,7 +92,7 @@ test "it returns path to ipfs file with cid as path" do test "save file", %{file_upload: file_upload} do with_mock Pleroma.HTTP, - post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] -> {:ok, %Tesla.Env{ status: 200, @@ -88,7 +107,7 @@ test "save file", %{file_upload: file_upload} do test "returns error", %{file_upload: file_upload} do with_mock Pleroma.HTTP, - post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] -> {:error, "IPFS Gateway upload failed"} end do assert capture_log(fn -> @@ -99,19 +118,19 @@ test "returns error", %{file_upload: file_upload} do test "returns error if JSON decode fails", %{file_upload: file_upload} do with_mock Pleroma.HTTP, [], - post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] -> {:ok, %Tesla.Env{status: 200, body: "invalid"}} end do assert capture_log(fn -> assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"} end) =~ - "Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError{data: \"invalid\", position: 0, token: nil}}" + "Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError" end end test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do with_mock Pleroma.HTTP, [], - post: fn "http://localhost:5001/api/v0/add", mp, [], params: ["cid-version": "1"] -> + post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] -> {:ok, %Tesla.Env{status: 200, body: "{\"key\": \"value\"}"}} end do assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"} @@ -120,9 +139,18 @@ test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_ end describe "delete_file/1" do + setup do + Config + |> expect(:get, fn [Pleroma.Uploaders.IPFS] -> + [post_gateway_url: "http://localhost:5001"] + end) + + :ok + end + test_with_mock "deletes file", Pleroma.HTTP, post: fn "http://localhost:5001/api/v0/files/rm", "", [], params: [arg: "image.jpg"] -> - {:ok, %{status_code: 204}} + {:ok, %{status: 204}} end do assert :ok = IPFS.delete_file("image.jpg") end