Merge branch 'ipfs_uploader' into 'develop'

feat: simple, but not stupid, uploader for IPFS

See merge request pleroma/pleroma!3654
This commit is contained in:
lain 2024-05-27 15:06:12 +00:00
commit d11ba9e85b
8 changed files with 288 additions and 2 deletions

View File

@ -0,0 +1 @@
Uploader: Add support for uploading attachments using IPFS

View File

@ -82,6 +82,10 @@
# region: "us-east-1", # may be required for Amazon AWS # region: "us-east-1", # may be required for Amazon AWS
scheme: "https://" scheme: "https://"
config :pleroma, Pleroma.Uploaders.IPFS,
post_gateway_url: nil,
get_gateway_url: nil
config :pleroma, :emoji, config :pleroma, :emoji,
shortcode_globs: ["/emoji/custom/**/*.png"], shortcode_globs: ["/emoji/custom/**/*.png"],
pack_extensions: [".png", ".gif"], pack_extensions: [".png", ".gif"],

View File

@ -136,6 +136,31 @@
} }
] ]
}, },
%{
group: :pleroma,
key: Pleroma.Uploaders.IPFS,
type: :group,
description: "IPFS uploader-related settings",
children: [
%{
key: :get_gateway_url,
type: :string,
description: "GET Gateway URL",
suggestions: [
"https://ipfs.mydomain.com/{CID}",
"https://{CID}.ipfs.mydomain.com/"
]
},
%{
key: :post_gateway_url,
type: :string,
description: "POST Gateway URL",
suggestions: [
"http://localhost:5001/"
]
}
]
},
%{ %{
group: :pleroma, group: :pleroma,
key: Pleroma.Uploaders.S3, key: Pleroma.Uploaders.S3,

View File

@ -153,6 +153,7 @@
config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock
config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock
config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock
config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock
peer_module = peer_module =
if String.to_integer(System.otp_release()) >= 25 do if String.to_integer(System.otp_release()) >= 25 do

View File

@ -661,6 +661,19 @@ config :ex_aws, :s3,
host: "s3.eu-central-1.amazonaws.com" host: "s3.eu-central-1.amazonaws.com"
``` ```
#### Pleroma.Uploaders.IPFS
* `post_gateway_url`: URL with port of POST Gateway (unauthenticated)
* `get_gateway_url`: URL of public GET Gateway
Example:
```elixir
config :pleroma, Pleroma.Uploaders.IPFS,
post_gateway_url: "http://localhost:5001",
get_gateway_url: "http://{CID}.ipfs.mydomain.com"
```
### Upload filters ### Upload filters
#### Pleroma.Upload.Filter.AnonymizeFilename #### Pleroma.Upload.Filter.AnonymizeFilename

View File

@ -239,8 +239,12 @@ defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do
"" ""
end end
[base_url, path] if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do
|> Path.join() String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path)
else
[base_url, path]
|> Path.join()
end
end end
defp url_from_spec(_upload, _base_url, {:url, url}), do: url defp url_from_spec(_upload, _base_url, {:url, url}), do: url
@ -277,6 +281,9 @@ def base_url do
Path.join([upload_base_url, bucket_with_namespace]) Path.join([upload_base_url, bucket_with_namespace])
end end
Pleroma.Uploaders.IPFS ->
@config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
_ -> _ ->
public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
end end

View File

@ -0,0 +1,77 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Uploaders.IPFS do
@behaviour Pleroma.Uploaders.Uploader
require Logger
alias Tesla.Multipart
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
defp get_final_url(method) do
config = @config_impl.get([__MODULE__])
post_base_url = Keyword.get(config, :post_gateway_url)
Path.join([post_base_url, method])
end
def put_file_endpoint do
get_final_url("/api/v0/add")
end
def delete_file_endpoint do
get_final_url("/api/v0/files/rm")
end
@placeholder "{CID}"
def placeholder, do: @placeholder
@impl true
def get_file(file) do
b_url = Pleroma.Upload.base_url()
if String.contains?(b_url, @placeholder) do
{:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}}
else
{:error, "IPFS Get URL doesn't contain 'cid' placeholder"}
end
end
@impl true
def put_file(%Pleroma.Upload{} = upload) do
mp =
Multipart.new()
|> Multipart.add_content_type_param("charset=utf-8")
|> Multipart.add_file(upload.tempfile)
case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do
{:ok, ret} ->
case Jason.decode(ret.body) do
{:ok, ret} ->
if Map.has_key?(ret, "Hash") do
{:ok, {:file, ret["Hash"]}}
else
{:error, "JSON doesn't contain Hash key"}
end
error ->
Logger.error("#{__MODULE__}: #{inspect(error)}")
{:error, "JSON decode failed"}
end
error ->
Logger.error("#{__MODULE__}: #{inspect(error)}")
{:error, "IPFS Gateway upload failed"}
end
end
@impl true
def delete_file(file) do
case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do
{:ok, %{status: 204}} -> :ok
error -> {:error, inspect(error)}
end
end
end

View File

@ -0,0 +1,158 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Uploaders.IPFSTest do
use Pleroma.DataCase
alias Pleroma.Uploaders.IPFS
alias Tesla.Multipart
import ExUnit.CaptureLog
import Mock
import Mox
alias Pleroma.UnstubbedConfigMock, as: Config
describe "get_final_url" do
setup do
Config
|> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
[post_gateway_url: "http://localhost:5001"]
end)
:ok
end
test "it returns the final url for put_file" do
assert IPFS.put_file_endpoint() == "http://localhost:5001/api/v0/add"
end
test "it returns the final url for delete_file" do
assert IPFS.delete_file_endpoint() == "http://localhost:5001/api/v0/files/rm"
end
end
describe "get_file/1" do
setup do
Config
|> expect(:get, fn [Pleroma.Upload, :uploader] -> Pleroma.Uploaders.IPFS end)
|> expect(:get, fn [Pleroma.Upload, :base_url] -> nil end)
|> expect(:get, fn [Pleroma.Uploaders.IPFS, :public_endpoint] -> nil end)
:ok
end
test "it returns path to ipfs file with cid as subdomain" do
Config
|> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] ->
"https://{CID}.ipfs.mydomain.com"
end)
assert IPFS.get_file("testcid") == {
:ok,
{:url, "https://testcid.ipfs.mydomain.com"}
}
end
test "it returns path to ipfs file with cid as path" do
Config
|> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] ->
"https://ipfs.mydomain.com/ipfs/{CID}"
end)
assert IPFS.get_file("testcid") == {
:ok,
{:url, "https://ipfs.mydomain.com/ipfs/testcid"}
}
end
end
describe "put_file/1" do
setup do
Config
|> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
[post_gateway_url: "http://localhost:5001"]
end)
file_upload = %Pleroma.Upload{
name: "image-tet.jpg",
content_type: "image/jpeg",
path: "test_folder/image-tet.jpg",
tempfile: Path.absname("test/instance_static/add/shortcode.png")
}
mp =
Multipart.new()
|> Multipart.add_content_type_param("charset=utf-8")
|> Multipart.add_file(file_upload.tempfile)
[file_upload: file_upload, mp: mp]
end
test "save file", %{file_upload: file_upload} do
with_mock Pleroma.HTTP,
post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
{:ok,
%Tesla.Env{
status: 200,
body:
"{\"Name\":\"image-tet.jpg\",\"Size\":\"5000\", \"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}"
}}
end do
assert IPFS.put_file(file_upload) ==
{:ok, {:file, "bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi"}}
end
end
test "returns error", %{file_upload: file_upload} do
with_mock Pleroma.HTTP,
post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
{:error, "IPFS Gateway upload failed"}
end do
assert capture_log(fn ->
assert IPFS.put_file(file_upload) == {:error, "IPFS Gateway upload failed"}
end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}"
end
end
test "returns error if JSON decode fails", %{file_upload: file_upload} do
with_mock Pleroma.HTTP, [],
post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
{:ok, %Tesla.Env{status: 200, body: "invalid"}}
end do
assert capture_log(fn ->
assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"}
end) =~
"Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError"
end
end
test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do
with_mock Pleroma.HTTP, [],
post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
{:ok, %Tesla.Env{status: 200, body: "{\"key\": \"value\"}"}}
end do
assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"}
end
end
end
describe "delete_file/1" do
setup do
Config
|> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
[post_gateway_url: "http://localhost:5001"]
end)
:ok
end
test_with_mock "deletes file", Pleroma.HTTP,
post: fn "http://localhost:5001/api/v0/files/rm", "", [], params: [arg: "image.jpg"] ->
{:ok, %{status: 204}}
end do
assert :ok = IPFS.delete_file("image.jpg")
end
end
end