Merge branch 'ipfs_uploader' into 'develop'
feat: simple, but not stupid, uploader for IPFS See merge request pleroma/pleroma!3654
This commit is contained in:
commit
d11ba9e85b
|
@ -0,0 +1 @@
|
||||||
|
Uploader: Add support for uploading attachments using IPFS
|
|
@ -82,6 +82,10 @@
|
||||||
# region: "us-east-1", # may be required for Amazon AWS
|
# region: "us-east-1", # may be required for Amazon AWS
|
||||||
scheme: "https://"
|
scheme: "https://"
|
||||||
|
|
||||||
|
config :pleroma, Pleroma.Uploaders.IPFS,
|
||||||
|
post_gateway_url: nil,
|
||||||
|
get_gateway_url: nil
|
||||||
|
|
||||||
config :pleroma, :emoji,
|
config :pleroma, :emoji,
|
||||||
shortcode_globs: ["/emoji/custom/**/*.png"],
|
shortcode_globs: ["/emoji/custom/**/*.png"],
|
||||||
pack_extensions: [".png", ".gif"],
|
pack_extensions: [".png", ".gif"],
|
||||||
|
|
|
@ -136,6 +136,31 @@
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
%{
|
||||||
|
group: :pleroma,
|
||||||
|
key: Pleroma.Uploaders.IPFS,
|
||||||
|
type: :group,
|
||||||
|
description: "IPFS uploader-related settings",
|
||||||
|
children: [
|
||||||
|
%{
|
||||||
|
key: :get_gateway_url,
|
||||||
|
type: :string,
|
||||||
|
description: "GET Gateway URL",
|
||||||
|
suggestions: [
|
||||||
|
"https://ipfs.mydomain.com/{CID}",
|
||||||
|
"https://{CID}.ipfs.mydomain.com/"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
%{
|
||||||
|
key: :post_gateway_url,
|
||||||
|
type: :string,
|
||||||
|
description: "POST Gateway URL",
|
||||||
|
suggestions: [
|
||||||
|
"http://localhost:5001/"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
%{
|
%{
|
||||||
group: :pleroma,
|
group: :pleroma,
|
||||||
key: Pleroma.Uploaders.S3,
|
key: Pleroma.Uploaders.S3,
|
||||||
|
|
|
@ -153,6 +153,7 @@
|
||||||
config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock
|
config :pleroma, Pleroma.Upload, config_impl: Pleroma.UnstubbedConfigMock
|
||||||
config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock
|
config :pleroma, Pleroma.ScheduledActivity, config_impl: Pleroma.UnstubbedConfigMock
|
||||||
config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock
|
config :pleroma, Pleroma.Web.RichMedia.Helpers, config_impl: Pleroma.StaticStubbedConfigMock
|
||||||
|
config :pleroma, Pleroma.Uploaders.IPFS, config_impl: Pleroma.UnstubbedConfigMock
|
||||||
|
|
||||||
peer_module =
|
peer_module =
|
||||||
if String.to_integer(System.otp_release()) >= 25 do
|
if String.to_integer(System.otp_release()) >= 25 do
|
||||||
|
|
|
@ -661,6 +661,19 @@ config :ex_aws, :s3,
|
||||||
host: "s3.eu-central-1.amazonaws.com"
|
host: "s3.eu-central-1.amazonaws.com"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### Pleroma.Uploaders.IPFS
|
||||||
|
|
||||||
|
* `post_gateway_url`: URL with port of POST Gateway (unauthenticated)
|
||||||
|
* `get_gateway_url`: URL of public GET Gateway
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
```elixir
|
||||||
|
config :pleroma, Pleroma.Uploaders.IPFS,
|
||||||
|
post_gateway_url: "http://localhost:5001",
|
||||||
|
get_gateway_url: "http://{CID}.ipfs.mydomain.com"
|
||||||
|
```
|
||||||
|
|
||||||
### Upload filters
|
### Upload filters
|
||||||
|
|
||||||
#### Pleroma.Upload.Filter.AnonymizeFilename
|
#### Pleroma.Upload.Filter.AnonymizeFilename
|
||||||
|
|
|
@ -239,8 +239,12 @@ defp url_from_spec(%__MODULE__{name: name}, base_url, {:file, path}) do
|
||||||
""
|
""
|
||||||
end
|
end
|
||||||
|
|
||||||
[base_url, path]
|
if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do
|
||||||
|> Path.join()
|
String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path)
|
||||||
|
else
|
||||||
|
[base_url, path]
|
||||||
|
|> Path.join()
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
defp url_from_spec(_upload, _base_url, {:url, url}), do: url
|
defp url_from_spec(_upload, _base_url, {:url, url}), do: url
|
||||||
|
@ -277,6 +281,9 @@ def base_url do
|
||||||
Path.join([upload_base_url, bucket_with_namespace])
|
Path.join([upload_base_url, bucket_with_namespace])
|
||||||
end
|
end
|
||||||
|
|
||||||
|
Pleroma.Uploaders.IPFS ->
|
||||||
|
@config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url])
|
||||||
|
|
||||||
_ ->
|
_ ->
|
||||||
public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
|
public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,77 @@
|
||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Uploaders.IPFS do
|
||||||
|
@behaviour Pleroma.Uploaders.Uploader
|
||||||
|
require Logger
|
||||||
|
|
||||||
|
alias Tesla.Multipart
|
||||||
|
|
||||||
|
@config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
|
||||||
|
|
||||||
|
defp get_final_url(method) do
|
||||||
|
config = @config_impl.get([__MODULE__])
|
||||||
|
post_base_url = Keyword.get(config, :post_gateway_url)
|
||||||
|
|
||||||
|
Path.join([post_base_url, method])
|
||||||
|
end
|
||||||
|
|
||||||
|
def put_file_endpoint do
|
||||||
|
get_final_url("/api/v0/add")
|
||||||
|
end
|
||||||
|
|
||||||
|
def delete_file_endpoint do
|
||||||
|
get_final_url("/api/v0/files/rm")
|
||||||
|
end
|
||||||
|
|
||||||
|
@placeholder "{CID}"
|
||||||
|
def placeholder, do: @placeholder
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def get_file(file) do
|
||||||
|
b_url = Pleroma.Upload.base_url()
|
||||||
|
|
||||||
|
if String.contains?(b_url, @placeholder) do
|
||||||
|
{:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}}
|
||||||
|
else
|
||||||
|
{:error, "IPFS Get URL doesn't contain 'cid' placeholder"}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def put_file(%Pleroma.Upload{} = upload) do
|
||||||
|
mp =
|
||||||
|
Multipart.new()
|
||||||
|
|> Multipart.add_content_type_param("charset=utf-8")
|
||||||
|
|> Multipart.add_file(upload.tempfile)
|
||||||
|
|
||||||
|
case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do
|
||||||
|
{:ok, ret} ->
|
||||||
|
case Jason.decode(ret.body) do
|
||||||
|
{:ok, ret} ->
|
||||||
|
if Map.has_key?(ret, "Hash") do
|
||||||
|
{:ok, {:file, ret["Hash"]}}
|
||||||
|
else
|
||||||
|
{:error, "JSON doesn't contain Hash key"}
|
||||||
|
end
|
||||||
|
|
||||||
|
error ->
|
||||||
|
Logger.error("#{__MODULE__}: #{inspect(error)}")
|
||||||
|
{:error, "JSON decode failed"}
|
||||||
|
end
|
||||||
|
|
||||||
|
error ->
|
||||||
|
Logger.error("#{__MODULE__}: #{inspect(error)}")
|
||||||
|
{:error, "IPFS Gateway upload failed"}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
@impl true
|
||||||
|
def delete_file(file) do
|
||||||
|
case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do
|
||||||
|
{:ok, %{status: 204}} -> :ok
|
||||||
|
error -> {:error, inspect(error)}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,158 @@
|
||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Uploaders.IPFSTest do
|
||||||
|
use Pleroma.DataCase
|
||||||
|
|
||||||
|
alias Pleroma.Uploaders.IPFS
|
||||||
|
alias Tesla.Multipart
|
||||||
|
|
||||||
|
import ExUnit.CaptureLog
|
||||||
|
import Mock
|
||||||
|
import Mox
|
||||||
|
|
||||||
|
alias Pleroma.UnstubbedConfigMock, as: Config
|
||||||
|
|
||||||
|
describe "get_final_url" do
|
||||||
|
setup do
|
||||||
|
Config
|
||||||
|
|> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
|
||||||
|
[post_gateway_url: "http://localhost:5001"]
|
||||||
|
end)
|
||||||
|
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
test "it returns the final url for put_file" do
|
||||||
|
assert IPFS.put_file_endpoint() == "http://localhost:5001/api/v0/add"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "it returns the final url for delete_file" do
|
||||||
|
assert IPFS.delete_file_endpoint() == "http://localhost:5001/api/v0/files/rm"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "get_file/1" do
|
||||||
|
setup do
|
||||||
|
Config
|
||||||
|
|> expect(:get, fn [Pleroma.Upload, :uploader] -> Pleroma.Uploaders.IPFS end)
|
||||||
|
|> expect(:get, fn [Pleroma.Upload, :base_url] -> nil end)
|
||||||
|
|> expect(:get, fn [Pleroma.Uploaders.IPFS, :public_endpoint] -> nil end)
|
||||||
|
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
test "it returns path to ipfs file with cid as subdomain" do
|
||||||
|
Config
|
||||||
|
|> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] ->
|
||||||
|
"https://{CID}.ipfs.mydomain.com"
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert IPFS.get_file("testcid") == {
|
||||||
|
:ok,
|
||||||
|
{:url, "https://testcid.ipfs.mydomain.com"}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
test "it returns path to ipfs file with cid as path" do
|
||||||
|
Config
|
||||||
|
|> expect(:get, fn [Pleroma.Uploaders.IPFS, :get_gateway_url] ->
|
||||||
|
"https://ipfs.mydomain.com/ipfs/{CID}"
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert IPFS.get_file("testcid") == {
|
||||||
|
:ok,
|
||||||
|
{:url, "https://ipfs.mydomain.com/ipfs/testcid"}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "put_file/1" do
|
||||||
|
setup do
|
||||||
|
Config
|
||||||
|
|> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
|
||||||
|
[post_gateway_url: "http://localhost:5001"]
|
||||||
|
end)
|
||||||
|
|
||||||
|
file_upload = %Pleroma.Upload{
|
||||||
|
name: "image-tet.jpg",
|
||||||
|
content_type: "image/jpeg",
|
||||||
|
path: "test_folder/image-tet.jpg",
|
||||||
|
tempfile: Path.absname("test/instance_static/add/shortcode.png")
|
||||||
|
}
|
||||||
|
|
||||||
|
mp =
|
||||||
|
Multipart.new()
|
||||||
|
|> Multipart.add_content_type_param("charset=utf-8")
|
||||||
|
|> Multipart.add_file(file_upload.tempfile)
|
||||||
|
|
||||||
|
[file_upload: file_upload, mp: mp]
|
||||||
|
end
|
||||||
|
|
||||||
|
test "save file", %{file_upload: file_upload} do
|
||||||
|
with_mock Pleroma.HTTP,
|
||||||
|
post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
|
||||||
|
{:ok,
|
||||||
|
%Tesla.Env{
|
||||||
|
status: 200,
|
||||||
|
body:
|
||||||
|
"{\"Name\":\"image-tet.jpg\",\"Size\":\"5000\", \"Hash\":\"bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi\"}"
|
||||||
|
}}
|
||||||
|
end do
|
||||||
|
assert IPFS.put_file(file_upload) ==
|
||||||
|
{:ok, {:file, "bafybeicrh7ltzx52yxcwrvxxckfmwhqdgsb6qym6dxqm2a4ymsakeshwoi"}}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns error", %{file_upload: file_upload} do
|
||||||
|
with_mock Pleroma.HTTP,
|
||||||
|
post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
|
||||||
|
{:error, "IPFS Gateway upload failed"}
|
||||||
|
end do
|
||||||
|
assert capture_log(fn ->
|
||||||
|
assert IPFS.put_file(file_upload) == {:error, "IPFS Gateway upload failed"}
|
||||||
|
end) =~ "Elixir.Pleroma.Uploaders.IPFS: {:error, \"IPFS Gateway upload failed\"}"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns error if JSON decode fails", %{file_upload: file_upload} do
|
||||||
|
with_mock Pleroma.HTTP, [],
|
||||||
|
post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
|
||||||
|
{:ok, %Tesla.Env{status: 200, body: "invalid"}}
|
||||||
|
end do
|
||||||
|
assert capture_log(fn ->
|
||||||
|
assert IPFS.put_file(file_upload) == {:error, "JSON decode failed"}
|
||||||
|
end) =~
|
||||||
|
"Elixir.Pleroma.Uploaders.IPFS: {:error, %Jason.DecodeError"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns error if JSON body doesn't contain Hash key", %{file_upload: file_upload} do
|
||||||
|
with_mock Pleroma.HTTP, [],
|
||||||
|
post: fn "http://localhost:5001/api/v0/add", _mp, [], params: ["cid-version": "1"] ->
|
||||||
|
{:ok, %Tesla.Env{status: 200, body: "{\"key\": \"value\"}"}}
|
||||||
|
end do
|
||||||
|
assert IPFS.put_file(file_upload) == {:error, "JSON doesn't contain Hash key"}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "delete_file/1" do
|
||||||
|
setup do
|
||||||
|
Config
|
||||||
|
|> expect(:get, fn [Pleroma.Uploaders.IPFS] ->
|
||||||
|
[post_gateway_url: "http://localhost:5001"]
|
||||||
|
end)
|
||||||
|
|
||||||
|
:ok
|
||||||
|
end
|
||||||
|
|
||||||
|
test_with_mock "deletes file", Pleroma.HTTP,
|
||||||
|
post: fn "http://localhost:5001/api/v0/files/rm", "", [], params: [arg: "image.jpg"] ->
|
||||||
|
{:ok, %{status: 204}}
|
||||||
|
end do
|
||||||
|
assert :ok = IPFS.delete_file("image.jpg")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
Loading…
Reference in New Issue