Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into qdrant-search-2
This commit is contained in:
commit
08e9d995f8
|
@ -1,8 +1,8 @@
|
|||
image: git.pleroma.social:5050/pleroma/pleroma/ci-base
|
||||
image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24
|
||||
|
||||
variables: &global_variables
|
||||
# Only used for the release
|
||||
ELIXIR_VER: 1.12.3
|
||||
ELIXIR_VER: 1.13.4
|
||||
POSTGRES_DB: pleroma_test
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
|
@ -72,7 +72,7 @@ check-changelog:
|
|||
tags:
|
||||
- amd64
|
||||
|
||||
build-1.12.3:
|
||||
build-1.13.4:
|
||||
extends:
|
||||
- .build_changes_policy
|
||||
- .using-ci-base
|
||||
|
@ -85,7 +85,7 @@ build-1.15.7-otp-25:
|
|||
- .build_changes_policy
|
||||
- .using-ci-base
|
||||
stage: build
|
||||
image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15
|
||||
image: git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25
|
||||
allow_failure: true
|
||||
script:
|
||||
- mix compile --force
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
ARG ELIXIR_IMG=hexpm/elixir
|
||||
ARG ELIXIR_VER=1.12.3
|
||||
ARG ERLANG_VER=24.2.1
|
||||
ARG ALPINE_VER=3.17.0
|
||||
ARG ELIXIR_VER=1.13.4
|
||||
ARG ERLANG_VER=24.3.4.15
|
||||
ARG ALPINE_VER=3.17.5
|
||||
|
||||
FROM ${ELIXIR_IMG}:${ELIXIR_VER}-erlang-${ERLANG_VER}-alpine-${ALPINE_VER} as build
|
||||
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
Elixir 1.13 is the minimum required version.
|
|
@ -0,0 +1 @@
|
|||
pleroma_ctl: Use realpath(1) instead of readlink(1)
|
|
@ -0,0 +1 @@
|
|||
Parsing of RichMedia TTLs for Amazon URLs when query parameters are nil
|
|
@ -0,0 +1 @@
|
|||
Monitoring of search backend health to control the processing of jobs in the search indexing Oban queue
|
|
@ -0,0 +1,8 @@
|
|||
FROM elixir:1.13.4-otp-24
|
||||
|
||||
# Single RUN statement, otherwise intermediate images are created
|
||||
# https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#run
|
||||
RUN apt-get update &&\
|
||||
apt-get install -y libmagic-dev cmake libimage-exiftool-perl ffmpeg &&\
|
||||
mix local.hex --force &&\
|
||||
mix local.rebar --force
|
|
@ -0,0 +1 @@
|
|||
docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.13.4-otp-24 --push .
|
|
@ -1 +1 @@
|
|||
docker buildx build --platform linux/amd64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push .
|
||||
docker buildx build --platform linux/amd64,linux/arm64 -t git.pleroma.social:5050/pleroma/pleroma/ci-base:elixir-1.15-otp25 --push .
|
||||
|
|
|
@ -579,7 +579,7 @@
|
|||
attachments_cleanup: 1,
|
||||
new_users_digest: 1,
|
||||
mute_expire: 5,
|
||||
search_indexing: 10,
|
||||
search_indexing: [limit: 10, paused: true],
|
||||
rich_media_expiration: 2
|
||||
],
|
||||
plugins: [Oban.Plugins.Pruner],
|
||||
|
|
|
@ -295,9 +295,7 @@ See [Admin-API](admin_api.md)
|
|||
"id": "9umDrYheeY451cQnEe",
|
||||
"name": "Read later",
|
||||
"emoji": "🕓",
|
||||
"source": {
|
||||
"emoji": "🕓"
|
||||
}
|
||||
"emoji_url": null
|
||||
}
|
||||
]
|
||||
```
|
||||
|
|
|
@ -14,7 +14,7 @@ Note: This article is potentially outdated because at this time we may not have
|
|||
|
||||
- PostgreSQL 11.0以上 (Ubuntu16.04では9.5しか提供されていないので,[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください)
|
||||
- `postgresql-contrib` 11.0以上 (同上)
|
||||
- Elixir 1.8 以上 ([Debianのリポジトリからインストールしないこと!!! ここからインストールすること!](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
|
||||
- Elixir 1.13 以上 ([Debianのリポジトリからインストールしないこと!!! ここからインストールすること!](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
|
||||
- `erlang-dev`
|
||||
- `erlang-nox`
|
||||
- `git`
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
## Required dependencies
|
||||
|
||||
* PostgreSQL >=11.0
|
||||
* Elixir >=1.11.0 <1.15
|
||||
* Elixir >=1.13.0 <1.15
|
||||
* Erlang OTP >=22.2.0 (supported: <27)
|
||||
* git
|
||||
* file / libmagic
|
||||
|
|
|
@ -109,7 +109,8 @@ def start(_type, _args) do
|
|||
streamer_registry() ++
|
||||
background_migrators() ++
|
||||
shout_child(shout_enabled?()) ++
|
||||
[Pleroma.Gopher.Server]
|
||||
[Pleroma.Gopher.Server] ++
|
||||
[Pleroma.Search.Healthcheck]
|
||||
|
||||
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
|
||||
# for other strategies and supported options
|
||||
|
|
|
@ -10,8 +10,12 @@ def remove_from_index(%Pleroma.Object{id: object_id}) do
|
|||
end
|
||||
|
||||
def search(query, options) do
|
||||
search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
|
||||
|
||||
search_module = Pleroma.Config.get([Pleroma.Search, :module])
|
||||
search_module.search(options[:for_user], query, options)
|
||||
end
|
||||
|
||||
def healthcheck_endpoints do
|
||||
search_module = Pleroma.Config.get([Pleroma.Search, :module])
|
||||
search_module.healthcheck_endpoints
|
||||
end
|
||||
end
|
||||
|
|
|
@ -54,6 +54,9 @@ def create_index, do: :ok
|
|||
@impl true
|
||||
def drop_index, do: :ok
|
||||
|
||||
@impl true
|
||||
def healthcheck_endpoints, do: nil
|
||||
|
||||
def maybe_restrict_author(query, %User{} = author) do
|
||||
Activity.Queries.by_author(query, author)
|
||||
end
|
||||
|
|
|
@ -0,0 +1,86 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
defmodule Pleroma.Search.Healthcheck do
|
||||
@doc """
|
||||
Monitors health of search backend to control processing of events based on health and availability.
|
||||
"""
|
||||
use GenServer
|
||||
require Logger
|
||||
|
||||
@queue :search_indexing
|
||||
@tick :timer.seconds(5)
|
||||
@timeout :timer.seconds(2)
|
||||
|
||||
def start_link(_) do
|
||||
GenServer.start_link(__MODULE__, [], name: __MODULE__)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def init(_) do
|
||||
state = %{healthy: false}
|
||||
{:ok, state, {:continue, :start}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_continue(:start, state) do
|
||||
tick()
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(:check, state) do
|
||||
urls = Pleroma.Search.healthcheck_endpoints()
|
||||
|
||||
new_state =
|
||||
if check(urls) do
|
||||
Oban.resume_queue(queue: @queue)
|
||||
Map.put(state, :healthy, true)
|
||||
else
|
||||
Oban.pause_queue(queue: @queue)
|
||||
Map.put(state, :healthy, false)
|
||||
end
|
||||
|
||||
maybe_log_state_change(state, new_state)
|
||||
|
||||
tick()
|
||||
{:noreply, new_state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call(:state, _from, state) do
|
||||
{:reply, state, state, :hibernate}
|
||||
end
|
||||
|
||||
def state, do: GenServer.call(__MODULE__, :state)
|
||||
|
||||
def check([]), do: true
|
||||
|
||||
def check(urls) when is_list(urls) do
|
||||
Enum.all?(
|
||||
urls,
|
||||
fn url ->
|
||||
case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do
|
||||
{:ok, %{status: 200}} -> true
|
||||
_ -> false
|
||||
end
|
||||
end
|
||||
)
|
||||
end
|
||||
|
||||
def check(_), do: true
|
||||
|
||||
defp tick do
|
||||
Process.send_after(self(), :check, @tick)
|
||||
end
|
||||
|
||||
defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do
|
||||
Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure")
|
||||
end
|
||||
|
||||
defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do
|
||||
Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass")
|
||||
end
|
||||
|
||||
defp maybe_log_state_change(_, _), do: :ok
|
||||
end
|
|
@ -184,4 +184,15 @@ def add_to_index(activity) do
|
|||
def remove_from_index(object) do
|
||||
meili_delete("/indexes/objects/documents/#{object.id}")
|
||||
end
|
||||
|
||||
@impl true
|
||||
def healthcheck_endpoints do
|
||||
endpoint =
|
||||
Config.get([Pleroma.Search.Meilisearch, :url])
|
||||
|> URI.parse()
|
||||
|> Map.put(:path, "/health")
|
||||
|> URI.to_string()
|
||||
|
||||
[endpoint]
|
||||
end
|
||||
end
|
||||
|
|
|
@ -31,4 +31,12 @@ defmodule Pleroma.Search.SearchBackend do
|
|||
Drop the index
|
||||
"""
|
||||
@callback drop_index() :: :ok | {:error, any()}
|
||||
|
||||
@doc """
|
||||
Healthcheck endpoints of search backend infrastructure to monitor for controlling
|
||||
processing of jobs in the Oban queue.
|
||||
|
||||
It is expected a 200 response is healthy and other responses are unhealthy.
|
||||
"""
|
||||
@callback healthcheck_endpoints :: list() | nil
|
||||
end
|
||||
|
|
|
@ -23,7 +23,7 @@ defp aws_signed_url?(image) when is_binary(image) and image != "" do
|
|||
%URI{host: host, query: query} = URI.parse(image)
|
||||
|
||||
is_binary(host) and String.contains?(host, "amazonaws.com") and
|
||||
String.contains?(query, "X-Amz-Expires")
|
||||
is_binary(query) and String.contains?(query, "X-Amz-Expires")
|
||||
end
|
||||
|
||||
defp aws_signed_url?(_), do: nil
|
||||
|
|
2
mix.exs
2
mix.exs
|
@ -5,7 +5,7 @@ def project do
|
|||
[
|
||||
app: :pleroma,
|
||||
version: version("2.6.52"),
|
||||
elixir: "~> 1.11",
|
||||
elixir: "~> 1.13",
|
||||
elixirc_paths: elixirc_paths(Mix.env()),
|
||||
compilers: Mix.compilers(),
|
||||
elixirc_options: [warnings_as_errors: warnings_as_errors()],
|
||||
|
|
|
@ -134,7 +134,7 @@ if [ -z "$1" ] || [ "$1" = "help" ]; then
|
|||
|
||||
"
|
||||
else
|
||||
SCRIPT=$(readlink -f "$0")
|
||||
SCRIPT=$(realpath "$0")
|
||||
SCRIPTPATH=$(dirname "$SCRIPT")
|
||||
|
||||
FULL_ARGS="$*"
|
||||
|
|
|
@ -0,0 +1,49 @@
|
|||
# Pleroma: A lightweight social networking server
|
||||
# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
|
||||
# SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
defmodule Pleroma.Search.HealthcheckTest do
|
||||
use Pleroma.DataCase
|
||||
|
||||
import Tesla.Mock
|
||||
|
||||
alias Pleroma.Search.Healthcheck
|
||||
|
||||
@good1 "http://good1.example.com/healthz"
|
||||
@good2 "http://good2.example.com/health"
|
||||
@bad "http://bad.example.com/healthy"
|
||||
|
||||
setup do
|
||||
mock(fn
|
||||
%{method: :get, url: @good1} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: ""
|
||||
}
|
||||
|
||||
%{method: :get, url: @good2} ->
|
||||
%Tesla.Env{
|
||||
status: 200,
|
||||
body: ""
|
||||
}
|
||||
|
||||
%{method: :get, url: @bad} ->
|
||||
%Tesla.Env{
|
||||
status: 503,
|
||||
body: ""
|
||||
}
|
||||
end)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
test "true for 200 responses" do
|
||||
assert Healthcheck.check([@good1])
|
||||
assert Healthcheck.check([@good1, @good2])
|
||||
end
|
||||
|
||||
test "false if any response is not a 200" do
|
||||
refute Healthcheck.check([@bad])
|
||||
refute Healthcheck.check([@good1, @bad])
|
||||
end
|
||||
end
|
|
@ -10,6 +10,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
|
|||
|
||||
alias Pleroma.UnstubbedConfigMock, as: ConfigMock
|
||||
alias Pleroma.Web.RichMedia.Card
|
||||
alias Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl
|
||||
|
||||
setup do
|
||||
ConfigMock
|
||||
|
@ -82,6 +83,12 @@ test "s3 signed url is parsed and correct ttl is set for rich media" do
|
|||
assert DateTime.diff(scheduled_at, timestamp_dt) == valid_till
|
||||
end
|
||||
|
||||
test "AWS URL for an image without expiration works" do
|
||||
og_data = %{"image" => "https://amazonaws.com/image.png"}
|
||||
|
||||
assert is_nil(AwsSignedUrl.ttl(og_data, ""))
|
||||
end
|
||||
|
||||
defp construct_s3_url(timestamp, valid_till) do
|
||||
"https://pleroma.s3.ap-southeast-1.amazonaws.com/sachin%20%281%29%20_a%20-%25%2Aasdasd%20BNN%20bnnn%20.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIBLWWK6RGDQXDLJQ%2F20190716%2Fap-southeast-1%2Fs3%2Faws4_request&X-Amz-Date=#{timestamp}&X-Amz-Expires=#{valid_till}&X-Amz-Signature=04ffd6b98634f4b1bbabc62e0fac4879093cd54a6eed24fe8eb38e8369526bbf&X-Amz-SignedHeaders=host"
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue