Merge branch 'simpler-oban-queues' into 'develop'

Oban queue simplification

See merge request pleroma/pleroma!4123
This commit is contained in:
feld 2024-05-27 19:02:53 +00:00
commit 38db406ce4
19 changed files with 63 additions and 34 deletions

View File

@ -0,0 +1 @@
Oban queues have refactored to simplify the queue design

View File

@ -582,24 +582,14 @@
log: false, log: false,
queues: [ queues: [
activity_expiration: 10, activity_expiration: 10,
token_expiration: 5,
filter_expiration: 1,
backup: 1,
federator_incoming: 5, federator_incoming: 5,
federator_outgoing: 5, federator_outgoing: 5,
ingestion_queue: 50, ingestion_queue: 50,
web_push: 50, web_push: 50,
mailer: 10,
transmogrifier: 20, transmogrifier: 20,
scheduled_activities: 10,
poll_notifications: 10,
background: 5, background: 5,
remote_fetcher: 2,
attachments_cleanup: 1,
new_users_digest: 1,
mute_expire: 5,
search_indexing: [limit: 10, paused: true], search_indexing: [limit: 10, paused: true],
rich_media_expiration: 2 slow: 1
], ],
plugins: [Oban.Plugins.Pruner], plugins: [Oban.Plugins.Pruner],
crontab: [ crontab: [

View File

@ -204,7 +204,7 @@ def due_activities(offset \\ 0) do
def job_query(scheduled_activity_id) do def job_query(scheduled_activity_id) do
from(j in Oban.Job, from(j in Oban.Job,
where: j.queue == "scheduled_activities", where: j.queue == "federator_outgoing",
where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id)) where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id))
) )
end end

View File

@ -44,7 +44,7 @@ def incoming_ap_doc(%{params: params, req_headers: req_headers}) do
end end
def incoming_ap_doc(%{"type" => "Delete"} = params) do def incoming_ap_doc(%{"type" => "Delete"} = params) do
ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3) ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow)
end end
def incoming_ap_doc(params) do def incoming_ap_doc(params) do

View File

@ -8,7 +8,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Repo alias Pleroma.Repo
use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup" use Pleroma.Workers.WorkerHelper, queue: "slow"
@impl Oban.Worker @impl Oban.Worker
def perform(%Job{ def perform(%Job{

View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.BackupWorker do defmodule Pleroma.Workers.BackupWorker do
use Oban.Worker, queue: :backup, max_attempts: 1 use Oban.Worker, queue: :slow, max_attempts: 1
alias Oban.Job alias Oban.Job
alias Pleroma.User.Backup alias Pleroma.User.Backup

View File

@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do
import Ecto.Query import Ecto.Query
use Pleroma.Workers.WorkerHelper, queue: "mailer" use Pleroma.Workers.WorkerHelper, queue: "background"
@impl Oban.Worker @impl Oban.Worker
def perform(_job) do def perform(_job) do

View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.MailerWorker do defmodule Pleroma.Workers.MailerWorker do
use Pleroma.Workers.WorkerHelper, queue: "mailer" use Pleroma.Workers.WorkerHelper, queue: "background"
@impl Oban.Worker @impl Oban.Worker
def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do

View File

@ -3,7 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.MuteExpireWorker do defmodule Pleroma.Workers.MuteExpireWorker do
use Pleroma.Workers.WorkerHelper, queue: "mute_expire" use Pleroma.Workers.WorkerHelper, queue: "background"
@impl Oban.Worker @impl Oban.Worker
def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do

View File

@ -6,7 +6,7 @@ defmodule Pleroma.Workers.PollWorker do
@moduledoc """ @moduledoc """
Generates notifications when a poll ends. Generates notifications when a poll ends.
""" """
use Pleroma.Workers.WorkerHelper, queue: "poll_notifications" use Pleroma.Workers.WorkerHelper, queue: "background"
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.Notification alias Pleroma.Notification

View File

@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
Worker which purges expired activity. Worker which purges expired activity.
""" """
use Oban.Worker, queue: :activity_expiration, max_attempts: 1, unique: [period: :infinity] use Oban.Worker, queue: :slow, max_attempts: 1, unique: [period: :infinity]
import Ecto.Query import Ecto.Query
@ -59,7 +59,7 @@ defp find_user(ap_id) do
def get_expiration(id) do def get_expiration(id) do
from(j in Oban.Job, from(j in Oban.Job,
where: j.state == "scheduled", where: j.state == "scheduled",
where: j.queue == "activity_expiration", where: j.queue == "slow",
where: fragment("?->>'activity_id' = ?", j.args, ^id) where: fragment("?->>'activity_id' = ?", j.args, ^id)
) )
|> Pleroma.Repo.one() |> Pleroma.Repo.one()

View File

@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do
Worker which purges expired filters Worker which purges expired filters
""" """
use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [period: :infinity] use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: :infinity]
import Ecto.Query import Ecto.Query
@ -38,7 +38,7 @@ def timeout(_job), do: :timer.seconds(5)
def get_expiration(id) do def get_expiration(id) do
from(j in Job, from(j in Job,
where: j.state == "scheduled", where: j.state == "scheduled",
where: j.queue == "filter_expiration", where: j.queue == "background",
where: fragment("?->'filter_id' = ?", j.args, ^id) where: fragment("?->'filter_id' = ?", j.args, ^id)
) )
|> Repo.one() |> Repo.one()

View File

@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredToken do
Worker which purges expired OAuth tokens Worker which purges expired OAuth tokens
""" """
use Oban.Worker, queue: :token_expiration, max_attempts: 1 use Oban.Worker, queue: :background, max_attempts: 1
@spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) :: @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) ::
{:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()} {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}

View File

@ -5,7 +5,7 @@
defmodule Pleroma.Workers.RemoteFetcherWorker do defmodule Pleroma.Workers.RemoteFetcherWorker do
alias Pleroma.Object.Fetcher alias Pleroma.Object.Fetcher
use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher" use Pleroma.Workers.WorkerHelper, queue: "background"
@impl Oban.Worker @impl Oban.Worker
def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do

View File

@ -6,7 +6,7 @@ defmodule Pleroma.Workers.RichMediaExpirationWorker do
alias Pleroma.Web.RichMedia.Card alias Pleroma.Web.RichMedia.Card
use Oban.Worker, use Oban.Worker,
queue: :rich_media_expiration queue: :background
@impl Oban.Worker @impl Oban.Worker
def perform(%Job{args: %{"url" => url} = _args}) do def perform(%Job{args: %{"url" => url} = _args}) do

View File

@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do
The worker to post scheduled activity. The worker to post scheduled activity.
""" """
use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing"
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.ScheduledActivity alias Pleroma.ScheduledActivity

View File

@ -0,0 +1,32 @@
defmodule Pleroma.Repo.Migrations.ObanQueuesRefactor do
use Ecto.Migration
@changed_queues [
{"attachments_cleanup", "slow"},
{"mailer", "background"},
{"mute_expire", "background"},
{"poll_notifications", "background"},
{"activity_expiration", "slow"},
{"filter_expiration", "background"},
{"token_expiration", "background"},
{"remote_fetcher", "background"},
{"rich_media_expiration", "background"}
]
def up do
Enum.each(@changed_queues, fn {old, new} ->
execute("UPDATE oban_jobs SET queue = '#{new}' WHERE queue = '#{old}';")
end)
# Handled special as reverting this would not be ideal and leaving it is harmless
execute(
"UPDATE oban_jobs SET queue = 'federator_outgoing' WHERE queue = 'scheduled_activities';"
)
end
def down do
# Just move all slow queue jobs to background queue if we are reverting
# as the slow queue will not be processing jobs
execute("UPDATE oban_jobs SET queue = 'background' WHERE queue = 'slow';")
end
end

View File

@ -31,8 +31,7 @@ test "scheduled activities with jobs when ScheduledActivity enabled" do
{:ok, sa1} = ScheduledActivity.create(user, attrs) {:ok, sa1} = ScheduledActivity.create(user, attrs)
{:ok, sa2} = ScheduledActivity.create(user, attrs) {:ok, sa2} = ScheduledActivity.create(user, attrs)
jobs = jobs = Repo.all(from(j in Oban.Job, where: j.queue == "federator_outgoing", select: j.args))
Repo.all(from(j in Oban.Job, where: j.queue == "scheduled_activities", select: j.args))
assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}] assert jobs == [%{"activity_id" => sa1.id}, %{"activity_id" => sa2.id}]
end end

View File

@ -3,6 +3,7 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do defmodule Pleroma.Web.MastodonAPI.ScheduledActivityControllerTest do
use Oban.Testing, repo: Pleroma.Repo
use Pleroma.Web.ConnCase, async: true use Pleroma.Web.ConnCase, async: true
alias Pleroma.Repo alias Pleroma.Repo
@ -78,7 +79,7 @@ test "updates a scheduled activity" do
} }
) )
job = Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) job = Repo.one(from(j in Oban.Job, where: j.queue == "federator_outgoing"))
assert job.args == %{"activity_id" => scheduled_activity.id} assert job.args == %{"activity_id" => scheduled_activity.id}
assert DateTime.truncate(job.scheduled_at, :second) == to_datetime(scheduled_at) assert DateTime.truncate(job.scheduled_at, :second) == to_datetime(scheduled_at)
@ -124,9 +125,11 @@ test "deletes a scheduled activity" do
} }
) )
job = Repo.one(from(j in Oban.Job, where: j.queue == "scheduled_activities")) assert_enqueued(
worker: Pleroma.Workers.ScheduledActivityWorker,
assert job.args == %{"activity_id" => scheduled_activity.id} args: %{"activity_id" => scheduled_activity.id},
queue: :federator_outgoing
)
res_conn = res_conn =
conn conn
@ -135,7 +138,11 @@ test "deletes a scheduled activity" do
assert %{} = json_response_and_validate_schema(res_conn, 200) assert %{} = json_response_and_validate_schema(res_conn, 200)
refute Repo.get(ScheduledActivity, scheduled_activity.id) refute Repo.get(ScheduledActivity, scheduled_activity.id)
refute Repo.get(Oban.Job, job.id)
refute_enqueued(
worker: Pleroma.Workers.ScheduledActivityWorker,
args: %{"activity_id" => scheduled_activity.id}
)
res_conn = res_conn =
conn conn