SimpleMRF silence: optimize, work okay with nil values in addressing

lib/pleroma/following_relationship.ex

@@ -95,7 +95,11 @@ def followers_query(%User{} = user) do
     |> where([r], r.state == ^:follow_accept)
   end
 
-  def followers_ap_ids(%User{} = user, from_ap_ids \\ nil) do
+  def followers_ap_ids(user, from_ap_ids \\ nil)
+
+  def followers_ap_ids(_, []), do: []
+
+  def followers_ap_ids(%User{} = user, from_ap_ids) do
     query =
       user
       |> followers_query()

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -117,14 +117,15 @@ defp check_silence(%{host: actor_host} = _actor_info, object) do
     object =
       with true <- MRF.subdomain_match?(silence, actor_host),
            user <- User.get_cached_by_ap_id(object["actor"]) do
-        to =
-          FollowingRelationship.followers_ap_ids(user, Map.get(object, "to", [])) ++
-            [user.follower_address]
+        # Don't use Map.get/3 intentionally, these must not be nil
+        fixed_to = object["to"] || []
+        fixed_cc = object["cc"] || []
 
-        cc = FollowingRelationship.followers_ap_ids(user, Map.get(object, "cc", []))
+        to = FollowingRelationship.followers_ap_ids(user, fixed_to)
+        cc = FollowingRelationship.followers_ap_ids(user, fixed_cc)
 
         object
-        |> Map.put("to", to)
+        |> Map.put("to", [user.follower_address] ++ to)
         |> Map.put("cc", cc)
       else
         _ -> object
@@ -133,8 +134,6 @@ defp check_silence(%{host: actor_host} = _actor_info, object) do
     {:ok, object}
   end
 
-  defp check_silence(_actor_info, object), do: {:ok, object}
-
   defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do
     report_removal =
       Config.get([:mrf_simple, :report_removal])