lists: Update domains in batches of 100

The asyncpg driver does not support an infinite amount of arguments in a
single query.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/dbl/lists.py b/src/dbl/lists.py
index 33f761676eb316ad8de3b5d9246d490a199d4fb8..0cc43b447b830f3f11e0e8cfb00f356f1d437fee 100644 (file)
--- a/src/dbl/lists.py
+++ b/src/dbl/lists.py
@@ -21,6 +21,7 @@
 import datetime
 import functools
 import io
+import itertools
 import logging
 import sqlalchemy.dialects.postgresql
 import sqlalchemy.orm
@@ -878,22 +879,23 @@ class List(sqlmodel.SQLModel, database.BackendMixin, table=True):
                )
 
                # De-list the redundant domains
-               await self.backend.db.execute(
-                       sqlmodel
-                       .update(
-                               domains.Domain,
-                       )
-                       .values(
-                               subsumed = True,
-                       )
-                       .where(
-                               domains.Domain.list == self,
-                               domains.Domain.removed_at == None,
-                               domains.Domain.name.in_(
-                                       redundant_names,
+               for batch in itertools.batched(redundant_names, 1000):
+                       await self.backend.db.execute(
+                               sqlmodel
+                               .update(
+                                       domains.Domain,
+                               )
+                               .values(
+                                       subsumed = True,
+                               )
+                               .where(
+                                       domains.Domain.list == self,
+                                       domains.Domain.removed_at == None,
+                                       domains.Domain.name.in_(
+                                               batch,
+                                       )
                                )
                        )
-               )
 
                # Update all stats afterwards
                if update_stats: