From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 10 Dec 2025 10:58:50 +0000 (+0000)
Subject: sources: Download the entire payload before processing it
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5a2afa52f3d03d1efc2fb7b0dade87a631fc3538;p=dnsbl.git

sources: Download the entire payload before processing it

Some web servers did not keep the connection open for long enough so
that we cannot keep processing all domains on the fly.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/dnsbl/sources.py b/src/dnsbl/sources.py
index 7433e55..9d9f5b8 100644
--- a/src/dnsbl/sources.py
+++ b/src/dnsbl/sources.py
@@ -21,6 +21,7 @@
 import datetime
 import email.utils
 import enum
+import io
 import logging
 import sqlalchemy.dialects.postgresql
 import sqlmodel
@@ -153,8 +154,19 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True):
 						log.debug("Source %s has not been changed, skipping processing" % self)
 						return False
 
+					buffer = io.StringIO()
+
+					# Read the entire payload into the buffer
+					for chunk in response.iter_text():
+						buffer.write(chunk)
+
+					# Rewind the buffer
+					buffer.seek(0)
+
 					# Add all domains
-					for line in response.iter_lines():
+					for line in buffer:
+						line = line.rstrip()
+
 						# Detect the format if still unknown
 						if format is None:
 							format = self._detect_format(line)