sources: Download the entire payload before processing it

Some web servers did not keep the connection open for long enough so
that we cannot keep processing all domains on the fly.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/dnsbl/sources.py b/src/dnsbl/sources.py
index 7433e55b7206fa36c521d3b156a567d11ceb4318..9d9f5b8bc2f67e7f58790b9d4b866f2d37d142cd 100644 (file)
--- a/src/dnsbl/sources.py
+++ b/src/dnsbl/sources.py
@@ -21,6 +21,7 @@
 import datetime
 import email.utils
 import enum
+import io
 import logging
 import sqlalchemy.dialects.postgresql
 import sqlmodel
@@ -153,8 +154,19 @@ class Source(sqlmodel.SQLModel, database.BackendMixin, table=True):
                                                log.debug("Source %s has not been changed, skipping processing" % self)
                                                return False
 
+                                       buffer = io.StringIO()
+
+                                       # Read the entire payload into the buffer
+                                       for chunk in response.iter_text():
+                                               buffer.write(chunk)
+
+                                       # Rewind the buffer
+                                       buffer.seek(0)
+
                                        # Add all domains
-                                       for line in response.iter_lines():
+                                       for line in buffer:
+                                               line = line.rstrip()
+
                                                # Detect the format if still unknown
                                                if format is None:
                                                        format = self._detect_format(line)