downloader: Seperate downloaded files into blocks and iterate over those

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/tools/downloader.py b/tools/downloader.py
index 0250669e70edea688e0422c662fb10cb8d8156de..d2b1511b1d38eb862d6ba80153348ed9e2f89b2e 100644 (file)
--- a/tools/downloader.py
+++ b/tools/downloader.py
@@ -81,9 +81,32 @@ class DownloaderContext(object):
 
        def __iter__(self):
                """
-                       Makes the object iterable by going through each line
+                       Makes the object iterable by going through each block
                """
-               return iter(self.body)
+               block = []
+
+               for line in self.body:
+                       # Convert to string
+                       for charset in ("utf-8", "latin1"):
+                               try:
+                                       line = line.decode(charset)
+                               except UnicodeDecodeError:
+                                       continue
+                               else:
+                                       break
+
+                       # Strip line-endings
+                       line = line.strip()
+
+                       if line:
+                               block.append(line)
+                               continue
+
+                       # End the block on an empty line
+                       yield block
+
+                       # Reset the block
+                       block = []
 
        @property
        def headers(self):
@@ -122,5 +145,7 @@ if __name__ == "__main__":
                print("Downloading %s..." % url)
                
                with d.request(url) as r:
-                       for line in r:
-                               print(line)
+                       for block in r:
+                               for line in block:
+                                       print(line)
+                               print()