From: Michael Tremer <michael.tremer@ipfire.org>
Date: Mon, 29 Jan 2018 20:33:21 +0000 (+0000)
Subject: downloader: Seperate downloaded files into blocks and iterate over those
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a145727ae4550f395e3d923afcfee9cd2ee8c523;p=location%2Flocation-database.git

downloader: Seperate downloaded files into blocks and iterate over those

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/tools/downloader.py b/tools/downloader.py
index 0250669..d2b1511 100644
--- a/tools/downloader.py
+++ b/tools/downloader.py
@@ -81,9 +81,32 @@ class DownloaderContext(object):
 
 	def __iter__(self):
 		"""
-			Makes the object iterable by going through each line
+			Makes the object iterable by going through each block
 		"""
-		return iter(self.body)
+		block = []
+
+		for line in self.body:
+			# Convert to string
+			for charset in ("utf-8", "latin1"):
+				try:
+					line = line.decode(charset)
+				except UnicodeDecodeError:
+					continue
+				else:
+					break
+
+			# Strip line-endings
+			line = line.strip()
+
+			if line:
+				block.append(line)
+				continue
+
+			# End the block on an empty line
+			yield block
+
+			# Reset the block
+			block = []
 
 	@property
 	def headers(self):
@@ -122,5 +145,7 @@ if __name__ == "__main__":
 		print("Downloading %s..." % url)
 		
 		with d.request(url) as r:
-			for line in r:
-				print(line)
+			for block in r:
+				for line in block:
+					print(line)
+				print()