From: Michael Tremer <michael.tremer@ipfire.org>
Date: Tue, 30 Jan 2018 15:14:36 +0000 (+0000)
Subject: Move some commonly used functions to util submodule
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=502af9365f5a45de0273481eb52a25bf074be061;p=location%2Flocation-database.git

Move some commonly used functions to util submodule

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/tools/base.py b/tools/base.py
index b4d3c38..e424a82 100644
--- a/tools/base.py
+++ b/tools/base.py
@@ -29,6 +29,7 @@ import re
 import sqlite3
 
 from . import downloader
+from . import util
 
 FILENAME_ASNUMS   = "asnums.txt"
 FILENMAE_NETWORKS = "networks.txt"
@@ -185,15 +186,6 @@ class RIRParser(object):
 		f.write("# Generated at %s\n" % self.start_time)
 		f.write("#\n\n")
 
-	def _split_line(self, line):
-		key, colon, val = line.partition(":")
-
-		# Strip any excess space
-		key = key.strip()
-		val = val.strip()
-
-		return key, val
-
 	def fetch_data(self):
 		if not self.rir.database_urls:
 			raise NotImplementedError("Database URLs not set")
@@ -249,7 +241,7 @@ class RIRParser(object):
 			logging.debug(line)
 
 			# Split line
-			key, val = self._split_line(line)
+			key, val = util.split_line(line)
 
 			if key == "inetnum":
 				start_address, delim, end_address = val.partition("-")
@@ -312,7 +304,7 @@ class RIRParser(object):
 			logging.debug(line)
 
 			# Split line
-			key, val = self._split_line(line)
+			key, val = util.split_line(line)
 
 			# Keep any significant data
 			if key in ("route6", "route"):
@@ -344,7 +336,7 @@ class RIRParser(object):
 			logging.debug(line)
 
 			# Split line
-			key, val = self._split_line(line)
+			key, val = util.split_line(line)
 
 			if key == "aut-num":
 				m = RE_AS.match(val)
@@ -376,7 +368,7 @@ class RIRParser(object):
 			logging.debug(line)
 
 			# Split line
-			key, val = self._split_line(line)
+			key, val = util.split_line(line)
 
 			if key in ("organisation", "org-name", "country"):
 				org[key] = val
diff --git a/tools/downloader.py b/tools/downloader.py
index af9cdf6..bd24ba0 100644
--- a/tools/downloader.py
+++ b/tools/downloader.py
@@ -24,6 +24,8 @@ import gzip
 import logging
 import urllib.request
 
+from . import util
+
 # Setup logger
 log = logging.getLogger("downloader")
 log.propagate = 1
@@ -83,35 +85,7 @@ class DownloaderContext(object):
 		"""
 			Makes the object iterable by going through each block
 		"""
-		block = []
-
-		for line in self.body:
-			# Convert to string
-			for charset in ("utf-8", "latin1"):
-				try:
-					line = line.decode(charset)
-				except UnicodeDecodeError:
-					continue
-				else:
-					break
-
-			# Strip line-endings
-			line = line.rstrip()
-
-			# Skip commented lines
-			if line.startswith("#"):
-				continue
-
-			if line:
-				block.append(line)
-				continue
-
-			# End the block on an empty line
-			if block:
-				yield block
-
-			# Reset the block
-			block = []
+		return util.iterate_over_blocks(self.body)
 
 	@property
 	def headers(self):
diff --git a/tools/util.py b/tools/util.py
new file mode 100644
index 0000000..9cdca70
--- /dev/null
+++ b/tools/util.py
@@ -0,0 +1,61 @@
+#!/usr/bin/python3
+###############################################################################
+#                                                                             #
+# location-database - A database to determine someone's                       #
+#                     location on the Internet                                #
+# Copyright (C) 2018 Michael Tremer                                           #
+#                                                                             #
+# This program is free software: you can redistribute it and/or modify        #
+# it under the terms of the GNU General Public License as published by        #
+# the Free Software Foundation, either version 3 of the License, or           #
+# (at your option) any later version.                                         #
+#                                                                             #
+# This program is distributed in the hope that it will be useful,             #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
+# GNU General Public License for more details.                                #
+#                                                                             #
+# You should have received a copy of the GNU General Public License           #
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
+#                                                                             #
+###############################################################################
+
+def iterate_over_blocks(f, charsets=("utf-8", "latin1")):
+	block = []
+
+	for line in f:
+		# Convert to string
+		for charset in charsets:
+			try:
+				line = line.decode(charset)
+			except UnicodeDecodeError:
+				continue
+			else:
+				break
+
+		# Strip line-endings
+		line = line.rstrip()
+
+		# Skip commented lines
+		if line.startswith("#"):
+			continue
+
+		if line:
+			block.append(line)
+			continue
+
+		# End the block on an empty line
+		if block:
+			yield block
+
+		# Reset the block
+		block = []
+
+def split_line(line):
+	key, colon, val = line.partition(":")
+
+	# Strip any excess space
+	key = key.strip()
+	val = val.strip()
+
+	return key, val