Move some commonly used functions to util submodule

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/tools/base.py b/tools/base.py
index b4d3c38bc328edb7fca560f4dd3924a4c0063cf9..e424a82b348b8e2dc6be0be7e1fbb40dbe00f99e 100644 (file)
--- a/tools/base.py
+++ b/tools/base.py
@@ -29,6 +29,7 @@ import re
 import sqlite3
 
 from . import downloader
+from . import util
 
 FILENAME_ASNUMS   = "asnums.txt"
 FILENMAE_NETWORKS = "networks.txt"
@@ -185,15 +186,6 @@ class RIRParser(object):
                f.write("# Generated at %s\n" % self.start_time)
                f.write("#\n\n")
 
-       def _split_line(self, line):
-               key, colon, val = line.partition(":")
-
-               # Strip any excess space
-               key = key.strip()
-               val = val.strip()
-
-               return key, val
-
        def fetch_data(self):
                if not self.rir.database_urls:
                        raise NotImplementedError("Database URLs not set")
@@ -249,7 +241,7 @@ class RIRParser(object):
                        logging.debug(line)
 
                        # Split line
-                       key, val = self._split_line(line)
+                       key, val = util.split_line(line)
 
                        if key == "inetnum":
                                start_address, delim, end_address = val.partition("-")
@@ -312,7 +304,7 @@ class RIRParser(object):
                        logging.debug(line)
 
                        # Split line
-                       key, val = self._split_line(line)
+                       key, val = util.split_line(line)
 
                        # Keep any significant data
                        if key in ("route6", "route"):
@@ -344,7 +336,7 @@ class RIRParser(object):
                        logging.debug(line)
 
                        # Split line
-                       key, val = self._split_line(line)
+                       key, val = util.split_line(line)
 
                        if key == "aut-num":
                                m = RE_AS.match(val)
@@ -376,7 +368,7 @@ class RIRParser(object):
                        logging.debug(line)
 
                        # Split line
-                       key, val = self._split_line(line)
+                       key, val = util.split_line(line)
 
                        if key in ("organisation", "org-name", "country"):
                                org[key] = val

diff --git a/tools/downloader.py b/tools/downloader.py
index af9cdf64a32dd76524faf80f60d7bcd0eef49ec1..bd24ba047bd6ec6ca27febaf8e2b32ea05b052a3 100644 (file)
--- a/tools/downloader.py
+++ b/tools/downloader.py
@@ -24,6 +24,8 @@ import gzip
 import logging
 import urllib.request
 
+from . import util
+
 # Setup logger
 log = logging.getLogger("downloader")
 log.propagate = 1
@@ -83,35 +85,7 @@ class DownloaderContext(object):
                """
                        Makes the object iterable by going through each block
                """
-               block = []
-
-               for line in self.body:
-                       # Convert to string
-                       for charset in ("utf-8", "latin1"):
-                               try:
-                                       line = line.decode(charset)
-                               except UnicodeDecodeError:
-                                       continue
-                               else:
-                                       break
-
-                       # Strip line-endings
-                       line = line.rstrip()
-
-                       # Skip commented lines
-                       if line.startswith("#"):
-                               continue
-
-                       if line:
-                               block.append(line)
-                               continue
-
-                       # End the block on an empty line
-                       if block:
-                               yield block
-
-                       # Reset the block
-                       block = []
+               return util.iterate_over_blocks(self.body)
 
        @property
        def headers(self):

diff --git a/tools/util.py b/tools/util.py
new file mode 100644 (file)
index 0000000..9cdca70
--- /dev/null
+++ b/tools/util.py
@@ -0,0 +1,61 @@
+#!/usr/bin/python3
+###############################################################################
+#                                                                             #
+# location-database - A database to determine someone's                       #
+#                     location on the Internet                                #
+# Copyright (C) 2018 Michael Tremer                                           #
+#                                                                             #
+# This program is free software: you can redistribute it and/or modify        #
+# it under the terms of the GNU General Public License as published by        #
+# the Free Software Foundation, either version 3 of the License, or           #
+# (at your option) any later version.                                         #
+#                                                                             #
+# This program is distributed in the hope that it will be useful,             #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
+# GNU General Public License for more details.                                #
+#                                                                             #
+# You should have received a copy of the GNU General Public License           #
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
+#                                                                             #
+###############################################################################
+
+def iterate_over_blocks(f, charsets=("utf-8", "latin1")):
+       block = []
+
+       for line in f:
+               # Convert to string
+               for charset in charsets:
+                       try:
+                               line = line.decode(charset)
+                       except UnicodeDecodeError:
+                               continue
+                       else:
+                               break
+
+               # Strip line-endings
+               line = line.rstrip()
+
+               # Skip commented lines
+               if line.startswith("#"):
+                       continue
+
+               if line:
+                       block.append(line)
+                       continue
+
+               # End the block on an empty line
+               if block:
+                       yield block
+
+               # Reset the block
+               block = []
+
+def split_line(line):
+       key, colon, val = line.partition(":")
+
+       # Strip any excess space
+       key = key.strip()
+       val = val.strip()
+
+       return key, val