python: Import downloader from database repository

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/Makefile.am b/Makefile.am
index e3a55d4f53e922196e3d85fc6a81f508766990c1..efb8288a455a5a5f6e4ebea4f3a4bf49270cf9ec 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -155,6 +155,7 @@ CLEANFILES += \
 dist_pkgpython_PYTHON = \
        src/python/__init__.py \
        src/python/i18n.py \
+       src/python/importer.py \
        src/python/logger.py
 
 pyexec_LTLIBRARIES = \

diff --git a/src/python/importer.py b/src/python/importer.py
new file mode 100644 (file)
index 0000000..4cf3aa5
--- /dev/null
+++ b/src/python/importer.py
@@ -0,0 +1,107 @@
+#!/usr/bin/python3
+###############################################################################
+#                                                                             #
+# libloc - A library to determine the location of someone on the Internet     #
+#                                                                             #
+# Copyright (C) 2020 IPFire Development Team <info@ipfire.org>                #
+#                                                                             #
+# This library is free software; you can redistribute it and/or               #
+# modify it under the terms of the GNU Lesser General Public                  #
+# License as published by the Free Software Foundation; either                #
+# version 2.1 of the License, or (at your option) any later version.          #
+#                                                                             #
+# This library is distributed in the hope that it will be useful,             #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
+# Lesser General Public License for more details.                             #
+#                                                                             #
+###############################################################################
+
+import gzip
+import logging
+import urllib.request
+
+# Initialise logging
+log = logging.getLogger("location.importer")
+log.propagate = 1
+
+class Downloader(object):
+       def __init__(self):
+               self.proxy = None
+
+       def set_proxy(self, url):
+               """
+                       Sets a HTTP proxy that is used to perform all requests
+               """
+               log.info("Using proxy %s" % url)
+               self.proxy = url
+
+       def request(self, url, data=None):
+               req = urllib.request.Request(url, data=data)
+
+               # Configure proxy
+               if self.proxy:
+                       req.set_proxy(self.proxy, "http")
+
+               return DownloaderContext(self, req)
+
+
+class DownloaderContext(object):
+       def __init__(self, downloader, request):
+               self.downloader = downloader
+               self.request = request
+
+               # Save the response object
+               self.response = None
+
+       def __enter__(self):
+               log.info("Retrieving %s..." % self.request.full_url)
+
+               # Send request
+               self.response = urllib.request.urlopen(self.request)
+
+               # Log the response headers
+               log.debug("Response Headers:")
+               for header in self.headers:
+                       log.debug("     %s: %s" % (header, self.get_header(header)))
+
+               return self
+
+       def __exit__(self, type, value, traceback):
+               pass
+
+       def __iter__(self):
+               """
+                       Makes the object iterable by going through each block
+               """
+               # Store body
+               body = self.body
+
+               while True:
+                       line = body.readline()
+                       if not line:
+                               break
+
+                       # Decode the line
+                       line = line.decode()
+
+                       # Strip the ending
+                       yield line.rstrip()
+
+       @property
+       def headers(self):
+               if self.response:
+                       return self.response.headers
+
+       def get_header(self, name):
+               if self.headers:
+                       return self.headers.get(name)
+
+       @property
+       def body(self):
+               """
+                       Returns a file-like object with the decoded content
+                       of the response.
+               """
+               # Return the response by default
+               return self.response

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index a5bbf4bbfeefc8bb9a5802045260f7866f34e999..1ced867db8cb6f7b3acaae4c1944be683af15d13 100644 (file)
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -23,6 +23,7 @@ import sys
 
 # Load our location module
 import location
+import location.importer
 from location.i18n import _
 
 # Initialise logging