src/python/location-downloader.in

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # libloc - A library to determine the location of someone on the Internet     #
   5 #                                                                             #
   6 # Copyright (C) 2019 IPFire Development Team <info@ipfire.org>                #
   7 #                                                                             #
   8 # This library is free software; you can redistribute it and/or               #
   9 # modify it under the terms of the GNU Lesser General Public                  #
  10 # License as published by the Free Software Foundation; either                #
  11 # version 2.1 of the License, or (at your option) any later version.          #
  12 #                                                                             #
  13 # This library is distributed in the hope that it will be useful,             #
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
  16 # Lesser General Public License for more details.                             #
  17 #                                                                             #
  18 ###############################################################################
  19
  20 import argparse
  21 import datetime
  22 import logging
  23 import lzma
  24 import os
  25 import random
  26 import shutil
  27 import sys
  28 import tempfile
  29 import time
  30 import urllib.error
  31 import urllib.parse
  32 import urllib.request
  33
  34 # Load our location module
  35 import location
  36 from location.i18n import _
  37
  38 DATABASE_FILENAME = "location.db.xz"
  39 MIRRORS = (
  40         "https://location.ipfire.org/databases/",
  41 )
  42
  43 # Initialise logging
  44 log = logging.getLogger("location.downloader")
  45 log.propagate = 1
  46
  47 class Downloader(object):
  48         def __init__(self, mirrors):
  49                 self.mirrors = list(mirrors)
  50
  51                 # Randomize mirrors
  52                 random.shuffle(self.mirrors)
  53
  54                 # Get proxies from environment
  55                 self.proxies = self._get_proxies()
  56
  57         def _get_proxies(self):
  58                 proxies = {}
  59
  60                 for protocol in ("https", "http"):
  61                         proxy = os.environ.get("%s_proxy" % protocol, None)
  62
  63                         if proxy:
  64                                 proxies[protocol] = proxy
  65
  66                 return proxies
  67
  68         def _make_request(self, url, baseurl=None, headers={}):
  69                 if baseurl:
  70                         url = urllib.parse.urljoin(baseurl, url)
  71
  72                 req = urllib.request.Request(url, method="GET")
  73
  74                 # Update headers
  75                 headers.update({
  76                         "User-Agent" : "location-downloader/@VERSION@",
  77                 })
  78
  79                 # Set headers
  80                 for header in headers:
  81                         req.add_header(header, headers[header])
  82
  83                 # Set proxies
  84                 for protocol in self.proxies:
  85                         req.set_proxy(self.proxies[protocol], protocol)
  86
  87                 return req
  88
  89         def _send_request(self, req, **kwargs):
  90                 # Log request headers
  91                 log.debug("HTTP %s Request to %s" % (req.method, req.host))
  92                 log.debug("     URL: %s" % req.full_url)
  93                 log.debug("     Headers:")
  94                 for k, v in req.header_items():
  95                         log.debug("             %s: %s" % (k, v))
  96
  97                 try:
  98                         res = urllib.request.urlopen(req, **kwargs)
  99
 100                 except urllib.error.HTTPError as e:
 101                         # Log response headers
 102                         log.debug("HTTP Response: %s" % e.code)
 103                         log.debug("     Headers:")
 104                         for header in e.headers:
 105                                 log.debug("             %s: %s" % (header, e.headers[header]))
 106
 107                         # Raise all other errors
 108                         raise e
 109
 110                 # Log response headers
 111                 log.debug("HTTP Response: %s" % res.code)
 112                 log.debug("     Headers:")
 113                 for k, v in res.getheaders():
 114                         log.debug("             %s: %s" % (k, v))
 115
 116                 return res
 117
 118         def download(self, url, public_key, timestamp=None, **kwargs):
 119                 headers = {}
 120
 121                 if timestamp:
 122                         headers["If-Modified-Since"] = timestamp.strftime(
 123                                 "%a, %d %b %Y %H:%M:%S GMT",
 124                         )
 125
 126                 t = tempfile.NamedTemporaryFile(delete=False)
 127                 with t:
 128                         # Try all mirrors
 129                         for mirror in self.mirrors:
 130                                 # Prepare HTTP request
 131                                 req = self._make_request(url, baseurl=mirror, headers=headers)
 132
 133                                 try:
 134                                         with self._send_request(req) as res:
 135                                                 decompressor = lzma.LZMADecompressor()
 136
 137                                                 # Read all data
 138                                                 while True:
 139                                                         buf = res.read(1024)
 140                                                         if not buf:
 141                                                                 break
 142
 143                                                         # Decompress data
 144                                                         buf = decompressor.decompress(buf)
 145                                                         if buf:
 146                                                                 t.write(buf)
 147
 148                                         # Write all data to disk
 149                                         t.flush()
 150
 151                                 # Catch decompression errors
 152                                 except lzma.LZMAError as e:
 153                                         log.warning("Could not decompress downloaded file: %s" % e)
 154                                         continue
 155
 156                                 except urllib.error.HTTPError as e:
 157                                         # The file on the server was too old
 158                                         if e.code == 304:
 159                                                 log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)
 160
 161                                         # Log any other HTTP errors
 162                                         else:
 163                                                 log.warning("%s reported: %s" % (mirror, e))
 164
 165                                         # Throw away any downloaded content and try again
 166                                         t.truncate()
 167
 168                                 else:
 169                                         # Check if the downloaded database is recent
 170                                         if not self._check_database(t, public_key, timestamp):
 171                                                 log.warning("Downloaded database is outdated. Trying next mirror...")
 172
 173                                                 # Throw away the data and try again
 174                                                 t.truncate()
 175                                                 continue
 176
 177                                         # Return temporary file
 178                                         return t
 179
 180                 raise FileNotFoundError(url)
 181
 182         def _check_database(self, f, public_key, timestamp=None):
 183                 """
 184                         Checks the downloaded database if it can be opened,
 185                         verified and if it is recent enough
 186                 """
 187                 log.debug("Opening downloaded database at %s" % f.name)
 188
 189                 db = location.Database(f.name)
 190
 191                 # Database is not recent
 192                 if timestamp and db.created_at < timestamp.timestamp():
 193                         return False
 194
 195                 log.info("Downloaded new database from %s" % (time.strftime(
 196                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 197                 )))
 198
 199                 # Verify the database
 200                 with open(public_key, "r") as f:
 201                         if not db.verify(f):
 202                                 log.error("Could not verify database")
 203                                 return False
 204
 205                 return True
 206
 207
 208 class CLI(object):
 209         def __init__(self):
 210                 self.downloader = Downloader(mirrors=MIRRORS)
 211
 212         def parse_cli(self):
 213                 parser = argparse.ArgumentParser(
 214                         description=_("Location Downloader Command Line Interface"),
 215                 )
 216                 subparsers = parser.add_subparsers()
 217
 218                 # Global configuration flags
 219                 parser.add_argument("--debug", action="store_true",
 220                         help=_("Enable debug output"))
 221                 parser.add_argument("--quiet", action="store_true",
 222                         help=_("Enable quiet mode"))
 223
 224                 # version
 225                 parser.add_argument("--version", action="version",
 226                         version="%(prog)s @VERSION@")
 227
 228                 # database
 229                 parser.add_argument("--database", "-d",
 230                         default="@databasedir@/database.db", help=_("Path to database"),
 231                 )
 232
 233                 # public key
 234                 parser.add_argument("--public-key", "-k",
 235                         default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
 236                 )
 237
 238                 # Update
 239                 update = subparsers.add_parser("update", help=_("Update database"))
 240                 update.set_defaults(func=self.handle_update)
 241
 242                 # Verify
 243                 verify = subparsers.add_parser("verify",
 244                         help=_("Verify the downloaded database"))
 245                 verify.set_defaults(func=self.handle_verify)
 246
 247                 args = parser.parse_args()
 248
 249                 # Configure logging
 250                 if args.debug:
 251                         location.logger.set_level(logging.DEBUG)
 252                 elif args.quiet:
 253                         location.logger.set_level(logging.WARNING)
 254
 255                 # Print usage if no action was given
 256                 if not "func" in args:
 257                         parser.print_usage()
 258                         sys.exit(2)
 259
 260                 return args
 261
 262         def run(self):
 263                 # Parse command line arguments
 264                 args = self.parse_cli()
 265
 266                 # Call function
 267                 ret = args.func(args)
 268
 269                 # Return with exit code
 270                 if ret:
 271                         sys.exit(ret)
 272
 273                 # Otherwise just exit
 274                 sys.exit(0)
 275
 276         def handle_update(self, ns):
 277                 # Fetch the version we need from DNS
 278                 t = location.discover_latest_version()
 279
 280                 # Parse timestamp into datetime format
 281                 try:
 282                         timestamp = datetime.datetime.fromtimestamp(t)
 283                 except:
 284                         raise
 285
 286                 # Open database
 287                 try:
 288                         db = location.Database(ns.database)
 289
 290                         # Check if we are already on the latest version
 291                         if db.created_at >= timestamp.timestamp():
 292                                 log.info("Already on the latest version")
 293                                 return
 294
 295                 except FileNotFoundError as e:
 296                         db = None
 297
 298                 # Try downloading a new database
 299                 try:
 300                         t = self.downloader.download(DATABASE_FILENAME,
 301                                 public_key=ns.public_key, timestamp=timestamp)
 302
 303                 # If no file could be downloaded, log a message
 304                 except FileNotFoundError as e:
 305                         log.error("Could not download a new database")
 306                         return 1
 307
 308                 # If we have not received a new file, there is nothing to do
 309                 if not t:
 310                         return 3
 311
 312                 # Write temporary file to destination
 313                 shutil.copyfile(t.name, ns.database)
 314
 315                 # Remove temporary file
 316                 os.unlink(t.name)
 317
 318                 return 0
 319
 320         def handle_verify(self, ns):
 321                 try:
 322                         db = location.Database(ns.database)
 323                 except FileNotFoundError as e:
 324                         log.error("%s: %s" % (ns.database, e))
 325                         return 127
 326
 327                 # Verify the database
 328                 with open(ns.public_key, "r") as f:
 329                         if not db.verify(f):
 330                                 log.error("Could not verify database")
 331                                 return 1
 332
 333                 # Success
 334                 log.debug("Database successfully verified")
 335                 return 0
 336
 337
 338 def main():
 339         # Run the command line interface
 340         c = CLI()
 341         c.run()
 342
 343 main()