src/python/location-downloader.in

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # libloc - A library to determine the location of someone on the Internet     #
   5 #                                                                             #
   6 # Copyright (C) 2019 IPFire Development Team <info@ipfire.org>                #
   7 #                                                                             #
   8 # This library is free software; you can redistribute it and/or               #
   9 # modify it under the terms of the GNU Lesser General Public                  #
  10 # License as published by the Free Software Foundation; either                #
  11 # version 2.1 of the License, or (at your option) any later version.          #
  12 #                                                                             #
  13 # This library is distributed in the hope that it will be useful,             #
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
  16 # Lesser General Public License for more details.                             #
  17 #                                                                             #
  18 ###############################################################################
  19
  20 import argparse
  21 import datetime
  22 import logging
  23 import lzma
  24 import os
  25 import random
  26 import shutil
  27 import stat
  28 import sys
  29 import tempfile
  30 import time
  31 import urllib.error
  32 import urllib.parse
  33 import urllib.request
  34
  35 # Load our location module
  36 import location
  37 from location.i18n import _
  38
  39 DATABASE_FILENAME = "location.db.xz"
  40 MIRRORS = (
  41         "https://location.ipfire.org/databases/",
  42 )
  43
  44 # Initialise logging
  45 log = logging.getLogger("location.downloader")
  46 log.propagate = 1
  47
  48 class Downloader(object):
  49         def __init__(self, version, mirrors):
  50                 self.version = version
  51                 self.mirrors = list(mirrors)
  52
  53                 # Randomize mirrors
  54                 random.shuffle(self.mirrors)
  55
  56                 # Get proxies from environment
  57                 self.proxies = self._get_proxies()
  58
  59         def _get_proxies(self):
  60                 proxies = {}
  61
  62                 for protocol in ("https", "http"):
  63                         proxy = os.environ.get("%s_proxy" % protocol, None)
  64
  65                         if proxy:
  66                                 proxies[protocol] = proxy
  67
  68                 return proxies
  69
  70         def _make_request(self, url, baseurl=None, headers={}):
  71                 if baseurl:
  72                         url = urllib.parse.urljoin(baseurl, url)
  73
  74                 req = urllib.request.Request(url, method="GET")
  75
  76                 # Update headers
  77                 headers.update({
  78                         "User-Agent" : "location-downloader/@VERSION@",
  79                 })
  80
  81                 # Set headers
  82                 for header in headers:
  83                         req.add_header(header, headers[header])
  84
  85                 # Set proxies
  86                 for protocol in self.proxies:
  87                         req.set_proxy(self.proxies[protocol], protocol)
  88
  89                 return req
  90
  91         def _send_request(self, req, **kwargs):
  92                 # Log request headers
  93                 log.debug("HTTP %s Request to %s" % (req.method, req.host))
  94                 log.debug("     URL: %s" % req.full_url)
  95                 log.debug("     Headers:")
  96                 for k, v in req.header_items():
  97                         log.debug("             %s: %s" % (k, v))
  98
  99                 try:
 100                         res = urllib.request.urlopen(req, **kwargs)
 101
 102                 except urllib.error.HTTPError as e:
 103                         # Log response headers
 104                         log.debug("HTTP Response: %s" % e.code)
 105                         log.debug("     Headers:")
 106                         for header in e.headers:
 107                                 log.debug("             %s: %s" % (header, e.headers[header]))
 108
 109                         # Raise all other errors
 110                         raise e
 111
 112                 # Log response headers
 113                 log.debug("HTTP Response: %s" % res.code)
 114                 log.debug("     Headers:")
 115                 for k, v in res.getheaders():
 116                         log.debug("             %s: %s" % (k, v))
 117
 118                 return res
 119
 120         def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs):
 121                 headers = {}
 122
 123                 if timestamp:
 124                         headers["If-Modified-Since"] = timestamp.strftime(
 125                                 "%a, %d %b %Y %H:%M:%S GMT",
 126                         )
 127
 128                 t = tempfile.NamedTemporaryFile(dir=tmpdir, delete=False)
 129                 with t:
 130                         # Try all mirrors
 131                         for mirror in self.mirrors:
 132                                 # Prepare HTTP request
 133                                 req = self._make_request(url, baseurl=mirror, headers=headers)
 134
 135                                 try:
 136                                         with self._send_request(req) as res:
 137                                                 decompressor = lzma.LZMADecompressor()
 138
 139                                                 # Read all data
 140                                                 while True:
 141                                                         buf = res.read(1024)
 142                                                         if not buf:
 143                                                                 break
 144
 145                                                         # Decompress data
 146                                                         buf = decompressor.decompress(buf)
 147                                                         if buf:
 148                                                                 t.write(buf)
 149
 150                                         # Write all data to disk
 151                                         t.flush()
 152
 153                                 # Catch decompression errors
 154                                 except lzma.LZMAError as e:
 155                                         log.warning("Could not decompress downloaded file: %s" % e)
 156                                         continue
 157
 158                                 except urllib.error.HTTPError as e:
 159                                         # The file on the server was too old
 160                                         if e.code == 304:
 161                                                 log.warning("%s is serving an outdated database. Trying next mirror..." % mirror)
 162
 163                                         # Log any other HTTP errors
 164                                         else:
 165                                                 log.warning("%s reported: %s" % (mirror, e))
 166
 167                                         # Throw away any downloaded content and try again
 168                                         t.truncate()
 169
 170                                 else:
 171                                         # Check if the downloaded database is recent
 172                                         if not self._check_database(t, public_key, timestamp):
 173                                                 log.warning("Downloaded database is outdated. Trying next mirror...")
 174
 175                                                 # Throw away the data and try again
 176                                                 t.truncate()
 177                                                 continue
 178
 179                                         # Make the file readable for everyone
 180                                         os.chmod(t.name, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH)
 181
 182                                         # Return temporary file
 183                                         return t
 184
 185                 raise FileNotFoundError(url)
 186
 187         def _check_database(self, f, public_key, timestamp=None):
 188                 """
 189                         Checks the downloaded database if it can be opened,
 190                         verified and if it is recent enough
 191                 """
 192                 log.debug("Opening downloaded database at %s" % f.name)
 193
 194                 db = location.Database(f.name)
 195
 196                 # Database is not recent
 197                 if timestamp and db.created_at < timestamp.timestamp():
 198                         return False
 199
 200                 log.info("Downloaded new database from %s" % (time.strftime(
 201                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 202                 )))
 203
 204                 # Verify the database
 205                 with open(public_key, "r") as f:
 206                         if not db.verify(f):
 207                                 log.error("Could not verify database")
 208                                 return False
 209
 210                 return True
 211
 212
 213 class CLI(object):
 214         def __init__(self):
 215                 # Which version are we downloading?
 216                 self.version = location.DATABASE_VERSION_LATEST
 217
 218                 self.downloader = Downloader(version=self.version, mirrors=MIRRORS)
 219
 220         def parse_cli(self):
 221                 parser = argparse.ArgumentParser(
 222                         description=_("Location Downloader Command Line Interface"),
 223                 )
 224                 subparsers = parser.add_subparsers()
 225
 226                 # Global configuration flags
 227                 parser.add_argument("--debug", action="store_true",
 228                         help=_("Enable debug output"))
 229                 parser.add_argument("--quiet", action="store_true",
 230                         help=_("Enable quiet mode"))
 231
 232                 # version
 233                 parser.add_argument("--version", action="version",
 234                         version="%(prog)s @VERSION@")
 235
 236                 # database
 237                 parser.add_argument("--database", "-d",
 238                         default="@databasedir@/database.db", help=_("Path to database"),
 239                 )
 240
 241                 # public key
 242                 parser.add_argument("--public-key", "-k",
 243                         default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
 244                 )
 245
 246                 # Update
 247                 update = subparsers.add_parser("update", help=_("Update database"))
 248                 update.set_defaults(func=self.handle_update)
 249
 250                 # Verify
 251                 verify = subparsers.add_parser("verify",
 252                         help=_("Verify the downloaded database"))
 253                 verify.set_defaults(func=self.handle_verify)
 254
 255                 args = parser.parse_args()
 256
 257                 # Configure logging
 258                 if args.debug:
 259                         location.logger.set_level(logging.DEBUG)
 260                 elif args.quiet:
 261                         location.logger.set_level(logging.WARNING)
 262
 263                 # Print usage if no action was given
 264                 if not "func" in args:
 265                         parser.print_usage()
 266                         sys.exit(2)
 267
 268                 return args
 269
 270         def run(self):
 271                 # Parse command line arguments
 272                 args = self.parse_cli()
 273
 274                 # Call function
 275                 ret = args.func(args)
 276
 277                 # Return with exit code
 278                 if ret:
 279                         sys.exit(ret)
 280
 281                 # Otherwise just exit
 282                 sys.exit(0)
 283
 284         def handle_update(self, ns):
 285                 # Fetch the timestamp we need from DNS
 286                 t = location.discover_latest_version(self.version)
 287
 288                 # Parse timestamp into datetime format
 289                 timestamp = datetime.datetime.fromtimestamp(t) if t else None
 290
 291                 # Open database
 292                 try:
 293                         db = location.Database(ns.database)
 294
 295                         # Check if we are already on the latest version
 296                         if timestamp and db.created_at >= timestamp.timestamp():
 297                                 log.info("Already on the latest version")
 298                                 return
 299
 300                 except FileNotFoundError as e:
 301                         db = None
 302
 303                 # Download the database into the correct directory
 304                 tmpdir = os.path.dirname(ns.database)
 305
 306                 # Try downloading a new database
 307                 try:
 308                         t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME),
 309                                 public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
 310
 311                 # If no file could be downloaded, log a message
 312                 except FileNotFoundError as e:
 313                         log.error("Could not download a new database")
 314                         return 1
 315
 316                 # If we have not received a new file, there is nothing to do
 317                 if not t:
 318                         return 3
 319
 320                 # Move temporary file to destination
 321                 shutil.move(t.name, ns.database)
 322
 323                 return 0
 324
 325         def handle_verify(self, ns):
 326                 try:
 327                         db = location.Database(ns.database)
 328                 except FileNotFoundError as e:
 329                         log.error("%s: %s" % (ns.database, e))
 330                         return 127
 331
 332                 # Verify the database
 333                 with open(ns.public_key, "r") as f:
 334                         if not db.verify(f):
 335                                 log.error("Could not verify database")
 336                                 return 1
 337
 338                 # Success
 339                 log.debug("Database successfully verified")
 340                 return 0
 341
 342
 343 def main():
 344         # Run the command line interface
 345         c = CLI()
 346         c.run()
 347
 348 main()