src/patches/libloc-0.9.1-move-location-downloader-functionality-into-location-query.patch

   1 commit a6f1e3463d4c2085c203ad58072d7a154b663904
   2 Author: Michael Tremer <michael.tremer@ipfire.org>
   3 Date:   Wed Jun 3 17:06:13 2020 +0000
   4
   5     Move location-downloader functionality into location-query
   6
   7     The commands are very long and confusion. Hence we merge this
   8     all into one command.
   9
  10     Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
  11
  12 diff --git a/Makefile.am b/Makefile.am
  13 index 31869e0..c0b1300 100644
  14 --- a/Makefile.am
  15 +++ b/Makefile.am
  16 @@ -146,6 +146,7 @@ CLEANFILES += \
  17  dist_pkgpython_PYTHON = \
  18         src/python/__init__.py \
  19         src/python/database.py \
  20 +       src/python/downloader.py \
  21         src/python/i18n.py \
  22         src/python/importer.py \
  23         src/python/logger.py
  24 @@ -234,19 +235,16 @@ uninstall-perl:
  25                 $(DESTDIR)/$(prefix)/man/man3/Location.3pm
  26
  27  bin_SCRIPTS = \
  28 -       src/python/location-downloader \
  29         src/python/location-exporter \
  30         src/python/location-importer \
  31         src/python/location-query
  32
  33  EXTRA_DIST += \
  34 -       src/python/location-downloader.in \
  35         src/python/location-exporter.in \
  36         src/python/location-importer.in \
  37         src/python/location-query.in
  38
  39  CLEANFILES += \
  40 -       src/python/location-downloader \
  41         src/python/location-exporter \
  42         src/python/location-importer \
  43         src/python/location-query
  44 diff --git a/src/python/location-downloader.in b/src/python/downloader.py
  45 similarity index 60%
  46 rename from src/python/location-downloader.in
  47 rename to src/python/downloader.py
  48 index bf0d682..c9e6e00 100644
  49 --- a/src/python/location-downloader.in
  50 +++ b/src/python/downloader.py
  51 @@ -3,7 +3,7 @@
  52  #                                                                             #
  53  # libloc - A library to determine the location of someone on the Internet     #
  54  #                                                                             #
  55 -# Copyright (C) 2019 IPFire Development Team <info@ipfire.org>                #
  56 +# Copyright (C) 2020 IPFire Development Team <info@ipfire.org>                #
  57  #                                                                             #
  58  # This library is free software; you can redistribute it and/or               #
  59  # modify it under the terms of the GNU Lesser General Public                  #
  60 @@ -17,24 +17,18 @@
  61  #                                                                             #
  62  ###############################################################################
  63
  64 -import argparse
  65 -import datetime
  66  import logging
  67  import lzma
  68  import os
  69  import random
  70 -import shutil
  71  import stat
  72 -import sys
  73  import tempfile
  74  import time
  75  import urllib.error
  76  import urllib.parse
  77  import urllib.request
  78
  79 -# Load our location module
  80 -import location
  81 -from location.i18n import _
  82 +from _location import Database, DATABASE_VERSION_LATEST
  83
  84  DATABASE_FILENAME = "location.db.xz"
  85  MIRRORS = (
  86 @@ -46,9 +40,11 @@ log = logging.getLogger("location.downloader")
  87  log.propagate = 1
  88
  89  class Downloader(object):
  90 -       def __init__(self, version, mirrors):
  91 +       def __init__(self, version=DATABASE_VERSION_LATEST, mirrors=None):
  92                 self.version = version
  93 -               self.mirrors = list(mirrors)
  94 +
  95 +               # Set mirrors or use defaults
  96 +               self.mirrors = list(mirrors or MIRRORS)
  97
  98                 # Randomize mirrors
  99                 random.shuffle(self.mirrors)
 100 @@ -117,9 +113,10 @@ class Downloader(object):
 101
 102                 return res
 103
 104 -       def download(self, url, public_key, timestamp=None, tmpdir=None, **kwargs):
 105 -               headers = {}
 106 +       def download(self, public_key, timestamp=None, tmpdir=None, **kwargs):
 107 +               url = "%s/%s" % (self.version, DATABASE_FILENAME)
 108
 109 +               headers = {}
 110                 if timestamp:
 111                         headers["If-Modified-Since"] = timestamp.strftime(
 112                                 "%a, %d %b %Y %H:%M:%S GMT",
 113 @@ -191,7 +188,7 @@ class Downloader(object):
 114                 """
 115                 log.debug("Opening downloaded database at %s" % f.name)
 116
 117 -               db = location.Database(f.name)
 118 +               db = Database(f.name)
 119
 120                 # Database is not recent
 121                 if timestamp and db.created_at < timestamp.timestamp():
 122 @@ -208,141 +205,3 @@ class Downloader(object):
 123                                 return False
 124
 125                 return True
 126 -
 127 -
 128 -class CLI(object):
 129 -       def __init__(self):
 130 -               # Which version are we downloading?
 131 -               self.version = location.DATABASE_VERSION_LATEST
 132 -
 133 -               self.downloader = Downloader(version=self.version, mirrors=MIRRORS)
 134 -
 135 -       def parse_cli(self):
 136 -               parser = argparse.ArgumentParser(
 137 -                       description=_("Location Downloader Command Line Interface"),
 138 -               )
 139 -               subparsers = parser.add_subparsers()
 140 -
 141 -               # Global configuration flags
 142 -               parser.add_argument("--debug", action="store_true",
 143 -                       help=_("Enable debug output"))
 144 -               parser.add_argument("--quiet", action="store_true",
 145 -                       help=_("Enable quiet mode"))
 146 -
 147 -               # version
 148 -               parser.add_argument("--version", action="version",
 149 -                       version="%(prog)s @VERSION@")
 150 -
 151 -               # database
 152 -               parser.add_argument("--database", "-d",
 153 -                       default="@databasedir@/database.db", help=_("Path to database"),
 154 -               )
 155 -
 156 -               # public key
 157 -               parser.add_argument("--public-key", "-k",
 158 -                       default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
 159 -               )
 160 -
 161 -               # Update
 162 -               update = subparsers.add_parser("update", help=_("Update database"))
 163 -               update.set_defaults(func=self.handle_update)
 164 -
 165 -               # Verify
 166 -               verify = subparsers.add_parser("verify",
 167 -                       help=_("Verify the downloaded database"))
 168 -               verify.set_defaults(func=self.handle_verify)
 169 -
 170 -               args = parser.parse_args()
 171 -
 172 -               # Configure logging
 173 -               if args.debug:
 174 -                       location.logger.set_level(logging.DEBUG)
 175 -               elif args.quiet:
 176 -                       location.logger.set_level(logging.WARNING)
 177 -
 178 -               # Print usage if no action was given
 179 -               if not "func" in args:
 180 -                       parser.print_usage()
 181 -                       sys.exit(2)
 182 -
 183 -               return args
 184 -
 185 -       def run(self):
 186 -               # Parse command line arguments
 187 -               args = self.parse_cli()
 188 -
 189 -               # Call function
 190 -               ret = args.func(args)
 191 -
 192 -               # Return with exit code
 193 -               if ret:
 194 -                       sys.exit(ret)
 195 -
 196 -               # Otherwise just exit
 197 -               sys.exit(0)
 198 -
 199 -       def handle_update(self, ns):
 200 -               # Fetch the timestamp we need from DNS
 201 -               t = location.discover_latest_version(self.version)
 202 -
 203 -               # Parse timestamp into datetime format
 204 -               timestamp = datetime.datetime.fromtimestamp(t) if t else None
 205 -
 206 -               # Open database
 207 -               try:
 208 -                       db = location.Database(ns.database)
 209 -
 210 -                       # Check if we are already on the latest version
 211 -                       if timestamp and db.created_at >= timestamp.timestamp():
 212 -                               log.info("Already on the latest version")
 213 -                               return
 214 -
 215 -               except FileNotFoundError as e:
 216 -                       db = None
 217 -
 218 -               # Download the database into the correct directory
 219 -               tmpdir = os.path.dirname(ns.database)
 220 -
 221 -               # Try downloading a new database
 222 -               try:
 223 -                       t = self.downloader.download("%s/%s" % (self.version, DATABASE_FILENAME),
 224 -                               public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
 225 -
 226 -               # If no file could be downloaded, log a message
 227 -               except FileNotFoundError as e:
 228 -                       log.error("Could not download a new database")
 229 -                       return 1
 230 -
 231 -               # If we have not received a new file, there is nothing to do
 232 -               if not t:
 233 -                       return 3
 234 -
 235 -               # Move temporary file to destination
 236 -               shutil.move(t.name, ns.database)
 237 -
 238 -               return 0
 239 -
 240 -       def handle_verify(self, ns):
 241 -               try:
 242 -                       db = location.Database(ns.database)
 243 -               except FileNotFoundError as e:
 244 -                       log.error("%s: %s" % (ns.database, e))
 245 -                       return 127
 246 -
 247 -               # Verify the database
 248 -               with open(ns.public_key, "r") as f:
 249 -                       if not db.verify(f):
 250 -                               log.error("Could not verify database")
 251 -                               return 1
 252 -
 253 -               # Success
 254 -               log.debug("Database successfully verified")
 255 -               return 0
 256 -
 257 -
 258 -def main():
 259 -       # Run the command line interface
 260 -       c = CLI()
 261 -       c.run()
 262 -
 263 -main()
 264 diff --git a/src/python/location-query.in b/src/python/location-query.in
 265 index dfdff8c..0291786 100644
 266 --- a/src/python/location-query.in
 267 +++ b/src/python/location-query.in
 268 @@ -18,16 +18,23 @@
 269  ###############################################################################
 270
 271  import argparse
 272 +import datetime
 273  import ipaddress
 274 +import logging
 275  import os
 276 +import shutil
 277  import socket
 278  import sys
 279  import time
 280
 281  # Load our location module
 282  import location
 283 +import location.downloader
 284  from location.i18n import _
 285
 286 +# Setup logging
 287 +log = logging.getLogger("location")
 288 +
 289  # Output formatters
 290
 291  class OutputFormatter(object):
 292 @@ -157,6 +164,15 @@ class CLI(object):
 293                 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
 294                 dump.set_defaults(func=self.handle_dump)
 295
 296 +               # Update
 297 +               update = subparsers.add_parser("update", help=_("Update database"))
 298 +               update.set_defaults(func=self.handle_update)
 299 +
 300 +               # Verify
 301 +               verify = subparsers.add_parser("verify",
 302 +                       help=_("Verify the downloaded database"))
 303 +               verify.set_defaults(func=self.handle_verify)
 304 +
 305                 # Get AS
 306                 get_as = subparsers.add_parser("get-as",
 307                         help=_("Get information about one or multiple Autonomous Systems"),
 308 @@ -423,6 +439,59 @@ class CLI(object):
 309                         for a in db.search_as(query):
 310                                 print(a)
 311
 312 +       def handle_update(self, db, ns):
 313 +               # Fetch the timestamp we need from DNS
 314 +               t = location.discover_latest_version()
 315 +
 316 +               # Parse timestamp into datetime format
 317 +               timestamp = datetime.datetime.fromtimestamp(t) if t else None
 318 +
 319 +               # Check the version of the local database
 320 +               if db and timestamp and db.created_at >= timestamp.timestamp():
 321 +                       log.info("Already on the latest version")
 322 +                       return
 323 +
 324 +               # Download the database into the correct directory
 325 +               tmpdir = os.path.dirname(ns.database)
 326 +
 327 +               # Create a downloader
 328 +               d = location.downloader.Downloader()
 329 +
 330 +               # Try downloading a new database
 331 +               try:
 332 +                       t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
 333 +
 334 +               # If no file could be downloaded, log a message
 335 +               except FileNotFoundError as e:
 336 +                       log.error("Could not download a new database")
 337 +                       return 1
 338 +
 339 +               # If we have not received a new file, there is nothing to do
 340 +               if not t:
 341 +                       return 3
 342 +
 343 +               # Move temporary file to destination
 344 +               shutil.move(t.name, ns.database)
 345 +
 346 +               return 0
 347 +
 348 +       def handle_verify(self, ns):
 349 +               try:
 350 +                       db = location.Database(ns.database)
 351 +               except FileNotFoundError as e:
 352 +                       log.error("%s: %s" % (ns.database, e))
 353 +                       return 127
 354 +
 355 +               # Verify the database
 356 +               with open(ns.public_key, "r") as f:
 357 +                       if not db.verify(f):
 358 +                               log.error("Could not verify database")
 359 +                               return 1
 360 +
 361 +               # Success
 362 +               log.debug("Database successfully verified")
 363 +               return 0
 364 +
 365         def __get_output_formatter(self, ns):
 366                 try:
 367                         cls = self.output_formats[ns.output_format]
 368 diff --git a/src/python/locationmodule.c b/src/python/locationmodule.c
 369 index a04cab7..5b72be9 100644
 370 --- a/src/python/locationmodule.c
 371 +++ b/src/python/locationmodule.c
 372 @@ -50,9 +50,9 @@ static PyObject* set_log_level(PyObject* m, PyObject* args) {
 373  }
 374
 375  static PyObject* discover_latest_version(PyObject* m, PyObject* args) {
 376 -       unsigned int version = 0;
 377 +       unsigned int version = LOC_DATABASE_VERSION_LATEST;
 378
 379 -       if (!PyArg_ParseTuple(args, "i", &version))
 380 +       if (!PyArg_ParseTuple(args, "|i", &version))
 381                 return NULL;
 382
 383         time_t t = 0;