src/python/location.in

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # libloc - A library to determine the location of someone on the Internet     #
   5 #                                                                             #
   6 # Copyright (C) 2017 IPFire Development Team <info@ipfire.org>                #
   7 #                                                                             #
   8 # This library is free software; you can redistribute it and/or               #
   9 # modify it under the terms of the GNU Lesser General Public                  #
  10 # License as published by the Free Software Foundation; either                #
  11 # version 2.1 of the License, or (at your option) any later version.          #
  12 #                                                                             #
  13 # This library is distributed in the hope that it will be useful,             #
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
  16 # Lesser General Public License for more details.                             #
  17 #                                                                             #
  18 ###############################################################################
  19
  20 import argparse
  21 import datetime
  22 import ipaddress
  23 import logging
  24 import os
  25 import re
  26 import shutil
  27 import socket
  28 import sys
  29 import time
  30
  31 # Load our location module
  32 import location
  33 import location.downloader
  34 import location.export
  35
  36 from location.i18n import _
  37
  38 # Setup logging
  39 log = logging.getLogger("location")
  40
  41 # Output formatters
  42
  43 class CLI(object):
  44         def parse_cli(self):
  45                 parser = argparse.ArgumentParser(
  46                         description=_("Location Database Command Line Interface"),
  47                 )
  48                 subparsers = parser.add_subparsers()
  49
  50                 # Global configuration flags
  51                 parser.add_argument("--debug", action="store_true",
  52                         help=_("Enable debug output"))
  53                 parser.add_argument("--quiet", action="store_true",
  54                         help=_("Enable quiet mode"))
  55
  56                 # version
  57                 parser.add_argument("--version", action="version",
  58                         version="%(prog)s @VERSION@")
  59
  60                 # database
  61                 parser.add_argument("--database", "-d",
  62                         default="@databasedir@/database.db", help=_("Path to database"),
  63                 )
  64
  65                 # public key
  66                 parser.add_argument("--public-key", "-k",
  67                         default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
  68                 )
  69
  70                 # Show the database version
  71                 version = subparsers.add_parser("version",
  72                         help=_("Show database version"))
  73                 version.set_defaults(func=self.handle_version)
  74
  75                 # lookup an IP address
  76                 lookup = subparsers.add_parser("lookup",
  77                         help=_("Lookup one or multiple IP addresses"),
  78                 )
  79                 lookup.add_argument("address", nargs="+")
  80                 lookup.set_defaults(func=self.handle_lookup)
  81
  82                 # Dump the whole database
  83                 dump = subparsers.add_parser("dump",
  84                         help=_("Dump the entire database"),
  85                 )
  86                 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
  87                 dump.set_defaults(func=self.handle_dump)
  88
  89                 # Update
  90                 update = subparsers.add_parser("update", help=_("Update database"))
  91                 update.add_argument("--cron",
  92                         help=_("Update the library only once per interval"),
  93                         choices=("daily", "weekly", "monthly"),
  94                 )
  95                 update.set_defaults(func=self.handle_update)
  96
  97                 # Verify
  98                 verify = subparsers.add_parser("verify",
  99                         help=_("Verify the downloaded database"))
 100                 verify.set_defaults(func=self.handle_verify)
 101
 102                 # Get AS
 103                 get_as = subparsers.add_parser("get-as",
 104                         help=_("Get information about one or multiple Autonomous Systems"),
 105                 )
 106                 get_as.add_argument("asn", nargs="+")
 107                 get_as.set_defaults(func=self.handle_get_as)
 108
 109                 # Search for AS
 110                 search_as = subparsers.add_parser("search-as",
 111                         help=_("Search for Autonomous Systems that match the string"),
 112                 )
 113                 search_as.add_argument("query", nargs=1)
 114                 search_as.set_defaults(func=self.handle_search_as)
 115
 116                 # List all networks in an AS
 117                 list_networks_by_as = subparsers.add_parser("list-networks-by-as",
 118                         help=_("Lists all networks in an AS"),
 119                 )
 120                 list_networks_by_as.add_argument("asn", nargs=1, type=int)
 121                 list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
 122                 list_networks_by_as.add_argument("--format",
 123                         choices=location.export.formats.keys(), default="list")
 124                 list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
 125
 126                 # List all networks in a country
 127                 list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
 128                         help=_("Lists all networks in a country"),
 129                 )
 130                 list_networks_by_cc.add_argument("country_code", nargs=1)
 131                 list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
 132                 list_networks_by_cc.add_argument("--format",
 133                         choices=location.export.formats.keys(), default="list")
 134                 list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
 135
 136                 # List all networks with flags
 137                 list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
 138                         help=_("Lists all networks with flags"),
 139                 )
 140                 list_networks_by_flags.add_argument("--anonymous-proxy",
 141                         action="store_true", help=_("Anonymous Proxies"),
 142                 )
 143                 list_networks_by_flags.add_argument("--satellite-provider",
 144                         action="store_true", help=_("Satellite Providers"),
 145                 )
 146                 list_networks_by_flags.add_argument("--anycast",
 147                         action="store_true", help=_("Anycasts"),
 148                 )
 149                 list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
 150                 list_networks_by_flags.add_argument("--format",
 151                         choices=location.export.formats.keys(), default="list")
 152                 list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
 153
 154                 # List countries
 155                 list_countries = subparsers.add_parser("list-countries",
 156                         help=_("Lists all countries"),
 157                 )
 158                 list_countries.add_argument("--show-name",
 159                         action="store_true", help=_("Show the name of the country"),
 160                 )
 161                 list_countries.add_argument("--show-continent",
 162                         action="store_true", help=_("Show the continent"),
 163                 )
 164                 list_countries.set_defaults(func=self.handle_list_countries)
 165
 166                 # Export
 167                 export = subparsers.add_parser("export",
 168                         help=_("Exports data in many formats to load it into packet filters"),
 169                 )
 170                 export.add_argument("--format", help=_("Output format"),
 171                         choices=location.export.formats.keys(), default="list")
 172                 export.add_argument("--directory", help=_("Output directory"), required=True)
 173                 export.add_argument("--family",
 174                         help=_("Specify address family"), choices=("ipv6", "ipv4"),
 175                 )
 176                 export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
 177                 export.set_defaults(func=self.handle_export)
 178
 179                 args = parser.parse_args()
 180
 181                 # Configure logging
 182                 if args.debug:
 183                         location.logger.set_level(logging.DEBUG)
 184                 elif args.quiet:
 185                         location.logger.set_level(logging.WARNING)
 186
 187                 # Print usage if no action was given
 188                 if not "func" in args:
 189                         parser.print_usage()
 190                         sys.exit(2)
 191
 192                 return args
 193
 194         def run(self):
 195                 # Parse command line arguments
 196                 args = self.parse_cli()
 197
 198                 # Open database
 199                 try:
 200                         db = location.Database(args.database)
 201                 except FileNotFoundError as e:
 202                         # Allow continuing without a database
 203                         if args.func == self.handle_update:
 204                                 db = None
 205
 206                         else:
 207                                 sys.stderr.write("location: Could not open database %s: %s\n" \
 208                                         % (args.database, e))
 209                                 sys.exit(1)
 210
 211                 # Translate family (if present)
 212                 if "family" in args:
 213                         if args.family == "ipv6":
 214                                 args.family = socket.AF_INET6
 215                         elif args.family == "ipv4":
 216                                 args.family = socket.AF_INET
 217                         else:
 218                                 args.family = 0
 219
 220                 # Call function
 221                 try:
 222                         ret = args.func(db, args)
 223
 224                 # Catch invalid inputs
 225                 except ValueError as e:
 226                         sys.stderr.write("%s\n" % e)
 227                         ret = 2
 228
 229                 # Return with exit code
 230                 if ret:
 231                         sys.exit(ret)
 232
 233                 # Otherwise just exit
 234                 sys.exit(0)
 235
 236         def handle_version(self, db, ns):
 237                 """
 238                         Print the version of the database
 239                 """
 240                 t = time.strftime(
 241                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 242                 )
 243
 244                 print(t)
 245
 246         def handle_lookup(self, db, ns):
 247                 ret = 0
 248
 249                 format = "  %-24s: %s"
 250
 251                 for address in ns.address:
 252                         try:
 253                                 network = db.lookup(address)
 254                         except ValueError:
 255                                 print(_("Invalid IP address: %s") % address, file=sys.stderr)
 256
 257                         args = {
 258                                 "address" : address,
 259                                 "network" : network,
 260                         }
 261
 262                         # Nothing found?
 263                         if not network:
 264                                 print(_("Nothing found for %(address)s") % args, file=sys.stderr)
 265                                 ret = 1
 266                                 continue
 267
 268                         print("%s:" % address)
 269                         print(format % (_("Network"), network))
 270
 271                         # Print country
 272                         if network.country_code:
 273                                 country = db.get_country(network.country_code)
 274
 275                                 print(format % (
 276                                         _("Country"),
 277                                         country.name if country else network.country_code),
 278                                 )
 279
 280                         # Print AS information
 281                         if network.asn:
 282                                 autonomous_system = db.get_as(network.asn)
 283
 284                                 print(format % (
 285                                         _("Autonomous System"),
 286                                         autonomous_system or "AS%s" % network.asn),
 287                                 )
 288
 289                         # Anonymous Proxy
 290                         if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
 291                                 print(format % (
 292                                         _("Anonymous Proxy"), _("yes"),
 293                                 ))
 294
 295                         # Satellite Provider
 296                         if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
 297                                 print(format % (
 298                                         _("Satellite Provider"), _("yes"),
 299                                 ))
 300
 301                         # Anycast
 302                         if network.has_flag(location.NETWORK_FLAG_ANYCAST):
 303                                 print(format % (
 304                                         _("Anycast"), _("yes"),
 305                                 ))
 306
 307                 return ret
 308
 309         def handle_dump(self, db, ns):
 310                 # Use output file or write to stdout
 311                 f = ns.output or sys.stdout
 312
 313                 # Format everything like this
 314                 format = "%-24s %s\n"
 315
 316                 # Write metadata
 317                 f.write("#\n# Location Database Export\n#\n")
 318
 319                 f.write("# Generated: %s\n" % time.strftime(
 320                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 321                 ))
 322
 323                 if db.vendor:
 324                         f.write("# Vendor:    %s\n" % db.vendor)
 325
 326                 if db.license:
 327                         f.write("# License:   %s\n" % db.license)
 328
 329                 f.write("#\n")
 330
 331                 if db.description:
 332                         for line in db.description.splitlines():
 333                                 line = "# %s" % line
 334                                 f.write("%s\n" % line.rstrip())
 335
 336                         f.write("#\n")
 337
 338                 # Iterate over all ASes
 339                 for a in db.ases:
 340                         f.write("\n")
 341                         f.write(format % ("aut-num:", "AS%s" % a.number))
 342                         f.write(format % ("name:", a.name))
 343
 344                 flags = {
 345                         location.NETWORK_FLAG_ANONYMOUS_PROXY    : "is-anonymous-proxy:",
 346                         location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
 347                         location.NETWORK_FLAG_ANYCAST            : "is-anycast:",
 348                 }
 349
 350                 # Iterate over all networks
 351                 for n in db.networks:
 352                         f.write("\n")
 353                         f.write(format % ("net:", n))
 354
 355                         if n.country_code:
 356                                 f.write(format % ("country:", n.country_code))
 357
 358                         if n.asn:
 359                                 f.write(format % ("aut-num:", n.asn))
 360
 361                         # Print all flags
 362                         for flag in flags:
 363                                 if n.has_flag(flag):
 364                                         f.write(format % (flags[flag], "yes"))
 365
 366         def handle_get_as(self, db, ns):
 367                 """
 368                         Gets information about Autonomous Systems
 369                 """
 370                 ret = 0
 371
 372                 for asn in ns.asn:
 373                         try:
 374                                 asn = int(asn)
 375                         except ValueError:
 376                                 print(_("Invalid ASN: %s") % asn, file=sys.stderr)
 377                                 ret = 1
 378                                 continue
 379
 380                         # Fetch AS from database
 381                         a = db.get_as(asn)
 382
 383                         # Nothing found
 384                         if not a:
 385                                 print(_("Could not find AS%s") % asn, file=sys.stderr)
 386                                 ret = 1
 387                                 continue
 388
 389                         print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
 390
 391                 return ret
 392
 393         def handle_search_as(self, db, ns):
 394                 for query in ns.query:
 395                         # Print all matches ASes
 396                         for a in db.search_as(query):
 397                                 print(a)
 398
 399         def handle_update(self, db, ns):
 400                 if ns.cron and db:
 401                         now = datetime.datetime.utcnow()
 402
 403                         # Parse the database timestamp
 404                         t = datetime.datetime.utcfromtimestamp(db.created_at)
 405
 406                         if ns.cron == "daily":
 407                                 delta = datetime.timedelta(days=1)
 408                         elif ns.cron == "weekly":
 409                                 delta = datetime.timedelta(days=7)
 410                         elif ns.cron == "monthly":
 411                                 delta = datetime.timedelta(days=30)
 412
 413                         # Check if the database has recently been updated
 414                         if t >= (now - delta):
 415                                 log.info(
 416                                         _("The datase has recently be updated recently (%s)") % \
 417                                                 format_timedelta(now - t),
 418                                 )
 419                                 return 3
 420
 421                 # Fetch the timestamp we need from DNS
 422                 t = location.discover_latest_version()
 423
 424                 # Parse timestamp into datetime format
 425                 timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
 426
 427                 # Check the version of the local database
 428                 if db and timestamp and db.created_at >= timestamp.timestamp():
 429                         log.info("Already on the latest version")
 430                         return
 431
 432                 # Download the database into the correct directory
 433                 tmpdir = os.path.dirname(ns.database)
 434
 435                 # Create a downloader
 436                 d = location.downloader.Downloader()
 437
 438                 # Try downloading a new database
 439                 try:
 440                         t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
 441
 442                 # If no file could be downloaded, log a message
 443                 except FileNotFoundError as e:
 444                         log.error("Could not download a new database")
 445                         return 1
 446
 447                 # If we have not received a new file, there is nothing to do
 448                 if not t:
 449                         return 3
 450
 451                 # Move temporary file to destination
 452                 shutil.move(t.name, ns.database)
 453
 454                 return 0
 455
 456         def handle_verify(self, ns):
 457                 try:
 458                         db = location.Database(ns.database)
 459                 except FileNotFoundError as e:
 460                         log.error("%s: %s" % (ns.database, e))
 461                         return 127
 462
 463                 # Verify the database
 464                 with open(ns.public_key, "r") as f:
 465                         if not db.verify(f):
 466                                 log.error("Could not verify database")
 467                                 return 1
 468
 469                 # Success
 470                 log.debug("Database successfully verified")
 471                 return 0
 472
 473         def __get_output_formatter(self, ns):
 474                 try:
 475                         cls = location.export.formats[ns.format]
 476                 except KeyError:
 477                         cls = location.export.OutputFormatter
 478
 479                 return cls
 480
 481         def handle_list_countries(self, db, ns):
 482                 for country in db.countries:
 483                         line = [
 484                                 country.code,
 485                         ]
 486
 487                         if ns.show_continent:
 488                                 line.append(country.continent_code)
 489
 490                         if ns.show_name:
 491                                 line.append(country.name)
 492
 493                         # Format the output
 494                         line = " ".join(line)
 495
 496                         # Print the output
 497                         print(line)
 498
 499         def handle_list_networks_by_as(self, db, ns):
 500                 writer = self.__get_output_formatter(ns)
 501
 502                 for asn in ns.asn:
 503                         f = writer(sys.stdout, prefix="AS%s" % asn)
 504
 505                         # Print all matching networks
 506                         for n in db.search_networks(asn=asn, family=ns.family):
 507                                 f.write(n)
 508
 509                         f.finish()
 510
 511         def handle_list_networks_by_cc(self, db, ns):
 512                 writer = self.__get_output_formatter(ns)
 513
 514                 for country_code in ns.country_code:
 515                         # Open standard output
 516                         f = writer(sys.stdout, prefix=country_code)
 517
 518                         # Print all matching networks
 519                         for n in db.search_networks(country_code=country_code, family=ns.family):
 520                                 f.write(n)
 521
 522                         f.finish()
 523
 524         def handle_list_networks_by_flags(self, db, ns):
 525                 flags = 0
 526
 527                 if ns.anonymous_proxy:
 528                         flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
 529
 530                 if ns.satellite_provider:
 531                         flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
 532
 533                 if ns.anycast:
 534                         flags |= location.NETWORK_FLAG_ANYCAST
 535
 536                 if not flags:
 537                         raise ValueError(_("You must at least pass one flag"))
 538
 539                 writer = self.__get_output_formatter(ns)
 540                 f = writer(sys.stdout, prefix="custom")
 541
 542                 for n in db.search_networks(flags=flags, family=ns.family):
 543                         f.write(n)
 544
 545                 f.finish()
 546
 547         def handle_export(self, db, ns):
 548                 countries, asns = [], []
 549
 550                 # Translate family
 551                 if ns.family:
 552                         families = [ ns.family ]
 553                 else:
 554                         families = [ socket.AF_INET6, socket.AF_INET ]
 555
 556                 for object in ns.objects:
 557                         m = re.match("^AS(\d+)$", object)
 558                         if m:
 559                                 object = int(m.group(1))
 560
 561                                 asns.append(object)
 562
 563                         elif location.country_code_is_valid(object) \
 564                                         or object in ("A1", "A2", "A3"):
 565                                 countries.append(object)
 566
 567                         else:
 568                                 log.warning("Invalid argument: %s" % object)
 569                                 continue
 570
 571                 # Default to exporting all countries
 572                 if not countries and not asns:
 573                         countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
 574
 575                 # Select the output format
 576                 writer = self.__get_output_formatter(ns)
 577
 578                 e = location.export.Exporter(db, writer)
 579                 e.export(ns.directory, countries=countries, asns=asns, families=families)
 580
 581
 582 def format_timedelta(t):
 583         s = []
 584
 585         if t.days:
 586                 s.append(
 587                         _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
 588                 )
 589
 590         hours = t.seconds // 3600
 591         if hours:
 592                 s.append(
 593                         _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
 594                 )
 595
 596         minutes = (t.seconds % 3600) // 60
 597         if minutes:
 598                 s.append(
 599                         _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
 600                 )
 601
 602         seconds = t.seconds % 60
 603         if t.seconds:
 604                 s.append(
 605                         _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
 606                 )
 607
 608         if not s:
 609                 return _("Now")
 610
 611         return _("%s ago") % ", ".join(s)
 612
 613 def main():
 614         # Run the command line interface
 615         c = CLI()
 616         c.run()
 617
 618 main()