src/python/location.in

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # libloc - A library to determine the location of someone on the Internet     #
   5 #                                                                             #
   6 # Copyright (C) 2017 IPFire Development Team <info@ipfire.org>                #
   7 #                                                                             #
   8 # This library is free software; you can redistribute it and/or               #
   9 # modify it under the terms of the GNU Lesser General Public                  #
  10 # License as published by the Free Software Foundation; either                #
  11 # version 2.1 of the License, or (at your option) any later version.          #
  12 #                                                                             #
  13 # This library is distributed in the hope that it will be useful,             #
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
  16 # Lesser General Public License for more details.                             #
  17 #                                                                             #
  18 ###############################################################################
  19
  20 import argparse
  21 import datetime
  22 import ipaddress
  23 import logging
  24 import os
  25 import re
  26 import shutil
  27 import socket
  28 import sys
  29 import time
  30
  31 # Load our location module
  32 import location
  33 import location.downloader
  34 import location.export
  35
  36 from location.i18n import _
  37
  38 # Setup logging
  39 log = logging.getLogger("location")
  40
  41 # Output formatters
  42
  43 class CLI(object):
  44         def parse_cli(self):
  45                 parser = argparse.ArgumentParser(
  46                         description=_("Location Database Command Line Interface"),
  47                 )
  48                 subparsers = parser.add_subparsers()
  49
  50                 # Global configuration flags
  51                 parser.add_argument("--debug", action="store_true",
  52                         help=_("Enable debug output"))
  53                 parser.add_argument("--quiet", action="store_true",
  54                         help=_("Enable quiet mode"))
  55
  56                 # version
  57                 parser.add_argument("--version", action="version",
  58                         version="%(prog)s @VERSION@")
  59
  60                 # database
  61                 parser.add_argument("--database", "-d",
  62                         default="@databasedir@/database.db", help=_("Path to database"),
  63                 )
  64
  65                 # public key
  66                 parser.add_argument("--public-key", "-k",
  67                         default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
  68                 )
  69
  70                 # Show the database version
  71                 version = subparsers.add_parser("version",
  72                         help=_("Show database version"))
  73                 version.set_defaults(func=self.handle_version)
  74
  75                 # lookup an IP address
  76                 lookup = subparsers.add_parser("lookup",
  77                         help=_("Lookup one or multiple IP addresses"),
  78                 )
  79                 lookup.add_argument("address", nargs="+")
  80                 lookup.set_defaults(func=self.handle_lookup)
  81
  82                 # Dump the whole database
  83                 dump = subparsers.add_parser("dump",
  84                         help=_("Dump the entire database"),
  85                 )
  86                 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
  87                 dump.set_defaults(func=self.handle_dump)
  88
  89                 # Update
  90                 update = subparsers.add_parser("update", help=_("Update database"))
  91                 update.add_argument("--cron",
  92                         help=_("Update the library only once per interval"),
  93                         choices=("daily", "weekly", "monthly"),
  94                 )
  95                 update.set_defaults(func=self.handle_update)
  96
  97                 # Verify
  98                 verify = subparsers.add_parser("verify",
  99                         help=_("Verify the downloaded database"))
 100                 verify.set_defaults(func=self.handle_verify)
 101
 102                 # Get AS
 103                 get_as = subparsers.add_parser("get-as",
 104                         help=_("Get information about one or multiple Autonomous Systems"),
 105                 )
 106                 get_as.add_argument("asn", nargs="+")
 107                 get_as.set_defaults(func=self.handle_get_as)
 108
 109                 # Search for AS
 110                 search_as = subparsers.add_parser("search-as",
 111                         help=_("Search for Autonomous Systems that match the string"),
 112                 )
 113                 search_as.add_argument("query", nargs=1)
 114                 search_as.set_defaults(func=self.handle_search_as)
 115
 116                 # List all networks in an AS
 117                 list_networks_by_as = subparsers.add_parser("list-networks-by-as",
 118                         help=_("Lists all networks in an AS"),
 119                 )
 120                 list_networks_by_as.add_argument("asn", nargs=1, type=int)
 121                 list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
 122                 list_networks_by_as.add_argument("--format",
 123                         choices=location.export.formats.keys(), default="list")
 124                 list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
 125
 126                 # List all networks in a country
 127                 list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
 128                         help=_("Lists all networks in a country"),
 129                 )
 130                 list_networks_by_cc.add_argument("country_code", nargs=1)
 131                 list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
 132                 list_networks_by_cc.add_argument("--format",
 133                         choices=location.export.formats.keys(), default="list")
 134                 list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
 135
 136                 # List all networks with flags
 137                 list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
 138                         help=_("Lists all networks with flags"),
 139                 )
 140                 list_networks_by_flags.add_argument("--anonymous-proxy",
 141                         action="store_true", help=_("Anonymous Proxies"),
 142                 )
 143                 list_networks_by_flags.add_argument("--satellite-provider",
 144                         action="store_true", help=_("Satellite Providers"),
 145                 )
 146                 list_networks_by_flags.add_argument("--anycast",
 147                         action="store_true", help=_("Anycasts"),
 148                 )
 149                 list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
 150                 list_networks_by_flags.add_argument("--format",
 151                         choices=location.export.formats.keys(), default="list")
 152                 list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
 153
 154                 # List countries
 155                 list_countries = subparsers.add_parser("list-countries",
 156                         help=_("Lists all countries"),
 157                 )
 158                 list_countries.add_argument("--show-name",
 159                         action="store_true", help=_("Show the name of the country"),
 160                 )
 161                 list_countries.add_argument("--show-continent",
 162                         action="store_true", help=_("Show the continent"),
 163                 )
 164                 list_countries.set_defaults(func=self.handle_list_countries)
 165
 166                 # Export
 167                 export = subparsers.add_parser("export",
 168                         help=_("Exports data in many formats to load it into packet filters"),
 169                 )
 170                 export.add_argument("--format", help=_("Output format"),
 171                         choices=location.export.formats.keys(), default="list")
 172                 export.add_argument("--directory", help=_("Output directory"), required=True)
 173                 export.add_argument("--family",
 174                         help=_("Specify address family"), choices=("ipv6", "ipv4"),
 175                 )
 176                 export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
 177                 export.set_defaults(func=self.handle_export)
 178
 179                 args = parser.parse_args()
 180
 181                 # Configure logging
 182                 if args.debug:
 183                         location.logger.set_level(logging.DEBUG)
 184                 elif args.quiet:
 185                         location.logger.set_level(logging.WARNING)
 186
 187                 # Print usage if no action was given
 188                 if not "func" in args:
 189                         parser.print_usage()
 190                         sys.exit(2)
 191
 192                 return args
 193
 194         def run(self):
 195                 # Parse command line arguments
 196                 args = self.parse_cli()
 197
 198                 # Open database
 199                 try:
 200                         db = location.Database(args.database)
 201                 except FileNotFoundError as e:
 202                         # Allow continuing without a database
 203                         if args.func == self.handle_update:
 204                                 db = None
 205
 206                         else:
 207                                 sys.stderr.write("location: Could not open database %s: %s\n" \
 208                                         % (args.database, e))
 209                                 sys.exit(1)
 210
 211                 # Translate family (if present)
 212                 if "family" in args:
 213                         if args.family == "ipv6":
 214                                 args.family = socket.AF_INET6
 215                         elif args.family == "ipv4":
 216                                 args.family = socket.AF_INET
 217                         else:
 218                                 args.family = 0
 219
 220                 # Call function
 221                 try:
 222                         ret = args.func(db, args)
 223
 224                 # Catch invalid inputs
 225                 except ValueError as e:
 226                         sys.stderr.write("%s\n" % e)
 227                         ret = 2
 228
 229                 # Return with exit code
 230                 if ret:
 231                         sys.exit(ret)
 232
 233                 # Otherwise just exit
 234                 sys.exit(0)
 235
 236         def handle_version(self, db, ns):
 237                 """
 238                         Print the version of the database
 239                 """
 240                 t = time.strftime(
 241                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 242                 )
 243
 244                 print(t)
 245
 246         def handle_lookup(self, db, ns):
 247                 ret = 0
 248
 249                 format = "  %-24s: %s"
 250
 251                 for address in ns.address:
 252                         try:
 253                                 network = db.lookup(address)
 254                         except ValueError:
 255                                 print(_("Invalid IP address: %s") % address, file=sys.stderr)
 256
 257                         args = {
 258                                 "address" : address,
 259                                 "network" : network,
 260                         }
 261
 262                         # Nothing found?
 263                         if not network:
 264                                 print(_("Nothing found for %(address)s") % args, file=sys.stderr)
 265                                 ret = 1
 266                                 continue
 267
 268                         print("%s:" % address)
 269                         print(format % (_("Network"), network))
 270
 271                         # Print country
 272                         if network.country_code:
 273                                 country = db.get_country(network.country_code)
 274
 275                                 print(format % (
 276                                         _("Country"),
 277                                         country.name if country else network.country_code),
 278                                 )
 279
 280                         # Print AS information
 281                         if network.asn:
 282                                 autonomous_system = db.get_as(network.asn)
 283
 284                                 print(format % (
 285                                         _("Autonomous System"),
 286                                         autonomous_system or "AS%s" % network.asn),
 287                                 )
 288
 289                         # Anonymous Proxy
 290                         if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
 291                                 print(format % (
 292                                         _("Anonymous Proxy"), _("yes"),
 293                                 ))
 294
 295                         # Satellite Provider
 296                         if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
 297                                 print(format % (
 298                                         _("Satellite Provider"), _("yes"),
 299                                 ))
 300
 301                         # Anycast
 302                         if network.has_flag(location.NETWORK_FLAG_ANYCAST):
 303                                 print(format % (
 304                                         _("Anycast"), _("yes"),
 305                                 ))
 306
 307                 return ret
 308
 309         def handle_dump(self, db, ns):
 310                 # Use output file or write to stdout
 311                 f = ns.output or sys.stdout
 312
 313                 # Format everything like this
 314                 format = "%-24s %s\n"
 315
 316                 # Write metadata
 317                 f.write("#\n# Location Database Export\n#\n")
 318
 319                 f.write("# Generated: %s\n" % time.strftime(
 320                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 321                 ))
 322
 323                 if db.vendor:
 324                         f.write("# Vendor:    %s\n" % db.vendor)
 325
 326                 if db.license:
 327                         f.write("# License:   %s\n" % db.license)
 328
 329                 f.write("#\n")
 330
 331                 if db.description:
 332                         for line in db.description.splitlines():
 333                                 line = "# %s" % line
 334                                 f.write("%s\n" % line.rstrip())
 335
 336                         f.write("#\n")
 337
 338                 # Iterate over all ASes
 339                 for a in db.ases:
 340                         f.write("\n")
 341                         f.write(format % ("aut-num:", "AS%s" % a.number))
 342                         f.write(format % ("name:", a.name))
 343
 344                 flags = {
 345                         location.NETWORK_FLAG_ANONYMOUS_PROXY    : "is-anonymous-proxy:",
 346                         location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
 347                         location.NETWORK_FLAG_ANYCAST            : "is-anycast:",
 348                 }
 349
 350                 # Iterate over all networks
 351                 for n in db.networks:
 352                         f.write("\n")
 353                         f.write(format % ("net:", n))
 354
 355                         if n.country_code:
 356                                 f.write(format % ("country:", n.country_code))
 357
 358                         if n.asn:
 359                                 f.write(format % ("aut-num:", n.asn))
 360
 361                         # Print all flags
 362                         for flag in flags:
 363                                 if n.has_flag(flag):
 364                                         f.write(format % (flags[flag], "yes"))
 365
 366         def handle_get_as(self, db, ns):
 367                 """
 368                         Gets information about Autonomous Systems
 369                 """
 370                 ret = 0
 371
 372                 for asn in ns.asn:
 373                         try:
 374                                 asn = int(asn)
 375                         except ValueError:
 376                                 print(_("Invalid ASN: %s") % asn, file=sys.stderr)
 377                                 ret = 1
 378                                 continue
 379
 380                         # Fetch AS from database
 381                         a = db.get_as(asn)
 382
 383                         # Nothing found
 384                         if not a:
 385                                 print(_("Could not find AS%s") % asn, file=sys.stderr)
 386                                 ret = 1
 387                                 continue
 388
 389                         print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
 390
 391                 return ret
 392
 393         def handle_search_as(self, db, ns):
 394                 for query in ns.query:
 395                         # Print all matches ASes
 396                         for a in db.search_as(query):
 397                                 print(a)
 398
 399         def handle_update(self, db, ns):
 400                 if ns.cron and db:
 401                         now = datetime.datetime.utcnow()
 402
 403                         # Parse the database timestamp
 404                         t = datetime.datetime.utcfromtimestamp(db.created_at)
 405
 406                         if ns.cron == "daily":
 407                                 delta = datetime.timedelta(days=1)
 408                         elif ns.cron == "weekly":
 409                                 delta = datetime.timedelta(days=7)
 410                         elif ns.cron == "monthly":
 411                                 delta = datetime.timedelta(days=30)
 412
 413                         # Check if the database has recently been updated
 414                         if t >= (now - delta):
 415                                 log.info(
 416                                         _("The database has been updated recently (%s)") % \
 417                                                 format_timedelta(now - t),
 418                                 )
 419                                 return 3
 420
 421                 # Fetch the timestamp we need from DNS
 422                 t = location.discover_latest_version()
 423
 424                 # Parse timestamp into datetime format
 425                 timestamp = datetime.datetime.utcfromtimestamp(t) if t else None
 426
 427                 # Check the version of the local database
 428                 if db and timestamp and db.created_at >= timestamp.timestamp():
 429                         log.info("Already on the latest version")
 430                         return
 431
 432                 # Download the database into the correct directory
 433                 tmpdir = os.path.dirname(ns.database)
 434
 435                 # Create a downloader
 436                 d = location.downloader.Downloader()
 437
 438                 # Try downloading a new database
 439                 try:
 440                         t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
 441
 442                 # If no file could be downloaded, log a message
 443                 except FileNotFoundError as e:
 444                         log.error("Could not download a new database")
 445                         return 1
 446
 447                 # If we have not received a new file, there is nothing to do
 448                 if not t:
 449                         return 3
 450
 451                 # Move temporary file to destination
 452                 shutil.move(t.name, ns.database)
 453
 454                 return 0
 455
 456         def handle_verify(self, db, ns):
 457                 # Verify the database
 458                 with open(ns.public_key, "r") as f:
 459                         if not db.verify(f):
 460                                 log.error("Could not verify database")
 461                                 return 1
 462
 463                 # Success
 464                 log.debug("Database successfully verified")
 465                 return 0
 466
 467         def __get_output_formatter(self, ns):
 468                 try:
 469                         cls = location.export.formats[ns.format]
 470                 except KeyError:
 471                         cls = location.export.OutputFormatter
 472
 473                 return cls
 474
 475         def handle_list_countries(self, db, ns):
 476                 for country in db.countries:
 477                         line = [
 478                                 country.code,
 479                         ]
 480
 481                         if ns.show_continent:
 482                                 line.append(country.continent_code)
 483
 484                         if ns.show_name:
 485                                 line.append(country.name)
 486
 487                         # Format the output
 488                         line = " ".join(line)
 489
 490                         # Print the output
 491                         print(line)
 492
 493         def handle_list_networks_by_as(self, db, ns):
 494                 writer = self.__get_output_formatter(ns)
 495
 496                 for asn in ns.asn:
 497                         f = writer(sys.stdout, prefix="AS%s" % asn)
 498
 499                         # Print all matching networks
 500                         for n in db.search_networks(asn=asn, family=ns.family):
 501                                 f.write(n)
 502
 503                         f.finish()
 504
 505         def handle_list_networks_by_cc(self, db, ns):
 506                 writer = self.__get_output_formatter(ns)
 507
 508                 for country_code in ns.country_code:
 509                         # Open standard output
 510                         f = writer(sys.stdout, prefix=country_code)
 511
 512                         # Print all matching networks
 513                         for n in db.search_networks(country_code=country_code, family=ns.family):
 514                                 f.write(n)
 515
 516                         f.finish()
 517
 518         def handle_list_networks_by_flags(self, db, ns):
 519                 flags = 0
 520
 521                 if ns.anonymous_proxy:
 522                         flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
 523
 524                 if ns.satellite_provider:
 525                         flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
 526
 527                 if ns.anycast:
 528                         flags |= location.NETWORK_FLAG_ANYCAST
 529
 530                 if not flags:
 531                         raise ValueError(_("You must at least pass one flag"))
 532
 533                 writer = self.__get_output_formatter(ns)
 534                 f = writer(sys.stdout, prefix="custom")
 535
 536                 for n in db.search_networks(flags=flags, family=ns.family):
 537                         f.write(n)
 538
 539                 f.finish()
 540
 541         def handle_export(self, db, ns):
 542                 countries, asns = [], []
 543
 544                 # Translate family
 545                 if ns.family:
 546                         families = [ ns.family ]
 547                 else:
 548                         families = [ socket.AF_INET6, socket.AF_INET ]
 549
 550                 for object in ns.objects:
 551                         m = re.match("^AS(\d+)$", object)
 552                         if m:
 553                                 object = int(m.group(1))
 554
 555                                 asns.append(object)
 556
 557                         elif location.country_code_is_valid(object) \
 558                                         or object in ("A1", "A2", "A3"):
 559                                 countries.append(object)
 560
 561                         else:
 562                                 log.warning("Invalid argument: %s" % object)
 563                                 continue
 564
 565                 # Default to exporting all countries
 566                 if not countries and not asns:
 567                         countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
 568
 569                 # Select the output format
 570                 writer = self.__get_output_formatter(ns)
 571
 572                 e = location.export.Exporter(db, writer)
 573                 e.export(ns.directory, countries=countries, asns=asns, families=families)
 574
 575
 576 def format_timedelta(t):
 577         s = []
 578
 579         if t.days:
 580                 s.append(
 581                         _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
 582                 )
 583
 584         hours = t.seconds // 3600
 585         if hours:
 586                 s.append(
 587                         _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
 588                 )
 589
 590         minutes = (t.seconds % 3600) // 60
 591         if minutes:
 592                 s.append(
 593                         _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
 594                 )
 595
 596         seconds = t.seconds % 60
 597         if t.seconds:
 598                 s.append(
 599                         _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
 600                 )
 601
 602         if not s:
 603                 return _("Now")
 604
 605         return _("%s ago") % ", ".join(s)
 606
 607 def main():
 608         # Run the command line interface
 609         c = CLI()
 610         c.run()
 611
 612 main()