src/python/location.in

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # libloc - A library to determine the location of someone on the Internet     #
   5 #                                                                             #
   6 # Copyright (C) 2017 IPFire Development Team <info@ipfire.org>                #
   7 #                                                                             #
   8 # This library is free software; you can redistribute it and/or               #
   9 # modify it under the terms of the GNU Lesser General Public                  #
  10 # License as published by the Free Software Foundation; either                #
  11 # version 2.1 of the License, or (at your option) any later version.          #
  12 #                                                                             #
  13 # This library is distributed in the hope that it will be useful,             #
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
  16 # Lesser General Public License for more details.                             #
  17 #                                                                             #
  18 ###############################################################################
  19
  20 import argparse
  21 import datetime
  22 import ipaddress
  23 import logging
  24 import os
  25 import re
  26 import shutil
  27 import socket
  28 import sys
  29 import time
  30
  31 # Load our location module
  32 import location
  33 import location.downloader
  34 import location.export
  35
  36 from location.i18n import _
  37
  38 # Setup logging
  39 log = logging.getLogger("location")
  40
  41 # Output formatters
  42
  43 class CLI(object):
  44         def parse_cli(self):
  45                 parser = argparse.ArgumentParser(
  46                         description=_("Location Database Command Line Interface"),
  47                 )
  48                 subparsers = parser.add_subparsers()
  49
  50                 # Global configuration flags
  51                 parser.add_argument("--debug", action="store_true",
  52                         help=_("Enable debug output"))
  53                 parser.add_argument("--quiet", action="store_true",
  54                         help=_("Enable quiet mode"))
  55
  56                 # version
  57                 parser.add_argument("--version", action="version",
  58                         version="%(prog)s @VERSION@")
  59
  60                 # database
  61                 parser.add_argument("--database", "-d",
  62                         default="@databasedir@/database.db", help=_("Path to database"),
  63                 )
  64
  65                 # public key
  66                 parser.add_argument("--public-key", "-k",
  67                         default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
  68                 )
  69
  70                 # Show the database version
  71                 version = subparsers.add_parser("version",
  72                         help=_("Show database version"))
  73                 version.set_defaults(func=self.handle_version)
  74
  75                 # lookup an IP address
  76                 lookup = subparsers.add_parser("lookup",
  77                         help=_("Lookup one or multiple IP addresses"),
  78                 )
  79                 lookup.add_argument("address", nargs="+")
  80                 lookup.set_defaults(func=self.handle_lookup)
  81
  82                 # Dump the whole database
  83                 dump = subparsers.add_parser("dump",
  84                         help=_("Dump the entire database"),
  85                 )
  86                 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
  87                 dump.set_defaults(func=self.handle_dump)
  88
  89                 # Update
  90                 update = subparsers.add_parser("update", help=_("Update database"))
  91                 update.add_argument("--cron",
  92                         help=_("Update the library only once per interval"),
  93                         choices=("daily", "weekly", "monthly"),
  94                 )
  95                 update.set_defaults(func=self.handle_update)
  96
  97                 # Verify
  98                 verify = subparsers.add_parser("verify",
  99                         help=_("Verify the downloaded database"))
 100                 verify.set_defaults(func=self.handle_verify)
 101
 102                 # Get AS
 103                 get_as = subparsers.add_parser("get-as",
 104                         help=_("Get information about one or multiple Autonomous Systems"),
 105                 )
 106                 get_as.add_argument("asn", nargs="+")
 107                 get_as.set_defaults(func=self.handle_get_as)
 108
 109                 # Search for AS
 110                 search_as = subparsers.add_parser("search-as",
 111                         help=_("Search for Autonomous Systems that match the string"),
 112                 )
 113                 search_as.add_argument("query", nargs=1)
 114                 search_as.set_defaults(func=self.handle_search_as)
 115
 116                 # List all networks in an AS
 117                 list_networks_by_as = subparsers.add_parser("list-networks-by-as",
 118                         help=_("Lists all networks in an AS"),
 119                 )
 120                 list_networks_by_as.add_argument("asn", nargs=1, type=int)
 121                 list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
 122                 list_networks_by_as.add_argument("--format",
 123                         choices=location.export.formats.keys(), default="list")
 124                 list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
 125
 126                 # List all networks in a country
 127                 list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
 128                         help=_("Lists all networks in a country"),
 129                 )
 130                 list_networks_by_cc.add_argument("country_code", nargs=1)
 131                 list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
 132                 list_networks_by_cc.add_argument("--format",
 133                         choices=location.export.formats.keys(), default="list")
 134                 list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
 135
 136                 # List all networks with flags
 137                 list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
 138                         help=_("Lists all networks with flags"),
 139                 )
 140                 list_networks_by_flags.add_argument("--anonymous-proxy",
 141                         action="store_true", help=_("Anonymous Proxies"),
 142                 )
 143                 list_networks_by_flags.add_argument("--satellite-provider",
 144                         action="store_true", help=_("Satellite Providers"),
 145                 )
 146                 list_networks_by_flags.add_argument("--anycast",
 147                         action="store_true", help=_("Anycasts"),
 148                 )
 149                 list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
 150                 list_networks_by_flags.add_argument("--format",
 151                         choices=location.export.formats.keys(), default="list")
 152                 list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
 153
 154                 # List countries
 155                 list_countries = subparsers.add_parser("list-countries",
 156                         help=_("Lists all countries"),
 157                 )
 158                 list_countries.add_argument("--show-name",
 159                         action="store_true", help=_("Show the name of the country"),
 160                 )
 161                 list_countries.add_argument("--show-continent",
 162                         action="store_true", help=_("Show the continent"),
 163                 )
 164                 list_countries.set_defaults(func=self.handle_list_countries)
 165
 166                 # Export
 167                 export = subparsers.add_parser("export",
 168                         help=_("Exports data in many formats to load it into packet filters"),
 169                 )
 170                 export.add_argument("--format", help=_("Output format"),
 171                         choices=location.export.formats.keys(), default="list")
 172                 export.add_argument("--directory", help=_("Output directory"), required=True)
 173                 export.add_argument("--family",
 174                         help=_("Specify address family"), choices=("ipv6", "ipv4"),
 175                 )
 176                 export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
 177                 export.set_defaults(func=self.handle_export)
 178
 179                 args = parser.parse_args()
 180
 181                 # Configure logging
 182                 if args.debug:
 183                         location.logger.set_level(logging.DEBUG)
 184                 elif args.quiet:
 185                         location.logger.set_level(logging.WARNING)
 186
 187                 # Print usage if no action was given
 188                 if not "func" in args:
 189                         parser.print_usage()
 190                         sys.exit(2)
 191
 192                 return args
 193
 194         def run(self):
 195                 # Parse command line arguments
 196                 args = self.parse_cli()
 197
 198                 # Open database
 199                 try:
 200                         db = location.Database(args.database)
 201                 except FileNotFoundError as e:
 202                         # Allow continuing without a database
 203                         if args.func == self.handle_update:
 204                                 db = None
 205
 206                         else:
 207                                 sys.stderr.write("location: Could not open database %s: %s\n" \
 208                                         % (args.database, e))
 209                                 sys.exit(1)
 210
 211                 # Translate family (if present)
 212                 if "family" in args:
 213                         if args.family == "ipv6":
 214                                 args.family = socket.AF_INET6
 215                         elif args.family == "ipv4":
 216                                 args.family = socket.AF_INET
 217                         else:
 218                                 args.family = 0
 219
 220                 # Call function
 221                 try:
 222                         ret = args.func(db, args)
 223
 224                 # Catch invalid inputs
 225                 except ValueError as e:
 226                         sys.stderr.write("%s\n" % e)
 227                         ret = 2
 228
 229                 # Return with exit code
 230                 if ret:
 231                         sys.exit(ret)
 232
 233                 # Otherwise just exit
 234                 sys.exit(0)
 235
 236         def handle_version(self, db, ns):
 237                 """
 238                         Print the version of the database
 239                 """
 240                 t = time.strftime(
 241                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 242                 )
 243
 244                 print(t)
 245
 246         def handle_lookup(self, db, ns):
 247                 ret = 0
 248
 249                 format = "  %-24s: %s"
 250
 251                 for address in ns.address:
 252                         try:
 253                                 network = db.lookup(address)
 254                         except ValueError:
 255                                 print(_("Invalid IP address: %s") % address, file=sys.stderr)
 256
 257                         args = {
 258                                 "address" : address,
 259                                 "network" : network,
 260                         }
 261
 262                         # Nothing found?
 263                         if not network:
 264                                 print(_("Nothing found for %(address)s") % args, file=sys.stderr)
 265                                 ret = 1
 266                                 continue
 267
 268                         print("%s:" % address)
 269                         print(format % (_("Network"), network))
 270
 271                         # Print country
 272                         if network.country_code:
 273                                 country = db.get_country(network.country_code)
 274
 275                                 print(format % (
 276                                         _("Country"),
 277                                         country.name if country else network.country_code),
 278                                 )
 279
 280                         # Print AS information
 281                         if network.asn:
 282                                 autonomous_system = db.get_as(network.asn)
 283
 284                                 print(format % (
 285                                         _("Autonomous System"),
 286                                         autonomous_system or "AS%s" % network.asn),
 287                                 )
 288
 289                         # Anonymous Proxy
 290                         if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
 291                                 print(format % (
 292                                         _("Anonymous Proxy"), _("yes"),
 293                                 ))
 294
 295                         # Satellite Provider
 296                         if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
 297                                 print(format % (
 298                                         _("Satellite Provider"), _("yes"),
 299                                 ))
 300
 301                         # Anycast
 302                         if network.has_flag(location.NETWORK_FLAG_ANYCAST):
 303                                 print(format % (
 304                                         _("Anycast"), _("yes"),
 305                                 ))
 306
 307                 return ret
 308
 309         def handle_dump(self, db, ns):
 310                 # Use output file or write to stdout
 311                 f = ns.output or sys.stdout
 312
 313                 # Format everything like this
 314                 format = "%-24s %s\n"
 315
 316                 # Write metadata
 317                 f.write("#\n# Location Database Export\n#\n")
 318
 319                 f.write("# Generated: %s\n" % time.strftime(
 320                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 321                 ))
 322
 323                 if db.vendor:
 324                         f.write("# Vendor:    %s\n" % db.vendor)
 325
 326                 if db.license:
 327                         f.write("# License:   %s\n" % db.license)
 328
 329                 f.write("#\n")
 330
 331                 if db.description:
 332                         for line in db.description.splitlines():
 333                                 line = "# %s" % line
 334                                 f.write("%s\n" % line.rstrip())
 335
 336                         f.write("#\n")
 337
 338                 # Iterate over all ASes
 339                 for a in db.ases:
 340                         f.write("\n")
 341                         f.write(format % ("aut-num:", "AS%s" % a.number))
 342                         f.write(format % ("name:", a.name))
 343
 344                 flags = {
 345                         location.NETWORK_FLAG_ANONYMOUS_PROXY    : "is-anonymous-proxy:",
 346                         location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
 347                         location.NETWORK_FLAG_ANYCAST            : "is-anycast:",
 348                 }
 349
 350                 # Iterate over all networks
 351                 for n in db.networks:
 352                         f.write("\n")
 353                         f.write(format % ("net:", n))
 354
 355                         if n.country_code:
 356                                 f.write(format % ("country:", n.country_code))
 357
 358                         if n.asn:
 359                                 f.write(format % ("aut-num:", n.asn))
 360
 361                         # Print all flags
 362                         for flag in flags:
 363                                 if n.has_flag(flag):
 364                                         f.write(format % (flags[flag], "yes"))
 365
 366         def handle_get_as(self, db, ns):
 367                 """
 368                         Gets information about Autonomous Systems
 369                 """
 370                 ret = 0
 371
 372                 for asn in ns.asn:
 373                         try:
 374                                 asn = int(asn)
 375                         except ValueError:
 376                                 print(_("Invalid ASN: %s") % asn, file=sys.stderr)
 377                                 ret = 1
 378                                 continue
 379
 380                         # Fetch AS from database
 381                         a = db.get_as(asn)
 382
 383                         # Nothing found
 384                         if not a:
 385                                 print(_("Could not find AS%s") % asn, file=sys.stderr)
 386                                 ret = 1
 387                                 continue
 388
 389                         print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
 390
 391                 return ret
 392
 393         def handle_search_as(self, db, ns):
 394                 for query in ns.query:
 395                         # Print all matches ASes
 396                         for a in db.search_as(query):
 397                                 print(a)
 398
 399         def handle_update(self, db, ns):
 400                 if ns.cron and db:
 401                         now = datetime.datetime.utcnow()
 402
 403                         # Parse the database timestamp
 404                         t = datetime.datetime.utcfromtimestamp(db.created_at)
 405
 406                         if ns.cron == "daily":
 407                                 delta = datetime.timedelta(days=1)
 408                         elif ns.cron == "weekly":
 409                                 delta = datetime.timedelta(days=7)
 410                         elif ns.cron == "monthly":
 411                                 delta = datetime.timedelta(days=30)
 412
 413                         # Check if the database has recently been updated
 414                         if t >= (now - delta):
 415                                 log.info(
 416                                         _("The database has been updated recently (%s)") % \
 417                                                 format_timedelta(now - t),
 418                                 )
 419                                 return 3
 420
 421                 # Fetch the timestamp we need from DNS
 422                 t = location.discover_latest_version()
 423
 424                 # Check the version of the local database
 425                 if db and t and db.created_at >= t:
 426                         log.info("Already on the latest version")
 427                         return
 428
 429                 # Download the database into the correct directory
 430                 tmpdir = os.path.dirname(ns.database)
 431
 432                 # Create a downloader
 433                 d = location.downloader.Downloader()
 434
 435                 # Try downloading a new database
 436                 try:
 437                         t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
 438
 439                 # If no file could be downloaded, log a message
 440                 except FileNotFoundError as e:
 441                         log.error("Could not download a new database")
 442                         return 1
 443
 444                 # If we have not received a new file, there is nothing to do
 445                 if not t:
 446                         return 3
 447
 448                 # Move temporary file to destination
 449                 shutil.move(t.name, ns.database)
 450
 451                 return 0
 452
 453         def handle_verify(self, db, ns):
 454                 # Verify the database
 455                 with open(ns.public_key, "r") as f:
 456                         if not db.verify(f):
 457                                 log.error("Could not verify database")
 458                                 return 1
 459
 460                 # Success
 461                 log.debug("Database successfully verified")
 462                 return 0
 463
 464         def __get_output_formatter(self, ns):
 465                 try:
 466                         cls = location.export.formats[ns.format]
 467                 except KeyError:
 468                         cls = location.export.OutputFormatter
 469
 470                 return cls
 471
 472         def handle_list_countries(self, db, ns):
 473                 for country in db.countries:
 474                         line = [
 475                                 country.code,
 476                         ]
 477
 478                         if ns.show_continent:
 479                                 line.append(country.continent_code)
 480
 481                         if ns.show_name:
 482                                 line.append(country.name)
 483
 484                         # Format the output
 485                         line = " ".join(line)
 486
 487                         # Print the output
 488                         print(line)
 489
 490         def handle_list_networks_by_as(self, db, ns):
 491                 writer = self.__get_output_formatter(ns)
 492
 493                 for asn in ns.asn:
 494                         f = writer(sys.stdout, prefix="AS%s" % asn)
 495
 496                         # Print all matching networks
 497                         for n in db.search_networks(asn=asn, family=ns.family):
 498                                 f.write(n)
 499
 500                         f.finish()
 501
 502         def handle_list_networks_by_cc(self, db, ns):
 503                 writer = self.__get_output_formatter(ns)
 504
 505                 for country_code in ns.country_code:
 506                         # Open standard output
 507                         f = writer(sys.stdout, prefix=country_code)
 508
 509                         # Print all matching networks
 510                         for n in db.search_networks(country_code=country_code, family=ns.family):
 511                                 f.write(n)
 512
 513                         f.finish()
 514
 515         def handle_list_networks_by_flags(self, db, ns):
 516                 flags = 0
 517
 518                 if ns.anonymous_proxy:
 519                         flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
 520
 521                 if ns.satellite_provider:
 522                         flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
 523
 524                 if ns.anycast:
 525                         flags |= location.NETWORK_FLAG_ANYCAST
 526
 527                 if not flags:
 528                         raise ValueError(_("You must at least pass one flag"))
 529
 530                 writer = self.__get_output_formatter(ns)
 531                 f = writer(sys.stdout, prefix="custom")
 532
 533                 for n in db.search_networks(flags=flags, family=ns.family):
 534                         f.write(n)
 535
 536                 f.finish()
 537
 538         def handle_export(self, db, ns):
 539                 countries, asns = [], []
 540
 541                 # Translate family
 542                 if ns.family:
 543                         families = [ ns.family ]
 544                 else:
 545                         families = [ socket.AF_INET6, socket.AF_INET ]
 546
 547                 for object in ns.objects:
 548                         m = re.match("^AS(\d+)$", object)
 549                         if m:
 550                                 object = int(m.group(1))
 551
 552                                 asns.append(object)
 553
 554                         elif location.country_code_is_valid(object) \
 555                                         or object in ("A1", "A2", "A3"):
 556                                 countries.append(object)
 557
 558                         else:
 559                                 log.warning("Invalid argument: %s" % object)
 560                                 continue
 561
 562                 # Default to exporting all countries
 563                 if not countries and not asns:
 564                         countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
 565
 566                 # Select the output format
 567                 writer = self.__get_output_formatter(ns)
 568
 569                 e = location.export.Exporter(db, writer)
 570                 e.export(ns.directory, countries=countries, asns=asns, families=families)
 571
 572
 573 def format_timedelta(t):
 574         s = []
 575
 576         if t.days:
 577                 s.append(
 578                         _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
 579                 )
 580
 581         hours = t.seconds // 3600
 582         if hours:
 583                 s.append(
 584                         _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
 585                 )
 586
 587         minutes = (t.seconds % 3600) // 60
 588         if minutes:
 589                 s.append(
 590                         _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
 591                 )
 592
 593         seconds = t.seconds % 60
 594         if t.seconds:
 595                 s.append(
 596                         _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
 597                 )
 598
 599         if not s:
 600                 return _("Now")
 601
 602         return _("%s ago") % ", ".join(s)
 603
 604 def main():
 605         # Run the command line interface
 606         c = CLI()
 607         c.run()
 608
 609 main()