src/python/location.in

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # libloc - A library to determine the location of someone on the Internet     #
   5 #                                                                             #
   6 # Copyright (C) 2017 IPFire Development Team <info@ipfire.org>                #
   7 #                                                                             #
   8 # This library is free software; you can redistribute it and/or               #
   9 # modify it under the terms of the GNU Lesser General Public                  #
  10 # License as published by the Free Software Foundation; either                #
  11 # version 2.1 of the License, or (at your option) any later version.          #
  12 #                                                                             #
  13 # This library is distributed in the hope that it will be useful,             #
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
  16 # Lesser General Public License for more details.                             #
  17 #                                                                             #
  18 ###############################################################################
  19
  20 import argparse
  21 import datetime
  22 import ipaddress
  23 import logging
  24 import os
  25 import re
  26 import shutil
  27 import socket
  28 import sys
  29 import time
  30
  31 # Load our location module
  32 import location
  33 import location.downloader
  34 import location.export
  35
  36 from location.i18n import _
  37
  38 # Setup logging
  39 log = logging.getLogger("location")
  40
  41 # Output formatters
  42
  43 class CLI(object):
  44         def parse_cli(self):
  45                 parser = argparse.ArgumentParser(
  46                         description=_("Location Database Command Line Interface"),
  47                 )
  48                 subparsers = parser.add_subparsers()
  49
  50                 # Global configuration flags
  51                 parser.add_argument("--debug", action="store_true",
  52                         help=_("Enable debug output"))
  53                 parser.add_argument("--quiet", action="store_true",
  54                         help=_("Enable quiet mode"))
  55
  56                 # version
  57                 parser.add_argument("--version", action="version",
  58                         version="%(prog)s @VERSION@")
  59
  60                 # database
  61                 parser.add_argument("--database", "-d",
  62                         default="@databasedir@/database.db", help=_("Path to database"),
  63                 )
  64
  65                 # public key
  66                 parser.add_argument("--public-key", "-k",
  67                         default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
  68                 )
  69
  70                 # Show the database version
  71                 version = subparsers.add_parser("version",
  72                         help=_("Show database version"))
  73                 version.set_defaults(func=self.handle_version)
  74
  75                 # lookup an IP address
  76                 lookup = subparsers.add_parser("lookup",
  77                         help=_("Lookup one or multiple IP addresses"),
  78                 )
  79                 lookup.add_argument("address", nargs="+")
  80                 lookup.set_defaults(func=self.handle_lookup)
  81
  82                 # Dump the whole database
  83                 dump = subparsers.add_parser("dump",
  84                         help=_("Dump the entire database"),
  85                 )
  86                 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
  87                 dump.set_defaults(func=self.handle_dump)
  88
  89                 # Update
  90                 update = subparsers.add_parser("update", help=_("Update database"))
  91                 update.add_argument("--cron",
  92                         help=_("Update the library only once per interval"),
  93                         choices=("daily", "weekly", "monthly"),
  94                 )
  95                 update.set_defaults(func=self.handle_update)
  96
  97                 # Verify
  98                 verify = subparsers.add_parser("verify",
  99                         help=_("Verify the downloaded database"))
 100                 verify.set_defaults(func=self.handle_verify)
 101
 102                 # Get AS
 103                 get_as = subparsers.add_parser("get-as",
 104                         help=_("Get information about one or multiple Autonomous Systems"),
 105                 )
 106                 get_as.add_argument("asn", nargs="+")
 107                 get_as.set_defaults(func=self.handle_get_as)
 108
 109                 # Search for AS
 110                 search_as = subparsers.add_parser("search-as",
 111                         help=_("Search for Autonomous Systems that match the string"),
 112                 )
 113                 search_as.add_argument("query", nargs=1)
 114                 search_as.set_defaults(func=self.handle_search_as)
 115
 116                 # List all networks in an AS
 117                 list_networks_by_as = subparsers.add_parser("list-networks-by-as",
 118                         help=_("Lists all networks in an AS"),
 119                 )
 120                 list_networks_by_as.add_argument("asn", nargs=1, type=int)
 121                 list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
 122                 list_networks_by_as.add_argument("--format",
 123                         choices=location.export.formats.keys(), default="list")
 124                 list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
 125
 126                 # List all networks in a country
 127                 list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
 128                         help=_("Lists all networks in a country"),
 129                 )
 130                 list_networks_by_cc.add_argument("country_code", nargs=1)
 131                 list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
 132                 list_networks_by_cc.add_argument("--format",
 133                         choices=location.export.formats.keys(), default="list")
 134                 list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
 135
 136                 # List all networks with flags
 137                 list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
 138                         help=_("Lists all networks with flags"),
 139                 )
 140                 list_networks_by_flags.add_argument("--anonymous-proxy",
 141                         action="store_true", help=_("Anonymous Proxies"),
 142                 )
 143                 list_networks_by_flags.add_argument("--satellite-provider",
 144                         action="store_true", help=_("Satellite Providers"),
 145                 )
 146                 list_networks_by_flags.add_argument("--anycast",
 147                         action="store_true", help=_("Anycasts"),
 148                 )
 149                 list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
 150                 list_networks_by_flags.add_argument("--format",
 151                         choices=location.export.formats.keys(), default="list")
 152                 list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
 153
 154                 # List countries
 155                 list_countries = subparsers.add_parser("list-countries",
 156                         help=_("Lists all countries"),
 157                 )
 158                 list_countries.add_argument("--show-name",
 159                         action="store_true", help=_("Show the name of the country"),
 160                 )
 161                 list_countries.add_argument("--show-continent",
 162                         action="store_true", help=_("Show the continent"),
 163                 )
 164                 list_countries.set_defaults(func=self.handle_list_countries)
 165
 166                 # Export
 167                 export = subparsers.add_parser("export",
 168                         help=_("Exports data in many formats to load it into packet filters"),
 169                 )
 170                 export.add_argument("--format", help=_("Output format"),
 171                         choices=location.export.formats.keys(), default="list")
 172                 export.add_argument("--directory", help=_("Output directory"), required=True)
 173                 export.add_argument("--family",
 174                         help=_("Specify address family"), choices=("ipv6", "ipv4"),
 175                 )
 176                 export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
 177                 export.set_defaults(func=self.handle_export)
 178
 179                 args = parser.parse_args()
 180
 181                 # Configure logging
 182                 if args.debug:
 183                         location.logger.set_level(logging.DEBUG)
 184                 elif args.quiet:
 185                         location.logger.set_level(logging.WARNING)
 186
 187                 # Print usage if no action was given
 188                 if not "func" in args:
 189                         parser.print_usage()
 190                         sys.exit(2)
 191
 192                 return args
 193
 194         def run(self):
 195                 # Parse command line arguments
 196                 args = self.parse_cli()
 197
 198                 # Open database
 199                 try:
 200                         db = location.Database(args.database)
 201                 except FileNotFoundError as e:
 202                         # Allow continuing without a database
 203                         if args.func == self.handle_update:
 204                                 db = None
 205
 206                         else:
 207                                 sys.stderr.write("location: Could not open database %s: %s\n" \
 208                                         % (args.database, e))
 209                                 sys.exit(1)
 210
 211                 # Translate family (if present)
 212                 if "family" in args:
 213                         if args.family == "ipv6":
 214                                 args.family = socket.AF_INET6
 215                         elif args.family == "ipv4":
 216                                 args.family = socket.AF_INET
 217                         else:
 218                                 args.family = 0
 219
 220                 # Call function
 221                 try:
 222                         ret = args.func(db, args)
 223
 224                 # Catch invalid inputs
 225                 except ValueError as e:
 226                         sys.stderr.write("%s\n" % e)
 227                         ret = 2
 228
 229                 # Return with exit code
 230                 if ret:
 231                         sys.exit(ret)
 232
 233                 # Otherwise just exit
 234                 sys.exit(0)
 235
 236         def handle_version(self, db, ns):
 237                 """
 238                         Print the version of the database
 239                 """
 240                 t = time.strftime(
 241                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 242                 )
 243
 244                 print(t)
 245
 246         def handle_lookup(self, db, ns):
 247                 ret = 0
 248
 249                 format = "  %-24s: %s"
 250
 251                 for address in ns.address:
 252                         try:
 253                                 network = db.lookup(address)
 254                         except ValueError:
 255                                 print(_("Invalid IP address: %s") % address, file=sys.stderr)
 256
 257                         args = {
 258                                 "address" : address,
 259                                 "network" : network,
 260                         }
 261
 262                         # Nothing found?
 263                         if not network:
 264                                 print(_("Nothing found for %(address)s") % args, file=sys.stderr)
 265                                 ret = 1
 266                                 continue
 267
 268                         print("%s:" % address)
 269                         print(format % (_("Network"), network))
 270
 271                         # Print country
 272                         if network.country_code:
 273                                 country = db.get_country(network.country_code)
 274
 275                                 print(format % (
 276                                         _("Country"),
 277                                         country.name if country else network.country_code),
 278                                 )
 279
 280                         # Print AS information
 281                         if network.asn:
 282                                 autonomous_system = db.get_as(network.asn)
 283
 284                                 print(format % (
 285                                         _("Autonomous System"),
 286                                         autonomous_system or "AS%s" % network.asn),
 287                                 )
 288
 289                         # Anonymous Proxy
 290                         if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
 291                                 print(format % (
 292                                         _("Anonymous Proxy"), _("yes"),
 293                                 ))
 294
 295                         # Satellite Provider
 296                         if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
 297                                 print(format % (
 298                                         _("Satellite Provider"), _("yes"),
 299                                 ))
 300
 301                         # Anycast
 302                         if network.has_flag(location.NETWORK_FLAG_ANYCAST):
 303                                 print(format % (
 304                                         _("Anycast"), _("yes"),
 305                                 ))
 306
 307                 return ret
 308
 309         def handle_dump(self, db, ns):
 310                 # Use output file or write to stdout
 311                 f = ns.output or sys.stdout
 312
 313                 # Format everything like this
 314                 format = "%-24s %s\n"
 315
 316                 # Write metadata
 317                 f.write("#\n# Location Database Export\n#\n")
 318
 319                 f.write("# Generated: %s\n" % time.strftime(
 320                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 321                 ))
 322
 323                 if db.vendor:
 324                         f.write("# Vendor:    %s\n" % db.vendor)
 325
 326                 if db.license:
 327                         f.write("# License:   %s\n" % db.license)
 328
 329                 f.write("#\n")
 330
 331                 if db.description:
 332                         for line in db.description.splitlines():
 333                                 line = "# %s" % line
 334                                 f.write("%s\n" % line.rstrip())
 335
 336                         f.write("#\n")
 337
 338                 # Iterate over all ASes
 339                 for a in db.ases:
 340                         f.write("\n")
 341                         f.write(format % ("aut-num:", "AS%s" % a.number))
 342                         f.write(format % ("name:", a.name))
 343
 344                 flags = {
 345                         location.NETWORK_FLAG_ANONYMOUS_PROXY    : "is-anonymous-proxy:",
 346                         location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
 347                         location.NETWORK_FLAG_ANYCAST            : "is-anycast:",
 348                 }
 349
 350                 # Iterate over all networks
 351                 for n in db.networks:
 352                         f.write("\n")
 353                         f.write(format % ("net:", n))
 354
 355                         if n.country_code:
 356                                 f.write(format % ("country:", n.country_code))
 357
 358                         if n.asn:
 359                                 f.write(format % ("aut-num:", n.asn))
 360
 361                         # Print all flags
 362                         for flag in flags:
 363                                 if n.has_flag(flag):
 364                                         f.write(format % (flags[flag], "yes"))
 365
 366         def handle_get_as(self, db, ns):
 367                 """
 368                         Gets information about Autonomous Systems
 369                 """
 370                 ret = 0
 371
 372                 for asn in ns.asn:
 373                         try:
 374                                 asn = int(asn)
 375                         except ValueError:
 376                                 print(_("Invalid ASN: %s") % asn, file=sys.stderr)
 377                                 ret = 1
 378                                 continue
 379
 380                         # Fetch AS from database
 381                         a = db.get_as(asn)
 382
 383                         # Nothing found
 384                         if not a:
 385                                 print(_("Could not find AS%s") % asn, file=sys.stderr)
 386                                 ret = 1
 387                                 continue
 388
 389                         print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
 390
 391                 return ret
 392
 393         def handle_search_as(self, db, ns):
 394                 for query in ns.query:
 395                         # Print all matches ASes
 396                         for a in db.search_as(query):
 397                                 print(a)
 398
 399         def handle_update(self, db, ns):
 400                 if ns.cron and db:
 401                         now = time.time()
 402
 403                         if ns.cron == "daily":
 404                                 delta = datetime.timedelta(days=1)
 405                         elif ns.cron == "weekly":
 406                                 delta = datetime.timedelta(days=7)
 407                         elif ns.cron == "monthly":
 408                                 delta = datetime.timedelta(days=30)
 409
 410                         delta = delta.total_seconds()
 411
 412                         # Check if the database has recently been updated
 413                         if db.created_at >= (now - delta):
 414                                 log.info(
 415                                         _("The database has been updated recently"),
 416                                 )
 417                                 return 3
 418
 419                 # Fetch the timestamp we need from DNS
 420                 t = location.discover_latest_version()
 421
 422                 # Check the version of the local database
 423                 if db and t and db.created_at >= t:
 424                         log.info("Already on the latest version")
 425                         return
 426
 427                 # Download the database into the correct directory
 428                 tmpdir = os.path.dirname(ns.database)
 429
 430                 # Create a downloader
 431                 d = location.downloader.Downloader()
 432
 433                 # Try downloading a new database
 434                 try:
 435                         t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
 436
 437                 # If no file could be downloaded, log a message
 438                 except FileNotFoundError as e:
 439                         log.error("Could not download a new database")
 440                         return 1
 441
 442                 # If we have not received a new file, there is nothing to do
 443                 if not t:
 444                         return 3
 445
 446                 # Move temporary file to destination
 447                 shutil.move(t.name, ns.database)
 448
 449                 return 0
 450
 451         def handle_verify(self, db, ns):
 452                 # Verify the database
 453                 with open(ns.public_key, "r") as f:
 454                         if not db.verify(f):
 455                                 log.error("Could not verify database")
 456                                 return 1
 457
 458                 # Success
 459                 log.debug("Database successfully verified")
 460                 return 0
 461
 462         def __get_output_formatter(self, ns):
 463                 try:
 464                         cls = location.export.formats[ns.format]
 465                 except KeyError:
 466                         cls = location.export.OutputFormatter
 467
 468                 return cls
 469
 470         def handle_list_countries(self, db, ns):
 471                 for country in db.countries:
 472                         line = [
 473                                 country.code,
 474                         ]
 475
 476                         if ns.show_continent:
 477                                 line.append(country.continent_code)
 478
 479                         if ns.show_name:
 480                                 line.append(country.name)
 481
 482                         # Format the output
 483                         line = " ".join(line)
 484
 485                         # Print the output
 486                         print(line)
 487
 488         def handle_list_networks_by_as(self, db, ns):
 489                 writer = self.__get_output_formatter(ns)
 490
 491                 for asn in ns.asn:
 492                         f = writer(sys.stdout, prefix="AS%s" % asn)
 493
 494                         # Print all matching networks
 495                         for n in db.search_networks(asn=asn, family=ns.family):
 496                                 f.write(n)
 497
 498                         f.finish()
 499
 500         def handle_list_networks_by_cc(self, db, ns):
 501                 writer = self.__get_output_formatter(ns)
 502
 503                 for country_code in ns.country_code:
 504                         # Open standard output
 505                         f = writer(sys.stdout, prefix=country_code)
 506
 507                         # Print all matching networks
 508                         for n in db.search_networks(country_code=country_code, family=ns.family):
 509                                 f.write(n)
 510
 511                         f.finish()
 512
 513         def handle_list_networks_by_flags(self, db, ns):
 514                 flags = 0
 515
 516                 if ns.anonymous_proxy:
 517                         flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
 518
 519                 if ns.satellite_provider:
 520                         flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
 521
 522                 if ns.anycast:
 523                         flags |= location.NETWORK_FLAG_ANYCAST
 524
 525                 if not flags:
 526                         raise ValueError(_("You must at least pass one flag"))
 527
 528                 writer = self.__get_output_formatter(ns)
 529                 f = writer(sys.stdout, prefix="custom")
 530
 531                 for n in db.search_networks(flags=flags, family=ns.family):
 532                         f.write(n)
 533
 534                 f.finish()
 535
 536         def handle_export(self, db, ns):
 537                 countries, asns = [], []
 538
 539                 # Translate family
 540                 if ns.family:
 541                         families = [ ns.family ]
 542                 else:
 543                         families = [ socket.AF_INET6, socket.AF_INET ]
 544
 545                 for object in ns.objects:
 546                         m = re.match("^AS(\d+)$", object)
 547                         if m:
 548                                 object = int(m.group(1))
 549
 550                                 asns.append(object)
 551
 552                         elif location.country_code_is_valid(object) \
 553                                         or object in ("A1", "A2", "A3"):
 554                                 countries.append(object)
 555
 556                         else:
 557                                 log.warning("Invalid argument: %s" % object)
 558                                 continue
 559
 560                 # Default to exporting all countries
 561                 if not countries and not asns:
 562                         countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
 563
 564                 # Select the output format
 565                 writer = self.__get_output_formatter(ns)
 566
 567                 e = location.export.Exporter(db, writer)
 568                 e.export(ns.directory, countries=countries, asns=asns, families=families)
 569
 570
 571 def format_timedelta(t):
 572         s = []
 573
 574         if t.days:
 575                 s.append(
 576                         _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
 577                 )
 578
 579         hours = t.seconds // 3600
 580         if hours:
 581                 s.append(
 582                         _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
 583                 )
 584
 585         minutes = (t.seconds % 3600) // 60
 586         if minutes:
 587                 s.append(
 588                         _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
 589                 )
 590
 591         seconds = t.seconds % 60
 592         if t.seconds:
 593                 s.append(
 594                         _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
 595                 )
 596
 597         if not s:
 598                 return _("Now")
 599
 600         return _("%s ago") % ", ".join(s)
 601
 602 def main():
 603         # Run the command line interface
 604         c = CLI()
 605         c.run()
 606
 607 main()