src/scripts/location.in

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # libloc - A library to determine the location of someone on the Internet     #
   5 #                                                                             #
   6 # Copyright (C) 2017-2021 IPFire Development Team <info@ipfire.org>           #
   7 #                                                                             #
   8 # This library is free software; you can redistribute it and/or               #
   9 # modify it under the terms of the GNU Lesser General Public                  #
  10 # License as published by the Free Software Foundation; either                #
  11 # version 2.1 of the License, or (at your option) any later version.          #
  12 #                                                                             #
  13 # This library is distributed in the hope that it will be useful,             #
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
  16 # Lesser General Public License for more details.                             #
  17 #                                                                             #
  18 ###############################################################################
  19
  20 import argparse
  21 import datetime
  22 import ipaddress
  23 import logging
  24 import os
  25 import re
  26 import shutil
  27 import socket
  28 import sys
  29 import time
  30
  31 # Load our location module
  32 import location
  33 import location.downloader
  34 import location.export
  35
  36 from location.i18n import _
  37
  38 # Setup logging
  39 log = logging.getLogger("location")
  40
  41 # Output formatters
  42
  43 class CLI(object):
  44         def parse_cli(self):
  45                 parser = argparse.ArgumentParser(
  46                         description=_("Location Database Command Line Interface"),
  47                 )
  48                 subparsers = parser.add_subparsers()
  49
  50                 # Global configuration flags
  51                 parser.add_argument("--debug", action="store_true",
  52                         help=_("Enable debug output"))
  53                 parser.add_argument("--quiet", action="store_true",
  54                         help=_("Enable quiet mode"))
  55
  56                 # version
  57                 parser.add_argument("--version", action="version",
  58                         version="%(prog)s @VERSION@")
  59
  60                 # database
  61                 parser.add_argument("--database", "-d",
  62                         default="@databasedir@/database.db", help=_("Path to database"),
  63                 )
  64
  65                 # public key
  66                 parser.add_argument("--public-key", "-k",
  67                         default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
  68                 )
  69
  70                 # Show the database version
  71                 version = subparsers.add_parser("version",
  72                         help=_("Show database version"))
  73                 version.set_defaults(func=self.handle_version)
  74
  75                 # lookup an IP address
  76                 lookup = subparsers.add_parser("lookup",
  77                         help=_("Lookup one or multiple IP addresses"),
  78                 )
  79                 lookup.add_argument("address", nargs="+")
  80                 lookup.set_defaults(func=self.handle_lookup)
  81
  82                 # Dump the whole database
  83                 dump = subparsers.add_parser("dump",
  84                         help=_("Dump the entire database"),
  85                 )
  86                 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
  87                 dump.set_defaults(func=self.handle_dump)
  88
  89                 # Update
  90                 update = subparsers.add_parser("update", help=_("Update database"))
  91                 update.add_argument("--cron",
  92                         help=_("Update the library only once per interval"),
  93                         choices=("daily", "weekly", "monthly"),
  94                 )
  95                 update.set_defaults(func=self.handle_update)
  96
  97                 # Verify
  98                 verify = subparsers.add_parser("verify",
  99                         help=_("Verify the downloaded database"))
 100                 verify.set_defaults(func=self.handle_verify)
 101
 102                 # Get AS
 103                 get_as = subparsers.add_parser("get-as",
 104                         help=_("Get information about one or multiple Autonomous Systems"),
 105                 )
 106                 get_as.add_argument("asn", nargs="+")
 107                 get_as.set_defaults(func=self.handle_get_as)
 108
 109                 # Search for AS
 110                 search_as = subparsers.add_parser("search-as",
 111                         help=_("Search for Autonomous Systems that match the string"),
 112                 )
 113                 search_as.add_argument("query", nargs=1)
 114                 search_as.set_defaults(func=self.handle_search_as)
 115
 116                 # List all networks in an AS
 117                 list_networks_by_as = subparsers.add_parser("list-networks-by-as",
 118                         help=_("Lists all networks in an AS"),
 119                 )
 120                 list_networks_by_as.add_argument("asn", nargs=1, type=int)
 121                 list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
 122                 list_networks_by_as.add_argument("--format",
 123                         choices=location.export.formats.keys(), default="list")
 124                 list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
 125
 126                 # List all networks in a country
 127                 list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
 128                         help=_("Lists all networks in a country"),
 129                 )
 130                 list_networks_by_cc.add_argument("country_code", nargs=1)
 131                 list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
 132                 list_networks_by_cc.add_argument("--format",
 133                         choices=location.export.formats.keys(), default="list")
 134                 list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
 135
 136                 # List all networks with flags
 137                 list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
 138                         help=_("Lists all networks with flags"),
 139                 )
 140                 list_networks_by_flags.add_argument("--anonymous-proxy",
 141                         action="store_true", help=_("Anonymous Proxies"),
 142                 )
 143                 list_networks_by_flags.add_argument("--satellite-provider",
 144                         action="store_true", help=_("Satellite Providers"),
 145                 )
 146                 list_networks_by_flags.add_argument("--anycast",
 147                         action="store_true", help=_("Anycasts"),
 148                 )
 149                 list_networks_by_flags.add_argument("--drop",
 150                         action="store_true", help=_("Hostile Networks safe to drop"),
 151                 )
 152                 list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
 153                 list_networks_by_flags.add_argument("--format",
 154                         choices=location.export.formats.keys(), default="list")
 155                 list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
 156
 157                 # List bogons
 158                 list_bogons = subparsers.add_parser("list-bogons",
 159                         help=_("Lists all bogons"),
 160                 )
 161                 list_bogons.add_argument("--family", choices=("ipv6", "ipv4"))
 162                 list_bogons.add_argument("--format",
 163                         choices=location.export.formats.keys(), default="list")
 164                 list_bogons.set_defaults(func=self.handle_list_bogons)
 165
 166                 # List countries
 167                 list_countries = subparsers.add_parser("list-countries",
 168                         help=_("Lists all countries"),
 169                 )
 170                 list_countries.add_argument("--show-name",
 171                         action="store_true", help=_("Show the name of the country"),
 172                 )
 173                 list_countries.add_argument("--show-continent",
 174                         action="store_true", help=_("Show the continent"),
 175                 )
 176                 list_countries.set_defaults(func=self.handle_list_countries)
 177
 178                 # Export
 179                 export = subparsers.add_parser("export",
 180                         help=_("Exports data in many formats to load it into packet filters"),
 181                 )
 182                 export.add_argument("--format", help=_("Output format"),
 183                         choices=location.export.formats.keys(), default="list")
 184                 export.add_argument("--directory", help=_("Output directory"))
 185                 export.add_argument("--family",
 186                         help=_("Specify address family"), choices=("ipv6", "ipv4"),
 187                 )
 188                 export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
 189                 export.set_defaults(func=self.handle_export)
 190
 191                 args = parser.parse_args()
 192
 193                 # Configure logging
 194                 if args.debug:
 195                         location.logger.set_level(logging.DEBUG)
 196                 elif args.quiet:
 197                         location.logger.set_level(logging.WARNING)
 198
 199                 # Print usage if no action was given
 200                 if not "func" in args:
 201                         parser.print_usage()
 202                         sys.exit(2)
 203
 204                 return args
 205
 206         def run(self):
 207                 # Parse command line arguments
 208                 args = self.parse_cli()
 209
 210                 # Open database
 211                 try:
 212                         db = location.Database(args.database)
 213                 except FileNotFoundError as e:
 214                         # Allow continuing without a database
 215                         if args.func == self.handle_update:
 216                                 db = None
 217
 218                         else:
 219                                 sys.stderr.write("location: Could not open database %s: %s\n" \
 220                                         % (args.database, e))
 221                                 sys.exit(1)
 222
 223                 # Translate family (if present)
 224                 if "family" in args:
 225                         if args.family == "ipv6":
 226                                 args.family = socket.AF_INET6
 227                         elif args.family == "ipv4":
 228                                 args.family = socket.AF_INET
 229                         else:
 230                                 args.family = 0
 231
 232                 # Call function
 233                 try:
 234                         ret = args.func(db, args)
 235
 236                 # Catch invalid inputs
 237                 except ValueError as e:
 238                         sys.stderr.write("%s\n" % e)
 239                         ret = 2
 240
 241                 # Catch any other exceptions
 242                 except Exception as e:
 243                         sys.stderr.write("%s\n" % e)
 244                         ret = 1
 245
 246                 # Return with exit code
 247                 if ret:
 248                         sys.exit(ret)
 249
 250                 # Otherwise just exit
 251                 sys.exit(0)
 252
 253         def handle_version(self, db, ns):
 254                 """
 255                         Print the version of the database
 256                 """
 257                 t = time.strftime(
 258                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 259                 )
 260
 261                 print(t)
 262
 263         def handle_lookup(self, db, ns):
 264                 ret = 0
 265
 266                 format = "  %-24s: %s"
 267
 268                 for address in ns.address:
 269                         try:
 270                                 network = db.lookup(address)
 271                         except ValueError:
 272                                 print(_("Invalid IP address: %s") % address, file=sys.stderr)
 273                                 return 2
 274
 275                         args = {
 276                                 "address" : address,
 277                                 "network" : network,
 278                         }
 279
 280                         # Nothing found?
 281                         if not network:
 282                                 print(_("Nothing found for %(address)s") % args, file=sys.stderr)
 283                                 ret = 1
 284                                 continue
 285
 286                         print("%s:" % address)
 287                         print(format % (_("Network"), network))
 288
 289                         # Print country
 290                         if network.country_code:
 291                                 country = db.get_country(network.country_code)
 292
 293                                 print(format % (
 294                                         _("Country"),
 295                                         country.name if country else network.country_code),
 296                                 )
 297
 298                         # Print AS information
 299                         if network.asn:
 300                                 autonomous_system = db.get_as(network.asn)
 301
 302                                 print(format % (
 303                                         _("Autonomous System"),
 304                                         autonomous_system or "AS%s" % network.asn),
 305                                 )
 306
 307                         # Anonymous Proxy
 308                         if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
 309                                 print(format % (
 310                                         _("Anonymous Proxy"), _("yes"),
 311                                 ))
 312
 313                         # Satellite Provider
 314                         if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
 315                                 print(format % (
 316                                         _("Satellite Provider"), _("yes"),
 317                                 ))
 318
 319                         # Anycast
 320                         if network.has_flag(location.NETWORK_FLAG_ANYCAST):
 321                                 print(format % (
 322                                         _("Anycast"), _("yes"),
 323                                 ))
 324
 325                         # Hostile Network
 326                         if network.has_flag(location.NETWORK_FLAG_DROP):
 327                                 print(format % (
 328                                         _("Hostile Network safe to drop"), _("yes"),
 329                                 ))
 330
 331                 return ret
 332
 333         def handle_dump(self, db, ns):
 334                 # Use output file or write to stdout
 335                 f = ns.output or sys.stdout
 336
 337                 # Format everything like this
 338                 format = "%-24s %s\n"
 339
 340                 # Write metadata
 341                 f.write("#\n# Location Database Export\n#\n")
 342
 343                 f.write("# Generated: %s\n" % time.strftime(
 344                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 345                 ))
 346
 347                 if db.vendor:
 348                         f.write("# Vendor:    %s\n" % db.vendor)
 349
 350                 if db.license:
 351                         f.write("# License:   %s\n" % db.license)
 352
 353                 f.write("#\n")
 354
 355                 if db.description:
 356                         for line in db.description.splitlines():
 357                                 line = "# %s" % line
 358                                 f.write("%s\n" % line.rstrip())
 359
 360                         f.write("#\n")
 361
 362                 # Iterate over all ASes
 363                 for a in db.ases:
 364                         f.write("\n")
 365                         f.write(format % ("aut-num:", "AS%s" % a.number))
 366                         f.write(format % ("name:", a.name))
 367
 368                 flags = {
 369                         location.NETWORK_FLAG_ANONYMOUS_PROXY    : "is-anonymous-proxy:",
 370                         location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
 371                         location.NETWORK_FLAG_ANYCAST            : "is-anycast:",
 372                         location.NETWORK_FLAG_DROP               : "drop:",
 373                 }
 374
 375                 # Iterate over all networks
 376                 for n in db.networks:
 377                         f.write("\n")
 378                         f.write(format % ("net:", n))
 379
 380                         if n.country_code:
 381                                 f.write(format % ("country:", n.country_code))
 382
 383                         if n.asn:
 384                                 f.write(format % ("aut-num:", n.asn))
 385
 386                         # Print all flags
 387                         for flag in flags:
 388                                 if n.has_flag(flag):
 389                                         f.write(format % (flags[flag], "yes"))
 390
 391         def handle_get_as(self, db, ns):
 392                 """
 393                         Gets information about Autonomous Systems
 394                 """
 395                 ret = 0
 396
 397                 for asn in ns.asn:
 398                         try:
 399                                 asn = int(asn)
 400                         except ValueError:
 401                                 print(_("Invalid ASN: %s") % asn, file=sys.stderr)
 402                                 ret = 1
 403                                 continue
 404
 405                         # Fetch AS from database
 406                         a = db.get_as(asn)
 407
 408                         # Nothing found
 409                         if not a:
 410                                 print(_("Could not find AS%s") % asn, file=sys.stderr)
 411                                 ret = 1
 412                                 continue
 413
 414                         print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
 415
 416                 return ret
 417
 418         def handle_search_as(self, db, ns):
 419                 for query in ns.query:
 420                         # Print all matches ASes
 421                         for a in db.search_as(query):
 422                                 print(a)
 423
 424         def handle_update(self, db, ns):
 425                 if ns.cron and db:
 426                         now = time.time()
 427
 428                         if ns.cron == "daily":
 429                                 delta = datetime.timedelta(days=1)
 430                         elif ns.cron == "weekly":
 431                                 delta = datetime.timedelta(days=7)
 432                         elif ns.cron == "monthly":
 433                                 delta = datetime.timedelta(days=30)
 434
 435                         delta = delta.total_seconds()
 436
 437                         # Check if the database has recently been updated
 438                         if db.created_at >= (now - delta):
 439                                 log.info(
 440                                         _("The database has been updated recently"),
 441                                 )
 442                                 return 3
 443
 444                 # Fetch the timestamp we need from DNS
 445                 t = location.discover_latest_version()
 446
 447                 # Check the version of the local database
 448                 if db and t and db.created_at >= t:
 449                         log.info("Already on the latest version")
 450                         return
 451
 452                 # Download the database into the correct directory
 453                 tmpdir = os.path.dirname(ns.database)
 454
 455                 # Create a downloader
 456                 d = location.downloader.Downloader()
 457
 458                 # Try downloading a new database
 459                 try:
 460                         t = d.download(public_key=ns.public_key, timestamp=t, tmpdir=tmpdir)
 461
 462                 # If no file could be downloaded, log a message
 463                 except FileNotFoundError as e:
 464                         log.error("Could not download a new database")
 465                         return 1
 466
 467                 # If we have not received a new file, there is nothing to do
 468                 if not t:
 469                         return 3
 470
 471                 # Move temporary file to destination
 472                 shutil.move(t.name, ns.database)
 473
 474                 return 0
 475
 476         def handle_verify(self, db, ns):
 477                 # Verify the database
 478                 with open(ns.public_key, "r") as f:
 479                         if not db.verify(f):
 480                                 log.error("Could not verify database")
 481                                 return 1
 482
 483                 # Success
 484                 log.info("Database successfully verified")
 485                 return 0
 486
 487         def __get_output_formatter(self, ns):
 488                 try:
 489                         cls = location.export.formats[ns.format]
 490                 except KeyError:
 491                         cls = location.export.OutputFormatter
 492
 493                 return cls
 494
 495         def handle_list_countries(self, db, ns):
 496                 for country in db.countries:
 497                         line = [
 498                                 country.code,
 499                         ]
 500
 501                         if ns.show_continent:
 502                                 line.append(country.continent_code)
 503
 504                         if ns.show_name:
 505                                 line.append(country.name)
 506
 507                         # Format the output
 508                         line = " ".join(line)
 509
 510                         # Print the output
 511                         print(line)
 512
 513         def handle_list_networks_by_as(self, db, ns):
 514                 writer = self.__get_output_formatter(ns)
 515
 516                 for asn in ns.asn:
 517                         f = writer("AS%s" % asn, f=sys.stdout)
 518
 519                         # Print all matching networks
 520                         for n in db.search_networks(asns=[asn], family=ns.family):
 521                                 f.write(n)
 522
 523                         f.finish()
 524
 525         def handle_list_networks_by_cc(self, db, ns):
 526                 writer = self.__get_output_formatter(ns)
 527
 528                 for country_code in ns.country_code:
 529                         # Open standard output
 530                         f = writer(country_code, f=sys.stdout)
 531
 532                         # Print all matching networks
 533                         for n in db.search_networks(country_codes=[country_code], family=ns.family):
 534                                 f.write(n)
 535
 536                         f.finish()
 537
 538         def handle_list_networks_by_flags(self, db, ns):
 539                 flags = 0
 540
 541                 if ns.anonymous_proxy:
 542                         flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
 543
 544                 if ns.satellite_provider:
 545                         flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
 546
 547                 if ns.anycast:
 548                         flags |= location.NETWORK_FLAG_ANYCAST
 549
 550                 if ns.drop:
 551                         flags |= location.NETWORK_FLAG_DROP
 552
 553                 if not flags:
 554                         raise ValueError(_("You must at least pass one flag"))
 555
 556                 writer = self.__get_output_formatter(ns)
 557                 f = writer("custom", f=sys.stdout)
 558
 559                 for n in db.search_networks(flags=flags, family=ns.family):
 560                         f.write(n)
 561
 562                 f.finish()
 563
 564         def handle_list_bogons(self, db, ns):
 565                 writer = self.__get_output_formatter(ns)
 566                 f = writer("bogons", f=sys.stdout)
 567
 568                 for n in db.list_bogons(family=ns.family):
 569                         f.write(n)
 570
 571                 f.finish()
 572
 573         def handle_export(self, db, ns):
 574                 countries, asns = [], []
 575
 576                 # Translate family
 577                 if ns.family:
 578                         families = [ ns.family ]
 579                 else:
 580                         families = [ socket.AF_INET6, socket.AF_INET ]
 581
 582                 for object in ns.objects:
 583                         m = re.match("^AS(\d+)$", object)
 584                         if m:
 585                                 object = int(m.group(1))
 586
 587                                 asns.append(object)
 588
 589                         elif location.country_code_is_valid(object) \
 590                                         or object in ("A1", "A2", "A3", "XD"):
 591                                 countries.append(object)
 592
 593                         else:
 594                                 log.warning("Invalid argument: %s" % object)
 595                                 continue
 596
 597                 # Default to exporting all countries
 598                 if not countries and not asns:
 599                         countries = ["A1", "A2", "A3", "XD"] + [country.code for country in db.countries]
 600
 601                 # Select the output format
 602                 writer = self.__get_output_formatter(ns)
 603
 604                 e = location.export.Exporter(db, writer)
 605                 e.export(ns.directory, countries=countries, asns=asns, families=families)
 606
 607
 608 def format_timedelta(t):
 609         s = []
 610
 611         if t.days:
 612                 s.append(
 613                         _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
 614                 )
 615
 616         hours = t.seconds // 3600
 617         if hours:
 618                 s.append(
 619                         _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
 620                 )
 621
 622         minutes = (t.seconds % 3600) // 60
 623         if minutes:
 624                 s.append(
 625                         _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
 626                 )
 627
 628         seconds = t.seconds % 60
 629         if t.seconds:
 630                 s.append(
 631                         _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
 632                 )
 633
 634         if not s:
 635                 return _("Now")
 636
 637         return _("%s ago") % ", ".join(s)
 638
 639 def main():
 640         # Run the command line interface
 641         c = CLI()
 642         c.run()
 643
 644 main()