src/python/location.in

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # libloc - A library to determine the location of someone on the Internet     #
   5 #                                                                             #
   6 # Copyright (C) 2017 IPFire Development Team <info@ipfire.org>                #
   7 #                                                                             #
   8 # This library is free software; you can redistribute it and/or               #
   9 # modify it under the terms of the GNU Lesser General Public                  #
  10 # License as published by the Free Software Foundation; either                #
  11 # version 2.1 of the License, or (at your option) any later version.          #
  12 #                                                                             #
  13 # This library is distributed in the hope that it will be useful,             #
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
  16 # Lesser General Public License for more details.                             #
  17 #                                                                             #
  18 ###############################################################################
  19
  20 import argparse
  21 import datetime
  22 import ipaddress
  23 import logging
  24 import os
  25 import re
  26 import shutil
  27 import socket
  28 import sys
  29 import time
  30
  31 # Load our location module
  32 import location
  33 import location.downloader
  34 import location.export
  35
  36 from location.i18n import _
  37
  38 # Setup logging
  39 log = logging.getLogger("location")
  40
  41 # Output formatters
  42
  43 class CLI(object):
  44         def parse_cli(self):
  45                 parser = argparse.ArgumentParser(
  46                         description=_("Location Database Command Line Interface"),
  47                 )
  48                 subparsers = parser.add_subparsers()
  49
  50                 # Global configuration flags
  51                 parser.add_argument("--debug", action="store_true",
  52                         help=_("Enable debug output"))
  53                 parser.add_argument("--quiet", action="store_true",
  54                         help=_("Enable quiet mode"))
  55
  56                 # version
  57                 parser.add_argument("--version", action="version",
  58                         version="%(prog)s @VERSION@")
  59
  60                 # database
  61                 parser.add_argument("--database", "-d",
  62                         default="@databasedir@/database.db", help=_("Path to database"),
  63                 )
  64
  65                 # public key
  66                 parser.add_argument("--public-key", "-k",
  67                         default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
  68                 )
  69
  70                 # Show the database version
  71                 version = subparsers.add_parser("version",
  72                         help=_("Show database version"))
  73                 version.set_defaults(func=self.handle_version)
  74
  75                 # lookup an IP address
  76                 lookup = subparsers.add_parser("lookup",
  77                         help=_("Lookup one or multiple IP addresses"),
  78                 )
  79                 lookup.add_argument("address", nargs="+")
  80                 lookup.set_defaults(func=self.handle_lookup)
  81
  82                 # Dump the whole database
  83                 dump = subparsers.add_parser("dump",
  84                         help=_("Dump the entire database"),
  85                 )
  86                 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
  87                 dump.set_defaults(func=self.handle_dump)
  88
  89                 # Update
  90                 update = subparsers.add_parser("update", help=_("Update database"))
  91                 update.add_argument("--cron",
  92                         help=_("Update the library only once per interval"),
  93                         choices=("daily", "weekly", "monthly"),
  94                 )
  95                 update.set_defaults(func=self.handle_update)
  96
  97                 # Verify
  98                 verify = subparsers.add_parser("verify",
  99                         help=_("Verify the downloaded database"))
 100                 verify.set_defaults(func=self.handle_verify)
 101
 102                 # Get AS
 103                 get_as = subparsers.add_parser("get-as",
 104                         help=_("Get information about one or multiple Autonomous Systems"),
 105                 )
 106                 get_as.add_argument("asn", nargs="+")
 107                 get_as.set_defaults(func=self.handle_get_as)
 108
 109                 # Search for AS
 110                 search_as = subparsers.add_parser("search-as",
 111                         help=_("Search for Autonomous Systems that match the string"),
 112                 )
 113                 search_as.add_argument("query", nargs=1)
 114                 search_as.set_defaults(func=self.handle_search_as)
 115
 116                 # List all networks in an AS
 117                 list_networks_by_as = subparsers.add_parser("list-networks-by-as",
 118                         help=_("Lists all networks in an AS"),
 119                 )
 120                 list_networks_by_as.add_argument("asn", nargs=1, type=int)
 121                 list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
 122                 list_networks_by_as.add_argument("--format",
 123                         choices=location.export.formats.keys(), default="list")
 124                 list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
 125
 126                 # List all networks in a country
 127                 list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
 128                         help=_("Lists all networks in a country"),
 129                 )
 130                 list_networks_by_cc.add_argument("country_code", nargs=1)
 131                 list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
 132                 list_networks_by_cc.add_argument("--format",
 133                         choices=location.export.formats.keys(), default="list")
 134                 list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
 135
 136                 # List all networks with flags
 137                 list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
 138                         help=_("Lists all networks with flags"),
 139                 )
 140                 list_networks_by_flags.add_argument("--anonymous-proxy",
 141                         action="store_true", help=_("Anonymous Proxies"),
 142                 )
 143                 list_networks_by_flags.add_argument("--satellite-provider",
 144                         action="store_true", help=_("Satellite Providers"),
 145                 )
 146                 list_networks_by_flags.add_argument("--anycast",
 147                         action="store_true", help=_("Anycasts"),
 148                 )
 149                 list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
 150                 list_networks_by_flags.add_argument("--format",
 151                         choices=location.export.formats.keys(), default="list")
 152                 list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
 153
 154                 # List countries
 155                 list_countries = subparsers.add_parser("list-countries",
 156                         help=_("Lists all countries"),
 157                 )
 158                 list_countries.add_argument("--show-name",
 159                         action="store_true", help=_("Show the name of the country"),
 160                 )
 161                 list_countries.add_argument("--show-continent",
 162                         action="store_true", help=_("Show the continent"),
 163                 )
 164                 list_countries.set_defaults(func=self.handle_list_countries)
 165
 166                 # Export
 167                 export = subparsers.add_parser("export",
 168                         help=_("Exports data in many formats to load it into packet filters"),
 169                 )
 170                 export.add_argument("--format", help=_("Output format"),
 171                         choices=location.export.formats.keys(), default="list")
 172                 export.add_argument("--directory", help=_("Output directory"), required=True)
 173                 export.add_argument("--family",
 174                         help=_("Specify address family"), choices=("ipv6", "ipv4"),
 175                 )
 176                 export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
 177                 export.set_defaults(func=self.handle_export)
 178
 179                 args = parser.parse_args()
 180
 181                 # Configure logging
 182                 if args.debug:
 183                         location.logger.set_level(logging.DEBUG)
 184                 elif args.quiet:
 185                         location.logger.set_level(logging.WARNING)
 186
 187                 # Print usage if no action was given
 188                 if not "func" in args:
 189                         parser.print_usage()
 190                         sys.exit(2)
 191
 192                 return args
 193
 194         def run(self):
 195                 # Parse command line arguments
 196                 args = self.parse_cli()
 197
 198                 # Open database
 199                 try:
 200                         db = location.Database(args.database)
 201                 except FileNotFoundError as e:
 202                         sys.stderr.write("location: Could not open database %s: %s\n" \
 203                                 % (args.database, e))
 204                         sys.exit(1)
 205
 206                 # Translate family (if present)
 207                 if "family" in args:
 208                         if args.family == "ipv6":
 209                                 args.family = socket.AF_INET6
 210                         elif args.family == "ipv4":
 211                                 args.family = socket.AF_INET
 212                         else:
 213                                 args.family = 0
 214
 215                 # Call function
 216                 try:
 217                         ret = args.func(db, args)
 218
 219                 # Catch invalid inputs
 220                 except ValueError as e:
 221                         sys.stderr.write("%s\n" % e)
 222                         ret = 2
 223
 224                 # Return with exit code
 225                 if ret:
 226                         sys.exit(ret)
 227
 228                 # Otherwise just exit
 229                 sys.exit(0)
 230
 231         def handle_version(self, db, ns):
 232                 """
 233                         Print the version of the database
 234                 """
 235                 t = time.strftime(
 236                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 237                 )
 238
 239                 print(t)
 240
 241         def handle_lookup(self, db, ns):
 242                 ret = 0
 243
 244                 format = "  %-24s: %s"
 245
 246                 for address in ns.address:
 247                         try:
 248                                 network = db.lookup(address)
 249                         except ValueError:
 250                                 print(_("Invalid IP address: %s") % address, file=sys.stderr)
 251
 252                         args = {
 253                                 "address" : address,
 254                                 "network" : network,
 255                         }
 256
 257                         # Nothing found?
 258                         if not network:
 259                                 print(_("Nothing found for %(address)s") % args, file=sys.stderr)
 260                                 ret = 1
 261                                 continue
 262
 263                         print("%s:" % address)
 264                         print(format % (_("Network"), network))
 265
 266                         # Print country
 267                         if network.country_code:
 268                                 country = db.get_country(network.country_code)
 269
 270                                 print(format % (
 271                                         _("Country"),
 272                                         country.name if country else network.country_code),
 273                                 )
 274
 275                         # Print AS information
 276                         if network.asn:
 277                                 autonomous_system = db.get_as(network.asn)
 278
 279                                 print(format % (
 280                                         _("Autonomous System"),
 281                                         autonomous_system or "AS%s" % network.asn),
 282                                 )
 283
 284                         # Anonymous Proxy
 285                         if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
 286                                 print(format % (
 287                                         _("Anonymous Proxy"), _("yes"),
 288                                 ))
 289
 290                         # Satellite Provider
 291                         if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
 292                                 print(format % (
 293                                         _("Satellite Provider"), _("yes"),
 294                                 ))
 295
 296                         # Anycast
 297                         if network.has_flag(location.NETWORK_FLAG_ANYCAST):
 298                                 print(format % (
 299                                         _("Anycast"), _("yes"),
 300                                 ))
 301
 302                 return ret
 303
 304         def handle_dump(self, db, ns):
 305                 # Use output file or write to stdout
 306                 f = ns.output or sys.stdout
 307
 308                 # Format everything like this
 309                 format = "%-24s %s\n"
 310
 311                 # Write metadata
 312                 f.write("#\n# Location Database Export\n#\n")
 313
 314                 f.write("# Generated: %s\n" % time.strftime(
 315                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 316                 ))
 317
 318                 if db.vendor:
 319                         f.write("# Vendor:    %s\n" % db.vendor)
 320
 321                 if db.license:
 322                         f.write("# License:   %s\n" % db.license)
 323
 324                 f.write("#\n")
 325
 326                 if db.description:
 327                         for line in db.description.splitlines():
 328                                 line = "# %s" % line
 329                                 f.write("%s\n" % line.rstrip())
 330
 331                         f.write("#\n")
 332
 333                 # Iterate over all ASes
 334                 for a in db.ases:
 335                         f.write("\n")
 336                         f.write(format % ("aut-num:", "AS%s" % a.number))
 337                         f.write(format % ("name:", a.name))
 338
 339                 flags = {
 340                         location.NETWORK_FLAG_ANONYMOUS_PROXY    : "is-anonymous-proxy:",
 341                         location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
 342                         location.NETWORK_FLAG_ANYCAST            : "is-anycast:",
 343                 }
 344
 345                 # Iterate over all networks
 346                 for n in db.networks:
 347                         f.write("\n")
 348                         f.write(format % ("net:", n))
 349
 350                         if n.country_code:
 351                                 f.write(format % ("country:", n.country_code))
 352
 353                         if n.asn:
 354                                 f.write(format % ("aut-num:", n.asn))
 355
 356                         # Print all flags
 357                         for flag in flags:
 358                                 if n.has_flag(flag):
 359                                         f.write(format % (flags[flag], "yes"))
 360
 361         def handle_get_as(self, db, ns):
 362                 """
 363                         Gets information about Autonomous Systems
 364                 """
 365                 ret = 0
 366
 367                 for asn in ns.asn:
 368                         try:
 369                                 asn = int(asn)
 370                         except ValueError:
 371                                 print(_("Invalid ASN: %s") % asn, file=sys.stderr)
 372                                 ret = 1
 373                                 continue
 374
 375                         # Fetch AS from database
 376                         a = db.get_as(asn)
 377
 378                         # Nothing found
 379                         if not a:
 380                                 print(_("Could not find AS%s") % asn, file=sys.stderr)
 381                                 ret = 1
 382                                 continue
 383
 384                         print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
 385
 386                 return ret
 387
 388         def handle_search_as(self, db, ns):
 389                 for query in ns.query:
 390                         # Print all matches ASes
 391                         for a in db.search_as(query):
 392                                 print(a)
 393
 394         def handle_update(self, db, ns):
 395                 if ns.cron and db:
 396                         now = datetime.datetime.utcnow()
 397
 398                         # Parse the database timestamp
 399                         t = datetime.datetime.fromtimestamp(db.created_at)
 400
 401                         if ns.cron == "daily":
 402                                 delta = datetime.timedelta(days=1)
 403                         elif ns.cron == "weekly":
 404                                 delta = datetime.timedelta(days=7)
 405                         elif ns.cron == "monthly":
 406                                 delta = datetime.timedelta(days=30)
 407
 408                         # Check if the database has recently been updated
 409                         if t >= (now - delta):
 410                                 log.info(
 411                                         _("The datase has recently be updated recently (%s)") % \
 412                                                 format_timedelta(now - t),
 413                                 )
 414                                 return
 415
 416                 # Fetch the timestamp we need from DNS
 417                 t = location.discover_latest_version()
 418
 419                 # Parse timestamp into datetime format
 420                 timestamp = datetime.datetime.fromtimestamp(t) if t else None
 421
 422                 # Check the version of the local database
 423                 if db and timestamp and db.created_at >= timestamp.timestamp():
 424                         log.info("Already on the latest version")
 425                         return
 426
 427                 # Download the database into the correct directory
 428                 tmpdir = os.path.dirname(ns.database)
 429
 430                 # Create a downloader
 431                 d = location.downloader.Downloader()
 432
 433                 # Try downloading a new database
 434                 try:
 435                         t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
 436
 437                 # If no file could be downloaded, log a message
 438                 except FileNotFoundError as e:
 439                         log.error("Could not download a new database")
 440                         return 1
 441
 442                 # If we have not received a new file, there is nothing to do
 443                 if not t:
 444                         return 3
 445
 446                 # Move temporary file to destination
 447                 shutil.move(t.name, ns.database)
 448
 449                 return 0
 450
 451         def handle_verify(self, ns):
 452                 try:
 453                         db = location.Database(ns.database)
 454                 except FileNotFoundError as e:
 455                         log.error("%s: %s" % (ns.database, e))
 456                         return 127
 457
 458                 # Verify the database
 459                 with open(ns.public_key, "r") as f:
 460                         if not db.verify(f):
 461                                 log.error("Could not verify database")
 462                                 return 1
 463
 464                 # Success
 465                 log.debug("Database successfully verified")
 466                 return 0
 467
 468         def __get_output_formatter(self, ns):
 469                 try:
 470                         cls = location.export.formats[ns.format]
 471                 except KeyError:
 472                         cls = location.export.OutputFormatter
 473
 474                 return cls
 475
 476         def handle_list_countries(self, db, ns):
 477                 for country in db.countries:
 478                         line = [
 479                                 country.code,
 480                         ]
 481
 482                         if ns.show_continent:
 483                                 line.append(country.continent_code)
 484
 485                         if ns.show_name:
 486                                 line.append(country.name)
 487
 488                         # Format the output
 489                         line = " ".join(line)
 490
 491                         # Print the output
 492                         print(line)
 493
 494         def handle_list_networks_by_as(self, db, ns):
 495                 writer = self.__get_output_formatter(ns)
 496
 497                 for asn in ns.asn:
 498                         f = writer(sys.stdout, prefix="AS%s" % asn)
 499
 500                         # Print all matching networks
 501                         for n in db.search_networks(asn=asn, family=ns.family):
 502                                 f.write(n)
 503
 504                         f.finish()
 505
 506         def handle_list_networks_by_cc(self, db, ns):
 507                 writer = self.__get_output_formatter(ns)
 508
 509                 for country_code in ns.country_code:
 510                         # Open standard output
 511                         f = writer(sys.stdout, prefix=country_code)
 512
 513                         # Print all matching networks
 514                         for n in db.search_networks(country_code=country_code, family=ns.family):
 515                                 f.write(n)
 516
 517                         f.finish()
 518
 519         def handle_list_networks_by_flags(self, db, ns):
 520                 flags = 0
 521
 522                 if ns.anonymous_proxy:
 523                         flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
 524
 525                 if ns.satellite_provider:
 526                         flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
 527
 528                 if ns.anycast:
 529                         flags |= location.NETWORK_FLAG_ANYCAST
 530
 531                 if not flags:
 532                         raise ValueError(_("You must at least pass one flag"))
 533
 534                 writer = self.__get_output_formatter(ns)
 535                 f = writer(sys.stdout, prefix="custom")
 536
 537                 for n in db.search_networks(flags=flags, family=ns.family):
 538                         f.write(n)
 539
 540                 f.finish()
 541
 542         def handle_export(self, db, ns):
 543                 countries, asns = [], []
 544
 545                 # Translate family
 546                 if ns.family:
 547                         families = [ ns.family ]
 548                 else:
 549                         families = [ socket.AF_INET6, socket.AF_INET ]
 550
 551                 for object in ns.objects:
 552                         m = re.match("^AS(\d+)$", object)
 553                         if m:
 554                                 object = int(m.group(1))
 555
 556                                 asns.append(object)
 557
 558                         elif location.country_code_is_valid(object) \
 559                                         or object in ("A1", "A2", "A3"):
 560                                 countries.append(object)
 561
 562                         else:
 563                                 log.warning("Invalid argument: %s" % object)
 564                                 continue
 565
 566                 # Default to exporting all countries
 567                 if not countries and not asns:
 568                         countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
 569
 570                 # Select the output format
 571                 writer = self.__get_output_formatter(ns)
 572
 573                 e = location.export.Exporter(db, writer)
 574                 e.export(ns.directory, countries=countries, asns=asns, families=families)
 575
 576
 577 def format_timedelta(t):
 578         s = []
 579
 580         if t.days:
 581                 s.append(
 582                         _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
 583                 )
 584
 585         hours = t.seconds // 3600
 586         if hours:
 587                 s.append(
 588                         _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
 589                 )
 590
 591         minutes = (t.seconds % 3600) // 60
 592         if minutes:
 593                 s.append(
 594                         _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
 595                 )
 596
 597         seconds = t.seconds % 60
 598         if t.seconds:
 599                 s.append(
 600                         _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
 601                 )
 602
 603         if not s:
 604                 return _("Now")
 605
 606         return _("%s ago") % ", ".join(s)
 607
 608 def main():
 609         # Run the command line interface
 610         c = CLI()
 611         c.run()
 612
 613 main()