src/python/location.in

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # libloc - A library to determine the location of someone on the Internet     #
   5 #                                                                             #
   6 # Copyright (C) 2017 IPFire Development Team <info@ipfire.org>                #
   7 #                                                                             #
   8 # This library is free software; you can redistribute it and/or               #
   9 # modify it under the terms of the GNU Lesser General Public                  #
  10 # License as published by the Free Software Foundation; either                #
  11 # version 2.1 of the License, or (at your option) any later version.          #
  12 #                                                                             #
  13 # This library is distributed in the hope that it will be useful,             #
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
  16 # Lesser General Public License for more details.                             #
  17 #                                                                             #
  18 ###############################################################################
  19
  20 import argparse
  21 import datetime
  22 import ipaddress
  23 import logging
  24 import os
  25 import re
  26 import shutil
  27 import socket
  28 import sys
  29 import time
  30
  31 # Load our location module
  32 import location
  33 import location.downloader
  34 import location.export
  35
  36 from location.i18n import _
  37
  38 # Setup logging
  39 log = logging.getLogger("location")
  40
  41 # Output formatters
  42
  43 class CLI(object):
  44         def parse_cli(self):
  45                 parser = argparse.ArgumentParser(
  46                         description=_("Location Database Command Line Interface"),
  47                 )
  48                 subparsers = parser.add_subparsers()
  49
  50                 # Global configuration flags
  51                 parser.add_argument("--debug", action="store_true",
  52                         help=_("Enable debug output"))
  53                 parser.add_argument("--quiet", action="store_true",
  54                         help=_("Enable quiet mode"))
  55
  56                 # version
  57                 parser.add_argument("--version", action="version",
  58                         version="%(prog)s @VERSION@")
  59
  60                 # database
  61                 parser.add_argument("--database", "-d",
  62                         default="@databasedir@/database.db", help=_("Path to database"),
  63                 )
  64
  65                 # public key
  66                 parser.add_argument("--public-key", "-k",
  67                         default="@databasedir@/signing-key.pem", help=_("Public Signing Key"),
  68                 )
  69
  70                 # Show the database version
  71                 version = subparsers.add_parser("version",
  72                         help=_("Show database version"))
  73                 version.set_defaults(func=self.handle_version)
  74
  75                 # lookup an IP address
  76                 lookup = subparsers.add_parser("lookup",
  77                         help=_("Lookup one or multiple IP addresses"),
  78                 )
  79                 lookup.add_argument("address", nargs="+")
  80                 lookup.set_defaults(func=self.handle_lookup)
  81
  82                 # Dump the whole database
  83                 dump = subparsers.add_parser("dump",
  84                         help=_("Dump the entire database"),
  85                 )
  86                 dump.add_argument("output", nargs="?", type=argparse.FileType("w"))
  87                 dump.set_defaults(func=self.handle_dump)
  88
  89                 # Update
  90                 update = subparsers.add_parser("update", help=_("Update database"))
  91                 update.add_argument("--cron",
  92                         help=_("Update the library only once per interval"),
  93                         choices=("daily", "weekly", "monthly"),
  94                 )
  95                 update.set_defaults(func=self.handle_update)
  96
  97                 # Verify
  98                 verify = subparsers.add_parser("verify",
  99                         help=_("Verify the downloaded database"))
 100                 verify.set_defaults(func=self.handle_verify)
 101
 102                 # Get AS
 103                 get_as = subparsers.add_parser("get-as",
 104                         help=_("Get information about one or multiple Autonomous Systems"),
 105                 )
 106                 get_as.add_argument("asn", nargs="+")
 107                 get_as.set_defaults(func=self.handle_get_as)
 108
 109                 # Search for AS
 110                 search_as = subparsers.add_parser("search-as",
 111                         help=_("Search for Autonomous Systems that match the string"),
 112                 )
 113                 search_as.add_argument("query", nargs=1)
 114                 search_as.set_defaults(func=self.handle_search_as)
 115
 116                 # List all networks in an AS
 117                 list_networks_by_as = subparsers.add_parser("list-networks-by-as",
 118                         help=_("Lists all networks in an AS"),
 119                 )
 120                 list_networks_by_as.add_argument("asn", nargs=1, type=int)
 121                 list_networks_by_as.add_argument("--family", choices=("ipv6", "ipv4"))
 122                 list_networks_by_as.add_argument("--format",
 123                         choices=location.export.formats.keys(), default="list")
 124                 list_networks_by_as.set_defaults(func=self.handle_list_networks_by_as)
 125
 126                 # List all networks in a country
 127                 list_networks_by_cc = subparsers.add_parser("list-networks-by-cc",
 128                         help=_("Lists all networks in a country"),
 129                 )
 130                 list_networks_by_cc.add_argument("country_code", nargs=1)
 131                 list_networks_by_cc.add_argument("--family", choices=("ipv6", "ipv4"))
 132                 list_networks_by_cc.add_argument("--format",
 133                         choices=location.export.formats.keys(), default="list")
 134                 list_networks_by_cc.set_defaults(func=self.handle_list_networks_by_cc)
 135
 136                 # List all networks with flags
 137                 list_networks_by_flags = subparsers.add_parser("list-networks-by-flags",
 138                         help=_("Lists all networks with flags"),
 139                 )
 140                 list_networks_by_flags.add_argument("--anonymous-proxy",
 141                         action="store_true", help=_("Anonymous Proxies"),
 142                 )
 143                 list_networks_by_flags.add_argument("--satellite-provider",
 144                         action="store_true", help=_("Satellite Providers"),
 145                 )
 146                 list_networks_by_flags.add_argument("--anycast",
 147                         action="store_true", help=_("Anycasts"),
 148                 )
 149                 list_networks_by_flags.add_argument("--family", choices=("ipv6", "ipv4"))
 150                 list_networks_by_flags.add_argument("--format",
 151                         choices=location.export.formats.keys(), default="list")
 152                 list_networks_by_flags.set_defaults(func=self.handle_list_networks_by_flags)
 153
 154                 # List countries
 155                 list_countries = subparsers.add_parser("list-countries",
 156                         help=_("Lists all countries"),
 157                 )
 158                 list_countries.add_argument("--show-name",
 159                         action="store_true", help=_("Show the name of the country"),
 160                 )
 161                 list_countries.add_argument("--show-continent",
 162                         action="store_true", help=_("Show the continent"),
 163                 )
 164                 list_countries.set_defaults(func=self.handle_list_countries)
 165
 166                 # Export
 167                 export = subparsers.add_parser("export",
 168                         help=_("Exports data in many formats to load it into packet filters"),
 169                 )
 170                 export.add_argument("--format", help=_("Output format"),
 171                         choices=location.export.formats.keys(), default="list")
 172                 export.add_argument("--directory", help=_("Output directory"), required=True)
 173                 export.add_argument("--family",
 174                         help=_("Specify address family"), choices=("ipv6", "ipv4"),
 175                 )
 176                 export.add_argument("objects", nargs="*", help=_("List country codes or ASNs to export"))
 177                 export.set_defaults(func=self.handle_export)
 178
 179                 args = parser.parse_args()
 180
 181                 # Configure logging
 182                 if args.debug:
 183                         location.logger.set_level(logging.DEBUG)
 184                 elif args.quiet:
 185                         location.logger.set_level(logging.WARNING)
 186
 187                 # Print usage if no action was given
 188                 if not "func" in args:
 189                         parser.print_usage()
 190                         sys.exit(2)
 191
 192                 return args
 193
 194         def run(self):
 195                 # Parse command line arguments
 196                 args = self.parse_cli()
 197
 198                 # Open database
 199                 try:
 200                         db = location.Database(args.database)
 201                 except FileNotFoundError as e:
 202                         # Allow continuing without a database
 203                         if args.func == self.handle_update:
 204                                 db = None
 205
 206                         else:
 207                                 sys.stderr.write("location: Could not open database %s: %s\n" \
 208                                         % (args.database, e))
 209                                 sys.exit(1)
 210
 211                 # Translate family (if present)
 212                 if "family" in args:
 213                         if args.family == "ipv6":
 214                                 args.family = socket.AF_INET6
 215                         elif args.family == "ipv4":
 216                                 args.family = socket.AF_INET
 217                         else:
 218                                 args.family = 0
 219
 220                 # Call function
 221                 try:
 222                         ret = args.func(db, args)
 223
 224                 # Catch invalid inputs
 225                 except ValueError as e:
 226                         sys.stderr.write("%s\n" % e)
 227                         ret = 2
 228
 229                 # Return with exit code
 230                 if ret:
 231                         sys.exit(ret)
 232
 233                 # Otherwise just exit
 234                 sys.exit(0)
 235
 236         def handle_version(self, db, ns):
 237                 """
 238                         Print the version of the database
 239                 """
 240                 t = time.strftime(
 241                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 242                 )
 243
 244                 print(t)
 245
 246         def handle_lookup(self, db, ns):
 247                 ret = 0
 248
 249                 format = "  %-24s: %s"
 250
 251                 for address in ns.address:
 252                         try:
 253                                 network = db.lookup(address)
 254                         except ValueError:
 255                                 print(_("Invalid IP address: %s") % address, file=sys.stderr)
 256                                 return 2
 257
 258                         args = {
 259                                 "address" : address,
 260                                 "network" : network,
 261                         }
 262
 263                         # Nothing found?
 264                         if not network:
 265                                 print(_("Nothing found for %(address)s") % args, file=sys.stderr)
 266                                 ret = 1
 267                                 continue
 268
 269                         print("%s:" % address)
 270                         print(format % (_("Network"), network))
 271
 272                         # Print country
 273                         if network.country_code:
 274                                 country = db.get_country(network.country_code)
 275
 276                                 print(format % (
 277                                         _("Country"),
 278                                         country.name if country else network.country_code),
 279                                 )
 280
 281                         # Print AS information
 282                         if network.asn:
 283                                 autonomous_system = db.get_as(network.asn)
 284
 285                                 print(format % (
 286                                         _("Autonomous System"),
 287                                         autonomous_system or "AS%s" % network.asn),
 288                                 )
 289
 290                         # Anonymous Proxy
 291                         if network.has_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY):
 292                                 print(format % (
 293                                         _("Anonymous Proxy"), _("yes"),
 294                                 ))
 295
 296                         # Satellite Provider
 297                         if network.has_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER):
 298                                 print(format % (
 299                                         _("Satellite Provider"), _("yes"),
 300                                 ))
 301
 302                         # Anycast
 303                         if network.has_flag(location.NETWORK_FLAG_ANYCAST):
 304                                 print(format % (
 305                                         _("Anycast"), _("yes"),
 306                                 ))
 307
 308                 return ret
 309
 310         def handle_dump(self, db, ns):
 311                 # Use output file or write to stdout
 312                 f = ns.output or sys.stdout
 313
 314                 # Format everything like this
 315                 format = "%-24s %s\n"
 316
 317                 # Write metadata
 318                 f.write("#\n# Location Database Export\n#\n")
 319
 320                 f.write("# Generated: %s\n" % time.strftime(
 321                         "%a, %d %b %Y %H:%M:%S GMT", time.gmtime(db.created_at),
 322                 ))
 323
 324                 if db.vendor:
 325                         f.write("# Vendor:    %s\n" % db.vendor)
 326
 327                 if db.license:
 328                         f.write("# License:   %s\n" % db.license)
 329
 330                 f.write("#\n")
 331
 332                 if db.description:
 333                         for line in db.description.splitlines():
 334                                 line = "# %s" % line
 335                                 f.write("%s\n" % line.rstrip())
 336
 337                         f.write("#\n")
 338
 339                 # Iterate over all ASes
 340                 for a in db.ases:
 341                         f.write("\n")
 342                         f.write(format % ("aut-num:", "AS%s" % a.number))
 343                         f.write(format % ("name:", a.name))
 344
 345                 flags = {
 346                         location.NETWORK_FLAG_ANONYMOUS_PROXY    : "is-anonymous-proxy:",
 347                         location.NETWORK_FLAG_SATELLITE_PROVIDER : "is-satellite-provider:",
 348                         location.NETWORK_FLAG_ANYCAST            : "is-anycast:",
 349                 }
 350
 351                 # Iterate over all networks
 352                 for n in db.networks:
 353                         f.write("\n")
 354                         f.write(format % ("net:", n))
 355
 356                         if n.country_code:
 357                                 f.write(format % ("country:", n.country_code))
 358
 359                         if n.asn:
 360                                 f.write(format % ("aut-num:", n.asn))
 361
 362                         # Print all flags
 363                         for flag in flags:
 364                                 if n.has_flag(flag):
 365                                         f.write(format % (flags[flag], "yes"))
 366
 367         def handle_get_as(self, db, ns):
 368                 """
 369                         Gets information about Autonomous Systems
 370                 """
 371                 ret = 0
 372
 373                 for asn in ns.asn:
 374                         try:
 375                                 asn = int(asn)
 376                         except ValueError:
 377                                 print(_("Invalid ASN: %s") % asn, file=sys.stderr)
 378                                 ret = 1
 379                                 continue
 380
 381                         # Fetch AS from database
 382                         a = db.get_as(asn)
 383
 384                         # Nothing found
 385                         if not a:
 386                                 print(_("Could not find AS%s") % asn, file=sys.stderr)
 387                                 ret = 1
 388                                 continue
 389
 390                         print(_("AS%(asn)s belongs to %(name)s") % { "asn" : a.number, "name" : a.name })
 391
 392                 return ret
 393
 394         def handle_search_as(self, db, ns):
 395                 for query in ns.query:
 396                         # Print all matches ASes
 397                         for a in db.search_as(query):
 398                                 print(a)
 399
 400         def handle_update(self, db, ns):
 401                 if ns.cron and db:
 402                         now = time.time()
 403
 404                         if ns.cron == "daily":
 405                                 delta = datetime.timedelta(days=1)
 406                         elif ns.cron == "weekly":
 407                                 delta = datetime.timedelta(days=7)
 408                         elif ns.cron == "monthly":
 409                                 delta = datetime.timedelta(days=30)
 410
 411                         delta = delta.total_seconds()
 412
 413                         # Check if the database has recently been updated
 414                         if db.created_at >= (now - delta):
 415                                 log.info(
 416                                         _("The database has been updated recently"),
 417                                 )
 418                                 return 3
 419
 420                 # Fetch the timestamp we need from DNS
 421                 t = location.discover_latest_version()
 422
 423                 # Check the version of the local database
 424                 if db and t and db.created_at >= t:
 425                         log.info("Already on the latest version")
 426                         return
 427
 428                 # Download the database into the correct directory
 429                 tmpdir = os.path.dirname(ns.database)
 430
 431                 # Create a downloader
 432                 d = location.downloader.Downloader()
 433
 434                 # Try downloading a new database
 435                 try:
 436                         t = d.download(public_key=ns.public_key, timestamp=timestamp, tmpdir=tmpdir)
 437
 438                 # If no file could be downloaded, log a message
 439                 except FileNotFoundError as e:
 440                         log.error("Could not download a new database")
 441                         return 1
 442
 443                 # If we have not received a new file, there is nothing to do
 444                 if not t:
 445                         return 3
 446
 447                 # Move temporary file to destination
 448                 shutil.move(t.name, ns.database)
 449
 450                 return 0
 451
 452         def handle_verify(self, db, ns):
 453                 # Verify the database
 454                 with open(ns.public_key, "r") as f:
 455                         if not db.verify(f):
 456                                 log.error("Could not verify database")
 457                                 return 1
 458
 459                 # Success
 460                 log.debug("Database successfully verified")
 461                 return 0
 462
 463         def __get_output_formatter(self, ns):
 464                 try:
 465                         cls = location.export.formats[ns.format]
 466                 except KeyError:
 467                         cls = location.export.OutputFormatter
 468
 469                 return cls
 470
 471         def handle_list_countries(self, db, ns):
 472                 for country in db.countries:
 473                         line = [
 474                                 country.code,
 475                         ]
 476
 477                         if ns.show_continent:
 478                                 line.append(country.continent_code)
 479
 480                         if ns.show_name:
 481                                 line.append(country.name)
 482
 483                         # Format the output
 484                         line = " ".join(line)
 485
 486                         # Print the output
 487                         print(line)
 488
 489         def handle_list_networks_by_as(self, db, ns):
 490                 writer = self.__get_output_formatter(ns)
 491
 492                 for asn in ns.asn:
 493                         f = writer(sys.stdout, prefix="AS%s" % asn)
 494
 495                         # Print all matching networks
 496                         for n in db.search_networks(asn=asn, family=ns.family):
 497                                 f.write(n)
 498
 499                         f.finish()
 500
 501         def handle_list_networks_by_cc(self, db, ns):
 502                 writer = self.__get_output_formatter(ns)
 503
 504                 for country_code in ns.country_code:
 505                         # Open standard output
 506                         f = writer(sys.stdout, prefix=country_code)
 507
 508                         # Print all matching networks
 509                         for n in db.search_networks(country_code=country_code, family=ns.family):
 510                                 f.write(n)
 511
 512                         f.finish()
 513
 514         def handle_list_networks_by_flags(self, db, ns):
 515                 flags = 0
 516
 517                 if ns.anonymous_proxy:
 518                         flags |= location.NETWORK_FLAG_ANONYMOUS_PROXY
 519
 520                 if ns.satellite_provider:
 521                         flags |= location.NETWORK_FLAG_SATELLITE_PROVIDER
 522
 523                 if ns.anycast:
 524                         flags |= location.NETWORK_FLAG_ANYCAST
 525
 526                 if not flags:
 527                         raise ValueError(_("You must at least pass one flag"))
 528
 529                 writer = self.__get_output_formatter(ns)
 530                 f = writer(sys.stdout, prefix="custom")
 531
 532                 for n in db.search_networks(flags=flags, family=ns.family):
 533                         f.write(n)
 534
 535                 f.finish()
 536
 537         def handle_export(self, db, ns):
 538                 countries, asns = [], []
 539
 540                 # Translate family
 541                 if ns.family:
 542                         families = [ ns.family ]
 543                 else:
 544                         families = [ socket.AF_INET6, socket.AF_INET ]
 545
 546                 for object in ns.objects:
 547                         m = re.match("^AS(\d+)$", object)
 548                         if m:
 549                                 object = int(m.group(1))
 550
 551                                 asns.append(object)
 552
 553                         elif location.country_code_is_valid(object) \
 554                                         or object in ("A1", "A2", "A3"):
 555                                 countries.append(object)
 556
 557                         else:
 558                                 log.warning("Invalid argument: %s" % object)
 559                                 continue
 560
 561                 # Default to exporting all countries
 562                 if not countries and not asns:
 563                         countries = ["A1", "A2", "A3"] + [country.code for country in db.countries]
 564
 565                 # Select the output format
 566                 writer = self.__get_output_formatter(ns)
 567
 568                 e = location.export.Exporter(db, writer)
 569                 e.export(ns.directory, countries=countries, asns=asns, families=families)
 570
 571
 572 def format_timedelta(t):
 573         s = []
 574
 575         if t.days:
 576                 s.append(
 577                         _("One Day", "%(days)s Days", t.days) % { "days" : t.days, }
 578                 )
 579
 580         hours = t.seconds // 3600
 581         if hours:
 582                 s.append(
 583                         _("One Hour", "%(hours)s Hours", hours) % { "hours" : hours, }
 584                 )
 585
 586         minutes = (t.seconds % 3600) // 60
 587         if minutes:
 588                 s.append(
 589                         _("One Minute", "%(minutes)s Minutes", minutes) % { "minutes" : minutes, }
 590                 )
 591
 592         seconds = t.seconds % 60
 593         if t.seconds:
 594                 s.append(
 595                         _("One Second", "%(seconds)s Seconds", seconds) % { "seconds" : seconds, }
 596                 )
 597
 598         if not s:
 599                 return _("Now")
 600
 601         return _("%s ago") % ", ".join(s)
 602
 603 def main():
 604         # Run the command line interface
 605         c = CLI()
 606         c.run()
 607
 608 main()