src/python/location-importer.in

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # libloc - A library to determine the location of someone on the Internet     #
   5 #                                                                             #
   6 # Copyright (C) 2020 IPFire Development Team <info@ipfire.org>                #
   7 #                                                                             #
   8 # This library is free software; you can redistribute it and/or               #
   9 # modify it under the terms of the GNU Lesser General Public                  #
  10 # License as published by the Free Software Foundation; either                #
  11 # version 2.1 of the License, or (at your option) any later version.          #
  12 #                                                                             #
  13 # This library is distributed in the hope that it will be useful,             #
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU           #
  16 # Lesser General Public License for more details.                             #
  17 #                                                                             #
  18 ###############################################################################
  19
  20 import argparse
  21 import ipaddress
  22 import logging
  23 import math
  24 import re
  25 import socket
  26 import sys
  27 import telnetlib
  28
  29 # Load our location module
  30 import location
  31 import location.database
  32 import location.importer
  33 from location.i18n import _
  34
  35 # Initialise logging
  36 log = logging.getLogger("location.importer")
  37 log.propagate = 1
  38
  39 class CLI(object):
  40         def parse_cli(self):
  41                 parser = argparse.ArgumentParser(
  42                         description=_("Location Importer Command Line Interface"),
  43                 )
  44                 subparsers = parser.add_subparsers()
  45
  46                 # Global configuration flags
  47                 parser.add_argument("--debug", action="store_true",
  48                         help=_("Enable debug output"))
  49                 parser.add_argument("--quiet", action="store_true",
  50                         help=_("Enable quiet mode"))
  51
  52                 # version
  53                 parser.add_argument("--version", action="version",
  54                         version="%(prog)s @VERSION@")
  55
  56                 # Database
  57                 parser.add_argument("--database-host", required=True,
  58                         help=_("Database Hostname"), metavar=_("HOST"))
  59                 parser.add_argument("--database-name", required=True,
  60                         help=_("Database Name"), metavar=_("NAME"))
  61                 parser.add_argument("--database-username", required=True,
  62                         help=_("Database Username"), metavar=_("USERNAME"))
  63                 parser.add_argument("--database-password", required=True,
  64                         help=_("Database Password"), metavar=_("PASSWORD"))
  65
  66                 # Write Database
  67                 write = subparsers.add_parser("write", help=_("Write database to file"))
  68                 write.set_defaults(func=self.handle_write)
  69                 write.add_argument("file", nargs=1, help=_("Database File"))
  70                 write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
  71                 write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key"))
  72                 write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
  73                 write.add_argument("--description", nargs="?", help=_("Sets a description"))
  74                 write.add_argument("--license", nargs="?", help=_("Sets the license"))
  75                 write.add_argument("--version", type=int, help=_("Database Format Version"))
  76
  77                 # Update WHOIS
  78                 update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
  79                 update_whois.set_defaults(func=self.handle_update_whois)
  80
  81                 # Update announcements
  82                 update_announcements = subparsers.add_parser("update-announcements",
  83                         help=_("Update BGP Annoucements"))
  84                 update_announcements.set_defaults(func=self.handle_update_announcements)
  85                 update_announcements.add_argument("server", nargs=1,
  86                         help=_("Route Server to connect to"), metavar=_("SERVER"))
  87
  88                 # Update overrides
  89                 update_overrides = subparsers.add_parser("update-overrides",
  90                         help=_("Update overrides"),
  91                 )
  92                 update_overrides.add_argument(
  93                         "files", nargs="+", help=_("Files to import"),
  94                 )
  95                 update_overrides.set_defaults(func=self.handle_update_overrides)
  96
  97                 # Import countries
  98                 import_countries = subparsers.add_parser("import-countries",
  99                         help=_("Import countries"),
 100                 )
 101                 import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"),
 102                         help=_("File to import"))
 103                 import_countries.set_defaults(func=self.handle_import_countries)
 104
 105                 args = parser.parse_args()
 106
 107                 # Configure logging
 108                 if args.debug:
 109                         location.logger.set_level(logging.DEBUG)
 110                 elif args.quiet:
 111                         location.logger.set_level(logging.WARNING)
 112
 113                 # Print usage if no action was given
 114                 if not "func" in args:
 115                         parser.print_usage()
 116                         sys.exit(2)
 117
 118                 return args
 119
 120         def run(self):
 121                 # Parse command line arguments
 122                 args = self.parse_cli()
 123
 124                 # Initialise database
 125                 self.db = self._setup_database(args)
 126
 127                 # Call function
 128                 ret = args.func(args)
 129
 130                 # Return with exit code
 131                 if ret:
 132                         sys.exit(ret)
 133
 134                 # Otherwise just exit
 135                 sys.exit(0)
 136
 137         def _setup_database(self, ns):
 138                 """
 139                         Initialise the database
 140                 """
 141                 # Connect to database
 142                 db = location.database.Connection(
 143                         host=ns.database_host, database=ns.database_name,
 144                         user=ns.database_username, password=ns.database_password,
 145                 )
 146
 147                 with db.transaction():
 148                         db.execute("""
 149                                 -- announcements
 150                                 CREATE TABLE IF NOT EXISTS announcements(network inet, autnum bigint,
 151                                         first_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP,
 152                                         last_seen_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP);
 153                                 CREATE UNIQUE INDEX IF NOT EXISTS announcements_networks ON announcements(network);
 154                                 CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
 155
 156                                 -- autnums
 157                                 CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
 158                                 CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
 159
 160                                 -- countries
 161                                 CREATE TABLE IF NOT EXISTS countries(
 162                                         country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
 163                                 CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
 164
 165                                 -- networks
 166                                 CREATE TABLE IF NOT EXISTS networks(network inet, country text);
 167                                 CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
 168                                 CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
 169
 170                                 -- overrides
 171                                 CREATE TABLE IF NOT EXISTS autnum_overrides(
 172                                         number bigint NOT NULL,
 173                                         name text,
 174                                         country text,
 175                                         is_anonymous_proxy boolean,
 176                                         is_satellite_provider boolean,
 177                                         is_anycast boolean
 178                                 );
 179                                 CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
 180                                         ON autnum_overrides(number);
 181
 182                                 CREATE TABLE IF NOT EXISTS network_overrides(
 183                                         network inet NOT NULL,
 184                                         country text,
 185                                         is_anonymous_proxy boolean,
 186                                         is_satellite_provider boolean,
 187                                         is_anycast boolean
 188                                 );
 189                                 CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
 190                                         ON network_overrides(network);
 191                         """)
 192
 193                 return db
 194
 195         def handle_write(self, ns):
 196                 """
 197                         Compiles a database in libloc format out of what is in the database
 198                 """
 199                 # Allocate a writer
 200                 writer = location.Writer(ns.signing_key, ns.backup_signing_key)
 201
 202                 # Set all metadata
 203                 if ns.vendor:
 204                         writer.vendor = ns.vendor
 205
 206                 if ns.description:
 207                         writer.description = ns.description
 208
 209                 if ns.license:
 210                         writer.license = ns.license
 211
 212                 # Add all Autonomous Systems
 213                 log.info("Writing Autonomous Systems...")
 214
 215                 # Select all ASes with a name
 216                 rows = self.db.query("""
 217                         SELECT
 218                                 autnums.number AS number,
 219                                 COALESCE(
 220                                         (SELECT overrides.name FROM autnum_overrides overrides
 221                                                 WHERE overrides.number = autnums.number),
 222                                         autnums.name
 223                                 ) AS name
 224                                 FROM autnums
 225                                 WHERE name <> %s ORDER BY number
 226                         """, "")
 227
 228                 for row in rows:
 229                         a = writer.add_as(row.number)
 230                         a.name = row.name
 231
 232                 # Add all networks
 233                 log.info("Writing networks...")
 234
 235                 # Select all known networks
 236                 rows = self.db.query("""
 237                         -- Get a (sorted) list of all known networks
 238                         WITH known_networks AS (
 239                                         SELECT network FROM announcements
 240                                 UNION
 241                                         SELECT network FROM networks
 242                                 ORDER BY network
 243                         )
 244
 245                         -- Return a list of those networks enriched with all
 246                         -- other information that we store in the database
 247                         SELECT
 248                                 DISTINCT ON (known_networks.network)
 249                                 known_networks.network AS network,
 250                                 announcements.autnum AS autnum,
 251
 252                                 -- Country
 253                                 COALESCE(
 254                                         (
 255                                                 SELECT country FROM network_overrides overrides
 256                                                         WHERE announcements.network <<= overrides.network
 257                                                         ORDER BY masklen(overrides.network) DESC
 258                                                         LIMIT 1
 259                                         ),
 260                                         (
 261                                                 SELECT country FROM autnum_overrides overrides
 262                                                         WHERE announcements.autnum = overrides.number
 263                                         ),
 264                                         networks.country
 265                                 ) AS country,
 266
 267                                 -- Flags
 268                                 COALESCE(
 269                                         (
 270                                                 SELECT is_anonymous_proxy FROM network_overrides overrides
 271                                                         WHERE announcements.network <<= overrides.network
 272                                                         ORDER BY masklen(overrides.network) DESC
 273                                                         LIMIT 1
 274                                         ),
 275                                         (
 276                                                 SELECT is_anonymous_proxy FROM autnum_overrides overrides
 277                                                         WHERE announcements.autnum = overrides.number
 278                                         ),
 279                                         FALSE
 280                                 ) AS is_anonymous_proxy,
 281                                 COALESCE(
 282                                         (
 283                                                 SELECT is_satellite_provider FROM network_overrides overrides
 284                                                         WHERE announcements.network <<= overrides.network
 285                                                         ORDER BY masklen(overrides.network) DESC
 286                                                         LIMIT 1
 287                                         ),
 288                                         (
 289                                                 SELECT is_satellite_provider FROM autnum_overrides overrides
 290                                                         WHERE announcements.autnum = overrides.number
 291                                         ),
 292                                         FALSE
 293                                 ) AS is_satellite_provider,
 294                                 COALESCE(
 295                                         (
 296                                                 SELECT is_anycast FROM network_overrides overrides
 297                                                         WHERE announcements.network <<= overrides.network
 298                                                         ORDER BY masklen(overrides.network) DESC
 299                                                         LIMIT 1
 300                                         ),
 301                                         (
 302                                                 SELECT is_anycast FROM autnum_overrides overrides
 303                                                         WHERE announcements.autnum = overrides.number
 304                                         ),
 305                                         FALSE
 306                                 ) AS is_anycast,
 307
 308                                 -- Must be part of returned values for ORDER BY clause
 309                                 masklen(announcements.network) AS sort_a,
 310                                 masklen(networks.network) AS sort_b
 311                         FROM known_networks
 312                                 LEFT JOIN announcements ON known_networks.network <<= announcements.network
 313                                 LEFT JOIN networks ON known_networks.network <<= networks.network
 314                         ORDER BY known_networks.network, sort_a DESC, sort_b DESC
 315                 """)
 316
 317                 for row in rows:
 318                         network = writer.add_network(row.network)
 319
 320                         # Save country
 321                         if row.country:
 322                                 network.country_code = row.country
 323
 324                         # Save ASN
 325                         if row.autnum:
 326                                 network.asn = row.autnum
 327
 328                         # Set flags
 329                         if row.is_anonymous_proxy:
 330                                 network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
 331
 332                         if row.is_satellite_provider:
 333                                 network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
 334
 335                         if row.is_anycast:
 336                                 network.set_flag(location.NETWORK_FLAG_ANYCAST)
 337
 338                 # Add all countries
 339                 log.info("Writing countries...")
 340                 rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
 341
 342                 for row in rows:
 343                         c = writer.add_country(row.country_code)
 344                         c.continent_code = row.continent_code
 345                         c.name = row.name
 346
 347                 # Write everything to file
 348                 log.info("Writing database to file...")
 349                 for file in ns.file:
 350                         writer.write(file)
 351
 352         def handle_update_whois(self, ns):
 353                 downloader = location.importer.Downloader()
 354
 355                 # Download all sources
 356                 with self.db.transaction():
 357                         # Create some temporary tables to store parsed data
 358                         self.db.execute("""
 359                                 CREATE TEMPORARY TABLE _autnums(number integer, organization text)
 360                                         ON COMMIT DROP;
 361                                 CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
 362
 363                                 CREATE TEMPORARY TABLE _organizations(handle text, name text)
 364                                         ON COMMIT DROP;
 365                                 CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
 366                         """)
 367
 368                         for source in location.importer.WHOIS_SOURCES:
 369                                 with downloader.request(source, return_blocks=True) as f:
 370                                         for block in f:
 371                                                 self._parse_block(block)
 372
 373                         self.db.execute("""
 374                                 INSERT INTO autnums(number, name)
 375                                         SELECT _autnums.number, _organizations.name FROM _autnums
 376                                                 LEFT JOIN _organizations ON _autnums.organization = _organizations.handle
 377                                 ON CONFLICT (number) DO UPDATE SET name = excluded.name;
 378                         """)
 379
 380                 # Download all extended sources
 381                 for source in location.importer.EXTENDED_SOURCES:
 382                         with self.db.transaction():
 383                                 # Download data
 384                                 with downloader.request(source) as f:
 385                                         for line in f:
 386                                                 self._parse_line(line)
 387
 388         def _parse_block(self, block):
 389                 # Get first line to find out what type of block this is
 390                 line = block[0]
 391
 392                 # aut-num
 393                 if line.startswith("aut-num:"):
 394                         return self._parse_autnum_block(block)
 395
 396                 # organisation
 397                 elif line.startswith("organisation:"):
 398                         return self._parse_org_block(block)
 399
 400         def _parse_autnum_block(self, block):
 401                 autnum = {}
 402                 for line in block:
 403                         # Split line
 404                         key, val = split_line(line)
 405
 406                         if key == "aut-num":
 407                                 m = re.match(r"^(AS|as)(\d+)", val)
 408                                 if m:
 409                                         autnum["asn"] = m.group(2)
 410
 411                         elif key == "org":
 412                                 autnum[key] = val
 413
 414                 # Skip empty objects
 415                 if not autnum:
 416                         return
 417
 418                 # Insert into database
 419                 self.db.execute("INSERT INTO _autnums(number, organization) \
 420                         VALUES(%s, %s) ON CONFLICT (number) DO UPDATE SET \
 421                                 organization = excluded.organization",
 422                         autnum.get("asn"), autnum.get("org"),
 423                 )
 424
 425         def _parse_org_block(self, block):
 426                 org = {}
 427                 for line in block:
 428                         # Split line
 429                         key, val = split_line(line)
 430
 431                         if key in ("organisation", "org-name"):
 432                                 org[key] = val
 433
 434                 # Skip empty objects
 435                 if not org:
 436                         return
 437
 438                 self.db.execute("INSERT INTO _organizations(handle, name) \
 439                         VALUES(%s, %s) ON CONFLICT (handle) DO \
 440                         UPDATE SET name = excluded.name",
 441                         org.get("organisation"), org.get("org-name"),
 442                 )
 443
 444         def _parse_line(self, line):
 445                 # Skip version line
 446                 if line.startswith("2"):
 447                         return
 448
 449                 # Skip comments
 450                 if line.startswith("#"):
 451                         return
 452
 453                 try:
 454                         registry, country_code, type, line = line.split("|", 3)
 455                 except:
 456                         log.warning("Could not parse line: %s" % line)
 457                         return
 458
 459                 # Skip any lines that are for stats only
 460                 if country_code == "*":
 461                         return
 462
 463                 if type in ("ipv6", "ipv4"):
 464                         return self._parse_ip_line(country_code, type, line)
 465
 466         def _parse_ip_line(self, country, type, line):
 467                 try:
 468                         address, prefix, date, status, organization = line.split("|")
 469                 except ValueError:
 470                         organization = None
 471
 472                         # Try parsing the line without organization
 473                         try:
 474                                 address, prefix, date, status = line.split("|")
 475                         except ValueError:
 476                                 log.warning("Unhandled line format: %s" % line)
 477                                 return
 478
 479                 # Skip anything that isn't properly assigned
 480                 if not status in ("assigned", "allocated"):
 481                         return
 482
 483                 # Cast prefix into an integer
 484                 try:
 485                         prefix = int(prefix)
 486                 except:
 487                         log.warning("Invalid prefix: %s" % prefix)
 488                         return
 489
 490                 # Fix prefix length for IPv4
 491                 if type == "ipv4":
 492                         prefix = 32 - int(math.log(prefix, 2))
 493
 494                 # Try to parse the address
 495                 try:
 496                         network = ipaddress.ip_network("%s/%s" % (address, prefix), strict=False)
 497                 except ValueError:
 498                         log.warning("Invalid IP address: %s" % address)
 499                         return
 500
 501                 self.db.execute("INSERT INTO networks(network, country) \
 502                         VALUES(%s, %s) ON CONFLICT (network) DO \
 503                         UPDATE SET country = excluded.country",
 504                         "%s" % network, country,
 505                 )
 506
 507         def handle_update_announcements(self, ns):
 508                 server = ns.server[0]
 509
 510                 with self.db.transaction():
 511                         if server.startswith("/"):
 512                                 self._handle_update_announcements_from_bird(server)
 513                         else:
 514                                 self._handle_update_announcements_from_telnet(server)
 515
 516                         # Purge anything we never want here
 517                         self.db.execute("""
 518                                 -- Delete default routes
 519                                 DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
 520
 521                                 -- Delete anything that is not global unicast address space
 522                                 DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
 523
 524                                 -- DELETE "current network" address space
 525                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
 526
 527                                 -- DELETE local loopback address space
 528                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
 529
 530                                 -- DELETE RFC 1918 address space
 531                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
 532                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
 533                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
 534
 535                                 -- DELETE test, benchmark and documentation address space
 536                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
 537                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
 538                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
 539                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
 540                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
 541
 542                                 -- DELETE CGNAT address space (RFC 6598)
 543                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
 544
 545                                 -- DELETE link local address space
 546                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
 547
 548                                 -- DELETE IPv6 to IPv4 (6to4) address space
 549                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
 550
 551                                 -- DELETE multicast and reserved address space
 552                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
 553                                 DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
 554
 555                                 -- Delete networks that are too small to be in the global routing table
 556                                 DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
 557                                 DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
 558
 559                                 -- Delete any non-public or reserved ASNs
 560                                 DELETE FROM announcements WHERE NOT (
 561                                         (autnum >= 1 AND autnum <= 23455)
 562                                         OR
 563                                         (autnum >= 23457 AND autnum <= 64495)
 564                                         OR
 565                                         (autnum >= 131072 AND autnum <= 4199999999)
 566                                 );
 567
 568                                 -- Delete everything that we have not seen for 14 days
 569                                 DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
 570                         """)
 571
 572         def _handle_update_announcements_from_bird(self, server):
 573                 # Pre-compile the regular expression for faster searching
 574                 route = re.compile(b"^\s(.+?)\s+.+?\[AS(.*?).\]$")
 575
 576                 log.info("Requesting routing table from Bird (%s)" % server)
 577
 578                 # Send command to list all routes
 579                 for line in self._bird_cmd(server, "show route"):
 580                         m = route.match(line)
 581                         if not m:
 582                                 log.debug("Could not parse line: %s" % line.decode())
 583                                 continue
 584
 585                         # Fetch the extracted network and ASN
 586                         network, autnum = m.groups()
 587
 588                         # Insert it into the database
 589                         self.db.execute("INSERT INTO announcements(network, autnum) \
 590                                 VALUES(%s, %s) ON CONFLICT (network) DO \
 591                                 UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
 592                                 network.decode(), autnum.decode(),
 593                         )
 594
 595         def _handle_update_announcements_from_telnet(self, server):
 596                 # Pre-compile regular expression for routes
 597                 route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
 598
 599                 with telnetlib.Telnet(server) as t:
 600                         # Enable debug mode
 601                         #if ns.debug:
 602                         #       t.set_debuglevel(10)
 603
 604                         # Wait for console greeting
 605                         greeting = t.read_until(b"> ", timeout=30)
 606                         if not greeting:
 607                                 log.error("Could not get a console prompt")
 608                                 return 1
 609
 610                         # Disable pagination
 611                         t.write(b"terminal length 0\n")
 612
 613                         # Wait for the prompt to return
 614                         t.read_until(b"> ")
 615
 616                         # Fetch the routing tables
 617                         for protocol in ("ipv6", "ipv4"):
 618                                 log.info("Requesting %s routing table" % protocol)
 619
 620                                 # Request the full unicast routing table
 621                                 t.write(b"show bgp %s unicast\n" % protocol.encode())
 622
 623                                 # Read entire header which ends with "Path"
 624                                 t.read_until(b"Path\r\n")
 625
 626                                 while True:
 627                                         # Try reading a full entry
 628                                         # Those might be broken across multiple lines but ends with i
 629                                         line = t.read_until(b"i\r\n", timeout=5)
 630                                         if not line:
 631                                                 break
 632
 633                                         # Show line for debugging
 634                                         #log.debug(repr(line))
 635
 636                                         # Try finding a route in here
 637                                         m = route.match(line)
 638                                         if m:
 639                                                 network, autnum = m.groups()
 640
 641                                                 # Convert network to string
 642                                                 network = network.decode()
 643
 644                                                 # Append /24 for IPv4 addresses
 645                                                 if not "/" in network and not ":" in network:
 646                                                         network = "%s/24" % network
 647
 648                                                 # Convert AS number to integer
 649                                                 autnum = int(autnum)
 650
 651                                                 log.info("Found announcement for %s by %s" % (network, autnum))
 652
 653                                                 self.db.execute("INSERT INTO announcements(network, autnum) \
 654                                                         VALUES(%s, %s) ON CONFLICT (network) DO \
 655                                                         UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
 656                                                         network, autnum,
 657                                                 )
 658
 659                                 log.info("Finished reading the %s routing table" % protocol)
 660
 661         def _bird_cmd(self, socket_path, command):
 662                 # Connect to the socket
 663                 s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
 664                 s.connect(socket_path)
 665
 666                 # Allocate some buffer
 667                 buffer = b""
 668
 669                 # Send the command
 670                 s.send(b"%s\n" % command.encode())
 671
 672                 while True:
 673                         # Fill up the buffer
 674                         buffer += s.recv(4096)
 675
 676                         while True:
 677                                 # Search for the next newline
 678                                 pos = buffer.find(b"\n")
 679
 680                                 # If we cannot find one, we go back and read more data
 681                                 if pos <= 0:
 682                                         break
 683
 684                                 # Cut after the newline character
 685                                 pos += 1
 686
 687                                 # Split the line we want and keep the rest in buffer
 688                                 line, buffer = buffer[:pos], buffer[pos:]
 689
 690                                 # Look for the end-of-output indicator
 691                                 if line == b"0000 \n":
 692                                         return
 693
 694                                 # Otherwise return the line
 695                                 yield line
 696
 697         def handle_update_overrides(self, ns):
 698                 with self.db.transaction():
 699                         # Drop all data that we have
 700                         self.db.execute("""
 701                                 TRUNCATE TABLE autnum_overrides;
 702                                 TRUNCATE TABLE network_overrides;
 703                         """)
 704
 705                         for file in ns.files:
 706                                 log.info("Reading %s..." % file)
 707
 708                                 with open(file, "rb") as f:
 709                                         for type, block in location.importer.read_blocks(f):
 710                                                 if type == "net":
 711                                                         network = block.get("net")
 712                                                         # Try to parse and normalise the network
 713                                                         try:
 714                                                                 network = ipaddress.ip_network(network, strict=False)
 715                                                         except ValueError as e:
 716                                                                 log.warning("Invalid IP network: %s: %s" % (network, e))
 717                                                                 continue
 718
 719                                                         # Prevent that we overwrite all networks
 720                                                         if network.prefixlen == 0:
 721                                                                 log.warning("Skipping %s: You cannot overwrite default" % network)
 722                                                                 continue
 723
 724                                                         self.db.execute("""
 725                                                                 INSERT INTO network_overrides(
 726                                                                         network,
 727                                                                         country,
 728                                                                         is_anonymous_proxy,
 729                                                                         is_satellite_provider,
 730                                                                         is_anycast
 731                                                                 ) VALUES (%s, %s, %s, %s, %s)
 732                                                                 ON CONFLICT (network) DO NOTHING""",
 733                                                                 "%s" % network,
 734                                                                 block.get("country"),
 735                                                                 self._parse_bool(block, "is-anonymous-proxy"),
 736                                                                 self._parse_bool(block, "is-satellite-provider"),
 737                                                                 self._parse_bool(block, "is-anycast"),
 738                                                         )
 739
 740                                                 elif type == "aut-num":
 741                                                         autnum = block.get("aut-num")
 742
 743                                                         # Check if AS number begins with "AS"
 744                                                         if not autnum.startswith("AS"):
 745                                                                 log.warning("Invalid AS number: %s" % autnum)
 746                                                                 continue
 747
 748                                                         # Strip "AS"
 749                                                         autnum = autnum[2:]
 750
 751                                                         self.db.execute("""
 752                                                                 INSERT INTO autnum_overrides(
 753                                                                         number,
 754                                                                         name,
 755                                                                         country,
 756                                                                         is_anonymous_proxy,
 757                                                                         is_satellite_provider,
 758                                                                         is_anycast
 759                                                                 ) VALUES(%s, %s, %s, %s, %s, %s)
 760                                                                 ON CONFLICT DO NOTHING""",
 761                                                                 autnum,
 762                                                                 block.get("name"),
 763                                                                 block.get("country"),
 764                                                                 self._parse_bool(block, "is-anonymous-proxy"),
 765                                                                 self._parse_bool(block, "is-satellite-provider"),
 766                                                                 self._parse_bool(block, "is-anycast"),
 767                                                         )
 768
 769                                                 else:
 770                                                         log.warning("Unsupport type: %s" % type)
 771
 772         @staticmethod
 773         def _parse_bool(block, key):
 774                 val = block.get(key)
 775
 776                 # There is no point to proceed when we got None
 777                 if val is None:
 778                         return
 779
 780                 # Convert to lowercase
 781                 val = val.lower()
 782
 783                 # True
 784                 if val in ("yes", "1"):
 785                         return True
 786
 787                 # False
 788                 if val in ("no", "0"):
 789                         return False
 790
 791                 # Default to None
 792                 return None
 793
 794         def handle_import_countries(self, ns):
 795                 with self.db.transaction():
 796                         # Drop all data that we have
 797                         self.db.execute("TRUNCATE TABLE countries")
 798
 799                         for file in ns.file:
 800                                 for line in file:
 801                                         line = line.rstrip()
 802
 803                                         # Ignore any comments
 804                                         if line.startswith("#"):
 805                                                 continue
 806
 807                                         try:
 808                                                 country_code, continent_code, name = line.split(maxsplit=2)
 809                                         except:
 810                                                 log.warning("Could not parse line: %s" % line)
 811                                                 continue
 812
 813                                         self.db.execute("INSERT INTO countries(country_code, name, continent_code) \
 814                                                 VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code)
 815
 816
 817 def split_line(line):
 818         key, colon, val = line.partition(":")
 819
 820         # Strip any excess space
 821         key = key.strip()
 822         val = val.strip()
 823
 824         return key, val
 825
 826 def main():
 827         # Run the command line interface
 828         c = CLI()
 829         c.run()
 830
 831 main()