+ def handle_update_announcements(self, ns):
+ server = ns.server[0]
+
+ # Pre-compile regular expression for routes
+ route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
+
+ with telnetlib.Telnet(server) as t:
+ # Enable debug mode
+ #if ns.debug:
+ # t.set_debuglevel(10)
+
+ # Wait for console greeting
+ greeting = t.read_until(b"> ", timeout=30)
+ if not greeting:
+ log.error("Could not get a console prompt")
+ return 1
+
+ # Disable pagination
+ t.write(b"terminal length 0\n")
+
+ # Wait for the prompt to return
+ t.read_until(b"> ")
+
+ # Fetch the routing tables
+ with self.db.transaction():
+ for protocol in ("ipv6", "ipv4"):
+ log.info("Requesting %s routing table" % protocol)
+
+ # Request the full unicast routing table
+ t.write(b"show bgp %s unicast\n" % protocol.encode())
+
+ # Read entire header which ends with "Path"
+ t.read_until(b"Path\r\n")
+
+ while True:
+ # Try reading a full entry
+ # Those might be broken across multiple lines but ends with i
+ line = t.read_until(b"i\r\n", timeout=5)
+ if not line:
+ break
+
+ # Show line for debugging
+ #log.debug(repr(line))
+
+ # Try finding a route in here
+ m = route.match(line)
+ if m:
+ network, autnum = m.groups()
+
+ # Convert network to string
+ network = network.decode()
+
+ # Append /24 for IPv4 addresses
+ if not "/" in network and not ":" in network:
+ network = "%s/24" % network
+
+ # Convert AS number to integer
+ autnum = int(autnum)
+
+ log.info("Found announcement for %s by %s" % (network, autnum))
+
+ self.db.execute("INSERT INTO announcements(network, autnum) \
+ VALUES(%s, %s) ON CONFLICT (network) DO \
+ UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
+ network, autnum,
+ )
+
+ log.info("Finished reading the %s routing table" % protocol)
+
+ # Purge anything we never want here
+ self.db.execute("""
+ -- Delete default routes
+ DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
+
+ -- Delete anything that is not global unicast address space
+ DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
+
+ -- DELETE "current network" address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
+
+ -- DELETE local loopback address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
+
+ -- DELETE RFC 1918 address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
+
+ -- DELETE test, benchmark and documentation address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
+
+ -- DELETE CGNAT address space (RFC 6598)
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
+
+ -- DELETE link local address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
+
+ -- DELETE IPv6 to IPv4 (6to4) address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
+
+ -- DELETE multicast and reserved address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
+
+ -- Delete networks that are too small to be in the global routing table
+ DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
+ DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
+
+ -- Delete any non-public or reserved ASNs
+ DELETE FROM announcements WHERE NOT (
+ (autnum >= 1 AND autnum <= 23455)
+ OR
+ (autnum >= 23457 AND autnum <= 64495)
+ OR
+ (autnum >= 131072 AND autnum <= 4199999999)
+ );
+
+ -- Delete everything that we have not seen for 14 days
+ DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
+ """)
+
+ def handle_update_overrides(self, ns):
+ with self.db.transaction():
+ # Drop all data that we have
+ self.db.execute("""
+ TRUNCATE TABLE autnum_overrides;
+ TRUNCATE TABLE network_overrides;
+ """)
+
+ for file in ns.files:
+ log.info("Reading %s..." % file)
+
+ with open(file, "rb") as f:
+ for type, block in location.importer.read_blocks(f):
+ if type == "net":
+ network = block.get("net")
+ # Try to parse and normalise the network
+ try:
+ network = ipaddress.ip_network(network, strict=False)
+ except ValueError as e:
+ log.warning("Invalid IP network: %s: %s" % (network, e))
+ continue
+
+ # Prevent that we overwrite all networks
+ if network.prefixlen == 0:
+ log.warning("Skipping %s: You cannot overwrite default" % network)
+ continue
+
+ self.db.execute("""
+ INSERT INTO network_overrides(
+ network,
+ country,
+ is_anonymous_proxy,
+ is_satellite_provider,
+ is_anycast
+ ) VALUES (%s, %s, %s, %s, %s)
+ ON CONFLICT (network) DO NOTHING""",
+ "%s" % network,
+ block.get("country"),
+ block.get("is-anonymous-proxy") == "yes",
+ block.get("is-satellite-provider") == "yes",
+ block.get("is-anycast") == "yes",
+ )
+
+ elif type == "aut-num":
+ autnum = block.get("aut-num")
+
+ # Check if AS number begins with "AS"
+ if not autnum.startswith("AS"):
+ log.warning("Invalid AS number: %s" % autnum)
+ continue
+
+ # Strip "AS"
+ autnum = autnum[2:]
+
+ self.db.execute("""
+ INSERT INTO autnum_overrides(
+ number,
+ name,
+ country,
+ is_anonymous_proxy,
+ is_satellite_provider,
+ is_anycast
+ ) VALUES(%s, %s, %s, %s, %s, %s)
+ ON CONFLICT DO NOTHING""",
+ autnum,
+ block.get("name"),
+ block.get("country"),
+ block.get("is-anonymous-proxy") == "yes",
+ block.get("is-satellite-provider") == "yes",
+ block.get("is-anycast") == "yes",
+ )
+
+ else:
+ log.warning("Unsupport type: %s" % type)
+