X-Git-Url: http://git.ipfire.org/?a=blobdiff_plain;f=src%2Fpython%2Flocation-importer.in;h=eb4a303b80aa0d375f3184e893f2cf8981a154ed;hb=b904896a954206e3862c9b9a67f4379113198361;hp=a4e24da1841601171d64d3f2ee3a14e14202b503;hpb=d7fc30572200478bb449bff15762056f343a10a9;p=people%2Fms%2Flibloc.git diff --git a/src/python/location-importer.in b/src/python/location-importer.in index a4e24da..eb4a303 100644 --- a/src/python/location-importer.in +++ b/src/python/location-importer.in @@ -45,6 +45,8 @@ class CLI(object): # Global configuration flags parser.add_argument("--debug", action="store_true", help=_("Enable debug output")) + parser.add_argument("--quiet", action="store_true", + help=_("Enable quiet mode")) # version parser.add_argument("--version", action="version", @@ -60,6 +62,16 @@ class CLI(object): parser.add_argument("--database-password", required=True, help=_("Database Password"), metavar=_("PASSWORD")) + # Write Database + write = subparsers.add_parser("write", help=_("Write database to file")) + write.set_defaults(func=self.handle_write) + write.add_argument("file", nargs=1, help=_("Database File")) + write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key")) + write.add_argument("--vendor", nargs="?", help=_("Sets the vendor")) + write.add_argument("--description", nargs="?", help=_("Sets a description")) + write.add_argument("--license", nargs="?", help=_("Sets the license")) + write.add_argument("--version", type=int, help=_("Database Format Version")) + # Update WHOIS update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information")) update_whois.set_defaults(func=self.handle_update_whois) @@ -82,9 +94,11 @@ class CLI(object): args = parser.parse_args() - # Enable debug logging + # Configure logging if args.debug: - log.setLevel(logging.DEBUG) + location.logger.set_level(logging.DEBUG) + elif args.quiet: + location.logger.set_level(logging.WARNING) # Print usage if no action was given if not "func" in args: @@ -130,7 +144,7 @@ class CLI(object): CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network)); -- autnums - CREATE TABLE IF NOT EXISTS autnums(number bigint, name text); + CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL); CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number); -- networks @@ -142,6 +156,7 @@ class CLI(object): CREATE TABLE IF NOT EXISTS autnum_overrides( number bigint NOT NULL, name text, + country text, is_anonymous_proxy boolean DEFAULT FALSE, is_satellite_provider boolean DEFAULT FALSE, is_anycast boolean DEFAULT FALSE @@ -162,6 +177,134 @@ class CLI(object): return db + def handle_write(self, ns): + """ + Compiles a database in libloc format out of what is in the database + """ + # Allocate a writer + writer = location.Writer(ns.signing_key) + + # Set all metadata + if ns.vendor: + writer.vendor = ns.vendor + + if ns.description: + writer.description = ns.description + + if ns.license: + writer.license = ns.license + + # Add all Autonomous Systems + log.info("Writing Autonomous Systems...") + + # Select all ASes with a name + rows = self.db.query(""" + SELECT + autnums.number AS number, + COALESCE( + (SELECT overrides.name FROM autnum_overrides overrides + WHERE overrides.number = autnums.number), + autnums.name + ) AS name + FROM autnums + WHERE name <> %s ORDER BY number + """, "") + + for row in rows: + a = writer.add_as(row.number) + a.name = row.name + + # Add all networks + log.info("Writing networks...") + + # Select all known networks + rows = self.db.query(""" + SELECT + DISTINCT ON (announcements.network) + announcements.network AS network, + announcements.autnum AS autnum, + + -- Country + COALESCE( + ( + SELECT country FROM network_overrides overrides + WHERE announcements.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT country FROM autnum_overrides overrides + WHERE announcements.autnum = overrides.number + ), + networks.country + ) AS country, + + -- Must be part of returned values for ORDER BY clause + masklen(networks.network) AS sort, + + -- Flags + COALESCE( + ( + SELECT is_anonymous_proxy FROM network_overrides overrides + WHERE announcements.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_anonymous_proxy FROM autnum_overrides overrides + WHERE announcements.autnum = overrides.number + ) + ) AS is_anonymous_proxy, + COALESCE( + ( + SELECT is_satellite_provider FROM network_overrides overrides + WHERE announcements.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_satellite_provider FROM autnum_overrides overrides + WHERE announcements.autnum = overrides.number + ) + ) AS is_satellite_provider, + COALESCE( + ( + SELECT is_anycast FROM network_overrides overrides + WHERE announcements.network <<= overrides.network + ORDER BY masklen(overrides.network) DESC + LIMIT 1 + ), + ( + SELECT is_anycast FROM autnum_overrides overrides + WHERE announcements.autnum = overrides.number + ) + ) AS is_anycast + FROM announcements + LEFT JOIN networks ON announcements.network <<= networks.network + ORDER BY announcements.network, sort DESC + """) + + for row in rows: + network = writer.add_network(row.network) + + # Save AS & country + network.asn, network.country_code = row.autnum, row.country + + # Set flags + if row.is_anonymous_proxy: + network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY) + + if row.is_satellite_provider: + network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER) + + if row.is_anycast: + network.set_flag(location.NETWORK_FLAG_ANYCAST) + + # Write everything to file + log.info("Writing database to file...") + for file in ns.file: + writer.write(file) + def handle_update_whois(self, ns): downloader = location.importer.Downloader() @@ -298,6 +441,7 @@ class CLI(object): prefix = int(prefix) except: log.warning("Invalid prefix: %s" % prefix) + return # Fix prefix length for IPv4 if type == "ipv4": @@ -320,7 +464,6 @@ class CLI(object): server = ns.server[0] # Pre-compile regular expression for routes - #route = re.compile(b"^\*>?\s[\si]?([^\s]+)[.\s]*?(\d+)\si$", re.MULTILINE) route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL) with telnetlib.Telnet(server) as t: @@ -329,8 +472,10 @@ class CLI(object): # t.set_debuglevel(10) # Wait for console greeting - greeting = t.read_until(b"> ") - log.debug(greeting.decode()) + greeting = t.read_until(b"> ", timeout=30) + if not greeting: + log.error("Could not get a console prompt") + return 1 # Disable pagination t.write(b"terminal length 0\n") @@ -367,6 +512,10 @@ class CLI(object): # Convert network to string network = network.decode() + # Append /24 for IPv4 addresses + if not "/" in network and not ":" in network: + network = "%s/24" % network + # Convert AS number to integer autnum = int(autnum) @@ -388,11 +537,37 @@ class CLI(object): -- Delete anything that is not global unicast address space DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3'; - -- DELETE RFC1918 address space + -- DELETE "current network" address space + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8'; + + -- DELETE local loopback address space + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8'; + + -- DELETE RFC 1918 address space DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8'; DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12'; DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16'; + -- DELETE test, benchmark and documentation address space + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24'; + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24'; + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15'; + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24'; + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24'; + + -- DELETE CGNAT address space (RFC 6598) + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10'; + + -- DELETE link local address space + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16'; + + -- DELETE IPv6 to IPv4 (6to4) address space + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24'; + + -- DELETE multicast and reserved address space + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4'; + DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4'; + -- Delete networks that are too small to be in the global routing table DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48; DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24; @@ -432,6 +607,11 @@ class CLI(object): log.warning("Invalid IP network: %s: %s" % (network, e)) continue + # Prevent that we overwrite all networks + if network.prefixlen == 0: + log.warning("Skipping %s: You cannot overwrite default" % network) + continue + self.db.execute(""" INSERT INTO network_overrides( network, @@ -439,7 +619,7 @@ class CLI(object): is_anonymous_proxy, is_satellite_provider, is_anycast - ) VALUES (%s, %s, %s, %s) + ) VALUES (%s, %s, %s, %s, %s) ON CONFLICT (network) DO NOTHING""", "%s" % network, block.get("country"), @@ -448,8 +628,8 @@ class CLI(object): block.get("is-anycast") == "yes", ) - elif type == "autnum": - autnum = block.get("autnum") + elif type == "aut-num": + autnum = block.get("aut-num") # Check if AS number begins with "AS" if not autnum.startswith("AS"): @@ -463,12 +643,15 @@ class CLI(object): INSERT INTO autnum_overrides( number, name, + country, is_anonymous_proxy, is_satellite_provider, is_anycast - ) VALUES(%s, %s, %s, %s, %s) + ) VALUES(%s, %s, %s, %s, %s, %s) ON CONFLICT DO NOTHING""", - autnum, block.get("name"), + autnum, + block.get("name"), + block.get("country"), block.get("is-anonymous-proxy") == "yes", block.get("is-satellite-provider") == "yes", block.get("is-anycast") == "yes",