# Global configuration flags
parser.add_argument("--debug", action="store_true",
help=_("Enable debug output"))
+ parser.add_argument("--quiet", action="store_true",
+ help=_("Enable quiet mode"))
# version
parser.add_argument("--version", action="version",
parser.add_argument("--database-password", required=True,
help=_("Database Password"), metavar=_("PASSWORD"))
+ # Write Database
+ write = subparsers.add_parser("write", help=_("Write database to file"))
+ write.set_defaults(func=self.handle_write)
+ write.add_argument("file", nargs=1, help=_("Database File"))
+ write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
+ write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
+ write.add_argument("--description", nargs="?", help=_("Sets a description"))
+ write.add_argument("--license", nargs="?", help=_("Sets the license"))
+ write.add_argument("--version", type=int, help=_("Database Format Version"))
+
# Update WHOIS
update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
update_whois.set_defaults(func=self.handle_update_whois)
args = parser.parse_args()
- # Enable debug logging
+ # Configure logging
if args.debug:
- log.setLevel(logging.DEBUG)
+ location.logger.set_level(logging.DEBUG)
+ elif args.quiet:
+ location.logger.set_level(logging.WARNING)
# Print usage if no action was given
if not "func" in args:
CREATE INDEX IF NOT EXISTS announcements_family ON announcements(family(network));
-- autnums
- CREATE TABLE IF NOT EXISTS autnums(number bigint, name text);
+ CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
-- networks
CREATE TABLE IF NOT EXISTS autnum_overrides(
number bigint NOT NULL,
name text,
+ country text,
is_anonymous_proxy boolean DEFAULT FALSE,
is_satellite_provider boolean DEFAULT FALSE,
is_anycast boolean DEFAULT FALSE
return db
+ def handle_write(self, ns):
+ """
+ Compiles a database in libloc format out of what is in the database
+ """
+ # Allocate a writer
+ writer = location.Writer(ns.signing_key)
+
+ # Set all metadata
+ if ns.vendor:
+ writer.vendor = ns.vendor
+
+ if ns.description:
+ writer.description = ns.description
+
+ if ns.license:
+ writer.license = ns.license
+
+ # Add all Autonomous Systems
+ log.info("Writing Autonomous Systems...")
+
+ # Select all ASes with a name
+ rows = self.db.query("""
+ SELECT
+ autnums.number AS number,
+ COALESCE(
+ (SELECT overrides.name FROM autnum_overrides overrides
+ WHERE overrides.number = autnums.number),
+ autnums.name
+ ) AS name
+ FROM autnums
+ WHERE name <> %s ORDER BY number
+ """, "")
+
+ for row in rows:
+ a = writer.add_as(row.number)
+ a.name = row.name
+
+ # Add all networks
+ log.info("Writing networks...")
+
+ # Select all known networks
+ rows = self.db.query("""
+ SELECT
+ DISTINCT ON (announcements.network)
+ announcements.network AS network,
+ announcements.autnum AS autnum,
+
+ -- Country
+ COALESCE(
+ (
+ SELECT country FROM network_overrides overrides
+ WHERE announcements.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT country FROM autnum_overrides overrides
+ WHERE announcements.autnum = overrides.number
+ ),
+ networks.country
+ ) AS country,
+
+ -- Must be part of returned values for ORDER BY clause
+ masklen(networks.network) AS sort,
+
+ -- Flags
+ COALESCE(
+ (
+ SELECT is_anonymous_proxy FROM network_overrides overrides
+ WHERE announcements.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT is_anonymous_proxy FROM autnum_overrides overrides
+ WHERE announcements.autnum = overrides.number
+ )
+ ) AS is_anonymous_proxy,
+ COALESCE(
+ (
+ SELECT is_satellite_provider FROM network_overrides overrides
+ WHERE announcements.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT is_satellite_provider FROM autnum_overrides overrides
+ WHERE announcements.autnum = overrides.number
+ )
+ ) AS is_satellite_provider,
+ COALESCE(
+ (
+ SELECT is_anycast FROM network_overrides overrides
+ WHERE announcements.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT is_anycast FROM autnum_overrides overrides
+ WHERE announcements.autnum = overrides.number
+ )
+ ) AS is_anycast
+ FROM announcements
+ LEFT JOIN networks ON announcements.network <<= networks.network
+ ORDER BY announcements.network, sort DESC
+ """)
+
+ for row in rows:
+ network = writer.add_network(row.network)
+
+ # Save AS & country
+ network.asn, network.country_code = row.autnum, row.country
+
+ # Set flags
+ if row.is_anonymous_proxy:
+ network.set_flag(location.NETWORK_FLAG_ANONYMOUS_PROXY)
+
+ if row.is_satellite_provider:
+ network.set_flag(location.NETWORK_FLAG_SATELLITE_PROVIDER)
+
+ if row.is_anycast:
+ network.set_flag(location.NETWORK_FLAG_ANYCAST)
+
+ # Write everything to file
+ log.info("Writing database to file...")
+ for file in ns.file:
+ writer.write(file)
+
def handle_update_whois(self, ns):
downloader = location.importer.Downloader()
prefix = int(prefix)
except:
log.warning("Invalid prefix: %s" % prefix)
+ return
# Fix prefix length for IPv4
if type == "ipv4":
server = ns.server[0]
# Pre-compile regular expression for routes
- #route = re.compile(b"^\*>?\s[\si]?([^\s]+)[.\s]*?(\d+)\si$", re.MULTILINE)
route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
with telnetlib.Telnet(server) as t:
# t.set_debuglevel(10)
# Wait for console greeting
- greeting = t.read_until(b"> ")
- log.debug(greeting.decode())
+ greeting = t.read_until(b"> ", timeout=30)
+ if not greeting:
+ log.error("Could not get a console prompt")
+ return 1
# Disable pagination
t.write(b"terminal length 0\n")
# Convert network to string
network = network.decode()
+ # Append /24 for IPv4 addresses
+ if not "/" in network and not ":" in network:
+ network = "%s/24" % network
+
# Convert AS number to integer
autnum = int(autnum)
-- Delete anything that is not global unicast address space
DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
- -- DELETE RFC1918 address space
+ -- DELETE "current network" address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
+
+ -- DELETE local loopback address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
+
+ -- DELETE RFC 1918 address space
DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
+ -- DELETE test, benchmark and documentation address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
+
+ -- DELETE CGNAT address space (RFC 6598)
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
+
+ -- DELETE link local address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
+
+ -- DELETE IPv6 to IPv4 (6to4) address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
+
+ -- DELETE multicast and reserved address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
+
-- Delete networks that are too small to be in the global routing table
DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
log.warning("Invalid IP network: %s: %s" % (network, e))
continue
+ # Prevent that we overwrite all networks
+ if network.prefixlen == 0:
+ log.warning("Skipping %s: You cannot overwrite default" % network)
+ continue
+
self.db.execute("""
INSERT INTO network_overrides(
network,
is_anonymous_proxy,
is_satellite_provider,
is_anycast
- ) VALUES (%s, %s, %s, %s)
+ ) VALUES (%s, %s, %s, %s, %s)
ON CONFLICT (network) DO NOTHING""",
"%s" % network,
block.get("country"),
block.get("is-anycast") == "yes",
)
- elif type == "autnum":
- autnum = block.get("autnum")
+ elif type == "aut-num":
+ autnum = block.get("aut-num")
# Check if AS number begins with "AS"
if not autnum.startswith("AS"):
INSERT INTO autnum_overrides(
number,
name,
+ country,
is_anonymous_proxy,
is_satellite_provider,
is_anycast
- ) VALUES(%s, %s, %s, %s, %s)
+ ) VALUES(%s, %s, %s, %s, %s, %s)
ON CONFLICT DO NOTHING""",
- autnum, block.get("name"),
+ autnum,
+ block.get("name"),
+ block.get("country"),
block.get("is-anonymous-proxy") == "yes",
block.get("is-satellite-provider") == "yes",
block.get("is-anycast") == "yes",
projects / people / ms / libloc.git / blobdiff