import logging
import math
import re
+import socket
import sys
import telnetlib
write.set_defaults(func=self.handle_write)
write.add_argument("file", nargs=1, help=_("Database File"))
write.add_argument("--signing-key", nargs="?", type=open, help=_("Signing Key"))
+ write.add_argument("--backup-signing-key", nargs="?", type=open, help=_("Backup Signing Key"))
write.add_argument("--vendor", nargs="?", help=_("Sets the vendor"))
write.add_argument("--description", nargs="?", help=_("Sets a description"))
write.add_argument("--license", nargs="?", help=_("Sets the license"))
+ write.add_argument("--version", type=int, help=_("Database Format Version"))
# Update WHOIS
update_whois = subparsers.add_parser("update-whois", help=_("Update WHOIS Information"))
)
update_overrides.set_defaults(func=self.handle_update_overrides)
+ # Import countries
+ import_countries = subparsers.add_parser("import-countries",
+ help=_("Import countries"),
+ )
+ import_countries.add_argument("file", nargs=1, type=argparse.FileType("r"),
+ help=_("File to import"))
+ import_countries.set_defaults(func=self.handle_import_countries)
+
args = parser.parse_args()
# Configure logging
CREATE TABLE IF NOT EXISTS autnums(number bigint, name text NOT NULL);
CREATE UNIQUE INDEX IF NOT EXISTS autnums_number ON autnums(number);
+ -- countries
+ CREATE TABLE IF NOT EXISTS countries(
+ country_code text NOT NULL, name text NOT NULL, continent_code text NOT NULL);
+ CREATE UNIQUE INDEX IF NOT EXISTS countries_country_code ON countries(country_code);
+
-- networks
CREATE TABLE IF NOT EXISTS networks(network inet, country text);
CREATE UNIQUE INDEX IF NOT EXISTS networks_network ON networks(network);
+ CREATE INDEX IF NOT EXISTS networks_family ON networks USING BTREE(family(network));
CREATE INDEX IF NOT EXISTS networks_search ON networks USING GIST(network inet_ops);
-- overrides
CREATE TABLE IF NOT EXISTS autnum_overrides(
number bigint NOT NULL,
name text,
- is_anonymous_proxy boolean DEFAULT FALSE,
- is_satellite_provider boolean DEFAULT FALSE,
- is_anycast boolean DEFAULT FALSE
+ country text,
+ is_anonymous_proxy boolean,
+ is_satellite_provider boolean,
+ is_anycast boolean
);
CREATE UNIQUE INDEX IF NOT EXISTS autnum_overrides_number
ON autnum_overrides(number);
CREATE TABLE IF NOT EXISTS network_overrides(
network inet NOT NULL,
country text,
- is_anonymous_proxy boolean DEFAULT FALSE,
- is_satellite_provider boolean DEFAULT FALSE,
- is_anycast boolean DEFAULT FALSE
+ is_anonymous_proxy boolean,
+ is_satellite_provider boolean,
+ is_anycast boolean
);
CREATE UNIQUE INDEX IF NOT EXISTS network_overrides_network
ON network_overrides(network);
Compiles a database in libloc format out of what is in the database
"""
# Allocate a writer
- writer = location.Writer(ns.signing_key)
+ writer = location.Writer(ns.signing_key, ns.backup_signing_key)
# Set all metadata
if ns.vendor:
# Select all known networks
rows = self.db.query("""
+ -- Get a (sorted) list of all known networks
+ WITH known_networks AS (
+ SELECT network FROM announcements
+ UNION
+ SELECT network FROM networks
+ UNION
+ SELECT network FROM network_overrides
+ ORDER BY network
+ )
+
+ -- Return a list of those networks enriched with all
+ -- other information that we store in the database
SELECT
- DISTINCT ON (announcements.network)
- announcements.network AS network,
+ DISTINCT ON (known_networks.network)
+ known_networks.network AS network,
announcements.autnum AS autnum,
- networks.country AS country,
- -- Must be part of returned values for ORDER BY clause
- masklen(networks.network) AS sort,
+ -- Country
+ COALESCE(
+ (
+ SELECT country FROM network_overrides overrides
+ WHERE announcements.network <<= overrides.network
+ ORDER BY masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT country FROM autnum_overrides overrides
+ WHERE announcements.autnum = overrides.number
+ ),
+ networks.country
+ ) AS country,
-- Flags
COALESCE(
(
SELECT is_anonymous_proxy FROM autnum_overrides overrides
WHERE announcements.autnum = overrides.number
- )
+ ),
+ FALSE
) AS is_anonymous_proxy,
COALESCE(
(
(
SELECT is_satellite_provider FROM autnum_overrides overrides
WHERE announcements.autnum = overrides.number
- )
+ ),
+ FALSE
) AS is_satellite_provider,
COALESCE(
(
(
SELECT is_anycast FROM autnum_overrides overrides
WHERE announcements.autnum = overrides.number
- )
- ) AS is_anycast
- FROM announcements
- LEFT JOIN networks ON announcements.network <<= networks.network
- ORDER BY announcements.network, sort DESC
+ ),
+ FALSE
+ ) AS is_anycast,
+
+ -- Must be part of returned values for ORDER BY clause
+ masklen(announcements.network) AS sort_a,
+ masklen(networks.network) AS sort_b
+ FROM known_networks
+ LEFT JOIN announcements ON known_networks.network <<= announcements.network
+ LEFT JOIN networks ON known_networks.network <<= networks.network
+ ORDER BY known_networks.network, sort_a DESC, sort_b DESC
""")
for row in rows:
network = writer.add_network(row.network)
- # Save AS & country
- network.asn, network.country_code = row.autnum, row.country
+ # Save country
+ if row.country:
+ network.country_code = row.country
+
+ # Save ASN
+ if row.autnum:
+ network.asn = row.autnum
# Set flags
if row.is_anonymous_proxy:
if row.is_anycast:
network.set_flag(location.NETWORK_FLAG_ANYCAST)
+ # Add all countries
+ log.info("Writing countries...")
+ rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
+
+ for row in rows:
+ c = writer.add_country(row.country_code)
+ c.continent_code = row.continent_code
+ c.name = row.name
+
# Write everything to file
log.info("Writing database to file...")
for file in ns.file:
ON COMMIT DROP;
CREATE UNIQUE INDEX _autnums_number ON _autnums(number);
- CREATE TEMPORARY TABLE _organizations(handle text, name text)
+ CREATE TEMPORARY TABLE _organizations(handle text, name text NOT NULL)
ON COMMIT DROP;
CREATE UNIQUE INDEX _organizations_handle ON _organizations(handle);
+
+ CREATE TEMPORARY TABLE _rirdata(network inet NOT NULL, country text NOT NULL)
+ ON COMMIT DROP;
+ CREATE INDEX _rirdata_search ON _rirdata USING BTREE(family(network), masklen(network));
+ CREATE UNIQUE INDEX _rirdata_network ON _rirdata(network);
+ """)
+
+ # Remove all previously imported content
+ self.db.execute("""
+ TRUNCATE TABLE networks;
""")
for source in location.importer.WHOIS_SOURCES:
for block in f:
self._parse_block(block)
+ # Process all parsed networks from every RIR we happen to have access to,
+ # insert the largest network chunks into the networks table immediately...
+ families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata ORDER BY family(network)")
+
+ for family in (row.family for row in families):
+ smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata WHERE family(network) = %s", family)
+
+ self.db.execute("INSERT INTO networks(network, country) \
+ SELECT network, country FROM _rirdata WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
+
+ # ... determine any other prefixes for this network family, ...
+ prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
+ WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
+
+ # ... and insert networks with this prefix in case they provide additional
+ # information (i. e. subnet of a larger chunk with a different country)
+ for prefix in (row.prefix for row in prefixes):
+ self.db.execute("""
+ WITH candidates AS (
+ SELECT
+ _rirdata.network,
+ _rirdata.country
+ FROM
+ _rirdata
+ WHERE
+ family(_rirdata.network) = %s
+ AND
+ masklen(_rirdata.network) = %s
+ ),
+ filtered AS (
+ SELECT
+ DISTINCT ON (c.network)
+ c.network,
+ c.country,
+ masklen(networks.network),
+ networks.country AS parent_country
+ FROM
+ candidates c
+ LEFT JOIN
+ networks
+ ON
+ c.network << networks.network
+ ORDER BY
+ c.network,
+ masklen(networks.network) DESC NULLS LAST
+ )
+ INSERT INTO
+ networks(network, country)
+ SELECT
+ network,
+ country
+ FROM
+ filtered
+ WHERE
+ parent_country IS NULL
+ OR
+ country <> parent_country
+ ON CONFLICT DO NOTHING""",
+ family, prefix,
+ )
+
self.db.execute("""
INSERT INTO autnums(number, name)
SELECT _autnums.number, _organizations.name FROM _autnums
- LEFT JOIN _organizations ON _autnums.organization = _organizations.handle
+ JOIN _organizations ON _autnums.organization = _organizations.handle
ON CONFLICT (number) DO UPDATE SET name = excluded.name;
""")
if line.startswith("aut-num:"):
return self._parse_autnum_block(block)
+ # inetnum
+ if line.startswith("inet6num:") or line.startswith("inetnum:"):
+ return self._parse_inetnum_block(block)
+
# organisation
elif line.startswith("organisation:"):
return self._parse_org_block(block)
autnum.get("asn"), autnum.get("org"),
)
+ def _parse_inetnum_block(self, block):
+ logging.debug("Parsing inetnum block:")
+
+ inetnum = {}
+ for line in block:
+ logging.debug(line)
+
+ # Split line
+ key, val = split_line(line)
+
+ if key == "inetnum":
+ start_address, delim, end_address = val.partition("-")
+
+ # Strip any excess space
+ start_address, end_address = start_address.rstrip(), end_address.strip()
+
+ # Convert to IP address
+ try:
+ start_address = ipaddress.ip_address(start_address)
+ end_address = ipaddress.ip_address(end_address)
+ except ValueError:
+ logging.warning("Could not parse line: %s" % line)
+ return
+
+ # Set prefix to default
+ prefix = 32
+
+ # Count number of addresses in this subnet
+ num_addresses = int(end_address) - int(start_address)
+ if num_addresses:
+ prefix -= math.log(num_addresses, 2)
+
+ inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
+
+ elif key == "inet6num":
+ inetnum[key] = val
+
+ elif key == "country":
+ if val == "UNITED STATES":
+ val = "US"
+
+ inetnum[key] = val.upper()
+
+ # Skip empty objects
+ if not inetnum or not "country" in inetnum:
+ return
+
+ network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
+ # Bail out in case we have processed a network covering the entire IP range, which
+ # is necessary to work around faulty (?) IPv6 network processing
+ if network.prefixlen == 0:
+ logging.warning("Skipping network covering the entire IP adress range: %s" % network)
+ return
+
+ # Bail out in case we have processed a network whose prefix length indicates it is
+ # not globally routable (we have decided not to process them at the moment, as they
+ # significantly enlarge our database without providing very helpful additional information)
+ if (network.prefixlen > 24 and network.version == 4) or (network.prefixlen > 48 and network.version == 6):
+ logging.info("Skipping network too small to be publicly announced: %s" % network)
+ return
+
+ # Bail out in case we have processed a non-public IP network
+ if network.is_private:
+ logging.warning("Skipping non-globally routable network: %s" % network)
+ return
+
+ self.db.execute("INSERT INTO _rirdata(network, country) \
+ VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+ "%s" % network, inetnum.get("country"),
+ )
+
def _parse_org_block(self, block):
org = {}
for line in block:
def handle_update_announcements(self, ns):
server = ns.server[0]
+ with self.db.transaction():
+ if server.startswith("/"):
+ self._handle_update_announcements_from_bird(server)
+ else:
+ self._handle_update_announcements_from_telnet(server)
+
+ # Purge anything we never want here
+ self.db.execute("""
+ -- Delete default routes
+ DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
+
+ -- Delete anything that is not global unicast address space
+ DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
+
+ -- DELETE "current network" address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
+
+ -- DELETE local loopback address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
+
+ -- DELETE RFC 1918 address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
+
+ -- DELETE test, benchmark and documentation address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
+
+ -- DELETE CGNAT address space (RFC 6598)
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
+
+ -- DELETE link local address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
+
+ -- DELETE IPv6 to IPv4 (6to4) address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
+
+ -- DELETE multicast and reserved address space
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
+ DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
+
+ -- Delete networks that are too small to be in the global routing table
+ DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
+ DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
+
+ -- Delete any non-public or reserved ASNs
+ DELETE FROM announcements WHERE NOT (
+ (autnum >= 1 AND autnum <= 23455)
+ OR
+ (autnum >= 23457 AND autnum <= 64495)
+ OR
+ (autnum >= 131072 AND autnum <= 4199999999)
+ );
+
+ -- Delete everything that we have not seen for 14 days
+ DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
+ """)
+
+ def _handle_update_announcements_from_bird(self, server):
+ # Pre-compile the regular expression for faster searching
+ route = re.compile(b"^\s(.+?)\s+.+?\[AS(.*?).\]$")
+
+ log.info("Requesting routing table from Bird (%s)" % server)
+
+ # Send command to list all routes
+ for line in self._bird_cmd(server, "show route"):
+ m = route.match(line)
+ if not m:
+ log.debug("Could not parse line: %s" % line.decode())
+ continue
+
+ # Fetch the extracted network and ASN
+ network, autnum = m.groups()
+
+ # Insert it into the database
+ self.db.execute("INSERT INTO announcements(network, autnum) \
+ VALUES(%s, %s) ON CONFLICT (network) DO \
+ UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
+ network.decode(), autnum.decode(),
+ )
+
+ def _handle_update_announcements_from_telnet(self, server):
# Pre-compile regular expression for routes
- #route = re.compile(b"^\*>?\s[\si]?([^\s]+)[.\s]*?(\d+)\si$", re.MULTILINE)
route = re.compile(b"^\*[\s\>]i([^\s]+).+?(\d+)\si\r\n", re.MULTILINE|re.DOTALL)
with telnetlib.Telnet(server) as t:
# t.set_debuglevel(10)
# Wait for console greeting
- greeting = t.read_until(b"> ")
- log.debug(greeting.decode())
+ greeting = t.read_until(b"> ", timeout=30)
+ if not greeting:
+ log.error("Could not get a console prompt")
+ return 1
# Disable pagination
t.write(b"terminal length 0\n")
t.read_until(b"> ")
# Fetch the routing tables
- with self.db.transaction():
- for protocol in ("ipv6", "ipv4"):
- log.info("Requesting %s routing table" % protocol)
+ for protocol in ("ipv6", "ipv4"):
+ log.info("Requesting %s routing table" % protocol)
- # Request the full unicast routing table
- t.write(b"show bgp %s unicast\n" % protocol.encode())
+ # Request the full unicast routing table
+ t.write(b"show bgp %s unicast\n" % protocol.encode())
- # Read entire header which ends with "Path"
- t.read_until(b"Path\r\n")
+ # Read entire header which ends with "Path"
+ t.read_until(b"Path\r\n")
- while True:
- # Try reading a full entry
- # Those might be broken across multiple lines but ends with i
- line = t.read_until(b"i\r\n", timeout=5)
- if not line:
- break
+ while True:
+ # Try reading a full entry
+ # Those might be broken across multiple lines but ends with i
+ line = t.read_until(b"i\r\n", timeout=5)
+ if not line:
+ break
- # Show line for debugging
- #log.debug(repr(line))
+ # Show line for debugging
+ #log.debug(repr(line))
- # Try finding a route in here
- m = route.match(line)
- if m:
- network, autnum = m.groups()
+ # Try finding a route in here
+ m = route.match(line)
+ if m:
+ network, autnum = m.groups()
- # Convert network to string
- network = network.decode()
+ # Convert network to string
+ network = network.decode()
- # Append /24 for IPv4 addresses
- if not "/" in network and not ":" in network:
- network = "%s/24" % network
+ # Append /24 for IPv4 addresses
+ if not "/" in network and not ":" in network:
+ network = "%s/24" % network
- # Convert AS number to integer
- autnum = int(autnum)
+ # Convert AS number to integer
+ autnum = int(autnum)
- log.info("Found announcement for %s by %s" % (network, autnum))
+ log.info("Found announcement for %s by %s" % (network, autnum))
- self.db.execute("INSERT INTO announcements(network, autnum) \
- VALUES(%s, %s) ON CONFLICT (network) DO \
- UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
- network, autnum,
- )
-
- log.info("Finished reading the %s routing table" % protocol)
-
- # Purge anything we never want here
- self.db.execute("""
- -- Delete default routes
- DELETE FROM announcements WHERE network = '::/0' OR network = '0.0.0.0/0';
+ self.db.execute("INSERT INTO announcements(network, autnum) \
+ VALUES(%s, %s) ON CONFLICT (network) DO \
+ UPDATE SET autnum = excluded.autnum, last_seen_at = CURRENT_TIMESTAMP",
+ network, autnum,
+ )
- -- Delete anything that is not global unicast address space
- DELETE FROM announcements WHERE family(network) = 6 AND NOT network <<= '2000::/3';
+ log.info("Finished reading the %s routing table" % protocol)
- -- DELETE "current network" address space
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '0.0.0.0/8';
+ def _bird_cmd(self, socket_path, command):
+ # Connect to the socket
+ s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ s.connect(socket_path)
- -- DELETE local loopback address space
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '127.0.0.0/8';
+ # Allocate some buffer
+ buffer = b""
- -- DELETE RFC 1918 address space
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '10.0.0.0/8';
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '172.16.0.0/12';
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.168.0.0/16';
+ # Send the command
+ s.send(b"%s\n" % command.encode())
- -- DELETE test, benchmark and documentation address space
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.0.0/24';
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.0.2.0/24';
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.18.0.0/15';
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '198.51.100.0/24';
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '203.0.113.0/24';
+ while True:
+ # Fill up the buffer
+ buffer += s.recv(4096)
- -- DELETE CGNAT address space (RFC 6598)
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '100.64.0.0/10';
+ while True:
+ # Search for the next newline
+ pos = buffer.find(b"\n")
- -- DELETE link local address space
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '169.254.0.0/16';
+ # If we cannot find one, we go back and read more data
+ if pos <= 0:
+ break
- -- DELETE IPv6 to IPv4 (6to4) address space
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '192.88.99.0/24';
+ # Cut after the newline character
+ pos += 1
- -- DELETE multicast and reserved address space
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '224.0.0.0/4';
- DELETE FROM announcements WHERE family(network) = 4 AND network <<= '240.0.0.0/4';
+ # Split the line we want and keep the rest in buffer
+ line, buffer = buffer[:pos], buffer[pos:]
- -- Delete networks that are too small to be in the global routing table
- DELETE FROM announcements WHERE family(network) = 6 AND masklen(network) > 48;
- DELETE FROM announcements WHERE family(network) = 4 AND masklen(network) > 24;
-
- -- Delete any non-public or reserved ASNs
- DELETE FROM announcements WHERE NOT (
- (autnum >= 1 AND autnum <= 23455)
- OR
- (autnum >= 23457 AND autnum <= 64495)
- OR
- (autnum >= 131072 AND autnum <= 4199999999)
- );
+ # Look for the end-of-output indicator
+ if line == b"0000 \n":
+ return
- -- Delete everything that we have not seen for 14 days
- DELETE FROM announcements WHERE last_seen_at <= CURRENT_TIMESTAMP - INTERVAL '14 days';
- """)
+ # Otherwise return the line
+ yield line
def handle_update_overrides(self, ns):
with self.db.transaction():
log.warning("Invalid IP network: %s: %s" % (network, e))
continue
+ # Prevent that we overwrite all networks
+ if network.prefixlen == 0:
+ log.warning("Skipping %s: You cannot overwrite default" % network)
+ continue
+
self.db.execute("""
INSERT INTO network_overrides(
network,
ON CONFLICT (network) DO NOTHING""",
"%s" % network,
block.get("country"),
- block.get("is-anonymous-proxy") == "yes",
- block.get("is-satellite-provider") == "yes",
- block.get("is-anycast") == "yes",
+ self._parse_bool(block, "is-anonymous-proxy"),
+ self._parse_bool(block, "is-satellite-provider"),
+ self._parse_bool(block, "is-anycast"),
)
- elif type == "autnum":
- autnum = block.get("autnum")
+ elif type == "aut-num":
+ autnum = block.get("aut-num")
# Check if AS number begins with "AS"
if not autnum.startswith("AS"):
INSERT INTO autnum_overrides(
number,
name,
+ country,
is_anonymous_proxy,
is_satellite_provider,
is_anycast
- ) VALUES(%s, %s, %s, %s, %s)
+ ) VALUES(%s, %s, %s, %s, %s, %s)
ON CONFLICT DO NOTHING""",
- autnum, block.get("name"),
- block.get("is-anonymous-proxy") == "yes",
- block.get("is-satellite-provider") == "yes",
- block.get("is-anycast") == "yes",
+ autnum,
+ block.get("name"),
+ block.get("country"),
+ self._parse_bool(block, "is-anonymous-proxy"),
+ self._parse_bool(block, "is-satellite-provider"),
+ self._parse_bool(block, "is-anycast"),
)
else:
log.warning("Unsupport type: %s" % type)
+ @staticmethod
+ def _parse_bool(block, key):
+ val = block.get(key)
+
+ # There is no point to proceed when we got None
+ if val is None:
+ return
+
+ # Convert to lowercase
+ val = val.lower()
+
+ # True
+ if val in ("yes", "1"):
+ return True
+
+ # False
+ if val in ("no", "0"):
+ return False
+
+ # Default to None
+ return None
+
+ def handle_import_countries(self, ns):
+ with self.db.transaction():
+ # Drop all data that we have
+ self.db.execute("TRUNCATE TABLE countries")
+
+ for file in ns.file:
+ for line in file:
+ line = line.rstrip()
+
+ # Ignore any comments
+ if line.startswith("#"):
+ continue
+
+ try:
+ country_code, continent_code, name = line.split(maxsplit=2)
+ except:
+ log.warning("Could not parse line: %s" % line)
+ continue
+
+ self.db.execute("INSERT INTO countries(country_code, name, continent_code) \
+ VALUES(%s, %s, %s) ON CONFLICT DO NOTHING", country_code, name, continent_code)
+
def split_line(line):
key, colon, val = line.partition(":")
projects / people / ms / libloc.git / blobdiff