# #
# libloc - A library to determine the location of someone on the Internet #
# #
-# Copyright (C) 2020-2022 IPFire Development Team <info@ipfire.org> #
+# Copyright (C) 2020-2024 IPFire Development Team <info@ipfire.org> #
# #
# This library is free software; you can redistribute it and/or #
# modify it under the terms of the GNU Lesser General Public #
import argparse
import concurrent.futures
+import csv
import http.client
import ipaddress
import json
(131072, 4199999999),
)
+# Configure the CSV parser for ARIN
+csv.register_dialect("arin", delimiter=",", quoting=csv.QUOTE_ALL, quotechar="\"")
class CLI(object):
def parse_cli(self):
help=_("Update Geofeeds"))
update_geofeeds.set_defaults(func=self.handle_update_geofeeds)
+ # Update feeds
+ update_feeds = subparsers.add_parser("update-feeds",
+ help=_("Update Feeds"))
+ update_feeds.set_defaults(func=self.handle_update_feeds)
+
# Update overrides
update_overrides = subparsers.add_parser("update-overrides",
help=_("Update overrides"),
CREATE INDEX IF NOT EXISTS network_geofeeds_url
ON network_geofeeds(url);
+ -- feeds
+ CREATE TABLE IF NOT EXISTS autnum_feeds(
+ number bigint NOT NULL,
+ source text NOT NULL,
+ name text,
+ country text,
+ is_anonymous_proxy boolean,
+ is_satellite_provider boolean,
+ is_anycast boolean,
+ is_drop boolean
+ );
+ CREATE UNIQUE INDEX IF NOT EXISTS autnum_feeds_unique
+ ON autnum_feeds(number, source);
+
+ CREATE TABLE IF NOT EXISTS network_feeds(
+ network inet NOT NULL,
+ source text NOT NULL,
+ country text,
+ is_anonymous_proxy boolean,
+ is_satellite_provider boolean,
+ is_anycast boolean,
+ is_drop boolean
+ );
+ CREATE UNIQUE INDEX IF NOT EXISTS network_feeds_unique
+ ON network_feeds(network, source);
+ CREATE INDEX IF NOT EXISTS network_feeds_search
+ ON network_feeds USING GIST(network inet_ops);
+
-- overrides
CREATE TABLE IF NOT EXISTS autnum_overrides(
number bigint NOT NULL,
return db
+ def fetch_countries(self):
+ """
+ Returns a list of all countries on the list
+ """
+ # Fetch all valid country codes to check parsed networks aganist...
+ countries = self.db.query("SELECT country_code FROM countries ORDER BY country_code")
+
+ return [country.country_code for country in countries]
+
def handle_write(self, ns):
"""
Compiles a database in libloc format out of what is in the database
UNION
SELECT network FROM networks
UNION
+ SELECT network FROM network_feeds
+ UNION
SELECT network FROM network_overrides
UNION
SELECT network FROM geofeed_networks
-- Country
COALESCE(
(
- SELECT country FROM network_overrides overrides
- WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
+ SELECT
+ country
+ FROM
+ network_overrides overrides
+ WHERE
+ networks.network <<= overrides.network
+ ORDER BY
+ masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT
+ country
+ FROM
+ autnum_overrides overrides
+ WHERE
+ networks.autnum = overrides.number
+ ),
+ (
+ SELECT
+ country
+ FROM
+ network_feeds feeds
+ WHERE
+ networks.network <<= feeds.network
+ ORDER BY
+ masklen(feeds.network) DESC
+ LIMIT 1
),
(
- SELECT country FROM autnum_overrides overrides
- WHERE networks.autnum = overrides.number
+ SELECT
+ country
+ FROM
+ autnum_feeds feeds
+ WHERE
+ networks.autnum = feeds.number
+ ORDER BY
+ source
+ LIMIT 1
),
(
SELECT
-- Flags
COALESCE(
(
- SELECT is_anonymous_proxy FROM network_overrides overrides
- WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
+ SELECT
+ is_anonymous_proxy
+ FROM
+ network_overrides overrides
+ WHERE
+ networks.network <<= overrides.network
+ ORDER BY
+ masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT
+ is_anonymous_proxy
+ FROM
+ network_feeds feeds
+ WHERE
+ networks.network <<= feeds.network
+ ORDER BY
+ masklen(feeds.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT
+ is_anonymous_proxy
+ FROM
+ autnum_feeds feeds
+ WHERE
+ networks.autnum = feeds.number
+ ORDER BY
+ source
+ LIMIT 1
),
(
- SELECT is_anonymous_proxy FROM autnum_overrides overrides
- WHERE networks.autnum = overrides.number
+ SELECT
+ is_anonymous_proxy
+ FROM
+ autnum_overrides overrides
+ WHERE
+ networks.autnum = overrides.number
),
FALSE
) AS is_anonymous_proxy,
COALESCE(
(
- SELECT is_satellite_provider FROM network_overrides overrides
- WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
+ SELECT
+ is_satellite_provider
+ FROM
+ network_overrides overrides
+ WHERE
+ networks.network <<= overrides.network
+ ORDER BY
+ masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT
+ is_satellite_provider
+ FROM
+ network_feeds feeds
+ WHERE
+ networks.network <<= feeds.network
+ ORDER BY
+ masklen(feeds.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT
+ is_satellite_provider
+ FROM
+ autnum_feeds feeds
+ WHERE
+ networks.autnum = feeds.number
+ ORDER BY
+ source
+ LIMIT 1
),
(
- SELECT is_satellite_provider FROM autnum_overrides overrides
- WHERE networks.autnum = overrides.number
+ SELECT
+ is_satellite_provider
+ FROM
+ autnum_overrides overrides
+ WHERE
+ networks.autnum = overrides.number
),
FALSE
) AS is_satellite_provider,
COALESCE(
(
- SELECT is_anycast FROM network_overrides overrides
- WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
+ SELECT
+ is_anycast
+ FROM
+ network_overrides overrides
+ WHERE
+ networks.network <<= overrides.network
+ ORDER BY
+ masklen(overrides.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT
+ is_anycast
+ FROM
+ network_feeds feeds
+ WHERE
+ networks.network <<= feeds.network
+ ORDER BY
+ masklen(feeds.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT
+ is_anycast
+ FROM
+ autnum_feeds feeds
+ WHERE
+ networks.autnum = feeds.number
+ ORDER BY
+ source
+ LIMIT 1
),
(
- SELECT is_anycast FROM autnum_overrides overrides
- WHERE networks.autnum = overrides.number
+ SELECT
+ is_anycast
+ FROM
+ autnum_overrides overrides
+ WHERE
+ networks.autnum = overrides.number
),
FALSE
) AS is_anycast,
COALESCE(
(
- SELECT is_drop FROM network_overrides overrides
- WHERE networks.network <<= overrides.network
- ORDER BY masklen(overrides.network) DESC
- LIMIT 1
+ SELECT
+ is_drop
+ FROM
+ network_overrides overrides
+ WHERE
+ networks.network <<= overrides.network
+ ORDER BY
+ masklen(overrides.network) DESC
+ LIMIT 1
),
(
- SELECT is_drop FROM autnum_overrides overrides
- WHERE networks.autnum = overrides.number
+ SELECT
+ is_drop
+ FROM
+ network_feeds feeds
+ WHERE
+ networks.network <<= feeds.network
+ ORDER BY
+ masklen(feeds.network) DESC
+ LIMIT 1
+ ),
+ (
+ SELECT
+ is_drop
+ FROM
+ autnum_feeds feeds
+ WHERE
+ networks.autnum = feeds.number
+ ORDER BY
+ source
+ LIMIT 1
+ ),
+ (
+ SELECT
+ is_drop
+ FROM
+ autnum_overrides overrides
+ WHERE
+ networks.autnum = overrides.number
),
FALSE
) AS is_drop
# Did we run successfully?
error = False
- # Fetch all valid country codes to check parsed networks aganist
- validcountries = self.countries
+ # Fetch all valid country codes to check parsed networks against
+ validcountries = self.fetch_countries()
# Iterate over all potential sources
for source in sorted(location.importer.SOURCES):
""")
# Remove all previously imported content
+ self.db.execute("DELETE FROM autnums WHERE source = %s", source)
self.db.execute("DELETE FROM networks WHERE source = %s", source)
try:
# Process all parsed networks from every RIR we happen to have access to,
# insert the largest network chunks into the networks table immediately...
- families = self.db.query("SELECT DISTINCT family(network) AS family FROM _rirdata \
- ORDER BY family(network)")
+ families = self.db.query("""
+ SELECT DISTINCT
+ family(network) AS family
+ FROM
+ _rirdata
+ ORDER BY
+ family(network)
+ """,
+ )
for family in (row.family for row in families):
# Fetch the smallest mask length in our data set
FROM
_rirdata
WHERE
- family(network) = %s""",
- family,
+ family(network) = %s
+ """, family,
)
# Copy all networks
family(network) = %s
ORDER BY
masklen(network) ASC
- OFFSET 1""",
- family,
+ OFFSET 1
+ """, family,
)
# ... and insert networks with this prefix in case they provide additional
parent_country IS NULL
OR
country <> parent_country
- ON CONFLICT DO NOTHING""",
- family, prefix,
+ ON CONFLICT DO NOTHING
+ """, family, prefix,
)
self.db.execute("""
- INSERT INTO autnums(number, name, source)
- SELECT _autnums.number, _organizations.name, _organizations.source FROM _autnums
- JOIN _organizations ON _autnums.organization = _organizations.handle
- ON CONFLICT (number) DO UPDATE SET name = excluded.name;
- """)
+ INSERT INTO
+ autnums
+ (
+ number,
+ name,
+ source
+ )
+ SELECT
+ _autnums.number,
+ _organizations.name,
+ _organizations.source
+ FROM
+ _autnums
+ JOIN
+ _organizations ON _autnums.organization = _organizations.handle
+ ON CONFLICT
+ (
+ number
+ )
+ DO UPDATE
+ SET name = excluded.name
+ """,
+ )
# Download and import (technical) AS names from ARIN
with self.db.transaction():
- self._import_as_names_from_arin()
+ self._import_as_names_from_arin(downloader)
# Return a non-zero exit code for errors
return 1 if error else 0
"%s" % network, country, [country], source_key,
)
- def _import_as_names_from_arin(self):
- downloader = location.importer.Downloader()
+ def _import_as_names_from_arin(self, downloader):
+ # Delete all previously imported content
+ self.db.execute("DELETE FROM autnums WHERE source = %s", "ARIN")
+
+ # Try to retrieve the feed from ftp.arin.net
+ feed = downloader.request_lines("https://ftp.arin.net/pub/resource_registry_service/asns.csv")
+
+ # Walk through the file
+ for line in csv.DictReader(feed, dialect="arin"):
+ log.debug("Processing object: %s" % line)
+
+ # Fetch status
+ status = line.get("Status")
- # XXX: Download AS names file from ARIN (note that these names appear to be quite
- # technical, not intended for human consumption, as description fields in
- # organisation handles for other RIRs are - however, this is what we have got,
- # and in some cases, it might be still better than nothing)
- for line in downloader.request_lines("https://ftp.arin.net/info/asn.txt"):
- # Valid lines start with a space, followed by the number of the Autonomous System ...
- if not line.startswith(" "):
+ # We are only interested in anything managed by ARIN
+ if not status == "Full Registry Services":
continue
- # Split line and check if there is a valid ASN in it...
- asn, name = line.split()[0:2]
+ # Fetch organization name
+ name = line.get("Org Name")
+ # Extract ASNs
+ first_asn = line.get("Start AS Number")
+ last_asn = line.get("End AS Number")
+
+ # Cast to a number
try:
- asn = int(asn)
- except ValueError:
- log.debug("Skipping ARIN AS names line not containing an integer for ASN")
+ first_asn = int(first_asn)
+ except TypeError as e:
+ log.warning("Could not parse ASN '%s'" % first_asn)
continue
- # Filter invalid ASNs...
- if not self._check_parsed_asn(asn):
+ try:
+ last_asn = int(last_asn)
+ except TypeError as e:
+ log.warning("Could not parse ASN '%s'" % last_asn)
continue
- # Skip any AS name that appears to be a placeholder for a different RIR or entity...
- if re.match(r"^(ASN-BLK|)(AFCONC|AFRINIC|APNIC|ASNBLK|LACNIC|RIPE|IANA)(?:\d?$|\-)", name):
- continue
+ # Check if the range is valid
+ if last_asn < first_asn:
+ log.warning("Invalid ASN range %s-%s" % (first_asn, last_asn))
- # Bail out in case the AS name contains anything we do not expect here...
- if re.search(r"[^a-zA-Z0-9-_]", name):
- log.debug("Skipping ARIN AS name for %s containing invalid characters: %s" % \
- (asn, name))
+ # Insert everything into the database
+ for asn in range(first_asn, last_asn + 1):
+ if not self._check_parsed_asn(asn):
+ log.warning("Skipping invalid ASN %s" % asn)
+ continue
- # Things look good here, run INSERT statement and skip this one if we already have
- # a (better?) name for this Autonomous System...
- self.db.execute("""
- INSERT INTO autnums(
- number,
- name,
- source
- ) VALUES (%s, %s, %s)
- ON CONFLICT (number) DO NOTHING""",
- asn,
- name,
- "ARIN",
- )
+ self.db.execute("""
+ INSERT INTO
+ autnums
+ (
+ number,
+ name,
+ source
+ )
+ VALUES
+ (
+ %s, %s, %s
+ )
+ ON CONFLICT
+ (
+ number
+ )
+ DO NOTHING
+ """, asn, name, "ARIN",
+ )
def handle_update_announcements(self, ns):
server = ns.server[0]
DELETE FROM network_overrides WHERE source = 'manual';
""")
- # Update overrides for various cloud providers big enough to publish their own IP
- # network allocation lists in a machine-readable format...
- self._update_overrides_for_aws()
-
- # Update overrides for Spamhaus DROP feeds...
- self._update_overrides_for_spamhaus_drop()
-
for file in ns.files:
log.info("Reading %s..." % file)
else:
log.warning("Unsupported type: %s" % type)
- def _update_overrides_for_aws(self):
- # Download Amazon AWS IP allocation file to create overrides...
+ def handle_update_feeds(self, ns):
+ """
+ Update any third-party feeds
+ """
+ success = True
+
+ # Create a downloader
downloader = location.importer.Downloader()
- try:
- # Fetch IP ranges
- f = downloader.retrieve("https://ip-ranges.amazonaws.com/ip-ranges.json")
+ feeds = (
+ # AWS IP Ranges
+ ("AWS-IP-RANGES", self._import_aws_ip_ranges, "https://ip-ranges.amazonaws.com/ip-ranges.json"),
+ )
- # Parse downloaded file
- aws_ip_dump = json.load(f)
- except Exception as e:
- log.error("unable to preprocess Amazon AWS IP ranges: %s" % e)
- return
+ # Walk through all feeds
+ for name, callback, url, *args in feeds:
+ try:
+ self._process_feed(downloader, name, callback, url, *args)
- # At this point, we can assume the downloaded file to be valid
- self.db.execute("""
- DELETE FROM network_overrides WHERE source = 'Amazon AWS IP feed';
- """)
+ # Log an error but continue if an exception occurs
+ except Exception as e:
+ log.error("Error processing feed '%s': %s" % (name, e))
+ success = False
+
+ # Spamhaus
+ self._update_feed_for_spamhaus_drop()
+
+ # Return status
+ return 0 if success else 1
- # XXX: Set up a dictionary for mapping a region name to a country. Unfortunately,
+ def _process_feed(self, downloader, name, callback, url, *args):
+ """
+ Processes one feed
+ """
+ # Open the URL
+ f = downloader.retrieve(url)
+
+ with self.db.transaction():
+ # Drop any previous content
+ self.db.execute("DELETE FROM autnum_feeds WHERE source = %s", name)
+ self.db.execute("DELETE FROM network_feeds WHERE source = %s", name)
+
+ # Call the callback to process the feed
+ return callback(name, f, *args)
+
+ def _import_aws_ip_ranges(self, name, f):
+ # Parse the feed
+ aws_ip_dump = json.load(f)
+
+ # Set up a dictionary for mapping a region name to a country. Unfortunately,
# there seems to be no machine-readable version available of this other than
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html
# (worse, it seems to be incomplete :-/ ); https://www.cloudping.cloud/endpoints
# was helpful here as well.
aws_region_country_map = {
- "af-south-1": "ZA",
- "ap-east-1": "HK",
- "ap-south-1": "IN",
- "ap-south-2": "IN",
- "ap-northeast-3": "JP",
- "ap-northeast-2": "KR",
- "ap-southeast-1": "SG",
- "ap-southeast-2": "AU",
- "ap-southeast-3": "MY",
- "ap-southeast-4": "AU",
- "ap-southeast-5": "NZ", # Auckland, NZ
- "ap-southeast-6": "AP", # XXX: Precise location not documented anywhere
- "ap-northeast-1": "JP",
- "ca-central-1": "CA",
- "ca-west-1": "CA",
- "eu-central-1": "DE",
- "eu-central-2": "CH",
- "eu-west-1": "IE",
- "eu-west-2": "GB",
- "eu-south-1": "IT",
- "eu-south-2": "ES",
- "eu-west-3": "FR",
- "eu-north-1": "SE",
- "il-central-1": "IL", # XXX: This one is not documented anywhere except for ip-ranges.json itself
- "me-central-1": "AE",
- "me-south-1": "BH",
- "sa-east-1": "BR"
- }
-
- # Fetch all valid country codes to check parsed networks aganist...
- rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
- validcountries = []
-
- for row in rows:
- validcountries.append(row.country_code)
-
- with self.db.transaction():
- for snetwork in aws_ip_dump["prefixes"] + aws_ip_dump["ipv6_prefixes"]:
- try:
- network = ipaddress.ip_network(snetwork.get("ip_prefix") or snetwork.get("ipv6_prefix"), strict=False)
- except ValueError:
- log.warning("Unable to parse line: %s" % snetwork)
- continue
+ # Africa
+ "af-south-1" : "ZA",
+
+ # Asia
+ "il-central-1" : "IL", # Tel Aviv
+
+ # Asia/Pacific
+ "ap-northeast-1" : "JP",
+ "ap-northeast-2" : "KR",
+ "ap-northeast-3" : "JP",
+ "ap-east-1" : "HK",
+ "ap-south-1" : "IN",
+ "ap-south-2" : "IN",
+ "ap-southeast-1" : "SG",
+ "ap-southeast-2" : "AU",
+ "ap-southeast-3" : "MY",
+ "ap-southeast-4" : "AU",
+ "ap-southeast-5" : "NZ", # Auckland, NZ
+ "ap-southeast-6" : "AP", # XXX: Precise location not documented anywhere
+
+ # Canada
+ "ca-central-1" : "CA",
+ "ca-west-1" : "CA",
+
+ # Europe
+ "eu-central-1" : "DE",
+ "eu-central-2" : "CH",
+ "eu-north-1" : "SE",
+ "eu-west-1" : "IE",
+ "eu-west-2" : "GB",
+ "eu-west-3" : "FR",
+ "eu-south-1" : "IT",
+ "eu-south-2" : "ES",
+
+ # Middle East
+ "me-central-1" : "AE",
+ "me-south-1" : "BH",
+
+ # South America
+ "sa-east-1" : "BR",
+
+ # Undocumented, likely located in Berlin rather than Frankfurt
+ "eusc-de-east-1" : "DE",
+ }
+
+ # Fetch all countries that we know of
+ countries = self.fetch_countries()
+
+ for snetwork in aws_ip_dump["prefixes"] + aws_ip_dump["ipv6_prefixes"]:
+ try:
+ network = ipaddress.ip_network(snetwork.get("ip_prefix") or snetwork.get("ipv6_prefix"), strict=False)
+ except ValueError:
+ log.warning("Unable to parse line: %s" % snetwork)
+ continue
- # Sanitize parsed networks...
- if not self._check_parsed_network(network):
- continue
+ # Sanitize parsed networks...
+ if not self._check_parsed_network(network):
+ continue
- # Determine region of this network...
- region = snetwork["region"]
- cc = None
- is_anycast = False
-
- # Any region name starting with "us-" will get "US" country code assigned straight away...
- if region.startswith("us-"):
- cc = "US"
- elif region.startswith("cn-"):
- # ... same goes for China ...
- cc = "CN"
- elif region == "GLOBAL":
- # ... funny region name for anycast-like networks ...
- is_anycast = True
- elif region in aws_region_country_map:
- # ... assign looked up country code otherwise ...
- cc = aws_region_country_map[region]
- else:
- # ... and bail out if we are missing something here
- log.warning("Unable to determine country code for line: %s" % snetwork)
- continue
+ # Determine region of this network...
+ region = snetwork["region"]
+ cc = None
+ is_anycast = False
+
+ # Any region name starting with "us-" will get "US" country code assigned straight away...
+ if region.startswith("us-"):
+ cc = "US"
+ elif region.startswith("cn-"):
+ # ... same goes for China ...
+ cc = "CN"
+ elif region == "GLOBAL":
+ # ... funny region name for anycast-like networks ...
+ is_anycast = True
+ elif region in aws_region_country_map:
+ # ... assign looked up country code otherwise ...
+ cc = aws_region_country_map[region]
+ else:
+ # ... and bail out if we are missing something here
+ log.warning("Unable to determine country code for line: %s" % snetwork)
+ continue
- # Skip networks with unknown country codes
- if not is_anycast and validcountries and cc not in validcountries:
- log.warning("Skipping Amazon AWS network with bogus country '%s': %s" % \
- (cc, network))
- return
+ # Skip networks with unknown country codes
+ if not is_anycast and countries and cc not in countries:
+ log.warning("Skipping Amazon AWS network with bogus country '%s': %s" % \
+ (cc, network))
+ return
- # Conduct SQL statement...
- self.db.execute("""
- INSERT INTO network_overrides(
- network,
- country,
- source,
- is_anonymous_proxy,
- is_satellite_provider,
- is_anycast
- ) VALUES (%s, %s, %s, %s, %s, %s)
- ON CONFLICT (network) DO NOTHING""",
- "%s" % network,
- cc,
- "Amazon AWS IP feed",
- None,
- None,
- is_anycast,
+ # Conduct SQL statement...
+ self.db.execute("""
+ INSERT INTO
+ network_feeds
+ (
+ network,
+ source,
+ country,
+ is_anycast
)
+ VALUES
+ (
+ %s, %s, %s, %s
+ )
+ ON CONFLICT (network, source) DO NOTHING
+ """, "%s" % network, "Amazon AWS IP feed", cc, is_anycast,
+ )
-
- def _update_overrides_for_spamhaus_drop(self):
+ def _update_feed_for_spamhaus_drop(self):
downloader = location.importer.Downloader()
ip_lists = [
]
asn_lists = [
- ("SPAMHAUS-ASNDROP", "https://www.spamhaus.org/drop/asndrop.txt")
+ ("SPAMHAUS-ASNDROP", "https://www.spamhaus.org/drop/asndrop.json")
]
for name, url in ip_lists:
# Conduct a very basic sanity check to rule out CDN issues causing bogus DROP
# downloads.
if len(fcontent) > 10:
- self.db.execute("DELETE FROM network_overrides WHERE source = %s", name)
+ self.db.execute("DELETE FROM network_feeds WHERE source = %s", name)
else:
log.warning("%s (%s) returned likely bogus file, ignored" % (name, url))
continue
# Conduct SQL statement...
self.db.execute("""
- INSERT INTO network_overrides(
+ INSERT INTO
+ network_feeds
+ (
network,
source,
is_drop
- ) VALUES (%s, %s, %s)
- ON CONFLICT (network) DO UPDATE SET is_drop = True""",
- "%s" % network,
- name,
- True
+ )
+ VALUES
+ (
+ %s, %s, %s
+ )""", "%s" % network, name, True,
)
for name, url in asn_lists:
# Conduct a very basic sanity check to rule out CDN issues causing bogus DROP
# downloads.
if len(fcontent) > 10:
- self.db.execute("DELETE FROM autnum_overrides WHERE source = %s", name)
+ self.db.execute("DELETE FROM autnum_feeds WHERE source = %s", name)
else:
log.warning("%s (%s) returned likely bogus file, ignored" % (name, url))
continue
# Iterate through every line, filter comments and add remaining ASNs to
# the override table in case they are valid...
- for sline in f.readlines():
+ for sline in fcontent:
# The response is assumed to be encoded in UTF-8...
sline = sline.decode("utf-8")
- # Comments start with a semicolon...
- if sline.startswith(";"):
+ # Load every line as a JSON object and try to obtain an ASN from it...
+ try:
+ lineobj = json.loads(sline)
+ except json.decoder.JSONDecodeError:
+ log.error("Unable to parse line as a JSON object: %s" % sline)
continue
- # Throw away anything after the first space...
- sline = sline.split()[0]
+ # Skip line contiaining file metadata
+ try:
+ type = lineobj["type"]
- # ... strip the "AS" prefix from it ...
- sline = sline.strip("AS")
+ if type == "metadata":
+ continue
+ except KeyError:
+ pass
- # ... and convert it into an integer. Voila.
- asn = int(sline)
+ try:
+ asn = lineobj["asn"]
+ as_name = lineobj["asname"]
+ except KeyError:
+ log.warning("Unable to extract necessary information from line: %s" % sline)
+ continue
# Filter invalid ASNs...
if not self._check_parsed_asn(asn):
# Conduct SQL statement...
self.db.execute("""
- INSERT INTO autnum_overrides(
+ INSERT INTO
+ autnum_feeds
+ (
number,
source,
is_drop
- ) VALUES (%s, %s, %s)
- ON CONFLICT (number) DO UPDATE SET is_drop = True""",
- "%s" % asn,
- name,
- True
+ )
+ VALUES
+ (
+ %s, %s, %s
+ )""", "%s" % asn, name, True,
)
@staticmethod
# Default to None
return None
- @property
- def countries(self):
- # Fetch all valid country codes to check parsed networks aganist
- rows = self.db.query("SELECT * FROM countries ORDER BY country_code")
-
- # Return all countries
- return [row.country_code for row in rows]
-
def handle_import_countries(self, ns):
with self.db.transaction():
# Drop all data that we have
projects / people / ms / libloc.git / blobdiff