importer: Import raw sources for inetnum's again

The extended feeds do not have enough detailed information
for us, so that we need to import inetnums from RIRs where
possible. Filtering private networks is necessary as RIR data
may contain 0.0.0.0/0 or similar entries for administrative
purposes or due to misfilings.

Special thanks goes to Michael for spending numerous hours
on this, setting up a testing environment and providing helpful
advice while debugging.

Partially fixes: #12458

Cc: Michael Tremer <michael.tremer@ipfire.org>
Signed-off-by: Peter Müller <peter.mueller@ipfire.org>
Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/python/importer.py b/src/python/importer.py
index de20f37857f9b09af8f914d51adcf0b352246472..f19db4bbd28f68e95485aad1ad627f9e7091367f 100644 (file)
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
        "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
 
        # Asia Pacific Network Information Centre
-       #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-       #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+       "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+       "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
        #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
        #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
        "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
        # XXX ???
 
        # Réseaux IP Européens
-       #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-       #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+       "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+       "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
        #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
        #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
        "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
 
 EXTENDED_SOURCES = (
        # African Network Information Centre
-       "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+       #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
 
        # Asia Pacific Network Information Centre
-       "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+       #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
 
        # American Registry for Internet Numbers
        "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
        "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
 
        # Réseaux IP Européens
-       "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+       #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
 )
 
 class Downloader(object):

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index 77952f2eef2a23e86080b9a2874a83d9e9fdbd68..e3a07a07c65eddb7a64f28b49b0ed6a8914c28b5 100644 (file)
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -393,6 +393,10 @@ class CLI(object):
                if line.startswith("aut-num:"):
                        return self._parse_autnum_block(block)
 
+               # inetnum
+               if line.startswith("inet6num:") or line.startswith("inetnum:"):
+                       return self._parse_inetnum_block(block)
+
                # organisation
                elif line.startswith("organisation:"):
                        return self._parse_org_block(block)
@@ -422,6 +426,65 @@ class CLI(object):
                        autnum.get("asn"), autnum.get("org"),
                )
 
+       def _parse_inetnum_block(self, block):
+               logging.debug("Parsing inetnum block:")
+
+               inetnum = {}
+               for line in block:
+                       logging.debug(line)
+
+                       # Split line
+                       key, val = split_line(line)
+
+                       if key == "inetnum":
+                               start_address, delim, end_address = val.partition("-")
+
+                               # Strip any excess space
+                               start_address, end_address = start_address.rstrip(), end_address.strip()
+
+                               # Convert to IP address
+                               try:
+                                       start_address = ipaddress.ip_address(start_address)
+                                       end_address   = ipaddress.ip_address(end_address)
+                               except ValueError:
+                                       logging.warning("Could not parse line: %s" % line)
+                                       return
+
+                               # Set prefix to default
+                               prefix = 32
+
+                               # Count number of addresses in this subnet
+                               num_addresses = int(end_address) - int(start_address)
+                               if num_addresses:
+                                       prefix -= math.log(num_addresses, 2)
+
+                               inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
+
+                       elif key == "inet6num":
+                               inetnum[key] = val
+
+                       elif key == "country":
+                               if val == "UNITED STATES":
+                                       val = "US"
+
+                               inetnum[key] = val.upper()
+
+               # Skip empty objects
+               if not inetnum:
+                       return
+
+               network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
+               # Bail out in case we have processed a non-public IP network
+               if network.is_private:
+                       logging.warning("Skipping non-globally routable network: %s" % network)
+                       return
+
+               self.db.execute("INSERT INTO networks(network, country) \
+                       VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+                       "%s" % network, inetnum.get("country"),
+               )
+
        def _parse_org_block(self, block):
                org = {}
                for line in block: