Revert "Revert "importer: Import raw sources for inetnum's again""

This reverts commit 92f6abf4e272672bb0a71cfe991261b95ebe2fef.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/python/importer.py b/src/python/importer.py
index de20f37857f9b09af8f914d51adcf0b352246472..f19db4bbd28f68e95485aad1ad627f9e7091367f 100644 (file)
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
        "https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
 
        # Asia Pacific Network Information Centre
-       #"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-       #"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+       "https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+       "https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
        #"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
        #"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
        "https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
        # XXX ???
 
        # Réseaux IP Européens
-       #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-       #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+       "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+       "https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
        #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
        #"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
        "https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
 
 EXTENDED_SOURCES = (
        # African Network Information Centre
-       "https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+       #"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
 
        # Asia Pacific Network Information Centre
-       "https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+       #"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
 
        # American Registry for Internet Numbers
        "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
        "http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
 
        # Réseaux IP Européens
-       "https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+       #"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
 )
 
 class Downloader(object):

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index 77952f2eef2a23e86080b9a2874a83d9e9fdbd68..e3a07a07c65eddb7a64f28b49b0ed6a8914c28b5 100644 (file)
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -393,6 +393,10 @@ class CLI(object):
                if line.startswith("aut-num:"):
                        return self._parse_autnum_block(block)
 
+               # inetnum
+               if line.startswith("inet6num:") or line.startswith("inetnum:"):
+                       return self._parse_inetnum_block(block)
+
                # organisation
                elif line.startswith("organisation:"):
                        return self._parse_org_block(block)
@@ -422,6 +426,65 @@ class CLI(object):
                        autnum.get("asn"), autnum.get("org"),
                )
 
+       def _parse_inetnum_block(self, block):
+               logging.debug("Parsing inetnum block:")
+
+               inetnum = {}
+               for line in block:
+                       logging.debug(line)
+
+                       # Split line
+                       key, val = split_line(line)
+
+                       if key == "inetnum":
+                               start_address, delim, end_address = val.partition("-")
+
+                               # Strip any excess space
+                               start_address, end_address = start_address.rstrip(), end_address.strip()
+
+                               # Convert to IP address
+                               try:
+                                       start_address = ipaddress.ip_address(start_address)
+                                       end_address   = ipaddress.ip_address(end_address)
+                               except ValueError:
+                                       logging.warning("Could not parse line: %s" % line)
+                                       return
+
+                               # Set prefix to default
+                               prefix = 32
+
+                               # Count number of addresses in this subnet
+                               num_addresses = int(end_address) - int(start_address)
+                               if num_addresses:
+                                       prefix -= math.log(num_addresses, 2)
+
+                               inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
+
+                       elif key == "inet6num":
+                               inetnum[key] = val
+
+                       elif key == "country":
+                               if val == "UNITED STATES":
+                                       val = "US"
+
+                               inetnum[key] = val.upper()
+
+               # Skip empty objects
+               if not inetnum:
+                       return
+
+               network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
+
+               # Bail out in case we have processed a non-public IP network
+               if network.is_private:
+                       logging.warning("Skipping non-globally routable network: %s" % network)
+                       return
+
+               self.db.execute("INSERT INTO networks(network, country) \
+                       VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
+                       "%s" % network, inetnum.get("country"),
+               )
+
        def _parse_org_block(self, block):
                org = {}
                for line in block: