python: Drop importing autnums from extended sources

There are 1:N mappings in there and we lack any additional context to
resolve them.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/python/importer.py b/src/python/importer.py
index 1321e0fb88520dcc758749210681be307a5ef1bf..a4a11f51ff4c4461ad442833a246a96dab1f2f5a 100644 (file)
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -38,7 +38,8 @@ WHOIS_SOURCES = (
        "https://ftp.apnic.net/apnic/whois/apnic.db.organisation.gz",
 
        # American Registry for Internet Numbers
-       "https://ftp.arin.net/pub/rr/arin.db",
+       # XXX there is nothing useful for us in here
+       #"https://ftp.arin.net/pub/rr/arin.db",
 
        # Latin America and Caribbean Network Information Centre
        # XXX ???

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index 835408a7153df4941595a18d7ba16503bd3e1a40..26e7ee171eee227d69433efa4d6ab6a31ec89475 100644 (file)
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -153,31 +153,11 @@ class CLI(object):
                # Download all extended sources
                for source in location.importer.EXTENDED_SOURCES:
                        with self.db.transaction():
-                               # Create some temporary tables to store parsed data
-                               self.db.execute("""
-                                       CREATE TEMPORARY TABLE _autnums(number integer, organization text)
-                                               ON COMMIT DROP;
-                                       CREATE INDEX _autnums_organization ON _autnums(organization);
-
-                                       CREATE TEMPORARY TABLE _inetnums(network inet, country text, organization text)
-                                               ON COMMIT DROP;
-                                       CREATE INDEX _inetnums_organization ON _inetnums(organization);
-                               """)
-
                                # Download data
                                with downloader.request(source) as f:
                                        for line in f:
                                                self._parse_line(line)
 
-                               # Store information in networks table
-                               self.db.execute("""
-                                       INSERT INTO networks(network, autnum, country)
-                                               SELECT _inetnums.network, _autnums.number, _inetnums.country FROM _inetnums
-                                                       LEFT JOIN _autnums ON _inetnums.organization = _autnums.organization
-                                                       ORDER BY _autnums.number
-                                       ON CONFLICT (network) DO NOTHING;
-                               """)
-
        def _parse_block(self, block):
                # Get first line to find out what type of block this is
                line = block[0]
@@ -256,13 +236,6 @@ class CLI(object):
                if type in ("ipv6", "ipv4"):
                        return self._parse_ip_line(country_code, type, line)
 
-               elif type == "asn":
-                       return self._parse_asn_line(country_code, line)
-
-               else:
-                       log.warning("Unknown line type: %s" % type)
-                       return
-
        def _parse_ip_line(self, country, type, line):
                try:
                        address, prefix, date, status, organization = line.split("|")
@@ -297,30 +270,12 @@ class CLI(object):
                        log.warning("Invalid IP address: %s" % address)
                        return
 
-               self.db.execute("INSERT INTO _inetnums(network, country, organization) \
-                       VALUES(%s, %s, %s)", "%s" % network, country, organization,
+               self.db.execute("INSERT INTO networks(network, country) \
+                       VALUES(%s, %s) ON CONFLICT (network) DO \
+                       UPDATE SET country = excluded.country",
+                       "%s" % network, country,
                )
 
-       def _parse_asn_line(self, country, line):
-               try:
-                       asn, dunno, date, status, org_id = line.split("|")
-               except ValueError:
-                       org_id = None
-
-                       # Try parsing the line without org_id
-                       try:
-                               asn, dunno, date, status = line.split("|")
-                       except ValueError:
-                               log.warning("Could not parse line: %s" % line)
-                               return
-
-               # Skip anything that isn't properly assigned
-               if not status in ("assigned", "allocated"):
-                       return
-
-               self.db.execute("INSERT INTO _autnums(number, organization) \
-                       VALUES(%s, %s)", asn, org_id)
-
 
 def split_line(line):
        key, colon, val = line.partition(":")