From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 16 Oct 2020 12:26:43 +0000 (+0000)
Subject: Revert "Revert "Revert "importer: Import raw sources for inetnum's again"""
X-Git-Tag: 0.9.5~115
X-Git-Url: http://git.ipfire.org/?p=people%2Fms%2Flibloc.git;a=commitdiff_plain;h=44341478233115b26bb27fdb24da5b0a1eedb173

Revert "Revert "Revert "importer: Import raw sources for inetnum's again"""

This reverts commit f532841e9197ce2f40aad8c086d786b2cb783a54.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/python/importer.py b/src/python/importer.py
index f19db4b..de20f37 100644
--- a/src/python/importer.py
+++ b/src/python/importer.py
@@ -30,8 +30,8 @@ WHOIS_SOURCES = (
 	"https://ftp.afrinic.net/pub/pub/dbase/afrinic.db.gz",
 
 	# Asia Pacific Network Information Centre
-	"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
-	"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
+	#"https://ftp.apnic.net/apnic/whois/apnic.db.inet6num.gz",
+	#"https://ftp.apnic.net/apnic/whois/apnic.db.inetnum.gz",
 	#"https://ftp.apnic.net/apnic/whois/apnic.db.route6.gz",
 	#"https://ftp.apnic.net/apnic/whois/apnic.db.route.gz",
 	"https://ftp.apnic.net/apnic/whois/apnic.db.aut-num.gz",
@@ -45,8 +45,8 @@ WHOIS_SOURCES = (
 	# XXX ???
 
 	# Réseaux IP Européens
-	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
-	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
+	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inet6num.gz",
+	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.inetnum.gz",
 	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route6.gz",
 	#"https://ftp.ripe.net/ripe/dbase/split/ripe.db.route.gz",
 	"https://ftp.ripe.net/ripe/dbase/split/ripe.db.aut-num.gz",
@@ -55,10 +55,10 @@ WHOIS_SOURCES = (
 
 EXTENDED_SOURCES = (
 	# African Network Information Centre
-	#"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
+	"https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest",
 
 	# Asia Pacific Network Information Centre
-	#"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
+	"https://ftp.apnic.net/apnic/stats/apnic/delegated-apnic-extended-latest",
 
 	# American Registry for Internet Numbers
 	"https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest",
@@ -67,7 +67,7 @@ EXTENDED_SOURCES = (
 	"http://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-latest",
 
 	# Réseaux IP Européens
-	#"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
+	"https://ftp.ripe.net/pub/stats/ripencc/delegated-ripencc-extended-latest",
 )
 
 class Downloader(object):
diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index b220eaf..e87d378 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -395,10 +395,6 @@ class CLI(object):
 		if line.startswith("aut-num:"):
 			return self._parse_autnum_block(block)
 
-		# inetnum
-		if line.startswith("inet6num:") or line.startswith("inetnum:"):
-			return self._parse_inetnum_block(block)
-
 		# organisation
 		elif line.startswith("organisation:"):
 			return self._parse_org_block(block)
@@ -428,65 +424,6 @@ class CLI(object):
 			autnum.get("asn"), autnum.get("org"),
 		)
 
-	def _parse_inetnum_block(self, block):
-		logging.debug("Parsing inetnum block:")
-
-		inetnum = {}
-		for line in block:
-			logging.debug(line)
-
-			# Split line
-			key, val = split_line(line)
-
-			if key == "inetnum":
-				start_address, delim, end_address = val.partition("-")
-
-				# Strip any excess space
-				start_address, end_address = start_address.rstrip(), end_address.strip()
-
-				# Convert to IP address
-				try:
-					start_address = ipaddress.ip_address(start_address)
-					end_address   = ipaddress.ip_address(end_address)
-				except ValueError:
-					logging.warning("Could not parse line: %s" % line)
-					return
-
-				# Set prefix to default
-				prefix = 32
-
-				# Count number of addresses in this subnet
-				num_addresses = int(end_address) - int(start_address)
-				if num_addresses:
-					prefix -= math.log(num_addresses, 2)
-
-				inetnum["inetnum"] = "%s/%.0f" % (start_address, prefix)
-
-			elif key == "inet6num":
-				inetnum[key] = val
-
-			elif key == "country":
-				if val == "UNITED STATES":
-					val = "US"
-
-				inetnum[key] = val.upper()
-
-		# Skip empty objects
-		if not inetnum:
-			return
-
-		network = ipaddress.ip_network(inetnum.get("inet6num") or inetnum.get("inetnum"), strict=False)
-
-		# Bail out in case we have processed a non-public IP network
-		if network.is_private:
-			logging.warning("Skipping non-globally routable network: %s" % network)
-			return
-
-		self.db.execute("INSERT INTO networks(network, country) \
-			VALUES(%s, %s) ON CONFLICT (network) DO UPDATE SET country = excluded.country",
-			"%s" % network, inetnum.get("country"),
-		)
-
 	def _parse_org_block(self, block):
 		org = {}
 		for line in block: