From 5bfa1bb4e82b3a2eee9cd547acf28c134d772ead Mon Sep 17 00:00:00 2001
From: Michael Tremer <michael.tremer@ipfire.org>
Date: Fri, 12 Aug 2022 15:47:29 +0000
Subject: [PATCH] importer: Tolerate that data might exist from other RIRs

Since we are breaking the import into smaller chunks now, it might be
that some data already exists in the database. This is now being
ignored and data won't be replaced.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---
 Makefile.am                      |  3 +-
 src/scripts/location-importer.in | 57 ++++++++++++++++++++++++++++----
 2 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/Makefile.am b/Makefile.am
index 38ce961..fc24614 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -55,7 +55,8 @@ databasedir = $(localstatedir)/lib/location
 pkgconfigdir = $(libdir)/pkgconfig
 
 # Overwrite Python path
-pkgpythondir = $(pythondir)/location
+#pkgpythondir = $(pythondir)/location
+pkgpythondir = /usr/lib/python3/dist-packages/location
 
 %: %.in Makefile
 	$(SED_PROCESS)
diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer.in
index cfc264d..22c7b35 100644
--- a/src/scripts/location-importer.in
+++ b/src/scripts/location-importer.in
@@ -457,16 +457,59 @@ class CLI(object):
 					ORDER BY family(network)")
 
 				for family in (row.family for row in families):
-					smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata \
-						WHERE family(network) = %s", family)
+					# Fetch the smallest mask length in our data set
+					smallest = self.db.get("""
+						SELECT
+							MIN(
+								masklen(network)
+							) AS prefix
+						FROM
+							_rirdata
+						WHERE
+							family(network) = %s""",
+						family,
+					)
 
-					self.db.execute("INSERT INTO networks(network, country, original_countries, source) \
-						SELECT network, country, original_countries, source FROM _rirdata \
-							WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
+					# Copy all networks
+					self.db.execute("""
+						INSERT INTO
+							networks
+						(
+							network,
+							country,
+							original_countries,
+							source
+						)
+						SELECT
+							network,
+							country,
+							original_countries,
+							source
+						FROM
+							_rirdata
+						WHERE
+							masklen(network) = %s
+						AND
+							family(network) = %s
+						ON CONFLICT DO
+							NOTHING""",
+						smallest.prefix,
+						family,
+					)
 
 					# ... determine any other prefixes for this network family, ...
-					prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
-						WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
+					prefixes = self.db.query("""
+						SELECT
+							DISTINCT masklen(network) AS prefix
+						FROM
+							_rirdata
+						WHERE
+							family(network) = %s
+						ORDER BY
+							masklen(network) ASC
+						OFFSET 1""",
+						family,
+					)
 
 					# ... and insert networks with this prefix in case they provide additional
 					# information (i. e. subnet of a larger chunk with a different country)
-- 
2.39.5