importer: Tolerate that data might exist from other RIRs

Since we are breaking the import into smaller chunks now, it might be
that some data already exists in the database. This is now being
ignored and data won't be replaced.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/Makefile.am b/Makefile.am
index 38ce961a0964fdd25c0ca311ddf567f87dc97ea1..fc24614d1033a70c9d8e82a8b267e8500e4f3b47 100644 (file)
--- a/Makefile.am
+++ b/Makefile.am
@@ -55,7 +55,8 @@ databasedir = $(localstatedir)/lib/location
 pkgconfigdir = $(libdir)/pkgconfig
 
 # Overwrite Python path
-pkgpythondir = $(pythondir)/location
+#pkgpythondir = $(pythondir)/location
+pkgpythondir = /usr/lib/python3/dist-packages/location
 
 %: %.in Makefile
        $(SED_PROCESS)

diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer.in
index cfc264d58f8d6c27dc839bec6039283054e394c0..22c7b35da82978f042d552130d590fdf8984c44e 100644 (file)
--- a/src/scripts/location-importer.in
+++ b/src/scripts/location-importer.in
@@ -457,16 +457,59 @@ class CLI(object):
                                        ORDER BY family(network)")
 
                                for family in (row.family for row in families):
-                                       smallest = self.db.get("SELECT MIN(masklen(network)) AS prefix FROM _rirdata \
-                                               WHERE family(network) = %s", family)
+                                       # Fetch the smallest mask length in our data set
+                                       smallest = self.db.get("""
+                                               SELECT
+                                                       MIN(
+                                                               masklen(network)
+                                                       ) AS prefix
+                                               FROM
+                                                       _rirdata
+                                               WHERE
+                                                       family(network) = %s""",
+                                               family,
+                                       )
 
-                                       self.db.execute("INSERT INTO networks(network, country, original_countries, source) \
-                                               SELECT network, country, original_countries, source FROM _rirdata \
-                                                       WHERE masklen(network) = %s AND family(network) = %s", smallest.prefix, family)
+                                       # Copy all networks
+                                       self.db.execute("""
+                                               INSERT INTO
+                                                       networks
+                                               (
+                                                       network,
+                                                       country,
+                                                       original_countries,
+                                                       source
+                                               )
+                                               SELECT
+                                                       network,
+                                                       country,
+                                                       original_countries,
+                                                       source
+                                               FROM
+                                                       _rirdata
+                                               WHERE
+                                                       masklen(network) = %s
+                                               AND
+                                                       family(network) = %s
+                                               ON CONFLICT DO
+                                                       NOTHING""",
+                                               smallest.prefix,
+                                               family,
+                                       )
 
                                        # ... determine any other prefixes for this network family, ...
-                                       prefixes = self.db.query("SELECT DISTINCT masklen(network) AS prefix FROM _rirdata \
-                                               WHERE family(network) = %s ORDER BY masklen(network) ASC OFFSET 1", family)
+                                       prefixes = self.db.query("""
+                                               SELECT
+                                                       DISTINCT masklen(network) AS prefix
+                                               FROM
+                                                       _rirdata
+                                               WHERE
+                                                       family(network) = %s
+                                               ORDER BY
+                                                       masklen(network) ASC
+                                               OFFSET 1""",
+                                               family,
+                                       )
 
                                        # ... and insert networks with this prefix in case they provide additional
                                        # information (i. e. subnet of a larger chunk with a different country)