From: Michael Tremer <michael.tremer@ipfire.org>
Date: Wed, 9 Mar 2022 10:29:11 +0000 (+0000)
Subject: importer: Improve performance of network export query
X-Git-Tag: 0.9.12~4
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=295a7774d98aa866da19acaafb9d04689d674440;p=location%2Flibloc.git

importer: Improve performance of network export query

This patch moves the subqueries out of the large query, so that the
database will materialize them for faster lookup.

We also drop the "UNION ALL" and replace it with just "UNION" because we
do not want any duplicate networks. That will save us many iterations
later on.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/python/location-importer.in b/src/python/location-importer.in
index 3b36184..bee9186 100644
--- a/src/python/location-importer.in
+++ b/src/python/location-importer.in
@@ -254,6 +254,35 @@ class CLI(object):
 
 		# Select all known networks
 		rows = self.db.query("""
+			WITH known_networks AS (
+				SELECT network FROM announcements
+				UNION
+				SELECT network FROM networks
+				UNION
+				SELECT network FROM network_overrides
+			),
+
+			ordered_networks AS (
+				SELECT
+					known_networks.network AS network,
+					announcements.autnum AS autnum,
+					networks.country AS country,
+
+					-- Must be part of returned values for ORDER BY clause
+					masklen(announcements.network) AS sort_a,
+					masklen(networks.network) AS sort_b
+				FROM
+					known_networks
+				LEFT JOIN
+					announcements ON known_networks.network <<= announcements.network
+				LEFT JOIN
+					networks ON known_networks.network <<= networks.network
+				ORDER BY
+					known_networks.network,
+					sort_a DESC,
+					sort_b DESC
+			)
+
 			-- Return a list of those networks enriched with all
 			-- other information that we store in the database
 			SELECT
@@ -329,31 +358,8 @@ class CLI(object):
 					),
 					FALSE
 				) AS is_drop
-			FROM (
-				SELECT
-					known_networks.network AS network,
-					announcements.autnum AS autnum,
-					networks.country AS country,
-
-					-- Must be part of returned values for ORDER BY clause
-					masklen(announcements.network) AS sort_a,
-					masklen(networks.network) AS sort_b
-				FROM (
-						SELECT network FROM announcements
-					UNION ALL
-						SELECT network FROM networks
-					UNION ALL
-						SELECT network FROM network_overrides
-					) known_networks
-				LEFT JOIN
-					announcements ON known_networks.network <<= announcements.network
-				LEFT JOIN
-					networks ON known_networks.network <<= networks.network
-				ORDER BY
-					known_networks.network,
-					sort_a DESC,
-					sort_b DESC
-			) networks
+			FROM
+				ordered_networks networks
 		""")
 
 		for row in rows: