From: Michael Tremer <michael.tremer@ipfire.org>
Date: Sat, 2 Mar 2024 09:23:30 +0000 (+0000)
Subject: importer: Create an extra table for feeds
X-Git-Tag: 0.9.18~141
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7808c8b7daaed7da152424b527b3fb39e482792d;p=location%2Flibloc.git

importer: Create an extra table for feeds

We are currently using the overrides table to import any third-party
feeds. That creates the problem that those tables can only hold one row
for each AS which we cannot easily merge without losing the source of
the information.

Therefore this patch creates a new table which allow us to store this
information per feed and we will use that information if there is no
overrides information.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer.in
index 6a99cf1..3a0d8d4 100644
--- a/src/scripts/location-importer.in
+++ b/src/scripts/location-importer.in
@@ -220,6 +220,20 @@ class CLI(object):
 				CREATE INDEX IF NOT EXISTS network_geofeeds_url
 					ON network_geofeeds(url);
 
+				-- feeds
+				CREATE TABLE IF NOT EXISTS autnum_feeds(
+					number bigint NOT NULL,
+					source text NOT NULL,
+					name text,
+					country text,
+					is_anonymous_proxy boolean,
+					is_satellite_provider boolean,
+					is_anycast boolean,
+					is_drop boolean
+				);
+				CREATE UNIQUE INDEX IF NOT EXISTS autnum_feeds_unique
+					ON autnum_feeds(number, source);
+
 				-- overrides
 				CREATE TABLE IF NOT EXISTS autnum_overrides(
 					number bigint NOT NULL,
@@ -350,6 +364,11 @@ class CLI(object):
 						SELECT country FROM autnum_overrides overrides
 							WHERE networks.autnum = overrides.number
 					),
+					(
+						SELECT country FROM autnum_feeds feeds
+							WHERE networks.autnum = feeds.number
+							ORDER BY source LIMIT 1
+					),
 					(
 						SELECT
 							geofeed_networks.country AS country
@@ -384,6 +403,11 @@ class CLI(object):
 							ORDER BY masklen(overrides.network) DESC
 							LIMIT 1
 					),
+					(
+						SELECT is_anonymous_proxy FROM autnum_feeds feeds
+							WHERE networks.autnum = feeds.number
+							ORDER BY source LIMIT 1
+					),
 					(
 						SELECT is_anonymous_proxy FROM autnum_overrides overrides
 							WHERE networks.autnum = overrides.number
@@ -397,6 +421,11 @@ class CLI(object):
 							ORDER BY masklen(overrides.network) DESC
 							LIMIT 1
 					),
+					(
+						SELECT is_satellite_provider FROM autnum_feeds feeds
+							WHERE networks.autnum = feeds.number
+							ORDER BY source LIMIT 1
+					),
 					(
 						SELECT is_satellite_provider FROM autnum_overrides overrides
 							WHERE networks.autnum = overrides.number
@@ -410,6 +439,11 @@ class CLI(object):
 							ORDER BY masklen(overrides.network) DESC
 							LIMIT 1
 					),
+					(
+						SELECT is_anycast FROM autnum_feeds feeds
+							WHERE networks.autnum = feeds.number
+							ORDER BY source LIMIT 1
+					),
 					(
 						SELECT is_anycast FROM autnum_overrides overrides
 							WHERE networks.autnum = overrides.number
@@ -423,6 +457,11 @@ class CLI(object):
 							ORDER BY masklen(overrides.network) DESC
 							LIMIT 1
 					),
+					(
+						SELECT is_drop FROM autnum_feeds feeds
+							WHERE networks.autnum = feeds.number
+							ORDER BY source LIMIT 1
+					),
 					(
 						SELECT is_drop FROM autnum_overrides overrides
 							WHERE networks.autnum = overrides.number