From: Michael Tremer <michael.tremer@ipfire.org>
Date: Mon, 4 Mar 2024 12:06:33 +0000 (+0000)
Subject: importer: No longer import Geofeeds concurrently
X-Git-Tag: 0.9.18~108
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=430c51c22ce28c82f03cecb4f2b4221cf70a4946;p=location%2Flibloc.git

importer: No longer import Geofeeds concurrently

This is kind of a problem since many servers have gone away, respond
very slowly, etc. But since we are moving towards psycopg3, we cannot do
this the old way any more.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/scripts/location-importer.in b/src/scripts/location-importer.in
index 7003f4f..6571f39 100644
--- a/src/scripts/location-importer.in
+++ b/src/scripts/location-importer.in
@@ -18,7 +18,6 @@
 ###############################################################################
 
 import argparse
-import concurrent.futures
 import csv
 import functools
 import http.client
@@ -1600,6 +1599,9 @@ class CLI(object):
 				yield line
 
 	def handle_update_geofeeds(self, ns):
+		# Create a downloader
+		downloader = location.importer.Downloader()
+
 		# Sync geofeeds
 		with self.db.transaction():
 			# Delete all geofeeds which are no longer linked
@@ -1647,18 +1649,10 @@ class CLI(object):
 				id
 		""")
 
-		# Create a downloader
-		downloader = location.importer.Downloader()
-
-		# Pass the downloader to the fetch_geofeed function
-		fetch_geofeed = functools.partial(self._fetch_geofeed, downloader)
-
-		with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
-			results = executor.map(fetch_geofeed, geofeeds)
-
-			# Fetch all results to raise any exceptions
-			for result in results:
-				pass
+		# Update all geofeeds
+		for geofeed in geofeeds:
+			with self.db.transaction():
+				self._fetch_geofeed(downloader, geofeed)
 
 		# Delete data from any feeds that did not update in the last two weeks
 		with self.db.transaction():