From 961106cf9228cf8a21f42489642e3c5668f43ca0 Mon Sep 17 00:00:00 2001
From: Stefan Schantl <stefan.schantl@ipfire.org>
Date: Fri, 14 Apr 2023 17:31:56 +0200
Subject: [PATCH] ipblocklist-functions.pl: Download and store blocklist only
 as plain file in cache directory

Do not longer convert into ipset compatible format - only parse and
store the file.

Signed-off-by: Stefan Schantl <stefan.schantl@ipfire.org>
---
 config/cfgroot/ipblocklist-functions.pl | 79 +++++++------------------
 1 file changed, 22 insertions(+), 57 deletions(-)

diff --git a/config/cfgroot/ipblocklist-functions.pl b/config/cfgroot/ipblocklist-functions.pl
index 1163d087b7..a53018f11d 100644
--- a/config/cfgroot/ipblocklist-functions.pl
+++ b/config/cfgroot/ipblocklist-functions.pl
@@ -83,14 +83,11 @@ sub get_cached_blocklist_file ($) {
 }
 
 #
-## The main download_and_create blocklist function.
+## The main download function.
 ##
-## Uses LWP to download a given blocklist. The If-Modified-Since header is
-## specified in the request so that only updated lists are downloaded (providing
-## that the server supports this functionality).
+## Uses the general downloader function to grab a given blocklist.
 ##
-## Once downloaded the list gets parsed, converted and stored in an ipset compatible
-## format.
+## Parses the response an stores the result as cached blocklist file.
 ##
 ## Parameters:
 ##   list      The name of the blocklist
@@ -102,7 +99,7 @@ sub get_cached_blocklist_file ($) {
 ##   empty_list - The downloaded blocklist is empty, or the parser was not able to parse
 ##                it correctly.
 #
-sub download_and_create_blocklist($) {
+sub download_blocklist ($) {
 	my ($list) = @_;
 
 	my %settings = (
@@ -134,14 +131,24 @@ sub download_and_create_blocklist($) {
 	# Return return codes from downloader.
 	return "not_modified" if ($response eq "not modified");
 
-	# Parse and loop through the downloaded list.
-	my @blocklist = ();
+	# Convert the response into an array to allow
+	# processing the items.
+	my @blocklist = split(/[\r\n]/, $response);
 
 	# Get the responsible parser for the current list.
 	my $parser = $parsers{$IPblocklist::List::sources{$list}{'parser'}};
 
-	# Loop through the grabbed raw list.
-	foreach my $line (split /[\r\n]+/, $response-) {
+	# Return if no parser could be omited.
+	return "no parser" unless ($parser);
+
+	# Get the name of the cached blocklist.
+	my $file = &get_cached_blocklist_file($list);
+
+	# Open the cached blocklist file for writing.
+	open (FILE, ">", $file) or die "Could not write to $file. $!\n";
+
+	# Loop through the response.
+	foreach my $line (@blocklist) {
 		# Remove newlines.
 		chomp $line;
 
@@ -153,60 +160,18 @@ sub download_and_create_blocklist($) {
 
 		# Check if we got a single address.
 		if ($address =~ m|/32|) {
-			# Add /32 as prefix.
+			# Remove /32 as prefix for single addresses.
 			$address =~ s|/32||;
 		}
 
-		# Push the address/network to the blocklist array.
-		push(@blocklist, $address);
-	}
-
-	# Check if the content could be parsed correctly and the blocklist
-	# contains at least one item.
-	unless(@blocklist) {
-		# No entries - exit and return "empty_list".
-		return "empty_list";
-	}
-
-	# Get amount of entries in the blocklist array.
-	my $list_entries = scalar(@blocklist);
-
-	# Optain the filename for this blocklist to save.
-	my $file = &get_ipset_db_file($list);
-
-	# Open the file for writing.
-	open(FILE, ">", "$file") or die "Could not write to $file. $!\n";
-
-	# Write file header.
-	print FILE "#Autogenerated file. Any custom changes will be overwritten!\n\n";
-
-	# Calculate the hashsize for better list performance.
-	my $hashsize = &_calculate_hashsize($list_entries);
-
-	# Simply set the limit of list elements to the double of current list elements.
-	my $maxelem = $list_entries *2;
-
-	# Add "v4" suffix to the list name.
-	$list = "$list" . "v4";
-
-	# Write line to create the set.
-	#
-	# We safely can use hash:net as type because it supports single addresses and networks.
-	print FILE "create $list hash:net family inet hashsize $hashsize maxelem $maxelem -exist\n";
-
-	# Write line to flush the set itself during loading.
-	print FILE "flush $list\n";
-
-	# Loop through the array which contains the blocklist.
-	foreach my $entry (@blocklist) {
-		# Add the entry to the list.
-		print FILE "add $list $entry\n";
+		# Write the address to the file.
+		print FILE "$address\n";
 	}
 
 	# Close the file handle.
 	close(FILE);
 
-	# Finished.
+	# Successfully finished return nothing
 	return;
 }
 
-- 
2.39.5