From: Stefan Schantl <stefan.schantl@ipfire.org>
Date: Fri, 14 Apr 2023 15:12:54 +0000 (+0200)
Subject: ipblocklist-functions.pl: Use download function from
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=03e83bcbc7d12ee3d6bb9c7310ee14e57f385e51;p=people%2Fstevee%2Fipfire-2.x.git

ipblocklist-functions.pl: Use download function from
general-functions.pl

Signed-off-by: Stefan Schantl <stefan.schantl@ipfire.org>
---

diff --git a/config/cfgroot/ipblocklist-functions.pl b/config/cfgroot/ipblocklist-functions.pl
index 8a65577ca1..b293f7098e 100644
--- a/config/cfgroot/ipblocklist-functions.pl
+++ b/config/cfgroot/ipblocklist-functions.pl
@@ -32,15 +32,21 @@ our $settings_dir = "/var/ipfire/ipblocklist";
 # Main settings file.
 our $settings_file = "$settings_dir/settings";
 
-# The file which keeps the time, when a blocklist last has been modified.
-our $modified_file = "$settings_dir/modified";
-
 # Location where the blocklists in ipset compatible format are stored.
 our $blocklist_dir = "/var/lib/ipblocklist";
 
 # File extension of the blocklist files.
 my $blocklist_file_extension = ".conf";
 
+# File to store Etags.
+our $etags_file = "$settings_dir/etags";
+
+# File to store the modified timestamps.
+our $modified_file = "$settings_dir/modified";
+
+# File extension of the cached blocklist files.
+my $cached_blocklist_file_extension = ".txt";
+
 # Hash which calls the correct parser functions.
 my %parsers = (
 	'ip-or-net-list' => \&parse_ip_or_net_list,
@@ -99,105 +105,34 @@ sub get_ipset_db_file($) {
 sub download_and_create_blocklist($) {
 	my ($list) = @_;
 
+	my %settings = (
+		"MAXSIZE" => 10_485_760,
+	);
+
 	# Check if the given blockist is known and data available.
 	unless($IPblocklist::List::sources{$list}) {
 		# No valid data for this blocklist - exit and return "1".
 		return 1;
 	}
 
-	# The allowed maximum download size in bytes.
-	my $max_dl_bytes = 10_485_760;
-
-	# The amount of download attempts before giving up and
-	# logging an error.
-	my $max_dl_attempts = 5;
-
-	# Read proxysettings.
-	my %proxysettings=();
-	&General::readhash("${General::swroot}/proxy/settings", \%proxysettings);
-
-	# Load required perl module to handle the download.
-	use LWP::UserAgent;
-
-	# Create a user agent for downloading the blacklist
-	# Limit the download size for safety
-	my $ua = LWP::UserAgent->new (
-		ssl_opts => {
-			SSL_ca_file     => '/etc/ssl/cert.pem',
-			verify_hostname => 1,
-		},
-
-		max_size => $max_dl_bytes,
-	);
-
-	# Set timeout to 10 seconds.
-	$ua->timeout(10);
-
-	# Check if an upstream proxy is configured.
-	if ($proxysettings{'UPSTREAM_PROXY'}) {
-		my $proxy_url;
-
-		$proxy_url = "http://";
-
-		# Check if the proxy requires authentication.
-		if (($proxysettings{'UPSTREAM_USER'}) && ($proxysettings{'UPSTREAM_PASSWORD'})) {
-			$proxy_url .= "$proxysettings{'UPSTREAM_USER'}\:$proxysettings{'UPSTREAM_PASSWORD'}\@";
-		}
-
-		# Add proxy server address and port.
-		$proxy_url .= $proxysettings{'UPSTREAM_PROXY'};
-
-		# Setup proxy settings.
-		$ua->proxy(['http', 'https'], $proxy_url);
-	}
-
-	# Gather the details, when a list got modified last time.
-	my %modified = ();
-
-	# Read-in data if the file exists.
-	&General::readhash($modified_file, \%modified ) if (-e $modified_file);
+	# Grab the URL to get the blocklist.
+	my $url = $IPblocklist::List::sources{$list}{'url'};
 
-	# Get the last modified time for this list.
-	my $last_modified = gmtime($modified{$list} || 0);
+	# Add the URL to the settings hash.
+	$settings{'URL'} = $url;
 
-	my $dl_attempt = 1;
-	my $response;
+	# Etag settings.
+	$settings{'ETAGSFILE'} = $etags_file;
+	$settings{'ETAGPREFIX'} = $list;
 
-	# Download and rety on failure loop.
-	while ($dl_attempt <= $max_dl_attempts) {
-		# Try to determine if there is a newer blocklist since last time and grab it.
-		$response = $ua->get($IPblocklist::List::sources{$list}{'url'}, 'If-Modified-Since' => $last_modified );
+	# Download the blocklist.
+	my $response = &General::downloader(%settings);
 
-		# Check if the download attempt was successfull.
-		if ($response->is_success) {
-			# We successfully grabbed the list - no more retries needed, break the loop.
-			# Further process the script code.
-			last;
-
-		# Exit, if the server responds with "Not modified (304).
-		} elsif ($response->code == 304) {
-			# Exit and return "not modified".
-			return "not_modified";
-
-		# Exit and log an erro
-		} elsif ($dl_attempt eq $max_dl_attempts) {
-			# Exit and return "dl_error".
-			return "dl_error";
-		}
-
-		# Increase download attempt counter.
-		$dl_attempt++;
-	}
-
-	# Update the timestamp for the new or modified list.
-	if($response->last_modified) {
-		$modified{$list} = $response->last_modified;
-	} else {
-		$modified{$list} = time();
-	}
+	# Abort if the response is empty
+	return "empty_list" unless ($response);
 
-	# Write-back the modified timestamps.
-	&General::writehash($modified_file, \%modified);
+	# Return return codes from downloader.
+	return "not_modified" if ($response eq "not modified");
 
 	# Parse and loop through the downloaded list.
 	my @blocklist = ();
@@ -206,7 +141,7 @@ sub download_and_create_blocklist($) {
 	my $parser = $parsers{$IPblocklist::List::sources{$list}{'parser'}};
 
 	# Loop through the grabbed raw list.
-	foreach my $line (split /[\r\n]+/, $response->content) {
+	foreach my $line (split /[\r\n]+/, $response-) {
 		# Remove newlines.
 		chomp $line;