ipblocklist-functions.pl: Use download function from

general-functions.pl

Signed-off-by: Stefan Schantl <stefan.schantl@ipfire.org>

diff --git a/config/cfgroot/ipblocklist-functions.pl b/config/cfgroot/ipblocklist-functions.pl
index 8a65577ca1ee51b68a806a57781a3c9e81f498e5..b293f7098ec4d9d35698c1dd8ea1d152f51c287a 100644 (file)
--- a/config/cfgroot/ipblocklist-functions.pl
+++ b/config/cfgroot/ipblocklist-functions.pl
@@ -32,15 +32,21 @@ our $settings_dir = "/var/ipfire/ipblocklist";
 # Main settings file.
 our $settings_file = "$settings_dir/settings";
 
-# The file which keeps the time, when a blocklist last has been modified.
-our $modified_file = "$settings_dir/modified";
-
 # Location where the blocklists in ipset compatible format are stored.
 our $blocklist_dir = "/var/lib/ipblocklist";
 
 # File extension of the blocklist files.
 my $blocklist_file_extension = ".conf";
 
+# File to store Etags.
+our $etags_file = "$settings_dir/etags";
+
+# File to store the modified timestamps.
+our $modified_file = "$settings_dir/modified";
+
+# File extension of the cached blocklist files.
+my $cached_blocklist_file_extension = ".txt";
+
 # Hash which calls the correct parser functions.
 my %parsers = (
        'ip-or-net-list' => \&parse_ip_or_net_list,
@@ -99,105 +105,34 @@ sub get_ipset_db_file($) {
 sub download_and_create_blocklist($) {
        my ($list) = @_;
 
+       my %settings = (
+               "MAXSIZE" => 10_485_760,
+       );
+
        # Check if the given blockist is known and data available.
        unless($IPblocklist::List::sources{$list}) {
                # No valid data for this blocklist - exit and return "1".
                return 1;
        }
 
-       # The allowed maximum download size in bytes.
-       my $max_dl_bytes = 10_485_760;
-
-       # The amount of download attempts before giving up and
-       # logging an error.
-       my $max_dl_attempts = 5;
-
-       # Read proxysettings.
-       my %proxysettings=();
-       &General::readhash("${General::swroot}/proxy/settings", \%proxysettings);
-
-       # Load required perl module to handle the download.
-       use LWP::UserAgent;
-
-       # Create a user agent for downloading the blacklist
-       # Limit the download size for safety
-       my $ua = LWP::UserAgent->new (
-               ssl_opts => {
-                       SSL_ca_file     => '/etc/ssl/cert.pem',
-                       verify_hostname => 1,
-               },
-
-               max_size => $max_dl_bytes,
-       );
-
-       # Set timeout to 10 seconds.
-       $ua->timeout(10);
-
-       # Check if an upstream proxy is configured.
-       if ($proxysettings{'UPSTREAM_PROXY'}) {
-               my $proxy_url;
-
-               $proxy_url = "http://";
-
-               # Check if the proxy requires authentication.
-               if (($proxysettings{'UPSTREAM_USER'}) && ($proxysettings{'UPSTREAM_PASSWORD'})) {
-                       $proxy_url .= "$proxysettings{'UPSTREAM_USER'}\:$proxysettings{'UPSTREAM_PASSWORD'}\@";
-               }
-
-               # Add proxy server address and port.
-               $proxy_url .= $proxysettings{'UPSTREAM_PROXY'};
-
-               # Setup proxy settings.
-               $ua->proxy(['http', 'https'], $proxy_url);
-       }
-
-       # Gather the details, when a list got modified last time.
-       my %modified = ();
-
-       # Read-in data if the file exists.
-       &General::readhash($modified_file, \%modified ) if (-e $modified_file);
+       # Grab the URL to get the blocklist.
+       my $url = $IPblocklist::List::sources{$list}{'url'};
 
-       # Get the last modified time for this list.
-       my $last_modified = gmtime($modified{$list} || 0);
+       # Add the URL to the settings hash.
+       $settings{'URL'} = $url;
 
-       my $dl_attempt = 1;
-       my $response;
+       # Etag settings.
+       $settings{'ETAGSFILE'} = $etags_file;
+       $settings{'ETAGPREFIX'} = $list;
 
-       # Download and rety on failure loop.
-       while ($dl_attempt <= $max_dl_attempts) {
-               # Try to determine if there is a newer blocklist since last time and grab it.
-               $response = $ua->get($IPblocklist::List::sources{$list}{'url'}, 'If-Modified-Since' => $last_modified );
+       # Download the blocklist.
+       my $response = &General::downloader(%settings);
 
-               # Check if the download attempt was successfull.
-               if ($response->is_success) {
-                       # We successfully grabbed the list - no more retries needed, break the loop.
-                       # Further process the script code.
-                       last;
-
-               # Exit, if the server responds with "Not modified (304).
-               } elsif ($response->code == 304) {
-                       # Exit and return "not modified".
-                       return "not_modified";
-
-               # Exit and log an erro
-               } elsif ($dl_attempt eq $max_dl_attempts) {
-                       # Exit and return "dl_error".
-                       return "dl_error";
-               }
-
-               # Increase download attempt counter.
-               $dl_attempt++;
-       }
-
-       # Update the timestamp for the new or modified list.
-       if($response->last_modified) {
-               $modified{$list} = $response->last_modified;
-       } else {
-               $modified{$list} = time();
-       }
+       # Abort if the response is empty
+       return "empty_list" unless ($response);
 
-       # Write-back the modified timestamps.
-       &General::writehash($modified_file, \%modified);
+       # Return return codes from downloader.
+       return "not_modified" if ($response eq "not modified");
 
        # Parse and loop through the downloaded list.
        my @blocklist = ();
@@ -206,7 +141,7 @@ sub download_and_create_blocklist($) {
        my $parser = $parsers{$IPblocklist::List::sources{$list}{'parser'}};
 
        # Loop through the grabbed raw list.
-       foreach my $line (split /[\r\n]+/, $response->content) {
+       foreach my $line (split /[\r\n]+/, $response-) {
                # Remove newlines.
                chomp $line;