}
#
-## This function is responsible for downloading the configured IDS rulesets or if no one is specified
-## all configured rulesets will be downloaded.
+## This function is responsible for downloading the ruleset for a given provider.
##
-## * At first it gathers all configured ruleset providers, initialize the downloader and sets an
-## upstream proxy if configured.
-## * After that, the given ruleset or in case all rulesets should be downloaded, it will determine wether it
-## is enabled or not.
+## * At first it initialize the downloader and sets an upstream proxy if configured.
## * The next step will be to generate the final download url, by obtaining the URL for the desired
-## ruleset, add the settings for the upstream proxy.
-## * Finally the function will grab all the rules files or tarballs from the servers.
+## ruleset and add the settings for the upstream proxy.
+## * Finally the function will grab the rule file or tarball from the server.
+## It tries to reduce the amount of download by using the "If-Modified-Since" HTTP header.
+#
+## Return codes:
+##
+## * "no url" - If no download URL could be gathered for the provider.
+## * "not modified" - In case the already stored rules file is up to date.
+## * "incomplete download" - When the remote file size differs from the downloaded file size.
+## * "$error" - The error message generated from the LWP::User Agent module.
#
sub downloadruleset ($) {
my ($provider) = @_;
- # If no provider is given default to "all".
- $provider //= 'all';
-
# The amount of download attempts before giving up and
# logging an error.
my $max_dl_attempts = 3;
- # Hash to store the providers and access id's, for which rules should be downloaded.
- my %sheduled_providers = ();
-
- # Get used provider settings.
- my %used_providers = ();
- &General::readhasharray("$providers_settings_file", \%used_providers);
-
- # Check if a ruleset has been configured.
- unless(%used_providers) {
- # Log that no ruleset has been configured and abort.
- &_log_to_syslog("No ruleset provider has been configured.");
-
- # Return "1".
- return 1;
- }
-
# Read proxysettings.
my %proxysettings=();
&General::readhash("${General::swroot}/proxy/settings", \%proxysettings);
$downloader->proxy(['http', 'https'], $proxy_url);
}
- # Loop through the hash of configured providers.
- foreach my $id ( keys %used_providers ) {
- # Skip providers which are not enabled.
- next if ($used_providers{$id}[3] ne "enabled");
-
- # Obtain the provider handle.
- my $provider_handle = $used_providers{$id}[0];
-
- # Handle update off all providers.
- if (($provider eq "all") || ($provider_handle eq "$provider")) {
- # Add provider handle and it's id to the hash of sheduled providers.
- $sheduled_providers{$provider_handle} = $id;
- }
- }
-
- # Loop through the hash of sheduled providers.
- foreach my $provider ( keys %sheduled_providers) {
- # Log download/update of the ruleset.
- &_log_to_syslog("Downloading ruleset for provider: $provider.");
-
- # Grab the download url for the provider.
- my $url = $IDS::Ruleset::Providers{$provider}{'dl_url'};
-
- # Check if the provider requires a subscription.
- if ($IDS::Ruleset::Providers{$provider}{'requires_subscription'} eq "True") {
- # Grab the previously stored access id for the provider from hash.
- my $id = $sheduled_providers{$provider};
-
- # Grab the subscription code.
- my $subscription_code = $used_providers{$id}[1];
+ # Log download/update of the ruleset.
+ &_log_to_syslog("Downloading ruleset for provider: $provider.");
- # Add the subscription code to the download url.
- $url =~ s/\<subscription_code\>/$subscription_code/g;
+ # Grab the download url for the provider.
+ my $url = $IDS::Ruleset::Providers{$provider}{'dl_url'};
- }
+ # Check if the provider requires a subscription.
+ if ($IDS::Ruleset::Providers{$provider}{'requires_subscription'} eq "True") {
+ # Grab the subscription code.
+ my $subscription_code = &get_subscription_code($provider);
- # Abort if no url could be determined for the provider.
- unless ($url) {
- # Log error and abort.
- &_log_to_syslog("Unable to gather a download URL for the selected ruleset provider.");
- return 1;
- }
+ # Add the subscription code to the download url.
+ $url =~ s/\<subscription_code\>/$subscription_code/g;
- # Pass the requested URL to the downloader.
- my $request = HTTP::Request->new(GET => $url);
+ }
- # Generate temporary file name, located in "/var/tmp" and with a suffix of ".tmp".
- # The downloaded file will be stored there until some sanity checks are performed.
- my $tmp = File::Temp->new( SUFFIX => ".tmp", DIR => "/var/tmp/", UNLINK => 0 );
- my $tmpfile = $tmp->filename();
+ # Abort if no url could be determined for the provider.
+ unless ($url) {
+ # Log error and abort.
+ &_log_to_syslog("Unable to gather a download URL for the selected ruleset provider.");
+ return "no url";
+ }
- # Call function to get the final path and filename for the downloaded file.
- my $dl_rulesfile = &_get_dl_rulesfile($provider);
+ # Pass the requested URL to the downloader.
+ my $request = HTTP::Request->new(GET => $url);
- # Check if the rulesfile already exits, because it has been downloaded in the past.
- #
- # In this case we are requesting the server if the remote file has been changed or not.
- # This will be done by sending the modification time in a special HTTP header.
- if (-f $dl_rulesfile) {
- # Call stat on the file.
- my $stat = stat($dl_rulesfile);
+ # Generate temporary file name, located in "/var/tmp" and with a suffix of ".tmp".
+ # The downloaded file will be stored there until some sanity checks are performed.
+ my $tmp = File::Temp->new( SUFFIX => ".tmp", DIR => "/var/tmp/", UNLINK => 0 );
+ my $tmpfile = $tmp->filename();
- # Omit the mtime of the existing file.
- my $mtime = $stat->mtime;
+ # Call function to get the final path and filename for the downloaded file.
+ my $dl_rulesfile = &_get_dl_rulesfile($provider);
- # Convert the timestamp into right format.
- my $http_date = time2str($mtime);
+ # Check if the rulesfile already exits, because it has been downloaded in the past.
+ #
+ # In this case we are requesting the server if the remote file has been changed or not.
+ # This will be done by sending the modification time in a special HTTP header.
+ if (-f $dl_rulesfile) {
+ # Call stat on the file.
+ my $stat = stat($dl_rulesfile);
- # Add the If-Modified-Since header to the request to ask the server if the
- # file has been modified.
- $request->header( 'If-Modified-Since' => "$http_date" );
- }
+ # Omit the mtime of the existing file.
+ my $mtime = $stat->mtime;
- my $dl_attempt = 1;
- my $response;
+ # Convert the timestamp into right format.
+ my $http_date = time2str($mtime);
- # Download and retry on failure.
- while ($dl_attempt <= $max_dl_attempts) {
- # Perform the request and save the output into the tmpfile.
- $response = $downloader->request($request, $tmpfile);
+ # Add the If-Modified-Since header to the request to ask the server if the
+ # file has been modified.
+ $request->header( 'If-Modified-Since' => "$http_date" );
+ }
- # Check if the download was successfull.
- if($response->is_success) {
- # Break loop.
- last;
+ my $dl_attempt = 1;
+ my $response;
- # Check if the server responds with 304 (Not Modified).
- } elsif ($response->code == 304) {
- # Log to syslog.
- &_log_to_syslog("Ruleset is up-to-date, no update required.");
+ # Download and retry on failure.
+ while ($dl_attempt <= $max_dl_attempts) {
+ # Perform the request and save the output into the tmpfile.
+ $response = $downloader->request($request, $tmpfile);
- # Nothing to do, the ruleset is up-to-date.
- return;
+ # Check if the download was successfull.
+ if($response->is_success) {
+ # Break loop.
+ last;
- # Check if we ran out of download re-tries.
- } elsif ($dl_attempt eq $max_dl_attempts) {
- # Obtain error.
- my $error = $response->content;
+ # Check if the server responds with 304 (Not Modified).
+ } elsif ($response->code == 304) {
+ # Log to syslog.
+ &_log_to_syslog("Ruleset is up-to-date, no update required.");
- # Log error message.
- &_log_to_syslog("Unable to download the ruleset. \($error\)");
+ # Return "not modified".
+ return "not modified";
- # Return "1" - false.
- return 1;
- }
+ # Check if we ran out of download re-tries.
+ } elsif ($dl_attempt eq $max_dl_attempts) {
+ # Obtain error.
+ my $error = $response->content;
- # Remove temporary file, if one exists.
- unlink("$tmpfile") if (-e "$tmpfile");
+ # Log error message.
+ &_log_to_syslog("Unable to download the ruleset. \($error\)");
- # Increase download attempt counter.
- $dl_attempt++;
+ # Return the error message from response..
+ return "$error";
}
- # Obtain the connection headers.
- my $headers = $response->headers;
+ # Remove temporary file, if one exists.
+ unlink("$tmpfile") if (-e "$tmpfile");
- # Get the timestamp from header, when the file has been modified the
- # last time.
- my $last_modified = $headers->last_modified;
+ # Increase download attempt counter.
+ $dl_attempt++;
+ }
- # Get the remote size of the downloaded file.
- my $remote_filesize = $headers->content_length;
+ # Obtain the connection headers.
+ my $headers = $response->headers;
- # Perform stat on the tmpfile.
- my $stat = stat($tmpfile);
+ # Get the timestamp from header, when the file has been modified the
+ # last time.
+ my $last_modified = $headers->last_modified;
- # Grab the local filesize of the downloaded tarball.
- my $local_filesize = $stat->size;
+ # Get the remote size of the downloaded file.
+ my $remote_filesize = $headers->content_length;
- # Check if both file sizes match.
- if (($remote_filesize) && ($remote_filesize ne $local_filesize)) {
- # Log error message.
- &_log_to_syslog("Unable to completely download the ruleset. ");
- &_log_to_syslog("Only got $local_filesize Bytes instead of $remote_filesize Bytes. ");
+ # Perform stat on the tmpfile.
+ my $stat = stat($tmpfile);
- # Delete temporary file.
- unlink("$tmpfile");
+ # Grab the local filesize of the downloaded tarball.
+ my $local_filesize = $stat->size;
- # Return "1" - false.
- return 1;
- }
+ # Check if both file sizes match.
+ if (($remote_filesize) && ($remote_filesize ne $local_filesize)) {
+ # Log error message.
+ &_log_to_syslog("Unable to completely download the ruleset. ");
+ &_log_to_syslog("Only got $local_filesize Bytes instead of $remote_filesize Bytes. ");
- # Check if a file name could be obtained.
- unless ($dl_rulesfile) {
- # Log error message.
- &_log_to_syslog("Unable to store the downloaded rules file. ");
+ # Delete temporary file.
+ unlink("$tmpfile");
- # Delete downloaded temporary file.
- unlink("$tmpfile");
+ # Return "1" - false.
+ return "incomplete download";
+ }
- # Return "1" - false.
- return 1;
- }
+ # Check if a file name could be obtained.
+ unless ($dl_rulesfile) {
+ # Log error message.
+ &_log_to_syslog("Unable to store the downloaded rules file. ");
- # Overwrite the may existing rulefile or tarball with the downloaded one.
- move("$tmpfile", "$dl_rulesfile");
+ # Delete downloaded temporary file.
+ unlink("$tmpfile");
- # Check if we got a last-modified value from the server.
- if ($last_modified) {
- # Assign the last-modified timestamp as mtime to the
- # rules file.
- utime(time(), "$last_modified", "$dl_rulesfile");
- }
+ # Return "1" - false.
+ return 1;
+ }
- # Delete temporary file.
- unlink("$tmpfile");
+ # Overwrite the may existing rulefile or tarball with the downloaded one.
+ move("$tmpfile", "$dl_rulesfile");
- # Set correct ownership for the tarball.
- set_ownership("$dl_rulesfile");
+ # Check if we got a last-modified value from the server.
+ if ($last_modified) {
+ # Assign the last-modified timestamp as mtime to the
+ # rules file.
+ utime(time(), "$last_modified", "$dl_rulesfile");
}
+ # Delete temporary file.
+ unlink("$tmpfile");
+
+ # Set correct ownership for the tarball.
+ set_ownership("$dl_rulesfile");
+
# If we got here, everything worked fine. Return nothing.
return;
}
close(FILE);
}
+#
+## Function to get the subscription code of a configured provider.
+#
+sub get_subscription_code($) {
+ my ($provider) = @_;
+
+ my %configured_providers = ();
+
+ # Read-in providers settings file.
+ &General::readhasharray($providers_settings_file, \%configured_providers);
+
+ # Loop through the hash of configured providers.
+ foreach my $id (keys %configured_providers) {
+ # Assign nice human-readable values to the data fields.
+ my $provider_handle = $configured_providers{$id}[0];
+ my $subscription_code = $configured_providers{$id}[1];
+
+ # Check if the current processed provider is the requested one.
+ if ($provider_handle eq $provider) {
+ # Return the obtained subscription code.
+ return $subscription_code;
+ }
+ }
+
+ # No subscription code found - return nothing.
+ return;
+}
+
#
## Function to get the ruleset date for a given provider.
##