ids-functions.pl: Change backend to use one file to load the used

rulefiles.

Suricata seems to struggle when using multiple and/or nested includes in
the same config section. This results in a only partially loaded
confguration where not all rulefiles are loaded and used.

Signed-off-by: Stefan Schantl <stefan.schantl@ipfire.org>

diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl
index ff84e1ad38bf733f04e596e0ece5bc0f3e4ead8c..cf896f340213b2a946660936726d86152c802fa0 100644 (file)
--- a/config/cfgroot/ids-functions.pl
+++ b/config/cfgroot/ids-functions.pl
@@ -63,10 +63,7 @@ use Sys::Syslog qw(:DEFAULT setlogsock);
 our $settingsdir = "${General::swroot}/suricata";
 
 # File where the main file for providers ruleset inclusion exists.
-our $suricata_used_providers_file = "$settingsdir/suricata-used-providers.yaml";
-
-# File for static ruleset inclusions.
-our $suricata_default_rulefiles_file = "$settingsdir/suricata-default-rules.yaml";
+our $suricata_used_rulesfiles_file = "$settingsdir/suricata-used-rulesfiles.yaml";
 
 # File where the addresses of the homenet are stored.
 our $homenet_file = "$settingsdir/suricata-homenet.yaml";
@@ -171,8 +168,7 @@ my %tr_app_layer_proto = (
 #
 sub check_and_create_filelayout() {
        # Check if the files exist and if not, create them.
-       unless (-f "$suricata_used_providers_file") { &create_empty_file($suricata_used_providers_file); }
-       unless (-f "$suricata_default_rulefiles_file") { &create_empty_file($suricata_default_rulefiles_file); }
+       unless (-f "$suricata_used_rulesfiles_file") { &create_empty_file($suricata_used_rulesfiles_file); }
        unless (-f "$ids_settings_file") { &create_empty_file($ids_settings_file); }
        unless (-f "$providers_settings_file") { &create_empty_file($providers_settings_file); }
        unless (-f "$whitelist_file" ) { &create_empty_file($whitelist_file); }
@@ -1446,83 +1442,19 @@ sub generate_http_ports_file() {
 }
 
 #
-## Function to generate and write the file for used rulefiles file for a given provider.
-##
-## The function requires as first argument a provider handle, and as second an array with files.
-#
-sub write_used_provider_rulefiles_file($@) {
-       my ($provider, @files) = @_;
-
-       # Get the path and file for the provider specific used rulefiles file.
-       my $used_provider_rulesfile_file = &get_used_provider_rulesfile_file($provider);
-
-       # Open file for used rulefiles.
-       open (FILE, ">", "$used_provider_rulesfile_file") or die "Could not write to $used_provider_rulesfile_file. $!\n";
-
-       # Write yaml header to the file.
-       print FILE "%YAML 1.1\n";
-       print FILE "---\n\n";
-
-       # Write header to file.
-       print FILE "#Autogenerated file. Any custom changes will be overwritten!\n";
-
-       # Loop through the array of given files.
-       foreach my $file (@files) {
-               # Check if the given filename exists and write it to the file of used rulefiles.
-               if(-f "$rulespath/$file") {
-                       print FILE " - $file\n";
-               }
-       }
-
-       # Close file after writing.
-       close(FILE);
-}
-
-#
-## Function to write the main file for provider rulesfiles inclusions.
+## Function to write the file that contains the rulefiles which are loaded by suricaa.
 ##
-## This function requires an array of provider handles.
+## This function requires an array of used provider handles.
 #
-sub write_main_used_rulefiles_file (@) {
+sub write_used_rulefiles_file (@) {
        my (@providers) = @_;
 
-       # Call function to write the static rulefiles file.
-       &_write_default_rulefiles_file();
-
-       # Open file for used rulefils inclusion.
-       open (FILE, ">", "$suricata_used_providers_file") or die "Could not write to $suricata_used_providers_file. $!\n";
-
-       # Write yaml header to the file.
-       print FILE "%YAML 1.1\n";
-       print FILE "---\n\n";
-
-       # Write header to file.
-       print FILE "#Autogenerated file. Any custom changes will be overwritten!\n";
-
-       # Loop through the list of given providers.
-       foreach my $provider (@providers) {
-               # Call function to get the providers used rulefiles file.
-               my $filename = &get_used_provider_rulesfile_file($provider);
-
-               # Check if the file exists and write it into the used rulefiles file.
-               if (-f $filename) {
-                       # Print the provider to the file.
-                       print FILE "include\: $filename\n";
-               }
-       }
-
-       # Close the filehandle after writing.
-       close(FILE);
-}
-
-sub _write_default_rulefiles_file () {
-       # Get enabled application layer protocols.
+       # Get the enabled application layer protocols.
        my @enabled_app_layer_protos = &get_suricata_enabled_app_layer_protos();
 
-       # Open file.
-       open (FILE, ">", $suricata_default_rulefiles_file) or die "Could not write to $suricata_default_rulefiles_file. $!\n";
+       # Open the file.
+       open (FILE, ">", $suricata_used_rulesfiles_file) or die "Could not write to $suricata_used_rulesfiles_file. $!\n";
 
-       # Write yaml header to the file.
        print FILE "%YAML 1.1\n";
        print FILE "---\n\n";
 
@@ -1566,17 +1498,38 @@ sub _write_default_rulefiles_file () {
                }
        }
 
+       # Loop through the array of enabled providers.
+       foreach my $provider (@providers) {
+               # Get the used rulefile for this provider.
+               my @used_rulesfiles = &get_provider_used_rulesfiles($provider);
+
+               # Check if there are
+               if(@used_rulesfiles) {
+                       # Add notice to the file.
+                       print FILE "\n#Used Rulesfiles for provider $provider.\n";
+
+                       # Loop through the array of used rulefiles.
+                       foreach my $enabled_rulesfile (@used_rulesfiles) {
+                               # Generate name and full path to the rulesfile.
+                               my $rulesfile = "$rulespath/$enabled_rulesfile";
+
+                               # Write the ruelsfile name to the file.
+                               print FILE " - $rulesfile\n";
+                       }
+               }
+       }
+
        # Close the file handle
        close(FILE);
 }
 
 #
-## Tiny function to generate the full path and name for the used_provider_rulesfile file of a given provider.
+## Tiny function to generate the full path and name for the file which stores the used rulefiles of a given provider.
 #
-sub get_used_provider_rulesfile_file ($) {
+sub get_provider_used_rulesfiles_file ($) {
        my ($provider) = @_;
 
-       my $filename = "$settingsdir/suricata\-$provider\-used\-rulefiles.yaml";
+       my $filename = "$settingsdir/$provider\-used\-rulesfiles";
 
        # Return the gernerated file.
        return $filename;
@@ -1909,44 +1862,35 @@ sub get_red_address() {
 #
 ## Function to get the used rules files of a given provider.
 #
-sub read_used_provider_rulesfiles($) {
+sub get_provider_used_rulesfiles($) {
        my ($provider) = @_;
 
+       # Hash to store the used rulefiles of the provider.
+       my %provider_rulefiles = ();
+
        # Array to store the used rulefiles.
        my @used_rulesfiles = ();
 
-       # Get the used rulesefile file for the provider.
-       my $rulesfile_file = &get_used_provider_rulesfile_file($provider);
+       # Get the filename which contains the used rulefiles for this provider.
+       my $used_rulesfiles_file = &get_provider_used_rulesfiles_file($provider);
 
-       # Check if the a used rulesfile exists for this provider.
-       if (-f $rulesfile_file) {
-               # Open the file or used rulefiles and read-in content.
-               open(FILE, $rulesfile_file) or die "Could not open $rulesfile_file. $!\n";
-
-               while (<FILE>) {
-                       # Assign the current line to a nice variable.
-                       my $line = $_;
-
-                       # Remove newlines.
-                       chomp($line);
+       # Read-in file, if it exists.
+       &General::readhash("$used_rulesfiles_file", \%provider_rulefiles) if (-f $used_rulesfiles_file);
 
-                       # Skip comments.
-                       next if ($line =~ /\#/);
+       # Loop through the hash of rulefiles which does the provider offer.
+       foreach my $rulefile (keys %provider_rulefiles) {
+               # Skip disabled rulefiles.
+               next unless($provider_rulefiles{$rulefile} eq "enabled");
 
-                       # Skip blank  lines.
-                       next if ($line =~ /^\s*$/);
+               # The General::readhash function does not allow dots as
+               # key value and limits the key "string" to the part before
+               # the dot, in case it contains one.
+               #
+               # So add the file extension for the rules file manually again.
+               $rulefile = "$rulefile.rules";
 
-                       # Gather the rulefile.
-                       if ($line =~ /.*- (.*)/) {
-                               my $rulefile = $1;
-
-                               # Add the rulefile to the array of used rulesfiles.
-                               push(@used_rulesfiles, $rulefile);
-                       }
-               }
-
-               # Close the file.
-               close(FILE);
+               # Add the enabled rulefile to the array of enabled rulefiles.
+               push(@used_rulesfiles, $rulefile);
        }
 
        # Return the array of used rulesfiles.

diff --git a/config/suricata/suricata.yaml b/config/suricata/suricata.yaml
index 6fbc7b3ee1bacbf2ab17815230205d1db0f3a551..03a7a83afc1bc965a1f9b4c03a31199f05d51e6f 100644 (file)
--- a/config/suricata/suricata.yaml
+++ b/config/suricata/suricata.yaml
@@ -47,10 +47,7 @@ vars:
 default-rule-path: /var/lib/suricata
 rule-files:
     # Include enabled ruleset files from external file.
-    include: /var/ipfire/suricata/suricata-used-providers.yaml
-
-    # Include default rules.
-    include: /var/ipfire/suricata/suricata-default-rules.yaml
+    include: /var/ipfire/suricata/suricata-used-rulesfiles.yaml
 
 classification-file: /usr/share/suricata/classification.config
 reference-config-file: /usr/share/suricata/reference.config