From 8d6714edc8c957214506bc483bc51edc06c94554 Mon Sep 17 00:00:00 2001 From: Stefan Schantl Date: Sat, 9 Apr 2022 13:11:18 +0200 Subject: [PATCH] ids-functions.pl: Change backend to use one file to load the used rulefiles. Suricata seems to struggle when using multiple and/or nested includes in the same config section. This results in a only partially loaded confguration where not all rulefiles are loaded and used. Signed-off-by: Stefan Schantl --- config/cfgroot/ids-functions.pl | 160 +++++++++++--------------------- config/suricata/suricata.yaml | 5 +- 2 files changed, 53 insertions(+), 112 deletions(-) diff --git a/config/cfgroot/ids-functions.pl b/config/cfgroot/ids-functions.pl index ff84e1ad38..cf896f3402 100644 --- a/config/cfgroot/ids-functions.pl +++ b/config/cfgroot/ids-functions.pl @@ -63,10 +63,7 @@ use Sys::Syslog qw(:DEFAULT setlogsock); our $settingsdir = "${General::swroot}/suricata"; # File where the main file for providers ruleset inclusion exists. -our $suricata_used_providers_file = "$settingsdir/suricata-used-providers.yaml"; - -# File for static ruleset inclusions. -our $suricata_default_rulefiles_file = "$settingsdir/suricata-default-rules.yaml"; +our $suricata_used_rulesfiles_file = "$settingsdir/suricata-used-rulesfiles.yaml"; # File where the addresses of the homenet are stored. our $homenet_file = "$settingsdir/suricata-homenet.yaml"; @@ -171,8 +168,7 @@ my %tr_app_layer_proto = ( # sub check_and_create_filelayout() { # Check if the files exist and if not, create them. - unless (-f "$suricata_used_providers_file") { &create_empty_file($suricata_used_providers_file); } - unless (-f "$suricata_default_rulefiles_file") { &create_empty_file($suricata_default_rulefiles_file); } + unless (-f "$suricata_used_rulesfiles_file") { &create_empty_file($suricata_used_rulesfiles_file); } unless (-f "$ids_settings_file") { &create_empty_file($ids_settings_file); } unless (-f "$providers_settings_file") { &create_empty_file($providers_settings_file); } unless (-f "$whitelist_file" ) { &create_empty_file($whitelist_file); } @@ -1446,83 +1442,19 @@ sub generate_http_ports_file() { } # -## Function to generate and write the file for used rulefiles file for a given provider. -## -## The function requires as first argument a provider handle, and as second an array with files. -# -sub write_used_provider_rulefiles_file($@) { - my ($provider, @files) = @_; - - # Get the path and file for the provider specific used rulefiles file. - my $used_provider_rulesfile_file = &get_used_provider_rulesfile_file($provider); - - # Open file for used rulefiles. - open (FILE, ">", "$used_provider_rulesfile_file") or die "Could not write to $used_provider_rulesfile_file. $!\n"; - - # Write yaml header to the file. - print FILE "%YAML 1.1\n"; - print FILE "---\n\n"; - - # Write header to file. - print FILE "#Autogenerated file. Any custom changes will be overwritten!\n"; - - # Loop through the array of given files. - foreach my $file (@files) { - # Check if the given filename exists and write it to the file of used rulefiles. - if(-f "$rulespath/$file") { - print FILE " - $file\n"; - } - } - - # Close file after writing. - close(FILE); -} - -# -## Function to write the main file for provider rulesfiles inclusions. +## Function to write the file that contains the rulefiles which are loaded by suricaa. ## -## This function requires an array of provider handles. +## This function requires an array of used provider handles. # -sub write_main_used_rulefiles_file (@) { +sub write_used_rulefiles_file (@) { my (@providers) = @_; - # Call function to write the static rulefiles file. - &_write_default_rulefiles_file(); - - # Open file for used rulefils inclusion. - open (FILE, ">", "$suricata_used_providers_file") or die "Could not write to $suricata_used_providers_file. $!\n"; - - # Write yaml header to the file. - print FILE "%YAML 1.1\n"; - print FILE "---\n\n"; - - # Write header to file. - print FILE "#Autogenerated file. Any custom changes will be overwritten!\n"; - - # Loop through the list of given providers. - foreach my $provider (@providers) { - # Call function to get the providers used rulefiles file. - my $filename = &get_used_provider_rulesfile_file($provider); - - # Check if the file exists and write it into the used rulefiles file. - if (-f $filename) { - # Print the provider to the file. - print FILE "include\: $filename\n"; - } - } - - # Close the filehandle after writing. - close(FILE); -} - -sub _write_default_rulefiles_file () { - # Get enabled application layer protocols. + # Get the enabled application layer protocols. my @enabled_app_layer_protos = &get_suricata_enabled_app_layer_protos(); - # Open file. - open (FILE, ">", $suricata_default_rulefiles_file) or die "Could not write to $suricata_default_rulefiles_file. $!\n"; + # Open the file. + open (FILE, ">", $suricata_used_rulesfiles_file) or die "Could not write to $suricata_used_rulesfiles_file. $!\n"; - # Write yaml header to the file. print FILE "%YAML 1.1\n"; print FILE "---\n\n"; @@ -1566,17 +1498,38 @@ sub _write_default_rulefiles_file () { } } + # Loop through the array of enabled providers. + foreach my $provider (@providers) { + # Get the used rulefile for this provider. + my @used_rulesfiles = &get_provider_used_rulesfiles($provider); + + # Check if there are + if(@used_rulesfiles) { + # Add notice to the file. + print FILE "\n#Used Rulesfiles for provider $provider.\n"; + + # Loop through the array of used rulefiles. + foreach my $enabled_rulesfile (@used_rulesfiles) { + # Generate name and full path to the rulesfile. + my $rulesfile = "$rulespath/$enabled_rulesfile"; + + # Write the ruelsfile name to the file. + print FILE " - $rulesfile\n"; + } + } + } + # Close the file handle close(FILE); } # -## Tiny function to generate the full path and name for the used_provider_rulesfile file of a given provider. +## Tiny function to generate the full path and name for the file which stores the used rulefiles of a given provider. # -sub get_used_provider_rulesfile_file ($) { +sub get_provider_used_rulesfiles_file ($) { my ($provider) = @_; - my $filename = "$settingsdir/suricata\-$provider\-used\-rulefiles.yaml"; + my $filename = "$settingsdir/$provider\-used\-rulesfiles"; # Return the gernerated file. return $filename; @@ -1909,44 +1862,35 @@ sub get_red_address() { # ## Function to get the used rules files of a given provider. # -sub read_used_provider_rulesfiles($) { +sub get_provider_used_rulesfiles($) { my ($provider) = @_; + # Hash to store the used rulefiles of the provider. + my %provider_rulefiles = (); + # Array to store the used rulefiles. my @used_rulesfiles = (); - # Get the used rulesefile file for the provider. - my $rulesfile_file = &get_used_provider_rulesfile_file($provider); + # Get the filename which contains the used rulefiles for this provider. + my $used_rulesfiles_file = &get_provider_used_rulesfiles_file($provider); - # Check if the a used rulesfile exists for this provider. - if (-f $rulesfile_file) { - # Open the file or used rulefiles and read-in content. - open(FILE, $rulesfile_file) or die "Could not open $rulesfile_file. $!\n"; - - while () { - # Assign the current line to a nice variable. - my $line = $_; - - # Remove newlines. - chomp($line); + # Read-in file, if it exists. + &General::readhash("$used_rulesfiles_file", \%provider_rulefiles) if (-f $used_rulesfiles_file); - # Skip comments. - next if ($line =~ /\#/); + # Loop through the hash of rulefiles which does the provider offer. + foreach my $rulefile (keys %provider_rulefiles) { + # Skip disabled rulefiles. + next unless($provider_rulefiles{$rulefile} eq "enabled"); - # Skip blank lines. - next if ($line =~ /^\s*$/); + # The General::readhash function does not allow dots as + # key value and limits the key "string" to the part before + # the dot, in case it contains one. + # + # So add the file extension for the rules file manually again. + $rulefile = "$rulefile.rules"; - # Gather the rulefile. - if ($line =~ /.*- (.*)/) { - my $rulefile = $1; - - # Add the rulefile to the array of used rulesfiles. - push(@used_rulesfiles, $rulefile); - } - } - - # Close the file. - close(FILE); + # Add the enabled rulefile to the array of enabled rulefiles. + push(@used_rulesfiles, $rulefile); } # Return the array of used rulesfiles. diff --git a/config/suricata/suricata.yaml b/config/suricata/suricata.yaml index 6fbc7b3ee1..03a7a83afc 100644 --- a/config/suricata/suricata.yaml +++ b/config/suricata/suricata.yaml @@ -47,10 +47,7 @@ vars: default-rule-path: /var/lib/suricata rule-files: # Include enabled ruleset files from external file. - include: /var/ipfire/suricata/suricata-used-providers.yaml - - # Include default rules. - include: /var/ipfire/suricata/suricata-default-rules.yaml + include: /var/ipfire/suricata/suricata-used-rulesfiles.yaml classification-file: /usr/share/suricata/classification.config reference-config-file: /usr/share/suricata/reference.config -- 2.39.5