[people/pmueller/ipfire-2.x.git] / config / cfgroot / ipblocklist-functions.pl

#!/usr/bin/perl -w
###############################################################################
#                                                                             #
# IPFire.org - A linux based firewall                                         #
# Copyright (C) 2007-2022  IPFire Team  <info@ipfire.org>                     #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 2 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
###############################################################################

package IPblocklist;

require '/var/ipfire/general-functions.pl';
require "${General::swroot}/ipblocklist/sources";

# The directory where all ipblocklist related files and settings are stored.
our $settings_dir = "/var/ipfire/ipblocklist";

# Main settings file.
our $settings_file = "$settings_dir/settings";

# The file which keeps the time, when a blocklist last has been modified.
our $modified_file = "$settings_dir/modified";

# Location where the blocklists in ipset compatible format are stored.
our $blocklist_dir = "/var/lib/ipblocklist";

# File extension of the blocklist files.
my $blocklist_file_extension = ".conf";

# Hash which calls the correct parser functions.
my %parsers = (
	'ip-or-net-list' => \&parse_ip_or_net_list,
	'dshield'        => \&parse_dshield
);

#
## Function to get all available blocklists.
#
sub get_blocklists () {
	my @blocklists;

	# Loop through the hash of blocklists.
	foreach my $blocklist ( keys %IPblocklist::List::sources ) {
		# Add the list to the array.
		push(@blocklists, $blocklist);
	}

	# Sort and return the array.
	return sort(@blocklists);
}

#
## Tiny function to get the full path and name of a given blocklist.
#
sub get_ipset_db_file($) {
	my ($set) = @_;

	# Generate the
	my $file = "$blocklist_dir/$set$blocklist_file_extension";

	# Return the file name.
	return $file;
}

#
## The main download_and_create blocklist function.
##
## Uses LWP to download a given blocklist. The If-Modified-Since header is
## specified in the request so that only updated lists are downloaded (providing
## that the server supports this functionality).
##
## Once downloaded the list gets parsed, converted and stored in an ipset compatible
## format.
##
## Parameters:
##   list      The name of the blocklist
##
## Returns:
##   nothing - On success
##   not_modified - In case the servers responds with "Not modified" (304)
##   dl_error - If the requested blocklist could not be downloaded.
##   empty_list - The downloaded blocklist is empty, or the parser was not able to parse
##                it correctly.
#
sub download_and_create_blocklist($) {
	my ($list) = @_;

	# Check if the given blockist is known and data available.
	unless($IPblocklist::List::sources{$list}) {
		# No valid data for this blocklist - exit and return "1".
		return 1;
	}

	# The allowed maximum download size in bytes.
	my $max_dl_bytes = 10_485_760;

	# The amount of download attempts before giving up and
	# logging an error.
	my $max_dl_attempts = 5;

	# Read proxysettings.
	my %proxysettings=();
	&General::readhash("${General::swroot}/proxy/settings", \%proxysettings);

	# Load required perl module to handle the download.
	use LWP::UserAgent;

	# Create a user agent for downloading the blacklist
	# Limit the download size for safety
	my $ua = LWP::UserAgent->new (
		ssl_opts => {
			SSL_ca_file     => '/etc/ssl/cert.pem',
			verify_hostname => 1,
		},

		max_size => $max_dl_bytes,
	);

	# Set timeout to 10 seconds.
	$ua->timeout(10);

	# Check if an upstream proxy is configured.
	if ($proxysettings{'UPSTREAM_PROXY'}) {
		my $proxy_url;

		$proxy_url = "http://";

		# Check if the proxy requires authentication.
		if (($proxysettings{'UPSTREAM_USER'}) && ($proxysettings{'UPSTREAM_PASSWORD'})) {
			$proxy_url .= "$proxysettings{'UPSTREAM_USER'}\:$proxysettings{'UPSTREAM_PASSWORD'}\@";
		}

		# Add proxy server address and port.
		$proxy_url .= $proxysettings{'UPSTREAM_PROXY'};

		# Setup proxy settings.
		$ua->proxy(['http', 'https'], $proxy_url);
	}

	# Gather the details, when a list got modified last time.
	my %modified = ();

	# Read-in data if the file exists.
	&General::readhash($modified_file, \%modified ) if (-e $modified_file);

	# Get the last modified time for this list.
	my $last_modified = gmtime($modified{$list} || 0);

	my $dl_attempt = 1;
	my $response;

	# Download and rety on failure loop.
	while ($dl_attempt <= $max_dl_attempts) {
		# Try to determine if there is a newer blocklist since last time and grab it.
		$response = $ua->get($IPblocklist::List::sources{$list}{'url'}, 'If-Modified-Since' => $last_modified );

		# Check if the download attempt was successfull.
		if ($response->is_success) {
			# We successfully grabbed the list - no more retries needed, break the loop.
			# Further process the script code.
			last;

		# Exit, if the server responds with "Not modified (304).
		} elsif ($response->code == 304) {
			# Exit and return "not modified".
			return "not_modified";

		# Exit and log an erro
		} elsif ($dl_attempt eq $max_dl_attempts) {
			# Exit and return "dl_error".
			return "dl_error";
		}

		# Increase download attempt counter.
		$dl_attempt++;
	}

	# Update the timestamp for the new or modified list.
	if($response->last_modified) {
		$modified{$list} = $response->last_modified;
	} else {
		$modified{$list} = time();
	}

	# Write-back the modified timestamps.
	&General::writehash($modified_file, \%modified);

	# Parse and loop through the downloaded list.
	my @blocklist = ();

	# Get the responsible parser for the current list.
	my $parser = $parsers{$IPblocklist::List::sources{$list}{'parser'}};

	# Loop through the grabbed raw list.
	foreach my $line (split /[\r\n]+/, $response->content) {
		# Remove newlines.
		chomp $line;

		# Call the parser and obtain the addess or network.
		my $address = &$parser($line);

		# Skip the line if it does not contain an address.
		next unless ($address and $address =~ m/\d+\.\d+\.\d+\.\d+/);

		# Check if we got a single address.
		if ($address =~ m|/32|) {
			# Add /32 as prefix.
			$address =~ s|/32||;
		}

		# Push the address/network to the blocklist array.
		push(@blocklist, $address);
	}

	# Check if the content could be parsed correctly and the blocklist
	# contains at least one item.
	unless(@blocklist) {
		# No entries - exit and return "empty_list".
		return "empty_list";
	}

	# Get amount of entries in the blocklist array.
	my $list_entries = scalar(@blocklist);

	# Optain the filename for this blocklist to save.
	my $file = &get_ipset_db_file($list);

	# Open the file for writing.
	open(FILE, ">", "$file") or die "Could not write to $file. $!\n";

	# Write file header.
	print FILE "#Autogenerated file. Any custom changes will be overwritten!\n\n";

	# Calculate the hashsize for better list performance.
	my $hashsize = &_calculate_hashsize($list_entries);

	# Simply set the limit of list elements to the double of current list elements.
	my $maxelem = $list_entries *2;

	# Write line to create the set.
	#
	# We safely can use hash:net as type because it supports single addresses and networks.
	print FILE "create $list hash:net family inet hashsize $hashsize maxelem $maxelem -exist\n";

	# Write line to flush the set itself during loading.
	print FILE "flush $list\n";

	# Loop through the array which contains the blocklist.
	foreach my $entry (@blocklist) {
		# Add the entry to the list.
		print FILE "add $list $entry\n";
	}

	# Close the file handle.
	close(FILE);

	# Finished.
	return;
}

#
## sub parse_ip_or_net_list( line )
##
## Parses an input line, looking for lines starting with an IP Address or
### Network specification.
##
## Parameters:
##   line  The line to parse
##
## Returns:
##   Either an IP Address or a null string
#
sub parse_ip_or_net_list( $ ) {
	my ($line) = @_;

	# Grab the IP address or network.
	$line =~ m|^(\d+\.\d+\.\d+\.\d+(?:/\d+)?)|;

	# Return the grabbed address.
	return $1;
}

#
## sub parse_dshield( line )
##
## Parses an input line removing comments.
##
## The format is:
## Start Addrs   End Addrs   Netmask   Nb Attacks   Network Name   Country   email
## We're only interested in the start address and netmask.
##
## Parameters:
##   line  The line to parse
##
## Returns:
##   Either and IP Address or a null string
#
sub parse_dshield( $ ) {
	my ($line) = @_;

	# Skip coments.
	return "" if ($line =~ m/^\s*#/);

	$line =~ s/#.*$//;

	#          |Start addrs                |   |End Addrs                |   |Mask
	$line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)\s+\d+\.\d+\.\d+\.\d+(?:/\d+)?\s+(\d+)|;

	# Return nothing if no start address could be grabbed.
	return unless ($1);

	# Add /32 as prefix for single addresses and return it.
	return "$1/32" unless ($2);

	# Return the obtained network.
	return "$1/$2";
}

#
## Helper function to proper calculate the hashsize.
#
sub _calculate_hashsize($) {
	my ($list_entries) = @_;

	my $hashsize = 1;
	$hashsize  <<= 1 while ($hashsize < $list_entries);

	# Return the calculated hashsize.
	return $hashsize;
}

#
## sub get_holdoff_rate(list)
##
## This function is used to get the holdoff rate in seconds for a desired provider,
## based on the configured rate limit in minutes (m), hours (h) or days (d) in the
## blacklist sources settings file.
##
#
sub get_holdoff_rate($) {
	my ($list) = @_;

	# Grab the configured lookup rate for the given list.
	my $rate = $IPblocklist::List::sources{$list}{'rate'};

	# Split the grabbed rate into value and unit.
	my ($value, $unit) = (uc $rate) =~ m/(\d+)([DHM]?)/;

	# Days
	if ($unit eq 'D') {
		$value *= 60 * 60 * 24;

	# Minutes
	} elsif ($unit eq 'M') {
		$value *= 60;

	# Everything else - assume hours.
	} else {
		$value *= 60 * 60;
	}

	# Sanity check - limit to range 5 min .. 1 week

	#        d    h    m    s
	$value =           5 * 60 if ($value < 5 * 60);
	$value = 7 * 24 * 60 * 60 if ($value > 7 * 24 * 60 * 60);

	return $value;
}

1;
Commit	Line	Data
f4c25a44 SS	1	#!/usr/bin/perl -w
	2	###############################################################################
	3	# #
	4	# IPFire.org - A linux based firewall #
	5	# Copyright (C) 2007-2022 IPFire Team <info@ipfire.org> #
	6	# #
	7	# This program is free software: you can redistribute it and/or modify #
	8	# it under the terms of the GNU General Public License as published by #
	9	# the Free Software Foundation, either version 2 of the License, or #
	10	# (at your option) any later version. #
	11	# #
	12	# This program is distributed in the hope that it will be useful, #
	13	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
	15	# GNU General Public License for more details. #
	16	# #
	17	# You should have received a copy of the GNU General Public License #
	18	# along with this program. If not, see <http://www.gnu.org/licenses/>. #
	19	# #
	20	###############################################################################
	21
	22	package IPblocklist;
	23
ba8455e4 SS	24	require '/var/ipfire/general-functions.pl';
	25	require "${General::swroot}/ipblocklist/sources";
	26
	27	# The directory where all ipblocklist related files and settings are stored.
	28	our $settings_dir = "/var/ipfire/ipblocklist";
	29
	30	# Main settings file.
	31	our $settings_file = "$settings_dir/settings";
	32
	33	# The file which keeps the time, when a blocklist last has been modified.
7f4829de	34	our $modified_file = "$settings_dir/modified";
f4c25a44	35
e64587a4	36	# Location where the blocklists in ipset compatible format are stored.
7f4829de	37	our $blocklist_dir = "/var/lib/ipblocklist";
e64587a4 SS	38
e64587a4 SS	39	# File extension of the blocklist files.
ba8455e4 SS	40	my $blocklist_file_extension = ".conf";
	41
	42	# Hash which calls the correct parser functions.
	43	my %parsers = (
	44	'ip-or-net-list' => \&parse_ip_or_net_list,
	45	'dshield' => \&parse_dshield
	46	);
e64587a4	47
f4c25a44 SS	48	#
	49	## Function to get all available blocklists.
	50	#
	51	sub get_blocklists () {
	52	my @blocklists;
	53
	54	# Loop through the hash of blocklists.
	55	foreach my $blocklist ( keys %IPblocklist::List::sources ) {
	56	# Add the list to the array.
	57	push(@blocklists, $blocklist);
	58	}
	59
	60	# Sort and return the array.
	61	return sort(@blocklists);
	62	}
	63
e64587a4 SS	64	#
	65	## Tiny function to get the full path and name of a given blocklist.
	66	#
	67	sub get_ipset_db_file($) {
	68	my ($set) = @_;
	69
	70	# Generate the
	71	my $file = "$blocklist_dir/$set$blocklist_file_extension";
	72
	73	# Return the file name.
	74	return $file;
	75	}
	76
ba8455e4 SS	77	#
	78	## The main download_and_create blocklist function.
	79	##
	80	## Uses LWP to download a given blocklist. The If-Modified-Since header is
	81	## specified in the request so that only updated lists are downloaded (providing
	82	## that the server supports this functionality).
	83	##
	84	## Once downloaded the list gets parsed, converted and stored in an ipset compatible
	85	## format.
	86	##
	87	## Parameters:
	88	## list The name of the blocklist
	89	##
	90	## Returns:
	91	## nothing - On success
	92	## not_modified - In case the servers responds with "Not modified" (304)
	93	## dl_error - If the requested blocklist could not be downloaded.
d7dd5653 SS	94	## empty_list - The downloaded blocklist is empty, or the parser was not able to parse
d7dd5653 SS	95	## it correctly.
ba8455e4 SS	96	#
	97	sub download_and_create_blocklist($) {
	98	my ($list) = @_;
	99
	100	# Check if the given blockist is known and data available.
	101	unless($IPblocklist::List::sources{$list}) {
	102	# No valid data for this blocklist - exit and return "1".
	103	return 1;
	104	}
	105
	106	# The allowed maximum download size in bytes.
	107	my $max_dl_bytes = 10_485_760;
	108
	109	# The amount of download attempts before giving up and
	110	# logging an error.
	111	my $max_dl_attempts = 5;
	112
	113	# Read proxysettings.
	114	my %proxysettings=();
	115	&General::readhash("${General::swroot}/proxy/settings", \%proxysettings);
	116
	117	# Load required perl module to handle the download.
	118	use LWP::UserAgent;
	119
	120	# Create a user agent for downloading the blacklist
	121	# Limit the download size for safety
	122	my $ua = LWP::UserAgent->new (
	123	ssl_opts => {
	124	SSL_ca_file => '/etc/ssl/cert.pem',
	125	verify_hostname => 1,
	126	},
	127
	128	max_size => $max_dl_bytes,
	129	);
	130
	131	# Set timeout to 10 seconds.
	132	$ua->timeout(10);
	133
	134	# Check if an upstream proxy is configured.
	135	if ($proxysettings{'UPSTREAM_PROXY'}) {
	136	my $proxy_url;
	137
	138	$proxy_url = "http://";
	139
	140	# Check if the proxy requires authentication.
	141	if (($proxysettings{'UPSTREAM_USER'}) && ($proxysettings{'UPSTREAM_PASSWORD'})) {
	142	$proxy_url .= "$proxysettings{'UPSTREAM_USER'}\:$proxysettings{'UPSTREAM_PASSWORD'}\@";
	143	}
	144
	145	# Add proxy server address and port.
	146	$proxy_url .= $proxysettings{'UPSTREAM_PROXY'};
	147
	148	# Setup proxy settings.
	149	$ua->proxy(['http', 'https'], $proxy_url);
	150	}
	151
	152	# Gather the details, when a list got modified last time.
	153	my %modified = ();
	154
	155	# Read-in data if the file exists.
	156	&General::readhash($modified_file, \%modified ) if (-e $modified_file);
	157
	158	# Get the last modified time for this list.
	159	my $last_modified = gmtime($modified{$list} \|\| 0);
160
161	my $dl_attempt = 1;
162	my $response;
163
164	# Download and rety on failure loop.
165	while ($dl_attempt <= $max_dl_attempts) {
166	# Try to determine if there is a newer blocklist since last time and grab it.
167	$response = $ua->get($IPblocklist::List::sources{$list}{'url'}, 'If-Modified-Since' => $last_modified );
168
169	# Check if the download attempt was successfull.
170	if ($response->is_success) {
171	# We successfully grabbed the list - no more retries needed, break the loop.
172	# Further process the script code.
173	last;
174
175	# Exit, if the server responds with "Not modified (304).
176	} elsif ($response->code == 304) {
177	# Exit and return "not modified".
178	return "not_modified";
179
180	# Exit and log an erro
181	} elsif ($dl_attempt eq $max_dl_attempts) {
182	# Exit and return "dl_error".
183	return "dl_error";
184	}
185
186	# Increase download attempt counter.
187	$dl_attempt++;
188	}
189
190	# Update the timestamp for the new or modified list.
d8113820 SS	191	if($response->last_modified) {
	192	$modified{$list} = $response->last_modified;
	193	} else {
	194	$modified{$list} = time();
	195	}
ba8455e4 SS	196
	197	# Write-back the modified timestamps.
	198	&General::writehash($modified_file, \%modified);
	199
	200	# Parse and loop through the downloaded list.
	201	my @blocklist = ();
	202
	203	# Get the responsible parser for the current list.
	204	my $parser = $parsers{$IPblocklist::List::sources{$list}{'parser'}};
	205
	206	# Loop through the grabbed raw list.
	207	foreach my $line (split /[\r\n]+/, $response->content) {
	208	# Remove newlines.
	209	chomp $line;
	210
	211	# Call the parser and obtain the addess or network.
	212	my $address = &$parser($line);
	213
	214	# Skip the line if it does not contain an address.
	215	next unless ($address and $address =~ m/\d+\.\d+\.\d+\.\d+/);
	216
	217	# Check if we got a single address.
	218	if ($address =~ m\|/32\|) {
	219	# Add /32 as prefix.
	220	$address =~ s\|/32\|\|;
	221	}
	222
	223	# Push the address/network to the blocklist array.
	224	push(@blocklist, $address);
	225	}
	226
d7dd5653 SS	227	# Check if the content could be parsed correctly and the blocklist
	228	# contains at least one item.
	229	unless(@blocklist) {
	230	# No entries - exit and return "empty_list".
	231	return "empty_list";
	232	}
	233
ba8455e4 SS	234	# Get amount of entries in the blocklist array.
	235	my $list_entries = scalar(@blocklist);
	236
	237	# Optain the filename for this blocklist to save.
	238	my $file = &get_ipset_db_file($list);
	239
	240	# Open the file for writing.
	241	open(FILE, ">", "$file") or die "Could not write to $file. $!\n";
	242
	243	# Write file header.
	244	print FILE "#Autogenerated file. Any custom changes will be overwritten!\n\n";
	245
	246	# Calculate the hashsize for better list performance.
	247	my $hashsize = &_calculate_hashsize($list_entries);
	248
	249	# Simply set the limit of list elements to the double of current list elements.
	250	my $maxelem = $list_entries *2;
	251
	252	# Write line to create the set.
	253	#
	254	# We safely can use hash:net as type because it supports single addresses and networks.
	255	print FILE "create $list hash:net family inet hashsize $hashsize maxelem $maxelem -exist\n";
	256
	257	# Write line to flush the set itself during loading.
	258	print FILE "flush $list\n";
	259
	260	# Loop through the array which contains the blocklist.
	261	foreach my $entry (@blocklist) {
	262	# Add the entry to the list.
	263	print FILE "add $list $entry\n";
	264	}
	265
	266	# Close the file handle.
	267	close(FILE);
	268
	269	# Finished.
	270	return;
	271	}
	272
ab017961 TF	273	#
	274	## sub parse_ip_or_net_list( line )
	275	##
	276	## Parses an input line, looking for lines starting with an IP Address or
	277	### Network specification.
	278	##
	279	## Parameters:
	280	## line The line to parse
	281	##
	282	## Returns:
	283	## Either an IP Address or a null string
	284	#
	285	sub parse_ip_or_net_list( $ ) {
	286	my ($line) = @_;
	287
	288	# Grab the IP address or network.
	289	$line =~ m\|^(\d+\.\d+\.\d+\.\d+(?:/\d+)?)\|;
	290
	291	# Return the grabbed address.
	292	return $1;
	293	}
ac9b5d8e TF	294
	295	#
	296	## sub parse_dshield( line )
	297	##
	298	## Parses an input line removing comments.
	299	##
	300	## The format is:
	301	## Start Addrs End Addrs Netmask Nb Attacks Network Name Country email
	302	## We're only interested in the start address and netmask.
	303	##
	304	## Parameters:
	305	## line The line to parse
	306	##
	307	## Returns:
	308	## Either and IP Address or a null string
	309	#
	310	sub parse_dshield( $ ) {
	311	my ($line) = @_;
	312
	313	# Skip coments.
	314	return "" if ($line =~ m/^\s*#/);
	315
	316	$line =~ s/#.*$//;
	317
	318	# \|Start addrs \| \|End Addrs \| \|Mask
	319	$line =~ m\|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)\s+\d+\.\d+\.\d+\.\d+(?:/\d+)?\s+(\d+)\|;
	320
	321	# Return nothing if no start address could be grabbed.
	322	return unless ($1);
	323
	324	# Add /32 as prefix for single addresses and return it.
	325	return "$1/32" unless ($2);
	326
	327	# Return the obtained network.
	328	return "$1/$2";
	329	}
0a4f60f2 TF	330
	331	#
	332	## Helper function to proper calculate the hashsize.
	333	#
	334	sub _calculate_hashsize($) {
	335	my ($list_entries) = @_;
	336
	337	my $hashsize = 1;
	338	$hashsize <<= 1 while ($hashsize < $list_entries);
	339
	340	# Return the calculated hashsize.
	341	return $hashsize;
	342	}
	343
8d12d12a TF	344	#
	345	## sub get_holdoff_rate(list)
	346	##
	347	## This function is used to get the holdoff rate in seconds for a desired provider,
	348	## based on the configured rate limit in minutes (m), hours (h) or days (d) in the
	349	## blacklist sources settings file.
	350	##
	351	#
	352	sub get_holdoff_rate($) {
	353	my ($list) = @_;
	354
	355	# Grab the configured lookup rate for the given list.
	356	my $rate = $IPblocklist::List::sources{$list}{'rate'};
	357
	358	# Split the grabbed rate into value and unit.
	359	my ($value, $unit) = (uc $rate) =~ m/(\d+)([DHM]?)/;
	360
	361	# Days
	362	if ($unit eq 'D') {
	363	$value = 60 60 * 24;
	364
	365	# Minutes
	366	} elsif ($unit eq 'M') {
	367	$value *= 60;
	368
	369	# Everything else - assume hours.
	370	} else {
	371	$value = 60 60;
	372	}
	373
	374	# Sanity check - limit to range 5 min .. 1 week
	375
	376	# d h m s
	377	$value = 5 * 60 if ($value < 5 * 60);
	378	$value = 7 * 24 * 60 * 60 if ($value > 7 * 24 * 60 * 60);
	379
	380	return $value;
	381	}
	382
f4c25a44	383	1;