2 ###############################################################################
4 # IPFire.org - A linux based firewall #
5 # Copyright (C) 2007-2022 IPFire Team <info@ipfire.org> #
7 # This program is free software: you can redistribute it and/or modify #
8 # it under the terms of the GNU General Public License as published by #
9 # the Free Software Foundation, either version 2 of the License, or #
10 # (at your option) any later version. #
12 # This program is distributed in the hope that it will be useful, #
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
15 # GNU General Public License for more details. #
17 # You should have received a copy of the GNU General Public License #
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
20 ###############################################################################
24 require '/var/ipfire/general-functions.pl';
25 require "${General::swroot}/ipblocklist/sources";
27 # The directory where all ipblocklist related files and settings are stored.
28 our $settings_dir = "/var/ipfire/ipblocklist";
31 our $settings_file = "$settings_dir/settings";
33 # The file which keeps the time, when a blocklist last has been modified.
34 our $modified_file = "$settings_dir/modified";
36 # Location where the blocklists in ipset compatible format are stored.
37 our $blocklist_dir = "/var/lib/ipblocklist";
39 # File extension of the blocklist files.
40 my $blocklist_file_extension = ".conf";
42 # Hash which calls the correct parser functions.
44 'ip-or-net-list' => \
&parse_ip_or_net_list
,
45 'dshield' => \
&parse_dshield
49 ## Function to get all available blocklists.
51 sub get_blocklists
() {
54 # Loop through the hash of blocklists.
55 foreach my $blocklist ( keys %IPblocklist::List
::sources
) {
56 # Add the list to the array.
57 push(@blocklists, $blocklist);
60 # Sort and return the array.
61 return sort(@blocklists);
65 ## Tiny function to get the full path and name of a given blocklist.
67 sub get_ipset_db_file
($) {
71 my $file = "$blocklist_dir/$set$blocklist_file_extension";
73 # Return the file name.
78 ## The main download_and_create blocklist function.
80 ## Uses LWP to download a given blocklist. The If-Modified-Since header is
81 ## specified in the request so that only updated lists are downloaded (providing
82 ## that the server supports this functionality).
84 ## Once downloaded the list gets parsed, converted and stored in an ipset compatible
88 ## list The name of the blocklist
91 ## nothing - On success
92 ## not_modified - In case the servers responds with "Not modified" (304)
93 ## dl_error - If the requested blocklist could not be downloaded.
94 ## empty_list - The downloaded blocklist is empty, or the parser was not able to parse
97 sub download_and_create_blocklist
($) {
100 # Check if the given blockist is known and data available.
101 unless($IPblocklist::List
::sources
{$list}) {
102 # No valid data for this blocklist - exit and return "1".
106 # The allowed maximum download size in bytes.
107 my $max_dl_bytes = 10_485_760
;
109 # The amount of download attempts before giving up and
111 my $max_dl_attempts = 5;
113 # Read proxysettings.
114 my %proxysettings=();
115 &General
::readhash
("${General::swroot}/proxy/settings", \
%proxysettings);
117 # Load required perl module to handle the download.
120 # Create a user agent for downloading the blacklist
121 # Limit the download size for safety
122 my $ua = LWP
::UserAgent
->new (
124 SSL_ca_file
=> '/etc/ssl/cert.pem',
125 verify_hostname
=> 1,
128 max_size
=> $max_dl_bytes,
131 # Set timeout to 10 seconds.
134 # Check if an upstream proxy is configured.
135 if ($proxysettings{'UPSTREAM_PROXY'}) {
138 $proxy_url = "http://";
140 # Check if the proxy requires authentication.
141 if (($proxysettings{'UPSTREAM_USER'}) && ($proxysettings{'UPSTREAM_PASSWORD'})) {
142 $proxy_url .= "$proxysettings{'UPSTREAM_USER'}\:$proxysettings{'UPSTREAM_PASSWORD'}\@";
145 # Add proxy server address and port.
146 $proxy_url .= $proxysettings{'UPSTREAM_PROXY'};
148 # Setup proxy settings.
149 $ua->proxy(['http', 'https'], $proxy_url);
152 # Gather the details, when a list got modified last time.
155 # Read-in data if the file exists.
156 &General
::readhash
($modified_file, \
%modified ) if (-e
$modified_file);
158 # Get the last modified time for this list.
159 my $last_modified = gmtime($modified{$list} || 0);
164 # Download and rety on failure loop.
165 while ($dl_attempt <= $max_dl_attempts) {
166 # Try to determine if there is a newer blocklist since last time and grab it.
167 $response = $ua->get($IPblocklist::List
::sources
{$list}{'url'}, 'If-Modified-Since' => $last_modified );
169 # Check if the download attempt was successfull.
170 if ($response->is_success) {
171 # We successfully grabbed the list - no more retries needed, break the loop.
172 # Further process the script code.
175 # Exit, if the server responds with "Not modified (304).
176 } elsif ($response->code == 304) {
177 # Exit and return "not modified".
178 return "not_modified";
180 # Exit and log an erro
181 } elsif ($dl_attempt eq $max_dl_attempts) {
182 # Exit and return "dl_error".
186 # Increase download attempt counter.
190 # Update the timestamp for the new or modified list.
191 if($response->last_modified) {
192 $modified{$list} = $response->last_modified;
194 $modified{$list} = time();
197 # Write-back the modified timestamps.
198 &General
::writehash
($modified_file, \
%modified);
200 # Parse and loop through the downloaded list.
203 # Get the responsible parser for the current list.
204 my $parser = $parsers{$IPblocklist::List
::sources
{$list}{'parser'}};
206 # Loop through the grabbed raw list.
207 foreach my $line (split /[\r\n]+/, $response->content) {
211 # Call the parser and obtain the addess or network.
212 my $address = &$parser($line);
214 # Skip the line if it does not contain an address.
215 next unless ($address and $address =~ m/\d+\.\d+\.\d+\.\d+/);
217 # Check if we got a single address.
218 if ($address =~ m
|/32|) {
223 # Push the address/network to the blocklist array.
224 push(@blocklist, $address);
227 # Check if the content could be parsed correctly and the blocklist
228 # contains at least one item.
230 # No entries - exit and return "empty_list".
234 # Get amount of entries in the blocklist array.
235 my $list_entries = scalar(@blocklist);
237 # Optain the filename for this blocklist to save.
238 my $file = &get_ipset_db_file
($list);
240 # Open the file for writing.
241 open(FILE
, ">", "$file") or die "Could not write to $file. $!\n";
244 print FILE
"#Autogenerated file. Any custom changes will be overwritten!\n\n";
246 # Calculate the hashsize for better list performance.
247 my $hashsize = &_calculate_hashsize
($list_entries);
249 # Simply set the limit of list elements to the double of current list elements.
250 my $maxelem = $list_entries *2;
252 # Write line to create the set.
254 # We safely can use hash:net as type because it supports single addresses and networks.
255 print FILE
"create $list hash:net family inet hashsize $hashsize maxelem $maxelem -exist\n";
257 # Write line to flush the set itself during loading.
258 print FILE
"flush $list\n";
260 # Loop through the array which contains the blocklist.
261 foreach my $entry (@blocklist) {
262 # Add the entry to the list.
263 print FILE
"add $list $entry\n";
266 # Close the file handle.
274 ## sub parse_ip_or_net_list( line )
276 ## Parses an input line, looking for lines starting with an IP Address or
277 ### Network specification.
280 ## line The line to parse
283 ## Either an IP Address or a null string
285 sub parse_ip_or_net_list
( $ ) {
288 # Grab the IP address or network.
289 $line =~ m
|^(\d
+\
.\d
+\
.\d
+\
.\d
+(?
:/\d
+)?
)|;
291 # Return the grabbed address.
296 ## sub parse_dshield( line )
298 ## Parses an input line removing comments.
301 ## Start Addrs End Addrs Netmask Nb Attacks Network Name Country email
302 ## We're only interested in the start address and netmask.
305 ## line The line to parse
308 ## Either and IP Address or a null string
310 sub parse_dshield
( $ ) {
314 return "" if ($line =~ m/^\s*#/);
318 # |Start addrs | |End Addrs | |Mask
319 $line =~ m
|(\d
+\
.\d
+\
.\d
+\
.\d
+(?
:/\d+)?)\s+\d+\.\d+\.\d+\.\d+(?:/\d
+)?\s
+(\d
+)|;
321 # Return nothing if no start address could be grabbed.
324 # Add /32 as prefix for single addresses and return it.
325 return "$1/32" unless ($2);
327 # Return the obtained network.
332 ## Helper function to proper calculate the hashsize.
334 sub _calculate_hashsize
($) {
335 my ($list_entries) = @_;
338 $hashsize <<= 1 while ($hashsize < $list_entries);
340 # Return the calculated hashsize.
345 ## sub get_holdoff_rate(list)
347 ## This function is used to get the holdoff rate in seconds for a desired provider,
348 ## based on the configured rate limit in minutes (m), hours (h) or days (d) in the
349 ## blacklist sources settings file.
352 sub get_holdoff_rate
($) {
355 # Grab the configured lookup rate for the given list.
356 my $rate = $IPblocklist::List
::sources
{$list}{'rate'};
358 # Split the grabbed rate into value and unit.
359 my ($value, $unit) = (uc $rate) =~ m/(\d+)([DHM]?)/;
363 $value *= 60 * 60 * 24;
366 } elsif ($unit eq 'M') {
369 # Everything else - assume hours.
374 # Sanity check - limit to range 5 min .. 1 week
377 $value = 5 * 60 if ($value < 5 * 60);
378 $value = 7 * 24 * 60 * 60 if ($value > 7 * 24 * 60 * 60);