]>
Commit | Line | Data |
---|---|---|
f4c25a44 SS |
1 | #!/usr/bin/perl -w |
2 | ############################################################################### | |
3 | # # | |
4 | # IPFire.org - A linux based firewall # | |
5 | # Copyright (C) 2007-2022 IPFire Team <info@ipfire.org> # | |
6 | # # | |
7 | # This program is free software: you can redistribute it and/or modify # | |
8 | # it under the terms of the GNU General Public License as published by # | |
9 | # the Free Software Foundation, either version 2 of the License, or # | |
10 | # (at your option) any later version. # | |
11 | # # | |
12 | # This program is distributed in the hope that it will be useful, # | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of # | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # | |
15 | # GNU General Public License for more details. # | |
16 | # # | |
17 | # You should have received a copy of the GNU General Public License # | |
18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. # | |
19 | # # | |
20 | ############################################################################### | |
21 | ||
22 | package IPblocklist; | |
23 | ||
ba8455e4 SS |
24 | require '/var/ipfire/general-functions.pl'; |
25 | require "${General::swroot}/ipblocklist/sources"; | |
26 | ||
27 | # The directory where all ipblocklist related files and settings are stored. | |
28 | our $settings_dir = "/var/ipfire/ipblocklist"; | |
29 | ||
30 | # Main settings file. | |
31 | our $settings_file = "$settings_dir/settings"; | |
32 | ||
33 | # The file which keeps the time, when a blocklist last has been modified. | |
7f4829de | 34 | our $modified_file = "$settings_dir/modified"; |
f4c25a44 | 35 | |
e64587a4 | 36 | # Location where the blocklists in ipset compatible format are stored. |
7f4829de | 37 | our $blocklist_dir = "/var/lib/ipblocklist"; |
e64587a4 SS |
38 | |
39 | # File extension of the blocklist files. | |
ba8455e4 SS |
40 | my $blocklist_file_extension = ".conf"; |
41 | ||
42 | # Hash which calls the correct parser functions. | |
43 | my %parsers = ( | |
44 | 'ip-or-net-list' => \&parse_ip_or_net_list, | |
45 | 'dshield' => \&parse_dshield | |
46 | ); | |
e64587a4 | 47 | |
f4c25a44 SS |
48 | # |
49 | ## Function to get all available blocklists. | |
50 | # | |
51 | sub get_blocklists () { | |
52 | my @blocklists; | |
53 | ||
54 | # Loop through the hash of blocklists. | |
55 | foreach my $blocklist ( keys %IPblocklist::List::sources ) { | |
56 | # Add the list to the array. | |
57 | push(@blocklists, $blocklist); | |
58 | } | |
59 | ||
60 | # Sort and return the array. | |
61 | return sort(@blocklists); | |
62 | } | |
63 | ||
e64587a4 SS |
64 | # |
65 | ## Tiny function to get the full path and name of a given blocklist. | |
66 | # | |
67 | sub get_ipset_db_file($) { | |
68 | my ($set) = @_; | |
69 | ||
70 | # Generate the | |
71 | my $file = "$blocklist_dir/$set$blocklist_file_extension"; | |
72 | ||
73 | # Return the file name. | |
74 | return $file; | |
75 | } | |
76 | ||
ba8455e4 SS |
77 | # |
78 | ## The main download_and_create blocklist function. | |
79 | ## | |
80 | ## Uses LWP to download a given blocklist. The If-Modified-Since header is | |
81 | ## specified in the request so that only updated lists are downloaded (providing | |
82 | ## that the server supports this functionality). | |
83 | ## | |
84 | ## Once downloaded the list gets parsed, converted and stored in an ipset compatible | |
85 | ## format. | |
86 | ## | |
87 | ## Parameters: | |
88 | ## list The name of the blocklist | |
89 | ## | |
90 | ## Returns: | |
91 | ## nothing - On success | |
92 | ## not_modified - In case the servers responds with "Not modified" (304) | |
93 | ## dl_error - If the requested blocklist could not be downloaded. | |
d7dd5653 SS |
94 | ## empty_list - The downloaded blocklist is empty, or the parser was not able to parse |
95 | ## it correctly. | |
ba8455e4 SS |
96 | # |
97 | sub download_and_create_blocklist($) { | |
98 | my ($list) = @_; | |
99 | ||
100 | # Check if the given blockist is known and data available. | |
101 | unless($IPblocklist::List::sources{$list}) { | |
102 | # No valid data for this blocklist - exit and return "1". | |
103 | return 1; | |
104 | } | |
105 | ||
106 | # The allowed maximum download size in bytes. | |
107 | my $max_dl_bytes = 10_485_760; | |
108 | ||
109 | # The amount of download attempts before giving up and | |
110 | # logging an error. | |
111 | my $max_dl_attempts = 5; | |
112 | ||
113 | # Read proxysettings. | |
114 | my %proxysettings=(); | |
115 | &General::readhash("${General::swroot}/proxy/settings", \%proxysettings); | |
116 | ||
117 | # Load required perl module to handle the download. | |
118 | use LWP::UserAgent; | |
119 | ||
120 | # Create a user agent for downloading the blacklist | |
121 | # Limit the download size for safety | |
122 | my $ua = LWP::UserAgent->new ( | |
123 | ssl_opts => { | |
124 | SSL_ca_file => '/etc/ssl/cert.pem', | |
125 | verify_hostname => 1, | |
126 | }, | |
127 | ||
128 | max_size => $max_dl_bytes, | |
129 | ); | |
130 | ||
131 | # Set timeout to 10 seconds. | |
132 | $ua->timeout(10); | |
133 | ||
134 | # Check if an upstream proxy is configured. | |
135 | if ($proxysettings{'UPSTREAM_PROXY'}) { | |
136 | my $proxy_url; | |
137 | ||
138 | $proxy_url = "http://"; | |
139 | ||
140 | # Check if the proxy requires authentication. | |
141 | if (($proxysettings{'UPSTREAM_USER'}) && ($proxysettings{'UPSTREAM_PASSWORD'})) { | |
142 | $proxy_url .= "$proxysettings{'UPSTREAM_USER'}\:$proxysettings{'UPSTREAM_PASSWORD'}\@"; | |
143 | } | |
144 | ||
145 | # Add proxy server address and port. | |
146 | $proxy_url .= $proxysettings{'UPSTREAM_PROXY'}; | |
147 | ||
148 | # Setup proxy settings. | |
149 | $ua->proxy(['http', 'https'], $proxy_url); | |
150 | } | |
151 | ||
152 | # Gather the details, when a list got modified last time. | |
153 | my %modified = (); | |
154 | ||
155 | # Read-in data if the file exists. | |
156 | &General::readhash($modified_file, \%modified ) if (-e $modified_file); | |
157 | ||
158 | # Get the last modified time for this list. | |
159 | my $last_modified = gmtime($modified{$list} || 0); | |
160 | ||
161 | my $dl_attempt = 1; | |
162 | my $response; | |
163 | ||
164 | # Download and rety on failure loop. | |
165 | while ($dl_attempt <= $max_dl_attempts) { | |
166 | # Try to determine if there is a newer blocklist since last time and grab it. | |
167 | $response = $ua->get($IPblocklist::List::sources{$list}{'url'}, 'If-Modified-Since' => $last_modified ); | |
168 | ||
169 | # Check if the download attempt was successfull. | |
170 | if ($response->is_success) { | |
171 | # We successfully grabbed the list - no more retries needed, break the loop. | |
172 | # Further process the script code. | |
173 | last; | |
174 | ||
175 | # Exit, if the server responds with "Not modified (304). | |
176 | } elsif ($response->code == 304) { | |
177 | # Exit and return "not modified". | |
178 | return "not_modified"; | |
179 | ||
180 | # Exit and log an erro | |
181 | } elsif ($dl_attempt eq $max_dl_attempts) { | |
182 | # Exit and return "dl_error". | |
183 | return "dl_error"; | |
184 | } | |
185 | ||
186 | # Increase download attempt counter. | |
187 | $dl_attempt++; | |
188 | } | |
189 | ||
190 | # Update the timestamp for the new or modified list. | |
d8113820 SS |
191 | if($response->last_modified) { |
192 | $modified{$list} = $response->last_modified; | |
193 | } else { | |
194 | $modified{$list} = time(); | |
195 | } | |
ba8455e4 SS |
196 | |
197 | # Write-back the modified timestamps. | |
198 | &General::writehash($modified_file, \%modified); | |
199 | ||
200 | # Parse and loop through the downloaded list. | |
201 | my @blocklist = (); | |
202 | ||
203 | # Get the responsible parser for the current list. | |
204 | my $parser = $parsers{$IPblocklist::List::sources{$list}{'parser'}}; | |
205 | ||
206 | # Loop through the grabbed raw list. | |
207 | foreach my $line (split /[\r\n]+/, $response->content) { | |
208 | # Remove newlines. | |
209 | chomp $line; | |
210 | ||
211 | # Call the parser and obtain the addess or network. | |
212 | my $address = &$parser($line); | |
213 | ||
214 | # Skip the line if it does not contain an address. | |
215 | next unless ($address and $address =~ m/\d+\.\d+\.\d+\.\d+/); | |
216 | ||
217 | # Check if we got a single address. | |
218 | if ($address =~ m|/32|) { | |
219 | # Add /32 as prefix. | |
220 | $address =~ s|/32||; | |
221 | } | |
222 | ||
223 | # Push the address/network to the blocklist array. | |
224 | push(@blocklist, $address); | |
225 | } | |
226 | ||
d7dd5653 SS |
227 | # Check if the content could be parsed correctly and the blocklist |
228 | # contains at least one item. | |
229 | unless(@blocklist) { | |
230 | # No entries - exit and return "empty_list". | |
231 | return "empty_list"; | |
232 | } | |
233 | ||
ba8455e4 SS |
234 | # Get amount of entries in the blocklist array. |
235 | my $list_entries = scalar(@blocklist); | |
236 | ||
237 | # Optain the filename for this blocklist to save. | |
238 | my $file = &get_ipset_db_file($list); | |
239 | ||
240 | # Open the file for writing. | |
241 | open(FILE, ">", "$file") or die "Could not write to $file. $!\n"; | |
242 | ||
243 | # Write file header. | |
244 | print FILE "#Autogenerated file. Any custom changes will be overwritten!\n\n"; | |
245 | ||
246 | # Calculate the hashsize for better list performance. | |
247 | my $hashsize = &_calculate_hashsize($list_entries); | |
248 | ||
249 | # Simply set the limit of list elements to the double of current list elements. | |
250 | my $maxelem = $list_entries *2; | |
251 | ||
404b5137 SS |
252 | # Add "v4" suffix to the list name. |
253 | $list = "$list" . "v4"; | |
254 | ||
ba8455e4 SS |
255 | # Write line to create the set. |
256 | # | |
257 | # We safely can use hash:net as type because it supports single addresses and networks. | |
258 | print FILE "create $list hash:net family inet hashsize $hashsize maxelem $maxelem -exist\n"; | |
259 | ||
260 | # Write line to flush the set itself during loading. | |
261 | print FILE "flush $list\n"; | |
262 | ||
263 | # Loop through the array which contains the blocklist. | |
264 | foreach my $entry (@blocklist) { | |
265 | # Add the entry to the list. | |
266 | print FILE "add $list $entry\n"; | |
267 | } | |
268 | ||
269 | # Close the file handle. | |
270 | close(FILE); | |
271 | ||
272 | # Finished. | |
273 | return; | |
274 | } | |
275 | ||
ab017961 TF |
276 | # |
277 | ## sub parse_ip_or_net_list( line ) | |
278 | ## | |
279 | ## Parses an input line, looking for lines starting with an IP Address or | |
280 | ### Network specification. | |
281 | ## | |
282 | ## Parameters: | |
283 | ## line The line to parse | |
284 | ## | |
285 | ## Returns: | |
286 | ## Either an IP Address or a null string | |
287 | # | |
288 | sub parse_ip_or_net_list( $ ) { | |
289 | my ($line) = @_; | |
290 | ||
291 | # Grab the IP address or network. | |
292 | $line =~ m|^(\d+\.\d+\.\d+\.\d+(?:/\d+)?)|; | |
293 | ||
294 | # Return the grabbed address. | |
295 | return $1; | |
296 | } | |
ac9b5d8e TF |
297 | |
298 | # | |
299 | ## sub parse_dshield( line ) | |
300 | ## | |
301 | ## Parses an input line removing comments. | |
302 | ## | |
303 | ## The format is: | |
304 | ## Start Addrs End Addrs Netmask Nb Attacks Network Name Country email | |
305 | ## We're only interested in the start address and netmask. | |
306 | ## | |
307 | ## Parameters: | |
308 | ## line The line to parse | |
309 | ## | |
310 | ## Returns: | |
311 | ## Either and IP Address or a null string | |
312 | # | |
313 | sub parse_dshield( $ ) { | |
314 | my ($line) = @_; | |
315 | ||
316 | # Skip coments. | |
317 | return "" if ($line =~ m/^\s*#/); | |
318 | ||
319 | $line =~ s/#.*$//; | |
320 | ||
321 | # |Start addrs | |End Addrs | |Mask | |
322 | $line =~ m|(\d+\.\d+\.\d+\.\d+(?:/\d+)?)\s+\d+\.\d+\.\d+\.\d+(?:/\d+)?\s+(\d+)|; | |
323 | ||
324 | # Return nothing if no start address could be grabbed. | |
325 | return unless ($1); | |
326 | ||
327 | # Add /32 as prefix for single addresses and return it. | |
328 | return "$1/32" unless ($2); | |
329 | ||
330 | # Return the obtained network. | |
331 | return "$1/$2"; | |
332 | } | |
0a4f60f2 TF |
333 | |
334 | # | |
335 | ## Helper function to proper calculate the hashsize. | |
336 | # | |
337 | sub _calculate_hashsize($) { | |
338 | my ($list_entries) = @_; | |
339 | ||
340 | my $hashsize = 1; | |
341 | $hashsize <<= 1 while ($hashsize < $list_entries); | |
342 | ||
343 | # Return the calculated hashsize. | |
344 | return $hashsize; | |
345 | } | |
346 | ||
8d12d12a TF |
347 | # |
348 | ## sub get_holdoff_rate(list) | |
349 | ## | |
350 | ## This function is used to get the holdoff rate in seconds for a desired provider, | |
351 | ## based on the configured rate limit in minutes (m), hours (h) or days (d) in the | |
352 | ## blacklist sources settings file. | |
353 | ## | |
354 | # | |
355 | sub get_holdoff_rate($) { | |
356 | my ($list) = @_; | |
357 | ||
358 | # Grab the configured lookup rate for the given list. | |
359 | my $rate = $IPblocklist::List::sources{$list}{'rate'}; | |
360 | ||
361 | # Split the grabbed rate into value and unit. | |
362 | my ($value, $unit) = (uc $rate) =~ m/(\d+)([DHM]?)/; | |
363 | ||
364 | # Days | |
365 | if ($unit eq 'D') { | |
366 | $value *= 60 * 60 * 24; | |
367 | ||
368 | # Minutes | |
369 | } elsif ($unit eq 'M') { | |
370 | $value *= 60; | |
371 | ||
372 | # Everything else - assume hours. | |
373 | } else { | |
374 | $value *= 60 * 60; | |
375 | } | |
376 | ||
377 | # Sanity check - limit to range 5 min .. 1 week | |
378 | ||
379 | # d h m s | |
380 | $value = 5 * 60 if ($value < 5 * 60); | |
381 | $value = 7 * 24 * 60 * 60 if ($value > 7 * 24 * 60 * 60); | |
382 | ||
383 | return $value; | |
384 | } | |
385 | ||
f4c25a44 | 386 | 1; |