updxlrator: Add support for tar.xz files
[people/teissler/ipfire-2.x.git] / config / updxlrator / updxlrator
CommitLineData
46c01c09
MT
1#!/usr/bin/perl
2#
3# This code is distributed under the terms of the GPL
4#
4f4092e3 5# (c) 2006-2009 marco.s - http://update-accelerator.advproxy.net
46c01c09 6#
4f4092e3
CS
7# Portions (c) 2008 by dotzball - http://www.blockouttraffic.de
8#
9# $Id: updxlrator,v 2.1 2009/01/10 00:00:00 marco.s Exp $
46c01c09 10#
4f6b044f
JIW
11# ChangeLog:
12#
13# 2012-10-26: nightshift - move curly bracket to capture AVG download source.
14# - Adding BIG HINT for new update source#
15#
46c01c09
MT
16
17use strict;
0a638396 18use HTTP::Date;
46c01c09
MT
19
20$|=1;
21
22my $swroot="/var/ipfire";
381f2e71 23my $updcachedir="/var/updatecache";
0a638396
CS
24my $apphome="/var/ipfire/updatexlrator";
25my $logfile="/var/log/updatexlrator/cache.log";
4f4092e3 26my $wget="/usr/bin/wget";
0a638396
CS
27my $debug=(-e "$apphome/debug");
28my $http_port='81';
46c01c09 29my %netsettings=();
0a638396 30my %proxysettings=();
46c01c09 31my %xlratorsettings=();
46c01c09
MT
32my $logging=0;
33my $passive_mode=0;
34my $maxusage=0;
35my $nice='';
36my @tmp=();
46c01c09 37my $request='';
0a638396
CS
38my $xlrator_url=0;
39my $source_url='';
46c01c09
MT
40my $hostaddr='';
41my $username='';
42my $method='';
0a638396
CS
43my $unique = 0;
44my $mirror = 1;
46c01c09
MT
45
46readhash("${swroot}/ethernet/settings", \%netsettings);
47
48if (-e "$swroot/updatexlrator/settings")
49{
50 &readhash("$swroot/updatexlrator/settings", \%xlratorsettings);
51 if ($xlratorsettings{'ENABLE_LOG'} eq 'on') { $logging=1; };
52 if ($xlratorsettings{'PASSIVE_MODE'} eq 'on') { $passive_mode=1; };
53 $maxusage=$xlratorsettings{'MAX_DISK_USAGE'};
646750c5 54 if ($xlratorsettings{'LOW_DOWNLOAD_PRIORITY'} eq 'on') { $nice='/bin/nice --adjustment=15 '; };
46c01c09
MT
55}
56if (!$maxusage) { $maxusage=75; };
57
4f4092e3
CS
58# dotzball: check for dead downloads
59system("$apphome/bin/checkdeaddl &");
46c01c09
MT
60
61while (<>) {
62
63 $request=$_;
46c01c09
MT
64
65 @tmp=split(/ /,$request);
66 chomp(@tmp);
67
0a638396
CS
68 $source_url = $tmp[0];
69 $hostaddr = $tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; }
70 $username = $tmp[2]; if ($username eq '') { $username='-'; };
71 $method = $tmp[3];
72
73 $xlrator_url = $source_url;
46c01c09
MT
74
75 if (($method eq 'GET') || ($method eq 'HEAD'))
76 {
77
78 # -----------------------------------------------------------
79 # Section: Windows Update / Windows Downloads
80 # -----------------------------------------------------------
81
82 if (
bc689853
JIW
83 (($source_url =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi|msp|msu|cab)$@i) ||
84 ($source_url =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|msi|msp|msu|cab)$@i))
0a638396
CS
85 && ($source_url !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i)
86 && ($source_url !~ m@\&@)
46c01c09
MT
87 )
88 {
0a638396 89 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique);
46c01c09
MT
90 }
91
92 # -----------------------------------------------------------
93 # Section: Adobe Downloads
94 # -----------------------------------------------------------
95
4f6b044f 96
0a638396
CS
97 if (
98 ($source_url =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|msi|bin|dmg|idx|gz)$@i) ||
99 ($source_url =~ m@^http://swupdl\.adobe\.com/updates/.*\.(exe|msi|bin|dmg|idx|gz|[a-z][a-z]_[A-Z][A-Z])$@i) ||
100 ($source_url =~ m@^http://swupmf\.adobe\.com/manifest/.*\.upd$@i)
101 )
102 {
103 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Adobe",$unique);
104 }
105
106 # -----------------------------------------------------------
107 # Section: Linux Downloads
108 # -----------------------------------------------------------
109
8dbba0e4 110 if (
77fa792f 111 ($source_url =~ m@^[h|f]t?tp://[^?]+\.(pkg\.tar\.xz|deb|rpm)$@i) ||
1235b1f1 112 ($source_url =~ m@^[h|f]t?tp://[^?]+/distfiles/[^?]+\.(tar\.gz|tar\.bz2|tar\.xz|tgz|zip|patch\.bz2|gz|docx|patch|pdf|exe)$@i)
8dbba0e4 113 )
46c01c09 114 {
0a638396
CS
115 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Linux",$mirror);
116 }
117
118 # -----------------------------------------------------------
119 # Section: Trend Micro Downloads
120 # -----------------------------------------------------------
121
122 if (
123 ($source_url =~ m@^http://[^/]*\.trendmicro\.com/activeupdate/.*@i) &&
124 ($source_url !~ m@.*/tmnotify\.dat$@i) &&
125 ($source_url !~ m@.*/ini_xml\.zip$@i) &&
126 ($source_url !~ m@.*/server\.ini$@i)
127 )
128 {
129 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"TrendMicro",$mirror);
46c01c09
MT
130 }
131
132 # -----------------------------------------------------------
133 # Section: Symantec Downloads
134 # -----------------------------------------------------------
135
0a638396 136 if ($source_url =~ m@^[h|f]tt?p://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|vdb|xdb)$@i)
46c01c09 137 {
0a638396 138 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Symantec",$unique);
46c01c09 139 }
0a638396 140
5b2a12ff 141 # -----------------------------------------------------------
0a638396 142 # Section: Apple Downloads
46c01c09 143 # -----------------------------------------------------------
0a638396
CS
144
145 if (
4f4092e3 146 (($source_url =~ m@^http://swcdn\.apple\.com/content/downloads/.*\.(tar|pkg)$@i) ||
0a638396
CS
147 ($source_url =~ m@^http://appldnld\.apple\.com\.edgesuite\.net/.*\.(exe|dmg)$@i) ||
148 ($source_url =~ m@^http://.*\.g.akamai.net/.*/3093/1/.*\.(tar|pkg|dmg|exe)$@i))
149 )
5b2a12ff 150 {
0a638396 151 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Apple",$unique);
5b2a12ff 152 }
0a638396
CS
153
154 # -----------------------------------------------------------
155 # Section: Avast Downloads
156 # -----------------------------------------------------------
157
5150d373 158 if ($source_url =~ m@^http://(ion|download)[\d]+\.avast\.com/.*\.(exe|vpu|vpx)$@i)
0a638396
CS
159 {
160 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avast",$mirror);
186e3d2c 161 }
0a638396 162
219c963e
JIW
163 # -----------------------------------------------------------
164 # Section: Mozilla Downloads
165 # -----------------------------------------------------------
166
167 if ($source_url=~ m@^http://.*\.mozilla\.net/.*\.((complete|partial)\.mar|exe)$@i)
168 {
169 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Mozilla",$unique);
170 }
171
0a638396 172 # -----------------------------------------------------------
7b59d838 173 # Section: Mcafee
0a638396 174 # -----------------------------------------------------------
186e3d2c 175
7b59d838
JIW
176 if ($source_url =~ m@^http://update\.nai\.com/.*\.(mcs|z|gem|dat|zip)$@i)
177 {
178 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"mcafee",$mirror);
179 }
186e3d2c 180
0a638396
CS
181 # -----------------------------------------------------------
182 # Section: Avira Downloads
183 # -----------------------------------------------------------
184
8dbba0e4
CS
185 if (
186 ($source_url =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i) ||
187 ($source_url =~ m@^http://80.190.130.19[4-5]/update/.*\.(htm|html|gz)$@i) ||
188 ($source_url =~ m@^http://62.146.64.14[6-7]/update/.*\.(htm|html|gz)$@i)
189 )
0a638396
CS
190 {
191 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avira",$mirror);
2dfb38e6 192 }
4ae86474
AF
193
194 # -----------------------------------------------------------
0a638396 195 # Section: IPFire Downloads
4ae86474
AF
196 # -----------------------------------------------------------
197
0a638396 198 if ($source_url =~ m@^[f|h]t?tp://.*\.(ipfire)$@i)
4ae86474 199 {
0a638396 200 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"IPFire",$mirror);
4ae86474
AF
201 }
202
0a638396
CS
203 # -----------------------------------------------------------
204 # Section: AVG Downloads
205 # -----------------------------------------------------------
46c01c09 206
4f4092e3
CS
207# if ($source_url =~ m@^http://[^/]*\.(grisoft|avg)\.com/.*\.(bin)$@i)
208# {
209# $xlrator_url = &check_cache($source_url,$hostaddr,$username,"AVG",$mirror);
210# }
46c01c09 211
4f6b044f
JIW
212# ----------- ADD NEW SOURCES BEFORE THIS LINE !!! ------------------
213 }
214
0a638396 215 $request="$xlrator_url $hostaddr $username $method\n";
46c01c09
MT
216
217 print $request;
218}
219
220# -------------------------------------------------------------------
221
222sub readhash
223{
224 my $filename = $_[0];
225 my $hash = $_[1];
226 my ($var, $val);
227
228 if (-e $filename)
229 {
230 open(FILE, $filename) or die "Unable to read file $filename";
231 while (<FILE>)
232 {
233 chop;
234 ($var, $val) = split /=/, $_, 2;
235 if ($var)
236 {
237 $val =~ s/^\'//g;
238 $val =~ s/\'$//g;
239
240 # Untaint variables read from hash
241 $var =~ /([A-Za-z0-9_-]*)/; $var = $1;
242 $val =~ /([\w\W]*)/; $val = $1;
243 $hash->{$var} = $val;
244 }
245 }
246 close FILE;
247 }
248}
249
250# -------------------------------------------------------------------
251
252sub writelog
253{
0a638396
CS
254 if ($logging)
255 {
256 open(LOGFILE,">>$logfile");
257 print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n";
258 close(LOGFILE);
259 }
46c01c09
MT
260}
261
262# -------------------------------------------------------------------
263
0a638396 264sub debuglog
46c01c09 265{
0a638396 266 if ($debug)
46c01c09 267 {
0a638396
CS
268 open(LOGFILE,">>/var/log/updatexlrator/debug.log");
269 my @now = localtime(time);
270 printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] [%s] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,"updxlrator",$_[0];
271 close(LOGFILE);
46c01c09 272 }
0a638396
CS
273}
274
275# -------------------------------------------------------------------
276
277sub setcachestatus
278{
279 open (FILE,">>$_[0]");
280 print FILE "$_[1]\n";
281 close FILE;
282}
283
284# -------------------------------------------------------------------
285
286sub diskfree
287{
288 open(DF,"/bin/df --block-size=1 $_[0]|");
289 my @dfdata = <DF>;
46c01c09 290 close DF;
0a638396
CS
291 shift(@dfdata);
292 chomp(@dfdata);
293 my $dfstr = join(' ',@dfdata);
294 my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
295 if ($free =~ m/^(\d+)$/)
296 {
297 return $free;
298 }
46c01c09
MT
299}
300
301# -------------------------------------------------------------------
302
303sub diskusage
304{
305 open(DF,"/bin/df $_[0]|");
0a638396
CS
306 my @dfdata = <DF>;
307 close DF;
308 shift(@dfdata);
309 chomp(@dfdata);
310 my $dfstr = join(' ',@dfdata);
311 my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
312 if ($percent =~ m/^(\d+)%$/)
46c01c09 313 {
0a638396
CS
314 $percent =~ s/%$//;
315 return $percent;
46c01c09 316 }
46c01c09
MT
317}
318
319# -------------------------------------------------------------------
320
0a638396 321sub getmtime
46c01c09 322{
0a638396 323 my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($_[0]);
46c01c09 324
0a638396
CS
325 return $mtime;
326}
46c01c09 327
0a638396 328# -------------------------------------------------------------------
46c01c09 329
0a638396
CS
330sub check_cache
331{
332 my $updsource="UPDCACHE";
333 my $updfile='';
334 my $cacheurl='';
335 my $vendorid='';
336 my $uuid='';
337 my @http_header=();
338 my $remote_size=0;
339 my $remote_mtime=0;
340 my $login='';
341 my $useragent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
342
343 my $sourceurl=$_[0];
344 my $cfmirror=$_[4];
345
346 $sourceurl =~ s@\%2f@/@ig;
347 $updfile = substr($sourceurl,rindex($sourceurl,"/")+1);
348 $updfile =~ s@\%20@ @ig;
349
350 if ($cfmirror)
46c01c09 351 {
0a638396
CS
352 $uuid = `echo $updfile | md5sum`;
353 } else {
354 $uuid = `echo $sourceurl | md5sum`;
46c01c09
MT
355 }
356
0a638396
CS
357 $uuid =~ s/[^0-9a-f]//g;
358 $uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/;
359
360 $vendorid = $_[3];
361 $vendorid =~ tr/A-Z/a-z/;
362
363 &debuglog("Processing URL $sourceurl");
364 &debuglog("Vendor ID is $vendorid");
365 &debuglog("UUID is $uuid");
366
367 if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'}))
46c01c09 368 {
0a638396
CS
369 $login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\"";
370 if ($proxysettings{'UPSTREAM_PASSWORD'})
46c01c09 371 {
0a638396 372 $login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\"";
46c01c09
MT
373 }
374 }
46c01c09 375
0a638396 376 if ($proxysettings{'UPSTREAM_PROXY'}) { &debuglog("Using upstream proxy $proxysettings{'UPSTREAM_PROXY'}"); }
46c01c09 377
0a638396 378 $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
4f4092e3 379 @http_header = `$wget $login --user-agent="$useragent" --spider -S $sourceurl 2>&1`;
0a638396 380 $ENV{'http_proxy'} = '';
46c01c09 381
0a638396 382 foreach (@http_header)
46c01c09 383 {
0a638396
CS
384 chomp;
385 if (/^\s*Content-Length:\s/) { $remote_size = $_; $remote_size =~ s/[^0-9]//g; }
386 if (/^\s*Last-Modified:\s/) { $remote_mtime = $_; $remote_mtime =~ s/^\s*Last-Modified:\s//; $remote_mtime = HTTP::Date::str2time($remote_mtime) }
46c01c09
MT
387 }
388
0a638396
CS
389 if (
390 (-e "$updcachedir/$vendorid/$uuid/$updfile") &&
391 ($remote_size == (-s "$updcachedir/$vendorid/$uuid/$updfile")) &&
392 ($remote_mtime == &getmtime("$updcachedir/$vendorid/$uuid/$updfile"))
393 )
46c01c09 394 {
0a638396
CS
395 &debuglog("File exists in cache and is up to date");
396 &debuglog("Retrieving file from cache ($updsource)");
397 &setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time);
0a638396 398 $cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile";
46c01c09
MT
399 }
400 else
401 {
0a638396
CS
402 if (-e "$updcachedir/$vendorid/$uuid/$updfile")
403 {
404 &debuglog("Local filesize: " . (-s "$updcachedir/$vendorid/$uuid/$updfile"));
405 &debuglog("Local timestamp: " . &getmtime("$updcachedir/$vendorid/$uuid/$updfile"));
406 } else { &debuglog("File not found in cache"); }
46c01c09 407 $updsource="DLSOURCE";
0a638396
CS
408 &debuglog("Remote filesize: $remote_size");
409 &debuglog("Remote timestamp: $remote_mtime");
410 &debuglog("Free disk space: " . &diskfree($updcachedir));
411 &debuglog("Disk usage: " . &diskusage($updcachedir) . "% (max. $maxusage%)");
412 if (-e "$updcachedir/download/$vendorid/$updfile") { &debuglog("File download/$vendorid/$updfile exists"); }
413 &debuglog("Retrieving file from source ($updsource)");
414 if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && ($remote_size <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$vendorid/$updfile"))
46c01c09 415 {
0a638396
CS
416 &debuglog("Running command $nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
417 system("$nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
46c01c09 418 }
0a638396 419 $cacheurl=$sourceurl;
46c01c09
MT
420 }
421
0a638396 422 &writelog($_[1],$_[2],$_[3],$updsource,$sourceurl);
46c01c09 423
0a638396 424 return $cacheurl;
46c01c09
MT
425}
426
427# -------------------------------------------------------------------