Started core37.
[people/pmueller/ipfire-2.x.git] / config / updxlrator / updxlrator
CommitLineData
46c01c09
MT
1#!/usr/bin/perl
2#
3# This code is distributed under the terms of the GPL
4#
4f4092e3 5# (c) 2006-2009 marco.s - http://update-accelerator.advproxy.net
46c01c09 6#
4f4092e3
CS
7# Portions (c) 2008 by dotzball - http://www.blockouttraffic.de
8#
9# $Id: updxlrator,v 2.1 2009/01/10 00:00:00 marco.s Exp $
46c01c09
MT
10#
11
12use strict;
0a638396 13use HTTP::Date;
46c01c09
MT
14
15$|=1;
16
17my $swroot="/var/ipfire";
381f2e71 18my $updcachedir="/var/updatecache";
0a638396
CS
19my $apphome="/var/ipfire/updatexlrator";
20my $logfile="/var/log/updatexlrator/cache.log";
4f4092e3 21my $wget="/usr/bin/wget";
0a638396
CS
22my $debug=(-e "$apphome/debug");
23my $http_port='81';
46c01c09 24my %netsettings=();
0a638396 25my %proxysettings=();
46c01c09 26my %xlratorsettings=();
46c01c09
MT
27my $logging=0;
28my $passive_mode=0;
29my $maxusage=0;
30my $nice='';
31my @tmp=();
46c01c09 32my $request='';
0a638396
CS
33my $xlrator_url=0;
34my $source_url='';
46c01c09
MT
35my $hostaddr='';
36my $username='';
37my $method='';
0a638396
CS
38my $unique = 0;
39my $mirror = 1;
46c01c09
MT
40
41readhash("${swroot}/ethernet/settings", \%netsettings);
42
43if (-e "$swroot/updatexlrator/settings")
44{
45 &readhash("$swroot/updatexlrator/settings", \%xlratorsettings);
46 if ($xlratorsettings{'ENABLE_LOG'} eq 'on') { $logging=1; };
47 if ($xlratorsettings{'PASSIVE_MODE'} eq 'on') { $passive_mode=1; };
48 $maxusage=$xlratorsettings{'MAX_DISK_USAGE'};
646750c5 49 if ($xlratorsettings{'LOW_DOWNLOAD_PRIORITY'} eq 'on') { $nice='/bin/nice --adjustment=15 '; };
46c01c09
MT
50}
51if (!$maxusage) { $maxusage=75; };
52
4f4092e3
CS
53# dotzball: check for dead downloads
54system("$apphome/bin/checkdeaddl &");
46c01c09
MT
55
56while (<>) {
57
58 $request=$_;
46c01c09
MT
59
60 @tmp=split(/ /,$request);
61 chomp(@tmp);
62
0a638396
CS
63 $source_url = $tmp[0];
64 $hostaddr = $tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; }
65 $username = $tmp[2]; if ($username eq '') { $username='-'; };
66 $method = $tmp[3];
67
68 $xlrator_url = $source_url;
46c01c09
MT
69
70 if (($method eq 'GET') || ($method eq 'HEAD'))
71 {
72
73 # -----------------------------------------------------------
74 # Section: Windows Update / Windows Downloads
75 # -----------------------------------------------------------
76
77 if (
0a638396
CS
78 (($source_url =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi|msp|cab)$@i) ||
79 ($source_url =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|msi|msp|cab)$@i))
80 && ($source_url !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i)
81 && ($source_url !~ m@\&@)
46c01c09
MT
82 )
83 {
0a638396 84 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique);
46c01c09
MT
85 }
86
87 # -----------------------------------------------------------
88 # Section: Adobe Downloads
89 # -----------------------------------------------------------
90
0a638396
CS
91 if (
92 ($source_url =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|msi|bin|dmg|idx|gz)$@i) ||
93 ($source_url =~ m@^http://swupdl\.adobe\.com/updates/.*\.(exe|msi|bin|dmg|idx|gz|[a-z][a-z]_[A-Z][A-Z])$@i) ||
94 ($source_url =~ m@^http://swupmf\.adobe\.com/manifest/.*\.upd$@i)
95 )
96 {
97 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Adobe",$unique);
98 }
99
100 # -----------------------------------------------------------
101 # Section: Linux Downloads
102 # -----------------------------------------------------------
103
104 if ($source_url =~ m@^[h|f]t?tp://[^?]+\.(deb|rpm)$@i)
46c01c09 105 {
0a638396
CS
106 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Linux",$mirror);
107 }
108
109 # -----------------------------------------------------------
110 # Section: Trend Micro Downloads
111 # -----------------------------------------------------------
112
113 if (
114 ($source_url =~ m@^http://[^/]*\.trendmicro\.com/activeupdate/.*@i) &&
115 ($source_url !~ m@.*/tmnotify\.dat$@i) &&
116 ($source_url !~ m@.*/ini_xml\.zip$@i) &&
117 ($source_url !~ m@.*/server\.ini$@i)
118 )
119 {
120 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"TrendMicro",$mirror);
46c01c09
MT
121 }
122
123 # -----------------------------------------------------------
124 # Section: Symantec Downloads
125 # -----------------------------------------------------------
126
0a638396 127 if ($source_url =~ m@^[h|f]tt?p://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|vdb|xdb)$@i)
46c01c09 128 {
0a638396 129 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Symantec",$unique);
46c01c09 130 }
0a638396 131
5b2a12ff 132 # -----------------------------------------------------------
0a638396 133 # Section: Apple Downloads
46c01c09 134 # -----------------------------------------------------------
0a638396
CS
135
136 if (
4f4092e3 137 (($source_url =~ m@^http://swcdn\.apple\.com/content/downloads/.*\.(tar|pkg)$@i) ||
0a638396
CS
138 ($source_url =~ m@^http://appldnld\.apple\.com\.edgesuite\.net/.*\.(exe|dmg)$@i) ||
139 ($source_url =~ m@^http://.*\.g.akamai.net/.*/3093/1/.*\.(tar|pkg|dmg|exe)$@i))
140 )
5b2a12ff 141 {
0a638396 142 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Apple",$unique);
5b2a12ff 143 }
0a638396
CS
144
145 # -----------------------------------------------------------
146 # Section: Avast Downloads
147 # -----------------------------------------------------------
148
149 if ($source_url =~ m@^http://download[\d]+\.avast\.com/.*\.(exe|vpu)$@i)
150 {
151 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avast",$mirror);
186e3d2c 152 }
0a638396
CS
153
154 # -----------------------------------------------------------
186e3d2c 155
0a638396
CS
156 # -----------------------------------------------------------
157 # Section: Avira Downloads
158 # -----------------------------------------------------------
159
160 if ($source_url =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i)
161 {
162 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avira",$mirror);
2dfb38e6 163 }
4ae86474
AF
164
165 # -----------------------------------------------------------
0a638396 166 # Section: IPFire Downloads
4ae86474
AF
167 # -----------------------------------------------------------
168
0a638396 169 if ($source_url =~ m@^[f|h]t?tp://.*\.(ipfire)$@i)
4ae86474 170 {
0a638396 171 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"IPFire",$mirror);
4ae86474
AF
172 }
173
0a638396 174 }
2dfb38e6 175
0a638396
CS
176 # -----------------------------------------------------------
177 # Section: AVG Downloads
178 # -----------------------------------------------------------
46c01c09 179
4f4092e3
CS
180# if ($source_url =~ m@^http://[^/]*\.(grisoft|avg)\.com/.*\.(bin)$@i)
181# {
182# $xlrator_url = &check_cache($source_url,$hostaddr,$username,"AVG",$mirror);
183# }
46c01c09 184
0a638396 185 $request="$xlrator_url $hostaddr $username $method\n";
46c01c09
MT
186
187 print $request;
188}
189
190# -------------------------------------------------------------------
191
192sub readhash
193{
194 my $filename = $_[0];
195 my $hash = $_[1];
196 my ($var, $val);
197
198 if (-e $filename)
199 {
200 open(FILE, $filename) or die "Unable to read file $filename";
201 while (<FILE>)
202 {
203 chop;
204 ($var, $val) = split /=/, $_, 2;
205 if ($var)
206 {
207 $val =~ s/^\'//g;
208 $val =~ s/\'$//g;
209
210 # Untaint variables read from hash
211 $var =~ /([A-Za-z0-9_-]*)/; $var = $1;
212 $val =~ /([\w\W]*)/; $val = $1;
213 $hash->{$var} = $val;
214 }
215 }
216 close FILE;
217 }
218}
219
220# -------------------------------------------------------------------
221
222sub writelog
223{
0a638396
CS
224 if ($logging)
225 {
226 open(LOGFILE,">>$logfile");
227 print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n";
228 close(LOGFILE);
229 }
46c01c09
MT
230}
231
232# -------------------------------------------------------------------
233
0a638396 234sub debuglog
46c01c09 235{
0a638396 236 if ($debug)
46c01c09 237 {
0a638396
CS
238 open(LOGFILE,">>/var/log/updatexlrator/debug.log");
239 my @now = localtime(time);
240 printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] [%s] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,"updxlrator",$_[0];
241 close(LOGFILE);
46c01c09 242 }
0a638396
CS
243}
244
245# -------------------------------------------------------------------
246
247sub setcachestatus
248{
249 open (FILE,">>$_[0]");
250 print FILE "$_[1]\n";
251 close FILE;
252}
253
254# -------------------------------------------------------------------
255
256sub diskfree
257{
258 open(DF,"/bin/df --block-size=1 $_[0]|");
259 my @dfdata = <DF>;
46c01c09 260 close DF;
0a638396
CS
261 shift(@dfdata);
262 chomp(@dfdata);
263 my $dfstr = join(' ',@dfdata);
264 my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
265 if ($free =~ m/^(\d+)$/)
266 {
267 return $free;
268 }
46c01c09
MT
269}
270
271# -------------------------------------------------------------------
272
273sub diskusage
274{
275 open(DF,"/bin/df $_[0]|");
0a638396
CS
276 my @dfdata = <DF>;
277 close DF;
278 shift(@dfdata);
279 chomp(@dfdata);
280 my $dfstr = join(' ',@dfdata);
281 my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
282 if ($percent =~ m/^(\d+)%$/)
46c01c09 283 {
0a638396
CS
284 $percent =~ s/%$//;
285 return $percent;
46c01c09 286 }
46c01c09
MT
287}
288
289# -------------------------------------------------------------------
290
0a638396 291sub getmtime
46c01c09 292{
0a638396 293 my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($_[0]);
46c01c09 294
0a638396
CS
295 return $mtime;
296}
46c01c09 297
0a638396 298# -------------------------------------------------------------------
46c01c09 299
0a638396
CS
300sub check_cache
301{
302 my $updsource="UPDCACHE";
303 my $updfile='';
304 my $cacheurl='';
305 my $vendorid='';
306 my $uuid='';
307 my @http_header=();
308 my $remote_size=0;
309 my $remote_mtime=0;
310 my $login='';
311 my $useragent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
312
313 my $sourceurl=$_[0];
314 my $cfmirror=$_[4];
315
316 $sourceurl =~ s@\%2f@/@ig;
317 $updfile = substr($sourceurl,rindex($sourceurl,"/")+1);
318 $updfile =~ s@\%20@ @ig;
319
320 if ($cfmirror)
46c01c09 321 {
0a638396
CS
322 $uuid = `echo $updfile | md5sum`;
323 } else {
324 $uuid = `echo $sourceurl | md5sum`;
46c01c09
MT
325 }
326
0a638396
CS
327 $uuid =~ s/[^0-9a-f]//g;
328 $uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/;
329
330 $vendorid = $_[3];
331 $vendorid =~ tr/A-Z/a-z/;
332
333 &debuglog("Processing URL $sourceurl");
334 &debuglog("Vendor ID is $vendorid");
335 &debuglog("UUID is $uuid");
336
337 if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'}))
46c01c09 338 {
0a638396
CS
339 $login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\"";
340 if ($proxysettings{'UPSTREAM_PASSWORD'})
46c01c09 341 {
0a638396 342 $login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\"";
46c01c09
MT
343 }
344 }
46c01c09 345
0a638396 346 if ($proxysettings{'UPSTREAM_PROXY'}) { &debuglog("Using upstream proxy $proxysettings{'UPSTREAM_PROXY'}"); }
46c01c09 347
0a638396 348 $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
4f4092e3 349 @http_header = `$wget $login --user-agent="$useragent" --spider -S $sourceurl 2>&1`;
0a638396 350 $ENV{'http_proxy'} = '';
46c01c09 351
0a638396 352 foreach (@http_header)
46c01c09 353 {
0a638396
CS
354 chomp;
355 if (/^\s*Content-Length:\s/) { $remote_size = $_; $remote_size =~ s/[^0-9]//g; }
356 if (/^\s*Last-Modified:\s/) { $remote_mtime = $_; $remote_mtime =~ s/^\s*Last-Modified:\s//; $remote_mtime = HTTP::Date::str2time($remote_mtime) }
46c01c09
MT
357 }
358
0a638396
CS
359 if (
360 (-e "$updcachedir/$vendorid/$uuid/$updfile") &&
361 ($remote_size == (-s "$updcachedir/$vendorid/$uuid/$updfile")) &&
362 ($remote_mtime == &getmtime("$updcachedir/$vendorid/$uuid/$updfile"))
363 )
46c01c09 364 {
0a638396
CS
365 &debuglog("File exists in cache and is up to date");
366 &debuglog("Retrieving file from cache ($updsource)");
367 &setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time);
0a638396 368 $cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile";
46c01c09
MT
369 }
370 else
371 {
0a638396
CS
372 if (-e "$updcachedir/$vendorid/$uuid/$updfile")
373 {
374 &debuglog("Local filesize: " . (-s "$updcachedir/$vendorid/$uuid/$updfile"));
375 &debuglog("Local timestamp: " . &getmtime("$updcachedir/$vendorid/$uuid/$updfile"));
376 } else { &debuglog("File not found in cache"); }
46c01c09 377 $updsource="DLSOURCE";
0a638396
CS
378 &debuglog("Remote filesize: $remote_size");
379 &debuglog("Remote timestamp: $remote_mtime");
380 &debuglog("Free disk space: " . &diskfree($updcachedir));
381 &debuglog("Disk usage: " . &diskusage($updcachedir) . "% (max. $maxusage%)");
382 if (-e "$updcachedir/download/$vendorid/$updfile") { &debuglog("File download/$vendorid/$updfile exists"); }
383 &debuglog("Retrieving file from source ($updsource)");
384 if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && ($remote_size <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$vendorid/$updfile"))
46c01c09 385 {
0a638396
CS
386 &debuglog("Running command $nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
387 system("$nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
46c01c09 388 }
0a638396 389 $cacheurl=$sourceurl;
46c01c09
MT
390 }
391
0a638396 392 &writelog($_[1],$_[2],$_[3],$updsource,$sourceurl);
46c01c09 393
0a638396 394 return $cacheurl;
46c01c09
MT
395}
396
397# -------------------------------------------------------------------