]> git.ipfire.org Git - people/dweismueller/ipfire-2.x.git/blob - config/updxlrator/updxlrator
Fixed Avira Update Caching and improved Gentoo Caching.
[people/dweismueller/ipfire-2.x.git] / config / updxlrator / updxlrator
1 #!/usr/bin/perl
2 #
3 # This code is distributed under the terms of the GPL
4 #
5 # (c) 2006-2009 marco.s - http://update-accelerator.advproxy.net
6 #
7 # Portions (c) 2008 by dotzball - http://www.blockouttraffic.de
8 #
9 # $Id: updxlrator,v 2.1 2009/01/10 00:00:00 marco.s Exp $
10 #
11
12 use strict;
13 use HTTP::Date;
14
15 $|=1;
16
17 my $swroot="/var/ipfire";
18 my $updcachedir="/var/updatecache";
19 my $apphome="/var/ipfire/updatexlrator";
20 my $logfile="/var/log/updatexlrator/cache.log";
21 my $wget="/usr/bin/wget";
22 my $debug=(-e "$apphome/debug");
23 my $http_port='81';
24 my %netsettings=();
25 my %proxysettings=();
26 my %xlratorsettings=();
27 my $logging=0;
28 my $passive_mode=0;
29 my $maxusage=0;
30 my $nice='';
31 my @tmp=();
32 my $request='';
33 my $xlrator_url=0;
34 my $source_url='';
35 my $hostaddr='';
36 my $username='';
37 my $method='';
38 my $unique = 0;
39 my $mirror = 1;
40
41 readhash("${swroot}/ethernet/settings", \%netsettings);
42
43 if (-e "$swroot/updatexlrator/settings")
44 {
45 &readhash("$swroot/updatexlrator/settings", \%xlratorsettings);
46 if ($xlratorsettings{'ENABLE_LOG'} eq 'on') { $logging=1; };
47 if ($xlratorsettings{'PASSIVE_MODE'} eq 'on') { $passive_mode=1; };
48 $maxusage=$xlratorsettings{'MAX_DISK_USAGE'};
49 if ($xlratorsettings{'LOW_DOWNLOAD_PRIORITY'} eq 'on') { $nice='/bin/nice --adjustment=15 '; };
50 }
51 if (!$maxusage) { $maxusage=75; };
52
53 # dotzball: check for dead downloads
54 system("$apphome/bin/checkdeaddl &");
55
56 while (<>) {
57
58 $request=$_;
59
60 @tmp=split(/ /,$request);
61 chomp(@tmp);
62
63 $source_url = $tmp[0];
64 $hostaddr = $tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; }
65 $username = $tmp[2]; if ($username eq '') { $username='-'; };
66 $method = $tmp[3];
67
68 $xlrator_url = $source_url;
69
70 if (($method eq 'GET') || ($method eq 'HEAD'))
71 {
72
73 # -----------------------------------------------------------
74 # Section: Windows Update / Windows Downloads
75 # -----------------------------------------------------------
76
77 if (
78 (($source_url =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi|msp|cab)$@i) ||
79 ($source_url =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|msi|msp|cab)$@i))
80 && ($source_url !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i)
81 && ($source_url !~ m@\&@)
82 )
83 {
84 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique);
85 }
86
87 # -----------------------------------------------------------
88 # Section: Adobe Downloads
89 # -----------------------------------------------------------
90
91 if (
92 ($source_url =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|msi|bin|dmg|idx|gz)$@i) ||
93 ($source_url =~ m@^http://swupdl\.adobe\.com/updates/.*\.(exe|msi|bin|dmg|idx|gz|[a-z][a-z]_[A-Z][A-Z])$@i) ||
94 ($source_url =~ m@^http://swupmf\.adobe\.com/manifest/.*\.upd$@i)
95 )
96 {
97 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Adobe",$unique);
98 }
99
100 # -----------------------------------------------------------
101 # Section: Linux Downloads
102 # -----------------------------------------------------------
103
104 if (
105 ($source_url =~ m@^[h|f]t?tp://[^?]+\.(deb|rpm)$@i) ||
106 ($source_url =~ m@^[h|f]t?tp://[^?]+/distfiles/[^?]+\.(tar\.gz|tar\.bz2|tgz|zip|patch\.bz2|gz|docx|patch|pdf|exe)$@i)
107 )
108 {
109 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Linux",$mirror);
110 }
111
112 # -----------------------------------------------------------
113 # Section: Trend Micro Downloads
114 # -----------------------------------------------------------
115
116 if (
117 ($source_url =~ m@^http://[^/]*\.trendmicro\.com/activeupdate/.*@i) &&
118 ($source_url !~ m@.*/tmnotify\.dat$@i) &&
119 ($source_url !~ m@.*/ini_xml\.zip$@i) &&
120 ($source_url !~ m@.*/server\.ini$@i)
121 )
122 {
123 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"TrendMicro",$mirror);
124 }
125
126 # -----------------------------------------------------------
127 # Section: Symantec Downloads
128 # -----------------------------------------------------------
129
130 if ($source_url =~ m@^[h|f]tt?p://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|vdb|xdb)$@i)
131 {
132 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Symantec",$unique);
133 }
134
135 # -----------------------------------------------------------
136 # Section: Apple Downloads
137 # -----------------------------------------------------------
138
139 if (
140 (($source_url =~ m@^http://swcdn\.apple\.com/content/downloads/.*\.(tar|pkg)$@i) ||
141 ($source_url =~ m@^http://appldnld\.apple\.com\.edgesuite\.net/.*\.(exe|dmg)$@i) ||
142 ($source_url =~ m@^http://.*\.g.akamai.net/.*/3093/1/.*\.(tar|pkg|dmg|exe)$@i))
143 )
144 {
145 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Apple",$unique);
146 }
147
148 # -----------------------------------------------------------
149 # Section: Avast Downloads
150 # -----------------------------------------------------------
151
152 if ($source_url =~ m@^http://download[\d]+\.avast\.com/.*\.(exe|vpu)$@i)
153 {
154 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avast",$mirror);
155 }
156
157 # -----------------------------------------------------------
158
159 # -----------------------------------------------------------
160 # Section: Avira Downloads
161 # -----------------------------------------------------------
162
163 if (
164 ($source_url =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i) ||
165 ($source_url =~ m@^http://80.190.130.19[4-5]/update/.*\.(htm|html|gz)$@i) ||
166 ($source_url =~ m@^http://62.146.64.14[6-7]/update/.*\.(htm|html|gz)$@i)
167 )
168 {
169 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avira",$mirror);
170 }
171
172 # -----------------------------------------------------------
173 # Section: IPFire Downloads
174 # -----------------------------------------------------------
175
176 if ($source_url =~ m@^[f|h]t?tp://.*\.(ipfire)$@i)
177 {
178 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"IPFire",$mirror);
179 }
180
181 }
182
183 # -----------------------------------------------------------
184 # Section: AVG Downloads
185 # -----------------------------------------------------------
186
187 # if ($source_url =~ m@^http://[^/]*\.(grisoft|avg)\.com/.*\.(bin)$@i)
188 # {
189 # $xlrator_url = &check_cache($source_url,$hostaddr,$username,"AVG",$mirror);
190 # }
191
192 $request="$xlrator_url $hostaddr $username $method\n";
193
194 print $request;
195 }
196
197 # -------------------------------------------------------------------
198
199 sub readhash
200 {
201 my $filename = $_[0];
202 my $hash = $_[1];
203 my ($var, $val);
204
205 if (-e $filename)
206 {
207 open(FILE, $filename) or die "Unable to read file $filename";
208 while (<FILE>)
209 {
210 chop;
211 ($var, $val) = split /=/, $_, 2;
212 if ($var)
213 {
214 $val =~ s/^\'//g;
215 $val =~ s/\'$//g;
216
217 # Untaint variables read from hash
218 $var =~ /([A-Za-z0-9_-]*)/; $var = $1;
219 $val =~ /([\w\W]*)/; $val = $1;
220 $hash->{$var} = $val;
221 }
222 }
223 close FILE;
224 }
225 }
226
227 # -------------------------------------------------------------------
228
229 sub writelog
230 {
231 if ($logging)
232 {
233 open(LOGFILE,">>$logfile");
234 print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n";
235 close(LOGFILE);
236 }
237 }
238
239 # -------------------------------------------------------------------
240
241 sub debuglog
242 {
243 if ($debug)
244 {
245 open(LOGFILE,">>/var/log/updatexlrator/debug.log");
246 my @now = localtime(time);
247 printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] [%s] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,"updxlrator",$_[0];
248 close(LOGFILE);
249 }
250 }
251
252 # -------------------------------------------------------------------
253
254 sub setcachestatus
255 {
256 open (FILE,">>$_[0]");
257 print FILE "$_[1]\n";
258 close FILE;
259 }
260
261 # -------------------------------------------------------------------
262
263 sub diskfree
264 {
265 open(DF,"/bin/df --block-size=1 $_[0]|");
266 my @dfdata = <DF>;
267 close DF;
268 shift(@dfdata);
269 chomp(@dfdata);
270 my $dfstr = join(' ',@dfdata);
271 my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
272 if ($free =~ m/^(\d+)$/)
273 {
274 return $free;
275 }
276 }
277
278 # -------------------------------------------------------------------
279
280 sub diskusage
281 {
282 open(DF,"/bin/df $_[0]|");
283 my @dfdata = <DF>;
284 close DF;
285 shift(@dfdata);
286 chomp(@dfdata);
287 my $dfstr = join(' ',@dfdata);
288 my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
289 if ($percent =~ m/^(\d+)%$/)
290 {
291 $percent =~ s/%$//;
292 return $percent;
293 }
294 }
295
296 # -------------------------------------------------------------------
297
298 sub getmtime
299 {
300 my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($_[0]);
301
302 return $mtime;
303 }
304
305 # -------------------------------------------------------------------
306
307 sub check_cache
308 {
309 my $updsource="UPDCACHE";
310 my $updfile='';
311 my $cacheurl='';
312 my $vendorid='';
313 my $uuid='';
314 my @http_header=();
315 my $remote_size=0;
316 my $remote_mtime=0;
317 my $login='';
318 my $useragent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
319
320 my $sourceurl=$_[0];
321 my $cfmirror=$_[4];
322
323 $sourceurl =~ s@\%2f@/@ig;
324 $updfile = substr($sourceurl,rindex($sourceurl,"/")+1);
325 $updfile =~ s@\%20@ @ig;
326
327 if ($cfmirror)
328 {
329 $uuid = `echo $updfile | md5sum`;
330 } else {
331 $uuid = `echo $sourceurl | md5sum`;
332 }
333
334 $uuid =~ s/[^0-9a-f]//g;
335 $uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/;
336
337 $vendorid = $_[3];
338 $vendorid =~ tr/A-Z/a-z/;
339
340 &debuglog("Processing URL $sourceurl");
341 &debuglog("Vendor ID is $vendorid");
342 &debuglog("UUID is $uuid");
343
344 if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'}))
345 {
346 $login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\"";
347 if ($proxysettings{'UPSTREAM_PASSWORD'})
348 {
349 $login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\"";
350 }
351 }
352
353 if ($proxysettings{'UPSTREAM_PROXY'}) { &debuglog("Using upstream proxy $proxysettings{'UPSTREAM_PROXY'}"); }
354
355 $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
356 @http_header = `$wget $login --user-agent="$useragent" --spider -S $sourceurl 2>&1`;
357 $ENV{'http_proxy'} = '';
358
359 foreach (@http_header)
360 {
361 chomp;
362 if (/^\s*Content-Length:\s/) { $remote_size = $_; $remote_size =~ s/[^0-9]//g; }
363 if (/^\s*Last-Modified:\s/) { $remote_mtime = $_; $remote_mtime =~ s/^\s*Last-Modified:\s//; $remote_mtime = HTTP::Date::str2time($remote_mtime) }
364 }
365
366 if (
367 (-e "$updcachedir/$vendorid/$uuid/$updfile") &&
368 ($remote_size == (-s "$updcachedir/$vendorid/$uuid/$updfile")) &&
369 ($remote_mtime == &getmtime("$updcachedir/$vendorid/$uuid/$updfile"))
370 )
371 {
372 &debuglog("File exists in cache and is up to date");
373 &debuglog("Retrieving file from cache ($updsource)");
374 &setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time);
375 $cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile";
376 }
377 else
378 {
379 if (-e "$updcachedir/$vendorid/$uuid/$updfile")
380 {
381 &debuglog("Local filesize: " . (-s "$updcachedir/$vendorid/$uuid/$updfile"));
382 &debuglog("Local timestamp: " . &getmtime("$updcachedir/$vendorid/$uuid/$updfile"));
383 } else { &debuglog("File not found in cache"); }
384 $updsource="DLSOURCE";
385 &debuglog("Remote filesize: $remote_size");
386 &debuglog("Remote timestamp: $remote_mtime");
387 &debuglog("Free disk space: " . &diskfree($updcachedir));
388 &debuglog("Disk usage: " . &diskusage($updcachedir) . "% (max. $maxusage%)");
389 if (-e "$updcachedir/download/$vendorid/$updfile") { &debuglog("File download/$vendorid/$updfile exists"); }
390 &debuglog("Retrieving file from source ($updsource)");
391 if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && ($remote_size <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$vendorid/$updfile"))
392 {
393 &debuglog("Running command $nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
394 system("$nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
395 }
396 $cacheurl=$sourceurl;
397 }
398
399 &writelog($_[1],$_[2],$_[3],$updsource,$sourceurl);
400
401 return $cacheurl;
402 }
403
404 # -------------------------------------------------------------------