config/updxlrator/updxlrator

   1 #!/usr/bin/perl
   2 #
   3 # This code is distributed under the terms of the GPL
   4 #
   5 # (c) 2006-2008 marco.s - http://update-accelerator.advproxy.net
   6 #
   7 # $Id: updxlrator,v 2.0 2008/04/06 00:00:00 marco.s Exp $
   8 #
   9
  10 use strict;
  11 use HTTP::Date;
  12
  13 $|=1;
  14
  15 my $swroot="/var/ipfire";
  16 my $updcachedir="/var/updatecache";
  17 my $apphome="/var/ipfire/updatexlrator";
  18 my $logfile="/var/log/updatexlrator/cache.log";
  19 my $debug=(-e "$apphome/debug");
  20 my $http_port='81';
  21 my %netsettings=();
  22 my %proxysettings=();
  23 my %xlratorsettings=();
  24 my $logging=0;
  25 my $passive_mode=0;
  26 my $maxusage=0;
  27 my $nice='';
  28 my @tmp=();
  29 my $request='';
  30 my $xlrator_url=0;
  31 my $source_url='';
  32 my $hostaddr='';
  33 my $username='';
  34 my $method='';
  35 my $unique = 0;
  36 my $mirror = 1;
  37
  38 readhash("${swroot}/ethernet/settings", \%netsettings);
  39
  40 if (-e "$swroot/updatexlrator/settings")
  41 {
  42         &readhash("$swroot/updatexlrator/settings", \%xlratorsettings);
  43         if ($xlratorsettings{'ENABLE_LOG'} eq 'on') { $logging=1; };
  44         if ($xlratorsettings{'PASSIVE_MODE'} eq 'on') { $passive_mode=1; };
  45         $maxusage=$xlratorsettings{'MAX_DISK_USAGE'};
  46         if ($xlratorsettings{'LOW_DOWNLOAD_PRIORITY'} eq 'on') { $nice='/bin/nice --adjustment=15 '; };
  47 }
  48 if (!$maxusage) { $maxusage=75; };
  49
  50
  51 while (<>) {
  52
  53         $request=$_;
  54
  55         @tmp=split(/ /,$request);
  56         chomp(@tmp);
  57
  58         $source_url = $tmp[0];
  59         $hostaddr   = $tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; }
  60         $username   = $tmp[2]; if ($username eq '') { $username='-'; };
  61         $method     = $tmp[3];
  62
  63         $xlrator_url = $source_url;
  64
  65         if (($method eq 'GET') || ($method eq 'HEAD'))
  66         {
  67
  68         # -----------------------------------------------------------
  69         #  Section: Windows Update / Windows Downloads
  70         # -----------------------------------------------------------
  71
  72         if (
  73             (($source_url =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi|msp|cab)$@i) ||
  74              ($source_url =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|msi|msp|cab)$@i))
  75         &&   ($source_url !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i)
  76         &&   ($source_url !~ m@\&@)
  77            )
  78         {
  79                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique);
  80         }
  81
  82         # -----------------------------------------------------------
  83         #  Section: Adobe Downloads
  84         # -----------------------------------------------------------
  85
  86         if (
  87             ($source_url =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|msi|bin|dmg|idx|gz)$@i) ||
  88             ($source_url =~ m@^http://swupdl\.adobe\.com/updates/.*\.(exe|msi|bin|dmg|idx|gz|[a-z][a-z]_[A-Z][A-Z])$@i) ||
  89             ($source_url =~ m@^http://swupmf\.adobe\.com/manifest/.*\.upd$@i)
  90            )
  91         {
  92                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Adobe",$unique);
  93         }
  94
  95         # -----------------------------------------------------------
  96         #  Section: Linux Downloads
  97         # -----------------------------------------------------------
  98
  99         if ($source_url =~ m@^[h|f]t?tp://[^?]+\.(deb|rpm)$@i)
 100         {
 101                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Linux",$mirror);
 102         }
 103
 104         # -----------------------------------------------------------
 105         #  Section: Trend Micro Downloads
 106         # -----------------------------------------------------------
 107
 108         if (
 109             ($source_url =~ m@^http://[^/]*\.trendmicro\.com/activeupdate/.*@i) &&
 110             ($source_url !~ m@.*/tmnotify\.dat$@i) &&
 111             ($source_url !~ m@.*/ini_xml\.zip$@i) &&
 112             ($source_url !~ m@.*/server\.ini$@i)
 113            )
 114         {
 115                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"TrendMicro",$mirror);
 116         }
 117
 118         # -----------------------------------------------------------
 119         #  Section: Symantec Downloads
 120         # -----------------------------------------------------------
 121
 122         if ($source_url =~ m@^[h|f]tt?p://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|vdb|xdb)$@i)
 123         {
 124                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Symantec",$unique);
 125         }
 126
 127         # -----------------------------------------------------------
 128         #  Section: Apple Downloads
 129         # -----------------------------------------------------------
 130
 131         if (
 132             (($source_url =~ m@^http://swcdn\.apple\.com/content/downloads/.*\.(tar)$@i) ||
 133              ($source_url =~ m@^http://appldnld\.apple\.com\.edgesuite\.net/.*\.(exe|dmg)$@i) ||
 134              ($source_url =~ m@^http://.*\.g.akamai.net/.*/3093/1/.*\.(tar|pkg|dmg|exe)$@i))
 135            )
 136         {
 137                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Apple",$unique);
 138         }
 139
 140         # -----------------------------------------------------------
 141         #  Section: Avast Downloads
 142         # -----------------------------------------------------------
 143
 144         if ($source_url =~ m@^http://download[\d]+\.avast\.com/.*\.(exe|vpu)$@i)
 145         {
 146                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avast",$mirror);
 147         }
 148
 149         # -----------------------------------------------------------
 150
 151         # -----------------------------------------------------------
 152         # Section: Avira Downloads
 153         # -----------------------------------------------------------
 154
 155         if ($source_url =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i)
 156         {
 157                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avira",$mirror);
 158         }
 159
 160         # -----------------------------------------------------------
 161         # Section: IPFire Downloads
 162         # -----------------------------------------------------------
 163
 164         if ($source_url =~ m@^[f|h]t?tp://.*\.(ipfire)$@i)
 165         {
 166                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"IPFire",$mirror);
 167         }
 168
 169         }
 170
 171         # -----------------------------------------------------------
 172         # Section: AVG Downloads
 173         # -----------------------------------------------------------
 174
 175         if ($source_url =~ m@^http://[^/]*\.(grisoft|avg)\.com/.*\.(bin)$@i)
 176         {
 177                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"AVG",$mirror);
 178         }
 179
 180         $request="$xlrator_url $hostaddr $username $method\n";
 181
 182         print $request;
 183 }
 184
 185 # -------------------------------------------------------------------
 186
 187 sub readhash
 188 {
 189         my $filename = $_[0];
 190         my $hash = $_[1];
 191         my ($var, $val);
 192
 193         if (-e $filename)
 194         {
 195                 open(FILE, $filename) or die "Unable to read file $filename";
 196                 while (<FILE>)
 197                 {
 198                         chop;
 199                         ($var, $val) = split /=/, $_, 2;
 200                         if ($var)
 201                         {
 202                                 $val =~ s/^\'//g;
 203                                 $val =~ s/\'$//g;
 204
 205                                 # Untaint variables read from hash
 206                                 $var =~ /([A-Za-z0-9_-]*)/; $var = $1;
 207                                 $val =~ /([\w\W]*)/; $val = $1;
 208                                 $hash->{$var} = $val;
 209                         }
 210                 }
 211                 close FILE;
 212         }
 213 }
 214
 215 # -------------------------------------------------------------------
 216
 217 sub writelog
 218 {
 219         if ($logging)
 220         {
 221                 open(LOGFILE,">>$logfile");
 222                 print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n";
 223                 close(LOGFILE);
 224         }
 225 }
 226
 227 # -------------------------------------------------------------------
 228
 229 sub debuglog
 230 {
 231         if ($debug)
 232         {
 233                 open(LOGFILE,">>/var/log/updatexlrator/debug.log");
 234                 my @now = localtime(time);
 235                 printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] [%s] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,"updxlrator",$_[0];
 236                 close(LOGFILE);
 237         }
 238 }
 239
 240 # -------------------------------------------------------------------
 241
 242 sub setcachestatus
 243 {
 244         open (FILE,">>$_[0]");
 245         print FILE "$_[1]\n";
 246         close FILE;
 247 }
 248
 249 # -------------------------------------------------------------------
 250
 251 sub diskfree
 252 {
 253         open(DF,"/bin/df --block-size=1 $_[0]|");
 254         my @dfdata = <DF>;
 255         close DF;
 256         shift(@dfdata);
 257         chomp(@dfdata);
 258         my $dfstr = join(' ',@dfdata);
 259         my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
 260         if ($free =~ m/^(\d+)$/)
 261         {
 262                 return $free;
 263         }
 264 }
 265
 266 # -------------------------------------------------------------------
 267
 268 sub diskusage
 269 {
 270         open(DF,"/bin/df $_[0]|");
 271         my @dfdata = <DF>;
 272         close DF;
 273         shift(@dfdata);
 274         chomp(@dfdata);
 275         my $dfstr = join(' ',@dfdata);
 276         my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
 277         if ($percent =~ m/^(\d+)%$/)
 278         {
 279                 $percent =~ s/%$//;
 280                 return $percent;
 281         }
 282 }
 283
 284 # -------------------------------------------------------------------
 285
 286 sub getmtime
 287 {
 288         my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($_[0]);
 289
 290         return $mtime;
 291 }
 292
 293 # -------------------------------------------------------------------
 294
 295 sub check_cache
 296 {
 297         my $updsource="UPDCACHE";
 298         my $updfile='';
 299         my $cacheurl='';
 300         my $vendorid='';
 301         my $uuid='';
 302         my @http_header=();
 303         my $remote_size=0;
 304         my $remote_mtime=0;
 305         my $login='';
 306         my $useragent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
 307
 308         my $sourceurl=$_[0];
 309         my $cfmirror=$_[4];
 310
 311         $sourceurl =~ s@\%2f@/@ig;
 312         $updfile = substr($sourceurl,rindex($sourceurl,"/")+1);
 313         $updfile =~ s@\%20@ @ig;
 314
 315         if ($cfmirror)
 316         {
 317                 $uuid = `echo $updfile | md5sum`;
 318         } else {
 319                 $uuid = `echo $sourceurl | md5sum`;
 320         }
 321
 322         $uuid =~ s/[^0-9a-f]//g;
 323         $uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/;
 324
 325         $vendorid = $_[3];
 326         $vendorid =~ tr/A-Z/a-z/;
 327
 328         &debuglog("Processing URL $sourceurl");
 329         &debuglog("Vendor ID is $vendorid");
 330         &debuglog("UUID is $uuid");
 331
 332         if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'}))
 333         {
 334                 $login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\"";
 335                 if ($proxysettings{'UPSTREAM_PASSWORD'})
 336                 {
 337                         $login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\"";
 338                 }
 339         }
 340
 341         if ($proxysettings{'UPSTREAM_PROXY'}) { &debuglog("Using upstream proxy $proxysettings{'UPSTREAM_PROXY'}"); }
 342
 343         $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
 344         @http_header = `wget $login --user-agent="$useragent" --spider -S $sourceurl 2>&1`;
 345         $ENV{'http_proxy'} = '';
 346
 347         foreach (@http_header)
 348         {
 349                 chomp;
 350                 if (/^\s*Content-Length:\s/) { $remote_size = $_; $remote_size =~ s/[^0-9]//g; }
 351                 if (/^\s*Last-Modified:\s/) { $remote_mtime = $_; $remote_mtime =~ s/^\s*Last-Modified:\s//; $remote_mtime = HTTP::Date::str2time($remote_mtime) }
 352         }
 353
 354         if (
 355                 (-e "$updcachedir/$vendorid/$uuid/$updfile") &&
 356                 ($remote_size == (-s "$updcachedir/$vendorid/$uuid/$updfile")) &&
 357                 ($remote_mtime == &getmtime("$updcachedir/$vendorid/$uuid/$updfile"))
 358            )
 359         {
 360                 &debuglog("File exists in cache and is up to date");
 361                 &debuglog("Retrieving file from cache ($updsource)");
 362                 &setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time);
 363                 $cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile";
 364         }
 365                 else
 366         {
 367                 if (-e "$updcachedir/$vendorid/$uuid/$updfile")
 368                 {
 369                         &debuglog("Local filesize: " . (-s "$updcachedir/$vendorid/$uuid/$updfile"));
 370                         &debuglog("Local timestamp: " . &getmtime("$updcachedir/$vendorid/$uuid/$updfile"));
 371                 } else { &debuglog("File not found in cache"); }
 372                 $updsource="DLSOURCE";
 373                 &debuglog("Remote filesize: $remote_size");
 374                 &debuglog("Remote timestamp: $remote_mtime");
 375                 &debuglog("Free disk space: " . &diskfree($updcachedir));
 376                 &debuglog("Disk usage: " . &diskusage($updcachedir) . "% (max. $maxusage%)");
 377                 if (-e "$updcachedir/download/$vendorid/$updfile") { &debuglog("File download/$vendorid/$updfile exists"); }
 378                 &debuglog("Retrieving file from source ($updsource)");
 379                 if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && ($remote_size <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$vendorid/$updfile"))
 380                 {
 381                         &debuglog("Running command $nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
 382                         system("$nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
 383                 }
 384                 $cacheurl=$sourceurl;
 385         }
 386
 387         &writelog($_[1],$_[2],$_[3],$updsource,$sourceurl);
 388
 389         return $cacheurl;
 390 }
 391
 392 # -------------------------------------------------------------------