config/updxlrator/updxlrator

   1 #!/usr/bin/perl
   2 #
   3 # This code is distributed under the terms of the GPL
   4 #
   5 # (c) 2006-2009 marco.s - http://update-accelerator.advproxy.net
   6 #
   7 # Portions (c) 2008 by dotzball - http://www.blockouttraffic.de
   8 #
   9 # $Id: updxlrator,v 2.1 2009/01/10 00:00:00 marco.s Exp $
  10 #
  11
  12 use strict;
  13 use HTTP::Date;
  14
  15 $|=1;
  16
  17 my $swroot="/var/ipfire";
  18 my $updcachedir="/var/updatecache";
  19 my $apphome="/var/ipfire/updatexlrator";
  20 my $logfile="/var/log/updatexlrator/cache.log";
  21 my $wget="/usr/bin/wget";
  22 my $debug=(-e "$apphome/debug");
  23 my $http_port='81';
  24 my %netsettings=();
  25 my %proxysettings=();
  26 my %xlratorsettings=();
  27 my $logging=0;
  28 my $passive_mode=0;
  29 my $maxusage=0;
  30 my $nice='';
  31 my @tmp=();
  32 my $request='';
  33 my $xlrator_url=0;
  34 my $source_url='';
  35 my $hostaddr='';
  36 my $username='';
  37 my $method='';
  38 my $unique = 0;
  39 my $mirror = 1;
  40
  41 readhash("${swroot}/ethernet/settings", \%netsettings);
  42
  43 if (-e "$swroot/updatexlrator/settings")
  44 {
  45         &readhash("$swroot/updatexlrator/settings", \%xlratorsettings);
  46         if ($xlratorsettings{'ENABLE_LOG'} eq 'on') { $logging=1; };
  47         if ($xlratorsettings{'PASSIVE_MODE'} eq 'on') { $passive_mode=1; };
  48         $maxusage=$xlratorsettings{'MAX_DISK_USAGE'};
  49         if ($xlratorsettings{'LOW_DOWNLOAD_PRIORITY'} eq 'on') { $nice='/bin/nice --adjustment=15 '; };
  50 }
  51 if (!$maxusage) { $maxusage=75; };
  52
  53 # dotzball: check for dead downloads
  54 system("$apphome/bin/checkdeaddl &");
  55
  56 while (<>) {
  57
  58         $request=$_;
  59
  60         @tmp=split(/ /,$request);
  61         chomp(@tmp);
  62
  63         $source_url = $tmp[0];
  64         $hostaddr   = $tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; }
  65         $username   = $tmp[2]; if ($username eq '') { $username='-'; };
  66         $method     = $tmp[3];
  67
  68         $xlrator_url = $source_url;
  69
  70         if (($method eq 'GET') || ($method eq 'HEAD'))
  71         {
  72
  73         # -----------------------------------------------------------
  74         #  Section: Windows Update / Windows Downloads
  75         # -----------------------------------------------------------
  76
  77         if (
  78             (($source_url =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi|msp|cab)$@i) ||
  79              ($source_url =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|msi|msp|cab)$@i))
  80         &&   ($source_url !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i)
  81         &&   ($source_url !~ m@\&@)
  82            )
  83         {
  84                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique);
  85         }
  86
  87         # -----------------------------------------------------------
  88         #  Section: Adobe Downloads
  89         # -----------------------------------------------------------
  90
  91         if (
  92             ($source_url =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|msi|bin|dmg|idx|gz)$@i) ||
  93             ($source_url =~ m@^http://swupdl\.adobe\.com/updates/.*\.(exe|msi|bin|dmg|idx|gz|[a-z][a-z]_[A-Z][A-Z])$@i) ||
  94             ($source_url =~ m@^http://swupmf\.adobe\.com/manifest/.*\.upd$@i)
  95            )
  96         {
  97                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Adobe",$unique);
  98         }
  99
 100         # -----------------------------------------------------------
 101         #  Section: Linux Downloads
 102         # -----------------------------------------------------------
 103
 104         if (
 105                 ($source_url =~ m@^[h|f]t?tp://[^?]+\.(deb|rpm)$@i) ||
 106                 ($source_url =~ m@^[h|f]t?tp://[^?]+/distfiles/[^?]+\.(tar\.gz|tar\.bz2|tgz|zip|patch\.bz2|gz|docx|patch|pdf|exe)$@i)
 107         )
 108         {
 109                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Linux",$mirror);
 110         }
 111
 112         # -----------------------------------------------------------
 113         #  Section: Trend Micro Downloads
 114         # -----------------------------------------------------------
 115
 116         if (
 117             ($source_url =~ m@^http://[^/]*\.trendmicro\.com/activeupdate/.*@i) &&
 118             ($source_url !~ m@.*/tmnotify\.dat$@i) &&
 119             ($source_url !~ m@.*/ini_xml\.zip$@i) &&
 120             ($source_url !~ m@.*/server\.ini$@i)
 121            )
 122         {
 123                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"TrendMicro",$mirror);
 124         }
 125
 126         # -----------------------------------------------------------
 127         #  Section: Symantec Downloads
 128         # -----------------------------------------------------------
 129
 130         if ($source_url =~ m@^[h|f]tt?p://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|vdb|xdb)$@i)
 131         {
 132                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Symantec",$unique);
 133         }
 134
 135         # -----------------------------------------------------------
 136         #  Section: Apple Downloads
 137         # -----------------------------------------------------------
 138
 139         if (
 140              (($source_url =~ m@^http://swcdn\.apple\.com/content/downloads/.*\.(tar|pkg)$@i) ||
 141              ($source_url =~ m@^http://appldnld\.apple\.com\.edgesuite\.net/.*\.(exe|dmg)$@i) ||
 142              ($source_url =~ m@^http://.*\.g.akamai.net/.*/3093/1/.*\.(tar|pkg|dmg|exe)$@i))
 143            )
 144         {
 145                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Apple",$unique);
 146         }
 147
 148         # -----------------------------------------------------------
 149         #  Section: Avast Downloads
 150         # -----------------------------------------------------------
 151
 152         if ($source_url =~ m@^http://(ion|download)[\d]+\.avast\.com/.*\.(exe|vpu|vpx)$@i)
 153         {
 154                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avast",$mirror);
 155         }
 156
 157         # -----------------------------------------------------------
 158         #  Section: Mozilla Downloads
 159         # -----------------------------------------------------------
 160
 161         if ($source_url=~ m@^http://.*\.mozilla\.net/.*\.((complete|partial)\.mar|exe)$@i)
 162         {
 163                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Mozilla",$unique);
 164         }
 165
 166         # -----------------------------------------------------------
 167         #  Section: Mcafee
 168         # -----------------------------------------------------------
 169
 170         if ($source_url =~ m@^http://update\.nai\.com/.*\.(mcs|z|gem|dat|zip)$@i)
 171         {
 172                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"mcafee",$mirror);
 173         }
 174
 175         # -----------------------------------------------------------
 176         # Section: Avira Downloads
 177         # -----------------------------------------------------------
 178
 179         if (
 180                 ($source_url =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i) ||
 181                 ($source_url =~ m@^http://80.190.130.19[4-5]/update/.*\.(htm|html|gz)$@i) ||
 182                 ($source_url =~ m@^http://62.146.64.14[6-7]/update/.*\.(htm|html|gz)$@i)
 183         )
 184         {
 185                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avira",$mirror);
 186         }
 187
 188         # -----------------------------------------------------------
 189         # Section: IPFire Downloads
 190         # -----------------------------------------------------------
 191
 192         if ($source_url =~ m@^[f|h]t?tp://.*\.(ipfire)$@i)
 193         {
 194                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"IPFire",$mirror);
 195         }
 196
 197         }
 198
 199         # -----------------------------------------------------------
 200         # Section: AVG Downloads
 201         # -----------------------------------------------------------
 202
 203 #       if ($source_url =~ m@^http://[^/]*\.(grisoft|avg)\.com/.*\.(bin)$@i)
 204 #       {
 205 #               $xlrator_url = &check_cache($source_url,$hostaddr,$username,"AVG",$mirror);
 206 #       }
 207
 208         $request="$xlrator_url $hostaddr $username $method\n";
 209
 210         print $request;
 211 }
 212
 213 # -------------------------------------------------------------------
 214
 215 sub readhash
 216 {
 217         my $filename = $_[0];
 218         my $hash = $_[1];
 219         my ($var, $val);
 220
 221         if (-e $filename)
 222         {
 223                 open(FILE, $filename) or die "Unable to read file $filename";
 224                 while (<FILE>)
 225                 {
 226                         chop;
 227                         ($var, $val) = split /=/, $_, 2;
 228                         if ($var)
 229                         {
 230                                 $val =~ s/^\'//g;
 231                                 $val =~ s/\'$//g;
 232
 233                                 # Untaint variables read from hash
 234                                 $var =~ /([A-Za-z0-9_-]*)/; $var = $1;
 235                                 $val =~ /([\w\W]*)/; $val = $1;
 236                                 $hash->{$var} = $val;
 237                         }
 238                 }
 239                 close FILE;
 240         }
 241 }
 242
 243 # -------------------------------------------------------------------
 244
 245 sub writelog
 246 {
 247         if ($logging)
 248         {
 249                 open(LOGFILE,">>$logfile");
 250                 print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n";
 251                 close(LOGFILE);
 252         }
 253 }
 254
 255 # -------------------------------------------------------------------
 256
 257 sub debuglog
 258 {
 259         if ($debug)
 260         {
 261                 open(LOGFILE,">>/var/log/updatexlrator/debug.log");
 262                 my @now = localtime(time);
 263                 printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] [%s] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,"updxlrator",$_[0];
 264                 close(LOGFILE);
 265         }
 266 }
 267
 268 # -------------------------------------------------------------------
 269
 270 sub setcachestatus
 271 {
 272         open (FILE,">>$_[0]");
 273         print FILE "$_[1]\n";
 274         close FILE;
 275 }
 276
 277 # -------------------------------------------------------------------
 278
 279 sub diskfree
 280 {
 281         open(DF,"/bin/df --block-size=1 $_[0]|");
 282         my @dfdata = <DF>;
 283         close DF;
 284         shift(@dfdata);
 285         chomp(@dfdata);
 286         my $dfstr = join(' ',@dfdata);
 287         my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
 288         if ($free =~ m/^(\d+)$/)
 289         {
 290                 return $free;
 291         }
 292 }
 293
 294 # -------------------------------------------------------------------
 295
 296 sub diskusage
 297 {
 298         open(DF,"/bin/df $_[0]|");
 299         my @dfdata = <DF>;
 300         close DF;
 301         shift(@dfdata);
 302         chomp(@dfdata);
 303         my $dfstr = join(' ',@dfdata);
 304         my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
 305         if ($percent =~ m/^(\d+)%$/)
 306         {
 307                 $percent =~ s/%$//;
 308                 return $percent;
 309         }
 310 }
 311
 312 # -------------------------------------------------------------------
 313
 314 sub getmtime
 315 {
 316         my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($_[0]);
 317
 318         return $mtime;
 319 }
 320
 321 # -------------------------------------------------------------------
 322
 323 sub check_cache
 324 {
 325         my $updsource="UPDCACHE";
 326         my $updfile='';
 327         my $cacheurl='';
 328         my $vendorid='';
 329         my $uuid='';
 330         my @http_header=();
 331         my $remote_size=0;
 332         my $remote_mtime=0;
 333         my $login='';
 334         my $useragent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
 335
 336         my $sourceurl=$_[0];
 337         my $cfmirror=$_[4];
 338
 339         $sourceurl =~ s@\%2f@/@ig;
 340         $updfile = substr($sourceurl,rindex($sourceurl,"/")+1);
 341         $updfile =~ s@\%20@ @ig;
 342
 343         if ($cfmirror)
 344         {
 345                 $uuid = `echo $updfile | md5sum`;
 346         } else {
 347                 $uuid = `echo $sourceurl | md5sum`;
 348         }
 349
 350         $uuid =~ s/[^0-9a-f]//g;
 351         $uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/;
 352
 353         $vendorid = $_[3];
 354         $vendorid =~ tr/A-Z/a-z/;
 355
 356         &debuglog("Processing URL $sourceurl");
 357         &debuglog("Vendor ID is $vendorid");
 358         &debuglog("UUID is $uuid");
 359
 360         if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'}))
 361         {
 362                 $login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\"";
 363                 if ($proxysettings{'UPSTREAM_PASSWORD'})
 364                 {
 365                         $login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\"";
 366                 }
 367         }
 368
 369         if ($proxysettings{'UPSTREAM_PROXY'}) { &debuglog("Using upstream proxy $proxysettings{'UPSTREAM_PROXY'}"); }
 370
 371         $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
 372         @http_header = `$wget $login --user-agent="$useragent" --spider -S $sourceurl 2>&1`;
 373         $ENV{'http_proxy'} = '';
 374
 375         foreach (@http_header)
 376         {
 377                 chomp;
 378                 if (/^\s*Content-Length:\s/) { $remote_size = $_; $remote_size =~ s/[^0-9]//g; }
 379                 if (/^\s*Last-Modified:\s/) { $remote_mtime = $_; $remote_mtime =~ s/^\s*Last-Modified:\s//; $remote_mtime = HTTP::Date::str2time($remote_mtime) }
 380         }
 381
 382         if (
 383                 (-e "$updcachedir/$vendorid/$uuid/$updfile") &&
 384                 ($remote_size == (-s "$updcachedir/$vendorid/$uuid/$updfile")) &&
 385                 ($remote_mtime == &getmtime("$updcachedir/$vendorid/$uuid/$updfile"))
 386            )
 387         {
 388                 &debuglog("File exists in cache and is up to date");
 389                 &debuglog("Retrieving file from cache ($updsource)");
 390                 &setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time);
 391                 $cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile";
 392         }
 393                 else
 394         {
 395                 if (-e "$updcachedir/$vendorid/$uuid/$updfile")
 396                 {
 397                         &debuglog("Local filesize: " . (-s "$updcachedir/$vendorid/$uuid/$updfile"));
 398                         &debuglog("Local timestamp: " . &getmtime("$updcachedir/$vendorid/$uuid/$updfile"));
 399                 } else { &debuglog("File not found in cache"); }
 400                 $updsource="DLSOURCE";
 401                 &debuglog("Remote filesize: $remote_size");
 402                 &debuglog("Remote timestamp: $remote_mtime");
 403                 &debuglog("Free disk space: " . &diskfree($updcachedir));
 404                 &debuglog("Disk usage: " . &diskusage($updcachedir) . "% (max. $maxusage%)");
 405                 if (-e "$updcachedir/download/$vendorid/$updfile") { &debuglog("File download/$vendorid/$updfile exists"); }
 406                 &debuglog("Retrieving file from source ($updsource)");
 407                 if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && ($remote_size <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$vendorid/$updfile"))
 408                 {
 409                         &debuglog("Running command $nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
 410                         system("$nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
 411                 }
 412                 $cacheurl=$sourceurl;
 413         }
 414
 415         &writelog($_[1],$_[2],$_[3],$updsource,$sourceurl);
 416
 417         return $cacheurl;
 418 }
 419
 420 # -------------------------------------------------------------------