config/updxlrator/updxlrator

   1 #!/usr/bin/perl
   2 #
   3 # This code is distributed under the terms of the GPL
   4 #
   5 # (c) 2006-2009 marco.s - http://update-accelerator.advproxy.net
   6 #
   7 # Portions (c) 2008 by dotzball - http://www.blockouttraffic.de
   8 #
   9 # $Id: updxlrator,v 2.1 2009/01/10 00:00:00 marco.s Exp $
  10 #
  11 # ChangeLog:
  12 #
  13 # 2012-10-26: nightshift - move curly bracket to capture AVG download source.
  14 #                        - Adding BIG HINT for new update source#
  15 #
  16
  17 use strict;
  18 use HTTP::Date;
  19
  20 $|=1;
  21
  22 my $swroot="/var/ipfire";
  23 my $updcachedir="/var/updatecache";
  24 my $apphome="/var/ipfire/updatexlrator";
  25 my $logfile="/var/log/updatexlrator/cache.log";
  26 my $wget="/usr/bin/wget";
  27 my $debug=(-e "$apphome/debug");
  28 my $http_port='81';
  29 my %netsettings=();
  30 my %proxysettings=();
  31 my %xlratorsettings=();
  32 my $logging=0;
  33 my $passive_mode=0;
  34 my $maxusage=0;
  35 my $nice='';
  36 my @tmp=();
  37 my $request='';
  38 my $xlrator_url=0;
  39 my $source_url='';
  40 my $hostaddr='';
  41 my $username='';
  42 my $method='';
  43 my $unique = 0;
  44 my $mirror = 1;
  45
  46 readhash("${swroot}/ethernet/settings", \%netsettings);
  47
  48 if (-e "$swroot/updatexlrator/settings")
  49 {
  50         &readhash("$swroot/updatexlrator/settings", \%xlratorsettings);
  51         if ($xlratorsettings{'ENABLE_LOG'} eq 'on') { $logging=1; };
  52         if ($xlratorsettings{'PASSIVE_MODE'} eq 'on') { $passive_mode=1; };
  53         $maxusage=$xlratorsettings{'MAX_DISK_USAGE'};
  54         if ($xlratorsettings{'LOW_DOWNLOAD_PRIORITY'} eq 'on') { $nice='/bin/nice --adjustment=15 '; };
  55 }
  56 if (!$maxusage) { $maxusage=75; };
  57
  58 # dotzball: check for dead downloads
  59 system("$apphome/bin/checkdeaddl &");
  60
  61 while (<>) {
  62
  63         $request=$_;
  64
  65         @tmp=split(/ /,$request);
  66         chomp(@tmp);
  67
  68         $source_url = $tmp[0];
  69         $hostaddr   = $tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; }
  70         $username   = $tmp[2]; if ($username eq '') { $username='-'; };
  71         $method     = $tmp[3];
  72
  73         $xlrator_url = "";
  74
  75         if (($method eq 'GET') || ($method eq 'HEAD'))
  76         {
  77
  78         # -----------------------------------------------------------
  79         #  Section: Windows Update / Windows Downloads
  80         # -----------------------------------------------------------
  81
  82         if (
  83             (($source_url =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi|msp|msu|cab)$@i) ||
  84              ($source_url =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|msi|msp|msu|cab)$@i))
  85         &&   ($source_url !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i)
  86         &&   ($source_url !~ m@\&@)
  87            )
  88         {
  89                 # NOTE: check_cache will change to $mirror instead of $unique if the filename contains an SHA1 hash
  90                 # and the URL is not found in cache!
  91                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique);
  92         }
  93
  94         # -----------------------------------------------------------
  95         #  Section: Adobe Downloads
  96         # -----------------------------------------------------------
  97
  98
  99         if (
 100             ($source_url =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|ms[ip]|bin|dmg|idx|gz)$@i) ||
 101             ($source_url =~ m@^http://swupdl\.adobe\.com/updates/.*\.(exe|ms[ip]|bin|dmg|idx|gz|[a-z][a-z]_[A-Z][A-Z])$@i) ||
 102             ($source_url =~ m@^http://swupmf\.adobe\.com/manifest/.*\.upd$@i)
 103            )
 104         {
 105                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Adobe",$unique);
 106         }
 107
 108         # -----------------------------------------------------------
 109         #  Section: Linux Downloads
 110         # -----------------------------------------------------------
 111
 112         if (
 113                 ($source_url =~ m@^[h|f]t?tp://[^?]+\.(pkg\.tar\.xz|deb|rpm)$@i) ||
 114                 ($source_url =~ m@^[h|f]t?tp://[^?]+/distfiles/[^?]+\.(tar\.gz|tar\.bz2|tar\.xz|tgz|zip|patch\.bz2|gz|docx|patch|pdf|exe)$@i)
 115         )
 116         {
 117                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Linux",$mirror);
 118         }
 119
 120         # -----------------------------------------------------------
 121         #  Section: Trend Micro Downloads
 122         # -----------------------------------------------------------
 123
 124         if (
 125             ($source_url =~ m@^http://[^/]*\.trendmicro\.com/activeupdate/.*@i) &&
 126             ($source_url !~ m@.*/tmnotify\.dat$@i) &&
 127             ($source_url !~ m@.*/ini_xml\.zip$@i) &&
 128             ($source_url !~ m@.*/server\.ini$@i)
 129            )
 130         {
 131                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"TrendMicro",$mirror);
 132         }
 133
 134         # -----------------------------------------------------------
 135         #  Section: Symantec Downloads
 136         # -----------------------------------------------------------
 137
 138         if ($source_url =~ m@^[h|f]tt?p://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|vdb|xdb)$@i)
 139         {
 140                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Symantec",$unique);
 141         }
 142
 143         # -----------------------------------------------------------
 144         #  Section: Apple Downloads
 145         # -----------------------------------------------------------
 146
 147         if (
 148              (($source_url =~ m@^http://swcdn\.apple\.com/content/downloads/.*\.(tar|pkg)$@i) ||
 149              ($source_url =~ m@^http://appldnld\.apple\.com\.edgesuite\.net/.*\.(exe|dmg)$@i) ||
 150              ($source_url =~ m@^http://.*\.g.akamai.net/.*/3093/1/.*\.(tar|pkg|dmg|exe)$@i))
 151            )
 152         {
 153                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Apple",$unique);
 154         }
 155
 156         # -----------------------------------------------------------
 157         #  Section: Avast Downloads
 158         # -----------------------------------------------------------
 159
 160         if ($source_url =~ m@^http://(ion|download)[\d]+\.avast\.com/.*\.(exe|vpu|vpx)$@i)
 161         {
 162                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avast",$mirror);
 163         }
 164
 165         # -----------------------------------------------------------
 166         #  Section: Mozilla Downloads
 167         # -----------------------------------------------------------
 168
 169         if ($source_url=~ m@^http://.*\.mozilla\.net/.*\.((complete|partial)\.mar|exe)$@i)
 170         {
 171                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Mozilla",$unique);
 172         }
 173
 174         # -----------------------------------------------------------
 175         #  Section: Mcafee
 176         # -----------------------------------------------------------
 177
 178         if ($source_url =~ m@^http://update\.nai\.com/.*\.(mcs|z|gem|dat|zip)$@i)
 179         {
 180                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"mcafee",$mirror);
 181         }
 182
 183         # -----------------------------------------------------------
 184         # Section: Avira Downloads
 185         # -----------------------------------------------------------
 186
 187         if (
 188                 ($source_url =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i) ||
 189                 ($source_url =~ m@^http://80.190.130.19[4-5]/update/.*\.(htm|html|gz)$@i) ||
 190                 ($source_url =~ m@^http://62.146.64.14[6-7]/update/.*\.(htm|html|gz)$@i)
 191         )
 192         {
 193                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avira",$mirror);
 194         }
 195
 196         # -----------------------------------------------------------
 197         # Section: IPFire Downloads
 198         # -----------------------------------------------------------
 199
 200         if ($source_url =~ m@^[f|h]t?tp://.*\.(ipfire)$@i)
 201         {
 202                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"IPFire",$mirror);
 203         }
 204
 205         # -----------------------------------------------------------
 206         # Section: AVG Downloads
 207         # -----------------------------------------------------------
 208
 209 #       if ($source_url =~ m@^http://[^/]*\.(grisoft|avg)\.com/.*\.(bin)$@i)
 210 #       {
 211 #               $xlrator_url = &check_cache($source_url,$hostaddr,$username,"AVG",$mirror);
 212 #       }
 213
 214 # ----------- ADD NEW SOURCES BEFORE THIS LINE !!! ------------------
 215         }
 216
 217         if ($xlrator_url) {
 218                 print "OK rewrite-url=\"$xlrator_url\"\n";
 219         } else {
 220                 print "ERR\n";
 221         }
 222 }
 223
 224 # -------------------------------------------------------------------
 225
 226 sub readhash
 227 {
 228         my $filename = $_[0];
 229         my $hash = $_[1];
 230         my ($var, $val);
 231
 232         if (-e $filename)
 233         {
 234                 open(FILE, $filename) or die "Unable to read file $filename";
 235                 while (<FILE>)
 236                 {
 237                         chop;
 238                         ($var, $val) = split /=/, $_, 2;
 239                         if ($var)
 240                         {
 241                                 $val =~ s/^\'//g;
 242                                 $val =~ s/\'$//g;
 243
 244                                 # Untaint variables read from hash
 245                                 $var =~ /([A-Za-z0-9_-]*)/; $var = $1;
 246                                 $val =~ /([\w\W]*)/; $val = $1;
 247                                 $hash->{$var} = $val;
 248                         }
 249                 }
 250                 close FILE;
 251         }
 252 }
 253
 254 # -------------------------------------------------------------------
 255
 256 sub writelog
 257 {
 258         if ($logging)
 259         {
 260                 open(LOGFILE,">>$logfile");
 261                 print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n";
 262                 close(LOGFILE);
 263         }
 264 }
 265
 266 # -------------------------------------------------------------------
 267
 268 sub debuglog
 269 {
 270         if ($debug)
 271         {
 272                 open(LOGFILE,">>/var/log/updatexlrator/debug.log");
 273                 my @now = localtime(time);
 274                 printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] [%s] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,"updxlrator",$_[0];
 275                 close(LOGFILE);
 276         }
 277 }
 278
 279 # -------------------------------------------------------------------
 280
 281 sub setcachestatus
 282 {
 283         open (FILE,">>$_[0]");
 284         print FILE "$_[1]\n";
 285         close FILE;
 286 }
 287
 288 # -------------------------------------------------------------------
 289
 290 sub diskfree
 291 {
 292         open(DF,"/bin/df --block-size=1 $_[0]|");
 293         my @dfdata = <DF>;
 294         close DF;
 295         shift(@dfdata);
 296         chomp(@dfdata);
 297         my $dfstr = join(' ',@dfdata);
 298         my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
 299         if ($free =~ m/^(\d+)$/)
 300         {
 301                 return $free;
 302         }
 303 }
 304
 305 # -------------------------------------------------------------------
 306
 307 sub diskusage
 308 {
 309         open(DF,"/bin/df $_[0]|");
 310         my @dfdata = <DF>;
 311         close DF;
 312         shift(@dfdata);
 313         chomp(@dfdata);
 314         my $dfstr = join(' ',@dfdata);
 315         my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
 316         if ($percent =~ m/^(\d+)%$/)
 317         {
 318                 $percent =~ s/%$//;
 319                 return $percent;
 320         }
 321 }
 322
 323 # -------------------------------------------------------------------
 324
 325 sub getmtime
 326 {
 327         my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($_[0]);
 328
 329         return $mtime;
 330 }
 331
 332 # -------------------------------------------------------------------
 333
 334 sub check_cache
 335 {
 336         my $updsource="UPDCACHE";
 337         my $updfile='';
 338         my $cacheurl='';
 339         my $vendorid='';
 340         my $uuid='';
 341         my @http_header=();
 342         my $remote_size=0;
 343         my $remote_mtime=0;
 344         my $login='';
 345         my $useragent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
 346
 347         my $sourceurl=$_[0];
 348         my $cfmirror=$_[4];
 349
 350         $sourceurl =~ s@\%2b@+@ig;
 351         $sourceurl =~ s@\%2f@/@ig;
 352         $sourceurl =~ s@\%7e@~@ig;
 353         $updfile = substr($sourceurl,rindex($sourceurl,"/")+1);
 354         $updfile =~ s@\%20@ @ig;
 355
 356         if ($cfmirror)
 357         {
 358                 $uuid = `echo $updfile | md5sum`;
 359         } else {
 360                 $uuid = `echo $sourceurl | md5sum`;
 361         }
 362
 363         $uuid =~ s/[^0-9a-f]//g;
 364         $uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/;
 365
 366         $vendorid = $_[3];
 367         $vendorid =~ tr/A-Z/a-z/;
 368
 369         &debuglog("Processing URL $sourceurl");
 370         &debuglog("Vendor ID is $vendorid");
 371         &debuglog("UUID is $uuid");
 372
 373         if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'}))
 374         {
 375                 $login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\"";
 376                 if ($proxysettings{'UPSTREAM_PASSWORD'})
 377                 {
 378                         $login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\"";
 379                 }
 380         }
 381
 382         if ($proxysettings{'UPSTREAM_PROXY'}) { &debuglog("Using upstream proxy $proxysettings{'UPSTREAM_PROXY'}"); }
 383
 384         $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
 385         @http_header = `$wget $login --user-agent="$useragent" --spider -S $sourceurl 2>&1`;
 386         $ENV{'http_proxy'} = '';
 387
 388         foreach (@http_header)
 389         {
 390                 chomp;
 391                 if (/^\s*Content-Length:\s/) { $remote_size = $_; $remote_size =~ s/[^0-9]//g; }
 392                 if (/^\s*Last-Modified:\s/) { $remote_mtime = $_; $remote_mtime =~ s/^\s*Last-Modified:\s//; $remote_mtime = HTTP::Date::str2time($remote_mtime) }
 393         }
 394
 395         if (
 396                 (-e "$updcachedir/$vendorid/$uuid/$updfile") &&
 397                 ($remote_size == (-s "$updcachedir/$vendorid/$uuid/$updfile")) &&
 398                 ($remote_mtime == &getmtime("$updcachedir/$vendorid/$uuid/$updfile"))
 399            )
 400         {
 401                 &debuglog("File exists in cache and is up to date");
 402                 &debuglog("Retrieving file from cache ($updsource) for $hostaddr");
 403                 &setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time);
 404                 $cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile";
 405         }
 406         elsif (
 407                 ($cfmirror == $unique) &&
 408                 ($vendorid == "microsoft") &&
 409                 ($source_url =~ m@.*[0-9a-f]{40}\.[^\.]+@i)
 410               )
 411         {
 412                         # Most Microsoft updates now have an SHA1 hash in the name. These should be treated as unique files.
 413                         # Since it wasn't found in the URL cache, switch to mirror mode and try again using just the filename.
 414                         &debuglog("SHA1: $vendorid $uuid not cached. Reprocessing as mirror $sourceurl");
 415                         $cacheurl = &check_cache($source_url,$hostaddr,$username,$vendorid,$mirror);
 416         }
 417                 else
 418         {
 419                 if (-e "$updcachedir/$vendorid/$uuid/$updfile")
 420                 {
 421                         &debuglog("Local filesize: " . (-s "$updcachedir/$vendorid/$uuid/$updfile"));
 422                         &debuglog("Local timestamp: " . &getmtime("$updcachedir/$vendorid/$uuid/$updfile"));
 423                 } else { &debuglog("File not found in cache"); }
 424                 $updsource="DLSOURCE";
 425                 &debuglog("Remote filesize: $remote_size");
 426                 &debuglog("Remote timestamp: $remote_mtime");
 427                 &debuglog("Free disk space: " . &diskfree($updcachedir));
 428                 &debuglog("Disk usage: " . &diskusage($updcachedir) . "% (max. $maxusage%)");
 429                 if (-e "$updcachedir/download/$vendorid/$updfile") { &debuglog("File download/$vendorid/$updfile exists"); }
 430                 &debuglog("Retrieving file from source ($updsource) for $hostaddr");
 431                 if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && ($remote_size <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$vendorid/$updfile"))
 432                 {
 433                         &debuglog("Running command $nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
 434                         system("$nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
 435                 }
 436                 $cacheurl=$sourceurl;
 437         }
 438
 439         &writelog($_[1],$_[2],$_[3],$updsource,$sourceurl);
 440
 441         return $cacheurl;
 442 }
 443
 444 # -------------------------------------------------------------------