config/updxlrator/updxlrator

   1 #!/usr/bin/perl
   2 #
   3 # This code is distributed under the terms of the GPL
   4 #
   5 # (c) 2006-2008 marco.s - http://update-accelerator.advproxy.net
   6 #
   7 # $Id: updxlrator,v 2.0 2008/04/06 00:00:00 marco.s Exp $
   8 #
   9
  10 use strict;
  11 use HTTP::Date;
  12
  13 $|=1;
  14
  15 my $swroot="/var/ipfire";
  16 my $updcachedir="/var/updatecache";
  17 my $apphome="/var/ipfire/updatexlrator";
  18 my $logfile="/var/log/updatexlrator/cache.log";
  19 my $wget="$apphome/bin/wget";
  20 my $debug=(-e "$apphome/debug");
  21 my $http_port='81';
  22 my %netsettings=();
  23 my %proxysettings=();
  24 my %xlratorsettings=();
  25 my $logging=0;
  26 my $passive_mode=0;
  27 my $maxusage=0;
  28 my $nice='';
  29 my @tmp=();
  30 my $request='';
  31 my $xlrator_url=0;
  32 my $source_url='';
  33 my $hostaddr='';
  34 my $username='';
  35 my $method='';
  36 my $unique = 0;
  37 my $mirror = 1;
  38
  39 readhash("${swroot}/ethernet/settings", \%netsettings);
  40
  41 if (-e "$swroot/updatexlrator/settings")
  42 {
  43         &readhash("$swroot/updatexlrator/settings", \%xlratorsettings);
  44         if ($xlratorsettings{'ENABLE_LOG'} eq 'on') { $logging=1; };
  45         if ($xlratorsettings{'PASSIVE_MODE'} eq 'on') { $passive_mode=1; };
  46         $maxusage=$xlratorsettings{'MAX_DISK_USAGE'};
  47         if ($xlratorsettings{'LOW_DOWNLOAD_PRIORITY'} eq 'on') { $nice='/usr/bin/nice --adjustment=15 '; };
  48 }
  49 if (!$maxusage) { $maxusage=75; };
  50
  51
  52 while (<>) {
  53
  54         $request=$_;
  55
  56         @tmp=split(/ /,$request);
  57         chomp(@tmp);
  58
  59         $source_url = $tmp[0];
  60         $hostaddr   = $tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; }
  61         $username   = $tmp[2]; if ($username eq '') { $username='-'; };
  62         $method     = $tmp[3];
  63
  64         $xlrator_url = $source_url;
  65
  66         if (($method eq 'GET') || ($method eq 'HEAD'))
  67         {
  68
  69         # -----------------------------------------------------------
  70         #  Section: Windows Update / Windows Downloads
  71         # -----------------------------------------------------------
  72
  73         if (
  74             (($source_url =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi|msp|cab)$@i) ||
  75              ($source_url =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|msi|msp|cab)$@i))
  76         &&   ($source_url !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i)
  77         &&   ($source_url !~ m@\&@)
  78            )
  79         {
  80                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique);
  81         }
  82
  83         # -----------------------------------------------------------
  84         #  Section: Adobe Downloads
  85         # -----------------------------------------------------------
  86
  87         if (
  88             ($source_url =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|msi|bin|dmg|idx|gz)$@i) ||
  89             ($source_url =~ m@^http://swupdl\.adobe\.com/updates/.*\.(exe|msi|bin|dmg|idx|gz|[a-z][a-z]_[A-Z][A-Z])$@i) ||
  90             ($source_url =~ m@^http://swupmf\.adobe\.com/manifest/.*\.upd$@i)
  91            )
  92         {
  93                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Adobe",$unique);
  94         }
  95
  96         # -----------------------------------------------------------
  97         #  Section: Linux Downloads
  98         # -----------------------------------------------------------
  99
 100         if ($source_url =~ m@^[h|f]t?tp://[^?]+\.(deb|rpm)$@i)
 101         {
 102                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Linux",$mirror);
 103         }
 104
 105         # -----------------------------------------------------------
 106         #  Section: Trend Micro Downloads
 107         # -----------------------------------------------------------
 108
 109         if (
 110             ($source_url =~ m@^http://[^/]*\.trendmicro\.com/activeupdate/.*@i) &&
 111             ($source_url !~ m@.*/tmnotify\.dat$@i) &&
 112             ($source_url !~ m@.*/ini_xml\.zip$@i) &&
 113             ($source_url !~ m@.*/server\.ini$@i)
 114            )
 115         {
 116                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"TrendMicro",$mirror);
 117         }
 118
 119         # -----------------------------------------------------------
 120         #  Section: Symantec Downloads
 121         # -----------------------------------------------------------
 122
 123         if ($source_url =~ m@^[h|f]tt?p://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|vdb|xdb)$@i)
 124         {
 125                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Symantec",$unique);
 126         }
 127
 128         # -----------------------------------------------------------
 129         #  Section: Apple Downloads
 130         # -----------------------------------------------------------
 131
 132         if (
 133             (($source_url =~ m@^http://swcdn\.apple\.com/content/downloads/.*\.(tar)$@i) ||
 134              ($source_url =~ m@^http://appldnld\.apple\.com\.edgesuite\.net/.*\.(exe|dmg)$@i) ||
 135              ($source_url =~ m@^http://.*\.g.akamai.net/.*/3093/1/.*\.(tar|pkg|dmg|exe)$@i))
 136            )
 137         {
 138                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Apple",$unique);
 139         }
 140
 141         # -----------------------------------------------------------
 142         #  Section: Avast Downloads
 143         # -----------------------------------------------------------
 144
 145         if ($source_url =~ m@^http://download[\d]+\.avast\.com/.*\.(exe|vpu)$@i)
 146         {
 147                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avast",$mirror);
 148         }
 149
 150         # -----------------------------------------------------------
 151
 152         # -----------------------------------------------------------
 153         # Section: Avira Downloads
 154         # -----------------------------------------------------------
 155
 156         if ($source_url =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i)
 157         {
 158                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avira",$mirror);
 159         }
 160
 161         # -----------------------------------------------------------
 162         # Section: IPFire Downloads
 163         # -----------------------------------------------------------
 164
 165         if ($source_url =~ m@^[f|h]t?tp://.*\.(ipfire)$@i)
 166         {
 167                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"IPFire",$mirror);
 168         }
 169
 170         }
 171
 172         # -----------------------------------------------------------
 173         # Section: AVG Downloads
 174         # -----------------------------------------------------------
 175
 176         if ($source_url =~ m@^http://[^/]*\.(grisoft|avg)\.com/.*\.(bin)$@i)
 177         {
 178                 $xlrator_url = &check_cache($source_url,$hostaddr,$username,"AVG",$mirror);
 179         }
 180
 181         $request="$xlrator_url $hostaddr $username $method\n";
 182
 183         print $request;
 184 }
 185
 186 # -------------------------------------------------------------------
 187
 188 sub readhash
 189 {
 190         my $filename = $_[0];
 191         my $hash = $_[1];
 192         my ($var, $val);
 193
 194         if (-e $filename)
 195         {
 196                 open(FILE, $filename) or die "Unable to read file $filename";
 197                 while (<FILE>)
 198                 {
 199                         chop;
 200                         ($var, $val) = split /=/, $_, 2;
 201                         if ($var)
 202                         {
 203                                 $val =~ s/^\'//g;
 204                                 $val =~ s/\'$//g;
 205
 206                                 # Untaint variables read from hash
 207                                 $var =~ /([A-Za-z0-9_-]*)/; $var = $1;
 208                                 $val =~ /([\w\W]*)/; $val = $1;
 209                                 $hash->{$var} = $val;
 210                         }
 211                 }
 212                 close FILE;
 213         }
 214 }
 215
 216 # -------------------------------------------------------------------
 217
 218 sub writelog
 219 {
 220         if ($logging)
 221         {
 222                 open(LOGFILE,">>$logfile");
 223                 print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n";
 224                 close(LOGFILE);
 225         }
 226 }
 227
 228 # -------------------------------------------------------------------
 229
 230 sub debuglog
 231 {
 232         if ($debug)
 233         {
 234                 open(LOGFILE,">>/var/log/updatexlrator/debug.log");
 235                 my @now = localtime(time);
 236                 printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] [%s] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,"updxlrator",$_[0];
 237                 close(LOGFILE);
 238         }
 239 }
 240
 241 # -------------------------------------------------------------------
 242
 243 sub setcachestatus
 244 {
 245         open (FILE,">>$_[0]");
 246         print FILE "$_[1]\n";
 247         close FILE;
 248 }
 249
 250 # -------------------------------------------------------------------
 251
 252 sub diskfree
 253 {
 254         open(DF,"/bin/df --block-size=1 $_[0]|");
 255         my @dfdata = <DF>;
 256         close DF;
 257         shift(@dfdata);
 258         chomp(@dfdata);
 259         my $dfstr = join(' ',@dfdata);
 260         my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
 261         if ($free =~ m/^(\d+)$/)
 262         {
 263                 return $free;
 264         }
 265 }
 266
 267 # -------------------------------------------------------------------
 268
 269 sub diskusage
 270 {
 271         open(DF,"/bin/df $_[0]|");
 272         my @dfdata = <DF>;
 273         close DF;
 274         shift(@dfdata);
 275         chomp(@dfdata);
 276         my $dfstr = join(' ',@dfdata);
 277         my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
 278         if ($percent =~ m/^(\d+)%$/)
 279         {
 280                 $percent =~ s/%$//;
 281                 return $percent;
 282         }
 283 }
 284
 285 # -------------------------------------------------------------------
 286
 287 sub getmtime
 288 {
 289         my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($_[0]);
 290
 291         return $mtime;
 292 }
 293
 294 # -------------------------------------------------------------------
 295
 296 sub check_cache
 297 {
 298         my $updsource="UPDCACHE";
 299         my $updfile='';
 300         my $cacheurl='';
 301         my $vendorid='';
 302         my $uuid='';
 303         my @http_header=();
 304         my $remote_size=0;
 305         my $remote_mtime=0;
 306         my $login='';
 307         my $useragent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
 308
 309         my $sourceurl=$_[0];
 310         my $cfmirror=$_[4];
 311
 312         $sourceurl =~ s@\%2f@/@ig;
 313         $updfile = substr($sourceurl,rindex($sourceurl,"/")+1);
 314         $updfile =~ s@\%20@ @ig;
 315
 316         if ($cfmirror)
 317         {
 318                 $uuid = `echo $updfile | md5sum`;
 319         } else {
 320                 $uuid = `echo $sourceurl | md5sum`;
 321         }
 322
 323         $uuid =~ s/[^0-9a-f]//g;
 324         $uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/;
 325
 326         $vendorid = $_[3];
 327         $vendorid =~ tr/A-Z/a-z/;
 328
 329         &debuglog("Processing URL $sourceurl");
 330         &debuglog("Vendor ID is $vendorid");
 331         &debuglog("UUID is $uuid");
 332
 333         if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'}))
 334         {
 335                 $login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\"";
 336                 if ($proxysettings{'UPSTREAM_PASSWORD'})
 337                 {
 338                         $login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\"";
 339                 }
 340         }
 341
 342         if ($proxysettings{'UPSTREAM_PROXY'}) { &debuglog("Using upstream proxy $proxysettings{'UPSTREAM_PROXY'}"); }
 343
 344         $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
 345         @http_header = `$wget $login --user-agent="$useragent" --spider -S $sourceurl 2>&1`;
 346         $ENV{'http_proxy'} = '';
 347
 348         foreach (@http_header)
 349         {
 350                 chomp;
 351                 if (/^\s*Content-Length:\s/) { $remote_size = $_; $remote_size =~ s/[^0-9]//g; }
 352                 if (/^\s*Last-Modified:\s/) { $remote_mtime = $_; $remote_mtime =~ s/^\s*Last-Modified:\s//; $remote_mtime = HTTP::Date::str2time($remote_mtime) }
 353         }
 354
 355         if (
 356                 (-e "$updcachedir/$vendorid/$uuid/$updfile") &&
 357                 ($remote_size == (-s "$updcachedir/$vendorid/$uuid/$updfile")) &&
 358                 ($remote_mtime == &getmtime("$updcachedir/$vendorid/$uuid/$updfile"))
 359            )
 360         {
 361                 &debuglog("File exists in cache and is up to date");
 362                 &debuglog("Retrieving file from cache ($updsource)");
 363                 &setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time);
 364                 system("$apphome/bin/setperms $vendorid/$uuid/access.log");
 365                 $cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile";
 366         }
 367                 else
 368         {
 369                 if (-e "$updcachedir/$vendorid/$uuid/$updfile")
 370                 {
 371                         &debuglog("Local filesize: " . (-s "$updcachedir/$vendorid/$uuid/$updfile"));
 372                         &debuglog("Local timestamp: " . &getmtime("$updcachedir/$vendorid/$uuid/$updfile"));
 373                 } else { &debuglog("File not found in cache"); }
 374                 $updsource="DLSOURCE";
 375                 &debuglog("Remote filesize: $remote_size");
 376                 &debuglog("Remote timestamp: $remote_mtime");
 377                 &debuglog("Free disk space: " . &diskfree($updcachedir));
 378                 &debuglog("Disk usage: " . &diskusage($updcachedir) . "% (max. $maxusage%)");
 379                 if (-e "$updcachedir/download/$vendorid/$updfile") { &debuglog("File download/$vendorid/$updfile exists"); }
 380                 &debuglog("Retrieving file from source ($updsource)");
 381                 if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && ($remote_size <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$vendorid/$updfile"))
 382                 {
 383                         &debuglog("Running command $nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
 384                         system("$nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
 385                 }
 386                 $cacheurl=$sourceurl;
 387         }
 388
 389         &writelog($_[1],$_[2],$_[3],$updsource,$sourceurl);
 390
 391         return $cacheurl;
 392 }
 393
 394 # -------------------------------------------------------------------