#
# This code is distributed under the terms of the GPL
#
-# (c) 2006 marco.s
+# (c) 2006-2008 marco.s - http://update-accelerator.advproxy.net
#
-# $Id: updxlrator,v 1.0 2006/10/03 00:00:00 marco.s Exp $
+# $Id: updxlrator,v 2.0 2008/04/06 00:00:00 marco.s Exp $
#
use strict;
-
-use IO::Socket;
+use HTTP::Date;
$|=1;
my $swroot="/var/ipfire";
my $updcachedir="/var/updatecache";
+my $apphome="/var/ipfire/updatexlrator";
+my $logfile="/var/log/updatexlrator/cache.log";
+my $wget="$apphome/bin/wget";
+my $debug=(-e "$apphome/debug");
+my $http_port='81';
my %netsettings=();
+my %proxysettings=();
my %xlratorsettings=();
-my $http_port="81";
-my $logfile="/var/log/updatexlrator/cache.log";
my $logging=0;
my $passive_mode=0;
my $maxusage=0;
my $nice='';
my @tmp=();
-my $now='';
my $request='';
-my $from_local_cache=0;
-my $dsturl='';
+my $xlrator_url=0;
+my $source_url='';
my $hostaddr='';
my $username='';
my $method='';
-my @metadata=();
-
-my $sfNoSource = "0";
-my $sfOk = "1";
-my $sfOutdated = "2";
-
-unless (-d "$updcachedir/metadata")
-{
- unless (-d "$updcachedir") { mkdir "$updcachedir"; }
- mkdir "$updcachedir/metadata";
- system("chown nobody.squid $updcachedir");
- system("chmod 775 $updcachedir");
- system("chown nobody.squid $updcachedir/metadata");
- system("chmod 775 $updcachedir/metadata");
-}
+my $unique = 0;
+my $mirror = 1;
readhash("${swroot}/ethernet/settings", \%netsettings);
if ($xlratorsettings{'ENABLE_LOG'} eq 'on') { $logging=1; };
if ($xlratorsettings{'PASSIVE_MODE'} eq 'on') { $passive_mode=1; };
$maxusage=$xlratorsettings{'MAX_DISK_USAGE'};
- if ($xlratorsettings{'LOW_DOWNLOAD_PRIORITY'} eq 'on') { $nice='/bin/nice --adjustment=15 '; };
+ if ($xlratorsettings{'LOW_DOWNLOAD_PRIORITY'} eq 'on') { $nice='/usr/bin/nice --adjustment=15 '; };
}
if (!$maxusage) { $maxusage=75; };
while (<>) {
$request=$_;
- $from_local_cache=0;
@tmp=split(/ /,$request);
chomp(@tmp);
- $dsturl =$tmp[0];
- $hostaddr=$tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; }
- $username=$tmp[2]; if ($username eq '') { $username='-'; };
- $method =$tmp[3];
+ $source_url = $tmp[0];
+ $hostaddr = $tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; }
+ $username = $tmp[2]; if ($username eq '') { $username='-'; };
+ $method = $tmp[3];
+
+ $xlrator_url = $source_url;
if (($method eq 'GET') || ($method eq 'HEAD'))
{
# -----------------------------------------------------------
if (
- (($dsturl =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi)$@i) ||
- ($dsturl =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|cab)$@i))
- && ($dsturl !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i)
- && ($dsturl !~ m@\&@)
+ (($source_url =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi|msp|cab)$@i) ||
+ ($source_url =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|msi|msp|cab)$@i))
+ && ($source_url !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i)
+ && ($source_url !~ m@\&@)
)
{
- $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Microsoft");
+ $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique);
}
# -----------------------------------------------------------
# Section: Adobe Downloads
# -----------------------------------------------------------
- if ($dsturl =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|bin|dmg|idx|gz)$@i)
+ if (
+ ($source_url =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|msi|bin|dmg|idx|gz)$@i) ||
+ ($source_url =~ m@^http://swupdl\.adobe\.com/updates/.*\.(exe|msi|bin|dmg|idx|gz|[a-z][a-z]_[A-Z][A-Z])$@i) ||
+ ($source_url =~ m@^http://swupmf\.adobe\.com/manifest/.*\.upd$@i)
+ )
+ {
+ $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Adobe",$unique);
+ }
+
+ # -----------------------------------------------------------
+ # Section: Linux Downloads
+ # -----------------------------------------------------------
+
+ if ($source_url =~ m@^[h|f]t?tp://[^?]+\.(deb|rpm)$@i)
{
- $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Adobe");
+ $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Linux",$mirror);
+ }
+
+ # -----------------------------------------------------------
+ # Section: Trend Micro Downloads
+ # -----------------------------------------------------------
+
+ if (
+ ($source_url =~ m@^http://[^/]*\.trendmicro\.com/activeupdate/.*@i) &&
+ ($source_url !~ m@.*/tmnotify\.dat$@i) &&
+ ($source_url !~ m@.*/ini_xml\.zip$@i) &&
+ ($source_url !~ m@.*/server\.ini$@i)
+ )
+ {
+ $xlrator_url = &check_cache($source_url,$hostaddr,$username,"TrendMicro",$mirror);
}
# -----------------------------------------------------------
# Section: Symantec Downloads
# -----------------------------------------------------------
- if ($dsturl =~ m@^[f|h]t?tp://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|xdb)$@i)
+ if ($source_url =~ m@^[h|f]tt?p://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|vdb|xdb)$@i)
{
- $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Symantec");
+ $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Symantec",$unique);
}
-
+
# -----------------------------------------------------------
- # Section: Avira Downloads
+ # Section: Apple Downloads
# -----------------------------------------------------------
-
- if ($dsturl =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i)
+
+ if (
+ (($source_url =~ m@^http://swcdn\.apple\.com/content/downloads/.*\.(tar)$@i) ||
+ ($source_url =~ m@^http://appldnld\.apple\.com\.edgesuite\.net/.*\.(exe|dmg)$@i) ||
+ ($source_url =~ m@^http://.*\.g.akamai.net/.*/3093/1/.*\.(tar|pkg|dmg|exe)$@i))
+ )
{
- $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Avira");
+ $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Apple",$unique);
}
-
- # -----------------------------------------------------------
- # Section: Avast Downloads
- # -----------------------------------------------------------
-
- if ($dsturl =~ m@^http://download[0-99]\.avast\.com/.*\.(exe|zip|vps|stamp|vpu)$@i)
- {
- $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Avast");
- }
-
- # -----------------------------------------------------------
- # Section: IPFire Downloads
- # -----------------------------------------------------------
-
- if ($dsturl =~ m@^[f|h]t?tp://.*\.(ipfire)$@i)
- {
- $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"IPFire");
- }
-
- # -----------------------------------------------------------
- # Section: Linux Downloads
- # -----------------------------------------------------------
-
- if ($dsturl =~ m@^[f|h]t?tp://.*\.(deb|rpm)$@i)
- {
- $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Linux");
+
+ # -----------------------------------------------------------
+ # Section: Avast Downloads
+ # -----------------------------------------------------------
+
+ if ($source_url =~ m@^http://download[\d]+\.avast\.com/.*\.(exe|vpu)$@i)
+ {
+ $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avast",$mirror);
}
+
+ # -----------------------------------------------------------
- # -----------------------------------------------------------
-
- # -----------------------------------------------------------
- # Section: Apple Downloads
- # -----------------------------------------------------------
-
- if ($dsturl =~ m@^[f|h]t?tp://swcdn\.apple.*\.(pkm|tar)$@i)
- {
- $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Apple");
+ # -----------------------------------------------------------
+ # Section: Avira Downloads
+ # -----------------------------------------------------------
+
+ if ($source_url =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i)
+ {
+ $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avira",$mirror);
}
# -----------------------------------------------------------
- # Section: AVG Downloads
+ # Section: IPFire Downloads
# -----------------------------------------------------------
- if ($dsturl =~ m@^http://[^/]*\.(grisoft|avg)\.com/.*\.(bin)$@i)
+ if ($source_url =~ m@^[f|h]t?tp://.*\.(ipfire)$@i)
{
- $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"AVG");
+ $xlrator_url = &check_cache($source_url,$hostaddr,$username,"IPFire",$mirror);
}
+ }
- # -----------------------------------------------------------
- # Section: Kaspersky Downloads
- # not working properly
- # -----------------------------------------------------------
-
- #if ($dsturl =~ m@^http://dnl-.*\.kaspersky-labs\.com\/(diffs|bases|AutoPatches).*$@i)
- #{
- #$from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Kaspersky");
- #}
-
- # -----------------------------------------------------------
+ # -----------------------------------------------------------
+ # Section: AVG Downloads
+ # -----------------------------------------------------------
+ if ($source_url =~ m@^http://[^/]*\.(grisoft|avg)\.com/.*\.(bin)$@i)
+ {
+ $xlrator_url = &check_cache($source_url,$hostaddr,$username,"AVG",$mirror);
}
- if ($from_local_cache) { $request="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/".substr($dsturl,rindex($dsturl,"/")+1)." $hostaddr $username $method\n"; }
+ $request="$xlrator_url $hostaddr $username $method\n";
print $request;
}
sub writelog
{
- open(LOGFILE,">>$logfile");
- print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n";
- close(LOGFILE);
+ if ($logging)
+ {
+ open(LOGFILE,">>$logfile");
+ print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n";
+ close(LOGFILE);
+ }
}
# -------------------------------------------------------------------
-sub diskfree
+sub debuglog
{
- open(DF,"/bin/df --block-size=1 $_[0]|");
- while(<DF>)
+ if ($debug)
{
- unless ($_ =~ m/^Filesystem/ )
- {
- my ($device,$size,$used,$free,$percent,$mount) = split;
- if ($free =~ m/^(\d+)$/)
- {
- close DF;
- return $free;
- }
- }
+ open(LOGFILE,">>/var/log/updatexlrator/debug.log");
+ my @now = localtime(time);
+ printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] [%s] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,"updxlrator",$_[0];
+ close(LOGFILE);
}
+}
+
+# -------------------------------------------------------------------
+
+sub setcachestatus
+{
+ open (FILE,">>$_[0]");
+ print FILE "$_[1]\n";
+ close FILE;
+}
+
+# -------------------------------------------------------------------
+
+sub diskfree
+{
+ open(DF,"/bin/df --block-size=1 $_[0]|");
+ my @dfdata = <DF>;
close DF;
+ shift(@dfdata);
+ chomp(@dfdata);
+ my $dfstr = join(' ',@dfdata);
+ my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
+ if ($free =~ m/^(\d+)$/)
+ {
+ return $free;
+ }
}
# -------------------------------------------------------------------
sub diskusage
{
open(DF,"/bin/df $_[0]|");
- while(<DF>)
+ my @dfdata = <DF>;
+ close DF;
+ shift(@dfdata);
+ chomp(@dfdata);
+ my $dfstr = join(' ',@dfdata);
+ my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
+ if ($percent =~ m/^(\d+)%$/)
{
- unless ($_ =~ m/^Filesystem/ )
- {
- my ($device,$size,$used,$free,$percent,$mount) = split;
- if ($percent =~ m/^(\d+)%$/)
- {
- close DF;
- $percent =~ s/%$//;
- return $percent;
- }
- }
+ $percent =~ s/%$//;
+ return $percent;
}
- close DF;
}
# -------------------------------------------------------------------
-sub getdownloadsize
+sub getmtime
{
- my $remote=0;
- my @response=();
- my $contentlength=0;
-
- my $url = $_[0];
+ my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($_[0]);
- $url =~ s@^(.*)://([^/]*)@@;
+ return $mtime;
+}
- my $proto = $1;
- my $fqhn = $2;
+# -------------------------------------------------------------------
- if ((-e "$swroot/red/active") && ($proto eq 'http'))
+sub check_cache
+{
+ my $updsource="UPDCACHE";
+ my $updfile='';
+ my $cacheurl='';
+ my $vendorid='';
+ my $uuid='';
+ my @http_header=();
+ my $remote_size=0;
+ my $remote_mtime=0;
+ my $login='';
+ my $useragent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
+
+ my $sourceurl=$_[0];
+ my $cfmirror=$_[4];
+
+ $sourceurl =~ s@\%2f@/@ig;
+ $updfile = substr($sourceurl,rindex($sourceurl,"/")+1);
+ $updfile =~ s@\%20@ @ig;
+
+ if ($cfmirror)
{
- $remote = IO::Socket::INET->new(
- PeerHost => $fqhn,
- PeerPort => 'http(80)',
- Timeout => 1
- );
+ $uuid = `echo $updfile | md5sum`;
+ } else {
+ $uuid = `echo $sourceurl | md5sum`;
}
- if ($remote)
+ $uuid =~ s/[^0-9a-f]//g;
+ $uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/;
+
+ $vendorid = $_[3];
+ $vendorid =~ tr/A-Z/a-z/;
+
+ &debuglog("Processing URL $sourceurl");
+ &debuglog("Vendor ID is $vendorid");
+ &debuglog("UUID is $uuid");
+
+ if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'}))
{
- print $remote "HEAD $url HTTP/1.0\n";
- print $remote "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)\n";
- print $remote "Host: $fqhn\n";
- print $remote "Accept: */*\n\n";
- while (<$remote>) { push(@response,$_); }
- close $remote;
- if ($response[0] =~ /^HTTP\/\d+\.\d+\s\d+\sOK\s*$/)
+ $login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\"";
+ if ($proxysettings{'UPSTREAM_PASSWORD'})
{
- foreach (@response)
- {
- if (/^Content-Length: /i)
- {
- s/^Content-Length: //i;
- $contentlength=$_;
- }
- }
+ $login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\"";
}
}
- return $contentlength;
-}
-# -------------------------------------------------------------------
+ if ($proxysettings{'UPSTREAM_PROXY'}) { &debuglog("Using upstream proxy $proxysettings{'UPSTREAM_PROXY'}"); }
-sub cache_access
-{
- my $updsource="UPDCACHE";
- my $updfile='';
- my $do_redirect=0;
-
- $_[0] =~ s@\%2f@/@ig;
- $updfile = substr($_[0],rindex($_[0],"/")+1);
+ $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
+ @http_header = `$wget $login --user-agent="$useragent" --spider -S $sourceurl 2>&1`;
+ $ENV{'http_proxy'} = '';
- if (!-e "$updcachedir/metadata/$updfile")
+ foreach (@http_header)
{
- open(FILE,">$updcachedir/metadata/$updfile");
- print FILE "$_[0]\n$_[3]\n$sfOutdated\n0\n";
- close(FILE);
+ chomp;
+ if (/^\s*Content-Length:\s/) { $remote_size = $_; $remote_size =~ s/[^0-9]//g; }
+ if (/^\s*Last-Modified:\s/) { $remote_mtime = $_; $remote_mtime =~ s/^\s*Last-Modified:\s//; $remote_mtime = HTTP::Date::str2time($remote_mtime) }
}
- if (-e "$updcachedir/$updfile")
+ if (
+ (-e "$updcachedir/$vendorid/$uuid/$updfile") &&
+ ($remote_size == (-s "$updcachedir/$vendorid/$uuid/$updfile")) &&
+ ($remote_mtime == &getmtime("$updcachedir/$vendorid/$uuid/$updfile"))
+ )
{
- open(FILE,">>$updcachedir/metadata/$updfile");
- print FILE time."\n";
- close(FILE);
- $do_redirect=1;
+ &debuglog("File exists in cache and is up to date");
+ &debuglog("Retrieving file from cache ($updsource)");
+ &setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time);
+ system("$apphome/bin/setperms $vendorid/$uuid/access.log");
+ $cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile";
}
else
{
+ if (-e "$updcachedir/$vendorid/$uuid/$updfile")
+ {
+ &debuglog("Local filesize: " . (-s "$updcachedir/$vendorid/$uuid/$updfile"));
+ &debuglog("Local timestamp: " . &getmtime("$updcachedir/$vendorid/$uuid/$updfile"));
+ } else { &debuglog("File not found in cache"); }
$updsource="DLSOURCE";
- if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && (&getdownloadsize <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$updfile"))
+ &debuglog("Remote filesize: $remote_size");
+ &debuglog("Remote timestamp: $remote_mtime");
+ &debuglog("Free disk space: " . &diskfree($updcachedir));
+ &debuglog("Disk usage: " . &diskusage($updcachedir) . "% (max. $maxusage%)");
+ if (-e "$updcachedir/download/$vendorid/$updfile") { &debuglog("File download/$vendorid/$updfile exists"); }
+ &debuglog("Retrieving file from source ($updsource)");
+ if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && ($remote_size <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$vendorid/$updfile"))
{
- system("$nice/var/ipfire/updatexlrator/bin/download $_[0] &");
+ &debuglog("Running command $nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
+ system("$nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
}
+ $cacheurl=$sourceurl;
}
- if ($logging) { &writelog($_[1],$_[2],$_[3],$updsource,$_[0]); }
+ &writelog($_[1],$_[2],$_[3],$updsource,$sourceurl);
- return $do_redirect;
+ return $cacheurl;
}
# -------------------------------------------------------------------
projects / people / pmueller / ipfire-2.x.git / blobdiff