X-Git-Url: http://git.ipfire.org/?p=people%2Fteissler%2Fipfire-2.x.git;a=blobdiff_plain;f=config%2Fupdxlrator%2Fupdxlrator;h=f93e2f2a5ac602d2e944195af76c39ca3ef27e18;hp=a7a7358492287fd705d2a3c21188b8628438e20e;hb=77fa792f93c8363d8199c02ec0b752639fe3ecec;hpb=69dcc42551e024d0a63a1d6749fb512c67c6b1ff diff --git a/config/updxlrator/updxlrator b/config/updxlrator/updxlrator index a7a735849..f93e2f2a5 100644 --- a/config/updxlrator/updxlrator +++ b/config/updxlrator/updxlrator @@ -2,50 +2,46 @@ # # This code is distributed under the terms of the GPL # -# (c) 2006 marco.s +# (c) 2006-2009 marco.s - http://update-accelerator.advproxy.net # -# $Id: updxlrator,v 1.0 2006/10/03 00:00:00 marco.s Exp $ +# Portions (c) 2008 by dotzball - http://www.blockouttraffic.de +# +# $Id: updxlrator,v 2.1 2009/01/10 00:00:00 marco.s Exp $ +# +# ChangeLog: +# +# 2012-10-26: nightshift - move curly bracket to capture AVG download source. +# - Adding BIG HINT for new update source# # use strict; - -use IO::Socket; +use HTTP::Date; $|=1; my $swroot="/var/ipfire"; -my $updcachedir="/srv/web/ipfire/html/updatecache"; +my $updcachedir="/var/updatecache"; +my $apphome="/var/ipfire/updatexlrator"; +my $logfile="/var/log/updatexlrator/cache.log"; +my $wget="/usr/bin/wget"; +my $debug=(-e "$apphome/debug"); +my $http_port='81'; my %netsettings=(); +my %proxysettings=(); my %xlratorsettings=(); -my $http_port="81"; -my $logfile="/var/log/updatexlrator/cache.log"; my $logging=0; my $passive_mode=0; my $maxusage=0; my $nice=''; my @tmp=(); -my $now=''; my $request=''; -my $from_local_cache=0; -my $dsturl=''; +my $xlrator_url=0; +my $source_url=''; my $hostaddr=''; my $username=''; my $method=''; -my @metadata=(); - -my $sfNoSource = "0"; -my $sfOk = "1"; -my $sfOutdated = "2"; - -unless (-d "$updcachedir/metadata") -{ - unless (-d "$updcachedir") { mkdir "$updcachedir"; } - mkdir "$updcachedir/metadata"; - system("chown nobody.squid $updcachedir"); - system("chmod 775 $updcachedir"); - system("chown nobody.squid $updcachedir/metadata"); - system("chmod 775 $updcachedir/metadata"); -} +my $unique = 0; +my $mirror = 1; readhash("${swroot}/ethernet/settings", \%netsettings); @@ -59,19 +55,22 @@ if (-e "$swroot/updatexlrator/settings") } if (!$maxusage) { $maxusage=75; }; +# dotzball: check for dead downloads +system("$apphome/bin/checkdeaddl &"); while (<>) { $request=$_; - $from_local_cache=0; @tmp=split(/ /,$request); chomp(@tmp); - $dsturl =$tmp[0]; - $hostaddr=$tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; } - $username=$tmp[2]; if ($username eq '') { $username='-'; }; - $method =$tmp[3]; + $source_url = $tmp[0]; + $hostaddr = $tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; } + $username = $tmp[2]; if ($username eq '') { $username='-'; }; + $method = $tmp[3]; + + $xlrator_url = $source_url; if (($method eq 'GET') || ($method eq 'HEAD')) { @@ -81,56 +80,139 @@ while (<>) { # ----------------------------------------------------------- if ( - (($dsturl =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi)$@i) || - ($dsturl =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|cab)$@i)) - && ($dsturl !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i) - && ($dsturl !~ m@\&@) + (($source_url =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi|msp|msu|cab)$@i) || + ($source_url =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|msi|msp|msu|cab)$@i)) + && ($source_url !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i) + && ($source_url !~ m@\&@) ) { - $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Microsoft"); + $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique); } # ----------------------------------------------------------- # Section: Adobe Downloads # ----------------------------------------------------------- - if ($dsturl =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|bin|dmg|idx|gz)$@i) + + if ( + ($source_url =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|msi|bin|dmg|idx|gz)$@i) || + ($source_url =~ m@^http://swupdl\.adobe\.com/updates/.*\.(exe|msi|bin|dmg|idx|gz|[a-z][a-z]_[A-Z][A-Z])$@i) || + ($source_url =~ m@^http://swupmf\.adobe\.com/manifest/.*\.upd$@i) + ) + { + $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Adobe",$unique); + } + + # ----------------------------------------------------------- + # Section: Linux Downloads + # ----------------------------------------------------------- + + if ( + ($source_url =~ m@^[h|f]t?tp://[^?]+\.(pkg\.tar\.xz|deb|rpm)$@i) || + ($source_url =~ m@^[h|f]t?tp://[^?]+/distfiles/[^?]+\.(tar\.gz|tar\.bz2|tgz|zip|patch\.bz2|gz|docx|patch|pdf|exe)$@i) + ) + { + $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Linux",$mirror); + } + + # ----------------------------------------------------------- + # Section: Trend Micro Downloads + # ----------------------------------------------------------- + + if ( + ($source_url =~ m@^http://[^/]*\.trendmicro\.com/activeupdate/.*@i) && + ($source_url !~ m@.*/tmnotify\.dat$@i) && + ($source_url !~ m@.*/ini_xml\.zip$@i) && + ($source_url !~ m@.*/server\.ini$@i) + ) { - $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Adobe"); + $xlrator_url = &check_cache($source_url,$hostaddr,$username,"TrendMicro",$mirror); } # ----------------------------------------------------------- # Section: Symantec Downloads # ----------------------------------------------------------- - if ($dsturl =~ m@^[f|h]t?tp://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|xdb)$@i) + if ($source_url =~ m@^[h|f]tt?p://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|vdb|xdb)$@i) { - $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Symantec"); + $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Symantec",$unique); } - + # ----------------------------------------------------------- - # Section: Avira Downloads + # Section: Apple Downloads # ----------------------------------------------------------- - - if ($dsturl =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i) + + if ( + (($source_url =~ m@^http://swcdn\.apple\.com/content/downloads/.*\.(tar|pkg)$@i) || + ($source_url =~ m@^http://appldnld\.apple\.com\.edgesuite\.net/.*\.(exe|dmg)$@i) || + ($source_url =~ m@^http://.*\.g.akamai.net/.*/3093/1/.*\.(tar|pkg|dmg|exe)$@i)) + ) { - $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Avira"); + $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Apple",$unique); } - - # ----------------------------------------------------------- - # Section: Avast Downloads - # ----------------------------------------------------------- - - if ($dsturl =~ m@^http://download[0-99]\.avast\.com/.*\.(exe|zip|vps|stamp|vpu)$@i) + + # ----------------------------------------------------------- + # Section: Avast Downloads + # ----------------------------------------------------------- + + if ($source_url =~ m@^http://(ion|download)[\d]+\.avast\.com/.*\.(exe|vpu|vpx)$@i) + { + $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avast",$mirror); + } + + # ----------------------------------------------------------- + # Section: Mozilla Downloads + # ----------------------------------------------------------- + + if ($source_url=~ m@^http://.*\.mozilla\.net/.*\.((complete|partial)\.mar|exe)$@i) { - $from_local_cache = &cache_access($dsturl,$hostaddr,$username,"Avast"); - } + $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Mozilla",$unique); + } + + # ----------------------------------------------------------- + # Section: Mcafee + # ----------------------------------------------------------- + + if ($source_url =~ m@^http://update\.nai\.com/.*\.(mcs|z|gem|dat|zip)$@i) + { + $xlrator_url = &check_cache($source_url,$hostaddr,$username,"mcafee",$mirror); + } - # ----------------------------------------------------------- + # ----------------------------------------------------------- + # Section: Avira Downloads + # ----------------------------------------------------------- + if ( + ($source_url =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i) || + ($source_url =~ m@^http://80.190.130.19[4-5]/update/.*\.(htm|html|gz)$@i) || + ($source_url =~ m@^http://62.146.64.14[6-7]/update/.*\.(htm|html|gz)$@i) + ) + { + $xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avira",$mirror); } - if ($from_local_cache) { $request="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/".substr($dsturl,rindex($dsturl,"/")+1)." $hostaddr $username $method\n"; } + # ----------------------------------------------------------- + # Section: IPFire Downloads + # ----------------------------------------------------------- + + if ($source_url =~ m@^[f|h]t?tp://.*\.(ipfire)$@i) + { + $xlrator_url = &check_cache($source_url,$hostaddr,$username,"IPFire",$mirror); + } + + # ----------------------------------------------------------- + # Section: AVG Downloads + # ----------------------------------------------------------- + +# if ($source_url =~ m@^http://[^/]*\.(grisoft|avg)\.com/.*\.(bin)$@i) +# { +# $xlrator_url = &check_cache($source_url,$hostaddr,$username,"AVG",$mirror); +# } + +# ----------- ADD NEW SOURCES BEFORE THIS LINE !!! ------------------ + } + + $request="$xlrator_url $hostaddr $username $method\n"; print $request; } @@ -169,29 +251,51 @@ sub readhash sub writelog { - open(LOGFILE,">>$logfile"); - print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n"; - close(LOGFILE); + if ($logging) + { + open(LOGFILE,">>$logfile"); + print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n"; + close(LOGFILE); + } } # ------------------------------------------------------------------- -sub diskfree +sub debuglog { - open(DF,"/bin/df --block-size=1 $_[0]|"); - while() + if ($debug) { - unless ($_ =~ m/^Filesystem/ ) - { - my ($device,$size,$used,$free,$percent,$mount) = split; - if ($free =~ m/^(\d+)$/) - { - close DF; - return $free; - } - } + open(LOGFILE,">>/var/log/updatexlrator/debug.log"); + my @now = localtime(time); + printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] [%s] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,"updxlrator",$_[0]; + close(LOGFILE); } +} + +# ------------------------------------------------------------------- + +sub setcachestatus +{ + open (FILE,">>$_[0]"); + print FILE "$_[1]\n"; + close FILE; +} + +# ------------------------------------------------------------------- + +sub diskfree +{ + open(DF,"/bin/df --block-size=1 $_[0]|"); + my @dfdata = ; close DF; + shift(@dfdata); + chomp(@dfdata); + my $dfstr = join(' ',@dfdata); + my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr); + if ($free =~ m/^(\d+)$/) + { + return $free; + } } # ------------------------------------------------------------------- @@ -199,106 +303,125 @@ sub diskfree sub diskusage { open(DF,"/bin/df $_[0]|"); - while() + my @dfdata = ; + close DF; + shift(@dfdata); + chomp(@dfdata); + my $dfstr = join(' ',@dfdata); + my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr); + if ($percent =~ m/^(\d+)%$/) { - unless ($_ =~ m/^Filesystem/ ) - { - my ($device,$size,$used,$free,$percent,$mount) = split; - if ($percent =~ m/^(\d+)%$/) - { - close DF; - $percent =~ s/%$//; - return $percent; - } - } + $percent =~ s/%$//; + return $percent; } - close DF; } # ------------------------------------------------------------------- -sub getdownloadsize +sub getmtime { - my $remote=0; - my @response=(); - my $contentlength=0; - - my $url = $_[0]; + my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($_[0]); - $url =~ s@^(.*)://([^/]*)@@; + return $mtime; +} - my $proto = $1; - my $fqhn = $2; +# ------------------------------------------------------------------- - if ((-e "$swroot/red/active") && ($proto eq 'http')) +sub check_cache +{ + my $updsource="UPDCACHE"; + my $updfile=''; + my $cacheurl=''; + my $vendorid=''; + my $uuid=''; + my @http_header=(); + my $remote_size=0; + my $remote_mtime=0; + my $login=''; + my $useragent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"; + + my $sourceurl=$_[0]; + my $cfmirror=$_[4]; + + $sourceurl =~ s@\%2f@/@ig; + $updfile = substr($sourceurl,rindex($sourceurl,"/")+1); + $updfile =~ s@\%20@ @ig; + + if ($cfmirror) { - $remote = IO::Socket::INET->new( - PeerHost => $fqhn, - PeerPort => 'http(80)', - Timeout => 1 - ); + $uuid = `echo $updfile | md5sum`; + } else { + $uuid = `echo $sourceurl | md5sum`; } - if ($remote) + $uuid =~ s/[^0-9a-f]//g; + $uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/; + + $vendorid = $_[3]; + $vendorid =~ tr/A-Z/a-z/; + + &debuglog("Processing URL $sourceurl"); + &debuglog("Vendor ID is $vendorid"); + &debuglog("UUID is $uuid"); + + if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'})) { - print $remote "HEAD $url HTTP/1.0\n"; - print $remote "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)\n"; - print $remote "Host: $fqhn\n"; - print $remote "Accept: */*\n\n"; - while (<$remote>) { push(@response,$_); } - close $remote; - if ($response[0] =~ /^HTTP\/\d+\.\d+\s\d+\sOK\s*$/) + $login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\""; + if ($proxysettings{'UPSTREAM_PASSWORD'}) { - foreach (@response) - { - if (/^Content-Length: /i) - { - s/^Content-Length: //i; - $contentlength=$_; - } - } + $login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\""; } } - return $contentlength; -} - -# ------------------------------------------------------------------- -sub cache_access -{ - my $updsource="UPDCACHE"; - my $updfile=''; - my $do_redirect=0; + if ($proxysettings{'UPSTREAM_PROXY'}) { &debuglog("Using upstream proxy $proxysettings{'UPSTREAM_PROXY'}"); } - $_[0] =~ s@\%2f@/@ig; - $updfile = substr($_[0],rindex($_[0],"/")+1); + $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'}; + @http_header = `$wget $login --user-agent="$useragent" --spider -S $sourceurl 2>&1`; + $ENV{'http_proxy'} = ''; - if (!-e "$updcachedir/metadata/$updfile") + foreach (@http_header) { - open(FILE,">$updcachedir/metadata/$updfile"); - print FILE "$_[0]\n$_[3]\n$sfOutdated\n0\n"; - close(FILE); + chomp; + if (/^\s*Content-Length:\s/) { $remote_size = $_; $remote_size =~ s/[^0-9]//g; } + if (/^\s*Last-Modified:\s/) { $remote_mtime = $_; $remote_mtime =~ s/^\s*Last-Modified:\s//; $remote_mtime = HTTP::Date::str2time($remote_mtime) } } - if (-e "$updcachedir/$updfile") + if ( + (-e "$updcachedir/$vendorid/$uuid/$updfile") && + ($remote_size == (-s "$updcachedir/$vendorid/$uuid/$updfile")) && + ($remote_mtime == &getmtime("$updcachedir/$vendorid/$uuid/$updfile")) + ) { - open(FILE,">>$updcachedir/metadata/$updfile"); - print FILE time."\n"; - close(FILE); - $do_redirect=1; + &debuglog("File exists in cache and is up to date"); + &debuglog("Retrieving file from cache ($updsource)"); + &setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time); + $cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile"; } else { + if (-e "$updcachedir/$vendorid/$uuid/$updfile") + { + &debuglog("Local filesize: " . (-s "$updcachedir/$vendorid/$uuid/$updfile")); + &debuglog("Local timestamp: " . &getmtime("$updcachedir/$vendorid/$uuid/$updfile")); + } else { &debuglog("File not found in cache"); } $updsource="DLSOURCE"; - if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && (&getdownloadsize <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$updfile")) + &debuglog("Remote filesize: $remote_size"); + &debuglog("Remote timestamp: $remote_mtime"); + &debuglog("Free disk space: " . &diskfree($updcachedir)); + &debuglog("Disk usage: " . &diskusage($updcachedir) . "% (max. $maxusage%)"); + if (-e "$updcachedir/download/$vendorid/$updfile") { &debuglog("File download/$vendorid/$updfile exists"); } + &debuglog("Retrieving file from source ($updsource)"); + if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && ($remote_size <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$vendorid/$updfile")) { - system("$nice/var/ipfire/updatexlrator/bin/download $_[0] &"); + &debuglog("Running command $nice$apphome/bin/download $vendorid $sourceurl $cfmirror &"); + system("$nice$apphome/bin/download $vendorid $sourceurl $cfmirror &"); } + $cacheurl=$sourceurl; } - if ($logging) { &writelog($_[1],$_[2],$_[3],$updsource,$_[0]); } + &writelog($_[1],$_[2],$_[3],$updsource,$sourceurl); - return $do_redirect; + return $cacheurl; } # -------------------------------------------------------------------