[people/pmueller/ipfire-2.x.git] / config / updxlrator / updxlrator

#!/usr/bin/perl
#
# This code is distributed under the terms of the GPL
#
# (c) 2006-2009 marco.s - http://update-accelerator.advproxy.net
#
# Portions (c) 2008 by dotzball - http://www.blockouttraffic.de
#
# $Id: updxlrator,v 2.1 2009/01/10 00:00:00 marco.s Exp $
#
# ChangeLog:
#
# 2012-10-26: nightshift - move curly bracket to capture AVG download source.
#			 - Adding BIG HINT for new update source#
#

use strict;
use HTTP::Date;

$|=1;

my $swroot="/var/ipfire";
my $updcachedir="/var/updatecache";
my $apphome="/var/ipfire/updatexlrator";
my $logfile="/var/log/updatexlrator/cache.log";
my $wget="/usr/bin/wget";
my $debug=(-e "$apphome/debug");
my $http_port='81';
my %netsettings=();
my %proxysettings=();
my %xlratorsettings=();
my $logging=0;
my $passive_mode=0;
my $maxusage=0;
my $nice='';
my @tmp=();
my $request='';
my $xlrator_url=0;
my $source_url='';
my $hostaddr='';
my $username='';
my $method='';
my $unique = 0;
my $mirror = 1;

readhash("${swroot}/ethernet/settings", \%netsettings);

if (-e "$swroot/updatexlrator/settings")
{
	&readhash("$swroot/updatexlrator/settings", \%xlratorsettings);
	if ($xlratorsettings{'ENABLE_LOG'} eq 'on') { $logging=1; };
	if ($xlratorsettings{'PASSIVE_MODE'} eq 'on') { $passive_mode=1; };
	$maxusage=$xlratorsettings{'MAX_DISK_USAGE'};
	if ($xlratorsettings{'LOW_DOWNLOAD_PRIORITY'} eq 'on') { $nice='/bin/nice --adjustment=15 '; };
}
if (!$maxusage) { $maxusage=75; };

# dotzball: check for dead downloads
system("$apphome/bin/checkdeaddl &");

while (<>) {

	$request=$_;

	@tmp=split(/ /,$request);
	chomp(@tmp);

	$source_url = $tmp[0];
	$hostaddr   = $tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; }
	$username   = $tmp[2]; if ($username eq '') { $username='-'; };
	$method     = $tmp[3];

	$xlrator_url = "";

	if (($method eq 'GET') || ($method eq 'HEAD')) 
	{

	# -----------------------------------------------------------
	#  Section: Windows Update / Windows Downloads
	# -----------------------------------------------------------

	if (
	    (($source_url =~ m@^http://[^/]*\.microsoft\.com/.*\.(exe|psf|msi|msp|msu|cab)$@i) ||
	     ($source_url =~ m@^http://[^/]*\.windowsupdate\.com/.*\.(exe|psf|msi|msp|msu|cab)$@i))
	&&   ($source_url !~ m@^http://[^/]*\.microsoft\.com/.*(/autoupd|selfupdate/).*\.cab@i)
	&&   ($source_url !~ m@\&@)
	   )
	{
		# NOTE: check_cache will change to $mirror instead of $unique if the filename contains an SHA1 hash
		# and the URL is not found in cache!
		$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique);
	}

	# -----------------------------------------------------------
	#  Section: Adobe Downloads
	# -----------------------------------------------------------


	if (
	    ($source_url =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe|msi|bin|dmg|idx|gz)$@i) ||
	    ($source_url =~ m@^http://swupdl\.adobe\.com/updates/.*\.(exe|msi|bin|dmg|idx|gz|[a-z][a-z]_[A-Z][A-Z])$@i) ||
	    ($source_url =~ m@^http://swupmf\.adobe\.com/manifest/.*\.upd$@i)
	   )
	{
		$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Adobe",$unique);
	}

	# -----------------------------------------------------------
	#  Section: Linux Downloads
	# -----------------------------------------------------------

	if (
		($source_url =~ m@^[h|f]t?tp://[^?]+\.(pkg\.tar\.xz|deb|rpm)$@i) ||
		($source_url =~ m@^[h|f]t?tp://[^?]+/distfiles/[^?]+\.(tar\.gz|tar\.bz2|tar\.xz|tgz|zip|patch\.bz2|gz|docx|patch|pdf|exe)$@i)
	)
	{
		$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Linux",$mirror);
	}

	# -----------------------------------------------------------
	#  Section: Trend Micro Downloads
	# -----------------------------------------------------------

	if (
	    ($source_url =~ m@^http://[^/]*\.trendmicro\.com/activeupdate/.*@i) &&
	    ($source_url !~ m@.*/tmnotify\.dat$@i) &&
	    ($source_url !~ m@.*/ini_xml\.zip$@i) &&
	    ($source_url !~ m@.*/server\.ini$@i)
	   )
	{
		$xlrator_url = &check_cache($source_url,$hostaddr,$username,"TrendMicro",$mirror);
	}

	# -----------------------------------------------------------
	#  Section: Symantec Downloads
	# -----------------------------------------------------------

	if ($source_url =~ m@^[h|f]tt?p://[^/]*\.symantec(liveupdate)?\.com/.*\.(exe|zip|vdb|xdb)$@i)
	{
		$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Symantec",$unique);
	}

	# -----------------------------------------------------------
	#  Section: Apple Downloads
	# -----------------------------------------------------------

	if (
	     (($source_url =~ m@^http://swcdn\.apple\.com/content/downloads/.*\.(tar|pkg)$@i) ||
	     ($source_url =~ m@^http://appldnld\.apple\.com\.edgesuite\.net/.*\.(exe|dmg)$@i) ||
	     ($source_url =~ m@^http://.*\.g.akamai.net/.*/3093/1/.*\.(tar|pkg|dmg|exe)$@i))
	   )
	{
		$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Apple",$unique);
	}

	# -----------------------------------------------------------
	#  Section: Avast Downloads
	# -----------------------------------------------------------

	if ($source_url =~ m@^http://(ion|download)[\d]+\.avast\.com/.*\.(exe|vpu|vpx)$@i)
	{
		$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avast",$mirror);
	}

        # -----------------------------------------------------------
        #  Section: Mozilla Downloads
        # -----------------------------------------------------------

	if ($source_url=~ m@^http://.*\.mozilla\.net/.*\.((complete|partial)\.mar|exe)$@i)
	{ 
		$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Mozilla",$unique);
	}

	# -----------------------------------------------------------
	#  Section: Mcafee
	# -----------------------------------------------------------
	
	if ($source_url =~ m@^http://update\.nai\.com/.*\.(mcs|z|gem|dat|zip)$@i)
	{
		$xlrator_url = &check_cache($source_url,$hostaddr,$username,"mcafee",$mirror);
	}
	
	# -----------------------------------------------------------
	# Section: Avira Downloads
	# -----------------------------------------------------------

	if (
		($source_url =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm|html|gz)$@i) ||
		($source_url =~ m@^http://80.190.130.19[4-5]/update/.*\.(htm|html|gz)$@i) ||
		($source_url =~ m@^http://62.146.64.14[6-7]/update/.*\.(htm|html|gz)$@i)
	)
	{
		$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avira",$mirror);
	}

	# -----------------------------------------------------------
	# Section: IPFire Downloads
	# -----------------------------------------------------------

	if ($source_url =~ m@^[f|h]t?tp://.*\.(ipfire)$@i)
	{
		$xlrator_url = &check_cache($source_url,$hostaddr,$username,"IPFire",$mirror);
	}

	# -----------------------------------------------------------
	# Section: AVG Downloads
	# -----------------------------------------------------------

#	if ($source_url =~ m@^http://[^/]*\.(grisoft|avg)\.com/.*\.(bin)$@i)
#	{
#		$xlrator_url = &check_cache($source_url,$hostaddr,$username,"AVG",$mirror);
#	}

# ----------- ADD NEW SOURCES BEFORE THIS LINE !!! ------------------
	}

	if ($xlrator_url) {
		print "OK rewrite-url=\"$xlrator_url\"\n";
	} else {
		print "ERR\n";
	}
}

# -------------------------------------------------------------------

sub readhash
{
	my $filename = $_[0];
	my $hash = $_[1];
	my ($var, $val);

	if (-e $filename)
	{
		open(FILE, $filename) or die "Unable to read file $filename";
		while (<FILE>)
		{
			chop;
			($var, $val) = split /=/, $_, 2;
			if ($var)
			{
				$val =~ s/^\'//g;
				$val =~ s/\'$//g;

				# Untaint variables read from hash
				$var =~ /([A-Za-z0-9_-]*)/; $var = $1;
				$val =~ /([\w\W]*)/; $val = $1;
				$hash->{$var} = $val;
			}
		}
		close FILE;
	}
}

# -------------------------------------------------------------------

sub writelog
{
	if ($logging)
	{
		open(LOGFILE,">>$logfile");
		print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n";
		close(LOGFILE);
	}
}

# -------------------------------------------------------------------

sub debuglog
{
	if ($debug)
	{
		open(LOGFILE,">>/var/log/updatexlrator/debug.log");
		my @now = localtime(time);
		printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] [%s] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,"updxlrator",$_[0];
		close(LOGFILE);
	}
}

# -------------------------------------------------------------------

sub setcachestatus
{
	open (FILE,">>$_[0]");
	print FILE "$_[1]\n";
	close FILE;
}

# -------------------------------------------------------------------

sub diskfree 
{ 
	open(DF,"/bin/df --block-size=1 $_[0]|");
	my @dfdata = <DF>;
	close DF;
	shift(@dfdata);
	chomp(@dfdata);
	my $dfstr = join(' ',@dfdata);
	my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
	if ($free =~ m/^(\d+)$/)
	{
        	return $free;
	}
}

# -------------------------------------------------------------------

sub diskusage
{
	open(DF,"/bin/df $_[0]|");
	my @dfdata = <DF>;
	close DF;
	shift(@dfdata);
	chomp(@dfdata);
	my $dfstr = join(' ',@dfdata);
	my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
	if ($percent =~ m/^(\d+)%$/)
	{
        	$percent =~ s/%$//;
	        return $percent;
	}
}

# -------------------------------------------------------------------

sub getmtime
{
	my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($_[0]);

	return $mtime;
}

# -------------------------------------------------------------------

sub check_cache 
{
	my $updsource="UPDCACHE";
	my $updfile='';
	my $cacheurl='';
	my $vendorid='';
	my $uuid='';
	my @http_header=();
	my $remote_size=0;
	my $remote_mtime=0;
	my $login='';
	my $useragent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";

	my $sourceurl=$_[0];
	my $cfmirror=$_[4];

	$sourceurl =~ s@\%2b@+@ig;
	$sourceurl =~ s@\%2f@/@ig;
	$sourceurl =~ s@\%7e@~@ig;
	$updfile = substr($sourceurl,rindex($sourceurl,"/")+1);
	$updfile =~ s@\%20@ @ig;

	if ($cfmirror)
	{
		$uuid = `echo $updfile | md5sum`;
	} else {
		$uuid = `echo $sourceurl | md5sum`;
	}

	$uuid =~ s/[^0-9a-f]//g;
	$uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/;

	$vendorid = $_[3];
	$vendorid =~ tr/A-Z/a-z/;

	&debuglog("Processing URL $sourceurl");
	&debuglog("Vendor ID is $vendorid");
	&debuglog("UUID is $uuid");

	if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'}))
	{
		$login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\"";
		if ($proxysettings{'UPSTREAM_PASSWORD'})
		{
			$login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\"";
		}
	}

	if ($proxysettings{'UPSTREAM_PROXY'}) { &debuglog("Using upstream proxy $proxysettings{'UPSTREAM_PROXY'}"); }

	$ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
	@http_header = `$wget $login --user-agent="$useragent" --spider -S $sourceurl 2>&1`;
	$ENV{'http_proxy'} = '';

	foreach (@http_header) 
	{
		chomp;
		if (/^\s*Content-Length:\s/) { $remote_size = $_; $remote_size =~ s/[^0-9]//g; }
		if (/^\s*Last-Modified:\s/) { $remote_mtime = $_; $remote_mtime =~ s/^\s*Last-Modified:\s//; $remote_mtime = HTTP::Date::str2time($remote_mtime) }
	}

	if (
		(-e "$updcachedir/$vendorid/$uuid/$updfile") && 
		($remote_size == (-s "$updcachedir/$vendorid/$uuid/$updfile")) &&
		($remote_mtime == &getmtime("$updcachedir/$vendorid/$uuid/$updfile"))
	   )
	{
		&debuglog("File exists in cache and is up to date");
		&debuglog("Retrieving file from cache ($updsource) for $hostaddr");
		&setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time);
		$cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile";
	}
	elsif (
		($cfmirror == $unique) &&
		($vendorid == "microsoft") &&
		($source_url =~ m@.*[0-9a-f]{40}\.[^\.]+@i)
	      )
	{
			# Most Microsoft updates now have an SHA1 hash in the name. These should be treated as unique files.
			# Since it wasn't found in the URL cache, switch to mirror mode and try again using just the filename.
			&debuglog("SHA1: $vendorid $uuid not cached. Reprocessing as mirror $sourceurl");
			$cacheurl = &check_cache($source_url,$hostaddr,$username,$vendorid,$mirror);
	}
		else
	{
		if (-e "$updcachedir/$vendorid/$uuid/$updfile")
		{
			&debuglog("Local filesize: " . (-s "$updcachedir/$vendorid/$uuid/$updfile"));
			&debuglog("Local timestamp: " . &getmtime("$updcachedir/$vendorid/$uuid/$updfile"));
		} else { &debuglog("File not found in cache"); }
		$updsource="DLSOURCE";
		&debuglog("Remote filesize: $remote_size");
		&debuglog("Remote timestamp: $remote_mtime");
		&debuglog("Free disk space: " . &diskfree($updcachedir));
		&debuglog("Disk usage: " . &diskusage($updcachedir) . "% (max. $maxusage%)");
		if (-e "$updcachedir/download/$vendorid/$updfile") { &debuglog("File download/$vendorid/$updfile exists"); }
		&debuglog("Retrieving file from source ($updsource) for $hostaddr");
		if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && ($remote_size <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$vendorid/$updfile"))
		{
			&debuglog("Running command $nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
			system("$nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
		}
		$cacheurl=$sourceurl;
	}

	&writelog($_[1],$_[2],$_[3],$updsource,$sourceurl);

	return $cacheurl;
}

# -------------------------------------------------------------------
Commit	Line	Data
46c01c09 MT	1	#!/usr/bin/perl
	2	#
	3	# This code is distributed under the terms of the GPL
	4	#
4f4092e3	5	# (c) 2006-2009 marco.s - http://update-accelerator.advproxy.net
46c01c09	6	#
4f4092e3 CS	7	# Portions (c) 2008 by dotzball - http://www.blockouttraffic.de
	8	#
	9	# $Id: updxlrator,v 2.1 2009/01/10 00:00:00 marco.s Exp $
46c01c09	10	#
4f6b044f JIW	11	# ChangeLog:
	12	#
	13	# 2012-10-26: nightshift - move curly bracket to capture AVG download source.
	14	# - Adding BIG HINT for new update source#
	15	#
46c01c09 MT	16
46c01c09 MT	17	use strict;
0a638396	18	use HTTP::Date;
46c01c09 MT	19
	20	$\|=1;
	21
	22	my $swroot="/var/ipfire";
381f2e71	23	my $updcachedir="/var/updatecache";
0a638396 CS	24	my $apphome="/var/ipfire/updatexlrator";
0a638396 CS	25	my $logfile="/var/log/updatexlrator/cache.log";
4f4092e3	26	my $wget="/usr/bin/wget";
0a638396 CS	27	my $debug=(-e "$apphome/debug");
0a638396 CS	28	my $http_port='81';
46c01c09	29	my %netsettings=();
0a638396	30	my %proxysettings=();
46c01c09	31	my %xlratorsettings=();
46c01c09 MT	32	my $logging=0;
	33	my $passive_mode=0;
	34	my $maxusage=0;
	35	my $nice='';
	36	my @tmp=();
46c01c09	37	my $request='';
0a638396 CS	38	my $xlrator_url=0;
0a638396 CS	39	my $source_url='';
46c01c09 MT	40	my $hostaddr='';
	41	my $username='';
	42	my $method='';
0a638396 CS	43	my $unique = 0;
0a638396 CS	44	my $mirror = 1;
46c01c09 MT	45
	46	readhash("${swroot}/ethernet/settings", \%netsettings);
	47
	48	if (-e "$swroot/updatexlrator/settings")
	49	{
	50	&readhash("$swroot/updatexlrator/settings", \%xlratorsettings);
	51	if ($xlratorsettings{'ENABLE_LOG'} eq 'on') { $logging=1; };
	52	if ($xlratorsettings{'PASSIVE_MODE'} eq 'on') { $passive_mode=1; };
	53	$maxusage=$xlratorsettings{'MAX_DISK_USAGE'};
646750c5	54	if ($xlratorsettings{'LOW_DOWNLOAD_PRIORITY'} eq 'on') { $nice='/bin/nice --adjustment=15 '; };
46c01c09 MT	55	}
	56	if (!$maxusage) { $maxusage=75; };
	57
4f4092e3 CS	58	# dotzball: check for dead downloads
4f4092e3 CS	59	system("$apphome/bin/checkdeaddl &");
46c01c09 MT	60
	61	while (<>) {
	62
	63	$request=$_;
46c01c09 MT	64
	65	@tmp=split(/ /,$request);
	66	chomp(@tmp);
	67
0a638396 CS	68	$source_url = $tmp[0];
	69	$hostaddr = $tmp[1]; while ($hostaddr =~ /.*\/$/) { chop $hostaddr; }
	70	$username = $tmp[2]; if ($username eq '') { $username='-'; };
	71	$method = $tmp[3];
	72
9663c20e	73	$xlrator_url = "";
46c01c09 MT	74
	75	if (($method eq 'GET') \|\| ($method eq 'HEAD'))
	76	{
	77
	78	# -----------------------------------------------------------
	79	# Section: Windows Update / Windows Downloads
	80	# -----------------------------------------------------------
	81
	82	if (
bc689853 JIW	83	(($source_url =~ m@^http://[^/]\.microsoft\.com/.\.(exe\|psf\|msi\|msp\|msu\|cab)$@i) \|\|
bc689853 JIW	84	($source_url =~ m@^http://[^/]\.windowsupdate\.com/.\.(exe\|psf\|msi\|msp\|msu\|cab)$@i))
0a638396 CS	85	&& ($source_url !~ m@^http://[^/]\.microsoft\.com/.(/autoupd\|selfupdate/).*\.cab@i)
0a638396 CS	86	&& ($source_url !~ m@\&@)
46c01c09 MT	87	)
46c01c09 MT	88	{
329631de JL	89	# NOTE: check_cache will change to $mirror instead of $unique if the filename contains an SHA1 hash
329631de JL	90	# and the URL is not found in cache!
0a638396	91	$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Microsoft",$unique);
46c01c09 MT	92	}
	93
	94	# -----------------------------------------------------------
	95	# Section: Adobe Downloads
	96	# -----------------------------------------------------------
	97
4f6b044f	98
0a638396 CS	99	if (
	100	($source_url =~ m@^http://(ar)?download\.adobe\.com/.*\.(exe\|msi\|bin\|dmg\|idx\|gz)$@i) \|\|
	101	($source_url =~ m@^http://swupdl\.adobe\.com/updates/.*\.(exe\|msi\|bin\|dmg\|idx\|gz\|[a-z][a-z]_[A-Z][A-Z])$@i) \|\|
	102	($source_url =~ m@^http://swupmf\.adobe\.com/manifest/.*\.upd$@i)
	103	)
	104	{
	105	$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Adobe",$unique);
	106	}
	107
	108	# -----------------------------------------------------------
	109	# Section: Linux Downloads
	110	# -----------------------------------------------------------
	111
8dbba0e4	112	if (
77fa792f	113	($source_url =~ m@^[h\|f]t?tp://[^?]+\.(pkg\.tar\.xz\|deb\|rpm)$@i) \|\|
1235b1f1	114	($source_url =~ m@^[h\|f]t?tp://[^?]+/distfiles/[^?]+\.(tar\.gz\|tar\.bz2\|tar\.xz\|tgz\|zip\|patch\.bz2\|gz\|docx\|patch\|pdf\|exe)$@i)
8dbba0e4	115	)
46c01c09	116	{
0a638396 CS	117	$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Linux",$mirror);
	118	}
	119
	120	# -----------------------------------------------------------
	121	# Section: Trend Micro Downloads
	122	# -----------------------------------------------------------
	123
	124	if (
	125	($source_url =~ m@^http://[^/]\.trendmicro\.com/activeupdate/.@i) &&
	126	($source_url !~ m@.*/tmnotify\.dat$@i) &&
	127	($source_url !~ m@.*/ini_xml\.zip$@i) &&
	128	($source_url !~ m@.*/server\.ini$@i)
	129	)
	130	{
	131	$xlrator_url = &check_cache($source_url,$hostaddr,$username,"TrendMicro",$mirror);
46c01c09 MT	132	}
	133
	134	# -----------------------------------------------------------
	135	# Section: Symantec Downloads
	136	# -----------------------------------------------------------
	137
0a638396	138	if ($source_url =~ m@^[h\|f]tt?p://[^/]\.symantec(liveupdate)?\.com/.\.(exe\|zip\|vdb\|xdb)$@i)
46c01c09	139	{
0a638396	140	$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Symantec",$unique);
46c01c09	141	}
0a638396	142
5b2a12ff	143	# -----------------------------------------------------------
0a638396	144	# Section: Apple Downloads
46c01c09	145	# -----------------------------------------------------------
0a638396 CS	146
0a638396 CS	147	if (
4f4092e3	148	(($source_url =~ m@^http://swcdn\.apple\.com/content/downloads/.*\.(tar\|pkg)$@i) \|\|
0a638396 CS	149	($source_url =~ m@^http://appldnld\.apple\.com\.edgesuite\.net/.*\.(exe\|dmg)$@i) \|\|
	150	($source_url =~ m@^http://.\.g.akamai.net/./3093/1/.*\.(tar\|pkg\|dmg\|exe)$@i))
	151	)
5b2a12ff	152	{
0a638396	153	$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Apple",$unique);
5b2a12ff	154	}
0a638396 CS	155
	156	# -----------------------------------------------------------
	157	# Section: Avast Downloads
	158	# -----------------------------------------------------------
	159
5150d373	160	if ($source_url =~ m@^http://(ion\|download)[\d]+\.avast\.com/.*\.(exe\|vpu\|vpx)$@i)
0a638396 CS	161	{
0a638396 CS	162	$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avast",$mirror);
186e3d2c	163	}
0a638396	164
219c963e JIW	165	# -----------------------------------------------------------
	166	# Section: Mozilla Downloads
	167	# -----------------------------------------------------------
	168
	169	if ($source_url=~ m@^http://.\.mozilla\.net/.\.((complete\|partial)\.mar\|exe)$@i)
	170	{
	171	$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Mozilla",$unique);
	172	}
	173
0a638396	174	# -----------------------------------------------------------
7b59d838	175	# Section: Mcafee
0a638396	176	# -----------------------------------------------------------
186e3d2c	177
7b59d838 JIW	178	if ($source_url =~ m@^http://update\.nai\.com/.*\.(mcs\|z\|gem\|dat\|zip)$@i)
	179	{
	180	$xlrator_url = &check_cache($source_url,$hostaddr,$username,"mcafee",$mirror);
	181	}
186e3d2c	182
0a638396 CS	183	# -----------------------------------------------------------
	184	# Section: Avira Downloads
	185	# -----------------------------------------------------------
	186
8dbba0e4 CS	187	if (
	188	($source_url =~ m@^http://dl[0-9]\.avgate\.net/.*\.(htm\|html\|gz)$@i) \|\|
	189	($source_url =~ m@^http://80.190.130.19[4-5]/update/.*\.(htm\|html\|gz)$@i) \|\|
	190	($source_url =~ m@^http://62.146.64.14[6-7]/update/.*\.(htm\|html\|gz)$@i)
	191	)
0a638396 CS	192	{
0a638396 CS	193	$xlrator_url = &check_cache($source_url,$hostaddr,$username,"Avira",$mirror);
2dfb38e6	194	}
4ae86474 AF	195
4ae86474 AF	196	# -----------------------------------------------------------
0a638396	197	# Section: IPFire Downloads
4ae86474 AF	198	# -----------------------------------------------------------
4ae86474 AF	199
0a638396	200	if ($source_url =~ m@^[f\|h]t?tp://.*\.(ipfire)$@i)
4ae86474	201	{
0a638396	202	$xlrator_url = &check_cache($source_url,$hostaddr,$username,"IPFire",$mirror);
4ae86474 AF	203	}
4ae86474 AF	204
0a638396 CS	205	# -----------------------------------------------------------
	206	# Section: AVG Downloads
	207	# -----------------------------------------------------------
46c01c09	208
4f4092e3 CS	209	# if ($source_url =~ m@^http://[^/]\.(grisoft\|avg)\.com/.\.(bin)$@i)
	210	# {
	211	# $xlrator_url = &check_cache($source_url,$hostaddr,$username,"AVG",$mirror);
	212	# }
46c01c09	213
4f6b044f JIW	214	# ----------- ADD NEW SOURCES BEFORE THIS LINE !!! ------------------
	215	}
	216
9663c20e MT	217	if ($xlrator_url) {
	218	print "OK rewrite-url=\"$xlrator_url\"\n";
	219	} else {
	220	print "ERR\n";
	221	}
46c01c09 MT	222	}
	223
	224	# -------------------------------------------------------------------
	225
	226	sub readhash
	227	{
	228	my $filename = $_[0];
	229	my $hash = $_[1];
	230	my ($var, $val);
	231
	232	if (-e $filename)
	233	{
	234	open(FILE, $filename) or die "Unable to read file $filename";
	235	while (<FILE>)
	236	{
	237	chop;
	238	($var, $val) = split /=/, $_, 2;
	239	if ($var)
	240	{
	241	$val =~ s/^\'//g;
	242	$val =~ s/\'$//g;
	243
	244	# Untaint variables read from hash
	245	$var =~ /([A-Za-z0-9_-]*)/; $var = $1;
	246	$val =~ /([\w\W]*)/; $val = $1;
	247	$hash->{$var} = $val;
	248	}
	249	}
	250	close FILE;
	251	}
	252	}
	253
	254	# -------------------------------------------------------------------
	255
	256	sub writelog
	257	{
0a638396 CS	258	if ($logging)
	259	{
	260	open(LOGFILE,">>$logfile");
	261	print LOGFILE time." $_[0] $_[1] $_[2] $_[3] $_[4]\n";
	262	close(LOGFILE);
	263	}
46c01c09 MT	264	}
	265
	266	# -------------------------------------------------------------------
	267
0a638396	268	sub debuglog
46c01c09	269	{
0a638396	270	if ($debug)
46c01c09	271	{
0a638396 CS	272	open(LOGFILE,">>/var/log/updatexlrator/debug.log");
	273	my @now = localtime(time);
	274	printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] [%s] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,"updxlrator",$_[0];
	275	close(LOGFILE);
46c01c09	276	}
0a638396 CS	277	}
	278
	279	# -------------------------------------------------------------------
	280
	281	sub setcachestatus
	282	{
	283	open (FILE,">>$_[0]");
	284	print FILE "$_[1]\n";
	285	close FILE;
	286	}
	287
	288	# -------------------------------------------------------------------
	289
	290	sub diskfree
	291	{
	292	open(DF,"/bin/df --block-size=1 $_[0]\|");
	293	my @dfdata = <DF>;
46c01c09	294	close DF;
0a638396 CS	295	shift(@dfdata);
	296	chomp(@dfdata);
	297	my $dfstr = join(' ',@dfdata);
	298	my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
	299	if ($free =~ m/^(\d+)$/)
	300	{
	301	return $free;
	302	}
46c01c09 MT	303	}
	304
	305	# -------------------------------------------------------------------
	306
	307	sub diskusage
	308	{
	309	open(DF,"/bin/df $_[0]\|");
0a638396 CS	310	my @dfdata = <DF>;
	311	close DF;
	312	shift(@dfdata);
	313	chomp(@dfdata);
	314	my $dfstr = join(' ',@dfdata);
	315	my ($device,$size,$used,$free,$percent,$mount) = split(' ',$dfstr);
	316	if ($percent =~ m/^(\d+)%$/)
46c01c09	317	{
0a638396 CS	318	$percent =~ s/%$//;
0a638396 CS	319	return $percent;
46c01c09	320	}
46c01c09 MT	321	}
	322
	323	# -------------------------------------------------------------------
	324
0a638396	325	sub getmtime
46c01c09	326	{
0a638396	327	my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,$ctime,$blksize,$blocks) = stat($_[0]);
46c01c09	328
0a638396 CS	329	return $mtime;
0a638396 CS	330	}
46c01c09	331
0a638396	332	# -------------------------------------------------------------------
46c01c09	333
0a638396 CS	334	sub check_cache
	335	{
	336	my $updsource="UPDCACHE";
	337	my $updfile='';
	338	my $cacheurl='';
	339	my $vendorid='';
	340	my $uuid='';
	341	my @http_header=();
	342	my $remote_size=0;
	343	my $remote_mtime=0;
	344	my $login='';
	345	my $useragent="Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)";
	346
	347	my $sourceurl=$_[0];
	348	my $cfmirror=$_[4];
	349
39c0b5b3	350	$sourceurl =~ s@\%2b@+@ig;
0a638396	351	$sourceurl =~ s@\%2f@/@ig;
39c0b5b3	352	$sourceurl =~ s@\%7e@~@ig;
0a638396 CS	353	$updfile = substr($sourceurl,rindex($sourceurl,"/")+1);
	354	$updfile =~ s@\%20@ @ig;
	355
	356	if ($cfmirror)
46c01c09	357	{
0a638396 CS	358	$uuid = `echo $updfile \| md5sum`;
	359	} else {
	360	$uuid = `echo $sourceurl \| md5sum`;
46c01c09 MT	361	}
46c01c09 MT	362
0a638396 CS	363	$uuid =~ s/[^0-9a-f]//g;
	364	$uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/;
	365
	366	$vendorid = $_[3];
	367	$vendorid =~ tr/A-Z/a-z/;
	368
	369	&debuglog("Processing URL $sourceurl");
	370	&debuglog("Vendor ID is $vendorid");
	371	&debuglog("UUID is $uuid");
	372
	373	if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'}))
46c01c09	374	{
0a638396 CS	375	$login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\"";
0a638396 CS	376	if ($proxysettings{'UPSTREAM_PASSWORD'})
46c01c09	377	{
0a638396	378	$login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\"";
46c01c09 MT	379	}
46c01c09 MT	380	}
46c01c09	381
0a638396	382	if ($proxysettings{'UPSTREAM_PROXY'}) { &debuglog("Using upstream proxy $proxysettings{'UPSTREAM_PROXY'}"); }
46c01c09	383
0a638396	384	$ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
4f4092e3	385	@http_header = `$wget $login --user-agent="$useragent" --spider -S $sourceurl 2>&1`;
0a638396	386	$ENV{'http_proxy'} = '';
46c01c09	387
0a638396	388	foreach (@http_header)
46c01c09	389	{
0a638396 CS	390	chomp;
	391	if (/^\s*Content-Length:\s/) { $remote_size = $_; $remote_size =~ s/[^0-9]//g; }
	392	if (/^\sLast-Modified:\s/) { $remote_mtime = $_; $remote_mtime =~ s/^\sLast-Modified:\s//; $remote_mtime = HTTP::Date::str2time($remote_mtime) }
46c01c09 MT	393	}
46c01c09 MT	394
0a638396 CS	395	if (
	396	(-e "$updcachedir/$vendorid/$uuid/$updfile") &&
	397	($remote_size == (-s "$updcachedir/$vendorid/$uuid/$updfile")) &&
	398	($remote_mtime == &getmtime("$updcachedir/$vendorid/$uuid/$updfile"))
	399	)
46c01c09	400	{
0a638396	401	&debuglog("File exists in cache and is up to date");
e331e22b	402	&debuglog("Retrieving file from cache ($updsource) for $hostaddr");
0a638396	403	&setcachestatus("$updcachedir/$vendorid/$uuid/access.log",time);
0a638396	404	$cacheurl="http://$netsettings{'GREEN_ADDRESS'}:$http_port/updatecache/$vendorid/$uuid/$updfile";
329631de JL	405	}
	406	elsif (
	407	($cfmirror == $unique) &&
	408	($vendorid == "microsoft") &&
	409	($source_url =~ m@.*[0-9a-f]{40}\.[^\.]+@i)
	410	)
	411	{
	412	# Most Microsoft updates now have an SHA1 hash in the name. These should be treated as unique files.
	413	# Since it wasn't found in the URL cache, switch to mirror mode and try again using just the filename.
	414	&debuglog("SHA1: $vendorid $uuid not cached. Reprocessing as mirror $sourceurl");
	415	$cacheurl = &check_cache($source_url,$hostaddr,$username,$vendorid,$mirror);
46c01c09 MT	416	}
	417	else
	418	{
0a638396 CS	419	if (-e "$updcachedir/$vendorid/$uuid/$updfile")
	420	{
	421	&debuglog("Local filesize: " . (-s "$updcachedir/$vendorid/$uuid/$updfile"));
	422	&debuglog("Local timestamp: " . &getmtime("$updcachedir/$vendorid/$uuid/$updfile"));
	423	} else { &debuglog("File not found in cache"); }
46c01c09	424	$updsource="DLSOURCE";
0a638396 CS	425	&debuglog("Remote filesize: $remote_size");
	426	&debuglog("Remote timestamp: $remote_mtime");
	427	&debuglog("Free disk space: " . &diskfree($updcachedir));
	428	&debuglog("Disk usage: " . &diskusage($updcachedir) . "% (max. $maxusage%)");
	429	if (-e "$updcachedir/download/$vendorid/$updfile") { &debuglog("File download/$vendorid/$updfile exists"); }
e331e22b	430	&debuglog("Retrieving file from source ($updsource) for $hostaddr");
0a638396	431	if ((!$passive_mode) && (&diskusage($updcachedir) <= $maxusage) && ($remote_size <= &diskfree($updcachedir)) && (!-e "$updcachedir/download/$vendorid/$updfile"))
46c01c09	432	{
0a638396 CS	433	&debuglog("Running command $nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
0a638396 CS	434	system("$nice$apphome/bin/download $vendorid $sourceurl $cfmirror &");
46c01c09	435	}
0a638396	436	$cacheurl=$sourceurl;
46c01c09 MT	437	}
46c01c09 MT	438
0a638396	439	&writelog($_[1],$_[2],$_[3],$updsource,$sourceurl);
46c01c09	440
0a638396	441	return $cacheurl;
46c01c09 MT	442	}
	443
	444	# -------------------------------------------------------------------