config/updxlrator/download

   1 #!/usr/bin/perl
   2 #
   3 # This code is distributed under the terms of the GPL
   4 #
   5 # (c) 2006-2008 marco.s - http://update-accelerator.advproxy.net
   6 #
   7 # Portions (c) 2008 by dotzball - http://www.blockouttraffic.de
   8 #
   9 # $Id: download,v 2.1 2008/07/23 00:00:00 marco.s Exp $
  10 #
  11
  12 use strict;
  13 use HTTP::Date;
  14
  15 require '/var/ipfire/updatexlrator/updxlrator-lib.pl';
  16
  17 my $logfile="/var/log/updatexlrator/download.log";
  18 my $logging=0;
  19 my $repository='/var/updatecache';
  20 my $login='';
  21 my $dlrate='';
  22 my $uuid='';
  23 my %xlratorsettings=();
  24 my %proxysettings=();
  25 my @http_header=();
  26 my $remote_size=0;
  27 my $remote_mtime=0;
  28 my $updatefile='';
  29 my $unique=0;
  30 my $mirror=1;
  31
  32 my %dlinfo=();
  33
  34 my $vendorid  = $ARGV[0]; if (!defined($vendorid)  || $vendorid  eq '') { exit; }
  35 my $sourceurl = $ARGV[1]; if (!defined($sourceurl) || $sourceurl eq '') { exit; }
  36 my $cfmirror  = $ARGV[2]; if (!defined($cfmirror)  || $cfmirror eq '') { exit; }
  37 my $restartdl = defined($ARGV[3]) ? $ARGV[3] : 0;
  38
  39 umask(0002);
  40
  41 $sourceurl =~ s@\%2b@+@ig;
  42 $sourceurl =~ s@\%2f@/@ig;
  43 $sourceurl =~ s@\%7e@~@ig;
  44 $updatefile = substr($sourceurl,rindex($sourceurl,"/")+1);
  45 $updatefile =~ s@\%20@ @ig;
  46 $vendorid =~ tr/A-Z/a-z/;
  47
  48 unless (-d "$repository/download/$vendorid")
  49 {
  50         system("mkdir -p $repository/download/$vendorid");
  51         chmod 0775, "$repository/download/$vendorid";
  52 }
  53
  54 if($restartdl == 0)
  55 {
  56         # this is a new download
  57         exit if (-e "$repository/download/$vendorid/$updatefile");
  58
  59         # hinder multiple downloads from starting simultaneously. Create empty "lock" file.
  60         # TODO: Another thread may sneak in between these two commands - so not fool-proof, but good enough?
  61         system("touch $repository/download/$vendorid/$updatefile");
  62
  63 }
  64 else
  65 {
  66         # this is a restart of a previous (unfinished) download
  67         # -> continue download
  68         &writelog("Continue download: $updatefile");
  69 }
  70
  71
  72 if ($cfmirror)
  73 {
  74         $uuid = `echo $updatefile | md5sum`;
  75 } else {
  76         $uuid = `echo $sourceurl  | md5sum`;
  77 }
  78
  79 $uuid =~ s/[^0-9a-f]//g;
  80 $uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/;
  81
  82 if (-e "$UPDXLT::swroot/updatexlrator/settings")
  83 {
  84         &UPDXLT::readhash("$UPDXLT::swroot/updatexlrator/settings", \%xlratorsettings);
  85         if ($xlratorsettings{'MAX_DOWNLOAD_RATE'} ne '') { $dlrate = "--limit-rate=" . int($xlratorsettings{'MAX_DOWNLOAD_RATE'} / 8) . "k" };
  86 }
  87
  88 if (-e "$UPDXLT::swroot/proxy/settings") { &UPDXLT::readhash("$UPDXLT::swroot/proxy/settings", \%proxysettings); }
  89
  90 if (-e "$UPDXLT::swroot/proxy/advanced/settings")
  91 {
  92         %proxysettings=();
  93         &UPDXLT::readhash("$UPDXLT::swroot/proxy/advanced/settings", \%proxysettings);
  94 }
  95
  96 if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'}))
  97 {
  98         $login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\"";
  99         if ($proxysettings{'UPSTREAM_PASSWORD'})
 100         {
 101                 $login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\"";
 102         }
 103 }
 104
 105 if ($xlratorsettings{'MAX_DOWNLOAD_RATE'} eq '')
 106 {
 107         &writelog("Retrieving file for local cache: $updatefile");
 108 } else {
 109         &writelog("Retrieving file for local cache at max. " . $xlratorsettings{'MAX_DOWNLOAD_RATE'} . " kbit/s: $updatefile");
 110 }
 111
 112 $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
 113 @http_header = `$UPDXLT::wget $login --user-agent="$UPDXLT::useragent" --spider -S $sourceurl 2>&1`;
 114 $ENV{'http_proxy'} = '';
 115
 116 foreach (@http_header)
 117 {
 118         chomp;
 119         if (/^\s*Content-Length:\s/) { s/[^0-9]//g; $remote_size=$_; &writelog("Remote file size: $_ bytes"); }
 120         if (/^\s*Last-Modified:\s/)
 121         {
 122                 s/^\s*Last-Modified:\s//;
 123                 $remote_mtime = HTTP::Date::str2time($_);
 124                 &writelog("Remote file date: $_");
 125         }
 126 }
 127
 128 $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'};
 129
 130 unless($restartdl)
 131 {
 132         # this is a new download
 133         # -> download from scratch
 134
 135         #already exited earlier if the file existed, and afterwards created this empty "lock", so if not empty now, another thread is already downloading it.
 136         exit if ( -s "$repository/download/$vendorid/$updatefile" );
 137         unlink "$repository/download/$vendorid/$updatefile.info";
 138 }
 139
 140 # save file informations while downloading
 141 $dlinfo{'VENDORID'} = $vendorid;
 142 $dlinfo{'SRCURL'} = $sourceurl;
 143 $dlinfo{'FILENAME'} = $updatefile;
 144 $dlinfo{'CFMIRROR'} = $cfmirror;
 145 $dlinfo{'REMOTETIME'} = $remote_mtime;
 146 $dlinfo{'REMOTESIZE'} = $remote_size;
 147 $dlinfo{'STATUS'} = "1";
 148 &UPDXLT::writehash("$repository/download/$vendorid/$updatefile.info", \%dlinfo);
 149
 150 my $cmd = "$UPDXLT::wget $login $dlrate --user-agent=\"$UPDXLT::useragent\" -q -P $repository/download/$vendorid --continue $sourceurl";
 151
 152 $_ = system("$cmd");
 153 $ENV{'http_proxy'} = '';
 154
 155 if ($_ == 0)
 156 {
 157         &writelog("Download finished with result code: OK");
 158
 159         unless (-d "$repository/$vendorid")
 160         {
 161                 system("mkdir -p $repository/$vendorid");
 162                 chmod 0775, "$repository/$vendorid";
 163         }
 164
 165         unless (-d "$repository/$vendorid/$uuid")
 166         {
 167                 system("mkdir -p $repository/$vendorid/$uuid");
 168                 chmod 0775, "$repository/$vendorid/$uuid";
 169         }
 170
 171         &writelog("Moving file to the cache directory: $vendorid/$uuid");
 172         $updatefile =~ s@ @\\ @ig;
 173         system("mv $repository/download/$vendorid/$updatefile $repository/$vendorid/$uuid");
 174         # Workaround for IPCop's mv bug:
 175         utime time,$remote_mtime,"$repository/$vendorid/$uuid/$updatefile";
 176         $updatefile =~ s@\\ @ @ig;
 177
 178         &UPDXLT::setcachestatus("$repository/$vendorid/$uuid/source.url",$sourceurl);
 179         &UPDXLT::setcachestatus("$repository/$vendorid/$uuid/status",$UPDXLT::sfOk);
 180         &UPDXLT::setcachestatus("$repository/$vendorid/$uuid/checkup.log",time);
 181         &UPDXLT::setcachestatus("$repository/$vendorid/$uuid/access.log",time);
 182
 183         # Update permissions of all files in the download directory.
 184         my @files = (
 185                 "$repository/$vendorid/$uuid/source.url",
 186                 "$repository/$vendorid/$uuid/status",
 187                 "$repository/$vendorid/$uuid/checkup.log",
 188                 "$repository/$vendorid/$uuid/access.log",
 189                 "$repository/$vendorid/$uuid/$updatefile"
 190         );
 191         chmod 0664, @files;
 192
 193         unlink ("$repository/download/$vendorid/$updatefile.info");
 194
 195 } else {
 196         &writelog("Download finished with result code: ERROR");
 197         if (-e "$repository/download/$vendorid/$updatefile") { unlink ("$repository/download/$vendorid/$updatefile"); }
 198 }
 199
 200
 201 # -------------------------------------------------------------------
 202
 203 sub writelog
 204 {
 205         if ($logging)
 206         {
 207                 open (LOGFILE,">>$logfile");
 208                 my @now = localtime(time);
 209                 printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,$_[0];
 210                 close LOGFILE;
 211         }
 212 }
 213
 214 # -------------------------------------------------------------------