]>
Commit | Line | Data |
---|---|---|
1 | #!/usr/bin/perl | |
2 | # | |
3 | # This code is distributed under the terms of the GPL | |
4 | # | |
5 | # (c) 2006-2008 marco.s - http://update-accelerator.advproxy.net | |
6 | # | |
7 | # Portions (c) 2008 by dotzball - http://www.blockouttraffic.de | |
8 | # | |
9 | # $Id: download,v 2.1 2008/07/23 00:00:00 marco.s Exp $ | |
10 | # | |
11 | ||
12 | use strict; | |
13 | use HTTP::Date; | |
14 | ||
15 | require '/var/ipfire/updatexlrator/updxlrator-lib.pl'; | |
16 | ||
17 | my $logfile="/var/log/updatexlrator/download.log"; | |
18 | my $logging=0; | |
19 | my $repository='/var/updatecache'; | |
20 | my $login=''; | |
21 | my $dlrate=''; | |
22 | my $uuid=''; | |
23 | my %xlratorsettings=(); | |
24 | my %proxysettings=(); | |
25 | my @http_header=(); | |
26 | my $remote_size=0; | |
27 | my $remote_mtime=0; | |
28 | my $updatefile=''; | |
29 | my $unique=0; | |
30 | my $mirror=1; | |
31 | ||
32 | my %dlinfo=(); | |
33 | my $wgetContinueFlag=""; | |
34 | ||
35 | my $vendorid = $ARGV[0]; if (!defined($vendorid) || $vendorid eq '') { exit; } | |
36 | my $sourceurl = $ARGV[1]; if (!defined($sourceurl) || $sourceurl eq '') { exit; } | |
37 | my $cfmirror = $ARGV[2]; if (!defined($cfmirror) || $cfmirror eq '') { exit; } | |
38 | my $restartdl = defined($ARGV[3]) ? $ARGV[3] : 0; | |
39 | ||
40 | umask(0002); | |
41 | ||
42 | $sourceurl =~ s@\%2b@+@ig; | |
43 | $sourceurl =~ s@\%2f@/@ig; | |
44 | $sourceurl =~ s@\%7e@~@ig; | |
45 | $updatefile = substr($sourceurl,rindex($sourceurl,"/")+1); | |
46 | $updatefile =~ s@\%20@ @ig; | |
47 | $vendorid =~ tr/A-Z/a-z/; | |
48 | ||
49 | unless (-d "$repository/download/$vendorid") | |
50 | { | |
51 | system("mkdir -p $repository/download/$vendorid"); | |
52 | system("chown -R nobody.squid $repository/download/$vendorid"); | |
53 | system("chmod 775 $repository/download/$vendorid"); | |
54 | } | |
55 | ||
56 | if($restartdl == 0) | |
57 | { | |
58 | # this is a new download | |
59 | exit if (-e "$repository/download/$vendorid/$updatefile"); | |
60 | ||
61 | # dotzball: Why is this necessary? | |
62 | system("touch $repository/download/$vendorid/$updatefile"); | |
63 | $wgetContinueFlag = "-nc"; | |
64 | ||
65 | } | |
66 | else | |
67 | { | |
68 | # this is a restart of a previous (unfinished) download | |
69 | # -> continue download | |
70 | $wgetContinueFlag = "-c"; | |
71 | &writelog("Continue download: $updatefile"); | |
72 | } | |
73 | ||
74 | ||
75 | if ($cfmirror) | |
76 | { | |
77 | $uuid = `echo $updatefile | md5sum`; | |
78 | } else { | |
79 | $uuid = `echo $sourceurl | md5sum`; | |
80 | } | |
81 | ||
82 | $uuid =~ s/[^0-9a-f]//g; | |
83 | $uuid =~ s/([a-f\d]{8})([a-f\d]{4})([a-f\d]{4})([a-f\d]{4})([a-f\d]{12})/$1-$2-$3-$4-$5/; | |
84 | ||
85 | if (-e "$UPDXLT::swroot/updatexlrator/settings") | |
86 | { | |
87 | &UPDXLT::readhash("$UPDXLT::swroot/updatexlrator/settings", \%xlratorsettings); | |
88 | if ($xlratorsettings{'MAX_DOWNLOAD_RATE'} ne '') { $dlrate = "--limit-rate=" . int($xlratorsettings{'MAX_DOWNLOAD_RATE'} / 8) . "k" }; | |
89 | } | |
90 | ||
91 | if (-e "$UPDXLT::swroot/proxy/settings") { &UPDXLT::readhash("$UPDXLT::swroot/proxy/settings", \%proxysettings); } | |
92 | ||
93 | if (-e "$UPDXLT::swroot/proxy/advanced/settings") | |
94 | { | |
95 | %proxysettings=(); | |
96 | &UPDXLT::readhash("$UPDXLT::swroot/proxy/advanced/settings", \%proxysettings); | |
97 | } | |
98 | ||
99 | if (($proxysettings{'UPSTREAM_PROXY'}) && ($proxysettings{'UPSTREAM_USER'})) | |
100 | { | |
101 | $login = "--proxy-user=\"$proxysettings{'UPSTREAM_USER'}\""; | |
102 | if ($proxysettings{'UPSTREAM_PASSWORD'}) | |
103 | { | |
104 | $login .= " --proxy-password=\"$proxysettings{'UPSTREAM_PASSWORD'}\""; | |
105 | } | |
106 | } | |
107 | ||
108 | if ($xlratorsettings{'MAX_DOWNLOAD_RATE'} eq '') | |
109 | { | |
110 | &writelog("Retrieving file for local cache: $updatefile"); | |
111 | } else { | |
112 | &writelog("Retrieving file for local cache at max. " . $xlratorsettings{'MAX_DOWNLOAD_RATE'} . " kBit/s: $updatefile"); | |
113 | } | |
114 | ||
115 | $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'}; | |
116 | @http_header = `$UPDXLT::wget $login --user-agent="$UPDXLT::useragent" --spider -S $sourceurl 2>&1`; | |
117 | $ENV{'http_proxy'} = ''; | |
118 | ||
119 | foreach (@http_header) | |
120 | { | |
121 | chomp; | |
122 | if (/^\s*Content-Length:\s/) { s/[^0-9]//g; $remote_size=$_; &writelog("Remote file size: $_ bytes"); } | |
123 | if (/^\s*Last-Modified:\s/) | |
124 | { | |
125 | s/^\s*Last-Modified:\s//; | |
126 | $remote_mtime = HTTP::Date::str2time($_); | |
127 | &writelog("Remote file date: $_"); | |
128 | } | |
129 | } | |
130 | ||
131 | $ENV{'http_proxy'} = $proxysettings{'UPSTREAM_PROXY'}; | |
132 | ||
133 | unless($restartdl) | |
134 | { | |
135 | # this is a new download | |
136 | # -> download from scratch | |
137 | unlink "$repository/download/$vendorid/$updatefile"; | |
138 | unlink "$repository/download/$vendorid/$updatefile.info"; | |
139 | } | |
140 | ||
141 | # save file informations while downloading | |
142 | $dlinfo{'VENDORID'} = $vendorid; | |
143 | $dlinfo{'SRCURL'} = $sourceurl; | |
144 | $dlinfo{'FILENAME'} = $updatefile; | |
145 | $dlinfo{'CFMIRROR'} = $cfmirror; | |
146 | $dlinfo{'REMOTETIME'} = $remote_mtime; | |
147 | $dlinfo{'REMOTESIZE'} = $remote_size; | |
148 | $dlinfo{'STATUS'} = "1"; | |
149 | &UPDXLT::writehash("$repository/download/$vendorid/$updatefile.info", \%dlinfo); | |
150 | ||
151 | my $cmd = "$UPDXLT::wget $login $dlrate --user-agent=\"$UPDXLT::useragent\" -q -P $repository/download/$vendorid $wgetContinueFlag $sourceurl"; | |
152 | ||
153 | $_ = system("$cmd"); | |
154 | $ENV{'http_proxy'} = ''; | |
155 | ||
156 | if ($_ == 0) | |
157 | { | |
158 | &writelog("Download finished with result code: OK"); | |
159 | ||
160 | unless (-d "$repository/$vendorid") | |
161 | { | |
162 | system("mkdir -p $repository/$vendorid"); | |
163 | system("chown -R nobody.squid $repository/$vendorid"); | |
164 | system("chmod 775 $repository/$vendorid"); | |
165 | } | |
166 | ||
167 | unless (-d "$repository/$vendorid/$uuid") | |
168 | { | |
169 | system("mkdir -p $repository/$vendorid/$uuid"); | |
170 | system("chown -R nobody.squid $repository/$vendorid/$uuid"); | |
171 | system("chmod 775 $repository/$vendorid/$uuid"); | |
172 | } | |
173 | ||
174 | &writelog("Moving file to the cache directory: $vendorid/$uuid"); | |
175 | $updatefile =~ s@ @\\ @ig; | |
176 | system("mv $repository/download/$vendorid/$updatefile $repository/$vendorid/$uuid"); | |
177 | # Workaround for IPCop's mv bug: | |
178 | utime time,$remote_mtime,"$repository/$vendorid/$uuid/$updatefile"; | |
179 | $updatefile =~ s@\\ @ @ig; | |
180 | ||
181 | &UPDXLT::setcachestatus("$repository/$vendorid/$uuid/source.url",$sourceurl); | |
182 | &UPDXLT::setcachestatus("$repository/$vendorid/$uuid/status",$UPDXLT::sfOk); | |
183 | &UPDXLT::setcachestatus("$repository/$vendorid/$uuid/checkup.log",time); | |
184 | &UPDXLT::setcachestatus("$repository/$vendorid/$uuid/access.log",time); | |
185 | ||
186 | system("chown -R nobody.squid $repository/$vendorid/$uuid/*"); | |
187 | system("chmod 775 $repository/$vendorid/$uuid/*"); | |
188 | ||
189 | unlink ("$repository/download/$vendorid/$updatefile.info"); | |
190 | ||
191 | } else { | |
192 | &writelog("Download finished with result code: ERROR"); | |
193 | if (-e "$repository/download/$vendorid/$updatefile") { unlink ("$repository/download/$vendorid/$updatefile"); } | |
194 | } | |
195 | ||
196 | ||
197 | # ------------------------------------------------------------------- | |
198 | ||
199 | sub writelog | |
200 | { | |
201 | if ($logging) | |
202 | { | |
203 | open (LOGFILE,">>$logfile"); | |
204 | my @now = localtime(time); | |
205 | printf LOGFILE "%04d-%02d-%02d %02d:%02d:%02d [%d] %s\n",$now[5]+1900,$now[4]+1,$now[3],$now[2],$now[1],$now[0],$$,$_[0]; | |
206 | close LOGFILE; | |
207 | } | |
208 | } | |
209 | ||
210 | # ------------------------------------------------------------------- |