]>
Commit | Line | Data |
---|---|---|
1de8761d | 1 | #!/usr/bin/python |
b792d887 MT |
2 | ############################################################################### |
3 | # # | |
4 | # Pakfire - The IPFire package management system # | |
5 | # Copyright (C) 2011 Pakfire development team # | |
6 | # # | |
7 | # This program is free software: you can redistribute it and/or modify # | |
8 | # it under the terms of the GNU General Public License as published by # | |
9 | # the Free Software Foundation, either version 3 of the License, or # | |
10 | # (at your option) any later version. # | |
11 | # # | |
12 | # This program is distributed in the hope that it will be useful, # | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of # | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # | |
15 | # GNU General Public License for more details. # | |
16 | # # | |
17 | # You should have received a copy of the GNU General Public License # | |
18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. # | |
19 | # # | |
20 | ############################################################################### | |
1de8761d MT |
21 | |
22 | import json | |
062699ee | 23 | import os |
aa14071d | 24 | import pycurl |
4f91860e | 25 | import random |
1de8761d | 26 | |
8b6bc023 MT |
27 | import logging |
28 | log = logging.getLogger("pakfire") | |
29 | ||
a6bd96bc | 30 | from config import _Config |
e57c5475 | 31 | |
aa14071d | 32 | import urlgrabber.grabber |
1de8761d | 33 | from urlgrabber.grabber import URLGrabber, URLGrabError |
4f91860e | 34 | from urlgrabber.mirror import MirrorGroup |
14ea3228 | 35 | from urlgrabber.progress import TextMeter |
1de8761d | 36 | |
a2d1644c | 37 | from pakfire.constants import * |
062699ee | 38 | from pakfire.i18n import _ |
1de8761d MT |
39 | |
40 | class PakfireGrabber(URLGrabber): | |
41 | """ | |
42 | Class to make some modifications on the urlgrabber configuration. | |
43 | """ | |
80104a80 | 44 | def __init__(self, pakfire, *args, **kwargs): |
14ea3228 MT |
45 | kwargs.update({ |
46 | "quote" : 0, | |
47 | "user_agent" : "pakfire/%s" % PAKFIRE_VERSION, | |
a6bd96bc MT |
48 | |
49 | "ssl_verify_host" : False, | |
50 | "ssl_verify_peer" : False, | |
14ea3228 MT |
51 | }) |
52 | ||
a6bd96bc | 53 | if isinstance(pakfire, _Config): |
e57c5475 MT |
54 | config = pakfire |
55 | else: | |
56 | config = pakfire.config | |
98733451 | 57 | self.config = config |
6a509182 | 58 | |
cfc16a71 | 59 | # Set throttle setting. |
a6bd96bc | 60 | bandwidth_throttle = config.get("downloader", "bandwidth_throttle") |
80104a80 MT |
61 | if bandwidth_throttle: |
62 | try: | |
63 | bandwidth_throttle = int(bandwidth_throttle) | |
64 | except ValueError: | |
8b6bc023 | 65 | log.error("Configuration value for bandwidth_throttle is invalid.") |
80104a80 MT |
66 | bandwidth_throttle = 0 |
67 | ||
68 | kwargs.update({ "throttle" : bandwidth_throttle }) | |
69 | ||
cfc16a71 | 70 | # Configure HTTP proxy. |
a6bd96bc | 71 | http_proxy = config.get("downloader", "http_proxy") |
cfc16a71 | 72 | if http_proxy: |
c611f46b | 73 | kwargs.update({ "proxies" : { "http" : http_proxy, "https" : http_proxy }}) |
cfc16a71 | 74 | |
14ea3228 MT |
75 | URLGrabber.__init__(self, *args, **kwargs) |
76 | ||
98733451 MT |
77 | def check_offline_mode(self): |
78 | offline = self.config.get("downloader", "offline") | |
79 | if not offline: | |
80 | return | |
81 | ||
82 | raise OfflineModeError | |
83 | ||
4efe0da7 | 84 | def urlread(self, filename, *args, **kwargs): |
98733451 MT |
85 | self.check_offline_mode() |
86 | ||
4efe0da7 MT |
87 | # This is for older versions of urlgrabber which are packaged in Debian |
88 | # and Ubuntu and cannot handle filenames as a normal Python string but need | |
89 | # a unicode string. | |
90 | return URLGrabber.urlread(self, filename.encode("utf-8"), *args, **kwargs) | |
91 | ||
0f8d6745 | 92 | def urlopen(self, filename, *args, **kwargs): |
98733451 MT |
93 | self.check_offline_mode() |
94 | ||
bbd51f58 MT |
95 | # This is for older versions of urlgrabber which are packaged in Debian |
96 | # and Ubuntu and cannot handle filenames as a normal Python string but need | |
97 | # a unicode string. | |
98 | return URLGrabber.urlopen(self, filename.encode("utf-8"), *args, **kwargs) | |
99 | ||
100 | def urlgrab(self, url, *args, **kwargs): | |
101 | self.check_offline_mode() | |
0f8d6745 | 102 | |
bbd51f58 MT |
103 | # This is for older versions of urlgrabber which are packaged in Debian |
104 | # and Ubuntu and cannot handle filenames as a normal Python string but need | |
105 | # a unicode string. | |
106 | return URLGrabber.urlgrab(self, url.encode("utf-8"), *args, **kwargs) | |
0f8d6745 | 107 | |
14ea3228 MT |
108 | |
109 | class PackageDownloader(PakfireGrabber): | |
80104a80 | 110 | def __init__(self, pakfire, *args, **kwargs): |
14ea3228 | 111 | kwargs.update({ |
ca38a577 | 112 | "progress_obj" : TextMeter(), |
14ea3228 MT |
113 | }) |
114 | ||
80104a80 | 115 | PakfireGrabber.__init__(self, pakfire, *args, **kwargs) |
14ea3228 MT |
116 | |
117 | ||
118 | class MetadataDownloader(PakfireGrabber): | |
80104a80 | 119 | def __init__(self, pakfire, *args, **kwargs): |
14ea3228 MT |
120 | kwargs.update({ |
121 | "http_headers" : (('Pragma', 'no-cache'),), | |
122 | }) | |
123 | ||
80104a80 | 124 | PakfireGrabber.__init__(self, pakfire, *args, **kwargs) |
14ea3228 MT |
125 | |
126 | ||
127 | class DatabaseDownloader(PackageDownloader): | |
80104a80 | 128 | def __init__(self, pakfire, *args, **kwargs): |
14ea3228 MT |
129 | kwargs.update({ |
130 | "http_headers" : (('Pragma', 'no-cache'),), | |
131 | }) | |
132 | ||
80104a80 | 133 | PackageDownloader.__init__(self, pakfire, *args, **kwargs) |
1de8761d | 134 | |
4f91860e | 135 | |
062699ee MT |
136 | class SourceDownloader(object): |
137 | def __init__(self, pakfire, mirrors=None): | |
138 | self.pakfire = pakfire | |
139 | ||
140 | self.grabber = PakfireGrabber( | |
141 | self.pakfire, | |
142 | progress_obj = TextMeter(), | |
143 | ) | |
144 | ||
145 | if mirrors: | |
146 | self.grabber = MirrorGroup(self.grabber, | |
4efe0da7 | 147 | [{ "mirror" : m.encode("utf-8") } for m in mirrors]) |
062699ee MT |
148 | |
149 | def download(self, files): | |
150 | existant_files = [] | |
151 | download_files = [] | |
152 | ||
153 | for file in files: | |
154 | filename = os.path.join(SOURCE_CACHE_DIR, file) | |
b76f5f47 | 155 | log.debug("Checking existance of %s..." % filename) |
062699ee | 156 | |
9ddb19b9 | 157 | if os.path.exists(filename) and os.path.getsize(filename): |
b76f5f47 | 158 | log.debug("...exists!") |
062699ee MT |
159 | existant_files.append(filename) |
160 | else: | |
b76f5f47 | 161 | log.debug("...does not exist!") |
062699ee MT |
162 | download_files.append(filename) |
163 | ||
164 | if download_files: | |
8b6bc023 | 165 | log.info(_("Downloading source files:")) |
062699ee | 166 | |
98733451 MT |
167 | if self.pakfire.offline: |
168 | raise OfflineModeError, _("Cannot download source code in offline mode.") | |
169 | ||
062699ee MT |
170 | # Create source download directory. |
171 | if not os.path.exists(SOURCE_CACHE_DIR): | |
172 | os.makedirs(SOURCE_CACHE_DIR) | |
173 | ||
174 | for filename in download_files: | |
175 | try: | |
176 | self.grabber.urlgrab(os.path.basename(filename), filename=filename) | |
177 | except URLGrabError, e: | |
08de9306 MT |
178 | # Remove partly downloaded file. |
179 | try: | |
180 | os.unlink(filename) | |
181 | except OSError: | |
182 | pass | |
183 | ||
062699ee MT |
184 | raise DownloadError, "%s %s" % (os.path.basename(filename), e) |
185 | ||
9ddb19b9 MT |
186 | # Check if the downloaded file was empty. |
187 | if os.path.getsize(filename) == 0: | |
188 | # Remove the file and raise an error. | |
189 | os.unlink(filename) | |
190 | ||
191 | raise DownloadError, _("Downloaded empty file: %s") \ | |
192 | % os.path.basename(filename) | |
193 | ||
8b6bc023 | 194 | log.info("") |
062699ee MT |
195 | |
196 | return existant_files + download_files | |
197 | ||
198 | ||
1de8761d | 199 | class Mirror(object): |
4f91860e | 200 | def __init__(self, url, location=None, preferred=False): |
1de8761d | 201 | # Save URL of the mirror in full format |
4f91860e | 202 | self.url = url |
1de8761d MT |
203 | |
204 | # Save the location (if given) | |
205 | self.location = location | |
206 | ||
207 | # Save preference | |
208 | self.preferred = False | |
209 | ||
210 | ||
211 | class MirrorList(object): | |
0f8d6745 | 212 | def __init__(self, pakfire, repo, mirrorlist): |
1de8761d MT |
213 | self.pakfire = pakfire |
214 | self.repo = repo | |
215 | ||
216 | self.__mirrors = [] | |
217 | ||
218 | # Save URL to more mirrors. | |
0f8d6745 MT |
219 | self.mirrorlist = mirrorlist |
220 | ||
221 | @property | |
222 | def base_mirror(self): | |
223 | if not self.repo.baseurl: | |
224 | return | |
225 | ||
226 | return Mirror(self.repo.baseurl, preferred=False) | |
1de8761d | 227 | |
5a99898b MT |
228 | @property |
229 | def distro(self): | |
230 | return self.repo.distro | |
231 | ||
1de8761d MT |
232 | @property |
233 | def cache(self): | |
234 | """ | |
235 | Shortcut to cache from repository. | |
236 | """ | |
237 | return self.repo.cache | |
238 | ||
239 | def update(self, force=False): | |
240 | # XXX should this be allowed? | |
241 | if not self.mirrorlist: | |
242 | return | |
243 | ||
c07a3ca7 MT |
244 | # If the system is not online, we cannot download anything. |
245 | if self.pakfire.offline: | |
246 | return | |
247 | ||
8b6bc023 | 248 | log.debug("Updating mirrorlist for repository '%s' (force=%s)" % (self.repo.name, force)) |
5a99898b MT |
249 | cache_filename = os.path.join("repodata", self.distro.sname, self.distro.release, |
250 | self.repo.name, self.distro.arch, "mirrors") | |
1de8761d MT |
251 | |
252 | # Force the update if no mirrorlist is available. | |
253 | if not self.cache.exists(cache_filename): | |
254 | force = True | |
255 | ||
256 | if not force and self.cache.exists(cache_filename): | |
257 | age = self.cache.age(cache_filename) | |
258 | ||
259 | # If the age could be determined and is higher than 24h, | |
260 | # we force an update. | |
261 | if age and age > TIME_24H: | |
262 | force = True | |
263 | ||
264 | if force: | |
80104a80 | 265 | g = MetadataDownloader(self.pakfire) |
1de8761d MT |
266 | |
267 | try: | |
268 | mirrordata = g.urlread(self.mirrorlist, limit=MIRRORLIST_MAXSIZE) | |
269 | except URLGrabError, e: | |
8b6bc023 | 270 | log.warning("Could not update the mirrorlist for repo '%s': %s" % (self.repo.name, e)) |
1de8761d MT |
271 | return |
272 | ||
273 | # XXX check for empty files or damaged output | |
274 | ||
275 | # Save new mirror data to cache. | |
276 | f = self.cache.open(cache_filename, "w") | |
277 | f.write(mirrordata) | |
278 | f.close() | |
279 | ||
280 | # Read mirrorlist from cache and parse it. | |
90919c62 | 281 | self.forget_mirrors() |
1de8761d MT |
282 | with self.cache.open(cache_filename) as f: |
283 | self.parse_mirrordata(f.read()) | |
284 | ||
285 | def parse_mirrordata(self, data): | |
286 | data = json.loads(data) | |
287 | ||
288 | for mirror in data["mirrors"]: | |
289 | self.add_mirror(**mirror) | |
290 | ||
291 | def add_mirror(self, *args, **kwargs): | |
292 | mirror = Mirror(*args, **kwargs) | |
293 | ||
294 | self.__mirrors.append(mirror) | |
295 | ||
90919c62 MT |
296 | def forget_mirrors(self): |
297 | self.__mirrors = [] | |
298 | ||
1de8761d MT |
299 | @property |
300 | def preferred(self): | |
301 | """ | |
302 | Return a generator for all mirrors that are preferred. | |
303 | """ | |
304 | for mirror in self.__mirrors: | |
305 | if mirror.preferred: | |
306 | yield mirror | |
307 | ||
4f91860e MT |
308 | @property |
309 | def non_preferred(self): | |
310 | """ | |
311 | Return a generator for all mirrors that are not preferred. | |
312 | """ | |
313 | for mirror in self.__mirrors: | |
314 | if not mirror.preferred: | |
315 | yield mirror | |
316 | ||
1de8761d MT |
317 | @property |
318 | def all(self): | |
319 | """ | |
320 | Return a generator for all mirrors. | |
321 | """ | |
322 | for mirror in self.__mirrors: | |
323 | yield mirror | |
324 | ||
4f91860e MT |
325 | def group(self, grabber): |
326 | """ | |
327 | Return a MirrorGroup object for the given grabber. | |
328 | """ | |
90919c62 MT |
329 | # Make sure the mirrorlist is up to date. |
330 | self.update() | |
331 | ||
4f91860e MT |
332 | # A list of mirrors that is passed to MirrorGroup. |
333 | mirrors = [] | |
334 | ||
335 | # Add all preferred mirrors at the first place and shuffle them | |
336 | # that we will start at a random place. | |
337 | for mirror in self.preferred: | |
9a1da36a | 338 | mirrors.append({ "mirror" : mirror.url.encode("utf-8") }) |
4f91860e MT |
339 | random.shuffle(mirrors) |
340 | ||
341 | # All other mirrors are added as well and will only be used if all | |
342 | # preferred mirrors did not work. | |
343 | for mirror in self.all: | |
9a1da36a MT |
344 | mirror = { "mirror" : mirror.url.encode("utf-8") } |
345 | if mirror in mirrors: | |
4f91860e MT |
346 | continue |
347 | ||
9a1da36a MT |
348 | mirrors.append(mirror) |
349 | ||
350 | # Always add the base mirror if any. | |
351 | base_mirror = self.base_mirror | |
352 | if base_mirror: | |
353 | mirror = { "mirror" : base_mirror.url.encode("utf-8") } | |
354 | if not mirror in mirrors: | |
355 | mirrors.append(mirror) | |
4f91860e MT |
356 | |
357 | return MirrorGroup(grabber, mirrors) | |
358 | ||
359 | ||
360 | ||
361 | class Downloader(object): | |
362 | def __init__(self, mirrors, files): | |
363 | self.grabber = PakfireGrabber() | |
364 | ||
365 | self.mirrorgroup = mirrors.group(self.grabber) | |
366 | ||
367 |