]>
Commit | Line | Data |
---|---|---|
1de8761d | 1 | #!/usr/bin/python |
b792d887 MT |
2 | ############################################################################### |
3 | # # | |
4 | # Pakfire - The IPFire package management system # | |
5 | # Copyright (C) 2011 Pakfire development team # | |
6 | # # | |
7 | # This program is free software: you can redistribute it and/or modify # | |
8 | # it under the terms of the GNU General Public License as published by # | |
9 | # the Free Software Foundation, either version 3 of the License, or # | |
10 | # (at your option) any later version. # | |
11 | # # | |
12 | # This program is distributed in the hope that it will be useful, # | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of # | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # | |
15 | # GNU General Public License for more details. # | |
16 | # # | |
17 | # You should have received a copy of the GNU General Public License # | |
18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. # | |
19 | # # | |
20 | ############################################################################### | |
1de8761d MT |
21 | |
22 | import json | |
23 | import logging | |
4f91860e | 24 | import random |
1de8761d | 25 | |
e57c5475 MT |
26 | from config import Config |
27 | ||
1de8761d | 28 | from urlgrabber.grabber import URLGrabber, URLGrabError |
4f91860e | 29 | from urlgrabber.mirror import MirrorGroup |
14ea3228 | 30 | from urlgrabber.progress import TextMeter |
1de8761d | 31 | |
a2d1644c | 32 | from pakfire.constants import * |
1de8761d MT |
33 | |
34 | class PakfireGrabber(URLGrabber): | |
35 | """ | |
36 | Class to make some modifications on the urlgrabber configuration. | |
37 | """ | |
80104a80 | 38 | def __init__(self, pakfire, *args, **kwargs): |
14ea3228 MT |
39 | kwargs.update({ |
40 | "quote" : 0, | |
41 | "user_agent" : "pakfire/%s" % PAKFIRE_VERSION, | |
42 | }) | |
43 | ||
e57c5475 MT |
44 | if isinstance(pakfire, Config): |
45 | config = pakfire | |
46 | else: | |
47 | config = pakfire.config | |
48 | ||
6a509182 | 49 | if config.get("offline"): |
6a509182 MT |
50 | raise OfflineModeError, "Cannot use %s in offline mode." % self.__class__.__name__ |
51 | ||
cfc16a71 | 52 | # Set throttle setting. |
e57c5475 | 53 | bandwidth_throttle = config.get("bandwidth_throttle") |
80104a80 MT |
54 | if bandwidth_throttle: |
55 | try: | |
56 | bandwidth_throttle = int(bandwidth_throttle) | |
57 | except ValueError: | |
58 | logging.error("Configuration value for bandwidth_throttle is invalid.") | |
59 | bandwidth_throttle = 0 | |
60 | ||
61 | kwargs.update({ "throttle" : bandwidth_throttle }) | |
62 | ||
cfc16a71 | 63 | # Configure HTTP proxy. |
e57c5475 | 64 | http_proxy = config.get("http_proxy") |
cfc16a71 MT |
65 | if http_proxy: |
66 | kwargs.update({ "proxies" : { "http" : http_proxy }}) | |
67 | ||
14ea3228 MT |
68 | URLGrabber.__init__(self, *args, **kwargs) |
69 | ||
70 | ||
71 | class PackageDownloader(PakfireGrabber): | |
80104a80 | 72 | def __init__(self, pakfire, *args, **kwargs): |
14ea3228 MT |
73 | kwargs.update({ |
74 | "progress_obj" : TextMeter(), | |
75 | }) | |
76 | ||
80104a80 | 77 | PakfireGrabber.__init__(self, pakfire, *args, **kwargs) |
14ea3228 MT |
78 | |
79 | ||
80 | class MetadataDownloader(PakfireGrabber): | |
80104a80 | 81 | def __init__(self, pakfire, *args, **kwargs): |
14ea3228 MT |
82 | kwargs.update({ |
83 | "http_headers" : (('Pragma', 'no-cache'),), | |
84 | }) | |
85 | ||
80104a80 | 86 | PakfireGrabber.__init__(self, pakfire, *args, **kwargs) |
14ea3228 MT |
87 | |
88 | ||
89 | class DatabaseDownloader(PackageDownloader): | |
80104a80 | 90 | def __init__(self, pakfire, *args, **kwargs): |
14ea3228 MT |
91 | kwargs.update({ |
92 | "http_headers" : (('Pragma', 'no-cache'),), | |
93 | }) | |
94 | ||
80104a80 | 95 | PackageDownloader.__init__(self, pakfire, *args, **kwargs) |
1de8761d | 96 | |
4f91860e | 97 | |
1de8761d | 98 | class Mirror(object): |
4f91860e | 99 | def __init__(self, url, location=None, preferred=False): |
1de8761d | 100 | # Save URL of the mirror in full format |
4f91860e | 101 | self.url = url |
1de8761d MT |
102 | |
103 | # Save the location (if given) | |
104 | self.location = location | |
105 | ||
106 | # Save preference | |
107 | self.preferred = False | |
108 | ||
109 | ||
110 | class MirrorList(object): | |
111 | def __init__(self, pakfire, repo): | |
112 | self.pakfire = pakfire | |
113 | self.repo = repo | |
114 | ||
115 | self.__mirrors = [] | |
116 | ||
117 | # Save URL to more mirrors. | |
118 | self.mirrorlist = repo.mirrorlist | |
119 | ||
120 | self.update(force=False) | |
121 | ||
122 | @property | |
123 | def cache(self): | |
124 | """ | |
125 | Shortcut to cache from repository. | |
126 | """ | |
127 | return self.repo.cache | |
128 | ||
129 | def update(self, force=False): | |
130 | # XXX should this be allowed? | |
131 | if not self.mirrorlist: | |
132 | return | |
133 | ||
c07a3ca7 MT |
134 | # If the system is not online, we cannot download anything. |
135 | if self.pakfire.offline: | |
136 | return | |
137 | ||
1de8761d MT |
138 | logging.debug("Updating mirrorlist for repository '%s' (force=%s)" % (self.repo.name, force)) |
139 | ||
140 | cache_filename = "mirrors/mirrorlist" | |
141 | ||
142 | # Force the update if no mirrorlist is available. | |
143 | if not self.cache.exists(cache_filename): | |
144 | force = True | |
145 | ||
146 | if not force and self.cache.exists(cache_filename): | |
147 | age = self.cache.age(cache_filename) | |
148 | ||
149 | # If the age could be determined and is higher than 24h, | |
150 | # we force an update. | |
151 | if age and age > TIME_24H: | |
152 | force = True | |
153 | ||
154 | if force: | |
80104a80 | 155 | g = MetadataDownloader(self.pakfire) |
1de8761d MT |
156 | |
157 | try: | |
158 | mirrordata = g.urlread(self.mirrorlist, limit=MIRRORLIST_MAXSIZE) | |
159 | except URLGrabError, e: | |
160 | logging.warning("Could not update the mirrorlist for repo '%s': %s" % (self.repo.name, e)) | |
161 | return | |
162 | ||
163 | # XXX check for empty files or damaged output | |
164 | ||
165 | # Save new mirror data to cache. | |
166 | f = self.cache.open(cache_filename, "w") | |
167 | f.write(mirrordata) | |
168 | f.close() | |
169 | ||
170 | # Read mirrorlist from cache and parse it. | |
171 | with self.cache.open(cache_filename) as f: | |
172 | self.parse_mirrordata(f.read()) | |
173 | ||
174 | def parse_mirrordata(self, data): | |
175 | data = json.loads(data) | |
176 | ||
177 | for mirror in data["mirrors"]: | |
178 | self.add_mirror(**mirror) | |
179 | ||
180 | def add_mirror(self, *args, **kwargs): | |
181 | mirror = Mirror(*args, **kwargs) | |
182 | ||
183 | self.__mirrors.append(mirror) | |
184 | ||
185 | @property | |
186 | def preferred(self): | |
187 | """ | |
188 | Return a generator for all mirrors that are preferred. | |
189 | """ | |
190 | for mirror in self.__mirrors: | |
191 | if mirror.preferred: | |
192 | yield mirror | |
193 | ||
4f91860e MT |
194 | @property |
195 | def non_preferred(self): | |
196 | """ | |
197 | Return a generator for all mirrors that are not preferred. | |
198 | """ | |
199 | for mirror in self.__mirrors: | |
200 | if not mirror.preferred: | |
201 | yield mirror | |
202 | ||
1de8761d MT |
203 | @property |
204 | def all(self): | |
205 | """ | |
206 | Return a generator for all mirrors. | |
207 | """ | |
208 | for mirror in self.__mirrors: | |
209 | yield mirror | |
210 | ||
4f91860e MT |
211 | def group(self, grabber): |
212 | """ | |
213 | Return a MirrorGroup object for the given grabber. | |
214 | """ | |
215 | # A list of mirrors that is passed to MirrorGroup. | |
216 | mirrors = [] | |
217 | ||
218 | # Add all preferred mirrors at the first place and shuffle them | |
219 | # that we will start at a random place. | |
220 | for mirror in self.preferred: | |
221 | mirrors.append(mirror.url) | |
222 | random.shuffle(mirrors) | |
223 | ||
224 | # All other mirrors are added as well and will only be used if all | |
225 | # preferred mirrors did not work. | |
226 | for mirror in self.all: | |
227 | if mirror.url in mirrors: | |
228 | continue | |
229 | ||
60285ce1 | 230 | mirrors.append({ "mirror" : mirror.url }) |
4f91860e MT |
231 | |
232 | return MirrorGroup(grabber, mirrors) | |
233 | ||
234 | ||
235 | ||
236 | class Downloader(object): | |
237 | def __init__(self, mirrors, files): | |
238 | self.grabber = PakfireGrabber() | |
239 | ||
240 | self.mirrorgroup = mirrors.group(self.grabber) | |
241 | ||
242 |