]>
Commit | Line | Data |
---|---|---|
940227cb MT |
1 | #!/usr/bin/python |
2 | ||
9068dba1 | 3 | import datetime |
940227cb | 4 | import logging |
0673d1b0 | 5 | import math |
54af860e | 6 | import os.path |
0673d1b0 | 7 | import random |
940227cb MT |
8 | import socket |
9 | import time | |
27bcdc3f | 10 | import ssl |
b2059099 | 11 | import tornado.gen |
940227cb | 12 | import tornado.httpclient |
9068dba1 | 13 | import tornado.netutil |
11347e46 | 14 | import urllib.parse |
940227cb | 15 | |
f110a9ff | 16 | from . import countries |
11347e46 | 17 | from .misc import Object |
95483f04 | 18 | from .decorators import * |
60024cc8 | 19 | |
9068dba1 | 20 | class Mirrors(Object): |
95483f04 MT |
21 | def _get_mirrors(self, query, *args): |
22 | res = self.db.query(query, *args) | |
23 | ||
24 | for row in res: | |
25 | yield Mirror(self.backend, row.id, data=row) | |
26 | ||
f110a9ff MT |
27 | def _get_mirror(self, query, *args): |
28 | res = self.db.get(query, *args) | |
29 | ||
30 | if res: | |
31 | return Mirror(self.backend, res.id, data=res) | |
32 | ||
95483f04 MT |
33 | def __iter__(self): |
34 | mirrors = self._get_mirrors("SELECT * FROM mirrors \ | |
35 | WHERE enabled IS TRUE ORDER BY hostname") | |
36 | ||
37 | return iter(mirrors) | |
38 | ||
b2059099 | 39 | @tornado.gen.coroutine |
940227cb | 40 | def check_all(self): |
95483f04 | 41 | for mirror in self: |
b2059099 MT |
42 | with self.db.transaction(): |
43 | yield mirror.check() | |
940227cb | 44 | |
f110a9ff MT |
45 | def get_for_download(self, filename, country_code=None): |
46 | # Try to find a good mirror for this country first | |
47 | if country_code: | |
48 | zone = countries.get_zone(country_code) | |
49 | ||
50 | mirror = self._get_mirror("SELECT mirrors.* FROM mirror_files files \ | |
51 | LEFT JOIN mirrors ON files.mirror = mirrors.id \ | |
52 | WHERE files.filename = %s \ | |
53 | AND mirrors.enabled IS TRUE AND mirrors.state = %s \ | |
54 | AND mirrors.country_code = ANY(%s) \ | |
55 | ORDER BY RANDOM() LIMIT 1", filename, "UP", | |
56 | countries.get_in_zone(zone)) | |
57 | ||
58 | if mirror: | |
59 | return mirror | |
60 | ||
61 | # Get a random mirror that serves the file | |
62 | return self._get_mirror("SELECT mirrors.* FROM mirror_files files \ | |
63 | LEFT JOIN mirrors ON files.mirror = mirrors.id \ | |
64 | WHERE files.filename = %s \ | |
65 | AND mirrors.enabled IS TRUE AND mirrors.state = %s \ | |
66 | ORDER BY RANDOM() LIMIT 1", filename, "UP") | |
5488a9f4 | 67 | |
f110a9ff MT |
68 | def get_by_hostname(self, hostname): |
69 | return self._get_mirror("SELECT * FROM mirrors \ | |
70 | WHERE hostname = %s", hostname) | |
5488a9f4 | 71 | |
940227cb | 72 | |
9068dba1 | 73 | class Mirror(Object): |
95483f04 MT |
74 | def init(self, id, data=None): |
75 | self.id = id | |
76 | self.data = data | |
940227cb | 77 | |
95483f04 MT |
78 | def __str__(self): |
79 | return self.hostname | |
119f55d7 | 80 | |
54af860e MT |
81 | def __repr__(self): |
82 | return "<%s %s>" % (self.__class__.__name__, self.url) | |
83 | ||
95483f04 MT |
84 | def __eq__(self, other): |
85 | if isinstance(other, self.__class__): | |
86 | return self.id == other.id | |
9068dba1 | 87 | |
95483f04 MT |
88 | def __lt__(self, other): |
89 | if isinstance(other, self.__class__): | |
90 | return self.hostname < other.hostname | |
940227cb | 91 | |
95483f04 MT |
92 | @lazy_property |
93 | def url(self): | |
10cdef58 | 94 | url = "%s://%s" % ("https" if self.supports_https else "http", self.hostname) |
95483f04 | 95 | |
940227cb MT |
96 | if not self.path.startswith("/"): |
97 | url += "/" | |
95483f04 | 98 | |
940227cb | 99 | url += "%s" % self.path |
95483f04 | 100 | |
940227cb MT |
101 | if not self.path.endswith("/"): |
102 | url += "/" | |
95483f04 | 103 | |
940227cb MT |
104 | return url |
105 | ||
9068dba1 MT |
106 | @property |
107 | def hostname(self): | |
95483f04 | 108 | return self.data.hostname |
9068dba1 MT |
109 | |
110 | @property | |
111 | def path(self): | |
95483f04 | 112 | return self.data.path |
940227cb | 113 | |
a69e87a1 MT |
114 | @property |
115 | def supports_https(self): | |
95483f04 | 116 | return self.data.supports_https |
10cdef58 | 117 | |
940227cb MT |
118 | @property |
119 | def address(self): | |
199b04e7 | 120 | for addr in self.addresses4: |
b898caea MT |
121 | return addr |
122 | ||
199b04e7 | 123 | for addr in self.addresses6: |
b898caea | 124 | return addr |
940227cb | 125 | |
9068dba1 MT |
126 | @property |
127 | def owner(self): | |
95483f04 | 128 | return self.data.owner |
9068dba1 | 129 | |
95483f04 | 130 | @lazy_property |
0673d1b0 | 131 | def location(self): |
95483f04 | 132 | return self.geoip.get_location(self.address) |
0673d1b0 MT |
133 | |
134 | @property | |
135 | def latitude(self): | |
9068dba1 MT |
136 | if self.location: |
137 | return self.location.latitude | |
0673d1b0 MT |
138 | |
139 | @property | |
140 | def longitude(self): | |
9068dba1 MT |
141 | if self.location: |
142 | return self.location.longitude | |
0673d1b0 | 143 | |
940227cb MT |
144 | @property |
145 | def country_code(self): | |
f110a9ff | 146 | return self.data.country_code |
940227cb | 147 | |
0673d1b0 MT |
148 | @property |
149 | def country_name(self): | |
95483f04 | 150 | return self.geoip.get_country_name(self.country_code) |
0673d1b0 | 151 | |
f110a9ff MT |
152 | @property |
153 | def zone(self): | |
154 | return countries.get_zone(self.country_name) | |
155 | ||
95483f04 | 156 | @lazy_property |
9068dba1 | 157 | def asn(self): |
95483f04 | 158 | return self.geoip.get_asn(self.address) |
0673d1b0 | 159 | |
940227cb MT |
160 | @property |
161 | def filelist(self): | |
162 | filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id) | |
163 | return [f.filename for f in filelist] | |
164 | ||
54af860e MT |
165 | @property |
166 | def prefix(self): | |
54af860e MT |
167 | return "" |
168 | ||
9068dba1 | 169 | def build_url(self, filename): |
11347e46 | 170 | return urllib.parse.urljoin(self.url, filename) |
9068dba1 MT |
171 | |
172 | @property | |
173 | def last_update(self): | |
95483f04 | 174 | return self.data.last_update |
9068dba1 MT |
175 | |
176 | @property | |
177 | def state(self): | |
95483f04 | 178 | return self.data.state |
9068dba1 | 179 | |
940227cb MT |
180 | def set_state(self, state): |
181 | logging.info("Setting state of %s to %s" % (self.hostname, state)) | |
182 | ||
183 | if self.state == state: | |
184 | return | |
185 | ||
9068dba1 | 186 | self.db.execute("UPDATE mirrors SET state = %s WHERE id = %s", state, self.id) |
940227cb MT |
187 | |
188 | # Reload changed settings | |
95483f04 | 189 | self.data["state"] = state |
940227cb | 190 | |
9068dba1 MT |
191 | @property |
192 | def enabled(self): | |
95483f04 | 193 | return self.data.enabled |
9068dba1 MT |
194 | |
195 | @property | |
196 | def disabled(self): | |
197 | return not self.enabled | |
198 | ||
b2059099 | 199 | @tornado.gen.coroutine |
940227cb MT |
200 | def check(self): |
201 | logging.info("Running check for mirror %s" % self.hostname) | |
202 | ||
3ead0979 MT |
203 | self.db.execute("UPDATE mirrors SET address = %s WHERE id = %s", |
204 | self.address, self.id) | |
205 | ||
b2059099 | 206 | success = yield self.check_timestamp() |
940227cb | 207 | |
b2059099 MT |
208 | if success: |
209 | yield self.check_filelist() | |
210 | ||
211 | def check_state(self, last_update): | |
940227cb MT |
212 | logging.debug("Checking state of mirror %s" % self.id) |
213 | ||
9068dba1 | 214 | if not self.enabled: |
940227cb | 215 | self.set_state("DOWN") |
9068dba1 MT |
216 | return |
217 | ||
218 | now = datetime.datetime.utcnow() | |
219 | ||
b2059099 | 220 | time_delta = now - last_update |
9068dba1 | 221 | time_diff = time_delta.total_seconds() |
940227cb | 222 | |
9068dba1 MT |
223 | time_down = self.settings.get_int("mirrors_time_down", 3*24*60*60) |
224 | if time_diff >= time_down: | |
940227cb | 225 | self.set_state("DOWN") |
9068dba1 | 226 | return |
940227cb | 227 | |
9068dba1 MT |
228 | time_outofsync = self.settings.get_int("mirrors_time_outofsync", 6*60*60) |
229 | if time_diff >= time_outofsync: | |
230 | self.set_state("OUTOFSYNC") | |
940227cb MT |
231 | return |
232 | ||
9068dba1 MT |
233 | self.set_state("UP") |
234 | ||
b2059099 | 235 | @tornado.gen.coroutine |
9068dba1 | 236 | def check_timestamp(self): |
940227cb MT |
237 | http = tornado.httpclient.AsyncHTTPClient() |
238 | ||
b2059099 MT |
239 | try: |
240 | response = yield http.fetch(self.url + ".timestamp", | |
241 | headers={ "Pragma" : "no-cache" }) | |
242 | except tornado.httpclient.HTTPError as e: | |
243 | logging.error("Error getting timestamp from %s: %s" % (self.hostname, e)) | |
244 | self.set_state("DOWN") | |
245 | return False | |
940227cb | 246 | |
27bcdc3f MT |
247 | except ssl.SSLError as e: |
248 | logging.error("SSL error when getting timestamp from %s: %s" % (self.hostname, e)) | |
249 | self.set_state("DOWN") | |
250 | return False | |
251 | ||
940227cb MT |
252 | if response.error: |
253 | logging.debug("Error getting timestamp from %s" % self.hostname) | |
a3ee39ce | 254 | self.set_state("DOWN") |
940227cb MT |
255 | return |
256 | ||
257 | try: | |
258 | timestamp = int(response.body.strip()) | |
259 | except ValueError: | |
260 | timestamp = 0 | |
261 | ||
ea324f48 | 262 | timestamp = datetime.datetime.utcfromtimestamp(timestamp) |
9068dba1 MT |
263 | |
264 | self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s", | |
940227cb MT |
265 | timestamp, self.id) |
266 | ||
b2059099 MT |
267 | # Update state |
268 | self.check_state(timestamp) | |
940227cb MT |
269 | |
270 | logging.info("Successfully updated timestamp from %s" % self.hostname) | |
b2059099 | 271 | return True |
940227cb | 272 | |
b2059099 | 273 | @tornado.gen.coroutine |
940227cb | 274 | def check_filelist(self): |
54af860e | 275 | # XXX need to remove data from disabled mirrors |
9068dba1 | 276 | if not self.enabled: |
940227cb MT |
277 | return |
278 | ||
279 | http = tornado.httpclient.AsyncHTTPClient() | |
280 | ||
b2059099 MT |
281 | try: |
282 | response = yield http.fetch(self.url + ".filelist", | |
283 | headers={ "Pragma" : "no-cache" }) | |
284 | except tornado.httpclient.HTTPError as e: | |
285 | logging.error("Error getting filelist from %s: %s" % (self.hostname, e)) | |
286 | self.set_state("DOWN") | |
287 | return | |
940227cb | 288 | |
940227cb | 289 | if response.error: |
b2059099 | 290 | logging.debug("Error getting filelist from %s" % self.hostname) |
940227cb MT |
291 | return |
292 | ||
b2059099 MT |
293 | # Drop the old filelist |
294 | self.db.execute("DELETE FROM mirror_files WHERE mirror = %s", self.id) | |
940227cb | 295 | |
b2059099 | 296 | # Add them all again |
940227cb | 297 | for file in response.body.splitlines(): |
b2059099 | 298 | file = os.path.join(self.prefix, file.decode()) |
56b9c1d8 | 299 | |
b2059099 MT |
300 | self.db.execute("INSERT INTO mirror_files(mirror, filename) \ |
301 | VALUES(%s, %s)", self.id, file) | |
940227cb MT |
302 | |
303 | logging.info("Successfully updated mirror filelist from %s" % self.hostname) | |
304 | ||
bd17b7d1 MT |
305 | @property |
306 | def development(self): | |
95483f04 | 307 | return self.data.get("mirrorlist_devel", False) |
bd17b7d1 MT |
308 | |
309 | @property | |
310 | def mirrorlist(self): | |
95483f04 | 311 | return self.data.get("mirrorlist", False) |
9068dba1 | 312 | |
f110a9ff | 313 | @lazy_property |
9068dba1 | 314 | def addresses(self): |
f110a9ff MT |
315 | try: |
316 | addrinfo = socket.getaddrinfo(self.hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM) | |
317 | except: | |
318 | raise Exception("Could not resolve %s" % self.hostname) | |
319 | ||
320 | ret = [] | |
321 | for family, socktype, proto, canonname, address in addrinfo: | |
322 | if family == socket.AF_INET: | |
323 | address, port = address | |
324 | elif family == socket.AF_INET6: | |
325 | address, port, flowid, scopeid = address | |
326 | ret.append((family, address)) | |
327 | ||
328 | return ret | |
9068dba1 MT |
329 | |
330 | @property | |
331 | def addresses6(self): | |
332 | return [address for family, address in self.addresses if family == socket.AF_INET6] | |
333 | ||
334 | @property | |
335 | def addresses4(self): | |
336 | return [address for family, address in self.addresses if family == socket.AF_INET] |