]>
Commit | Line | Data |
---|---|---|
940227cb MT |
1 | #!/usr/bin/python |
2 | ||
9068dba1 | 3 | import datetime |
940227cb | 4 | import logging |
0673d1b0 | 5 | import math |
54af860e | 6 | import os.path |
0673d1b0 | 7 | import random |
940227cb MT |
8 | import socket |
9 | import time | |
b2059099 | 10 | import tornado.gen |
940227cb | 11 | import tornado.httpclient |
9068dba1 | 12 | import tornado.netutil |
11347e46 | 13 | import urllib.parse |
940227cb | 14 | |
f110a9ff | 15 | from . import countries |
11347e46 | 16 | from .misc import Object |
95483f04 | 17 | from .decorators import * |
60024cc8 | 18 | |
9068dba1 | 19 | class Mirrors(Object): |
95483f04 MT |
20 | def _get_mirrors(self, query, *args): |
21 | res = self.db.query(query, *args) | |
22 | ||
23 | for row in res: | |
24 | yield Mirror(self.backend, row.id, data=row) | |
25 | ||
f110a9ff MT |
26 | def _get_mirror(self, query, *args): |
27 | res = self.db.get(query, *args) | |
28 | ||
29 | if res: | |
30 | return Mirror(self.backend, res.id, data=res) | |
31 | ||
95483f04 MT |
32 | def __iter__(self): |
33 | mirrors = self._get_mirrors("SELECT * FROM mirrors \ | |
34 | WHERE enabled IS TRUE ORDER BY hostname") | |
35 | ||
36 | return iter(mirrors) | |
37 | ||
b2059099 | 38 | @tornado.gen.coroutine |
940227cb | 39 | def check_all(self): |
95483f04 | 40 | for mirror in self: |
b2059099 MT |
41 | with self.db.transaction(): |
42 | yield mirror.check() | |
940227cb | 43 | |
f110a9ff MT |
44 | def get_for_download(self, filename, country_code=None): |
45 | # Try to find a good mirror for this country first | |
46 | if country_code: | |
47 | zone = countries.get_zone(country_code) | |
48 | ||
49 | mirror = self._get_mirror("SELECT mirrors.* FROM mirror_files files \ | |
50 | LEFT JOIN mirrors ON files.mirror = mirrors.id \ | |
51 | WHERE files.filename = %s \ | |
52 | AND mirrors.enabled IS TRUE AND mirrors.state = %s \ | |
53 | AND mirrors.country_code = ANY(%s) \ | |
54 | ORDER BY RANDOM() LIMIT 1", filename, "UP", | |
55 | countries.get_in_zone(zone)) | |
56 | ||
57 | if mirror: | |
58 | return mirror | |
59 | ||
60 | # Get a random mirror that serves the file | |
61 | return self._get_mirror("SELECT mirrors.* FROM mirror_files files \ | |
62 | LEFT JOIN mirrors ON files.mirror = mirrors.id \ | |
63 | WHERE files.filename = %s \ | |
64 | AND mirrors.enabled IS TRUE AND mirrors.state = %s \ | |
65 | ORDER BY RANDOM() LIMIT 1", filename, "UP") | |
5488a9f4 | 66 | |
f110a9ff MT |
67 | def get_by_hostname(self, hostname): |
68 | return self._get_mirror("SELECT * FROM mirrors \ | |
69 | WHERE hostname = %s", hostname) | |
5488a9f4 | 70 | |
940227cb | 71 | |
9068dba1 | 72 | class Mirror(Object): |
95483f04 MT |
73 | def init(self, id, data=None): |
74 | self.id = id | |
75 | self.data = data | |
940227cb | 76 | |
95483f04 MT |
77 | def __str__(self): |
78 | return self.hostname | |
119f55d7 | 79 | |
54af860e MT |
80 | def __repr__(self): |
81 | return "<%s %s>" % (self.__class__.__name__, self.url) | |
82 | ||
95483f04 MT |
83 | def __eq__(self, other): |
84 | if isinstance(other, self.__class__): | |
85 | return self.id == other.id | |
9068dba1 | 86 | |
95483f04 MT |
87 | def __lt__(self, other): |
88 | if isinstance(other, self.__class__): | |
89 | return self.hostname < other.hostname | |
940227cb | 90 | |
95483f04 MT |
91 | @lazy_property |
92 | def url(self): | |
10cdef58 | 93 | url = "%s://%s" % ("https" if self.supports_https else "http", self.hostname) |
95483f04 | 94 | |
940227cb MT |
95 | if not self.path.startswith("/"): |
96 | url += "/" | |
95483f04 | 97 | |
940227cb | 98 | url += "%s" % self.path |
95483f04 | 99 | |
940227cb MT |
100 | if not self.path.endswith("/"): |
101 | url += "/" | |
95483f04 | 102 | |
940227cb MT |
103 | return url |
104 | ||
9068dba1 MT |
105 | @property |
106 | def hostname(self): | |
95483f04 | 107 | return self.data.hostname |
9068dba1 MT |
108 | |
109 | @property | |
110 | def path(self): | |
95483f04 | 111 | return self.data.path |
940227cb | 112 | |
a69e87a1 MT |
113 | @property |
114 | def supports_https(self): | |
95483f04 | 115 | return self.data.supports_https |
10cdef58 | 116 | |
940227cb MT |
117 | @property |
118 | def address(self): | |
199b04e7 | 119 | for addr in self.addresses4: |
b898caea MT |
120 | return addr |
121 | ||
199b04e7 | 122 | for addr in self.addresses6: |
b898caea | 123 | return addr |
940227cb | 124 | |
9068dba1 MT |
125 | @property |
126 | def owner(self): | |
95483f04 | 127 | return self.data.owner |
9068dba1 | 128 | |
95483f04 | 129 | @lazy_property |
0673d1b0 | 130 | def location(self): |
95483f04 | 131 | return self.geoip.get_location(self.address) |
0673d1b0 MT |
132 | |
133 | @property | |
134 | def latitude(self): | |
9068dba1 MT |
135 | if self.location: |
136 | return self.location.latitude | |
0673d1b0 MT |
137 | |
138 | @property | |
139 | def longitude(self): | |
9068dba1 MT |
140 | if self.location: |
141 | return self.location.longitude | |
0673d1b0 | 142 | |
940227cb MT |
143 | @property |
144 | def country_code(self): | |
f110a9ff | 145 | return self.data.country_code |
940227cb | 146 | |
0673d1b0 MT |
147 | @property |
148 | def country_name(self): | |
95483f04 | 149 | return self.geoip.get_country_name(self.country_code) |
0673d1b0 | 150 | |
f110a9ff MT |
151 | @property |
152 | def zone(self): | |
153 | return countries.get_zone(self.country_name) | |
154 | ||
95483f04 | 155 | @lazy_property |
9068dba1 | 156 | def asn(self): |
95483f04 | 157 | return self.geoip.get_asn(self.address) |
0673d1b0 | 158 | |
940227cb MT |
159 | @property |
160 | def filelist(self): | |
161 | filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id) | |
162 | return [f.filename for f in filelist] | |
163 | ||
54af860e MT |
164 | @property |
165 | def prefix(self): | |
54af860e MT |
166 | return "" |
167 | ||
9068dba1 | 168 | def build_url(self, filename): |
11347e46 | 169 | return urllib.parse.urljoin(self.url, filename) |
9068dba1 MT |
170 | |
171 | @property | |
172 | def last_update(self): | |
95483f04 | 173 | return self.data.last_update |
9068dba1 MT |
174 | |
175 | @property | |
176 | def state(self): | |
95483f04 | 177 | return self.data.state |
9068dba1 | 178 | |
940227cb MT |
179 | def set_state(self, state): |
180 | logging.info("Setting state of %s to %s" % (self.hostname, state)) | |
181 | ||
182 | if self.state == state: | |
183 | return | |
184 | ||
9068dba1 | 185 | self.db.execute("UPDATE mirrors SET state = %s WHERE id = %s", state, self.id) |
940227cb MT |
186 | |
187 | # Reload changed settings | |
95483f04 | 188 | self.data["state"] = state |
940227cb | 189 | |
9068dba1 MT |
190 | @property |
191 | def enabled(self): | |
95483f04 | 192 | return self.data.enabled |
9068dba1 MT |
193 | |
194 | @property | |
195 | def disabled(self): | |
196 | return not self.enabled | |
197 | ||
b2059099 | 198 | @tornado.gen.coroutine |
940227cb MT |
199 | def check(self): |
200 | logging.info("Running check for mirror %s" % self.hostname) | |
201 | ||
3ead0979 MT |
202 | self.db.execute("UPDATE mirrors SET address = %s WHERE id = %s", |
203 | self.address, self.id) | |
204 | ||
b2059099 | 205 | success = yield self.check_timestamp() |
940227cb | 206 | |
b2059099 MT |
207 | if success: |
208 | yield self.check_filelist() | |
209 | ||
210 | def check_state(self, last_update): | |
940227cb MT |
211 | logging.debug("Checking state of mirror %s" % self.id) |
212 | ||
9068dba1 | 213 | if not self.enabled: |
940227cb | 214 | self.set_state("DOWN") |
9068dba1 MT |
215 | return |
216 | ||
217 | now = datetime.datetime.utcnow() | |
218 | ||
b2059099 | 219 | time_delta = now - last_update |
9068dba1 | 220 | time_diff = time_delta.total_seconds() |
940227cb | 221 | |
9068dba1 MT |
222 | time_down = self.settings.get_int("mirrors_time_down", 3*24*60*60) |
223 | if time_diff >= time_down: | |
940227cb | 224 | self.set_state("DOWN") |
9068dba1 | 225 | return |
940227cb | 226 | |
9068dba1 MT |
227 | time_outofsync = self.settings.get_int("mirrors_time_outofsync", 6*60*60) |
228 | if time_diff >= time_outofsync: | |
229 | self.set_state("OUTOFSYNC") | |
940227cb MT |
230 | return |
231 | ||
9068dba1 MT |
232 | self.set_state("UP") |
233 | ||
b2059099 | 234 | @tornado.gen.coroutine |
9068dba1 | 235 | def check_timestamp(self): |
940227cb MT |
236 | http = tornado.httpclient.AsyncHTTPClient() |
237 | ||
b2059099 MT |
238 | try: |
239 | response = yield http.fetch(self.url + ".timestamp", | |
240 | headers={ "Pragma" : "no-cache" }) | |
241 | except tornado.httpclient.HTTPError as e: | |
242 | logging.error("Error getting timestamp from %s: %s" % (self.hostname, e)) | |
243 | self.set_state("DOWN") | |
244 | return False | |
940227cb | 245 | |
940227cb MT |
246 | if response.error: |
247 | logging.debug("Error getting timestamp from %s" % self.hostname) | |
a3ee39ce | 248 | self.set_state("DOWN") |
940227cb MT |
249 | return |
250 | ||
251 | try: | |
252 | timestamp = int(response.body.strip()) | |
253 | except ValueError: | |
254 | timestamp = 0 | |
255 | ||
ea324f48 | 256 | timestamp = datetime.datetime.utcfromtimestamp(timestamp) |
9068dba1 MT |
257 | |
258 | self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s", | |
940227cb MT |
259 | timestamp, self.id) |
260 | ||
b2059099 MT |
261 | # Update state |
262 | self.check_state(timestamp) | |
940227cb MT |
263 | |
264 | logging.info("Successfully updated timestamp from %s" % self.hostname) | |
b2059099 | 265 | return True |
940227cb | 266 | |
b2059099 | 267 | @tornado.gen.coroutine |
940227cb | 268 | def check_filelist(self): |
54af860e | 269 | # XXX need to remove data from disabled mirrors |
9068dba1 | 270 | if not self.enabled: |
940227cb MT |
271 | return |
272 | ||
273 | http = tornado.httpclient.AsyncHTTPClient() | |
274 | ||
b2059099 MT |
275 | try: |
276 | response = yield http.fetch(self.url + ".filelist", | |
277 | headers={ "Pragma" : "no-cache" }) | |
278 | except tornado.httpclient.HTTPError as e: | |
279 | logging.error("Error getting filelist from %s: %s" % (self.hostname, e)) | |
280 | self.set_state("DOWN") | |
281 | return | |
940227cb | 282 | |
940227cb | 283 | if response.error: |
b2059099 | 284 | logging.debug("Error getting filelist from %s" % self.hostname) |
940227cb MT |
285 | return |
286 | ||
b2059099 MT |
287 | # Drop the old filelist |
288 | self.db.execute("DELETE FROM mirror_files WHERE mirror = %s", self.id) | |
940227cb | 289 | |
b2059099 | 290 | # Add them all again |
940227cb | 291 | for file in response.body.splitlines(): |
b2059099 | 292 | file = os.path.join(self.prefix, file.decode()) |
56b9c1d8 | 293 | |
b2059099 MT |
294 | self.db.execute("INSERT INTO mirror_files(mirror, filename) \ |
295 | VALUES(%s, %s)", self.id, file) | |
940227cb MT |
296 | |
297 | logging.info("Successfully updated mirror filelist from %s" % self.hostname) | |
298 | ||
bd17b7d1 MT |
299 | @property |
300 | def development(self): | |
95483f04 | 301 | return self.data.get("mirrorlist_devel", False) |
bd17b7d1 MT |
302 | |
303 | @property | |
304 | def mirrorlist(self): | |
95483f04 | 305 | return self.data.get("mirrorlist", False) |
9068dba1 | 306 | |
f110a9ff | 307 | @lazy_property |
9068dba1 | 308 | def addresses(self): |
f110a9ff MT |
309 | try: |
310 | addrinfo = socket.getaddrinfo(self.hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM) | |
311 | except: | |
312 | raise Exception("Could not resolve %s" % self.hostname) | |
313 | ||
314 | ret = [] | |
315 | for family, socktype, proto, canonname, address in addrinfo: | |
316 | if family == socket.AF_INET: | |
317 | address, port = address | |
318 | elif family == socket.AF_INET6: | |
319 | address, port, flowid, scopeid = address | |
320 | ret.append((family, address)) | |
321 | ||
322 | return ret | |
9068dba1 MT |
323 | |
324 | @property | |
325 | def addresses6(self): | |
326 | return [address for family, address in self.addresses if family == socket.AF_INET6] | |
327 | ||
328 | @property | |
329 | def addresses4(self): | |
330 | return [address for family, address in self.addresses if family == socket.AF_INET] |