]>
Commit | Line | Data |
---|---|---|
940227cb MT |
1 | #!/usr/bin/python |
2 | ||
9068dba1 | 3 | import datetime |
940227cb | 4 | import logging |
7a9e9176 | 5 | import iso3166 |
0673d1b0 | 6 | import math |
54af860e | 7 | import os.path |
0673d1b0 | 8 | import random |
940227cb MT |
9 | import socket |
10 | import time | |
27bcdc3f | 11 | import ssl |
940227cb | 12 | import tornado.httpclient |
1bb68b0a | 13 | import tornado.iostream |
9068dba1 | 14 | import tornado.netutil |
11347e46 | 15 | import urllib.parse |
940227cb | 16 | |
f110a9ff | 17 | from . import countries |
440aba92 | 18 | from . import util |
11347e46 | 19 | from .misc import Object |
95483f04 | 20 | from .decorators import * |
60024cc8 | 21 | |
9068dba1 | 22 | class Mirrors(Object): |
95483f04 MT |
23 | def _get_mirrors(self, query, *args): |
24 | res = self.db.query(query, *args) | |
25 | ||
26 | for row in res: | |
27 | yield Mirror(self.backend, row.id, data=row) | |
28 | ||
f110a9ff MT |
29 | def _get_mirror(self, query, *args): |
30 | res = self.db.get(query, *args) | |
31 | ||
32 | if res: | |
33 | return Mirror(self.backend, res.id, data=res) | |
34 | ||
95483f04 MT |
35 | def __iter__(self): |
36 | mirrors = self._get_mirrors("SELECT * FROM mirrors \ | |
37 | WHERE enabled IS TRUE ORDER BY hostname") | |
38 | ||
39 | return iter(mirrors) | |
40 | ||
9fdf4fb7 | 41 | async def check_all(self): |
95483f04 | 42 | for mirror in self: |
b2059099 | 43 | with self.db.transaction(): |
9fdf4fb7 | 44 | await mirror.check() |
940227cb | 45 | |
f110a9ff MT |
46 | def get_for_download(self, filename, country_code=None): |
47 | # Try to find a good mirror for this country first | |
48 | if country_code: | |
49 | zone = countries.get_zone(country_code) | |
50 | ||
51 | mirror = self._get_mirror("SELECT mirrors.* FROM mirror_files files \ | |
52 | LEFT JOIN mirrors ON files.mirror = mirrors.id \ | |
53 | WHERE files.filename = %s \ | |
54 | AND mirrors.enabled IS TRUE AND mirrors.state = %s \ | |
55 | AND mirrors.country_code = ANY(%s) \ | |
56 | ORDER BY RANDOM() LIMIT 1", filename, "UP", | |
57 | countries.get_in_zone(zone)) | |
58 | ||
59 | if mirror: | |
60 | return mirror | |
61 | ||
62 | # Get a random mirror that serves the file | |
63 | return self._get_mirror("SELECT mirrors.* FROM mirror_files files \ | |
64 | LEFT JOIN mirrors ON files.mirror = mirrors.id \ | |
65 | WHERE files.filename = %s \ | |
66 | AND mirrors.enabled IS TRUE AND mirrors.state = %s \ | |
67 | ORDER BY RANDOM() LIMIT 1", filename, "UP") | |
5488a9f4 | 68 | |
f110a9ff MT |
69 | def get_by_hostname(self, hostname): |
70 | return self._get_mirror("SELECT * FROM mirrors \ | |
71 | WHERE hostname = %s", hostname) | |
5488a9f4 | 72 | |
7a9e9176 MT |
73 | def get_by_countries(self): |
74 | mirrors = {} | |
75 | ||
76 | for m in self: | |
77 | try: | |
78 | mirrors[m.country].append(m) | |
79 | except KeyError: | |
80 | mirrors[m.country] = [m] | |
81 | ||
82 | return mirrors | |
83 | ||
940227cb | 84 | |
9068dba1 | 85 | class Mirror(Object): |
95483f04 MT |
86 | def init(self, id, data=None): |
87 | self.id = id | |
88 | self.data = data | |
940227cb | 89 | |
95483f04 MT |
90 | def __str__(self): |
91 | return self.hostname | |
119f55d7 | 92 | |
54af860e MT |
93 | def __repr__(self): |
94 | return "<%s %s>" % (self.__class__.__name__, self.url) | |
95 | ||
95483f04 MT |
96 | def __eq__(self, other): |
97 | if isinstance(other, self.__class__): | |
98 | return self.id == other.id | |
9068dba1 | 99 | |
95483f04 MT |
100 | def __lt__(self, other): |
101 | if isinstance(other, self.__class__): | |
102 | return self.hostname < other.hostname | |
940227cb | 103 | |
440aba92 MT |
104 | def __hash__(self): |
105 | return self.id | |
106 | ||
95483f04 MT |
107 | @lazy_property |
108 | def url(self): | |
10cdef58 | 109 | url = "%s://%s" % ("https" if self.supports_https else "http", self.hostname) |
95483f04 | 110 | |
940227cb MT |
111 | if not self.path.startswith("/"): |
112 | url += "/" | |
95483f04 | 113 | |
940227cb | 114 | url += "%s" % self.path |
95483f04 | 115 | |
940227cb MT |
116 | if not self.path.endswith("/"): |
117 | url += "/" | |
95483f04 | 118 | |
940227cb MT |
119 | return url |
120 | ||
9068dba1 MT |
121 | @property |
122 | def hostname(self): | |
95483f04 | 123 | return self.data.hostname |
9068dba1 | 124 | |
440aba92 MT |
125 | @lazy_property |
126 | def address(self): | |
127 | """ | |
128 | Returns the stored address | |
129 | """ | |
130 | if self.data.address: | |
131 | return util.Address(self.backend, self.data.address) | |
132 | ||
9068dba1 MT |
133 | @property |
134 | def path(self): | |
95483f04 | 135 | return self.data.path |
940227cb | 136 | |
a69e87a1 MT |
137 | @property |
138 | def supports_https(self): | |
95483f04 | 139 | return self.data.supports_https |
10cdef58 | 140 | |
9068dba1 MT |
141 | @property |
142 | def owner(self): | |
95483f04 | 143 | return self.data.owner |
9068dba1 | 144 | |
0673d1b0 | 145 | @property |
7a9e9176 MT |
146 | def country(self): |
147 | return iso3166.countries.get(self.country_code) | |
148 | ||
940227cb MT |
149 | @property |
150 | def country_code(self): | |
440aba92 MT |
151 | if self.data.country_code: |
152 | return self.data.country_code | |
940227cb | 153 | |
440aba92 MT |
154 | if self.address: |
155 | return self.address.country_code | |
0673d1b0 | 156 | |
f110a9ff MT |
157 | @property |
158 | def zone(self): | |
159 | return countries.get_zone(self.country_name) | |
160 | ||
440aba92 | 161 | @property |
9068dba1 | 162 | def asn(self): |
440aba92 MT |
163 | if self.address: |
164 | return self.address.asn | |
0673d1b0 | 165 | |
940227cb MT |
166 | @property |
167 | def filelist(self): | |
168 | filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id) | |
169 | return [f.filename for f in filelist] | |
170 | ||
54af860e MT |
171 | @property |
172 | def prefix(self): | |
54af860e MT |
173 | return "" |
174 | ||
9068dba1 | 175 | def build_url(self, filename): |
11347e46 | 176 | return urllib.parse.urljoin(self.url, filename) |
9068dba1 MT |
177 | |
178 | @property | |
179 | def last_update(self): | |
95483f04 | 180 | return self.data.last_update |
9068dba1 MT |
181 | |
182 | @property | |
183 | def state(self): | |
95483f04 | 184 | return self.data.state |
9068dba1 | 185 | |
940227cb | 186 | def set_state(self, state): |
fb51c9c7 | 187 | logging.debug("Setting state of %s to %s" % (self.hostname, state)) |
940227cb MT |
188 | |
189 | if self.state == state: | |
190 | return | |
191 | ||
9068dba1 | 192 | self.db.execute("UPDATE mirrors SET state = %s WHERE id = %s", state, self.id) |
940227cb MT |
193 | |
194 | # Reload changed settings | |
95483f04 | 195 | self.data["state"] = state |
940227cb | 196 | |
9068dba1 MT |
197 | @property |
198 | def enabled(self): | |
95483f04 | 199 | return self.data.enabled |
9068dba1 MT |
200 | |
201 | @property | |
202 | def disabled(self): | |
203 | return not self.enabled | |
204 | ||
9fdf4fb7 | 205 | async def check(self): |
fb51c9c7 | 206 | logging.debug("Running check for mirror %s" % self.hostname) |
940227cb | 207 | |
3ead0979 | 208 | self.db.execute("UPDATE mirrors SET address = %s WHERE id = %s", |
440aba92 | 209 | await self.resolve(), self.id) |
3ead0979 | 210 | |
9fdf4fb7 | 211 | success = await self.check_timestamp() |
b2059099 | 212 | if success: |
9fdf4fb7 | 213 | await self.check_filelist() |
b2059099 MT |
214 | |
215 | def check_state(self, last_update): | |
940227cb MT |
216 | logging.debug("Checking state of mirror %s" % self.id) |
217 | ||
9068dba1 | 218 | if not self.enabled: |
940227cb | 219 | self.set_state("DOWN") |
9068dba1 MT |
220 | return |
221 | ||
222 | now = datetime.datetime.utcnow() | |
223 | ||
b2059099 | 224 | time_delta = now - last_update |
9068dba1 | 225 | time_diff = time_delta.total_seconds() |
940227cb | 226 | |
9068dba1 MT |
227 | time_down = self.settings.get_int("mirrors_time_down", 3*24*60*60) |
228 | if time_diff >= time_down: | |
940227cb | 229 | self.set_state("DOWN") |
9068dba1 | 230 | return |
940227cb | 231 | |
9068dba1 MT |
232 | time_outofsync = self.settings.get_int("mirrors_time_outofsync", 6*60*60) |
233 | if time_diff >= time_outofsync: | |
234 | self.set_state("OUTOFSYNC") | |
940227cb MT |
235 | return |
236 | ||
9068dba1 MT |
237 | self.set_state("UP") |
238 | ||
9fdf4fb7 | 239 | async def check_timestamp(self): |
940227cb MT |
240 | http = tornado.httpclient.AsyncHTTPClient() |
241 | ||
b2059099 | 242 | try: |
9fdf4fb7 | 243 | response = await http.fetch(self.url + ".timestamp", |
b2059099 MT |
244 | headers={ "Pragma" : "no-cache" }) |
245 | except tornado.httpclient.HTTPError as e: | |
0661e16b | 246 | logging.warning("Error getting timestamp from %s: %s" % (self.hostname, e)) |
b2059099 MT |
247 | self.set_state("DOWN") |
248 | return False | |
940227cb | 249 | |
27bcdc3f | 250 | except ssl.SSLError as e: |
0661e16b | 251 | logging.warning("SSL error when getting timestamp from %s: %s" % (self.hostname, e)) |
27bcdc3f MT |
252 | self.set_state("DOWN") |
253 | return False | |
254 | ||
1bb68b0a | 255 | except tornado.iostream.StreamClosedError as e: |
0661e16b | 256 | logging.warning("Connection closed unexpectedly for %s: %s" % (self.hostname, e)) |
1bb68b0a MT |
257 | self.set_state("DOWN") |
258 | return False | |
259 | ||
260 | except OSError as e: | |
0661e16b | 261 | logging.warning("Could not connect to %s: %s" % (self.hostname, e)) |
1bb68b0a MT |
262 | self.set_state("DOWN") |
263 | return False | |
264 | ||
940227cb | 265 | if response.error: |
0661e16b | 266 | logging.warning("Error getting timestamp from %s" % self.hostname) |
a3ee39ce | 267 | self.set_state("DOWN") |
940227cb MT |
268 | return |
269 | ||
270 | try: | |
271 | timestamp = int(response.body.strip()) | |
272 | except ValueError: | |
273 | timestamp = 0 | |
274 | ||
ea324f48 | 275 | timestamp = datetime.datetime.utcfromtimestamp(timestamp) |
9068dba1 MT |
276 | |
277 | self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s", | |
940227cb MT |
278 | timestamp, self.id) |
279 | ||
b2059099 MT |
280 | # Update state |
281 | self.check_state(timestamp) | |
940227cb | 282 | |
fb51c9c7 MT |
283 | logging.debug("Successfully updated timestamp from %s" % self.hostname) |
284 | ||
b2059099 | 285 | return True |
940227cb | 286 | |
9fdf4fb7 | 287 | async def check_filelist(self): |
54af860e | 288 | # XXX need to remove data from disabled mirrors |
9068dba1 | 289 | if not self.enabled: |
940227cb MT |
290 | return |
291 | ||
292 | http = tornado.httpclient.AsyncHTTPClient() | |
293 | ||
b2059099 | 294 | try: |
9fdf4fb7 | 295 | response = await http.fetch(self.url + ".filelist", |
b2059099 MT |
296 | headers={ "Pragma" : "no-cache" }) |
297 | except tornado.httpclient.HTTPError as e: | |
5c704007 | 298 | logging.warning("Error getting filelist from %s: %s" % (self.hostname, e)) |
b2059099 MT |
299 | self.set_state("DOWN") |
300 | return | |
940227cb | 301 | |
940227cb | 302 | if response.error: |
b2059099 | 303 | logging.debug("Error getting filelist from %s" % self.hostname) |
940227cb MT |
304 | return |
305 | ||
b2059099 MT |
306 | # Drop the old filelist |
307 | self.db.execute("DELETE FROM mirror_files WHERE mirror = %s", self.id) | |
940227cb | 308 | |
b2059099 | 309 | # Add them all again |
940227cb | 310 | for file in response.body.splitlines(): |
b2059099 | 311 | file = os.path.join(self.prefix, file.decode()) |
56b9c1d8 | 312 | |
b2059099 MT |
313 | self.db.execute("INSERT INTO mirror_files(mirror, filename) \ |
314 | VALUES(%s, %s)", self.id, file) | |
940227cb | 315 | |
fb51c9c7 | 316 | logging.debug("Successfully updated mirror filelist from %s" % self.hostname) |
940227cb | 317 | |
bd17b7d1 MT |
318 | @property |
319 | def development(self): | |
95483f04 | 320 | return self.data.get("mirrorlist_devel", False) |
bd17b7d1 MT |
321 | |
322 | @property | |
323 | def mirrorlist(self): | |
95483f04 | 324 | return self.data.get("mirrorlist", False) |
9068dba1 | 325 | |
440aba92 MT |
326 | async def resolve(self): |
327 | """ | |
328 | Returns a single IP address of this mirror | |
329 | """ | |
330 | addresses = await self.backend.resolver.resolve(self.hostname, 0) | |
f110a9ff | 331 | |
440aba92 MT |
332 | # Return the first address |
333 | for family, address in addresses: | |
334 | host, port = address | |
9068dba1 | 335 | |
440aba92 | 336 | return host |