]>
Commit | Line | Data |
---|---|---|
940227cb MT |
1 | #!/usr/bin/python |
2 | ||
9068dba1 | 3 | import datetime |
940227cb | 4 | import logging |
0673d1b0 | 5 | import math |
54af860e | 6 | import os.path |
0673d1b0 | 7 | import random |
940227cb MT |
8 | import socket |
9 | import time | |
27bcdc3f | 10 | import ssl |
940227cb | 11 | import tornado.httpclient |
1bb68b0a | 12 | import tornado.iostream |
9068dba1 | 13 | import tornado.netutil |
11347e46 | 14 | import urllib.parse |
940227cb | 15 | |
f110a9ff | 16 | from . import countries |
11347e46 | 17 | from .misc import Object |
95483f04 | 18 | from .decorators import * |
60024cc8 | 19 | |
9068dba1 | 20 | class Mirrors(Object): |
95483f04 MT |
21 | def _get_mirrors(self, query, *args): |
22 | res = self.db.query(query, *args) | |
23 | ||
24 | for row in res: | |
25 | yield Mirror(self.backend, row.id, data=row) | |
26 | ||
f110a9ff MT |
27 | def _get_mirror(self, query, *args): |
28 | res = self.db.get(query, *args) | |
29 | ||
30 | if res: | |
31 | return Mirror(self.backend, res.id, data=res) | |
32 | ||
95483f04 MT |
33 | def __iter__(self): |
34 | mirrors = self._get_mirrors("SELECT * FROM mirrors \ | |
35 | WHERE enabled IS TRUE ORDER BY hostname") | |
36 | ||
37 | return iter(mirrors) | |
38 | ||
9fdf4fb7 | 39 | async def check_all(self): |
95483f04 | 40 | for mirror in self: |
b2059099 | 41 | with self.db.transaction(): |
9fdf4fb7 | 42 | await mirror.check() |
940227cb | 43 | |
f110a9ff MT |
44 | def get_for_download(self, filename, country_code=None): |
45 | # Try to find a good mirror for this country first | |
46 | if country_code: | |
47 | zone = countries.get_zone(country_code) | |
48 | ||
49 | mirror = self._get_mirror("SELECT mirrors.* FROM mirror_files files \ | |
50 | LEFT JOIN mirrors ON files.mirror = mirrors.id \ | |
51 | WHERE files.filename = %s \ | |
52 | AND mirrors.enabled IS TRUE AND mirrors.state = %s \ | |
53 | AND mirrors.country_code = ANY(%s) \ | |
54 | ORDER BY RANDOM() LIMIT 1", filename, "UP", | |
55 | countries.get_in_zone(zone)) | |
56 | ||
57 | if mirror: | |
58 | return mirror | |
59 | ||
60 | # Get a random mirror that serves the file | |
61 | return self._get_mirror("SELECT mirrors.* FROM mirror_files files \ | |
62 | LEFT JOIN mirrors ON files.mirror = mirrors.id \ | |
63 | WHERE files.filename = %s \ | |
64 | AND mirrors.enabled IS TRUE AND mirrors.state = %s \ | |
65 | ORDER BY RANDOM() LIMIT 1", filename, "UP") | |
5488a9f4 | 66 | |
f110a9ff MT |
67 | def get_by_hostname(self, hostname): |
68 | return self._get_mirror("SELECT * FROM mirrors \ | |
69 | WHERE hostname = %s", hostname) | |
5488a9f4 | 70 | |
940227cb | 71 | |
9068dba1 | 72 | class Mirror(Object): |
95483f04 MT |
73 | def init(self, id, data=None): |
74 | self.id = id | |
75 | self.data = data | |
940227cb | 76 | |
95483f04 MT |
77 | def __str__(self): |
78 | return self.hostname | |
119f55d7 | 79 | |
54af860e MT |
80 | def __repr__(self): |
81 | return "<%s %s>" % (self.__class__.__name__, self.url) | |
82 | ||
95483f04 MT |
83 | def __eq__(self, other): |
84 | if isinstance(other, self.__class__): | |
85 | return self.id == other.id | |
9068dba1 | 86 | |
95483f04 MT |
87 | def __lt__(self, other): |
88 | if isinstance(other, self.__class__): | |
89 | return self.hostname < other.hostname | |
940227cb | 90 | |
95483f04 MT |
91 | @lazy_property |
92 | def url(self): | |
10cdef58 | 93 | url = "%s://%s" % ("https" if self.supports_https else "http", self.hostname) |
95483f04 | 94 | |
940227cb MT |
95 | if not self.path.startswith("/"): |
96 | url += "/" | |
95483f04 | 97 | |
940227cb | 98 | url += "%s" % self.path |
95483f04 | 99 | |
940227cb MT |
100 | if not self.path.endswith("/"): |
101 | url += "/" | |
95483f04 | 102 | |
940227cb MT |
103 | return url |
104 | ||
9068dba1 MT |
105 | @property |
106 | def hostname(self): | |
95483f04 | 107 | return self.data.hostname |
9068dba1 MT |
108 | |
109 | @property | |
110 | def path(self): | |
95483f04 | 111 | return self.data.path |
940227cb | 112 | |
a69e87a1 MT |
113 | @property |
114 | def supports_https(self): | |
95483f04 | 115 | return self.data.supports_https |
10cdef58 | 116 | |
940227cb MT |
117 | @property |
118 | def address(self): | |
199b04e7 | 119 | for addr in self.addresses4: |
b898caea MT |
120 | return addr |
121 | ||
199b04e7 | 122 | for addr in self.addresses6: |
b898caea | 123 | return addr |
940227cb | 124 | |
9068dba1 MT |
125 | @property |
126 | def owner(self): | |
95483f04 | 127 | return self.data.owner |
9068dba1 | 128 | |
95483f04 | 129 | @lazy_property |
0673d1b0 | 130 | def location(self): |
95483f04 | 131 | return self.geoip.get_location(self.address) |
0673d1b0 MT |
132 | |
133 | @property | |
134 | def latitude(self): | |
9068dba1 MT |
135 | if self.location: |
136 | return self.location.latitude | |
0673d1b0 MT |
137 | |
138 | @property | |
139 | def longitude(self): | |
9068dba1 MT |
140 | if self.location: |
141 | return self.location.longitude | |
0673d1b0 | 142 | |
940227cb MT |
143 | @property |
144 | def country_code(self): | |
f110a9ff | 145 | return self.data.country_code |
940227cb | 146 | |
0673d1b0 MT |
147 | @property |
148 | def country_name(self): | |
95483f04 | 149 | return self.geoip.get_country_name(self.country_code) |
0673d1b0 | 150 | |
f110a9ff MT |
151 | @property |
152 | def zone(self): | |
153 | return countries.get_zone(self.country_name) | |
154 | ||
95483f04 | 155 | @lazy_property |
9068dba1 | 156 | def asn(self): |
95483f04 | 157 | return self.geoip.get_asn(self.address) |
0673d1b0 | 158 | |
940227cb MT |
159 | @property |
160 | def filelist(self): | |
161 | filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id) | |
162 | return [f.filename for f in filelist] | |
163 | ||
54af860e MT |
164 | @property |
165 | def prefix(self): | |
54af860e MT |
166 | return "" |
167 | ||
9068dba1 | 168 | def build_url(self, filename): |
11347e46 | 169 | return urllib.parse.urljoin(self.url, filename) |
9068dba1 MT |
170 | |
171 | @property | |
172 | def last_update(self): | |
95483f04 | 173 | return self.data.last_update |
9068dba1 MT |
174 | |
175 | @property | |
176 | def state(self): | |
95483f04 | 177 | return self.data.state |
9068dba1 | 178 | |
940227cb | 179 | def set_state(self, state): |
fb51c9c7 | 180 | logging.debug("Setting state of %s to %s" % (self.hostname, state)) |
940227cb MT |
181 | |
182 | if self.state == state: | |
183 | return | |
184 | ||
9068dba1 | 185 | self.db.execute("UPDATE mirrors SET state = %s WHERE id = %s", state, self.id) |
940227cb MT |
186 | |
187 | # Reload changed settings | |
95483f04 | 188 | self.data["state"] = state |
940227cb | 189 | |
9068dba1 MT |
190 | @property |
191 | def enabled(self): | |
95483f04 | 192 | return self.data.enabled |
9068dba1 MT |
193 | |
194 | @property | |
195 | def disabled(self): | |
196 | return not self.enabled | |
197 | ||
9fdf4fb7 | 198 | async def check(self): |
fb51c9c7 | 199 | logging.debug("Running check for mirror %s" % self.hostname) |
940227cb | 200 | |
3ead0979 MT |
201 | self.db.execute("UPDATE mirrors SET address = %s WHERE id = %s", |
202 | self.address, self.id) | |
203 | ||
9fdf4fb7 | 204 | success = await self.check_timestamp() |
b2059099 | 205 | if success: |
9fdf4fb7 | 206 | await self.check_filelist() |
b2059099 MT |
207 | |
208 | def check_state(self, last_update): | |
940227cb MT |
209 | logging.debug("Checking state of mirror %s" % self.id) |
210 | ||
9068dba1 | 211 | if not self.enabled: |
940227cb | 212 | self.set_state("DOWN") |
9068dba1 MT |
213 | return |
214 | ||
215 | now = datetime.datetime.utcnow() | |
216 | ||
b2059099 | 217 | time_delta = now - last_update |
9068dba1 | 218 | time_diff = time_delta.total_seconds() |
940227cb | 219 | |
9068dba1 MT |
220 | time_down = self.settings.get_int("mirrors_time_down", 3*24*60*60) |
221 | if time_diff >= time_down: | |
940227cb | 222 | self.set_state("DOWN") |
9068dba1 | 223 | return |
940227cb | 224 | |
9068dba1 MT |
225 | time_outofsync = self.settings.get_int("mirrors_time_outofsync", 6*60*60) |
226 | if time_diff >= time_outofsync: | |
227 | self.set_state("OUTOFSYNC") | |
940227cb MT |
228 | return |
229 | ||
9068dba1 MT |
230 | self.set_state("UP") |
231 | ||
9fdf4fb7 | 232 | async def check_timestamp(self): |
940227cb MT |
233 | http = tornado.httpclient.AsyncHTTPClient() |
234 | ||
b2059099 | 235 | try: |
9fdf4fb7 | 236 | response = await http.fetch(self.url + ".timestamp", |
b2059099 MT |
237 | headers={ "Pragma" : "no-cache" }) |
238 | except tornado.httpclient.HTTPError as e: | |
0661e16b | 239 | logging.warning("Error getting timestamp from %s: %s" % (self.hostname, e)) |
b2059099 MT |
240 | self.set_state("DOWN") |
241 | return False | |
940227cb | 242 | |
27bcdc3f | 243 | except ssl.SSLError as e: |
0661e16b | 244 | logging.warning("SSL error when getting timestamp from %s: %s" % (self.hostname, e)) |
27bcdc3f MT |
245 | self.set_state("DOWN") |
246 | return False | |
247 | ||
1bb68b0a | 248 | except tornado.iostream.StreamClosedError as e: |
0661e16b | 249 | logging.warning("Connection closed unexpectedly for %s: %s" % (self.hostname, e)) |
1bb68b0a MT |
250 | self.set_state("DOWN") |
251 | return False | |
252 | ||
253 | except OSError as e: | |
0661e16b | 254 | logging.warning("Could not connect to %s: %s" % (self.hostname, e)) |
1bb68b0a MT |
255 | self.set_state("DOWN") |
256 | return False | |
257 | ||
940227cb | 258 | if response.error: |
0661e16b | 259 | logging.warning("Error getting timestamp from %s" % self.hostname) |
a3ee39ce | 260 | self.set_state("DOWN") |
940227cb MT |
261 | return |
262 | ||
263 | try: | |
264 | timestamp = int(response.body.strip()) | |
265 | except ValueError: | |
266 | timestamp = 0 | |
267 | ||
ea324f48 | 268 | timestamp = datetime.datetime.utcfromtimestamp(timestamp) |
9068dba1 MT |
269 | |
270 | self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s", | |
940227cb MT |
271 | timestamp, self.id) |
272 | ||
b2059099 MT |
273 | # Update state |
274 | self.check_state(timestamp) | |
940227cb | 275 | |
fb51c9c7 MT |
276 | logging.debug("Successfully updated timestamp from %s" % self.hostname) |
277 | ||
b2059099 | 278 | return True |
940227cb | 279 | |
9fdf4fb7 | 280 | async def check_filelist(self): |
54af860e | 281 | # XXX need to remove data from disabled mirrors |
9068dba1 | 282 | if not self.enabled: |
940227cb MT |
283 | return |
284 | ||
285 | http = tornado.httpclient.AsyncHTTPClient() | |
286 | ||
b2059099 | 287 | try: |
9fdf4fb7 | 288 | response = await http.fetch(self.url + ".filelist", |
b2059099 MT |
289 | headers={ "Pragma" : "no-cache" }) |
290 | except tornado.httpclient.HTTPError as e: | |
5c704007 | 291 | logging.warning("Error getting filelist from %s: %s" % (self.hostname, e)) |
b2059099 MT |
292 | self.set_state("DOWN") |
293 | return | |
940227cb | 294 | |
940227cb | 295 | if response.error: |
b2059099 | 296 | logging.debug("Error getting filelist from %s" % self.hostname) |
940227cb MT |
297 | return |
298 | ||
b2059099 MT |
299 | # Drop the old filelist |
300 | self.db.execute("DELETE FROM mirror_files WHERE mirror = %s", self.id) | |
940227cb | 301 | |
b2059099 | 302 | # Add them all again |
940227cb | 303 | for file in response.body.splitlines(): |
b2059099 | 304 | file = os.path.join(self.prefix, file.decode()) |
56b9c1d8 | 305 | |
b2059099 MT |
306 | self.db.execute("INSERT INTO mirror_files(mirror, filename) \ |
307 | VALUES(%s, %s)", self.id, file) | |
940227cb | 308 | |
fb51c9c7 | 309 | logging.debug("Successfully updated mirror filelist from %s" % self.hostname) |
940227cb | 310 | |
bd17b7d1 MT |
311 | @property |
312 | def development(self): | |
95483f04 | 313 | return self.data.get("mirrorlist_devel", False) |
bd17b7d1 MT |
314 | |
315 | @property | |
316 | def mirrorlist(self): | |
95483f04 | 317 | return self.data.get("mirrorlist", False) |
9068dba1 | 318 | |
f110a9ff | 319 | @lazy_property |
9068dba1 | 320 | def addresses(self): |
b064747a | 321 | addrinfo = socket.getaddrinfo(self.hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM) |
f110a9ff MT |
322 | |
323 | ret = [] | |
324 | for family, socktype, proto, canonname, address in addrinfo: | |
325 | if family == socket.AF_INET: | |
326 | address, port = address | |
327 | elif family == socket.AF_INET6: | |
328 | address, port, flowid, scopeid = address | |
329 | ret.append((family, address)) | |
330 | ||
331 | return ret | |
9068dba1 MT |
332 | |
333 | @property | |
334 | def addresses6(self): | |
335 | return [address for family, address in self.addresses if family == socket.AF_INET6] | |
336 | ||
337 | @property | |
338 | def addresses4(self): | |
339 | return [address for family, address in self.addresses if family == socket.AF_INET] |