]>
git.ipfire.org Git - ipfire.org.git/blob - src/backend/mirrors.py
12 import tornado
.httpclient
13 import tornado
.iostream
14 import tornado
.netutil
17 from . import countries
19 from .misc
import Object
20 from .decorators
import *
22 class Mirrors(Object
):
23 def _get_mirrors(self
, query
, *args
):
24 res
= self
.db
.query(query
, *args
)
27 yield Mirror(self
.backend
, row
.id, data
=row
)
29 def _get_mirror(self
, query
, *args
):
30 res
= self
.db
.get(query
, *args
)
33 return Mirror(self
.backend
, res
.id, data
=res
)
36 mirrors
= self
._get
_mirrors
("SELECT * FROM mirrors \
37 WHERE enabled IS TRUE ORDER BY hostname")
41 async def check_all(self
):
43 with self
.db
.transaction():
46 def get_for_download(self
, filename
, country_code
=None):
47 # Try to find a good mirror for this country first
49 zone
= countries
.get_zone(country_code
)
51 mirror
= self
._get
_mirror
("SELECT mirrors.* FROM mirror_files files \
52 LEFT JOIN mirrors ON files.mirror = mirrors.id \
53 WHERE files.filename = %s \
54 AND mirrors.enabled IS TRUE AND mirrors.state = %s \
55 AND mirrors.country_code = ANY(%s) \
56 ORDER BY RANDOM() LIMIT 1", filename
, "UP",
57 countries
.get_in_zone(zone
))
62 # Get a random mirror that serves the file
63 return self
._get
_mirror
("SELECT mirrors.* FROM mirror_files files \
64 LEFT JOIN mirrors ON files.mirror = mirrors.id \
65 WHERE files.filename = %s \
66 AND mirrors.enabled IS TRUE AND mirrors.state = %s \
67 ORDER BY RANDOM() LIMIT 1", filename
, "UP")
69 def get_by_hostname(self
, hostname
):
70 return self
._get
_mirror
("SELECT * FROM mirrors \
71 WHERE hostname = %s", hostname
)
73 def get_by_countries(self
):
78 mirrors
[m
.country
].append(m
)
80 mirrors
[m
.country
] = [m
]
86 def init(self
, id, data
=None):
94 return "<%s %s>" % (self
.__class
__.__name
__, self
.url
)
96 def __eq__(self
, other
):
97 if isinstance(other
, self
.__class
__):
98 return self
.id == other
.id
100 def __lt__(self
, other
):
101 if isinstance(other
, self
.__class
__):
102 return self
.hostname
< other
.hostname
109 url
= "%s://%s" % ("https" if self
.supports_https
else "http", self
.hostname
)
111 if not self
.path
.startswith("/"):
114 url
+= "%s" % self
.path
116 if not self
.path
.endswith("/"):
123 return self
.data
.hostname
128 Returns the stored address
130 if self
.data
.address
:
131 return util
.Address(self
.backend
, self
.data
.address
)
135 return self
.data
.path
138 def supports_https(self
):
139 return self
.data
.supports_https
143 return self
.data
.owner
147 return iso3166
.countries
.get(self
.country_code
)
150 def country_code(self
):
151 if self
.data
.country_code
:
152 return self
.data
.country_code
155 return self
.address
.country_code
159 return countries
.get_zone(self
.country_name
)
164 return self
.address
.asn
168 filelist
= self
.db
.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self
.id)
169 return [f
.filename
for f
in filelist
]
175 def build_url(self
, filename
):
176 return urllib
.parse
.urljoin(self
.url
, filename
)
179 def last_update(self
):
180 return self
.data
.last_update
184 return self
.data
.state
186 def set_state(self
, state
):
187 logging
.debug("Setting state of %s to %s" % (self
.hostname
, state
))
189 if self
.state
== state
:
192 self
.db
.execute("UPDATE mirrors SET state = %s WHERE id = %s", state
, self
.id)
194 # Reload changed settings
195 self
.data
["state"] = state
199 return self
.data
.enabled
203 return not self
.enabled
205 async def check(self
):
206 logging
.debug("Running check for mirror %s" % self
.hostname
)
208 self
.db
.execute("UPDATE mirrors SET address = %s WHERE id = %s",
209 await self
.resolve(), self
.id)
211 success
= await self
.check_timestamp()
213 await self
.check_filelist()
215 def check_state(self
, last_update
):
216 logging
.debug("Checking state of mirror %s" % self
.id)
219 self
.set_state("DOWN")
222 now
= datetime
.datetime
.utcnow()
224 time_delta
= now
- last_update
225 time_diff
= time_delta
.total_seconds()
227 time_down
= self
.settings
.get_int("mirrors_time_down", 3*24*60*60)
228 if time_diff
>= time_down
:
229 self
.set_state("DOWN")
232 time_outofsync
= self
.settings
.get_int("mirrors_time_outofsync", 6*60*60)
233 if time_diff
>= time_outofsync
:
234 self
.set_state("OUTOFSYNC")
239 async def check_timestamp(self
):
240 http
= tornado
.httpclient
.AsyncHTTPClient()
243 response
= await http
.fetch(self
.url
+ ".timestamp",
244 headers
={ "Pragma" : "no-cache" })
245 except tornado
.httpclient
.HTTPError
as e
:
246 logging
.warning("Error getting timestamp from %s: %s" % (self
.hostname
, e
))
247 self
.set_state("DOWN")
250 except ssl
.SSLError
as e
:
251 logging
.warning("SSL error when getting timestamp from %s: %s" % (self
.hostname
, e
))
252 self
.set_state("DOWN")
255 except tornado
.iostream
.StreamClosedError
as e
:
256 logging
.warning("Connection closed unexpectedly for %s: %s" % (self
.hostname
, e
))
257 self
.set_state("DOWN")
261 logging
.warning("Could not connect to %s: %s" % (self
.hostname
, e
))
262 self
.set_state("DOWN")
266 logging
.warning("Error getting timestamp from %s" % self
.hostname
)
267 self
.set_state("DOWN")
271 timestamp
= int(response
.body
.strip())
275 timestamp
= datetime
.datetime
.utcfromtimestamp(timestamp
)
277 self
.db
.execute("UPDATE mirrors SET last_update = %s WHERE id = %s",
281 self
.check_state(timestamp
)
283 logging
.debug("Successfully updated timestamp from %s" % self
.hostname
)
287 async def check_filelist(self
):
288 # XXX need to remove data from disabled mirrors
292 http
= tornado
.httpclient
.AsyncHTTPClient()
295 response
= await http
.fetch(self
.url
+ ".filelist",
296 headers
={ "Pragma" : "no-cache" })
297 except tornado
.httpclient
.HTTPError
as e
:
298 logging
.warning("Error getting filelist from %s: %s" % (self
.hostname
, e
))
299 self
.set_state("DOWN")
303 logging
.debug("Error getting filelist from %s" % self
.hostname
)
306 # Drop the old filelist
307 self
.db
.execute("DELETE FROM mirror_files WHERE mirror = %s", self
.id)
310 for file in response
.body
.splitlines():
311 file = os
.path
.join(self
.prefix
, file.decode())
313 self
.db
.execute("INSERT INTO mirror_files(mirror, filename) \
314 VALUES(%s, %s)", self
.id, file)
316 logging
.debug("Successfully updated mirror filelist from %s" % self
.hostname
)
319 def development(self
):
320 return self
.data
.get("mirrorlist_devel", False)
323 def mirrorlist(self
):
324 return self
.data
.get("mirrorlist", False)
326 async def resolve(self
):
328 Returns a single IP address of this mirror
330 addresses
= await self
.backend
.resolver
.resolve(self
.hostname
, 0)
332 # Return the first address
333 for family
, address
in addresses
: