]>
git.ipfire.org Git - ipfire.org.git/blob - src/backend/mirrors.py
11 import tornado
.httpclient
12 import tornado
.iostream
13 import tornado
.netutil
16 from . import countries
17 from .misc
import Object
18 from .decorators
import *
20 class Mirrors(Object
):
21 def _get_mirrors(self
, query
, *args
):
22 res
= self
.db
.query(query
, *args
)
25 yield Mirror(self
.backend
, row
.id, data
=row
)
27 def _get_mirror(self
, query
, *args
):
28 res
= self
.db
.get(query
, *args
)
31 return Mirror(self
.backend
, res
.id, data
=res
)
34 mirrors
= self
._get
_mirrors
("SELECT * FROM mirrors \
35 WHERE enabled IS TRUE ORDER BY hostname")
39 async def check_all(self
):
41 with self
.db
.transaction():
44 def get_for_download(self
, filename
, country_code
=None):
45 # Try to find a good mirror for this country first
47 zone
= countries
.get_zone(country_code
)
49 mirror
= self
._get
_mirror
("SELECT mirrors.* FROM mirror_files files \
50 LEFT JOIN mirrors ON files.mirror = mirrors.id \
51 WHERE files.filename = %s \
52 AND mirrors.enabled IS TRUE AND mirrors.state = %s \
53 AND mirrors.country_code = ANY(%s) \
54 ORDER BY RANDOM() LIMIT 1", filename
, "UP",
55 countries
.get_in_zone(zone
))
60 # Get a random mirror that serves the file
61 return self
._get
_mirror
("SELECT mirrors.* FROM mirror_files files \
62 LEFT JOIN mirrors ON files.mirror = mirrors.id \
63 WHERE files.filename = %s \
64 AND mirrors.enabled IS TRUE AND mirrors.state = %s \
65 ORDER BY RANDOM() LIMIT 1", filename
, "UP")
67 def get_by_hostname(self
, hostname
):
68 return self
._get
_mirror
("SELECT * FROM mirrors \
69 WHERE hostname = %s", hostname
)
73 def init(self
, id, data
=None):
81 return "<%s %s>" % (self
.__class
__.__name
__, self
.url
)
83 def __eq__(self
, other
):
84 if isinstance(other
, self
.__class
__):
85 return self
.id == other
.id
87 def __lt__(self
, other
):
88 if isinstance(other
, self
.__class
__):
89 return self
.hostname
< other
.hostname
93 url
= "%s://%s" % ("https" if self
.supports_https
else "http", self
.hostname
)
95 if not self
.path
.startswith("/"):
98 url
+= "%s" % self
.path
100 if not self
.path
.endswith("/"):
107 return self
.data
.hostname
111 return self
.data
.path
114 def supports_https(self
):
115 return self
.data
.supports_https
119 for addr
in self
.addresses4
:
122 for addr
in self
.addresses6
:
127 return self
.data
.owner
131 return self
.geoip
.get_location(self
.address
)
136 return self
.location
.latitude
141 return self
.location
.longitude
144 def country_code(self
):
145 return self
.data
.country_code
148 def country_name(self
):
149 return self
.geoip
.get_country_name(self
.country_code
)
153 return countries
.get_zone(self
.country_name
)
157 return self
.geoip
.get_asn(self
.address
)
161 filelist
= self
.db
.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self
.id)
162 return [f
.filename
for f
in filelist
]
168 def build_url(self
, filename
):
169 return urllib
.parse
.urljoin(self
.url
, filename
)
172 def last_update(self
):
173 return self
.data
.last_update
177 return self
.data
.state
179 def set_state(self
, state
):
180 logging
.debug("Setting state of %s to %s" % (self
.hostname
, state
))
182 if self
.state
== state
:
185 self
.db
.execute("UPDATE mirrors SET state = %s WHERE id = %s", state
, self
.id)
187 # Reload changed settings
188 self
.data
["state"] = state
192 return self
.data
.enabled
196 return not self
.enabled
198 async def check(self
):
199 logging
.debug("Running check for mirror %s" % self
.hostname
)
201 self
.db
.execute("UPDATE mirrors SET address = %s WHERE id = %s",
202 self
.address
, self
.id)
204 success
= await self
.check_timestamp()
206 await self
.check_filelist()
208 def check_state(self
, last_update
):
209 logging
.debug("Checking state of mirror %s" % self
.id)
212 self
.set_state("DOWN")
215 now
= datetime
.datetime
.utcnow()
217 time_delta
= now
- last_update
218 time_diff
= time_delta
.total_seconds()
220 time_down
= self
.settings
.get_int("mirrors_time_down", 3*24*60*60)
221 if time_diff
>= time_down
:
222 self
.set_state("DOWN")
225 time_outofsync
= self
.settings
.get_int("mirrors_time_outofsync", 6*60*60)
226 if time_diff
>= time_outofsync
:
227 self
.set_state("OUTOFSYNC")
232 async def check_timestamp(self
):
233 http
= tornado
.httpclient
.AsyncHTTPClient()
236 response
= await http
.fetch(self
.url
+ ".timestamp",
237 headers
={ "Pragma" : "no-cache" })
238 except tornado
.httpclient
.HTTPError
as e
:
239 logging
.error("Error getting timestamp from %s: %s" % (self
.hostname
, e
))
240 self
.set_state("DOWN")
243 except ssl
.SSLError
as e
:
244 logging
.error("SSL error when getting timestamp from %s: %s" % (self
.hostname
, e
))
245 self
.set_state("DOWN")
248 except tornado
.iostream
.StreamClosedError
as e
:
249 logging
.error("Connection closed unexpectedly for %s: %s" % (self
.hostname
, e
))
250 self
.set_state("DOWN")
254 logging
.error("Could not connect to %s: %s" % (self
.hostname
, e
))
255 self
.set_state("DOWN")
259 logging
.debug("Error getting timestamp from %s" % self
.hostname
)
260 self
.set_state("DOWN")
264 timestamp
= int(response
.body
.strip())
268 timestamp
= datetime
.datetime
.utcfromtimestamp(timestamp
)
270 self
.db
.execute("UPDATE mirrors SET last_update = %s WHERE id = %s",
274 self
.check_state(timestamp
)
276 logging
.debug("Successfully updated timestamp from %s" % self
.hostname
)
280 async def check_filelist(self
):
281 # XXX need to remove data from disabled mirrors
285 http
= tornado
.httpclient
.AsyncHTTPClient()
288 response
= await http
.fetch(self
.url
+ ".filelist",
289 headers
={ "Pragma" : "no-cache" })
290 except tornado
.httpclient
.HTTPError
as e
:
291 logging
.error("Error getting filelist from %s: %s" % (self
.hostname
, e
))
292 self
.set_state("DOWN")
296 logging
.debug("Error getting filelist from %s" % self
.hostname
)
299 # Drop the old filelist
300 self
.db
.execute("DELETE FROM mirror_files WHERE mirror = %s", self
.id)
303 for file in response
.body
.splitlines():
304 file = os
.path
.join(self
.prefix
, file.decode())
306 self
.db
.execute("INSERT INTO mirror_files(mirror, filename) \
307 VALUES(%s, %s)", self
.id, file)
309 logging
.debug("Successfully updated mirror filelist from %s" % self
.hostname
)
312 def development(self
):
313 return self
.data
.get("mirrorlist_devel", False)
316 def mirrorlist(self
):
317 return self
.data
.get("mirrorlist", False)
321 addrinfo
= socket
.getaddrinfo(self
.hostname
, 0, socket
.AF_UNSPEC
, socket
.SOCK_STREAM
)
324 for family
, socktype
, proto
, canonname
, address
in addrinfo
:
325 if family
== socket
.AF_INET
:
326 address
, port
= address
327 elif family
== socket
.AF_INET6
:
328 address
, port
, flowid
, scopeid
= address
329 ret
.append((family
, address
))
334 def addresses6(self
):
335 return [address
for family
, address
in self
.addresses
if family
== socket
.AF_INET6
]
338 def addresses4(self
):
339 return [address
for family
, address
in self
.addresses
if family
== socket
.AF_INET
]