]>
git.ipfire.org Git - people/shoehn/ipfire.org.git/blob - webapp/backend/mirrors.py
3 from __future__
import division
12 import tornado
.httpclient
13 import tornado
.netutil
16 from misc
import Object
18 class Downloads(Object
):
21 ret
= self
.db
.get("SELECT COUNT(*) AS total FROM log_download")
27 ret
= self
.db
.get("SELECT COUNT(*) AS today FROM log_download WHERE date::date = NOW()::date")
33 ret
= self
.db
.get("SELECT COUNT(*) AS yesterday FROM log_download WHERE date::date = (NOW() - INTERVAL '1 day')::date")
39 ret
= self
.db
.query("WITH downloads AS (SELECT * FROM log_download \
40 WHERE DATE(date) BETWEEN (NOW()::date - INTERVAL '30 days') AND DATE(NOW())) \
41 SELECT DATE(date) AS date, COUNT(*) AS count FROM downloads \
42 GROUP BY DATE(date) ORDER BY date")
46 def get_countries(self
, duration
="all"):
47 query
= "SELECT country_code, count(country_code) AS count FROM log_download"
49 if duration
== "today":
50 query
+= " WHERE date::date = NOW()::date"
52 query
+= " GROUP BY country_code ORDER BY count DESC"
54 results
= self
.db
.query(query
)
57 count
= sum([o
.count
for o
in results
])
60 ret
.append((res
.country_code
, res
.count
/ count
))
64 def get_mirror_load(self
, duration
="all"):
65 query
= "SELECT mirror, COUNT(mirror) AS count FROM log_download"
67 if duration
== "today":
68 query
+= " WHERE date::date = NOW()::date"
70 query
+= " GROUP BY mirror ORDER BY count DESC"
72 results
= self
.db
.query(query
)
75 count
= sum([o
.count
for o
in results
])
78 mirror
= self
.mirrors
.get(res
.mirror
)
79 ret
[mirror
.hostname
] = res
.count
/ count
84 class Mirrors(Object
):
86 for mirror
in self
.get_all():
90 return Mirror(self
.backend
, id)
93 res
= self
.db
.query("SELECT * FROM mirrors WHERE enabled = %s", True)
97 mirror
= Mirror(self
.backend
, row
.id, row
)
98 mirrors
.append(mirror
)
100 return MirrorSet(self
.backend
, sorted(mirrors
))
102 def get_all_up(self
):
103 res
= self
.db
.query("SELECT * FROM mirrors WHERE enabled = %s AND state = %s \
104 ORDER BY hostname", True, "UP")
108 m
= Mirror(self
.backend
, row
.id, row
)
111 return MirrorSet(self
.backend
, mirrors
)
113 def get_by_hostname(self
, hostname
):
114 ret
= self
.db
.get("SELECT * FROM mirrors WHERE hostname = %s", hostname
)
117 return Mirror(self
.backend
, ret
.id, ret
)
119 def get_with_file(self
, filename
, country
=None):
120 # XXX quick and dirty solution - needs a performance boost
121 mirror_ids
= [m
.mirror
for m
in self
.db
.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename
)]
124 # # Sort out all mirrors that are not preferred to the given country
125 # for mirror in self.get_for_country(country):
126 # if not mirror.id in mirror_ids:
127 # mirror_ids.remove(mirror.id)
130 for mirror_id
in mirror_ids
:
131 mirror
= self
.get(mirror_id
)
132 if not mirror
.state
== "UP":
134 mirrors
.append(mirror
)
138 def get_for_location(self
, location
, max_distance
=4000, filename
=None):
143 res
= self
.db
.query("\
144 WITH client AS (SELECT point(%s, %s) AS location) \
145 SELECT * FROM mirrors WHERE mirrors.state = %s \
146 AND mirrors.id IN ( \
147 SELECT mirror FROM mirror_files WHERE filename = %s \
148 ) AND mirrors.id IN ( \
149 SELECT id FROM mirrors_locations, client \
150 WHERE geodistance(mirrors_locations.location, client.location) <= %s \
152 location
.latitude
, location
.longitude
, "UP", filename
, max_distance
)
154 res
= self
.db
.query("\
155 WITH client AS (SELECT point(%s, %s) AS location) \
156 SELECT * FROM mirrors WHERE mirrors.state = %s AND mirrors.id IN ( \
157 SELECT id FROM mirrors_locations, client \
158 WHERE geodistance(mirrors_locations.location, client.location) <= %s \
160 location
.latitude
, location
.longitude
, "UP", max_distance
)
164 mirror
= Mirror(self
.backend
, row
.id, row
)
165 mirrors
.append(mirror
)
167 return sorted(mirrors
, reverse
=True)
169 def get_all_files(self
):
172 for mirror
in self
.get_all():
173 if not mirror
.state
== "UP":
176 for file in mirror
.filelist
:
177 if not file in files
:
182 def get_random(self
, filename
=None):
184 ret
= self
.db
.get("SELECT * FROM mirrors WHERE state = %s \
185 AND mirrors.id IN (SELECT mirror FROM mirror_files \
186 WHERE filename = %s) ORDER BY RANDOM() LIMIT 1", "UP", filename
)
188 ret
= self
.db
.get("SELECT * FROM mirrors WHERE state = %s \
189 ORDER BY RANDOM() LIMIT 1", "UP")
192 return Mirror(self
.backend
, ret
.id, ret
)
194 def file_exists(self
, filename
):
195 ret
= self
.db
.get("SELECT 1 FROM mirror_files \
196 WHERE filename = %s LIMIT 1", filename
)
204 class MirrorSet(Object
):
205 def __init__(self
, backend
, mirrors
):
206 Object
.__init
__(self
, backend
)
208 self
._mirrors
= mirrors
210 def __add__(self
, other
):
213 for mirror
in self
._mirrors
+ other
._mirrors
:
214 if mirror
in mirrors
:
217 mirrors
.append(mirror
)
219 return MirrorSet(self
.backend
, mirrors
)
221 def __sub__(self
, other
):
222 mirrors
= self
._mirrors
[:]
224 for mirror
in other
._mirrors
:
225 if mirror
in mirrors
:
226 mirrors
.remove(mirror
)
228 return MirrorSet(self
.backend
, mirrors
)
231 return iter(self
._mirrors
)
234 return len(self
._mirrors
)
237 return "<MirrorSet %s>" % ", ".join([m
.hostname
for m
in self
._mirrors
])
239 def get_with_file(self
, filename
):
240 with_file
= [m
.mirror
for m
in self
.db
.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename
)]
243 for mirror
in self
._mirrors
:
244 if mirror
.id in with_file
:
245 mirrors
.append(mirror
)
247 return MirrorSet(self
.backend
, mirrors
)
249 def get_random(self
):
251 for mirror
in self
._mirrors
:
252 for i
in range(0, mirror
.priority
):
253 mirrors
.append(mirror
)
255 return random
.choice(mirrors
)
257 def get_for_location(self
, location
):
262 while len(mirrors
) <= 3 and distance
<= 8000:
263 for mirror
in self
._mirrors
:
264 if mirror
in mirrors
:
267 mirror_distance
= mirror
.distance_to(location
)
268 if mirror_distance
is None:
271 if mirror_distance
<= distance
:
272 mirrors
.append(mirror
)
276 return MirrorSet(self
.backend
, mirrors
)
278 def get_with_state(self
, state
):
281 for mirror
in self
._mirrors
:
282 if mirror
.state
== state
:
283 mirrors
.append(mirror
)
285 return MirrorSet(self
.backend
, mirrors
)
288 class Mirror(Object
):
289 def __init__(self
, backend
, id, data
=None):
290 Object
.__init
__(self
, backend
)
297 self
._info
= self
.db
.get("SELECT * FROM mirrors WHERE id = %s", self
.id)
298 self
._info
["url"] = self
.generate_url()
300 self
.__location
= None
301 self
.__country
_name
= None
304 return "<%s %s>" % (self
.__class
__.__name
__, self
.url
)
306 def __cmp__(self
, other
):
307 ret
= cmp(self
.country_code
, other
.country_code
)
310 ret
= cmp(self
.hostname
, other
.hostname
)
314 def generate_url(self
):
315 url
= "http://%s" % self
.hostname
316 if not self
.path
.startswith("/"):
318 url
+= "%s" % self
.path
319 if not self
.path
.endswith("/"):
325 return self
._info
.hostname
329 return self
._info
.path
333 for addr
in self
.addresses4
:
336 for addr
in self
.addresses6
:
341 return self
._info
.owner
345 if self
.__location
is None:
346 self
.__location
= self
.geoip
.get_location(self
.address
)
348 return self
.__location
353 return self
.location
.latitude
358 return self
.location
.longitude
361 def coordinates(self
):
362 return (self
.latitude
, self
.longitude
)
365 def coordiante_str(self
):
368 for i
in self
.coordinates
:
369 coordinates
.append("%s" % i
)
371 return ",".join(coordinates
)
374 def country_code(self
):
376 return self
.location
.country
379 def country_name(self
):
380 if self
.__country
_name
is None:
381 self
.__country
_name
= self
.geoip
.get_country_name(self
.country_code
)
383 return self
.__country
_name
386 def location_str(self
):
389 if self
._info
.location
:
390 location
.append(self
._info
.location
)
393 location
.append(self
.location
.city
)
394 location
.append(self
.country_name
)
396 return ", ".join([s
for s
in location
if s
])
400 if not hasattr(self
, "__asn"):
401 self
.__asn
= self
.geoip
.get_asn(self
.address
)
407 filelist
= self
.db
.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self
.id)
408 return [f
.filename
for f
in filelist
]
416 return self
._info
.url
418 def build_url(self
, filename
):
419 return urlparse
.urljoin(self
.url
, filename
)
422 def last_update(self
):
423 return self
._info
.last_update
427 return self
._info
.state
429 def set_state(self
, state
):
430 logging
.info("Setting state of %s to %s" % (self
.hostname
, state
))
432 if self
.state
== state
:
435 self
.db
.execute("UPDATE mirrors SET state = %s WHERE id = %s", state
, self
.id)
437 # Reload changed settings
438 if hasattr(self
, "_info"):
439 self
._info
["state"] = state
443 return self
._info
.enabled
447 return not self
.enabled
450 logging
.info("Running check for mirror %s" % self
.hostname
)
452 self
.db
.execute("UPDATE mirrors SET address = %s WHERE id = %s",
453 self
.address
, self
.id)
455 self
.check_timestamp()
456 self
.check_filelist()
458 def check_state(self
):
459 logging
.debug("Checking state of mirror %s" % self
.id)
462 self
.set_state("DOWN")
465 now
= datetime
.datetime
.utcnow()
467 time_delta
= now
- self
.last_update
468 time_diff
= time_delta
.total_seconds()
470 time_down
= self
.settings
.get_int("mirrors_time_down", 3*24*60*60)
471 if time_diff
>= time_down
:
472 self
.set_state("DOWN")
475 time_outofsync
= self
.settings
.get_int("mirrors_time_outofsync", 6*60*60)
476 if time_diff
>= time_outofsync
:
477 self
.set_state("OUTOFSYNC")
482 def check_timestamp(self
):
483 http
= tornado
.httpclient
.AsyncHTTPClient()
485 http
.fetch(self
.url
+ ".timestamp",
486 headers
={ "Pragma" : "no-cache" },
487 callback
=self
.__check
_timestamp
_response
)
489 def __check_timestamp_response(self
, response
):
491 logging
.debug("Error getting timestamp from %s" % self
.hostname
)
492 self
.set_state("DOWN")
496 timestamp
= int(response
.body
.strip())
500 timestamp
= datetime
.datetime
.utcfromtimestamp(timestamp
)
502 self
.db
.execute("UPDATE mirrors SET last_update = %s WHERE id = %s",
505 # Reload changed settings
506 if hasattr(self
, "_info"):
507 self
._info
["timestamp"] = timestamp
511 logging
.info("Successfully updated timestamp from %s" % self
.hostname
)
513 def check_filelist(self
):
514 # XXX need to remove data from disabled mirrors
518 http
= tornado
.httpclient
.AsyncHTTPClient()
520 http
.fetch(self
.url
+ ".filelist",
521 headers
={ "Pragma" : "no-cache" },
522 callback
=self
.__check
_filelist
_response
)
524 def __check_filelist_response(self
, response
):
526 logging
.debug("Error getting timestamp from %s" % self
.hostname
)
529 files
= self
.filelist
531 for file in response
.body
.splitlines():
532 file = os
.path
.join(self
.prefix
, file)
538 self
.db
.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)",
542 self
.db
.execute("DELETE FROM mirror_files WHERE mirror=%s AND filename=%s",
545 logging
.info("Successfully updated mirror filelist from %s" % self
.hostname
)
548 def prefer_for_countries(self
):
549 countries
= self
._info
.get("prefer_for_countries", "")
551 return sorted(countries
.split(", "))
556 def prefer_for_countries_names(self
):
557 countries
= [self
.geoip
.get_country_name(c
.upper()) for c
in self
.prefer_for_countries
]
559 return sorted(countries
)
561 def distance_to(self
, location
, ignore_preference
=False):
567 country_code
= location
.country
.lower()
569 if not ignore_preference
and country_code
in self
.prefer_for_countries
:
572 # http://www.movable-type.co.uk/scripts/latlong.html
574 if self
.latitude
is None:
577 if self
.longitude
is None:
581 delta_lat
= math
.radians(self
.latitude
- location
.latitude
)
582 delta_lon
= math
.radians(self
.longitude
- location
.longitude
)
584 lat1
= math
.radians(self
.latitude
)
585 lat2
= math
.radians(location
.latitude
)
587 a
= math
.sin(delta_lat
/ 2) ** 2
588 a
+= math
.cos(lat1
) * math
.cos(lat2
) * (math
.sin(delta_lon
/ 2) ** 2)
591 b2
= math
.sqrt(1 - a
)
593 c
= 2 * math
.atan2(b1
, b2
)
597 def traffic(self
, since
):
598 # XXX needs to be done better
601 for entry
in self
.db
.query("SELECT filename, filesize FROM files"):
602 files
[entry
.filename
] = entry
.filesize
604 query
= "SELECT COUNT(filename) as count, filename FROM log_download WHERE mirror = %s"
605 query
+= " AND date >= %s GROUP BY filename"
608 for entry
in self
.db
.query(query
, self
.id, since
):
609 if files
.has_key(entry
.filename
):
610 traffic
+= entry
.count
* files
[entry
.filename
]
616 return self
._info
.get("priority", 10)
619 def development(self
):
620 return self
._info
.get("mirrorlist_devel", False)
623 def mirrorlist(self
):
624 return self
._info
.get("mirrorlist", False)
628 if not hasattr(self
, "__addresses"):
630 addrinfo
= socket
.getaddrinfo(self
.hostname
, 0, socket
.AF_UNSPEC
, socket
.SOCK_STREAM
)
632 raise Exception("Could not resolve %s" % self
.hostname
)
635 for family
, socktype
, proto
, canonname
, address
in addrinfo
:
636 if family
== socket
.AF_INET
:
637 address
, port
= address
638 elif family
== socket
.AF_INET6
:
639 address
, port
, flowid
, scopeid
= address
640 ret
.append((family
, address
))
642 self
.__addresses
= ret
644 return self
.__addresses
647 def addresses6(self
):
648 return [address
for family
, address
in self
.addresses
if family
== socket
.AF_INET6
]
651 def addresses4(self
):
652 return [address
for family
, address
in self
.addresses
if family
== socket
.AF_INET
]