]>
git.ipfire.org Git - people/shoehn/ipfire.org.git/blob - webapp/backend/mirrors.py
3 from __future__
import division
12 import tornado
.httpclient
13 import tornado
.netutil
16 from misc
import Object
18 class Downloads(Object
):
21 ret
= self
.db
.get("SELECT COUNT(*) AS total FROM log_download")
27 ret
= self
.db
.get("SELECT COUNT(*) AS today FROM log_download WHERE date::date = NOW()::date")
33 ret
= self
.db
.get("SELECT COUNT(*) AS yesterday FROM log_download WHERE date::date = (NOW() - INTERVAL '1 day')::date")
39 ret
= self
.db
.query("SELECT date::date AS date, COUNT(*) AS downloads FROM log_download"
40 " WHERE date::date BETWEEN (NOW() - INTERVAL '30 days')::date AND NOW()::date GROUP BY date::date")
44 def get_countries(self
, duration
="all"):
45 query
= "SELECT country_code, count(country_code) AS count FROM log_download"
47 if duration
== "today":
48 query
+= " WHERE date::date = NOW()::date"
50 query
+= " GROUP BY country_code ORDER BY count DESC"
52 results
= self
.db
.query(query
)
55 count
= sum([o
.count
for o
in results
])
58 ret
[res
.country_code
] = res
.count
/ count
62 def get_mirror_load(self
, duration
="all"):
63 query
= "SELECT mirror, COUNT(mirror) AS count FROM log_download"
65 if duration
== "today":
66 query
+= " WHERE date::date = NOW()::date"
68 query
+= " GROUP BY mirror ORDER BY count DESC"
70 results
= self
.db
.query(query
)
73 count
= sum([o
.count
for o
in results
])
76 mirror
= self
.mirrors
.get(res
.mirror
)
77 ret
[mirror
.hostname
] = res
.count
/ count
82 class Mirrors(Object
):
84 for mirror
in self
.get_all():
88 return Mirror(self
.backend
, id)
91 res
= self
.db
.query("SELECT * FROM mirrors WHERE enabled = %s", True)
95 mirror
= Mirror(self
.backend
, row
.id, row
)
96 mirrors
.append(mirror
)
98 return MirrorSet(self
.backend
, sorted(mirrors
))
100 def get_all_up(self
):
101 res
= self
.db
.query("SELECT * FROM mirrors WHERE enabled = %s AND state = %s \
102 ORDER BY hostname", True, "UP")
106 m
= Mirror(self
.backend
, row
.id, row
)
109 return MirrorSet(self
.backend
, mirrors
)
111 def get_by_hostname(self
, hostname
):
112 mirror
= self
.db
.get("SELECT id FROM mirrors WHERE hostname=%s", hostname
)
114 return Mirror(self
.backend
, mirror
.id)
116 def get_with_file(self
, filename
, country
=None):
117 # XXX quick and dirty solution - needs a performance boost
118 mirror_ids
= [m
.mirror
for m
in self
.db
.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename
)]
121 # # Sort out all mirrors that are not preferred to the given country
122 # for mirror in self.get_for_country(country):
123 # if not mirror.id in mirror_ids:
124 # mirror_ids.remove(mirror.id)
127 for mirror_id
in mirror_ids
:
128 mirror
= self
.get(mirror_id
)
129 if not mirror
.state
== "UP":
131 mirrors
.append(mirror
)
135 def get_for_country(self
, country
):
136 # XXX need option for random order
137 mirrors
= self
.db
.query("SELECT id FROM mirrors WHERE prefer_for_countries LIKE %s", country
)
139 for mirror
in mirrors
:
140 yield self
.get(mirror
.id)
142 def get_for_location(self
, location
):
149 all_mirrors
= self
.get_all()
151 while all_mirrors
and len(mirrors
) <= 3 and distance
<= 8000:
152 for mirror
in all_mirrors
:
153 mirror_distance
= mirror
.distance_to(location
)
154 if mirror_distance
is None:
157 if mirror_distance
<= distance
:
158 mirrors
.append(mirror
)
159 all_mirrors
.remove(mirror
)
165 def get_all_files(self
):
168 for mirror
in self
.get_all():
169 if not mirror
.state
== "UP":
172 for file in mirror
.filelist
:
173 if not file in files
:
179 class MirrorSet(Object
):
180 def __init__(self
, backend
, mirrors
):
181 Object
.__init
__(self
, backend
)
183 self
._mirrors
= mirrors
185 def __add__(self
, other
):
188 for mirror
in self
._mirrors
+ other
._mirrors
:
189 if mirror
in mirrors
:
192 mirrors
.append(mirror
)
194 return MirrorSet(self
.backend
, mirrors
)
196 def __sub__(self
, other
):
197 mirrors
= self
._mirrors
[:]
199 for mirror
in other
._mirrors
:
200 if mirror
in mirrors
:
201 mirrors
.remove(mirror
)
203 return MirrorSet(self
.backend
, mirrors
)
206 return iter(self
._mirrors
)
209 return len(self
._mirrors
)
212 return "<MirrorSet %s>" % ", ".join([m
.hostname
for m
in self
._mirrors
])
214 def get_with_file(self
, filename
):
215 with_file
= [m
.mirror
for m
in self
.db
.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename
)]
218 for mirror
in self
._mirrors
:
219 if mirror
.id in with_file
:
220 mirrors
.append(mirror
)
222 return MirrorSet(self
.backend
, mirrors
)
224 def get_random(self
):
226 for mirror
in self
._mirrors
:
227 for i
in range(0, mirror
.priority
):
228 mirrors
.append(mirror
)
230 return random
.choice(mirrors
)
232 def get_for_country(self
, country
):
235 for mirror
in self
._mirrors
:
236 if country
in mirror
.prefer_for_countries
:
237 mirrors
.append(mirror
)
239 return MirrorSet(self
.backend
, mirrors
)
241 def get_for_location(self
, location
):
246 while len(mirrors
) <= 3 and distance
<= 8000:
247 for mirror
in self
._mirrors
:
248 if mirror
in mirrors
:
251 mirror_distance
= mirror
.distance_to(location
)
252 if mirror_distance
is None:
255 if mirror_distance
<= distance
:
256 mirrors
.append(mirror
)
260 return MirrorSet(self
.backend
, mirrors
)
262 def get_with_state(self
, state
):
265 for mirror
in self
._mirrors
:
266 if mirror
.state
== state
:
267 mirrors
.append(mirror
)
269 return MirrorSet(self
.backend
, mirrors
)
272 class Mirror(Object
):
273 def __init__(self
, backend
, id, data
=None):
274 Object
.__init
__(self
, backend
)
281 self
._info
= self
.db
.get("SELECT * FROM mirrors WHERE id = %s", self
.id)
282 self
._info
["url"] = self
.generate_url()
284 self
.__location
= None
285 self
.__country
_name
= None
288 return "<%s %s>" % (self
.__class
__.__name
__, self
.url
)
290 def __cmp__(self
, other
):
291 ret
= cmp(self
.country_code
, other
.country_code
)
294 ret
= cmp(self
.hostname
, other
.hostname
)
298 def generate_url(self
):
299 url
= "http://%s" % self
.hostname
300 if not self
.path
.startswith("/"):
302 url
+= "%s" % self
.path
303 if not self
.path
.endswith("/"):
309 return self
._info
.hostname
313 return self
._info
.path
317 return socket
.gethostbyname(self
.hostname
)
321 return self
._info
.owner
325 if self
.__location
is None:
326 self
.__location
= self
.geoip
.get_location(self
.address
)
328 return self
.__location
333 return self
.location
.latitude
338 return self
.location
.longitude
341 def coordinates(self
):
342 return (self
.latitude
, self
.longitude
)
345 def coordiante_str(self
):
348 for i
in self
.coordinates
:
349 coordinates
.append("%s" % i
)
351 return ",".join(coordinates
)
354 def country_code(self
):
356 return self
.location
.country
359 def country_name(self
):
360 if self
.__country
_name
is None:
361 self
.__country
_name
= self
.geoip
.get_country_name(self
.country_code
)
363 return self
.__country
_name
366 def location_str(self
):
369 if self
._info
.location
:
370 location
.append(self
._info
.location
)
373 location
.append(self
.location
.city
)
374 location
.append(self
.country_name
)
376 return ", ".join([s
for s
in location
if s
])
380 if not hasattr(self
, "__asn"):
381 self
.__asn
= self
.geoip
.get_asn(self
.address
)
387 filelist
= self
.db
.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self
.id)
388 return [f
.filename
for f
in filelist
]
396 return self
._info
.url
398 def build_url(self
, filename
):
399 return urlparse
.urljoin(self
.url
, filename
)
402 def last_update(self
):
403 return self
._info
.last_update
407 return self
._info
.state
409 def set_state(self
, state
):
410 logging
.info("Setting state of %s to %s" % (self
.hostname
, state
))
412 if self
.state
== state
:
415 self
.db
.execute("UPDATE mirrors SET state = %s WHERE id = %s", state
, self
.id)
417 # Reload changed settings
418 if hasattr(self
, "_info"):
419 self
._info
["state"] = state
423 return self
._info
.enabled
427 return not self
.enabled
430 logging
.info("Running check for mirror %s" % self
.hostname
)
432 self
.check_timestamp()
433 self
.check_filelist()
435 def check_state(self
):
436 logging
.debug("Checking state of mirror %s" % self
.id)
439 self
.set_state("DOWN")
442 now
= datetime
.datetime
.utcnow()
444 time_delta
= now
- self
.last_update
445 time_diff
= time_delta
.total_seconds()
447 time_down
= self
.settings
.get_int("mirrors_time_down", 3*24*60*60)
448 if time_diff
>= time_down
:
449 self
.set_state("DOWN")
452 time_outofsync
= self
.settings
.get_int("mirrors_time_outofsync", 6*60*60)
453 if time_diff
>= time_outofsync
:
454 self
.set_state("OUTOFSYNC")
459 def check_timestamp(self
):
460 http
= tornado
.httpclient
.AsyncHTTPClient()
462 http
.fetch(self
.url
+ ".timestamp",
463 headers
={ "Pragma" : "no-cache" },
464 callback
=self
.__check
_timestamp
_response
)
466 def __check_timestamp_response(self
, response
):
468 logging
.debug("Error getting timestamp from %s" % self
.hostname
)
469 self
.set_state("DOWN")
473 timestamp
= int(response
.body
.strip())
477 timestamp
= datetime
.datetime
.utcfromtimestamp(timestamp
)
479 self
.db
.execute("UPDATE mirrors SET last_update = %s WHERE id = %s",
482 # Reload changed settings
483 if hasattr(self
, "_info"):
484 self
._info
["timestamp"] = timestamp
488 logging
.info("Successfully updated timestamp from %s" % self
.hostname
)
490 def check_filelist(self
):
491 # XXX need to remove data from disabled mirrors
495 http
= tornado
.httpclient
.AsyncHTTPClient()
497 http
.fetch(self
.url
+ ".filelist",
498 headers
={ "Pragma" : "no-cache" },
499 callback
=self
.__check
_filelist
_response
)
501 def __check_filelist_response(self
, response
):
503 logging
.debug("Error getting timestamp from %s" % self
.hostname
)
506 files
= self
.filelist
508 for file in response
.body
.splitlines():
509 file = os
.path
.join(self
.prefix
, file)
515 self
.db
.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)",
519 self
.db
.execute("DELETE FROM mirror_files WHERE mirror=%s AND filename=%s",
522 logging
.info("Successfully updated mirror filelist from %s" % self
.hostname
)
525 def prefer_for_countries(self
):
526 countries
= self
._info
.get("prefer_for_countries", "")
528 return sorted(countries
.split(", "))
533 def prefer_for_countries_names(self
):
534 countries
= [self
.geoip
.get_country_name(c
.upper()) for c
in self
.prefer_for_countries
]
536 return sorted(countries
)
538 def distance_to(self
, location
, ignore_preference
=False):
544 country_code
= location
.country
.lower()
546 if not ignore_preference
and country_code
in self
.prefer_for_countries
:
549 # http://www.movable-type.co.uk/scripts/latlong.html
551 if self
.latitude
is None:
554 if self
.longitude
is None:
558 delta_lat
= math
.radians(self
.latitude
- location
.latitude
)
559 delta_lon
= math
.radians(self
.longitude
- location
.longitude
)
561 lat1
= math
.radians(self
.latitude
)
562 lat2
= math
.radians(location
.latitude
)
564 a
= math
.sin(delta_lat
/ 2) ** 2
565 a
+= math
.cos(lat1
) * math
.cos(lat2
) * (math
.sin(delta_lon
/ 2) ** 2)
568 b2
= math
.sqrt(1 - a
)
570 c
= 2 * math
.atan2(b1
, b2
)
574 def traffic(self
, since
):
575 # XXX needs to be done better
578 for entry
in self
.db
.query("SELECT filename, filesize FROM files"):
579 files
[entry
.filename
] = entry
.filesize
581 query
= "SELECT COUNT(filename) as count, filename FROM log_download WHERE mirror = %s"
582 query
+= " AND date >= %s GROUP BY filename"
585 for entry
in self
.db
.query(query
, self
.id, since
):
586 if files
.has_key(entry
.filename
):
587 traffic
+= entry
.count
* files
[entry
.filename
]
593 return self
._info
.get("priority", 10)
596 def development(self
):
597 return self
._info
.get("mirrorlist_devel", False)
600 def mirrorlist(self
):
601 return self
._info
.get("mirrorlist", False)
605 if not hasattr(self
, "__addresses"):
606 addrinfo
= socket
.getaddrinfo(self
.hostname
, 0, socket
.AF_UNSPEC
, socket
.SOCK_STREAM
)
609 for family
, socktype
, proto
, canonname
, address
in addrinfo
:
610 if family
== socket
.AF_INET
:
611 address
, port
= address
612 elif family
== socket
.AF_INET6
:
613 address
, port
, flowid
, scopeid
= address
614 ret
.append((family
, address
))
616 self
.__addresses
= ret
618 return self
.__addresses
621 def addresses6(self
):
622 return [address
for family
, address
in self
.addresses
if family
== socket
.AF_INET6
]
625 def addresses4(self
):
626 return [address
for family
, address
in self
.addresses
if family
== socket
.AF_INET
]