]>
git.ipfire.org Git - ipfire.org.git/blob - webapp/backend/mirrors.py
60bd7793f997efbc244883740bf86bc546ccd69f
3 from __future__
import division
12 import tornado
.httpclient
13 import tornado
.netutil
16 from misc
import Object
18 class Downloads(Object
):
21 ret
= self
.db
.get("SELECT COUNT(*) AS total FROM log_download")
27 ret
= self
.db
.get("SELECT COUNT(*) AS today FROM log_download WHERE date::date = NOW()::date")
33 ret
= self
.db
.get("SELECT COUNT(*) AS yesterday FROM log_download WHERE date::date = (NOW() - INTERVAL '1 day')::date")
39 ret
= self
.db
.query("WITH downloads AS (SELECT * FROM log_download \
40 WHERE DATE(date) BETWEEN (NOW()::date - INTERVAL '30 days') AND DATE(NOW())) \
41 SELECT DATE(date) AS date, COUNT(*) AS count FROM downloads \
42 GROUP BY DATE(date) ORDER BY date")
46 def get_countries(self
, duration
="all"):
47 query
= "SELECT country_code, count(country_code) AS count FROM log_download"
49 if duration
== "today":
50 query
+= " WHERE date::date = NOW()::date"
52 query
+= " GROUP BY country_code ORDER BY count DESC"
54 results
= self
.db
.query(query
)
57 count
= sum([o
.count
for o
in results
])
60 ret
.append((res
.country_code
, res
.count
/ count
))
64 def get_mirror_load(self
, duration
="all"):
65 query
= "SELECT mirror, COUNT(mirror) AS count FROM log_download"
67 if duration
== "today":
68 query
+= " WHERE date::date = NOW()::date"
70 query
+= " GROUP BY mirror ORDER BY count DESC"
72 results
= self
.db
.query(query
)
75 count
= sum([o
.count
for o
in results
])
78 mirror
= self
.mirrors
.get(res
.mirror
)
79 ret
[mirror
.hostname
] = res
.count
/ count
84 class Mirrors(Object
):
86 for mirror
in self
.get_all():
90 return Mirror(self
.backend
, id)
93 res
= self
.db
.query("SELECT * FROM mirrors WHERE enabled = %s", True)
97 mirror
= Mirror(self
.backend
, row
.id, row
)
98 mirrors
.append(mirror
)
100 return MirrorSet(self
.backend
, sorted(mirrors
))
102 def get_by_hostname(self
, hostname
):
103 ret
= self
.db
.get("SELECT * FROM mirrors WHERE hostname = %s", hostname
)
106 return Mirror(self
.backend
, ret
.id, ret
)
108 def get_with_file(self
, filename
, country
=None):
109 # XXX quick and dirty solution - needs a performance boost
110 mirror_ids
= [m
.mirror
for m
in self
.db
.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename
)]
113 # # Sort out all mirrors that are not preferred to the given country
114 # for mirror in self.get_for_country(country):
115 # if not mirror.id in mirror_ids:
116 # mirror_ids.remove(mirror.id)
119 for mirror_id
in mirror_ids
:
120 mirror
= self
.get(mirror_id
)
121 if not mirror
.state
== "UP":
123 mirrors
.append(mirror
)
127 def get_for_location(self
, location
, max_distance
=4000, filename
=None):
132 res
= self
.db
.query("\
133 WITH client AS (SELECT point(%s, %s) AS location) \
134 SELECT * FROM mirrors WHERE mirrors.state = %s \
135 AND mirrors.id IN ( \
136 SELECT mirror FROM mirror_files WHERE filename = %s \
137 ) AND mirrors.id IN ( \
138 SELECT id FROM mirrors_locations, client \
139 WHERE geodistance(mirrors_locations.location, client.location) <= %s \
141 location
.latitude
, location
.longitude
, "UP", filename
, max_distance
)
143 res
= self
.db
.query("\
144 WITH client AS (SELECT point(%s, %s) AS location) \
145 SELECT * FROM mirrors WHERE mirrors.state = %s AND mirrors.id IN ( \
146 SELECT id FROM mirrors_locations, client \
147 WHERE geodistance(mirrors_locations.location, client.location) <= %s \
149 location
.latitude
, location
.longitude
, "UP", max_distance
)
153 mirror
= Mirror(self
.backend
, row
.id, row
)
154 mirrors
.append(mirror
)
156 return sorted(mirrors
, reverse
=True)
158 def get_all_files(self
):
161 for mirror
in self
.get_all():
162 if not mirror
.state
== "UP":
165 for file in mirror
.filelist
:
166 if not file in files
:
171 def get_random(self
, filename
=None):
173 ret
= self
.db
.get("SELECT * FROM mirrors WHERE state = %s \
174 AND mirrors.id IN (SELECT mirror FROM mirror_files \
175 WHERE filename = %s) ORDER BY RANDOM() LIMIT 1", "UP", filename
)
177 ret
= self
.db
.get("SELECT * FROM mirrors WHERE state = %s \
178 ORDER BY RANDOM() LIMIT 1", "UP")
181 return Mirror(self
.backend
, ret
.id, ret
)
183 def file_exists(self
, filename
):
184 ret
= self
.db
.get("SELECT 1 FROM mirror_files \
185 WHERE filename = %s LIMIT 1", filename
)
193 class MirrorSet(Object
):
194 def __init__(self
, backend
, mirrors
):
195 Object
.__init
__(self
, backend
)
197 self
._mirrors
= mirrors
199 def __add__(self
, other
):
202 for mirror
in self
._mirrors
+ other
._mirrors
:
203 if mirror
in mirrors
:
206 mirrors
.append(mirror
)
208 return MirrorSet(self
.backend
, mirrors
)
210 def __sub__(self
, other
):
211 mirrors
= self
._mirrors
[:]
213 for mirror
in other
._mirrors
:
214 if mirror
in mirrors
:
215 mirrors
.remove(mirror
)
217 return MirrorSet(self
.backend
, mirrors
)
220 return iter(self
._mirrors
)
223 return len(self
._mirrors
)
226 return "<MirrorSet %s>" % ", ".join([m
.hostname
for m
in self
._mirrors
])
228 def get_with_file(self
, filename
):
229 with_file
= [m
.mirror
for m
in self
.db
.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename
)]
232 for mirror
in self
._mirrors
:
233 if mirror
.id in with_file
:
234 mirrors
.append(mirror
)
236 return MirrorSet(self
.backend
, mirrors
)
238 def get_random(self
):
240 for mirror
in self
._mirrors
:
241 for i
in range(0, mirror
.priority
):
242 mirrors
.append(mirror
)
244 return random
.choice(mirrors
)
246 def get_for_location(self
, location
):
251 while len(mirrors
) <= 3 and distance
<= 8000:
252 for mirror
in self
._mirrors
:
253 if mirror
in mirrors
:
256 mirror_distance
= mirror
.distance_to(location
)
257 if mirror_distance
is None:
260 if mirror_distance
<= distance
:
261 mirrors
.append(mirror
)
265 return MirrorSet(self
.backend
, mirrors
)
267 def get_with_state(self
, state
):
270 for mirror
in self
._mirrors
:
271 if mirror
.state
== state
:
272 mirrors
.append(mirror
)
274 return MirrorSet(self
.backend
, mirrors
)
277 class Mirror(Object
):
278 def __init__(self
, backend
, id, data
=None):
279 Object
.__init
__(self
, backend
)
286 self
._info
= self
.db
.get("SELECT * FROM mirrors WHERE id = %s", self
.id)
287 self
._info
["url"] = self
.generate_url()
289 self
.__location
= None
290 self
.__country
_name
= None
293 return "<%s %s>" % (self
.__class
__.__name
__, self
.url
)
295 def __cmp__(self
, other
):
296 ret
= cmp(self
.country_code
, other
.country_code
)
299 ret
= cmp(self
.hostname
, other
.hostname
)
303 def generate_url(self
):
304 url
= "%s://%s" % ("https" if self
.supports_https
else "http", self
.hostname
)
305 if not self
.path
.startswith("/"):
307 url
+= "%s" % self
.path
308 if not self
.path
.endswith("/"):
314 return self
._info
.hostname
318 return self
._info
.path
321 def supports_https(self
):
322 return self
._info
.supports_https
326 for addr
in self
.addresses4
:
329 for addr
in self
.addresses6
:
334 return self
._info
.owner
338 if self
.__location
is None:
339 self
.__location
= self
.geoip
.get_location(self
.address
)
341 return self
.__location
346 return self
.location
.latitude
351 return self
.location
.longitude
354 def coordinates(self
):
355 return (self
.latitude
, self
.longitude
)
358 def coordiante_str(self
):
361 for i
in self
.coordinates
:
362 coordinates
.append("%s" % i
)
364 return ",".join(coordinates
)
367 def country_code(self
):
369 return self
.location
.country
372 def country_name(self
):
373 if self
.__country
_name
is None:
374 self
.__country
_name
= self
.geoip
.get_country_name(self
.country_code
)
376 return self
.__country
_name
379 def location_str(self
):
382 if self
._info
.location
:
383 location
.append(self
._info
.location
)
386 location
.append(self
.location
.city
)
387 location
.append(self
.country_name
)
389 return ", ".join([s
for s
in location
if s
])
393 if not hasattr(self
, "__asn"):
394 self
.__asn
= self
.geoip
.get_asn(self
.address
)
400 filelist
= self
.db
.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self
.id)
401 return [f
.filename
for f
in filelist
]
409 return self
._info
.url
411 def build_url(self
, filename
):
412 return urlparse
.urljoin(self
.url
, filename
)
415 def last_update(self
):
416 return self
._info
.last_update
420 return self
._info
.state
422 def set_state(self
, state
):
423 logging
.info("Setting state of %s to %s" % (self
.hostname
, state
))
425 if self
.state
== state
:
428 self
.db
.execute("UPDATE mirrors SET state = %s WHERE id = %s", state
, self
.id)
430 # Reload changed settings
431 if hasattr(self
, "_info"):
432 self
._info
["state"] = state
436 return self
._info
.enabled
440 return not self
.enabled
443 logging
.info("Running check for mirror %s" % self
.hostname
)
445 self
.db
.execute("UPDATE mirrors SET address = %s WHERE id = %s",
446 self
.address
, self
.id)
448 self
.check_timestamp()
449 self
.check_filelist()
451 def check_state(self
):
452 logging
.debug("Checking state of mirror %s" % self
.id)
455 self
.set_state("DOWN")
458 now
= datetime
.datetime
.utcnow()
460 time_delta
= now
- self
.last_update
461 time_diff
= time_delta
.total_seconds()
463 time_down
= self
.settings
.get_int("mirrors_time_down", 3*24*60*60)
464 if time_diff
>= time_down
:
465 self
.set_state("DOWN")
468 time_outofsync
= self
.settings
.get_int("mirrors_time_outofsync", 6*60*60)
469 if time_diff
>= time_outofsync
:
470 self
.set_state("OUTOFSYNC")
475 def check_timestamp(self
):
476 http
= tornado
.httpclient
.AsyncHTTPClient()
478 http
.fetch(self
.url
+ ".timestamp",
479 headers
={ "Pragma" : "no-cache" },
480 callback
=self
.__check
_timestamp
_response
)
482 def __check_timestamp_response(self
, response
):
484 logging
.debug("Error getting timestamp from %s" % self
.hostname
)
485 self
.set_state("DOWN")
489 timestamp
= int(response
.body
.strip())
493 timestamp
= datetime
.datetime
.utcfromtimestamp(timestamp
)
495 self
.db
.execute("UPDATE mirrors SET last_update = %s WHERE id = %s",
498 # Reload changed settings
499 if hasattr(self
, "_info"):
500 self
._info
["timestamp"] = timestamp
504 logging
.info("Successfully updated timestamp from %s" % self
.hostname
)
506 def check_filelist(self
):
507 # XXX need to remove data from disabled mirrors
511 http
= tornado
.httpclient
.AsyncHTTPClient()
513 http
.fetch(self
.url
+ ".filelist",
514 headers
={ "Pragma" : "no-cache" },
515 callback
=self
.__check
_filelist
_response
)
517 def __check_filelist_response(self
, response
):
519 logging
.debug("Error getting timestamp from %s" % self
.hostname
)
522 files
= self
.filelist
524 for file in response
.body
.splitlines():
525 file = os
.path
.join(self
.prefix
, file)
531 self
.db
.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)",
535 self
.db
.execute("DELETE FROM mirror_files WHERE mirror=%s AND filename=%s",
538 logging
.info("Successfully updated mirror filelist from %s" % self
.hostname
)
541 def prefer_for_countries(self
):
542 countries
= self
._info
.get("prefer_for_countries", "")
544 return sorted(countries
.split(", "))
549 def prefer_for_countries_names(self
):
550 countries
= [self
.geoip
.get_country_name(c
.upper()) for c
in self
.prefer_for_countries
]
552 return sorted(countries
)
554 def distance_to(self
, location
, ignore_preference
=False):
560 country_code
= location
.country
.lower()
562 if not ignore_preference
and country_code
in self
.prefer_for_countries
:
565 # http://www.movable-type.co.uk/scripts/latlong.html
567 if self
.latitude
is None:
570 if self
.longitude
is None:
574 delta_lat
= math
.radians(self
.latitude
- location
.latitude
)
575 delta_lon
= math
.radians(self
.longitude
- location
.longitude
)
577 lat1
= math
.radians(self
.latitude
)
578 lat2
= math
.radians(location
.latitude
)
580 a
= math
.sin(delta_lat
/ 2) ** 2
581 a
+= math
.cos(lat1
) * math
.cos(lat2
) * (math
.sin(delta_lon
/ 2) ** 2)
584 b2
= math
.sqrt(1 - a
)
586 c
= 2 * math
.atan2(b1
, b2
)
590 def traffic(self
, since
):
591 # XXX needs to be done better
594 for entry
in self
.db
.query("SELECT filename, filesize FROM files"):
595 files
[entry
.filename
] = entry
.filesize
597 query
= "SELECT COUNT(filename) as count, filename FROM log_download WHERE mirror = %s"
598 query
+= " AND date >= %s GROUP BY filename"
601 for entry
in self
.db
.query(query
, self
.id, since
):
602 if files
.has_key(entry
.filename
):
603 traffic
+= entry
.count
* files
[entry
.filename
]
609 return self
._info
.get("priority", 10)
612 def development(self
):
613 return self
._info
.get("mirrorlist_devel", False)
616 def mirrorlist(self
):
617 return self
._info
.get("mirrorlist", False)
621 if not hasattr(self
, "__addresses"):
623 addrinfo
= socket
.getaddrinfo(self
.hostname
, 0, socket
.AF_UNSPEC
, socket
.SOCK_STREAM
)
625 raise Exception("Could not resolve %s" % self
.hostname
)
628 for family
, socktype
, proto
, canonname
, address
in addrinfo
:
629 if family
== socket
.AF_INET
:
630 address
, port
= address
631 elif family
== socket
.AF_INET6
:
632 address
, port
, flowid
, scopeid
= address
633 ret
.append((family
, address
))
635 self
.__addresses
= ret
637 return self
.__addresses
640 def addresses6(self
):
641 return [address
for family
, address
in self
.addresses
if family
== socket
.AF_INET6
]
644 def addresses4(self
):
645 return [address
for family
, address
in self
.addresses
if family
== socket
.AF_INET
]