]>
git.ipfire.org Git - ipfire.org.git/blob - www/webapp/backend/mirrors.py
9 import tornado
.httpclient
11 from databases
import Databases
12 from geoip
import GeoIP
13 from memcached
import Memcached
14 from misc
import Singleton
16 class Downloads(object):
17 __metaclass__
= Singleton
21 return Databases().webapp
29 ret
= self
.db
.get("SELECT COUNT(*) AS total FROM log_download")
35 ret
= self
.db
.get("SELECT COUNT(*) AS today FROM log_download WHERE date >= NOW() - 1000000")
41 ret
= self
.db
.get("SELECT COUNT(*) AS yesterday FROM log_download WHERE DATE(date) = DATE(NOW())-1")
47 ret
= self
.db
.query("SELECT DATE(date) AS date, COUNT(*) AS downloads FROM log_download"
48 " WHERE DATE(date) BETWEEN DATE(NOW()) - 31 AND DATE(NOW()) GROUP BY DATE(date)")
52 def get_countries(self
, duration
="all"):
53 query
= "SELECT country_code, count(country_code) AS count FROM log_download"
55 if duration
== "today":
56 query
+= " WHERE date >= NOW() - 1000000"
58 query
+= " GROUP BY country_code ORDER BY count DESC"
60 results
= self
.db
.query(query
)
63 count
= sum([o
.count
for o
in results
])
65 ret
[res
.country_code
] = float(res
.count
) / count
69 def get_mirror_load(self
, duration
="all"):
70 query
= "SELECT mirror, COUNT(mirror) AS count FROM log_download"
72 if duration
== "today":
73 query
+= " WHERE date >= NOW() - 1000000"
75 query
+= " GROUP BY mirror ORDER BY count DESC"
77 results
= self
.db
.query(query
)
80 count
= sum([o
.count
for o
in results
])
82 mirror
= self
.mirrors
.get(res
.mirror
)
83 ret
[mirror
.hostname
] = float(res
.count
) / count
88 class Mirrors(object):
89 __metaclass__
= Singleton
93 return Databases().webapp
100 return [Mirror(m
.id) for m
in self
.db
.query("SELECT id FROM mirrors WHERE disabled = 'N' ORDER BY state,hostname")]
103 for mirror
in self
.list():
110 return MirrorSet(self
.list())
112 def get_by_hostname(self
, hostname
):
113 mirror
= self
.db
.get("SELECT id FROM mirrors WHERE hostname=%s", hostname
)
115 return Mirror(mirror
.id)
117 def get_with_file(self
, filename
, country
=None):
118 # XXX quick and dirty solution - needs a performance boost
119 mirror_ids
= [m
.mirror
for m
in self
.db
.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename
)]
122 # # Sort out all mirrors that are not preferred to the given country
123 # for mirror in self.get_for_country(country):
124 # if not mirror.id in mirror_ids:
125 # mirror_ids.remove(mirror.id)
128 for mirror_id
in mirror_ids
:
129 mirror
= self
.get(mirror_id
)
130 if not mirror
.state
== "UP":
132 mirrors
.append(mirror
)
134 logging
.debug("%s" % mirrors
)
138 def get_for_country(self
, country
):
139 # XXX need option for random order
140 mirrors
= self
.db
.query("SELECT id FROM mirrors WHERE prefer_for_countries LIKE %s", country
)
142 for mirror
in mirrors
:
143 yield self
.get(mirror
.id)
145 def get_for_location(self
, addr
):
149 all_mirrors
= self
.list()
151 while all_mirrors
and len(mirrors
) <= 2 and distance
<= 270:
152 for mirror
in all_mirrors
:
153 if mirror
.distance_to(addr
) <= distance
:
154 mirrors
.append(mirror
)
155 all_mirrors
.remove(mirror
)
161 def get_all_files(self
):
164 for mirror
in self
.list():
165 if not mirror
.state
== "UP":
168 for file in mirror
.filelist
:
169 if not file in files
:
175 class MirrorSet(object):
176 def __init__(self
, mirrors
):
177 self
._mirrors
= mirrors
179 def __add__(self
, other
):
182 for mirror
in self
._mirrors
+ other
._mirrors
:
183 if mirror
in mirrors
:
186 mirrors
.append(mirror
)
188 return MirrorSet(mirrors
)
190 def __sub__(self
, other
):
191 mirrors
= self
._mirrors
[:]
193 for mirror
in other
._mirrors
:
194 if mirror
in mirrors
:
195 mirrors
.remove(mirror
)
197 return MirrorSet(mirrors
)
200 return iter(self
._mirrors
)
203 return len(self
._mirrors
)
206 return "<MirrorSet %s>" % ", ".join([m
.hostname
for m
in self
._mirrors
])
212 def get_with_file(self
, filename
):
213 with_file
= [m
.mirror
for m
in self
.db
.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename
)]
216 for mirror
in self
._mirrors
:
217 if mirror
.id in with_file
:
218 mirrors
.append(mirror
)
220 return MirrorSet(mirrors
)
222 def get_random(self
):
224 for mirror
in self
._mirrors
:
225 for i
in range(0, mirror
.priority
):
226 mirrors
.append(mirror
)
228 return random
.choice(mirrors
)
230 def get_for_country(self
, country
):
233 for mirror
in self
._mirrors
:
234 if country
in mirror
.prefer_for_countries
:
235 mirrors
.append(mirror
)
237 return MirrorSet(mirrors
)
239 def get_for_location(self
, addr
):
244 while len(mirrors
) <= 2 and distance
<= 270:
245 for mirror
in self
._mirrors
:
246 if mirror
in mirrors
:
249 if mirror
.distance_to(addr
) <= distance
:
250 mirrors
.append(mirror
)
254 return MirrorSet(mirrors
)
256 def get_with_state(self
, state
):
259 for mirror
in self
._mirrors
:
260 if mirror
.state
== state
:
261 mirrors
.append(mirror
)
263 return MirrorSet(mirrors
)
266 class Mirror(object):
267 def __init__(self
, id):
273 return "<%s %s>" % (self
.__class
__.__name
__, self
.url
)
275 def __cmp__(self
, other
):
276 return cmp(self
.id, other
.id)
280 return Databases().webapp
282 def reload(self
, force
=False):
283 memcached
= Memcached()
284 mem_id
= "mirror-%s" % self
.id
287 memcached
.delete(mem_id
)
289 self
._info
= memcached
.get(mem_id
)
291 self
._info
= self
.db
.get("SELECT * FROM mirrors WHERE id=%s", self
.id)
292 self
._info
["url"] = self
.generate_url()
294 memcached
.set(mem_id
, self
._info
, 60)
296 def generate_url(self
):
297 url
= "http://%s" % self
.hostname
298 if not self
.path
.startswith("/"):
300 url
+= "%s" % self
.path
301 if not self
.path
.endswith("/"):
305 def __getattr__(self
, key
):
307 return self
._info
[key
]
309 raise AttributeError(key
)
313 return socket
.gethostbyname(self
.hostname
)
317 if not hasattr(self
, "__location"):
318 self
.__location
= GeoIP().get_all(self
.address
)
320 return self
.__location
324 return self
.location
.latitude
328 return self
.location
.longitude
331 def coordinates(self
):
332 return (self
.latitude
, self
.longitude
)
335 def coordiante_str(self
):
338 for i
in self
.coordinates
:
339 coordinates
.append("%s" % i
)
341 return ",".join(coordinates
)
344 def country_code(self
):
345 return GeoIP().get_country(self
.address
).lower() or "unknown"
348 def country_name(self
):
349 return GeoIP().get_country_name(self
.country_code
)
353 if self
._info
["city"]:
354 return self
._info
["city"]
356 return self
.location
.city
359 def location_str(self
):
360 s
= self
.country_name
362 s
= "%s, %s" % (self
.city
, s
)
368 filelist
= self
.db
.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self
.id)
369 return [f
.filename
for f
in filelist
]
373 if self
.type.startswith("pakfire"):
378 def set_state(self
, state
):
379 logging
.info("Setting state of %s to %s" % (self
.hostname
, state
))
381 if self
.state
== state
:
384 self
.db
.execute("UPDATE mirrors SET state=%s WHERE id=%s",
387 # Reload changed settings
388 self
.reload(force
=True)
391 logging
.info("Running check for mirror %s" % self
.hostname
)
393 self
.check_timestamp()
394 self
.check_filelist()
396 def check_state(self
):
397 logging
.debug("Checking state of mirror %s" % self
.id)
399 if self
.disabled
== "Y":
400 self
.set_state("DOWN")
402 time_diff
= time
.time() - self
.last_update
403 if time_diff
> 3*24*60*60: # XXX get this into Settings
404 self
.set_state("DOWN")
405 elif time_diff
> 6*60*60:
406 self
.set_state("OUTOFSYNC")
410 def check_timestamp(self
):
411 if self
.releases
== "N":
414 http
= tornado
.httpclient
.AsyncHTTPClient()
416 http
.fetch(self
.url
+ ".timestamp",
417 headers
={ "Pragma" : "no-cache" },
418 callback
=self
.__check
_timestamp
_response
)
420 def __check_timestamp_response(self
, response
):
422 logging
.debug("Error getting timestamp from %s" % self
.hostname
)
423 self
.set_state("DOWN")
427 timestamp
= int(response
.body
.strip())
431 self
.db
.execute("UPDATE mirrors SET last_update=%s WHERE id=%s",
434 # Reload changed settings
435 self
.reload(force
=True)
439 logging
.info("Successfully updated timestamp from %s" % self
.hostname
)
441 def check_filelist(self
):
442 # XXX need to remove data from disabled mirrors
443 if self
.releases
== "N" or self
.disabled
== "Y" or self
.type != "full":
446 http
= tornado
.httpclient
.AsyncHTTPClient()
448 http
.fetch(self
.url
+ ".filelist",
449 headers
={ "Pragma" : "no-cache" },
450 callback
=self
.__check
_filelist
_response
)
452 def __check_filelist_response(self
, response
):
454 logging
.debug("Error getting timestamp from %s" % self
.hostname
)
457 files
= self
.filelist
459 for file in response
.body
.splitlines():
460 file = os
.path
.join(self
.prefix
, file)
466 self
.db
.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)",
470 self
.db
.execute("DELETE FROM mirror_files WHERE mirror=%s AND filename=%s",
473 logging
.info("Successfully updated mirror filelist from %s" % self
.hostname
)
476 def prefer_for_countries(self
):
477 countries
= self
._info
.get("prefer_for_countries", "")
479 return sorted(countries
.split(", "))
484 def prefer_for_countries_names(self
):
485 return sorted([GeoIP().get_country_name(c
) for c
in self
.prefer_for_countries
])
487 def distance_to(self
, addr
):
488 location
= GeoIP().get_all(addr
)
492 if location
.country_code
.lower() in self
.prefer_for_countries
:
496 self
.latitude
- location
.latitude
,
497 self
.longitude
- location
.longitude
501 for i
in distance_vector
:
504 return math
.sqrt(distance
)
506 def traffic(self
, since
):
507 # XXX needs to be done better
510 for entry
in self
.db
.query("SELECT filename, filesize FROM files"):
511 files
[entry
.filename
] = entry
.filesize
513 query
= "SELECT COUNT(filename) as count, filename FROM log_download WHERE mirror = %s"
514 query
+= " AND date >= %s GROUP BY filename"
517 for entry
in self
.db
.query(query
, self
.id, since
):
518 if files
.has_key(entry
.filename
):
519 traffic
+= entry
.count
* files
[entry
.filename
]
525 return self
._info
.get("priority", 10)