]>
git.ipfire.org Git - ipfire.org.git/blob - webapp/backend/mirrors.py
260cff0d0a425340019b425c13a2b2edec31256a
9 import tornado
.httpclient
11 from databases
import Databases
12 from geoip
import GeoIP
13 from memcached
import Memcached
14 from misc
import Singleton
16 class Downloads(object):
17 __metaclass__
= Singleton
21 return Databases().webapp
29 ret
= self
.db
.get("SELECT COUNT(*) AS total FROM log_download")
35 ret
= self
.db
.get("SELECT COUNT(*) AS today FROM log_download WHERE date >= NOW() - 1000000")
41 ret
= self
.db
.get("SELECT COUNT(*) AS yesterday FROM log_download WHERE DATE(date) = DATE(NOW())-1")
47 ret
= self
.db
.query("SELECT DATE(date) AS date, COUNT(*) AS downloads FROM log_download"
48 " WHERE DATE(date) BETWEEN DATE(NOW()) - 31 AND DATE(NOW()) GROUP BY DATE(date)")
52 def get_countries(self
, duration
="all"):
53 query
= "SELECT country_code, count(country_code) AS count FROM log_download"
55 if duration
== "today":
56 query
+= " WHERE date >= NOW() - 1000000"
58 query
+= " GROUP BY country_code ORDER BY count DESC"
60 results
= self
.db
.query(query
)
63 count
= sum([o
.count
for o
in results
])
65 ret
[res
.country_code
] = float(res
.count
) / count
69 def get_mirror_load(self
, duration
="all"):
70 query
= "SELECT mirror, COUNT(mirror) AS count FROM log_download"
72 if duration
== "today":
73 query
+= " WHERE date >= NOW() - 1000000"
75 query
+= " GROUP BY mirror ORDER BY count DESC"
77 results
= self
.db
.query(query
)
80 count
= sum([o
.count
for o
in results
])
82 mirror
= self
.mirrors
.get(res
.mirror
)
83 ret
[mirror
.hostname
] = float(res
.count
) / count
88 class Mirrors(object):
89 __metaclass__
= Singleton
93 return Databases().webapp
100 return [Mirror(m
.id) for m
in self
.db
.query("SELECT id FROM mirrors WHERE disabled = 'N' ORDER BY state,hostname")]
103 for mirror
in self
.list():
110 return MirrorSet(self
.list())
112 def get_all_up(self
):
113 res
= self
.db
.query("SELECT * FROM mirrors WHERE disabled = %s AND state = %s ORDER BY hostname", "N", "UP")
117 m
= Mirror(row
.id, row
)
120 return MirrorSet(mirrors
)
122 def get_by_hostname(self
, hostname
):
123 mirror
= self
.db
.get("SELECT id FROM mirrors WHERE hostname=%s", hostname
)
125 return Mirror(mirror
.id)
127 def get_with_file(self
, filename
, country
=None):
128 # XXX quick and dirty solution - needs a performance boost
129 mirror_ids
= [m
.mirror
for m
in self
.db
.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename
)]
132 # # Sort out all mirrors that are not preferred to the given country
133 # for mirror in self.get_for_country(country):
134 # if not mirror.id in mirror_ids:
135 # mirror_ids.remove(mirror.id)
138 for mirror_id
in mirror_ids
:
139 mirror
= self
.get(mirror_id
)
140 if not mirror
.state
== "UP":
142 mirrors
.append(mirror
)
144 logging
.debug("%s" % mirrors
)
148 def get_for_country(self
, country
):
149 # XXX need option for random order
150 mirrors
= self
.db
.query("SELECT id FROM mirrors WHERE prefer_for_countries LIKE %s", country
)
152 for mirror
in mirrors
:
153 yield self
.get(mirror
.id)
155 def get_for_location(self
, addr
):
159 all_mirrors
= self
.list()
161 location
= GeoIP().get_all(addr
)
165 while all_mirrors
and len(mirrors
) <= 2 and distance
<= 270:
166 for mirror
in all_mirrors
:
167 if mirror
.distance_to(location
) <= distance
:
168 mirrors
.append(mirror
)
169 all_mirrors
.remove(mirror
)
175 def get_all_files(self
):
178 for mirror
in self
.list():
179 if not mirror
.state
== "UP":
182 for file in mirror
.filelist
:
183 if not file in files
:
189 class MirrorSet(object):
190 def __init__(self
, mirrors
):
191 self
._mirrors
= mirrors
193 def __add__(self
, other
):
196 for mirror
in self
._mirrors
+ other
._mirrors
:
197 if mirror
in mirrors
:
200 mirrors
.append(mirror
)
202 return MirrorSet(mirrors
)
204 def __sub__(self
, other
):
205 mirrors
= self
._mirrors
[:]
207 for mirror
in other
._mirrors
:
208 if mirror
in mirrors
:
209 mirrors
.remove(mirror
)
211 return MirrorSet(mirrors
)
214 return iter(self
._mirrors
)
217 return len(self
._mirrors
)
220 return "<MirrorSet %s>" % ", ".join([m
.hostname
for m
in self
._mirrors
])
226 def get_with_file(self
, filename
):
227 with_file
= [m
.mirror
for m
in self
.db
.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename
)]
230 for mirror
in self
._mirrors
:
231 if mirror
.id in with_file
:
232 mirrors
.append(mirror
)
234 return MirrorSet(mirrors
)
236 def get_random(self
):
238 for mirror
in self
._mirrors
:
239 for i
in range(0, mirror
.priority
):
240 mirrors
.append(mirror
)
242 return random
.choice(mirrors
)
244 def get_for_country(self
, country
):
247 for mirror
in self
._mirrors
:
248 if country
in mirror
.prefer_for_countries
:
249 mirrors
.append(mirror
)
251 return MirrorSet(mirrors
)
253 def get_for_location(self
, addr
):
258 location
= GeoIP().get_all(addr
)
260 while len(mirrors
) <= 2 and distance
<= 270:
261 for mirror
in self
._mirrors
:
262 if mirror
in mirrors
:
265 if mirror
.distance_to(location
) <= distance
:
266 mirrors
.append(mirror
)
270 return MirrorSet(mirrors
)
272 def get_with_state(self
, state
):
275 for mirror
in self
._mirrors
:
276 if mirror
.state
== state
:
277 mirrors
.append(mirror
)
279 return MirrorSet(mirrors
)
282 class Mirror(object):
283 def __init__(self
, id, data
=None):
289 self
._info
= self
.db
.get("SELECT * FROM mirrors WHERE id = %s", self
.id)
290 self
._info
["url"] = self
.generate_url()
292 self
.__location
= None
293 self
.__country
_name
= None
296 return "<%s %s>" % (self
.__class
__.__name
__, self
.url
)
298 def __cmp__(self
, other
):
299 return cmp(self
.id, other
.id)
303 return Databases().webapp
305 def generate_url(self
):
306 url
= "http://%s" % self
.hostname
307 if not self
.path
.startswith("/"):
309 url
+= "%s" % self
.path
310 if not self
.path
.endswith("/"):
314 def __getattr__(self
, key
):
316 return self
._info
[key
]
318 raise AttributeError(key
)
322 return socket
.gethostbyname(self
.hostname
)
326 if self
.__location
is None:
327 self
.__location
= GeoIP().get_all(self
.address
)
329 return self
.__location
333 return self
.location
.latitude
337 return self
.location
.longitude
340 def coordinates(self
):
341 return (self
.latitude
, self
.longitude
)
344 def coordiante_str(self
):
347 for i
in self
.coordinates
:
348 coordinates
.append("%s" % i
)
350 return ",".join(coordinates
)
353 def country_code(self
):
354 return self
.location
.country_code
.lower() or "unknown"
357 def country_name(self
):
358 if self
.__country
_name
is None:
359 self
.__country
_name
= GeoIP().get_country_name(self
.country_code
)
361 return self
.__country
_name
365 if self
._info
["city"]:
366 return self
._info
["city"]
368 return self
.location
.city
371 def location_str(self
):
372 s
= self
.country_name
374 s
= "%s, %s" % (self
.city
, s
)
380 filelist
= self
.db
.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self
.id)
381 return [f
.filename
for f
in filelist
]
385 if self
.type.startswith("pakfire"):
390 def set_state(self
, state
):
391 logging
.info("Setting state of %s to %s" % (self
.hostname
, state
))
393 if self
.state
== state
:
396 self
.db
.execute("UPDATE mirrors SET state=%s WHERE id=%s",
399 # Reload changed settings
400 if hasattr(self
, "_info"):
401 self
._info
["state"] = state
404 logging
.info("Running check for mirror %s" % self
.hostname
)
406 self
.check_timestamp()
407 self
.check_filelist()
409 def check_state(self
):
410 logging
.debug("Checking state of mirror %s" % self
.id)
412 if self
.disabled
== "Y":
413 self
.set_state("DOWN")
415 time_diff
= time
.time() - self
.last_update
416 if time_diff
> 3*24*60*60: # XXX get this into Settings
417 self
.set_state("DOWN")
418 elif time_diff
> 6*60*60:
419 self
.set_state("OUTOFSYNC")
423 def check_timestamp(self
):
424 if not self
.type == "full":
427 http
= tornado
.httpclient
.AsyncHTTPClient()
429 http
.fetch(self
.url
+ ".timestamp",
430 headers
={ "Pragma" : "no-cache" },
431 callback
=self
.__check
_timestamp
_response
)
433 def __check_timestamp_response(self
, response
):
435 logging
.debug("Error getting timestamp from %s" % self
.hostname
)
436 self
.set_state("DOWN")
440 timestamp
= int(response
.body
.strip())
444 self
.db
.execute("UPDATE mirrors SET last_update=%s WHERE id=%s",
447 # Reload changed settings
448 if hasattr(self
, "_info"):
449 self
._info
["timestamp"] = timestamp
453 logging
.info("Successfully updated timestamp from %s" % self
.hostname
)
455 def check_filelist(self
):
456 # XXX need to remove data from disabled mirrors
457 if self
.disabled
== "Y" or self
.type != "full":
460 http
= tornado
.httpclient
.AsyncHTTPClient()
462 http
.fetch(self
.url
+ ".filelist",
463 headers
={ "Pragma" : "no-cache" },
464 callback
=self
.__check
_filelist
_response
)
466 def __check_filelist_response(self
, response
):
468 logging
.debug("Error getting timestamp from %s" % self
.hostname
)
471 files
= self
.filelist
473 for file in response
.body
.splitlines():
474 file = os
.path
.join(self
.prefix
, file)
480 self
.db
.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)",
484 self
.db
.execute("DELETE FROM mirror_files WHERE mirror=%s AND filename=%s",
487 logging
.info("Successfully updated mirror filelist from %s" % self
.hostname
)
490 def prefer_for_countries(self
):
491 countries
= self
._info
.get("prefer_for_countries", "")
493 return sorted(countries
.split(", "))
498 def prefer_for_countries_names(self
):
499 return sorted([GeoIP().get_country_name(c
) for c
in self
.prefer_for_countries
])
501 def distance_to(self
, location
, ignore_preference
=False):
505 if not ignore_preference
and location
.country_code
.lower() in self
.prefer_for_countries
:
509 self
.latitude
- location
.latitude
,
510 self
.longitude
- location
.longitude
514 for i
in distance_vector
:
517 return math
.sqrt(distance
)
519 def traffic(self
, since
):
520 # XXX needs to be done better
523 for entry
in self
.db
.query("SELECT filename, filesize FROM files"):
524 files
[entry
.filename
] = entry
.filesize
526 query
= "SELECT COUNT(filename) as count, filename FROM log_download WHERE mirror = %s"
527 query
+= " AND date >= %s GROUP BY filename"
530 for entry
in self
.db
.query(query
, self
.id, since
):
531 if files
.has_key(entry
.filename
):
532 traffic
+= entry
.count
* files
[entry
.filename
]
538 return self
._info
.get("priority", 10)
540 def is_pakfire2(self
):
541 return self
.type in ("full", "pakfire2")
544 def development(self
):
545 return self
._info
.get("development", "N") == "Y"
548 def mirrorlist(self
):
549 return self
._info
.get("mirrorlist", "N") == "Y"