X-Git-Url: http://git.ipfire.org/?p=people%2Fshoehn%2Fipfire.org.git;a=blobdiff_plain;f=webapp%2Fbackend%2Fmirrors.py;h=265e5160b82994f1a18d7b12d037822a372611f5;hp=260cff0d0a425340019b425c13a2b2edec31256a;hb=9068dba187b011290bd8bb947d3695fa90e71e27;hpb=864437e25485c45a366ce87bec54158cdd0118d2 diff --git a/webapp/backend/mirrors.py b/webapp/backend/mirrors.py index 260cff0..265e516 100644 --- a/webapp/backend/mirrors.py +++ b/webapp/backend/mirrors.py @@ -1,5 +1,8 @@ #!/usr/bin/python +from __future__ import division + +import datetime import logging import math import os.path @@ -7,23 +10,12 @@ import random import socket import time import tornado.httpclient +import tornado.netutil +import urlparse -from databases import Databases -from geoip import GeoIP -from memcached import Memcached -from misc import Singleton - -class Downloads(object): - __metaclass__ = Singleton - - @property - def db(self): - return Databases().webapp - - @property - def mirrors(self): - return Mirrors() +from misc import Object +class Downloads(Object): @property def total(self): ret = self.db.get("SELECT COUNT(*) AS total FROM log_download") @@ -32,20 +24,20 @@ class Downloads(object): @property def today(self): - ret = self.db.get("SELECT COUNT(*) AS today FROM log_download WHERE date >= NOW() - 1000000") + ret = self.db.get("SELECT COUNT(*) AS today FROM log_download WHERE date::date = NOW()::date") return ret.today @property def yesterday(self): - ret = self.db.get("SELECT COUNT(*) AS yesterday FROM log_download WHERE DATE(date) = DATE(NOW())-1") + ret = self.db.get("SELECT COUNT(*) AS yesterday FROM log_download WHERE date::date = (NOW() - INTERVAL '1 day')::date") return ret.yesterday @property def daily_map(self): - ret = self.db.query("SELECT DATE(date) AS date, COUNT(*) AS downloads FROM log_download" - " WHERE DATE(date) BETWEEN DATE(NOW()) - 31 AND DATE(NOW()) GROUP BY DATE(date)") + ret = self.db.query("SELECT date::date AS date, COUNT(*) AS downloads FROM log_download" + " WHERE date::date BETWEEN (NOW() - INTERVAL '30 days')::date AND NOW()::date GROUP BY date::date") return ret @@ -53,7 +45,7 @@ class Downloads(object): query = "SELECT country_code, count(country_code) AS count FROM log_download" if duration == "today": - query += " WHERE date >= NOW() - 1000000" + query += " WHERE date::date = NOW()::date" query += " GROUP BY country_code ORDER BY count DESC" @@ -61,8 +53,9 @@ class Downloads(object): ret = {} count = sum([o.count for o in results]) - for res in results: - ret[res.country_code] = float(res.count) / count + if count: + for res in results: + ret[res.country_code] = res.count / count return ret @@ -70,7 +63,7 @@ class Downloads(object): query = "SELECT mirror, COUNT(mirror) AS count FROM log_download" if duration == "today": - query += " WHERE date >= NOW() - 1000000" + query += " WHERE date::date = NOW()::date" query += " GROUP BY mirror ORDER BY count DESC" @@ -78,51 +71,47 @@ class Downloads(object): ret = {} count = sum([o.count for o in results]) - for res in results: - mirror = self.mirrors.get(res.mirror) - ret[mirror.hostname] = float(res.count) / count + if count: + for res in results: + mirror = self.mirrors.get(res.mirror) + ret[mirror.hostname] = res.count / count return ret -class Mirrors(object): - __metaclass__ = Singleton - - @property - def db(self): - return Databases().webapp - - @property - def memcached(self): - return Memcached() - - def list(self): - return [Mirror(m.id) for m in self.db.query("SELECT id FROM mirrors WHERE disabled = 'N' ORDER BY state,hostname")] - +class Mirrors(Object): def check_all(self): - for mirror in self.list(): + for mirror in self.get_all(): mirror.check() def get(self, id): - return Mirror(id) + return Mirror(self.backend, id) def get_all(self): - return MirrorSet(self.list()) + res = self.db.query("SELECT * FROM mirrors WHERE enabled = %s", True) + + mirrors = [] + for row in res: + mirror = Mirror(self.backend, row.id, row) + mirrors.append(mirror) + + return MirrorSet(self.backend, sorted(mirrors)) def get_all_up(self): - res = self.db.query("SELECT * FROM mirrors WHERE disabled = %s AND state = %s ORDER BY hostname", "N", "UP") + res = self.db.query("SELECT * FROM mirrors WHERE enabled = %s AND state = %s \ + ORDER BY hostname", True, "UP") mirrors = [] for row in res: - m = Mirror(row.id, row) + m = Mirror(self.backend, row.id, row) mirrors.append(m) - return MirrorSet(mirrors) + return MirrorSet(self.backend, mirrors) def get_by_hostname(self, hostname): mirror = self.db.get("SELECT id FROM mirrors WHERE hostname=%s", hostname) - return Mirror(mirror.id) + return Mirror(self.backend, mirror.id) def get_with_file(self, filename, country=None): # XXX quick and dirty solution - needs a performance boost @@ -141,8 +130,6 @@ class Mirrors(object): continue mirrors.append(mirror) - logging.debug("%s" % mirrors) - return mirrors def get_for_country(self, country): @@ -152,19 +139,22 @@ class Mirrors(object): for mirror in mirrors: yield self.get(mirror.id) - def get_for_location(self, addr): - distance = 10 - - mirrors = [] - all_mirrors = self.list() - - location = GeoIP().get_all(addr) + def get_for_location(self, location): if not location: return None - while all_mirrors and len(mirrors) <= 2 and distance <= 270: + distance = 2500 + + mirrors = [] + all_mirrors = self.get_all() + + while all_mirrors and len(mirrors) <= 3 and distance <= 8000: for mirror in all_mirrors: - if mirror.distance_to(location) <= distance: + mirror_distance = mirror.distance_to(location) + if mirror_distance is None: + continue + + if mirror_distance<= distance: mirrors.append(mirror) all_mirrors.remove(mirror) @@ -175,7 +165,7 @@ class Mirrors(object): def get_all_files(self): files = [] - for mirror in self.list(): + for mirror in self.get_all(): if not mirror.state == "UP": continue @@ -186,8 +176,10 @@ class Mirrors(object): return files -class MirrorSet(object): - def __init__(self, mirrors): +class MirrorSet(Object): + def __init__(self, backend, mirrors): + Object.__init__(self, backend) + self._mirrors = mirrors def __add__(self, other): @@ -199,7 +191,7 @@ class MirrorSet(object): mirrors.append(mirror) - return MirrorSet(mirrors) + return MirrorSet(self.backend, mirrors) def __sub__(self, other): mirrors = self._mirrors[:] @@ -208,7 +200,7 @@ class MirrorSet(object): if mirror in mirrors: mirrors.remove(mirror) - return MirrorSet(mirrors) + return MirrorSet(self.backend, mirrors) def __iter__(self): return iter(self._mirrors) @@ -219,10 +211,6 @@ class MirrorSet(object): def __str__(self): return "" % ", ".join([m.hostname for m in self._mirrors]) - @property - def db(self): - return Mirrors().db - def get_with_file(self, filename): with_file = [m.mirror for m in self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)] @@ -231,7 +219,7 @@ class MirrorSet(object): if mirror.id in with_file: mirrors.append(mirror) - return MirrorSet(mirrors) + return MirrorSet(self.backend, mirrors) def get_random(self): mirrors = [] @@ -248,26 +236,28 @@ class MirrorSet(object): if country in mirror.prefer_for_countries: mirrors.append(mirror) - return MirrorSet(mirrors) - - def get_for_location(self, addr): - distance = 10 + return MirrorSet(self.backend, mirrors) + def get_for_location(self, location): + distance = 2500 mirrors = [] - location = GeoIP().get_all(addr) if location: - while len(mirrors) <= 2 and distance <= 270: + while len(mirrors) <= 3 and distance <= 8000: for mirror in self._mirrors: if mirror in mirrors: continue - if mirror.distance_to(location) <= distance: + mirror_distance = mirror.distance_to(location) + if mirror_distance is None: + continue + + if mirror_distance <= distance: mirrors.append(mirror) distance *= 1.2 - return MirrorSet(mirrors) + return MirrorSet(self.backend, mirrors) def get_with_state(self, state): mirrors = [] @@ -276,11 +266,13 @@ class MirrorSet(object): if mirror.state == state: mirrors.append(mirror) - return MirrorSet(mirrors) + return MirrorSet(self.backend, mirrors) + +class Mirror(Object): + def __init__(self, backend, id, data=None): + Object.__init__(self, backend) -class Mirror(object): - def __init__(self, id, data=None): self.id = id if data: @@ -296,11 +288,12 @@ class Mirror(object): return "<%s %s>" % (self.__class__.__name__, self.url) def __cmp__(self, other): - return cmp(self.id, other.id) + ret = cmp(self.country_code, other.country_code) - @property - def db(self): - return Databases().webapp + if not ret: + ret = cmp(self.hostname, other.hostname) + + return ret def generate_url(self): url = "http://%s" % self.hostname @@ -311,30 +304,38 @@ class Mirror(object): url += "/" return url - def __getattr__(self, key): - try: - return self._info[key] - except KeyError: - raise AttributeError(key) + @property + def hostname(self): + return self._info.hostname + + @property + def path(self): + return self._info.path @property def address(self): return socket.gethostbyname(self.hostname) + @property + def owner(self): + return self._info.owner + @property def location(self): if self.__location is None: - self.__location = GeoIP().get_all(self.address) + self.__location = self.geoip.get_location(self.address) return self.__location @property def latitude(self): - return self.location.latitude + if self.location: + return self.location.latitude @property def longitude(self): - return self.location.longitude + if self.location: + return self.location.longitude @property def coordinates(self): @@ -351,29 +352,35 @@ class Mirror(object): @property def country_code(self): - return self.location.country_code.lower() or "unknown" + if self.location: + return self.location.country @property def country_name(self): if self.__country_name is None: - self.__country_name = GeoIP().get_country_name(self.country_code) + self.__country_name = self.geoip.get_country_name(self.country_code) return self.__country_name @property - def city(self): - if self._info["city"]: - return self._info["city"] + def location_str(self): + location = [] + + if self._info.location: + location.append(self._info.location) - return self.location.city + elif self.location: + location.append(self.location.city) + location.append(self.country_name) + + return ", ".join([s for s in location if s]) @property - def location_str(self): - s = self.country_name - if self.city: - s = "%s, %s" % (self.city, s) + def asn(self): + if not hasattr(self, "__asn"): + self.__asn = self.geoip.get_asn(self.address) - return s + return self.__asn @property def filelist(self): @@ -382,24 +389,43 @@ class Mirror(object): @property def prefix(self): - if self.type.startswith("pakfire"): - return self.type - return "" + @property + def url(self): + return self._info.url + + def build_url(self, filename): + return urlparse.urljoin(self.url, filename) + + @property + def last_update(self): + return self._info.last_update + + @property + def state(self): + return self._info.state + def set_state(self, state): logging.info("Setting state of %s to %s" % (self.hostname, state)) if self.state == state: return - self.db.execute("UPDATE mirrors SET state=%s WHERE id=%s", - state, self.id) + self.db.execute("UPDATE mirrors SET state = %s WHERE id = %s", state, self.id) # Reload changed settings if hasattr(self, "_info"): self._info["state"] = state + @property + def enabled(self): + return self._info.enabled + + @property + def disabled(self): + return not self.enabled + def check(self): logging.info("Running check for mirror %s" % self.hostname) @@ -409,21 +435,28 @@ class Mirror(object): def check_state(self): logging.debug("Checking state of mirror %s" % self.id) - if self.disabled == "Y": + if not self.enabled: self.set_state("DOWN") + return + + now = datetime.datetime.utcnow() + + time_delta = now - self.last_update + time_diff = time_delta.total_seconds() - time_diff = time.time() - self.last_update - if time_diff > 3*24*60*60: # XXX get this into Settings + time_down = self.settings.get_int("mirrors_time_down", 3*24*60*60) + if time_diff >= time_down: self.set_state("DOWN") - elif time_diff > 6*60*60: - self.set_state("OUTOFSYNC") - else: - self.set_state("UP") + return - def check_timestamp(self): - if not self.type == "full": + time_outofsync = self.settings.get_int("mirrors_time_outofsync", 6*60*60) + if time_diff >= time_outofsync: + self.set_state("OUTOFSYNC") return + self.set_state("UP") + + def check_timestamp(self): http = tornado.httpclient.AsyncHTTPClient() http.fetch(self.url + ".timestamp", @@ -441,7 +474,9 @@ class Mirror(object): except ValueError: timestamp = 0 - self.db.execute("UPDATE mirrors SET last_update=%s WHERE id=%s", + timestamp = datetime.datetime.fromtimestamp(timestamp) + + self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s", timestamp, self.id) # Reload changed settings @@ -454,7 +489,7 @@ class Mirror(object): def check_filelist(self): # XXX need to remove data from disabled mirrors - if self.disabled == "Y" or self.type != "full": + if not self.enabled: return http = tornado.httpclient.AsyncHTTPClient() @@ -496,25 +531,45 @@ class Mirror(object): @property def prefer_for_countries_names(self): - return sorted([GeoIP().get_country_name(c) for c in self.prefer_for_countries]) + countries = [self.geoip.get_country_name(c.upper()) for c in self.prefer_for_countries] + + return sorted(countries) def distance_to(self, location, ignore_preference=False): if not location: - return 0 + return None - if not ignore_preference and location.country_code.lower() in self.prefer_for_countries: + country_code = None + if location.country: + country_code = location.country.lower() + + if not ignore_preference and country_code in self.prefer_for_countries: return 0 - distance_vector = ( - self.latitude - location.latitude, - self.longitude - location.longitude - ) + # http://www.movable-type.co.uk/scripts/latlong.html + + if self.latitude is None: + return None + + if self.longitude is None: + return None + + earth = 6371 # km + delta_lat = math.radians(self.latitude - location.latitude) + delta_lon = math.radians(self.longitude - location.longitude) + + lat1 = math.radians(self.latitude) + lat2 = math.radians(location.latitude) + + a = math.sin(delta_lat / 2) ** 2 + a += math.cos(lat1) * math.cos(lat2) * (math.sin(delta_lon / 2) ** 2) - distance = 0 - for i in distance_vector: - distance += i**2 + b1 = math.sqrt(a) + b2 = math.sqrt(1 - a) - return math.sqrt(distance) + c = 2 * math.atan2(b1, b2) + + return c * earth def traffic(self, since): # XXX needs to be done better @@ -537,13 +592,35 @@ class Mirror(object): def priority(self): return self._info.get("priority", 10) - def is_pakfire2(self): - return self.type in ("full", "pakfire2") - @property def development(self): - return self._info.get("development", "N") == "Y" + return self._info.get("development", False) @property def mirrorlist(self): - return self._info.get("mirrorlist", "N") == "Y" + return self._info.get("mirrorlist", False) + + @property + def addresses(self): + if not hasattr(self, "__addresses"): + addrinfo = socket.getaddrinfo(self.hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM) + + ret = [] + for family, socktype, proto, canonname, address in addrinfo: + if family == socket.AF_INET: + address, port = address + elif family == socket.AF_INET6: + address, port, flowid, scopeid = address + ret.append((family, address)) + + self.__addresses = ret + + return self.__addresses + + @property + def addresses6(self): + return [address for family, address in self.addresses if family == socket.AF_INET6] + + @property + def addresses4(self): + return [address for family, address in self.addresses if family == socket.AF_INET]