#!/usr/bin/python
+from __future__ import division
+
+import datetime
import logging
import math
import os.path
import socket
import time
import tornado.httpclient
+import tornado.netutil
+import urlparse
-from databases import Databases
-from geoip import GeoIP
-from memcached import Memcached
-from misc import Singleton
-
-class Downloads(object):
- __metaclass__ = Singleton
-
- @property
- def db(self):
- return Databases().webapp
-
- @property
- def mirrors(self):
- return Mirrors()
+from misc import Object
+class Downloads(Object):
@property
def total(self):
ret = self.db.get("SELECT COUNT(*) AS total FROM log_download")
@property
def today(self):
- ret = self.db.get("SELECT COUNT(*) AS today FROM log_download WHERE date >= NOW() - 1000000")
+ ret = self.db.get("SELECT COUNT(*) AS today FROM log_download WHERE date::date = NOW()::date")
return ret.today
@property
def yesterday(self):
- ret = self.db.get("SELECT COUNT(*) AS yesterday FROM log_download WHERE DATE(date) = DATE(NOW())-1")
+ ret = self.db.get("SELECT COUNT(*) AS yesterday FROM log_download WHERE date::date = (NOW() - INTERVAL '1 day')::date")
return ret.yesterday
@property
def daily_map(self):
- ret = self.db.query("SELECT DATE(date) AS date, COUNT(*) AS downloads FROM log_download"
- " WHERE DATE(date) BETWEEN DATE(NOW()) - 31 AND DATE(NOW()) GROUP BY DATE(date)")
+ ret = self.db.query("SELECT date::date AS date, COUNT(*) AS downloads FROM log_download"
+ " WHERE date::date BETWEEN (NOW() - INTERVAL '30 days')::date AND NOW()::date GROUP BY date::date")
return ret
query = "SELECT country_code, count(country_code) AS count FROM log_download"
if duration == "today":
- query += " WHERE date >= NOW() - 1000000"
+ query += " WHERE date::date = NOW()::date"
query += " GROUP BY country_code ORDER BY count DESC"
ret = {}
count = sum([o.count for o in results])
- for res in results:
- ret[res.country_code] = float(res.count) / count
+ if count:
+ for res in results:
+ ret[res.country_code] = res.count / count
return ret
query = "SELECT mirror, COUNT(mirror) AS count FROM log_download"
if duration == "today":
- query += " WHERE date >= NOW() - 1000000"
+ query += " WHERE date::date = NOW()::date"
query += " GROUP BY mirror ORDER BY count DESC"
ret = {}
count = sum([o.count for o in results])
- for res in results:
- mirror = self.mirrors.get(res.mirror)
- ret[mirror.hostname] = float(res.count) / count
+ if count:
+ for res in results:
+ mirror = self.mirrors.get(res.mirror)
+ ret[mirror.hostname] = res.count / count
return ret
-class Mirrors(object):
- __metaclass__ = Singleton
-
- @property
- def db(self):
- return Databases().webapp
-
- @property
- def memcached(self):
- return Memcached()
-
- def list(self):
- return [Mirror(m.id) for m in self.db.query("SELECT id FROM mirrors WHERE disabled = 'N' ORDER BY state,hostname")]
-
+class Mirrors(Object):
def check_all(self):
- for mirror in self.list():
+ for mirror in self.get_all():
mirror.check()
def get(self, id):
- return Mirror(id)
+ return Mirror(self.backend, id)
def get_all(self):
- return MirrorSet(self.list())
+ res = self.db.query("SELECT * FROM mirrors WHERE enabled = %s", True)
+
+ mirrors = []
+ for row in res:
+ mirror = Mirror(self.backend, row.id, row)
+ mirrors.append(mirror)
+
+ return MirrorSet(self.backend, sorted(mirrors))
def get_all_up(self):
- res = self.db.query("SELECT * FROM mirrors WHERE disabled = %s AND state = %s ORDER BY hostname", "N", "UP")
+ res = self.db.query("SELECT * FROM mirrors WHERE enabled = %s AND state = %s \
+ ORDER BY hostname", True, "UP")
mirrors = []
for row in res:
- m = Mirror(row.id, row)
+ m = Mirror(self.backend, row.id, row)
mirrors.append(m)
- return MirrorSet(mirrors)
+ return MirrorSet(self.backend, mirrors)
def get_by_hostname(self, hostname):
mirror = self.db.get("SELECT id FROM mirrors WHERE hostname=%s", hostname)
- return Mirror(mirror.id)
+ return Mirror(self.backend, mirror.id)
def get_with_file(self, filename, country=None):
# XXX quick and dirty solution - needs a performance boost
continue
mirrors.append(mirror)
- logging.debug("%s" % mirrors)
-
return mirrors
def get_for_country(self, country):
for mirror in mirrors:
yield self.get(mirror.id)
- def get_for_location(self, addr):
- distance = 10
-
- mirrors = []
- all_mirrors = self.list()
-
- location = GeoIP().get_all(addr)
+ def get_for_location(self, location):
if not location:
return None
- while all_mirrors and len(mirrors) <= 2 and distance <= 270:
+ distance = 2500
+
+ mirrors = []
+ all_mirrors = self.get_all()
+
+ while all_mirrors and len(mirrors) <= 3 and distance <= 8000:
for mirror in all_mirrors:
- if mirror.distance_to(location) <= distance:
+ mirror_distance = mirror.distance_to(location)
+ if mirror_distance is None:
+ continue
+
+ if mirror_distance<= distance:
mirrors.append(mirror)
all_mirrors.remove(mirror)
def get_all_files(self):
files = []
- for mirror in self.list():
+ for mirror in self.get_all():
if not mirror.state == "UP":
continue
return files
-class MirrorSet(object):
- def __init__(self, mirrors):
+class MirrorSet(Object):
+ def __init__(self, backend, mirrors):
+ Object.__init__(self, backend)
+
self._mirrors = mirrors
def __add__(self, other):
mirrors.append(mirror)
- return MirrorSet(mirrors)
+ return MirrorSet(self.backend, mirrors)
def __sub__(self, other):
mirrors = self._mirrors[:]
if mirror in mirrors:
mirrors.remove(mirror)
- return MirrorSet(mirrors)
+ return MirrorSet(self.backend, mirrors)
def __iter__(self):
return iter(self._mirrors)
def __str__(self):
return "<MirrorSet %s>" % ", ".join([m.hostname for m in self._mirrors])
- @property
- def db(self):
- return Mirrors().db
-
def get_with_file(self, filename):
with_file = [m.mirror for m in self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)]
if mirror.id in with_file:
mirrors.append(mirror)
- return MirrorSet(mirrors)
+ return MirrorSet(self.backend, mirrors)
def get_random(self):
mirrors = []
if country in mirror.prefer_for_countries:
mirrors.append(mirror)
- return MirrorSet(mirrors)
-
- def get_for_location(self, addr):
- distance = 10
+ return MirrorSet(self.backend, mirrors)
+ def get_for_location(self, location):
+ distance = 2500
mirrors = []
- location = GeoIP().get_all(addr)
if location:
- while len(mirrors) <= 2 and distance <= 270:
+ while len(mirrors) <= 3 and distance <= 8000:
for mirror in self._mirrors:
if mirror in mirrors:
continue
- if mirror.distance_to(location) <= distance:
+ mirror_distance = mirror.distance_to(location)
+ if mirror_distance is None:
+ continue
+
+ if mirror_distance <= distance:
mirrors.append(mirror)
distance *= 1.2
- return MirrorSet(mirrors)
+ return MirrorSet(self.backend, mirrors)
def get_with_state(self, state):
mirrors = []
if mirror.state == state:
mirrors.append(mirror)
- return MirrorSet(mirrors)
+ return MirrorSet(self.backend, mirrors)
+
+class Mirror(Object):
+ def __init__(self, backend, id, data=None):
+ Object.__init__(self, backend)
-class Mirror(object):
- def __init__(self, id, data=None):
self.id = id
if data:
return "<%s %s>" % (self.__class__.__name__, self.url)
def __cmp__(self, other):
- return cmp(self.id, other.id)
+ ret = cmp(self.country_code, other.country_code)
- @property
- def db(self):
- return Databases().webapp
+ if not ret:
+ ret = cmp(self.hostname, other.hostname)
+
+ return ret
def generate_url(self):
url = "http://%s" % self.hostname
url += "/"
return url
- def __getattr__(self, key):
- try:
- return self._info[key]
- except KeyError:
- raise AttributeError(key)
+ @property
+ def hostname(self):
+ return self._info.hostname
+
+ @property
+ def path(self):
+ return self._info.path
@property
def address(self):
return socket.gethostbyname(self.hostname)
+ @property
+ def owner(self):
+ return self._info.owner
+
@property
def location(self):
if self.__location is None:
- self.__location = GeoIP().get_all(self.address)
+ self.__location = self.geoip.get_location(self.address)
return self.__location
@property
def latitude(self):
- return self.location.latitude
+ if self.location:
+ return self.location.latitude
@property
def longitude(self):
- return self.location.longitude
+ if self.location:
+ return self.location.longitude
@property
def coordinates(self):
@property
def country_code(self):
- return self.location.country_code.lower() or "unknown"
+ if self.location:
+ return self.location.country
@property
def country_name(self):
if self.__country_name is None:
- self.__country_name = GeoIP().get_country_name(self.country_code)
+ self.__country_name = self.geoip.get_country_name(self.country_code)
return self.__country_name
@property
- def city(self):
- if self._info["city"]:
- return self._info["city"]
+ def location_str(self):
+ location = []
+
+ if self._info.location:
+ location.append(self._info.location)
- return self.location.city
+ elif self.location:
+ location.append(self.location.city)
+ location.append(self.country_name)
+
+ return ", ".join([s for s in location if s])
@property
- def location_str(self):
- s = self.country_name
- if self.city:
- s = "%s, %s" % (self.city, s)
+ def asn(self):
+ if not hasattr(self, "__asn"):
+ self.__asn = self.geoip.get_asn(self.address)
- return s
+ return self.__asn
@property
def filelist(self):
@property
def prefix(self):
- if self.type.startswith("pakfire"):
- return self.type
-
return ""
+ @property
+ def url(self):
+ return self._info.url
+
+ def build_url(self, filename):
+ return urlparse.urljoin(self.url, filename)
+
+ @property
+ def last_update(self):
+ return self._info.last_update
+
+ @property
+ def state(self):
+ return self._info.state
+
def set_state(self, state):
logging.info("Setting state of %s to %s" % (self.hostname, state))
if self.state == state:
return
- self.db.execute("UPDATE mirrors SET state=%s WHERE id=%s",
- state, self.id)
+ self.db.execute("UPDATE mirrors SET state = %s WHERE id = %s", state, self.id)
# Reload changed settings
if hasattr(self, "_info"):
self._info["state"] = state
+ @property
+ def enabled(self):
+ return self._info.enabled
+
+ @property
+ def disabled(self):
+ return not self.enabled
+
def check(self):
logging.info("Running check for mirror %s" % self.hostname)
def check_state(self):
logging.debug("Checking state of mirror %s" % self.id)
- if self.disabled == "Y":
+ if not self.enabled:
self.set_state("DOWN")
+ return
+
+ now = datetime.datetime.utcnow()
+
+ time_delta = now - self.last_update
+ time_diff = time_delta.total_seconds()
- time_diff = time.time() - self.last_update
- if time_diff > 3*24*60*60: # XXX get this into Settings
+ time_down = self.settings.get_int("mirrors_time_down", 3*24*60*60)
+ if time_diff >= time_down:
self.set_state("DOWN")
- elif time_diff > 6*60*60:
- self.set_state("OUTOFSYNC")
- else:
- self.set_state("UP")
+ return
- def check_timestamp(self):
- if not self.type == "full":
+ time_outofsync = self.settings.get_int("mirrors_time_outofsync", 6*60*60)
+ if time_diff >= time_outofsync:
+ self.set_state("OUTOFSYNC")
return
+ self.set_state("UP")
+
+ def check_timestamp(self):
http = tornado.httpclient.AsyncHTTPClient()
http.fetch(self.url + ".timestamp",
except ValueError:
timestamp = 0
- self.db.execute("UPDATE mirrors SET last_update=%s WHERE id=%s",
+ timestamp = datetime.datetime.fromtimestamp(timestamp)
+
+ self.db.execute("UPDATE mirrors SET last_update = %s WHERE id = %s",
timestamp, self.id)
# Reload changed settings
def check_filelist(self):
# XXX need to remove data from disabled mirrors
- if self.disabled == "Y" or self.type != "full":
+ if not self.enabled:
return
http = tornado.httpclient.AsyncHTTPClient()
@property
def prefer_for_countries_names(self):
- return sorted([GeoIP().get_country_name(c) for c in self.prefer_for_countries])
+ countries = [self.geoip.get_country_name(c.upper()) for c in self.prefer_for_countries]
+
+ return sorted(countries)
def distance_to(self, location, ignore_preference=False):
if not location:
- return 0
+ return None
- if not ignore_preference and location.country_code.lower() in self.prefer_for_countries:
+ country_code = None
+ if location.country:
+ country_code = location.country.lower()
+
+ if not ignore_preference and country_code in self.prefer_for_countries:
return 0
- distance_vector = (
- self.latitude - location.latitude,
- self.longitude - location.longitude
- )
+ # http://www.movable-type.co.uk/scripts/latlong.html
+
+ if self.latitude is None:
+ return None
+
+ if self.longitude is None:
+ return None
+
+ earth = 6371 # km
+ delta_lat = math.radians(self.latitude - location.latitude)
+ delta_lon = math.radians(self.longitude - location.longitude)
+
+ lat1 = math.radians(self.latitude)
+ lat2 = math.radians(location.latitude)
+
+ a = math.sin(delta_lat / 2) ** 2
+ a += math.cos(lat1) * math.cos(lat2) * (math.sin(delta_lon / 2) ** 2)
- distance = 0
- for i in distance_vector:
- distance += i**2
+ b1 = math.sqrt(a)
+ b2 = math.sqrt(1 - a)
- return math.sqrt(distance)
+ c = 2 * math.atan2(b1, b2)
+
+ return c * earth
def traffic(self, since):
# XXX needs to be done better
def priority(self):
return self._info.get("priority", 10)
- def is_pakfire2(self):
- return self.type in ("full", "pakfire2")
-
@property
def development(self):
- return self._info.get("development", "N") == "Y"
+ return self._info.get("development", False)
@property
def mirrorlist(self):
- return self._info.get("mirrorlist", "N") == "Y"
+ return self._info.get("mirrorlist", False)
+
+ @property
+ def addresses(self):
+ if not hasattr(self, "__addresses"):
+ addrinfo = socket.getaddrinfo(self.hostname, 0, socket.AF_UNSPEC, socket.SOCK_STREAM)
+
+ ret = []
+ for family, socktype, proto, canonname, address in addrinfo:
+ if family == socket.AF_INET:
+ address, port = address
+ elif family == socket.AF_INET6:
+ address, port, flowid, scopeid = address
+ ret.append((family, address))
+
+ self.__addresses = ret
+
+ return self.__addresses
+
+ @property
+ def addresses6(self):
+ return [address for family, address in self.addresses if family == socket.AF_INET6]
+
+ @property
+ def addresses4(self):
+ return [address for family, address in self.addresses if family == socket.AF_INET]
projects / people / shoehn / ipfire.org.git / blobdiff