From 0673d1b06069c363f2a829a1dfad53e4352849d0 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Tue, 18 Jan 2011 23:07:32 +0100 Subject: [PATCH] mirrors: New functions for geo load-balancing. --- www/static/css/style.css | 1 + www/templates/mirrors-item.html | 8 +- www/templates/mirrors.html | 25 +-- www/templates/modules/mirrors-table.html | 15 ++ www/translations/de_DE.csv | 3 + www/webapp/__init__.py | 3 +- www/webapp/backend/geoip.py | 41 +++- www/webapp/backend/mirrors.py | 226 ++++++++++++++++++++++- www/webapp/handlers_download.py | 20 +- www/webapp/handlers_mirrors.py | 29 ++- www/webapp/ui_modules.py | 5 + 11 files changed, 323 insertions(+), 53 deletions(-) create mode 100644 www/templates/modules/mirrors-table.html diff --git a/www/static/css/style.css b/www/static/css/style.css index a2660d98..21f096f7 100644 --- a/www/static/css/style.css +++ b/www/static/css/style.css @@ -886,6 +886,7 @@ table.mirrors td { table.mirrors td.hostname { text-align: right; padding-left: 2em; + width: 10em; } table.mirrors td.down { diff --git a/www/templates/mirrors-item.html b/www/templates/mirrors-item.html index f81ed7c5..81cbaa4e 100644 --- a/www/templates/mirrors-item.html +++ b/www/templates/mirrors-item.html @@ -28,6 +28,12 @@ {{ _("Owner") }} {{ item.owner }} + {% if item.prefer_for_countries %} + + {{ _("Preferred for") }}: + {{ locale.list(item.prefer_for_countries_names) }} + + {% end %}

@@ -43,7 +49,7 @@ {{ _("The mirror %s is located in %s.") % (item.hostname, item.location_str) }}
{{ _(

{% end block %} diff --git a/www/templates/mirrors.html b/www/templates/mirrors.html index 0ec796a8..fadaf44d 100644 --- a/www/templates/mirrors.html +++ b/www/templates/mirrors.html @@ -16,21 +16,16 @@
-

{{ _("List of servers") }}

- - {% for mirror in mirrors %} - - - {% if not mirror.state == "UP" %} - - {% else %} - - {% end %} - - {% end %} -
- {{ mirror.hostname }} - {{ _("Last update") }}: {{ locale.format_date(mirror.last_update) }}. 
+ {% if other_mirrors %} +

{{ _("Mirror servers nearby") }}

+ {{ modules.MirrorsTable(preferred_mirrors) }} + +

{{ _("Worldwide mirror servers") }}

+ {{ modules.MirrorsTable(other_mirrors) }} + {% else %} +

{{ _("Worldwide mirror servers") }}

+ {{ modules.MirrorsTable(preferred_mirrors) }} + {% end %} {% end block %} {% block sidebar %} diff --git a/www/templates/modules/mirrors-table.html b/www/templates/modules/mirrors-table.html new file mode 100644 index 00000000..ce446173 --- /dev/null +++ b/www/templates/modules/mirrors-table.html @@ -0,0 +1,15 @@ + + {% for mirror in mirrors %} + + + + + {% end %} +
+ {{ mirror.hostname }} + + {{ mirror.country_code }} + {{ mirror.location_str }} +
+
+ diff --git a/www/translations/de_DE.csv b/www/translations/de_DE.csv index c0396a09..6741200f 100644 --- a/www/translations/de_DE.csv +++ b/www/translations/de_DE.csv @@ -256,3 +256,6 @@ "Go to the wiki","Zum Wiki" "Documentation","Dokumentation" "Configuration","Konfiguration" +"Mirror servers nearby","Mirror-Server in der Nähe" +"Worldwide mirror servers","Weltweite Mirror-Server" +"Preferred for","Bevorzugt für" diff --git a/www/webapp/__init__.py b/www/webapp/__init__.py index 7274e109..fb1bca7a 100644 --- a/www/webapp/__init__.py +++ b/www/webapp/__init__.py @@ -21,13 +21,14 @@ class Application(tornado.web.Application): def __init__(self): settings = dict( cookie_secret = "aXBmaXJlY29va2llc2VjcmV0Cg==", - debug = False, + debug = True, gzip = True, login_url = "/login", template_path = os.path.join(BASEDIR, "templates"), ui_modules = { "Menu" : MenuModule, "MirrorItem" : MirrorItemModule, + "MirrorsTable" : MirrorsTableModule, "NewsItem" : NewsItemModule, "NewsLine" : NewsLineModule, "PlanetEntry" : PlanetEntryModule, diff --git a/www/webapp/backend/geoip.py b/www/webapp/backend/geoip.py index 9a68fe39..123bc772 100644 --- a/www/webapp/backend/geoip.py +++ b/www/webapp/backend/geoip.py @@ -3,6 +3,7 @@ import re from databases import Databases +from memcached import Memcached from misc import Singleton class GeoIP(object): @@ -15,6 +16,10 @@ class GeoIP(object): def db(self): return Databases().geoip + @property + def memcached(self): + return Memcached() + def __encode_ip(self, addr): # We get a tuple if there were proxy headers. addr = addr.split(", ") @@ -27,15 +32,37 @@ class GeoIP(object): return int(((int(a1) * 256 + int(a2)) * 256 + int(a3)) * 256 + int(a4) + 100) def get_country(self, addr): - return self.db.get("SELECT * FROM ip_group_country WHERE ip_start <= %s \ - ORDER BY ip_start DESC LIMIT 1;", self.__encode_ip(addr)).country_code.lower() + addr = self.__encode_ip(addr) + + mem_id = "geoip-country-%s" % addr + ret = self.memcached.get(mem_id) + + if not ret: + ret = self.db.get("SELECT * FROM ip_group_country WHERE ip_start <= %s \ + ORDER BY ip_start DESC LIMIT 1;", addr).country_code.lower() + self.memcached.set(mem_id, ret, 3600) + + return ret def get_all(self, addr): - # XXX should be done with a join - location = self.db.get("SELECT location FROM ip_group_city WHERE ip_start <= %s \ - ORDER BY ip_start DESC LIMIT 1;", self.__encode_ip(addr)).location - - return self.db.get("SELECT * FROM locations WHERE id = %s", int(location)) + addr = self.__encode_ip(addr) + + mem_id = "geoip-all-%s" % addr + ret = self.memcached.get(mem_id) + + if not ret: + # XXX should be done with a join + location = self.db.get("SELECT location FROM ip_group_city WHERE ip_start <= %s \ + ORDER BY ip_start DESC LIMIT 1;", addr).location + + ret = self.db.get("SELECT * FROM locations WHERE id = %s", int(location)) + self.memcached.set(mem_id, ret, 3600) + + # If location was not determinable + if ret.latitude == 0 and ret.longitude == 0: + return None + + return ret def get_country_name(self, code): name = "Unknown" diff --git a/www/webapp/backend/mirrors.py b/www/webapp/backend/mirrors.py index 1de46312..cc5af681 100644 --- a/www/webapp/backend/mirrors.py +++ b/www/webapp/backend/mirrors.py @@ -1,13 +1,16 @@ #!/usr/bin/python import logging +import math import os.path +import random import socket import time import tornado.httpclient from databases import Databases from geoip import GeoIP +from memcached import Memcached from misc import Singleton class Mirrors(object): @@ -17,6 +20,10 @@ class Mirrors(object): def db(self): return Databases().webapp + @property + def memcached(self): + return Memcached() + def list(self): return [Mirror(m.id) for m in self.db.query("SELECT id FROM mirrors ORDER BY state")] @@ -27,6 +34,9 @@ class Mirrors(object): def get(self, id): return Mirror(id) + def get_all(self): + return MirrorSet(self.list()) + def get_by_hostname(self, hostname): mirror = self.db.get("SELECT id FROM mirrors WHERE hostname=%s", hostname) @@ -60,6 +70,22 @@ class Mirrors(object): for mirror in mirrors: yield self.get(mirror.id) + def get_for_location(self, addr): + distance = 10 + + mirrors = [] + all_mirrors = self.list() + + while all_mirrors and len(mirrors) <= 2 and distance <= 270: + for mirror in all_mirrors: + if mirror.distance_to(addr) <= distance: + mirrors.append(mirror) + all_mirrors.remove(mirror) + + distance *= 1.2 + + return mirrors + def get_all_files(self): files = [] @@ -74,6 +100,97 @@ class Mirrors(object): return files +class MirrorSet(object): + def __init__(self, mirrors): + self._mirrors = mirrors + + def __add__(self, other): + mirrors = [] + + for mirror in self._mirrors + other._mirrors: + if mirror in mirrors: + continue + + mirrors.append(mirror) + + return MirrorSet(mirrors) + + def __sub__(self, other): + mirrors = self._mirrors[:] + + for mirror in other._mirrors: + if mirror in mirrors: + mirrors.remove(mirror) + + return MirrorSet(mirrors) + + def __iter__(self): + return iter(self._mirrors) + + def __len__(self): + return len(self._mirrors) + + def __str__(self): + return "" % ", ".join([m.hostname for m in self._mirrors]) + + @property + def db(self): + return Mirrors().db + + def get_with_file(self, filename): + with_file = [m.mirror for m in self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)] + + mirrors = [] + for mirror in self._mirrors: + if mirror.id in with_file: + mirrors.append(mirror) + + return MirrorSet(mirrors) + + def get_random(self): + mirrors = [] + for mirror in self._mirrors: + for i in range(0, mirror.priority + 1): + mirrors.append(mirror) + + return random.choice(mirrors) + + def get_for_country(self, country): + mirrors = [] + + for mirror in self._mirrors: + if country in mirror.prefer_for_countries: + mirrors.append(mirror) + + return MirrorSet(mirrors) + + def get_for_location(self, addr): + distance = 10 + + mirrors = [] + + while len(mirrors) <= 2 and distance <= 270: + for mirror in self._mirrors: + if mirror in mirrors: + continue + + if mirror.distance_to(addr) <= distance: + mirrors.append(mirror) + + distance *= 1.2 + + return MirrorSet(mirrors) + + def get_with_state(self, state): + mirrors = [] + + for mirror in self._mirrors: + if mirror.state == state: + mirrors.append(mirror) + + return MirrorSet(mirrors) + + class Mirror(object): def __init__(self, id): self.id = id @@ -91,8 +208,15 @@ class Mirror(object): return Databases().webapp def reload(self): - self._info = self.db.get("SELECT * FROM mirrors WHERE id=%s", self.id) - self._info["url"] = self.generate_url() + memcached = Memcached() + mem_id = "mirror-%s" % self.id + + self._info = memcached.get(mem_id) + if not self._info: + self._info = self.db.get("SELECT * FROM mirrors WHERE id=%s", self.id) + self._info["url"] = self.generate_url() + + memcached.set(mem_id, self._info, 60) def generate_url(self): url = "http://%s" % self.hostname @@ -113,10 +237,57 @@ class Mirror(object): def address(self): return socket.gethostbyname(self.hostname) + @property + def location(self): + if not hasattr(self, "__location"): + self.__location = GeoIP().get_all(self.address) + + return self.__location + + @property + def latitude(self): + return self.location.latitude + + @property + def longitude(self): + return self.location.longitude + + @property + def coordinates(self): + return (self.latitude, self.longitude) + + @property + def coordiante_str(self): + coordinates = [] + + for i in self.coordinates: + coordinates.append("%s" % i) + + return ",".join(coordinates) + @property def country_code(self): return GeoIP().get_country(self.address).lower() or "unknown" + @property + def country_name(self): + return GeoIP().get_country_name(self.country_code) + + @property + def city(self): + if self._info["city"]: + return self._info["city"] + + return self.location.city + + @property + def location_str(self): + s = self.country_name + if self.city: + s = "%s, %s" % (self.city, s) + + return s + @property def filelist(self): filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id) @@ -227,12 +398,53 @@ class Mirror(object): @property def prefer_for_countries(self): - return self._info.get("prefer_for_countries", "").split() + countries = self._info.get("prefer_for_countries", "") + if countries: + return sorted(countries.split(", ")) + return [] + + @property + def prefer_for_countries_names(self): + return sorted([GeoIP().get_country_name(c) for c in self.prefer_for_countries]) + def distance_to(self, addr): + location = GeoIP().get_all(addr) + if not location: + return 0 -if __name__ == "__main__": - m = Mirrors() + if location.country_code.lower() in self.prefer_for_countries: + return 0 + + distance_vector = ( + self.latitude - location.latitude, + self.longitude - location.longitude + ) + + distance = 0 + for i in distance_vector: + distance += i**2 + + return math.sqrt(distance) + + def traffic(self, since): + # XXX needs to be done better + + files = {} + for entry in self.db.query("SELECT filename, filesize FROM files"): + files[entry.filename] = entry.filesize + + query = "SELECT COUNT(filename) as count, filename FROM log_download WHERE mirror = %s" + query += " AND date >= %s GROUP BY filename" + + traffic = 0 + for entry in self.db.query(query, self.id, since): + if files.has_key(entry.filename): + traffic += entry.count * files[entry.filename] + + return traffic + + @property + def priority(self): + return self._info.get("priority", 1) * 10 - for mirror in m.list(): - print mirror.hostname, mirror.country_code diff --git a/www/webapp/handlers_download.py b/www/webapp/handlers_download.py index d8b1dc3d..84874c34 100644 --- a/www/webapp/handlers_download.py +++ b/www/webapp/handlers_download.py @@ -68,19 +68,25 @@ class DownloadDevelopmentHandler(BaseHandler): class DownloadFileHandler(BaseHandler): def get(self, filename): - country_code = self.geoip.get_country(self.request.remote_ip) - self.set_header("Pragma", "no-cache") - self.set_header("X-Mirror-Client-Country", country_code) - mirrors = self.mirrors.get_with_file(filename, country=country_code) - if not mirrors: - self.mirrors.get_with_file(filename) + # Get all mirrors... + mirrors = self.mirrors.get_all() + mirrors = mirrors.get_with_file(filename) + mirrors = mirrors.get_with_state("UP") if not mirrors: raise tornado.web.HTTPError(404, "File not found: %s" % filename) - mirror = random.choice(mirrors) + # Find mirrors located near to the user. + # If we have not found any, we use all. + if len(mirrors) <= 3: + #mirrors_nearby = mirrors.get_for_location(self.request.remote_ip) + mirrors_nearby = mirrors.get_for_location("193.59.194.101") + if mirrors_nearby: + mirrors = mirrors_nearby + + mirror = mirrors.get_random() self.redirect(mirror.url + filename[len(mirror.prefix):]) diff --git a/www/webapp/handlers_mirrors.py b/www/webapp/handlers_mirrors.py index 86d2e8ad..8ae911df 100644 --- a/www/webapp/handlers_mirrors.py +++ b/www/webapp/handlers_mirrors.py @@ -7,30 +7,29 @@ from handlers_base import * class MirrorIndexHandler(BaseHandler): def get(self): - mirrors = self.mirrors.list() + ip_addr = self.get_argument("addr", self.request.remote_ip) - self.render("mirrors.html", mirrors=mirrors) + # Get a list of all mirrors. + all_mirrors = self.mirrors.get_all() + + # Choose the preferred ones by their location. + preferred_mirrors = all_mirrors.get_for_location(ip_addr) + + # Remove the preferred ones from the list of the rest. + other_mirrors = all_mirrors - preferred_mirrors + + self.render("mirrors.html", + preferred_mirrors=preferred_mirrors, other_mirrors=other_mirrors) class MirrorItemHandler(BaseHandler): def get(self, id): + _ = self.locale.translate + mirror = self.mirrors.get(id) if not mirror: raise tornado.web.HTTPError(404) - ip = socket.gethostbyname(mirror.hostname) - mirror.location = self.geoip.get_all(ip) - - # Shortcut for coordiantes - mirror.coordiantes = "%s,%s" % \ - (mirror.location.latitude, mirror.location.longitude) - - # Nice string for the user - mirror.location_str = mirror.location.country_code - if mirror.location.city: - mirror.location_str = "%s, %s" % \ - (mirror.location.city, mirror.location_str) - self.render("mirrors-item.html", item=mirror) diff --git a/www/webapp/ui_modules.py b/www/webapp/ui_modules.py index ba836087..01b9c86e 100644 --- a/www/webapp/ui_modules.py +++ b/www/webapp/ui_modules.py @@ -224,3 +224,8 @@ class StasyGeoTableModule(UIModule): countries.append(country) return self.render_string("modules/stasy-table-geo.html", countries=countries) + + +class MirrorsTableModule(UIModule): + def render(self, mirrors): + return self.render_string("modules/mirrors-table.html", mirrors=mirrors) -- 2.39.2