From: Michael Tremer Date: Sat, 23 Jan 2010 13:29:47 +0000 (+0100) Subject: Add mirrorlist and rewrite load balancer. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=3add293ad7466a3a9aa927542e816148065bded3;p=ipfire.org.git Add mirrorlist and rewrite load balancer. --- diff --git a/www/mirrors.json b/www/mirrors.json new file mode 100644 index 00000000..13186c16 --- /dev/null +++ b/www/mirrors.json @@ -0,0 +1,161 @@ +[ + { + "name" : "IPFire Project", + "location" : { + "city" : "Cologne", + "country" : "Germany", + "country_code" : "de" + }, + "hostname" : "mirror1.ipfire.org", + "path" : "", + "serves" : { + "isos" : true, + "pakfire2" : true, + "pakfire3" : true + } + }, + + { + "name" : "Ronald Wiesinger", + "location" : { + "city" : "Vienna", + "country" : "Austria", + "country_code" : "at" + }, + "hostname" : "www.rowie.at", + "path" : "/ipfire", + "serves" : { + "isos" : true, + "pakfire2" : false, + "pakfire3" : false + } + }, + + { + "name" : "Jan Paul Tücking", + "location" : { + "city" : "Karlsruhe", + "country" : "Germany", + "country_code" : "de" + }, + "hostname" : "ipfire.earl-net.com", + "path" : "", + "serves" : { + "isos" : true, + "pakfire2" : false, + "pakfire3" : false + } + }, + + { + "name" : "Markus Villwock", + "location" : { + "city" : "Hannover", + "country" : "Germany", + "country_code" : "de" + }, + "hostname" : "kraefte.net", + "path" : "/ipfire", + "serves" : { + "isos" : true, + "pakfire2" : true, + "pakfire3" : false + } + }, + + { + "name" : "ISP42", + "location" : { + "city" : "Hannover", + "country" : "Germany", + "country_code" : "de" + }, + "hostname" : "mirror2.ipfire.org", + "path" : "", + "serves" : { + "isos" : true, + "pakfire2" : true, + "pakfire3" : true + } + }, + + { + "name" : "Peter Schälchli", + "location" : { + "city" : "Paris", + "country" : "France", + "country_code" : "fr" + }, + "hostname" : "mirror3.ipfire.org", + "path" : "", + "serves" : { + "isos" : true, + "pakfire2" : true, + "pakfire3" : false + } + }, + + { + "name" : "Peter Schälchli", + "location" : { + "city" : "Cologne", + "country" : "Germany", + "country_code" : "de" + }, + "hostname" : "mirror5.ipfire.org", + "path" : "", + "serves" : { + "isos" : true, + "pakfire2" : true, + "pakfire3" : false + } + }, + + { + "name" : "Kim Barthel", + "location" : { + "city" : "Stuttgart", + "country" : "Germany", + "country_code" : "de" + }, + "hostname" : "ipfire.kbarthel.de", + "path" : "", + "serves" : { + "isos" : true, + "pakfire2" : false, + "pakfire3" : false + } + }, + + { + "name" : "Kim Barthel", + "location" : { + "city" : "Stuttgart", + "country" : "Germany", + "country_code" : "de" + }, + "hostname" : "ipfire.1l0v3u.com", + "path" : "", + "serves" : { + "isos" : true, + "pakfire2" : false, + "pakfire3" : false + } + }, + + { + "name" : "Sebastian Winter", + "location" : { + "city" : "Minden", + "country" : "Germany", + "country_code" : "de" + }, + "hostname" : "ipfiremirror.wintertech.de", + "path" : "", + "serves" : { + "isos" : true, + "pakfire2" : false, + "pakfire3" : false + } + } +] diff --git a/www/static/css/style.css b/www/static/css/style.css index 2f596a4e..dde854ad 100644 --- a/www/static/css/style.css +++ b/www/static/css/style.css @@ -1067,3 +1067,38 @@ table.download-torrents td { table.download-torrents td.seeds,td.peers { text-align: right; } + +table.download-mirrors { + margin-bottom: 25px; + margin-left: 15px; + margin-top: 25px; + width: 700px; +} + +table.download-mirrors tr { + height: 32px; +} + +table.download-mirrors tr.legend { + text-align: right; +} + +table.download-mirrors td { + padding-left: 10px; + padding-right: 10px; +} + +table.download-mirrors tr.unreachable, td.unreachable { + border: 1px solid #f55; + background-color: #f99; +} + +table.download-mirrors tr.reachable, td.reachable { + border: 1px solid #5f5; + background-color: #9f9; +} + +table.download-mirrors td.latency { + width: 70px; + text-align: right; +} diff --git a/www/templates/downloads-mirrors.html b/www/templates/downloads-mirrors.html new file mode 100644 index 00000000..ed009888 --- /dev/null +++ b/www/templates/downloads-mirrors.html @@ -0,0 +1,75 @@ +{% extends "base.html" %} + +{% block content %} +
+ +

{{ _("IPFire Mirrors") }}

+ + {% if lang == "de" %} +

+ Diese Seite zeigt eine Liste der Mirror-Server des IPFire-Projektes. +

+ +

+ Bei einem Download wird einer der Server zufällig aus der Liste + gewählt und der User umgeleitet. +

+ + + {% else %} +

+ This page is an overview about our mirror servers. +

+ +

+ When a user downloads a file, one of the servers is arbitrarily + choosen und the user gets reditected. +

+ + + {% end %} + + + + + + + + {% for mirror in mirrors.all %} + + + + + + {% end %} + + + + + + + + + + + +
{{ _("Owner (Hostname)") }}{{ _("Location") }}{{ _("Latency") }}
{{ mirror.name }} ({{ mirror.hostname }}) + {{ mirror.location[ + {{ mirror.location["country"] }}, {{ mirror.location["city"] }} + {{ mirror.latency }} ms
 
{{ _("Legend") }}:{{ _("Server is reachable") }}
 {{ _("Server is unreachable") }}
+ +
+
+ +{% end block %} diff --git a/www/translations/de_DE.csv b/www/translations/de_DE.csv index 4d0346fd..68727c11 100644 --- a/www/translations/de_DE.csv +++ b/www/translations/de_DE.csv @@ -67,3 +67,10 @@ "USB FDD Image","USB Floppy Image" "USB HDD Image","USB Harddisk Image" "Use this image to burn a CD and install IPFire from it.","Brennen Sie dieses Image und booten Sie die Installation davon." +"Owner (Hostname)","Eigentümer (Hostname)" +"Priority","Priorität" +"Location","Standort" +"Latency","Latenz" +"Legend","Legende" +"Server is reachable","Server ist erreichbar" +"Server is unreachable","Server ist nicht erreichbar" diff --git a/www/webapp.py b/www/webapp.py index 5e4d6909..13a2bb3b 100755 --- a/www/webapp.py +++ b/www/webapp.py @@ -9,4 +9,12 @@ application = Application() if __name__ == "__main__": http_server = tornado.httpserver.HTTPServer(application) http_server.listen(8080) - tornado.ioloop.IOLoop.instance().start() + + try: + tornado.ioloop.IOLoop.instance().start() + except KeyboardInterrupt: + # Shutdown mirror monitoring + from webapp.mirrors import mirrors + mirrors.shutdown() + + raise diff --git a/www/webapp/__init__.py b/www/webapp/__init__.py index 71a8e8b1..a8e34c50 100644 --- a/www/webapp/__init__.py +++ b/www/webapp/__init__.py @@ -63,6 +63,7 @@ class Application(tornado.web.Application): (r"/[A-Za-z]{2}/downloads?", DownloadHandler), (r"/[A-Za-z]{2}/downloads?/all", DownloadAllHandler), (r"/[A-Za-z]{2}/downloads?/development", DownloadDevelopmentHandler), + (r"/[A-Za-z]{2}/downloads?/mirrors", DownloadMirrorHandler), (r"/[A-Za-z]{2}/downloads?/torrents", DownloadTorrentHandler), # API (r"/api/cluster_info", ApiClusterInfoHandler), @@ -72,10 +73,10 @@ class Application(tornado.web.Application): # download.ipfire.org self.add_handlers(r"download\.ipfire\.org", [ - (r"/", MainHandler), - (r"/[A-Za-z]{2}/?", MainHandler), - (r"/[A-Za-z]{2}/index", DownloadHandler), - ] + static_handlers) + (r"/", tornado.web.RedirectHandler, { "url" : "http://www.ipfire.org/" }), + (r"/(favicon\.ico)", tornado.web.StaticFileHandler, dict(path = static_path)), + (r"/(.*)", DownloadFileHandler), + ]) # source.ipfire.org self.add_handlers(r"source\.ipfire\.org", [ @@ -100,6 +101,10 @@ class Application(tornado.web.Application): ] + static_handlers) # ipfire.org - self.add_handlers(r"ipfire\.org", [ + self.add_handlers(r".*", [ (r".*", tornado.web.RedirectHandler, { "url" : "http://www.ipfire.org" }) ]) + + def __del__(self): + from mirrors import mirrors + mirrors.stop() diff --git a/www/webapp/handlers.py b/www/webapp/handlers.py index e2cb209a..58f1a0fb 100644 --- a/www/webapp/handlers.py +++ b/www/webapp/handlers.py @@ -18,6 +18,7 @@ import tornado.web from banners import banners from helpers import size from info import info +from mirrors import mirrors from news import news from releases import releases @@ -126,6 +127,11 @@ class DownloadTorrentHandler(BaseHandler): tracker=urlparse.urlparse(response.request.url).netloc) +class DownloadMirrorHandler(BaseHandler): + def get(self): + self.render("downloads-mirrors.html", mirrors=mirrors) + + class StaticHandler(BaseHandler): @property def static_path(self): @@ -273,3 +279,18 @@ class SourceDownloadHandler(BaseHandler): self.write(file.read()) finally: file.close() + + +class DownloadFileHandler(BaseHandler): + def get(self, path): + for mirror in mirrors.with_file(path): + if not mirror.reachable: + continue + + self.redirect(mirror.url + path) + return + + raise tornado.web.HTTPError(404) + + def get_error_html(self, status_code, **kwargs): + return tornado.web.RequestHandler.get_error_html(self, status_code, **kwargs) diff --git a/www/webapp/helpers.py b/www/webapp/helpers.py index 3e1df309..62717cc4 100644 --- a/www/webapp/helpers.py +++ b/www/webapp/helpers.py @@ -1,5 +1,7 @@ #!/usr/bin/python +import subprocess + class Item(object): def __init__(self, **args): self.args = args @@ -29,3 +31,35 @@ def _stringify(d): for key in d.keys(): ret[str(key)] = d[key] return ret + +def ping(host, count=5, wait=10): + cmd = subprocess.Popen( + ["ping", "-c%d" % count, "-w%d" % wait, host], + stdout = subprocess.PIPE, + stderr = subprocess.PIPE, + ) + + latency = None + + out, error = cmd.communicate() + + for line in out.split("\n"): + if not line.startswith("rtt"): + continue + + line = line.split() + if len(line) < 4: + break + + rtts = line[3].split("/") + if len(rtts) < 4: + break + + latency = "%.1f" % float(rtts[1]) + + return latency + + +if __name__ == "__main__": + print ping("www.ipfire.org") + print ping("www.rowie.at") diff --git a/www/webapp/mirrors.py b/www/webapp/mirrors.py new file mode 100644 index 00000000..58ba28c8 --- /dev/null +++ b/www/webapp/mirrors.py @@ -0,0 +1,197 @@ +#!/usr/bin/python + +import tornado.httpclient + +import random +import simplejson +import threading +import time + +from helpers import Item, _stringify, ping + +class Mirrors(threading.Thread): + def __init__(self, filename): + threading.Thread.__init__(self, name="Mirror Monitor") + + self.items = [] + self.load(filename) + + self.__running = True + + self.start() + + def load(self, filename): + f = open(filename) + data = f.read() + f.close() + + for item in simplejson.loads(data): + self.items.append(MirrorItem(**_stringify(item))) + + @property + def all(self): + return sorted(self.items) + + @property + def random(self): + # Doesnt work :( + #return random.shuffle(self.items) + ret = [] + items = self.items[:] + while items: + rnd = random.randint(0, len(items)-1) + ret.append(items.pop(rnd)) + return ret + + @property + def reachable(self): + ret = [] + for mirror in self.items: + if not mirror.reachable: + continue + ret.append(mirror) + return ret + + @property + def unreachable(self): + ret = [] + for mirror in self.all: + if mirror in self.reachable: + continue + ret.append(mirror) + return ret + + def pickone(self, reachable=False): + mirrors = self.items + if reachable: + mirrors = self.reachable + if not mirrors: + return None + return random.choice(mirrors) + + def with_file(self, path): + ret = [] + for mirror in self.random: + if not mirror["serves"]["isos"]: + continue + if path in mirror.files: + ret.append(mirror) + return ret + + def shutdown(self): + self.__running = False + + def run(self): + for mirror in self.random: + if not self.__running: + return + mirror.update() + + count = 0 + while self.__running: + if not count: + count = 300 # 30 secs + mirror = self.pickone() + if mirror: + mirror.update() + + time.sleep(0.1) + count -= 1 + + +class MirrorItem(Item): + def __init__(self, *args, **kwargs): + Item.__init__(self, *args, **kwargs) + + self.filelist = MirrorFilelist(self) + self.latency = "N/A" + + def __cmp__(self, other): + return cmp(self.name, other.name) + + def update(self): + self.latency = ping(self.hostname) or "N/A" + if self.filelist.outdated: + self.filelist.update() + + @property + def reachable(self): + return not self.latency == "N/A" + + @property + def url(self): + ret = "http://" + self.hostname + if not self.path.startswith("/"): + ret += "/" + ret += self.path + if not ret.endswith("/"): + ret += "/" + return ret + + @property + def files(self): + return self.filelist.files + + def has_file(self, path): + return path in self.files + + +class MirrorFilelist(object): + def __init__(self, mirror): + self.mirror = mirror + + self.__files = [] + self.__time = 0 + + #self.update(now=True) + + def update(self, now=False): + args = {} + + if now: + while not self.mirror.reachable: + time.sleep(10) + + http = tornado.httpclient.HTTPClient() + + if not now: + http = tornado.httpclient.AsyncHTTPClient() + args["callback"] = self.on_response + + try: + reponse = http.fetch(self.mirror.url + ".filelist", **args) + except tornado.httpclient.HTTPError: + self.__time = time.time() + return + + if now: + self.on_response(reponse) + + def on_response(self, response): + self.__files = [] + self.__time = time.time() + + if not response.code == 200: + return + + # If invalid html content... + if response.body.startswith(" 60*60 + + @property + def files(self): + #if self.outdated: + # self.update() + return self.__files + + +mirrors = Mirrors("mirrors.json")