]> git.ipfire.org Git - ipfire.org.git/commitdiff
Add new download load balancer.
authorMichael Tremer <michael.tremer@ipfire.org>
Sun, 2 Jan 2011 12:55:18 +0000 (13:55 +0100)
committerMichael Tremer <michael.tremer@ipfire.org>
Sun, 2 Jan 2011 12:55:18 +0000 (13:55 +0100)
www/webapp/__init__.py
www/webapp/backend/mirrors.py
www/webapp/backend/stasy.py
www/webapp/handlers.py
www/webapp/handlers_base.py
www/webapp/handlers_download.py

index 903cbac377b38842cafb742c118f2c3d637a45d3..f7ca0ad95b53d678c0b7e54fccc14b558d1555e2 100644 (file)
@@ -85,15 +85,8 @@ class Application(tornado.web.Application):
                #       (r"/author/(.*)", NewsAuthorHandler),
                #] + static_handlers)
 
-               # download.ipfire.org
-               self.add_handlers(r"download\.ipfire\.org", [
-                       (r"/", tornado.web.RedirectHandler, { "url" : "http://www.ipfire.org/" }),
-                       (r"/(favicon\.ico)", tornado.web.StaticFileHandler, dict(path = static_path)),
-                       (r"/(.*)", DownloadFileHandler),
-               ])
-
                # downloads.ipfire.org
-               self.add_handlers(r"downloads\.ipfire\.org", [
+               self.add_handlers(r"downloads?\.ipfire\.org", [
                        (r"/", DownloadsIndexHandler),
                        (r"/latest", DownloadsLatestHandler),
                        (r"/release/([0-9]+)", DownloadsReleaseHandler),
@@ -101,7 +94,9 @@ class Application(tornado.web.Application):
                        (r"/development", DownloadsDevelopmentHandler),
                        (r"/mirrors", tornado.web.RedirectHandler, { "url" : "http://mirrors.ipfire.org/" }),
                        (r"/source", tornado.web.RedirectHandler, { "url" : "http://source.ipfire.org/" }),
-               ] + static_handlers)
+               ] + static_handlers + [
+                       (r"/(.*)", DownloadFileHandler),
+               ])
 
                # mirrors.ipfire.org
                self.add_handlers(r"mirrors\.ipfire\.org", [
@@ -129,6 +124,7 @@ class Application(tornado.web.Application):
                        (r"/stats/cpuflags", StasyStatsCPUFlagsHandler),
                        (r"/stats/geo", StasyStatsGeoHandler),
                        (r"/stats/memory", StasyStatsMemoryHandler),
+                       (r"/stats/network", StasyStatsNetworkHandler),
                        (r"/stats/oses", StasyStatsOSesHandler),
                        (r"/stats/virtual", StasyStatsVirtualHandler),
                ] + static_handlers)
index 67db5f8625ca6b55ac68b46e2bce1d6201a3f225..415a9f1df10e6367facec94340f8fba6de0f5d0d 100644 (file)
@@ -1,6 +1,7 @@
 #!/usr/bin/python
 
 import logging
+import os.path
 import socket
 import time
 import tornado.httpclient
@@ -24,16 +25,40 @@ class Mirrors(object):
                        mirror.check()
 
        def get(self, id):
-               return Mirror( id)
+               return Mirror(id)
 
        def get_by_hostname(self, hostname):
                mirror = self.db.get("SELECT id FROM mirrors WHERE hostname=%s", hostname)
 
                return Mirror(mirror.id)
 
-       def get_with_file(self, filename):
-               return [Mirror(m.mirror) for m in \
-                       self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)]
+       def get_with_file(self, filename, country=None):
+               # XXX quick and dirty solution - needs a performance boost
+               mirror_ids = [m.mirror for m in self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)]
+
+               #if country:
+               #       # Sort out all mirrors that are not preferred to the given country
+               #       for mirror in self.get_for_country(country):
+               #               if not mirror.id in mirror_ids:
+               #                       mirror_ids.remove(mirror.id)
+
+               mirrors = []
+               for mirror_id in mirror_ids:
+                       mirror = self.get(mirror_id)
+                       if not mirror.state == "UP":
+                               continue
+                       mirrors.append(mirror)
+
+               logging.debug("%s" % mirrors)
+
+               return mirrors
+
+       def get_for_country(self, country):
+               # XXX need option for random order
+               mirrors = self.db.query("SELECT id FROM mirrors WHERE prefer_for_countries LIKE %s", country)
+
+               for mirror in mirrors:
+                       yield self.get(mirror.id)
 
 
 class Mirror(object):
@@ -42,6 +67,12 @@ class Mirror(object):
 
                self.reload()
 
+       def __repr__(self):
+               return "<%s %s>" % (self.__class__.__name__, self.url)
+
+       def __cmp__(self, other):
+               return cmp(self.id, other.id)
+
        @property
        def db(self):
                return Databases().webapp
@@ -78,6 +109,13 @@ class Mirror(object):
                filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id)
                return [f.filename for f in filelist]
 
+       @property
+       def prefix(self):
+               if self.type.startswith("pakfire"):
+                       return self.type
+
+               return ""
+
        def set_state(self, state):
                logging.info("Setting state of %s to %s" % (self.hostname, state))
 
@@ -117,7 +155,7 @@ class Mirror(object):
                http = tornado.httpclient.AsyncHTTPClient()
 
                http.fetch(self.url + ".timestamp",
-                       headers={"Pragma" : "no-cache", },
+                       headers={ "Pragma" : "no-cache" },
                        callback=self.__check_timestamp_response)
 
        def __check_timestamp_response(self, response):
@@ -141,13 +179,14 @@ class Mirror(object):
                logging.info("Successfully updated timestamp from %s" % self.hostname)
 
        def check_filelist(self):
-               if self.releases == "N":
+               # XXX need to remove data from disabled mirrors
+               if self.releases == "N" or self.disabled == "Y" or self.type != "full":
                        return
 
                http = tornado.httpclient.AsyncHTTPClient()
 
                http.fetch(self.url + ".filelist",
-                       headers={"Pragma" : "no-cache", },
+                       headers={ "Pragma" : "no-cache" },
                        callback=self.__check_filelist_response)
 
        def __check_filelist_response(self, response):
@@ -159,10 +198,15 @@ class Mirror(object):
 
                for file in response.body.splitlines():
                        self.db.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)",
-                                       self.id, file)
+                                       self.id, os.path.join(self.prefix, file))
 
                logging.info("Successfully updated mirror filelist from %s" % self.hostname)
 
+       @property
+       def prefer_for_countries(self):
+               return self._info.get("prefer_for_countries", "").split()
+
+
 
 if __name__ == "__main__":
        m = Mirrors()
index d5e5aec159cc49ac19eec81d62f934797e6b1e95..7c01753ed6ebed39fa19536e788c67e452cedc70 100644 (file)
@@ -307,6 +307,7 @@ class Stasy(object):
                return self.query({ "public_id" : public_id }).count() >= 1
 
        def get_profile(self, public_id):
+               p = None
                # XXX should only find one object in the end
                for p in self.query({ "public_id" : public_id }):
                        p = Profile(p)
index ad67f3d16d6ee002dd69dfdd82c8de57390656ce..afc4315b0090deb5f925010b3882f9259aadeb40 100644 (file)
@@ -53,7 +53,7 @@ class IndexHandler(BaseHandler):
        def get(self):
                # Get a list of the most recent news items and put them on the page.            
                latest_news = self.news.get_latest(limit=1, locale=self.locale)
-               recent_news = self.news.get_latest(limit=2, locale=self.locale, offset=1)
+               recent_news = self.news.get_latest(limit=3, locale=self.locale, offset=1)
 
                return self.render("index.html",
                        latest_news=latest_news, recent_news=recent_news)
index 602b9fef49317d5a4728b613895174a91c80aba5..15d795ca24ab6cd26c12aea27a58989cf9de81c1 100644 (file)
@@ -21,9 +21,11 @@ class BaseHandler(tornado.web.RequestHandler):
                # another output that guessed.
                locale = self.get_argument("locale", None)
                if locale:
-                       locale = tornado.locale.get(locale)
-                       for locale in ALLOWED_LOCALES:
-                               return locale
+                       for l in ALLOWED_LOCALES:
+                               if not l.code.startswith(locale):
+                                       continue
+
+                               return l
 
                # If no locale was provided we guess what the browser sends us
                locale = self.get_browser_locale()
index 80338216bda1044ea1610b219c75947efdf6e936..e0ccd895c0bdacd3d13f4914384ff5b039f8d56e 100644 (file)
@@ -1,9 +1,11 @@
 #!/usr/bin/python
 
+import logging
 import random
-
 import tornado.web
 
+import backend
+
 from handlers_base import *
 
 class DownloadsIndexHandler(BaseHandler):
@@ -66,10 +68,21 @@ class DownloadDevelopmentHandler(BaseHandler):
 
 class DownloadFileHandler(BaseHandler):
        def get(self, filename):
-               mirrors = self.mirrors.get_with_file(filename)
+               country_code = self.geoip.get_country(self.request.remote_ip)
+
+               self.set_header("Pragma", "no-cache")
+               self.set_header("X-Mirror-Client-Country", country_code)
+
+               mirrors = self.mirrors.get_with_file(filename, country=country_code)
+               if not mirrors:
+                       self.mirrors.get_with_file(filename)
+
+               if not mirrors:
+                       raise tornado.web.HTTPError(404, "File not found: %s" % filename)
 
-               # Choose a random one
-               # XXX need better metric here
                mirror = random.choice(mirrors)
 
-               self.redirect(mirror.url + filename)
+               self.redirect(mirror.url + filename[len(mirror.prefix):])
+
+               self.mirrors.db.execute("INSERT INTO log_download(filename, mirror, country_code) VALUES(%s, %s, %s)",
+                       filename, mirror.id, country_code)