From 54af860e3bde8d56ce54a1a0b9bec60faaa0b4a7 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sun, 2 Jan 2011 13:55:18 +0100 Subject: [PATCH] Add new download load balancer. --- www/webapp/__init__.py | 14 +++----- www/webapp/backend/mirrors.py | 60 ++++++++++++++++++++++++++++----- www/webapp/backend/stasy.py | 1 + www/webapp/handlers.py | 2 +- www/webapp/handlers_base.py | 8 +++-- www/webapp/handlers_download.py | 23 ++++++++++--- 6 files changed, 82 insertions(+), 26 deletions(-) diff --git a/www/webapp/__init__.py b/www/webapp/__init__.py index 903cbac..f7ca0ad 100644 --- a/www/webapp/__init__.py +++ b/www/webapp/__init__.py @@ -85,15 +85,8 @@ class Application(tornado.web.Application): # (r"/author/(.*)", NewsAuthorHandler), #] + static_handlers) - # download.ipfire.org - self.add_handlers(r"download\.ipfire\.org", [ - (r"/", tornado.web.RedirectHandler, { "url" : "http://www.ipfire.org/" }), - (r"/(favicon\.ico)", tornado.web.StaticFileHandler, dict(path = static_path)), - (r"/(.*)", DownloadFileHandler), - ]) - # downloads.ipfire.org - self.add_handlers(r"downloads\.ipfire\.org", [ + self.add_handlers(r"downloads?\.ipfire\.org", [ (r"/", DownloadsIndexHandler), (r"/latest", DownloadsLatestHandler), (r"/release/([0-9]+)", DownloadsReleaseHandler), @@ -101,7 +94,9 @@ class Application(tornado.web.Application): (r"/development", DownloadsDevelopmentHandler), (r"/mirrors", tornado.web.RedirectHandler, { "url" : "http://mirrors.ipfire.org/" }), (r"/source", tornado.web.RedirectHandler, { "url" : "http://source.ipfire.org/" }), - ] + static_handlers) + ] + static_handlers + [ + (r"/(.*)", DownloadFileHandler), + ]) # mirrors.ipfire.org self.add_handlers(r"mirrors\.ipfire\.org", [ @@ -129,6 +124,7 @@ class Application(tornado.web.Application): (r"/stats/cpuflags", StasyStatsCPUFlagsHandler), (r"/stats/geo", StasyStatsGeoHandler), (r"/stats/memory", StasyStatsMemoryHandler), + (r"/stats/network", StasyStatsNetworkHandler), (r"/stats/oses", StasyStatsOSesHandler), (r"/stats/virtual", StasyStatsVirtualHandler), ] + static_handlers) diff --git a/www/webapp/backend/mirrors.py b/www/webapp/backend/mirrors.py index 67db5f8..415a9f1 100644 --- a/www/webapp/backend/mirrors.py +++ b/www/webapp/backend/mirrors.py @@ -1,6 +1,7 @@ #!/usr/bin/python import logging +import os.path import socket import time import tornado.httpclient @@ -24,16 +25,40 @@ class Mirrors(object): mirror.check() def get(self, id): - return Mirror( id) + return Mirror(id) def get_by_hostname(self, hostname): mirror = self.db.get("SELECT id FROM mirrors WHERE hostname=%s", hostname) return Mirror(mirror.id) - def get_with_file(self, filename): - return [Mirror(m.mirror) for m in \ - self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)] + def get_with_file(self, filename, country=None): + # XXX quick and dirty solution - needs a performance boost + mirror_ids = [m.mirror for m in self.db.query("SELECT mirror FROM mirror_files WHERE filename=%s", filename)] + + #if country: + # # Sort out all mirrors that are not preferred to the given country + # for mirror in self.get_for_country(country): + # if not mirror.id in mirror_ids: + # mirror_ids.remove(mirror.id) + + mirrors = [] + for mirror_id in mirror_ids: + mirror = self.get(mirror_id) + if not mirror.state == "UP": + continue + mirrors.append(mirror) + + logging.debug("%s" % mirrors) + + return mirrors + + def get_for_country(self, country): + # XXX need option for random order + mirrors = self.db.query("SELECT id FROM mirrors WHERE prefer_for_countries LIKE %s", country) + + for mirror in mirrors: + yield self.get(mirror.id) class Mirror(object): @@ -42,6 +67,12 @@ class Mirror(object): self.reload() + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, self.url) + + def __cmp__(self, other): + return cmp(self.id, other.id) + @property def db(self): return Databases().webapp @@ -78,6 +109,13 @@ class Mirror(object): filelist = self.db.query("SELECT filename FROM mirror_files WHERE mirror=%s ORDER BY filename", self.id) return [f.filename for f in filelist] + @property + def prefix(self): + if self.type.startswith("pakfire"): + return self.type + + return "" + def set_state(self, state): logging.info("Setting state of %s to %s" % (self.hostname, state)) @@ -117,7 +155,7 @@ class Mirror(object): http = tornado.httpclient.AsyncHTTPClient() http.fetch(self.url + ".timestamp", - headers={"Pragma" : "no-cache", }, + headers={ "Pragma" : "no-cache" }, callback=self.__check_timestamp_response) def __check_timestamp_response(self, response): @@ -141,13 +179,14 @@ class Mirror(object): logging.info("Successfully updated timestamp from %s" % self.hostname) def check_filelist(self): - if self.releases == "N": + # XXX need to remove data from disabled mirrors + if self.releases == "N" or self.disabled == "Y" or self.type != "full": return http = tornado.httpclient.AsyncHTTPClient() http.fetch(self.url + ".filelist", - headers={"Pragma" : "no-cache", }, + headers={ "Pragma" : "no-cache" }, callback=self.__check_filelist_response) def __check_filelist_response(self, response): @@ -159,10 +198,15 @@ class Mirror(object): for file in response.body.splitlines(): self.db.execute("INSERT INTO mirror_files(mirror, filename) VALUES(%s, %s)", - self.id, file) + self.id, os.path.join(self.prefix, file)) logging.info("Successfully updated mirror filelist from %s" % self.hostname) + @property + def prefer_for_countries(self): + return self._info.get("prefer_for_countries", "").split() + + if __name__ == "__main__": m = Mirrors() diff --git a/www/webapp/backend/stasy.py b/www/webapp/backend/stasy.py index d5e5aec..7c01753 100644 --- a/www/webapp/backend/stasy.py +++ b/www/webapp/backend/stasy.py @@ -307,6 +307,7 @@ class Stasy(object): return self.query({ "public_id" : public_id }).count() >= 1 def get_profile(self, public_id): + p = None # XXX should only find one object in the end for p in self.query({ "public_id" : public_id }): p = Profile(p) diff --git a/www/webapp/handlers.py b/www/webapp/handlers.py index ad67f3d..afc4315 100644 --- a/www/webapp/handlers.py +++ b/www/webapp/handlers.py @@ -53,7 +53,7 @@ class IndexHandler(BaseHandler): def get(self): # Get a list of the most recent news items and put them on the page. latest_news = self.news.get_latest(limit=1, locale=self.locale) - recent_news = self.news.get_latest(limit=2, locale=self.locale, offset=1) + recent_news = self.news.get_latest(limit=3, locale=self.locale, offset=1) return self.render("index.html", latest_news=latest_news, recent_news=recent_news) diff --git a/www/webapp/handlers_base.py b/www/webapp/handlers_base.py index 602b9fe..15d795c 100644 --- a/www/webapp/handlers_base.py +++ b/www/webapp/handlers_base.py @@ -21,9 +21,11 @@ class BaseHandler(tornado.web.RequestHandler): # another output that guessed. locale = self.get_argument("locale", None) if locale: - locale = tornado.locale.get(locale) - for locale in ALLOWED_LOCALES: - return locale + for l in ALLOWED_LOCALES: + if not l.code.startswith(locale): + continue + + return l # If no locale was provided we guess what the browser sends us locale = self.get_browser_locale() diff --git a/www/webapp/handlers_download.py b/www/webapp/handlers_download.py index 8033821..e0ccd89 100644 --- a/www/webapp/handlers_download.py +++ b/www/webapp/handlers_download.py @@ -1,9 +1,11 @@ #!/usr/bin/python +import logging import random - import tornado.web +import backend + from handlers_base import * class DownloadsIndexHandler(BaseHandler): @@ -66,10 +68,21 @@ class DownloadDevelopmentHandler(BaseHandler): class DownloadFileHandler(BaseHandler): def get(self, filename): - mirrors = self.mirrors.get_with_file(filename) + country_code = self.geoip.get_country(self.request.remote_ip) + + self.set_header("Pragma", "no-cache") + self.set_header("X-Mirror-Client-Country", country_code) + + mirrors = self.mirrors.get_with_file(filename, country=country_code) + if not mirrors: + self.mirrors.get_with_file(filename) + + if not mirrors: + raise tornado.web.HTTPError(404, "File not found: %s" % filename) - # Choose a random one - # XXX need better metric here mirror = random.choice(mirrors) - self.redirect(mirror.url + filename) + self.redirect(mirror.url + filename[len(mirror.prefix):]) + + self.mirrors.db.execute("INSERT INTO log_download(filename, mirror, country_code) VALUES(%s, %s, %s)", + filename, mirror.id, country_code) -- 2.39.5