From: Michael Tremer Date: Sat, 7 Aug 2010 14:36:12 +0000 (+0200) Subject: Some very big changes I cannot break down to distinct commits. X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=feb02477234bd1494c7e82aec088898013f55d7f;p=ipfire.org.git Some very big changes I cannot break down to distinct commits. --- diff --git a/www/mirrors.json b/www/mirrors.json index bb0d54dd..073eae11 100644 --- a/www/mirrors.json +++ b/www/mirrors.json @@ -1,6 +1,6 @@ [ { - "name" : "IPFire Project", + "owner" : "IPFire Project", "location" : { "city" : "Cologne", "country" : "Germany", @@ -32,7 +32,7 @@ }, { - "name" : "Jan Paul Tücking", + "owner" : "Jan Paul Tücking", "location" : { "city" : "Falkenstein/Vogtland", "country" : "Germany", @@ -48,7 +48,7 @@ }, { - "name" : "Markus Villwock", + "owner" : "Markus Villwock", "location" : { "city" : "Hannover", "country" : "Germany", @@ -64,7 +64,7 @@ }, { - "name" : "Kim Barthel", + "owner" : "Kim Barthel", "location" : { "city" : "Stuttgart", "country" : "Germany", @@ -80,7 +80,7 @@ }, { - "name" : "Robert Möker", + "owner" : "Robert Möker", "location" : { "city" : "Frankfurt am Main", "country" : "Germany", @@ -96,7 +96,7 @@ }, { - "name" : "SWITCH", + "owner" : "SWITCH", "location" : { "city" : "Zurich", "country" : "Switzerland", @@ -112,18 +112,18 @@ }, { - "name" : "Peter Schälchli", - "location" : { - "city" : "Paris", - "country" : "France", - "country_code" : "fr" - }, - "hostname" : "mirror6.ipfire.org", - "path" : "", - "serves" : { - "isos" : true, - "pakfire2" : true, - "pakfire3" : false - } - } + "owner" : "Peter Schälchli", + "location" : { + "city" : "Paris", + "country" : "France", + "country_code" : "fr" + }, + "hostname" : "mirror6.ipfire.org", + "path" : "", + "serves" : { + "isos" : true, + "pakfire2" : true, + "pakfire3" : false + } + } ] diff --git a/www/static/css/style.css b/www/static/css/style.css index 4bb6ece2..c38b3326 100644 --- a/www/static/css/style.css +++ b/www/static/css/style.css @@ -1092,16 +1092,23 @@ table.download-mirrors td { padding-right: 10px; } +/* table.download-mirrors tr.unreachable, td.unreachable { border: 1px solid #f55; background-color: #f99; } +*/ -table.download-mirrors tr.reachable, td.reachable { +table.download-mirrors tr.ok, td.ok { border: 1px solid #5f5; background-color: #9f9; } +table.download-mirrors tr.outdated, td.outdated { + border: 1px solid #55f; + background-color: #99f; +} + table.download-mirrors td.latency { width: 70px; text-align: right; diff --git a/www/templates/admin-accounts-edit.html b/www/templates/admin-accounts-edit.html new file mode 100644 index 00000000..6c693cb7 --- /dev/null +++ b/www/templates/admin-accounts-edit.html @@ -0,0 +1,9 @@ +{% extends "admin-base.html" %} + +{% block content %} +
+

{{ _("Account Administrator") }}

+ + {{ user.realname }} +
+{% end %} diff --git a/www/templates/admin-accounts.html b/www/templates/admin-accounts.html new file mode 100644 index 00000000..cd3224b2 --- /dev/null +++ b/www/templates/admin-accounts.html @@ -0,0 +1,19 @@ +{% extends "admin-base.html" %} + +{% block content %} +
+

{{ _("Account Administrator") }}

+ +

+ {{ _("Create new account") }} +

+ + +
+{% end %} diff --git a/www/templates/admin-base.html b/www/templates/admin-base.html new file mode 100644 index 00000000..5303fca8 --- /dev/null +++ b/www/templates/admin-base.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} + +{% block title %}{{ _("IPFire Admin Area") }}{% end %} + +{% block languages %}{% end %} + +{% block sidebar %} +

{{ _("Options") }}

+ + +{% end %} diff --git a/www/templates/admin-index.html b/www/templates/admin-index.html new file mode 100644 index 00000000..e0ee87ef --- /dev/null +++ b/www/templates/admin-index.html @@ -0,0 +1,7 @@ +{% extends "admin-base.html" %} + +{% block content %} +
+

{{ _("Admin Area") }}

+
+{% end %} diff --git a/www/templates/admin-planet-compose.html b/www/templates/admin-planet-compose.html new file mode 100644 index 00000000..4999bcd2 --- /dev/null +++ b/www/templates/admin-planet-compose.html @@ -0,0 +1,54 @@ +{% extends "admin-base.html" %} + +{% block content %} +
+

{{ _("Compose new entry") }}

+
+ {{ xsrf_form_html() }} + + + + + + + + + + + + + + +
{{ _("Title") }}
  + +
+ + +
+
+
+ +
+{% end block %} + +{% block javascript %} + +{% end block %} + +{% block sidebar %} +

Syntax info

+ +{% end block %} diff --git a/www/templates/admin-planet.html b/www/templates/admin-planet.html new file mode 100644 index 00000000..d26caa4a --- /dev/null +++ b/www/templates/admin-planet.html @@ -0,0 +1,26 @@ +{% extends "admin-base.html" %} + +{% block content %} +
+

{{ _("Planet Administrator") }}

+ +

+ {{ _("Compose new entry") }} +

+ + + + + + + + {% for entry in entries %} + + + + + + {% end %} +
{{ _("Author") }}{{ _("Title") }} 
{{ entry.author.realname }}{{ entry.title }}{{ _("Edit") }}
+
+{% end %} diff --git a/www/templates/downloads-mirrors.html b/www/templates/downloads-mirrors.html index 5058f99e..6e1b8056 100644 --- a/www/templates/downloads-mirrors.html +++ b/www/templates/downloads-mirrors.html @@ -47,29 +47,32 @@ {{ _("Owner (Hostname)") }} {{ _("Location") }} - {{ _("Latency") }} + {{ _("Last update") }} + {{ _("Contains") }} {% for mirror in mirrors.all %} - - {{ mirror.name }} ({{ mirror.hostname }}) + + {{ mirror.owner }} ({{ mirror.hostname }}) {{ mirror.location[ {{ mirror.location["country"] }}, {{ mirror.location["city"] }} - {{ mirror.latency }} ms + + + {{ mirror.filelist_compare(mirrors.master.files) }}% {% end %} -   +   {{ _("Legend") }}: - {{ _("Server is reachable") }} + {{ _("Server is okay") }}   - {{ _("Server is unreachable") }} + {{ _("Server is outdated") }} diff --git a/www/webapp.py b/www/webapp.py index 41e53caa..bb5c1673 100755 --- a/www/webapp.py +++ b/www/webapp.py @@ -1,20 +1,43 @@ #!/usr/bin/python2.6 +import daemon +import logging +import logging.handlers +import os +import signal +import sys + import tornado.httpserver import tornado.ioloop +import tornado.options from webapp import Application -application = Application() + +tornado.options.parse_command_line() + +def setupLogging(): + formatter = logging.Formatter("%(asctime)s %(levelname)8s %(message)s") + + #handler = logging.handlers.RotatingFileHandler("webapp.log", + # maxBytes=10*1024**2, backupCount=5) + handler = logging.FileHandler("webapp.log") + + handler.setFormatter(formatter) + logging.getLogger().addHandler(handler) if __name__ == "__main__": - http_server = tornado.httpserver.HTTPServer(application, xheaders=True) - http_server.listen(8001) + setupLogging() + app = Application() + + context = daemon.DaemonContext( + working_directory=os.getcwd(), + stdout=sys.stdout, stderr=sys.stderr, # XXX causes errors... + ) - try: - tornado.ioloop.IOLoop.instance().start() - except: - # Shutdown mirror monitoring - from webapp.mirrors import mirrors - mirrors.shutdown() + context.signal_map = { + signal.SIGHUP : app.reload, + signal.SIGTERM : app.stop, + } - raise + with context: + app.run() diff --git a/www/webapp/__init__.py b/www/webapp/__init__.py index 200c0974..82478dd3 100644 --- a/www/webapp/__init__.py +++ b/www/webapp/__init__.py @@ -1,22 +1,22 @@ #/usr/bin/python +import logging import os.path import simplejson -simplejson._default_decoder = simplejson.JSONDecoder(encoding="latin-1") - +import tornado.httpserver import tornado.locale import tornado.options import tornado.web -from db import HashDatabase, UserDatabase +import datastore + from handlers import * from ui_modules import * BASEDIR = os.path.join(os.path.dirname(__file__), "..") tornado.locale.load_translations(os.path.join(BASEDIR, "translations")) -tornado.options.enable_pretty_logging() class Application(tornado.web.Application): def __init__(self): @@ -42,10 +42,7 @@ class Application(tornado.web.Application): tornado.web.Application.__init__(self, **settings) - # Initialize database connections - self.hash_db = HashDatabase() - self.planet_db = tornado.database.Connection("172.28.1.150", "planet", user="planet") - self.user_db = UserDatabase() + self.ds = datastore.DataStore(self) self.settings["static_path"] = static_path = os.path.join(BASEDIR, "static") static_handlers = [ @@ -136,6 +133,41 @@ class Application(tornado.web.Application): (r".*", tornado.web.RedirectHandler, { "url" : "http://www.ipfire.org" }) ]) + logging.info("Successfully initialied application") + + self.__running = True + def __del__(self): - from mirrors import mirrors - mirrors.stop() + logging.info("Shutting down application") + + @property + def db(self): + return self.ds.db + + @property + def ioloop(self): + return tornado.ioloop.IOLoop.instance() + + def stop(self, *args): + logging.debug("Caught shutdown signal") + self.ioloop.stop() + + self.__running = False + + def run(self, port=8001): + logging.debug("Going to background") + + http_server = tornado.httpserver.HTTPServer(self, xheaders=True) + http_server.listen(port) + + self.ioloop.start() + + while self.__running: + time.sleep(1) + + self.ds.trigger() + + def reload(self): + logging.debug("Caught reload signal") + + self.ds.reload() diff --git a/www/webapp/banners.py b/www/webapp/banners.py deleted file mode 100644 index 65aeac8d..00000000 --- a/www/webapp/banners.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/python - -import random - -from helpers import Item, _stringify, json_loads - -class Banners(object): - def __init__(self, filename=None): - self.items = [] - - if filename: - self.load(filename) - - def load(self, filename): - f = open(filename) - data = f.read() - f.close() - - for item in json_loads(data): - self.items.append(Item(**_stringify(item))) - - def get(self): - if self.items: - return random.choice(self.items) - - -banners = Banners("banners.json") diff --git a/www/webapp/datastore/__init__.py b/www/webapp/datastore/__init__.py new file mode 100644 index 00000000..de5051fd --- /dev/null +++ b/www/webapp/datastore/__init__.py @@ -0,0 +1,40 @@ +#!/usr/bin/python + +import banners +#import builds +#import info +import menu +#import mirrors +import news +import releases +import tracker + +from connections import * + +class Databases(object): + def __init__(self, application): + self.application = application + + self.hashes = HashConnection() + self.planet = PlanetConnection() + self.tracker = TrackerConnection() + + +class DataStore(object): + def __init__(self, application): + self.application = application + self.db = Databases(self.application) + + self.reload() + + def reload(self): + self.banners = banners.Banners(self.application, "banners.json") +# self.info = info.Info(self.application, "info.json") + self.menu = menu.Menu(self.application, "menu.json") +# self.mirrors = mirrors.Mirrors(self.application, "mirrors.json") + self.news = news.News(self.application, "news.json") + self.releases = releases.Releases(self.application, "releases.json") + self.tracker = tracker.Tracker(self.application) + + def trigger(self): + pass diff --git a/www/webapp/datastore/banners.py b/www/webapp/datastore/banners.py new file mode 100644 index 00000000..566ba388 --- /dev/null +++ b/www/webapp/datastore/banners.py @@ -0,0 +1,22 @@ +#!/usr/bin/python + +import random +import simplejson + +from tornado.database import Row + +class Banners(object): + def __init__(self, application, filename=None): + self.application = application + self.items = [] + + if filename: + self.load(filename) + + def load(self, filename): + with open(filename) as f: + self.items = [Row(i) for i in simplejson.load(f)] + + def get(self): + if self.items: + return random.choice(self.items) diff --git a/www/webapp/builds.py b/www/webapp/datastore/builds.py similarity index 98% rename from www/webapp/builds.py rename to www/webapp/datastore/builds.py index 49dbaceb..e94eb0b7 100644 --- a/www/webapp/builds.py +++ b/www/webapp/datastore/builds.py @@ -4,9 +4,8 @@ import os import time from helpers import size -from info import info -def find(): +def find(info): ret = [] for item in info["nightly_builds"]: path = item.get("path", None) diff --git a/www/webapp/datastore/config.py b/www/webapp/datastore/config.py new file mode 100644 index 00000000..b44d0fa6 --- /dev/null +++ b/www/webapp/datastore/config.py @@ -0,0 +1,4 @@ +#/usr/bin/python + +class Config(object): + pass diff --git a/www/webapp/datastore/connections.py b/www/webapp/datastore/connections.py new file mode 100644 index 00000000..34a2089e --- /dev/null +++ b/www/webapp/datastore/connections.py @@ -0,0 +1,19 @@ +#!/usr/bin/python + +import tornado.database + +MYSQL_SERVER = "mysql.ipfire.org" + +class HashConnection(tornado.database.Connection): + def __init__(self): + tornado.database.Connection.__init__(self, MYSQL_SERVER, "hashes", user="webapp") + + +class PlanetConnection(tornado.database.Connection): + def __init__(self): + tornado.database.Connection.__init__(self, MYSQL_SERVER, "planet", user="webapp") + + +class TrackerConnection(tornado.database.Connection): + def __init__(self): + tornado.database.Connection.__init__(self, MYSQL_SERVER, "tracker", user="webapp") diff --git a/www/webapp/info.py b/www/webapp/datastore/info.py similarity index 74% rename from www/webapp/info.py rename to www/webapp/datastore/info.py index 3ea75b0c..27f55cc0 100644 --- a/www/webapp/info.py +++ b/www/webapp/datastore/info.py @@ -3,7 +3,8 @@ from helpers import json_loads class Info(dict): - def __init__(self, filename): + def __init__(self, application, filename): + self.application = application self.load(filename) def load(self, filename): @@ -11,6 +12,3 @@ class Info(dict): for key, val in json_loads(f.read()).items(): self[key] = val f.close() - - -info = Info("info.json") diff --git a/www/webapp/datastore/menu.py b/www/webapp/datastore/menu.py new file mode 100644 index 00000000..3f7d34ca --- /dev/null +++ b/www/webapp/datastore/menu.py @@ -0,0 +1,23 @@ +#!/usr/bin/python + +import simplejson + +from tornado.database import Row + +class Menu(object): + def __init__(self, application, filename=None): + self.application = application + self.items = {} + + if filename: + self.load(filename) + + def load(self, filename): + with open(filename) as f: + for url, items in simplejson.load(f).items(): + self.items[url] = [] + for item in items: + self.items[url].append(Row(item)) + + def get(self, url): + return self.items.get(url, []) diff --git a/www/webapp/datastore/mirrors.py b/www/webapp/datastore/mirrors.py new file mode 100644 index 00000000..f801f7e6 --- /dev/null +++ b/www/webapp/datastore/mirrors.py @@ -0,0 +1,232 @@ +#!/usr/bin/python + +import tornado.httpclient + +import GeoIP +import random +import time + +from threading import Thread + +from helpers import Item, _stringify, ping, json_loads + +import logging + +geoip_cache = GeoIP.new(GeoIP.GEOIP_MEMORY_CACHE) + +continents = { + "america" : [ "us", ], + "asia" : [ "cn", "jp", ], + "australia" : [ "au", ], + "europe" : [ "at", "de", "es", "fr", ], +} + +def continent_by_ip(ip): + country = geoip_cache.country_code_by_addr(ip) + if not country: + return + + country = country.lower() + + for continent, countries in continents.items(): + if not country in countries: + continue + return continent + +class Mirrors(object): + def __init__(self, application, filename): + self.application = application + self.items = [] + self.load(filename) + + def load(self, filename): + f = open(filename) + data = f.read() + f.close() + + for item in json_loads(data): + self.items.append(MirrorItem(**_stringify(item))) + + @property + def master(self): + return self.items[0] + + @property + def all(self): + return sorted(self.items) + + @property + def random(self): + mirrors = self.get() + random.shuffle(mirrors) + return mirrors + + @property + def reachable(self): + return self.filter_reachable(self.all) + + @property + def unreachable(self): + return self.filter_unreachable(self.all) + + def filter_continent(self, mirrors, continent): + try: + countries = continents[continent] + except KeyError: + return [] + + ret = [] + for mirror in mirrors: + if mirror.location["country_code"] in countries: + ret.append(mirror) + + return ret + + def filter_file(self, mirrors, path): + ret = [] + for mirror in mirrors: + if not mirror["serves"]["isos"]: + continue + if path in mirror.files: + ret.append(mirror) + return ret + + def filter_reachable(self, mirrors): + return [m for m in mirrors if m.reachable] + + def filter_unreachable(self, mirrors): + return [m for m in mirrors if not m.reachable] + + def get(self, continent=None, file=None, unreachable=False, reachable=False): + mirrors = self.all + + if continent: + mirrors = self.filter_continent(mirrors, continent) + if file: + mirrors = self.filter_file(mirrors, file) + if reachable: + mirrors = self.filter_reachable(mirrors) + if unreachable: + mirrors = self.filter_unreachable(mirrors) + + return mirrors + + def pickone(self, **kwargs): + if kwargs.has_key("ip"): + kwargs["continent"] = continent_by_ip(kwargs.pop("ip")) + + mirrors = self.get(**kwargs) + + # If we did not get any mirrors we try a global one + if not mirrors: + del kwargs["continent"] + mirrors = self.get(**kwargs) + + if not mirrors: + return None + + return random.choice(mirrors) + + # To be removed + def with_file(self, path): + return self.get(file=path) + + trigger_counter = 0 + + def trigger(self): + if not self.trigger_counter: + self.trigger_counter = 300 + + for mirror in self.all: + mirror.update() + + self.trigger_counter -= 1 + + +class MirrorItem(object): + def __init__(self, **kwargs): + self.items = kwargs + + self.files = [] + self.timestamp = 0 + + logging.debug("Initialized mirror %s" % self.hostname) + + def __cmp__(self, other): + return cmp(self.owner, other.owner) + + def __repr__(self): + return "<%s %s>" % (self.__class__.__name__, self.hostname) + + def __getattr__(self, key): + try: + return self.items[key] + except KeyError: + raise AttributeError + + def html_class(self): + if time.time() - self.timestamp > 60*60*24: + return "outdated" + return "ok" + + def update(self): + self.update_filelist() + self.update_timestamp() + + def update_filelist(self): + http = tornado.httpclient.AsyncHTTPClient() + + http.fetch(self.url + ".filelist", + callback=self.__update_filelist_response) + + def __update_filelist_response(self, response): + if response.error: + logging.debug("Error getting filelist from %s" % self.hostname) + return + + if not response.code == 200: + return + + logging.debug("Got filelist from %s" % self.hostname) + + self.files = [] + + for line in response.body.splitlines(): + self.files.append(line) + + def update_timestamp(self): + http = tornado.httpclient.AsyncHTTPClient() + + http.fetch(self.url + ".timestamp", + callback=self.__update_timestamp_response) + + def __update_timestamp_response(self, response): + if response.error: + logging.debug("Error getting timestamp from %s" % self.hostname) + + data = response.body.strip() + try: + self.timestamp = int(data) + except ValueError: + self.timestamp = 0 + + @property + def url(self): + ret = "http://" + self.hostname + if not self.path.startswith("/"): + ret += "/" + ret += self.path + if not ret.endswith("/"): + ret += "/" + return ret + + def filelist_compare(self, other_files): + own_files = [f for f in self.files if f in other_files] + + try: + return (len(own_files) * 100) / len(other_files) + except ZeroDivisionError: + return 0 + + def has_file(self, path): + return path in self.files diff --git a/www/webapp/news.py b/www/webapp/datastore/news.py similarity index 65% rename from www/webapp/news.py rename to www/webapp/datastore/news.py index 65257808..2ebb2564 100644 --- a/www/webapp/news.py +++ b/www/webapp/datastore/news.py @@ -1,9 +1,12 @@ #!/usr/bin/python -from helpers import Item, _stringify, json_loads +import simplejson + +from tornado.database import Row class News(object): - def __init__(self, filename=None): + def __init__(self, application, filename=None): + self.application = application self.items = [] if filename: @@ -16,10 +19,10 @@ class News(object): data = data.replace("\n", "").replace("\t", " ") - json = json_loads(data) + json = simplejson.loads(data) for key in sorted(json.keys()): json[key]["id"] = key - self.items.append(NewsItem(**_stringify(json[key]))) + self.items.append(Row(json[key])) def get(self, limit=None): ret = self.items[:] @@ -27,8 +30,3 @@ class News(object): if limit: ret = ret[:limit] return ret - - -NewsItem = Item - -news = News("news.json") diff --git a/www/webapp/releases.py b/www/webapp/datastore/releases.py similarity index 77% rename from www/webapp/releases.py rename to www/webapp/datastore/releases.py index e065296c..86e005c6 100644 --- a/www/webapp/releases.py +++ b/www/webapp/datastore/releases.py @@ -1,8 +1,10 @@ #!/usr/bin/python -from helpers import Item, _stringify, json_loads +import simplejson -class ReleaseItem(Item): +from tornado.database import Row + +class ReleaseItem(Row): options = { "iso" : { "prio" : 10, @@ -48,12 +50,18 @@ class ReleaseItem(Item): }, } + def __init__(self, info): + self.info = info + + def __getattr__(self, key): + return self.info[key] + @property def downloads(self): ret = [] - for fileitem in self.args["files"]: + for fileitem in self.info["files"]: filetype = fileitem["type"] - ret.append(Item( + ret.append(Row( desc = self.options[filetype]["desc"], file = fileitem["name"], hash = fileitem.get("hash", None), @@ -66,21 +74,6 @@ class ReleaseItem(Item): ret.sort(lambda a, b: cmp(a.prio, b.prio)) return ret - - for option in self.options.keys(): - if not self.args["files"].has_key(option): - continue - - ret.append(Item( - desc = self.options[option]["desc"], - file = self.args["files"][option], - prio = self.options[option]["prio"], - type = option, - url = self.options[option]["url"] + self.args["files"][option], - )) - - ret.sort(lambda a, b: cmp(a.prio, b.prio)) - return ret @property def iso(self): @@ -104,19 +97,16 @@ class ReleaseItem(Item): class Releases(object): - def __init__(self, filename="releases.json"): + def __init__(self, application, filename="releases.json"): + self.application = application self.items = [] if filename: self.load(filename) def load(self, filename): - f = open(filename) - data = f.read() - f.close() - - for item in json_loads(data): - self.items.append(ReleaseItem(**_stringify(item))) + with open(filename) as f: + self.items = [ReleaseItem(i) for i in simplejson.load(f)] @property def all(self): @@ -171,15 +161,3 @@ class Releases(object): if item.torrent: ret.append(item) return ret - - -releases = Releases("releases.json") - -if __name__ == "__main__": - r = Releases() - - print r.stable - print r.development - print r.latest - print r.online - print r.offline diff --git a/www/webapp/torrent.py b/www/webapp/datastore/tracker.py similarity index 97% rename from www/webapp/torrent.py rename to www/webapp/datastore/tracker.py index b005dc74..c1572148 100644 --- a/www/webapp/torrent.py +++ b/www/webapp/datastore/tracker.py @@ -2,9 +2,6 @@ import time -import tornado.database - - def decode_hex(s): ret = [] for c in s: @@ -25,12 +22,12 @@ class Tracker(object): numwant = 50 - def __init__(self): - self.db = tornado.database.Connection( - host="172.28.1.150", - database="tracker", - user="tracker", - ) + def __init__(self, application): + self.application = application + + @property + def db(self): + return self.application.db.tracker def _fetch(self, hash, limit=None, random=False, completed=False, no_peer_id=False): query = "SELECT * FROM peers WHERE last_update >= %d" % self.since @@ -146,9 +143,6 @@ class Tracker(object): return int(time.time() - self.interval) -tracker = Tracker() - - ##### This is borrowed from the bittorrent client libary ##### def decode_int(x, f): diff --git a/www/webapp/db.py b/www/webapp/db.py index fdab672d..c4661965 100644 --- a/www/webapp/db.py +++ b/www/webapp/db.py @@ -131,3 +131,12 @@ class User(object): @property def realname(self): return self.obj["cn"][0] + + +class Databases(object): + def __init__(self, application): + self.application = application + + self.hashes = HashDatabase() + self.planet = PlanetDatabase() + self.users = UserDatabase() diff --git a/www/webapp/handlers.py b/www/webapp/handlers.py index 1eb5b55b..eaa5ec12 100644 --- a/www/webapp/handlers.py +++ b/www/webapp/handlers.py @@ -18,19 +18,10 @@ import tornado.httpclient import tornado.locale import tornado.web -from banners import banners -from helpers import size, Item -from info import info -from mirrors import mirrors -from news import news -from releases import releases -from torrent import tracker, bencode, bdecode, decode_hex - -import builds -import menu -import cluster +from helpers import size +from datastore.tracker import bencode, bdecode, decode_hex + import markdown -#import uriel class BaseHandler(tornado.web.RequestHandler): def get_user_locale(self): @@ -43,12 +34,14 @@ class BaseHandler(tornado.web.RequestHandler): @property def render_args(self): return { - "banner" : banners.get(), + "banner" : self.ds.banners.get(), + "hostname" : self.request.host, "lang" : self.locale.code[:2], "langs" : [l[:2] for l in tornado.locale.get_supported_locales(None)], "lang_link" : self.lang_link, "link" : self.link, "title" : "no title given", + "time_ago" : self.time_ago, "server" : self.request.host.replace("ipfire", "ipfire"), "uri" : self.request.uri, "year" : time.strftime("%Y"), @@ -57,7 +50,6 @@ class BaseHandler(tornado.web.RequestHandler): def render(self, *args, **kwargs): nargs = self.render_args nargs.update(kwargs) - nargs["hostname"] = self.request.host tornado.web.RequestHandler.render(self, *args, **nargs) def link(self, s): @@ -66,6 +58,18 @@ class BaseHandler(tornado.web.RequestHandler): def lang_link(self, lang): return "/%s/%s" % (lang, self.request.uri[4:]) + def time_ago(self, stamp): + if not stamp: + return "N/A" + + ago = time.time() - stamp + for unit in ("s", "m", "h", "d", "M"): + if ago < 60: + break + ago /= 60 + + return "<%d%s" % (ago, unit) + def get_error_html(self, status_code, **kwargs): if status_code in (404, 500): render_args = self.render_args @@ -79,16 +83,12 @@ class BaseHandler(tornado.web.RequestHandler): return tornado.web.RequestHandler.get_error_html(self, status_code, **kwargs) @property - def hash_db(self): - return self.application.hash_db - - @property - def planet_db(self): - return self.application.planet_db + def db(self): + return self.application.db @property - def user_db(self): - return self.application.user_db + def ds(self): + return self.application.ds class MainHandler(BaseHandler): @@ -99,17 +99,17 @@ class MainHandler(BaseHandler): class DownloadHandler(BaseHandler): def get(self): - self.render("downloads.html", release=releases.latest) + self.render("downloads.html", release=self.ds.releases.latest) class DownloadAllHandler(BaseHandler): def get(self): - self.render("downloads-all.html", releases=releases) + self.render("downloads-all.html", releases=self.ds.releases) class DownloadDevelopmentHandler(BaseHandler): def get(self): - self.render("downloads-development.html", releases=releases) + self.render("downloads-development.html", releases=self.ds.releases) class DownloadTorrentHandler(BaseHandler): @@ -121,7 +121,7 @@ class DownloadTorrentHandler(BaseHandler): http.fetch(self.tracker_url, callback=self.async_callback(self.on_response)) def on_response(self, response): - torrents = releases.torrents + torrents = self.ds.releases.torrents hashes = {} if response.code == 200: for line in response.body.split("\n"): @@ -140,9 +140,29 @@ class DownloadTorrentHandler(BaseHandler): tracker=urlparse.urlparse(response.request.url).netloc) +class DownloadTorrentHandler(BaseHandler): + @property + def tracker(self): + return self.ds.tracker + + def get(self): + releases = self.ds.releases.torrents + + hashes = {} + for hash in [release.torrent.hash for release in releases if release.torrent]: + hashes[hash] = { + "peers" : self.tracker.get_peers(hash), + "seeds" : self.tracker.get_seeds(hash), + } + + self.render("downloads-torrents.html", + hashes=hashes, + releases=releases) + + class DownloadMirrorHandler(BaseHandler): def get(self): - self.render("downloads-mirrors.html", mirrors=mirrors) + self.render("downloads-mirrors.html", mirrors=self.ds.mirrors) class StaticHandler(BaseHandler): @@ -169,12 +189,12 @@ class StaticHandler(BaseHandler): class IndexHandler(BaseHandler): def get(self): - self.render("index.html", news=news) + self.render("index.html", news=self.ds.news) class NewsHandler(BaseHandler): def get(self): - self.render("news.html", news=news) + self.render("news.html", news=self.ds.news) class BuildHandler(BaseHandler): @@ -185,7 +205,7 @@ class BuildHandler(BaseHandler): ">24h" : [], } - for build in builds.find(): + for build in self.ds.builds.find(self.ds.info): if (time.time() - float(build.get("date"))) < 12*60*60: self.builds["<12h"].append(build) elif (time.time() - float(build.get("date"))) < 24*60*60: @@ -221,7 +241,7 @@ class SourceHandler(BaseHandler): if file in [f["name"] for f in fileobjects]: continue - hash = self.hash_db.get_hash(os.path.join(dir, file)) + hash = self.db.hash.get_hash(os.path.join(dir, file)) if not hash: hash = "0000000000000000000000000000000000000000" @@ -262,7 +282,7 @@ class SourceDownloadHandler(BaseHandler): if mime_type: self.set_header("Content-Type", mime_type) - hash = self.hash_db.get_hash(path) + hash = self.db.hash.get_hash(path) if hash: self.set_header("X-Hash-Sha1", "%s" % hash) @@ -277,7 +297,7 @@ class SourceDownloadHandler(BaseHandler): class DownloadFileHandler(BaseHandler): def get(self, path): - for mirror in mirrors.with_file(path): + for mirror in self.ds.mirrors.with_file(path): if not mirror.reachable: continue @@ -304,7 +324,6 @@ class RSSHandler(BaseHandler): self.render("rss.xml", items=items, lang=lang) -<<<<<<< HEAD class TrackerBaseHandler(tornado.web.RequestHandler): def get_hexencoded_argument(self, name, all=False): try: @@ -327,6 +346,7 @@ class TrackerBaseHandler(tornado.web.RequestHandler): self.write(bencode({"failure reason" : error_message })) self.finish() + class TrackerAnnounceHandler(TrackerBaseHandler): def get(self): self.set_header("Content-Type", "text/plain") @@ -395,11 +415,12 @@ class TrackerScrapeHandler(TrackerBaseHandler): self.write(bencode(tracker.scrape(hashes=info_hashes))) self.finish() -======= + + class PlanetBaseHandler(BaseHandler): @property def db(self): - return self.application.planet_db + return self.db.planet class PlanetMainHandler(PlanetBaseHandler): @@ -445,4 +466,120 @@ class PlanetPostingHandler(PlanetBaseHandler): entry.author = user self.render("planet-posting.html", entry=entry, user=user) ->>>>>>> planet + + +class AdminBaseHandler(BaseHandler): + def render(self, *args, **kwargs): + + return BaseHandler.render(self, *args, **kwargs) + + +class AdminIndexHandler(AdminBaseHandler): + def get(self): + self.render("admin-index.html") + + +class AdminApiPlanetRenderMarkupHandler(AdminBaseHandler): + def get(self): + text = self.get_argument("text", "") + + # Render markup + self.write(markdown.markdown(text)) + self.finish() + + +class AdminPlanetHandler(AdminBaseHandler): + def get(self): + entries = self.planet_db.query("SELECT * FROM entries ORDER BY published DESC") + + for entry in entries: + entry.author = self.user_db.get_user_by_id(entry.author_id) + + self.render("admin-planet.html", entries=entries) + + +class AdminPlanetComposeHandler(AdminBaseHandler): + #@tornado.web.authenticated + def get(self, id=None): + if id: + entry = self.planet_db.get("SELECT * FROM entries WHERE id = '%s'", int(id)) + else: + entry = tornado.database.Row(id="", title="", text="") + + self.render("admin-planet-compose.html", entry=entry) + + #@tornado.web.authenticated + def post(self, id=None): + id = self.get_argument("id", id) + title = self.get_argument("title") + text = self.get_argument("text") + + if id: + entry = self.planet_db.get("SELECT * FROM entries WHERE id = %s", id) + if not entry: + raise tornado.web.HTTPError(404) + + self.planet_db.execute("UPDATE entries SET title = %s, text = %s " + "WHERE id = %s", title, text, id) + + slug = entry.slug + + else: + slug = unicodedata.normalize("NFKD", title).encode("ascii", "ignore") + slug = re.sub(r"[^\w]+", " ", slug) + slug = "-".join(slug.lower().strip().split()) + + if not slug: + slug = "entry" + + while True: + e = self.planet_db.get("SELECT * FROM entries WHERE slug = %s", slug) + if not e: + break + slug += "-" + + self.planet_db.execute("INSERT INTO entries(author_id, title, slug, text, published) " + "VALUES(%s, %s, %s, %s, UTC_TIMESTAMP())", 500, title, slug, text) + + self.redirect("/planet") + + +class AdminPlanetEditHandler(AdminPlanetComposeHandler): + pass + + +class AdminAccountsHandler(AdminBaseHandler): + def get(self): + users = self.user_db.users + + self.render("admin-accounts.html", accounts=users) + + +class AdminAccountsEditHandler(AdminBaseHandler): + def get(self, id): + user = self.user_db.get_user_by_id(id) + + if not user: + raise tornado.web.HTTPError(404) + + self.render("admin-accounts-edit.html", user=user) + + +class AuthLoginHandler(BaseHandler): + def get(self): + self.render("admin-login.html") + + def post(self): + #name = self.get_attribute("name") + #password = self.get_attribute("password") + + pass + + #if self.user_db.check_password(name, password): + # self.set_secure_cookie("user", int(user.id)) + + +class AuthLogoutHandler(BaseHandler): + def get(self): + self.clear_cookie("user") + self.redirect("/") diff --git a/www/webapp/helpers.py b/www/webapp/helpers.py index d634fcb9..5e4b25db 100644 --- a/www/webapp/helpers.py +++ b/www/webapp/helpers.py @@ -3,19 +3,6 @@ import simplejson import subprocess -class Item(object): - def __init__(self, **args): - self.args = args - - def __getattr__(self, key): - return self.args[key] - - def __getitem__(self, key): - return self.args[key] - - def __setitem__(self, key, val): - self.args[key] = val - def size(s): suffixes = ["B", "K", "M", "G", "T",] @@ -26,15 +13,6 @@ def size(s): return "%.0f%s" % (s, suffixes[idx]) -def json_loads(s): - return simplejson.loads(s.decode("utf-8")) - -def _stringify(d): - ret = {} - for key in d.keys(): - ret[str(key)] = d[key] - return ret - def ping(host, count=5, wait=10): cmd = subprocess.Popen( ["ping", "-c%d" % count, "-w%d" % wait, host], @@ -61,8 +39,3 @@ def ping(host, count=5, wait=10): latency = "%.1f" % float(rtts[1]) return latency - - -if __name__ == "__main__": - print ping("www.ipfire.org") - print ping("www.rowie.at") diff --git a/www/webapp/menu.py b/www/webapp/menu.py deleted file mode 100644 index bce95a1a..00000000 --- a/www/webapp/menu.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/python - -import simplejson - -from helpers import Item, _stringify - -class Menu(object): - def __init__(self, filename=None): - self.items = {} - - if filename: - self.load(filename) - - def load(self, filename): - f = open(filename) - data = f.read() - f.close() - - for url, items in simplejson.loads(data).items(): - self.items[url] = [] - for item in items: - self.items[url].append(MenuItem(**_stringify(item))) - - def get(self, url): - return self.items.get(url, []) - - -class MenuItem(Item): - def __init__(self, **args): - Item.__init__(self, **args) - self.active = False - - # Parse submenu - if self.args.has_key("subs"): - self.args["items"] = [] - for sub in self.args["subs"]: - self.args["items"].append(MenuItem(**_stringify(sub))) - del self.args["subs"] - diff --git a/www/webapp/mirrors.py b/www/webapp/mirrors.py deleted file mode 100644 index df19627b..00000000 --- a/www/webapp/mirrors.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/python - -import tornado.httpclient - -import random -import threading -import time - -from helpers import Item, _stringify, ping, json_loads - -class Mirrors(threading.Thread): - def __init__(self, filename): - threading.Thread.__init__(self, name="Mirror Monitor") - - self.items = [] - self.load(filename) - - self.__running = True - - self.start() - - def load(self, filename): - f = open(filename) - data = f.read() - f.close() - - for item in json_loads(data): - self.items.append(MirrorItem(**_stringify(item))) - - @property - def all(self): - return sorted(self.items) - - @property - def random(self): - # Doesnt work :( - #return random.shuffle(self.items) - ret = [] - items = self.items[:] - while items: - rnd = random.randint(0, len(items)-1) - ret.append(items.pop(rnd)) - return ret - - @property - def reachable(self): - ret = [] - for mirror in self.items: - if not mirror.reachable: - continue - ret.append(mirror) - return ret - - @property - def unreachable(self): - ret = [] - for mirror in self.all: - if mirror in self.reachable: - continue - ret.append(mirror) - return ret - - def pickone(self, reachable=False): - mirrors = self.items - if reachable: - mirrors = self.reachable - if not mirrors: - return None - return random.choice(mirrors) - - def with_file(self, path): - ret = [] - for mirror in self.random: - if not mirror["serves"]["isos"]: - continue - if path in mirror.files: - ret.append(mirror) - return ret - - def shutdown(self): - self.__running = False - - def run(self): - for mirror in self.random: - if not self.__running: - return - mirror.update() - - count = 0 - while self.__running: - if not count: - count = 300 # 30 secs - mirror = self.pickone() - if mirror: - mirror.update() - - time.sleep(0.1) - count -= 1 - - -class MirrorItem(Item): - def __init__(self, *args, **kwargs): - Item.__init__(self, *args, **kwargs) - - self.filelist = MirrorFilelist(self) - self.latency = "N/A" - - def __cmp__(self, other): - return cmp(self.name, other.name) - - def update(self): - self.latency = ping(self.hostname) or "N/A" - if self.filelist.outdated: - self.filelist.update() - - @property - def reachable(self): - return not self.latency == "N/A" - - @property - def url(self): - ret = "http://" + self.hostname - if not self.path.startswith("/"): - ret += "/" - ret += self.path - if not ret.endswith("/"): - ret += "/" - return ret - - @property - def files(self): - return self.filelist.files - - def has_file(self, path): - return path in self.files - - -class MirrorFilelist(object): - def __init__(self, mirror): - self.mirror = mirror - - self.__files = [] - self.__time = 0 - - #self.update(now=True) - - def update(self, now=False): - args = {} - - if now: - while not self.mirror.reachable: - time.sleep(10) - - http = tornado.httpclient.HTTPClient() - - if not now: - http = tornado.httpclient.AsyncHTTPClient() - args["callback"] = self.on_response - - try: - reponse = http.fetch(self.mirror.url + ".filelist", **args) - except tornado.httpclient.HTTPError: - self.__time = time.time() - return - - if now: - self.on_response(reponse) - - def on_response(self, response): - self.__files = [] - self.__time = time.time() - - if not response.code == 200: - return - - # If invalid html content... - if response.body.startswith(" 60*60 - - @property - def files(self): - #if self.outdated: - # self.update() - return self.__files - - -mirrors = Mirrors("mirrors.json") diff --git a/www/webapp/ui_modules.py b/www/webapp/ui_modules.py index e601051f..43992a48 100644 --- a/www/webapp/ui_modules.py +++ b/www/webapp/ui_modules.py @@ -1,12 +1,9 @@ #!/usr/bin/python -import tornado.web - import markdown -import menu -import releases +import tornado.web -from helpers import Item +from tornado.database import Row class UIModule(tornado.web.UIModule): def render_string(self, *args, **kwargs): @@ -35,10 +32,8 @@ class MenuItemModule(UIModule): class MenuModule(UIModule): - def render(self, menuclass=None): - if not menuclass: - menuclass = menu.Menu("menu.json") - + def render(self): + menuclass = self.handler.application.ds.menu host = self.request.host.lower().split(':')[0] return self.render_string("modules/menu.html", menuitems=menuclass.get(host)) @@ -46,7 +41,7 @@ class MenuModule(UIModule): class NewsItemModule(UIModule): def render(self, item): - item = Item(**item.args.copy()) + item = Row(item.copy()) for attr in ("subject", "content"): if type(item[attr]) != type({}): continue @@ -66,11 +61,9 @@ class SidebarItemModule(UIModule): class SidebarReleaseModule(UIModule): - releases = releases.Releases() - def render(self): return self.render_string("modules/sidebar-release.html", - releases=self.releases) + releases=self.handler.application.ds.releases) class ReleaseItemModule(UIModule):