From a6dc0bad05f27ffd7da7e65ab4688db077cb8e04 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sun, 13 Oct 2013 19:16:40 +0200 Subject: [PATCH] Update RSS feed generation code. --- templates/feeds/base.xml | 23 ++++++ templates/feeds/news.xml | 5 ++ templates/feeds/planet.xml | 5 ++ templates/rss.xml | 25 ------ webapp/__init__.py | 9 +++ webapp/backend/__init__.py | 2 + webapp/backend/accounts.py | 22 +++--- webapp/backend/base.py | 17 ++++ webapp/backend/misc.py | 17 ++++ webapp/backend/planet.py | 125 ++++++++++++++++-------------- webapp/handlers_base.py | 8 +- webapp/handlers_planet.py | 12 +-- webapp/handlers_rss.py | 154 +++++++++++-------------------------- 13 files changed, 211 insertions(+), 213 deletions(-) create mode 100644 templates/feeds/base.xml create mode 100644 templates/feeds/news.xml create mode 100644 templates/feeds/planet.xml delete mode 100644 templates/rss.xml create mode 100644 webapp/backend/base.py diff --git a/templates/feeds/base.xml b/templates/feeds/base.xml new file mode 100644 index 00000000..a0b560f5 --- /dev/null +++ b/templates/feeds/base.xml @@ -0,0 +1,23 @@ + + + + {% block title %}{% end block %} + {% block url %}{% end block %} + {% block description %}{% end block %} + {{ lang }} + {% block copyright %}IPFire Team{% end block %} + Thu, 8 Nov 2007 00:00:00 +0200 + {% for item in items %} + + {{ item.title }} + {{ item.url }} + {{ item.author.email }} ({{ item.author.name }}) + {{ item.url }} + {{ item.published.strftime("%a, %d %b %Y %H:%M:%S +0200") }} + + + + + {% end %} + + diff --git a/templates/feeds/news.xml b/templates/feeds/news.xml new file mode 100644 index 00000000..1b3a860a --- /dev/null +++ b/templates/feeds/news.xml @@ -0,0 +1,5 @@ +{% extends "base.xml" %} + +{% block title %}IPFire.org - News{% end block%} +{% block url %}http://www.ipfire.org/{% end block %} +{% block description %}IPFire News Feed{% end block %} diff --git a/templates/feeds/planet.xml b/templates/feeds/planet.xml new file mode 100644 index 00000000..e42aea9c --- /dev/null +++ b/templates/feeds/planet.xml @@ -0,0 +1,5 @@ +{% extends "base.xml" %} + +{% block title %}planet.ipfire.org{% end block%} +{% block url %}http://planet.ipfire.org/{% end block %} +{% block description %}IPFire Planet Feed{% end block %} diff --git a/templates/rss.xml b/templates/rss.xml deleted file mode 100644 index 1d6ee4b3..00000000 --- a/templates/rss.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - {{ title }} - {{ url }} - {{ description }} - {{ lang }} - IPFire Team - Thu, 8 Nov 2007 00:00:00 +0200 - {% for item in items %} - - {{ escape(item.title) }} - {{ item.url }} - {{ item.author.mail }} ({{ item.author.name }}) - {{ item.url }} - {{ item.date.strftime("%a, %d %b %Y %H:%M:%S +0200") }} - - - - - {% end %} - - diff --git a/webapp/__init__.py b/webapp/__init__.py index 4abd435f..30d83fb8 100644 --- a/webapp/__init__.py +++ b/webapp/__init__.py @@ -22,6 +22,8 @@ tornado.locale.load_gettext_translations(os.path.join(BASEDIR, "translations"), class Application(tornado.web.Application): def __init__(self): + self.__backend = None + settings = dict( cookie_secret = "aXBmaXJlY29va2llc2VjcmV0Cg==", debug = options.debug, @@ -246,6 +248,13 @@ class Application(tornado.web.Application): def __del__(self): logging.info("Shutting down application") + @property + def backend(self): + if self.__backend is None: + self.__backend = backend.Backend() + + return self.__backend + @property def ioloop(self): return tornado.ioloop.IOLoop.instance() diff --git a/webapp/backend/__init__.py b/webapp/backend/__init__.py index 2889e729..17f42d43 100644 --- a/webapp/backend/__init__.py +++ b/webapp/backend/__init__.py @@ -6,6 +6,8 @@ from tornado.options import define, options, parse_command_line define("debug", default=False, help="Run in debug mode", type=bool) parse_command_line() +from base import Backend + from ads import Advertisements from accounts import Accounts from banners import Banners diff --git a/webapp/backend/accounts.py b/webapp/backend/accounts.py index 9be734e9..10976615 100644 --- a/webapp/backend/accounts.py +++ b/webapp/backend/accounts.py @@ -6,17 +6,16 @@ import ldap import logging import urllib -from misc import Singleton +from misc import Object from settings import Settings -class Accounts(object): - __metaclass__ = Singleton - +class Accounts(Object): @property def settings(self): return Settings() - def __init__(self): + def __init__(self, backend): + Object.__init__(self, backend) self.__db = None self._init() @@ -51,7 +50,7 @@ class Accounts(object): for dn, attrs in results: #if attrs["loginShell"] == ["/bin/bash"]: - self._accounts[dn] = Account(dn) + self._accounts[dn] = Account(self.backend, dn) def list(self): return sorted(self._accounts.values()) @@ -68,8 +67,9 @@ class Accounts(object): search = find -class Account(object): - def __init__(self, dn): +class Account(Object): + def __init__(self, backend, dn): + Object.__init__(self, backend) self.dn = dn self.__attributes = {} @@ -82,7 +82,7 @@ class Account(object): @property def db(self): - return Accounts().db + return self.accounts.db @property def attributes(self): @@ -148,6 +148,10 @@ class Account(object): def is_admin(self): return True # XXX todo + @property + def name(self): + return self.cn + @property def email(self): name = self.cn.lower() diff --git a/webapp/backend/base.py b/webapp/backend/base.py new file mode 100644 index 00000000..58ee5ca8 --- /dev/null +++ b/webapp/backend/base.py @@ -0,0 +1,17 @@ +#!/usr/bin/python + +import tornado.database + +import accounts +import planet + +MYSQL_SERVER = "mysql-master.ipfire.org" +MYSQL_DB = "webapp" +MYSQL_USER = "webapp" + +class Backend(object): + def __init__(self): + self.db = tornado.database.Connection(MYSQL_SERVER, MYSQL_DB, user=MYSQL_USER) + + self.accounts = accounts.Accounts(self) + self.planet = planet.Planet(self) diff --git a/webapp/backend/misc.py b/webapp/backend/misc.py index 7279a2e3..22dce10b 100644 --- a/webapp/backend/misc.py +++ b/webapp/backend/misc.py @@ -1,5 +1,22 @@ #!/usr/bin/python +class Object(object): + def __init__(self, backend): + self.backend = backend + + @property + def db(self): + return self.backend.db + + @property + def accounts(self): + return self.backend.accounts + + @property + def planet(self): + return self.backend.planet + + class Singleton(type): """ A class for using the singleton pattern diff --git a/webapp/backend/planet.py b/webapp/backend/planet.py index 0be57123..e8551865 100644 --- a/webapp/backend/planet.py +++ b/webapp/backend/planet.py @@ -8,44 +8,33 @@ import unicodedata from accounts import Accounts from databases import Databases -from misc import Singleton +from misc import Object -class PlanetEntry(object): - def __init__(self, db, entry=None): - self.db = db +class PlanetEntry(Object): + def __init__(self, backend, data): + Object.__init__(self, backend) - if entry: - self.__entry = entry - else: - self.__entry = tornado.database.Row({ - "id" : None, - "title" : "", - "markdown" : "", - "tags" : [], - }) - - def set(self, key, val): - self.__entry[key] = val - - @property - def planet(self): - return Planet() + self.data = data @property def id(self): - return self.__entry.id + return self.data.id @property def slug(self): - return self.__entry.slug + return self.data.slug @property def title(self): - return self.__entry.title + return self.data.title + + @property + def url(self): + return "http://planet.ipfire.org/post/%s" % self.slug @property def published(self): - return self.__entry.published + return self.data.published @property def year(self): @@ -57,11 +46,18 @@ class PlanetEntry(object): @property def updated(self): - return self.__entry.updated + return self.data.updated @property def markdown(self): - return self.__entry.markdown + return self.data.markdown + + @property + def markup(self): + if self.data.markup: + return self.data.markup + + return self.render(self.markdown) @property def abstract(self): @@ -72,11 +68,15 @@ class PlanetEntry(object): @property def text(self): - return self.render(self.markdown) + # Compat for markup + return self.markup @property def author(self): - return Accounts().search(self.__entry.author_id) + if not hasattr(self, "__author"): + self.__author = self.accounts.search(self.data.author_id) + + return self.__author # Tags @@ -104,17 +104,11 @@ class PlanetEntry(object): tags = property(get_tags, set_tags) -class Planet(object): - __metaclass__ = Singleton - - @property - def db(self): - return Databases().webapp - +class Planet(Object): def get_authors(self): authors = [] - for author in self.db.query("SELECT DISTINCT author_id FROM planet"): - author = Accounts().search(author.author_id) + for author in self.db.query("SELECT DISTINCT author_id FROM planet WHERE status = %s", "published"): + author = self.accounts.search(author.author_id) if author: authors.append(author) @@ -122,19 +116,19 @@ class Planet(object): def get_years(self): res = self.db.query("SELECT DISTINCT YEAR(published) AS year \ - FROM planet ORDER BY year DESC") + FROM planet WHERE status = %s ORDER BY year DESC", "published") return [row.year for row in res] def get_entry_by_slug(self, slug): entry = self.db.get("SELECT * FROM planet WHERE slug = %s", slug) if entry: - return PlanetEntry(self.db, entry) + return PlanetEntry(self.backend, entry) def get_entry_by_id(self, id): entry = self.db.get("SELECT * FROM planet WHERE id = %s", id) if entry: - return PlanetEntry(self.db, entry) + return PlanetEntry(self.backend, entry) def _limit_and_offset_query(self, limit=None, offset=None): query = " " @@ -147,38 +141,53 @@ class Planet(object): return query - def get_entries(self, limit=3, offset=None): - query = "SELECT * FROM planet WHERE acknowledged='Y' ORDER BY published DESC" + def get_entries(self, limit=3, offset=None, status="published", author_id=None): + query = "SELECT * FROM planet" + args, clauses = [], [] - # Respect limit and offset - query += self._limit_and_offset_query(limit=limit, offset=offset) + if status: + clauses.append("status = %s") + args.append(status) - entries = [] - for entry in self.db.query(query): - entries.append(PlanetEntry(self.db, entry)) + if author_id: + clauses.append("author_id = %s") + args.append(author_id) - return entries + if clauses: + query += " WHERE %s" % " AND ".join(clauses) - def get_entries_by_author(self, author_id, limit=None, offset=None): - query = "SELECT * FROM planet WHERE author_id = '%s'" % author_id - query += " AND acknowledged='Y' ORDER BY published DESC" + query += " ORDER BY published DESC" + + # Respect limit and offset + if limit: + if offset: + query += " LIMIT %s,%s" + args += [offset, limit,] + else: + query += " LIMIT %s" + args.append(limit) - # Respect limit and offset - query += self._limit_and_offset_query(limit=limit, offset=offset) + entries = [] + for entry in self.db.query(query, *args): + entry = PlanetEntry(self.backend, entry) + entries.append(entry) - entries = self.db.query(query) + return entries - return [PlanetEntry(self.db, e) for e in entries] + def get_entries_by_author(self, author_id, limit=None, offset=None): + return self.get_entries(limit=limit, offset=offset, author_id=author_id) def get_entries_by_year(self, year): entries = self.db.query("SELECT * FROM planet \ - WHERE YEAR(published) = %s ORDER BY published DESC", year) + WHERE status = %s AND YEAR(published) = %s ORDER BY published DESC", + "published", year) - return [PlanetEntry(self.db, e) for e in entries] + return [PlanetEntry(self.backend, e) for e in entries] def render(self, text, limit=0): if limit and len(text) >= limit: text = text[:limit] + "..." + return textile.textile(text) def _generate_slug(self, title): @@ -228,7 +237,7 @@ class Planet(object): args.append(len(tags)) entries = self.db.query(query % " OR ".join(clauses), *args) - return [PlanetEntry(self.db, e) for e in entries] + return [PlanetEntry(self.backend, e) for e in entries] def search_autocomplete(self, what): tags = what.split() diff --git a/webapp/handlers_base.py b/webapp/handlers_base.py index 893ad1c2..9ff886aa 100644 --- a/webapp/handlers_base.py +++ b/webapp/handlers_base.py @@ -103,13 +103,17 @@ class BaseHandler(tornado.web.RequestHandler): return ret + @property + def backend(self): + return self.application.backend + @property def advertisements(self): return backend.Advertisements() @property def accounts(self): - return backend.Accounts() + return self.backend.accounts @property def banners(self): @@ -149,7 +153,7 @@ class BaseHandler(tornado.web.RequestHandler): @property def planet(self): - return backend.Planet() + return self.backend.planet @property def wishlist(self): diff --git a/webapp/handlers_planet.py b/webapp/handlers_planet.py index ec0d8d9b..4a6fe5cb 100644 --- a/webapp/handlers_planet.py +++ b/webapp/handlers_planet.py @@ -4,18 +4,8 @@ import tornado.web from handlers_base import * -import backend - -from backend.databases import Databases - class PlanetBaseHandler(BaseHandler): - @property - def db(self): - return Databases().webapp - - @property - def planet(self): - return backend.Planet() + pass class PlanetMainHandler(PlanetBaseHandler): diff --git a/webapp/handlers_rss.py b/webapp/handlers_rss.py index a30c008f..21ae4acb 100644 --- a/webapp/handlers_rss.py +++ b/webapp/handlers_rss.py @@ -1,5 +1,6 @@ #!/usr/bin/python +import logging import textile import tornado.database @@ -8,147 +9,84 @@ import backend from handlers_base import * class RSSHandler(BaseHandler): - _cache_prefix = "" - - @property - def cache_prefix(self): - return self._cache_prefix + _default_limit = 10 + _default_offset = 0 def prepare(self): self.set_header("Content-Type", "application/rss+xml") - def get(self): - offset = int(self.get_argument("offset", 0)) - limit = int(self.get_argument("limit", 10)) + @property + def limit(self): + value = self.get_argument("limit", None) + + try: + return int(value) + except (TypeError, ValueError): + return self._default_limit + + @property + def offset(self): + value = self.get_argument("offset", None) + + try: + return int(value) + except (TypeError, ValueError): + return self._default_offset + + def get(self, *args, **kwargs): + url = "%s%s" % (self.request.host, self.request.path) rss_id = "rss-%s-locale=%s-limit=%s-offset=%s" % \ - (self.cache_prefix, self.locale.code, limit, offset) + (url, self.locale.code, self.limit, self.offset) - items = self.memcached.get(rss_id) - if not items: - items = self.generate() - - self.memcached.set(rss_id, items, 15) + rss = self.memcached.get(rss_id) + if not rss: + logging.debug("Generating RSS feed (%s)..." % rss_id) + rss = self.generate(*args, **kwargs) - self.render("rss.xml", items=items, title=self.title, - url=self.url, description=self.description) + self.memcached.set(rss_id, rss, 900) + + self.finish(rss) def generate(self): raise NotImplementedError class RSSNewsHandler(RSSHandler): - _cache_prefix = "news" - - title = "IPFire.org - News" - url = "http://www.ipfire.org/" - description = "IPFire News Feed" - def generate(self): - offset = int(self.get_argument("offset", 0)) - limit = int(self.get_argument("limit", 10)) - - news = self.news.get_latest( - locale=self.locale, - limit=limit, - offset=offset, - ) + news = self.news.get_latest(locale=self.locale, + limit=self.limit, offset=self.offset) items = [] for n in news: # Get author information - author = self.get_account(n.author_id) - n.author = tornado.database.Row( - name = author.cn, - mail = author.email, - ) + n.author = self.get_account(n.author_id) # Render text n.text = textile.textile(n.text) item = tornado.database.Row({ - "title" : n.title, - "author" : n.author, - "date" : n.date, - "url" : "http://www.ipfire.org/news/%s" % n.slug, - "text" : n.text, + "title" : n.title, + "author" : n.author, + "published" : n.date, + "url" : "http://www.ipfire.org/news/%s" % n.slug, + "markup" : n.text, }) items.append(item) - return items + return self.render_string("feeds/news.xml", items=items) class RSSPlanetAllHandler(RSSHandler): - _cache_prefix = "planet" - - title = "IPFire.org - Planet" - url = "http://planet.ipfire.org/" - description = "IPFire Planet Feed" - def generate(self): - offset = int(self.get_argument("offset", 0)) - limit = int(self.get_argument("limit", 10)) + items = self.planet.get_entries(limit=self.limit, offset=self.offset) - news = self.planet.get_entries( - limit=limit, - offset=offset, - ) - - items = [] - for n in news: - # Get author information - author = tornado.database.Row( - name = n.author.cn, - mail = n.author.email, - ) - - item = tornado.database.Row({ - "title" : n.title, - "author" : author, - "date" : n.published, - "url" : "http://planet.ipfire.org/post/%s" % n.slug, - "text" : textile.textile(n.text), - }) - items.append(item) - - return items + return self.render_string("feeds/planet.xml", items=items) class RSSPlanetUserHandler(RSSPlanetAllHandler): - @property - def cache_prefix(self): - return "%s-user=%s" % (self._cache_prefix, self.user) - - def get(self, user): - self.user = user - - return RSSPlanetAllHandler.get(self) - - def generate(self): - offset = int(self.get_argument("offset", 0)) - limit = int(self.get_argument("limit", 10)) - - news = self.planet.get_entries_by_author( - self.user, - limit=limit, - offset=offset, - ) - - items = [] - for n in news: - # Get author information - author = tornado.database.Row( - name = n.author.cn, - mail = n.author.email, - ) - - item = tornado.database.Row({ - "title" : n.title, - "author" : author, - "date" : n.published, - "url" : "http://planet.ipfire.org/post/%s" % n.slug, - "text" : textile.textile(n.text), - }) - items.append(item) + def generate(self, user): + items = self.planet.get_entries_by_author(user, + limit=self.limit, offset=self.offset) - return items + return self.render_string("feeds/planet.xml", items=items) -- 2.47.3