--- /dev/null
+recursive-include demos *.py *.yaml *.html *.css *.png *.js *.xml *.sql README
+include tornado/epoll.c
Tornado
=======
Tornado is an open source version of the scalable, non-blocking web server
-and and tools that power FriendFeed. The FriendFeed application is written
-using a web framework that looks a bit like web.py or Google's webapp, but
-with additional tools and optimizations to take advantage of the underlying
-non-blocking infrastructure.
-
-Documentation and downloads are available at http://www.tornadoweb.org/
+and and tools that power FriendFeed. Documentation and downloads are
+available at http://www.tornadoweb.org/
Tornado is licensed under the Apache Licence, Version 2.0
(http://www.apache.org/licenses/LICENSE-2.0.html).
-
Installation
============
To install:
On Ubuntu Linux, you can install the packages with:
sudo apt-get install python-pycurl python-simplejson
-
--- /dev/null
+Running the Tornado AppEngine example
+=====================================
+This example is designed to run in Google AppEngine, so there are a couple
+of steps to get it running. You can download the Google AppEngine Python
+development environment at http://code.google.com/appengine/downloads.html.
+
+1. Link or copy the tornado code directory into this directory:
+
+ ln -s ../../tornado tornado
+
+ AppEngine doesn't use the Python modules installed on this machine.
+ You need to have the 'tornado' module copied or linked for AppEngine
+ to find it.
+
+3. Install and run dev_appserver
+
+ If you don't already have the App Engine SDK, download it from
+ http://code.google.com/appengine/downloads.html
+
+ To start the tornado demo, run the dev server on this directory:
+
+ dev_appserver.py .
+
+4. Visit http://localhost:8080/ in your browser
+
+ If you sign in as an administrator, you will be able to create and
+ edit blog posts. If you sign in as anybody else, you will only see
+ the existing blog posts.
+
+
+If you want to deploy the blog in production:
+
+1. Register a new appengine application and put its id in app.yaml
+
+ First register a new application at http://appengine.google.com/.
+ Then edit app.yaml in this directory and change the "application"
+ setting from "tornado-appenginge" to your new application id.
+
+2. Deploy to App Engine
+
+ If you registered an application id, you can now upload your new
+ Tornado blog by running this command:
+
+ appcfg update .
+
+ After that, visit application_id.appspot.com, where application_id
+ is the application you registered.
+
--- /dev/null
+application: tornado-appengine
+version: 1
+runtime: python
+api_version: 1
+
+handlers:
+- url: /static/
+ static_dir: static
+
+- url: /.*
+ script: blog.py
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import functools
+import markdown
+import os.path
+import re
+import tornado.web
+import tornado.wsgi
+import unicodedata
+import wsgiref.handlers
+
+from google.appengine.api import users
+from google.appengine.ext import db
+
+
+class Entry(db.Model):
+ """A single blog entry."""
+ author = db.UserProperty()
+ title = db.StringProperty(required=True)
+ slug = db.StringProperty(required=True)
+ markdown = db.TextProperty(required=True)
+ html = db.TextProperty(required=True)
+ published = db.DateTimeProperty(auto_now_add=True)
+ updated = db.DateTimeProperty(auto_now=True)
+
+
+def administrator(method):
+ """Decorate with this method to restrict to site admins."""
+ @functools.wraps(method)
+ def wrapper(self, *args, **kwargs):
+ if not self.current_user:
+ if self.request.method == "GET":
+ self.redirect(self.get_login_url())
+ return
+ raise web.HTTPError(403)
+ elif not self.current_user.administrator:
+ if self.request.method == "GET":
+ self.redirect("/")
+ return
+ raise web.HTTPError(403)
+ else:
+ return method(self, *args, **kwargs)
+ return wrapper
+
+
+class BaseHandler(tornado.web.RequestHandler):
+ """Implements Google Accounts authentication methods."""
+ def get_current_user(self):
+ user = users.get_current_user()
+ if user: user.administrator = users.is_current_user_admin()
+ return user
+
+ def get_login_url(self):
+ return users.create_login_url(self.request.uri)
+
+ def render_string(self, template_name, **kwargs):
+ # Let the templates access the users module to generate login URLs
+ return tornado.web.RequestHandler.render_string(
+ self, template_name, users=users, **kwargs)
+
+
+class HomeHandler(BaseHandler):
+ def get(self):
+ entries = db.Query(Entry).order('-published').fetch(limit=5)
+ if not entries:
+ if not self.current_user or self.current_user.administrator:
+ self.redirect("/compose")
+ return
+ self.render("home.html", entries=entries)
+
+
+class EntryHandler(BaseHandler):
+ def get(self, slug):
+ entry = db.Query(Entry).filter("slug =", slug).get()
+ if not entry: raise tornado.web.HTTPError(404)
+ self.render("entry.html", entry=entry)
+
+
+class ArchiveHandler(BaseHandler):
+ def get(self):
+ entries = db.Query(Entry).order('-published')
+ self.render("archive.html", entries=entries)
+
+
+class FeedHandler(BaseHandler):
+ def get(self):
+ entries = db.Query(Entry).order('-published').fetch(limit=10)
+ self.set_header("Content-Type", "application/atom+xml")
+ self.render("feed.xml", entries=entries)
+
+
+class ComposeHandler(BaseHandler):
+ @administrator
+ def get(self):
+ key = self.get_argument("key", None)
+ entry = Entry.get(key) if key else None
+ self.render("compose.html", entry=entry)
+
+ @administrator
+ def post(self):
+ key = self.get_argument("key", None)
+ if key:
+ entry = Entry.get(key)
+ entry.title = self.get_argument("title")
+ entry.markdown = self.get_argument("markdown")
+ entry.html = markdown.markdown(self.get_argument("markdown"))
+ else:
+ title = self.get_argument("title")
+ slug = unicodedata.normalize("NFKD", title).encode(
+ "ascii", "ignore")
+ slug = re.sub(r"[^\w]+", " ", slug)
+ slug = "-".join(slug.lower().strip().split())
+ if not slug: slug = "entry"
+ while True:
+ existing = db.Query(Entry).filter("slug =", slug).get()
+ if not existing or str(existing.key()) == key:
+ break
+ slug += "-2"
+ entry = Entry(
+ author=self.current_user,
+ title=title,
+ slug=slug,
+ markdown=self.get_argument("markdown"),
+ html=markdown.markdown(self.get_argument("markdown")),
+ )
+ entry.put()
+ self.redirect("/entry/" + entry.slug)
+
+
+class EntryModule(tornado.web.UIModule):
+ def render(self, entry):
+ return self.render_string("modules/entry.html", entry=entry)
+
+
+settings = {
+ "blog_title": u"Tornado Blog",
+ "template_path": os.path.join(os.path.dirname(__file__), "templates"),
+ "ui_modules": {"Entry": EntryModule},
+ "xsrf_cookies": True,
+}
+application = tornado.wsgi.WSGIApplication([
+ (r"/", HomeHandler),
+ (r"/archive", ArchiveHandler),
+ (r"/feed", FeedHandler),
+ (r"/entry/([^/]+)", EntryHandler),
+ (r"/compose", ComposeHandler),
+], **settings)
+
+
+def main():
+ wsgiref.handlers.CGIHandler().run(application)
+
+
+if __name__ == "__main__":
+ main()
--- /dev/null
+#!/usr/bin/env python
+# Copyright (c) 2007-2008 ActiveState Corp.
+# License: MIT (http://www.opensource.org/licenses/mit-license.php)
+
+r"""A fast and complete Python implementation of Markdown.
+
+[from http://daringfireball.net/projects/markdown/]
+> Markdown is a text-to-HTML filter; it translates an easy-to-read /
+> easy-to-write structured text format into HTML. Markdown's text
+> format is most similar to that of plain text email, and supports
+> features such as headers, *emphasis*, code blocks, blockquotes, and
+> links.
+>
+> Markdown's syntax is designed not as a generic markup language, but
+> specifically to serve as a front-end to (X)HTML. You can use span-level
+> HTML tags anywhere in a Markdown document, and you can use block level
+> HTML tags (like <div> and <table> as well).
+
+Module usage:
+
+ >>> import markdown2
+ >>> markdown2.markdown("*boo!*") # or use `html = markdown_path(PATH)`
+ u'<p><em>boo!</em></p>\n'
+
+ >>> markdowner = Markdown()
+ >>> markdowner.convert("*boo!*")
+ u'<p><em>boo!</em></p>\n'
+ >>> markdowner.convert("**boom!**")
+ u'<p><strong>boom!</strong></p>\n'
+
+This implementation of Markdown implements the full "core" syntax plus a
+number of extras (e.g., code syntax coloring, footnotes) as described on
+<http://code.google.com/p/python-markdown2/wiki/Extras>.
+"""
+
+cmdln_desc = """A fast and complete Python implementation of Markdown, a
+text-to-HTML conversion tool for web writers.
+"""
+
+# Dev Notes:
+# - There is already a Python markdown processor
+# (http://www.freewisdom.org/projects/python-markdown/).
+# - Python's regex syntax doesn't have '\z', so I'm using '\Z'. I'm
+# not yet sure if there implications with this. Compare 'pydoc sre'
+# and 'perldoc perlre'.
+
+__version_info__ = (1, 0, 1, 14) # first three nums match Markdown.pl
+__version__ = '1.0.1.14'
+__author__ = "Trent Mick"
+
+import os
+import sys
+from pprint import pprint
+import re
+import logging
+try:
+ from hashlib import md5
+except ImportError:
+ from md5 import md5
+import optparse
+from random import random
+import codecs
+
+
+
+#---- Python version compat
+
+if sys.version_info[:2] < (2,4):
+ from sets import Set as set
+ def reversed(sequence):
+ for i in sequence[::-1]:
+ yield i
+ def _unicode_decode(s, encoding, errors='xmlcharrefreplace'):
+ return unicode(s, encoding, errors)
+else:
+ def _unicode_decode(s, encoding, errors='strict'):
+ return s.decode(encoding, errors)
+
+
+#---- globals
+
+DEBUG = False
+log = logging.getLogger("markdown")
+
+DEFAULT_TAB_WIDTH = 4
+
+# Table of hash values for escaped characters:
+def _escape_hash(s):
+ # Lame attempt to avoid possible collision with someone actually
+ # using the MD5 hexdigest of one of these chars in there text.
+ # Other ideas: random.random(), uuid.uuid()
+ #return md5(s).hexdigest() # Markdown.pl effectively does this.
+ return 'md5-'+md5(s).hexdigest()
+g_escape_table = dict([(ch, _escape_hash(ch))
+ for ch in '\\`*_{}[]()>#+-.!'])
+
+
+
+#---- exceptions
+
+class MarkdownError(Exception):
+ pass
+
+
+
+#---- public api
+
+def markdown_path(path, encoding="utf-8",
+ html4tags=False, tab_width=DEFAULT_TAB_WIDTH,
+ safe_mode=None, extras=None, link_patterns=None,
+ use_file_vars=False):
+ text = codecs.open(path, 'r', encoding).read()
+ return Markdown(html4tags=html4tags, tab_width=tab_width,
+ safe_mode=safe_mode, extras=extras,
+ link_patterns=link_patterns,
+ use_file_vars=use_file_vars).convert(text)
+
+def markdown(text, html4tags=False, tab_width=DEFAULT_TAB_WIDTH,
+ safe_mode=None, extras=None, link_patterns=None,
+ use_file_vars=False):
+ return Markdown(html4tags=html4tags, tab_width=tab_width,
+ safe_mode=safe_mode, extras=extras,
+ link_patterns=link_patterns,
+ use_file_vars=use_file_vars).convert(text)
+
+class Markdown(object):
+ # The dict of "extras" to enable in processing -- a mapping of
+ # extra name to argument for the extra. Most extras do not have an
+ # argument, in which case the value is None.
+ #
+ # This can be set via (a) subclassing and (b) the constructor
+ # "extras" argument.
+ extras = None
+
+ urls = None
+ titles = None
+ html_blocks = None
+ html_spans = None
+ html_removed_text = "[HTML_REMOVED]" # for compat with markdown.py
+
+ # Used to track when we're inside an ordered or unordered list
+ # (see _ProcessListItems() for details):
+ list_level = 0
+
+ _ws_only_line_re = re.compile(r"^[ \t]+$", re.M)
+
+ def __init__(self, html4tags=False, tab_width=4, safe_mode=None,
+ extras=None, link_patterns=None, use_file_vars=False):
+ if html4tags:
+ self.empty_element_suffix = ">"
+ else:
+ self.empty_element_suffix = " />"
+ self.tab_width = tab_width
+
+ # For compatibility with earlier markdown2.py and with
+ # markdown.py's safe_mode being a boolean,
+ # safe_mode == True -> "replace"
+ if safe_mode is True:
+ self.safe_mode = "replace"
+ else:
+ self.safe_mode = safe_mode
+
+ if self.extras is None:
+ self.extras = {}
+ elif not isinstance(self.extras, dict):
+ self.extras = dict([(e, None) for e in self.extras])
+ if extras:
+ if not isinstance(extras, dict):
+ extras = dict([(e, None) for e in extras])
+ self.extras.update(extras)
+ assert isinstance(self.extras, dict)
+ self._instance_extras = self.extras.copy()
+ self.link_patterns = link_patterns
+ self.use_file_vars = use_file_vars
+ self._outdent_re = re.compile(r'^(\t|[ ]{1,%d})' % tab_width, re.M)
+
+ def reset(self):
+ self.urls = {}
+ self.titles = {}
+ self.html_blocks = {}
+ self.html_spans = {}
+ self.list_level = 0
+ self.extras = self._instance_extras.copy()
+ if "footnotes" in self.extras:
+ self.footnotes = {}
+ self.footnote_ids = []
+
+ def convert(self, text):
+ """Convert the given text."""
+ # Main function. The order in which other subs are called here is
+ # essential. Link and image substitutions need to happen before
+ # _EscapeSpecialChars(), so that any *'s or _'s in the <a>
+ # and <img> tags get encoded.
+
+ # Clear the global hashes. If we don't clear these, you get conflicts
+ # from other articles when generating a page which contains more than
+ # one article (e.g. an index page that shows the N most recent
+ # articles):
+ self.reset()
+
+ if not isinstance(text, unicode):
+ #TODO: perhaps shouldn't presume UTF-8 for string input?
+ text = unicode(text, 'utf-8')
+
+ if self.use_file_vars:
+ # Look for emacs-style file variable hints.
+ emacs_vars = self._get_emacs_vars(text)
+ if "markdown-extras" in emacs_vars:
+ splitter = re.compile("[ ,]+")
+ for e in splitter.split(emacs_vars["markdown-extras"]):
+ if '=' in e:
+ ename, earg = e.split('=', 1)
+ try:
+ earg = int(earg)
+ except ValueError:
+ pass
+ else:
+ ename, earg = e, None
+ self.extras[ename] = earg
+
+ # Standardize line endings:
+ text = re.sub("\r\n|\r", "\n", text)
+
+ # Make sure $text ends with a couple of newlines:
+ text += "\n\n"
+
+ # Convert all tabs to spaces.
+ text = self._detab(text)
+
+ # Strip any lines consisting only of spaces and tabs.
+ # This makes subsequent regexen easier to write, because we can
+ # match consecutive blank lines with /\n+/ instead of something
+ # contorted like /[ \t]*\n+/ .
+ text = self._ws_only_line_re.sub("", text)
+
+ if self.safe_mode:
+ text = self._hash_html_spans(text)
+
+ # Turn block-level HTML blocks into hash entries
+ text = self._hash_html_blocks(text, raw=True)
+
+ # Strip link definitions, store in hashes.
+ if "footnotes" in self.extras:
+ # Must do footnotes first because an unlucky footnote defn
+ # looks like a link defn:
+ # [^4]: this "looks like a link defn"
+ text = self._strip_footnote_definitions(text)
+ text = self._strip_link_definitions(text)
+
+ text = self._run_block_gamut(text)
+
+ if "footnotes" in self.extras:
+ text = self._add_footnotes(text)
+
+ text = self._unescape_special_chars(text)
+
+ if self.safe_mode:
+ text = self._unhash_html_spans(text)
+
+ text += "\n"
+ return text
+
+ _emacs_oneliner_vars_pat = re.compile(r"-\*-\s*([^\r\n]*?)\s*-\*-", re.UNICODE)
+ # This regular expression is intended to match blocks like this:
+ # PREFIX Local Variables: SUFFIX
+ # PREFIX mode: Tcl SUFFIX
+ # PREFIX End: SUFFIX
+ # Some notes:
+ # - "[ \t]" is used instead of "\s" to specifically exclude newlines
+ # - "(\r\n|\n|\r)" is used instead of "$" because the sre engine does
+ # not like anything other than Unix-style line terminators.
+ _emacs_local_vars_pat = re.compile(r"""^
+ (?P<prefix>(?:[^\r\n|\n|\r])*?)
+ [\ \t]*Local\ Variables:[\ \t]*
+ (?P<suffix>.*?)(?:\r\n|\n|\r)
+ (?P<content>.*?\1End:)
+ """, re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE)
+
+ def _get_emacs_vars(self, text):
+ """Return a dictionary of emacs-style local variables.
+
+ Parsing is done loosely according to this spec (and according to
+ some in-practice deviations from this):
+ http://www.gnu.org/software/emacs/manual/html_node/emacs/Specifying-File-Variables.html#Specifying-File-Variables
+ """
+ emacs_vars = {}
+ SIZE = pow(2, 13) # 8kB
+
+ # Search near the start for a '-*-'-style one-liner of variables.
+ head = text[:SIZE]
+ if "-*-" in head:
+ match = self._emacs_oneliner_vars_pat.search(head)
+ if match:
+ emacs_vars_str = match.group(1)
+ assert '\n' not in emacs_vars_str
+ emacs_var_strs = [s.strip() for s in emacs_vars_str.split(';')
+ if s.strip()]
+ if len(emacs_var_strs) == 1 and ':' not in emacs_var_strs[0]:
+ # While not in the spec, this form is allowed by emacs:
+ # -*- Tcl -*-
+ # where the implied "variable" is "mode". This form
+ # is only allowed if there are no other variables.
+ emacs_vars["mode"] = emacs_var_strs[0].strip()
+ else:
+ for emacs_var_str in emacs_var_strs:
+ try:
+ variable, value = emacs_var_str.strip().split(':', 1)
+ except ValueError:
+ log.debug("emacs variables error: malformed -*- "
+ "line: %r", emacs_var_str)
+ continue
+ # Lowercase the variable name because Emacs allows "Mode"
+ # or "mode" or "MoDe", etc.
+ emacs_vars[variable.lower()] = value.strip()
+
+ tail = text[-SIZE:]
+ if "Local Variables" in tail:
+ match = self._emacs_local_vars_pat.search(tail)
+ if match:
+ prefix = match.group("prefix")
+ suffix = match.group("suffix")
+ lines = match.group("content").splitlines(0)
+ #print "prefix=%r, suffix=%r, content=%r, lines: %s"\
+ # % (prefix, suffix, match.group("content"), lines)
+
+ # Validate the Local Variables block: proper prefix and suffix
+ # usage.
+ for i, line in enumerate(lines):
+ if not line.startswith(prefix):
+ log.debug("emacs variables error: line '%s' "
+ "does not use proper prefix '%s'"
+ % (line, prefix))
+ return {}
+ # Don't validate suffix on last line. Emacs doesn't care,
+ # neither should we.
+ if i != len(lines)-1 and not line.endswith(suffix):
+ log.debug("emacs variables error: line '%s' "
+ "does not use proper suffix '%s'"
+ % (line, suffix))
+ return {}
+
+ # Parse out one emacs var per line.
+ continued_for = None
+ for line in lines[:-1]: # no var on the last line ("PREFIX End:")
+ if prefix: line = line[len(prefix):] # strip prefix
+ if suffix: line = line[:-len(suffix)] # strip suffix
+ line = line.strip()
+ if continued_for:
+ variable = continued_for
+ if line.endswith('\\'):
+ line = line[:-1].rstrip()
+ else:
+ continued_for = None
+ emacs_vars[variable] += ' ' + line
+ else:
+ try:
+ variable, value = line.split(':', 1)
+ except ValueError:
+ log.debug("local variables error: missing colon "
+ "in local variables entry: '%s'" % line)
+ continue
+ # Do NOT lowercase the variable name, because Emacs only
+ # allows "mode" (and not "Mode", "MoDe", etc.) in this block.
+ value = value.strip()
+ if value.endswith('\\'):
+ value = value[:-1].rstrip()
+ continued_for = variable
+ else:
+ continued_for = None
+ emacs_vars[variable] = value
+
+ # Unquote values.
+ for var, val in emacs_vars.items():
+ if len(val) > 1 and (val.startswith('"') and val.endswith('"')
+ or val.startswith('"') and val.endswith('"')):
+ emacs_vars[var] = val[1:-1]
+
+ return emacs_vars
+
+ # Cribbed from a post by Bart Lateur:
+ # <http://www.nntp.perl.org/group/perl.macperl.anyperl/154>
+ _detab_re = re.compile(r'(.*?)\t', re.M)
+ def _detab_sub(self, match):
+ g1 = match.group(1)
+ return g1 + (' ' * (self.tab_width - len(g1) % self.tab_width))
+ def _detab(self, text):
+ r"""Remove (leading?) tabs from a file.
+
+ >>> m = Markdown()
+ >>> m._detab("\tfoo")
+ ' foo'
+ >>> m._detab(" \tfoo")
+ ' foo'
+ >>> m._detab("\t foo")
+ ' foo'
+ >>> m._detab(" foo")
+ ' foo'
+ >>> m._detab(" foo\n\tbar\tblam")
+ ' foo\n bar blam'
+ """
+ if '\t' not in text:
+ return text
+ return self._detab_re.subn(self._detab_sub, text)[0]
+
+ _block_tags_a = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math|ins|del'
+ _strict_tag_block_re = re.compile(r"""
+ ( # save in \1
+ ^ # start of line (with re.M)
+ <(%s) # start tag = \2
+ \b # word break
+ (.*\n)*? # any number of lines, minimally matching
+ </\2> # the matching end tag
+ [ \t]* # trailing spaces/tabs
+ (?=\n+|\Z) # followed by a newline or end of document
+ )
+ """ % _block_tags_a,
+ re.X | re.M)
+
+ _block_tags_b = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math'
+ _liberal_tag_block_re = re.compile(r"""
+ ( # save in \1
+ ^ # start of line (with re.M)
+ <(%s) # start tag = \2
+ \b # word break
+ (.*\n)*? # any number of lines, minimally matching
+ .*</\2> # the matching end tag
+ [ \t]* # trailing spaces/tabs
+ (?=\n+|\Z) # followed by a newline or end of document
+ )
+ """ % _block_tags_b,
+ re.X | re.M)
+
+ def _hash_html_block_sub(self, match, raw=False):
+ html = match.group(1)
+ if raw and self.safe_mode:
+ html = self._sanitize_html(html)
+ key = _hash_text(html)
+ self.html_blocks[key] = html
+ return "\n\n" + key + "\n\n"
+
+ def _hash_html_blocks(self, text, raw=False):
+ """Hashify HTML blocks
+
+ We only want to do this for block-level HTML tags, such as headers,
+ lists, and tables. That's because we still want to wrap <p>s around
+ "paragraphs" that are wrapped in non-block-level tags, such as anchors,
+ phrase emphasis, and spans. The list of tags we're looking for is
+ hard-coded.
+
+ @param raw {boolean} indicates if these are raw HTML blocks in
+ the original source. It makes a difference in "safe" mode.
+ """
+ if '<' not in text:
+ return text
+
+ # Pass `raw` value into our calls to self._hash_html_block_sub.
+ hash_html_block_sub = _curry(self._hash_html_block_sub, raw=raw)
+
+ # First, look for nested blocks, e.g.:
+ # <div>
+ # <div>
+ # tags for inner block must be indented.
+ # </div>
+ # </div>
+ #
+ # The outermost tags must start at the left margin for this to match, and
+ # the inner nested divs must be indented.
+ # We need to do this before the next, more liberal match, because the next
+ # match will start at the first `<div>` and stop at the first `</div>`.
+ text = self._strict_tag_block_re.sub(hash_html_block_sub, text)
+
+ # Now match more liberally, simply from `\n<tag>` to `</tag>\n`
+ text = self._liberal_tag_block_re.sub(hash_html_block_sub, text)
+
+ # Special case just for <hr />. It was easier to make a special
+ # case than to make the other regex more complicated.
+ if "<hr" in text:
+ _hr_tag_re = _hr_tag_re_from_tab_width(self.tab_width)
+ text = _hr_tag_re.sub(hash_html_block_sub, text)
+
+ # Special case for standalone HTML comments:
+ if "<!--" in text:
+ start = 0
+ while True:
+ # Delimiters for next comment block.
+ try:
+ start_idx = text.index("<!--", start)
+ except ValueError, ex:
+ break
+ try:
+ end_idx = text.index("-->", start_idx) + 3
+ except ValueError, ex:
+ break
+
+ # Start position for next comment block search.
+ start = end_idx
+
+ # Validate whitespace before comment.
+ if start_idx:
+ # - Up to `tab_width - 1` spaces before start_idx.
+ for i in range(self.tab_width - 1):
+ if text[start_idx - 1] != ' ':
+ break
+ start_idx -= 1
+ if start_idx == 0:
+ break
+ # - Must be preceded by 2 newlines or hit the start of
+ # the document.
+ if start_idx == 0:
+ pass
+ elif start_idx == 1 and text[0] == '\n':
+ start_idx = 0 # to match minute detail of Markdown.pl regex
+ elif text[start_idx-2:start_idx] == '\n\n':
+ pass
+ else:
+ break
+
+ # Validate whitespace after comment.
+ # - Any number of spaces and tabs.
+ while end_idx < len(text):
+ if text[end_idx] not in ' \t':
+ break
+ end_idx += 1
+ # - Must be following by 2 newlines or hit end of text.
+ if text[end_idx:end_idx+2] not in ('', '\n', '\n\n'):
+ continue
+
+ # Escape and hash (must match `_hash_html_block_sub`).
+ html = text[start_idx:end_idx]
+ if raw and self.safe_mode:
+ html = self._sanitize_html(html)
+ key = _hash_text(html)
+ self.html_blocks[key] = html
+ text = text[:start_idx] + "\n\n" + key + "\n\n" + text[end_idx:]
+
+ if "xml" in self.extras:
+ # Treat XML processing instructions and namespaced one-liner
+ # tags as if they were block HTML tags. E.g., if standalone
+ # (i.e. are their own paragraph), the following do not get
+ # wrapped in a <p> tag:
+ # <?foo bar?>
+ #
+ # <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="chapter_1.md"/>
+ _xml_oneliner_re = _xml_oneliner_re_from_tab_width(self.tab_width)
+ text = _xml_oneliner_re.sub(hash_html_block_sub, text)
+
+ return text
+
+ def _strip_link_definitions(self, text):
+ # Strips link definitions from text, stores the URLs and titles in
+ # hash references.
+ less_than_tab = self.tab_width - 1
+
+ # Link defs are in the form:
+ # [id]: url "optional title"
+ _link_def_re = re.compile(r"""
+ ^[ ]{0,%d}\[(.+)\]: # id = \1
+ [ \t]*
+ \n? # maybe *one* newline
+ [ \t]*
+ <?(.+?)>? # url = \2
+ [ \t]*
+ (?:
+ \n? # maybe one newline
+ [ \t]*
+ (?<=\s) # lookbehind for whitespace
+ ['"(]
+ ([^\n]*) # title = \3
+ ['")]
+ [ \t]*
+ )? # title is optional
+ (?:\n+|\Z)
+ """ % less_than_tab, re.X | re.M | re.U)
+ return _link_def_re.sub(self._extract_link_def_sub, text)
+
+ def _extract_link_def_sub(self, match):
+ id, url, title = match.groups()
+ key = id.lower() # Link IDs are case-insensitive
+ self.urls[key] = self._encode_amps_and_angles(url)
+ if title:
+ self.titles[key] = title.replace('"', '"')
+ return ""
+
+ def _extract_footnote_def_sub(self, match):
+ id, text = match.groups()
+ text = _dedent(text, skip_first_line=not text.startswith('\n')).strip()
+ normed_id = re.sub(r'\W', '-', id)
+ # Ensure footnote text ends with a couple newlines (for some
+ # block gamut matches).
+ self.footnotes[normed_id] = text + "\n\n"
+ return ""
+
+ def _strip_footnote_definitions(self, text):
+ """A footnote definition looks like this:
+
+ [^note-id]: Text of the note.
+
+ May include one or more indented paragraphs.
+
+ Where,
+ - The 'note-id' can be pretty much anything, though typically it
+ is the number of the footnote.
+ - The first paragraph may start on the next line, like so:
+
+ [^note-id]:
+ Text of the note.
+ """
+ less_than_tab = self.tab_width - 1
+ footnote_def_re = re.compile(r'''
+ ^[ ]{0,%d}\[\^(.+)\]: # id = \1
+ [ \t]*
+ ( # footnote text = \2
+ # First line need not start with the spaces.
+ (?:\s*.*\n+)
+ (?:
+ (?:[ ]{%d} | \t) # Subsequent lines must be indented.
+ .*\n+
+ )*
+ )
+ # Lookahead for non-space at line-start, or end of doc.
+ (?:(?=^[ ]{0,%d}\S)|\Z)
+ ''' % (less_than_tab, self.tab_width, self.tab_width),
+ re.X | re.M)
+ return footnote_def_re.sub(self._extract_footnote_def_sub, text)
+
+
+ _hr_res = [
+ re.compile(r"^[ ]{0,2}([ ]?\*[ ]?){3,}[ \t]*$", re.M),
+ re.compile(r"^[ ]{0,2}([ ]?\-[ ]?){3,}[ \t]*$", re.M),
+ re.compile(r"^[ ]{0,2}([ ]?\_[ ]?){3,}[ \t]*$", re.M),
+ ]
+
+ def _run_block_gamut(self, text):
+ # These are all the transformations that form block-level
+ # tags like paragraphs, headers, and list items.
+
+ text = self._do_headers(text)
+
+ # Do Horizontal Rules:
+ hr = "\n<hr"+self.empty_element_suffix+"\n"
+ for hr_re in self._hr_res:
+ text = hr_re.sub(hr, text)
+
+ text = self._do_lists(text)
+
+ if "pyshell" in self.extras:
+ text = self._prepare_pyshell_blocks(text)
+
+ text = self._do_code_blocks(text)
+
+ text = self._do_block_quotes(text)
+
+ # We already ran _HashHTMLBlocks() before, in Markdown(), but that
+ # was to escape raw HTML in the original Markdown source. This time,
+ # we're escaping the markup we've just created, so that we don't wrap
+ # <p> tags around block-level tags.
+ text = self._hash_html_blocks(text)
+
+ text = self._form_paragraphs(text)
+
+ return text
+
+ def _pyshell_block_sub(self, match):
+ lines = match.group(0).splitlines(0)
+ _dedentlines(lines)
+ indent = ' ' * self.tab_width
+ s = ('\n' # separate from possible cuddled paragraph
+ + indent + ('\n'+indent).join(lines)
+ + '\n\n')
+ return s
+
+ def _prepare_pyshell_blocks(self, text):
+ """Ensure that Python interactive shell sessions are put in
+ code blocks -- even if not properly indented.
+ """
+ if ">>>" not in text:
+ return text
+
+ less_than_tab = self.tab_width - 1
+ _pyshell_block_re = re.compile(r"""
+ ^([ ]{0,%d})>>>[ ].*\n # first line
+ ^(\1.*\S+.*\n)* # any number of subsequent lines
+ ^\n # ends with a blank line
+ """ % less_than_tab, re.M | re.X)
+
+ return _pyshell_block_re.sub(self._pyshell_block_sub, text)
+
+ def _run_span_gamut(self, text):
+ # These are all the transformations that occur *within* block-level
+ # tags like paragraphs, headers, and list items.
+
+ text = self._do_code_spans(text)
+
+ text = self._escape_special_chars(text)
+
+ # Process anchor and image tags.
+ text = self._do_links(text)
+
+ # Make links out of things like `<http://example.com/>`
+ # Must come after _do_links(), because you can use < and >
+ # delimiters in inline links like [this](<url>).
+ text = self._do_auto_links(text)
+
+ if "link-patterns" in self.extras:
+ text = self._do_link_patterns(text)
+
+ text = self._encode_amps_and_angles(text)
+
+ text = self._do_italics_and_bold(text)
+
+ # Do hard breaks:
+ text = re.sub(r" {2,}\n", " <br%s\n" % self.empty_element_suffix, text)
+
+ return text
+
+ # "Sorta" because auto-links are identified as "tag" tokens.
+ _sorta_html_tokenize_re = re.compile(r"""
+ (
+ # tag
+ </?
+ (?:\w+) # tag name
+ (?:\s+(?:[\w-]+:)?[\w-]+=(?:".*?"|'.*?'))* # attributes
+ \s*/?>
+ |
+ # auto-link (e.g., <http://www.activestate.com/>)
+ <\w+[^>]*>
+ |
+ <!--.*?--> # comment
+ |
+ <\?.*?\?> # processing instruction
+ )
+ """, re.X)
+
+ def _escape_special_chars(self, text):
+ # Python markdown note: the HTML tokenization here differs from
+ # that in Markdown.pl, hence the behaviour for subtle cases can
+ # differ (I believe the tokenizer here does a better job because
+ # it isn't susceptible to unmatched '<' and '>' in HTML tags).
+ # Note, however, that '>' is not allowed in an auto-link URL
+ # here.
+ escaped = []
+ is_html_markup = False
+ for token in self._sorta_html_tokenize_re.split(text):
+ if is_html_markup:
+ # Within tags/HTML-comments/auto-links, encode * and _
+ # so they don't conflict with their use in Markdown for
+ # italics and strong. We're replacing each such
+ # character with its corresponding MD5 checksum value;
+ # this is likely overkill, but it should prevent us from
+ # colliding with the escape values by accident.
+ escaped.append(token.replace('*', g_escape_table['*'])
+ .replace('_', g_escape_table['_']))
+ else:
+ escaped.append(self._encode_backslash_escapes(token))
+ is_html_markup = not is_html_markup
+ return ''.join(escaped)
+
+ def _hash_html_spans(self, text):
+ # Used for safe_mode.
+
+ def _is_auto_link(s):
+ if ':' in s and self._auto_link_re.match(s):
+ return True
+ elif '@' in s and self._auto_email_link_re.match(s):
+ return True
+ return False
+
+ tokens = []
+ is_html_markup = False
+ for token in self._sorta_html_tokenize_re.split(text):
+ if is_html_markup and not _is_auto_link(token):
+ sanitized = self._sanitize_html(token)
+ key = _hash_text(sanitized)
+ self.html_spans[key] = sanitized
+ tokens.append(key)
+ else:
+ tokens.append(token)
+ is_html_markup = not is_html_markup
+ return ''.join(tokens)
+
+ def _unhash_html_spans(self, text):
+ for key, sanitized in self.html_spans.items():
+ text = text.replace(key, sanitized)
+ return text
+
+ def _sanitize_html(self, s):
+ if self.safe_mode == "replace":
+ return self.html_removed_text
+ elif self.safe_mode == "escape":
+ replacements = [
+ ('&', '&'),
+ ('<', '<'),
+ ('>', '>'),
+ ]
+ for before, after in replacements:
+ s = s.replace(before, after)
+ return s
+ else:
+ raise MarkdownError("invalid value for 'safe_mode': %r (must be "
+ "'escape' or 'replace')" % self.safe_mode)
+
+ _tail_of_inline_link_re = re.compile(r'''
+ # Match tail of: [text](/url/) or [text](/url/ "title")
+ \( # literal paren
+ [ \t]*
+ (?P<url> # \1
+ <.*?>
+ |
+ .*?
+ )
+ [ \t]*
+ ( # \2
+ (['"]) # quote char = \3
+ (?P<title>.*?)
+ \3 # matching quote
+ )? # title is optional
+ \)
+ ''', re.X | re.S)
+ _tail_of_reference_link_re = re.compile(r'''
+ # Match tail of: [text][id]
+ [ ]? # one optional space
+ (?:\n[ ]*)? # one optional newline followed by spaces
+ \[
+ (?P<id>.*?)
+ \]
+ ''', re.X | re.S)
+
+ def _do_links(self, text):
+ """Turn Markdown link shortcuts into XHTML <a> and <img> tags.
+
+ This is a combination of Markdown.pl's _DoAnchors() and
+ _DoImages(). They are done together because that simplified the
+ approach. It was necessary to use a different approach than
+ Markdown.pl because of the lack of atomic matching support in
+ Python's regex engine used in $g_nested_brackets.
+ """
+ MAX_LINK_TEXT_SENTINEL = 3000 # markdown2 issue 24
+
+ # `anchor_allowed_pos` is used to support img links inside
+ # anchors, but not anchors inside anchors. An anchor's start
+ # pos must be `>= anchor_allowed_pos`.
+ anchor_allowed_pos = 0
+
+ curr_pos = 0
+ while True: # Handle the next link.
+ # The next '[' is the start of:
+ # - an inline anchor: [text](url "title")
+ # - a reference anchor: [text][id]
+ # - an inline img: 
+ # - a reference img: ![text][id]
+ # - a footnote ref: [^id]
+ # (Only if 'footnotes' extra enabled)
+ # - a footnote defn: [^id]: ...
+ # (Only if 'footnotes' extra enabled) These have already
+ # been stripped in _strip_footnote_definitions() so no
+ # need to watch for them.
+ # - a link definition: [id]: url "title"
+ # These have already been stripped in
+ # _strip_link_definitions() so no need to watch for them.
+ # - not markup: [...anything else...
+ try:
+ start_idx = text.index('[', curr_pos)
+ except ValueError:
+ break
+ text_length = len(text)
+
+ # Find the matching closing ']'.
+ # Markdown.pl allows *matching* brackets in link text so we
+ # will here too. Markdown.pl *doesn't* currently allow
+ # matching brackets in img alt text -- we'll differ in that
+ # regard.
+ bracket_depth = 0
+ for p in range(start_idx+1, min(start_idx+MAX_LINK_TEXT_SENTINEL,
+ text_length)):
+ ch = text[p]
+ if ch == ']':
+ bracket_depth -= 1
+ if bracket_depth < 0:
+ break
+ elif ch == '[':
+ bracket_depth += 1
+ else:
+ # Closing bracket not found within sentinel length.
+ # This isn't markup.
+ curr_pos = start_idx + 1
+ continue
+ link_text = text[start_idx+1:p]
+
+ # Possibly a footnote ref?
+ if "footnotes" in self.extras and link_text.startswith("^"):
+ normed_id = re.sub(r'\W', '-', link_text[1:])
+ if normed_id in self.footnotes:
+ self.footnote_ids.append(normed_id)
+ result = '<sup class="footnote-ref" id="fnref-%s">' \
+ '<a href="#fn-%s">%s</a></sup>' \
+ % (normed_id, normed_id, len(self.footnote_ids))
+ text = text[:start_idx] + result + text[p+1:]
+ else:
+ # This id isn't defined, leave the markup alone.
+ curr_pos = p+1
+ continue
+
+ # Now determine what this is by the remainder.
+ p += 1
+ if p == text_length:
+ return text
+
+ # Inline anchor or img?
+ if text[p] == '(': # attempt at perf improvement
+ match = self._tail_of_inline_link_re.match(text, p)
+ if match:
+ # Handle an inline anchor or img.
+ is_img = start_idx > 0 and text[start_idx-1] == "!"
+ if is_img:
+ start_idx -= 1
+
+ url, title = match.group("url"), match.group("title")
+ if url and url[0] == '<':
+ url = url[1:-1] # '<url>' -> 'url'
+ # We've got to encode these to avoid conflicting
+ # with italics/bold.
+ url = url.replace('*', g_escape_table['*']) \
+ .replace('_', g_escape_table['_'])
+ if title:
+ title_str = ' title="%s"' \
+ % title.replace('*', g_escape_table['*']) \
+ .replace('_', g_escape_table['_']) \
+ .replace('"', '"')
+ else:
+ title_str = ''
+ if is_img:
+ result = '<img src="%s" alt="%s"%s%s' \
+ % (url, link_text.replace('"', '"'),
+ title_str, self.empty_element_suffix)
+ curr_pos = start_idx + len(result)
+ text = text[:start_idx] + result + text[match.end():]
+ elif start_idx >= anchor_allowed_pos:
+ result_head = '<a href="%s"%s>' % (url, title_str)
+ result = '%s%s</a>' % (result_head, link_text)
+ # <img> allowed from curr_pos on, <a> from
+ # anchor_allowed_pos on.
+ curr_pos = start_idx + len(result_head)
+ anchor_allowed_pos = start_idx + len(result)
+ text = text[:start_idx] + result + text[match.end():]
+ else:
+ # Anchor not allowed here.
+ curr_pos = start_idx + 1
+ continue
+
+ # Reference anchor or img?
+ else:
+ match = self._tail_of_reference_link_re.match(text, p)
+ if match:
+ # Handle a reference-style anchor or img.
+ is_img = start_idx > 0 and text[start_idx-1] == "!"
+ if is_img:
+ start_idx -= 1
+ link_id = match.group("id").lower()
+ if not link_id:
+ link_id = link_text.lower() # for links like [this][]
+ if link_id in self.urls:
+ url = self.urls[link_id]
+ # We've got to encode these to avoid conflicting
+ # with italics/bold.
+ url = url.replace('*', g_escape_table['*']) \
+ .replace('_', g_escape_table['_'])
+ title = self.titles.get(link_id)
+ if title:
+ title = title.replace('*', g_escape_table['*']) \
+ .replace('_', g_escape_table['_'])
+ title_str = ' title="%s"' % title
+ else:
+ title_str = ''
+ if is_img:
+ result = '<img src="%s" alt="%s"%s%s' \
+ % (url, link_text.replace('"', '"'),
+ title_str, self.empty_element_suffix)
+ curr_pos = start_idx + len(result)
+ text = text[:start_idx] + result + text[match.end():]
+ elif start_idx >= anchor_allowed_pos:
+ result = '<a href="%s"%s>%s</a>' \
+ % (url, title_str, link_text)
+ result_head = '<a href="%s"%s>' % (url, title_str)
+ result = '%s%s</a>' % (result_head, link_text)
+ # <img> allowed from curr_pos on, <a> from
+ # anchor_allowed_pos on.
+ curr_pos = start_idx + len(result_head)
+ anchor_allowed_pos = start_idx + len(result)
+ text = text[:start_idx] + result + text[match.end():]
+ else:
+ # Anchor not allowed here.
+ curr_pos = start_idx + 1
+ else:
+ # This id isn't defined, leave the markup alone.
+ curr_pos = match.end()
+ continue
+
+ # Otherwise, it isn't markup.
+ curr_pos = start_idx + 1
+
+ return text
+
+
+ _setext_h_re = re.compile(r'^(.+)[ \t]*\n(=+|-+)[ \t]*\n+', re.M)
+ def _setext_h_sub(self, match):
+ n = {"=": 1, "-": 2}[match.group(2)[0]]
+ demote_headers = self.extras.get("demote-headers")
+ if demote_headers:
+ n = min(n + demote_headers, 6)
+ return "<h%d>%s</h%d>\n\n" \
+ % (n, self._run_span_gamut(match.group(1)), n)
+
+ _atx_h_re = re.compile(r'''
+ ^(\#{1,6}) # \1 = string of #'s
+ [ \t]*
+ (.+?) # \2 = Header text
+ [ \t]*
+ (?<!\\) # ensure not an escaped trailing '#'
+ \#* # optional closing #'s (not counted)
+ \n+
+ ''', re.X | re.M)
+ def _atx_h_sub(self, match):
+ n = len(match.group(1))
+ demote_headers = self.extras.get("demote-headers")
+ if demote_headers:
+ n = min(n + demote_headers, 6)
+ return "<h%d>%s</h%d>\n\n" \
+ % (n, self._run_span_gamut(match.group(2)), n)
+
+ def _do_headers(self, text):
+ # Setext-style headers:
+ # Header 1
+ # ========
+ #
+ # Header 2
+ # --------
+ text = self._setext_h_re.sub(self._setext_h_sub, text)
+
+ # atx-style headers:
+ # # Header 1
+ # ## Header 2
+ # ## Header 2 with closing hashes ##
+ # ...
+ # ###### Header 6
+ text = self._atx_h_re.sub(self._atx_h_sub, text)
+
+ return text
+
+
+ _marker_ul_chars = '*+-'
+ _marker_any = r'(?:[%s]|\d+\.)' % _marker_ul_chars
+ _marker_ul = '(?:[%s])' % _marker_ul_chars
+ _marker_ol = r'(?:\d+\.)'
+
+ def _list_sub(self, match):
+ lst = match.group(1)
+ lst_type = match.group(3) in self._marker_ul_chars and "ul" or "ol"
+ result = self._process_list_items(lst)
+ if self.list_level:
+ return "<%s>\n%s</%s>\n" % (lst_type, result, lst_type)
+ else:
+ return "<%s>\n%s</%s>\n\n" % (lst_type, result, lst_type)
+
+ def _do_lists(self, text):
+ # Form HTML ordered (numbered) and unordered (bulleted) lists.
+
+ for marker_pat in (self._marker_ul, self._marker_ol):
+ # Re-usable pattern to match any entire ul or ol list:
+ less_than_tab = self.tab_width - 1
+ whole_list = r'''
+ ( # \1 = whole list
+ ( # \2
+ [ ]{0,%d}
+ (%s) # \3 = first list item marker
+ [ \t]+
+ )
+ (?:.+?)
+ ( # \4
+ \Z
+ |
+ \n{2,}
+ (?=\S)
+ (?! # Negative lookahead for another list item marker
+ [ \t]*
+ %s[ \t]+
+ )
+ )
+ )
+ ''' % (less_than_tab, marker_pat, marker_pat)
+
+ # We use a different prefix before nested lists than top-level lists.
+ # See extended comment in _process_list_items().
+ #
+ # Note: There's a bit of duplication here. My original implementation
+ # created a scalar regex pattern as the conditional result of the test on
+ # $g_list_level, and then only ran the $text =~ s{...}{...}egmx
+ # substitution once, using the scalar as the pattern. This worked,
+ # everywhere except when running under MT on my hosting account at Pair
+ # Networks. There, this caused all rebuilds to be killed by the reaper (or
+ # perhaps they crashed, but that seems incredibly unlikely given that the
+ # same script on the same server ran fine *except* under MT. I've spent
+ # more time trying to figure out why this is happening than I'd like to
+ # admit. My only guess, backed up by the fact that this workaround works,
+ # is that Perl optimizes the substition when it can figure out that the
+ # pattern will never change, and when this optimization isn't on, we run
+ # afoul of the reaper. Thus, the slightly redundant code to that uses two
+ # static s/// patterns rather than one conditional pattern.
+
+ if self.list_level:
+ sub_list_re = re.compile("^"+whole_list, re.X | re.M | re.S)
+ text = sub_list_re.sub(self._list_sub, text)
+ else:
+ list_re = re.compile(r"(?:(?<=\n\n)|\A\n?)"+whole_list,
+ re.X | re.M | re.S)
+ text = list_re.sub(self._list_sub, text)
+
+ return text
+
+ _list_item_re = re.compile(r'''
+ (\n)? # leading line = \1
+ (^[ \t]*) # leading whitespace = \2
+ (%s) [ \t]+ # list marker = \3
+ ((?:.+?) # list item text = \4
+ (\n{1,2})) # eols = \5
+ (?= \n* (\Z | \2 (%s) [ \t]+))
+ ''' % (_marker_any, _marker_any),
+ re.M | re.X | re.S)
+
+ _last_li_endswith_two_eols = False
+ def _list_item_sub(self, match):
+ item = match.group(4)
+ leading_line = match.group(1)
+ leading_space = match.group(2)
+ if leading_line or "\n\n" in item or self._last_li_endswith_two_eols:
+ item = self._run_block_gamut(self._outdent(item))
+ else:
+ # Recursion for sub-lists:
+ item = self._do_lists(self._outdent(item))
+ if item.endswith('\n'):
+ item = item[:-1]
+ item = self._run_span_gamut(item)
+ self._last_li_endswith_two_eols = (len(match.group(5)) == 2)
+ return "<li>%s</li>\n" % item
+
+ def _process_list_items(self, list_str):
+ # Process the contents of a single ordered or unordered list,
+ # splitting it into individual list items.
+
+ # The $g_list_level global keeps track of when we're inside a list.
+ # Each time we enter a list, we increment it; when we leave a list,
+ # we decrement. If it's zero, we're not in a list anymore.
+ #
+ # We do this because when we're not inside a list, we want to treat
+ # something like this:
+ #
+ # I recommend upgrading to version
+ # 8. Oops, now this line is treated
+ # as a sub-list.
+ #
+ # As a single paragraph, despite the fact that the second line starts
+ # with a digit-period-space sequence.
+ #
+ # Whereas when we're inside a list (or sub-list), that line will be
+ # treated as the start of a sub-list. What a kludge, huh? This is
+ # an aspect of Markdown's syntax that's hard to parse perfectly
+ # without resorting to mind-reading. Perhaps the solution is to
+ # change the syntax rules such that sub-lists must start with a
+ # starting cardinal number; e.g. "1." or "a.".
+ self.list_level += 1
+ self._last_li_endswith_two_eols = False
+ list_str = list_str.rstrip('\n') + '\n'
+ list_str = self._list_item_re.sub(self._list_item_sub, list_str)
+ self.list_level -= 1
+ return list_str
+
+ def _get_pygments_lexer(self, lexer_name):
+ try:
+ from pygments import lexers, util
+ except ImportError:
+ return None
+ try:
+ return lexers.get_lexer_by_name(lexer_name)
+ except util.ClassNotFound:
+ return None
+
+ def _color_with_pygments(self, codeblock, lexer, **formatter_opts):
+ import pygments
+ import pygments.formatters
+
+ class HtmlCodeFormatter(pygments.formatters.HtmlFormatter):
+ def _wrap_code(self, inner):
+ """A function for use in a Pygments Formatter which
+ wraps in <code> tags.
+ """
+ yield 0, "<code>"
+ for tup in inner:
+ yield tup
+ yield 0, "</code>"
+
+ def wrap(self, source, outfile):
+ """Return the source with a code, pre, and div."""
+ return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
+
+ formatter = HtmlCodeFormatter(cssclass="codehilite", **formatter_opts)
+ return pygments.highlight(codeblock, lexer, formatter)
+
+ def _code_block_sub(self, match):
+ codeblock = match.group(1)
+ codeblock = self._outdent(codeblock)
+ codeblock = self._detab(codeblock)
+ codeblock = codeblock.lstrip('\n') # trim leading newlines
+ codeblock = codeblock.rstrip() # trim trailing whitespace
+
+ if "code-color" in self.extras and codeblock.startswith(":::"):
+ lexer_name, rest = codeblock.split('\n', 1)
+ lexer_name = lexer_name[3:].strip()
+ lexer = self._get_pygments_lexer(lexer_name)
+ codeblock = rest.lstrip("\n") # Remove lexer declaration line.
+ if lexer:
+ formatter_opts = self.extras['code-color'] or {}
+ colored = self._color_with_pygments(codeblock, lexer,
+ **formatter_opts)
+ return "\n\n%s\n\n" % colored
+
+ codeblock = self._encode_code(codeblock)
+ return "\n\n<pre><code>%s\n</code></pre>\n\n" % codeblock
+
+ def _do_code_blocks(self, text):
+ """Process Markdown `<pre><code>` blocks."""
+ code_block_re = re.compile(r'''
+ (?:\n\n|\A)
+ ( # $1 = the code block -- one or more lines, starting with a space/tab
+ (?:
+ (?:[ ]{%d} | \t) # Lines must start with a tab or a tab-width of spaces
+ .*\n+
+ )+
+ )
+ ((?=^[ ]{0,%d}\S)|\Z) # Lookahead for non-space at line-start, or end of doc
+ ''' % (self.tab_width, self.tab_width),
+ re.M | re.X)
+
+ return code_block_re.sub(self._code_block_sub, text)
+
+
+ # Rules for a code span:
+ # - backslash escapes are not interpreted in a code span
+ # - to include one or or a run of more backticks the delimiters must
+ # be a longer run of backticks
+ # - cannot start or end a code span with a backtick; pad with a
+ # space and that space will be removed in the emitted HTML
+ # See `test/tm-cases/escapes.text` for a number of edge-case
+ # examples.
+ _code_span_re = re.compile(r'''
+ (?<!\\)
+ (`+) # \1 = Opening run of `
+ (?!`) # See Note A test/tm-cases/escapes.text
+ (.+?) # \2 = The code block
+ (?<!`)
+ \1 # Matching closer
+ (?!`)
+ ''', re.X | re.S)
+
+ def _code_span_sub(self, match):
+ c = match.group(2).strip(" \t")
+ c = self._encode_code(c)
+ return "<code>%s</code>" % c
+
+ def _do_code_spans(self, text):
+ # * Backtick quotes are used for <code></code> spans.
+ #
+ # * You can use multiple backticks as the delimiters if you want to
+ # include literal backticks in the code span. So, this input:
+ #
+ # Just type ``foo `bar` baz`` at the prompt.
+ #
+ # Will translate to:
+ #
+ # <p>Just type <code>foo `bar` baz</code> at the prompt.</p>
+ #
+ # There's no arbitrary limit to the number of backticks you
+ # can use as delimters. If you need three consecutive backticks
+ # in your code, use four for delimiters, etc.
+ #
+ # * You can use spaces to get literal backticks at the edges:
+ #
+ # ... type `` `bar` `` ...
+ #
+ # Turns to:
+ #
+ # ... type <code>`bar`</code> ...
+ return self._code_span_re.sub(self._code_span_sub, text)
+
+ def _encode_code(self, text):
+ """Encode/escape certain characters inside Markdown code runs.
+ The point is that in code, these characters are literals,
+ and lose their special Markdown meanings.
+ """
+ replacements = [
+ # Encode all ampersands; HTML entities are not
+ # entities within a Markdown code span.
+ ('&', '&'),
+ # Do the angle bracket song and dance:
+ ('<', '<'),
+ ('>', '>'),
+ # Now, escape characters that are magic in Markdown:
+ ('*', g_escape_table['*']),
+ ('_', g_escape_table['_']),
+ ('{', g_escape_table['{']),
+ ('}', g_escape_table['}']),
+ ('[', g_escape_table['[']),
+ (']', g_escape_table[']']),
+ ('\\', g_escape_table['\\']),
+ ]
+ for before, after in replacements:
+ text = text.replace(before, after)
+ return text
+
+ _strong_re = re.compile(r"(\*\*|__)(?=\S)(.+?[*_]*)(?<=\S)\1", re.S)
+ _em_re = re.compile(r"(\*|_)(?=\S)(.+?)(?<=\S)\1", re.S)
+ _code_friendly_strong_re = re.compile(r"\*\*(?=\S)(.+?[*_]*)(?<=\S)\*\*", re.S)
+ _code_friendly_em_re = re.compile(r"\*(?=\S)(.+?)(?<=\S)\*", re.S)
+ def _do_italics_and_bold(self, text):
+ # <strong> must go first:
+ if "code-friendly" in self.extras:
+ text = self._code_friendly_strong_re.sub(r"<strong>\1</strong>", text)
+ text = self._code_friendly_em_re.sub(r"<em>\1</em>", text)
+ else:
+ text = self._strong_re.sub(r"<strong>\2</strong>", text)
+ text = self._em_re.sub(r"<em>\2</em>", text)
+ return text
+
+
+ _block_quote_re = re.compile(r'''
+ ( # Wrap whole match in \1
+ (
+ ^[ \t]*>[ \t]? # '>' at the start of a line
+ .+\n # rest of the first line
+ (.+\n)* # subsequent consecutive lines
+ \n* # blanks
+ )+
+ )
+ ''', re.M | re.X)
+ _bq_one_level_re = re.compile('^[ \t]*>[ \t]?', re.M);
+
+ _html_pre_block_re = re.compile(r'(\s*<pre>.+?</pre>)', re.S)
+ def _dedent_two_spaces_sub(self, match):
+ return re.sub(r'(?m)^ ', '', match.group(1))
+
+ def _block_quote_sub(self, match):
+ bq = match.group(1)
+ bq = self._bq_one_level_re.sub('', bq) # trim one level of quoting
+ bq = self._ws_only_line_re.sub('', bq) # trim whitespace-only lines
+ bq = self._run_block_gamut(bq) # recurse
+
+ bq = re.sub('(?m)^', ' ', bq)
+ # These leading spaces screw with <pre> content, so we need to fix that:
+ bq = self._html_pre_block_re.sub(self._dedent_two_spaces_sub, bq)
+
+ return "<blockquote>\n%s\n</blockquote>\n\n" % bq
+
+ def _do_block_quotes(self, text):
+ if '>' not in text:
+ return text
+ return self._block_quote_re.sub(self._block_quote_sub, text)
+
+ def _form_paragraphs(self, text):
+ # Strip leading and trailing lines:
+ text = text.strip('\n')
+
+ # Wrap <p> tags.
+ grafs = re.split(r"\n{2,}", text)
+ for i, graf in enumerate(grafs):
+ if graf in self.html_blocks:
+ # Unhashify HTML blocks
+ grafs[i] = self.html_blocks[graf]
+ else:
+ # Wrap <p> tags.
+ graf = self._run_span_gamut(graf)
+ grafs[i] = "<p>" + graf.lstrip(" \t") + "</p>"
+
+ return "\n\n".join(grafs)
+
+ def _add_footnotes(self, text):
+ if self.footnotes:
+ footer = [
+ '<div class="footnotes">',
+ '<hr' + self.empty_element_suffix,
+ '<ol>',
+ ]
+ for i, id in enumerate(self.footnote_ids):
+ if i != 0:
+ footer.append('')
+ footer.append('<li id="fn-%s">' % id)
+ footer.append(self._run_block_gamut(self.footnotes[id]))
+ backlink = ('<a href="#fnref-%s" '
+ 'class="footnoteBackLink" '
+ 'title="Jump back to footnote %d in the text.">'
+ '↩</a>' % (id, i+1))
+ if footer[-1].endswith("</p>"):
+ footer[-1] = footer[-1][:-len("</p>")] \
+ + ' ' + backlink + "</p>"
+ else:
+ footer.append("\n<p>%s</p>" % backlink)
+ footer.append('</li>')
+ footer.append('</ol>')
+ footer.append('</div>')
+ return text + '\n\n' + '\n'.join(footer)
+ else:
+ return text
+
+ # Ampersand-encoding based entirely on Nat Irons's Amputator MT plugin:
+ # http://bumppo.net/projects/amputator/
+ _ampersand_re = re.compile(r'&(?!#?[xX]?(?:[0-9a-fA-F]+|\w+);)')
+ _naked_lt_re = re.compile(r'<(?![a-z/?\$!])', re.I)
+ _naked_gt_re = re.compile(r'''(?<![a-z?!/'"-])>''', re.I)
+
+ def _encode_amps_and_angles(self, text):
+ # Smart processing for ampersands and angle brackets that need
+ # to be encoded.
+ text = self._ampersand_re.sub('&', text)
+
+ # Encode naked <'s
+ text = self._naked_lt_re.sub('<', text)
+
+ # Encode naked >'s
+ # Note: Other markdown implementations (e.g. Markdown.pl, PHP
+ # Markdown) don't do this.
+ text = self._naked_gt_re.sub('>', text)
+ return text
+
+ def _encode_backslash_escapes(self, text):
+ for ch, escape in g_escape_table.items():
+ text = text.replace("\\"+ch, escape)
+ return text
+
+ _auto_link_re = re.compile(r'<((https?|ftp):[^\'">\s]+)>', re.I)
+ def _auto_link_sub(self, match):
+ g1 = match.group(1)
+ return '<a href="%s">%s</a>' % (g1, g1)
+
+ _auto_email_link_re = re.compile(r"""
+ <
+ (?:mailto:)?
+ (
+ [-.\w]+
+ \@
+ [-\w]+(\.[-\w]+)*\.[a-z]+
+ )
+ >
+ """, re.I | re.X | re.U)
+ def _auto_email_link_sub(self, match):
+ return self._encode_email_address(
+ self._unescape_special_chars(match.group(1)))
+
+ def _do_auto_links(self, text):
+ text = self._auto_link_re.sub(self._auto_link_sub, text)
+ text = self._auto_email_link_re.sub(self._auto_email_link_sub, text)
+ return text
+
+ def _encode_email_address(self, addr):
+ # Input: an email address, e.g. "foo@example.com"
+ #
+ # Output: the email address as a mailto link, with each character
+ # of the address encoded as either a decimal or hex entity, in
+ # the hopes of foiling most address harvesting spam bots. E.g.:
+ #
+ # <a href="mailto:foo@e
+ # xample.com">foo
+ # @example.com</a>
+ #
+ # Based on a filter by Matthew Wickline, posted to the BBEdit-Talk
+ # mailing list: <http://tinyurl.com/yu7ue>
+ chars = [_xml_encode_email_char_at_random(ch)
+ for ch in "mailto:" + addr]
+ # Strip the mailto: from the visible part.
+ addr = '<a href="%s">%s</a>' \
+ % (''.join(chars), ''.join(chars[7:]))
+ return addr
+
+ def _do_link_patterns(self, text):
+ """Caveat emptor: there isn't much guarding against link
+ patterns being formed inside other standard Markdown links, e.g.
+ inside a [link def][like this].
+
+ Dev Notes: *Could* consider prefixing regexes with a negative
+ lookbehind assertion to attempt to guard against this.
+ """
+ link_from_hash = {}
+ for regex, repl in self.link_patterns:
+ replacements = []
+ for match in regex.finditer(text):
+ if hasattr(repl, "__call__"):
+ href = repl(match)
+ else:
+ href = match.expand(repl)
+ replacements.append((match.span(), href))
+ for (start, end), href in reversed(replacements):
+ escaped_href = (
+ href.replace('"', '"') # b/c of attr quote
+ # To avoid markdown <em> and <strong>:
+ .replace('*', g_escape_table['*'])
+ .replace('_', g_escape_table['_']))
+ link = '<a href="%s">%s</a>' % (escaped_href, text[start:end])
+ hash = md5(link).hexdigest()
+ link_from_hash[hash] = link
+ text = text[:start] + hash + text[end:]
+ for hash, link in link_from_hash.items():
+ text = text.replace(hash, link)
+ return text
+
+ def _unescape_special_chars(self, text):
+ # Swap back in all the special characters we've hidden.
+ for ch, hash in g_escape_table.items():
+ text = text.replace(hash, ch)
+ return text
+
+ def _outdent(self, text):
+ # Remove one level of line-leading tabs or spaces
+ return self._outdent_re.sub('', text)
+
+
+class MarkdownWithExtras(Markdown):
+ """A markdowner class that enables most extras:
+
+ - footnotes
+ - code-color (only has effect if 'pygments' Python module on path)
+
+ These are not included:
+ - pyshell (specific to Python-related documenting)
+ - code-friendly (because it *disables* part of the syntax)
+ - link-patterns (because you need to specify some actual
+ link-patterns anyway)
+ """
+ extras = ["footnotes", "code-color"]
+
+
+#---- internal support functions
+
+# From http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52549
+def _curry(*args, **kwargs):
+ function, args = args[0], args[1:]
+ def result(*rest, **kwrest):
+ combined = kwargs.copy()
+ combined.update(kwrest)
+ return function(*args + rest, **combined)
+ return result
+
+# Recipe: regex_from_encoded_pattern (1.0)
+def _regex_from_encoded_pattern(s):
+ """'foo' -> re.compile(re.escape('foo'))
+ '/foo/' -> re.compile('foo')
+ '/foo/i' -> re.compile('foo', re.I)
+ """
+ if s.startswith('/') and s.rfind('/') != 0:
+ # Parse it: /PATTERN/FLAGS
+ idx = s.rfind('/')
+ pattern, flags_str = s[1:idx], s[idx+1:]
+ flag_from_char = {
+ "i": re.IGNORECASE,
+ "l": re.LOCALE,
+ "s": re.DOTALL,
+ "m": re.MULTILINE,
+ "u": re.UNICODE,
+ }
+ flags = 0
+ for char in flags_str:
+ try:
+ flags |= flag_from_char[char]
+ except KeyError:
+ raise ValueError("unsupported regex flag: '%s' in '%s' "
+ "(must be one of '%s')"
+ % (char, s, ''.join(flag_from_char.keys())))
+ return re.compile(s[1:idx], flags)
+ else: # not an encoded regex
+ return re.compile(re.escape(s))
+
+# Recipe: dedent (0.1.2)
+def _dedentlines(lines, tabsize=8, skip_first_line=False):
+ """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines
+
+ "lines" is a list of lines to dedent.
+ "tabsize" is the tab width to use for indent width calculations.
+ "skip_first_line" is a boolean indicating if the first line should
+ be skipped for calculating the indent width and for dedenting.
+ This is sometimes useful for docstrings and similar.
+
+ Same as dedent() except operates on a sequence of lines. Note: the
+ lines list is modified **in-place**.
+ """
+ DEBUG = False
+ if DEBUG:
+ print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\
+ % (tabsize, skip_first_line)
+ indents = []
+ margin = None
+ for i, line in enumerate(lines):
+ if i == 0 and skip_first_line: continue
+ indent = 0
+ for ch in line:
+ if ch == ' ':
+ indent += 1
+ elif ch == '\t':
+ indent += tabsize - (indent % tabsize)
+ elif ch in '\r\n':
+ continue # skip all-whitespace lines
+ else:
+ break
+ else:
+ continue # skip all-whitespace lines
+ if DEBUG: print "dedent: indent=%d: %r" % (indent, line)
+ if margin is None:
+ margin = indent
+ else:
+ margin = min(margin, indent)
+ if DEBUG: print "dedent: margin=%r" % margin
+
+ if margin is not None and margin > 0:
+ for i, line in enumerate(lines):
+ if i == 0 and skip_first_line: continue
+ removed = 0
+ for j, ch in enumerate(line):
+ if ch == ' ':
+ removed += 1
+ elif ch == '\t':
+ removed += tabsize - (removed % tabsize)
+ elif ch in '\r\n':
+ if DEBUG: print "dedent: %r: EOL -> strip up to EOL" % line
+ lines[i] = lines[i][j:]
+ break
+ else:
+ raise ValueError("unexpected non-whitespace char %r in "
+ "line %r while removing %d-space margin"
+ % (ch, line, margin))
+ if DEBUG:
+ print "dedent: %r: %r -> removed %d/%d"\
+ % (line, ch, removed, margin)
+ if removed == margin:
+ lines[i] = lines[i][j+1:]
+ break
+ elif removed > margin:
+ lines[i] = ' '*(removed-margin) + lines[i][j+1:]
+ break
+ else:
+ if removed:
+ lines[i] = lines[i][removed:]
+ return lines
+
+def _dedent(text, tabsize=8, skip_first_line=False):
+ """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text
+
+ "text" is the text to dedent.
+ "tabsize" is the tab width to use for indent width calculations.
+ "skip_first_line" is a boolean indicating if the first line should
+ be skipped for calculating the indent width and for dedenting.
+ This is sometimes useful for docstrings and similar.
+
+ textwrap.dedent(s), but don't expand tabs to spaces
+ """
+ lines = text.splitlines(1)
+ _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line)
+ return ''.join(lines)
+
+
+class _memoized(object):
+ """Decorator that caches a function's return value each time it is called.
+ If called later with the same arguments, the cached value is returned, and
+ not re-evaluated.
+
+ http://wiki.python.org/moin/PythonDecoratorLibrary
+ """
+ def __init__(self, func):
+ self.func = func
+ self.cache = {}
+ def __call__(self, *args):
+ try:
+ return self.cache[args]
+ except KeyError:
+ self.cache[args] = value = self.func(*args)
+ return value
+ except TypeError:
+ # uncachable -- for instance, passing a list as an argument.
+ # Better to not cache than to blow up entirely.
+ return self.func(*args)
+ def __repr__(self):
+ """Return the function's docstring."""
+ return self.func.__doc__
+
+
+def _xml_oneliner_re_from_tab_width(tab_width):
+ """Standalone XML processing instruction regex."""
+ return re.compile(r"""
+ (?:
+ (?<=\n\n) # Starting after a blank line
+ | # or
+ \A\n? # the beginning of the doc
+ )
+ ( # save in $1
+ [ ]{0,%d}
+ (?:
+ <\?\w+\b\s+.*?\?> # XML processing instruction
+ |
+ <\w+:\w+\b\s+.*?/> # namespaced single tag
+ )
+ [ \t]*
+ (?=\n{2,}|\Z) # followed by a blank line or end of document
+ )
+ """ % (tab_width - 1), re.X)
+_xml_oneliner_re_from_tab_width = _memoized(_xml_oneliner_re_from_tab_width)
+
+def _hr_tag_re_from_tab_width(tab_width):
+ return re.compile(r"""
+ (?:
+ (?<=\n\n) # Starting after a blank line
+ | # or
+ \A\n? # the beginning of the doc
+ )
+ ( # save in \1
+ [ ]{0,%d}
+ <(hr) # start tag = \2
+ \b # word break
+ ([^<>])*? #
+ /?> # the matching end tag
+ [ \t]*
+ (?=\n{2,}|\Z) # followed by a blank line or end of document
+ )
+ """ % (tab_width - 1), re.X)
+_hr_tag_re_from_tab_width = _memoized(_hr_tag_re_from_tab_width)
+
+
+def _xml_encode_email_char_at_random(ch):
+ r = random()
+ # Roughly 10% raw, 45% hex, 45% dec.
+ # '@' *must* be encoded. I [John Gruber] insist.
+ # Issue 26: '_' must be encoded.
+ if r > 0.9 and ch not in "@_":
+ return ch
+ elif r < 0.45:
+ # The [1:] is to drop leading '0': 0x63 -> x63
+ return '&#%s;' % hex(ord(ch))[1:]
+ else:
+ return '&#%s;' % ord(ch)
+
+def _hash_text(text):
+ return 'md5:'+md5(text.encode("utf-8")).hexdigest()
+
+
+#---- mainline
+
+class _NoReflowFormatter(optparse.IndentedHelpFormatter):
+ """An optparse formatter that does NOT reflow the description."""
+ def format_description(self, description):
+ return description or ""
+
+def _test():
+ import doctest
+ doctest.testmod()
+
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv
+ if not logging.root.handlers:
+ logging.basicConfig()
+
+ usage = "usage: %prog [PATHS...]"
+ version = "%prog "+__version__
+ parser = optparse.OptionParser(prog="markdown2", usage=usage,
+ version=version, description=cmdln_desc,
+ formatter=_NoReflowFormatter())
+ parser.add_option("-v", "--verbose", dest="log_level",
+ action="store_const", const=logging.DEBUG,
+ help="more verbose output")
+ parser.add_option("--encoding",
+ help="specify encoding of text content")
+ parser.add_option("--html4tags", action="store_true", default=False,
+ help="use HTML 4 style for empty element tags")
+ parser.add_option("-s", "--safe", metavar="MODE", dest="safe_mode",
+ help="sanitize literal HTML: 'escape' escapes "
+ "HTML meta chars, 'replace' replaces with an "
+ "[HTML_REMOVED] note")
+ parser.add_option("-x", "--extras", action="append",
+ help="Turn on specific extra features (not part of "
+ "the core Markdown spec). Supported values: "
+ "'code-friendly' disables _/__ for emphasis; "
+ "'code-color' adds code-block syntax coloring; "
+ "'link-patterns' adds auto-linking based on patterns; "
+ "'footnotes' adds the footnotes syntax;"
+ "'xml' passes one-liner processing instructions and namespaced XML tags;"
+ "'pyshell' to put unindented Python interactive shell sessions in a <code> block.")
+ parser.add_option("--use-file-vars",
+ help="Look for and use Emacs-style 'markdown-extras' "
+ "file var to turn on extras. See "
+ "<http://code.google.com/p/python-markdown2/wiki/Extras>.")
+ parser.add_option("--link-patterns-file",
+ help="path to a link pattern file")
+ parser.add_option("--self-test", action="store_true",
+ help="run internal self-tests (some doctests)")
+ parser.add_option("--compare", action="store_true",
+ help="run against Markdown.pl as well (for testing)")
+ parser.set_defaults(log_level=logging.INFO, compare=False,
+ encoding="utf-8", safe_mode=None, use_file_vars=False)
+ opts, paths = parser.parse_args()
+ log.setLevel(opts.log_level)
+
+ if opts.self_test:
+ return _test()
+
+ if opts.extras:
+ extras = {}
+ for s in opts.extras:
+ splitter = re.compile("[,;: ]+")
+ for e in splitter.split(s):
+ if '=' in e:
+ ename, earg = e.split('=', 1)
+ try:
+ earg = int(earg)
+ except ValueError:
+ pass
+ else:
+ ename, earg = e, None
+ extras[ename] = earg
+ else:
+ extras = None
+
+ if opts.link_patterns_file:
+ link_patterns = []
+ f = open(opts.link_patterns_file)
+ try:
+ for i, line in enumerate(f.readlines()):
+ if not line.strip(): continue
+ if line.lstrip().startswith("#"): continue
+ try:
+ pat, href = line.rstrip().rsplit(None, 1)
+ except ValueError:
+ raise MarkdownError("%s:%d: invalid link pattern line: %r"
+ % (opts.link_patterns_file, i+1, line))
+ link_patterns.append(
+ (_regex_from_encoded_pattern(pat), href))
+ finally:
+ f.close()
+ else:
+ link_patterns = None
+
+ from os.path import join, dirname, abspath, exists
+ markdown_pl = join(dirname(dirname(abspath(__file__))), "test",
+ "Markdown.pl")
+ for path in paths:
+ if opts.compare:
+ print "==== Markdown.pl ===="
+ perl_cmd = 'perl %s "%s"' % (markdown_pl, path)
+ o = os.popen(perl_cmd)
+ perl_html = o.read()
+ o.close()
+ sys.stdout.write(perl_html)
+ print "==== markdown2.py ===="
+ html = markdown_path(path, encoding=opts.encoding,
+ html4tags=opts.html4tags,
+ safe_mode=opts.safe_mode,
+ extras=extras, link_patterns=link_patterns,
+ use_file_vars=opts.use_file_vars)
+ sys.stdout.write(
+ html.encode(sys.stdout.encoding or "utf-8", 'xmlcharrefreplace'))
+ if opts.compare:
+ test_dir = join(dirname(dirname(abspath(__file__))), "test")
+ if exists(join(test_dir, "test_markdown2.py")):
+ sys.path.insert(0, test_dir)
+ from test_markdown2 import norm_html_from_html
+ norm_html = norm_html_from_html(html)
+ norm_perl_html = norm_html_from_html(perl_html)
+ else:
+ norm_html = html
+ norm_perl_html = perl_html
+ print "==== match? %r ====" % (norm_perl_html == norm_html)
+
+
+if __name__ == "__main__":
+ sys.exit( main(sys.argv) )
+
--- /dev/null
+/*
+ * Copyright 2009 Facebook
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+body {
+ background: white;
+ color: black;
+ margin: 15px;
+ margin-top: 0;
+}
+
+body,
+input,
+textarea {
+ font-family: Georgia, serif;
+ font-size: 12pt;
+}
+
+table {
+ border-collapse: collapse;
+ border: 0;
+}
+
+td {
+ border: 0;
+ padding: 0;
+}
+
+h1,
+h2,
+h3,
+h4 {
+ font-family: "Helvetica Nue", Helvetica, Arial, sans-serif;
+ margin: 0;
+}
+
+h1 {
+ font-size: 20pt;
+}
+
+pre,
+code {
+ font-family: monospace;
+ color: #060;
+}
+
+pre {
+ margin-left: 1em;
+ padding-left: 1em;
+ border-left: 1px solid silver;
+ line-height: 14pt;
+}
+
+a,
+a code {
+ color: #00c;
+}
+
+#body {
+ max-width: 800px;
+ margin: auto;
+}
+
+#header {
+ background-color: #3b5998;
+ padding: 5px;
+ padding-left: 10px;
+ padding-right: 10px;
+ margin-bottom: 1em;
+}
+
+#header,
+#header a {
+ color: white;
+}
+
+#header h1 a {
+ text-decoration: none;
+}
+
+#footer,
+#content {
+ margin-left: 10px;
+ margin-right: 10px;
+}
+
+#footer {
+ margin-top: 3em;
+}
+
+.entry h1 a {
+ color: black;
+ text-decoration: none;
+}
+
+.entry {
+ margin-bottom: 2em;
+}
+
+.entry .date {
+ margin-top: 3px;
+}
+
+.entry p {
+ margin: 0;
+ margin-bottom: 1em;
+}
+
+.entry .body {
+ margin-top: 1em;
+ line-height: 16pt;
+}
+
+.compose td {
+ vertical-align: middle;
+ padding-bottom: 5px;
+}
+
+.compose td.field {
+ padding-right: 10px;
+}
+
+.compose .title,
+.compose .submit {
+ font-family: "Helvetica Nue", Helvetica, Arial, sans-serif;
+ font-weight: bold;
+}
+
+.compose .title {
+ font-size: 20pt;
+}
+
+.compose .title,
+.compose .markdown {
+ width: 100%;
+}
+
+.compose .markdown {
+ height: 500px;
+ line-height: 16pt;
+}
--- /dev/null
+{% extends "base.html" %}
+
+{% block head %}
+ <style type="text/css">
+ ul.archive {
+ list-style-type: none;
+ margin: 0;
+ padding: 0;
+ }
+
+ ul.archive li {
+ margin-bottom: 1em;
+ }
+
+ ul.archive .title {
+ font-family: "Helvetica Nue", Helvetica, Arial, sans-serif;
+ font-size: 14pt;
+ }
+ </style>
+{% end %}
+
+{% block body %}
+ <ul class="archive">
+ {% for entry in entries %}
+ <li>
+ <div class="title"><a href="/entry/{{ entry.slug }}">{{ escape(entry.title) }}</a></div>
+ <div class="date">{{ locale.format_date(entry.published, full_format=True, shorter=True) }}</div>
+ </li>
+ {% end %}
+ </ul>
+{% end %}
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+ <title>{{ escape(handler.settings["blog_title"]) }}</title>
+ <link rel="stylesheet" href="/static/blog.css" type="text/css"/>
+ <link rel="alternate" href="/feed" type="application/atom+xml" title="{{ escape(handler.settings["blog_title"]) }}"/>
+ {% block head %}{% end %}
+ </head>
+ <body>
+ <div id="body">
+ <div id="header">
+ <div style="float:right">
+ {% if not current_user %}
+ {{ _('<a href="%(url)s">Sign in</a> to compose/edit') % {"url": escape(users.create_login_url(request.uri))} }}
+ {% else %}
+ {% if current_user.administrator %}
+ <a href="/compose">{{ _("New post") }}</a> -
+ {% end %}
+ <a href="{{ escape(users.create_logout_url(request.uri)) }}">{{ _("Sign out") }}</a>
+ {% end %}
+ </div>
+ <h1><a href="/">{{ escape(handler.settings["blog_title"]) }}</a></h1>
+ </div>
+ <div id="content">{% block body %}{% end %}</div>
+ </div>
+ {% block bottom %}{% end %}
+ </body>
+</html>
--- /dev/null
+{% extends "base.html" %}
+
+{% block body %}
+ <form action="{{ request.path }}" method="post" class="compose">
+ <div style="margin-bottom:5px"><input name="title" type="text" class="title" value="{{ escape(entry.title) if entry else "" }}"/></div>
+ <div style="margin-bottom:5px"><textarea name="markdown" rows="30" cols="40" class="markdown">{{ escape(entry.markdown) if entry else "" }}</textarea></div>
+ <div>
+ <div style="float:right"><a href="http://daringfireball.net/projects/markdown/syntax">{{ _("Syntax documentation") }}</a></div>
+ <input type="submit" value="{{ _("Save changes") if entry else _("Publish post") }}" class="submit"/>
+ <a href="{{ "/entry/" + entry.slug if entry else "/" }}">{{ _("Cancel") }}</a>
+ </div>
+ {% if entry %}
+ <input type="hidden" name="key" value="{{ str(entry.key()) }}"/>
+ {% end %}
+ {{ xsrf_form_html() }}
+ </form>
+{% end %}
+
+{% block bottom %}
+ <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js" type="text/javascript"></script>
+ <script type="text/javascript">
+ //<![CDATA[
+
+ $(function() {
+ $("input[name=title]").select();
+ $("form.compose").submit(function() {
+ var required = ["title", "markdown"];
+ var form = $(this).get(0);
+ for (var i = 0; i < required.length; i++) {
+ if (!form[required[i]].value) {
+ $(form[required[i]]).select();
+ return false;
+ }
+ }
+ return true;
+ });
+ });
+
+ //]]>
+ </script>
+{% end %}
+
--- /dev/null
+{% extends "base.html" %}
+
+{% block body %}
+ {{ modules.Entry(entry) }}
+{% end %}
--- /dev/null
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ {% set date_format = "%Y-%m-%dT%H:%M:%SZ" %}
+ <title>{{ escape(handler.settings["blog_title"]) }}</title>
+ {% if len(entries) > 0 %}
+ <updated>{{ max(e.updated for e in entries).strftime(date_format) }}</updated>
+ {% else %}
+ <updated>{{ datetime.datetime.utcnow().strftime(date_format) }}</updated>
+ {% end %}
+ <id>http://{{ request.host }}/</id>
+ <link rel="alternate" href="http://{{ request.host }}/" title="{{ escape(handler.settings["blog_title"]) }}" type="text/html"/>
+ <link rel="self" href="{{ request.full_url() }}" title="{{ escape(handler.settings["blog_title"]) }}" type="application/atom+xml"/>
+ <author><name>{{ escape(handler.settings["blog_title"]) }}</name></author>
+ {% for entry in entries %}
+ <entry>
+ <id>http://{{ request.host }}/entry/{{ entry.slug }}</id>
+ <title type="text">{{ escape(entry.title) }}</title>
+ <link href="http://{{ request.host }}/entry/{{ entry.slug }}" rel="alternate" type="text/html"/>
+ <updated>{{ entry.updated.strftime(date_format) }}</updated>
+ <published>{{ entry.published.strftime(date_format) }}</published>
+ <content type="xhtml" xml:base="http://{{ request.host }}/">
+ <div xmlns="http://www.w3.org/1999/xhtml">{{ entry.html }}</div>
+ </content>
+ </entry>
+ {% end %}
+</feed>
--- /dev/null
+{% extends "base.html" %}
+
+{% block body %}
+ {% for entry in entries %}
+ {{ modules.Entry(entry) }}
+ {% end %}
+ <div><a href="/archive">{{ _("Archive") }}</a></div>
+{% end %}
--- /dev/null
+<div class="entry">
+ <h1><a href="/entry/{{ entry.slug }}">{{ escape(entry.title) }}</a></h1>
+ <div class="date">{{ locale.format_date(entry.published, full_format=True, shorter=True) }}</div>
+ <div class="body">{{ entry.html }}</div>
+ {% if current_user and current_user.administrator %}
+ <div class="admin"><a href="/compose?key={{ str(entry.key()) }}">{{ _("Edit this post") }}</a></div>
+ {% end %}
+</div>
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import tornado.auth
+import tornado.escape
+import tornado.httpserver
+import tornado.ioloop
+import tornado.options
+import tornado.web
+
+from tornado.options import define, options
+
+define("port", default=8888, help="run on the given port", type=int)
+
+
+class Application(tornado.web.Application):
+ def __init__(self):
+ handlers = [
+ (r"/", MainHandler),
+ (r"/auth/login", AuthHandler),
+ ]
+ settings = dict(
+ cookie_secret="32oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
+ login_url="/auth/login",
+ google_consumer_key="www.tornadoweb.org",
+ google_consumer_secret="ZcyJGvEEFn82+h9/PWgBeB0E",
+ )
+ tornado.web.Application.__init__(self, handlers, **settings)
+
+
+class BaseHandler(tornado.web.RequestHandler):
+ def get_current_user(self):
+ user_json = self.get_secure_cookie("user")
+ if not user_json: return None
+ return tornado.escape.json_decode(user_json)
+
+
+class MainHandler(BaseHandler):
+ @tornado.web.authenticated
+ def get(self):
+ name = tornado.escape.xhtml_escape(self.current_user["name"])
+ self.write("Hello, " + name)
+
+
+class AuthHandler(BaseHandler, tornado.auth.GoogleMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("openid.mode", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authenticate_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "Google auth failed")
+ self.set_secure_cookie("user", tornado.escape.json_encode(user))
+ self.redirect("/")
+
+
+def main():
+ tornado.options.parse_command_line()
+ http_server = tornado.httpserver.HTTPServer(Application())
+ http_server.listen(options.port)
+ tornado.ioloop.IOLoop.instance().start()
+
+
+if __name__ == "__main__":
+ main()
--- /dev/null
+Running the Tornado Blog example app
+====================================
+This demo is a simple blogging engine that uses MySQL to store posts and
+Google Accounts for author authentication. Since it depends on MySQL, you
+need to set up MySQL and the database schema for the demo to run.
+
+1. Install prerequisites and build tornado
+
+ See http://www.tornadoweb.org/ for installation instructions. If you can
+ run the "helloworld" example application, your environment is set up
+ correctly.
+
+2. Install MySQL if needed
+
+ Consult the documentation for your platform. Under Ubuntu Linux you
+ can run "apt-get install mysql". Under OS X you can download the
+ MySQL PKG file from http://dev.mysql.com/downloads/mysql/
+
+3. Connect to MySQL and create a database and user for the blog.
+
+ Connect to MySQL as a user that can create databases and users:
+ mysql -u root
+
+ Create a database named "blog":
+ mysql> CREATE DATABASE blog;
+
+ Allow the "blog" user to connect with the password "blog":
+ mysql> GRANT ALL PRIVILEGES ON blog.* TO 'blog'@'localhost' IDENTIFIED BY 'blog';
+
+4. Create the tables in your new database.
+
+ You can use the provided schema.sql file by running this command:
+ mysql --user=blog --password=blog --database=blog < schema.sql
+
+ You can run the above command again later if you want to delete the
+ contents of the blog and start over after testing.
+
+5. Run the blog example
+
+ With the default user, password, and database you can just run:
+ ./blog.py
+
+ If you've changed anything, you can alter the default MySQL settings
+ with arguments on the command line, e.g.:
+ ./blog.py --mysql_user=casey --mysql_password=happiness --mysql_database=foodblog
+
+6. Visit your new blog
+
+ Open http://localhost:8888/ in your web browser. You will be redirected to
+ a Google account sign-in page because the blog uses Google accounts for
+ authentication.
+
+ Currently the first user to connect will automatically be given the
+ ability to create and edit posts.
+
+ Once you've created one blog post, subsequent users will not be
+ prompted to sign in.
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import markdown
+import os.path
+import re
+import tornado.auth
+import tornado.database
+import tornado.httpserver
+import tornado.ioloop
+import tornado.options
+import tornado.web
+import unicodedata
+
+from tornado.options import define, options
+
+define("port", default=8888, help="run on the given port", type=int)
+define("mysql_host", default="127.0.0.1:3306", help="blog database host")
+define("mysql_database", default="blog", help="blog database name")
+define("mysql_user", default="blog", help="blog database user")
+define("mysql_password", default="blog", help="blog database password")
+
+
+class Application(tornado.web.Application):
+ def __init__(self):
+ handlers = [
+ (r"/", HomeHandler),
+ (r"/archive", ArchiveHandler),
+ (r"/feed", FeedHandler),
+ (r"/entry/([^/]+)", EntryHandler),
+ (r"/compose", ComposeHandler),
+ (r"/auth/login", AuthLoginHandler),
+ (r"/auth/logout", AuthLogoutHandler),
+ ]
+ settings = dict(
+ blog_title=u"Tornado Blog",
+ template_path=os.path.join(os.path.dirname(__file__), "templates"),
+ static_path=os.path.join(os.path.dirname(__file__), "static"),
+ ui_modules={"Entry": EntryModule},
+ xsrf_cookies=True,
+ cookie_secret="11oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
+ login_url="/auth/login",
+ )
+ tornado.web.Application.__init__(self, handlers, **settings)
+
+ # Have one global connection to the blog DB across all handlers
+ self.db = tornado.database.Connection(
+ host=options.mysql_host, database=options.mysql_database,
+ user=options.mysql_user, password=options.mysql_password)
+
+
+class BaseHandler(tornado.web.RequestHandler):
+ @property
+ def db(self):
+ return self.application.db
+
+ def get_current_user(self):
+ user_id = self.get_secure_cookie("user")
+ if not user_id: return None
+ return self.db.get("SELECT * FROM authors WHERE id = %s", int(user_id))
+
+
+class HomeHandler(BaseHandler):
+ def get(self):
+ entries = self.db.query("SELECT * FROM entries ORDER BY published "
+ "DESC LIMIT 5")
+ if not entries:
+ self.redirect("/compose")
+ return
+ self.render("home.html", entries=entries)
+
+
+class EntryHandler(BaseHandler):
+ def get(self, slug):
+ entry = self.db.get("SELECT * FROM entries WHERE slug = %s", slug)
+ if not entry: raise tornado.web.HTTPError(404)
+ self.render("entry.html", entry=entry)
+
+
+class ArchiveHandler(BaseHandler):
+ def get(self):
+ entries = self.db.query("SELECT * FROM entries ORDER BY published "
+ "DESC")
+ self.render("archive.html", entries=entries)
+
+
+class FeedHandler(BaseHandler):
+ def get(self):
+ entries = self.db.query("SELECT * FROM entries ORDER BY published "
+ "DESC LIMIT 10")
+ self.set_header("Content-Type", "application/atom+xml")
+ self.render("feed.xml", entries=entries)
+
+
+class ComposeHandler(BaseHandler):
+ @tornado.web.authenticated
+ def get(self):
+ id = self.get_argument("id", None)
+ entry = None
+ if id:
+ entry = self.db.get("SELECT * FROM entries WHERE id = %s", int(id))
+ self.render("compose.html", entry=entry)
+
+ @tornado.web.authenticated
+ def post(self):
+ id = self.get_argument("id", None)
+ title = self.get_argument("title")
+ text = self.get_argument("markdown")
+ html = markdown.markdown(text)
+ if id:
+ entry = self.db.get("SELECT * FROM entries WHERE id = %s", int(id))
+ if not entry: raise tornado.web.HTTPError(404)
+ slug = entry.slug
+ self.db.execute(
+ "UPDATE entries SET title = %s, markdown = %s, html = %s "
+ "WHERE id = %s", title, text, html, int(id))
+ else:
+ slug = unicodedata.normalize("NFKD", title).encode(
+ "ascii", "ignore")
+ slug = re.sub(r"[^\w]+", " ", slug)
+ slug = "-".join(slug.lower().strip().split())
+ if not slug: slug = "entry"
+ while True:
+ e = self.db.get("SELECT * FROM entries WHERE slug = %s", slug)
+ if not e: break
+ slug += "-2"
+ self.db.execute(
+ "INSERT INTO entries (author_id,title,slug,markdown,html,"
+ "published) VALUES (%s,%s,%s,%s,%s,UTC_TIMESTAMP())",
+ self.current_user.id, title, slug, text, html)
+ self.redirect("/entry/" + slug)
+
+
+class AuthLoginHandler(BaseHandler, tornado.auth.GoogleMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("openid.mode", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authenticate_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "Google auth failed")
+ author = self.db.get("SELECT * FROM authors WHERE email = %s",
+ user["email"])
+ if not author:
+ # Auto-create first author
+ any_author = self.db.get("SELECT * FROM authors LIMIT 1")
+ if not any_author:
+ author_id = self.db.execute(
+ "INSERT INTO authors (email,name) VALUES (%s,%s)",
+ user["email"], user["name"])
+ else:
+ self.redirect("/")
+ return
+ else:
+ author_id = author["id"]
+ self.set_secure_cookie("user", str(author_id))
+ self.redirect(self.get_argument("next", "/"))
+
+
+class AuthLogoutHandler(BaseHandler):
+ def get(self):
+ self.clear_cookie("user")
+ self.redirect(self.get_argument("next", "/"))
+
+
+class EntryModule(tornado.web.UIModule):
+ def render(self, entry):
+ return self.render_string("modules/entry.html", entry=entry)
+
+
+def main():
+ tornado.options.parse_command_line()
+ http_server = tornado.httpserver.HTTPServer(Application())
+ http_server.listen(options.port)
+ tornado.ioloop.IOLoop.instance().start()
+
+
+if __name__ == "__main__":
+ main()
--- /dev/null
+#!/usr/bin/env python
+# Copyright (c) 2007-2008 ActiveState Corp.
+# License: MIT (http://www.opensource.org/licenses/mit-license.php)
+
+r"""A fast and complete Python implementation of Markdown.
+
+[from http://daringfireball.net/projects/markdown/]
+> Markdown is a text-to-HTML filter; it translates an easy-to-read /
+> easy-to-write structured text format into HTML. Markdown's text
+> format is most similar to that of plain text email, and supports
+> features such as headers, *emphasis*, code blocks, blockquotes, and
+> links.
+>
+> Markdown's syntax is designed not as a generic markup language, but
+> specifically to serve as a front-end to (X)HTML. You can use span-level
+> HTML tags anywhere in a Markdown document, and you can use block level
+> HTML tags (like <div> and <table> as well).
+
+Module usage:
+
+ >>> import markdown2
+ >>> markdown2.markdown("*boo!*") # or use `html = markdown_path(PATH)`
+ u'<p><em>boo!</em></p>\n'
+
+ >>> markdowner = Markdown()
+ >>> markdowner.convert("*boo!*")
+ u'<p><em>boo!</em></p>\n'
+ >>> markdowner.convert("**boom!**")
+ u'<p><strong>boom!</strong></p>\n'
+
+This implementation of Markdown implements the full "core" syntax plus a
+number of extras (e.g., code syntax coloring, footnotes) as described on
+<http://code.google.com/p/python-markdown2/wiki/Extras>.
+"""
+
+cmdln_desc = """A fast and complete Python implementation of Markdown, a
+text-to-HTML conversion tool for web writers.
+"""
+
+# Dev Notes:
+# - There is already a Python markdown processor
+# (http://www.freewisdom.org/projects/python-markdown/).
+# - Python's regex syntax doesn't have '\z', so I'm using '\Z'. I'm
+# not yet sure if there implications with this. Compare 'pydoc sre'
+# and 'perldoc perlre'.
+
+__version_info__ = (1, 0, 1, 14) # first three nums match Markdown.pl
+__version__ = '1.0.1.14'
+__author__ = "Trent Mick"
+
+import os
+import sys
+from pprint import pprint
+import re
+import logging
+try:
+ from hashlib import md5
+except ImportError:
+ from md5 import md5
+import optparse
+from random import random
+import codecs
+
+
+
+#---- Python version compat
+
+if sys.version_info[:2] < (2,4):
+ from sets import Set as set
+ def reversed(sequence):
+ for i in sequence[::-1]:
+ yield i
+ def _unicode_decode(s, encoding, errors='xmlcharrefreplace'):
+ return unicode(s, encoding, errors)
+else:
+ def _unicode_decode(s, encoding, errors='strict'):
+ return s.decode(encoding, errors)
+
+
+#---- globals
+
+DEBUG = False
+log = logging.getLogger("markdown")
+
+DEFAULT_TAB_WIDTH = 4
+
+# Table of hash values for escaped characters:
+def _escape_hash(s):
+ # Lame attempt to avoid possible collision with someone actually
+ # using the MD5 hexdigest of one of these chars in there text.
+ # Other ideas: random.random(), uuid.uuid()
+ #return md5(s).hexdigest() # Markdown.pl effectively does this.
+ return 'md5-'+md5(s).hexdigest()
+g_escape_table = dict([(ch, _escape_hash(ch))
+ for ch in '\\`*_{}[]()>#+-.!'])
+
+
+
+#---- exceptions
+
+class MarkdownError(Exception):
+ pass
+
+
+
+#---- public api
+
+def markdown_path(path, encoding="utf-8",
+ html4tags=False, tab_width=DEFAULT_TAB_WIDTH,
+ safe_mode=None, extras=None, link_patterns=None,
+ use_file_vars=False):
+ text = codecs.open(path, 'r', encoding).read()
+ return Markdown(html4tags=html4tags, tab_width=tab_width,
+ safe_mode=safe_mode, extras=extras,
+ link_patterns=link_patterns,
+ use_file_vars=use_file_vars).convert(text)
+
+def markdown(text, html4tags=False, tab_width=DEFAULT_TAB_WIDTH,
+ safe_mode=None, extras=None, link_patterns=None,
+ use_file_vars=False):
+ return Markdown(html4tags=html4tags, tab_width=tab_width,
+ safe_mode=safe_mode, extras=extras,
+ link_patterns=link_patterns,
+ use_file_vars=use_file_vars).convert(text)
+
+class Markdown(object):
+ # The dict of "extras" to enable in processing -- a mapping of
+ # extra name to argument for the extra. Most extras do not have an
+ # argument, in which case the value is None.
+ #
+ # This can be set via (a) subclassing and (b) the constructor
+ # "extras" argument.
+ extras = None
+
+ urls = None
+ titles = None
+ html_blocks = None
+ html_spans = None
+ html_removed_text = "[HTML_REMOVED]" # for compat with markdown.py
+
+ # Used to track when we're inside an ordered or unordered list
+ # (see _ProcessListItems() for details):
+ list_level = 0
+
+ _ws_only_line_re = re.compile(r"^[ \t]+$", re.M)
+
+ def __init__(self, html4tags=False, tab_width=4, safe_mode=None,
+ extras=None, link_patterns=None, use_file_vars=False):
+ if html4tags:
+ self.empty_element_suffix = ">"
+ else:
+ self.empty_element_suffix = " />"
+ self.tab_width = tab_width
+
+ # For compatibility with earlier markdown2.py and with
+ # markdown.py's safe_mode being a boolean,
+ # safe_mode == True -> "replace"
+ if safe_mode is True:
+ self.safe_mode = "replace"
+ else:
+ self.safe_mode = safe_mode
+
+ if self.extras is None:
+ self.extras = {}
+ elif not isinstance(self.extras, dict):
+ self.extras = dict([(e, None) for e in self.extras])
+ if extras:
+ if not isinstance(extras, dict):
+ extras = dict([(e, None) for e in extras])
+ self.extras.update(extras)
+ assert isinstance(self.extras, dict)
+ self._instance_extras = self.extras.copy()
+ self.link_patterns = link_patterns
+ self.use_file_vars = use_file_vars
+ self._outdent_re = re.compile(r'^(\t|[ ]{1,%d})' % tab_width, re.M)
+
+ def reset(self):
+ self.urls = {}
+ self.titles = {}
+ self.html_blocks = {}
+ self.html_spans = {}
+ self.list_level = 0
+ self.extras = self._instance_extras.copy()
+ if "footnotes" in self.extras:
+ self.footnotes = {}
+ self.footnote_ids = []
+
+ def convert(self, text):
+ """Convert the given text."""
+ # Main function. The order in which other subs are called here is
+ # essential. Link and image substitutions need to happen before
+ # _EscapeSpecialChars(), so that any *'s or _'s in the <a>
+ # and <img> tags get encoded.
+
+ # Clear the global hashes. If we don't clear these, you get conflicts
+ # from other articles when generating a page which contains more than
+ # one article (e.g. an index page that shows the N most recent
+ # articles):
+ self.reset()
+
+ if not isinstance(text, unicode):
+ #TODO: perhaps shouldn't presume UTF-8 for string input?
+ text = unicode(text, 'utf-8')
+
+ if self.use_file_vars:
+ # Look for emacs-style file variable hints.
+ emacs_vars = self._get_emacs_vars(text)
+ if "markdown-extras" in emacs_vars:
+ splitter = re.compile("[ ,]+")
+ for e in splitter.split(emacs_vars["markdown-extras"]):
+ if '=' in e:
+ ename, earg = e.split('=', 1)
+ try:
+ earg = int(earg)
+ except ValueError:
+ pass
+ else:
+ ename, earg = e, None
+ self.extras[ename] = earg
+
+ # Standardize line endings:
+ text = re.sub("\r\n|\r", "\n", text)
+
+ # Make sure $text ends with a couple of newlines:
+ text += "\n\n"
+
+ # Convert all tabs to spaces.
+ text = self._detab(text)
+
+ # Strip any lines consisting only of spaces and tabs.
+ # This makes subsequent regexen easier to write, because we can
+ # match consecutive blank lines with /\n+/ instead of something
+ # contorted like /[ \t]*\n+/ .
+ text = self._ws_only_line_re.sub("", text)
+
+ if self.safe_mode:
+ text = self._hash_html_spans(text)
+
+ # Turn block-level HTML blocks into hash entries
+ text = self._hash_html_blocks(text, raw=True)
+
+ # Strip link definitions, store in hashes.
+ if "footnotes" in self.extras:
+ # Must do footnotes first because an unlucky footnote defn
+ # looks like a link defn:
+ # [^4]: this "looks like a link defn"
+ text = self._strip_footnote_definitions(text)
+ text = self._strip_link_definitions(text)
+
+ text = self._run_block_gamut(text)
+
+ if "footnotes" in self.extras:
+ text = self._add_footnotes(text)
+
+ text = self._unescape_special_chars(text)
+
+ if self.safe_mode:
+ text = self._unhash_html_spans(text)
+
+ text += "\n"
+ return text
+
+ _emacs_oneliner_vars_pat = re.compile(r"-\*-\s*([^\r\n]*?)\s*-\*-", re.UNICODE)
+ # This regular expression is intended to match blocks like this:
+ # PREFIX Local Variables: SUFFIX
+ # PREFIX mode: Tcl SUFFIX
+ # PREFIX End: SUFFIX
+ # Some notes:
+ # - "[ \t]" is used instead of "\s" to specifically exclude newlines
+ # - "(\r\n|\n|\r)" is used instead of "$" because the sre engine does
+ # not like anything other than Unix-style line terminators.
+ _emacs_local_vars_pat = re.compile(r"""^
+ (?P<prefix>(?:[^\r\n|\n|\r])*?)
+ [\ \t]*Local\ Variables:[\ \t]*
+ (?P<suffix>.*?)(?:\r\n|\n|\r)
+ (?P<content>.*?\1End:)
+ """, re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE)
+
+ def _get_emacs_vars(self, text):
+ """Return a dictionary of emacs-style local variables.
+
+ Parsing is done loosely according to this spec (and according to
+ some in-practice deviations from this):
+ http://www.gnu.org/software/emacs/manual/html_node/emacs/Specifying-File-Variables.html#Specifying-File-Variables
+ """
+ emacs_vars = {}
+ SIZE = pow(2, 13) # 8kB
+
+ # Search near the start for a '-*-'-style one-liner of variables.
+ head = text[:SIZE]
+ if "-*-" in head:
+ match = self._emacs_oneliner_vars_pat.search(head)
+ if match:
+ emacs_vars_str = match.group(1)
+ assert '\n' not in emacs_vars_str
+ emacs_var_strs = [s.strip() for s in emacs_vars_str.split(';')
+ if s.strip()]
+ if len(emacs_var_strs) == 1 and ':' not in emacs_var_strs[0]:
+ # While not in the spec, this form is allowed by emacs:
+ # -*- Tcl -*-
+ # where the implied "variable" is "mode". This form
+ # is only allowed if there are no other variables.
+ emacs_vars["mode"] = emacs_var_strs[0].strip()
+ else:
+ for emacs_var_str in emacs_var_strs:
+ try:
+ variable, value = emacs_var_str.strip().split(':', 1)
+ except ValueError:
+ log.debug("emacs variables error: malformed -*- "
+ "line: %r", emacs_var_str)
+ continue
+ # Lowercase the variable name because Emacs allows "Mode"
+ # or "mode" or "MoDe", etc.
+ emacs_vars[variable.lower()] = value.strip()
+
+ tail = text[-SIZE:]
+ if "Local Variables" in tail:
+ match = self._emacs_local_vars_pat.search(tail)
+ if match:
+ prefix = match.group("prefix")
+ suffix = match.group("suffix")
+ lines = match.group("content").splitlines(0)
+ #print "prefix=%r, suffix=%r, content=%r, lines: %s"\
+ # % (prefix, suffix, match.group("content"), lines)
+
+ # Validate the Local Variables block: proper prefix and suffix
+ # usage.
+ for i, line in enumerate(lines):
+ if not line.startswith(prefix):
+ log.debug("emacs variables error: line '%s' "
+ "does not use proper prefix '%s'"
+ % (line, prefix))
+ return {}
+ # Don't validate suffix on last line. Emacs doesn't care,
+ # neither should we.
+ if i != len(lines)-1 and not line.endswith(suffix):
+ log.debug("emacs variables error: line '%s' "
+ "does not use proper suffix '%s'"
+ % (line, suffix))
+ return {}
+
+ # Parse out one emacs var per line.
+ continued_for = None
+ for line in lines[:-1]: # no var on the last line ("PREFIX End:")
+ if prefix: line = line[len(prefix):] # strip prefix
+ if suffix: line = line[:-len(suffix)] # strip suffix
+ line = line.strip()
+ if continued_for:
+ variable = continued_for
+ if line.endswith('\\'):
+ line = line[:-1].rstrip()
+ else:
+ continued_for = None
+ emacs_vars[variable] += ' ' + line
+ else:
+ try:
+ variable, value = line.split(':', 1)
+ except ValueError:
+ log.debug("local variables error: missing colon "
+ "in local variables entry: '%s'" % line)
+ continue
+ # Do NOT lowercase the variable name, because Emacs only
+ # allows "mode" (and not "Mode", "MoDe", etc.) in this block.
+ value = value.strip()
+ if value.endswith('\\'):
+ value = value[:-1].rstrip()
+ continued_for = variable
+ else:
+ continued_for = None
+ emacs_vars[variable] = value
+
+ # Unquote values.
+ for var, val in emacs_vars.items():
+ if len(val) > 1 and (val.startswith('"') and val.endswith('"')
+ or val.startswith('"') and val.endswith('"')):
+ emacs_vars[var] = val[1:-1]
+
+ return emacs_vars
+
+ # Cribbed from a post by Bart Lateur:
+ # <http://www.nntp.perl.org/group/perl.macperl.anyperl/154>
+ _detab_re = re.compile(r'(.*?)\t', re.M)
+ def _detab_sub(self, match):
+ g1 = match.group(1)
+ return g1 + (' ' * (self.tab_width - len(g1) % self.tab_width))
+ def _detab(self, text):
+ r"""Remove (leading?) tabs from a file.
+
+ >>> m = Markdown()
+ >>> m._detab("\tfoo")
+ ' foo'
+ >>> m._detab(" \tfoo")
+ ' foo'
+ >>> m._detab("\t foo")
+ ' foo'
+ >>> m._detab(" foo")
+ ' foo'
+ >>> m._detab(" foo\n\tbar\tblam")
+ ' foo\n bar blam'
+ """
+ if '\t' not in text:
+ return text
+ return self._detab_re.subn(self._detab_sub, text)[0]
+
+ _block_tags_a = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math|ins|del'
+ _strict_tag_block_re = re.compile(r"""
+ ( # save in \1
+ ^ # start of line (with re.M)
+ <(%s) # start tag = \2
+ \b # word break
+ (.*\n)*? # any number of lines, minimally matching
+ </\2> # the matching end tag
+ [ \t]* # trailing spaces/tabs
+ (?=\n+|\Z) # followed by a newline or end of document
+ )
+ """ % _block_tags_a,
+ re.X | re.M)
+
+ _block_tags_b = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math'
+ _liberal_tag_block_re = re.compile(r"""
+ ( # save in \1
+ ^ # start of line (with re.M)
+ <(%s) # start tag = \2
+ \b # word break
+ (.*\n)*? # any number of lines, minimally matching
+ .*</\2> # the matching end tag
+ [ \t]* # trailing spaces/tabs
+ (?=\n+|\Z) # followed by a newline or end of document
+ )
+ """ % _block_tags_b,
+ re.X | re.M)
+
+ def _hash_html_block_sub(self, match, raw=False):
+ html = match.group(1)
+ if raw and self.safe_mode:
+ html = self._sanitize_html(html)
+ key = _hash_text(html)
+ self.html_blocks[key] = html
+ return "\n\n" + key + "\n\n"
+
+ def _hash_html_blocks(self, text, raw=False):
+ """Hashify HTML blocks
+
+ We only want to do this for block-level HTML tags, such as headers,
+ lists, and tables. That's because we still want to wrap <p>s around
+ "paragraphs" that are wrapped in non-block-level tags, such as anchors,
+ phrase emphasis, and spans. The list of tags we're looking for is
+ hard-coded.
+
+ @param raw {boolean} indicates if these are raw HTML blocks in
+ the original source. It makes a difference in "safe" mode.
+ """
+ if '<' not in text:
+ return text
+
+ # Pass `raw` value into our calls to self._hash_html_block_sub.
+ hash_html_block_sub = _curry(self._hash_html_block_sub, raw=raw)
+
+ # First, look for nested blocks, e.g.:
+ # <div>
+ # <div>
+ # tags for inner block must be indented.
+ # </div>
+ # </div>
+ #
+ # The outermost tags must start at the left margin for this to match, and
+ # the inner nested divs must be indented.
+ # We need to do this before the next, more liberal match, because the next
+ # match will start at the first `<div>` and stop at the first `</div>`.
+ text = self._strict_tag_block_re.sub(hash_html_block_sub, text)
+
+ # Now match more liberally, simply from `\n<tag>` to `</tag>\n`
+ text = self._liberal_tag_block_re.sub(hash_html_block_sub, text)
+
+ # Special case just for <hr />. It was easier to make a special
+ # case than to make the other regex more complicated.
+ if "<hr" in text:
+ _hr_tag_re = _hr_tag_re_from_tab_width(self.tab_width)
+ text = _hr_tag_re.sub(hash_html_block_sub, text)
+
+ # Special case for standalone HTML comments:
+ if "<!--" in text:
+ start = 0
+ while True:
+ # Delimiters for next comment block.
+ try:
+ start_idx = text.index("<!--", start)
+ except ValueError, ex:
+ break
+ try:
+ end_idx = text.index("-->", start_idx) + 3
+ except ValueError, ex:
+ break
+
+ # Start position for next comment block search.
+ start = end_idx
+
+ # Validate whitespace before comment.
+ if start_idx:
+ # - Up to `tab_width - 1` spaces before start_idx.
+ for i in range(self.tab_width - 1):
+ if text[start_idx - 1] != ' ':
+ break
+ start_idx -= 1
+ if start_idx == 0:
+ break
+ # - Must be preceded by 2 newlines or hit the start of
+ # the document.
+ if start_idx == 0:
+ pass
+ elif start_idx == 1 and text[0] == '\n':
+ start_idx = 0 # to match minute detail of Markdown.pl regex
+ elif text[start_idx-2:start_idx] == '\n\n':
+ pass
+ else:
+ break
+
+ # Validate whitespace after comment.
+ # - Any number of spaces and tabs.
+ while end_idx < len(text):
+ if text[end_idx] not in ' \t':
+ break
+ end_idx += 1
+ # - Must be following by 2 newlines or hit end of text.
+ if text[end_idx:end_idx+2] not in ('', '\n', '\n\n'):
+ continue
+
+ # Escape and hash (must match `_hash_html_block_sub`).
+ html = text[start_idx:end_idx]
+ if raw and self.safe_mode:
+ html = self._sanitize_html(html)
+ key = _hash_text(html)
+ self.html_blocks[key] = html
+ text = text[:start_idx] + "\n\n" + key + "\n\n" + text[end_idx:]
+
+ if "xml" in self.extras:
+ # Treat XML processing instructions and namespaced one-liner
+ # tags as if they were block HTML tags. E.g., if standalone
+ # (i.e. are their own paragraph), the following do not get
+ # wrapped in a <p> tag:
+ # <?foo bar?>
+ #
+ # <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="chapter_1.md"/>
+ _xml_oneliner_re = _xml_oneliner_re_from_tab_width(self.tab_width)
+ text = _xml_oneliner_re.sub(hash_html_block_sub, text)
+
+ return text
+
+ def _strip_link_definitions(self, text):
+ # Strips link definitions from text, stores the URLs and titles in
+ # hash references.
+ less_than_tab = self.tab_width - 1
+
+ # Link defs are in the form:
+ # [id]: url "optional title"
+ _link_def_re = re.compile(r"""
+ ^[ ]{0,%d}\[(.+)\]: # id = \1
+ [ \t]*
+ \n? # maybe *one* newline
+ [ \t]*
+ <?(.+?)>? # url = \2
+ [ \t]*
+ (?:
+ \n? # maybe one newline
+ [ \t]*
+ (?<=\s) # lookbehind for whitespace
+ ['"(]
+ ([^\n]*) # title = \3
+ ['")]
+ [ \t]*
+ )? # title is optional
+ (?:\n+|\Z)
+ """ % less_than_tab, re.X | re.M | re.U)
+ return _link_def_re.sub(self._extract_link_def_sub, text)
+
+ def _extract_link_def_sub(self, match):
+ id, url, title = match.groups()
+ key = id.lower() # Link IDs are case-insensitive
+ self.urls[key] = self._encode_amps_and_angles(url)
+ if title:
+ self.titles[key] = title.replace('"', '"')
+ return ""
+
+ def _extract_footnote_def_sub(self, match):
+ id, text = match.groups()
+ text = _dedent(text, skip_first_line=not text.startswith('\n')).strip()
+ normed_id = re.sub(r'\W', '-', id)
+ # Ensure footnote text ends with a couple newlines (for some
+ # block gamut matches).
+ self.footnotes[normed_id] = text + "\n\n"
+ return ""
+
+ def _strip_footnote_definitions(self, text):
+ """A footnote definition looks like this:
+
+ [^note-id]: Text of the note.
+
+ May include one or more indented paragraphs.
+
+ Where,
+ - The 'note-id' can be pretty much anything, though typically it
+ is the number of the footnote.
+ - The first paragraph may start on the next line, like so:
+
+ [^note-id]:
+ Text of the note.
+ """
+ less_than_tab = self.tab_width - 1
+ footnote_def_re = re.compile(r'''
+ ^[ ]{0,%d}\[\^(.+)\]: # id = \1
+ [ \t]*
+ ( # footnote text = \2
+ # First line need not start with the spaces.
+ (?:\s*.*\n+)
+ (?:
+ (?:[ ]{%d} | \t) # Subsequent lines must be indented.
+ .*\n+
+ )*
+ )
+ # Lookahead for non-space at line-start, or end of doc.
+ (?:(?=^[ ]{0,%d}\S)|\Z)
+ ''' % (less_than_tab, self.tab_width, self.tab_width),
+ re.X | re.M)
+ return footnote_def_re.sub(self._extract_footnote_def_sub, text)
+
+
+ _hr_res = [
+ re.compile(r"^[ ]{0,2}([ ]?\*[ ]?){3,}[ \t]*$", re.M),
+ re.compile(r"^[ ]{0,2}([ ]?\-[ ]?){3,}[ \t]*$", re.M),
+ re.compile(r"^[ ]{0,2}([ ]?\_[ ]?){3,}[ \t]*$", re.M),
+ ]
+
+ def _run_block_gamut(self, text):
+ # These are all the transformations that form block-level
+ # tags like paragraphs, headers, and list items.
+
+ text = self._do_headers(text)
+
+ # Do Horizontal Rules:
+ hr = "\n<hr"+self.empty_element_suffix+"\n"
+ for hr_re in self._hr_res:
+ text = hr_re.sub(hr, text)
+
+ text = self._do_lists(text)
+
+ if "pyshell" in self.extras:
+ text = self._prepare_pyshell_blocks(text)
+
+ text = self._do_code_blocks(text)
+
+ text = self._do_block_quotes(text)
+
+ # We already ran _HashHTMLBlocks() before, in Markdown(), but that
+ # was to escape raw HTML in the original Markdown source. This time,
+ # we're escaping the markup we've just created, so that we don't wrap
+ # <p> tags around block-level tags.
+ text = self._hash_html_blocks(text)
+
+ text = self._form_paragraphs(text)
+
+ return text
+
+ def _pyshell_block_sub(self, match):
+ lines = match.group(0).splitlines(0)
+ _dedentlines(lines)
+ indent = ' ' * self.tab_width
+ s = ('\n' # separate from possible cuddled paragraph
+ + indent + ('\n'+indent).join(lines)
+ + '\n\n')
+ return s
+
+ def _prepare_pyshell_blocks(self, text):
+ """Ensure that Python interactive shell sessions are put in
+ code blocks -- even if not properly indented.
+ """
+ if ">>>" not in text:
+ return text
+
+ less_than_tab = self.tab_width - 1
+ _pyshell_block_re = re.compile(r"""
+ ^([ ]{0,%d})>>>[ ].*\n # first line
+ ^(\1.*\S+.*\n)* # any number of subsequent lines
+ ^\n # ends with a blank line
+ """ % less_than_tab, re.M | re.X)
+
+ return _pyshell_block_re.sub(self._pyshell_block_sub, text)
+
+ def _run_span_gamut(self, text):
+ # These are all the transformations that occur *within* block-level
+ # tags like paragraphs, headers, and list items.
+
+ text = self._do_code_spans(text)
+
+ text = self._escape_special_chars(text)
+
+ # Process anchor and image tags.
+ text = self._do_links(text)
+
+ # Make links out of things like `<http://example.com/>`
+ # Must come after _do_links(), because you can use < and >
+ # delimiters in inline links like [this](<url>).
+ text = self._do_auto_links(text)
+
+ if "link-patterns" in self.extras:
+ text = self._do_link_patterns(text)
+
+ text = self._encode_amps_and_angles(text)
+
+ text = self._do_italics_and_bold(text)
+
+ # Do hard breaks:
+ text = re.sub(r" {2,}\n", " <br%s\n" % self.empty_element_suffix, text)
+
+ return text
+
+ # "Sorta" because auto-links are identified as "tag" tokens.
+ _sorta_html_tokenize_re = re.compile(r"""
+ (
+ # tag
+ </?
+ (?:\w+) # tag name
+ (?:\s+(?:[\w-]+:)?[\w-]+=(?:".*?"|'.*?'))* # attributes
+ \s*/?>
+ |
+ # auto-link (e.g., <http://www.activestate.com/>)
+ <\w+[^>]*>
+ |
+ <!--.*?--> # comment
+ |
+ <\?.*?\?> # processing instruction
+ )
+ """, re.X)
+
+ def _escape_special_chars(self, text):
+ # Python markdown note: the HTML tokenization here differs from
+ # that in Markdown.pl, hence the behaviour for subtle cases can
+ # differ (I believe the tokenizer here does a better job because
+ # it isn't susceptible to unmatched '<' and '>' in HTML tags).
+ # Note, however, that '>' is not allowed in an auto-link URL
+ # here.
+ escaped = []
+ is_html_markup = False
+ for token in self._sorta_html_tokenize_re.split(text):
+ if is_html_markup:
+ # Within tags/HTML-comments/auto-links, encode * and _
+ # so they don't conflict with their use in Markdown for
+ # italics and strong. We're replacing each such
+ # character with its corresponding MD5 checksum value;
+ # this is likely overkill, but it should prevent us from
+ # colliding with the escape values by accident.
+ escaped.append(token.replace('*', g_escape_table['*'])
+ .replace('_', g_escape_table['_']))
+ else:
+ escaped.append(self._encode_backslash_escapes(token))
+ is_html_markup = not is_html_markup
+ return ''.join(escaped)
+
+ def _hash_html_spans(self, text):
+ # Used for safe_mode.
+
+ def _is_auto_link(s):
+ if ':' in s and self._auto_link_re.match(s):
+ return True
+ elif '@' in s and self._auto_email_link_re.match(s):
+ return True
+ return False
+
+ tokens = []
+ is_html_markup = False
+ for token in self._sorta_html_tokenize_re.split(text):
+ if is_html_markup and not _is_auto_link(token):
+ sanitized = self._sanitize_html(token)
+ key = _hash_text(sanitized)
+ self.html_spans[key] = sanitized
+ tokens.append(key)
+ else:
+ tokens.append(token)
+ is_html_markup = not is_html_markup
+ return ''.join(tokens)
+
+ def _unhash_html_spans(self, text):
+ for key, sanitized in self.html_spans.items():
+ text = text.replace(key, sanitized)
+ return text
+
+ def _sanitize_html(self, s):
+ if self.safe_mode == "replace":
+ return self.html_removed_text
+ elif self.safe_mode == "escape":
+ replacements = [
+ ('&', '&'),
+ ('<', '<'),
+ ('>', '>'),
+ ]
+ for before, after in replacements:
+ s = s.replace(before, after)
+ return s
+ else:
+ raise MarkdownError("invalid value for 'safe_mode': %r (must be "
+ "'escape' or 'replace')" % self.safe_mode)
+
+ _tail_of_inline_link_re = re.compile(r'''
+ # Match tail of: [text](/url/) or [text](/url/ "title")
+ \( # literal paren
+ [ \t]*
+ (?P<url> # \1
+ <.*?>
+ |
+ .*?
+ )
+ [ \t]*
+ ( # \2
+ (['"]) # quote char = \3
+ (?P<title>.*?)
+ \3 # matching quote
+ )? # title is optional
+ \)
+ ''', re.X | re.S)
+ _tail_of_reference_link_re = re.compile(r'''
+ # Match tail of: [text][id]
+ [ ]? # one optional space
+ (?:\n[ ]*)? # one optional newline followed by spaces
+ \[
+ (?P<id>.*?)
+ \]
+ ''', re.X | re.S)
+
+ def _do_links(self, text):
+ """Turn Markdown link shortcuts into XHTML <a> and <img> tags.
+
+ This is a combination of Markdown.pl's _DoAnchors() and
+ _DoImages(). They are done together because that simplified the
+ approach. It was necessary to use a different approach than
+ Markdown.pl because of the lack of atomic matching support in
+ Python's regex engine used in $g_nested_brackets.
+ """
+ MAX_LINK_TEXT_SENTINEL = 3000 # markdown2 issue 24
+
+ # `anchor_allowed_pos` is used to support img links inside
+ # anchors, but not anchors inside anchors. An anchor's start
+ # pos must be `>= anchor_allowed_pos`.
+ anchor_allowed_pos = 0
+
+ curr_pos = 0
+ while True: # Handle the next link.
+ # The next '[' is the start of:
+ # - an inline anchor: [text](url "title")
+ # - a reference anchor: [text][id]
+ # - an inline img: 
+ # - a reference img: ![text][id]
+ # - a footnote ref: [^id]
+ # (Only if 'footnotes' extra enabled)
+ # - a footnote defn: [^id]: ...
+ # (Only if 'footnotes' extra enabled) These have already
+ # been stripped in _strip_footnote_definitions() so no
+ # need to watch for them.
+ # - a link definition: [id]: url "title"
+ # These have already been stripped in
+ # _strip_link_definitions() so no need to watch for them.
+ # - not markup: [...anything else...
+ try:
+ start_idx = text.index('[', curr_pos)
+ except ValueError:
+ break
+ text_length = len(text)
+
+ # Find the matching closing ']'.
+ # Markdown.pl allows *matching* brackets in link text so we
+ # will here too. Markdown.pl *doesn't* currently allow
+ # matching brackets in img alt text -- we'll differ in that
+ # regard.
+ bracket_depth = 0
+ for p in range(start_idx+1, min(start_idx+MAX_LINK_TEXT_SENTINEL,
+ text_length)):
+ ch = text[p]
+ if ch == ']':
+ bracket_depth -= 1
+ if bracket_depth < 0:
+ break
+ elif ch == '[':
+ bracket_depth += 1
+ else:
+ # Closing bracket not found within sentinel length.
+ # This isn't markup.
+ curr_pos = start_idx + 1
+ continue
+ link_text = text[start_idx+1:p]
+
+ # Possibly a footnote ref?
+ if "footnotes" in self.extras and link_text.startswith("^"):
+ normed_id = re.sub(r'\W', '-', link_text[1:])
+ if normed_id in self.footnotes:
+ self.footnote_ids.append(normed_id)
+ result = '<sup class="footnote-ref" id="fnref-%s">' \
+ '<a href="#fn-%s">%s</a></sup>' \
+ % (normed_id, normed_id, len(self.footnote_ids))
+ text = text[:start_idx] + result + text[p+1:]
+ else:
+ # This id isn't defined, leave the markup alone.
+ curr_pos = p+1
+ continue
+
+ # Now determine what this is by the remainder.
+ p += 1
+ if p == text_length:
+ return text
+
+ # Inline anchor or img?
+ if text[p] == '(': # attempt at perf improvement
+ match = self._tail_of_inline_link_re.match(text, p)
+ if match:
+ # Handle an inline anchor or img.
+ is_img = start_idx > 0 and text[start_idx-1] == "!"
+ if is_img:
+ start_idx -= 1
+
+ url, title = match.group("url"), match.group("title")
+ if url and url[0] == '<':
+ url = url[1:-1] # '<url>' -> 'url'
+ # We've got to encode these to avoid conflicting
+ # with italics/bold.
+ url = url.replace('*', g_escape_table['*']) \
+ .replace('_', g_escape_table['_'])
+ if title:
+ title_str = ' title="%s"' \
+ % title.replace('*', g_escape_table['*']) \
+ .replace('_', g_escape_table['_']) \
+ .replace('"', '"')
+ else:
+ title_str = ''
+ if is_img:
+ result = '<img src="%s" alt="%s"%s%s' \
+ % (url, link_text.replace('"', '"'),
+ title_str, self.empty_element_suffix)
+ curr_pos = start_idx + len(result)
+ text = text[:start_idx] + result + text[match.end():]
+ elif start_idx >= anchor_allowed_pos:
+ result_head = '<a href="%s"%s>' % (url, title_str)
+ result = '%s%s</a>' % (result_head, link_text)
+ # <img> allowed from curr_pos on, <a> from
+ # anchor_allowed_pos on.
+ curr_pos = start_idx + len(result_head)
+ anchor_allowed_pos = start_idx + len(result)
+ text = text[:start_idx] + result + text[match.end():]
+ else:
+ # Anchor not allowed here.
+ curr_pos = start_idx + 1
+ continue
+
+ # Reference anchor or img?
+ else:
+ match = self._tail_of_reference_link_re.match(text, p)
+ if match:
+ # Handle a reference-style anchor or img.
+ is_img = start_idx > 0 and text[start_idx-1] == "!"
+ if is_img:
+ start_idx -= 1
+ link_id = match.group("id").lower()
+ if not link_id:
+ link_id = link_text.lower() # for links like [this][]
+ if link_id in self.urls:
+ url = self.urls[link_id]
+ # We've got to encode these to avoid conflicting
+ # with italics/bold.
+ url = url.replace('*', g_escape_table['*']) \
+ .replace('_', g_escape_table['_'])
+ title = self.titles.get(link_id)
+ if title:
+ title = title.replace('*', g_escape_table['*']) \
+ .replace('_', g_escape_table['_'])
+ title_str = ' title="%s"' % title
+ else:
+ title_str = ''
+ if is_img:
+ result = '<img src="%s" alt="%s"%s%s' \
+ % (url, link_text.replace('"', '"'),
+ title_str, self.empty_element_suffix)
+ curr_pos = start_idx + len(result)
+ text = text[:start_idx] + result + text[match.end():]
+ elif start_idx >= anchor_allowed_pos:
+ result = '<a href="%s"%s>%s</a>' \
+ % (url, title_str, link_text)
+ result_head = '<a href="%s"%s>' % (url, title_str)
+ result = '%s%s</a>' % (result_head, link_text)
+ # <img> allowed from curr_pos on, <a> from
+ # anchor_allowed_pos on.
+ curr_pos = start_idx + len(result_head)
+ anchor_allowed_pos = start_idx + len(result)
+ text = text[:start_idx] + result + text[match.end():]
+ else:
+ # Anchor not allowed here.
+ curr_pos = start_idx + 1
+ else:
+ # This id isn't defined, leave the markup alone.
+ curr_pos = match.end()
+ continue
+
+ # Otherwise, it isn't markup.
+ curr_pos = start_idx + 1
+
+ return text
+
+
+ _setext_h_re = re.compile(r'^(.+)[ \t]*\n(=+|-+)[ \t]*\n+', re.M)
+ def _setext_h_sub(self, match):
+ n = {"=": 1, "-": 2}[match.group(2)[0]]
+ demote_headers = self.extras.get("demote-headers")
+ if demote_headers:
+ n = min(n + demote_headers, 6)
+ return "<h%d>%s</h%d>\n\n" \
+ % (n, self._run_span_gamut(match.group(1)), n)
+
+ _atx_h_re = re.compile(r'''
+ ^(\#{1,6}) # \1 = string of #'s
+ [ \t]*
+ (.+?) # \2 = Header text
+ [ \t]*
+ (?<!\\) # ensure not an escaped trailing '#'
+ \#* # optional closing #'s (not counted)
+ \n+
+ ''', re.X | re.M)
+ def _atx_h_sub(self, match):
+ n = len(match.group(1))
+ demote_headers = self.extras.get("demote-headers")
+ if demote_headers:
+ n = min(n + demote_headers, 6)
+ return "<h%d>%s</h%d>\n\n" \
+ % (n, self._run_span_gamut(match.group(2)), n)
+
+ def _do_headers(self, text):
+ # Setext-style headers:
+ # Header 1
+ # ========
+ #
+ # Header 2
+ # --------
+ text = self._setext_h_re.sub(self._setext_h_sub, text)
+
+ # atx-style headers:
+ # # Header 1
+ # ## Header 2
+ # ## Header 2 with closing hashes ##
+ # ...
+ # ###### Header 6
+ text = self._atx_h_re.sub(self._atx_h_sub, text)
+
+ return text
+
+
+ _marker_ul_chars = '*+-'
+ _marker_any = r'(?:[%s]|\d+\.)' % _marker_ul_chars
+ _marker_ul = '(?:[%s])' % _marker_ul_chars
+ _marker_ol = r'(?:\d+\.)'
+
+ def _list_sub(self, match):
+ lst = match.group(1)
+ lst_type = match.group(3) in self._marker_ul_chars and "ul" or "ol"
+ result = self._process_list_items(lst)
+ if self.list_level:
+ return "<%s>\n%s</%s>\n" % (lst_type, result, lst_type)
+ else:
+ return "<%s>\n%s</%s>\n\n" % (lst_type, result, lst_type)
+
+ def _do_lists(self, text):
+ # Form HTML ordered (numbered) and unordered (bulleted) lists.
+
+ for marker_pat in (self._marker_ul, self._marker_ol):
+ # Re-usable pattern to match any entire ul or ol list:
+ less_than_tab = self.tab_width - 1
+ whole_list = r'''
+ ( # \1 = whole list
+ ( # \2
+ [ ]{0,%d}
+ (%s) # \3 = first list item marker
+ [ \t]+
+ )
+ (?:.+?)
+ ( # \4
+ \Z
+ |
+ \n{2,}
+ (?=\S)
+ (?! # Negative lookahead for another list item marker
+ [ \t]*
+ %s[ \t]+
+ )
+ )
+ )
+ ''' % (less_than_tab, marker_pat, marker_pat)
+
+ # We use a different prefix before nested lists than top-level lists.
+ # See extended comment in _process_list_items().
+ #
+ # Note: There's a bit of duplication here. My original implementation
+ # created a scalar regex pattern as the conditional result of the test on
+ # $g_list_level, and then only ran the $text =~ s{...}{...}egmx
+ # substitution once, using the scalar as the pattern. This worked,
+ # everywhere except when running under MT on my hosting account at Pair
+ # Networks. There, this caused all rebuilds to be killed by the reaper (or
+ # perhaps they crashed, but that seems incredibly unlikely given that the
+ # same script on the same server ran fine *except* under MT. I've spent
+ # more time trying to figure out why this is happening than I'd like to
+ # admit. My only guess, backed up by the fact that this workaround works,
+ # is that Perl optimizes the substition when it can figure out that the
+ # pattern will never change, and when this optimization isn't on, we run
+ # afoul of the reaper. Thus, the slightly redundant code to that uses two
+ # static s/// patterns rather than one conditional pattern.
+
+ if self.list_level:
+ sub_list_re = re.compile("^"+whole_list, re.X | re.M | re.S)
+ text = sub_list_re.sub(self._list_sub, text)
+ else:
+ list_re = re.compile(r"(?:(?<=\n\n)|\A\n?)"+whole_list,
+ re.X | re.M | re.S)
+ text = list_re.sub(self._list_sub, text)
+
+ return text
+
+ _list_item_re = re.compile(r'''
+ (\n)? # leading line = \1
+ (^[ \t]*) # leading whitespace = \2
+ (%s) [ \t]+ # list marker = \3
+ ((?:.+?) # list item text = \4
+ (\n{1,2})) # eols = \5
+ (?= \n* (\Z | \2 (%s) [ \t]+))
+ ''' % (_marker_any, _marker_any),
+ re.M | re.X | re.S)
+
+ _last_li_endswith_two_eols = False
+ def _list_item_sub(self, match):
+ item = match.group(4)
+ leading_line = match.group(1)
+ leading_space = match.group(2)
+ if leading_line or "\n\n" in item or self._last_li_endswith_two_eols:
+ item = self._run_block_gamut(self._outdent(item))
+ else:
+ # Recursion for sub-lists:
+ item = self._do_lists(self._outdent(item))
+ if item.endswith('\n'):
+ item = item[:-1]
+ item = self._run_span_gamut(item)
+ self._last_li_endswith_two_eols = (len(match.group(5)) == 2)
+ return "<li>%s</li>\n" % item
+
+ def _process_list_items(self, list_str):
+ # Process the contents of a single ordered or unordered list,
+ # splitting it into individual list items.
+
+ # The $g_list_level global keeps track of when we're inside a list.
+ # Each time we enter a list, we increment it; when we leave a list,
+ # we decrement. If it's zero, we're not in a list anymore.
+ #
+ # We do this because when we're not inside a list, we want to treat
+ # something like this:
+ #
+ # I recommend upgrading to version
+ # 8. Oops, now this line is treated
+ # as a sub-list.
+ #
+ # As a single paragraph, despite the fact that the second line starts
+ # with a digit-period-space sequence.
+ #
+ # Whereas when we're inside a list (or sub-list), that line will be
+ # treated as the start of a sub-list. What a kludge, huh? This is
+ # an aspect of Markdown's syntax that's hard to parse perfectly
+ # without resorting to mind-reading. Perhaps the solution is to
+ # change the syntax rules such that sub-lists must start with a
+ # starting cardinal number; e.g. "1." or "a.".
+ self.list_level += 1
+ self._last_li_endswith_two_eols = False
+ list_str = list_str.rstrip('\n') + '\n'
+ list_str = self._list_item_re.sub(self._list_item_sub, list_str)
+ self.list_level -= 1
+ return list_str
+
+ def _get_pygments_lexer(self, lexer_name):
+ try:
+ from pygments import lexers, util
+ except ImportError:
+ return None
+ try:
+ return lexers.get_lexer_by_name(lexer_name)
+ except util.ClassNotFound:
+ return None
+
+ def _color_with_pygments(self, codeblock, lexer, **formatter_opts):
+ import pygments
+ import pygments.formatters
+
+ class HtmlCodeFormatter(pygments.formatters.HtmlFormatter):
+ def _wrap_code(self, inner):
+ """A function for use in a Pygments Formatter which
+ wraps in <code> tags.
+ """
+ yield 0, "<code>"
+ for tup in inner:
+ yield tup
+ yield 0, "</code>"
+
+ def wrap(self, source, outfile):
+ """Return the source with a code, pre, and div."""
+ return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
+
+ formatter = HtmlCodeFormatter(cssclass="codehilite", **formatter_opts)
+ return pygments.highlight(codeblock, lexer, formatter)
+
+ def _code_block_sub(self, match):
+ codeblock = match.group(1)
+ codeblock = self._outdent(codeblock)
+ codeblock = self._detab(codeblock)
+ codeblock = codeblock.lstrip('\n') # trim leading newlines
+ codeblock = codeblock.rstrip() # trim trailing whitespace
+
+ if "code-color" in self.extras and codeblock.startswith(":::"):
+ lexer_name, rest = codeblock.split('\n', 1)
+ lexer_name = lexer_name[3:].strip()
+ lexer = self._get_pygments_lexer(lexer_name)
+ codeblock = rest.lstrip("\n") # Remove lexer declaration line.
+ if lexer:
+ formatter_opts = self.extras['code-color'] or {}
+ colored = self._color_with_pygments(codeblock, lexer,
+ **formatter_opts)
+ return "\n\n%s\n\n" % colored
+
+ codeblock = self._encode_code(codeblock)
+ return "\n\n<pre><code>%s\n</code></pre>\n\n" % codeblock
+
+ def _do_code_blocks(self, text):
+ """Process Markdown `<pre><code>` blocks."""
+ code_block_re = re.compile(r'''
+ (?:\n\n|\A)
+ ( # $1 = the code block -- one or more lines, starting with a space/tab
+ (?:
+ (?:[ ]{%d} | \t) # Lines must start with a tab or a tab-width of spaces
+ .*\n+
+ )+
+ )
+ ((?=^[ ]{0,%d}\S)|\Z) # Lookahead for non-space at line-start, or end of doc
+ ''' % (self.tab_width, self.tab_width),
+ re.M | re.X)
+
+ return code_block_re.sub(self._code_block_sub, text)
+
+
+ # Rules for a code span:
+ # - backslash escapes are not interpreted in a code span
+ # - to include one or or a run of more backticks the delimiters must
+ # be a longer run of backticks
+ # - cannot start or end a code span with a backtick; pad with a
+ # space and that space will be removed in the emitted HTML
+ # See `test/tm-cases/escapes.text` for a number of edge-case
+ # examples.
+ _code_span_re = re.compile(r'''
+ (?<!\\)
+ (`+) # \1 = Opening run of `
+ (?!`) # See Note A test/tm-cases/escapes.text
+ (.+?) # \2 = The code block
+ (?<!`)
+ \1 # Matching closer
+ (?!`)
+ ''', re.X | re.S)
+
+ def _code_span_sub(self, match):
+ c = match.group(2).strip(" \t")
+ c = self._encode_code(c)
+ return "<code>%s</code>" % c
+
+ def _do_code_spans(self, text):
+ # * Backtick quotes are used for <code></code> spans.
+ #
+ # * You can use multiple backticks as the delimiters if you want to
+ # include literal backticks in the code span. So, this input:
+ #
+ # Just type ``foo `bar` baz`` at the prompt.
+ #
+ # Will translate to:
+ #
+ # <p>Just type <code>foo `bar` baz</code> at the prompt.</p>
+ #
+ # There's no arbitrary limit to the number of backticks you
+ # can use as delimters. If you need three consecutive backticks
+ # in your code, use four for delimiters, etc.
+ #
+ # * You can use spaces to get literal backticks at the edges:
+ #
+ # ... type `` `bar` `` ...
+ #
+ # Turns to:
+ #
+ # ... type <code>`bar`</code> ...
+ return self._code_span_re.sub(self._code_span_sub, text)
+
+ def _encode_code(self, text):
+ """Encode/escape certain characters inside Markdown code runs.
+ The point is that in code, these characters are literals,
+ and lose their special Markdown meanings.
+ """
+ replacements = [
+ # Encode all ampersands; HTML entities are not
+ # entities within a Markdown code span.
+ ('&', '&'),
+ # Do the angle bracket song and dance:
+ ('<', '<'),
+ ('>', '>'),
+ # Now, escape characters that are magic in Markdown:
+ ('*', g_escape_table['*']),
+ ('_', g_escape_table['_']),
+ ('{', g_escape_table['{']),
+ ('}', g_escape_table['}']),
+ ('[', g_escape_table['[']),
+ (']', g_escape_table[']']),
+ ('\\', g_escape_table['\\']),
+ ]
+ for before, after in replacements:
+ text = text.replace(before, after)
+ return text
+
+ _strong_re = re.compile(r"(\*\*|__)(?=\S)(.+?[*_]*)(?<=\S)\1", re.S)
+ _em_re = re.compile(r"(\*|_)(?=\S)(.+?)(?<=\S)\1", re.S)
+ _code_friendly_strong_re = re.compile(r"\*\*(?=\S)(.+?[*_]*)(?<=\S)\*\*", re.S)
+ _code_friendly_em_re = re.compile(r"\*(?=\S)(.+?)(?<=\S)\*", re.S)
+ def _do_italics_and_bold(self, text):
+ # <strong> must go first:
+ if "code-friendly" in self.extras:
+ text = self._code_friendly_strong_re.sub(r"<strong>\1</strong>", text)
+ text = self._code_friendly_em_re.sub(r"<em>\1</em>", text)
+ else:
+ text = self._strong_re.sub(r"<strong>\2</strong>", text)
+ text = self._em_re.sub(r"<em>\2</em>", text)
+ return text
+
+
+ _block_quote_re = re.compile(r'''
+ ( # Wrap whole match in \1
+ (
+ ^[ \t]*>[ \t]? # '>' at the start of a line
+ .+\n # rest of the first line
+ (.+\n)* # subsequent consecutive lines
+ \n* # blanks
+ )+
+ )
+ ''', re.M | re.X)
+ _bq_one_level_re = re.compile('^[ \t]*>[ \t]?', re.M);
+
+ _html_pre_block_re = re.compile(r'(\s*<pre>.+?</pre>)', re.S)
+ def _dedent_two_spaces_sub(self, match):
+ return re.sub(r'(?m)^ ', '', match.group(1))
+
+ def _block_quote_sub(self, match):
+ bq = match.group(1)
+ bq = self._bq_one_level_re.sub('', bq) # trim one level of quoting
+ bq = self._ws_only_line_re.sub('', bq) # trim whitespace-only lines
+ bq = self._run_block_gamut(bq) # recurse
+
+ bq = re.sub('(?m)^', ' ', bq)
+ # These leading spaces screw with <pre> content, so we need to fix that:
+ bq = self._html_pre_block_re.sub(self._dedent_two_spaces_sub, bq)
+
+ return "<blockquote>\n%s\n</blockquote>\n\n" % bq
+
+ def _do_block_quotes(self, text):
+ if '>' not in text:
+ return text
+ return self._block_quote_re.sub(self._block_quote_sub, text)
+
+ def _form_paragraphs(self, text):
+ # Strip leading and trailing lines:
+ text = text.strip('\n')
+
+ # Wrap <p> tags.
+ grafs = re.split(r"\n{2,}", text)
+ for i, graf in enumerate(grafs):
+ if graf in self.html_blocks:
+ # Unhashify HTML blocks
+ grafs[i] = self.html_blocks[graf]
+ else:
+ # Wrap <p> tags.
+ graf = self._run_span_gamut(graf)
+ grafs[i] = "<p>" + graf.lstrip(" \t") + "</p>"
+
+ return "\n\n".join(grafs)
+
+ def _add_footnotes(self, text):
+ if self.footnotes:
+ footer = [
+ '<div class="footnotes">',
+ '<hr' + self.empty_element_suffix,
+ '<ol>',
+ ]
+ for i, id in enumerate(self.footnote_ids):
+ if i != 0:
+ footer.append('')
+ footer.append('<li id="fn-%s">' % id)
+ footer.append(self._run_block_gamut(self.footnotes[id]))
+ backlink = ('<a href="#fnref-%s" '
+ 'class="footnoteBackLink" '
+ 'title="Jump back to footnote %d in the text.">'
+ '↩</a>' % (id, i+1))
+ if footer[-1].endswith("</p>"):
+ footer[-1] = footer[-1][:-len("</p>")] \
+ + ' ' + backlink + "</p>"
+ else:
+ footer.append("\n<p>%s</p>" % backlink)
+ footer.append('</li>')
+ footer.append('</ol>')
+ footer.append('</div>')
+ return text + '\n\n' + '\n'.join(footer)
+ else:
+ return text
+
+ # Ampersand-encoding based entirely on Nat Irons's Amputator MT plugin:
+ # http://bumppo.net/projects/amputator/
+ _ampersand_re = re.compile(r'&(?!#?[xX]?(?:[0-9a-fA-F]+|\w+);)')
+ _naked_lt_re = re.compile(r'<(?![a-z/?\$!])', re.I)
+ _naked_gt_re = re.compile(r'''(?<![a-z?!/'"-])>''', re.I)
+
+ def _encode_amps_and_angles(self, text):
+ # Smart processing for ampersands and angle brackets that need
+ # to be encoded.
+ text = self._ampersand_re.sub('&', text)
+
+ # Encode naked <'s
+ text = self._naked_lt_re.sub('<', text)
+
+ # Encode naked >'s
+ # Note: Other markdown implementations (e.g. Markdown.pl, PHP
+ # Markdown) don't do this.
+ text = self._naked_gt_re.sub('>', text)
+ return text
+
+ def _encode_backslash_escapes(self, text):
+ for ch, escape in g_escape_table.items():
+ text = text.replace("\\"+ch, escape)
+ return text
+
+ _auto_link_re = re.compile(r'<((https?|ftp):[^\'">\s]+)>', re.I)
+ def _auto_link_sub(self, match):
+ g1 = match.group(1)
+ return '<a href="%s">%s</a>' % (g1, g1)
+
+ _auto_email_link_re = re.compile(r"""
+ <
+ (?:mailto:)?
+ (
+ [-.\w]+
+ \@
+ [-\w]+(\.[-\w]+)*\.[a-z]+
+ )
+ >
+ """, re.I | re.X | re.U)
+ def _auto_email_link_sub(self, match):
+ return self._encode_email_address(
+ self._unescape_special_chars(match.group(1)))
+
+ def _do_auto_links(self, text):
+ text = self._auto_link_re.sub(self._auto_link_sub, text)
+ text = self._auto_email_link_re.sub(self._auto_email_link_sub, text)
+ return text
+
+ def _encode_email_address(self, addr):
+ # Input: an email address, e.g. "foo@example.com"
+ #
+ # Output: the email address as a mailto link, with each character
+ # of the address encoded as either a decimal or hex entity, in
+ # the hopes of foiling most address harvesting spam bots. E.g.:
+ #
+ # <a href="mailto:foo@e
+ # xample.com">foo
+ # @example.com</a>
+ #
+ # Based on a filter by Matthew Wickline, posted to the BBEdit-Talk
+ # mailing list: <http://tinyurl.com/yu7ue>
+ chars = [_xml_encode_email_char_at_random(ch)
+ for ch in "mailto:" + addr]
+ # Strip the mailto: from the visible part.
+ addr = '<a href="%s">%s</a>' \
+ % (''.join(chars), ''.join(chars[7:]))
+ return addr
+
+ def _do_link_patterns(self, text):
+ """Caveat emptor: there isn't much guarding against link
+ patterns being formed inside other standard Markdown links, e.g.
+ inside a [link def][like this].
+
+ Dev Notes: *Could* consider prefixing regexes with a negative
+ lookbehind assertion to attempt to guard against this.
+ """
+ link_from_hash = {}
+ for regex, repl in self.link_patterns:
+ replacements = []
+ for match in regex.finditer(text):
+ if hasattr(repl, "__call__"):
+ href = repl(match)
+ else:
+ href = match.expand(repl)
+ replacements.append((match.span(), href))
+ for (start, end), href in reversed(replacements):
+ escaped_href = (
+ href.replace('"', '"') # b/c of attr quote
+ # To avoid markdown <em> and <strong>:
+ .replace('*', g_escape_table['*'])
+ .replace('_', g_escape_table['_']))
+ link = '<a href="%s">%s</a>' % (escaped_href, text[start:end])
+ hash = md5(link).hexdigest()
+ link_from_hash[hash] = link
+ text = text[:start] + hash + text[end:]
+ for hash, link in link_from_hash.items():
+ text = text.replace(hash, link)
+ return text
+
+ def _unescape_special_chars(self, text):
+ # Swap back in all the special characters we've hidden.
+ for ch, hash in g_escape_table.items():
+ text = text.replace(hash, ch)
+ return text
+
+ def _outdent(self, text):
+ # Remove one level of line-leading tabs or spaces
+ return self._outdent_re.sub('', text)
+
+
+class MarkdownWithExtras(Markdown):
+ """A markdowner class that enables most extras:
+
+ - footnotes
+ - code-color (only has effect if 'pygments' Python module on path)
+
+ These are not included:
+ - pyshell (specific to Python-related documenting)
+ - code-friendly (because it *disables* part of the syntax)
+ - link-patterns (because you need to specify some actual
+ link-patterns anyway)
+ """
+ extras = ["footnotes", "code-color"]
+
+
+#---- internal support functions
+
+# From http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52549
+def _curry(*args, **kwargs):
+ function, args = args[0], args[1:]
+ def result(*rest, **kwrest):
+ combined = kwargs.copy()
+ combined.update(kwrest)
+ return function(*args + rest, **combined)
+ return result
+
+# Recipe: regex_from_encoded_pattern (1.0)
+def _regex_from_encoded_pattern(s):
+ """'foo' -> re.compile(re.escape('foo'))
+ '/foo/' -> re.compile('foo')
+ '/foo/i' -> re.compile('foo', re.I)
+ """
+ if s.startswith('/') and s.rfind('/') != 0:
+ # Parse it: /PATTERN/FLAGS
+ idx = s.rfind('/')
+ pattern, flags_str = s[1:idx], s[idx+1:]
+ flag_from_char = {
+ "i": re.IGNORECASE,
+ "l": re.LOCALE,
+ "s": re.DOTALL,
+ "m": re.MULTILINE,
+ "u": re.UNICODE,
+ }
+ flags = 0
+ for char in flags_str:
+ try:
+ flags |= flag_from_char[char]
+ except KeyError:
+ raise ValueError("unsupported regex flag: '%s' in '%s' "
+ "(must be one of '%s')"
+ % (char, s, ''.join(flag_from_char.keys())))
+ return re.compile(s[1:idx], flags)
+ else: # not an encoded regex
+ return re.compile(re.escape(s))
+
+# Recipe: dedent (0.1.2)
+def _dedentlines(lines, tabsize=8, skip_first_line=False):
+ """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines
+
+ "lines" is a list of lines to dedent.
+ "tabsize" is the tab width to use for indent width calculations.
+ "skip_first_line" is a boolean indicating if the first line should
+ be skipped for calculating the indent width and for dedenting.
+ This is sometimes useful for docstrings and similar.
+
+ Same as dedent() except operates on a sequence of lines. Note: the
+ lines list is modified **in-place**.
+ """
+ DEBUG = False
+ if DEBUG:
+ print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\
+ % (tabsize, skip_first_line)
+ indents = []
+ margin = None
+ for i, line in enumerate(lines):
+ if i == 0 and skip_first_line: continue
+ indent = 0
+ for ch in line:
+ if ch == ' ':
+ indent += 1
+ elif ch == '\t':
+ indent += tabsize - (indent % tabsize)
+ elif ch in '\r\n':
+ continue # skip all-whitespace lines
+ else:
+ break
+ else:
+ continue # skip all-whitespace lines
+ if DEBUG: print "dedent: indent=%d: %r" % (indent, line)
+ if margin is None:
+ margin = indent
+ else:
+ margin = min(margin, indent)
+ if DEBUG: print "dedent: margin=%r" % margin
+
+ if margin is not None and margin > 0:
+ for i, line in enumerate(lines):
+ if i == 0 and skip_first_line: continue
+ removed = 0
+ for j, ch in enumerate(line):
+ if ch == ' ':
+ removed += 1
+ elif ch == '\t':
+ removed += tabsize - (removed % tabsize)
+ elif ch in '\r\n':
+ if DEBUG: print "dedent: %r: EOL -> strip up to EOL" % line
+ lines[i] = lines[i][j:]
+ break
+ else:
+ raise ValueError("unexpected non-whitespace char %r in "
+ "line %r while removing %d-space margin"
+ % (ch, line, margin))
+ if DEBUG:
+ print "dedent: %r: %r -> removed %d/%d"\
+ % (line, ch, removed, margin)
+ if removed == margin:
+ lines[i] = lines[i][j+1:]
+ break
+ elif removed > margin:
+ lines[i] = ' '*(removed-margin) + lines[i][j+1:]
+ break
+ else:
+ if removed:
+ lines[i] = lines[i][removed:]
+ return lines
+
+def _dedent(text, tabsize=8, skip_first_line=False):
+ """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text
+
+ "text" is the text to dedent.
+ "tabsize" is the tab width to use for indent width calculations.
+ "skip_first_line" is a boolean indicating if the first line should
+ be skipped for calculating the indent width and for dedenting.
+ This is sometimes useful for docstrings and similar.
+
+ textwrap.dedent(s), but don't expand tabs to spaces
+ """
+ lines = text.splitlines(1)
+ _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line)
+ return ''.join(lines)
+
+
+class _memoized(object):
+ """Decorator that caches a function's return value each time it is called.
+ If called later with the same arguments, the cached value is returned, and
+ not re-evaluated.
+
+ http://wiki.python.org/moin/PythonDecoratorLibrary
+ """
+ def __init__(self, func):
+ self.func = func
+ self.cache = {}
+ def __call__(self, *args):
+ try:
+ return self.cache[args]
+ except KeyError:
+ self.cache[args] = value = self.func(*args)
+ return value
+ except TypeError:
+ # uncachable -- for instance, passing a list as an argument.
+ # Better to not cache than to blow up entirely.
+ return self.func(*args)
+ def __repr__(self):
+ """Return the function's docstring."""
+ return self.func.__doc__
+
+
+def _xml_oneliner_re_from_tab_width(tab_width):
+ """Standalone XML processing instruction regex."""
+ return re.compile(r"""
+ (?:
+ (?<=\n\n) # Starting after a blank line
+ | # or
+ \A\n? # the beginning of the doc
+ )
+ ( # save in $1
+ [ ]{0,%d}
+ (?:
+ <\?\w+\b\s+.*?\?> # XML processing instruction
+ |
+ <\w+:\w+\b\s+.*?/> # namespaced single tag
+ )
+ [ \t]*
+ (?=\n{2,}|\Z) # followed by a blank line or end of document
+ )
+ """ % (tab_width - 1), re.X)
+_xml_oneliner_re_from_tab_width = _memoized(_xml_oneliner_re_from_tab_width)
+
+def _hr_tag_re_from_tab_width(tab_width):
+ return re.compile(r"""
+ (?:
+ (?<=\n\n) # Starting after a blank line
+ | # or
+ \A\n? # the beginning of the doc
+ )
+ ( # save in \1
+ [ ]{0,%d}
+ <(hr) # start tag = \2
+ \b # word break
+ ([^<>])*? #
+ /?> # the matching end tag
+ [ \t]*
+ (?=\n{2,}|\Z) # followed by a blank line or end of document
+ )
+ """ % (tab_width - 1), re.X)
+_hr_tag_re_from_tab_width = _memoized(_hr_tag_re_from_tab_width)
+
+
+def _xml_encode_email_char_at_random(ch):
+ r = random()
+ # Roughly 10% raw, 45% hex, 45% dec.
+ # '@' *must* be encoded. I [John Gruber] insist.
+ # Issue 26: '_' must be encoded.
+ if r > 0.9 and ch not in "@_":
+ return ch
+ elif r < 0.45:
+ # The [1:] is to drop leading '0': 0x63 -> x63
+ return '&#%s;' % hex(ord(ch))[1:]
+ else:
+ return '&#%s;' % ord(ch)
+
+def _hash_text(text):
+ return 'md5:'+md5(text.encode("utf-8")).hexdigest()
+
+
+#---- mainline
+
+class _NoReflowFormatter(optparse.IndentedHelpFormatter):
+ """An optparse formatter that does NOT reflow the description."""
+ def format_description(self, description):
+ return description or ""
+
+def _test():
+ import doctest
+ doctest.testmod()
+
+def main(argv=None):
+ if argv is None:
+ argv = sys.argv
+ if not logging.root.handlers:
+ logging.basicConfig()
+
+ usage = "usage: %prog [PATHS...]"
+ version = "%prog "+__version__
+ parser = optparse.OptionParser(prog="markdown2", usage=usage,
+ version=version, description=cmdln_desc,
+ formatter=_NoReflowFormatter())
+ parser.add_option("-v", "--verbose", dest="log_level",
+ action="store_const", const=logging.DEBUG,
+ help="more verbose output")
+ parser.add_option("--encoding",
+ help="specify encoding of text content")
+ parser.add_option("--html4tags", action="store_true", default=False,
+ help="use HTML 4 style for empty element tags")
+ parser.add_option("-s", "--safe", metavar="MODE", dest="safe_mode",
+ help="sanitize literal HTML: 'escape' escapes "
+ "HTML meta chars, 'replace' replaces with an "
+ "[HTML_REMOVED] note")
+ parser.add_option("-x", "--extras", action="append",
+ help="Turn on specific extra features (not part of "
+ "the core Markdown spec). Supported values: "
+ "'code-friendly' disables _/__ for emphasis; "
+ "'code-color' adds code-block syntax coloring; "
+ "'link-patterns' adds auto-linking based on patterns; "
+ "'footnotes' adds the footnotes syntax;"
+ "'xml' passes one-liner processing instructions and namespaced XML tags;"
+ "'pyshell' to put unindented Python interactive shell sessions in a <code> block.")
+ parser.add_option("--use-file-vars",
+ help="Look for and use Emacs-style 'markdown-extras' "
+ "file var to turn on extras. See "
+ "<http://code.google.com/p/python-markdown2/wiki/Extras>.")
+ parser.add_option("--link-patterns-file",
+ help="path to a link pattern file")
+ parser.add_option("--self-test", action="store_true",
+ help="run internal self-tests (some doctests)")
+ parser.add_option("--compare", action="store_true",
+ help="run against Markdown.pl as well (for testing)")
+ parser.set_defaults(log_level=logging.INFO, compare=False,
+ encoding="utf-8", safe_mode=None, use_file_vars=False)
+ opts, paths = parser.parse_args()
+ log.setLevel(opts.log_level)
+
+ if opts.self_test:
+ return _test()
+
+ if opts.extras:
+ extras = {}
+ for s in opts.extras:
+ splitter = re.compile("[,;: ]+")
+ for e in splitter.split(s):
+ if '=' in e:
+ ename, earg = e.split('=', 1)
+ try:
+ earg = int(earg)
+ except ValueError:
+ pass
+ else:
+ ename, earg = e, None
+ extras[ename] = earg
+ else:
+ extras = None
+
+ if opts.link_patterns_file:
+ link_patterns = []
+ f = open(opts.link_patterns_file)
+ try:
+ for i, line in enumerate(f.readlines()):
+ if not line.strip(): continue
+ if line.lstrip().startswith("#"): continue
+ try:
+ pat, href = line.rstrip().rsplit(None, 1)
+ except ValueError:
+ raise MarkdownError("%s:%d: invalid link pattern line: %r"
+ % (opts.link_patterns_file, i+1, line))
+ link_patterns.append(
+ (_regex_from_encoded_pattern(pat), href))
+ finally:
+ f.close()
+ else:
+ link_patterns = None
+
+ from os.path import join, dirname, abspath, exists
+ markdown_pl = join(dirname(dirname(abspath(__file__))), "test",
+ "Markdown.pl")
+ for path in paths:
+ if opts.compare:
+ print "==== Markdown.pl ===="
+ perl_cmd = 'perl %s "%s"' % (markdown_pl, path)
+ o = os.popen(perl_cmd)
+ perl_html = o.read()
+ o.close()
+ sys.stdout.write(perl_html)
+ print "==== markdown2.py ===="
+ html = markdown_path(path, encoding=opts.encoding,
+ html4tags=opts.html4tags,
+ safe_mode=opts.safe_mode,
+ extras=extras, link_patterns=link_patterns,
+ use_file_vars=opts.use_file_vars)
+ sys.stdout.write(
+ html.encode(sys.stdout.encoding or "utf-8", 'xmlcharrefreplace'))
+ if opts.compare:
+ test_dir = join(dirname(dirname(abspath(__file__))), "test")
+ if exists(join(test_dir, "test_markdown2.py")):
+ sys.path.insert(0, test_dir)
+ from test_markdown2 import norm_html_from_html
+ norm_html = norm_html_from_html(html)
+ norm_perl_html = norm_html_from_html(perl_html)
+ else:
+ norm_html = html
+ norm_perl_html = perl_html
+ print "==== match? %r ====" % (norm_perl_html == norm_html)
+
+
+if __name__ == "__main__":
+ sys.exit( main(sys.argv) )
+
--- /dev/null
+-- Copyright 2009 FriendFeed
+--
+-- Licensed under the Apache License, Version 2.0 (the "License"); you may
+-- not use this file except in compliance with the License. You may obtain
+-- a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+-- WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+-- License for the specific language governing permissions and limitations
+-- under the License.
+
+-- To create the database:
+-- CREATE DATABASE blog;
+-- GRANT ALL PRIVILEGES ON blog.* TO 'blog'@'localhost' IDENTIFIED BY 'blog';
+--
+-- To reload the tables:
+-- mysql --user=blog --password=blog --database=blog < schema.sql
+
+SET SESSION storage_engine = "InnoDB";
+SET SESSION time_zone = "+0:00";
+ALTER DATABASE CHARACTER SET "utf8";
+
+DROP TABLE IF EXISTS entries;
+CREATE TABLE entries (
+ id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ author_id INT NOT NULL REFERENCES authors(id),
+ slug VARCHAR(100) NOT NULL UNIQUE,
+ title VARCHAR(512) NOT NULL,
+ markdown MEDIUMTEXT NOT NULL,
+ html MEDIUMTEXT NOT NULL,
+ published DATETIME NOT NULL,
+ updated TIMESTAMP NOT NULL,
+ KEY (published)
+);
+
+DROP TABLE IF EXISTS authors;
+CREATE TABLE authors (
+ id INT NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ email VARCHAR(100) NOT NULL UNIQUE,
+ name VARCHAR(100) NOT NULL
+);
--- /dev/null
+/*
+ * Copyright 2009 Facebook
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+body {
+ background: white;
+ color: black;
+ margin: 15px;
+ margin-top: 0;
+}
+
+body,
+input,
+textarea {
+ font-family: Georgia, serif;
+ font-size: 12pt;
+}
+
+table {
+ border-collapse: collapse;
+ border: 0;
+}
+
+td {
+ border: 0;
+ padding: 0;
+}
+
+h1,
+h2,
+h3,
+h4 {
+ font-family: "Helvetica Nue", Helvetica, Arial, sans-serif;
+ margin: 0;
+}
+
+h1 {
+ font-size: 20pt;
+}
+
+pre,
+code {
+ font-family: monospace;
+ color: #060;
+}
+
+pre {
+ margin-left: 1em;
+ padding-left: 1em;
+ border-left: 1px solid silver;
+ line-height: 14pt;
+}
+
+a,
+a code {
+ color: #00c;
+}
+
+#body {
+ max-width: 800px;
+ margin: auto;
+}
+
+#header {
+ background-color: #3b5998;
+ padding: 5px;
+ padding-left: 10px;
+ padding-right: 10px;
+ margin-bottom: 1em;
+}
+
+#header,
+#header a {
+ color: white;
+}
+
+#header h1 a {
+ text-decoration: none;
+}
+
+#footer,
+#content {
+ margin-left: 10px;
+ margin-right: 10px;
+}
+
+#footer {
+ margin-top: 3em;
+}
+
+.entry h1 a {
+ color: black;
+ text-decoration: none;
+}
+
+.entry {
+ margin-bottom: 2em;
+}
+
+.entry .date {
+ margin-top: 3px;
+}
+
+.entry p {
+ margin: 0;
+ margin-bottom: 1em;
+}
+
+.entry .body {
+ margin-top: 1em;
+ line-height: 16pt;
+}
+
+.compose td {
+ vertical-align: middle;
+ padding-bottom: 5px;
+}
+
+.compose td.field {
+ padding-right: 10px;
+}
+
+.compose .title,
+.compose .submit {
+ font-family: "Helvetica Nue", Helvetica, Arial, sans-serif;
+ font-weight: bold;
+}
+
+.compose .title {
+ font-size: 20pt;
+}
+
+.compose .title,
+.compose .markdown {
+ width: 100%;
+}
+
+.compose .markdown {
+ height: 500px;
+ line-height: 16pt;
+}
--- /dev/null
+{% extends "base.html" %}
+
+{% block head %}
+ <style type="text/css">
+ ul.archive {
+ list-style-type: none;
+ margin: 0;
+ padding: 0;
+ }
+
+ ul.archive li {
+ margin-bottom: 1em;
+ }
+
+ ul.archive .title {
+ font-family: "Helvetica Nue", Helvetica, Arial, sans-serif;
+ font-size: 14pt;
+ }
+ </style>
+{% end %}
+
+{% block body %}
+ <ul class="archive">
+ {% for entry in entries %}
+ <li>
+ <div class="title"><a href="/entry/{{ entry.slug }}">{{ escape(entry.title) }}</a></div>
+ <div class="date">{{ locale.format_date(entry.published, full_format=True, shorter=True) }}</div>
+ </li>
+ {% end %}
+ </ul>
+{% end %}
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+ <title>{{ escape(handler.settings["blog_title"]) }}</title>
+ <link rel="stylesheet" href="{{ static_url("blog.css") }}" type="text/css"/>
+ <link rel="alternate" href="/feed" type="application/atom+xml" title="{{ escape(handler.settings["blog_title"]) }}"/>
+ {% block head %}{% end %}
+ </head>
+ <body>
+ <div id="body">
+ <div id="header">
+ <div style="float:right">
+ {% if current_user %}
+ <a href="/compose">{{ _("New post") }}</a> -
+ <a href="/auth/logout?next={{ url_escape(request.uri) }}">{{ _("Sign out") }}</a>
+ {% else %}
+ {{ _('<a href="%(url)s">Sign in</a> to compose/edit') % {"url": "/auth/login?next=" + url_escape(request.uri)} }}
+ {% end %}
+ </div>
+ <h1><a href="/">{{ escape(handler.settings["blog_title"]) }}</a></h1>
+ </div>
+ <div id="content">{% block body %}{% end %}</div>
+ </div>
+ {% block bottom %}{% end %}
+ </body>
+</html>
--- /dev/null
+{% extends "base.html" %}
+
+{% block body %}
+ <form action="{{ request.path }}" method="post" class="compose">
+ <div style="margin-bottom:5px"><input name="title" type="text" class="title" value="{{ escape(entry.title) if entry else "" }}"/></div>
+ <div style="margin-bottom:5px"><textarea name="markdown" rows="30" cols="40" class="markdown">{{ escape(entry.markdown) if entry else "" }}</textarea></div>
+ <div>
+ <div style="float:right"><a href="http://daringfireball.net/projects/markdown/syntax">{{ _("Syntax documentation") }}</a></div>
+ <input type="submit" value="{{ _("Save changes") if entry else _("Publish post") }}" class="submit"/>
+ <a href="{{ "/entry/" + entry.slug if entry else "/" }}">{{ _("Cancel") }}</a>
+ </div>
+ {% if entry %}
+ <input type="hidden" name="id" value="{{ entry.id }}"/>
+ {% end %}
+ {{ xsrf_form_html() }}
+ </form>
+{% end %}
+
+{% block bottom %}
+ <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js" type="text/javascript"></script>
+ <script type="text/javascript">
+ //<![CDATA[
+
+ $(function() {
+ $("input[name=title]").select();
+ $("form.compose").submit(function() {
+ var required = ["title", "markdown"];
+ var form = $(this).get(0);
+ for (var i = 0; i < required.length; i++) {
+ if (!form[required[i]].value) {
+ $(form[required[i]]).select();
+ return false;
+ }
+ }
+ return true;
+ });
+ });
+
+ //]]>
+ </script>
+{% end %}
+
--- /dev/null
+{% extends "base.html" %}
+
+{% block body %}
+ {{ modules.Entry(entry) }}
+{% end %}
--- /dev/null
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+ {% set date_format = "%Y-%m-%dT%H:%M:%SZ" %}
+ <title>{{ escape(handler.settings["blog_title"]) }}</title>
+ {% if len(entries) > 0 %}
+ <updated>{{ max(e.updated for e in entries).strftime(date_format) }}</updated>
+ {% else %}
+ <updated>{{ datetime.datetime.utcnow().strftime(date_format) }}</updated>
+ {% end %}
+ <id>http://{{ request.host }}/</id>
+ <link rel="alternate" href="http://{{ request.host }}/" title="{{ escape(handler.settings["blog_title"]) }}" type="text/html"/>
+ <link rel="self" href="{{ request.full_url() }}" title="{{ escape(handler.settings["blog_title"]) }}" type="application/atom+xml"/>
+ <author><name>{{ escape(handler.settings["blog_title"]) }}</name></author>
+ {% for entry in entries %}
+ <entry>
+ <id>http://{{ request.host }}/entry/{{ entry.slug }}</id>
+ <title type="text">{{ escape(entry.title) }}</title>
+ <link href="http://{{ request.host }}/entry/{{ entry.slug }}" rel="alternate" type="text/html"/>
+ <updated>{{ entry.updated.strftime(date_format) }}</updated>
+ <published>{{ entry.published.strftime(date_format) }}</published>
+ <content type="xhtml" xml:base="http://{{ request.host }}/">
+ <div xmlns="http://www.w3.org/1999/xhtml">{{ entry.html }}</div>
+ </content>
+ </entry>
+ {% end %}
+</feed>
--- /dev/null
+{% extends "base.html" %}
+
+{% block body %}
+ {% for entry in entries %}
+ {{ modules.Entry(entry) }}
+ {% end %}
+ <div><a href="/archive">{{ _("Archive") }}</a></div>
+{% end %}
--- /dev/null
+<div class="entry">
+ <h1><a href="/entry/{{ entry.slug }}">{{ escape(entry.title) }}</a></h1>
+ <div class="date">{{ locale.format_date(entry.published, full_format=True, shorter=True) }}</div>
+ <div class="body">{{ entry.html }}</div>
+ {% if current_user %}
+ <div class="admin"><a href="/compose?id={{ entry.id }}">{{ _("Edit this post") }}</a></div>
+ {% end %}
+</div>
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import tornado.auth
+import tornado.escape
+import tornado.httpserver
+import tornado.ioloop
+import tornado.options
+import tornado.web
+import os.path
+import uuid
+
+from tornado.options import define, options
+
+define("port", default=8888, help="run on the given port", type=int)
+
+
+class Application(tornado.web.Application):
+ def __init__(self):
+ handlers = [
+ (r"/", MainHandler),
+ (r"/auth/login", AuthLoginHandler),
+ (r"/auth/logout", AuthLogoutHandler),
+ (r"/a/message/new", MessageNewHandler),
+ (r"/a/message/updates", MessageUpdatesHandler),
+ ]
+ settings = dict(
+ cookie_secret="43oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
+ login_url="/auth/login",
+ template_path=os.path.join(os.path.dirname(__file__), "templates"),
+ static_path=os.path.join(os.path.dirname(__file__), "static"),
+ xsrf_cookies=True,
+ )
+ tornado.web.Application.__init__(self, handlers, **settings)
+
+
+class BaseHandler(tornado.web.RequestHandler):
+ def get_current_user(self):
+ user_json = self.get_secure_cookie("user")
+ if not user_json: return None
+ return tornado.escape.json_decode(user_json)
+
+
+class MainHandler(BaseHandler):
+ @tornado.web.authenticated
+ def get(self):
+ self.render("index.html", messages=MessageMixin.cache)
+
+
+class MessageMixin(object):
+ waiters = []
+ cache = []
+ cache_size = 200
+
+ def wait_for_messages(self, callback, cursor=None):
+ cls = MessageMixin
+ if cursor:
+ index = 0
+ for i in xrange(len(cls.cache)):
+ index = len(cls.cache) - i - 1
+ if cls.cache[index]["id"] == cursor: break
+ recent = cls.cache[index + 1:]
+ if recent:
+ callback(recent)
+ return
+ cls.waiters.append(callback)
+
+ def new_messages(self, messages):
+ cls = MessageMixin
+ logging.info("Sending new message to %r listeners", len(cls.waiters))
+ for callback in cls.waiters:
+ try:
+ callback(messages)
+ except:
+ logging.error("Error in waiter callback", exc_info=True)
+ cls.waiters = []
+ cls.cache.extend(messages)
+ if len(cls.cache) > self.cache_size:
+ cls.cache = cls.cache[-self.cache_size:]
+
+
+class MessageNewHandler(BaseHandler, MessageMixin):
+ @tornado.web.authenticated
+ def post(self):
+ message = {
+ "id": str(uuid.uuid4()),
+ "from": self.current_user["first_name"],
+ "body": self.get_argument("body"),
+ }
+ message["html"] = self.render_string("message.html", message=message)
+ if self.get_argument("next", None):
+ self.redirect(self.get_argument("next"))
+ else:
+ self.write(message)
+ self.new_messages([message])
+
+
+class MessageUpdatesHandler(BaseHandler, MessageMixin):
+ @tornado.web.authenticated
+ @tornado.web.asynchronous
+ def post(self):
+ cursor = self.get_argument("cursor", None)
+ self.wait_for_messages(self.async_callback(self.on_new_messages),
+ cursor=cursor)
+
+ def on_new_messages(self, messages):
+ # Closed client connection
+ if self.request.connection.stream.closed():
+ return
+ self.finish(dict(messages=messages))
+
+
+class AuthLoginHandler(BaseHandler, tornado.auth.GoogleMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("openid.mode", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authenticate_redirect(ax_attrs=["name"])
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "Google auth failed")
+ self.set_secure_cookie("user", tornado.escape.json_encode(user))
+ self.redirect("/")
+
+
+class AuthLogoutHandler(BaseHandler, tornado.auth.FacebookMixin):
+ def get(self):
+ self.clear_cookie("user")
+ self.redirect(self.get_argument("next", "/"))
+
+
+def main():
+ tornado.options.parse_command_line()
+ http_server = tornado.httpserver.HTTPServer(Application())
+ http_server.listen(options.port)
+ tornado.ioloop.IOLoop.instance().start()
+
+
+if __name__ == "__main__":
+ main()
--- /dev/null
+/*
+ * Copyright 2009 FriendFeed
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+body {
+ background: white;
+ margin: 10px;
+}
+
+body,
+input {
+ font-family: sans-serif;
+ font-size: 10pt;
+ color: black;
+}
+
+table {
+ border-collapse: collapse;
+ border: 0;
+}
+
+td {
+ border: 0;
+ padding: 0;
+}
+
+#body {
+ position: absolute;
+ bottom: 10px;
+ left: 10px;
+}
+
+#input {
+ margin-top: 0.5em;
+}
+
+#inbox .message {
+ padding-top: 0.25em;
+}
--- /dev/null
+// Copyright 2009 FriendFeed
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations
+// under the License.
+
+$(document).ready(function() {
+ if (!window.console) window.console = {};
+ if (!window.console.log) window.console.log = function() {};
+
+ $("#messageform").live("submit", function() {
+ newMessage($(this));
+ return false;
+ });
+ $("#messageform").live("keypress", function(e) {
+ if (e.keyCode == 13) {
+ newMessage($(this));
+ return false;
+ }
+ });
+ $("#message").select();
+ updater.poll();
+});
+
+function newMessage(form) {
+ var message = form.formToDict();
+ var disabled = form.find("input[type=submit]");
+ disabled.disable();
+ $.postJSON("/a/message/new", message, function(response) {
+ updater.showMessage(response);
+ if (message.id) {
+ form.parent().remove();
+ } else {
+ form.find("input[type=text]").val("").select();
+ disabled.enable();
+ }
+ });
+}
+
+function getCookie(name) {
+ var r = document.cookie.match("\\b" + name + "=([^;]*)\\b");
+ return r ? r[1] : undefined;
+}
+
+jQuery.postJSON = function(url, args, callback) {
+ args._xsrf = getCookie("_xsrf");
+ $.ajax({url: url, data: $.param(args), dataType: "text", type: "POST",
+ success: function(response) {
+ if (callback) callback(eval("(" + response + ")"));
+ }, error: function(response) {
+ console.log("ERROR:", response)
+ }});
+};
+
+jQuery.fn.formToDict = function() {
+ var fields = this.serializeArray();
+ var json = {}
+ for (var i = 0; i < fields.length; i++) {
+ json[fields[i].name] = fields[i].value;
+ }
+ if (json.next) delete json.next;
+ return json;
+};
+
+jQuery.fn.disable = function() {
+ this.enable(false);
+ return this;
+};
+
+jQuery.fn.enable = function(opt_enable) {
+ if (arguments.length && !opt_enable) {
+ this.attr("disabled", "disabled");
+ } else {
+ this.removeAttr("disabled");
+ }
+ return this;
+};
+
+var updater = {
+ errorSleepTime: 500,
+ cursor: null,
+
+ poll: function() {
+ var args = {"_xsrf": getCookie("_xsrf")};
+ if (updater.cursor) args.cursor = updater.cursor;
+ $.ajax({url: "/a/message/updates", type: "POST", dataType: "text",
+ data: $.param(args), success: updater.onSuccess,
+ error: updater.onError});
+ },
+
+ onSuccess: function(response) {
+ try {
+ updater.newMessages(eval("(" + response + ")"));
+ } catch (e) {
+ updater.onError();
+ return;
+ }
+ updater.errorSleepTime = 500;
+ window.setTimeout(updater.poll, 0);
+ },
+
+ onError: function(response) {
+ updater.errorSleepTime *= 2;
+ console.log("Poll error; sleeping for", updater.errorSleepTime, "ms");
+ window.setTimeout(updater.poll, updater.errorSleepTime);
+ },
+
+ newMessages: function(response) {
+ if (!response.messages) return;
+ updater.cursor = response.cursor;
+ var messages = response.messages;
+ updater.cursor = messages[messages.length - 1].id;
+ console.log(messages.length, "new messages, cursor:", updater.cursor);
+ for (var i = 0; i < messages.length; i++) {
+ updater.showMessage(messages[i]);
+ }
+ },
+
+ showMessage: function(message) {
+ var existing = $("#m" + message.id);
+ if (existing.length > 0) return;
+ var node = $(message.html);
+ node.hide();
+ $("#inbox").append(node);
+ node.slideDown();
+ }
+};
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+ <title>Tornado Chat Demo</title>
+ <link rel="stylesheet" href="{{ static_url("chat.css") }}" type="text/css"/>
+ </head>
+ <body>
+ <div style="float:right">
+ <b>{{ escape(current_user["name"]) }}</b> -
+ <a href="/auth/logout">{{ _("Sign out") }}</a>
+ </div>
+ <div id="body">
+ <div id="inbox">
+ {% for message in messages %}
+ {% include "message.html" %}
+ {% end %}
+ </div>
+ <div id="input">
+ <form action="/a/message/new" method="post" id="messageform">
+ <table>
+ <tr>
+ <td><input name="body" id="message" style="width:500px"/></td>
+ <td style="padding-left:5px">
+ <input type="submit" value="{{ _("Post") }}"/>
+ <input type="hidden" name="next" value="{{ request.path }}"/>
+ {{ xsrf_form_html() }}
+ </td>
+ </tr>
+ </table>
+ </form>
+ </div>
+ </div>
+ <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3/jquery.min.js" type="text/javascript"></script>
+ <script src="{{ static_url("chat.js") }}" type="text/javascript"></script>
+ </body>
+</html>
--- /dev/null
+<div class="message" id="m{{ message["id"] }}"><b>{{ message["from"] }}: </b>{{ escape(message["body"]) }}</div>
--- /dev/null
+Running the Tornado AppEngine example
+=====================================
+To work with the provided Facebook api key, this example must be
+accessed at http://localhost:8888/ to match the Connect URL set in the
+example application.
+
+To use any other domain, a new Facebook application must be registered
+with a Connect URL set to that domain.
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import os.path
+import tornado.auth
+import tornado.escape
+import tornado.httpserver
+import tornado.ioloop
+import tornado.options
+import tornado.web
+import uimodules
+
+from tornado.options import define, options
+
+define("port", default=8888, help="run on the given port", type=int)
+define("facebook_api_key", help="your Facebook application API key",
+ default="9e2ada1b462142c4dfcc8e894ea1e37c")
+define("facebook_secret", help="your Facebook application secret",
+ default="32fc6114554e3c53d5952594510021e2")
+
+
+class Application(tornado.web.Application):
+ def __init__(self):
+ handlers = [
+ (r"/", MainHandler),
+ (r"/auth/login", AuthLoginHandler),
+ (r"/auth/logout", AuthLogoutHandler),
+ ]
+ settings = dict(
+ cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
+ login_url="/auth/login",
+ template_path=os.path.join(os.path.dirname(__file__), "templates"),
+ static_path=os.path.join(os.path.dirname(__file__), "static"),
+ xsrf_cookies=True,
+ facebook_api_key=options.facebook_api_key,
+ facebook_secret=options.facebook_secret,
+ ui_modules= {"Post": PostModule},
+ debug=True,
+ )
+ tornado.web.Application.__init__(self, handlers, **settings)
+
+
+class BaseHandler(tornado.web.RequestHandler):
+ def get_current_user(self):
+ user_json = self.get_secure_cookie("user")
+ if not user_json: return None
+ return tornado.escape.json_decode(user_json)
+
+
+class MainHandler(BaseHandler, tornado.auth.FacebookMixin):
+ @tornado.web.authenticated
+ @tornado.web.asynchronous
+ def get(self):
+ self.facebook_request(
+ method="stream.get",
+ callback=self.async_callback(self._on_stream),
+ session_key=self.current_user["session_key"])
+
+ def _on_stream(self, stream):
+ if stream is None:
+ # Session may have expired
+ self.redirect("/auth/login")
+ return
+ # Turn profiles into a dict mapping id => profile
+ stream["profiles"] = dict((p["id"], p) for p in stream["profiles"])
+ self.render("stream.html", stream=stream)
+
+
+class AuthLoginHandler(BaseHandler, tornado.auth.FacebookMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("session", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authorize_redirect("read_stream")
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "Facebook auth failed")
+ self.set_secure_cookie("user", tornado.escape.json_encode(user))
+ self.redirect(self.get_argument("next", "/"))
+
+
+class AuthLogoutHandler(BaseHandler, tornado.auth.FacebookMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ self.clear_cookie("user")
+ if not self.current_user:
+ self.redirect(self.get_argument("next", "/"))
+ return
+ self.facebook_request(
+ method="auth.revokeAuthorization",
+ callback=self.async_callback(self._on_deauthorize),
+ session_key=self.current_user["session_key"])
+
+ def _on_deauthorize(self, response):
+ self.redirect(self.get_argument("next", "/"))
+
+
+class PostModule(tornado.web.UIModule):
+ def render(self, post, actor):
+ return self.render_string("modules/post.html", post=post, actor=actor)
+
+
+def main():
+ tornado.options.parse_command_line()
+ http_server = tornado.httpserver.HTTPServer(Application())
+ http_server.listen(options.port)
+ tornado.ioloop.IOLoop.instance().start()
+
+
+if __name__ == "__main__":
+ main()
--- /dev/null
+/*
+ * Copyright 2009 Facebook
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+body {
+ background: white;
+ color: black;
+ margin: 15px;
+}
+
+body,
+input,
+textarea {
+ font-family: "Lucida Grande", Tahoma, Verdana, sans-serif;
+ font-size: 10pt;
+}
+
+table {
+ border-collapse: collapse;
+ border: 0;
+}
+
+td {
+ border: 0;
+ padding: 0;
+}
+
+img {
+ border: 0;
+}
+
+a {
+ text-decoration: none;
+ color: #3b5998;
+}
+
+a:hover {
+ text-decoration: underline;
+}
+
+.post {
+ border-bottom: 1px solid #eeeeee;
+ min-height: 50px;
+ padding-bottom: 10px;
+ margin-top: 10px;
+}
+
+.post .picture {
+ float: left;
+}
+
+.post .picture img {
+ height: 50px;
+ width: 50px;
+}
+
+.post .body {
+ margin-left: 60px;
+}
+
+.post .media img {
+ border: 1px solid #cccccc;
+ padding: 3px;
+}
+
+.post .media:hover img {
+ border: 1px solid #3b5998;
+}
+
+.post a.actor {
+ font-weight: bold;
+}
+
+.post .meta {
+ font-size: 11px;
+}
+
+.post a.permalink {
+ color: #777777;
+}
+
+#body {
+ max-width: 700px;
+ margin: auto;
+}
--- /dev/null
+<div class="post">
+ <div class="picture">
+ <a href="{{ actor["url"] }}"><img src="{{ actor["pic_square"] }}"/></a>
+ </div>
+ <div class="body">
+ <a href="{{ actor["url"] }}" class="actor">{{ escape(actor["name"]) }}</a>
+ {% if post["message"] %}
+ <span class="message">{{ escape(post["message"]) }}</span>
+ {% end %}
+ {% if post["attachment"] %}
+ <div class="attachment">
+ {% if post["attachment"].get("name") %}
+ <div class="name"><a href="{{ post["attachment"]["href"] }}">{{ escape(post["attachment"]["name"]) }}</a></div>
+ {% end %}
+ {% if post["attachment"].get("description") %}
+ <div class="description">{{ post["attachment"]["description"] }}</div>
+ {% end %}
+ {% for media in filter(lambda m: m.get("src") and m["type"] in ("photo", "link"), post["attachment"]["media"]) %}
+ <span class="media">
+ <a href="{{ media["href"] }}"><img src="{{ media["src"] }}" alt="{{ escape(media.get("alt", "")) }}"/></a>
+ </span>
+ {% end %}
+ </div>
+ {% end %}
+ <div class="meta">
+ <a href="{{ post["permalink"] }}" class="permalink">{{ locale.format_date(post["created_time"]) }}</a>
+ </div>
+ </div>
+</div>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+ <title>Tornado Facebook Stream Demo</title>
+ <link rel="stylesheet" href="{{ static_url("facebook.css") }}" type="text/css"/>
+ </head>
+ <body>
+ <div id="body">
+ <div style="float:right">
+ <b>{{ escape(current_user["name"]) }}</b> -
+ <a href="/auth/logout">{{ _("Sign out") }}</a>
+ </div>
+ <div style="margin-bottom:1em"><a href="/">{{ _("Refresh stream") }}</a></div>
+ <div id="stream">
+ {% for post in stream["posts"] %}
+ {{ modules.Post(post, stream["profiles"][post["actor_id"]]) }}
+ {% end %}
+ </div>
+ </div>
+ </body>
+</html>
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import tornado.web
+
+
+class Entry(tornado.web.UIModule):
+ def render(self):
+ return '<div>ENTRY</div>'
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import tornado.httpserver
+import tornado.ioloop
+import tornado.options
+import tornado.web
+import logging
+
+from tornado.options import define, options
+
+define("port", default=8888, help="run on the given port", type=int)
+
+
+class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ self.write("Hello, world")
+
+
+def main():
+ tornado.options.parse_command_line()
+ application = tornado.web.Application([
+ (r"/", MainHandler),
+ ])
+ http_server = tornado.httpserver.HTTPServer(application)
+ http_server.listen(options.port)
+ tornado.ioloop.IOLoop.instance().start()
+
+
+if __name__ == "__main__":
+ main()
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import distutils.core
+import sys
+
+# Build the epoll extension for Linux systems with Python < 2.6
+extensions = []
+major, minor = sys.version_info[:2]
+python_26 = (major > 2 or (major == 2 and minor >= 6))
+if "linux" in sys.platform.lower() and not python_26:
+ extensions.append(distutils.core.Extension(
+ "tornado.epoll", ["tornado/epoll.c"]))
+
+distutils.core.setup(
+ name="tornado",
+ version="0.1",
+ packages = ["tornado"],
+ ext_modules = extensions,
+ author="Facebook",
+ author_email="python-tornado@googlegroups.com",
+ url="http://www.tornadoweb.org/",
+ license="http://www.apache.org/licenses/LICENSE-2.0",
+ description="Tornado is an open source version of the scalable, non-blocking web server and and tools that power FriendFeed",
+)
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""The Tornado web server and tools."""
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Implementations of various third-party authentication schemes.
+
+All the classes in this file are class Mixins designed to be used with
+web.py RequestHandler classes. The primary methods for each service are
+authenticate_redirect(), authorize_redirect(), and get_authenticated_user().
+The former should be called to redirect the user to, e.g., the OpenID
+authentication page on the third party service, and the latter should
+be called upon return to get the user data from the data returned by
+the third party service.
+
+They all take slightly different arguments due to the fact all these
+services implement authentication and authorization slightly differently.
+See the individual service classes below for complete documentation.
+
+Example usage for Google OpenID:
+
+class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("openid.mode", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authenticate_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "Google auth failed")
+ # Save the user with, e.g., set_secure_cookie()
+
+"""
+
+import base64
+import binascii
+import cgi
+import hashlib
+import hmac
+import httpclient
+import escape
+import logging
+import time
+import urllib
+import urlparse
+import uuid
+
+
+class OpenIdMixin(object):
+ """Abstract implementation of OpenID and Attribute Exchange.
+
+ See GoogleMixin below for example implementations.
+ """
+ def authenticate_redirect(self, callback_uri=None,
+ ax_attrs=["name","email","language","username"]):
+ """Returns the authentication URL for this service.
+
+ After authentication, the service will redirect back to the given
+ callback URI.
+
+ We request the given attributes for the authenticated user by
+ default (name, email, language, and username). If you don't need
+ all those attributes for your app, you can request fewer with
+ the ax_attrs keyword argument.
+ """
+ callback_uri = callback_uri or self.request.path
+ args = self._openid_args(callback_uri, ax_attrs=ax_attrs)
+ self.redirect(self._OPENID_ENDPOINT + "?" + urllib.urlencode(args))
+
+ def get_authenticated_user(self, callback):
+ """Fetches the authenticated user data upon redirect.
+
+ This method should be called by the handler that receives the
+ redirect from the authenticate_redirect() or authorize_redirect()
+ methods.
+ """
+ # Verify the OpenID response via direct request to the OP
+ args = dict((k, v[-1]) for k, v in self.request.arguments.iteritems())
+ args["openid.mode"] = u"check_authentication"
+ url = self._OPENID_ENDPOINT + "?" + urllib.urlencode(args)
+ http = httpclient.AsyncHTTPClient()
+ http.fetch(url, self.async_callback(
+ self._on_authentication_verified, callback))
+
+ def _openid_args(self, callback_uri, ax_attrs=[], oauth_scope=None):
+ url = urlparse.urljoin(self.request.full_url(), callback_uri)
+ args = {
+ "openid.ns": "http://specs.openid.net/auth/2.0",
+ "openid.claimed_id":
+ "http://specs.openid.net/auth/2.0/identifier_select",
+ "openid.identity":
+ "http://specs.openid.net/auth/2.0/identifier_select",
+ "openid.return_to": url,
+ "openid.realm": "http://" + self.request.host + "/",
+ "openid.mode": "checkid_setup",
+ }
+ if ax_attrs:
+ args.update({
+ "openid.ns.ax": "http://openid.net/srv/ax/1.0",
+ "openid.ax.mode": "fetch_request",
+ })
+ ax_attrs = set(ax_attrs)
+ required = []
+ if "name" in ax_attrs:
+ ax_attrs -= set(["name", "firstname", "fullname", "lastname"])
+ required += ["firstname", "fullname", "lastname"]
+ args.update({
+ "openid.ax.type.firstname":
+ "http://axschema.org/namePerson/first",
+ "openid.ax.type.fullname":
+ "http://axschema.org/namePerson",
+ "openid.ax.type.lastname":
+ "http://axschema.org/namePerson/last",
+ })
+ known_attrs = {
+ "email": "http://axschema.org/contact/email",
+ "language": "http://axschema.org/pref/language",
+ "username": "http://axschema.org/namePerson/friendly",
+ }
+ for name in ax_attrs:
+ args["openid.ax.type." + name] = known_attrs[name]
+ required.append(name)
+ args["openid.ax.required"] = ",".join(required)
+ if oauth_scope:
+ args.update({
+ "openid.ns.oauth":
+ "http://specs.openid.net/extensions/oauth/1.0",
+ "openid.oauth.consumer": self.request.host.split(":")[0],
+ "openid.oauth.scope": oauth_scope,
+ })
+ return args
+
+ def _on_authentication_verified(self, callback, response):
+ if response.error or u"is_valid:true" not in response.body:
+ logging.warning("Invalid OpenID response: %s", response.error or
+ response.body)
+ callback(None)
+ return
+
+ # Make sure we got back at least an email from attribute exchange
+ ax_ns = None
+ for name, values in self.request.arguments.iteritems():
+ if name.startswith("openid.ns.") and \
+ values[-1] == u"http://openid.net/srv/ax/1.0":
+ ax_ns = name[10:]
+ break
+ def get_ax_arg(uri):
+ if not ax_ns: return u""
+ prefix = "openid." + ax_ns + ".type."
+ ax_name = None
+ for name, values in self.request.arguments.iteritems():
+ if values[-1] == uri and name.startswith(prefix):
+ part = name[len(prefix):]
+ ax_name = "openid." + ax_ns + ".value." + part
+ break
+ if not ax_name: return u""
+ return self.get_argument(ax_name, u"")
+
+ email = get_ax_arg("http://axschema.org/contact/email")
+ name = get_ax_arg("http://axschema.org/namePerson")
+ first_name = get_ax_arg("http://axschema.org/namePerson/first")
+ last_name = get_ax_arg("http://axschema.org/namePerson/last")
+ username = get_ax_arg("http://axschema.org/namePerson/friendly")
+ locale = get_ax_arg("http://axschema.org/pref/language").lower()
+ user = dict()
+ name_parts = []
+ if first_name:
+ user["first_name"] = first_name
+ name_parts.append(first_name)
+ if last_name:
+ user["last_name"] = last_name
+ name_parts.append(last_name)
+ if name:
+ user["name"] = name
+ elif name_parts:
+ user["name"] = u" ".join(name_parts)
+ elif email:
+ user["name"] = email.split("@")[0]
+ if email: user["email"] = email
+ if locale: user["locale"] = locale
+ if username: user["username"] = username
+ callback(user)
+
+
+class OAuthMixin(object):
+ """Abstract implementation of OAuth.
+
+ See TwitterMixin and FriendFeedMixin below for example implementations.
+ """
+ def authorize_redirect(self, callback_uri=None):
+ """Redirects the user to obtain OAuth authorization for this service.
+
+ Twitter and FriendFeed both require that you register a Callback
+ URL with your application. You should call this method to log the
+ user in, and then call get_authenticated_user() in the handler
+ you registered as your Callback URL to complete the authorization
+ process.
+
+ This method sets a cookie called _oauth_request_token which is
+ subsequently used (and cleared) in get_authenticated_user for
+ security purposes.
+ """
+ if callback_uri and getattr(self, "_OAUTH_NO_CALLBACKS", False):
+ raise Exception("This service does not support oauth_callback")
+ http = httpclient.AsyncHTTPClient()
+ http.fetch(self._oauth_request_token_url(), self.async_callback(
+ self._on_request_token, self._OAUTH_AUTHORIZE_URL, callback_uri))
+
+ def get_authenticated_user(self, callback):
+ """Gets the OAuth authorized user and access token on callback.
+
+ This method should be called from the handler for your registered
+ OAuth Callback URL to complete the registration process. We call
+ callback with the authenticated user, which in addition to standard
+ attributes like 'name' includes the 'access_key' attribute, which
+ contains the OAuth access you can use to make authorized requests
+ to this service on behalf of the user.
+ """
+ request_key = self.get_argument("oauth_token")
+ request_cookie = self.get_cookie("_oauth_request_token")
+ if not request_cookie:
+ logging.warning("Missing OAuth request token cookie")
+ callback(None)
+ return
+ cookie_key, cookie_secret = request_cookie.split("|")
+ if cookie_key != request_key:
+ logging.warning("Request token does not match cookie")
+ callback(None)
+ return
+ token = dict(key=cookie_key, secret=cookie_secret)
+ http = httpclient.AsyncHTTPClient()
+ http.fetch(self._oauth_access_token_url(token), self.async_callback(
+ self._on_access_token, callback))
+
+ def _oauth_request_token_url(self):
+ consumer_token = self._oauth_consumer_token()
+ url = self._OAUTH_REQUEST_TOKEN_URL
+ args = dict(
+ oauth_consumer_key=consumer_token["key"],
+ oauth_signature_method="HMAC-SHA1",
+ oauth_timestamp=str(int(time.time())),
+ oauth_nonce=binascii.b2a_hex(uuid.uuid4().bytes),
+ oauth_version="1.0",
+ )
+ signature = _oauth_signature(consumer_token, "GET", url, args)
+ args["oauth_signature"] = signature
+ return url + "?" + urllib.urlencode(args)
+
+ def _on_request_token(self, authorize_url, callback_uri, response):
+ if response.error:
+ raise Exception("Could not get request token")
+ request_token = _oauth_parse_response(response.body)
+ data = "|".join([request_token["key"], request_token["secret"]])
+ self.set_cookie("_oauth_request_token", data)
+ args = dict(oauth_token=request_token["key"])
+ if callback_uri:
+ args["oauth_callback"] = urlparse.urljoin(
+ self.request.full_url(), callback_uri)
+ self.redirect(authorize_url + "?" + urllib.urlencode(args))
+
+ def _oauth_access_token_url(self, request_token):
+ consumer_token = self._oauth_consumer_token()
+ url = self._OAUTH_ACCESS_TOKEN_URL
+ args = dict(
+ oauth_consumer_key=consumer_token["key"],
+ oauth_token=request_token["key"],
+ oauth_signature_method="HMAC-SHA1",
+ oauth_timestamp=str(int(time.time())),
+ oauth_nonce=binascii.b2a_hex(uuid.uuid4().bytes),
+ oauth_version="1.0",
+ )
+ signature = _oauth_signature(consumer_token, "GET", url, args,
+ request_token)
+ args["oauth_signature"] = signature
+ return url + "?" + urllib.urlencode(args)
+
+ def _on_access_token(self, callback, response):
+ if response.error:
+ logging.warning("Could not fetch access token")
+ callback(None)
+ return
+ access_token = _oauth_parse_response(response.body)
+ user = self._oauth_get_user(access_token, self.async_callback(
+ self._on_oauth_get_user, access_token, callback))
+
+ def _oauth_get_user(self, access_token, callback):
+ raise NotImplementedError()
+
+ def _on_oauth_get_user(self, access_token, callback, user):
+ if not user:
+ callback(None)
+ return
+ user["access_token"] = access_token
+ callback(user)
+
+ def _oauth_request_parameters(self, url, access_token, parameters={},
+ method="GET"):
+ """Returns the OAuth parameters as a dict for the given request.
+
+ parameters should include all POST arguments and query string arguments
+ that will be sent with the request.
+ """
+ consumer_token = self._oauth_consumer_token()
+ base_args = dict(
+ oauth_consumer_key=consumer_token["key"],
+ oauth_token=access_token["key"],
+ oauth_signature_method="HMAC-SHA1",
+ oauth_timestamp=str(int(time.time())),
+ oauth_nonce=binascii.b2a_hex(uuid.uuid4().bytes),
+ oauth_version="1.0",
+ )
+ args = {}
+ args.update(base_args)
+ args.update(parameters)
+ signature = _oauth_signature(consumer_token, method, url, args,
+ access_token)
+ base_args["oauth_signature"] = signature
+ return base_args
+
+
+class TwitterMixin(OAuthMixin):
+ """Twitter OAuth authentication.
+
+ To authenticate with Twitter, register your application with
+ Twitter at http://twitter.com/apps. Then copy your Consumer Key and
+ Consumer Secret to the application settings 'twitter_consumer_key' and
+ 'twitter_consumer_secret'. Use this Mixin on the handler for the URL
+ you registered as your application's Callback URL.
+
+ When your application is set up, you can use this Mixin like this
+ to authenticate the user with Twitter and get access to their stream:
+
+ class TwitterHandler(tornado.web.RequestHandler,
+ tornado.auth.TwitterMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("oauth_token", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authorize_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "Twitter auth failed")
+ # Save the user using, e.g., set_secure_cookie()
+
+ The user object returned by get_authenticated_user() includes the
+ attributes 'username', 'name', and all of the custom Twitter user
+ attributes describe at
+ http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-users%C2%A0show
+ in addition to 'access_token'. You should save the access token with
+ the user; it is required to make requests on behalf of the user later
+ with twitter_request().
+ """
+ _OAUTH_REQUEST_TOKEN_URL = "http://twitter.com/oauth/request_token"
+ _OAUTH_ACCESS_TOKEN_URL = "http://twitter.com/oauth/access_token"
+ _OAUTH_AUTHORIZE_URL = "http://twitter.com/oauth/authorize"
+ _OAUTH_AUTHENTICATE_URL = "http://twitter.com/oauth/authenticate"
+ _OAUTH_NO_CALLBACKS = True
+
+ def authenticate_redirect(self):
+ """Just like authorize_redirect(), but auto-redirects if authorized.
+
+ This is generally the right interface to use if you are using
+ Twitter for single-sign on.
+ """
+ http = httpclient.AsyncHTTPClient()
+ http.fetch(self._oauth_request_token_url(), self.async_callback(
+ self._on_request_token, self._OAUTH_AUTHENTICATE_URL, None))
+
+ def twitter_request(self, path, callback, access_token=None,
+ post_args=None, **args):
+ """Fetches the given API path, e.g., "/statuses/user_timeline/btaylor"
+
+ The path should not include the format (we automatically append
+ ".json" and parse the JSON output).
+
+ If the request is a POST, post_args should be provided. Query
+ string arguments should be given as keyword arguments.
+
+ All the Twitter methods are documented at
+ http://apiwiki.twitter.com/Twitter-API-Documentation.
+
+ Many methods require an OAuth access token which you can obtain
+ through authorize_redirect() and get_authenticated_user(). The
+ user returned through that process includes an 'access_token'
+ attribute that can be used to make authenticated requests via
+ this method. Example usage:
+
+ class MainHandler(tornado.web.RequestHandler,
+ tornado.auth.TwitterMixin):
+ @tornado.web.authenticated
+ @tornado.web.asynchronous
+ def get(self):
+ self.twitter_request(
+ "/statuses/update",
+ post_args={"status": "Testing Tornado Web Server"},
+ access_token=user["access_token"],
+ callback=self.async_callback(self._on_post))
+
+ def _on_post(self, new_entry):
+ if not new_entry:
+ # Call failed; perhaps missing permission?
+ self.authorize_redirect()
+ return
+ self.finish("Posted a message!")
+
+ """
+ # Add the OAuth resource request signature if we have credentials
+ url = "http://twitter.com" + path + ".json"
+ if access_token:
+ all_args = {}
+ all_args.update(args)
+ all_args.update(post_args or {})
+ consumer_token = self._oauth_consumer_token()
+ method = "POST" if post_args is not None else "GET"
+ oauth = self._oauth_request_parameters(
+ url, access_token, all_args, method=method)
+ args.update(oauth)
+ if args: url += "?" + urllib.urlencode(args)
+ callback = self.async_callback(self._on_twitter_request, callback)
+ http = httpclient.AsyncHTTPClient()
+ if post_args is not None:
+ http.fetch(url, method="POST", body=urllib.urlencode(post_args),
+ callback=callback)
+ else:
+ http.fetch(url, callback=callback)
+
+ def _on_twitter_request(self, callback, response):
+ if response.error:
+ logging.warning("Error response %s fetching %s", response.error,
+ response.request.url)
+ callback(None)
+ return
+ callback(escape.json_decode(response.body))
+
+ def _oauth_consumer_token(self):
+ self.require_setting("twitter_consumer_key", "Twitter OAuth")
+ self.require_setting("twitter_consumer_secret", "Twitter OAuth")
+ return dict(
+ key=self.settings["twitter_consumer_key"],
+ secret=self.settings["twitter_consumer_secret"])
+
+ def _oauth_get_user(self, access_token, callback):
+ callback = self.async_callback(self._parse_user_response, callback)
+ self.twitter_request(
+ "/users/show/" + access_token["screen_name"],
+ access_token=access_token, callback=callback)
+
+ def _parse_user_response(self, callback, user):
+ if user:
+ user["username"] = user["screen_name"]
+ callback(user)
+
+
+class FriendFeedMixin(OAuthMixin):
+ """FriendFeed OAuth authentication.
+
+ To authenticate with FriendFeed, register your application with
+ FriendFeed at http://friendfeed.com/api/applications. Then
+ copy your Consumer Key and Consumer Secret to the application settings
+ 'friendfeed_consumer_key' and 'friendfeed_consumer_secret'. Use
+ this Mixin on the handler for the URL you registered as your
+ application's Callback URL.
+
+ When your application is set up, you can use this Mixin like this
+ to authenticate the user with FriendFeed and get access to their feed:
+
+ class FriendFeedHandler(tornado.web.RequestHandler,
+ tornado.auth.FriendFeedMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("oauth_token", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authorize_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "FriendFeed auth failed")
+ # Save the user using, e.g., set_secure_cookie()
+
+ The user object returned by get_authenticated_user() includes the
+ attributes 'username', 'name', and 'description' in addition to
+ 'access_token'. You should save the access token with the user;
+ it is required to make requests on behalf of the user later with
+ friendfeed_request().
+ """
+ _OAUTH_REQUEST_TOKEN_URL = "https://friendfeed.com/account/oauth/request_token"
+ _OAUTH_ACCESS_TOKEN_URL = "https://friendfeed.com/account/oauth/access_token"
+ _OAUTH_AUTHORIZE_URL = "https://friendfeed.com/account/oauth/authorize"
+ _OAUTH_NO_CALLBACKS = True
+
+ def friendfeed_request(self, path, callback, access_token=None,
+ post_args=None, **args):
+ """Fetches the given relative API path, e.g., "/bret/friends"
+
+ If the request is a POST, post_args should be provided. Query
+ string arguments should be given as keyword arguments.
+
+ All the FriendFeed methods are documented at
+ http://friendfeed.com/api/documentation.
+
+ Many methods require an OAuth access token which you can obtain
+ through authorize_redirect() and get_authenticated_user(). The
+ user returned through that process includes an 'access_token'
+ attribute that can be used to make authenticated requests via
+ this method. Example usage:
+
+ class MainHandler(tornado.web.RequestHandler,
+ tornado.auth.FriendFeedMixin):
+ @tornado.web.authenticated
+ @tornado.web.asynchronous
+ def get(self):
+ self.friendfeed_request(
+ "/entry",
+ post_args={"body": "Testing Tornado Web Server"},
+ access_token=self.current_user["access_token"],
+ callback=self.async_callback(self._on_post))
+
+ def _on_post(self, new_entry):
+ if not new_entry:
+ # Call failed; perhaps missing permission?
+ self.authorize_redirect()
+ return
+ self.finish("Posted a message!")
+
+ """
+ # Add the OAuth resource request signature if we have credentials
+ url = "http://friendfeed-api.com/v2" + path
+ if access_token:
+ all_args = {}
+ all_args.update(args)
+ all_args.update(post_args or {})
+ consumer_token = self._oauth_consumer_token()
+ method = "POST" if post_args is not None else "GET"
+ oauth = self._oauth_request_parameters(
+ url, access_token, all_args, method=method)
+ args.update(oauth)
+ if args: url += "?" + urllib.urlencode(args)
+ callback = self.async_callback(self._on_friendfeed_request, callback)
+ http = httpclient.AsyncHTTPClient()
+ if post_args is not None:
+ http.fetch(url, method="POST", body=urllib.urlencode(post_args),
+ callback=callback)
+ else:
+ http.fetch(url, callback=callback)
+
+ def _on_friendfeed_request(self, callback, response):
+ if response.error:
+ logging.warning("Error response %s fetching %s", response.error,
+ response.request.url)
+ callback(None)
+ return
+ callback(escape.json_decode(response.body))
+
+ def _oauth_consumer_token(self):
+ self.require_setting("friendfeed_consumer_key", "FriendFeed OAuth")
+ self.require_setting("friendfeed_consumer_secret", "FriendFeed OAuth")
+ return dict(
+ key=self.settings["friendfeed_consumer_key"],
+ secret=self.settings["friendfeed_consumer_secret"])
+
+ def _oauth_get_user(self, access_token, callback):
+ callback = self.async_callback(self._parse_user_response, callback)
+ self.friendfeed_request(
+ "/feedinfo/" + access_token["username"],
+ include="id,name,description", access_token=access_token,
+ callback=callback)
+
+ def _parse_user_response(self, callback, user):
+ if user:
+ user["username"] = user["id"]
+ callback(user)
+
+
+class GoogleMixin(OpenIdMixin, OAuthMixin):
+ """Google Open ID / OAuth authentication.
+
+ No application registration is necessary to use Google for authentication
+ or to access Google resources on behalf of a user. To authenticate with
+ Google, redirect with authenticate_redirect(). On return, parse the
+ response with get_authenticated_user(). We send a dict containing the
+ values for the user, including 'email', 'name', and 'locale'.
+ Example usage:
+
+ class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("openid.mode", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authenticate_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "Google auth failed")
+ # Save the user with, e.g., set_secure_cookie()
+
+ """
+ _OPENID_ENDPOINT = "https://www.google.com/accounts/o8/ud"
+ _OAUTH_ACCESS_TOKEN_URL = "https://www.google.com/accounts/OAuthGetAccessToken"
+
+ def authorize_redirect(self, oauth_scope, callback_uri=None,
+ ax_attrs=["name","email","language","username"]):
+ """Authenticates and authorizes for the given Google resource.
+
+ Some of the available resources are:
+
+ Gmail Contacts - http://www.google.com/m8/feeds/
+ Calendar - http://www.google.com/calendar/feeds/
+ Finance - http://finance.google.com/finance/feeds/
+
+ You can authorize multiple resources by separating the resource
+ URLs with a space.
+ """
+ callback_uri = callback_uri or self.request.path
+ args = self._openid_args(callback_uri, ax_attrs=ax_attrs,
+ oauth_scope=oauth_scope)
+ self.redirect(self._OPENID_ENDPOINT + "?" + urllib.urlencode(args))
+
+ def get_authenticated_user(self, callback):
+ """Fetches the authenticated user data upon redirect."""
+ # Look to see if we are doing combined OpenID/OAuth
+ oauth_ns = ""
+ for name, values in self.request.arguments.iteritems():
+ if name.startswith("openid.ns.") and \
+ values[-1] == u"http://specs.openid.net/extensions/oauth/1.0":
+ oauth_ns = name[10:]
+ break
+ token = self.get_argument("openid." + oauth_ns + ".request_token", "")
+ if token:
+ http = httpclient.AsyncHTTPClient()
+ token = dict(key=token, secret="")
+ http.fetch(self._oauth_access_token_url(token),
+ self.async_callback(self._on_access_token, callback))
+ else:
+ OpenIdMixin.get_authenticated_user(self, callback)
+
+ def _oauth_consumer_token(self):
+ self.require_setting("google_consumer_key", "Google OAuth")
+ self.require_setting("google_consumer_secret", "Google OAuth")
+ return dict(
+ key=self.settings["google_consumer_key"],
+ secret=self.settings["google_consumer_secret"])
+
+ def _oauth_get_user(self, access_token, callback):
+ OpenIdMixin.get_authenticated_user(self, callback)
+
+
+class FacebookMixin(object):
+ """Facebook Connect authentication.
+
+ To authenticate with Facebook, register your application with
+ Facebook at http://www.facebook.com/developers/apps.php. Then
+ copy your API Key and Application Secret to the application settings
+ 'facebook_api_key' and 'facebook_secret'.
+
+ When your application is set up, you can use this Mixin like this
+ to authenticate the user with Facebook:
+
+ class FacebookHandler(tornado.web.RequestHandler,
+ tornado.auth.FacebookMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("auth_token", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authenticate_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ raise tornado.web.HTTPError(500, "Facebook auth failed")
+ # Save the user using, e.g., set_secure_cookie()
+
+ The user object returned by get_authenticated_user() includes the
+ attributes 'facebook_uid' and 'name' in addition to session attributes
+ like 'session_key'. You should save the session key with the user; it is
+ required to make requests on behalf of the user later with
+ facebook_request().
+ """
+ def authenticate_redirect(self, callback_uri=None, cancel_uri=None,
+ extended_permissions=None):
+ """Authenticates/installs this app for the current user."""
+ self.require_setting("facebook_api_key", "Facebook Connect")
+ callback_uri = callback_uri or self.request.path
+ args = {
+ "api_key": self.settings["facebook_api_key"],
+ "v": "1.0",
+ "fbconnect": "true",
+ "display": "page",
+ "next": urlparse.urljoin(self.request.full_url(), callback_uri),
+ "return_session": "true",
+ }
+ if cancel_uri:
+ args["cancel_url"] = urlparse.urljoin(
+ self.request.full_url(), cancel_uri)
+ if extended_permissions:
+ if isinstance(extended_permissions, basestring):
+ extended_permissions = [extended_permissions]
+ args["req_perms"] = ",".join(extended_permissions)
+ self.redirect("http://www.facebook.com/login.php?" +
+ urllib.urlencode(args))
+
+ def authorize_redirect(self, extended_permissions, callback_uri=None,
+ cancel_uri=None):
+ """Redirects to an authorization request for the given FB resource.
+
+ The available resource names are listed at
+ http://wiki.developers.facebook.com/index.php/Extended_permission.
+ The most common resource types include:
+
+ publish_stream
+ read_stream
+ email
+ sms
+
+ extended_permissions can be a single permission name or a list of
+ names. To get the session secret and session key, call
+ get_authenticated_user() just as you would with
+ authenticate_redirect().
+ """
+ self.authenticate_redirect(callback_uri, cancel_uri,
+ extended_permissions)
+
+ def get_authenticated_user(self, callback):
+ """Fetches the authenticated Facebook user.
+
+ The authenticated user includes the special Facebook attributes
+ 'session_key' and 'facebook_uid' in addition to the standard
+ user attributes like 'name'.
+ """
+ self.require_setting("facebook_api_key", "Facebook Connect")
+ session = escape.json_decode(self.get_argument("session"))
+ self.facebook_request(
+ method="facebook.users.getInfo",
+ callback=self.async_callback(
+ self._on_get_user_info, callback, session),
+ session_key=session["session_key"],
+ uids=session["uid"],
+ fields="uid,first_name,last_name,name,locale,pic_square")
+
+ def facebook_request(self, method, callback, **args):
+ """Makes a Facebook API REST request.
+
+ We automatically include the Facebook API key and signature, but
+ it is the callers responsibility to include 'session_key' and any
+ other required arguments to the method.
+
+ The available Facebook methods are documented here:
+ http://wiki.developers.facebook.com/index.php/API
+
+ Here is an example for the stream.get() method:
+
+ class MainHandler(tornado.web.RequestHandler,
+ tornado.auth.FacebookMixin):
+ @tornado.web.authenticated
+ @tornado.web.asynchronous
+ def get(self):
+ self.facebook_request(
+ method="stream.get",
+ callback=self.async_callback(self._on_stream),
+ session_key=self.current_user["session_key"])
+
+ def _on_stream(self, stream):
+ if stream is None:
+ # Not authorized to read the stream yet?
+ self.redirect(self.authorize_redirect("read_stream"))
+ return
+ self.render("stream.html", stream=stream)
+
+ """
+ self.require_setting("facebook_api_key", "Facebook Connect")
+ self.require_setting("facebook_secret", "Facebook Connect")
+ if not method.startswith("facebook."):
+ method = "facebook." + method
+ args["api_key"] = self.settings["facebook_api_key"]
+ args["v"] = "1.0"
+ args["method"] = method
+ args["call_id"] = str(long(time.time() * 1e6))
+ args["format"] = "json"
+ args["sig"] = self._signature(args)
+ url = "http://api.facebook.com/restserver.php?" + \
+ urllib.urlencode(args)
+ http = httpclient.AsyncHTTPClient()
+ http.fetch(url, callback=self.async_callback(
+ self._parse_response, callback))
+
+ def _on_get_user_info(self, callback, session, users):
+ if users is None:
+ callback(None)
+ return
+ callback({
+ "name": users[0]["name"],
+ "first_name": users[0]["first_name"],
+ "last_name": users[0]["last_name"],
+ "uid": users[0]["uid"],
+ "locale": users[0]["locale"],
+ "session_key": session["session_key"],
+ "session_expires": session["expires"],
+ })
+
+ def _parse_response(self, callback, response):
+ if response.error:
+ logging.warning("HTTP error from Facebook: %s", response.error)
+ callback(None)
+ return
+ try:
+ json = escape.json_decode(response.body)
+ except:
+ logging.warning("Invalid JSON from Facebook: %r", response.body)
+ callback(None)
+ return
+ if isinstance(json, dict) and json.get("error_code"):
+ logging.warning("Facebook error: %d: %r", json["error_code"],
+ json.get("error_msg"))
+ callback(None)
+ return
+ callback(json)
+
+ def _signature(self, args):
+ parts = ["%s=%s" % (n, args[n]) for n in sorted(args.keys())]
+ body = "".join(parts) + self.settings["facebook_secret"]
+ if isinstance(body, unicode): body = body.encode("utf-8")
+ return hashlib.md5(body).hexdigest()
+
+
+def _oauth_signature(consumer_token, method, url, parameters={}, token=None):
+ """Calculates the HMAC-SHA1 OAuth signature for the given request.
+
+ See http://oauth.net/core/1.0/#signing_process
+ """
+ parts = urlparse.urlparse(url)
+ scheme, netloc, path = parts[:3]
+ normalized_url = scheme.lower() + "://" + netloc.lower() + path
+
+ base_elems = []
+ base_elems.append(method.upper())
+ base_elems.append(normalized_url)
+ base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v)))
+ for k, v in sorted(parameters.items())))
+ base_string = "&".join(_oauth_escape(e) for e in base_elems)
+
+ key_elems = [consumer_token["secret"]]
+ key_elems.append(token["secret"] if token else "")
+ key = "&".join(key_elems)
+
+ hash = hmac.new(key, base_string, hashlib.sha1)
+ return binascii.b2a_base64(hash.digest())[:-1]
+
+
+def _oauth_escape(val):
+ if isinstance(val, unicode):
+ val = val.encode("utf-8")
+ return urllib.quote(val, safe="~")
+
+
+def _oauth_parse_response(body):
+ p = cgi.parse_qs(body, keep_blank_values=False)
+ token = dict(key=p["oauth_token"][0], secret=p["oauth_token_secret"][0])
+
+ # Add the extra parameters the Provider included to the token
+ special = ("oauth_token", "oauth_token_secret")
+ token.update((k, p[k][0]) for k in p if k not in special)
+ return token
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A lightweight wrapper around MySQLdb."""
+
+import copy
+import MySQLdb
+import MySQLdb.constants
+import MySQLdb.converters
+import MySQLdb.cursors
+import itertools
+
+
+class Connection(object):
+ """A lightweight wrapper around MySQLdb DB-API connections.
+
+ The main value we provide is wrapping rows in a dict/object so that
+ columns can be accessed by name. Typical usage:
+
+ db = database.Connection("localhost", "mydatabase")
+ for article in db.query("SELECT * FROM articles"):
+ print article.title
+
+ Cursors are hidden by the implementation, but other than that, the methods
+ are very similar to the DB-API.
+
+ We explicitly set the timezone to UTC and the character encoding to
+ UTF-8 on all connections to avoid time zone and encoding errors.
+ """
+ def __init__(self, host, database, user=None, password=None):
+ self.host = host
+ self.database = database
+
+ args = dict(conv=CONVERSIONS, use_unicode=True, charset="utf8",
+ db=database, init_command='SET time_zone = "+0:00"',
+ sql_mode="TRADITIONAL")
+ if user is not None:
+ args["user"] = user
+ if password is not None:
+ args["passwd"] = password
+
+ # We accept a path to a MySQL socket file or a host(:port) string
+ if "/" in host:
+ args["unix_socket"] = host
+ else:
+ self.socket = None
+ pair = host.split(":")
+ if len(pair) == 2:
+ args["host"] = pair[0]
+ args["port"] = int(pair[1])
+ else:
+ args["host"] = host
+ args["port"] = 3306
+
+ self._db = None
+ self._db = MySQLdb.connect(**args)
+ self._db.autocommit(True)
+
+ def __del__(self):
+ if self._db is not None:
+ self.close()
+
+ def close(self):
+ """Closes this database connection."""
+ self._db.close()
+ self._db = None
+
+ def iter(self, query, *parameters):
+ """Returns an iterator for the given query and parameters."""
+ cursor = MySQLdb.cursors.SSCursor(self._db)
+ try:
+ cursor.execute(query, parameters)
+ column_names = [d[0] for d in cursor.description]
+ for row in cursor:
+ yield Row(zip(column_names, row))
+ finally:
+ cursor.close()
+
+ def query(self, query, *parameters):
+ """Returns a row list for the given query and parameters."""
+ cursor = self._db.cursor()
+ try:
+ cursor.execute(query, parameters)
+ column_names = [d[0] for d in cursor.description]
+ return [Row(itertools.izip(column_names, row)) for row in cursor]
+ finally:
+ cursor.close()
+
+ def get(self, query, *parameters):
+ """Returns the first row returned for the given query."""
+ rows = self.query(query, *parameters)
+ if not rows:
+ return None
+ elif len(rows) > 1:
+ raise Exception("Multiple rows returned for Database.get() query")
+ else:
+ return rows[0]
+
+ def execute(self, query, *parameters):
+ """Executes the given query, returning the lastrowid from the query."""
+ cursor = self._db.cursor()
+ try:
+ cursor.execute(query, parameters)
+ return cursor.lastrowid
+ finally:
+ cursor.close()
+
+ def executemany(self, query, parameters):
+ """Executes the given query against all the given param sequences.
+
+ We return the lastrowid from the query.
+ """
+ cursor = self._db.cursor()
+ try:
+ cursor.executemany(query, parameters)
+ return cursor.lastrowid
+ finally:
+ cursor.close()
+
+
+class Row(dict):
+ """A dict that allows for object-like property access syntax."""
+ def __getattr__(self, name):
+ try:
+ return self[name]
+ except KeyError:
+ raise AttributeError(name)
+
+
+# Fix the access conversions to properly recognize unicode/binary
+FIELD_TYPE = MySQLdb.constants.FIELD_TYPE
+FLAG = MySQLdb.constants.FLAG
+CONVERSIONS = copy.deepcopy(MySQLdb.converters.conversions)
+for field_type in \
+ [FIELD_TYPE.BLOB, FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING] + \
+ ([FIELD_TYPE.VARCHAR] if 'VARCHAR' in vars(FIELD_TYPE) else []):
+ CONVERSIONS[field_type].insert(0, (FLAG.BINARY, str))
+
+
+# Alias some common MySQL exceptions
+IntegrityError = MySQLdb.IntegrityError
+OperationalError = MySQLdb.OperationalError
--- /dev/null
+/*
+ * Copyright 2009 Facebook
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "Python.h"
+#include <string.h>
+#include <sys/epoll.h>
+
+#define MAX_EVENTS 24
+
+/*
+ * Simple wrapper around epoll_create.
+ */
+static PyObject* _epoll_create(void) {
+ int fd = epoll_create(MAX_EVENTS);
+ if (fd == -1) {
+ PyErr_SetFromErrno(PyExc_Exception);
+ return NULL;
+ }
+
+ return PyInt_FromLong(fd);
+}
+
+/*
+ * Simple wrapper around epoll_ctl. We throw an exception if the call fails
+ * rather than returning the error code since it is an infrequent (and likely
+ * catastrophic) event when it does happen.
+ */
+static PyObject* _epoll_ctl(PyObject* self, PyObject* args) {
+ int epfd, op, fd, events;
+ struct epoll_event event;
+
+ if (!PyArg_ParseTuple(args, "iiiI", &epfd, &op, &fd, &events)) {
+ return NULL;
+ }
+
+ memset(&event, 0, sizeof(event));
+ event.events = events;
+ event.data.fd = fd;
+ if (epoll_ctl(epfd, op, fd, &event) == -1) {
+ PyErr_SetFromErrno(PyExc_OSError);
+ return NULL;
+ }
+
+ Py_INCREF(Py_None);
+ return Py_None;
+}
+
+/*
+ * Simple wrapper around epoll_wait. We return None if the call times out and
+ * throw an exception if an error occurs. Otherwise, we return a list of
+ * (fd, event) tuples.
+ */
+static PyObject* _epoll_wait(PyObject* self, PyObject* args) {
+ struct epoll_event events[MAX_EVENTS];
+ int epfd, timeout, num_events, i;
+ PyObject* list;
+ PyObject* tuple;
+
+ if (!PyArg_ParseTuple(args, "ii", &epfd, &timeout)) {
+ return NULL;
+ }
+
+ Py_BEGIN_ALLOW_THREADS
+ num_events = epoll_wait(epfd, events, MAX_EVENTS, timeout);
+ Py_END_ALLOW_THREADS
+ if (num_events == -1) {
+ PyErr_SetFromErrno(PyExc_Exception);
+ return NULL;
+ }
+
+ list = PyList_New(num_events);
+ for (i = 0; i < num_events; i++) {
+ tuple = PyTuple_New(2);
+ PyTuple_SET_ITEM(tuple, 0, PyInt_FromLong(events[i].data.fd));
+ PyTuple_SET_ITEM(tuple, 1, PyInt_FromLong(events[i].events));
+ PyList_SET_ITEM(list, i, tuple);
+ }
+ return list;
+}
+
+/*
+ * Our method declararations
+ */
+static PyMethodDef kEpollMethods[] = {
+ {"epoll_create", (PyCFunction)_epoll_create, METH_NOARGS,
+ "Create an epoll file descriptor"},
+ {"epoll_ctl", _epoll_ctl, METH_VARARGS,
+ "Control an epoll file descriptor"},
+ {"epoll_wait", _epoll_wait, METH_VARARGS,
+ "Wait for events on an epoll file descriptor"},
+ {NULL, NULL, 0, NULL}
+};
+
+/*
+ * Module initialization
+ */
+PyMODINIT_FUNC initepoll(void) {
+ Py_InitModule("epoll", kEpollMethods);
+}
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Escaping/unescaping methods for HTML, JSON, URLs, and others."""
+
+import htmlentitydefs
+import re
+import xml.sax.saxutils
+import urllib
+
+try:
+ import json
+ assert hasattr(json, "loads") and hasattr(json, "dumps")
+ _json_decode = lambda s: json.loads(s)
+ _json_encode = lambda v: json.dumps(v)
+except:
+ try:
+ import simplejson
+ _json_decode = lambda s: simplejson.loads(_unicode(s))
+ _json_encode = lambda v: simplejson.dumps(v)
+ except ImportError:
+ try:
+ # For Google AppEngine
+ from django.utils import simplejson
+ _json_decode = lambda s: simplejson.loads(_unicode(s))
+ _json_encode = lambda v: simplejson.dumps(v)
+ except ImportError:
+ raise Exception("A JSON parser is required, e.g., simplejson at "
+ "http://pypi.python.org/pypi/simplejson/")
+
+
+def xhtml_escape(value):
+ """Escapes a string so it is valid within XML or XHTML."""
+ return utf8(xml.sax.saxutils.escape(value))
+
+
+def xhtml_unescape(value):
+ """Un-escapes an XML-escaped string."""
+ return re.sub(r"&(#?)(\w+?);", _convert_entity, _unicode(value))
+
+
+def json_encode(value):
+ """JSON-encodes the given Python object."""
+ return _json_encode(value)
+
+
+def json_decode(value):
+ """Returns Python objects for the given JSON string."""
+ return _json_decode(value)
+
+
+def squeeze(value):
+ """Replace all sequences of whitespace chars with a single space."""
+ return re.sub(r"[\x00-\x20]+", " ", value).strip()
+
+
+def url_escape(value):
+ """Returns a valid URL-encoded version of the given value."""
+ return urllib.quote_plus(utf8(value))
+
+
+def url_unescape(value):
+ """Decodes the given value from a URL."""
+ return _unicode(urllib.unquote_plus(value))
+
+
+def utf8(value):
+ if isinstance(value, unicode):
+ return value.encode("utf-8")
+ assert isinstance(value, str)
+ return value
+
+
+def _unicode(value):
+ if isinstance(value, str):
+ return value.decode("utf-8")
+ assert isinstance(value, unicode)
+ return value
+
+
+def _convert_entity(m):
+ if m.group(1) == "#":
+ try:
+ return unichr(int(m.group(2)))
+ except ValueError:
+ return "&#%s;" % m.group(2)
+ try:
+ return _HTML_UNICODE_MAP[m.group(2)]
+ except KeyError:
+ return "&%s;" % m.group(2)
+
+
+def _build_unicode_map():
+ unicode_map = {}
+ for name, value in htmlentitydefs.name2codepoint.iteritems():
+ unicode_map[name] = unichr(value)
+ return unicode_map
+
+_HTML_UNICODE_MAP = _build_unicode_map()
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Blocking and non-blocking HTTP client implementations using pycurl."""
+
+import calendar
+import collections
+import cStringIO
+import email.utils
+import errno
+import functools
+import httplib
+import ioloop
+import logging
+import pycurl
+import time
+
+
+class HTTPClient(object):
+ """A blocking HTTP client backed with pycurl.
+
+ Typical usage looks like this:
+
+ http_client = httpclient.HTTPClient()
+ try:
+ response = http_client.fetch("http://www.google.com/")
+ print response.body
+ except httpclient.HTTPError, e:
+ print "Error:", e
+
+ fetch() can take a string URL or an HTTPRequest instance, which offers
+ more options, like executing POST/PUT/DELETE requests.
+ """
+ def __init__(self, max_simultaneous_connections=None):
+ self._curl = _curl_create(max_simultaneous_connections)
+
+ def __del__(self):
+ self._curl.close()
+
+ def fetch(self, request, **kwargs):
+ """Executes an HTTPRequest, returning an HTTPResponse.
+
+ If an error occurs during the fetch, we raise an HTTPError.
+ """
+ if not isinstance(request, HTTPRequest):
+ request = HTTPRequest(url=request, **kwargs)
+ buffer = cStringIO.StringIO()
+ headers = {}
+ try:
+ _curl_setup_request(self._curl, request, buffer, headers)
+ self._curl.perform()
+ code = self._curl.getinfo(pycurl.HTTP_CODE)
+ if code < 200 or code >= 300:
+ raise HTTPError(code)
+ effective_url = self._curl.getinfo(pycurl.EFFECTIVE_URL)
+ return HTTPResponse(
+ request=request, code=code, headers=headers,
+ body=buffer.getvalue(), effective_url=effective_url)
+ except pycurl.error, e:
+ raise CurlError(*e)
+ finally:
+ buffer.close()
+
+
+class AsyncHTTPClient(object):
+ """An non-blocking HTTP client backed with pycurl.
+
+ Example usage:
+
+ import ioloop
+
+ def handle_request(response):
+ if response.error:
+ print "Error:", response.error
+ else:
+ print response.body
+ ioloop.IOLoop.instance().stop()
+
+ http_client = httpclient.AsyncHTTPClient()
+ http_client.fetch("http://www.google.com/", handle_request)
+ ioloop.IOLoop.instance().start()
+
+ fetch() can take a string URL or an HTTPRequest instance, which offers
+ more options, like executing POST/PUT/DELETE requests.
+
+ The keyword argument max_clients to the AsyncHTTPClient constructor
+ determines the maximum number of simultaneous fetch() operations that
+ can execute in parallel on each IOLoop.
+ """
+ _ASYNC_CLIENTS = {}
+
+ def __new__(cls, io_loop=None, max_clients=10,
+ max_simultaneous_connections=None):
+ # There is one client per IOLoop since they share curl instances
+ io_loop = io_loop or ioloop.IOLoop.instance()
+ if id(io_loop) in cls._ASYNC_CLIENTS:
+ return cls._ASYNC_CLIENTS[id(io_loop)]
+ else:
+ instance = super(AsyncHTTPClient, cls).__new__(cls)
+ instance.io_loop = io_loop
+ instance._multi = pycurl.CurlMulti()
+ instance._curls = [_curl_create(max_simultaneous_connections)
+ for i in xrange(max_clients)]
+ instance._free_list = instance._curls[:]
+ instance._requests = collections.deque()
+ instance._fds = {}
+ instance._events = {}
+ instance._added_perform_callback = False
+ instance._timeout = None
+ cls._ASYNC_CLIENTS[id(io_loop)] = instance
+ return instance
+
+ def fetch(self, request, callback, **kwargs):
+ """Executes an HTTPRequest, calling callback with an HTTPResponse.
+
+ If an error occurs during the fetch, the HTTPResponse given to the
+ callback has a non-None error attribute that contains the exception
+ encountered during the request. You can call response.reraise() to
+ throw the exception (if any) in the callback.
+ """
+ if not isinstance(request, HTTPRequest):
+ request = HTTPRequest(url=request, **kwargs)
+ self._requests.append((request, callback))
+ self._add_perform_callback()
+
+ def _add_perform_callback(self):
+ if not self._added_perform_callback:
+ self.io_loop.add_callback(self._perform)
+ self._added_perform_callback = True
+
+ def _handle_events(self, fd, events):
+ self._events[fd] = events
+ self._add_perform_callback()
+
+ def _handle_timeout(self):
+ self._timeout = None
+ self._perform()
+
+ def _perform(self):
+ self._added_perform_callback = False
+
+ while True:
+ while True:
+ ret, num_handles = self._multi.perform()
+ if ret != pycurl.E_CALL_MULTI_PERFORM:
+ break
+
+ # Handle completed fetches
+ completed = 0
+ while True:
+ num_q, ok_list, err_list = self._multi.info_read()
+ for curl in ok_list:
+ self._finish(curl)
+ completed += 1
+ for curl, errnum, errmsg in err_list:
+ self._finish(curl, errnum, errmsg)
+ completed += 1
+ if num_q == 0:
+ break
+
+ # Start fetching new URLs
+ started = 0
+ while self._free_list and self._requests:
+ started += 1
+ curl = self._free_list.pop()
+ (request, callback) = self._requests.popleft()
+ curl.info = {
+ "headers": {},
+ "buffer": cStringIO.StringIO(),
+ "request": request,
+ "callback": callback,
+ "start_time": time.time(),
+ }
+ _curl_setup_request(curl, request, curl.info["buffer"],
+ curl.info["headers"])
+ self._multi.add_handle(curl)
+
+ if not started and not completed:
+ break
+
+ if self._timeout is not None:
+ self.io_loop.remove_timeout(self._timeout)
+ self._timeout = None
+
+ if num_handles:
+ self._timeout = self.io_loop.add_timeout(
+ time.time() + 0.2, self._handle_timeout)
+
+ # Wait for more I/O
+ fds = {}
+ (readable, writable, exceptable) = self._multi.fdset()
+ for fd in readable:
+ fds[fd] = fds.get(fd, 0) | 0x1 | 0x2
+ for fd in writable:
+ fds[fd] = fds.get(fd, 0) | 0x4
+ for fd in exceptable:
+ fds[fd] = fds.get(fd, 0) | 0x8 | 0x10
+
+ for fd in self._fds:
+ if fd not in fds:
+ self.io_loop.remove_handler(fd)
+
+ for fd, events in fds.iteritems():
+ old_events = self._fds.get(fd, None)
+ if old_events is None:
+ self.io_loop.add_handler(fd, self._handle_events, events)
+ elif old_events != events:
+ try:
+ self.io_loop.update_handler(fd, events)
+ except OSError, e:
+ if e[0] == errno.ENOENT:
+ self.io_loop.add_handler(fd, self._handle_events,
+ events)
+ else:
+ raise
+ self._fds = fds
+
+ def _finish(self, curl, curl_error=None, curl_message=None):
+ info = curl.info
+ curl.info = None
+ self._multi.remove_handle(curl)
+ self._free_list.append(curl)
+ if curl_error:
+ error = CurlError(curl_error, curl_message)
+ code = error.code
+ body = None
+ effective_url = None
+ else:
+ error = None
+ code = curl.getinfo(pycurl.HTTP_CODE)
+ body = info["buffer"].getvalue()
+ effective_url = curl.getinfo(pycurl.EFFECTIVE_URL)
+ info["buffer"].close()
+ info["callback"](HTTPResponse(
+ request=info["request"], code=code, headers=info["headers"],
+ body=body, effective_url=effective_url, error=error,
+ request_time=time.time() - info["start_time"]))
+
+
+class HTTPRequest(object):
+ def __init__(self, url, method="GET", headers={}, body=None,
+ auth_username=None, auth_password=None,
+ connect_timeout=None, request_timeout=None,
+ if_modified_since=None, follow_redirects=True,
+ max_redirects=5, user_agent=None, use_gzip=True,
+ network_interface=None):
+ if if_modified_since:
+ timestamp = calendar.timegm(if_modified_since.utctimetuple())
+ headers["If-Modified-Since"] = email.utils.formatdate(
+ timestamp, localtime=False, usegmt=True)
+ if "Pragma" not in headers:
+ headers["Pragma"] = ""
+ self.url = _utf8(url)
+ self.method = method
+ self.headers = headers
+ self.body = body
+ self.auth_username = _utf8(auth_username)
+ self.auth_password = _utf8(auth_password)
+ self.connect_timeout = connect_timeout or 20.0
+ self.request_timeout = request_timeout or 20.0
+ self.follow_redirects = follow_redirects
+ self.max_redirects = max_redirects
+ self.user_agent = user_agent
+ self.use_gzip = use_gzip
+ self.network_interface = network_interface
+
+
+class HTTPResponse(object):
+ def __init__(self, request, code, headers={}, body="", effective_url=None,
+ error=None, request_time=None):
+ self.request = request
+ self.code = code
+ self.headers = headers
+ self.body = body
+ if effective_url is None:
+ self.effective_url = request.url
+ else:
+ self.effective_url = effective_url
+ if error is None:
+ if self.code < 200 or self.code >= 300:
+ self.error = HTTPError(self.code)
+ else:
+ self.error = None
+ else:
+ self.error = error
+ self.request_time = request_time
+
+ def rethrow(self):
+ if self.error:
+ raise self.error
+
+ def __repr__(self):
+ args = ",".join("%s=%r" % i for i in self.__dict__.iteritems())
+ return "%s(%s)" % (self.__class__.__name__, args)
+
+
+class HTTPError(Exception):
+ def __init__(self, code, message=None):
+ self.code = code
+ Exception.__init__(self, "HTTP %d" % self.code)
+
+ def __repr__(self):
+ return "%s(%r)" % (self.__class__.__name__, self.code)
+
+ def __str__(self):
+ return "%d: %s" % (self.code, httplib.responses.get(
+ self.code, "Unknown"))
+
+
+class CurlError(HTTPError):
+ def __init__(self, errno, message):
+ HTTPError.__init__(self, 599, message)
+ self.errno = errno
+
+ def __repr__(self):
+ return "%s(%r,%r)" % (self.__class__.__name__, self.errno,
+ self.message)
+
+ def __str__(self):
+ return "%d: %s" % (self.errno, self.message)
+
+
+def _curl_create(max_simultaneous_connections=None):
+ curl = pycurl.Curl()
+ if logging.getLogger().isEnabledFor(logging.DEBUG):
+ curl.setopt(pycurl.VERBOSE, 1)
+ curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug)
+ curl.setopt(pycurl.MAXCONNECTS, max_simultaneous_connections or 5)
+ return curl
+
+
+def _curl_setup_request(curl, request, buffer, headers):
+ curl.setopt(pycurl.URL, request.url)
+ curl.setopt(pycurl.HTTPHEADER,
+ ["%s: %s" % i for i in request.headers.iteritems()])
+ try:
+ curl.setopt(pycurl.HEADERFUNCTION,
+ functools.partial(_curl_header_callback, headers))
+ except:
+ # Old version of curl; response will not include headers
+ pass
+ curl.setopt(pycurl.WRITEFUNCTION, buffer.write)
+ curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
+ curl.setopt(pycurl.MAXREDIRS, request.max_redirects)
+ curl.setopt(pycurl.CONNECTTIMEOUT, int(request.connect_timeout))
+ curl.setopt(pycurl.TIMEOUT, int(request.request_timeout))
+ if request.user_agent:
+ curl.setopt(pycurl.USERAGENT, request.user_agent)
+ else:
+ curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
+ if request.network_interface:
+ curl.setopt(pycurl.INTERFACE, request.network_interface)
+ if request.use_gzip:
+ curl.setopt(pycurl.ENCODING, "gzip,deflate")
+ else:
+ curl.setopt(pycurl.ENCODING, "none")
+
+ # Set the request method through curl's retarded interface which makes
+ # up names for every single method
+ curl_options = {
+ "GET": pycurl.HTTPGET,
+ "POST": pycurl.POST,
+ "PUT": pycurl.UPLOAD,
+ "HEAD": pycurl.NOBODY,
+ }
+ for o in curl_options.values():
+ curl.setopt(o, False)
+ curl.setopt(curl_options[request.method], True)
+
+ # Handle curl's cryptic options for every individual HTTP method
+ if request.method in ("POST", "PUT"):
+ request_buffer = cStringIO.StringIO(request.body)
+ curl.setopt(pycurl.READFUNCTION, request_buffer.read)
+ if request.method == "POST":
+ def ioctl(cmd):
+ if cmd == curl.IOCMD_RESTARTREAD:
+ request_buffer.seek(0)
+ curl.setopt(pycurl.IOCTLFUNCTION, ioctl)
+ curl.setopt(pycurl.POSTFIELDSIZE, len(request.body))
+ else:
+ curl.setopt(pycurl.INFILESIZE, len(request.body))
+
+ if request.auth_username and request.auth_password:
+ userpwd = "%s:%s" % (request.auth_username, request.auth_password)
+ curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
+ curl.setopt(pycurl.USERPWD, userpwd)
+ logging.info("%s %s (username: %r)", request.method, request.url,
+ request.auth_username)
+ else:
+ curl.unsetopt(pycurl.USERPWD)
+ logging.info("%s %s", request.method, request.url)
+
+
+def _curl_header_callback(headers, header_line):
+ if header_line.startswith("HTTP/"):
+ headers.clear()
+ return
+ if header_line == "\r\n":
+ return
+ parts = header_line.split(": ")
+ if len(parts) != 2:
+ logging.warning("Invalid HTTP response header line %r", header_line)
+ return
+ headers[parts[0].strip()] = parts[1].strip()
+
+
+def _curl_debug(debug_type, debug_msg):
+ debug_types = ('I', '<', '>', '<', '>')
+ if debug_type == 0:
+ logging.debug('%s', debug_msg.strip())
+ elif debug_type in (1, 2):
+ for line in debug_msg.splitlines():
+ logging.debug('%s %s', debug_types[debug_type], line)
+ elif debug_type == 4:
+ logging.debug('%s %r', debug_types[debug_type], debug_msg)
+
+
+def _utf8(value):
+ if value is None:
+ return value
+ if isinstance(value, unicode):
+ return value.encode("utf-8")
+ assert isinstance(value, str)
+ return value
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A non-blocking, single-threaded HTTP server."""
+
+import cgi
+import errno
+import fcntl
+import functools
+import ioloop
+import iostream
+import logging
+import socket
+import time
+import urlparse
+
+
+class HTTPServer(object):
+ """A non-blocking, single-threaded HTTP server.
+
+ A server is defined by a request callback that takes an HTTPRequest
+ instance as an argument and writes a valid HTTP response with
+ request.write(). request.finish() finishes the request (but does not
+ necessarily close the connection in the case of HTTP/1.1 keep-alive
+ requests). A simple example server that echoes back the URI you
+ requested:
+
+ import httpserver
+ import ioloop
+
+ def handle_request(request):
+ message = "You requested %s\n" % request.uri
+ request.write("HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n%s" % (
+ len(message), message))
+ request.finish()
+
+ http_server = httpserver.HTTPServer(handle_request)
+ http_server.listen(8888)
+ ioloop.IOLoop.instance().start()
+
+ HTTPServer is a very basic connection handler. Beyond parsing the
+ HTTP request body and headers, the only HTTP semantics implemented
+ in HTTPServer is HTTP/1.1 keep-alive connections. We do not, however,
+ implement chunked encoding, so the request callback must provide a
+ Content-Length header or implement chunked encoding for HTTP/1.1
+ requests for the server to run correctly for HTTP/1.1 clients. If
+ the request handler is unable to do this, you can provide the
+ no_keep_alive argument to the HTTPServer constructor, which will
+ ensure the connection is closed on every request no matter what HTTP
+ version the client is using.
+
+ If xheaders is True, we support the X-Real-Ip and X-Scheme headers,
+ which override the remote IP and HTTP scheme for all requests. These
+ headers are useful when running Tornado behind a reverse proxy or
+ load balancer.
+ """
+ def __init__(self, request_callback, no_keep_alive=False, io_loop=None,
+ xheaders=False):
+ self.request_callback = request_callback
+ self.no_keep_alive = no_keep_alive
+ self.io_loop = io_loop or ioloop.IOLoop.instance()
+ self.xheaders = xheaders
+ self._socket = None
+
+ def listen(self, port):
+ assert not self._socket
+ self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
+ flags = fcntl.fcntl(self._socket.fileno(), fcntl.F_GETFD)
+ flags |= fcntl.FD_CLOEXEC
+ fcntl.fcntl(self._socket.fileno(), fcntl.F_SETFD, flags)
+ self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ self._socket.setblocking(0)
+ self._socket.bind(("", port))
+ self._socket.listen(128)
+ self.io_loop.add_handler(self._socket.fileno(), self._handle_events,
+ self.io_loop.READ)
+
+ def _handle_events(self, fd, events):
+ while True:
+ try:
+ connection, address = self._socket.accept()
+ except socket.error, e:
+ if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+ return
+ raise
+ try:
+ stream = iostream.IOStream(connection, io_loop=self.io_loop)
+ HTTPConnection(stream, address, self.request_callback,
+ self.no_keep_alive, self.xheaders)
+ except:
+ logging.error("Error in connection callback", exc_info=True)
+
+
+class HTTPConnection(object):
+ """Handles a connection to an HTTP client, executing HTTP requests.
+
+ We parse HTTP headers and bodies, and execute the request callback
+ until the HTTP conection is closed.
+ """
+ def __init__(self, stream, address, request_callback, no_keep_alive=False,
+ xheaders=False):
+ self.stream = stream
+ self.address = address
+ self.request_callback = request_callback
+ self.no_keep_alive = no_keep_alive
+ self.xheaders = xheaders
+ self._request = None
+ self._request_finished = False
+ self.stream.read_until("\r\n\r\n", self._on_headers)
+
+ def write(self, chunk):
+ assert self._request, "Request closed"
+ self.stream.write(chunk, self._on_write_complete)
+
+ def finish(self):
+ assert self._request, "Request closed"
+ self._request_finished = True
+ if not self.stream.writing():
+ self._finish_request()
+
+ def _on_write_complete(self):
+ if self._request_finished:
+ self._finish_request()
+
+ def _finish_request(self):
+ if self.no_keep_alive:
+ disconnect = True
+ else:
+ connection_header = self._request.headers.get("Connection")
+ if self._request.supports_http_1_1():
+ disconnect = connection_header == "close"
+ elif ("Content-Length" in self._request.headers
+ or self._request.method in ("HEAD", "GET")):
+ disconnect = connection_header != "Keep-Alive"
+ else:
+ disconnect = True
+ self._request = None
+ self._request_finished = False
+ if disconnect:
+ self.stream.close()
+ return
+ self.stream.read_until("\r\n\r\n", self._on_headers)
+
+ def _on_headers(self, data):
+ eol = data.find("\r\n")
+ start_line = data[:eol]
+ method, uri, version = start_line.split(" ")
+ if not version.startswith("HTTP/"):
+ raise Exception("Malformed HTTP version in HTTP Request-Line")
+ headers = HTTPHeaders.parse(data[eol:])
+ self._request = HTTPRequest(
+ connection=self, method=method, uri=uri, version=version,
+ headers=headers, remote_ip=self.address[0])
+
+ content_length = headers.get("Content-Length")
+ if content_length:
+ content_length = int(content_length)
+ if content_length > self.stream.max_buffer_size:
+ raise Exception("Content-Length too long")
+ if headers.get("Expect") == "100-continue":
+ self.stream.write("HTTP/1.1 100 (Continue)\r\n\r\n")
+ self.stream.read_bytes(content_length, self._on_request_body)
+ return
+
+ self.request_callback(self._request)
+
+ def _on_request_body(self, data):
+ self._request.body = data
+ content_type = self._request.headers.get("Content-Type", "")
+ if self._request.method == "POST":
+ if content_type.startswith("application/x-www-form-urlencoded"):
+ arguments = cgi.parse_qs(self._request.body)
+ for name, values in arguments.iteritems():
+ values = [v for v in values if v]
+ if values:
+ self._request.arguments.setdefault(name, []).extend(
+ values)
+ elif content_type.startswith("multipart/form-data"):
+ boundary = content_type[30:]
+ if boundary: self._parse_mime_body(boundary, data)
+ self.request_callback(self._request)
+
+ def _parse_mime_body(self, boundary, data):
+ if data.endswith("\r\n"):
+ footer_length = len(boundary) + 6
+ else:
+ footer_length = len(boundary) + 4
+ parts = data[:-footer_length].split("--" + boundary + "\r\n")
+ for part in parts:
+ if not part: continue
+ eoh = part.find("\r\n\r\n")
+ if eoh == -1:
+ logging.warning("multipart/form-data missing headers")
+ continue
+ headers = HTTPHeaders.parse(part[:eoh])
+ name_header = headers.get("Content-Disposition", "")
+ if not name_header.startswith("form-data;") or \
+ not part.endswith("\r\n"):
+ logging.warning("Invalid multipart/form-data")
+ continue
+ value = part[eoh + 4:-2]
+ name_values = {}
+ for name_part in name_header[10:].split(";"):
+ name, name_value = name_part.strip().split("=", 1)
+ name_values[name] = name_value.strip('"').decode("utf-8")
+ if not name_values.get("name"):
+ logging.warning("multipart/form-data value missing name")
+ continue
+ name = name_values["name"]
+ if name_values.get("filename"):
+ ctype = headers.get("Content-Type", "application/unknown")
+ self._request.files.setdefault(name, []).append(dict(
+ filename=name_values["filename"], body=value,
+ content_type=ctype))
+ else:
+ self._request.arguments.setdefault(name, []).append(value)
+
+
+class HTTPRequest(object):
+ """A single HTTP request.
+
+ GET/POST arguments are available in the arguments property, which
+ maps arguments names to lists of values (to support multiple values
+ for individual names). Names and values are both unicode always.
+
+ File uploads are available in the files property, which maps file
+ names to list of files. Each file is a dictionary of the form
+ {"filename":..., "content_type":..., "body":...}. The content_type
+ comes from the provided HTTP header and should not be trusted
+ outright given that it can be easily forged.
+
+ An HTTP request is attached to a single HTTP connection, which can
+ be accessed through the "connection" attribute. Since connections
+ are typically kept open in HTTP/1.1, multiple requests can be handled
+ sequentially on a single connection.
+ """
+ def __init__(self, method, uri, version="HTTP/1.0", headers=None,
+ body=None, remote_ip=None, protocol=None, host=None,
+ files=None, connection=None):
+ self.method = method
+ self.uri = uri
+ self.version = version
+ self.headers = headers or HTTPHeaders()
+ self.body = body or ""
+ if connection and connection.xheaders:
+ self.remote_ip = headers.get("X-Real-Ip", remote_ip)
+ self.protocol = headers.get("X-Scheme", protocol) or "http"
+ else:
+ self.remote_ip = remote_ip
+ self.protocol = protocol or "http"
+ self.host = host or headers.get("Host") or "127.0.0.1"
+ self.files = files or {}
+ self.connection = connection
+ self._start_time = time.time()
+ self._finish_time = None
+
+ scheme, netloc, path, query, fragment = urlparse.urlsplit(uri)
+ self.path = path
+ self.query = query
+ arguments = cgi.parse_qs(query)
+ self.arguments = {}
+ for name, values in arguments.iteritems():
+ values = [v for v in values if v]
+ if values: self.arguments[name] = values
+
+ def supports_http_1_1(self):
+ """Returns True if this request supports HTTP/1.1 semantics"""
+ return self.version == "HTTP/1.1"
+
+ def write(self, chunk):
+ """Writes the given chunk to the response stream."""
+ assert isinstance(chunk, str)
+ self.connection.write(chunk)
+
+ def finish(self):
+ """Finishes this HTTP request on the open connection."""
+ self.connection.finish()
+ self._finish_time = time.time()
+
+ def full_url(self):
+ """Reconstructs the full URL for this request."""
+ return self.protocol + "://" + self.host + self.uri
+
+ def request_time(self):
+ """Returns the amount of time it took for this request to execute."""
+ if self._finish_time is None:
+ return time.time() - self._start_time
+ else:
+ return self._finish_time - self._start_time
+
+ def __repr__(self):
+ attrs = ("protocol", "host", "method", "uri", "version", "remote_ip",
+ "remote_ip", "body")
+ args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs])
+ return "%s(%s, headers=%s)" % (
+ self.__class__.__name__, args, dict(self.headers))
+
+
+class HTTPHeaders(dict):
+ """A dictionary that maintains Http-Header-Case for all keys."""
+ def __setitem__(self, name, value):
+ dict.__setitem__(self, self._normalize_name(name), value)
+
+ def __getitem__(self, name):
+ return dict.__getitem__(self, self._normalize_name(name))
+
+ def _normalize_name(self, name):
+ return intern("-".join([w.capitalize() for w in name.split("-")]))
+
+ @classmethod
+ def parse(cls, headers_string):
+ headers = cls()
+ for line in headers_string.splitlines():
+ if line:
+ name, value = line.split(": ", 1)
+ headers[name] = value
+ return headers
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A level-triggered I/O loop for non-blocking sockets."""
+
+import bisect
+import errno
+import fcntl
+import logging
+import os
+import select
+import time
+
+
+class IOLoop(object):
+ """A level-triggered I/O loop.
+
+ We use epoll if it is available, or else we fall back on select(). If
+ you are implementing a system that needs to handle 1000s of simultaneous
+ connections, you should use Linux and either compile our epoll module or
+ use Python 2.6+ to get epoll support.
+
+ Example usage for a simple TCP server:
+
+ import errno
+ import functools
+ import ioloop
+ import socket
+
+ def connection_ready(sock, fd, events):
+ while True:
+ try:
+ connection, address = sock.accept()
+ except socket.error, e:
+ if e[0] not in (errno.EWOULDBLOCK, errno.EAGAIN):
+ raise
+ return
+ connection.setblocking(0)
+ handle_connection(connection, address)
+
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ sock.setblocking(0)
+ sock.bind(("", port))
+ sock.listen(128)
+
+ io_loop = ioloop.IOLoop.instance()
+ callback = functools.partial(connection_ready, sock)
+ io_loop.add_handler(sock.fileno(), callback, io_loop.READ)
+ io_loop.start()
+
+ """
+ # Constants from the epoll module
+ _EPOLLIN = 0x001
+ _EPOLLPRI = 0x002
+ _EPOLLOUT = 0x004
+ _EPOLLERR = 0x008
+ _EPOLLHUP = 0x010
+ _EPOLLRDHUP = 0x2000
+ _EPOLLONESHOT = (1 << 30)
+ _EPOLLET = (1 << 31)
+
+ # Our events map exactly to the epoll events
+ NONE = 0
+ READ = _EPOLLIN
+ WRITE = _EPOLLOUT
+ ERROR = _EPOLLERR | _EPOLLHUP | _EPOLLRDHUP
+
+ def __init__(self, impl=None):
+ self._impl = impl or _poll()
+ self._handlers = {}
+ self._events = {}
+ self._callbacks = set()
+ self._timeouts = []
+ self._running = False
+
+ # Create a pipe that we send bogus data to when we want to wake
+ # the I/O loop when it is idle
+ r, w = os.pipe()
+ self._set_nonblocking(r)
+ self._set_nonblocking(w)
+ self._waker_reader = os.fdopen(r, "r", 0)
+ self._waker_writer = os.fdopen(w, "w", 0)
+ self.add_handler(r, self._read_waker, self.WRITE)
+
+ @classmethod
+ def instance(cls):
+ """Returns a global IOLoop instance.
+
+ Most single-threaded applications have a single, global IOLoop.
+ Use this method instead of passing around IOLoop instances
+ throughout your code.
+
+ A common pattern for classes that depend on IOLoops is to use
+ a default argument to enable programs with multiple IOLoops
+ but not require the argument for simpler applications:
+
+ class MyClass(object):
+ def __init__(self, io_loop=None):
+ self.io_loop = io_loop or IOLoop.instance()
+ """
+ if not hasattr(cls, "_instance"):
+ cls._instance = cls()
+ return cls._instance
+
+ def add_handler(self, fd, handler, events):
+ """Registers the given handler to receive the given events for fd."""
+ self._handlers[fd] = handler
+ self._impl.register(fd, events | self.ERROR)
+
+ def update_handler(self, fd, events):
+ """Changes the events we listen for fd."""
+ self._impl.modify(fd, events | self.ERROR)
+
+ def remove_handler(self, fd):
+ """Stop listening for events on fd."""
+ self._handlers.pop(fd, None)
+ self._events.pop(fd, None)
+ try:
+ self._impl.unregister(fd)
+ except OSError:
+ logging.debug("Error deleting fd from IOLoop", exc_info=True)
+
+ def start(self):
+ """Starts the I/O loop.
+
+ The loop will run until one of the I/O handlers calls stop(), which
+ will make the loop stop after the current event iteration completes.
+ """
+ self._running = True
+ while True:
+ # Never use an infinite timeout here - it can stall epoll
+ poll_timeout = 0.2
+
+ # Prevent IO event starvation by delaying new callbacks
+ # to the next iteration of the event loop.
+ callbacks = list(self._callbacks)
+ for callback in callbacks:
+ # A callback can add or remove other callbacks
+ if callback in self._callbacks:
+ self._callbacks.remove(callback)
+ self._run_callback(callback)
+
+ if self._callbacks:
+ poll_timeout = 0.0
+
+ if self._timeouts:
+ now = time.time()
+ while self._timeouts and self._timeouts[0].deadline <= now:
+ timeout = self._timeouts.pop(0)
+ self._run_callback(timeout.callback)
+ if self._timeouts:
+ milliseconds = self._timeouts[0].deadline - now
+ poll_timeout = min(milliseconds, poll_timeout)
+
+ if not self._running:
+ break
+
+ try:
+ event_pairs = self._impl.poll(poll_timeout)
+ except Exception, e:
+ if e.args == (4, "Interrupted system call"):
+ logging.warning("Interrupted system call", exc_info=1)
+ continue
+ else:
+ raise
+
+ # Pop one fd at a time from the set of pending fds and run
+ # its handler. Since that handler may perform actions on
+ # other file descriptors, there may be reentrant calls to
+ # this IOLoop that update self._events
+ self._events.update(event_pairs)
+ while self._events:
+ fd, events = self._events.popitem()
+ try:
+ self._handlers[fd](fd, events)
+ except KeyboardInterrupt:
+ raise
+ except OSError, e:
+ if e[0] == errno.EPIPE:
+ # Happens when the client closes the connection
+ pass
+ else:
+ logging.error("Exception in I/O handler for fd %d",
+ fd, exc_info=True)
+ except:
+ logging.error("Exception in I/O handler for fd %d",
+ fd, exc_info=True)
+
+ def stop(self):
+ """Stop the loop after the current event loop iteration is complete."""
+ self._running = False
+ self._wake()
+
+ def running(self):
+ """Returns true if this IOLoop is currently running."""
+ return self._running
+
+ def add_timeout(self, deadline, callback):
+ """Calls the given callback at the time deadline from the I/O loop."""
+ timeout = _Timeout(deadline, callback)
+ bisect.insort(self._timeouts, timeout)
+ return timeout
+
+ def remove_timeout(self, timeout):
+ self._timeouts.remove(timeout)
+
+ def add_callback(self, callback):
+ """Calls the given callback on the next I/O loop iteration."""
+ self._callbacks.add(callback)
+ self._wake()
+
+ def remove_callback(self, callback):
+ """Removes the given callback from the next I/O loop iteration."""
+ self._callbacks.pop(callback)
+
+ def _wake(self):
+ try:
+ self._waker_writer.write("x")
+ except IOError:
+ pass
+
+ def _run_callback(self, callback):
+ try:
+ callback()
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except:
+ logging.error("Exception in callback %r", callback, exc_info=True)
+
+ def _read_waker(self, fd, events):
+ try:
+ while True:
+ self._waker_reader.read()
+ except IOError:
+ pass
+
+ def _set_nonblocking(self, fd):
+ flags = fcntl.fcntl(fd, fcntl.F_GETFL)
+ fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
+
+
+class _Timeout(object):
+ """An IOLoop timeout, a UNIX timestamp and a callback"""
+ def __init__(self, deadline, callback):
+ self.deadline = deadline
+ self.callback = callback
+
+ def __cmp__(self, other):
+ return cmp((self.deadline, id(self.callback)),
+ (other.deadline, id(other.callback)))
+
+
+class _EPoll(object):
+ """An epoll-based event loop using our C module for Python 2.5 systems"""
+ _EPOLL_CTL_ADD = 1
+ _EPOLL_CTL_DEL = 2
+ _EPOLL_CTL_MOD = 3
+
+ def __init__(self):
+ self._epoll_fd = epoll.epoll_create()
+
+ def register(self, fd, events):
+ epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_ADD, fd, events)
+
+ def modify(self, fd, events):
+ epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_MOD, fd, events)
+
+ def unregister(self, fd):
+ epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_DEL, fd, 0)
+
+ def poll(self, timeout):
+ return epoll.epoll_wait(self._epoll_fd, int(timeout * 1000))
+
+
+class _Select(object):
+ """A simple, select()-based IOLoop implementation for non-Linux systems"""
+ def __init__(self):
+ self.read_fds = set()
+ self.write_fds = set()
+ self.error_fds = set()
+ self.fd_sets = (self.read_fds, self.write_fds, self.error_fds)
+
+ def register(self, fd, events):
+ if events & IOLoop.READ: self.read_fds.add(fd)
+ if events & IOLoop.WRITE: self.write_fds.add(fd)
+ if events & IOLoop.ERROR: self.error_fds.add(fd)
+
+ def modify(self, fd, events):
+ self.unregister(fd)
+ self.register(fd, events)
+
+ def unregister(self, fd):
+ self.read_fds.discard(fd)
+ self.write_fds.discard(fd)
+ self.error_fds.discard(fd)
+
+ def poll(self, timeout):
+ readable, writeable, errors = select.select(
+ self.read_fds, self.write_fds, self.error_fds, timeout)
+ events = {}
+ for fd in readable:
+ events[fd] = events.get(fd, 0) | IOLoop.READ
+ for fd in writeable:
+ events[fd] = events.get(fd, 0) | IOLoop.WRITE
+ for fd in errors:
+ events[fd] = events.get(fd, 0) | IOLoop.ERROR
+ return events.items()
+
+
+# Choose a poll implementation. Use epoll if it is available, fall back to
+# select() for non-Linux platforms
+if hasattr(select, "epoll"):
+ # Python 2.6+ on Linux
+ _poll = select.epoll
+else:
+ try:
+ # Linux systems with our C module installed
+ import epoll
+ _poll = _EPoll
+ except:
+ # All other systems
+ import sys
+ if "linux" in sys.platform:
+ logging.warning("epoll module not found; using select()")
+ _poll = _Select
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A utility class to write to and read from a non-blocking socket."""
+
+import errno
+import ioloop
+import logging
+import socket
+
+
+class IOStream(object):
+ """A utility class to write to and read from a non-blocking socket.
+
+ We support three methods: write(), read_until(), and read_bytes().
+ All of the methods take callbacks (since writing and reading are
+ non-blocking and asynchronous). read_until() reads the socket until
+ a given delimiter, and read_bytes() reads until a specified number
+ of bytes have been read from the socket.
+
+ A very simple (and broken) HTTP client using this class:
+
+ import ioloop
+ import iostream
+ import socket
+
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
+ s.connect(("friendfeed.com", 80))
+ stream = IOStream(s)
+
+ def on_headers(data):
+ headers = {}
+ for line in data.split("\r\n"):
+ parts = line.split(":")
+ if len(parts) == 2:
+ headers[parts[0].strip()] = parts[1].strip()
+ stream.read_bytes(int(headers["Content-Length"]), on_body)
+
+ def on_body(data):
+ print data
+ stream.close()
+ ioloop.IOLoop.instance().stop()
+
+ stream.write("GET / HTTP/1.0\r\n\r\n")
+ stream.read_until("\r\n\r\n", on_headers)
+ ioloop.IOLoop.instance().start()
+
+ """
+ def __init__(self, socket, io_loop=None, max_buffer_size=104857600,
+ read_chunk_size=4096):
+ self.socket = socket
+ self.socket.setblocking(False)
+ self.io_loop = io_loop or ioloop.IOLoop.instance()
+ self.max_buffer_size = max_buffer_size
+ self.read_chunk_size = read_chunk_size
+ self._read_buffer = ""
+ self._write_buffer = ""
+ self._read_delimiter = None
+ self._read_bytes = None
+ self._read_callback = None
+ self._write_callback = None
+ self._close_callback = None
+ self._state = self.io_loop.ERROR
+ self.io_loop.add_handler(
+ self.socket.fileno(), self._handle_events, self._state)
+
+ def read_until(self, delimiter, callback):
+ """Call callback when we read the given delimiter."""
+ assert not self._read_callback, "Already reading"
+ loc = self._read_buffer.find(delimiter)
+ if loc != -1:
+ callback(self._consume(loc + len(delimiter)))
+ return
+ self._check_closed()
+ self._read_delimiter = delimiter
+ self._read_callback = callback
+ self._add_io_state(self.io_loop.READ)
+
+ def read_bytes(self, num_bytes, callback):
+ """Call callback when we read the given number of bytes."""
+ assert not self._read_callback, "Already reading"
+ if len(self._read_buffer) >= num_bytes:
+ callback(self._consume(num_bytes))
+ return
+ self._check_closed()
+ self._read_bytes = num_bytes
+ self._read_callback = callback
+ self._add_io_state(self.io_loop.READ)
+
+ def write(self, data, callback=None):
+ """Write the given data to this stream.
+
+ If callback is given, we call it when all of the buffered write
+ data has been successfully written to the stream. If there was
+ previously buffered write data and an old write callback, that
+ callback is simply overwritten with this new callback.
+ """
+ self._check_closed()
+ self._write_buffer += data
+ self._add_io_state(self.io_loop.WRITE)
+ self._write_callback = callback
+
+ def set_close_callback(self, callback):
+ """Call the given callback when the stream is closed."""
+ self._close_callback = callback
+
+ def close(self):
+ """Close this stream."""
+ if self.socket is not None:
+ self.io_loop.remove_handler(self.socket.fileno())
+ self.socket.close()
+ self.socket = None
+ if self._close_callback: self._close_callback()
+
+ def reading(self):
+ """Returns true if we are currently reading from the stream."""
+ return self._read_callback is not None
+
+ def writing(self):
+ """Returns true if we are currently writing to the stream."""
+ return len(self._write_buffer) > 0
+
+ def closed(self):
+ return self.socket is None
+
+ def _handle_events(self, fd, events):
+ if not self.socket:
+ logging.warning("Got events for closed stream %d", fd)
+ return
+ if events & self.io_loop.READ:
+ self._handle_read()
+ if not self.socket:
+ return
+ if events & self.io_loop.WRITE:
+ self._handle_write()
+ if not self.socket:
+ return
+ if events & self.io_loop.ERROR:
+ self.close()
+ return
+ state = self.io_loop.ERROR
+ if self._read_delimiter or self._read_bytes:
+ state |= self.io_loop.READ
+ if self._write_buffer:
+ state |= self.io_loop.WRITE
+ if state != self._state:
+ self._state = state
+ self.io_loop.update_handler(self.socket.fileno(), self._state)
+
+ def _handle_read(self):
+ try:
+ chunk = self.socket.recv(self.read_chunk_size)
+ except socket.error, e:
+ if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+ return
+ else:
+ logging.warning("Read error on %d: %s",
+ self.socket.fileno(), e)
+ self.close()
+ return
+ if not chunk:
+ self.close()
+ return
+ self._read_buffer += chunk
+ if len(self._read_buffer) >= self.max_buffer_size:
+ logging.error("Reached maximum read buffer size")
+ self.close()
+ return
+ if self._read_bytes:
+ if len(self._read_buffer) >= self._read_bytes:
+ num_bytes = self._read_bytes
+ callback = self._read_callback
+ self._read_callback = None
+ self._read_bytes = None
+ callback(self._consume(num_bytes))
+ elif self._read_delimiter:
+ loc = self._read_buffer.find(self._read_delimiter)
+ if loc != -1:
+ callback = self._read_callback
+ delimiter_len = len(self._read_delimiter)
+ self._read_callback = None
+ self._read_delimiter = None
+ callback(self._consume(loc + delimiter_len))
+
+ def _handle_write(self):
+ while self._write_buffer:
+ try:
+ num_bytes = self.socket.send(self._write_buffer)
+ self._write_buffer = self._write_buffer[num_bytes:]
+ except socket.error, e:
+ if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+ break
+ else:
+ logging.warning("Read error on %d: %s",
+ self.socket.fileno(), e)
+ self.close()
+ return
+ if not self._write_buffer and self._write_callback:
+ callback = self._write_callback
+ self._write_callback = None
+ callback()
+
+ def _consume(self, loc):
+ result = self._read_buffer[:loc]
+ self._read_buffer = self._read_buffer[loc:]
+ return result
+
+ def _check_closed(self):
+ if not self.socket:
+ raise IOError("Stream is closed")
+
+ def _add_io_state(self, state):
+ if not self._state & state:
+ self._state = self._state | state
+ self.io_loop.update_handler(self.socket.fileno(), self._state)
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Translation methods for generating localized strings.
+
+To load a locale and generate a translated string:
+
+ user_locale = locale.get("es_LA")
+ print user_locale.translate("Sign out")
+
+locale.get() returns the closest matching locale, not necessarily the
+specific locale you requested. You can support pluralization with
+additional arguments to translate(), e.g.:
+
+ people = [...]
+ message = user_locale.translate(
+ "%(list)s is online", "%(list)s are online", len(people))
+ print message % {"list": user_locale.list(people)}
+
+The first string is chosen if len(people) == 1, otherwise the second
+string is chosen.
+"""
+
+import csv
+import datetime
+import logging
+import os
+import os.path
+import re
+
+_default_locale = "en_US"
+_translations = {}
+_supported_locales = frozenset([_default_locale])
+
+
+def get(*locale_codes):
+ """Returns the closest match for the given locale codes.
+
+ We iterate over all given locale codes in order. If we have a tight
+ or a loose match for the code (e.g., "en" for "en_US"), we return
+ the locale. Otherwise we move to the next code in the list.
+
+ By default we return en_US if no translations are found for any of
+ the specified locales. You can change the default locale with
+ set_default_locale() below.
+ """
+ return Locale.get_closest(*locale_codes)
+
+
+def set_default_locale(code):
+ """Sets the default locale, used in get_closest_locale().
+
+ The default locale is assumed to be the language used for all strings
+ in the system. The translations loaded from disk are mappings from
+ the default locale to the destination locale. Consequently, you don't
+ need to create a translation file for the default locale.
+ """
+ global _default_locale
+ global _supported_locales
+ _default_locale = code
+ _supported_locales = frozenset(_translations.keys() + [_default_locale])
+
+
+def load_translations(directory):
+ """Loads translations from CSV files in a directory.
+
+ Translations are strings with optional Python-style named placeholders
+ (e.g., "My name is %(name)s") and their associated translations.
+
+ The directory should have translation files of the form LOCALE.csv,
+ e.g. es_GT.csv. The CSV files should have two or three columns: string,
+ translation, and an optional plural indicator. Plural indicators should
+ be one of "plural" or "singular". A given string can have both singular
+ and plural forms. For example "%(name)s liked this" may have a
+ different verb conjugation depending on whether %(name)s is one
+ name or a list of names. There should be two rows in the CSV file for
+ that string, one with plural indicator "singular", and one "plural".
+ For strings with no verbs that would change on translation, simply
+ use "unknown" or the empty string (or don't include the column at all).
+
+ Example translation es_LA.csv:
+
+ "I love you","Te amo"
+ "%(name)s liked this","A %(name)s les gust\xf3 esto","plural"
+ "%(name)s liked this","A %(name)s le gust\xf3 esto","singular"
+
+ """
+ global _translations
+ global _supported_locales
+ _translations = {}
+ for path in os.listdir(directory):
+ if not path.endswith(".csv"): continue
+ locale, extension = path.split(".")
+ if locale not in LOCALE_NAMES:
+ logging.error("Unrecognized locale %r (path: %s)", locale,
+ os.path.join(directory, path))
+ continue
+ f = open(os.path.join(directory, path), "r")
+ _translations[locale] = {}
+ for i, row in enumerate(csv.reader(f)):
+ if not row or len(row) < 2: continue
+ row = [c.decode("utf-8").strip() for c in row]
+ english, translation = row[:2]
+ if len(row) > 2:
+ plural = row[2] or "unknown"
+ else:
+ plural = "unknown"
+ if plural not in ("plural", "singular", "unknown"):
+ logging.error("Unrecognized plural indicator %r in %s line %d",
+ plural, path, i + 1)
+ continue
+ _translations[locale].setdefault(plural, {})[english] = translation
+ f.close()
+ _supported_locales = frozenset(_translations.keys() + [_default_locale])
+ logging.info("Supported locales: %s", sorted(_supported_locales))
+
+
+def get_supported_locales(cls):
+ """Returns a list of all the supported locale codes."""
+ return _supported_locales
+
+
+class Locale(object):
+ @classmethod
+ def get_closest(cls, *locale_codes):
+ """Returns the closest match for the given locale code."""
+ for code in locale_codes:
+ if not code: continue
+ code = code.replace("-", "_")
+ parts = code.split("_")
+ if len(parts) > 2:
+ continue
+ elif len(parts) == 2:
+ code = parts[0].lower() + "_" + parts[1].upper()
+ if code in _supported_locales:
+ return cls.get(code)
+ if parts[0].lower() in _supported_locales:
+ return cls.get(parts[0].lower())
+ return cls.get(_default_locale)
+
+ @classmethod
+ def get(cls, code):
+ """Returns the Locale for the given locale code.
+
+ If it is not supported, we raise an exception.
+ """
+ if not hasattr(cls, "_cache"):
+ cls._cache = {}
+ if code not in cls._cache:
+ assert code in _supported_locales
+ translations = _translations.get(code, {})
+ cls._cache[code] = Locale(code, translations)
+ return cls._cache[code]
+
+ def __init__(self, code, translations):
+ self.code = code
+ self.name = LOCALE_NAMES.get(code, {}).get("name", u"Unknown")
+ self.rtl = False
+ for prefix in ["fa", "ar", "he"]:
+ if self.code.startswith(prefix):
+ self.rtl = True
+ break
+ self.translations = translations
+
+ # Initialize strings for date formatting
+ _ = self.translate
+ self._months = [
+ _("January"), _("February"), _("March"), _("April"),
+ _("May"), _("June"), _("July"), _("August"),
+ _("September"), _("October"), _("November"), _("December")]
+ self._weekdays = [
+ _("Monday"), _("Tuesday"), _("Wednesday"), _("Thursday"),
+ _("Friday"), _("Saturday"), _("Sunday")]
+
+ def translate(self, message, plural_message=None, count=None):
+ """Returns the translation for the given message for this locale.
+
+ If plural_message is given, you must also provide count. We return
+ plural_message when count != 1, and we return the singular form
+ for the given message when count == 1.
+ """
+ if plural_message is not None:
+ assert count is not None
+ if count != 1:
+ message = plural_message
+ message_dict = self.translations.get("plural", {})
+ else:
+ message_dict = self.translations.get("singular", {})
+ else:
+ message_dict = self.translations.get("unknown", {})
+ return message_dict.get(message, message)
+
+ def format_date(self, date, gmt_offset=0, relative=True, shorter=False,
+ full_format=False):
+ """Formats the given date (which should be GMT).
+
+ By default, we return a relative time (e.g., "2 minutes ago"). You
+ can return an absolute date string with relative=False.
+
+ You can force a full format date ("July 10, 1980") with
+ full_format=True.
+ """
+ if self.code.startswith("ru"):
+ relative = False
+ if type(date) in (int, long, float):
+ date = datetime.datetime.utcfromtimestamp(date)
+ now = datetime.datetime.utcnow()
+ # Round down to now. Due to click skew, things are somethings
+ # slightly in the future.
+ if date > now: date = now
+ local_date = date - datetime.timedelta(minutes=gmt_offset)
+ local_now = now - datetime.timedelta(minutes=gmt_offset)
+ local_yesterday = local_now - datetime.timedelta(hours=24)
+ difference = now - date
+ seconds = difference.seconds
+ days = difference.days
+
+ _ = self.translate
+ format = None
+ if not full_format:
+ if relative and days == 0:
+ if seconds < 50:
+ return _("1 second ago", "%(seconds)d seconds ago",
+ seconds) % { "seconds": seconds }
+
+ if seconds < 50 * 60:
+ minutes = round(seconds / 60.0)
+ return _("1 minute ago", "%(minutes)d minutes ago",
+ minutes) % { "minutes": minutes }
+
+ hours = round(seconds / (60.0 * 60))
+ return _("1 hour ago", "%(hours)d hours ago",
+ hours) % { "hours": hours }
+
+ if days == 0:
+ format = _("%(time)s")
+ elif days == 1 and local_date.day == local_yesterday.day and \
+ relative:
+ format = _("yesterday") if shorter else \
+ _("yesterday at %(time)s")
+ elif days < 5:
+ format = _("%(weekday)s") if shorter else \
+ _("%(weekday)s at %(time)s")
+ elif days < 334: # 11mo, since confusing for same month last year
+ format = _("%(month_name)s %(day)s") if shorter else \
+ _("%(month_name)s %(day)s at %(time)s")
+
+ if format is None:
+ format = _("%(month_name)s %(day)s, %(year)s") if shorter else \
+ _("%(month_name)s %(day)s, %(year)s at %(time)s")
+
+ tfhour_clock = self.code not in ("en", "en_US", "zh_CN")
+ if tfhour_clock:
+ str_time = "%d:%02d" % (local_date.hour, local_date.minute)
+ elif self.code == "zh_CN":
+ str_time = "%s%d:%02d" % (
+ (u'\u4e0a\u5348', u'\u4e0b\u5348')[local_date.hour >= 12],
+ local_date.hour % 12 or 12, local_date.minute)
+ else:
+ str_time = "%d:%02d %s" % (
+ local_date.hour % 12 or 12, local_date.minute,
+ ("am", "pm")[local_date.hour >= 12])
+
+ return format % {
+ "month_name": self._months[local_date.month - 1],
+ "weekday": self._weekdays[local_date.weekday()],
+ "day": str(local_date.day),
+ "year": str(local_date.year),
+ "time": str_time
+ }
+
+ def format_day(self, date, gmt_offset=0, dow=True):
+ """Formats the given date as a day of week.
+
+ Example: "Monday, January 22". You can remove the day of week with
+ dow=False.
+ """
+ local_date = date - datetime.timedelta(minutes=gmt_offset)
+ _ = self.translate
+ if dow:
+ return _("%(weekday)s, %(month_name)s %(day)s") % {
+ "month_name": self._months[local_date.month - 1],
+ "weekday": self.WEEKDAYS[local_date.weekday()],
+ "day": str(local_date.day),
+ }
+ else:
+ return _("%(month_name)s %(day)s") % {
+ "month_name": self._months[local_date.month - 1],
+ "day": str(local_date.day),
+ }
+
+ def list(self, parts):
+ """Returns a comma-separated list for the given list of parts.
+
+ The format is, e.g., "A, B and C", "A and B" or just "A" for lists
+ of size 1.
+ """
+ _ = self.translate
+ if len(parts) == 0: return ""
+ if len(parts) == 1: return parts[0]
+ comma = u' \u0648 ' if self.code.startswith("fa") else u", "
+ return _("%(commas)s and %(last)s") % {
+ "commas": comma.join(parts[:-1]),
+ "last": parts[len(parts) - 1],
+ }
+
+ def friendly_number(self, value):
+ """Returns a comma-separated number for the given integer."""
+ if self.code not in ("en", "en_US"):
+ return str(value)
+ value = str(value)
+ parts = []
+ while value:
+ parts.append(value[-3:])
+ value = value[:-3]
+ return ",".join(reversed(parts))
+
+
+LOCALE_NAMES = {
+ "af_ZA": {"name_en": u"Afrikaans", "name": u"Afrikaans"},
+ "ar_AR": {"name_en": u"Arabic", "name": u"\u0627\u0644\u0639\u0631\u0628\u064a\u0629"},
+ "bg_BG": {"name_en": u"Bulgarian", "name": u"\u0411\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438"},
+ "bn_IN": {"name_en": u"Bengali", "name": u"\u09ac\u09be\u0982\u09b2\u09be"},
+ "bs_BA": {"name_en": u"Bosnian", "name": u"Bosanski"},
+ "ca_ES": {"name_en": u"Catalan", "name": u"Catal\xe0"},
+ "cs_CZ": {"name_en": u"Czech", "name": u"\u010ce\u0161tina"},
+ "cy_GB": {"name_en": u"Welsh", "name": u"Cymraeg"},
+ "da_DK": {"name_en": u"Danish", "name": u"Dansk"},
+ "de_DE": {"name_en": u"German", "name": u"Deutsch"},
+ "el_GR": {"name_en": u"Greek", "name": u"\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"},
+ "en_GB": {"name_en": u"English (UK)", "name": u"English (UK)"},
+ "en_US": {"name_en": u"English (US)", "name": u"English (US)"},
+ "es_ES": {"name_en": u"Spanish (Spain)", "name": u"Espa\xf1ol (Espa\xf1a)"},
+ "es_LA": {"name_en": u"Spanish", "name": u"Espa\xf1ol"},
+ "et_EE": {"name_en": u"Estonian", "name": u"Eesti"},
+ "eu_ES": {"name_en": u"Basque", "name": u"Euskara"},
+ "fa_IR": {"name_en": u"Persian", "name": u"\u0641\u0627\u0631\u0633\u06cc"},
+ "fi_FI": {"name_en": u"Finnish", "name": u"Suomi"},
+ "fr_CA": {"name_en": u"French (Canada)", "name": u"Fran\xe7ais (Canada)"},
+ "fr_FR": {"name_en": u"French", "name": u"Fran\xe7ais"},
+ "ga_IE": {"name_en": u"Irish", "name": u"Gaeilge"},
+ "gl_ES": {"name_en": u"Galician", "name": u"Galego"},
+ "he_IL": {"name_en": u"Hebrew", "name": u"\u05e2\u05d1\u05e8\u05d9\u05ea"},
+ "hi_IN": {"name_en": u"Hindi", "name": u"\u0939\u093f\u0928\u094d\u0926\u0940"},
+ "hr_HR": {"name_en": u"Croatian", "name": u"Hrvatski"},
+ "hu_HU": {"name_en": u"Hungarian", "name": u"Magyar"},
+ "id_ID": {"name_en": u"Indonesian", "name": u"Bahasa Indonesia"},
+ "is_IS": {"name_en": u"Icelandic", "name": u"\xcdslenska"},
+ "it_IT": {"name_en": u"Italian", "name": u"Italiano"},
+ "ja_JP": {"name_en": u"Japanese", "name": u"\xe6\xe6\xe8"},
+ "ko_KR": {"name_en": u"Korean", "name": u"\xed\xea\xec"},
+ "lt_LT": {"name_en": u"Lithuanian", "name": u"Lietuvi\u0173"},
+ "lv_LV": {"name_en": u"Latvian", "name": u"Latvie\u0161u"},
+ "mk_MK": {"name_en": u"Macedonian", "name": u"\u041c\u0430\u043a\u0435\u0434\u043e\u043d\u0441\u043a\u0438"},
+ "ml_IN": {"name_en": u"Malayalam", "name": u"\u0d2e\u0d32\u0d2f\u0d3e\u0d33\u0d02"},
+ "ms_MY": {"name_en": u"Malay", "name": u"Bahasa Melayu"},
+ "nb_NO": {"name_en": u"Norwegian (bokmal)", "name": u"Norsk (bokm\xe5l)"},
+ "nl_NL": {"name_en": u"Dutch", "name": u"Nederlands"},
+ "nn_NO": {"name_en": u"Norwegian (nynorsk)", "name": u"Norsk (nynorsk)"},
+ "pa_IN": {"name_en": u"Punjabi", "name": u"\u0a2a\u0a70\u0a1c\u0a3e\u0a2c\u0a40"},
+ "pl_PL": {"name_en": u"Polish", "name": u"Polski"},
+ "pt_BR": {"name_en": u"Portuguese (Brazil)", "name": u"Portugu\xeas (Brasil)"},
+ "pt_PT": {"name_en": u"Portuguese (Portugal)", "name": u"Portugu\xeas (Portugal)"},
+ "ro_RO": {"name_en": u"Romanian", "name": u"Rom\xe2n\u0103"},
+ "ru_RU": {"name_en": u"Russian", "name": u"\u0420\u0443\u0441\u0441\u043a\u0438\u0439"},
+ "sk_SK": {"name_en": u"Slovak", "name": u"Sloven\u010dina"},
+ "sl_SI": {"name_en": u"Slovenian", "name": u"Sloven\u0161\u010dina"},
+ "sq_AL": {"name_en": u"Albanian", "name": u"Shqip"},
+ "sr_RS": {"name_en": u"Serbian", "name": u"\u0421\u0440\u043f\u0441\u043a\u0438"},
+ "sv_SE": {"name_en": u"Swedish", "name": u"Svenska"},
+ "sw_KE": {"name_en": u"Swahili", "name": u"Kiswahili"},
+ "ta_IN": {"name_en": u"Tamil", "name": u"\u0ba4\u0bae\u0bbf\u0bb4\u0bcd"},
+ "te_IN": {"name_en": u"Telugu", "name": u"\u0c24\u0c46\u0c32\u0c41\u0c17\u0c41"},
+ "th_TH": {"name_en": u"Thai", "name": u"\u0e20\u0e32\u0e29\u0e32\u0e44\u0e17\u0e22"},
+ "tl_PH": {"name_en": u"Filipino", "name": u"Filipino"},
+ "tr_TR": {"name_en": u"Turkish", "name": u"T\xfcrk\xe7e"},
+ "uk_UA": {"name_en": u"Ukraini ", "name": u"\u0423\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430"},
+ "vi_VN": {"name_en": u"Vietnamese", "name": u"Ti\u1ebfng Vi\u1ec7t"},
+ "zh_CN": {"name_en": u"Chinese (Simplified)", "name": u"\xe4\xe6(\xe7\xe4)"},
+ "zh_HK": {"name_en": u"Chinese (Hong Kong)", "name": u"\xe4\xe6(\xe9\xe6)"},
+ "zh_TW": {"name_en": u"Chinese (Taiwan)", "name": u"\xe4\xe6(\xe5\xe7)"},
+}
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A command line parsing module that lets modules define their own options.
+
+Each module defines its own options, e.g.,
+
+ from tornado.options import define, options
+
+ define("mysql_host", default="127.0.0.1:3306", help="Main user DB")
+ define("memcache_hosts", default="127.0.0.1:11011", multiple=True,
+ help="Main user memcache servers")
+
+ def connect():
+ db = database.Connection(options.mysql_host)
+ ...
+
+The main() method of your application does not need to be aware of all of
+the options used throughout your program; they are all automatically loaded
+when the modules are loaded. Your main() method can parse the command line
+or parse a config file with:
+
+ import tornado.options
+ tornado.options.parse_config_file("/etc/server.conf")
+ tornado.options.parse_command_line()
+
+Command line formats are what you would expect ("--myoption=myvalue").
+Config files are just Python files. Global names become options, e.g.,
+
+ myoption = "myvalue"
+ myotheroption = "myothervalue"
+
+We support datetimes, timedeltas, ints, and floats (just pass a 'type'
+kwarg to define). We also accept multi-value options. See the documentation
+for define() below.
+"""
+
+import datetime
+import logging
+import re
+import sys
+import time
+
+# For pretty log messages, if available
+try:
+ import curses
+except:
+ curses = None
+
+
+def define(name, default=None, type=str, help=None, metavar=None,
+ multiple=False):
+ """Defines a new command line option.
+
+ If type is given (one of str, float, int, datetime, or timedelta),
+ we parse the command line arguments based on the given type. If
+ multiple is True, we accept comma-separated values, and the option
+ value is always a list.
+
+ For multi-value integers, we also accept the syntax x:y, which
+ turns into range(x, y) - very useful for long integer ranges.
+
+ help and metavar are used to construct the automatically generated
+ command line help string. The help message is formatted like:
+
+ --name=METAVAR help string
+
+ Command line option names must be unique globally. They can be parsed
+ from the command line with parse_command_line() or parsed from a
+ config file with parse_config_file.
+ """
+ if name in options:
+ raise Error("Option %r already defined in %s", name,
+ options[name].file_name)
+ frame = sys._getframe(0)
+ options_file = frame.f_code.co_filename
+ file_name = frame.f_back.f_code.co_filename
+ if file_name == options_file: file_name = ""
+ options[name] = _Option(name, file_name=file_name, default=default,
+ type=type, help=help, metavar=metavar,
+ multiple=multiple)
+
+
+def parse_command_line(args=None):
+ """Parses all options given on the command line.
+
+ We return all command line arguments that are not options as a list.
+ """
+ if args is None: args = sys.argv
+ for i in xrange(1, len(args)):
+ # All things after the last option are command line arguments
+ if not args[i].startswith("-"):
+ return args[i:]
+ if args[i] == "--":
+ continue
+ arg = args[i].lstrip("-")
+ name, equals, value = arg.partition("=")
+ name = name.replace('-', '_')
+ if not name in options:
+ print_help()
+ raise Error('Unrecognized command line option: %r' % name)
+ option = options[name]
+ if not equals:
+ if option.type == bool:
+ value = "true"
+ else:
+ raise Error('Option %r requires a value' % name)
+ option.parse(value)
+ if options.help:
+ print_help()
+ sys.exit(0)
+
+ # Set up log level and pretty console logging by default
+ logging.getLogger().setLevel(getattr(logging, options.logging.upper()))
+ enable_pretty_logging()
+
+ return []
+
+
+def parse_config_file(path, overwrite=True):
+ """Parses and loads the Python config file at the given path."""
+ config = {}
+ execfile(path, config, config)
+ for name in config:
+ if name in options:
+ options[name].set(config[name])
+
+
+def print_help(file=sys.stdout):
+ """Prints all the command line options to stdout."""
+ print >> file, "Usage: %s [OPTIONS]" % sys.argv[0]
+ print >> file, ""
+ print >> file, "Options:"
+ by_file = {}
+ for option in options.itervalues():
+ by_file.setdefault(option.file_name, []).append(option)
+
+ for filename, o in sorted(by_file.items()):
+ if filename: print >> file, filename
+ o.sort(key=lambda option: option.name)
+ for option in o:
+ prefix = option.name
+ if option.metavar:
+ prefix += "=" + option.metavar
+ print >> file, " --%-30s %s" % (prefix, option.help or "")
+ print >> file
+
+
+class _Options(dict):
+ """Our global program options, an dictionary with object-like access."""
+ @classmethod
+ def instance(cls):
+ if not hasattr(cls, "_instance"):
+ cls._instance = cls()
+ return cls._instance
+
+ def __getattr__(self, name):
+ if isinstance(self.get(name), _Option):
+ return self[name].value()
+ raise Error("Unrecognized option %r" % name)
+
+
+class _Option(object):
+ def __init__(self, name, default=None, type=str, help=None, metavar=None,
+ multiple=False, file_name=None):
+ if default is None and multiple:
+ default = []
+ self.name = name
+ self.type = type
+ self.help = help
+ self.metavar = metavar
+ self.multiple = multiple
+ self.file_name = file_name
+ self.default = default
+ self._value = None
+
+ def value(self):
+ return self.default if self._value is None else self._value
+
+ def parse(self, value):
+ _parse = {
+ datetime.datetime: self._parse_datetime,
+ datetime.timedelta: self._parse_timedelta,
+ bool: self._parse_bool,
+ str: self._parse_string,
+ }.get(self.type, self.type)
+ if self.multiple:
+ if self._value is None:
+ self._value = []
+ for part in value.split(","):
+ if self.type in (int, long):
+ # allow ranges of the form X:Y (inclusive at both ends)
+ lo, _, hi = part.partition(":")
+ lo = _parse(lo)
+ hi = _parse(hi) if hi else lo
+ self._value.extend(range(lo, hi+1))
+ else:
+ self._value.append(_parse(part))
+ else:
+ self._value = _parse(value)
+ return self.value()
+
+ def set(self, value):
+ if self.multiple:
+ if not isinstance(value, list):
+ raise Error("Option %r is required to be a list of %s" %
+ (self.name, self.type.__name__))
+ for item in value:
+ if item != None and not isinstance(item, self.type):
+ raise Error("Option %r is required to be a list of %s" %
+ (self.name, self.type.__name__))
+ else:
+ if value != None and not isinstance(value, self.type):
+ raise Error("Option %r is required to be a %s" %
+ (self.name, self.type.__name__))
+ self._value = value
+
+ # Supported date/time formats in our options
+ _DATETIME_FORMATS = [
+ "%a %b %d %H:%M:%S %Y",
+ "%Y-%m-%d %H:%M:%S",
+ "%Y-%m-%d %H:%M",
+ "%Y-%m-%dT%H:%M",
+ "%Y%m%d %H:%M:%S",
+ "%Y%m%d %H:%M",
+ "%Y-%m-%d",
+ "%Y%m%d",
+ "%H:%M:%S",
+ "%H:%M",
+ ]
+
+ def _parse_datetime(self, value):
+ for format in self._DATETIME_FORMATS:
+ try:
+ return datetime.datetime.strptime(value, format)
+ except ValueError:
+ pass
+ raise Error('Unrecognized date/time format: %r' % value)
+
+ _TIMEDELTA_ABBREVS = [
+ ('hours', ['h']),
+ ('minutes', ['m', 'min']),
+ ('seconds', ['s', 'sec']),
+ ('milliseconds', ['ms']),
+ ('microseconds', ['us']),
+ ('days', ['d']),
+ ('weeks', ['w']),
+ ]
+
+ _TIMEDELTA_ABBREV_DICT = dict(
+ (abbrev, full) for full, abbrevs in _TIMEDELTA_ABBREVS
+ for abbrev in abbrevs)
+
+ _FLOAT_PATTERN = r'[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?'
+
+ _TIMEDELTA_PATTERN = re.compile(
+ r'\s*(%s)\s*(\w*)\s*' % _FLOAT_PATTERN, re.IGNORECASE)
+
+ def _parse_timedelta(self, value):
+ try:
+ sum = datetime.timedelta()
+ start = 0
+ while start < len(value):
+ m = self._TIMEDELTA_PATTERN.match(value, start)
+ if not m:
+ raise Exception()
+ num = float(m.group(1))
+ units = m.group(2) or 'seconds'
+ units = self._TIMEDELTA_ABBREV_DICT.get(units, units)
+ sum += datetime.timedelta(**{units: num})
+ start = m.end()
+ return sum
+ except:
+ raise
+
+ def _parse_bool(self, value):
+ return value.lower() not in ("false", "0", "f")
+
+ def _parse_string(self, value):
+ return value.decode("utf-8")
+
+
+class Error(Exception):
+ pass
+
+
+def enable_pretty_logging():
+ """Turns on colored logging output for stderr if we are in a tty."""
+ if not curses: return
+ try:
+ if not sys.stderr.isatty(): return
+ curses.setupterm()
+ except:
+ return
+ channel = logging.StreamHandler()
+ channel.setFormatter(_ColorLogFormatter())
+ logging.getLogger().addHandler(channel)
+
+
+class _ColorLogFormatter(logging.Formatter):
+ def __init__(self, *args, **kwargs):
+ logging.Formatter.__init__(self, *args, **kwargs)
+ fg_color = curses.tigetstr("setaf") or curses.tigetstr("setf") or ""
+ self._colors = {
+ logging.DEBUG: curses.tparm(fg_color, 4), # Blue
+ logging.INFO: curses.tparm(fg_color, 2), # Green
+ logging.WARNING: curses.tparm(fg_color, 3), # Yellow
+ logging.ERROR: curses.tparm(fg_color, 1), # Red
+ }
+ self._normal = curses.tigetstr("sgr0")
+
+ def format(self, record):
+ try:
+ record.message = record.getMessage()
+ except Exception, e:
+ record.message = "Bad message (%r): %r" % (e, record.__dict__)
+ record.asctime = time.strftime(
+ "%y%m%d %H:%M:%S", self.converter(record.created))
+ prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \
+ record.__dict__
+ color = self._colors.get(record.levelno, self._normal)
+ formatted = color + prefix + self._normal + " " + record.message
+ if record.exc_info:
+ if not record.exc_text:
+ record.exc_text = self.formatException(record.exc_info)
+ if record.exc_text:
+ formatted = formatted.rstrip() + "\n" + record.exc_text
+ return formatted.replace("\n", "\n ")
+
+
+options = _Options.instance()
+
+
+# Default options
+define("help", type=bool, help="show this help information")
+define("logging", default="info", help="set the Python log level",
+ metavar="info|warning|error")
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Implementation of an S3-like storage server based on local files.
+
+Useful to test features that will eventually run on S3, or if you want to
+run something locally that was once running on S3.
+
+We don't support all the features of S3, but it does work with the
+standard S3 client for the most basic semantics. To use the standard
+S3 client with this module:
+
+ c = S3.AWSAuthConnection("", "", server="localhost", port=8888,
+ is_secure=False)
+ c.create_bucket("mybucket")
+ c.put("mybucket", "mykey", "a value")
+ print c.get("mybucket", "mykey").body
+
+"""
+
+import bisect
+import datetime
+import escape
+import hashlib
+import httpserver
+import ioloop
+import os
+import os.path
+import urllib
+import web
+
+
+def start(port, root_directory="/tmp/s3", bucket_depth=0):
+ """Starts the mock S3 server on the given port at the given path."""
+ application = S3Application(root_directory, bucket_depth)
+ http_server = httpserver.HTTPServer(application)
+ http_server.listen(port)
+ ioloop.IOLoop.instance().start()
+
+
+class S3Application(web.Application):
+ """Implementation of an S3-like storage server based on local files.
+
+ If bucket depth is given, we break files up into multiple directories
+ to prevent hitting file system limits for number of files in each
+ directories. 1 means one level of directories, 2 means 2, etc.
+ """
+ def __init__(self, root_directory, bucket_depth=0):
+ web.Application.__init__(self, [
+ (r"/", RootHandler),
+ (r"/([^/]+)/(.+)", ObjectHandler),
+ (r"/([^/]+)/", BucketHandler),
+ ])
+ self.directory = os.path.abspath(root_directory)
+ if not os.path.exists(self.directory):
+ os.makedirs(self.directory)
+ self.bucket_depth = bucket_depth
+
+
+class BaseRequestHandler(web.RequestHandler):
+ SUPPORTED_METHODS = ("PUT", "GET", "DELETE")
+
+ def render_xml(self, value):
+ assert isinstance(value, dict) and len(value) == 1
+ self.set_header("Content-Type", "application/xml; charset=UTF-8")
+ name = value.keys()[0]
+ parts = []
+ parts.append('<' + escape.utf8(name) +
+ ' xmlns="http://doc.s3.amazonaws.com/2006-03-01">')
+ self._render_parts(value.values()[0], parts)
+ parts.append('</' + escape.utf8(name) + '>')
+ self.finish('<?xml version="1.0" encoding="UTF-8"?>\n' +
+ ''.join(parts))
+
+ def _render_parts(self, value, parts=[]):
+ if isinstance(value, basestring):
+ parts.append(escape.xhtml_escape(value))
+ elif isinstance(value, int) or isinstance(value, long):
+ parts.append(str(value))
+ elif isinstance(value, datetime.datetime):
+ parts.append(value.strftime("%Y-%m-%dT%H:%M:%S.000Z"))
+ elif isinstance(value, dict):
+ for name, subvalue in value.iteritems():
+ if not isinstance(subvalue, list):
+ subvalue = [subvalue]
+ for subsubvalue in subvalue:
+ parts.append('<' + escape.utf8(name) + '>')
+ self._render_parts(subsubvalue, parts)
+ parts.append('</' + escape.utf8(name) + '>')
+ else:
+ raise Exception("Unknown S3 value type %r", value)
+
+ def _object_path(self, bucket, object_name):
+ if self.application.bucket_depth < 1:
+ return os.path.abspath(os.path.join(
+ self.application.directory, bucket, object_name))
+ hash = hashlib.md5(object_name).hexdigest()
+ path = os.path.abspath(os.path.join(
+ self.application.directory, bucket))
+ for i in range(self.application.bucket_depth):
+ path = os.path.join(path, hash[:2 * (i + 1)])
+ return os.path.join(path, object_name)
+
+
+class RootHandler(BaseRequestHandler):
+ def get(self):
+ names = os.listdir(self.application.directory)
+ buckets = []
+ for name in names:
+ path = os.path.join(self.application.directory, name)
+ info = os.stat(path)
+ buckets.append({
+ "Name": name,
+ "CreationDate": datetime.datetime.utcfromtimestamp(
+ info.st_ctime),
+ })
+ self.render_xml({"ListAllMyBucketsResult": {
+ "Buckets": {"Bucket": buckets},
+ }})
+
+
+class BucketHandler(BaseRequestHandler):
+ def get(self, bucket_name):
+ prefix = self.get_argument("prefix", u"")
+ marker = self.get_argument("marker", u"")
+ max_keys = int(self.get_argument("max-keys", 50000))
+ path = os.path.abspath(os.path.join(self.application.directory,
+ bucket_name))
+ terse = int(self.get_argument("terse", 0))
+ if not path.startswith(self.application.directory) or \
+ not os.path.isdir(path):
+ raise web.HTTPError(404)
+ object_names = []
+ for root, dirs, files in os.walk(path):
+ for file_name in files:
+ object_names.append(os.path.join(root, file_name))
+ skip = len(path) + 1
+ for i in range(self.application.bucket_depth):
+ skip += 2 * (i + 1) + 1
+ object_names = [n[skip:] for n in object_names]
+ object_names.sort()
+ contents = []
+
+ start_pos = 0
+ if marker:
+ start_pos = bisect.bisect_right(object_names, marker, start_pos)
+ if prefix:
+ start_pos = bisect.bisect_left(object_names, prefix, start_pos)
+
+ truncated = False
+ for object_name in object_names[start_pos:]:
+ if not object_name.startswith(prefix):
+ break
+ if len(contents) >= max_keys:
+ truncated = True
+ break
+ object_path = self._object_path(bucket_name, object_name)
+ c = {"Key": object_name}
+ if not terse:
+ info = os.stat(object_path)
+ c.update({
+ "LastModified": datetime.datetime.utcfromtimestamp(
+ info.st_mtime),
+ "Size": info.st_size,
+ })
+ contents.append(c)
+ marker = object_name
+ self.render_xml({"ListBucketResult": {
+ "Name": bucket_name,
+ "Prefix": prefix,
+ "Marker": marker,
+ "MaxKeys": max_keys,
+ "IsTruncated": truncated,
+ "Contents": contents,
+ }})
+
+ def put(self, bucket_name):
+ path = os.path.abspath(os.path.join(
+ self.application.directory, bucket_name))
+ if not path.startswith(self.application.directory) or \
+ os.path.exists(path):
+ raise web.HTTPError(403)
+ os.makedirs(path)
+ self.finish()
+
+ def delete(self, bucket_name):
+ path = os.path.abspath(os.path.join(
+ self.application.directory, bucket_name))
+ if not path.startswith(self.application.directory) or \
+ not os.path.isdir(path):
+ raise web.HTTPError(404)
+ if len(os.listdir(path)) > 0:
+ raise web.HTTPError(403)
+ os.rmdir(path)
+ self.set_status(204)
+ self.finish()
+
+
+class ObjectHandler(BaseRequestHandler):
+ def get(self, bucket, object_name):
+ object_name = urllib.unquote(object_name)
+ path = self._object_path(bucket, object_name)
+ if not path.startswith(self.application.directory) or \
+ not os.path.isfile(path):
+ raise web.HTTPError(404)
+ info = os.stat(path)
+ self.set_header("Content-Type", "application/unknown")
+ self.set_header("Last-Modified", datetime.datetime.utcfromtimestamp(
+ info.st_mtime))
+ object_file = open(path, "r")
+ try:
+ self.finish(object_file.read())
+ finally:
+ object_file.close()
+
+ def put(self, bucket, object_name):
+ object_name = urllib.unquote(object_name)
+ bucket_dir = os.path.abspath(os.path.join(
+ self.application.directory, bucket))
+ if not bucket_dir.startswith(self.application.directory) or \
+ not os.path.isdir(bucket_dir):
+ raise web.HTTPError(404)
+ path = self._object_path(bucket, object_name)
+ if not path.startswith(bucket_dir) or os.path.isdir(path):
+ raise web.HTTPError(403)
+ directory = os.path.dirname(path)
+ if not os.path.exists(directory):
+ os.makedirs(directory)
+ object_file = open(path, "w")
+ object_file.write(self.request.body)
+ object_file.close()
+ self.finish()
+
+ def delete(self, bucket, object_name):
+ object_name = urllib.unquote(object_name)
+ path = self._object_path(bucket, object_name)
+ if not path.startswith(self.application.directory) or \
+ not os.path.isfile(path):
+ raise web.HTTPError(404)
+ os.unlink(path)
+ self.set_status(204)
+ self.finish()
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A simple template system that compiles templates to Python code.
+
+Basic usage looks like:
+
+ t = template.Template("<html>{{ myvalue }}</html>")
+ print t.generate(myvalue="XXX")
+
+Loader is a class that loads templates from a root directory and caches
+the compiled templates:
+
+ loader = template.Loader("/home/btaylor")
+ print loader.load("test.html").generate(myvalue="XXX")
+
+We compile all templates to raw Python. Error-reporting is currently... uh,
+interesting. Syntax for the templates
+
+ ### base.html
+ <html>
+ <head>
+ <title>{% block title %}Default title{% end %}</title>
+ </head>
+ <body>
+ <ul>
+ {% for student in students %}
+ {% block student %}
+ <li>{{ escape(student.name) }}</li>
+ {% end %}
+ {% end %}
+ </ul>
+ </body>
+ </html>
+
+ ### bold.html
+ {% extends "base.html" %}
+
+ {% block title %}A bolder title{% end %}
+
+ {% block student %}
+ <li><span style="bold">{{ escape(student.name) }}</span></li>
+ {% block %}
+
+Unlike most other template systems, we do not put any restrictions on the
+expressions you can include in your statements. if and for blocks get
+translated exactly into Python, do you can do complex expressions like:
+
+ {% for student in [p for p in people if p.student and p.age > 23] %}
+ <li>{{ escape(student.name) }}</li>
+ {% end %}
+
+Translating directly to Python means you can apply functions to expressions
+easily, like the escape() function in the examples above. You can pass
+functions in to your template just like any other variable:
+
+ ### Python code
+ def add(x, y):
+ return x + y
+ template.execute(add=add)
+
+ ### The template
+ {{ add(1, 2) }}
+
+We provide the functions escape(), url_escape(), json_encode(), and squeeze()
+to all templates by default.
+"""
+
+from __future__ import with_statement
+
+import cStringIO
+import datetime
+import escape
+import logging
+import os.path
+import re
+
+
+class Template(object):
+ """A compiled template.
+
+ We compile into Python from the given template_string. You can generate
+ the template from variables with generate().
+ """
+ def __init__(self, template_string, name="<string>", loader=None,
+ compress_whitespace=None):
+ self.name = name
+ if compress_whitespace is None:
+ compress_whitespace = name.endswith(".html") or \
+ name.endswith(".js")
+ reader = _TemplateReader(name, template_string)
+ self.file = _File(_parse(reader))
+ self.code = self._generate_python(loader, compress_whitespace)
+ try:
+ self.compiled = compile(self.code, self.name, "exec")
+ except:
+ formatted_code = _format_code(self.code).rstrip()
+ logging.error("%s code:\n%s", self.name, formatted_code)
+ raise
+
+ def generate(self, **kwargs):
+ """Generate this template with the given arguments."""
+ namespace = {
+ "escape": escape.xhtml_escape,
+ "url_escape": escape.url_escape,
+ "json_encode": escape.json_encode,
+ "squeeze": escape.squeeze,
+ "datetime": datetime,
+ }
+ namespace.update(kwargs)
+ exec self.compiled in namespace
+ execute = namespace["_execute"]
+ try:
+ return execute()
+ except:
+ formatted_code = _format_code(self.code).rstrip()
+ logging.error("%s code:\n%s", self.name, formatted_code)
+ raise
+
+ def _generate_python(self, loader, compress_whitespace):
+ buffer = cStringIO.StringIO()
+ try:
+ named_blocks = {}
+ ancestors = self._get_ancestors(loader)
+ ancestors.reverse()
+ for ancestor in ancestors:
+ ancestor.find_named_blocks(loader, named_blocks)
+ self.file.find_named_blocks(loader, named_blocks)
+ writer = _CodeWriter(buffer, named_blocks, loader, self,
+ compress_whitespace)
+ ancestors[0].generate(writer)
+ return buffer.getvalue()
+ finally:
+ buffer.close()
+
+ def _get_ancestors(self, loader):
+ ancestors = [self.file]
+ for chunk in self.file.body.chunks:
+ if isinstance(chunk, _ExtendsBlock):
+ if not loader:
+ raise ParseError("{% extends %} block found, but no "
+ "template loader")
+ template = loader.load(chunk.name, self.name)
+ ancestors.extend(template._get_ancestors(loader))
+ return ancestors
+
+
+class Loader(object):
+ """A template loader that loads from a single root directory.
+
+ You must use a template loader to use template constructs like
+ {% extends %} and {% include %}. Loader caches all templates after
+ they are loaded the first time.
+ """
+ def __init__(self, root_directory):
+ self.root = os.path.abspath(root_directory)
+ self.templates = {}
+
+ def load(self, name, parent_path=None):
+ if parent_path and not parent_path.startswith("<") and \
+ not parent_path.startswith("/") and \
+ not name.startswith("/"):
+ current_path = os.path.join(self.root, parent_path)
+ file_dir = os.path.dirname(os.path.abspath(current_path))
+ relative_path = os.path.abspath(os.path.join(file_dir, name))
+ if relative_path.startswith(self.root):
+ name = relative_path[len(self.root) + 1:]
+ if name not in self.templates:
+ path = os.path.join(self.root, name)
+ f = open(path, "r")
+ self.templates[name] = Template(f.read(), name=name, loader=self)
+ f.close()
+ return self.templates[name]
+
+
+class _Node(object):
+ def each_child(self):
+ return ()
+
+ def generate(self, writer):
+ raise NotImplementedError()
+
+ def find_named_blocks(self, loader, named_blocks):
+ for child in self.each_child():
+ child.find_named_blocks(loader, named_blocks)
+
+
+class _File(_Node):
+ def __init__(self, body):
+ self.body = body
+
+ def generate(self, writer):
+ writer.write_line("def _execute():")
+ with writer.indent():
+ writer.write_line("_buffer = []")
+ self.body.generate(writer)
+ writer.write_line("return ''.join(_buffer)")
+
+ def each_child(self):
+ return (self.body,)
+
+
+
+class _ChunkList(_Node):
+ def __init__(self, chunks):
+ self.chunks = chunks
+
+ def generate(self, writer):
+ for chunk in self.chunks:
+ chunk.generate(writer)
+
+ def each_child(self):
+ return self.chunks
+
+
+class _NamedBlock(_Node):
+ def __init__(self, name, body=None):
+ self.name = name
+ self.body = body
+
+ def each_child(self):
+ return (self.body,)
+
+ def generate(self, writer):
+ writer.named_blocks[self.name].generate(writer)
+
+ def find_named_blocks(self, loader, named_blocks):
+ named_blocks[self.name] = self.body
+ _Node.find_named_blocks(self, loader, named_blocks)
+
+
+class _ExtendsBlock(_Node):
+ def __init__(self, name):
+ self.name = name
+
+
+class _IncludeBlock(_Node):
+ def __init__(self, name, reader):
+ self.name = name
+ self.template_name = reader.name
+
+ def find_named_blocks(self, loader, named_blocks):
+ included = loader.load(self.name, self.template_name)
+ included.file.find_named_blocks(loader, named_blocks)
+
+ def generate(self, writer):
+ included = writer.loader.load(self.name, self.template_name)
+ old = writer.current_template
+ writer.current_template = included
+ included.file.body.generate(writer)
+ writer.current_template = old
+
+
+class _ApplyBlock(_Node):
+ def __init__(self, method, body=None):
+ self.method = method
+ self.body = body
+
+ def each_child(self):
+ return (self.body,)
+
+ def generate(self, writer):
+ method_name = "apply%d" % writer.apply_counter
+ writer.apply_counter += 1
+ writer.write_line("def %s():" % method_name)
+ with writer.indent():
+ writer.write_line("_buffer = []")
+ self.body.generate(writer)
+ writer.write_line("return ''.join(_buffer)")
+ writer.write_line("_buffer.append(%s(%s()))" % (
+ self.method, method_name))
+
+
+class _ControlBlock(_Node):
+ def __init__(self, statement, body=None):
+ self.statement = statement
+ self.body = body
+
+ def each_child(self):
+ return (self.body,)
+
+ def generate(self, writer):
+ writer.write_line("%s:" % self.statement)
+ with writer.indent():
+ self.body.generate(writer)
+
+
+class _IntermediateControlBlock(_Node):
+ def __init__(self, statement):
+ self.statement = statement
+
+ def generate(self, writer):
+ writer.write_line("%s:" % self.statement, writer.indent_size() - 1)
+
+
+class _Statement(_Node):
+ def __init__(self, statement):
+ self.statement = statement
+
+ def generate(self, writer):
+ writer.write_line(self.statement)
+
+
+class _Expression(_Node):
+ def __init__(self, expression):
+ self.expression = expression
+
+ def generate(self, writer):
+ writer.write_line("_tmp = %s" % self.expression)
+ writer.write_line("if isinstance(_tmp, str): _buffer.append(_tmp)")
+ writer.write_line("elif isinstance(_tmp, unicode): "
+ "_buffer.append(_tmp.encode('utf-8'))")
+ writer.write_line("else: _buffer.append(str(_tmp))")
+
+
+class _Text(_Node):
+ def __init__(self, value):
+ self.value = value
+
+ def generate(self, writer):
+ value = self.value
+
+ # Compress lots of white space to a single character. If the whitespace
+ # breaks a line, have it continue to break a line, but just with a
+ # single \n character
+ if writer.compress_whitespace and "<pre>" not in value:
+ value = re.sub(r"([\t ]+)", " ", value)
+ value = re.sub(r"(\s*\n\s*)", "\n", value)
+
+ if value:
+ writer.write_line('_buffer.append(%r)' % value)
+
+
+class ParseError(Exception):
+ """Raised for template syntax errors."""
+ pass
+
+
+class _CodeWriter(object):
+ def __init__(self, file, named_blocks, loader, current_template,
+ compress_whitespace):
+ self.file = file
+ self.named_blocks = named_blocks
+ self.loader = loader
+ self.current_template = current_template
+ self.compress_whitespace = compress_whitespace
+ self.apply_counter = 0
+ self._indent = 0
+
+ def indent(self):
+ return self
+
+ def indent_size(self):
+ return self._indent
+
+ def __enter__(self):
+ self._indent += 1
+ return self
+
+ def __exit__(self, *args):
+ assert self._indent > 0
+ self._indent -= 1
+
+ def write_line(self, line, indent=None):
+ if indent == None:
+ indent = self._indent
+ for i in xrange(indent):
+ self.file.write(" ")
+ print >> self.file, line
+
+
+class _TemplateReader(object):
+ def __init__(self, name, text):
+ self.name = name
+ self.text = text
+ self.line = 0
+ self.pos = 0
+
+ def find(self, needle, start=0, end=None):
+ assert start >= 0, start
+ pos = self.pos
+ start += pos
+ if end is None:
+ index = self.text.find(needle, start)
+ else:
+ end += pos
+ assert end >= start
+ index = self.text.find(needle, start, end)
+ if index != -1:
+ index -= pos
+ return index
+
+ def consume(self, count=None):
+ if count is None:
+ count = len(self.text) - self.pos
+ newpos = self.pos + count
+ self.line += self.text.count("\n", self.pos, newpos)
+ s = self.text[self.pos:newpos]
+ self.pos = newpos
+ return s
+
+ def remaining(self):
+ return len(self.text) - self.pos
+
+ def __len__(self):
+ return self.remaining()
+
+ def __getitem__(self, key):
+ if type(key) is slice:
+ size = len(self)
+ start, stop, step = slice.indices(size)
+ if start is None: start = self.pos
+ else: start += self.pos
+ if stop is not None: stop += self.pos
+ return self.text[slice(start, stop, step)]
+ elif key < 0:
+ return self.text[key]
+ else:
+ return self.text[self.pos + key]
+
+ def __str__(self):
+ return self.text[self.pos:]
+
+
+def _format_code(code):
+ lines = code.splitlines()
+ format = "%%%dd %%s\n" % len(repr(len(lines) + 1))
+ return "".join([format % (i + 1, line) for (i, line) in enumerate(lines)])
+
+
+def _parse(reader, in_block=None):
+ body = _ChunkList([])
+ while True:
+ # Find next template directive
+ curly = 0
+ while True:
+ curly = reader.find("{", curly)
+ if curly == -1 or curly + 1 == reader.remaining():
+ # EOF
+ if in_block:
+ raise ParseError("Missing {%% end %%} block for %s" %
+ in_block)
+ body.chunks.append(_Text(reader.consume()))
+ return body
+ # If the first curly brace is not the start of a special token,
+ # start searching from the character after it
+ if reader[curly + 1] not in ("{", "%"):
+ curly += 1
+ continue
+ break
+
+ # Append any text before the special token
+ if curly > 0:
+ body.chunks.append(_Text(reader.consume(curly)))
+
+ start_brace = reader.consume(2)
+ line = reader.line
+
+ # Expression
+ if start_brace == "{{":
+ end = reader.find("}}")
+ if end == -1 or reader.find("\n", 0, end) != -1:
+ raise ParseError("Missing end expression }} on line %d" % line)
+ contents = reader.consume(end).strip()
+ reader.consume(2)
+ if not contents:
+ raise ParseError("Empty expression on line %d" % line)
+ body.chunks.append(_Expression(contents))
+ continue
+
+ # Block
+ assert start_brace == "{%", start_brace
+ end = reader.find("%}")
+ if end == -1 or reader.find("\n", 0, end) != -1:
+ raise ParseError("Missing end block %%} on line %d" % line)
+ contents = reader.consume(end).strip()
+ reader.consume(2)
+ if not contents:
+ raise ParseError("Empty block tag ({%% %%}) on line %d" % line)
+
+ operator, space, suffix = contents.partition(" ")
+ suffix = suffix.strip()
+
+ # Intermediate ("else", "elif", etc) blocks
+ intermediate_blocks = {
+ "else": set(["if", "for", "while"]),
+ "elif": set(["if"]),
+ "except": set(["try"]),
+ "finally": set(["try"]),
+ }
+ allowed_parents = intermediate_blocks.get(operator)
+ if allowed_parents is not None:
+ if not in_block:
+ raise ParseError("%s outside %s block" %
+ (operator, allowed_parents))
+ if in_block not in allowed_parents:
+ raise ParseError("%s block cannot be attached to %s block" % (operator, in_block))
+ body.chunks.append(_IntermediateControlBlock(contents))
+ continue
+
+ # End tag
+ elif operator == "end":
+ if not in_block:
+ raise ParseError("Extra {%% end %%} block on line %d" % line)
+ return body
+
+ elif operator in ("extends", "include", "set", "import", "comment"):
+ if operator == "comment":
+ continue
+ if operator == "extends":
+ suffix = suffix.strip('"').strip("'")
+ if not suffix:
+ raise ParseError("extends missing file path on line %d" % line)
+ block = _ExtendsBlock(suffix)
+ elif operator == "import":
+ if not suffix:
+ raise ParseError("import missing statement on line %d" % line)
+ block = _Statement(contents)
+ elif operator == "include":
+ suffix = suffix.strip('"').strip("'")
+ if not suffix:
+ raise ParseError("include missing file path on line %d" % line)
+ block = _IncludeBlock(suffix, reader)
+ elif operator == "set":
+ if not suffix:
+ raise ParseError("set missing statement on line %d" % line)
+ block = _Statement(suffix)
+ body.chunks.append(block)
+ continue
+
+ elif operator in ("apply", "block", "try", "if", "for", "while"):
+ # parse inner body recursively
+ block_body = _parse(reader, operator)
+ if operator == "apply":
+ if not suffix:
+ raise ParseError("apply missing method name on line %d" % line)
+ block = _ApplyBlock(suffix, block_body)
+ elif operator == "block":
+ if not suffix:
+ raise ParseError("block missing name on line %d" % line)
+ block = _NamedBlock(suffix, block_body)
+ else:
+ block = _ControlBlock(contents, block_body)
+ body.chunks.append(block)
+ continue
+
+ else:
+ raise ParseError("unknown operator: %r" % operator)
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""The Tornado web framework.
+
+The Tornado web framework looks a bit like web.py (http://webpy.org/) or
+Google's webapp (http://code.google.com/appengine/docs/python/tools/webapp/),
+but with additional tools and optimizations to take advantage of the
+Tornado non-blocking web server and tools.
+
+Here is the canonical "Hello, world" example app:
+
+ import tornado.httpserver
+ import tornado.ioloop
+ import tornado.web
+
+ class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ self.write("Hello, world")
+
+ if __name__ == "__main__":
+ application = tornado.web.Application([
+ (r"/", MainHandler),
+ ])
+ http_server = tornado.httpserver.HTTPServer(application)
+ http_server.listen(8888)
+ tornado.ioloop.IOLoop.instance().start()
+
+See the Tornado walkthrough on Google Code for more details and a good
+getting started guide.
+"""
+
+import base64
+import binascii
+import calendar
+import Cookie
+import datetime
+import email.utils
+import escape
+import functools
+import hashlib
+import hmac
+import httplib
+import locale
+import logging
+import mimetypes
+import os.path
+import re
+import stat
+import sys
+import template
+import time
+import types
+import urllib
+import urlparse
+import uuid
+
+
+class RequestHandler(object):
+ """Subclass this class and define get() or post() to make a handler.
+
+ If you want to support more methods than the standard GET/HEAD/POST, you
+ should override the class variable SUPPORTED_METHODS in your
+ RequestHandler class.
+ """
+ SUPPORTED_METHODS = ("GET", "HEAD", "POST", "DELETE", "PUT")
+
+ def __init__(self, application, request, transforms=None):
+ self.application = application
+ self.request = request
+ self._headers_written = False
+ self._finished = False
+ self._auto_finish = True
+ self._transforms = transforms or []
+ self.ui = _O((n, self._ui_method(m)) for n, m in
+ application.ui_methods.iteritems())
+ self.ui["modules"] = _O((n, self._ui_module(n, m)) for n, m in
+ application.ui_modules.iteritems())
+ self.clear()
+
+ @property
+ def settings(self):
+ return self.application.settings
+
+ def head(self, *args, **kwargs):
+ raise HTTPError(405)
+
+ def get(self, *args, **kwargs):
+ raise HTTPError(405)
+
+ def post(self, *args, **kwargs):
+ raise HTTPError(405)
+
+ def delete(self, *args, **kwargs):
+ raise HTTPError(405)
+
+ def put(self, *args, **kwargs):
+ raise HTTPError(405)
+
+ def prepare(self):
+ """Called before the actual handler method.
+
+ Useful to override in a handler if you want a common bottleneck for
+ all of your requests.
+ """
+ pass
+
+ def clear(self):
+ """Resets all headers and content for this response."""
+ self._headers = {
+ "Server": "TornadoServer/0.1",
+ "Content-Type": "text/html; charset=UTF-8",
+ }
+ if not self.request.supports_http_1_1():
+ if self.request.headers.get("Connection") == "Keep-Alive":
+ self.set_header("Connection", "Keep-Alive")
+ self._write_buffer = []
+ self._status_code = 200
+
+ def set_status(self, status_code):
+ """Sets the status code for our response."""
+ assert status_code in httplib.responses
+ self._status_code = status_code
+
+ def set_header(self, name, value):
+ """Sets the given response header name and value.
+
+ If a datetime is given, we automatically format it according to the
+ HTTP specification. If the value is not a string, we convert it to
+ a string. All header values are then encoded as UTF-8.
+ """
+ if isinstance(value, datetime.datetime):
+ t = calendar.timegm(value.utctimetuple())
+ value = email.utils.formatdate(t, localtime=False, usegmt=True)
+ elif isinstance(value, int) or isinstance(value, long):
+ value = str(value)
+ else:
+ value = _utf8(value)
+ # If \n is allowed into the header, it is possible to inject
+ # additional headers or split the request. Also cap length to
+ # prevent obviously erroneous values.
+ safe_value = re.sub(r"[\x00-\x1f]", " ", value)[:4000]
+ if safe_value != value:
+ raise ValueError("Unsafe header value %r", value)
+ self._headers[name] = value
+
+ _ARG_DEFAULT = []
+ def get_argument(self, name, default=_ARG_DEFAULT, strip=True):
+ """Returns the value of the argument with the given name.
+
+ If default is not provided, the argument is considered to be
+ required, and we throw an HTTP 404 exception if it is missing.
+
+ The returned value is always unicode.
+ """
+ values = self.request.arguments.get(name, None)
+ if values is None:
+ if default is self._ARG_DEFAULT:
+ raise HTTPError(404, "Missing argument %s" % name)
+ return default
+ # Get rid of any weird control chars
+ value = re.sub(r"[\x00-\x08\x0e-\x1f]", " ", values[-1])
+ value = _unicode(value)
+ if strip: value = value.strip()
+ return value
+
+ @property
+ def cookies(self):
+ """A dictionary of Cookie.Morsel objects."""
+ if not hasattr(self, "_cookies"):
+ self._cookies = Cookie.BaseCookie()
+ if "Cookie" in self.request.headers:
+ try:
+ self._cookies.load(self.request.headers["Cookie"])
+ except:
+ self.clear_all_cookies()
+ return self._cookies
+
+ def get_cookie(self, name, default=None):
+ """Gets the value of the cookie with the given name, else default."""
+ if name in self.cookies:
+ return self.cookies[name].value
+ return default
+
+ def set_cookie(self, name, value, domain=None, expires=None, path="/",
+ expires_days=None):
+ """Sets the given cookie name/value with the given options."""
+ name = _utf8(name)
+ value = _utf8(value)
+ if re.search(r"[\x00-\x20]", name + value):
+ # Don't let us accidentally inject bad stuff
+ raise ValueError("Invalid cookie %r: %r" % (name, value))
+ if not hasattr(self, "_new_cookies"):
+ self._new_cookies = []
+ new_cookie = Cookie.BaseCookie()
+ self._new_cookies.append(new_cookie)
+ new_cookie[name] = value
+ if domain:
+ new_cookie[name]["domain"] = domain
+ if expires_days is not None and not expires:
+ expires = datetime.datetime.utcnow() + datetime.timedelta(
+ days=expires_days)
+ if expires:
+ timestamp = calendar.timegm(expires.utctimetuple())
+ new_cookie[name]["expires"] = email.utils.formatdate(
+ timestamp, localtime=False, usegmt=True)
+ if path:
+ new_cookie[name]["path"] = path
+
+ def clear_cookie(self, name, path="/", domain=None):
+ """Deletes the cookie with the given name."""
+ expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
+ self.set_cookie(name, value="", path=path, expires=expires,
+ domain=domain)
+
+ def clear_all_cookies(self):
+ """Deletes all the cookies the user sent with this request."""
+ for name in self.cookies.iterkeys():
+ self.clear_cookie(name)
+
+ def set_secure_cookie(self, name, value, expires_days=30, **kwargs):
+ """Signs and timestamps a cookie so it cannot be forged.
+
+ You must specify the 'cookie_secret' setting in your Application
+ to use this method. It should be a long, random sequence of bytes
+ to be used as the HMAC secret for the signature.
+
+ To read a cookie set with this method, use get_secure_cookie().
+ """
+ timestamp = str(int(time.time()))
+ value = base64.b64encode(value)
+ signature = self._cookie_signature(value, timestamp)
+ value = "|".join([value, timestamp, signature])
+ self.set_cookie(name, value, expires_days=expires_days, **kwargs)
+
+ def get_secure_cookie(self, name):
+ """Returns the given signed cookie if it validates, or None."""
+ value = self.get_cookie(name)
+ if not value: return None
+ parts = value.split("|")
+ if len(parts) != 3: return None
+ if self._cookie_signature(parts[0], parts[1]) != parts[2]:
+ logging.warning("Invalid cookie signature %r", value)
+ return None
+ timestamp = int(parts[1])
+ if timestamp < time.time() - 31 * 86400:
+ logging.warning("Expired cookie %r", value)
+ return None
+ try:
+ return base64.b64decode(parts[0])
+ except:
+ return None
+
+ def _cookie_signature(self, *parts):
+ self.require_setting("cookie_secret", "secure cookies")
+ hash = hmac.new(self.application.settings["cookie_secret"],
+ digestmod=hashlib.sha1)
+ for part in parts: hash.update(part)
+ return hash.hexdigest()
+
+ def redirect(self, url, permanent=False):
+ """Sends a redirect to the given (optionally relative) URL."""
+ if self._headers_written:
+ raise Exception("Cannot redirect after headers have been written")
+ self.set_status(301 if permanent else 302)
+ # Remove whitespace
+ url = re.sub(r"[\x00-\x20]+", "", _utf8(url))
+ self.set_header("Location", urlparse.urljoin(self.request.uri, url))
+ self.finish()
+
+ def write(self, chunk):
+ """Writes the given chunk to the output buffer.
+
+ To write the output to the network, use the flush() method below.
+
+ If the given chunk is a dictionary, we write it as JSON and set
+ the Content-Type of the response to be text/javascript.
+ """
+ assert not self._finished
+ if isinstance(chunk, dict):
+ chunk = escape.json_encode(chunk)
+ self.set_header("Content-Type", "text/javascript; charset=UTF-8")
+ chunk = _utf8(chunk)
+ self._write_buffer.append(chunk)
+
+ def render(self, template_name, **kwargs):
+ """Renders the template with the given arguments as the response."""
+ html = self.render_string(template_name, **kwargs)
+
+ # Insert the additional JS and CSS added by the modules on the page
+ js_embed = []
+ js_files = []
+ css_embed = []
+ css_files = []
+ html_heads = []
+ for module in getattr(self, "_active_modules", {}).itervalues():
+ embed_part = module.embedded_javascript()
+ if embed_part: js_embed.append(_utf8(embed_part))
+ file_part = module.javascript_files()
+ if file_part:
+ if isinstance(file_part, basestring):
+ js_files.append(file_part)
+ else:
+ js_files.extend(file_part)
+ embed_part = module.embedded_css()
+ if embed_part: css_embed.append(_utf8(embed_part))
+ file_part = module.css_files()
+ if file_part:
+ if isinstance(file_part, basestring):
+ css_files.append(file_part)
+ else:
+ css_files.extend(file_part)
+ head_part = module.html_head()
+ if head_part: html_heads.append(_utf8(head_part))
+ if js_embed:
+ js_embed = '<script type="text/javascript">\n//<![CDATA[\n' + \
+ '\n'.join(js_embed) + '\n//]]>\n</script>'
+ sloc = html.rindex('</body>')
+ html = html[:sloc] + js_embed + '\n' + html[sloc:]
+ if js_files:
+ paths = set()
+ for path in js_files:
+ if not path.startswith("/") and not path.startswith("http:"):
+ paths.add(self.static_url(path))
+ else:
+ paths.add(path)
+ js_embed = ''.join('<script src="' + escape.xhtml_escape(p) +
+ '" type="text/javascript"></script>'
+ for p in paths)
+ sloc = html.rindex('</body>')
+ html = html[:sloc] + js_embed + '\n' + html[sloc:]
+ if css_embed:
+ css_embed = '<style type="text/css">\n' + '\n'.join(css_embed) + \
+ '\n</style>'
+ hloc = html.index('</head>')
+ html = html[:hloc] + css_embed + '\n' + html[hloc:]
+ if css_files:
+ paths = set()
+ for path in css_files:
+ if not path.startswith("/") and not path.startswith("http:"):
+ paths.add(self.static_url(path))
+ else:
+ paths.add(path)
+ css_embed = ''.join('<link href="' + escape.xhtml_escape(p) + '" '
+ 'type="text/css" rel="stylesheet"/>'
+ for p in paths)
+ hloc = html.index('</head>')
+ html = html[:hloc] + css_embed + '\n' + html[hloc:]
+ if html_heads:
+ hloc = html.index('</head>')
+ html = html[:hloc] + ''.join(html_heads) + '\n' + html[hloc:]
+
+ self.finish(html)
+
+ def render_string(self, template_name, **kwargs):
+ """Generate the given template with the given arguments.
+
+ We return the generated string. To generate and write a template
+ as a response, use render() above.
+ """
+ # If no template_path is specified, use the path of the calling file
+ template_path = self.application.settings.get("template_path")
+ if not template_path:
+ frame = sys._getframe(0)
+ web_file = frame.f_code.co_filename
+ while frame.f_code.co_filename == web_file:
+ frame = frame.f_back
+ template_path = os.path.dirname(frame.f_code.co_filename)
+ if not getattr(RequestHandler, "_templates", None):
+ RequestHandler._templates = {}
+ if template_path not in RequestHandler._templates:
+ RequestHandler._templates[template_path] = template.Loader(
+ template_path)
+ t = RequestHandler._templates[template_path].load(template_name)
+ args = dict(
+ handler=self,
+ request=self.request,
+ current_user=self.current_user,
+ locale=self.locale,
+ _=self.locale.translate,
+ static_url=self.static_url,
+ xsrf_form_html=self.xsrf_form_html,
+ )
+ args.update(self.ui)
+ args.update(kwargs)
+ return t.generate(**args)
+
+ def flush(self, include_footers=False):
+ """Flushes the current output buffer to the nextwork."""
+ if self.application._wsgi:
+ raise Exception("WSGI applications do not support flush()")
+ if not self._headers_written:
+ self._headers_written = True
+ headers = self._generate_headers()
+ else:
+ headers = ""
+
+ # Ignore the chunk and only write the headers for HEAD requests
+ if self.request.method == "HEAD":
+ if headers: self.request.write(headers)
+ return
+
+ if self._write_buffer:
+ chunk = "".join(self._write_buffer)
+ self._write_buffer = []
+ if chunk:
+ # Don't write out empty chunks because that means
+ # END-OF-STREAM with chunked encoding
+ for transform in self._transforms:
+ chunk = transform.transform_chunk(chunk)
+ else:
+ chunk = ""
+ if include_footers:
+ footers = []
+ for transform in self._transforms:
+ footer = transform.footer()
+ if footer: chunk += footer
+
+ if headers or chunk:
+ self.request.write(headers + chunk)
+
+ def finish(self, chunk=None):
+ """Finishes this response, ending the HTTP request."""
+ assert not self._finished
+ if chunk: self.write(chunk)
+
+ # Automatically support ETags and add the Content-Length header if
+ # we have not flushed any content yet.
+ if not self._headers_written:
+ if self._status_code == 200 and self.request.method == "GET":
+ hasher = hashlib.sha1()
+ for part in self._write_buffer:
+ hasher.update(part)
+ etag = '"%s"' % hasher.hexdigest()
+ inm = self.request.headers.get("If-None-Match")
+ if inm and inm.find(etag) != -1:
+ self._write_buffer = []
+ self.set_status(304)
+ else:
+ self.set_header("Etag", etag)
+ if "Content-Length" not in self._headers:
+ content_length = sum(len(part) for part in self._write_buffer)
+ self.set_header("Content-Length", content_length)
+
+ if not self.application._wsgi:
+ self.flush(include_footers=True)
+ self.request.finish()
+ self._log()
+ self._finished = True
+
+ def send_error(self, status_code=500):
+ """Sends the given HTTP error code to the browser.
+
+ We also send the error HTML for the given error code as returned by
+ get_error_html. Override that method if you want custom error pages
+ for your application.
+ """
+ if self._headers_written:
+ logging.error("Cannot send error response after headers written")
+ if not self._finished:
+ self.finish()
+ return
+ self.clear()
+ self.set_status(status_code)
+ message = self.get_error_html(status_code)
+ self.finish(message)
+
+ def get_error_html(self, status_code):
+ """Override to implement custom error pages."""
+ return "<html><title>%(code)d: %(message)s</title>" \
+ "<body>%(code)d: %(message)s</body></html>" % {
+ "code": status_code,
+ "message": httplib.responses[status_code],
+ }
+
+ @property
+ def locale(self):
+ """The local for the current session.
+
+ Determined by either get_user_locale, which you can override to
+ set the locale based on, e.g., a user preference stored in a
+ database, or get_browser_locale, which uses the Accept-Language
+ header.
+ """
+ if not hasattr(self, "_locale"):
+ self._locale = self.get_user_locale()
+ if not self._locale:
+ self._locale = self.get_browser_locale()
+ assert self._locale
+ return self._locale
+
+ def get_user_locale(self):
+ """Override to determine the locale from the authenticated user.
+
+ If None is returned, we use the Accept-Language header.
+ """
+ return None
+
+ def get_browser_locale(self, default="en_US"):
+ """Determines the user's locale from Accept-Language header.
+
+ See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4
+ """
+ if "Accept-Language" in self.request.headers:
+ languages = self.request.headers["Accept-Language"].split(",")
+ locales = []
+ for language in languages:
+ parts = language.strip().split(";")
+ if len(parts) > 1 and parts[1].startswith("q="):
+ try:
+ score = float(parts[1][2:])
+ except (ValueError, TypeError):
+ score = 0.0
+ else:
+ score = 1.0
+ locales.append((parts[0], score))
+ if locales:
+ locales.sort(key=lambda (l, s): s, reverse=True)
+ codes = [l[0] for l in locales]
+ return locale.get(*codes)
+ return locale.get(default)
+
+ @property
+ def current_user(self):
+ """The authenticated user for this request.
+
+ Determined by either get_current_user, which you can override to
+ set the user based on, e.g., a cookie. If that method is not
+ overridden, this method always returns None.
+
+ We lazy-load the current user the first time this method is called
+ and cache the result after that.
+ """
+ if not hasattr(self, "_current_user"):
+ self._current_user = self.get_current_user()
+ return self._current_user
+
+ def get_current_user(self):
+ """Override to determine the current user from, e.g., a cookie."""
+ return None
+
+ def get_login_url(self):
+ """Override to customize the login URL based on the request.
+
+ By default, we use the 'login_url' application setting.
+ """
+ self.require_setting("login_url", "@tornado.web.authenticated")
+ return self.application.settings["login_url"]
+
+ @property
+ def xsrf_token(self):
+ """The XSRF-prevention token for the current user/session.
+
+ To prevent cross-site request forgery, we set an '_xsrf' cookie
+ and include the same '_xsrf' value as an argument with all POST
+ requests. If the two do not match, we reject the form submission
+ as a potential forgery.
+
+ See http://en.wikipedia.org/wiki/Cross-site_request_forgery
+ """
+ if not hasattr(self, "_xsrf_token"):
+ token = self.get_cookie("_xsrf")
+ if not token:
+ token = binascii.b2a_hex(uuid.uuid4().bytes)
+ expires_days = 30 if self.current_user else None
+ self.set_cookie("_xsrf", token, expires_days=expires_days)
+ self._xsrf_token = token
+ return self._xsrf_token
+
+ def check_xsrf_cookie(self):
+ """Verifies that the '_xsrf' cookie matches the '_xsrf' argument.
+
+ To prevent cross-site request forgery, we set an '_xsrf' cookie
+ and include the same '_xsrf' value as an argument with all POST
+ requests. If the two do not match, we reject the form submission
+ as a potential forgery.
+
+ See http://en.wikipedia.org/wiki/Cross-site_request_forgery
+ """
+ token = self.get_argument("_xsrf", None)
+ if not token:
+ raise HTTPError(403, "'_xsrf' argument missing from POST")
+ if self.xsrf_token != token:
+ raise HTTPError(403, "XSRF cookie does not match POST argument")
+
+ def xsrf_form_html(self):
+ """An HTML <input/> element to be included with all POST forms.
+
+ It defines the _xsrf input value, which we check on all POST
+ requests to prevent cross-site request forgery. If you have set
+ the 'xsrf_cookies' application setting, you must include this
+ HTML within all of your HTML forms.
+
+ See check_xsrf_cookie() above for more information.
+ """
+ return '<input type="hidden" name="_xsrf" value="' + \
+ escape.xhtml_escape(self.xsrf_token) + '"/>'
+
+ def static_url(self, path):
+ """Returns a static URL for the given relative static file path.
+
+ This method requires you set the 'static_path' setting in your
+ application (which specifies the root directory of your static
+ files).
+
+ We append ?v=<signature> to the returned URL, which makes our
+ static file handler set an infinite expiration header on the
+ returned content. The signature is based on the content of the
+ file.
+
+ If this handler has a "include_host" attribute, we include the
+ full host for every static URL, including the "http://". Set
+ this attribute for handlers whose output needs non-relative static
+ path names.
+ """
+ self.require_setting("static_path", "static_url")
+ if not hasattr(RequestHandler, "_static_hashes"):
+ RequestHandler._static_hashes = {}
+ hashes = RequestHandler._static_hashes
+ if path not in hashes:
+ try:
+ f = open(os.path.join(
+ self.application.settings["static_path"], path))
+ hashes[path] = hashlib.md5(f.read()).hexdigest()
+ f.close()
+ except:
+ logging.error("Could not open static file %r", path)
+ hashes[path] = None
+ base = self.request.protocol + "://" + self.request.host \
+ if getattr(self, "include_host", False) else ""
+ if hashes.get(path):
+ return base + "/static/" + path + "?v=" + hashes[path][:5]
+ else:
+ return base + "/static/" + path
+
+ def async_callback(self, callback, *args, **kwargs):
+ """Wrap callbacks with this if they are used on asynchronous requests.
+
+ Catches exceptions and properly finishes the request.
+ """
+ if callback is None:
+ return None
+ if args or kwargs:
+ callback = functools.partial(callback, *args, **kwargs)
+ def wrapper(*args, **kwargs):
+ try:
+ return callback(*args, **kwargs)
+ except Exception, e:
+ if self._headers_written:
+ logging.error("Exception after headers written",
+ exc_info=True)
+ else:
+ self._handle_request_exception(e)
+ return wrapper
+
+ def require_setting(self, name, feature="this feature"):
+ """Raises an exception if the given app setting is not defined."""
+ if not self.application.settings.get(name):
+ raise Exception("You must define the '%s' setting in your "
+ "application to use %s" % (name, feature))
+
+ def _execute(self, transforms, *args, **kwargs):
+ """Executes this request with the given output transforms."""
+ self._transforms = transforms
+ try:
+ if self.request.method not in self.SUPPORTED_METHODS:
+ raise HTTPError(405)
+ # If XSRF cookies are turned on, reject form submissions without
+ # the proper cookie
+ if self.request.method == "POST" and \
+ self.application.settings.get("xsrf_cookies"):
+ self.check_xsrf_cookie()
+ self.prepare()
+ if not self._finished:
+ getattr(self, self.request.method.lower())(*args, **kwargs)
+ if self._auto_finish and not self._finished:
+ self.finish()
+ except Exception, e:
+ self._handle_request_exception(e)
+
+ def _generate_headers(self):
+ for transform in self._transforms:
+ headers = transform.transform_headers(self._headers)
+ lines = [self.request.version + " " + str(self._status_code) + " " +
+ httplib.responses[self._status_code]]
+ lines.extend(["%s: %s" % (n, v) for n, v in self._headers.iteritems()])
+ for cookie_dict in getattr(self, "_new_cookies", []):
+ for cookie in cookie_dict.values():
+ lines.append("Set-Cookie: " + cookie.OutputString(None))
+ return "\r\n".join(lines) + "\r\n\r\n"
+
+ def _log(self):
+ if self._status_code < 400:
+ log_method = logging.info
+ elif self._status_code < 500:
+ log_method = logging.warning
+ else:
+ log_method = logging.error
+ request_time = 1000.0 * self.request.request_time()
+ log_method("%d %s %.2fms", self._status_code,
+ self._request_summary(), request_time)
+
+ def _request_summary(self):
+ return self.request.method + " " + self.request.uri + " (" + \
+ self.request.remote_ip + ")"
+
+ def _handle_request_exception(self, e):
+ if isinstance(e, HTTPError):
+ if e.log_message:
+ format = "%d %s: " + e.log_message
+ args = [e.status_code, self._request_summary()] + list(e.args)
+ logging.warning(format, *args)
+ if e.status_code not in httplib.responses:
+ logging.error("Bad HTTP status code: %d", e.status_code)
+ self.send_error(500)
+ else:
+ self.send_error(e.status_code)
+ else:
+ logging.error("Uncaught exception %s\n%r", self._request_summary(),
+ self.request, exc_info=e)
+ self.send_error(500)
+
+ def _ui_module(self, name, module):
+ def render(*args, **kwargs):
+ if not hasattr(self, "_active_modules"):
+ self._active_modules = {}
+ if name not in self._active_modules:
+ self._active_modules[name] = module(self)
+ rendered = self._active_modules[name].render(*args, **kwargs)
+ return rendered
+ return render
+
+ def _ui_method(self, method):
+ return lambda *args, **kwargs: method(self, *args, **kwargs)
+
+
+def asynchronous(method):
+ """Wrap request handler methods with this if they are asynchronous.
+
+ If this decorator is given, the response is not finished when the
+ method returns. It is up to the request handler to call self.finish()
+ to finish the HTTP request. Without this decorator, the request is
+ automatically finished when the get() or post() method returns.
+
+ class MyRequestHandler(web.RequestHandler):
+ @web.asynchronous
+ def get(self):
+ http = httpclient.AsyncHTTPClient()
+ http.fetch("http://friendfeed.com/", self._on_download)
+
+ def _on_download(self, response):
+ self.write("Downloaded!")
+ self.finish()
+
+ """
+ @functools.wraps(method)
+ def wrapper(self, *args, **kwargs):
+ if self.application._wsgi:
+ raise Exception("@asynchronous is not supported for WSGI apps")
+ self._auto_finish = False
+ return method(self, *args, **kwargs)
+ return wrapper
+
+
+class Application(object):
+ """A collection of request handlers that make up a web application.
+
+ Instances of this class are callable and can be passed directly to
+ HTTPServer to serve the application:
+
+ application = web.Application([
+ (r"/", MainPageHandler),
+ ])
+ http_server = httpserver.HTTPServer(application)
+ http_server.listen(8080)
+ ioloop.IOLoop.instance().start()
+
+ The constructor for this class takes in a list of (regexp, request_class)
+ tuples. When we receive requests, we iterate over the list in order and
+ instantiate an instance of the first request class whose regexp matches
+ the request path.
+
+ Each tuple can contain an optional third element, which should be a
+ dictionary if it is present. That dictionary is passed as keyword
+ arguments to the contructor of the handler. This pattern is used
+ for the StaticFileHandler below:
+
+ application = web.Application([
+ (r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
+ ])
+
+ We support virtual hosts with the add_handlers method, which takes in
+ a host regular expression as the first argument:
+
+ application.add_handlers(r"www\.myhost\.com", [
+ (r"/article/([0-9]+)", ArticleHandler),
+ ])
+
+ You can serve static files by sending the static_path setting as a
+ keyword argument. We will serve those files from the /static/ URI,
+ and we will serve /favicon.ico and /robots.txt from the same directory.
+ """
+ def __init__(self, handlers=None, default_host="", transforms=None,
+ **settings):
+ if transforms is None:
+ self.transforms = [ChunkedTransferEncoding]
+ else:
+ self.transforms = transforms
+ self.handlers = []
+ self.default_host = default_host
+ self.settings = settings
+ self.ui_modules = {}
+ self.ui_methods = {}
+ self._wsgi = False
+ self._load_ui_modules(settings.get("ui_modules", {}))
+ self._load_ui_methods(settings.get("ui_methods", {}))
+ if self.settings.get("static_path"):
+ path = self.settings["static_path"]
+ handlers = list(handlers or [])
+ handlers.extend([
+ (r"/static/(.*)", StaticFileHandler, dict(path=path)),
+ (r"/(favicon\.ico)", StaticFileHandler, dict(path=path)),
+ (r"/(robots\.txt)", StaticFileHandler, dict(path=path)),
+ ])
+ if handlers: self.add_handlers(".*$", handlers)
+
+ def add_handlers(self, host_pattern, host_handlers):
+ """Appends the given handlers to our handler list."""
+ if not host_pattern.endswith("$"):
+ host_pattern += "$"
+ handlers = []
+ self.handlers.append((re.compile(host_pattern), handlers))
+
+ for handler_tuple in host_handlers:
+ assert len(handler_tuple) in (2, 3)
+ pattern = handler_tuple[0]
+ handler = handler_tuple[1]
+ if len(handler_tuple) == 3:
+ kwargs = handler_tuple[2]
+ else:
+ kwargs = {}
+ if not pattern.endswith("$"):
+ pattern += "$"
+ handlers.append((re.compile(pattern), handler, kwargs))
+
+ def add_transform(self, transform_class):
+ """Adds the given OutputTransform to our transform list."""
+ self.transforms.append(transform_class)
+
+ def _get_host_handlers(self, request):
+ host = request.host.lower().split(':')[0]
+ for pattern, handlers in self.handlers:
+ if pattern.match(host):
+ return handlers
+ # Look for default host if not behind load balancer (for debugging)
+ if "X-Real-Ip" not in request.headers:
+ for pattern, handlers in self.handlers:
+ if pattern.match(self.default_host):
+ return handlers
+ return None
+
+ def _load_ui_methods(self, methods):
+ if type(methods) is types.ModuleType:
+ self._load_ui_methods(dict((n, getattr(methods, n))
+ for n in dir(methods)))
+ elif isinstance(methods, list):
+ for m in list: self._load_ui_methods(m)
+ else:
+ for name, fn in methods.iteritems():
+ if not name.startswith("_") and hasattr(fn, "__call__") \
+ and name[0].lower() == name[0]:
+ self.ui_methods[name] = fn
+
+ def _load_ui_modules(self, modules):
+ if type(modules) is types.ModuleType:
+ self._load_ui_modules(dict((n, getattr(modules, n))
+ for n in dir(modules)))
+ elif isinstance(modules, list):
+ for m in list: self._load_ui_modules(m)
+ else:
+ assert isinstance(modules, dict)
+ for name, cls in modules.iteritems():
+ try:
+ if issubclass(cls, UIModule):
+ self.ui_modules[name] = cls
+ except TypeError:
+ pass
+
+ def __call__(self, request):
+ """Called by HTTPServer to execute the request."""
+ transforms = [t(request) for t in self.transforms]
+ handler = None
+ args = []
+ handlers = self._get_host_handlers(request)
+ if not handlers:
+ handler = RedirectHandler(
+ request, "http://" + self.default_host + "/")
+ else:
+ for pattern, handler_class, kwargs in handlers:
+ match = pattern.match(request.path)
+ if match:
+ handler = handler_class(self, request, **kwargs)
+ args = match.groups()
+ break
+ if not handler:
+ handler = ErrorHandler(self, request, 404)
+
+ # In debug mode, re-compile templates and reload static files on every
+ # request so you don't need to restart to see changes
+ if self.settings.get("debug"):
+ RequestHandler._templates = None
+ RequestHandler._static_hashes = {}
+
+ handler._execute(transforms, *args)
+ return handler
+
+
+class HTTPError(Exception):
+ """An exception that will turn into an HTTP error response."""
+ def __init__(self, status_code, log_message=None, *args):
+ self.status_code = status_code
+ self.log_message = log_message
+ self.args = args
+
+ def __str__(self):
+ message = "HTTP %d: %s" % (
+ self.status_code, httplib.responses[self.status_code])
+ if self.log_message:
+ return message + " (" + (self.log_message % self.args) + ")"
+ else:
+ return message
+
+
+class ErrorHandler(RequestHandler):
+ """Generates an error response with status_code for all requests."""
+ def __init__(self, application, request, status_code):
+ RequestHandler.__init__(self, application, request)
+ self.set_status(status_code)
+
+ def prepare(self):
+ raise HTTPError(self._status_code)
+
+
+class RedirectHandler(RequestHandler):
+ """Redirects the client to the given URL for all GET requests.
+
+ You should provide the keyword argument "url" to the handler, e.g.:
+
+ application = web.Application([
+ (r"/oldpath", web.RedirectHandler, {"url": "/newpath"}),
+ ])
+ """
+ def __init__(self, application, request, url, permanent=True):
+ RequestHandler.__init__(self, application, request)
+ self._url = url
+ self._permanent = permanent
+
+ def get(self):
+ self.redirect(self._url, permanent=self._permanent)
+
+
+class StaticFileHandler(RequestHandler):
+ """A simple handler that can serve static content from a directory.
+
+ To map a path to this handler for a static data directory /var/www,
+ you would add a line to your application like:
+
+ application = web.Application([
+ (r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}),
+ ])
+
+ The local root directory of the content should be passed as the "path"
+ argument to the handler.
+
+ To support aggressive browser caching, if the argument "v" is given
+ with the path, we set an infinite HTTP expiration header. So, if you
+ want browsers to cache a file indefinitely, send them to, e.g.,
+ /static/images/myimage.png?v=xxx.
+ """
+ def __init__(self, application, request, path):
+ RequestHandler.__init__(self, application, request)
+ self.root = os.path.abspath(path) + "/"
+
+ def head(self, path):
+ self.get(path, include_body=False)
+
+ def get(self, path, include_body=True):
+ abspath = os.path.abspath(os.path.join(self.root, path))
+ if not abspath.startswith(self.root):
+ raise HTTPError(403, "%s is not in root static directory", path)
+ if not os.path.exists(abspath):
+ raise HTTPError(404)
+ if not os.path.isfile(abspath):
+ raise HTTPError(403, "%s is not a file", path)
+
+ # Check the If-Modified-Since, and don't send the result if the
+ # content has not been modified
+ stat_result = os.stat(abspath)
+ modified = datetime.datetime.fromtimestamp(stat_result[stat.ST_MTIME])
+ ims_value = self.request.headers.get("If-Modified-Since")
+ if ims_value is not None:
+ date_tuple = email.utils.parsedate(ims_value)
+ if_since = datetime.datetime.fromtimestamp(time.mktime(date_tuple))
+ if if_since >= modified:
+ self.set_status(304)
+ return
+
+ self.set_header("Last-Modified", modified)
+ self.set_header("Content-Length", stat_result[stat.ST_SIZE])
+ if "v" in self.request.arguments:
+ self.set_header("Expires", datetime.datetime.utcnow() + \
+ datetime.timedelta(days=365*10))
+ self.set_header("Cache-Control", "max-age=" + str(86400*365*10))
+ else:
+ self.set_header("Cache-Control", "public")
+ mime_type, encoding = mimetypes.guess_type(abspath)
+ if mime_type:
+ self.set_header("Content-Type", mime_type)
+
+ if not include_body:
+ return
+ file = open(abspath, "r")
+ try:
+ self.write(file.read())
+ finally:
+ file.close()
+
+
+class OutputTransform(object):
+ """A transform modifies the result of an HTTP request (e.g., GZip encoding)
+
+ A new transform instance is created for every request. The sequence of
+ calls is:
+
+ t = Transform(request) # Constructor
+ # Request processing
+ headers = t.transform_headers(headers)
+ # Write headers
+ for block in result:
+ write(t.transform_chunk(block)
+ write(t.footer())
+
+ See the ChunkedTransferEncoding example below if you want to implement a
+ new Transform.
+ """
+ def __init__(self, request):
+ pass
+
+ def transform_headers(self, headers):
+ return headers
+
+ def transform_chunk(self, block):
+ return block
+
+ def footer(self):
+ return None
+
+
+class ChunkedTransferEncoding(OutputTransform):
+ """Applies the chunked transfer encoding to the response.
+
+ See http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1
+ """
+ def __init__(self, request):
+ self._chunking = request.supports_http_1_1()
+
+ def transform_headers(self, headers):
+ if self._chunking:
+ # No need to chunk the output if a Content-Length is specified
+ if "Content-Length" in headers or "Transfer-Encoding" in headers:
+ self._chunking = False
+ else:
+ headers["Transfer-Encoding"] = "chunked"
+ return headers
+
+ def transform_chunk(self, block):
+ if self._chunking:
+ return ("%x" % len(block)) + "\r\n" + block + "\r\n"
+ else:
+ return block
+
+ def footer(self):
+ if self._chunking:
+ return "0\r\n\r\n"
+ else:
+ return None
+
+
+def authenticated(method):
+ """Decorate methods with this to require that the user be logged in."""
+ @functools.wraps(method)
+ def wrapper(self, *args, **kwargs):
+ if not self.current_user:
+ if self.request.method == "GET":
+ url = self.get_login_url()
+ if "?" not in url:
+ url += "?" + urllib.urlencode(dict(next=self.request.uri))
+ self.redirect(url)
+ return
+ raise HTTPError(403)
+ return method(self, *args, **kwargs)
+ return wrapper
+
+
+class UIModule(object):
+ """A UI re-usable, modular unit on a page.
+
+ UI modules often execute additional queries, and they can include
+ additional CSS and JavaScript that will be included in the output
+ page, which is automatically inserted on page render.
+ """
+ def __init__(self, handler):
+ self.handler = handler
+ self.request = handler.request
+ self.ui = handler.ui
+ self.current_user = handler.current_user
+ self.locale = handler.locale
+
+ def render(self, *args, **kwargs):
+ raise NotImplementedError()
+
+ def embedded_javascript(self):
+ """Returns a JavaScript string that will be embedded in the page."""
+ return None
+
+ def javascript_files(self):
+ """Returns a list of JavaScript files required by this module."""
+ return None
+
+ def embedded_css(self):
+ """Returns a CSS string that will be embedded in the page."""
+ return None
+
+ def css_files(self):
+ """Returns a list of JavaScript files required by this module."""
+ return None
+
+ def html_head(self):
+ """Returns a CSS string that will be put in the <head/> element"""
+ return None
+
+ def render_string(self, path, **kwargs):
+ return self.handler.render_string(path, **kwargs)
+
+
+def _utf8(s):
+ if isinstance(s, unicode):
+ return s.encode("utf-8")
+ assert isinstance(s, str)
+ return s
+
+
+def _unicode(s):
+ if isinstance(s, str):
+ try:
+ return s.decode("utf-8")
+ except UnicodeDecodeError:
+ raise HTTPError(400, "Non-utf8 argument")
+ assert isinstance(s, unicode)
+ return s
+
+
+class _O(dict):
+ """Makes a dictionary behave like an object."""
+ def __getattr__(self, name):
+ try:
+ return self[name]
+ except KeyError:
+ raise AttributeError(name)
+
+ def __setattr__(self, name, value):
+ self[name] = value
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Facebook
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""WSGI support for the Tornado web framework.
+
+We export WSGIApplication, which is very similar to web.Application, except
+no asynchronous methods are supported (since WSGI does not support
+non-blocking requests properly). If you call self.flush() or other
+asynchronous methods in your request handlers running in a WSGIApplication,
+we throw an exception.
+
+Example usage:
+
+ import tornado.web
+ import tornado.wsgi
+ import wsgiref.simple_server
+
+ class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ self.write("Hello, world")
+
+ if __name__ == "__main__":
+ application = tornado.wsgi.WSGIApplication([
+ (r"/", MainHandler),
+ ])
+ server = wsgiref.simple_server.make_server('', 8888, application)
+ server.serve_forever()
+
+See the 'appengine' demo for an example of using this module to run
+a Tornado app on Google AppEngine.
+
+Since no asynchronous methods are available for WSGI applications, the
+httpclient and auth modules are both not available for WSGI applications.
+"""
+
+import cgi
+import httplib
+import logging
+import time
+import urllib
+import web
+
+
+class WSGIApplication(web.Application):
+ """A WSGI-equivalent of web.Application.
+
+ We support the same interface, but handlers running in a WSGIApplication
+ do not support flush() or asynchronous methods.
+ """
+ def __init__(self, handlers=None, default_host="", **settings):
+ web.Application.__init__(self, handlers, default_host, transforms=[],
+ **settings)
+ self._wsgi = True
+
+ def __call__(self, environ, start_response):
+ handler = web.Application.__call__(self, HTTPRequest(environ))
+ assert handler._finished
+ status = str(handler._status_code) + " " + \
+ httplib.responses[handler._status_code]
+ headers = handler._headers.items()
+ for cookie_dict in getattr(handler, "_new_cookies", []):
+ for cookie in cookie_dict.values():
+ headers.append(("Set-Cookie", cookie.OutputString(None)))
+ start_response(status, headers)
+ return handler._write_buffer
+
+
+class HTTPRequest(object):
+ """Mimics httpserver.HTTPRequest for WSGI applications."""
+ def __init__(self, environ):
+ """Parses the given WSGI environ to construct the request."""
+ self.method = environ["REQUEST_METHOD"]
+ self.path = urllib.quote(environ.get("SCRIPT_NAME", ""))
+ self.path += urllib.quote(environ.get("PATH_INFO", ""))
+ self.uri = self.path
+ self.arguments = {}
+ self.query = environ.get("QUERY_STRING", "")
+ if self.query:
+ self.uri += "?" + self.query
+ arguments = cgi.parse_qs(self.query)
+ for name, values in arguments.iteritems():
+ values = [v for v in values if v]
+ if values: self.arguments[name] = values
+ self.version = "HTTP/1.1"
+ self.headers = HTTPHeaders()
+ if environ.get("CONTENT_TYPE"):
+ self.headers["Content-Type"] = environ["CONTENT_TYPE"]
+ if environ.get("CONTENT_LENGTH"):
+ self.headers["Content-Length"] = int(environ["CONTENT_LENGTH"])
+ for key in environ:
+ if key.startswith("HTTP_"):
+ self.headers[key[5:].replace("_", "-")] = environ[key]
+ if self.headers.get("Content-Length"):
+ self.body = environ["wsgi.input"].read()
+ else:
+ self.body = ""
+ self.protocol = environ["wsgi.url_scheme"]
+ self.remote_ip = environ.get("REMOTE_ADDR", "")
+ if environ.get("HTTP_HOST"):
+ self.host = environ["HTTP_HOST"]
+ else:
+ self.host = environ["SERVER_NAME"]
+
+ # Parse request body
+ self.files = {}
+ content_type = self.headers.get("Content-Type", "")
+ if content_type.startswith("application/x-www-form-urlencoded"):
+ for name, values in cgi.parse_qs(self.body).iteritems():
+ self.arguments.setdefault(name, []).extend(values)
+ elif content_type.startswith("multipart/form-data"):
+ boundary = content_type[30:]
+ if boundary: self._parse_mime_body(boundary, data)
+
+ self._start_time = time.time()
+ self._finish_time = None
+
+ def supports_http_1_1(self):
+ """Returns True if this request supports HTTP/1.1 semantics"""
+ return self.version == "HTTP/1.1"
+
+ def full_url(self):
+ """Reconstructs the full URL for this request."""
+ return self.protocol + "://" + self.host + self.uri
+
+ def request_time(self):
+ """Returns the amount of time it took for this request to execute."""
+ if self._finish_time is None:
+ return time.time() - self._start_time
+ else:
+ return self._finish_time - self._start_time
+
+ def _parse_mime_body(self, boundary):
+ if self.body.endswith("\r\n"):
+ footer_length = len(boundary) + 6
+ else:
+ footer_length = len(boundary) + 4
+ parts = self.body[:-footer_length].split("--" + boundary + "\r\n")
+ for part in parts:
+ if not part: continue
+ eoh = part.find("\r\n\r\n")
+ if eoh == -1:
+ logging.warning("multipart/form-data missing headers")
+ continue
+ headers = HTTPHeaders.parse(part[:eoh])
+ name_header = headers.get("Content-Disposition", "")
+ if not name_header.startswith("form-data;") or \
+ not part.endswith("\r\n"):
+ logging.warning("Invalid multipart/form-data")
+ continue
+ value = part[eoh + 4:-2]
+ name_values = {}
+ for name_part in name_header[10:].split(";"):
+ name, name_value = name_part.strip().split("=", 1)
+ name_values[name] = name_value.strip('"').decode("utf-8")
+ if not name_values.get("name"):
+ logging.warning("multipart/form-data value missing name")
+ continue
+ name = name_values["name"]
+ if name_values.get("filename"):
+ ctype = headers.get("Content-Type", "application/unknown")
+ self.files.setdefault(name, []).append(dict(
+ filename=name_values["filename"], body=value,
+ content_type=ctype))
+ else:
+ self.arguments.setdefault(name, []).append(value)
+
+
+class HTTPHeaders(dict):
+ """A dictionary that maintains Http-Header-Case for all keys."""
+ def __setitem__(self, name, value):
+ dict.__setitem__(self, self._normalize_name(name), value)
+
+ def __getitem__(self, name):
+ return dict.__getitem__(self, self._normalize_name(name))
+
+ def _normalize_name(self, name):
+ return intern("-".join([w.capitalize() for w in name.split("-")]))
--- /dev/null
+application: python-tornado
+version: 1
+runtime: python
+api_version: 1
+
+handlers:
+- url: /static/
+ static_dir: static
+
+- url: /robots\.txt
+ static_files: static/robots.txt
+ upload: static/robots.txt
+
+- url: /favicon\.ico
+ static_files: static/favicon.ico
+ upload: static/favicon.ico
+
+- url: /.*
+ script: website.py
--- /dev/null
+"""
+Python Markdown
+===============
+
+Python Markdown converts Markdown to HTML and can be used as a library or
+called from the command line.
+
+## Basic usage as a module:
+
+ import markdown
+ md = Markdown()
+ html = md.convert(your_text_string)
+
+## Basic use from the command line:
+
+ python markdown.py source.txt > destination.html
+
+Run "python markdown.py --help" to see more options.
+
+## Extensions
+
+See <http://www.freewisdom.org/projects/python-markdown/> for more
+information and instructions on how to extend the functionality of
+Python Markdown. Read that before you try modifying this file.
+
+## Authors and License
+
+Started by [Manfred Stienstra](http://www.dwerg.net/). Continued and
+maintained by [Yuri Takhteyev](http://www.freewisdom.org), [Waylan
+Limberg](http://achinghead.com/) and [Artem Yunusov](http://blog.splyer.com).
+
+Contact: markdown@freewisdom.org
+
+Copyright 2007, 2008 The Python Markdown Project (v. 1.7 and later)
+Copyright 200? Django Software Foundation (OrderedDict implementation)
+Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b)
+Copyright 2004 Manfred Stienstra (the original version)
+
+License: BSD (see docs/LICENSE for details).
+"""
+
+version = "2.0"
+version_info = (2,0,0, "Final")
+
+import re
+import codecs
+import sys
+import warnings
+import logging
+from logging import DEBUG, INFO, WARN, ERROR, CRITICAL
+
+
+"""
+CONSTANTS
+=============================================================================
+"""
+
+"""
+Constants you might want to modify
+-----------------------------------------------------------------------------
+"""
+
+# default logging level for command-line use
+COMMAND_LINE_LOGGING_LEVEL = CRITICAL
+TAB_LENGTH = 4 # expand tabs to this many spaces
+ENABLE_ATTRIBUTES = True # @id = xyz -> <... id="xyz">
+SMART_EMPHASIS = True # this_or_that does not become this<i>or</i>that
+DEFAULT_OUTPUT_FORMAT = 'xhtml1' # xhtml or html4 output
+HTML_REMOVED_TEXT = "[HTML_REMOVED]" # text used instead of HTML in safe mode
+BLOCK_LEVEL_ELEMENTS = re.compile("p|div|h[1-6]|blockquote|pre|table|dl|ol|ul"
+ "|script|noscript|form|fieldset|iframe|math"
+ "|ins|del|hr|hr/|style|li|dt|dd|thead|tbody"
+ "|tr|th|td")
+DOC_TAG = "div" # Element used to wrap document - later removed
+
+# Placeholders
+STX = u'\u0002' # Use STX ("Start of text") for start-of-placeholder
+ETX = u'\u0003' # Use ETX ("End of text") for end-of-placeholder
+INLINE_PLACEHOLDER_PREFIX = STX+"klzzwxh:"
+INLINE_PLACEHOLDER = INLINE_PLACEHOLDER_PREFIX + "%s" + ETX
+AMP_SUBSTITUTE = STX+"amp"+ETX
+
+
+"""
+Constants you probably do not need to change
+-----------------------------------------------------------------------------
+"""
+
+RTL_BIDI_RANGES = ( (u'\u0590', u'\u07FF'),
+ # Hebrew (0590-05FF), Arabic (0600-06FF),
+ # Syriac (0700-074F), Arabic supplement (0750-077F),
+ # Thaana (0780-07BF), Nko (07C0-07FF).
+ (u'\u2D30', u'\u2D7F'), # Tifinagh
+ )
+
+
+"""
+AUXILIARY GLOBAL FUNCTIONS
+=============================================================================
+"""
+
+
+def message(level, text):
+ """ A wrapper method for logging debug messages. """
+ logger = logging.getLogger('MARKDOWN')
+ if logger.handlers:
+ # The logger is configured
+ logger.log(level, text)
+ if level > WARN:
+ sys.exit(0)
+ elif level > WARN:
+ raise MarkdownException, text
+ else:
+ warnings.warn(text, MarkdownWarning)
+
+
+def isBlockLevel(tag):
+ """Check if the tag is a block level HTML tag."""
+ return BLOCK_LEVEL_ELEMENTS.match(tag)
+
+"""
+MISC AUXILIARY CLASSES
+=============================================================================
+"""
+
+class AtomicString(unicode):
+ """A string which should not be further processed."""
+ pass
+
+
+class MarkdownException(Exception):
+ """ A Markdown Exception. """
+ pass
+
+
+class MarkdownWarning(Warning):
+ """ A Markdown Warning. """
+ pass
+
+
+"""
+OVERALL DESIGN
+=============================================================================
+
+Markdown processing takes place in four steps:
+
+1. A bunch of "preprocessors" munge the input text.
+2. BlockParser() parses the high-level structural elements of the
+ pre-processed text into an ElementTree.
+3. A bunch of "treeprocessors" are run against the ElementTree. One such
+ treeprocessor runs InlinePatterns against the ElementTree, detecting inline
+ markup.
+4. Some post-processors are run against the text after the ElementTree has
+ been serialized into text.
+5. The output is written to a string.
+
+Those steps are put together by the Markdown() class.
+
+"""
+
+import preprocessors
+import blockprocessors
+import treeprocessors
+import inlinepatterns
+import postprocessors
+import blockparser
+import etree_loader
+import odict
+
+# Extensions should use "markdown.etree" instead of "etree" (or do `from
+# markdown import etree`). Do not import it by yourself.
+
+etree = etree_loader.importETree()
+
+# Adds the ability to output html4
+import html4
+
+
+class Markdown:
+ """Convert Markdown to HTML."""
+
+ def __init__(self,
+ extensions=[],
+ extension_configs={},
+ safe_mode = False,
+ output_format=DEFAULT_OUTPUT_FORMAT):
+ """
+ Creates a new Markdown instance.
+
+ Keyword arguments:
+
+ * extensions: A list of extensions.
+ If they are of type string, the module mdx_name.py will be loaded.
+ If they are a subclass of markdown.Extension, they will be used
+ as-is.
+ * extension-configs: Configuration setting for extensions.
+ * safe_mode: Disallow raw html. One of "remove", "replace" or "escape".
+ * output_format: Format of output. Supported formats are:
+ * "xhtml1": Outputs XHTML 1.x. Default.
+ * "xhtml": Outputs latest supported version of XHTML (currently XHTML 1.1).
+ * "html4": Outputs HTML 4
+ * "html": Outputs latest supported version of HTML (currently HTML 4).
+ Note that it is suggested that the more specific formats ("xhtml1"
+ and "html4") be used as "xhtml" or "html" may change in the future
+ if it makes sense at that time.
+
+ """
+
+ self.safeMode = safe_mode
+ self.registeredExtensions = []
+ self.docType = ""
+ self.stripTopLevelTags = True
+
+ # Preprocessors
+ self.preprocessors = odict.OrderedDict()
+ self.preprocessors["html_block"] = \
+ preprocessors.HtmlBlockPreprocessor(self)
+ self.preprocessors["reference"] = \
+ preprocessors.ReferencePreprocessor(self)
+ # footnote preprocessor will be inserted with "<reference"
+
+ # Block processors - ran by the parser
+ self.parser = blockparser.BlockParser()
+ self.parser.blockprocessors['empty'] = \
+ blockprocessors.EmptyBlockProcessor(self.parser)
+ self.parser.blockprocessors['indent'] = \
+ blockprocessors.ListIndentProcessor(self.parser)
+ self.parser.blockprocessors['code'] = \
+ blockprocessors.CodeBlockProcessor(self.parser)
+ self.parser.blockprocessors['hashheader'] = \
+ blockprocessors.HashHeaderProcessor(self.parser)
+ self.parser.blockprocessors['setextheader'] = \
+ blockprocessors.SetextHeaderProcessor(self.parser)
+ self.parser.blockprocessors['hr'] = \
+ blockprocessors.HRProcessor(self.parser)
+ self.parser.blockprocessors['olist'] = \
+ blockprocessors.OListProcessor(self.parser)
+ self.parser.blockprocessors['ulist'] = \
+ blockprocessors.UListProcessor(self.parser)
+ self.parser.blockprocessors['quote'] = \
+ blockprocessors.BlockQuoteProcessor(self.parser)
+ self.parser.blockprocessors['paragraph'] = \
+ blockprocessors.ParagraphProcessor(self.parser)
+
+
+ #self.prePatterns = []
+
+ # Inline patterns - Run on the tree
+ self.inlinePatterns = odict.OrderedDict()
+ self.inlinePatterns["backtick"] = \
+ inlinepatterns.BacktickPattern(inlinepatterns.BACKTICK_RE)
+ self.inlinePatterns["escape"] = \
+ inlinepatterns.SimpleTextPattern(inlinepatterns.ESCAPE_RE)
+ self.inlinePatterns["reference"] = \
+ inlinepatterns.ReferencePattern(inlinepatterns.REFERENCE_RE, self)
+ self.inlinePatterns["link"] = \
+ inlinepatterns.LinkPattern(inlinepatterns.LINK_RE, self)
+ self.inlinePatterns["image_link"] = \
+ inlinepatterns.ImagePattern(inlinepatterns.IMAGE_LINK_RE, self)
+ self.inlinePatterns["image_reference"] = \
+ inlinepatterns.ImageReferencePattern(inlinepatterns.IMAGE_REFERENCE_RE, self)
+ self.inlinePatterns["autolink"] = \
+ inlinepatterns.AutolinkPattern(inlinepatterns.AUTOLINK_RE, self)
+ self.inlinePatterns["automail"] = \
+ inlinepatterns.AutomailPattern(inlinepatterns.AUTOMAIL_RE, self)
+ self.inlinePatterns["linebreak2"] = \
+ inlinepatterns.SubstituteTagPattern(inlinepatterns.LINE_BREAK_2_RE, 'br')
+ self.inlinePatterns["linebreak"] = \
+ inlinepatterns.SubstituteTagPattern(inlinepatterns.LINE_BREAK_RE, 'br')
+ self.inlinePatterns["html"] = \
+ inlinepatterns.HtmlPattern(inlinepatterns.HTML_RE, self)
+ self.inlinePatterns["entity"] = \
+ inlinepatterns.HtmlPattern(inlinepatterns.ENTITY_RE, self)
+ self.inlinePatterns["not_strong"] = \
+ inlinepatterns.SimpleTextPattern(inlinepatterns.NOT_STRONG_RE)
+ self.inlinePatterns["strong_em"] = \
+ inlinepatterns.DoubleTagPattern(inlinepatterns.STRONG_EM_RE, 'strong,em')
+ self.inlinePatterns["strong"] = \
+ inlinepatterns.SimpleTagPattern(inlinepatterns.STRONG_RE, 'strong')
+ self.inlinePatterns["emphasis"] = \
+ inlinepatterns.SimpleTagPattern(inlinepatterns.EMPHASIS_RE, 'em')
+ self.inlinePatterns["emphasis2"] = \
+ inlinepatterns.SimpleTagPattern(inlinepatterns.EMPHASIS_2_RE, 'em')
+ # The order of the handlers matters!!!
+
+
+ # Tree processors - run once we have a basic parse.
+ self.treeprocessors = odict.OrderedDict()
+ self.treeprocessors["inline"] = treeprocessors.InlineProcessor(self)
+ self.treeprocessors["prettify"] = \
+ treeprocessors.PrettifyTreeprocessor(self)
+
+ # Postprocessors - finishing touches.
+ self.postprocessors = odict.OrderedDict()
+ self.postprocessors["raw_html"] = \
+ postprocessors.RawHtmlPostprocessor(self)
+ self.postprocessors["amp_substitute"] = \
+ postprocessors.AndSubstitutePostprocessor()
+ # footnote postprocessor will be inserted with ">amp_substitute"
+
+ # Map format keys to serializers
+ self.output_formats = {
+ 'html' : html4.to_html_string,
+ 'html4' : html4.to_html_string,
+ 'xhtml' : etree.tostring,
+ 'xhtml1': etree.tostring,
+ }
+
+ self.references = {}
+ self.htmlStash = preprocessors.HtmlStash()
+ self.registerExtensions(extensions = extensions,
+ configs = extension_configs)
+ self.set_output_format(output_format)
+ self.reset()
+
+ def registerExtensions(self, extensions, configs):
+ """
+ Register extensions with this instance of Markdown.
+
+ Keyword aurguments:
+
+ * extensions: A list of extensions, which can either
+ be strings or objects. See the docstring on Markdown.
+ * configs: A dictionary mapping module names to config options.
+
+ """
+ for ext in extensions:
+ if isinstance(ext, basestring):
+ ext = load_extension(ext, configs.get(ext, []))
+ try:
+ ext.extendMarkdown(self, globals())
+ except AttributeError:
+ message(ERROR, "Incorrect type! Extension '%s' is "
+ "neither a string or an Extension." %(repr(ext)))
+
+
+ def registerExtension(self, extension):
+ """ This gets called by the extension """
+ self.registeredExtensions.append(extension)
+
+ def reset(self):
+ """
+ Resets all state variables so that we can start with a new text.
+ """
+ self.htmlStash.reset()
+ self.references.clear()
+
+ for extension in self.registeredExtensions:
+ extension.reset()
+
+ def set_output_format(self, format):
+ """ Set the output format for the class instance. """
+ try:
+ self.serializer = self.output_formats[format.lower()]
+ except KeyError:
+ message(CRITICAL, 'Invalid Output Format: "%s". Use one of %s.' \
+ % (format, self.output_formats.keys()))
+
+ def convert(self, source):
+ """
+ Convert markdown to serialized XHTML or HTML.
+
+ Keyword arguments:
+
+ * source: Source text as a Unicode string.
+
+ """
+
+ # Fixup the source text
+ if not source.strip():
+ return u"" # a blank unicode string
+ try:
+ source = unicode(source)
+ except UnicodeDecodeError:
+ message(CRITICAL, 'UnicodeDecodeError: Markdown only accepts unicode or ascii input.')
+ return u""
+
+ source = source.replace(STX, "").replace(ETX, "")
+ source = source.replace("\r\n", "\n").replace("\r", "\n") + "\n\n"
+ source = re.sub(r'\n\s+\n', '\n\n', source)
+ source = source.expandtabs(TAB_LENGTH)
+
+ # Split into lines and run the line preprocessors.
+ self.lines = source.split("\n")
+ for prep in self.preprocessors.values():
+ self.lines = prep.run(self.lines)
+
+ # Parse the high-level elements.
+ root = self.parser.parseDocument(self.lines).getroot()
+
+ # Run the tree-processors
+ for treeprocessor in self.treeprocessors.values():
+ newRoot = treeprocessor.run(root)
+ if newRoot:
+ root = newRoot
+
+ # Serialize _properly_. Strip top-level tags.
+ output, length = codecs.utf_8_decode(self.serializer(root, encoding="utf8"))
+ if self.stripTopLevelTags:
+ start = output.index('<%s>'%DOC_TAG)+len(DOC_TAG)+2
+ end = output.rindex('</%s>'%DOC_TAG)
+ output = output[start:end].strip()
+
+ # Run the text post-processors
+ for pp in self.postprocessors.values():
+ output = pp.run(output)
+
+ return output.strip()
+
+ def convertFile(self, input=None, output=None, encoding=None):
+ """Converts a markdown file and returns the HTML as a unicode string.
+
+ Decodes the file using the provided encoding (defaults to utf-8),
+ passes the file content to markdown, and outputs the html to either
+ the provided stream or the file with provided name, using the same
+ encoding as the source file.
+
+ **Note:** This is the only place that decoding and encoding of unicode
+ takes place in Python-Markdown. (All other code is unicode-in /
+ unicode-out.)
+
+ Keyword arguments:
+
+ * input: Name of source text file.
+ * output: Name of output file. Writes to stdout if `None`.
+ * encoding: Encoding of input and output files. Defaults to utf-8.
+
+ """
+
+ encoding = encoding or "utf-8"
+
+ # Read the source
+ input_file = codecs.open(input, mode="r", encoding=encoding)
+ text = input_file.read()
+ input_file.close()
+ text = text.lstrip(u'\ufeff') # remove the byte-order mark
+
+ # Convert
+ html = self.convert(text)
+
+ # Write to file or stdout
+ if isinstance(output, (str, unicode)):
+ output_file = codecs.open(output, "w", encoding=encoding)
+ output_file.write(html)
+ output_file.close()
+ else:
+ output.write(html.encode(encoding))
+
+
+"""
+Extensions
+-----------------------------------------------------------------------------
+"""
+
+class Extension:
+ """ Base class for extensions to subclass. """
+ def __init__(self, configs = {}):
+ """Create an instance of an Extention.
+
+ Keyword arguments:
+
+ * configs: A dict of configuration setting used by an Extension.
+ """
+ self.config = configs
+
+ def getConfig(self, key):
+ """ Return a setting for the given key or an empty string. """
+ if key in self.config:
+ return self.config[key][0]
+ else:
+ return ""
+
+ def getConfigInfo(self):
+ """ Return all config settings as a list of tuples. """
+ return [(key, self.config[key][1]) for key in self.config.keys()]
+
+ def setConfig(self, key, value):
+ """ Set a config setting for `key` with the given `value`. """
+ self.config[key][0] = value
+
+ def extendMarkdown(self, md, md_globals):
+ """
+ Add the various proccesors and patterns to the Markdown Instance.
+
+ This method must be overriden by every extension.
+
+ Keyword arguments:
+
+ * md: The Markdown instance.
+
+ * md_globals: Global variables in the markdown module namespace.
+
+ """
+ pass
+
+
+def load_extension(ext_name, configs = []):
+ """Load extension by name, then return the module.
+
+ The extension name may contain arguments as part of the string in the
+ following format: "extname(key1=value1,key2=value2)"
+
+ """
+
+ # Parse extensions config params (ignore the order)
+ configs = dict(configs)
+ pos = ext_name.find("(") # find the first "("
+ if pos > 0:
+ ext_args = ext_name[pos+1:-1]
+ ext_name = ext_name[:pos]
+ pairs = [x.split("=") for x in ext_args.split(",")]
+ configs.update([(x.strip(), y.strip()) for (x, y) in pairs])
+
+ # Setup the module names
+ ext_module = 'markdown.extensions'
+ module_name_new_style = '.'.join([ext_module, ext_name])
+ module_name_old_style = '_'.join(['mdx', ext_name])
+
+ # Try loading the extention first from one place, then another
+ try: # New style (markdown.extensons.<extension>)
+ module = __import__(module_name_new_style, {}, {}, [ext_module])
+ except ImportError:
+ try: # Old style (mdx.<extension>)
+ module = __import__(module_name_old_style)
+ except ImportError:
+ message(WARN, "Failed loading extension '%s' from '%s' or '%s'"
+ % (ext_name, module_name_new_style, module_name_old_style))
+ # Return None so we don't try to initiate none-existant extension
+ return None
+
+ # If the module is loaded successfully, we expect it to define a
+ # function called makeExtension()
+ try:
+ return module.makeExtension(configs.items())
+ except AttributeError:
+ message(CRITICAL, "Failed to initiate extension '%s'" % ext_name)
+
+
+def load_extensions(ext_names):
+ """Loads multiple extensions"""
+ extensions = []
+ for ext_name in ext_names:
+ extension = load_extension(ext_name)
+ if extension:
+ extensions.append(extension)
+ return extensions
+
+
+"""
+EXPORTED FUNCTIONS
+=============================================================================
+
+Those are the two functions we really mean to export: markdown() and
+markdownFromFile().
+"""
+
+def markdown(text,
+ extensions = [],
+ safe_mode = False,
+ output_format = DEFAULT_OUTPUT_FORMAT):
+ """Convert a markdown string to HTML and return HTML as a unicode string.
+
+ This is a shortcut function for `Markdown` class to cover the most
+ basic use case. It initializes an instance of Markdown, loads the
+ necessary extensions and runs the parser on the given text.
+
+ Keyword arguments:
+
+ * text: Markdown formatted text as Unicode or ASCII string.
+ * extensions: A list of extensions or extension names (may contain config args).
+ * safe_mode: Disallow raw html. One of "remove", "replace" or "escape".
+ * output_format: Format of output. Supported formats are:
+ * "xhtml1": Outputs XHTML 1.x. Default.
+ * "xhtml": Outputs latest supported version of XHTML (currently XHTML 1.1).
+ * "html4": Outputs HTML 4
+ * "html": Outputs latest supported version of HTML (currently HTML 4).
+ Note that it is suggested that the more specific formats ("xhtml1"
+ and "html4") be used as "xhtml" or "html" may change in the future
+ if it makes sense at that time.
+
+ Returns: An HTML document as a string.
+
+ """
+ md = Markdown(extensions=load_extensions(extensions),
+ safe_mode=safe_mode,
+ output_format=output_format)
+ return md.convert(text)
+
+
+def markdownFromFile(input = None,
+ output = None,
+ extensions = [],
+ encoding = None,
+ safe_mode = False,
+ output_format = DEFAULT_OUTPUT_FORMAT):
+ """Read markdown code from a file and write it to a file or a stream."""
+ md = Markdown(extensions=load_extensions(extensions),
+ safe_mode=safe_mode,
+ output_format=output_format)
+ md.convertFile(input, output, encoding)
+
+
+
--- /dev/null
+
+import markdown
+
+class State(list):
+ """ Track the current and nested state of the parser.
+
+ This utility class is used to track the state of the BlockParser and
+ support multiple levels if nesting. It's just a simple API wrapped around
+ a list. Each time a state is set, that state is appended to the end of the
+ list. Each time a state is reset, that state is removed from the end of
+ the list.
+
+ Therefore, each time a state is set for a nested block, that state must be
+ reset when we back out of that level of nesting or the state could be
+ corrupted.
+
+ While all the methods of a list object are available, only the three
+ defined below need be used.
+
+ """
+
+ def set(self, state):
+ """ Set a new state. """
+ self.append(state)
+
+ def reset(self):
+ """ Step back one step in nested state. """
+ self.pop()
+
+ def isstate(self, state):
+ """ Test that top (current) level is of given state. """
+ if len(self):
+ return self[-1] == state
+ else:
+ return False
+
+class BlockParser:
+ """ Parse Markdown blocks into an ElementTree object.
+
+ A wrapper class that stitches the various BlockProcessors together,
+ looping through them and creating an ElementTree object.
+ """
+
+ def __init__(self):
+ self.blockprocessors = markdown.odict.OrderedDict()
+ self.state = State()
+
+ def parseDocument(self, lines):
+ """ Parse a markdown document into an ElementTree.
+
+ Given a list of lines, an ElementTree object (not just a parent Element)
+ is created and the root element is passed to the parser as the parent.
+ The ElementTree object is returned.
+
+ This should only be called on an entire document, not pieces.
+
+ """
+ # Create a ElementTree from the lines
+ self.root = markdown.etree.Element(markdown.DOC_TAG)
+ self.parseChunk(self.root, '\n'.join(lines))
+ return markdown.etree.ElementTree(self.root)
+
+ def parseChunk(self, parent, text):
+ """ Parse a chunk of markdown text and attach to given etree node.
+
+ While the ``text`` argument is generally assumed to contain multiple
+ blocks which will be split on blank lines, it could contain only one
+ block. Generally, this method would be called by extensions when
+ block parsing is required.
+
+ The ``parent`` etree Element passed in is altered in place.
+ Nothing is returned.
+
+ """
+ self.parseBlocks(parent, text.split('\n\n'))
+
+ def parseBlocks(self, parent, blocks):
+ """ Process blocks of markdown text and attach to given etree node.
+
+ Given a list of ``blocks``, each blockprocessor is stepped through
+ until there are no blocks left. While an extension could potentially
+ call this method directly, it's generally expected to be used internally.
+
+ This is a public method as an extension may need to add/alter additional
+ BlockProcessors which call this method to recursively parse a nested
+ block.
+
+ """
+ while blocks:
+ for processor in self.blockprocessors.values():
+ if processor.test(parent, blocks[0]):
+ processor.run(parent, blocks)
+ break
+
+
--- /dev/null
+"""
+CORE MARKDOWN BLOCKPARSER
+=============================================================================
+
+This parser handles basic parsing of Markdown blocks. It doesn't concern itself
+with inline elements such as **bold** or *italics*, but rather just catches
+blocks, lists, quotes, etc.
+
+The BlockParser is made up of a bunch of BlockProssors, each handling a
+different type of block. Extensions may add/replace/remove BlockProcessors
+as they need to alter how markdown blocks are parsed.
+
+"""
+
+import re
+import markdown
+
+class BlockProcessor:
+ """ Base class for block processors.
+
+ Each subclass will provide the methods below to work with the source and
+ tree. Each processor will need to define it's own ``test`` and ``run``
+ methods. The ``test`` method should return True or False, to indicate
+ whether the current block should be processed by this processor. If the
+ test passes, the parser will call the processors ``run`` method.
+
+ """
+
+ def __init__(self, parser=None):
+ self.parser = parser
+
+ def lastChild(self, parent):
+ """ Return the last child of an etree element. """
+ if len(parent):
+ return parent[-1]
+ else:
+ return None
+
+ def detab(self, text):
+ """ Remove a tab from the front of each line of the given text. """
+ newtext = []
+ lines = text.split('\n')
+ for line in lines:
+ if line.startswith(' '*markdown.TAB_LENGTH):
+ newtext.append(line[markdown.TAB_LENGTH:])
+ elif not line.strip():
+ newtext.append('')
+ else:
+ break
+ return '\n'.join(newtext), '\n'.join(lines[len(newtext):])
+
+ def looseDetab(self, text, level=1):
+ """ Remove a tab from front of lines but allowing dedented lines. """
+ lines = text.split('\n')
+ for i in range(len(lines)):
+ if lines[i].startswith(' '*markdown.TAB_LENGTH*level):
+ lines[i] = lines[i][markdown.TAB_LENGTH*level:]
+ return '\n'.join(lines)
+
+ def test(self, parent, block):
+ """ Test for block type. Must be overridden by subclasses.
+
+ As the parser loops through processors, it will call the ``test`` method
+ on each to determine if the given block of text is of that type. This
+ method must return a boolean ``True`` or ``False``. The actual method of
+ testing is left to the needs of that particular block type. It could
+ be as simple as ``block.startswith(some_string)`` or a complex regular
+ expression. As the block type may be different depending on the parent
+ of the block (i.e. inside a list), the parent etree element is also
+ provided and may be used as part of the test.
+
+ Keywords:
+
+ * ``parent``: A etree element which will be the parent of the block.
+ * ``block``: A block of text from the source which has been split at
+ blank lines.
+ """
+ pass
+
+ def run(self, parent, blocks):
+ """ Run processor. Must be overridden by subclasses.
+
+ When the parser determines the appropriate type of a block, the parser
+ will call the corresponding processor's ``run`` method. This method
+ should parse the individual lines of the block and append them to
+ the etree.
+
+ Note that both the ``parent`` and ``etree`` keywords are pointers
+ to instances of the objects which should be edited in place. Each
+ processor must make changes to the existing objects as there is no
+ mechanism to return new/different objects to replace them.
+
+ This means that this method should be adding SubElements or adding text
+ to the parent, and should remove (``pop``) or add (``insert``) items to
+ the list of blocks.
+
+ Keywords:
+
+ * ``parent``: A etree element which is the parent of the current block.
+ * ``blocks``: A list of all remaining blocks of the document.
+ """
+ pass
+
+
+class ListIndentProcessor(BlockProcessor):
+ """ Process children of list items.
+
+ Example:
+ * a list item
+ process this part
+
+ or this part
+
+ """
+
+ INDENT_RE = re.compile(r'^(([ ]{%s})+)'% markdown.TAB_LENGTH)
+ ITEM_TYPES = ['li']
+ LIST_TYPES = ['ul', 'ol']
+
+ def test(self, parent, block):
+ return block.startswith(' '*markdown.TAB_LENGTH) and \
+ not self.parser.state.isstate('detabbed') and \
+ (parent.tag in self.ITEM_TYPES or \
+ (len(parent) and parent[-1] and \
+ (parent[-1].tag in self.LIST_TYPES)
+ )
+ )
+
+ def run(self, parent, blocks):
+ block = blocks.pop(0)
+ level, sibling = self.get_level(parent, block)
+ block = self.looseDetab(block, level)
+
+ self.parser.state.set('detabbed')
+ if parent.tag in self.ITEM_TYPES:
+ # The parent is already a li. Just parse the child block.
+ self.parser.parseBlocks(parent, [block])
+ elif sibling.tag in self.ITEM_TYPES:
+ # The sibling is a li. Use it as parent.
+ self.parser.parseBlocks(sibling, [block])
+ elif len(sibling) and sibling[-1].tag in self.ITEM_TYPES:
+ # The parent is a list (``ol`` or ``ul``) which has children.
+ # Assume the last child li is the parent of this block.
+ if sibling[-1].text:
+ # If the parent li has text, that text needs to be moved to a p
+ block = '%s\n\n%s' % (sibling[-1].text, block)
+ sibling[-1].text = ''
+ self.parser.parseChunk(sibling[-1], block)
+ else:
+ self.create_item(sibling, block)
+ self.parser.state.reset()
+
+ def create_item(self, parent, block):
+ """ Create a new li and parse the block with it as the parent. """
+ li = markdown.etree.SubElement(parent, 'li')
+ self.parser.parseBlocks(li, [block])
+
+ def get_level(self, parent, block):
+ """ Get level of indent based on list level. """
+ # Get indent level
+ m = self.INDENT_RE.match(block)
+ if m:
+ indent_level = len(m.group(1))/markdown.TAB_LENGTH
+ else:
+ indent_level = 0
+ if self.parser.state.isstate('list'):
+ # We're in a tightlist - so we already are at correct parent.
+ level = 1
+ else:
+ # We're in a looselist - so we need to find parent.
+ level = 0
+ # Step through children of tree to find matching indent level.
+ while indent_level > level:
+ child = self.lastChild(parent)
+ if child and (child.tag in self.LIST_TYPES or child.tag in self.ITEM_TYPES):
+ if child.tag in self.LIST_TYPES:
+ level += 1
+ parent = child
+ else:
+ # No more child levels. If we're short of indent_level,
+ # we have a code block. So we stop here.
+ break
+ return level, parent
+
+
+class CodeBlockProcessor(BlockProcessor):
+ """ Process code blocks. """
+
+ def test(self, parent, block):
+ return block.startswith(' '*markdown.TAB_LENGTH)
+
+ def run(self, parent, blocks):
+ sibling = self.lastChild(parent)
+ block = blocks.pop(0)
+ theRest = ''
+ if sibling and sibling.tag == "pre" and len(sibling) \
+ and sibling[0].tag == "code":
+ # The previous block was a code block. As blank lines do not start
+ # new code blocks, append this block to the previous, adding back
+ # linebreaks removed from the split into a list.
+ code = sibling[0]
+ block, theRest = self.detab(block)
+ code.text = markdown.AtomicString('%s\n%s\n' % (code.text, block.rstrip()))
+ else:
+ # This is a new codeblock. Create the elements and insert text.
+ pre = markdown.etree.SubElement(parent, 'pre')
+ code = markdown.etree.SubElement(pre, 'code')
+ block, theRest = self.detab(block)
+ code.text = markdown.AtomicString('%s\n' % block.rstrip())
+ if theRest:
+ # This block contained unindented line(s) after the first indented
+ # line. Insert these lines as the first block of the master blocks
+ # list for future processing.
+ blocks.insert(0, theRest)
+
+
+class BlockQuoteProcessor(BlockProcessor):
+
+ RE = re.compile(r'(^|\n)[ ]{0,3}>[ ]?(.*)')
+
+ def test(self, parent, block):
+ return bool(self.RE.search(block))
+
+ def run(self, parent, blocks):
+ block = blocks.pop(0)
+ m = self.RE.search(block)
+ if m:
+ before = block[:m.start()] # Lines before blockquote
+ # Pass lines before blockquote in recursively for parsing forst.
+ self.parser.parseBlocks(parent, [before])
+ # Remove ``> `` from begining of each line.
+ block = '\n'.join([self.clean(line) for line in
+ block[m.start():].split('\n')])
+ sibling = self.lastChild(parent)
+ if sibling and sibling.tag == "blockquote":
+ # Previous block was a blockquote so set that as this blocks parent
+ quote = sibling
+ else:
+ # This is a new blockquote. Create a new parent element.
+ quote = markdown.etree.SubElement(parent, 'blockquote')
+ # Recursively parse block with blockquote as parent.
+ self.parser.parseChunk(quote, block)
+
+ def clean(self, line):
+ """ Remove ``>`` from beginning of a line. """
+ m = self.RE.match(line)
+ if line.strip() == ">":
+ return ""
+ elif m:
+ return m.group(2)
+ else:
+ return line
+
+class OListProcessor(BlockProcessor):
+ """ Process ordered list blocks. """
+
+ TAG = 'ol'
+ # Detect an item (``1. item``). ``group(1)`` contains contents of item.
+ RE = re.compile(r'^[ ]{0,3}\d+\.[ ](.*)')
+ # Detect items on secondary lines. they can be of either list type.
+ CHILD_RE = re.compile(r'^[ ]{0,3}((\d+\.)|[*+-])[ ](.*)')
+ # Detect indented (nested) items of either type
+ INDENT_RE = re.compile(r'^[ ]{4,7}((\d+\.)|[*+-])[ ].*')
+
+ def test(self, parent, block):
+ return bool(self.RE.match(block))
+
+ def run(self, parent, blocks):
+ # Check fr multiple items in one block.
+ items = self.get_items(blocks.pop(0))
+ sibling = self.lastChild(parent)
+ if sibling and sibling.tag in ['ol', 'ul']:
+ # Previous block was a list item, so set that as parent
+ lst = sibling
+ # make sure previous item is in a p.
+ if len(lst) and lst[-1].text and not len(lst[-1]):
+ p = markdown.etree.SubElement(lst[-1], 'p')
+ p.text = lst[-1].text
+ lst[-1].text = ''
+ # parse first block differently as it gets wrapped in a p.
+ li = markdown.etree.SubElement(lst, 'li')
+ self.parser.state.set('looselist')
+ firstitem = items.pop(0)
+ self.parser.parseBlocks(li, [firstitem])
+ self.parser.state.reset()
+ else:
+ # This is a new list so create parent with appropriate tag.
+ lst = markdown.etree.SubElement(parent, self.TAG)
+ self.parser.state.set('list')
+ # Loop through items in block, recursively parsing each with the
+ # appropriate parent.
+ for item in items:
+ if item.startswith(' '*markdown.TAB_LENGTH):
+ # Item is indented. Parse with last item as parent
+ self.parser.parseBlocks(lst[-1], [item])
+ else:
+ # New item. Create li and parse with it as parent
+ li = markdown.etree.SubElement(lst, 'li')
+ self.parser.parseBlocks(li, [item])
+ self.parser.state.reset()
+
+ def get_items(self, block):
+ """ Break a block into list items. """
+ items = []
+ for line in block.split('\n'):
+ m = self.CHILD_RE.match(line)
+ if m:
+ # This is a new item. Append
+ items.append(m.group(3))
+ elif self.INDENT_RE.match(line):
+ # This is an indented (possibly nested) item.
+ if items[-1].startswith(' '*markdown.TAB_LENGTH):
+ # Previous item was indented. Append to that item.
+ items[-1] = '%s\n%s' % (items[-1], line)
+ else:
+ items.append(line)
+ else:
+ # This is another line of previous item. Append to that item.
+ items[-1] = '%s\n%s' % (items[-1], line)
+ return items
+
+
+class UListProcessor(OListProcessor):
+ """ Process unordered list blocks. """
+
+ TAG = 'ul'
+ RE = re.compile(r'^[ ]{0,3}[*+-][ ](.*)')
+
+
+class HashHeaderProcessor(BlockProcessor):
+ """ Process Hash Headers. """
+
+ # Detect a header at start of any line in block
+ RE = re.compile(r'(^|\n)(?P<level>#{1,6})(?P<header>.*?)#*(\n|$)')
+
+ def test(self, parent, block):
+ return bool(self.RE.search(block))
+
+ def run(self, parent, blocks):
+ block = blocks.pop(0)
+ m = self.RE.search(block)
+ if m:
+ before = block[:m.start()] # All lines before header
+ after = block[m.end():] # All lines after header
+ if before:
+ # As the header was not the first line of the block and the
+ # lines before the header must be parsed first,
+ # recursively parse this lines as a block.
+ self.parser.parseBlocks(parent, [before])
+ # Create header using named groups from RE
+ h = markdown.etree.SubElement(parent, 'h%d' % len(m.group('level')))
+ h.text = m.group('header').strip()
+ if after:
+ # Insert remaining lines as first block for future parsing.
+ blocks.insert(0, after)
+ else:
+ # This should never happen, but just in case...
+ message(CRITICAL, "We've got a problem header!")
+
+
+class SetextHeaderProcessor(BlockProcessor):
+ """ Process Setext-style Headers. """
+
+ # Detect Setext-style header. Must be first 2 lines of block.
+ RE = re.compile(r'^.*?\n[=-]{3,}', re.MULTILINE)
+
+ def test(self, parent, block):
+ return bool(self.RE.match(block))
+
+ def run(self, parent, blocks):
+ lines = blocks.pop(0).split('\n')
+ # Determine level. ``=`` is 1 and ``-`` is 2.
+ if lines[1].startswith('='):
+ level = 1
+ else:
+ level = 2
+ h = markdown.etree.SubElement(parent, 'h%d' % level)
+ h.text = lines[0].strip()
+ if len(lines) > 2:
+ # Block contains additional lines. Add to master blocks for later.
+ blocks.insert(0, '\n'.join(lines[2:]))
+
+
+class HRProcessor(BlockProcessor):
+ """ Process Horizontal Rules. """
+
+ RE = r'[ ]{0,3}(?P<ch>[*_-])[ ]?((?P=ch)[ ]?){2,}[ ]*'
+ # Detect hr on any line of a block.
+ SEARCH_RE = re.compile(r'(^|\n)%s(\n|$)' % RE)
+ # Match a hr on a single line of text.
+ MATCH_RE = re.compile(r'^%s$' % RE)
+
+ def test(self, parent, block):
+ return bool(self.SEARCH_RE.search(block))
+
+ def run(self, parent, blocks):
+ lines = blocks.pop(0).split('\n')
+ prelines = []
+ # Check for lines in block before hr.
+ for line in lines:
+ m = self.MATCH_RE.match(line)
+ if m:
+ break
+ else:
+ prelines.append(line)
+ if len(prelines):
+ # Recursively parse lines before hr so they get parsed first.
+ self.parser.parseBlocks(parent, ['\n'.join(prelines)])
+ # create hr
+ hr = markdown.etree.SubElement(parent, 'hr')
+ # check for lines in block after hr.
+ lines = lines[len(prelines)+1:]
+ if len(lines):
+ # Add lines after hr to master blocks for later parsing.
+ blocks.insert(0, '\n'.join(lines))
+
+
+class EmptyBlockProcessor(BlockProcessor):
+ """ Process blocks and start with an empty line. """
+
+ # Detect a block that only contains whitespace
+ # or only whitespace on the first line.
+ RE = re.compile(r'^\s*\n')
+
+ def test(self, parent, block):
+ return bool(self.RE.match(block))
+
+ def run(self, parent, blocks):
+ block = blocks.pop(0)
+ m = self.RE.match(block)
+ if m:
+ # Add remaining line to master blocks for later.
+ blocks.insert(0, block[m.end():])
+ sibling = self.lastChild(parent)
+ if sibling and sibling.tag == 'pre' and sibling[0] and \
+ sibling[0].tag == 'code':
+ # Last block is a codeblock. Append to preserve whitespace.
+ sibling[0].text = markdown.AtomicString('%s/n/n/n' % sibling[0].text )
+
+
+class ParagraphProcessor(BlockProcessor):
+ """ Process Paragraph blocks. """
+
+ def test(self, parent, block):
+ return True
+
+ def run(self, parent, blocks):
+ block = blocks.pop(0)
+ if block.strip():
+ # Not a blank block. Add to parent, otherwise throw it away.
+ if self.parser.state.isstate('list'):
+ # The parent is a tight-list. Append to parent.text
+ if parent.text:
+ parent.text = '%s\n%s' % (parent.text, block)
+ else:
+ parent.text = block.lstrip()
+ else:
+ # Create a regular paragraph
+ p = markdown.etree.SubElement(parent, 'p')
+ p.text = block.lstrip()
--- /dev/null
+"""
+COMMAND-LINE SPECIFIC STUFF
+=============================================================================
+
+The rest of the code is specifically for handling the case where Python
+Markdown is called from the command line.
+"""
+
+import markdown
+import sys
+import logging
+from logging import DEBUG, INFO, WARN, ERROR, CRITICAL
+
+EXECUTABLE_NAME_FOR_USAGE = "python markdown.py"
+""" The name used in the usage statement displayed for python versions < 2.3.
+(With python 2.3 and higher the usage statement is generated by optparse
+and uses the actual name of the executable called.) """
+
+OPTPARSE_WARNING = """
+Python 2.3 or higher required for advanced command line options.
+For lower versions of Python use:
+
+ %s INPUT_FILE > OUTPUT_FILE
+
+""" % EXECUTABLE_NAME_FOR_USAGE
+
+def parse_options():
+ """
+ Define and parse `optparse` options for command-line usage.
+ """
+
+ try:
+ optparse = __import__("optparse")
+ except:
+ if len(sys.argv) == 2:
+ return {'input': sys.argv[1],
+ 'output': None,
+ 'safe': False,
+ 'extensions': [],
+ 'encoding': None }, CRITICAL
+ else:
+ print OPTPARSE_WARNING
+ return None, None
+
+ parser = optparse.OptionParser(usage="%prog INPUTFILE [options]")
+ parser.add_option("-f", "--file", dest="filename", default=sys.stdout,
+ help="write output to OUTPUT_FILE",
+ metavar="OUTPUT_FILE")
+ parser.add_option("-e", "--encoding", dest="encoding",
+ help="encoding for input and output files",)
+ parser.add_option("-q", "--quiet", default = CRITICAL,
+ action="store_const", const=CRITICAL+10, dest="verbose",
+ help="suppress all messages")
+ parser.add_option("-v", "--verbose",
+ action="store_const", const=INFO, dest="verbose",
+ help="print info messages")
+ parser.add_option("-s", "--safe", dest="safe", default=False,
+ metavar="SAFE_MODE",
+ help="safe mode ('replace', 'remove' or 'escape' user's HTML tag)")
+ parser.add_option("-o", "--output_format", dest="output_format",
+ default='xhtml1', metavar="OUTPUT_FORMAT",
+ help="Format of output. One of 'xhtml1' (default) or 'html4'.")
+ parser.add_option("--noisy",
+ action="store_const", const=DEBUG, dest="verbose",
+ help="print debug messages")
+ parser.add_option("-x", "--extension", action="append", dest="extensions",
+ help = "load extension EXTENSION", metavar="EXTENSION")
+
+ (options, args) = parser.parse_args()
+
+ if not len(args) == 1:
+ parser.print_help()
+ return None, None
+ else:
+ input_file = args[0]
+
+ if not options.extensions:
+ options.extensions = []
+
+ return {'input': input_file,
+ 'output': options.filename,
+ 'safe_mode': options.safe,
+ 'extensions': options.extensions,
+ 'encoding': options.encoding,
+ 'output_format': options.output_format}, options.verbose
+
+def run():
+ """Run Markdown from the command line."""
+
+ # Parse options and adjust logging level if necessary
+ options, logging_level = parse_options()
+ if not options: sys.exit(0)
+ if logging_level: logging.getLogger('MARKDOWN').setLevel(logging_level)
+
+ # Run
+ markdown.markdownFromFile(**options)
--- /dev/null
+
+from markdown import message, CRITICAL
+import sys
+
+## Import
+def importETree():
+ """Import the best implementation of ElementTree, return a module object."""
+ etree_in_c = None
+ try: # Is it Python 2.5+ with C implemenation of ElementTree installed?
+ import xml.etree.cElementTree as etree_in_c
+ except ImportError:
+ try: # Is it Python 2.5+ with Python implementation of ElementTree?
+ import xml.etree.ElementTree as etree
+ except ImportError:
+ try: # An earlier version of Python with cElementTree installed?
+ import cElementTree as etree_in_c
+ except ImportError:
+ try: # An earlier version of Python with Python ElementTree?
+ import elementtree.ElementTree as etree
+ except ImportError:
+ message(CRITICAL, "Failed to import ElementTree")
+ sys.exit(1)
+ if etree_in_c and etree_in_c.VERSION < "1.0":
+ message(CRITICAL, "For cElementTree version 1.0 or higher is required.")
+ sys.exit(1)
+ elif etree_in_c :
+ return etree_in_c
+ elif etree.VERSION < "1.1":
+ message(CRITICAL, "For ElementTree version 1.1 or higher is required")
+ sys.exit(1)
+ else :
+ return etree
+
--- /dev/null
+"""
+Table of Contents Extension for Python-Markdown
+* * *
+
+(c) 2008 [Jack Miller](http://codezen.org)
+
+Dependencies:
+* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/)
+
+"""
+import markdown
+from markdown import etree
+import re
+
+class TocTreeprocessor(markdown.treeprocessors.Treeprocessor):
+ # Iterator wrapper to get parent and child all at once
+ def iterparent(self, root):
+ for parent in root.getiterator():
+ for child in parent:
+ yield parent, child
+
+ def run(self, doc):
+ div = etree.Element("div")
+ div.attrib["class"] = "toc"
+ last_li = None
+
+ # Add title to the div
+ if self.config["title"][0]:
+ header = etree.SubElement(div, "span")
+ header.attrib["class"] = "toctitle"
+ header.text = self.config["title"][0]
+
+ level = 0
+ list_stack=[div]
+ header_rgx = re.compile("[Hh][123456]")
+
+ # Get a list of id attributes
+ used_ids = []
+ for c in doc.getiterator():
+ if "id" in c.attrib:
+ used_ids.append(c.attrib["id"])
+
+ for (p, c) in self.iterparent(doc):
+ if not c.text:
+ continue
+
+ # To keep the output from screwing up the
+ # validation by putting a <div> inside of a <p>
+ # we actually replace the <p> in its entirety.
+ # We do not allow the marker inside a header as that
+ # would causes an enless loop of placing a new TOC
+ # inside previously generated TOC.
+
+ if c.text.find(self.config["marker"][0]) > -1 and not header_rgx.match(c.tag):
+ for i in range(len(p)):
+ if p[i] == c:
+ p[i] = div
+ break
+
+ if header_rgx.match(c.tag):
+ tag_level = int(c.tag[-1])
+
+ # Regardless of how many levels we jumped
+ # only one list should be created, since
+ # empty lists containing lists are illegal.
+
+ if tag_level < level:
+ list_stack.pop()
+ level = tag_level
+
+ if tag_level > level:
+ newlist = etree.Element("ul")
+ if last_li:
+ last_li.append(newlist)
+ else:
+ list_stack[-1].append(newlist)
+ list_stack.append(newlist)
+ level = tag_level
+
+ # Do not override pre-existing ids
+ if not "id" in c.attrib:
+ id = self.config["slugify"][0](c.text)
+ if id in used_ids:
+ ctr = 1
+ while "%s_%d" % (id, ctr) in used_ids:
+ ctr += 1
+ id = "%s_%d" % (id, ctr)
+ used_ids.append(id)
+ c.attrib["id"] = id
+ else:
+ id = c.attrib["id"]
+
+ # List item link, to be inserted into the toc div
+ last_li = etree.Element("li")
+ link = etree.SubElement(last_li, "a")
+ link.text = c.text
+ link.attrib["href"] = '#' + id
+
+ if int(self.config["anchorlink"][0]):
+ anchor = etree.SubElement(c, "a")
+ anchor.text = c.text
+ anchor.attrib["href"] = "#" + id
+ anchor.attrib["class"] = "toclink"
+ c.text = ""
+
+ list_stack[-1].append(last_li)
+
+class TocExtension(markdown.Extension):
+ def __init__(self, configs):
+ self.config = { "marker" : ["[TOC]",
+ "Text to find and replace with Table of Contents -"
+ "Defaults to \"[TOC]\""],
+ "slugify" : [self.slugify,
+ "Function to generate anchors based on header text-"
+ "Defaults to a built in slugify function."],
+ "title" : [None,
+ "Title to insert into TOC <div> - "
+ "Defaults to None"],
+ "anchorlink" : [0,
+ "1 if header should be a self link"
+ "Defaults to 0"]}
+
+ for key, value in configs:
+ self.setConfig(key, value)
+
+ # This is exactly the same as Django's slugify
+ def slugify(self, value):
+ """ Slugify a string, to make it URL friendly. """
+ import unicodedata
+ value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore')
+ value = unicode(re.sub('[^\w\s-]', '', value).strip().lower())
+ return re.sub('[-\s]+','-',value)
+
+ def extendMarkdown(self, md, md_globals):
+ tocext = TocTreeprocessor(md)
+ tocext.config = self.config
+ md.treeprocessors.add("toc", tocext, "_begin")
+
+def makeExtension(configs={}):
+ return TocExtension(configs=configs)
--- /dev/null
+# markdown/html4.py
+#
+# Add html4 serialization to older versions of Elementree
+# Taken from ElementTree 1.3 preview with slight modifications
+#
+# Copyright (c) 1999-2007 by Fredrik Lundh. All rights reserved.
+#
+# fredrik@pythonware.com
+# http://www.pythonware.com
+#
+# --------------------------------------------------------------------
+# The ElementTree toolkit is
+#
+# Copyright (c) 1999-2007 by Fredrik Lundh
+#
+# By obtaining, using, and/or copying this software and/or its
+# associated documentation, you agree that you have read, understood,
+# and will comply with the following terms and conditions:
+#
+# Permission to use, copy, modify, and distribute this software and
+# its associated documentation for any purpose and without fee is
+# hereby granted, provided that the above copyright notice appears in
+# all copies, and that both that copyright notice and this permission
+# notice appear in supporting documentation, and that the name of
+# Secret Labs AB or the author not be used in advertising or publicity
+# pertaining to distribution of the software without specific, written
+# prior permission.
+#
+# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
+# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
+# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
+# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
+# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+# OF THIS SOFTWARE.
+# --------------------------------------------------------------------
+
+
+import markdown
+ElementTree = markdown.etree.ElementTree
+QName = markdown.etree.QName
+Comment = markdown.etree.Comment
+PI = markdown.etree.PI
+ProcessingInstruction = markdown.etree.ProcessingInstruction
+
+HTML_EMPTY = ("area", "base", "basefont", "br", "col", "frame", "hr",
+ "img", "input", "isindex", "link", "meta" "param")
+
+try:
+ HTML_EMPTY = set(HTML_EMPTY)
+except NameError:
+ pass
+
+_namespace_map = {
+ # "well-known" namespace prefixes
+ "http://www.w3.org/XML/1998/namespace": "xml",
+ "http://www.w3.org/1999/xhtml": "html",
+ "http://www.w3.org/1999/02/22-rdf-syntax-ns#": "rdf",
+ "http://schemas.xmlsoap.org/wsdl/": "wsdl",
+ # xml schema
+ "http://www.w3.org/2001/XMLSchema": "xs",
+ "http://www.w3.org/2001/XMLSchema-instance": "xsi",
+ # dublic core
+ "http://purl.org/dc/elements/1.1/": "dc",
+}
+
+
+def _raise_serialization_error(text):
+ raise TypeError(
+ "cannot serialize %r (type %s)" % (text, type(text).__name__)
+ )
+
+def _encode(text, encoding):
+ try:
+ return text.encode(encoding, "xmlcharrefreplace")
+ except (TypeError, AttributeError):
+ _raise_serialization_error(text)
+
+def _escape_cdata(text, encoding):
+ # escape character data
+ try:
+ # it's worth avoiding do-nothing calls for strings that are
+ # shorter than 500 character, or so. assume that's, by far,
+ # the most common case in most applications.
+ if "&" in text:
+ text = text.replace("&", "&")
+ if "<" in text:
+ text = text.replace("<", "<")
+ if ">" in text:
+ text = text.replace(">", ">")
+ return text.encode(encoding, "xmlcharrefreplace")
+ except (TypeError, AttributeError):
+ _raise_serialization_error(text)
+
+
+def _escape_attrib(text, encoding):
+ # escape attribute value
+ try:
+ if "&" in text:
+ text = text.replace("&", "&")
+ if "<" in text:
+ text = text.replace("<", "<")
+ if ">" in text:
+ text = text.replace(">", ">")
+ if "\"" in text:
+ text = text.replace("\"", """)
+ if "\n" in text:
+ text = text.replace("\n", " ")
+ return text.encode(encoding, "xmlcharrefreplace")
+ except (TypeError, AttributeError):
+ _raise_serialization_error(text)
+
+def _escape_attrib_html(text, encoding):
+ # escape attribute value
+ try:
+ if "&" in text:
+ text = text.replace("&", "&")
+ if ">" in text:
+ text = text.replace(">", ">")
+ if "\"" in text:
+ text = text.replace("\"", """)
+ return text.encode(encoding, "xmlcharrefreplace")
+ except (TypeError, AttributeError):
+ _raise_serialization_error(text)
+
+
+def _serialize_html(write, elem, encoding, qnames, namespaces):
+ tag = elem.tag
+ text = elem.text
+ if tag is Comment:
+ write("<!--%s-->" % _escape_cdata(text, encoding))
+ elif tag is ProcessingInstruction:
+ write("<?%s?>" % _escape_cdata(text, encoding))
+ else:
+ tag = qnames[tag]
+ if tag is None:
+ if text:
+ write(_escape_cdata(text, encoding))
+ for e in elem:
+ _serialize_html(write, e, encoding, qnames, None)
+ else:
+ write("<" + tag)
+ items = elem.items()
+ if items or namespaces:
+ items.sort() # lexical order
+ for k, v in items:
+ if isinstance(k, QName):
+ k = k.text
+ if isinstance(v, QName):
+ v = qnames[v.text]
+ else:
+ v = _escape_attrib_html(v, encoding)
+ # FIXME: handle boolean attributes
+ write(" %s=\"%s\"" % (qnames[k], v))
+ if namespaces:
+ items = namespaces.items()
+ items.sort(key=lambda x: x[1]) # sort on prefix
+ for v, k in items:
+ if k:
+ k = ":" + k
+ write(" xmlns%s=\"%s\"" % (
+ k.encode(encoding),
+ _escape_attrib(v, encoding)
+ ))
+ write(">")
+ tag = tag.lower()
+ if text:
+ if tag == "script" or tag == "style":
+ write(_encode(text, encoding))
+ else:
+ write(_escape_cdata(text, encoding))
+ for e in elem:
+ _serialize_html(write, e, encoding, qnames, None)
+ if tag not in HTML_EMPTY:
+ write("</" + tag + ">")
+ if elem.tail:
+ write(_escape_cdata(elem.tail, encoding))
+
+def write_html(root, f,
+ # keyword arguments
+ encoding="us-ascii",
+ default_namespace=None):
+ assert root is not None
+ if not hasattr(f, "write"):
+ f = open(f, "wb")
+ write = f.write
+ if not encoding:
+ encoding = "us-ascii"
+ qnames, namespaces = _namespaces(
+ root, encoding, default_namespace
+ )
+ _serialize_html(
+ write, root, encoding, qnames, namespaces
+ )
+
+# --------------------------------------------------------------------
+# serialization support
+
+def _namespaces(elem, encoding, default_namespace=None):
+ # identify namespaces used in this tree
+
+ # maps qnames to *encoded* prefix:local names
+ qnames = {None: None}
+
+ # maps uri:s to prefixes
+ namespaces = {}
+ if default_namespace:
+ namespaces[default_namespace] = ""
+
+ def encode(text):
+ return text.encode(encoding)
+
+ def add_qname(qname):
+ # calculate serialized qname representation
+ try:
+ if qname[:1] == "{":
+ uri, tag = qname[1:].split("}", 1)
+ prefix = namespaces.get(uri)
+ if prefix is None:
+ prefix = _namespace_map.get(uri)
+ if prefix is None:
+ prefix = "ns%d" % len(namespaces)
+ if prefix != "xml":
+ namespaces[uri] = prefix
+ if prefix:
+ qnames[qname] = encode("%s:%s" % (prefix, tag))
+ else:
+ qnames[qname] = encode(tag) # default element
+ else:
+ if default_namespace:
+ # FIXME: can this be handled in XML 1.0?
+ raise ValueError(
+ "cannot use non-qualified names with "
+ "default_namespace option"
+ )
+ qnames[qname] = encode(qname)
+ except TypeError:
+ _raise_serialization_error(qname)
+
+ # populate qname and namespaces table
+ try:
+ iterate = elem.iter
+ except AttributeError:
+ iterate = elem.getiterator # cET compatibility
+ for elem in iterate():
+ tag = elem.tag
+ if isinstance(tag, QName) and tag.text not in qnames:
+ add_qname(tag.text)
+ elif isinstance(tag, basestring):
+ if tag not in qnames:
+ add_qname(tag)
+ elif tag is not None and tag is not Comment and tag is not PI:
+ _raise_serialization_error(tag)
+ for key, value in elem.items():
+ if isinstance(key, QName):
+ key = key.text
+ if key not in qnames:
+ add_qname(key)
+ if isinstance(value, QName) and value.text not in qnames:
+ add_qname(value.text)
+ text = elem.text
+ if isinstance(text, QName) and text.text not in qnames:
+ add_qname(text.text)
+ return qnames, namespaces
+
+def to_html_string(element, encoding=None):
+ class dummy:
+ pass
+ data = []
+ file = dummy()
+ file.write = data.append
+ write_html(ElementTree(element).getroot(),file,encoding)
+ return "".join(data)
--- /dev/null
+"""
+INLINE PATTERNS
+=============================================================================
+
+Inline patterns such as *emphasis* are handled by means of auxiliary
+objects, one per pattern. Pattern objects must be instances of classes
+that extend markdown.Pattern. Each pattern object uses a single regular
+expression and needs support the following methods:
+
+ pattern.getCompiledRegExp() # returns a regular expression
+
+ pattern.handleMatch(m) # takes a match object and returns
+ # an ElementTree element or just plain text
+
+All of python markdown's built-in patterns subclass from Pattern,
+but you can add additional patterns that don't.
+
+Also note that all the regular expressions used by inline must
+capture the whole block. For this reason, they all start with
+'^(.*)' and end with '(.*)!'. In case with built-in expression
+Pattern takes care of adding the "^(.*)" and "(.*)!".
+
+Finally, the order in which regular expressions are applied is very
+important - e.g. if we first replace http://.../ links with <a> tags
+and _then_ try to replace inline html, we would end up with a mess.
+So, we apply the expressions in the following order:
+
+* escape and backticks have to go before everything else, so
+ that we can preempt any markdown patterns by escaping them.
+
+* then we handle auto-links (must be done before inline html)
+
+* then we handle inline HTML. At this point we will simply
+ replace all inline HTML strings with a placeholder and add
+ the actual HTML to a hash.
+
+* then inline images (must be done before links)
+
+* then bracketed links, first regular then reference-style
+
+* finally we apply strong and emphasis
+"""
+
+import markdown
+import re
+from urlparse import urlparse, urlunparse
+import sys
+if sys.version >= "3.0":
+ from html import entities as htmlentitydefs
+else:
+ import htmlentitydefs
+
+"""
+The actual regular expressions for patterns
+-----------------------------------------------------------------------------
+"""
+
+NOBRACKET = r'[^\]\[]*'
+BRK = ( r'\[('
+ + (NOBRACKET + r'(\[')*6
+ + (NOBRACKET+ r'\])*')*6
+ + NOBRACKET + r')\]' )
+NOIMG = r'(?<!\!)'
+
+BACKTICK_RE = r'(?<!\\)(`+)(.+?)(?<!`)\2(?!`)' # `e=f()` or ``e=f("`")``
+ESCAPE_RE = r'\\(.)' # \<
+EMPHASIS_RE = r'(\*)([^\*]*)\2' # *emphasis*
+STRONG_RE = r'(\*{2}|_{2})(.*?)\2' # **strong**
+STRONG_EM_RE = r'(\*{3}|_{3})(.*?)\2' # ***strong***
+
+if markdown.SMART_EMPHASIS:
+ EMPHASIS_2_RE = r'(?<!\S)(_)(\S.*?)\2' # _emphasis_
+else:
+ EMPHASIS_2_RE = r'(_)(.*?)\2' # _emphasis_
+
+LINK_RE = NOIMG + BRK + \
+r'''\(\s*(<.*?>|((?:(?:\(.*?\))|[^\(\)]))*?)\s*((['"])(.*)\12)?\)'''
+# [text](url) or [text](<url>)
+
+IMAGE_LINK_RE = r'\!' + BRK + r'\s*\((<.*?>|([^\)]*))\)'
+#  or 
+REFERENCE_RE = NOIMG + BRK+ r'\s*\[([^\]]*)\]' # [Google][3]
+IMAGE_REFERENCE_RE = r'\!' + BRK + '\s*\[([^\]]*)\]' # ![alt text][2]
+NOT_STRONG_RE = r'( \* )' # stand-alone * or _
+AUTOLINK_RE = r'<((?:f|ht)tps?://[^>]*)>' # <http://www.123.com>
+AUTOMAIL_RE = r'<([^> \!]*@[^> ]*)>' # <me@example.com>
+
+HTML_RE = r'(\<([a-zA-Z/][^\>]*?|\!--.*?--)\>)' # <...>
+ENTITY_RE = r'(&[\#a-zA-Z0-9]*;)' # &
+LINE_BREAK_RE = r' \n' # two spaces at end of line
+LINE_BREAK_2_RE = r' $' # two spaces at end of text
+
+
+def dequote(string):
+ """Remove quotes from around a string."""
+ if ( ( string.startswith('"') and string.endswith('"'))
+ or (string.startswith("'") and string.endswith("'")) ):
+ return string[1:-1]
+ else:
+ return string
+
+ATTR_RE = re.compile("\{@([^\}]*)=([^\}]*)}") # {@id=123}
+
+def handleAttributes(text, parent):
+ """Set values of an element based on attribute definitions ({@id=123})."""
+ def attributeCallback(match):
+ parent.set(match.group(1), match.group(2).replace('\n', ' '))
+ return ATTR_RE.sub(attributeCallback, text)
+
+
+"""
+The pattern classes
+-----------------------------------------------------------------------------
+"""
+
+class Pattern:
+ """Base class that inline patterns subclass. """
+
+ def __init__ (self, pattern, markdown_instance=None):
+ """
+ Create an instant of an inline pattern.
+
+ Keyword arguments:
+
+ * pattern: A regular expression that matches a pattern
+
+ """
+ self.pattern = pattern
+ self.compiled_re = re.compile("^(.*?)%s(.*?)$" % pattern, re.DOTALL)
+
+ # Api for Markdown to pass safe_mode into instance
+ self.safe_mode = False
+ if markdown_instance:
+ self.markdown = markdown_instance
+
+ def getCompiledRegExp (self):
+ """ Return a compiled regular expression. """
+ return self.compiled_re
+
+ def handleMatch(self, m):
+ """Return a ElementTree element from the given match.
+
+ Subclasses should override this method.
+
+ Keyword arguments:
+
+ * m: A re match object containing a match of the pattern.
+
+ """
+ pass
+
+ def type(self):
+ """ Return class name, to define pattern type """
+ return self.__class__.__name__
+
+BasePattern = Pattern # for backward compatibility
+
+class SimpleTextPattern (Pattern):
+ """ Return a simple text of group(2) of a Pattern. """
+ def handleMatch(self, m):
+ text = m.group(2)
+ if text == markdown.INLINE_PLACEHOLDER_PREFIX:
+ return None
+ return text
+
+class SimpleTagPattern (Pattern):
+ """
+ Return element of type `tag` with a text attribute of group(3)
+ of a Pattern.
+
+ """
+ def __init__ (self, pattern, tag):
+ Pattern.__init__(self, pattern)
+ self.tag = tag
+
+ def handleMatch(self, m):
+ el = markdown.etree.Element(self.tag)
+ el.text = m.group(3)
+ return el
+
+
+class SubstituteTagPattern (SimpleTagPattern):
+ """ Return a eLement of type `tag` with no children. """
+ def handleMatch (self, m):
+ return markdown.etree.Element(self.tag)
+
+
+class BacktickPattern (Pattern):
+ """ Return a `<code>` element containing the matching text. """
+ def __init__ (self, pattern):
+ Pattern.__init__(self, pattern)
+ self.tag = "code"
+
+ def handleMatch(self, m):
+ el = markdown.etree.Element(self.tag)
+ el.text = markdown.AtomicString(m.group(3).strip())
+ return el
+
+
+class DoubleTagPattern (SimpleTagPattern):
+ """Return a ElementTree element nested in tag2 nested in tag1.
+
+ Useful for strong emphasis etc.
+
+ """
+ def handleMatch(self, m):
+ tag1, tag2 = self.tag.split(",")
+ el1 = markdown.etree.Element(tag1)
+ el2 = markdown.etree.SubElement(el1, tag2)
+ el2.text = m.group(3)
+ return el1
+
+
+class HtmlPattern (Pattern):
+ """ Store raw inline html and return a placeholder. """
+ def handleMatch (self, m):
+ rawhtml = m.group(2)
+ inline = True
+ place_holder = self.markdown.htmlStash.store(rawhtml)
+ return place_holder
+
+
+class LinkPattern (Pattern):
+ """ Return a link element from the given match. """
+ def handleMatch(self, m):
+ el = markdown.etree.Element("a")
+ el.text = m.group(2)
+ title = m.group(11)
+ href = m.group(9)
+
+ if href:
+ if href[0] == "<":
+ href = href[1:-1]
+ el.set("href", self.sanitize_url(href.strip()))
+ else:
+ el.set("href", "")
+
+ if title:
+ title = dequote(title) #.replace('"', """)
+ el.set("title", title)
+ return el
+
+ def sanitize_url(self, url):
+ """
+ Sanitize a url against xss attacks in "safe_mode".
+
+ Rather than specifically blacklisting `javascript:alert("XSS")` and all
+ its aliases (see <http://ha.ckers.org/xss.html>), we whitelist known
+ safe url formats. Most urls contain a network location, however some
+ are known not to (i.e.: mailto links). Script urls do not contain a
+ location. Additionally, for `javascript:...`, the scheme would be
+ "javascript" but some aliases will appear to `urlparse()` to have no
+ scheme. On top of that relative links (i.e.: "foo/bar.html") have no
+ scheme. Therefore we must check "path", "parameters", "query" and
+ "fragment" for any literal colons. We don't check "scheme" for colons
+ because it *should* never have any and "netloc" must allow the form:
+ `username:password@host:port`.
+
+ """
+ locless_schemes = ['', 'mailto', 'news']
+ scheme, netloc, path, params, query, fragment = url = urlparse(url)
+ safe_url = False
+ if netloc != '' or scheme in locless_schemes:
+ safe_url = True
+
+ for part in url[2:]:
+ if ":" in part:
+ safe_url = False
+
+ if self.markdown.safeMode and not safe_url:
+ return ''
+ else:
+ return urlunparse(url)
+
+class ImagePattern(LinkPattern):
+ """ Return a img element from the given match. """
+ def handleMatch(self, m):
+ el = markdown.etree.Element("img")
+ src_parts = m.group(9).split()
+ if src_parts:
+ src = src_parts[0]
+ if src[0] == "<" and src[-1] == ">":
+ src = src[1:-1]
+ el.set('src', self.sanitize_url(src))
+ else:
+ el.set('src', "")
+ if len(src_parts) > 1:
+ el.set('title', dequote(" ".join(src_parts[1:])))
+
+ if markdown.ENABLE_ATTRIBUTES:
+ truealt = handleAttributes(m.group(2), el)
+ else:
+ truealt = m.group(2)
+
+ el.set('alt', truealt)
+ return el
+
+class ReferencePattern(LinkPattern):
+ """ Match to a stored reference and return link element. """
+ def handleMatch(self, m):
+ if m.group(9):
+ id = m.group(9).lower()
+ else:
+ # if we got something like "[Google][]"
+ # we'll use "google" as the id
+ id = m.group(2).lower()
+
+ if not id in self.markdown.references: # ignore undefined refs
+ return None
+ href, title = self.markdown.references[id]
+
+ text = m.group(2)
+ return self.makeTag(href, title, text)
+
+ def makeTag(self, href, title, text):
+ el = markdown.etree.Element('a')
+
+ el.set('href', self.sanitize_url(href))
+ if title:
+ el.set('title', title)
+
+ el.text = text
+ return el
+
+
+class ImageReferencePattern (ReferencePattern):
+ """ Match to a stored reference and return img element. """
+ def makeTag(self, href, title, text):
+ el = markdown.etree.Element("img")
+ el.set("src", self.sanitize_url(href))
+ if title:
+ el.set("title", title)
+ el.set("alt", text)
+ return el
+
+
+class AutolinkPattern (Pattern):
+ """ Return a link Element given an autolink (`<http://example/com>`). """
+ def handleMatch(self, m):
+ el = markdown.etree.Element("a")
+ el.set('href', m.group(2))
+ el.text = markdown.AtomicString(m.group(2))
+ return el
+
+class AutomailPattern (Pattern):
+ """
+ Return a mailto link Element given an automail link (`<foo@example.com>`).
+ """
+ def handleMatch(self, m):
+ el = markdown.etree.Element('a')
+ email = m.group(2)
+ if email.startswith("mailto:"):
+ email = email[len("mailto:"):]
+
+ def codepoint2name(code):
+ """Return entity definition by code, or the code if not defined."""
+ entity = htmlentitydefs.codepoint2name.get(code)
+ if entity:
+ return "%s%s;" % (markdown.AMP_SUBSTITUTE, entity)
+ else:
+ return "%s#%d;" % (markdown.AMP_SUBSTITUTE, code)
+
+ letters = [codepoint2name(ord(letter)) for letter in email]
+ el.text = markdown.AtomicString(''.join(letters))
+
+ mailto = "mailto:" + email
+ mailto = "".join([markdown.AMP_SUBSTITUTE + '#%d;' %
+ ord(letter) for letter in mailto])
+ el.set('href', mailto)
+ return el
+
--- /dev/null
+class OrderedDict(dict):
+ """
+ A dictionary that keeps its keys in the order in which they're inserted.
+
+ Copied from Django's SortedDict with some modifications.
+
+ """
+ def __new__(cls, *args, **kwargs):
+ instance = super(OrderedDict, cls).__new__(cls, *args, **kwargs)
+ instance.keyOrder = []
+ return instance
+
+ def __init__(self, data=None):
+ if data is None:
+ data = {}
+ super(OrderedDict, self).__init__(data)
+ if isinstance(data, dict):
+ self.keyOrder = data.keys()
+ else:
+ self.keyOrder = []
+ for key, value in data:
+ if key not in self.keyOrder:
+ self.keyOrder.append(key)
+
+ def __deepcopy__(self, memo):
+ from copy import deepcopy
+ return self.__class__([(key, deepcopy(value, memo))
+ for key, value in self.iteritems()])
+
+ def __setitem__(self, key, value):
+ super(OrderedDict, self).__setitem__(key, value)
+ if key not in self.keyOrder:
+ self.keyOrder.append(key)
+
+ def __delitem__(self, key):
+ super(OrderedDict, self).__delitem__(key)
+ self.keyOrder.remove(key)
+
+ def __iter__(self):
+ for k in self.keyOrder:
+ yield k
+
+ def pop(self, k, *args):
+ result = super(OrderedDict, self).pop(k, *args)
+ try:
+ self.keyOrder.remove(k)
+ except ValueError:
+ # Key wasn't in the dictionary in the first place. No problem.
+ pass
+ return result
+
+ def popitem(self):
+ result = super(OrderedDict, self).popitem()
+ self.keyOrder.remove(result[0])
+ return result
+
+ def items(self):
+ return zip(self.keyOrder, self.values())
+
+ def iteritems(self):
+ for key in self.keyOrder:
+ yield key, super(OrderedDict, self).__getitem__(key)
+
+ def keys(self):
+ return self.keyOrder[:]
+
+ def iterkeys(self):
+ return iter(self.keyOrder)
+
+ def values(self):
+ return [super(OrderedDict, self).__getitem__(k) for k in self.keyOrder]
+
+ def itervalues(self):
+ for key in self.keyOrder:
+ yield super(OrderedDict, self).__getitem__(key)
+
+ def update(self, dict_):
+ for k, v in dict_.items():
+ self.__setitem__(k, v)
+
+ def setdefault(self, key, default):
+ if key not in self.keyOrder:
+ self.keyOrder.append(key)
+ return super(OrderedDict, self).setdefault(key, default)
+
+ def value_for_index(self, index):
+ """Return the value of the item at the given zero-based index."""
+ return self[self.keyOrder[index]]
+
+ def insert(self, index, key, value):
+ """Insert the key, value pair before the item with the given index."""
+ if key in self.keyOrder:
+ n = self.keyOrder.index(key)
+ del self.keyOrder[n]
+ if n < index:
+ index -= 1
+ self.keyOrder.insert(index, key)
+ super(OrderedDict, self).__setitem__(key, value)
+
+ def copy(self):
+ """Return a copy of this object."""
+ # This way of initializing the copy means it works for subclasses, too.
+ obj = self.__class__(self)
+ obj.keyOrder = self.keyOrder[:]
+ return obj
+
+ def __repr__(self):
+ """
+ Replace the normal dict.__repr__ with a version that returns the keys
+ in their sorted order.
+ """
+ return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()])
+
+ def clear(self):
+ super(OrderedDict, self).clear()
+ self.keyOrder = []
+
+ def index(self, key):
+ """ Return the index of a given key. """
+ return self.keyOrder.index(key)
+
+ def index_for_location(self, location):
+ """ Return index or None for a given location. """
+ if location == '_begin':
+ i = 0
+ elif location == '_end':
+ i = None
+ elif location.startswith('<') or location.startswith('>'):
+ i = self.index(location[1:])
+ if location.startswith('>'):
+ if i >= len(self):
+ # last item
+ i = None
+ else:
+ i += 1
+ else:
+ raise ValueError('Not a valid location: "%s". Location key '
+ 'must start with a ">" or "<".' % location)
+ return i
+
+ def add(self, key, value, location):
+ """ Insert by key location. """
+ i = self.index_for_location(location)
+ if i is not None:
+ self.insert(i, key, value)
+ else:
+ self.__setitem__(key, value)
+
+ def link(self, key, location):
+ """ Change location of an existing item. """
+ n = self.keyOrder.index(key)
+ del self.keyOrder[n]
+ i = self.index_for_location(location)
+ try:
+ if i is not None:
+ self.keyOrder.insert(i, key)
+ else:
+ self.keyOrder.append(key)
+ except Error:
+ # restore to prevent data loss and reraise
+ self.keyOrder.insert(n, key)
+ raise Error
--- /dev/null
+"""
+POST-PROCESSORS
+=============================================================================
+
+Markdown also allows post-processors, which are similar to preprocessors in
+that they need to implement a "run" method. However, they are run after core
+processing.
+
+"""
+
+
+import markdown
+
+class Processor:
+ def __init__(self, markdown_instance=None):
+ if markdown_instance:
+ self.markdown = markdown_instance
+
+class Postprocessor(Processor):
+ """
+ Postprocessors are run after the ElementTree it converted back into text.
+
+ Each Postprocessor implements a "run" method that takes a pointer to a
+ text string, modifies it as necessary and returns a text string.
+
+ Postprocessors must extend markdown.Postprocessor.
+
+ """
+
+ def run(self, text):
+ """
+ Subclasses of Postprocessor should implement a `run` method, which
+ takes the html document as a single text string and returns a
+ (possibly modified) string.
+
+ """
+ pass
+
+
+class RawHtmlPostprocessor(Postprocessor):
+ """ Restore raw html to the document. """
+
+ def run(self, text):
+ """ Iterate over html stash and restore "safe" html. """
+ for i in range(self.markdown.htmlStash.html_counter):
+ html, safe = self.markdown.htmlStash.rawHtmlBlocks[i]
+ if self.markdown.safeMode and not safe:
+ if str(self.markdown.safeMode).lower() == 'escape':
+ html = self.escape(html)
+ elif str(self.markdown.safeMode).lower() == 'remove':
+ html = ''
+ else:
+ html = markdown.HTML_REMOVED_TEXT
+ if safe or not self.markdown.safeMode:
+ text = text.replace("<p>%s</p>" %
+ (markdown.preprocessors.HTML_PLACEHOLDER % i),
+ html + "\n")
+ text = text.replace(markdown.preprocessors.HTML_PLACEHOLDER % i,
+ html)
+ return text
+
+ def escape(self, html):
+ """ Basic html escaping """
+ html = html.replace('&', '&')
+ html = html.replace('<', '<')
+ html = html.replace('>', '>')
+ return html.replace('"', '"')
+
+
+class AndSubstitutePostprocessor(Postprocessor):
+ """ Restore valid entities """
+ def __init__(self):
+ pass
+
+ def run(self, text):
+ text = text.replace(markdown.AMP_SUBSTITUTE, "&")
+ return text
--- /dev/null
+
+"""
+PRE-PROCESSORS
+=============================================================================
+
+Preprocessors work on source text before we start doing anything too
+complicated.
+"""
+
+import re
+import markdown
+
+HTML_PLACEHOLDER_PREFIX = markdown.STX+"wzxhzdk:"
+HTML_PLACEHOLDER = HTML_PLACEHOLDER_PREFIX + "%d" + markdown.ETX
+
+class Processor:
+ def __init__(self, markdown_instance=None):
+ if markdown_instance:
+ self.markdown = markdown_instance
+
+class Preprocessor (Processor):
+ """
+ Preprocessors are run after the text is broken into lines.
+
+ Each preprocessor implements a "run" method that takes a pointer to a
+ list of lines of the document, modifies it as necessary and returns
+ either the same pointer or a pointer to a new list.
+
+ Preprocessors must extend markdown.Preprocessor.
+
+ """
+ def run(self, lines):
+ """
+ Each subclass of Preprocessor should override the `run` method, which
+ takes the document as a list of strings split by newlines and returns
+ the (possibly modified) list of lines.
+
+ """
+ pass
+
+class HtmlStash:
+ """
+ This class is used for stashing HTML objects that we extract
+ in the beginning and replace with place-holders.
+ """
+
+ def __init__ (self):
+ """ Create a HtmlStash. """
+ self.html_counter = 0 # for counting inline html segments
+ self.rawHtmlBlocks=[]
+
+ def store(self, html, safe=False):
+ """
+ Saves an HTML segment for later reinsertion. Returns a
+ placeholder string that needs to be inserted into the
+ document.
+
+ Keyword arguments:
+
+ * html: an html segment
+ * safe: label an html segment as safe for safemode
+
+ Returns : a placeholder string
+
+ """
+ self.rawHtmlBlocks.append((html, safe))
+ placeholder = HTML_PLACEHOLDER % self.html_counter
+ self.html_counter += 1
+ return placeholder
+
+ def reset(self):
+ self.html_counter = 0
+ self.rawHtmlBlocks = []
+
+
+class HtmlBlockPreprocessor(Preprocessor):
+ """Remove html blocks from the text and store them for later retrieval."""
+
+ right_tag_patterns = ["</%s>", "%s>"]
+
+ def _get_left_tag(self, block):
+ return block[1:].replace(">", " ", 1).split()[0].lower()
+
+ def _get_right_tag(self, left_tag, block):
+ for p in self.right_tag_patterns:
+ tag = p % left_tag
+ i = block.rfind(tag)
+ if i > 2:
+ return tag.lstrip("<").rstrip(">"), i + len(p)-2 + len(left_tag)
+ return block.rstrip()[-len(left_tag)-2:-1].lower(), len(block)
+
+ def _equal_tags(self, left_tag, right_tag):
+ if left_tag == 'div' or left_tag[0] in ['?', '@', '%']: # handle PHP, etc.
+ return True
+ if ("/" + left_tag) == right_tag:
+ return True
+ if (right_tag == "--" and left_tag == "--"):
+ return True
+ elif left_tag == right_tag[1:] \
+ and right_tag[0] != "<":
+ return True
+ else:
+ return False
+
+ def _is_oneliner(self, tag):
+ return (tag in ['hr', 'hr/'])
+
+ def run(self, lines):
+ text = "\n".join(lines)
+ new_blocks = []
+ text = text.split("\n\n")
+ items = []
+ left_tag = ''
+ right_tag = ''
+ in_tag = False # flag
+
+ while text:
+ block = text[0]
+ if block.startswith("\n"):
+ block = block[1:]
+ text = text[1:]
+
+ if block.startswith("\n"):
+ block = block[1:]
+
+ if not in_tag:
+ if block.startswith("<"):
+ left_tag = self._get_left_tag(block)
+ right_tag, data_index = self._get_right_tag(left_tag, block)
+
+ if data_index < len(block):
+ text.insert(0, block[data_index:])
+ block = block[:data_index]
+
+ if not (markdown.isBlockLevel(left_tag) \
+ or block[1] in ["!", "?", "@", "%"]):
+ new_blocks.append(block)
+ continue
+
+ if self._is_oneliner(left_tag):
+ new_blocks.append(block.strip())
+ continue
+
+ if block[1] == "!":
+ # is a comment block
+ left_tag = "--"
+ right_tag, data_index = self._get_right_tag(left_tag, block)
+ # keep checking conditions below and maybe just append
+
+ if block.rstrip().endswith(">") \
+ and self._equal_tags(left_tag, right_tag):
+ new_blocks.append(
+ self.markdown.htmlStash.store(block.strip()))
+ continue
+ else: #if not block[1] == "!":
+ # if is block level tag and is not complete
+
+ if markdown.isBlockLevel(left_tag) or left_tag == "--" \
+ and not block.rstrip().endswith(">"):
+ items.append(block.strip())
+ in_tag = True
+ else:
+ new_blocks.append(
+ self.markdown.htmlStash.store(block.strip()))
+
+ continue
+
+ new_blocks.append(block)
+
+ else:
+ items.append(block.strip())
+
+ right_tag, data_index = self._get_right_tag(left_tag, block)
+
+ if self._equal_tags(left_tag, right_tag):
+ # if find closing tag
+ in_tag = False
+ new_blocks.append(
+ self.markdown.htmlStash.store('\n\n'.join(items)))
+ items = []
+
+ if items:
+ new_blocks.append(self.markdown.htmlStash.store('\n\n'.join(items)))
+ new_blocks.append('\n')
+
+ new_text = "\n\n".join(new_blocks)
+ return new_text.split("\n")
+
+
+class ReferencePreprocessor(Preprocessor):
+ """ Remove reference definitions from text and store for later use. """
+
+ RE = re.compile(r'^(\ ?\ ?\ ?)\[([^\]]*)\]:\s*([^ ]*)(.*)$', re.DOTALL)
+
+ def run (self, lines):
+ new_text = [];
+ for line in lines:
+ m = self.RE.match(line)
+ if m:
+ id = m.group(2).strip().lower()
+ t = m.group(4).strip() # potential title
+ if not t:
+ self.markdown.references[id] = (m.group(3), t)
+ elif (len(t) >= 2
+ and (t[0] == t[-1] == "\""
+ or t[0] == t[-1] == "\'"
+ or (t[0] == "(" and t[-1] == ")") ) ):
+ self.markdown.references[id] = (m.group(3), t[1:-1])
+ else:
+ new_text.append(line)
+ else:
+ new_text.append(line)
+
+ return new_text #+ "\n"
--- /dev/null
+import markdown
+import re
+
+def isString(s):
+ """ Check if it's string """
+ return isinstance(s, unicode) or isinstance(s, str)
+
+class Processor:
+ def __init__(self, markdown_instance=None):
+ if markdown_instance:
+ self.markdown = markdown_instance
+
+class Treeprocessor(Processor):
+ """
+ Treeprocessors are run on the ElementTree object before serialization.
+
+ Each Treeprocessor implements a "run" method that takes a pointer to an
+ ElementTree, modifies it as necessary and returns an ElementTree
+ object.
+
+ Treeprocessors must extend markdown.Treeprocessor.
+
+ """
+ def run(self, root):
+ """
+ Subclasses of Treeprocessor should implement a `run` method, which
+ takes a root ElementTree. This method can return another ElementTree
+ object, and the existing root ElementTree will be replaced, or it can
+ modify the current tree and return None.
+ """
+ pass
+
+
+class InlineProcessor(Treeprocessor):
+ """
+ A Treeprocessor that traverses a tree, applying inline patterns.
+ """
+
+ def __init__ (self, md):
+ self.__placeholder_prefix = markdown.INLINE_PLACEHOLDER_PREFIX
+ self.__placeholder_suffix = markdown.ETX
+ self.__placeholder_length = 4 + len(self.__placeholder_prefix) \
+ + len(self.__placeholder_suffix)
+ self.__placeholder_re = re.compile(markdown.INLINE_PLACEHOLDER % r'([0-9]{4})')
+ self.markdown = md
+
+ def __makePlaceholder(self, type):
+ """ Generate a placeholder """
+ id = "%04d" % len(self.stashed_nodes)
+ hash = markdown.INLINE_PLACEHOLDER % id
+ return hash, id
+
+ def __findPlaceholder(self, data, index):
+ """
+ Extract id from data string, start from index
+
+ Keyword arguments:
+
+ * data: string
+ * index: index, from which we start search
+
+ Returns: placeholder id and string index, after the found placeholder.
+ """
+
+ m = self.__placeholder_re.search(data, index)
+ if m:
+ return m.group(1), m.end()
+ else:
+ return None, index + 1
+
+ def __stashNode(self, node, type):
+ """ Add node to stash """
+ placeholder, id = self.__makePlaceholder(type)
+ self.stashed_nodes[id] = node
+ return placeholder
+
+ def __handleInline(self, data, patternIndex=0):
+ """
+ Process string with inline patterns and replace it
+ with placeholders
+
+ Keyword arguments:
+
+ * data: A line of Markdown text
+ * patternIndex: The index of the inlinePattern to start with
+
+ Returns: String with placeholders.
+
+ """
+ if not isinstance(data, markdown.AtomicString):
+ startIndex = 0
+ while patternIndex < len(self.markdown.inlinePatterns):
+ data, matched, startIndex = self.__applyPattern(
+ self.markdown.inlinePatterns.value_for_index(patternIndex),
+ data, patternIndex, startIndex)
+ if not matched:
+ patternIndex += 1
+ return data
+
+ def __processElementText(self, node, subnode, isText=True):
+ """
+ Process placeholders in Element.text or Element.tail
+ of Elements popped from self.stashed_nodes.
+
+ Keywords arguments:
+
+ * node: parent node
+ * subnode: processing node
+ * isText: bool variable, True - it's text, False - it's tail
+
+ Returns: None
+
+ """
+ if isText:
+ text = subnode.text
+ subnode.text = None
+ else:
+ text = subnode.tail
+ subnode.tail = None
+
+ childResult = self.__processPlaceholders(text, subnode)
+
+ if not isText and node is not subnode:
+ pos = node.getchildren().index(subnode)
+ node.remove(subnode)
+ else:
+ pos = 0
+
+ childResult.reverse()
+ for newChild in childResult:
+ node.insert(pos, newChild)
+
+ def __processPlaceholders(self, data, parent):
+ """
+ Process string with placeholders and generate ElementTree tree.
+
+ Keyword arguments:
+
+ * data: string with placeholders instead of ElementTree elements.
+ * parent: Element, which contains processing inline data
+
+ Returns: list with ElementTree elements with applied inline patterns.
+ """
+ def linkText(text):
+ if text:
+ if result:
+ if result[-1].tail:
+ result[-1].tail += text
+ else:
+ result[-1].tail = text
+ else:
+ if parent.text:
+ parent.text += text
+ else:
+ parent.text = text
+
+ result = []
+ strartIndex = 0
+ while data:
+ index = data.find(self.__placeholder_prefix, strartIndex)
+ if index != -1:
+ id, phEndIndex = self.__findPlaceholder(data, index)
+
+ if id in self.stashed_nodes:
+ node = self.stashed_nodes.get(id)
+
+ if index > 0:
+ text = data[strartIndex:index]
+ linkText(text)
+
+ if not isString(node): # it's Element
+ for child in [node] + node.getchildren():
+ if child.tail:
+ if child.tail.strip():
+ self.__processElementText(node, child, False)
+ if child.text:
+ if child.text.strip():
+ self.__processElementText(child, child)
+ else: # it's just a string
+ linkText(node)
+ strartIndex = phEndIndex
+ continue
+
+ strartIndex = phEndIndex
+ result.append(node)
+
+ else: # wrong placeholder
+ end = index + len(prefix)
+ linkText(data[strartIndex:end])
+ strartIndex = end
+ else:
+ text = data[strartIndex:]
+ linkText(text)
+ data = ""
+
+ return result
+
+ def __applyPattern(self, pattern, data, patternIndex, startIndex=0):
+ """
+ Check if the line fits the pattern, create the necessary
+ elements, add it to stashed_nodes.
+
+ Keyword arguments:
+
+ * data: the text to be processed
+ * pattern: the pattern to be checked
+ * patternIndex: index of current pattern
+ * startIndex: string index, from which we starting search
+
+ Returns: String with placeholders instead of ElementTree elements.
+
+ """
+ match = pattern.getCompiledRegExp().match(data[startIndex:])
+ leftData = data[:startIndex]
+
+ if not match:
+ return data, False, 0
+
+ node = pattern.handleMatch(match)
+
+ if node is None:
+ return data, True, len(leftData) + match.span(len(match.groups()))[0]
+
+ if not isString(node):
+ if not isinstance(node.text, markdown.AtomicString):
+ # We need to process current node too
+ for child in [node] + node.getchildren():
+ if not isString(node):
+ if child.text:
+ child.text = self.__handleInline(child.text,
+ patternIndex + 1)
+ if child.tail:
+ child.tail = self.__handleInline(child.tail,
+ patternIndex)
+
+ placeholder = self.__stashNode(node, pattern.type())
+
+ return "%s%s%s%s" % (leftData,
+ match.group(1),
+ placeholder, match.groups()[-1]), True, 0
+
+ def run(self, tree):
+ """Apply inline patterns to a parsed Markdown tree.
+
+ Iterate over ElementTree, find elements with inline tag, apply inline
+ patterns and append newly created Elements to tree. If you don't
+ want process your data with inline paterns, instead of normal string,
+ use subclass AtomicString:
+
+ node.text = markdown.AtomicString("data won't be processed with inline patterns")
+
+ Arguments:
+
+ * markdownTree: ElementTree object, representing Markdown tree.
+
+ Returns: ElementTree object with applied inline patterns.
+
+ """
+ self.stashed_nodes = {}
+
+ stack = [tree]
+
+ while stack:
+ currElement = stack.pop()
+ insertQueue = []
+ for child in currElement.getchildren():
+ if child.text and not isinstance(child.text, markdown.AtomicString):
+ text = child.text
+ child.text = None
+ lst = self.__processPlaceholders(self.__handleInline(
+ text), child)
+ stack += lst
+ insertQueue.append((child, lst))
+
+ if child.getchildren():
+ stack.append(child)
+
+ for element, lst in insertQueue:
+ if element.text:
+ element.text = \
+ markdown.inlinepatterns.handleAttributes(element.text,
+ element)
+ i = 0
+ for newChild in lst:
+ # Processing attributes
+ if newChild.tail:
+ newChild.tail = \
+ markdown.inlinepatterns.handleAttributes(newChild.tail,
+ element)
+ if newChild.text:
+ newChild.text = \
+ markdown.inlinepatterns.handleAttributes(newChild.text,
+ newChild)
+ element.insert(i, newChild)
+ i += 1
+ return tree
+
+
+class PrettifyTreeprocessor(Treeprocessor):
+ """ Add linebreaks to the html document. """
+
+ def _prettifyETree(self, elem):
+ """ Recursively add linebreaks to ElementTree children. """
+
+ i = "\n"
+ if markdown.isBlockLevel(elem.tag) and elem.tag not in ['code', 'pre']:
+ if (not elem.text or not elem.text.strip()) \
+ and len(elem) and markdown.isBlockLevel(elem[0].tag):
+ elem.text = i
+ for e in elem:
+ if markdown.isBlockLevel(e.tag):
+ self._prettifyETree(e)
+ if not elem.tail or not elem.tail.strip():
+ elem.tail = i
+ if not elem.tail or not elem.tail.strip():
+ elem.tail = i
+
+ def run(self, root):
+ """ Add linebreaks to ElementTree root object. """
+
+ self._prettifyETree(root)
+ # Do <br />'s seperately as they are often in the middle of
+ # inline content and missed by _prettifyETree.
+ brs = root.getiterator('br')
+ for br in brs:
+ if not br.tail or not br.tail.strip():
+ br.tail = '\n'
+ else:
+ br.tail = '\n%s' % br.tail
--- /dev/null
+body {
+ background: white;
+ color: black;
+ font-family: Georgia, serif;
+ font-size: 11pt;
+ margin: 10px;
+ margin-top: 15px;
+ margin-bottom: 15px;
+}
+
+h1,
+h2,
+h3,
+h4 {
+ font-family: Calibri, sans-serif;
+ margin: 0;
+}
+
+img {
+ border: 0;
+}
+
+pre,
+code {
+ color: #060;
+}
+
+a,
+a code {
+ color: #216093;
+}
+
+table {
+ border-collapse: collapse;
+ border: 0;
+}
+
+td {
+ border: 0;
+ padding: 0;
+}
+
+#body {
+ margin: auto;
+ max-width: 850px;
+}
+
+#header {
+ margin-bottom: 15px;
+ margin-right: 30px;
+}
+
+#content,
+#footer {
+ margin-left: 31px;
+ margin-right: 31px;
+}
+
+#content p,
+#content li,
+#footer {
+ line-height: 16pt;
+}
+
+#content pre {
+ line-height: 14pt;
+ margin: 17pt;
+ padding-left: 1em;
+ border-left: 1px solid #ccc;
+}
+
+#footer {
+ margin-top: 5em;
+}
+
+#header .logo {
+ line-height: 0;
+ padding-bottom: 5px;
+ padding-right: 15px;
+}
+
+#header .logo img {
+ width: 286px;
+ height: 72px;
+}
+
+#header .title {
+ vertical-align: bottom;
+}
+
+#header .title h1 {
+ font-size: 35px;
+ font-weight: normal;
+}
+
+#header .title h1,
+#header .title h1 a {
+ color: #666;
+}
+
+#content h1,
+#content h2,
+#content h3 {
+ color: #4d8cbf;
+ margin-bottom: 2pt;
+ margin-top: 17pt;
+}
+
+#content h2 {
+ font-size: 19pt;
+}
+
+#content h3 {
+ font-size: 15pt;
+}
+
+#content p {
+ margin: 0;
+ margin-bottom: 1em;
+}
--- /dev/null
+User-Agent: *
+Disallow:
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+ <title>{% block title %}Tornado Web Server{% end %}</title>
+ <link rel="stylesheet" href="/static/base.css" type="text/css"/>
+ {% block head %}{% end %}
+ </head>
+ <body>
+ <div id="body">
+ <div id="header">
+ <table>
+ <tr>
+ <td class="logo"><a href="/"><img src="/static/tornado.png" alt="Tornado"/></a></td>
+ <td class="title">{% block headertitle %}{% end %}</td>
+ </tr>
+ </table>
+ </div>
+ <div id="content">{% block body %}{% end %}</div>
+ <div id="footer">
+ <div>Tornado is one of <a href="http://developer.facebook.com/opensource.php">Facebook's open source technolgies</a>. It is available under the <a href="http://www.apache.org/licenses/LICENSE-2.0.html">Apache Licence, Version 2.0</a>.</div>
+ <div>This web site and all documentation is licensed under <a href="http://creativecommons.org/licenses/by/3.0/">Creative Commons 3.0</a>.</div>
+ </div>
+ </div>
+ {% block bottom %}{% end %}
+ </body>
+</html>
--- /dev/null
+{% extends "base.html" %}
+
+{% block title %}Tornado Web Server Documentation{% end %}
+
+{% block headertitle %}<h1>documentation</h1>{% end %}
+
+{% block body %}
+ {{ markdown("documentation.txt", toc=True) }}
+{% end %}
--- /dev/null
+Overview
+--------
+[FriendFeed](http://friendfeed.com/)'s web server is a relatively simple,
+non-blocking web server written in Python. The FriendFeed application is
+written using a web framework that looks a bit like
+[web.py](http://webpy.org/) or Google's
+[webapp](http://code.google.com/appengine/docs/python/tools/webapp/),
+but with additional tools and optimizations to take advantage of the
+non-blocking web server and tools.
+
+[Tornado](http://github.com/facebook/tornado) is an open source
+version of this web server and some of the tools we use most often at
+FriendFeed. The framework is distinct from most mainstream web server
+frameworks (and certainly most Python frameworks) because it is
+non-blocking and reasonably fast. Because it is non-blocking
+and uses [epoll](http://www.kernel.org/doc/man-pages/online/pages/man4/epoll.4.html), it can handle 1000s of simultaneous standing connections,
+which means the framework is ideal for real-time web services. We built the
+web server specifically to handle FriendFeed's real-time features —
+every active user of FriendFeed maintains an open connection to the
+FriendFeed servers. (For more information on scaling servers to support
+thousands of clients, see
+[The C10K problem](http://www.kegel.com/c10k.html).)
+
+Here is the canonical "Hello, world" example app:
+
+ import tornado.httpserver
+ import tornado.ioloop
+ import tornado.web
+
+ class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ self.write("Hello, world")
+
+ application = tornado.web.Application([
+ (r"/", MainHandler),
+ ])
+
+ if __name__ == "__main__":
+ http_server = tornado.httpserver.HTTPServer(application)
+ http_server.listen(8888)
+ tornado.ioloop.IOLoop.instance().start()
+
+See [Tornado walkthrough](#tornado-walkthrough) below for a detailed
+walkthrough of the `tornado.web` package.
+
+We attempted to clean up the code base to reduce interdependencies between
+modules, so you should (theoretically) be able to use any of the modules
+independently in your project without using the whole package.
+
+
+Download
+--------
+Download the most recent version of Tornado from GitHub:
+
+> [tornado-0.1.tar.gz](/static/tornado-0.1.tar.gz)
+
+You can also [browse the source](http://github.com/facebook/tornado) on GitHub.
+
+To install Tornado:
+
+ tar xvzf tornado-1.0.tar.gz
+ cd tornado-1.0
+ python setup.py build
+ sudo python setup.py install
+
+After installation, you should be able to run any of the demos in the `demos`
+directory included with the Tornado package.
+
+ ./demos/helloworld/helloworld.py
+
+
+Module index
+------------
+The most important module is [`web`](http://github.com/facebook/tornado/blob/master/tornado/web.py), which is the web framework
+that includes most of the meat of the Tornado package. The other modules
+are tools that make `web` more useful. See
+[Tornado walkthrough](#tornado-walkthrough) below for a detailed
+walkthrough of the `web` package.
+
+### Main modules
+ * [`web`](http://github.com/facebook/tornado/blob/master/tornado/web.py) - The web framework on which FriendFeed is built. `web` incorporates most of the important features of Tornado
+ * [`escape`](http://github.com/facebook/tornado/blob/master/tornado/escape.py) - XHTML, JSON, and URL encoding/decoding methods
+ * [`database`](http://github.com/facebook/tornado/blob/master/tornado/database.py) - A simple wrapper around `MySQLdb` to make MySQL easier to use
+ * [`template`](http://github.com/facebook/tornado/blob/master/tornado/template.py) - A Python-based web templating language
+ * [`httpclient`](http://github.com/facebook/tornado/blob/master/tornado/httpclient.py) - A non-blocking HTTP client designed to work with `web` and `httpserver`
+ * [`auth`](http://github.com/facebook/tornado/blob/master/tornado/auth.py) - Implementation of third party authentication and authorization schemes (Google OpenID/OAuth, Facebook Platform, Yahoo BBAuth, FriendFeed OpenID/OAuth, Twitter OAuth)
+ * [`locale`](http://github.com/facebook/tornado/blob/master/tornado/locale.py) - Localization/translation support
+ * [`options`](http://github.com/facebook/tornado/blob/master/tornado/options.py) - Command line and config file parsing, optimized for server environments
+
+### Low-level modules
+ * [`httpserver`](http://github.com/facebook/tornado/blob/master/tornado/httpserver.py) - A very simple HTTP server built on which `web` is built
+ * [`iostream`](http://github.com/facebook/tornado/blob/master/tornado/iostream.py) - A simple wrapper around non-blocking sockets to aide common reading and writing patterns
+ * [`ioloop`](http://github.com/facebook/tornado/blob/master/tornado/ioloop.py) - Core I/O loop
+
+### Random modules
+ * [`s3server`](http://github.com/facebook/tornado/blob/master/tornado/s3server.py) - A web server that implements most of the [Amazon S3](http://aws.amazon.com/s3/) interface, backed by local file storage
+
+
+Tornado walkthrough
+-------------------
+
+### Request handlers and request arguments
+
+A Tornado web application maps URLs or URL patterns to subclasses of
+`tornado.web.RequestHandler`. Those classes define `get()` or `post()`
+methods to handle HTTP `GET` or `POST` requests to that URL.
+
+This code maps the root URL `/` to `MainHandler` and the URL pattern
+`/story/([0-9]+)` to `StoryHandler`. Regular expression groups are passed
+as arguments to the `RequestHandler` methods:
+
+ class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ self.write("You requested the main page")
+
+ class StoryHandler(tornado.web.RequestHandler):
+ def get(self, story_id):
+ self.write("You requested the story " + story_id)
+
+ application = tornado.web.Application([
+ (r"/", MainHandler),
+ (r"/story/([0-9]+)", StoryHandler),
+ ])
+
+You can get query string arguments and parse `POST` bodies with the
+`get_argument()` method:
+
+ class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ self.write('<html><body><form action="/" method="post">'
+ '<input type="text" name="message">'
+ '<input type="submit" value="Submit">'
+ '</form></body></html>')
+
+ def post(self):
+ self.set_header("Content-Type", "text/plain")
+ self.write("You wrote " + self.get_argument("message"))
+
+If you want to send an error response to the client, e.g., 403 Unauthorized,
+you can just raise a `tornado.web.HTTPError` exception:
+
+ if not self.user_is_logged_in():
+ raise tornado.web.HTTPError(403)
+
+The request handler can access the object representing the current request
+with `self.request`. The `HTTPRequest` object includes a number of useful
+attribute, including:
+
+ * `arguments` - all of the `GET` and `POST` arguments
+ * `files` - all of the uploaded files (via `multipart/form-data` POST requests)
+ * `path` - the request path (everything before the `?`)
+ * `headers` - the request headers
+
+See the class definition for `HTTPRequest` in `httpserver` for a complete list
+of attributes.
+
+
+### Templates
+
+You can use any template language supported by Python, but Tornado ships
+with its own templating language that is a lot faster and more flexible
+than many of the most popular templating systems out there. See the
+[`template`](http://github.com/facebook/tornado/blob/master/tornado/template.py) module documentation for complete documentation.
+
+A Tornado template is just HTML (or any other text-based format) with
+Python control sequences and expressions embedded within the markup:
+
+ <html>
+ <head>
+ <title>{{ title }}</title>
+ </head>
+ <body>
+ <ul>
+ {% for item in items %}
+ <li>{{ escape(item) }}</li>
+ {% end %}
+ </ul>
+ </body>
+ </html>
+
+If you saved this template as "template.html" and put it in the same
+directory as your Python file, you could render this template with:
+
+ class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ items = ["Item 1", "Item 2", "Item 3"]
+ self.render("template.html", title="My title", items=items)
+
+Tornado templates support *control statements* and *expressions*. Control
+statements are surronded by `{%` and `%}`, e.g., `{% if len(items) > 2 %}`.
+Expressions are surrounded by `{{` and `}}`, e.g., `{{ items[0] }}`.
+
+Control statements more or less map exactly to Python statements. We support
+`if`, `for`, `while`, and `try`, all of which are terminated with `{% end %}`.
+We also support *template inheritance* using the `extends` and `block`
+statements, which are described in detail in the documentation for the
+[`template` module](http://github.com/facebook/tornado/blob/master/tornado/template.py).
+
+Expressions can be any Python expression, including function calls. We
+support the functions `escape`, `url_escape`, and `json_encode` by default,
+and you can pass other functions into the template simply by passing them
+as keyword arguments to the template render function:
+
+ class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ self.render("template.html", add=self.add)
+
+ def add(self, x, y):
+ return x + y
+
+When you are building a real application, you are going to want to use
+all of the features of Tornado templates, especially template inheritance.
+Read all about those features in the [`template` module](http://github.com/facebook/tornado/blob/master/tornado/template.py)
+section.
+
+Under the hood, Tornado templates are translated directly to Python.
+The expressions you include in your template are copied verbatim into
+a Python function representing your template. We don't try to prevent
+anything in the template language; we created it explicitly to provide
+the flexibility that other, stricter templating systems prevent.
+Consequently, if you write random stuff inside of your template expressions,
+you will get random Python errors when you execute the template.
+
+
+### Cookies and secure cookies
+
+You can set cookies in the user's browser with the `set_cookie` method:
+
+ class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ if not self.get_cookie("mycookie"):
+ self.set_cookie("mycookie", "myvalue")
+ self.write("Your cookie was not set yet!")
+ else:
+ self.write("Your cookie was set!")
+
+Cookies are easily forged by malicious clients. If you need to set cookies
+to, e.g., save the user ID of the currently logged in user, you need to
+sign your cookies to prevent forgery. Tornado supports this out of the
+box with the `set_secure_cookie` and `get_secure_cookie` methods. To use
+these methods, you need to specify a secret key named `cookie_secret` when
+you create your application. You can pass in application settings as keyword
+arguments to your application:
+
+ application = tornado.web.Application([
+ (r"/", MainHandler),
+ ], cookie_secret="61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=")
+
+Signed cookies contain the encoded value of the cookie in addition to a
+timestamp and an [HMAC](http://en.wikipedia.org/wiki/HMAC) signature. If the
+cookie is old or if the signature doesn't match, `get_secure_cookie` will
+return `None` just as if the cookie isn't set. The secure version of the
+example above:
+
+ class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ if not self.get_secure_cookie("mycookie"):
+ self.set_secure_cookie("mycookie", "myvalue")
+ self.write("Your cookie was not set yet!")
+ else:
+ self.write("Your cookie was set!")
+
+
+### User authentication
+
+The currently authenticated user is available in every request handler
+as `self.current_user`, and in every template as `current_user`. By
+default, `current_user` is `None`.
+
+To implement user authentication in your application, you need to
+override the `get_current_user()` method in your request handlers to
+determine the current user based on, e.g., the value of a cookie.
+Here is an example that lets users log into the application simply
+by specifying a nickname, which is then saved in a cookie:
+
+ class BaseHandler(tornado.web.RequestHandler):
+ def get_current_user(self):
+ return self.get_secure_cookie("user")
+
+ class MainHandler(BaseHandler):
+ def get(self):
+ if not self.current_user:
+ self.redirect("/login")
+ return
+ name = tornado.escape.xhtml_escape(self.current_user)
+ self.write("Hello, " + name)
+
+ class LoginHandler(BaseHandler):
+ def get(self):
+ self.write('<html><body><form action="/login" method="post">'
+ 'Name: <input type="text" name="name">'
+ '<input type="submit" value="Sign in">'
+ '</form></body></html>')
+
+ def post(self):
+ self.set_secure_cookie("user", self.get_argument("name"))
+ self.redirect("/")
+
+ application = tornado.web.Application([
+ (r"/", MainHandler),
+ (r"/login", LoginHandler),
+ ], cookie_secret="61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=")
+
+You can require that the user be logged in using the
+[Python decorator](http://www.python.org/dev/peps/pep-0318/)
+`tornado.web.authenticated`. If a request goes to a method with this
+decorator, and the user is not logged in, they will be redirected to
+`login_url` (another application setting). The example above could
+be rewritten:
+
+ class MainHandler(BaseHandler):
+ @tornado.web.authenticated
+ def get(self):
+ name = tornado.escape.xhtml_escape(self.current_user)
+ self.write("Hello, " + name)
+
+ settings = {
+ "cookie_secret": "61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
+ "login_url": "/login",
+ }
+ application = tornado.web.Application([
+ (r"/", MainHandler),
+ (r"/login", LoginHandler),
+ ], **settings)
+
+If you decorate `post()` methods with the `authenticated` decorator, and
+the user is not logged in, the server will send a `403` response.
+
+Tornado comes with built-in support for third-party authentication schemes
+like Google OAuth. See the [`auth` module](http://github.com/facebook/tornado/blob/master/tornado/auth.py) for more details. Check
+out the Tornado Blog example application for a complete example that
+uses authentication (and stores user data in a MySQL database).
+
+
+### Cross-site request forgery protection
+
+[Cross-site request forgery](http://en.wikipedia.org/wiki/Cross-site_request_forgery), or XSRF, is a common problem for personalized web applcations. See the
+[Wikipedia article](http://en.wikipedia.org/wiki/Cross-site_request_forgery)
+for more information on how XSRF works.
+
+The generally accepted solution to prevent XSRF is to cookie every user
+with an unpredictable value and include that value as an additional
+argument with every form submission on your site. If the cookie and the
+value in the form submission do not match, then the request is likely
+forged.
+
+Tornado comes with built-in XSRF protection. To include it in your site,
+include the application setting `xsrf_cookies`:
+
+ settings = {
+ "cookie_secret": "61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
+ "login_url": "/login",
+ "xsrf_cookies": True,
+ }
+ application = tornado.web.Application([
+ (r"/", MainHandler),
+ (r"/login", LoginHandler),
+ ], **settings)
+
+If `xsrf_cookies` is set, the Tornado web application will set the `_xsrf`
+cookie for all users and reject all `POST` requests hat do not contain a
+correct `_xsrf` value. If you turn this setting on, you need to instrument
+all forms that submit via `POST` to contain this field. You can do this with
+the special function `xsrf_form_html()`, available in all templates:
+
+ <form method="/login" method="post">
+ {{ xsrf_form_html() }}
+ <div>Username: <input type="text" name="username"/></div>
+ <div>Password: <input type="password" name="password"/></div>
+ <div><input type="submit" value="Sign in"/></div>
+ </form>
+
+If you submit AJAX `POST` requests, you will also need to instrument your
+JavaScript to include the `_xsrf` value with each request. This is the
+[jQuery](http://jquery.com/) function we use at FriendFeed for AJAX `POST`
+requests that automatically adds the `_xsrf` value to all requests:
+
+ function getCookie(name) {
+ var r = document.cookie.match("\\b" + name + "=([^;]*)\\b");
+ return r ? r[1] : undefined;
+ }
+
+ jQuery.postJSON = function(url, args, callback) {
+ args._xsrf = getCookie("_xsrf");
+ $.ajax({url: url, data: $.param(args), dataType: "text", type: "POST",
+ success: function(response) {
+ callback(eval("(" + response + ")"));
+ }});
+ };
+
+
+### Static files and aggressive file caching
+
+You can serve static files from Tornado by specifying the `static_path`
+setting in your application:
+
+ settings = {
+ "static_path": os.path.join(os.path.dirname(__file__), "static"),
+ "cookie_secret": "61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
+ "login_url": "/login",
+ "xsrf_cookies": True,
+ }
+ application = tornado.web.Application([
+ (r"/", MainHandler),
+ (r"/login", LoginHandler),
+ ], **settings)
+
+This setting will automatically make all requests that start with `/static/`
+serve from that static directory, e.g., [http://localhost:8888/static/foo.png](http://localhost:8888/static/foo.png)
+will serve the file `foo.png` from the specified static directory. We
+also automatically serve `/robots.txt` and `/favicon.ico` from the static
+directory (even though they don't start with the `/static/` prefix).
+
+To improve performance, it is generally a good idea for browsers to
+cache static resources aggressively so browsers won't send unnecessary
+`If-Modified-Since` or `Etag` requests that might block the rendering of
+the page. Tornado supports this out of the box with *static content
+versioning*.
+
+To use this feature, use the `static_url()` method in your templates rather
+than typing the URL of the static file directly in your HTML:
+
+ <html>
+ <head>
+ <title>FriendFeed - {{ _("Home") }}</title>
+ </head>
+ <body>
+ <div><img src="{{ static_url("images/logo.png") }}"/></div>
+ </body>
+ </html>
+
+The `static_url()` function will translate that relative path to a URI
+that looks like `/static/images/logo.png?v=aae54`. The `v` argument is
+a hash of the content in `logo.png`, and its presence makes the Tornado
+server send cache headers to the user's browser that will make the browser
+cache the content indefinitely.
+
+Since the `v` argument is based on the content of the file, if you update
+a file and restart your server, it will start sending a new `v` value,
+so the user's browser will automatically fetch the new file. If the file's
+contents don't change, the browser will continue to use a locally cached
+copy without ever checking for updates on the server, significantly
+improving rendering performance.
+
+In production, you probably want to serve static files from a more
+optimized static file server like [nginx](http://nginx.net/). You can
+configure most any web server to support these caching semantics. Here
+is the nginx configuration we use at FriendFeed:
+
+ location /static/ {
+ root /var/friendfeed/static;
+ if ($query_string) {
+ expires max;
+ }
+ }
+
+
+### Localization
+
+The locale of the current user (whether they are logged in or not) is
+always available as `self.locale` in the request handler and as `locale`
+in templates. The name of the locale (e.g., `en_US`) is available as
+`locale.name`, and you can translate strings with the `locale.translate`
+method. Templates also have the global function call `_()` available
+for string translation. The translate function has two forms:
+
+ _("Translate this string")
+
+which translates the string directly based on the current locale, and
+
+ _("A person liked this", "%(num)d people liked this", len(people)) % {"num": len(people)}
+
+which translates a string that can be singular or plural based on the value
+of the third argument. In the example above, a translation of the first
+string will be returned if `len(people)` is `1`, or a translation of the
+second string will be returned otherwise.
+
+The most common pattern for translations is to use Python named placeholders
+for variables (the `%(num)d` in the example above) since placeholders can
+move around on translation.
+
+Here is a properly localized template:
+
+ <html>
+ <head>
+ <title>FriendFeed - {{ _("Sign in") }}</title>
+ </head>
+ <body>
+ <form action="{{ request.path }}" method="post">
+ <div>{{ _("Username") }} <input type="text" name="username"/></div>
+ <div>{{ _("Password") }} <input type="password" name="password"/></div>
+ <div><input type="submit" value="{{ _("Sign in") }}"/></div>
+ {{ xsrf_form_html() }}
+ </form>
+ </body>
+ </html>
+
+By default, we detect the user's locale using the `Accept-Language` header
+sent by the user's browser. We choose `en_US` if we can't find an appropriate
+`Accept-Language` value. If you let user's set their locale as a preference,
+you can override this default locale selection by overriding `get_user_locale`
+in your request handler:
+
+ class BaseHandler(tornado.web.RequestHandler):
+ def get_current_user(self):
+ user_id = self.get_secure_cookie("user")
+ if not user_id: return None
+ return self.backend.get_user_by_id(user_id)
+
+ def get_user_locale(self):
+ if "locale" not in self.current_user.prefs:
+ # Use the Accept-Language header
+ return None
+ return self.current_user.prefs["locale"]
+
+If `get_user_locale` returns `None`, we fall back on the `Accept-Language`
+header.
+
+You can load all the translations for your application using the
+`tornado.locale.load_translations` method. It takes in the name of the
+directory which should contain CSV files named after the locales whose
+translations they contain, e.g., `es_GT.csv` or `fr_CA.csv`. The method
+loads all the translations from those CSV files and infers the list of
+supported locales based on the presence of each CSV file. You typically
+call this method once in the `main()` method of your server:
+
+ def main():
+ tornado.locale.load_translations(
+ os.path.join(os.path.dirname(__file__), "translations"))
+ start_server()
+
+You can get the list of supported locales in your application with
+`tornado.locale.get_supported_locales()`. The user's locale is chosen to
+be the closest match based on the supported locales. For example, if the
+user's locale is `es_GT`, and the `es` locale is supported, `self.locale`
+will be `es` for that request. We fall back on `en_US` if no close match
+can be found.
+
+See the [`locale` module](http://github.com/facebook/tornado/blob/master/tornado/locale.py) documentation for detailed information
+on the CSV format and other localization methods.
+
+
+### UI modules
+
+Tornado supports *UI modules* to make it easy to support standard, reusable
+UI widgets across your application. UI modules are like special functional
+calls to render components of your page, and they can come packaged with
+their own CSS and JavaScript.
+
+For example, if you are implementing a blog, and you want to have
+blog entries appear on both the blog home page and on each blog entry page,
+you can make an `Entry` module to render them on both pages. First, create
+a Python module for your UI modules, e.g., `uimodules.py`:
+
+ class Entry(tornado.web.UIModule):
+ def render(self, entry, show_comments=False):
+ return self.render_string(
+ "module-entry.html", show_comments=show_comments)
+
+Tell Tornado to use `uimodules.py` using the `ui_modules` setting in your
+application:
+
+ class HomeHandler(tornado.web.RequestHandler):
+ def get(self):
+ entries = self.db.query("SELECT * FROM entries ORDER BY date DESC")
+ self.render("home.html", entries=entries)
+
+ class EntryHandler(tornado.web.RequestHandler):
+ def get(self, entry_id):
+ entry = self.db.get("SELECT * FROM entries WHERE id = %s", entry_id)
+ if not entry: raise tornado.web.HTTPError(404)
+ self.render("entry.html", entry=entry)
+
+ settings = {
+ "ui_modules": uimodules,
+ }
+ application = tornado.web.Application([
+ (r"/", HomeHandler),
+ (r"/entry/([0-9]+)", EntryHandler),
+ ], **settings)
+
+Within `home.html`, you reference the `Entry` module rather than printing
+the HTML directly:
+
+ {% for entry in entries %}
+ {{ modules.Entry(entry) }}
+ {% end %}
+
+Within `entry.html`, you reference the `Entry` module with the
+`show_comments` argument to show the expanded form of the entry:
+
+ {{ modules.Entry(entry, show_comments=True) }}
+
+Modules can include custom CSS and JavaScript functions by overriding
+the `embedded_css`, `embedded_javascript`, `javascript_file`, or
+`css_file` methods:
+
+ class Entry(tornado.web.UIModule):
+ def embedded_css(self):
+ return ".entry { margin-bottom: 1em; }"
+
+ def render(self, entry, show_comments=False):
+ return self.render_string(
+ "module-entry.html", show_comments=show_comments)
+
+Module CSS and JavaScript will be included once no matter how many times
+a module is used on a page. CSS is always included in the `<head>` of the
+page, and JavaScript is always included just before the `</body>` tag
+at the end of the page.
+
+
+### Non-blocking, asynchronous requests
+
+When a request handler is executed, the request is automatically finished.
+Since Tornado uses a non-blocking I/O style, you can override this default
+behavior if you want a request to remain open after the main request handler
+method returns using the `tonado.web.asynchronous` decorator.
+
+When you use this decorator, it is your responsibility to call
+`self.finish()` to finish the HTTP request, or the user's browser
+will simply hang:
+
+ class MainHandler(tornado.web.RequestHandler):
+ @tonado.web.asynchronous
+ def get(self):
+ self.write("Hello, world")
+ self.finish()
+
+Here is a real example that makes a call to the FriendFeed API using
+Tornado's built-in asynchronous HTTP client:
+
+ class MainHandler(tornado.web.RequestHandler):
+ @tornado.web.asynchronous
+ def get(self):
+ http = tornado.httpclient.AsyncHTTPClient()
+ http.fetch("http://friendfeed-api.com/v2/feed/bret",
+ callback=self.async_callback(self.on_response))
+
+ def on_response(self, response):
+ if response.error: raise tornado.web.HTTPError(500)
+ json = tornado.escape.json_decode(response.body)
+ self.write("Fetched " + str(len(json["entries"])) + " entries "
+ "from the FriendFeed API")
+ self.finish()
+
+When `get()` returns, the request has not finished. When the HTTP client
+eventually calls `on_response()`, the request is still open, and the response
+is finally flushed to the client with the call to `self.finish()`.
+
+If you make calls to asynchronous library functions that require a callback
+(like the HTTP `fetch` function above), you should always wrap your
+callbacks with `self.async_callback`. This simple wrapper ensures that if
+your callback function raises an exception or has a programming error,
+a proper HTTP error response will be sent to the browser, and the connection
+will be properly closed.
+
+For a more advanced asynchronous example, take a look at the `chat` example
+application, which implements an AJAX chat room using
+[long polling](http://en.wikipedia.org/wiki/Push_technology#Long_polling).
+
+
+### Third party authentication
+
+Tornado's `auth` module implements the authentication and authorization
+protocols for a number of the most popular sites on the web, including
+Google/Gmail, Facebook, Twitter, Yahoo, and FriendFeed. The module includes
+methods to log users in via these sites and, where applicable, methods to
+authorize access to the service so you can, e.g., download a user's address
+book or publish a Twitter message on their behalf.
+
+Here is an example handler that uses Google for authentication, saving
+the Google credentials in a cookie for later access:
+
+ class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin):
+ @tornado.web.asynchronous
+ def get(self):
+ if self.get_argument("openid.mode", None):
+ self.get_authenticated_user(self.async_callback(self._on_auth))
+ return
+ self.authenticate_redirect()
+
+ def _on_auth(self, user):
+ if not user:
+ self.authenticate_redirect()
+ return
+ # Save the user with, e.g., set_secure_cookie()
+
+See the `auth` module documentation for more details.
+
+
+Performance
+-----------
+Web application performance is generally bound by architecture, not frontend
+performance. That said, Tornado is pretty fast relative to most popular
+Python web frameworks.
+
+We ran a few remedial load tests on a simple "Hello, world" application
+in each of the most popular Python web frameworks
+([Django](http://www.djangoproject.com/), [web.py](http://webpy.org/), and
+[CherryPy](http://www.cherrypy.org/)) to get the baseline performance of
+each relative to Tornado. We used Apache/mod_wsgi for Django and web.py
+and ran CherryPy as a standalone server, which was our impression of how
+each framework is typically run in production environments. We ran 4
+single-threaded Tornado frontends behind an [nginx](http://nginx.net/)
+reverse proxy, which is how we recommend running Tornado in production
+(our load test machine had four cores, and we recommend 1 frontend per
+core).
+
+We load tested each with Apache Benchmark (`ab`) on the a separate machine
+with the command
+
+ ab -n 100000 -c 25 http://10.0.1.x/
+
+The results (requests per second) on a 2.4GHz AMD Opteron processor with
+4 cores:
+
+<div style="text-align:center;margin-top:2em;margin-bottom:2em"><img src="http://chart.apis.google.com/chart?chxt=y&chd=t%3A100%2C40%2C27%2C25%2C9&chco=609bcc&chm=t+8213%2C000000%2C0%2C0%2C11%7Ct+3353%2C000000%2C0%2C1%2C11%7Ct+2223%2C000000%2C0%2C2%2C11%7Ct+2066%2C000000%2C0%2C3%2C11%7Ct+785%2C000000%2C0%2C4%2C11&chs=600x175&cht=bhs&chtt=Web+server+requests%2Fsec+%28AMD+Opteron%2C+2.4GHz%2C+4+cores%29&chxl=0%3A%7CCherryPy+%28standalone%29%7Cweb.py+%28Apache%2Fmod_wsgi%29%7CDjango+%28Apache%2Fmod_wsgi%29%7CTornado+%281+single-threaded+frontend%29%7CTornado+%28nginx%3B+4+frontends%29%7C"/></div>
+
+In our tests, Tornado consistently had 4X the throughput of the next fastest
+framework, and even a single standalone Tornado frontend got 33% more
+throughput even though it only used one of the four cores.
+
+Not very scientific, but at a high level, it should give you a sense that we
+have cared about performance as we built Tornado, and it shouldn't add too
+much latency to your apps relative to most Python web development frameworks.
+
+
+Running Tornado in production
+-----------------------------
+At FriendFeed, we use [nginx](http://nginx.net/) as a load balancer
+and static file server. We run multiple instances of the Tornado web
+server on multiple frontend machines. We typically run one Tornado frontend
+per core on the machine (sometimes more depending on utilization).
+
+This is a barebones nginx config file that is structurally similar to the
+one we use at FriendFeed. It assumes nginx and the Tornado servers
+are running on the same machine, and the four Tornado servers
+are running on ports 8000 - 8003:
+
+ user nginx;
+ worker_processes 1;
+
+ error_log /var/log/nginx/error.log;
+ pid /var/run/nginx.pid;
+
+ events {
+ worker_connections 1024;
+ use epoll;
+ }
+
+ http {
+ # Enumerate all the Tornado servers here
+ upstream frontends {
+ server 127.0.0.1:8000;
+ server 127.0.0.1:8001;
+ server 127.0.0.1:8002;
+ server 127.0.0.1:8003;
+ }
+
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+
+ access_log /var/log/nginx/access.log;
+
+ keepalive_timeout 65;
+ proxy_read_timeout 200;
+ sendfile on;
+ tcp_nopush on;
+ tcp_nodelay on;
+ gzip on;
+ gzip_min_length 1000;
+ gzip_proxied any;
+ gzip_types text/plain text/html text/css text/xml
+ application/x-javascript application/xml
+ application/atom+xml text/javascript;
+
+ # Only retry if there was a communication error, not a timeout
+ # on the Tornado server (to avoid propagating "queries of death"
+ # to all frontends)
+ proxy_next_upstream error;
+
+ server {
+ listen 80;
+
+ # Allow file uploads
+ client_max_body_size 50M;
+
+ location ^~ /static/ {
+ root /var/www;
+ if ($query_string) {
+ expires max;
+ }
+ }
+ location = /favicon.ico {
+ rewrite (.*) /static/favicon.ico;
+ }
+ location = /robots.txt {
+ rewrite (.*) /static/robots.txt;
+ }
+
+ location / {
+ proxy_pass_header Server;
+ proxy_set_header Host $http_host;
+ proxy_redirect false;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Scheme $scheme;
+ proxy_pass http://frontends;
+ }
+ }
+ }
+
+
+WSGI and Google AppEngine
+-------------------------
+Tornado comes with limited support for [WSGI](http://wsgi.org/). However,
+since WSGI does not support non-blocking requests, you cannot use any
+of the asynchronous/non-blocking features of Tornado in your application
+if you choose to use WSGI instead of Tornado's HTTP server. Some of the
+features that are not available in WSGI applications:
+`@tornado.web.asynchronous`, the `httpclient` module, and the `auth` module.
+
+You can create a valid WSGI application from your Tornado request handlers
+by using `WSGIApplication` in the `wsgi` module instead of using
+`tornado.web.Application`. Here is an example that uses the built-in WSGI
+`CGIHandler` to make a valid
+[Google AppEngine](http://code.google.com/appengine/) application:
+
+ import tornado.web
+ import tornado.wsgi
+ import wsgiref.handlers
+
+ class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ self.write("Hello, world")
+
+ if __name__ == "__main__":
+ application = tornado.wsgi.WSGIApplication([
+ (r"/", MainHandler),
+ ])
+ wsgiref.handlers.CGIHandler().run(application)
+
+See the `appengine` example application for a full-featured AppEngine
+app built on Tornado.
+
+
+Caveats and support
+-------------------
+Tornado was refactored from the [FriendFeed](http://friendfeed.com/)
+code base to reduce dependencies. This refactoring may have introduced bugs.
+Likewise, because the FriendFeed servers have always run
+[behind nginx](#running-tornado-in-production), Tornado has not been
+extensively tested HTTP/1.1 clients beyond Firefox. Tornado currently
+does not attempt to handle multi-line headers and some types of malformed
+input.
+
+You can discuss Tornado and report bugs on [the Tornado developer mailing list](http://groups.google.com/group/python-tornado).
--- /dev/null
+{% extends "base.html" %}
+
+{% block body %}
+ <p><a href="http://www.tornadoserver.org/">Tornado</a> is an open source version of the scalable, non-blocking web server and and tools that power <a href="http://friendfeed.com/">FriendFeed</a>. The FriendFeed application is written using a web framework that looks a bit like <a href="http://webpy.org/">web.py</a> or <a href="http://code.google.com/appengine/docs/python/tools/webapp/">Google's webapp</a>, but with additional tools and optimizations to take advantage of the underlying non-blocking infrastructure.</p>
+ <p>The framework is distinct from most mainstream web server frameworks (and certainly most Python frameworks) because it is non-blocking and reasonably fast. Because it is non-blocking and uses <a href="http://www.kernel.org/doc/man-pages/online/pages/man4/epoll.4.html">epoll</a>, it can handle thousands of simultaneous standing connections, which means it is ideal for real-time web services. We built the web server specifically to handle FriendFeed's real-time features — every active user of FriendFeed maintains an open connection to the FriendFeed servers. (For more information on scaling servers to support thousands of clients, see The <a href="http://www.kegel.com/c10k.html">C10K problem</a>.)</p>
+ <p>See the <a href="/documentation">Tornado documentation</a> for a detailed walkthrough of the framework.</p>
+
+ <h2>Download and install</h2>
+ <p><b>Download:</b> <a href="/static/tornado-0.1.tar.gz">tornado-0.1.tar.gz</a></p>
+ <pre><code>tar xvzf python-tornado-0.1.tar.gz
+cd python-tornado-0.1
+python setup.py build
+sudo python setup.py install</code></pre>
+ <p>The Tornado source code is <a href="http://github.com/facebook/tornado">hosted on GitHub</a>.</p>
+
+ <h3>Prerequisites</h3>
+ <p>Tornado has been tested on Python 2.5 and 2.6. To use all of the features of Tornado, you need to have <a href="http://pycurl.sourceforge.net/">PycURL</a> and a JSON library like <a href="http://pypi.python.org/pypi/simplejson/">simplejson</a> installed. Complete installation instructions for Mac OS X and Ubuntu are included below for convenience.</p>
+ <p style="font-weight:bold">Mac OS X 10.5/10.6</p>
+ <pre><code>sudo easy_install setuptools pycurl==7.16.2.1 simplejson</code></pre>
+
+ <p style="font-weight:bold">Ubuntu Linux</p>
+ <pre><code>sudo apt-get install python-pycurl python-simplejson</code></pre>
+
+ <h2>Hello, world</h2>
+ <p>Here is the canonical "Hello, world" example app for Tornado:</p>
+ <pre><code>import tornado.httpserver
+import tornado.ioloop
+import tornado.web
+
+class MainHandler(tornado.web.RequestHandler):
+ def get(self):
+ self.write("Hello, world")
+
+application = tornado.web.Application([
+ (r"/", MainHandler),
+])
+
+if __name__ == "__main__":
+ http_server = tornado.httpserver.HTTPServer(application)
+ http_server.listen(8888)
+ tornado.ioloop.IOLoop.instance().start()</code></pre>
+ <p>See the <a href="/documentation">Tornado documentation</a> for a detailed walkthrough of the framework.</p>
+
+ <h2>Discussion and support</h2>
+ <p>You can discuss Tornado and report bugs on <a href="http://groups.google.com/group/python-tornado">the Tornado developer mailing list</a>.
+
+{% end %}
+
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright 2009 Bret Taylor
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import markdown
+import os
+import os.path
+import time
+import tornado.web
+import tornado.wsgi
+import wsgiref.handlers
+
+
+class ContentHandler(tornado.web.RequestHandler):
+ def get(self, path):
+ paths = ("documentation", "index")
+ if not path: path = "index"
+ if path not in paths:
+ raise tornado.web.HTTPError(404)
+ self.render(path + ".html", markdown=self.markdown)
+
+ def markdown(self, path, toc=False):
+ if not hasattr(ContentHandler, "_md") or self.settings.get("debug"):
+ ContentHandler._md = {}
+ if path not in ContentHandler._md:
+ full_path = os.path.join(self.settings["template_path"], path)
+ f = open(full_path, "r")
+ contents = f.read().decode("utf-8")
+ f.close()
+ if toc: contents = u"[TOC]\n\n" + contents
+ md = markdown.Markdown(extensions=["toc"] if toc else [])
+ ContentHandler._md[path] = md.convert(contents).encode("utf-8")
+ return ContentHandler._md[path]
+
+
+settings = {
+ "template_path": os.path.join(os.path.dirname(__file__), "templates"),
+ "xsrf_cookies": True,
+ "debug": os.environ.get("SERVER_SOFTWARE", "").startswith("Development/"),
+}
+application = tornado.wsgi.WSGIApplication([
+ (r"/([a-z]*)", ContentHandler),
+], **settings)
+
+
+def main():
+ wsgiref.handlers.CGIHandler().run(application)
+
+
+if __name__ == "__main__":
+ main()
projects / thirdparty / tornado.git / commitdiff
