From 8657f3bb9b6d9e6a005994ebdc191ec49e4fb65c Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 21 Oct 2006 20:57:10 +0000 Subject: [PATCH] merge of new documentation generation system --- CHANGES | 2 + README | 8 - doc/build/README | 10 + doc/build/compile_docstrings.py | 41 ---- doc/build/components/autohandler | 5 + doc/build/components/base.myt | 38 +++ doc/build/components/content_layout.myt | 15 ++ doc/build/components/doclib.myt | 187 -------------- doc/build/components/formatting.myt | 261 ++------------------ doc/build/components/index.myt | 2 - doc/build/components/nav.myt | 106 ++++++++ doc/build/components/printsection.myt | 155 ------------ doc/build/components/pydoc.myt | 60 ++--- doc/build/components/section_wrapper.myt | 42 ---- doc/build/components/toc.myt | 95 ++++++++ doc/build/content/docstrings.myt | 16 +- doc/build/content/document_base.myt | 42 ---- doc/build/content/index.myt | 7 + doc/build/content/metadata.txt | 10 +- doc/build/content/tutorial.txt | 2 +- doc/build/gen_docstrings.py | 74 ++++++ doc/build/genhtml.py | 61 ++++- doc/build/lib/docstring.py | 7 +- doc/build/lib/documentgen.py | 44 ---- doc/build/lib/toc.py | 70 ++++++ doc/build/read_markdown.py | 230 ++++++++++++++++++ doc/build/runhtml.py | 17 +- doc/build/txt2myt.py | 296 ----------------------- 28 files changed, 790 insertions(+), 1113 deletions(-) create mode 100644 doc/build/README delete mode 100644 doc/build/compile_docstrings.py create mode 100644 doc/build/components/base.myt create mode 100644 doc/build/components/content_layout.myt delete mode 100644 doc/build/components/doclib.myt delete mode 100644 doc/build/components/index.myt create mode 100644 doc/build/components/nav.myt delete mode 100644 doc/build/components/printsection.myt delete mode 100644 doc/build/components/section_wrapper.myt create mode 100644 doc/build/components/toc.myt delete mode 100644 doc/build/content/document_base.myt create mode 100644 doc/build/content/index.myt create mode 100644 doc/build/gen_docstrings.py delete mode 100644 doc/build/lib/documentgen.py create mode 100644 doc/build/lib/toc.py create mode 100644 doc/build/read_markdown.py delete mode 100644 doc/build/txt2myt.py diff --git a/CHANGES b/CHANGES index 95bd4bc65d..3cd19c9d18 100644 --- a/CHANGES +++ b/CHANGES @@ -10,6 +10,8 @@ Test suite includes "--log-info" and "--log-debug" arguments which work independently of --verbose/--quiet. Logging added to orm to allow tracking of mapper configurations, row iteration. + - the documentation-generation system has been overhauled to be + much simpler in design and more integrated with Markdown - Specific Databases: - SQLite: - sqlite boolean datatype converts False/True to 0/1 by default diff --git a/README b/README index 8c398b7c44..f4726e164f 100644 --- a/README +++ b/README @@ -11,14 +11,6 @@ an svn-tagged build. Documentation is available in HTML format in the ./doc/ directory. -The "raw" format of the documentation is Markdown with a few extra syntaxes -added in; those files are present in ./doc/build/content/. - -To fully generate the documentation into both Myghty and HTML format: - - cd ./doc/build/ - python genhtml.py - Information running unit tests is in README.unittests. good luck ! diff --git a/doc/build/README b/doc/build/README new file mode 100644 index 0000000000..ce7570f5ca --- /dev/null +++ b/doc/build/README @@ -0,0 +1,10 @@ +Documentation exists in its original format as Markdown files in the ./content directory. + +To generate documentation: + + python genhtml.py + +This generates the Markdown files into Myghty templates as an interim step and then into HTML. It also +creates two pickled datafiles corresponding to the table of contents and all the generated docstrings +for the SQLAlchemy sourcecode. + diff --git a/doc/build/compile_docstrings.py b/doc/build/compile_docstrings.py deleted file mode 100644 index 4ddc8cb719..0000000000 --- a/doc/build/compile_docstrings.py +++ /dev/null @@ -1,41 +0,0 @@ -import cPickle as pickle -import sys, os - -sys.path = ['../../lib', './lib/'] + sys.path - -import docstring - -import sqlalchemy.schema as schema -import sqlalchemy.engine as engine -import sqlalchemy.engine.strategies as strategies -import sqlalchemy.sql as sql -import sqlalchemy.pool as pool -import sqlalchemy.orm as orm -import sqlalchemy.exceptions as exceptions -import sqlalchemy.ext.proxy as proxy -import sqlalchemy.ext.sessioncontext as sessioncontext -import sqlalchemy.mods.threadlocal as threadlocal -import sqlalchemy.ext.selectresults as selectresults - -objects = [] -def make_doc(obj, classes=None, functions=None): - objects.append(docstring.ObjectDoc(obj, classes=classes, functions=functions)) - -make_doc(obj=sql) -make_doc(obj=schema) -make_doc(obj=engine) -make_doc(obj=engine.url) -make_doc(obj=orm, classes=[orm.MapperExtension]) -make_doc(obj=orm.mapperlib, classes=[orm.mapperlib.Mapper]) -make_doc(obj=orm.query, classes=[orm.query.Query, orm.query.OperationContext, orm.query.QueryContext, orm.query.SelectionContext]) -make_doc(obj=orm.session, classes=[orm.session.Session, orm.session.SessionTransaction]) -make_doc(obj=pool, classes=[pool.DBProxy, pool.Pool, pool.QueuePool, pool.SingletonThreadPool]) -make_doc(obj=sessioncontext) -make_doc(obj=threadlocal) -make_doc(obj=selectresults) -make_doc(obj=exceptions) -make_doc(obj=proxy) - - -output = os.path.join(os.getcwd(), 'content', "compiled_docstrings.pickle") -pickle.dump(objects, file(output, 'w')) diff --git a/doc/build/components/autohandler b/doc/build/components/autohandler index 38f964fcb4..223b7ff615 100644 --- a/doc/build/components/autohandler +++ b/doc/build/components/autohandler @@ -2,6 +2,11 @@ <& REQUEST:title &> + + + + + diff --git a/doc/build/components/base.myt b/doc/build/components/base.myt new file mode 100644 index 0000000000..7d6e7bae60 --- /dev/null +++ b/doc/build/components/base.myt @@ -0,0 +1,38 @@ +<%doc>base.myt - common to all documentation pages. intentionally separate from autohandler, which can be swapped +out for a different one +<%args> + extension="myt" + +<%python scope="init"> + if m.cache_self(key=m.request_component.file): + return + # bootstrap TOC structure from request args, or pickled file if not present. + import cPickle as pickle + import os, time + m.log("base.myt generating from table of contents for file %s" % m.request_component.file) + toc = m.request_args.get('toc') + if toc is None: + filename = os.path.join(os.path.dirname(m.request_component.file), 'table_of_contents.pickle') + toc = pickle.load(file(filename)) + version = toc.version + last_updated = toc.last_updated + +<%method title> + <% m.request_component.attributes.get('title') %> + + +
 
+ +
+ +
+ +

<% toc.root.doctitle %>

+
Version: <% version %> Last Updated: <% time.strftime('%x %X', time.localtime(last_updated)) %>
+
+ +% m.call_next(toc=toc, extension=extension) + +
+ + diff --git a/doc/build/components/content_layout.myt b/doc/build/components/content_layout.myt new file mode 100644 index 0000000000..c44a6f38de --- /dev/null +++ b/doc/build/components/content_layout.myt @@ -0,0 +1,15 @@ +<%doc>defines the default layout for normal documentation pages (not including the index) +<%args> + extension="myt" + toc + +<%flags>inherit="base.myt" +<%init> + current = toc.get_by_file(m.request_component.attributes['filename']) + + + +<& nav.myt:topnav, item=current, extension=extension &> +
+% m.call_next(toc=toc, extension=extension) +
diff --git a/doc/build/components/doclib.myt b/doc/build/components/doclib.myt deleted file mode 100644 index 12efa1dabb..0000000000 --- a/doc/build/components/doclib.myt +++ /dev/null @@ -1,187 +0,0 @@ - -<%python scope="global"> -import sys, string, re - -# datastructure that will store the whole contents of the documentation -class TOCElement: - def __init__(self, filename, name, description, parent = None, ext = None, header = None, last_updated = 0, htmldescription=None, altlink=None): - self.filename = filename - self.name = name - self.parent = parent - self.path = self._create_path() - self.header = header - self.altlink = altlink - if self.parent is not None: - self.root = parent.root - self.root.pathlookup[self.path] = self - - if self.parent.filename != self.filename: - self.root.filelookup[self.filename] = self - self.isTop = True - else: - self.root = self - self.pathlookup = {} - self.pathlookup[''] = self - self.filelookup = {} - self.filelookup[filename] = self - - if ext is not None: - self.ext = ext - else: - self.ext = self.root.ext - - self.last_updated = last_updated - self.description = description - self.htmldescription = htmldescription - self.content = None - self.previous = None - self.next = None - self.children = [] - if parent: - if len(parent.children): - self.previous = parent.children[-1] - parent.children[-1].next = self - parent.children.append(self) - if last_updated > parent.last_updated: - parent.last_updated = self.last_updated - - def get_file(self, name): - name = re.sub("\.\w+$", "", name) - return self.root.filelookup[name] - - def lookup(self, path): - return self.root.pathlookup[path] - - def get_link(self, includefile = True, anchor = True): - if includefile: - if anchor: - return "%s%s#%s" % (self.filename, self.ext, self.path) - else: - return "%s%s" % (self.filename, self.ext) - else: - if anchor: - return "#" + self.path - else: - return "" - - - def _create_path(self): - elem = self - tokens = [] - while elem.parent is not None: - tokens.insert(0, elem.name) - elem = elem.parent - path = string.join(tokens, '_') - return path - - - - - -<%python scope="request"> - current = Value() - filename = Value() - - - -<%args scope="request"> - paged = 'yes' - - -<%python scope="init"> - - try: - a = r - isdynamic = True - ext = ".myt" - except: - isdynamic = False - ext = ".html" - - request_comp = m.request_comp() - - if isdynamic and not m.interpreter.attributes.get('docs_static_cache', False): - page_cache = True - else: - page_cache = False - - # for dynamic page, cache the output of the final page - - if page_cache: - if m.cache_self(key="doc_%s" % paged, component = request_comp): - return - - list_comp = m.fetch_next() - files = request_comp.attributes['files'] - title = request_comp.attributes.setdefault('title', "Documentation") - version = request_comp.attributes['version'] - wrapper = request_comp.attributes['wrapper'] - index = request_comp.attributes['index'] - onepage = request_comp.attributes['onepage'] - - - - def buildtoc(): - root = TOCElement("", "root", "root element", ext = ext) - current.assign(root) - - for file in files: - filename.assign(file) - comp = m.fetch_component(file + ".myt") - - main = m.scomp(comp) - - return root - - if not page_cache: - # non-dynamic (i.e. command-line) page, cache the datastructure so successive - # pages are fast (disables auto-recompiling) - cache = m.get_cache(list_comp) - - toc = cache.get_value('toc', createfunc = buildtoc) - - else: - toc = buildtoc() - - last_updated = toc.last_updated - m.comp(wrapper, isdynamic=isdynamic, ext = ext, toc = toc, comp = request_comp, onepage = onepage, paged = paged, title = title, version = version, index=index, last_updated = last_updated) - - - -<%method title> -<% m.request_comp().get_attribute('title', inherit = True) or "Documentation" %> - - -<%method item> - <%doc>stores an item in the table of contents - <%args> - # name should be a URL friendly name used for hyperlinking the section - name - - # description is the heading for the item - description - - htmldescription = None - escapedesc = False - - header = None - altlink=None - - <%python scope="init"> - if escapedesc: - description = m.apply_escapes(description, ['h']) - - current(TOCElement(filename(), name, description, current(), header = header, last_updated = m.caller.component_source.last_modified, htmldescription=htmldescription, altlink=altlink)) - current().content = m.content() - current(current().parent) - - - -<%method current> -<%init>return current() - - - - - - diff --git a/doc/build/components/formatting.myt b/doc/build/components/formatting.myt index ee9fb3958d..da1c81df05 100644 --- a/doc/build/components/formatting.myt +++ b/doc/build/components/formatting.myt @@ -1,196 +1,53 @@ -<%doc>formatting.myt - library of HTML formatting functions to operate on a TOCElement tree +<%doc>formatting.myt - Provides section formatting elements, syntax-highlighted code blocks, and other special filters. <%global> import string, re import highlight - -<%method printtoc> -<%args> - root - includefile - current = None - full = False - children = True - - -% header = False - -% -% header = True -

<% i.header %>

- - - -<%def printtocelement> -<%doc>prints a TOCElement as a table of contents item and prints its immediate child items - <%args> - item - includefile - bold = False - full = False - children = True - - -
  • " href="<% item.get_link(includefile, anchor = (not includefile)) %>"><% item.description %>
    • - -% if children: - -% - - -<%def printsmtocelem> - <%args> - item - includefile - children = False - -
  • <% item.description %>
    • - -% if children: - -% - - - - - -<%method printitem> -<%doc>prints the description and contents of a TOC element and recursively prints its child items - +<%method section> +<%doc>Main section formatting element. <%args> - item - indentlevel = 0 - includefile - omitheader = False - root = None + toc + path + description=None +<%init> + item = toc.get_by_path(path) + if item is None: + raise "path: " + path + -% if root is None: root = item - -% if not omitheader: -% if item.altlink: - -% -% - -
    +
    <%python> - regexp = re.compile(r"__FORMAT:LINK{(?:\@path=(.+?))?(?:\@xtra=(.+?))?(?:\@text=(.+?))?(?:\@href=(.+?))?(?:\@class=(.+?))?}") - def link(matchobj): - path = matchobj.group(1) - xtra = matchobj.group(2) - text = matchobj.group(3) - href = matchobj.group(4) - class_ = matchobj.group(5) - - if class_ is not None: - class_ = 'class="%s"' % class_ - else: - class_ = '' - - if href: - return '%s' % (href, class_, text or href) - else: - try: - element = item.lookup(path) - if xtra is not None: - return '%s' % (element.get_link(includefile), xtra, class_, text or xtra) - else: - return '%s' % (element.get_link(includefile), class_, text or element.description) - except KeyError: - if xtra is not None: - return '%s' % (text or xtra) - else: - return '%s' % text or path - + content = m.content() re2 = re.compile(r"'''PYESC(.+?)PYESC'''", re.S) - content = regexp.sub(link, item.content) content = re2.sub(lambda m: m.group(1), content) - description = regexp.sub(link, item.htmldescription or item.description) -% if not omitheader: -

    <% description %>

    +% if item.depth > 1: +

    <% description or item.description %>

    % -
    - -<%python> - #m.write(item.content) - m.write(content) - +
    + <% content %>
    -% for i in item.children: - <& printitem, item=i, indentlevel=indentlevel + 1, includefile = includefile, root=root &> -% - -% if root is not None and len(item.children) == 0: - back to section top +% if item.depth > 1: +% if (item.next and item.next.depth >= item.depth): + back to section top % - -% if indentlevel == 0: -% if includefile: - <& SELF:pagenav, item=item, includefile=includefile &> -% else: -
    -% # +% else: + back to section top + <& nav.myt:pagenav, item=item &> % -
    -<%method pagenav> -<%args> - item - includefile - -
    -
    - -% if not includefile: - Top | -% - -% if item.previous is not None: - Previous: <& SELF:itemlink, item=item.previous, includefile = includefile &> -% # end if - -% if item.next is not None: -% if item.previous is not None: - | -% # end if - - Next: <& SELF:itemlink, item=item.next, includefile = includefile &> -% # end if - -
    -
    - <%method formatplain> <%filter> @@ -202,58 +59,8 @@ <% m.content() | h%> -<%method itemlink trim="both"> - <%args> - item - includefile - - <% item.description %> - - -<%method paramtable> - - <% m.content() %> -
    - - -<%method member_doc> - <%args> - name = "" - link = "" - type = None - - - -
    - - <% name %> -
    <% m.content() %>
    -
    - - - - - -<%method function_doc> - <%args> - name = "" - link = "" - alt = None - arglist = [] - rettype = None - - - -
    - - <% name %>(<% string.join(map(lambda k: "%s" % k, arglist), ", ")%>) -
    <% m.content() %>
    -
    - - - <%method codeline trim="both"> <% m.content() %> @@ -300,24 +107,8 @@ <% content %>
    -<%method link trim="both"> - <%args> - path = None - param = None - method = None - member = None - text = None - href = None - class_ = None - - <%init> - if href is None and path is None: - path = m.comp('doclib.myt:current').path - - extra = (param or method or member) - -__FORMAT:LINK{<% path and "@path=" + path or "" %><% extra and "@xtra=" + extra or "" %><% text and "@text=" + text or "" %><% href and "@href=" + href or "" %><% class_ and "@class=" + class_ or "" %>} - + + <%method popboxlink trim="both"> <%args> @@ -354,7 +145,7 @@ javascript:togglePopbox('<% name %>', '<% show %>', '<% hide %>') <%init> href = m.scomp('SELF:popboxlink') - '''PYESC<& SELF:link, href=href, text=link, class_="codepoplink" &>PYESC''' + '''PYESC<& nav.myt:link, href=href, text=link, class_="codepoplink" &>PYESC''' <%method codepopper trim="both"> diff --git a/doc/build/components/index.myt b/doc/build/components/index.myt deleted file mode 100644 index c55dfc2def..0000000000 --- a/doc/build/components/index.myt +++ /dev/null @@ -1,2 +0,0 @@ -<%flags>inherit="document_base.myt" - diff --git a/doc/build/components/nav.myt b/doc/build/components/nav.myt new file mode 100644 index 0000000000..a045df8ce7 --- /dev/null +++ b/doc/build/components/nav.myt @@ -0,0 +1,106 @@ +<%doc>nav.myt - Provides page navigation elements that are derived from toc.TOCElement structures, including +individual hyperlinks as well as navigational toolbars and table-of-content listings. + +<%method itemlink trim="both"> + <%args> + item + anchor=True + + <%args scope="request"> + extension='myt' + + <% item.description %> + + +<%method toclink trim="both"> + <%args> + toc + path + description=None + extension + + <%init> + item = toc.get_by_path(path) + if description is None: + if item: + description = item.description + else: + description = path + +% if item: + <% description %> +% else: + <% description %> +% + + + +<%method link trim="both"> + <%args> + href + text + class_ + + ><% text %> + + +<%method topnav> + <%args> + item + extension + +
    + +
    + +Table of Contents + +
    +% if item.previous is not None: +Previous: <& itemlink, item=item.previous, anchor=False &> +% + +% if item.previous is not None and item.next is not None: +  |   +% + +% if item.next is not None: + +Next: <& itemlink, item=item.next, anchor=False &> +% + +
    +
    + +
    +
    <% item.description %>
    +
    + <& toc.myt:printtoc, root=item, current=None, full=True, extension=extension, anchor_toplevel=True &> +
    +
    + +
    + + +<%method pagenav> +<%args> + item + +
    +
    + +% if item.previous is not None: + Previous: <& itemlink, item=item.previous &> +% # end if + +% if item.next is not None: +% if item.previous is not None: + | +% # end if + + Next: <& itemlink, item=item.next &> +% # end if + +
    +
    + diff --git a/doc/build/components/printsection.myt b/doc/build/components/printsection.myt deleted file mode 100644 index cd5a26ff36..0000000000 --- a/doc/build/components/printsection.myt +++ /dev/null @@ -1,155 +0,0 @@ -<%args> - toc - paged - comp - isdynamic - index - ext - onepage - - -<%python scope="init"> - # get the item being requested by this embedded component from the documentation tree - try: - current = toc.get_file(comp.get_name()) - except KeyError: - current = None - - -% if paged == 'yes': -% if current is None: - <& toc, includefile = True, **ARGS &> -% else: - - <& topnav, item=current, **ARGS &> -
    - <& formatting.myt:printitem, item=current, includefile = True, omitheader = True &> -
    -% # if/else -% else: - <& toc, includefile = False, **ARGS &> -
    -% for i in toc.children: -
    - - -
    - <& topnav, item=i, **ARGS &> -
    - - <& formatting.myt:printitem, item=i, includefile = False, omitheader = True &> -
    -% # for i -
    - -% # if/else - - -<%method topnav> - <%args> - isdynamic - paged - item - index - ext - onepage - -% ispaged = (paged =='yes') - -
    - -<& topnavcontrol, **ARGS &> - -
    - -Table of Contents - -
    -% if item.previous is not None: -Previous: <& formatting.myt:itemlink, item=item.previous, includefile=ispaged &> -% - -% if item.previous is not None and item.next is not None: -  |   -% - -% if item.next is not None: - -Next: <& formatting.myt:itemlink, item=item.next, includefile=ispaged &> -% - -
    -
    - - -
    -
    <% item.description %>
    -
    - <& formatting.myt:printtoc, root = item, includefile = False, current = None, full = True &> -
    -
    - -
    - - -<%method topnavcontrol> - <%args> - isdynamic - paged - index - ext - onepage - -% ispaged = (paged =='yes') - -
    -% if ispaged: - View: Paged  |  One Page -% else: - View: Paged  |  One Page -% -
    - - - -<%method toc> - <%args> - toc - includefile = True - isdynamic - paged - index - ext - onepage - - - -
    - <& topnavcontrol, **ARGS &> - - -

    Table of Contents

    -    - (view full table) -

    - -
    - <& formatting.myt:printtoc, root = toc, includefile = includefile, current = None, full = False, children=False &> -
    - -
    - - -
    - -

    Table of Contents: Full

    -    - (view brief table) -

    - -
    - <& formatting.myt:printtoc, root = toc, includefile = includefile, current = None, full = True, children=True &> -
    - -
    - diff --git a/doc/build/components/pydoc.myt b/doc/build/components/pydoc.myt index bf7a6952a0..b787ae74f8 100644 --- a/doc/build/components/pydoc.myt +++ b/doc/build/components/pydoc.myt @@ -1,44 +1,42 @@ +<%doc>pydoc.myt - provides formatting functions for printing docstring.AbstractDoc generated python documentation objects. + <%global> - import docstring, string, sys +import docstring <%method obj_doc> <%args> obj + toc + extension - -<%python> +<%init> if obj.isclass: - s = [] links = [] for elem in obj.inherits: if isinstance(elem, docstring.ObjectDoc): - links.append("%s" % (str(elem.id), elem.name)) - s.append(elem.name) + links.append(m.scomp("nav.myt:toclink", toc=toc, path=elem.toc_path, extension=extension, description=elem.name)) else: links.append(str(elem)) - s.append(str(elem)) - description = "class " + obj.classname + "(%s)" % (','.join(s)) htmldescription = "class " + obj.classname + "(%s)" % (','.join(links)) else: - description = obj.description htmldescription = obj.description - - -<&|doclib.myt:item, name=obj.name, description=description, htmldescription=htmldescription, altlink=str(obj.id) &> + + + +<&|formatting.myt:section, toc=toc, path=obj.toc_path, description=htmldescription &> + <&|formatting.myt:formatplain&><% obj.doc %> % if not obj.isclass and obj.functions: -<&|doclib.myt:item, name="modfunc", description="Module Functions" &> -<&|formatting.myt:paramtable&> + % for func in obj.functions: <& SELF:function_doc, func=func &> % - - + % else: + % if obj.functions: -<&|formatting.myt:paramtable&> % for func in obj.functions: % if isinstance(func, docstring.FunctionDoc): <& SELF:function_doc, func=func &> @@ -46,26 +44,26 @@ else: <& SELF:property_doc, prop=func &> % % - % % % if obj.classes: -<&|formatting.myt:paramtable&> % for class_ in obj.classes: - <& SELF:obj_doc, obj=class_ &> + <& SELF:obj_doc, obj=class_, toc=toc, extension=extension &> % - % - <%method function_doc> <%args>func - <&|formatting.myt:function_doc, name=func.name, link=func.link, arglist=func.arglist &> - <&|formatting.myt:formatplain&><% func.doc %> - +
    + + <% func.name %>(<% ", ".join(map(lambda k: "%s" % k, func.arglist))%>) +
    + <&|formatting.myt:formatplain&><% func.doc %> +
    +
    @@ -73,7 +71,13 @@ else: <%args> prop - <&|formatting.myt:member_doc, name=prop.name, link=prop.link &> - <&|formatting.myt:formatplain&><% prop.doc %> - +
    + + <% prop.name %> +
    + <&|formatting.myt:formatplain&><% prop.doc %> +
    +
    + + diff --git a/doc/build/components/section_wrapper.myt b/doc/build/components/section_wrapper.myt deleted file mode 100644 index b4e7c524e8..0000000000 --- a/doc/build/components/section_wrapper.myt +++ /dev/null @@ -1,42 +0,0 @@ -<%python scope="global"> - import string, re, time - - - -<%args> - comp - toc - title - version = None - last_updated = None - index - paged - onepage - isdynamic - ext - - - - - - - - - -
     
    - -
    - -
    - -

    <% title %>

    -% if version is not None: -
    Version: <% version %> Last Updated: <% time.strftime('%x %X', time.localtime(last_updated)) %>
    -% -
    - -<& printsection.myt, paged = paged, toc = toc, comp = comp, isdynamic=isdynamic, index=index, ext=ext,onepage=onepage &> - -
    - - diff --git a/doc/build/components/toc.myt b/doc/build/components/toc.myt new file mode 100644 index 0000000000..6580a39c50 --- /dev/null +++ b/doc/build/components/toc.myt @@ -0,0 +1,95 @@ +<%doc>toc.myt - prints full table of contents listings given toc.TOCElement strucures + +<%method toc> + <%args> + toc + extension + + + +
    + + +

    Table of Contents

    +    + (view full table) +

    + +
    + <& printtoc, root = toc, current = None, full = False, children=False, extension=extension, anchor_toplevel=False &> +
    + +
    + + +
    + +

    Table of Contents: Full

    +    + (view brief table) +

    + +
    + <& printtoc, root = toc, current = None, full = True, children=True, extension=extension, anchor_toplevel=False &> +
    + +
    + + + +<%method printtoc> +<%args> + root + current = None + full = False + children = True + extension + anchor_toplevel=False + + + + + +<%def printtocelement> +<%doc>prints a TOCElement as a table of contents item and prints its immediate child items + <%args> + item + bold = False + full = False + children = True + extension + anchor_toplevel + + +
  • " href="<% item.get_link(extension=extension, anchor=anchor_toplevel) %>"><% item.description %>
    • + +% if children: + +% + + +<%def printsmtocelem> + <%args> + item + children = False + extension + +
  • <% item.description %>
    • + +% if children: + +% + + diff --git a/doc/build/content/docstrings.myt b/doc/build/content/docstrings.myt index a30a5fb636..685c0314fa 100644 --- a/doc/build/content/docstrings.myt +++ b/doc/build/content/docstrings.myt @@ -1,6 +1,12 @@ -<%flags>inherit='document_base.myt' -<%attr>title='Modules and Classes' -<&|doclib.myt:item, name="docstrings", description="Modules and Classes" &> +<%flags>inherit='content_layout.myt' +<%attr> + title='Modules and Classes' + filename='docstrings' + +<%args> + toc + extension + <%init> import cPickle as pickle import os @@ -9,7 +15,5 @@ % for obj in data: -<& pydoc.myt:obj_doc, obj=obj &> +<& pydoc.myt:obj_doc, obj=obj, toc=toc, extension=extension &> % - - \ No newline at end of file diff --git a/doc/build/content/document_base.myt b/doc/build/content/document_base.myt deleted file mode 100644 index e4750c7563..0000000000 --- a/doc/build/content/document_base.myt +++ /dev/null @@ -1,42 +0,0 @@ -<%flags>inherit="doclib.myt" - -<%python scope="global"> - - files = [ - 'tutorial', - 'dbengine', - 'metadata', - 'sqlconstruction', - 'datamapping', - 'unitofwork', - 'adv_datamapping', - 'types', - 'pooling', - 'plugins', - 'docstrings', - ] - - - -<%attr> - files=files - wrapper='section_wrapper.myt' - onepage='documentation' - index='index' - title='SQLAlchemy 0.3 Documentation' - version = '0.3.0' - - -<%method title> -% try: -# avoid inheritance via attr instead of attributes - <% m.base_component.attr['title'] %> - <% self.owner.attr['title'] %> -% except KeyError: - <% self.owner.attr['title'] %> -% - - - - - - diff --git a/doc/build/content/index.myt b/doc/build/content/index.myt new file mode 100644 index 0000000000..524a43b1a8 --- /dev/null +++ b/doc/build/content/index.myt @@ -0,0 +1,7 @@ +<%flags>inherit='base.myt' +<%args> + extension + toc + + +<& toc.myt:toc, toc=toc, extension=extension &> diff --git a/doc/build/content/metadata.txt b/doc/build/content/metadata.txt index 4cae58cb4b..cc677cd89a 100644 --- a/doc/build/content/metadata.txt +++ b/doc/build/content/metadata.txt @@ -463,13 +463,13 @@ A table with a sequence looks like: Column("description", String(40)), Column("createdate", DateTime()) ) - -The Sequence is used with Postgres or Oracle to indicate the name of a database sequence that will be used to create default values for a column. When a table with a Sequence on a column is created in the database by SQLAlchemy, the database sequence object is also created. Similarly, the database sequence is dropped when the table is dropped. Sequences are typically used with primary key columns. When using Postgres, if an integer primary key column defines no explicit Sequence or other default method, SQLAlchemy will create the column with the SERIAL keyword, and will pre-execute a sequence named "tablename_columnname_seq" in order to retrieve new primary key values, if they were not otherwise explicitly stated. Oracle, which has no "auto-increment" keyword, requires that a Sequence be created for a table if automatic primary key generation is desired. - + +The Sequence is used with Postgres or Oracle to indicate the name of a database sequence that will be used to create default values for a column. When a table with a Sequence on a column is created in the database by SQLAlchemy, the database sequence object is also created. Similarly, the database sequence is dropped when the table is dropped. Sequences are typically used with primary key columns. When using Postgres, if an integer primary key column defines no explicit Sequence or other default method, SQLAlchemy will create the column with the SERIAL keyword, and will pre-execute a sequence named "tablename_columnname_seq" in order to retrieve new primary key values, if they were not otherwise explicitly stated. Oracle, which has no "auto-increment" keyword, requires that a Sequence be created for a table if automatic primary key generation is desired. + A Sequence object can be defined on a Table that is then used for a non-sequence-supporting database. In that case, the Sequence object is simply ignored. Note that a Sequence object is **entirely optional for all databases except Oracle**, as other databases offer options for auto-creating primary key values, such as AUTOINCREMENT, SERIAL, etc. SQLAlchemy will use these default methods for creating primary key values if no Sequence is present on the table metadata. - + A sequence can also be specified with `optional=True` which indicates the Sequence should only be used on a database that requires an explicit sequence, and not those that supply some other method of providing integer values. At the moment, it essentially means "use this sequence only with Oracle and not Postgres". - + ### Defining Constraints and Indexes {@name=constraints} #### UNIQUE Constraint diff --git a/doc/build/content/tutorial.txt b/doc/build/content/tutorial.txt index d17c82db60..2cce781ea6 100644 --- a/doc/build/content/tutorial.txt +++ b/doc/build/content/tutorial.txt @@ -3,7 +3,7 @@ Tutorial This tutorial provides a relatively simple walking tour through the basic concepts of SQLAlchemy. You may wish to skip it and dive into the [main manual][manual] which is more reference-oriented. The examples in this tutorial comprise a fully working interactive Python session, and are guaranteed to be functioning courtesy of [doctest][]. [doctest]: http://www.python.org/doc/lib/module-doctest.html -[manual]: rel:howtoread +[manual]: rel:metadata Installation ------------ diff --git a/doc/build/gen_docstrings.py b/doc/build/gen_docstrings.py new file mode 100644 index 0000000000..67b60cddcb --- /dev/null +++ b/doc/build/gen_docstrings.py @@ -0,0 +1,74 @@ +from toc import TOCElement +import docstring + +import sqlalchemy.schema as schema +import sqlalchemy.engine as engine +import sqlalchemy.engine.strategies as strategies +import sqlalchemy.sql as sql +import sqlalchemy.pool as pool +import sqlalchemy.orm as orm +import sqlalchemy.exceptions as exceptions +import sqlalchemy.ext.proxy as proxy +import sqlalchemy.ext.sessioncontext as sessioncontext +import sqlalchemy.mods.threadlocal as threadlocal +import sqlalchemy.ext.selectresults as selectresults + +def make_doc(obj, classes=None, functions=None): + """generate a docstring.ObjectDoc structure for an individual module, list of classes, and list of functions.""" + return docstring.ObjectDoc(obj, classes=classes, functions=functions) + +def make_all_docs(): + """generate a docstring.AbstractDoc structure.""" + objects = [ + make_doc(obj=sql), + make_doc(obj=schema), + make_doc(obj=engine), + make_doc(obj=engine.url), + make_doc(obj=orm, classes=[orm.MapperExtension]), + make_doc(obj=orm.mapperlib, classes=[orm.mapperlib.Mapper]), + make_doc(obj=orm.query, classes=[orm.query.Query, orm.query.QueryContext, orm.query.SelectionContext]), + make_doc(obj=orm.session, classes=[orm.session.Session, orm.session.SessionTransaction]), + make_doc(obj=pool, classes=[pool.DBProxy, pool.Pool, pool.QueuePool, pool.SingletonThreadPool]), + make_doc(obj=sessioncontext), + make_doc(obj=threadlocal), + make_doc(obj=selectresults), + make_doc(obj=exceptions), + make_doc(obj=proxy), + ] + return objects + +def create_docstring_toc(data, root): + """given a docstring.AbstractDoc structure, create new TOCElement nodes corresponding + to the elements and cross-reference them back to the doc structure.""" + root = TOCElement("docstrings", name="docstrings", description="Generated Documentation", parent=root) + def create_obj_toc(obj, toc): + if obj.isclass: + s = [] + links = [] + for elem in obj.inherits: + if isinstance(elem, docstring.ObjectDoc): + links.append("%s" % (str(elem.id), elem.name)) + s.append(elem.name) + else: + links.append(str(elem)) + s.append(str(elem)) + description = "class " + obj.classname + "(%s)" % (','.join(s)) + htmldescription = "class " + obj.classname + "(%s)" % (','.join(links)) + else: + description = obj.description + htmldescription = obj.description + + toc = TOCElement("docstrings", obj.name, description, parent=toc) + obj.toc_path = toc.path + + if not obj.isclass and obj.functions: + TOCElement("docstrings", name="modfunc", description="Module Functions", parent=toc) + + if obj.classes: + for class_ in obj.classes: + create_obj_toc(class_, toc) + + for obj in data: + create_obj_toc(obj, root) + return data + diff --git a/doc/build/genhtml.py b/doc/build/genhtml.py index 2a274b03ab..015f65bdf0 100644 --- a/doc/build/genhtml.py +++ b/doc/build/genhtml.py @@ -1,24 +1,65 @@ #!/usr/bin/env python -import sys,re,os +import sys,re,os,shutil +import myghty.interp +import myghty.exception as exception +import cPickle as pickle -print "Running txt2myt.py..." -execfile("txt2myt.py") +sys.path = ['../../lib', './lib/'] + sys.path -print "Generating docstring data" -execfile("compile_docstrings.py") +import gen_docstrings, read_markdown, toc + +files = [ + 'index', + 'tutorial', + 'dbengine', + 'metadata', + 'sqlconstruction', + 'datamapping', + 'unitofwork', + 'adv_datamapping', + 'types', + 'pooling', + 'plugins', + 'docstrings' + ] + +title='SQLAlchemy 0.3 Documentation' +version = '0.3.0' + +root = toc.TOCElement('', 'root', '', version=version, doctitle=title) + +shutil.copy('./content/index.myt', './output/index.myt') +shutil.copy('./content/docstrings.myt', './output/docstrings.myt') + +read_markdown.parse_markdown_files(root, files) +docstrings = gen_docstrings.make_all_docs() +gen_docstrings.create_docstring_toc(docstrings, root) + +pickle.dump(docstrings, file('./output/compiled_docstrings.pickle', 'w')) +pickle.dump(root, file('./output/table_of_contents.pickle', 'w')) component_root = [ {'components': './components'}, - {'content' : './content'} + {'output' :'./output'} ] -doccomp = ['document_base.myt'] output = os.path.dirname(os.getcwd()) -sys.path = ['./lib/'] + sys.path +interp = myghty.interp.Interpreter(component_root = component_root, output_encoding='utf-8') -import documentgen +def genfile(name, toc): + infile = name + ".myt" + outname = os.path.join(os.getcwd(), '../', name + ".html") + outfile = file(outname, 'w') + print infile, '->', outname + interp.execute(infile, out_buffer=outfile, request_args={'toc':toc,'extension':'html'}, raise_error=True) + +try: + for filename in files: + genfile(filename, root) +except exception.Error, e: + sys.stderr.write(e.textformat()) -documentgen.genall(doccomp, component_root, output) + diff --git a/doc/build/lib/docstring.py b/doc/build/lib/docstring.py index b6879c7ade..b4ddcff4ef 100644 --- a/doc/build/lib/docstring.py +++ b/doc/build/lib/docstring.py @@ -1,6 +1,8 @@ -import re, types, string, inspect +""" +defines a pickleable, recursive "generated python documentation" datastructure. +""" -"""sucks a module and its contents into a simple documentation object, suitable for pickling""" +import re, types, string, inspect allobjects = {} @@ -9,6 +11,7 @@ class AbstractDoc(object): allobjects[id(obj)] = self self.id = id(obj) self.allobjects = allobjects + self.toc_path = None class ObjectDoc(AbstractDoc): def __init__(self, obj, functions=None, classes=None): diff --git a/doc/build/lib/documentgen.py b/doc/build/lib/documentgen.py deleted file mode 100644 index 7e1783d1dd..0000000000 --- a/doc/build/lib/documentgen.py +++ /dev/null @@ -1,44 +0,0 @@ -import sys, re, os -import myghty.interp -import myghty.exception as exception - -# document generation library - -def genall(comps, component_root, output_dir): - interp = myghty.interp.Interpreter( component_root = component_root) - - try: - for comp in comps: - gendoc(comp, interp, output_dir = output_dir) - except exception.Error, e: - sys.stderr.write(e.textformat()) - - -def gendoc(doccomp, interp, output_dir): - component = interp.load(doccomp) - files = component.get_attribute('files') - index = component.get_attribute('index') - onepage = component.get_attribute('onepage') - - genfile(index + ".myt", interp, output_dir) - - for file in files: - file += '.myt' - genfile(file, interp, output_dir) - - genfile(index + ".myt", interp, output_dir, outfile = onepage + ".html", args = {'paged':'no'}) - - - -def genfile(file, interp, output_dir, outfile = None, args = {}): - if outfile is None: - outfile = re.sub(r"\..+$", "%s" % '.html', file) - - outfile = os.path.join(output_dir, outfile) - print "%s -> %s" % (file, outfile) - outbuf = open(outfile, "w") - - interp.execute(file, out_buffer = outbuf, request_args = args, raise_error = True) - - outbuf.close() - diff --git a/doc/build/lib/toc.py b/doc/build/lib/toc.py new file mode 100644 index 0000000000..ebf7e95471 --- /dev/null +++ b/doc/build/lib/toc.py @@ -0,0 +1,70 @@ +""" +defines a pickleable, recursive "table of contents" datastructure. + +TOCElements define a name, a description, and also a uniquely-identifying "path" which is +used to generate hyperlinks between document sections. +""" +import time + +toc_by_file = {} +toc_by_path = {} + +class TOCElement(object): + def __init__(self, filename, name, description, parent=None, version=None, last_updated=None, doctitle=None, **kwargs): + self.filename = filename + self.name = name + self.description = description + self.parent = parent + self.content = None + self.toc_by_path = toc_by_path + self.toc_by_file = toc_by_file + self.last_updated = time.time() + self.version = version + self.doctitle = doctitle + (self.path, self.depth) = self._create_path() + #print "NEW TOC:", self.path + for key, value in kwargs.iteritems(): + setattr(self, key, value) + + toc_by_path[self.path] = self + + self.is_top = (self.parent is not None and self.parent.filename != self.filename) or self.parent is None + if self.is_top: + toc_by_file[self.filename] = self + + self.root = self.parent or self + + self.content = None + self.previous = None + self.next = None + self.children = [] + if parent: + if len(parent.children): + self.previous = parent.children[-1] + parent.children[-1].next = self + parent.children.append(self) + + def get_page_root(self): + return self.toc_by_file[self.filename] + + def get_by_path(self, path): + return self.toc_by_path.get(path) + + def get_by_file(self, filename): + return self.toc_by_file[filename] + + def get_link(self, extension='html', anchor=True): + if anchor: + return "%s.%s#%s" % (self.filename, extension, self.path) + else: + return "%s.%s" % (self.filename, extension) + + def _create_path(self): + elem = self + tokens = [] + depth = 0 + while elem.parent is not None: + tokens.insert(0, elem.name) + elem = elem.parent + depth +=1 + return ('_'.join(tokens), depth) diff --git a/doc/build/read_markdown.py b/doc/build/read_markdown.py new file mode 100644 index 0000000000..fdc6175b29 --- /dev/null +++ b/doc/build/read_markdown.py @@ -0,0 +1,230 @@ +import sys, re, os +from toc import TOCElement + +try: + import elementtree.ElementTree as et +except: + raise "This module requires ElementTree to run (http://effbot.org/zone/element-index.htm)" + +import markdown + +def dump_tree(elem, stream): + if elem.tag.startswith('MYGHTY:'): + dump_myghty_tag(elem, stream) + else: + stream.write("<%s %s>" % (elem.tag, " ".join("%s=%s" % (key, repr(val)) for key, val in elem.attrib.iteritems()))) + if elem.text: + stream.write(elem.text) + for child in elem: + dump_tree(child, stream) + if child.tail: + stream.write(child.tail) + stream.write("" % elem.tag) + +def dump_myghty_tag(elem, stream): + tag = elem.tag[7:] + params = ', '.join(['%s=%s' % i for i in elem.items()]) + pipe = '' + if elem.text or len(elem): + pipe = '|' + comma = '' + if params: + comma = ', ' + stream.write('<&%s%s%s%s&>' % (pipe, tag, comma, params)) + if pipe: + if elem.text: + stream.write(elem.text) + for n in elem: + dump_tree(n, stream) + if n.tail: + stream.write(n.tail) + stream.write("") + +def create_toc(filename, tree, tocroot): + title = [None] + current = [tocroot] + level = [0] + def process(tree): + while True: + i = find_header_index(tree) + if i is None: + return + node = tree[i] + taglevel = int(node.tag[1]) + start, end = i, end_of_header(tree, taglevel, i+1) + content = tree[start+1:end] + description = node.text.strip() + if title[0] is None: + title[0] = description + name = node.get('name') + if name is None: + name = description.split()[0].lower() + + taglevel = node.tag[1] + if taglevel > level[0]: + current[0] = TOCElement(filename, name, description, current[0]) + elif taglevel == level[0]: + current[0] = TOCElement(filename, name, description, current[0].parent) + else: + current[0] = TOCElement(filename, name, description, current[0].parent.parent) + + level[0] = taglevel + + tag = et.Element("MYGHTY:formatting.myt:section", path=literal(current[0].path), toc="toc") + tag.text = (node.tail or "") + '\n' + tag.tail = '\n' + tag[:] = content + tree[start:end] = [tag] + + process(tag) + + process(tree) + return (title[0], tocroot.get_by_file(filename)) + +def literal(s): + return '"%s"' % s + +def index(parent, item): + for n, i in enumerate(parent): + if i is item: + return n + +def find_header_index(tree): + for i, node in enumerate(tree): + if is_header(node): + return i + +def is_header(node): + t = node.tag + return (isinstance(t, str) and len(t) == 2 and t[0] == 'h' + and t[1] in '123456789') + +def end_of_header(tree, level, start): + for i, node in enumerate(tree[start:]): + if is_header(node) and int(node.tag[1]) <= level: + return start + i + return len(tree) + +def process_rel_href(tree): + parent = get_parent_map(tree) + for a in tree.findall('.//a'): + m = re.match(r'(bold)?rel\:(.+)', a.get('href')) + if m: + (bold, path) = m.group(1,2) + text = a.text + if text == path: + tag = et.Element("MYGHTY:nav.myt:toclink", path=literal(path), toc="toc", extension="extension") + else: + tag = et.Element("MYGHTY:nav.myt:toclink", path=literal(path), description=literal(text), toc="toc", extension="extension") + a_parent = parent[a] + if bold: + bold = et.Element('strong') + bold.tail = a.tail + bold.append(tag) + a_parent[index(a_parent, a)] = bold + else: + tag.tail = a.tail + a_parent[index(a_parent, a)] = tag + +def replace_pre_with_myt(tree): + def splice_code_tag(pre, text, type=None, title=None): + doctest_directives = re.compile(r'#\s*doctest:\s*[+-]\w+(,[+-]\w+)*\s*$', re.M) + text = re.sub(doctest_directives, '', text) + # process '>>>' to have quotes around it, to work with the myghty python + # syntax highlighter which uses the tokenize module + text = re.sub(r'>>> ', r'">>>" ', text) + + # indent two spaces. among other things, this helps comment lines "# " from being + # consumed as Myghty comments. + text = re.compile(r'^(?!<&)', re.M).sub(' ', text) + + sqlre = re.compile(r'{sql}(.*?)((?:SELECT|INSERT|DELETE|UPDATE|CREATE|DROP|PRAGMA|DESCRIBE).*?)\n\s*(\n|$)', re.S) + if sqlre.search(text) is not None: + use_sliders = False + else: + use_sliders = True + + text = sqlre.sub(r"<&formatting.myt:poplink&>\1\n<&|formatting.myt:codepopper, link='sql'&>\2\n\n", text) + + sqlre2 = re.compile(r'{opensql}(.*?)((?:SELECT|INSERT|DELETE|UPDATE|CREATE|DROP).*?)\n\s*(\n|$)', re.S) + text = sqlre2.sub(r"<&|formatting.myt:poppedcode &>\1\n\2\n\n", text) + + opts = {} + if type == 'python': + opts['syntaxtype'] = literal('python') + else: + opts['syntaxtype'] = None + + if title is not None: + opts['title'] = literal(title) + + if use_sliders: + opts['use_sliders'] = True + + tag = et.Element("MYGHTY:formatting.myt:code", **opts) + tag.text = text + + pre_parent = parents[pre] + tag.tail = pre.tail + pre_parent[reverse_parent(pre_parent, pre)] = tag + + parents = get_parent_map(tree) + + for precode in tree.findall('.//pre/code'): + m = re.match(r'\{(python|code)(?: title="(.*?)"){0,1}\}', precode.text.lstrip()) + if m: + code = m.group(1) + title = m.group(2) + text = precode.text.lstrip() + text = re.sub(r'{(python|code).*?}(\n\s*)?', '', text) + splice_code_tag(parents[precode], text, type=code, title=title) + elif precode.text.lstrip().startswith('>>> '): + splice_code_tag(parents[precode], precode.text) + +def reverse_parent(parent, item): + for n, i in enumerate(parent): + if i is item: + return n + +def get_parent_map(tree): + return dict([(c, p) for p in tree.getiterator() for c in p]) + +def header(toc, title, filename): + return """#encoding: utf-8 +<%%flags> + inherit='content_layout.myt' + +<%%args> + toc + extension + +<%%attr> + title='%s - %s' + filename = '%s' + +<%%doc>This file is generated. Edit the .txt files instead of this one. +""" % (toc.root.doctitle, title, filename) + +class utf8stream(object): + def __init__(self, stream): + self.stream = stream + def write(self, str): + self.stream.write(str.encode('utf8')) + +def parse_markdown_files(toc, files): + for inname in files: + infile = 'content/%s.txt' % inname + if not os.access(infile, os.F_OK): + continue + html = markdown.markdown(file(infile).read()) + tree = et.fromstring("" + html + "") + (title, toc_element) = create_toc(inname, tree, toc) + replace_pre_with_myt(tree) + process_rel_href(tree) + outname = 'output/%s.myt' % inname + print infile, '->', outname + outfile = utf8stream(file(outname, 'w')) + outfile.write(header(toc, title, inname)) + dump_tree(tree, outfile) + + diff --git a/doc/build/runhtml.py b/doc/build/runhtml.py index 591e3b7f7a..c5b34e4bf1 100755 --- a/doc/build/runhtml.py +++ b/doc/build/runhtml.py @@ -1,15 +1,13 @@ #!/usr/bin/env python import sys,re,os -print "Running txt2myt.py..." -execfile("txt2myt.py") +"""starts an HTTP server which will serve generated .myt files from the ./components and +./output directories.""" -print "Generating docstring data" -execfile("compile_docstrings.py") component_root = [ {'components': './components'}, - {'content' : './content'} + {'content' : './output'} ] doccomp = ['document_base.myt'] output = os.path.dirname(os.getcwd()) @@ -18,15 +16,16 @@ sys.path = ['./lib/'] + sys.path import myghty.http.HTTPServerHandler as HTTPServerHandler +port = 8080 httpd = HTTPServerHandler.HTTPServer( - port = 8080, - + port = port, handlers = [ - {'.*(?:\.myt|/$)' : HTTPServerHandler.HSHandler(path_translate=[(r'^/$', r'/index.myt')], data_dir = './cache', component_root = component_root)}, + {'.*(?:\.myt|/$)' : HTTPServerHandler.HSHandler(path_translate=[(r'^/$', r'/index.myt')], data_dir = './cache', component_root = component_root, output_encoding='utf-8')}, ], docroot = [{'.*' : '../'}], ) - + +print "Listening on %d" % port httpd.serve_forever() diff --git a/doc/build/txt2myt.py b/doc/build/txt2myt.py deleted file mode 100644 index 3c81137a32..0000000000 --- a/doc/build/txt2myt.py +++ /dev/null @@ -1,296 +0,0 @@ -"""txt2myt.py -- translate all .txt files in a `content` directory to -.myt template, assuming .txt conform to Markdown syntax. -""" - -import sys -import string -import re - -try: - import elementtree.ElementTree as et -except: - raise "This module requires ElementTree to run (http://effbot.org/zone/element-index.htm)" - -sys.path.insert(0, './lib') -import markdown - -class MyElementTree(et.ElementTree): - def _write(self, file, node, encoding, namespaces): - """With support for myghty 'tags'""" - if node.tag is MyghtyTag: - params = ', '.join(['%s="%s"' % i for i in node.items()]) - pipe = '' - if node.text or len(node): - pipe = '|' - comma = '' - if params: - comma = ', ' - file.write('<&%s%s%s%s&>' % (pipe, node.name, comma, params)) - if pipe: - if node.text: - file.write(node.text) - for n in node: - self._write(file, n, encoding, namespaces) - file.write("") - if node.tail: - file.write(node.tail) - else: - et.ElementTree._write(self, file, node, encoding, namespaces) - -# The same as et.dump, but using MyElementTree -def dump(elem): - # debugging - if not isinstance(elem, et.ElementTree): - elem = MyElementTree(elem) - elem.write(sys.stdout) - tail = elem.getroot().tail - if not tail or tail[-1] != "\n": - sys.stdout.write("\n") - -# The same as et.tostring, but using MyElementTree -def tostring(element, encoding=None): - class dummy: - pass - data = [] - file = dummy() - file.write = data.append - MyElementTree(element).write(file, encoding) - return string.join(data, "") - -def MyghtyTag(name_, attrib_={}, **extra): - """Can be used with ElementTree in places where Element is required""" - element = et.Element(MyghtyTag, attrib_, **extra) - element.name = name_ - return element - -CODE_BLOCK = 'formatting.myt:code' -DOCTEST_DIRECTIVES = re.compile(r'#\s*doctest:\s*[+-]\w+(,[+-]\w+)*\s*$', re.M) -LINK = 'formatting.myt:link' -LINK_REG = re.compile(r'(bold)?rel\:(.+)') -SECTION = 'doclib.myt:item' - -def process_code_blocks(tree): - """Replace 
    ...
    with Myghty tags, if it contains: - * '{python}' - or - * '>>> ' - - Note: '{python}' will be removed - Note: also remove all doctest directives - """ - parent = get_parent_map(tree) - - def replace_pre_with_myt(pre, text, type=None, title=None): - text = re.sub(DOCTEST_DIRECTIVES, '', text) - # process '>>>' to have quotes around it, to work with the myghty python - # syntax highlighter which uses the tokenize module - text = re.sub(r'>>> ', r'">>>" ', text) - - # indent two spaces. among other things, this helps comment lines "# " from being - # consumed as Myghty comments. - text = re.compile(r'^(?!<&)', re.M).sub(' ', text) - - sqlre = re.compile(r'{sql}(.*?)((?:SELECT|INSERT|DELETE|UPDATE|CREATE|DROP|PRAGMA|DESCRIBE).*?)\n\s*(\n|$)', re.S) - if sqlre.search(text) is not None: - use_sliders = False - else: - use_sliders = True - - text = sqlre.sub(r"<&formatting.myt:poplink&>\1\n<&|formatting.myt:codepopper, link='sql'&>\2\n\n", text) - - sqlre2 = re.compile(r'{opensql}(.*?)((?:SELECT|INSERT|DELETE|UPDATE|CREATE|DROP).*?)\n\s*(\n|$)', re.S) - text = sqlre2.sub(r"<&|formatting.myt:poppedcode &>\1\n\2\n\n", text) - - pre_parent = parent[pre] - opts = {} - if type == 'python': - opts['syntaxtype'] = 'python' - else: - opts['syntaxtype'] = None - - if title is not None: - opts['title'] = title - - if use_sliders: - opts['use_sliders'] = True - - tag = MyghtyTag(CODE_BLOCK, opts) - tag.text = text - tag.tail = pre.tail - pre_parent[index(pre_parent, pre)] = tag - - for precode in tree.findall('.//pre/code'): - m = re.match(r'\{(python|code)(?: title="(.*?)"){0,1}\}', precode.text.lstrip()) - if m: - code = m.group(1) - title = m.group(2) - text = precode.text.lstrip() - text = re.sub(r'{(python|code).*?}(\n\s*)?', '', text) - replace_pre_with_myt(parent[precode], text, type=code, title=title) - elif precode.text.lstrip().startswith('>>> '): - replace_pre_with_myt(parent[precode], precode.text) - -def process_rel_href(tree): - """Replace all YYY with Myghty tags - - If XXX is the same as YYY, text attribute for Myghty tag will not be - provided. - """ - parent = get_parent_map(tree) - for a in tree.findall('.//a'): - m = LINK_REG.match(a.get('href')) - if m: - (bold, path) = m.group(1,2) - text = a.text - if text == path: - tag = MyghtyTag(LINK, path=path) - else: - tag = MyghtyTag(LINK, path=path, text=text) - a_parent = parent[a] - if bold: - bold = et.Element('strong') - bold.tail = a.tail - bold.append(tag) - a_parent[index(a_parent, a)] = bold - else: - tag.tail = a.tail - a_parent[index(a_parent, a)] = tag - -def process_headers(tree): - """Replace all

    ,

    ... with Mighty tags - """ - title = [None] - def process(tree): - while True: - i = find_header_index(tree) - if i is None: - return - node = tree[i] - level = int(node.tag[1]) - start, end = i, end_of_header(tree, level, i+1) - content = tree[start+1:end] - description = node.text.strip() - if title[0] is None: - title[0] = description - name = node.get('name') - if name is None: - name = description.split()[0].lower() - - tag = MyghtyTag(SECTION, name=name, description=description) - tag.text = (node.tail or "") + '\n' - tag.tail = '\n' - tag[:] = content - tree[start:end] = [tag] - - process(tag) - - process(tree) - return title[0] - -def index(parent, item): - for n, i in enumerate(parent): - if i is item: - return n - -def find_header_index(tree): - for i, node in enumerate(tree): - if is_header(node): - return i - -def is_header(node): - t = node.tag - return (isinstance(t, str) and len(t) == 2 and t[0] == 'h' - and t[1] in '123456789') - -def end_of_header(tree, level, start): - for i, node in enumerate(tree[start:]): - if is_header(node) and int(node.tag[1]) <= level: - return start + i - return len(tree) - -def get_parent_map(tree): - return dict([(c, p) for p in tree.getiterator() for c in p]) - -def html2myghtydoc(html): - """Convert HTML to Myghty template (for SQLAlchemy's doc framework) - - html -- string containing HTML document, which: - * should be without surrounding or tags - * should have only one top-level header (usually

    ). The text of the - header will be used as a title of Myghty template. - - Things converter treats specially: - * YYY will result in: - <&formatting.myt:link, path="XXX", text="YYY"&> - - If XXX is the same as YYY, text attribute will be absent. E.g. - XXX will result in: - <&formatting.myt:link, path="XXX"&> - - * All header tags (

    ,

    ...) will be replaced. E.g. - -

    Great title

    Section content goes here

    - - will turn into: - - <&|doclib.myt:item, name="great", description="Great title"&> -

    Section content goes here

    - - - Note that by default the value of `name` parameter is a lower-case version - of the first word of the title. If you want to override it, supply - `name` attribute to header tag, e.g.: -

    Great title

    ... - will turn into - <&|doclib.myt:item, name="title", description="Great title"&>... - - (Note that to give this attribute in Markdown, you can - write: {@name=title}.) - - * If you have {python} marker inside , which is inside 
    , it will
-      replaced with Myghty tag. E.g.:
-
-      
    {python} print 'hello, world!'
    
-
-      will turn into
-
-      <&|formatting.myt:code&>print 'hello, world!'
-
-      You don't need to write {python} marker if you use examples starting with
-      Python prompt: >>>
-
-      If you use doctest directives, they will be removed from output.
-
-      (Note that 
     and  is what Markdown outputs for pre-formatted
-      code blocks.)
-    """
-
-    tree = et.fromstring("" + html + "")
-
-    process_code_blocks(tree)
-    process_rel_href(tree)
-    title = process_headers(tree)
-
-    header = "<%flags>inherit='document_base.myt'\n"
-    header += "<%attr>title='" + title + "'\n"
-    header += "\n"
-
-    # discard surrounding  tag
-    body = ''.join([tostring(e) for e in tree[:]])
-
-    return header + body
-
-if __name__ == '__main__':
-    import glob
-    filenames = sys.argv[1:]
-    if len(filenames) == 0:
-        filenames = glob.glob('content/*.txt')
-    for inname in filenames:
-        outname = inname[:-3] + 'myt'
-        print inname, '->', outname
-        input = file(inname).read()
-        html = markdown.markdown(input)
-        #file(inname[:-3] + "html", 'w').write(html)
-        myt = html2myghtydoc(html)
-        file(outname, 'w').write(myt)
-- 
2.47.2