]>
Commit | Line | Data |
---|---|---|
4078ee54 PM |
1 | # coding=utf-8 |
2 | # | |
3 | # QEMU qapidoc QAPI file parsing extension | |
4 | # | |
5 | # Copyright (c) 2020 Linaro | |
6 | # | |
7 | # This work is licensed under the terms of the GNU GPLv2 or later. | |
8 | # See the COPYING file in the top-level directory. | |
9 | ||
10 | """ | |
11 | qapidoc is a Sphinx extension that implements the qapi-doc directive | |
12 | ||
13 | The purpose of this extension is to read the documentation comments | |
14 | in QAPI schema files, and insert them all into the current document. | |
15 | ||
16 | It implements one new rST directive, "qapi-doc::". | |
17 | Each qapi-doc:: directive takes one argument, which is the | |
18 | pathname of the schema file to process, relative to the source tree. | |
19 | ||
20 | The docs/conf.py file must set the qapidoc_srctree config value to | |
21 | the root of the QEMU source tree. | |
22 | ||
23 | The Sphinx documentation on writing extensions is at: | |
24 | https://www.sphinx-doc.org/en/master/development/index.html | |
25 | """ | |
26 | ||
27 | import os | |
28 | import re | |
29 | ||
30 | from docutils import nodes | |
31 | from docutils.statemachine import ViewList | |
32 | from docutils.parsers.rst import directives, Directive | |
33 | from sphinx.errors import ExtensionError | |
34 | from sphinx.util.nodes import nested_parse_with_titles | |
35 | import sphinx | |
36 | from qapi.gen import QAPISchemaVisitor | |
46f49468 JS |
37 | from qapi.error import QAPIError, QAPISemError |
38 | from qapi.schema import QAPISchema | |
4078ee54 PM |
39 | |
40 | ||
41 | # Sphinx up to 1.6 uses AutodocReporter; 1.7 and later | |
42 | # use switch_source_input. Check borrowed from kerneldoc.py. | |
43 | Use_SSI = sphinx.__version__[:3] >= '1.7' | |
44 | if Use_SSI: | |
45 | from sphinx.util.docutils import switch_source_input | |
46 | else: | |
47 | from sphinx.ext.autodoc import AutodocReporter | |
48 | ||
49 | ||
50 | __version__ = '1.0' | |
51 | ||
52 | ||
53 | # Function borrowed from pydash, which is under the MIT license | |
54 | def intersperse(iterable, separator): | |
55 | """Yield the members of *iterable* interspersed with *separator*.""" | |
56 | iterable = iter(iterable) | |
57 | yield next(iterable) | |
58 | for item in iterable: | |
59 | yield separator | |
60 | yield item | |
61 | ||
62 | ||
63 | class QAPISchemaGenRSTVisitor(QAPISchemaVisitor): | |
64 | """A QAPI schema visitor which generates docutils/Sphinx nodes | |
65 | ||
66 | This class builds up a tree of docutils/Sphinx nodes corresponding | |
67 | to documentation for the various QAPI objects. To use it, first | |
68 | create a QAPISchemaGenRSTVisitor object, and call its | |
69 | visit_begin() method. Then you can call one of the two methods | |
70 | 'freeform' (to add documentation for a freeform documentation | |
71 | chunk) or 'symbol' (to add documentation for a QAPI symbol). These | |
72 | will cause the visitor to build up the tree of document | |
73 | nodes. Once you've added all the documentation via 'freeform' and | |
74 | 'symbol' method calls, you can call 'get_document_nodes' to get | |
75 | the final list of document nodes (in a form suitable for returning | |
76 | from a Sphinx directive's 'run' method). | |
77 | """ | |
78 | def __init__(self, sphinx_directive): | |
79 | self._cur_doc = None | |
80 | self._sphinx_directive = sphinx_directive | |
81 | self._top_node = nodes.section() | |
82 | self._active_headings = [self._top_node] | |
83 | ||
84 | def _make_dlitem(self, term, defn): | |
85 | """Return a dlitem node with the specified term and definition. | |
86 | ||
87 | term should be a list of Text and literal nodes. | |
88 | defn should be one of: | |
89 | - a string, which will be handed to _parse_text_into_node | |
90 | - a list of Text and literal nodes, which will be put into | |
91 | a paragraph node | |
92 | """ | |
93 | dlitem = nodes.definition_list_item() | |
94 | dlterm = nodes.term('', '', *term) | |
95 | dlitem += dlterm | |
96 | if defn: | |
97 | dldef = nodes.definition() | |
98 | if isinstance(defn, list): | |
99 | dldef += nodes.paragraph('', '', *defn) | |
100 | else: | |
101 | self._parse_text_into_node(defn, dldef) | |
102 | dlitem += dldef | |
103 | return dlitem | |
104 | ||
105 | def _make_section(self, title): | |
106 | """Return a section node with optional title""" | |
107 | section = nodes.section(ids=[self._sphinx_directive.new_serialno()]) | |
108 | if title: | |
109 | section += nodes.title(title, title) | |
110 | return section | |
111 | ||
112 | def _nodes_for_ifcond(self, ifcond, with_if=True): | |
113 | """Return list of Text, literal nodes for the ifcond | |
114 | ||
d806f89f | 115 | Return a list which gives text like ' (If: condition)'. |
4078ee54 PM |
116 | If with_if is False, we don't return the "(If: " and ")". |
117 | """ | |
d806f89f MAL |
118 | |
119 | doc = ifcond.docgen() | |
120 | if not doc: | |
121 | return [] | |
122 | doc = nodes.literal('', doc) | |
4078ee54 | 123 | if not with_if: |
d806f89f | 124 | return [doc] |
4078ee54 PM |
125 | |
126 | nodelist = [nodes.Text(' ('), nodes.strong('', 'If: ')] | |
d806f89f | 127 | nodelist.append(doc) |
4078ee54 PM |
128 | nodelist.append(nodes.Text(')')) |
129 | return nodelist | |
130 | ||
131 | def _nodes_for_one_member(self, member): | |
132 | """Return list of Text, literal nodes for this member | |
133 | ||
134 | Return a list of doctree nodes which give text like | |
135 | 'name: type (optional) (If: ...)' suitable for use as the | |
136 | 'term' part of a definition list item. | |
137 | """ | |
138 | term = [nodes.literal('', member.name)] | |
139 | if member.type.doc_type(): | |
140 | term.append(nodes.Text(': ')) | |
141 | term.append(nodes.literal('', member.type.doc_type())) | |
142 | if member.optional: | |
143 | term.append(nodes.Text(' (optional)')) | |
33aa3267 | 144 | if member.ifcond.is_present(): |
4078ee54 PM |
145 | term.extend(self._nodes_for_ifcond(member.ifcond)) |
146 | return term | |
147 | ||
d1da8af8 | 148 | def _nodes_for_variant_when(self, branches, variant): |
4078ee54 PM |
149 | """Return list of Text, literal nodes for variant 'when' clause |
150 | ||
151 | Return a list of doctree nodes which give text like | |
152 | 'when tagname is variant (If: ...)' suitable for use in | |
d1da8af8 | 153 | the 'branches' part of a definition list. |
4078ee54 PM |
154 | """ |
155 | term = [nodes.Text(' when '), | |
d1da8af8 | 156 | nodes.literal('', branches.tag_member.name), |
4078ee54 PM |
157 | nodes.Text(' is '), |
158 | nodes.literal('', '"%s"' % variant.name)] | |
33aa3267 | 159 | if variant.ifcond.is_present(): |
4078ee54 PM |
160 | term.extend(self._nodes_for_ifcond(variant.ifcond)) |
161 | return term | |
162 | ||
d1da8af8 | 163 | def _nodes_for_members(self, doc, what, base=None, branches=None): |
4078ee54 PM |
164 | """Return list of doctree nodes for the table of members""" |
165 | dlnode = nodes.definition_list() | |
166 | for section in doc.args.values(): | |
167 | term = self._nodes_for_one_member(section.member) | |
168 | # TODO drop fallbacks when undocumented members are outlawed | |
169 | if section.text: | |
170 | defn = section.text | |
4078ee54 PM |
171 | else: |
172 | defn = [nodes.Text('Not documented')] | |
173 | ||
174 | dlnode += self._make_dlitem(term, defn) | |
175 | ||
176 | if base: | |
177 | dlnode += self._make_dlitem([nodes.Text('The members of '), | |
178 | nodes.literal('', base.doc_type())], | |
179 | None) | |
180 | ||
d1da8af8 MA |
181 | if branches: |
182 | for v in branches.variants: | |
e51e80cc MA |
183 | if v.type.name == 'q_empty': |
184 | continue | |
185 | assert not v.type.is_implicit() | |
186 | term = [nodes.Text('The members of '), | |
187 | nodes.literal('', v.type.doc_type())] | |
d1da8af8 | 188 | term.extend(self._nodes_for_variant_when(branches, v)) |
e51e80cc | 189 | dlnode += self._make_dlitem(term, None) |
4078ee54 PM |
190 | |
191 | if not dlnode.children: | |
192 | return [] | |
193 | ||
194 | section = self._make_section(what) | |
195 | section += dlnode | |
196 | return [section] | |
197 | ||
198 | def _nodes_for_enum_values(self, doc): | |
199 | """Return list of doctree nodes for the table of enum values""" | |
200 | seen_item = False | |
201 | dlnode = nodes.definition_list() | |
202 | for section in doc.args.values(): | |
203 | termtext = [nodes.literal('', section.member.name)] | |
33aa3267 | 204 | if section.member.ifcond.is_present(): |
4078ee54 PM |
205 | termtext.extend(self._nodes_for_ifcond(section.member.ifcond)) |
206 | # TODO drop fallbacks when undocumented members are outlawed | |
207 | if section.text: | |
208 | defn = section.text | |
209 | else: | |
210 | defn = [nodes.Text('Not documented')] | |
211 | ||
212 | dlnode += self._make_dlitem(termtext, defn) | |
213 | seen_item = True | |
214 | ||
215 | if not seen_item: | |
216 | return [] | |
217 | ||
218 | section = self._make_section('Values') | |
219 | section += dlnode | |
220 | return [section] | |
221 | ||
222 | def _nodes_for_arguments(self, doc, boxed_arg_type): | |
223 | """Return list of doctree nodes for the arguments section""" | |
224 | if boxed_arg_type: | |
225 | assert not doc.args | |
226 | section = self._make_section('Arguments') | |
227 | dlnode = nodes.definition_list() | |
228 | dlnode += self._make_dlitem( | |
229 | [nodes.Text('The members of '), | |
230 | nodes.literal('', boxed_arg_type.name)], | |
231 | None) | |
232 | section += dlnode | |
233 | return [section] | |
234 | ||
235 | return self._nodes_for_members(doc, 'Arguments') | |
236 | ||
237 | def _nodes_for_features(self, doc): | |
238 | """Return list of doctree nodes for the table of features""" | |
239 | seen_item = False | |
240 | dlnode = nodes.definition_list() | |
241 | for section in doc.features.values(): | |
573e2223 MA |
242 | dlnode += self._make_dlitem( |
243 | [nodes.literal('', section.member.name)], section.text) | |
4078ee54 PM |
244 | seen_item = True |
245 | ||
246 | if not seen_item: | |
247 | return [] | |
248 | ||
249 | section = self._make_section('Features') | |
250 | section += dlnode | |
251 | return [section] | |
252 | ||
253 | def _nodes_for_example(self, exampletext): | |
254 | """Return list of doctree nodes for a code example snippet""" | |
255 | return [nodes.literal_block(exampletext, exampletext)] | |
256 | ||
257 | def _nodes_for_sections(self, doc): | |
258 | """Return list of doctree nodes for additional sections""" | |
259 | nodelist = [] | |
260 | for section in doc.sections: | |
31c54b92 | 261 | if section.tag and section.tag == 'TODO': |
f57e1d05 MA |
262 | # Hide TODO: sections |
263 | continue | |
31c54b92 MA |
264 | snode = self._make_section(section.tag) |
265 | if section.tag and section.tag.startswith('Example'): | |
4078ee54 PM |
266 | snode += self._nodes_for_example(section.text) |
267 | else: | |
268 | self._parse_text_into_node(section.text, snode) | |
269 | nodelist.append(snode) | |
270 | return nodelist | |
271 | ||
272 | def _nodes_for_if_section(self, ifcond): | |
273 | """Return list of doctree nodes for the "If" section""" | |
274 | nodelist = [] | |
33aa3267 | 275 | if ifcond.is_present(): |
4078ee54 | 276 | snode = self._make_section('If') |
2d18b4ca JS |
277 | snode += nodes.paragraph( |
278 | '', '', *self._nodes_for_ifcond(ifcond, with_if=False) | |
279 | ) | |
4078ee54 PM |
280 | nodelist.append(snode) |
281 | return nodelist | |
282 | ||
283 | def _add_doc(self, typ, sections): | |
284 | """Add documentation for a command/object/enum... | |
285 | ||
286 | We assume we're documenting the thing defined in self._cur_doc. | |
287 | typ is the type of thing being added ("Command", "Object", etc) | |
288 | ||
289 | sections is a list of nodes for sections to add to the definition. | |
290 | """ | |
291 | ||
292 | doc = self._cur_doc | |
293 | snode = nodes.section(ids=[self._sphinx_directive.new_serialno()]) | |
294 | snode += nodes.title('', '', *[nodes.literal(doc.symbol, doc.symbol), | |
295 | nodes.Text(' (' + typ + ')')]) | |
296 | self._parse_text_into_node(doc.body.text, snode) | |
297 | for s in sections: | |
298 | if s is not None: | |
299 | snode += s | |
300 | self._add_node_to_current_heading(snode) | |
301 | ||
302 | def visit_enum_type(self, name, info, ifcond, features, members, prefix): | |
303 | doc = self._cur_doc | |
304 | self._add_doc('Enum', | |
305 | self._nodes_for_enum_values(doc) | |
306 | + self._nodes_for_features(doc) | |
307 | + self._nodes_for_sections(doc) | |
308 | + self._nodes_for_if_section(ifcond)) | |
309 | ||
310 | def visit_object_type(self, name, info, ifcond, features, | |
d1da8af8 | 311 | base, members, branches): |
4078ee54 PM |
312 | doc = self._cur_doc |
313 | if base and base.is_implicit(): | |
314 | base = None | |
315 | self._add_doc('Object', | |
d1da8af8 | 316 | self._nodes_for_members(doc, 'Members', base, branches) |
4078ee54 PM |
317 | + self._nodes_for_features(doc) |
318 | + self._nodes_for_sections(doc) | |
319 | + self._nodes_for_if_section(ifcond)) | |
320 | ||
321 | def visit_alternate_type(self, name, info, ifcond, features, variants): | |
322 | doc = self._cur_doc | |
323 | self._add_doc('Alternate', | |
324 | self._nodes_for_members(doc, 'Members') | |
325 | + self._nodes_for_features(doc) | |
326 | + self._nodes_for_sections(doc) | |
327 | + self._nodes_for_if_section(ifcond)) | |
328 | ||
329 | def visit_command(self, name, info, ifcond, features, arg_type, | |
330 | ret_type, gen, success_response, boxed, allow_oob, | |
04f22362 | 331 | allow_preconfig, coroutine): |
4078ee54 PM |
332 | doc = self._cur_doc |
333 | self._add_doc('Command', | |
334 | self._nodes_for_arguments(doc, | |
335 | arg_type if boxed else None) | |
336 | + self._nodes_for_features(doc) | |
337 | + self._nodes_for_sections(doc) | |
338 | + self._nodes_for_if_section(ifcond)) | |
339 | ||
340 | def visit_event(self, name, info, ifcond, features, arg_type, boxed): | |
341 | doc = self._cur_doc | |
342 | self._add_doc('Event', | |
343 | self._nodes_for_arguments(doc, | |
344 | arg_type if boxed else None) | |
345 | + self._nodes_for_features(doc) | |
346 | + self._nodes_for_sections(doc) | |
347 | + self._nodes_for_if_section(ifcond)) | |
348 | ||
349 | def symbol(self, doc, entity): | |
350 | """Add documentation for one symbol to the document tree | |
351 | ||
352 | This is the main entry point which causes us to add documentation | |
353 | nodes for a symbol (which could be a 'command', 'object', 'event', | |
354 | etc). We do this by calling 'visit' on the schema entity, which | |
355 | will then call back into one of our visit_* methods, depending | |
356 | on what kind of thing this symbol is. | |
357 | """ | |
358 | self._cur_doc = doc | |
359 | entity.visit(self) | |
360 | self._cur_doc = None | |
361 | ||
362 | def _start_new_heading(self, heading, level): | |
363 | """Start a new heading at the specified heading level | |
364 | ||
365 | Create a new section whose title is 'heading' and which is placed | |
366 | in the docutils node tree as a child of the most recent level-1 | |
367 | heading. Subsequent document sections (commands, freeform doc chunks, | |
368 | etc) will be placed as children of this new heading section. | |
369 | """ | |
370 | if len(self._active_headings) < level: | |
371 | raise QAPISemError(self._cur_doc.info, | |
372 | 'Level %d subheading found outside a ' | |
373 | 'level %d heading' | |
374 | % (level, level - 1)) | |
375 | snode = self._make_section(heading) | |
376 | self._active_headings[level - 1] += snode | |
377 | self._active_headings = self._active_headings[:level] | |
378 | self._active_headings.append(snode) | |
379 | ||
380 | def _add_node_to_current_heading(self, node): | |
381 | """Add the node to whatever the current active heading is""" | |
382 | self._active_headings[-1] += node | |
383 | ||
384 | def freeform(self, doc): | |
385 | """Add a piece of 'freeform' documentation to the document tree | |
386 | ||
387 | A 'freeform' document chunk doesn't relate to any particular | |
388 | symbol (for instance, it could be an introduction). | |
389 | ||
390 | If the freeform document starts with a line of the form | |
391 | '= Heading text', this is a section or subsection heading, with | |
392 | the heading level indicated by the number of '=' signs. | |
393 | """ | |
394 | ||
395 | # QAPIDoc documentation says free-form documentation blocks | |
396 | # must have only a body section, nothing else. | |
397 | assert not doc.sections | |
398 | assert not doc.args | |
399 | assert not doc.features | |
400 | self._cur_doc = doc | |
401 | ||
402 | text = doc.body.text | |
403 | if re.match(r'=+ ', text): | |
404 | # Section/subsection heading (if present, will always be | |
405 | # the first line of the block) | |
406 | (heading, _, text) = text.partition('\n') | |
407 | (leader, _, heading) = heading.partition(' ') | |
408 | self._start_new_heading(heading, len(leader)) | |
409 | if text == '': | |
410 | return | |
411 | ||
412 | node = self._make_section(None) | |
413 | self._parse_text_into_node(text, node) | |
414 | self._add_node_to_current_heading(node) | |
415 | self._cur_doc = None | |
416 | ||
417 | def _parse_text_into_node(self, doctext, node): | |
418 | """Parse a chunk of QAPI-doc-format text into the node | |
419 | ||
420 | The doc comment can contain most inline rST markup, including | |
421 | bulleted and enumerated lists. | |
422 | As an extra permitted piece of markup, @var will be turned | |
423 | into ``var``. | |
424 | """ | |
425 | ||
426 | # Handle the "@var means ``var`` case | |
427 | doctext = re.sub(r'@([\w-]+)', r'``\1``', doctext) | |
428 | ||
429 | rstlist = ViewList() | |
430 | for line in doctext.splitlines(): | |
431 | # The reported line number will always be that of the start line | |
432 | # of the doc comment, rather than the actual location of the error. | |
433 | # Being more precise would require overhaul of the QAPIDoc class | |
434 | # to track lines more exactly within all the sub-parts of the doc | |
435 | # comment, as well as counting lines here. | |
436 | rstlist.append(line, self._cur_doc.info.fname, | |
437 | self._cur_doc.info.line) | |
438 | # Append a blank line -- in some cases rST syntax errors get | |
439 | # attributed to the line after one with actual text, and if there | |
440 | # isn't anything in the ViewList corresponding to that then Sphinx | |
441 | # 1.6's AutodocReporter will then misidentify the source/line location | |
442 | # in the error message (usually attributing it to the top-level | |
443 | # .rst file rather than the offending .json file). The extra blank | |
444 | # line won't affect the rendered output. | |
445 | rstlist.append("", self._cur_doc.info.fname, self._cur_doc.info.line) | |
446 | self._sphinx_directive.do_parse(rstlist, node) | |
447 | ||
448 | def get_document_nodes(self): | |
449 | """Return the list of docutils nodes which make up the document""" | |
450 | return self._top_node.children | |
451 | ||
452 | ||
453 | class QAPISchemaGenDepVisitor(QAPISchemaVisitor): | |
454 | """A QAPI schema visitor which adds Sphinx dependencies each module | |
455 | ||
456 | This class calls the Sphinx note_dependency() function to tell Sphinx | |
457 | that the generated documentation output depends on the input | |
458 | schema file associated with each module in the QAPI input. | |
459 | """ | |
460 | def __init__(self, env, qapidir): | |
461 | self._env = env | |
462 | self._qapidir = qapidir | |
463 | ||
464 | def visit_module(self, name): | |
35f15acb | 465 | if name != "./builtin": |
4078ee54 PM |
466 | qapifile = self._qapidir + '/' + name |
467 | self._env.note_dependency(os.path.abspath(qapifile)) | |
468 | super().visit_module(name) | |
469 | ||
470 | ||
471 | class QAPIDocDirective(Directive): | |
472 | """Extract documentation from the specified QAPI .json file""" | |
473 | required_argument = 1 | |
474 | optional_arguments = 1 | |
475 | option_spec = { | |
476 | 'qapifile': directives.unchanged_required | |
477 | } | |
478 | has_content = False | |
479 | ||
480 | def new_serialno(self): | |
481 | """Return a unique new ID string suitable for use as a node's ID""" | |
482 | env = self.state.document.settings.env | |
483 | return 'qapidoc-%d' % env.new_serialno('qapidoc') | |
484 | ||
485 | def run(self): | |
486 | env = self.state.document.settings.env | |
487 | qapifile = env.config.qapidoc_srctree + '/' + self.arguments[0] | |
488 | qapidir = os.path.dirname(qapifile) | |
489 | ||
490 | try: | |
491 | schema = QAPISchema(qapifile) | |
492 | ||
493 | # First tell Sphinx about all the schema files that the | |
494 | # output documentation depends on (including 'qapifile' itself) | |
495 | schema.visit(QAPISchemaGenDepVisitor(env, qapidir)) | |
496 | ||
497 | vis = QAPISchemaGenRSTVisitor(self) | |
498 | vis.visit_begin(schema) | |
499 | for doc in schema.docs: | |
500 | if doc.symbol: | |
501 | vis.symbol(doc, schema.lookup_entity(doc.symbol)) | |
502 | else: | |
503 | vis.freeform(doc) | |
504 | return vis.get_document_nodes() | |
505 | except QAPIError as err: | |
506 | # Launder QAPI parse errors into Sphinx extension errors | |
507 | # so they are displayed nicely to the user | |
c375f05e | 508 | raise ExtensionError(str(err)) from err |
4078ee54 PM |
509 | |
510 | def do_parse(self, rstlist, node): | |
511 | """Parse rST source lines and add them to the specified node | |
512 | ||
513 | Take the list of rST source lines rstlist, parse them as | |
514 | rST, and add the resulting docutils nodes as children of node. | |
515 | The nodes are parsed in a way that allows them to include | |
516 | subheadings (titles) without confusing the rendering of | |
517 | anything else. | |
518 | """ | |
519 | # This is from kerneldoc.py -- it works around an API change in | |
520 | # Sphinx between 1.6 and 1.7. Unlike kerneldoc.py, we use | |
521 | # sphinx.util.nodes.nested_parse_with_titles() rather than the | |
522 | # plain self.state.nested_parse(), and so we can drop the saving | |
523 | # of title_styles and section_level that kerneldoc.py does, | |
524 | # because nested_parse_with_titles() does that for us. | |
525 | if Use_SSI: | |
526 | with switch_source_input(self.state, rstlist): | |
527 | nested_parse_with_titles(self.state, rstlist, node) | |
528 | else: | |
529 | save = self.state.memo.reporter | |
530 | self.state.memo.reporter = AutodocReporter( | |
531 | rstlist, self.state.memo.reporter) | |
532 | try: | |
533 | nested_parse_with_titles(self.state, rstlist, node) | |
534 | finally: | |
535 | self.state.memo.reporter = save | |
536 | ||
537 | ||
538 | def setup(app): | |
539 | """ Register qapi-doc directive with Sphinx""" | |
540 | app.add_config_value('qapidoc_srctree', None, 'env') | |
541 | app.add_directive('qapi-doc', QAPIDocDirective) | |
542 | ||
543 | return dict( | |
544 | version=__version__, | |
545 | parallel_read_safe=True, | |
546 | parallel_write_safe=True | |
547 | ) |