tools/update-dbus-docs.py

   1 #!/usr/bin/env python3
   2 # SPDX-License-Identifier: LGPL-2.1-or-later
   3 # pylint: disable=superfluous-parens,consider-using-with
   4
   5 import argparse
   6 import collections
   7 import sys
   8 import os
   9 import subprocess
  10 import io
  11
  12 try:
  13     from lxml import etree
  14 except ModuleNotFoundError as e:
  15     etree = e
  16
  17 try:
  18     from shlex import join as shlex_join
  19 except ImportError as e:
  20     shlex_join = e
  21
  22 try:
  23     from shlex import quote as shlex_quote
  24 except ImportError as e:
  25     shlex_quote = e
  26
  27 class NoCommand(Exception):
  28     pass
  29
  30 BORING_INTERFACES = [
  31     'org.freedesktop.DBus.Peer',
  32     'org.freedesktop.DBus.Introspectable',
  33     'org.freedesktop.DBus.Properties',
  34 ]
  35 RED = '\x1b[31m'
  36 GREEN = '\x1b[32m'
  37 YELLOW = '\x1b[33m'
  38 RESET = '\x1b[39m'
  39
  40 arguments = None
  41
  42 def xml_parser():
  43     return etree.XMLParser(no_network=True,
  44                            remove_comments=False,
  45                            strip_cdata=False,
  46                            resolve_entities=False)
  47
  48 def print_method(declarations, elem, *, prefix, file, is_signal=False):
  49     name = elem.get('name')
  50     klass = 'signal' if is_signal else 'method'
  51     declarations[klass].append(name)
  52
  53     # @org.freedesktop.systemd1.Privileged("true")
  54     # SetShowStatus(in  s mode);
  55
  56     for anno in elem.findall('./annotation'):
  57         anno_name = anno.get('name')
  58         anno_value = anno.get('value')
  59         print(f'''{prefix}@{anno_name}("{anno_value}")''', file=file)
  60
  61     print(f'''{prefix}{name}(''', file=file, end='')
  62     lead = ',\n' + prefix + ' ' * len(name) + ' '
  63
  64     for num, arg in enumerate(elem.findall('./arg')):
  65         argname = arg.get('name')
  66
  67         if argname is None:
  68             if arguments.print_errors:
  69                 print(f'method {name}: argument {num+1} has no name', file=sys.stderr)
  70             argname = 'UNNAMED'
  71
  72         argtype = arg.get('type')
  73         if not is_signal:
  74             direction = arg.get('direction')
  75             print(f'''{lead if num > 0 else ''}{direction:3} {argtype} {argname}''', file=file, end='')
  76         else:
  77             print(f'''{lead if num > 0 else ''}{argtype} {argname}''', file=file, end='')
  78
  79     print(');', file=file)
  80
  81 ACCESS_MAP = {
  82     'read' : 'readonly',
  83     'write' : 'readwrite',
  84 }
  85
  86 def value_ellipsis(prop_type):
  87     if prop_type == 's':
  88         return "'...'"
  89     if prop_type[0] == 'a':
  90         inner = value_ellipsis(prop_type[1:])
  91         return f"[{inner}{', ...' if inner != '...' else ''}]"
  92     return '...'
  93
  94 def print_property(declarations, elem, *, prefix, file):
  95     prop_name = elem.get('name')
  96     prop_type = elem.get('type')
  97     prop_access = elem.get('access')
  98
  99     declarations['property'].append(prop_name)
 100
 101     # @org.freedesktop.DBus.Property.EmitsChangedSignal("false")
 102     # @org.freedesktop.systemd1.Privileged("true")
 103     # readwrite b EnableWallMessages = false;
 104
 105     for anno in elem.findall('./annotation'):
 106         anno_name = anno.get('name')
 107         anno_value = anno.get('value')
 108         print(f'''{prefix}@{anno_name}("{anno_value}")''', file=file)
 109
 110     prop_access = ACCESS_MAP.get(prop_access, prop_access)
 111     print(f'''{prefix}{prop_access} {prop_type} {prop_name} = {value_ellipsis(prop_type)};''', file=file)
 112
 113 def print_interface(iface, *, prefix, file, print_boring, only_interface, declarations):
 114     name = iface.get('name')
 115
 116     is_boring = (name in BORING_INTERFACES or
 117                  only_interface is not None and name != only_interface)
 118
 119     if is_boring and print_boring:
 120         print(f'''{prefix}interface {name} {{ ... }};''', file=file)
 121
 122     elif not is_boring and not print_boring:
 123         print(f'''{prefix}interface {name} {{''', file=file)
 124         prefix2 = prefix + '  '
 125
 126         for num, elem in enumerate(iface.findall('./method')):
 127             if num == 0:
 128                 print(f'''{prefix2}methods:''', file=file)
 129             print_method(declarations, elem, prefix=prefix2 + '  ', file=file)
 130
 131         for num, elem in enumerate(iface.findall('./signal')):
 132             if num == 0:
 133                 print(f'''{prefix2}signals:''', file=file)
 134             print_method(declarations, elem, prefix=prefix2 + '  ', file=file, is_signal=True)
 135
 136         for num, elem in enumerate(iface.findall('./property')):
 137             if num == 0:
 138                 print(f'''{prefix2}properties:''', file=file)
 139             print_property(declarations, elem, prefix=prefix2 + '  ', file=file)
 140
 141         print(f'''{prefix}}};''', file=file)
 142
 143 def check_documented(document, declarations, stats, interface, missing_version):
 144     missing = []
 145
 146     sections = document.findall("refsect1")
 147     history_section = document.find("refsect1[title = 'History']")
 148     if history_section is not None:
 149         sections.remove(history_section)
 150
 151     for klass, items in declarations.items():
 152         stats['total'] += len(items)
 153
 154         for item in items:
 155             if klass in ('method', 'signal'):
 156                 elem = 'function'
 157                 item_repr = f'{item}()'
 158
 159                 # Find all functions/signals in <function> elements that are not
 160                 # suffixed with '()' and fix them
 161                 for section in sections:
 162                     element = section.find(f".//{elem}[. = '{item}']")
 163                     if element is not None:
 164                         element.text = item_repr
 165
 166             elif klass == 'property':
 167                 elem = 'varname'
 168                 item_repr = item
 169             else:
 170                 assert False, (klass, item)
 171
 172             predicate = f".//{elem}[. = '{item_repr}']"
 173             if not any(section.find(predicate) is not None for section in sections):
 174                 if arguments.print_errors:
 175                     print(f'{klass} {item} is not documented :(')
 176                 missing.append((klass, item))
 177
 178             if history_section is None or history_section.find(predicate) is None:
 179                 missing_version.append(f"{interface}.{item_repr}")
 180
 181     stats['missing'] += len(missing)
 182
 183     return missing
 184
 185 def xml_to_text(destination, xml, *, only_interface=None):
 186     file = io.StringIO()
 187
 188     declarations = collections.defaultdict(list)
 189     interfaces = []
 190
 191     print(f'''node {destination} {{''', file=file)
 192
 193     for print_boring in [False, True]:
 194         for iface in xml.findall('./interface'):
 195             print_interface(iface, prefix='  ', file=file,
 196                             print_boring=print_boring,
 197                             only_interface=only_interface,
 198                             declarations=declarations)
 199             name = iface.get('name')
 200             if not name in BORING_INTERFACES:
 201                 interfaces.append(name)
 202
 203     print('''};''', file=file)
 204
 205     return file.getvalue(), declarations, interfaces
 206
 207 def subst_output(document, programlisting, stats, missing_version):
 208     executable = programlisting.get('executable', None)
 209     if executable is None:
 210         # Not our thing
 211         return
 212     executable = programlisting.get('executable')
 213     node = programlisting.get('node')
 214     interface = programlisting.get('interface')
 215
 216     argv = [f'{arguments.build_dir}/{executable}', f'--bus-introspect={interface}']
 217     if isinstance(shlex_join, Exception):
 218         print(f'COMMAND: {" ".join(shlex_quote(arg) for arg in argv)}')
 219     else:
 220         print(f'COMMAND: {shlex_join(argv)}')
 221
 222     try:
 223         out = subprocess.check_output(argv, universal_newlines=True)
 224     except FileNotFoundError:
 225         print(f'{executable} not found, ignoring', file=sys.stderr)
 226         return
 227
 228     xml = etree.fromstring(out, parser=xml_parser())
 229
 230     new_text, declarations, interfaces = xml_to_text(node, xml, only_interface=interface)
 231     programlisting.text = '\n' + new_text + '    '
 232
 233     if declarations:
 234         missing = check_documented(document, declarations, stats, interface, missing_version)
 235         parent = programlisting.getparent()
 236
 237         # delete old comments
 238         for child in parent:
 239             if child.tag is etree.Comment and 'Autogenerated' in child.text:
 240                 parent.remove(child)
 241             if child.tag is etree.Comment and 'not documented' in child.text:
 242                 parent.remove(child)
 243             if child.tag == "variablelist" and child.attrib.get("generated", False) == "True":
 244                 parent.remove(child)
 245
 246         # insert pointer for systemd-directives generation
 247         the_tail = programlisting.tail #tail is erased by addnext, so save it here.
 248         prev_element = etree.Comment("Autogenerated cross-references for systemd.directives, do not edit")
 249         programlisting.addnext(prev_element)
 250         programlisting.tail = the_tail
 251
 252         for interface in interfaces:
 253             variablelist = etree.Element("variablelist")
 254             variablelist.attrib['class'] = 'dbus-interface'
 255             variablelist.attrib['generated'] = 'True'
 256             variablelist.attrib['extra-ref'] = interface
 257
 258             prev_element.addnext(variablelist)
 259             prev_element.tail = the_tail
 260             prev_element = variablelist
 261
 262         for decl_type,decl_list in declarations.items():
 263             for declaration in decl_list:
 264                 variablelist = etree.Element("variablelist")
 265                 variablelist.attrib['class'] = 'dbus-'+decl_type
 266                 variablelist.attrib['generated'] = 'True'
 267                 if decl_type in ('method', 'signal'):
 268                     variablelist.attrib['extra-ref'] = declaration + '()'
 269                 else:
 270                     variablelist.attrib['extra-ref'] = declaration
 271
 272                 prev_element.addnext(variablelist)
 273                 prev_element.tail = the_tail
 274                 prev_element = variablelist
 275
 276         last_element = etree.Comment("End of Autogenerated section")
 277         prev_element.addnext(last_element)
 278         prev_element.tail = the_tail
 279         last_element.tail = the_tail
 280
 281         # insert comments for undocumented items
 282         for item in reversed(missing):
 283             comment = etree.Comment(f'{item[0]} {item[1]} is not documented!')
 284             comment.tail = programlisting.tail
 285             parent.insert(parent.index(programlisting) + 1, comment)
 286
 287 def process(page, missing_version):
 288     src = open(page).read()
 289     xml = etree.fromstring(src, parser=xml_parser())
 290
 291     # print('parsing {}'.format(name), file=sys.stderr)
 292     if xml.tag != 'refentry':
 293         return None
 294
 295     stats = collections.Counter()
 296
 297     pls = xml.findall('.//programlisting')
 298     for pl in pls:
 299         subst_output(xml, pl, stats, missing_version)
 300
 301     out_text = etree.tostring(xml, encoding='unicode')
 302     # massage format to avoid some lxml whitespace handling idiosyncrasies
 303     # https://bugs.launchpad.net/lxml/+bug/526799
 304     out_text = (src[:src.find('<refentryinfo')] +
 305                 out_text[out_text.find('<refentryinfo'):] +
 306                 '\n')
 307
 308     if not arguments.test:
 309         with open(page, 'w') as out:
 310             out.write(out_text)
 311
 312     return { "stats" : stats, "modified" : out_text != src }
 313
 314 def parse_args():
 315     p = argparse.ArgumentParser()
 316     p.add_argument('--test', action='store_true',
 317                    help='only verify that everything is up2date')
 318     p.add_argument('--build-dir', default='build')
 319     p.add_argument('pages', nargs='+')
 320     opts = p.parse_args()
 321     opts.print_errors = not opts.test
 322     return opts
 323
 324 def main():
 325     # pylint: disable=global-statement
 326     global arguments
 327     arguments = parse_args()
 328
 329     for item in (etree, shlex_quote):
 330         if isinstance(item, Exception):
 331             print(item, file=sys.stderr)
 332             sys.exit(77 if arguments.test else 1)
 333
 334     if not os.path.exists(f'{arguments.build_dir}/systemd'):
 335         sys.exit(f"{arguments.build_dir}/systemd doesn't exist. Use --build-dir=.")
 336
 337     missing_version = []
 338     stats = {page.split('/')[-1] : process(page, missing_version) for page in arguments.pages}
 339
 340     ignore_list = open(os.path.join(os.path.dirname(__file__), 'dbus_ignorelist')).read().split()
 341     missing_version = [x for x in missing_version if x not in ignore_list]
 342
 343     for missing in missing_version:
 344         print(f"{RED}Missing version information for {missing}{RESET}")
 345
 346     if missing_version:
 347         sys.exit(1)
 348
 349     # Let's print all statistics at the end
 350     mlen = max(len(page) for page in stats)
 351     total = sum((item['stats'] for item in stats.values()), collections.Counter())
 352     total = 'total', { "stats" : total, "modified" : False }
 353     modified = []
 354     classification = 'OUTDATED' if arguments.test else 'MODIFIED'
 355     for page, info in sorted(stats.items()) + [total]:
 356         m = info['stats']['missing']
 357         t = info['stats']['total']
 358         p = page + ':'
 359         c = classification if info['modified'] else ''
 360         if c:
 361             modified.append(page)
 362         color = RED if m > t/2 else (YELLOW if m else GREEN)
 363         print(f'{color}{p:{mlen + 1}} {t - m}/{t} {c}{RESET}')
 364
 365     if arguments.test and modified:
 366         sys.exit(f'Outdated pages: {", ".join(modified)}\n'
 367                  f'Hint: ninja -C {arguments.build_dir} update-dbus-docs')
 368
 369 if __name__ == '__main__':
 370     main()