[thirdparty/systemd.git] / tools / update-dbus-docs.py

#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1+

import collections
import sys
import shlex
import subprocess
import io
import pprint
from lxml import etree

PARSER = etree.XMLParser(no_network=True,
                         remove_comments=False,
                         strip_cdata=False,
                         resolve_entities=False)

PRINT_ERRORS = True

class NoCommand(Exception):
    pass

def find_command(lines):
    acc = []
    for num, line in enumerate(lines):
        # skip empty leading line
        if num == 0 and not line:
            continue
        cont = line.endswith('\\')
        if cont:
            line = line[:-1].rstrip()
        acc.append(line if not acc else line.lstrip())
        if not cont:
            break
    joined = ' '.join(acc)
    if not joined.startswith('$ '):
        raise NoCommand
    return joined[2:], lines[:num+1] + [''], lines[-1]

BORING_INTERFACES = [
    'org.freedesktop.DBus.Peer',
    'org.freedesktop.DBus.Introspectable',
    'org.freedesktop.DBus.Properties',
]

def print_method(declarations, elem, *, prefix, file, is_signal=False):
    name = elem.get('name')
    klass = 'signal' if is_signal else 'method'
    declarations[klass].append(name)

    print(f'''{prefix}{name}(''', file=file, end='')
    lead = ',\n' + prefix + ' ' * len(name) + ' '

    for num, arg in enumerate(elem.findall('./arg')):
        argname = arg.get('name')

        if argname is None:
            if PRINT_ERRORS:
                print(f'method {name}: argument {num+1} has no name', file=sys.stderr)
            argname = 'UNNAMED'

        type = arg.get('type')
        if not is_signal:
            direction = arg.get('direction')
            print(f'''{lead if num > 0 else ''}{direction:3} {type} {argname}''', file=file, end='')
        else:
            print(f'''{lead if num > 0 else ''}{type} {argname}''', file=file, end='')

    print(f');', file=file)

ACCESS_MAP = {
    'read' : 'readonly',
    'write' : 'readwrite',
}

def value_ellipsis(type):
    if type == 's':
        return "'...'";
    if type[0] == 'a':
        inner = value_ellipsis(type[1:])
        return f"[{inner}{', ...' if inner != '...' else ''}]";
    return '...'

def print_property(declarations, elem, *, prefix, file):
    name = elem.get('name')
    type = elem.get('type')
    access = elem.get('access')

    declarations['property'].append(name)

    # @org.freedesktop.DBus.Property.EmitsChangedSignal("false")
    # @org.freedesktop.systemd1.Privileged("true")
    # readwrite b EnableWallMessages = false;

    for anno in elem.findall('./annotation'):
        anno_name = anno.get('name')
        anno_value = anno.get('value')
        print(f'''{prefix}@{anno_name}("{anno_value}")''', file=file)

    access = ACCESS_MAP.get(access, access)
    print(f'''{prefix}{access} {type} {name} = {value_ellipsis(type)};''', file=file)

def print_interface(iface, *, prefix, file, print_boring, only_interface, declarations):
    name = iface.get('name')

    is_boring = (name in BORING_INTERFACES or
                 only_interface is not None and name != only_interface)

    if is_boring and print_boring:
        print(f'''{prefix}interface {name} {{ ... }};''', file=file)

    elif not is_boring and not print_boring:
        print(f'''{prefix}interface {name} {{''', file=file)
        prefix2 = prefix + '  '

        for num, elem in enumerate(iface.findall('./method')):
            if num == 0:
                print(f'''{prefix2}methods:''', file=file)
            print_method(declarations, elem, prefix=prefix2 + '  ', file=file)

        for num, elem in enumerate(iface.findall('./signal')):
            if num == 0:
                print(f'''{prefix2}signals:''', file=file)
            print_method(declarations, elem, prefix=prefix2 + '  ', file=file, is_signal=True)

        for num, elem in enumerate(iface.findall('./property')):
            if num == 0:
                print(f'''{prefix2}properties:''', file=file)
            print_property(declarations, elem, prefix=prefix2 + '  ', file=file)

        print(f'''{prefix}}};''', file=file)

def document_has_elem_with_text(document, elem, item_repr):
    predicate = f".//{elem}" # [text() = 'foo'] doesn't seem supported :(
    for loc in document.findall(predicate):
        if loc.text == item_repr:
            return True
    else:
        return False

def check_documented(document, declarations):
    missing = []
    for klass, items in declarations.items():
        for item in items:
            if klass == 'method':
                elem = 'function'
                item_repr = f'{item}()'
            elif klass == 'signal':
                elem = 'function'
                item_repr = item
            elif klass == 'property':
                elem = 'varname'
                item_repr = item
            else:
                assert False, (klass, item)

            if not document_has_elem_with_text(document, elem, item_repr):
                if PRINT_ERRORS:
                    print(f'{klass} {item} is not documented :(')
                missing.append((klass, item))

    return missing

def xml_to_text(destination, xml, *, only_interface=None):
    file = io.StringIO()

    declarations = collections.defaultdict(list)
    interfaces = []

    print(f'''node {destination} {{''', file=file)

    for print_boring in [False, True]:
        for iface in xml.findall('./interface'):
            print_interface(iface, prefix='  ', file=file,
                            print_boring=print_boring,
                            only_interface=only_interface,
                            declarations=declarations)
            name = iface.get('name')
            if not name in BORING_INTERFACES:
                interfaces.append(name)

    print(f'''}};''', file=file)

    return file.getvalue(), declarations, interfaces

def subst_output(document, programlisting):
    try:
        cmd, prefix_lines, footer = find_command(programlisting.text.splitlines())
    except NoCommand:
        return

    only_interface = programlisting.get('interface', None)

    argv = shlex.split(cmd)
    argv += ['--xml']
    print(f'COMMAND: {shlex.join(argv)}')

    object_idx = argv.index('--object-path')
    object_path = argv[object_idx + 1]

    try:
        out = subprocess.check_output(argv, text=True)
    except subprocess.CalledProcessError:
        print('command failed, ignoring', file=sys.stderr)
        return

    xml = etree.fromstring(out, parser=PARSER)

    new_text, declarations, interfaces = xml_to_text(object_path, xml, only_interface=only_interface)

    programlisting.text = '\n'.join(prefix_lines) + '\n' + new_text + footer

    if declarations:
        missing = check_documented(document, declarations)
        parent = programlisting.getparent()

        # delete old comments
        for child in parent:
            if (child.tag == etree.Comment
                and 'Autogenerated' in child.text):
                parent.remove(child)
            if (child.tag == etree.Comment
                and 'not documented' in child.text):
                parent.remove(child)
            if (child.tag == "variablelist"
                and child.attrib.get("generated",False) == "True"):
                parent.remove(child)

        # insert pointer for systemd-directives generation
        the_tail = programlisting.tail #tail is erased by addnext, so save it here.
        prev_element = etree.Comment("Autogenerated cross-references for systemd.directives, do not edit")
        programlisting.addnext(prev_element)
        programlisting.tail = the_tail

        for interface in interfaces:
            variablelist = etree.Element("variablelist")
            variablelist.attrib['class'] = 'dbus-interface'
            variablelist.attrib['generated'] = 'True'
            variablelist.attrib['extra-ref'] = interface

            prev_element.addnext(variablelist)
            prev_element.tail = the_tail
            prev_element = variablelist

        for decl_type,decl_list in declarations.items():
            for declaration in decl_list:
                variablelist = etree.Element("variablelist")
                variablelist.attrib['class'] = 'dbus-'+decl_type
                variablelist.attrib['generated'] = 'True'
                if decl_type == 'method' :
                    variablelist.attrib['extra-ref'] = declaration + '()'
                else:
                    variablelist.attrib['extra-ref'] = declaration

                prev_element.addnext(variablelist)
                prev_element.tail = the_tail
                prev_element = variablelist

        last_element = etree.Comment("End of Autogenerated section")
        prev_element.addnext(last_element)
        prev_element.tail = the_tail
        last_element.tail = the_tail

        # insert comments for undocumented items
        for item in reversed(missing):
            comment = etree.Comment(f'{item[0]} {item[1]} is not documented!')
            comment.tail = programlisting.tail
            parent.insert(parent.index(programlisting) + 1, comment)

def process(page):
    src = open(page).read()
    xml = etree.fromstring(src, parser=PARSER)

    # print('parsing {}'.format(name), file=sys.stderr)
    if xml.tag != 'refentry':
        return

    pls = xml.findall('.//programlisting')
    for pl in pls:
        subst_output(xml, pl)

    out_text = etree.tostring(xml, encoding='unicode')
    # massage format to avoid some lxml whitespace handling idiosyncracies
    # https://bugs.launchpad.net/lxml/+bug/526799
    out_text = (src[:src.find('<refentryinfo')] +
                out_text[out_text.find('<refentryinfo'):] +
                '\n')

    with open(page, 'w') as out:
        out.write(out_text)

if __name__ == '__main__':
    pages = sys.argv[1:]

    for page in pages:
        process(page)
Commit	Line	Data
e5dd26cc ZJS	1	#!/usr/bin/env python3
	2	# SPDX-License-Identifier: LGPL-2.1+
	3
	4	import collections
	5	import sys
	6	import shlex
	7	import subprocess
	8	import io
	9	import pprint
	10	from lxml import etree
	11
	12	PARSER = etree.XMLParser(no_network=True,
	13	remove_comments=False,
	14	strip_cdata=False,
	15	resolve_entities=False)
	16
	17	PRINT_ERRORS = True
	18
	19	class NoCommand(Exception):
	20	pass
	21
	22	def find_command(lines):
	23	acc = []
	24	for num, line in enumerate(lines):
	25	# skip empty leading line
	26	if num == 0 and not line:
	27	continue
	28	cont = line.endswith('\\')
	29	if cont:
	30	line = line[:-1].rstrip()
	31	acc.append(line if not acc else line.lstrip())
	32	if not cont:
	33	break
	34	joined = ' '.join(acc)
	35	if not joined.startswith('$ '):
	36	raise NoCommand
	37	return joined[2:], lines[:num+1] + [''], lines[-1]
	38
	39	BORING_INTERFACES = [
	40	'org.freedesktop.DBus.Peer',
	41	'org.freedesktop.DBus.Introspectable',
	42	'org.freedesktop.DBus.Properties',
	43	]
	44
	45	def print_method(declarations, elem, *, prefix, file, is_signal=False):
	46	name = elem.get('name')
	47	klass = 'signal' if is_signal else 'method'
	48	declarations[klass].append(name)
	49
	50	print(f'''{prefix}{name}(''', file=file, end='')
	51	lead = ',\n' + prefix + ' ' * len(name) + ' '
	52
	53	for num, arg in enumerate(elem.findall('./arg')):
	54	argname = arg.get('name')
	55
	56	if argname is None:
	57	if PRINT_ERRORS:
	58	print(f'method {name}: argument {num+1} has no name', file=sys.stderr)
	59	argname = 'UNNAMED'
	60
	61	type = arg.get('type')
	62	if not is_signal:
	63	direction = arg.get('direction')
	64	print(f'''{lead if num > 0 else ''}{direction:3} {type} {argname}''', file=file, end='')
65	else:
66	print(f'''{lead if num > 0 else ''}{type} {argname}''', file=file, end='')
67
68	print(f');', file=file)
69
70	ACCESS_MAP = {
71	'read' : 'readonly',
72	'write' : 'readwrite',
73	}
74
75	def value_ellipsis(type):
76	if type == 's':
77	return "'...'";
78	if type[0] == 'a':
79	inner = value_ellipsis(type[1:])
80	return f"[{inner}{', ...' if inner != '...' else ''}]";
81	return '...'
82
83	def print_property(declarations, elem, *, prefix, file):
84	name = elem.get('name')
85	type = elem.get('type')
86	access = elem.get('access')
87
88	declarations['property'].append(name)
89
90	# @org.freedesktop.DBus.Property.EmitsChangedSignal("false")
91	# @org.freedesktop.systemd1.Privileged("true")
92	# readwrite b EnableWallMessages = false;
93
94	for anno in elem.findall('./annotation'):
95	anno_name = anno.get('name')
96	anno_value = anno.get('value')
97	print(f'''{prefix}@{anno_name}("{anno_value}")''', file=file)
98
99	access = ACCESS_MAP.get(access, access)
100	print(f'''{prefix}{access} {type} {name} = {value_ellipsis(type)};''', file=file)
101
08fe1b6c	102	def print_interface(iface, *, prefix, file, print_boring, only_interface, declarations):
e5dd26cc ZJS	103	name = iface.get('name')
e5dd26cc ZJS	104
08fe1b6c ZJS	105	is_boring = (name in BORING_INTERFACES or
	106	only_interface is not None and name != only_interface)
	107
e5dd26cc ZJS	108	if is_boring and print_boring:
e5dd26cc ZJS	109	print(f'''{prefix}interface {name} {{ ... }};''', file=file)
08fe1b6c	110
e5dd26cc ZJS	111	elif not is_boring and not print_boring:
	112	print(f'''{prefix}interface {name} {{''', file=file)
	113	prefix2 = prefix + ' '
	114
	115	for num, elem in enumerate(iface.findall('./method')):
	116	if num == 0:
	117	print(f'''{prefix2}methods:''', file=file)
	118	print_method(declarations, elem, prefix=prefix2 + ' ', file=file)
	119
	120	for num, elem in enumerate(iface.findall('./signal')):
	121	if num == 0:
	122	print(f'''{prefix2}signals:''', file=file)
	123	print_method(declarations, elem, prefix=prefix2 + ' ', file=file, is_signal=True)
	124
	125	for num, elem in enumerate(iface.findall('./property')):
	126	if num == 0:
	127	print(f'''{prefix2}properties:''', file=file)
	128	print_property(declarations, elem, prefix=prefix2 + ' ', file=file)
	129
	130	print(f'''{prefix}}};''', file=file)
	131
	132	def document_has_elem_with_text(document, elem, item_repr):
	133	predicate = f".//{elem}" # [text() = 'foo'] doesn't seem supported :(
	134	for loc in document.findall(predicate):
	135	if loc.text == item_repr:
	136	return True
	137	else:
	138	return False
	139
	140	def check_documented(document, declarations):
	141	missing = []
	142	for klass, items in declarations.items():
	143	for item in items:
	144	if klass == 'method':
	145	elem = 'function'
	146	item_repr = f'{item}()'
	147	elif klass == 'signal':
	148	elem = 'function'
	149	item_repr = item
	150	elif klass == 'property':
	151	elem = 'varname'
	152	item_repr = item
	153	else:
	154	assert False, (klass, item)
	155
	156	if not document_has_elem_with_text(document, elem, item_repr):
	157	if PRINT_ERRORS:
	158	print(f'{klass} {item} is not documented :(')
	159	missing.append((klass, item))
	160
	161	return missing
	162
08fe1b6c	163	def xml_to_text(destination, xml, *, only_interface=None):
e5dd26cc ZJS	164	file = io.StringIO()
	165
	166	declarations = collections.defaultdict(list)
f92c8d1c	167	interfaces = []
e5dd26cc ZJS	168
	169	print(f'''node {destination} {{''', file=file)
	170
	171	for print_boring in [False, True]:
	172	for iface in xml.findall('./interface'):
	173	print_interface(iface, prefix=' ', file=file,
	174	print_boring=print_boring,
08fe1b6c	175	only_interface=only_interface,
e5dd26cc	176	declarations=declarations)
f92c8d1c JR	177	name = iface.get('name')
	178	if not name in BORING_INTERFACES:
	179	interfaces.append(name)
e5dd26cc ZJS	180
	181	print(f'''}};''', file=file)
	182
f92c8d1c	183	return file.getvalue(), declarations, interfaces
e5dd26cc ZJS	184
	185	def subst_output(document, programlisting):
	186	try:
	187	cmd, prefix_lines, footer = find_command(programlisting.text.splitlines())
	188	except NoCommand:
	189	return
	190
08fe1b6c ZJS	191	only_interface = programlisting.get('interface', None)
08fe1b6c ZJS	192
e5dd26cc ZJS	193	argv = shlex.split(cmd)
	194	argv += ['--xml']
	195	print(f'COMMAND: {shlex.join(argv)}')
	196
	197	object_idx = argv.index('--object-path')
	198	object_path = argv[object_idx + 1]
	199
	200	try:
	201	out = subprocess.check_output(argv, text=True)
	202	except subprocess.CalledProcessError:
	203	print('command failed, ignoring', file=sys.stderr)
	204	return
	205
	206	xml = etree.fromstring(out, parser=PARSER)
	207
f92c8d1c	208	new_text, declarations, interfaces = xml_to_text(object_path, xml, only_interface=only_interface)
e5dd26cc ZJS	209
	210	programlisting.text = '\n'.join(prefix_lines) + '\n' + new_text + footer
	211
	212	if declarations:
	213	missing = check_documented(document, declarations)
	214	parent = programlisting.getparent()
	215
	216	# delete old comments
	217	for child in parent:
f92c8d1c JR	218	if (child.tag == etree.Comment
	219	and 'Autogenerated' in child.text):
	220	parent.remove(child)
e5dd26cc ZJS	221	if (child.tag == etree.Comment
	222	and 'not documented' in child.text):
	223	parent.remove(child)
f92c8d1c JR	224	if (child.tag == "variablelist"
	225	and child.attrib.get("generated",False) == "True"):
	226	parent.remove(child)
	227
	228	# insert pointer for systemd-directives generation
	229	the_tail = programlisting.tail #tail is erased by addnext, so save it here.
	230	prev_element = etree.Comment("Autogenerated cross-references for systemd.directives, do not edit")
	231	programlisting.addnext(prev_element)
	232	programlisting.tail = the_tail
	233
	234	for interface in interfaces:
	235	variablelist = etree.Element("variablelist")
	236	variablelist.attrib['class'] = 'dbus-interface'
	237	variablelist.attrib['generated'] = 'True'
	238	variablelist.attrib['extra-ref'] = interface
	239
	240	prev_element.addnext(variablelist)
	241	prev_element.tail = the_tail
	242	prev_element = variablelist
	243
	244	for decl_type,decl_list in declarations.items():
	245	for declaration in decl_list:
	246	variablelist = etree.Element("variablelist")
	247	variablelist.attrib['class'] = 'dbus-'+decl_type
	248	variablelist.attrib['generated'] = 'True'
	249	if decl_type == 'method' :
	250	variablelist.attrib['extra-ref'] = declaration + '()'
	251	else:
	252	variablelist.attrib['extra-ref'] = declaration
	253
	254	prev_element.addnext(variablelist)
	255	prev_element.tail = the_tail
	256	prev_element = variablelist
	257
	258	last_element = etree.Comment("End of Autogenerated section")
	259	prev_element.addnext(last_element)
	260	prev_element.tail = the_tail
	261	last_element.tail = the_tail
e5dd26cc ZJS	262
	263	# insert comments for undocumented items
	264	for item in reversed(missing):
	265	comment = etree.Comment(f'{item[0]} {item[1]} is not documented!')
	266	comment.tail = programlisting.tail
	267	parent.insert(parent.index(programlisting) + 1, comment)
	268
	269	def process(page):
	270	src = open(page).read()
	271	xml = etree.fromstring(src, parser=PARSER)
	272
	273	# print('parsing {}'.format(name), file=sys.stderr)
	274	if xml.tag != 'refentry':
	275	return
	276
	277	pls = xml.findall('.//programlisting')
	278	for pl in pls:
	279	subst_output(xml, pl)
	280
	281	out_text = etree.tostring(xml, encoding='unicode')
	282	# massage format to avoid some lxml whitespace handling idiosyncracies
	283	# https://bugs.launchpad.net/lxml/+bug/526799
	284	out_text = (src[:src.find('<refentryinfo')] +
	285	out_text[out_text.find('<refentryinfo'):] +
	286	'\n')
	287
	288	with open(page, 'w') as out:
	289	out.write(out_text)
	290
	291	if __name__ == '__main__':
	292	pages = sys.argv[1:]
	293
	294	for page in pages:
	295	process(page)