[thirdparty/systemd.git] / tools / update-dbus-docs.py

#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1+

import argparse
import collections
import sys
import os
import shlex
import subprocess
import io
from lxml import etree

PARSER = etree.XMLParser(no_network=True,
                         remove_comments=False,
                         strip_cdata=False,
                         resolve_entities=False)

PRINT_ERRORS = True

class NoCommand(Exception):
    pass

BORING_INTERFACES = [
    'org.freedesktop.DBus.Peer',
    'org.freedesktop.DBus.Introspectable',
    'org.freedesktop.DBus.Properties',
]

def print_method(declarations, elem, *, prefix, file, is_signal=False):
    name = elem.get('name')
    klass = 'signal' if is_signal else 'method'
    declarations[klass].append(name)

    print(f'''{prefix}{name}(''', file=file, end='')
    lead = ',\n' + prefix + ' ' * len(name) + ' '

    for num, arg in enumerate(elem.findall('./arg')):
        argname = arg.get('name')

        if argname is None:
            if PRINT_ERRORS:
                print(f'method {name}: argument {num+1} has no name', file=sys.stderr)
            argname = 'UNNAMED'

        type = arg.get('type')
        if not is_signal:
            direction = arg.get('direction')
            print(f'''{lead if num > 0 else ''}{direction:3} {type} {argname}''', file=file, end='')
        else:
            print(f'''{lead if num > 0 else ''}{type} {argname}''', file=file, end='')

    print(f');', file=file)

ACCESS_MAP = {
    'read' : 'readonly',
    'write' : 'readwrite',
}

def value_ellipsis(type):
    if type == 's':
        return "'...'";
    if type[0] == 'a':
        inner = value_ellipsis(type[1:])
        return f"[{inner}{', ...' if inner != '...' else ''}]";
    return '...'

def print_property(declarations, elem, *, prefix, file):
    name = elem.get('name')
    type = elem.get('type')
    access = elem.get('access')

    declarations['property'].append(name)

    # @org.freedesktop.DBus.Property.EmitsChangedSignal("false")
    # @org.freedesktop.systemd1.Privileged("true")
    # readwrite b EnableWallMessages = false;

    for anno in elem.findall('./annotation'):
        anno_name = anno.get('name')
        anno_value = anno.get('value')
        print(f'''{prefix}@{anno_name}("{anno_value}")''', file=file)

    access = ACCESS_MAP.get(access, access)
    print(f'''{prefix}{access} {type} {name} = {value_ellipsis(type)};''', file=file)

def print_interface(iface, *, prefix, file, print_boring, only_interface, declarations):
    name = iface.get('name')

    is_boring = (name in BORING_INTERFACES or
                 only_interface is not None and name != only_interface)

    if is_boring and print_boring:
        print(f'''{prefix}interface {name} {{ ... }};''', file=file)

    elif not is_boring and not print_boring:
        print(f'''{prefix}interface {name} {{''', file=file)
        prefix2 = prefix + '  '

        for num, elem in enumerate(iface.findall('./method')):
            if num == 0:
                print(f'''{prefix2}methods:''', file=file)
            print_method(declarations, elem, prefix=prefix2 + '  ', file=file)

        for num, elem in enumerate(iface.findall('./signal')):
            if num == 0:
                print(f'''{prefix2}signals:''', file=file)
            print_method(declarations, elem, prefix=prefix2 + '  ', file=file, is_signal=True)

        for num, elem in enumerate(iface.findall('./property')):
            if num == 0:
                print(f'''{prefix2}properties:''', file=file)
            print_property(declarations, elem, prefix=prefix2 + '  ', file=file)

        print(f'''{prefix}}};''', file=file)

def document_has_elem_with_text(document, elem, item_repr):
    predicate = f".//{elem}" # [text() = 'foo'] doesn't seem supported :(
    for loc in document.findall(predicate):
        if loc.text == item_repr:
            return True
    else:
        return False

def check_documented(document, declarations, stats):
    missing = []
    for klass, items in declarations.items():
        stats['total'] += len(items)

        for item in items:
            if klass == 'method':
                elem = 'function'
                item_repr = f'{item}()'
            elif klass == 'signal':
                elem = 'function'
                item_repr = item
            elif klass == 'property':
                elem = 'varname'
                item_repr = item
            else:
                assert False, (klass, item)

            if not document_has_elem_with_text(document, elem, item_repr):
                if PRINT_ERRORS:
                    print(f'{klass} {item} is not documented :(')
                missing.append((klass, item))

    stats['missing'] += len(missing)

    return missing

def xml_to_text(destination, xml, *, only_interface=None):
    file = io.StringIO()

    declarations = collections.defaultdict(list)
    interfaces = []

    print(f'''node {destination} {{''', file=file)

    for print_boring in [False, True]:
        for iface in xml.findall('./interface'):
            print_interface(iface, prefix='  ', file=file,
                            print_boring=print_boring,
                            only_interface=only_interface,
                            declarations=declarations)
            name = iface.get('name')
            if not name in BORING_INTERFACES:
                interfaces.append(name)

    print(f'''}};''', file=file)

    return file.getvalue(), declarations, interfaces

def subst_output(document, programlisting, stats):
    executable = programlisting.get('executable', None)
    if executable is None:
        # Not our thing
        return
    executable = programlisting.get('executable')
    node = programlisting.get('node')
    interface = programlisting.get('interface')

    argv = [f'{opts.build_dir}/{executable}', f'--bus-introspect={interface}']
    print(f'COMMAND: {shlex.join(argv)}')

    try:
        out = subprocess.check_output(argv, text=True)
    except FileNotFoundError:
        print(f'{executable} not found, ignoring', file=sys.stderr)
        return

    xml = etree.fromstring(out, parser=PARSER)

    new_text, declarations, interfaces = xml_to_text(node, xml, only_interface=interface)
    programlisting.text = '\n' + new_text + '    '

    if declarations:
        missing = check_documented(document, declarations, stats)
        parent = programlisting.getparent()

        # delete old comments
        for child in parent:
            if (child.tag == etree.Comment
                and 'Autogenerated' in child.text):
                parent.remove(child)
            if (child.tag == etree.Comment
                and 'not documented' in child.text):
                parent.remove(child)
            if (child.tag == "variablelist"
                and child.attrib.get("generated",False) == "True"):
                parent.remove(child)

        # insert pointer for systemd-directives generation
        the_tail = programlisting.tail #tail is erased by addnext, so save it here.
        prev_element = etree.Comment("Autogenerated cross-references for systemd.directives, do not edit")
        programlisting.addnext(prev_element)
        programlisting.tail = the_tail

        for interface in interfaces:
            variablelist = etree.Element("variablelist")
            variablelist.attrib['class'] = 'dbus-interface'
            variablelist.attrib['generated'] = 'True'
            variablelist.attrib['extra-ref'] = interface

            prev_element.addnext(variablelist)
            prev_element.tail = the_tail
            prev_element = variablelist

        for decl_type,decl_list in declarations.items():
            for declaration in decl_list:
                variablelist = etree.Element("variablelist")
                variablelist.attrib['class'] = 'dbus-'+decl_type
                variablelist.attrib['generated'] = 'True'
                if decl_type == 'method' :
                    variablelist.attrib['extra-ref'] = declaration + '()'
                else:
                    variablelist.attrib['extra-ref'] = declaration

                prev_element.addnext(variablelist)
                prev_element.tail = the_tail
                prev_element = variablelist

        last_element = etree.Comment("End of Autogenerated section")
        prev_element.addnext(last_element)
        prev_element.tail = the_tail
        last_element.tail = the_tail

        # insert comments for undocumented items
        for item in reversed(missing):
            comment = etree.Comment(f'{item[0]} {item[1]} is not documented!')
            comment.tail = programlisting.tail
            parent.insert(parent.index(programlisting) + 1, comment)

def process(page):
    src = open(page).read()
    xml = etree.fromstring(src, parser=PARSER)

    # print('parsing {}'.format(name), file=sys.stderr)
    if xml.tag != 'refentry':
        return

    stats = collections.Counter()

    pls = xml.findall('.//programlisting')
    for pl in pls:
        subst_output(xml, pl, stats)

    out_text = etree.tostring(xml, encoding='unicode')
    # massage format to avoid some lxml whitespace handling idiosyncrasies
    # https://bugs.launchpad.net/lxml/+bug/526799
    out_text = (src[:src.find('<refentryinfo')] +
                out_text[out_text.find('<refentryinfo'):] +
                '\n')

    with open(page, 'w') as out:
        out.write(out_text)

    return stats

def parse_args():
    p = argparse.ArgumentParser()
    p.add_argument('--build-dir', default='build')
    p.add_argument('pages', nargs='+')
    return p.parse_args()

if __name__ == '__main__':
    opts = parse_args()

    if not os.path.exists(f'{opts.build_dir}/systemd'):
        exit(f"{opts.build_dir}/systemd doesn't exist. Use --build-dir=.")

    stats = {page.split('/')[-1] : process(page) for page in opts.pages}

    # Let's print all statistics at the end
    mlen = max(len(page) for page in stats)
    total = 'total', sum(stats.values(), start=collections.Counter())
    for page, counts in sorted(stats.items()) + [total]:
        m = counts['missing']
        t = counts['total']
        p = page + ':'
        print(f'{p:{mlen + 1}} {t - m}/{t}')
Commit	Line	Data
e5dd26cc ZJS	1	#!/usr/bin/env python3
	2	# SPDX-License-Identifier: LGPL-2.1+
	3
0f5cea02	4	import argparse
e5dd26cc ZJS	5	import collections
e5dd26cc ZJS	6	import sys
c351d568	7	import os
e5dd26cc ZJS	8	import shlex
	9	import subprocess
	10	import io
e5dd26cc ZJS	11	from lxml import etree
	12
	13	PARSER = etree.XMLParser(no_network=True,
	14	remove_comments=False,
	15	strip_cdata=False,
	16	resolve_entities=False)
	17
	18	PRINT_ERRORS = True
	19
	20	class NoCommand(Exception):
	21	pass
	22
e5dd26cc ZJS	23	BORING_INTERFACES = [
	24	'org.freedesktop.DBus.Peer',
	25	'org.freedesktop.DBus.Introspectable',
	26	'org.freedesktop.DBus.Properties',
	27	]
	28
	29	def print_method(declarations, elem, *, prefix, file, is_signal=False):
	30	name = elem.get('name')
	31	klass = 'signal' if is_signal else 'method'
	32	declarations[klass].append(name)
	33
	34	print(f'''{prefix}{name}(''', file=file, end='')
	35	lead = ',\n' + prefix + ' ' * len(name) + ' '
	36
	37	for num, arg in enumerate(elem.findall('./arg')):
	38	argname = arg.get('name')
	39
	40	if argname is None:
	41	if PRINT_ERRORS:
	42	print(f'method {name}: argument {num+1} has no name', file=sys.stderr)
	43	argname = 'UNNAMED'
	44
	45	type = arg.get('type')
	46	if not is_signal:
	47	direction = arg.get('direction')
	48	print(f'''{lead if num > 0 else ''}{direction:3} {type} {argname}''', file=file, end='')
	49	else:
	50	print(f'''{lead if num > 0 else ''}{type} {argname}''', file=file, end='')
	51
	52	print(f');', file=file)
	53
	54	ACCESS_MAP = {
	55	'read' : 'readonly',
	56	'write' : 'readwrite',
	57	}
	58
	59	def value_ellipsis(type):
	60	if type == 's':
	61	return "'...'";
	62	if type[0] == 'a':
	63	inner = value_ellipsis(type[1:])
	64	return f"[{inner}{', ...' if inner != '...' else ''}]";
	65	return '...'
	66
	67	def print_property(declarations, elem, *, prefix, file):
	68	name = elem.get('name')
	69	type = elem.get('type')
	70	access = elem.get('access')
	71
	72	declarations['property'].append(name)
	73
	74	# @org.freedesktop.DBus.Property.EmitsChangedSignal("false")
	75	# @org.freedesktop.systemd1.Privileged("true")
	76	# readwrite b EnableWallMessages = false;
	77
	78	for anno in elem.findall('./annotation'):
	79	anno_name = anno.get('name')
	80	anno_value = anno.get('value')
	81	print(f'''{prefix}@{anno_name}("{anno_value}")''', file=file)
	82
	83	access = ACCESS_MAP.get(access, access)
	84	print(f'''{prefix}{access} {type} {name} = {value_ellipsis(type)};''', file=file)
	85
08fe1b6c	86	def print_interface(iface, *, prefix, file, print_boring, only_interface, declarations):
e5dd26cc ZJS	87	name = iface.get('name')
e5dd26cc ZJS	88
08fe1b6c ZJS	89	is_boring = (name in BORING_INTERFACES or
	90	only_interface is not None and name != only_interface)
	91
e5dd26cc ZJS	92	if is_boring and print_boring:
e5dd26cc ZJS	93	print(f'''{prefix}interface {name} {{ ... }};''', file=file)
08fe1b6c	94
e5dd26cc ZJS	95	elif not is_boring and not print_boring:
	96	print(f'''{prefix}interface {name} {{''', file=file)
	97	prefix2 = prefix + ' '
	98
	99	for num, elem in enumerate(iface.findall('./method')):
	100	if num == 0:
	101	print(f'''{prefix2}methods:''', file=file)
	102	print_method(declarations, elem, prefix=prefix2 + ' ', file=file)
	103
	104	for num, elem in enumerate(iface.findall('./signal')):
	105	if num == 0:
	106	print(f'''{prefix2}signals:''', file=file)
	107	print_method(declarations, elem, prefix=prefix2 + ' ', file=file, is_signal=True)
	108
	109	for num, elem in enumerate(iface.findall('./property')):
	110	if num == 0:
	111	print(f'''{prefix2}properties:''', file=file)
	112	print_property(declarations, elem, prefix=prefix2 + ' ', file=file)
	113
	114	print(f'''{prefix}}};''', file=file)
	115
	116	def document_has_elem_with_text(document, elem, item_repr):
	117	predicate = f".//{elem}" # [text() = 'foo'] doesn't seem supported :(
	118	for loc in document.findall(predicate):
	119	if loc.text == item_repr:
	120	return True
	121	else:
	122	return False
	123
af4c7dc2	124	def check_documented(document, declarations, stats):
e5dd26cc ZJS	125	missing = []
e5dd26cc ZJS	126	for klass, items in declarations.items():
af4c7dc2 ZJS	127	stats['total'] += len(items)
af4c7dc2 ZJS	128
e5dd26cc ZJS	129	for item in items:
	130	if klass == 'method':
	131	elem = 'function'
	132	item_repr = f'{item}()'
	133	elif klass == 'signal':
	134	elem = 'function'
	135	item_repr = item
	136	elif klass == 'property':
	137	elem = 'varname'
	138	item_repr = item
	139	else:
	140	assert False, (klass, item)
	141
	142	if not document_has_elem_with_text(document, elem, item_repr):
	143	if PRINT_ERRORS:
	144	print(f'{klass} {item} is not documented :(')
	145	missing.append((klass, item))
	146
af4c7dc2 ZJS	147	stats['missing'] += len(missing)
af4c7dc2 ZJS	148
e5dd26cc ZJS	149	return missing
e5dd26cc ZJS	150
08fe1b6c	151	def xml_to_text(destination, xml, *, only_interface=None):
e5dd26cc ZJS	152	file = io.StringIO()
	153
	154	declarations = collections.defaultdict(list)
f92c8d1c	155	interfaces = []
e5dd26cc ZJS	156
	157	print(f'''node {destination} {{''', file=file)
	158
	159	for print_boring in [False, True]:
	160	for iface in xml.findall('./interface'):
	161	print_interface(iface, prefix=' ', file=file,
	162	print_boring=print_boring,
08fe1b6c	163	only_interface=only_interface,
e5dd26cc	164	declarations=declarations)
f92c8d1c JR	165	name = iface.get('name')
	166	if not name in BORING_INTERFACES:
	167	interfaces.append(name)
e5dd26cc ZJS	168
	169	print(f'''}};''', file=file)
	170
f92c8d1c	171	return file.getvalue(), declarations, interfaces
e5dd26cc	172
af4c7dc2	173	def subst_output(document, programlisting, stats):
c351d568 ZJS	174	executable = programlisting.get('executable', None)
	175	if executable is None:
	176	# Not our thing
e5dd26cc	177	return
c351d568 ZJS	178	executable = programlisting.get('executable')
	179	node = programlisting.get('node')
	180	interface = programlisting.get('interface')
e5dd26cc	181
0f5cea02	182	argv = [f'{opts.build_dir}/{executable}', f'--bus-introspect={interface}']
e5dd26cc ZJS	183	print(f'COMMAND: {shlex.join(argv)}')
e5dd26cc ZJS	184
e5dd26cc ZJS	185	try:
e5dd26cc ZJS	186	out = subprocess.check_output(argv, text=True)
c351d568 ZJS	187	except FileNotFoundError:
c351d568 ZJS	188	print(f'{executable} not found, ignoring', file=sys.stderr)
e5dd26cc ZJS	189	return
	190
	191	xml = etree.fromstring(out, parser=PARSER)
	192
c351d568 ZJS	193	new_text, declarations, interfaces = xml_to_text(node, xml, only_interface=interface)
c351d568 ZJS	194	programlisting.text = '\n' + new_text + ' '
e5dd26cc ZJS	195
e5dd26cc ZJS	196	if declarations:
af4c7dc2	197	missing = check_documented(document, declarations, stats)
e5dd26cc ZJS	198	parent = programlisting.getparent()
	199
	200	# delete old comments
	201	for child in parent:
f92c8d1c JR	202	if (child.tag == etree.Comment
	203	and 'Autogenerated' in child.text):
	204	parent.remove(child)
e5dd26cc ZJS	205	if (child.tag == etree.Comment
	206	and 'not documented' in child.text):
	207	parent.remove(child)
f92c8d1c JR	208	if (child.tag == "variablelist"
	209	and child.attrib.get("generated",False) == "True"):
	210	parent.remove(child)
	211
	212	# insert pointer for systemd-directives generation
	213	the_tail = programlisting.tail #tail is erased by addnext, so save it here.
	214	prev_element = etree.Comment("Autogenerated cross-references for systemd.directives, do not edit")
	215	programlisting.addnext(prev_element)
	216	programlisting.tail = the_tail
	217
	218	for interface in interfaces:
	219	variablelist = etree.Element("variablelist")
	220	variablelist.attrib['class'] = 'dbus-interface'
	221	variablelist.attrib['generated'] = 'True'
	222	variablelist.attrib['extra-ref'] = interface
	223
	224	prev_element.addnext(variablelist)
	225	prev_element.tail = the_tail
	226	prev_element = variablelist
	227
	228	for decl_type,decl_list in declarations.items():
	229	for declaration in decl_list:
	230	variablelist = etree.Element("variablelist")
	231	variablelist.attrib['class'] = 'dbus-'+decl_type
	232	variablelist.attrib['generated'] = 'True'
	233	if decl_type == 'method' :
	234	variablelist.attrib['extra-ref'] = declaration + '()'
	235	else:
	236	variablelist.attrib['extra-ref'] = declaration
	237
	238	prev_element.addnext(variablelist)
	239	prev_element.tail = the_tail
	240	prev_element = variablelist
	241
	242	last_element = etree.Comment("End of Autogenerated section")
	243	prev_element.addnext(last_element)
	244	prev_element.tail = the_tail
	245	last_element.tail = the_tail
e5dd26cc ZJS	246
	247	# insert comments for undocumented items
	248	for item in reversed(missing):
	249	comment = etree.Comment(f'{item[0]} {item[1]} is not documented!')
	250	comment.tail = programlisting.tail
	251	parent.insert(parent.index(programlisting) + 1, comment)
	252
	253	def process(page):
	254	src = open(page).read()
	255	xml = etree.fromstring(src, parser=PARSER)
	256
	257	# print('parsing {}'.format(name), file=sys.stderr)
	258	if xml.tag != 'refentry':
	259	return
	260
af4c7dc2 ZJS	261	stats = collections.Counter()
af4c7dc2 ZJS	262
e5dd26cc ZJS	263	pls = xml.findall('.//programlisting')
e5dd26cc ZJS	264	for pl in pls:
af4c7dc2	265	subst_output(xml, pl, stats)
e5dd26cc ZJS	266
e5dd26cc ZJS	267	out_text = etree.tostring(xml, encoding='unicode')
86b52a39	268	# massage format to avoid some lxml whitespace handling idiosyncrasies
e5dd26cc ZJS	269	# https://bugs.launchpad.net/lxml/+bug/526799
	270	out_text = (src[:src.find('<refentryinfo')] +
	271	out_text[out_text.find('<refentryinfo'):] +
	272	'\n')
	273
	274	with open(page, 'w') as out:
	275	out.write(out_text)
	276
af4c7dc2 ZJS	277	return stats
af4c7dc2 ZJS	278
0f5cea02 ZJS	279	def parse_args():
	280	p = argparse.ArgumentParser()
	281	p.add_argument('--build-dir', default='build')
	282	p.add_argument('pages', nargs='+')
	283	return p.parse_args()
e5dd26cc	284
0f5cea02 ZJS	285	if __name__ == '__main__':
0f5cea02 ZJS	286	opts = parse_args()
c351d568	287
0f5cea02 ZJS	288	if not os.path.exists(f'{opts.build_dir}/systemd'):
0f5cea02 ZJS	289	exit(f"{opts.build_dir}/systemd doesn't exist. Use --build-dir=.")
c351d568	290
0f5cea02	291	stats = {page.split('/')[-1] : process(page) for page in opts.pages}
af4c7dc2 ZJS	292
	293	# Let's print all statistics at the end
	294	mlen = max(len(page) for page in stats)
	295	total = 'total', sum(stats.values(), start=collections.Counter())
	296	for page, counts in sorted(stats.items()) + [total]:
	297	m = counts['missing']
	298	t = counts['total']
	299	p = page + ':'
	300	print(f'{p:{mlen + 1}} {t - m}/{t}')