tools/make-directive-index.py

   1 #!/usr/bin/env python3
   2 # SPDX-License-Identifier: LGPL-2.1+
   3
   4 import sys
   5 import collections
   6 import re
   7 from xml_helper import xml_parse, xml_print, tree
   8 from copy import deepcopy
   9
  10 TEMPLATE = '''\
  11 <refentry id="systemd.directives" conditional="HAVE_PYTHON">
  12
  13         <refentryinfo>
  14                 <title>systemd.directives</title>
  15                 <productname>systemd</productname>
  16         </refentryinfo>
  17
  18         <refmeta>
  19                 <refentrytitle>systemd.directives</refentrytitle>
  20                 <manvolnum>7</manvolnum>
  21         </refmeta>
  22
  23         <refnamediv>
  24                 <refname>systemd.directives</refname>
  25                 <refpurpose>Index of configuration directives</refpurpose>
  26         </refnamediv>
  27
  28         <refsect1>
  29                 <title>Unit directives</title>
  30
  31                 <para>Directives for configuring units, used in unit
  32                 files.</para>
  33
  34                 <variablelist id='unit-directives' />
  35         </refsect1>
  36
  37         <refsect1>
  38                 <title>Options on the kernel command line</title>
  39
  40                 <para>Kernel boot options for configuring the behaviour of the
  41                 systemd process.</para>
  42
  43                 <variablelist id='kernel-commandline-options' />
  44         </refsect1>
  45
  46         <refsect1>
  47                 <title>Environment variables</title>
  48
  49                 <para>Environment variables understood by the systemd manager
  50                 and other programs and environment variable-compatible settings.</para>
  51
  52                 <variablelist id='environment-variables' />
  53         </refsect1>
  54
  55         <refsect1>
  56                 <title>EFI variables</title>
  57
  58                 <para>EFI variables understood by
  59                 <citerefentry><refentrytitle>systemd-boot</refentrytitle><manvolnum>7</manvolnum></citerefentry>
  60                 and other programs.</para>
  61
  62                 <variablelist id='efi-variables' />
  63         </refsect1>
  64
  65         <refsect1>
  66                 <title>UDEV directives</title>
  67
  68                 <para>Directives for configuring systemd units through the
  69                 udev database.</para>
  70
  71                 <variablelist id='udev-directives' />
  72         </refsect1>
  73
  74         <refsect1>
  75                 <title>Network directives</title>
  76
  77                 <para>Directives for configuring network links through the
  78                 net-setup-link udev builtin and networks through
  79                 systemd-networkd.</para>
  80
  81                 <variablelist id='network-directives' />
  82         </refsect1>
  83
  84         <refsect1>
  85                 <title>Journal fields</title>
  86
  87                 <para>Fields in the journal events with a well known meaning.</para>
  88
  89                 <variablelist id='journal-directives' />
  90         </refsect1>
  91
  92         <refsect1>
  93                 <title>PAM configuration directives</title>
  94
  95                 <para>Directives for configuring PAM behaviour.</para>
  96
  97                 <variablelist id='pam-directives' />
  98         </refsect1>
  99
 100         <refsect1>
 101                 <title><filename>/etc/crypttab</filename> and
 102                 <filename>/etc/fstab</filename> options</title>
 103
 104                 <para>Options which influence mounted filesystems and
 105                 encrypted volumes.</para>
 106
 107                 <variablelist id='fstab-options' />
 108         </refsect1>
 109
 110         <refsect1>
 111                 <title><citerefentry><refentrytitle>systemd.nspawn</refentrytitle><manvolnum>5</manvolnum></citerefentry>
 112                 directives</title>
 113
 114                 <para>Directives for configuring systemd-nspawn containers.</para>
 115
 116                 <variablelist id='nspawn-directives' />
 117         </refsect1>
 118
 119         <refsect1>
 120                 <title>Program configuration options</title>
 121
 122                 <para>Directives for configuring the behaviour of the
 123                 systemd process and other tools through configuration files.</para>
 124
 125                 <variablelist id='config-directives' />
 126         </refsect1>
 127
 128         <refsect1>
 129                 <title>Command line options</title>
 130
 131                 <para>Command-line options accepted by programs in the
 132                 systemd suite.</para>
 133
 134                 <variablelist id='options' />
 135         </refsect1>
 136
 137         <refsect1>
 138                 <title>Constants</title>
 139
 140                 <para>Various constant used and/or defined by systemd.</para>
 141
 142                 <variablelist id='constants' />
 143         </refsect1>
 144
 145         <refsect1>
 146                 <title>Miscellaneous options and directives</title>
 147
 148                 <para>Other configuration elements which don't fit in
 149                 any of the above groups.</para>
 150
 151                 <variablelist id='miscellaneous' />
 152         </refsect1>
 153
 154         <refsect1>
 155                 <title>Files and directories</title>
 156
 157                 <para>Paths and file names referred to in the
 158                 documentation.</para>
 159
 160                 <variablelist id='filenames' />
 161         </refsect1>
 162
 163         <refsect1>
 164                 <title>D-Bus interfaces</title>
 165
 166                 <para>Interfaces exposed over D-Bus.</para>
 167
 168                 <variablelist id='dbus-interface' />
 169         </refsect1>
 170
 171         <refsect1>
 172                 <title>D-Bus methods</title>
 173
 174                 <para>Methods exposed in the D-Bus interface.</para>
 175
 176                 <variablelist id='dbus-method' />
 177         </refsect1>
 178
 179         <refsect1>
 180                 <title>D-Bus properties</title>
 181
 182                 <para>Properties exposed in the D-Bus interface.</para>
 183
 184                 <variablelist id='dbus-property' />
 185         </refsect1>
 186
 187         <refsect1>
 188                 <title>D-Bus signals</title>
 189
 190                 <para>Signals emitted in the D-Bus interface.</para>
 191
 192                 <variablelist id='dbus-signal' />
 193         </refsect1>
 194
 195         <refsect1>
 196                 <title>Colophon</title>
 197                 <para id='colophon' />
 198         </refsect1>
 199 </refentry>
 200 '''
 201
 202 COLOPHON = '''\
 203 This index contains {count} entries in {sections} sections,
 204 referring to {pages} individual manual pages.
 205 '''
 206
 207 def _extract_directives(directive_groups, formatting, page):
 208     t = xml_parse(page)
 209     section = t.find('./refmeta/manvolnum').text
 210     pagename = t.find('./refmeta/refentrytitle').text
 211
 212     storopt = directive_groups['options']
 213     for variablelist in t.iterfind('.//variablelist'):
 214         klass = variablelist.attrib.get('class')
 215         searchpath = variablelist.attrib.get('xpath','./varlistentry/term/varname')
 216         storvar = directive_groups[klass or 'miscellaneous']
 217         # <option>s go in OPTIONS, unless class is specified
 218         for xpath, stor in ((searchpath, storvar),
 219                             ('./varlistentry/term/option',
 220                              storvar if klass else storopt)):
 221             for name in variablelist.iterfind(xpath):
 222                 text = re.sub(r'([= ]).*', r'\1', name.text).rstrip()
 223                 if text.startswith('-'):
 224                     # for options, merge options with and without mandatory arg
 225                     text = text.partition('=')[0]
 226                 stor[text].append((pagename, section))
 227                 if text not in formatting:
 228                     # use element as formatted display
 229                     if name.text[-1] in "= '":
 230                         name.clear()
 231                     else:
 232                         name.tail = ''
 233                     name.text = text
 234                     formatting[text] = name
 235         extra = variablelist.attrib.get('extra-ref')
 236         if extra:
 237             stor[extra].append((pagename, section))
 238             if extra not in formatting:
 239                 elt = tree.Element("varname")
 240                 elt.text= extra
 241                 formatting[extra] = elt
 242
 243     storfile = directive_groups['filenames']
 244     for xpath, absolute_only in (('.//refsynopsisdiv//filename', False),
 245                                  ('.//refsynopsisdiv//command', False),
 246                                  ('.//filename', True)):
 247         for name in t.iterfind(xpath):
 248             if absolute_only and not (name.text and name.text.startswith('/')):
 249                 continue
 250             if name.attrib.get('index') == 'false':
 251                 continue
 252             name.tail = ''
 253             if name.text:
 254                 if name.text.endswith('*'):
 255                     name.text = name.text[:-1]
 256                 if not name.text.startswith('.'):
 257                     text = name.text.partition(' ')[0]
 258                     if text != name.text:
 259                         name.clear()
 260                         name.text = text
 261                     if text.endswith('/'):
 262                         text = text[:-1]
 263                     storfile[text].append((pagename, section))
 264                     if text not in formatting:
 265                         # use element as formatted display
 266                         formatting[text] = name
 267             else:
 268                 text = ' '.join(name.itertext())
 269                 storfile[text].append((pagename, section))
 270                 formatting[text] = name
 271
 272     storfile = directive_groups['constants']
 273     for name in t.iterfind('.//constant'):
 274         if name.attrib.get('index') == 'false':
 275             continue
 276         name.tail = ''
 277         if name.text.startswith('('): # a cast, strip it
 278             name.text = name.text.partition(' ')[2]
 279         storfile[name.text].append((pagename, section))
 280         formatting[name.text] = name
 281
 282 def _make_section(template, name, directives, formatting):
 283     varlist = template.find(".//*[@id='{}']".format(name))
 284     for varname, manpages in sorted(directives.items()):
 285         entry = tree.SubElement(varlist, 'varlistentry')
 286         term = tree.SubElement(entry, 'term')
 287         display = deepcopy(formatting[varname])
 288         term.append(display)
 289
 290         para = tree.SubElement(tree.SubElement(entry, 'listitem'), 'para')
 291
 292         b = None
 293         for manpage, manvolume in sorted(set(manpages)):
 294             if b is not None:
 295                 b.tail = ', '
 296             b = tree.SubElement(para, 'citerefentry')
 297             c = tree.SubElement(b, 'refentrytitle')
 298             c.text = manpage
 299             c.attrib['target'] = varname
 300             d = tree.SubElement(b, 'manvolnum')
 301             d.text = manvolume
 302         entry.tail = '\n\n'
 303
 304 def _make_colophon(template, groups):
 305     count = 0
 306     pages = set()
 307     for group in groups:
 308         count += len(group)
 309         for pagelist in group.values():
 310             pages |= set(pagelist)
 311
 312     para = template.find(".//para[@id='colophon']")
 313     para.text = COLOPHON.format(count=count,
 314                                 sections=len(groups),
 315                                 pages=len(pages))
 316
 317 def _make_page(template, directive_groups, formatting):
 318     """Create an XML tree from directive_groups.
 319
 320     directive_groups = {
 321        'class': {'variable': [('manpage', 'manvolume'), ...],
 322                  'variable2': ...},
 323        ...
 324     }
 325     """
 326     for name, directives in directive_groups.items():
 327         _make_section(template, name, directives, formatting)
 328
 329     _make_colophon(template, directive_groups.values())
 330
 331     return template
 332
 333 def make_page(*xml_files):
 334     "Extract directives from xml_files and return XML index tree."
 335     template = tree.fromstring(TEMPLATE)
 336     names = [vl.get('id') for vl in template.iterfind('.//variablelist')]
 337     directive_groups = {name:collections.defaultdict(list)
 338                         for name in names}
 339     formatting = {}
 340     for page in xml_files:
 341         try:
 342             _extract_directives(directive_groups, formatting, page)
 343         except Exception:
 344             raise ValueError("failed to process " + page)
 345
 346     return _make_page(template, directive_groups, formatting)
 347
 348 if __name__ == '__main__':
 349     with open(sys.argv[1], 'wb') as f:
 350         f.write(xml_print(make_page(*sys.argv[2:])))