Revert "tools/make-directive-index: parallelize"

This reverts commit a2031de849da52aa85b7e4326c0112ed7e5b5672.

The patch itself seems OK, but it exposes a bug in lxml or libxml2-2.9.12 which
was just released. This is being resolved in
https://gitlab.gnome.org/GNOME/libxml2/-/issues/255, but it might be while. So
let's revert this for now to unbreak our CI.

Fixes #19601.

diff --git a/tools/make-directive-index.py b/tools/make-directive-index.py
index 6aaad6700c1098f02306d46d83973033fd83f345..bbdc5572f53a968ae41ecf5bbb8d5c6275628846 100755 (executable)
--- a/tools/make-directive-index.py
+++ b/tools/make-directive-index.py
@@ -4,21 +4,18 @@
 import sys
 import collections
 import re
-import concurrent.futures
 from xml_helper import xml_parse, xml_print, tree
+from copy import deepcopy
 
 COLOPHON = '''\
 This index contains {count} entries in {sections} sections,
 referring to {pages} individual manual pages.
 '''
 
-def _extract_directives(page, names):
-    directive_groups = {name:collections.defaultdict(set) for name in names}
-
+def _extract_directives(directive_groups, formatting, page):
     t = xml_parse(page)
     section = t.find('./refmeta/manvolnum').text
     pagename = t.find('./refmeta/refentrytitle').text
-    formatting = {}
 
     storopt = directive_groups['options']
     for variablelist in t.iterfind('.//variablelist'):
@@ -34,7 +31,7 @@ def _extract_directives(page, names):
                 if text.startswith('-'):
                     # for options, merge options with and without mandatory arg
                     text = text.partition('=')[0]
-                stor[text].add((pagename, section))
+                stor[text].append((pagename, section))
                 if text not in formatting:
                     # use element as formatted display
                     if name.text[-1] in "= '":
@@ -45,7 +42,7 @@ def _extract_directives(page, names):
                     formatting[text] = name
         extra = variablelist.attrib.get('extra-ref')
         if extra:
-            stor[extra].add((pagename, section))
+            stor[extra].append((pagename, section))
             if extra not in formatting:
                 elt = tree.Element("varname")
                 elt.text= extra
@@ -71,13 +68,13 @@ def _extract_directives(page, names):
                         name.text = text
                     if text.endswith('/'):
                         text = text[:-1]
-                    storfile[text].add((pagename, section))
+                    storfile[text].append((pagename, section))
                     if text not in formatting:
                         # use element as formatted display
                         formatting[text] = name
             else:
                 text = ' '.join(name.itertext())
-                storfile[text].add((pagename, section))
+                storfile[text].append((pagename, section))
                 formatting[text] = name
 
     storfile = directive_groups['constants']
@@ -87,7 +84,7 @@ def _extract_directives(page, names):
         name.tail = ''
         if name.text.startswith('('): # a cast, strip it
             name.text = name.text.partition(' ')[2]
-        storfile[name.text].add((pagename, section))
+        storfile[name.text].append((pagename, section))
         formatting[name.text] = name
 
     storfile = directive_groups['specifiers']
@@ -96,30 +93,18 @@ def _extract_directives(page, names):
             continue
         if name.attrib.get('index') == 'false':
             continue
-        storfile[name.text].add((pagename, section))
+        storfile[name.text].append((pagename, section))
         formatting[name.text] = name
     for name in t.iterfind(".//literal[@class='specifiers']"):
-        storfile[name.text].add((pagename, section))
+        storfile[name.text].append((pagename, section))
         formatting[name.text] = name
 
-    # Serialize to allow pickling
-    formatting = {name:xml_print(value) for name, value in formatting.items()}
-
-    return directive_groups, formatting
-
-def extract_directives(arg):
-    page, names = arg
-    try:
-        return _extract_directives(page, names)
-    except Exception:
-        raise ValueError("Failed to process {}".format(page))
-
 def _make_section(template, name, directives, formatting):
     varlist = template.find(".//*[@id='{}']".format(name))
     for varname, manpages in sorted(directives.items()):
         entry = tree.SubElement(varlist, 'varlistentry')
         term = tree.SubElement(entry, 'term')
-        display = tree.fromstring(formatting[varname])
+        display = deepcopy(formatting[varname])
         term.append(display)
 
         para = tree.SubElement(tree.SubElement(entry, 'listitem'), 'para')
@@ -169,26 +154,20 @@ def make_page(template_path, xml_files):
     "Extract directives from xml_files and return XML index tree."
     template = xml_parse(template_path)
     names = [vl.get('id') for vl in template.iterfind('.//variablelist')]
-
-    with concurrent.futures.ProcessPoolExecutor() as pool:
-        args = ((xml_file, names) for xml_file in xml_files)
-        results = list(pool.map(extract_directives, args))
-
-    directive_groups = {name:collections.defaultdict(set) for name in names}
+    directive_groups = {name:collections.defaultdict(list)
+                        for name in names}
     formatting = {}
-    for d_g, f in reversed(results):
-        for group, mapping in d_g.items():
-            for name, value in mapping.items():
-                directive_groups[group][name].update(value)
-
-        formatting.update(f)
+    for page in xml_files:
+        try:
+            _extract_directives(directive_groups, formatting, page)
+        except Exception:
+            raise ValueError("failed to process " + page)
 
     return _make_page(template, directive_groups, formatting)
 
-def main(output, template_path, *xml_files):
-    with open(output, 'wb') as f:
+if __name__ == '__main__':
+    with open(sys.argv[1], 'wb') as f:
+        template_path = sys.argv[2]
+        xml_files = sys.argv[3:]
         xml = make_page(template_path, xml_files)
         f.write(xml_print(xml))
-
-if __name__ == '__main__':
-    main(*sys.argv[1:])