scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2018 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 """
  36
  37 import argparse
  38 import datetime
  39 import email.mime.text
  40 import email.utils
  41 import json
  42 import os
  43 import re
  44 import shutil
  45 import smtplib
  46 import stat
  47 import subprocess
  48 import sys
  49 import time
  50 import urllib.request
  51
  52 try:
  53     os.cpu_count
  54 except:
  55     import multiprocessing
  56     os.cpu_count = lambda: multiprocessing.cpu_count()
  57
  58 try:
  59     re.fullmatch
  60 except:
  61     re.fullmatch = lambda p,s,f=0: re.match(p+"\\Z",s,f)
  62
  63 try:
  64     subprocess.run
  65 except:
  66     class _CompletedProcess:
  67         def __init__(self, args, returncode, stdout=None, stderr=None):
  68             self.args = args
  69             self.returncode = returncode
  70             self.stdout = stdout
  71             self.stderr = stderr
  72
  73     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  74         assert(timeout is None)
  75         with subprocess.Popen(*popenargs, **kwargs) as process:
  76             try:
  77                 stdout, stderr = process.communicate(input)
  78             except:
  79                 process.kill()
  80                 process.wait()
  81                 raise
  82             returncode = process.poll()
  83             if check and returncode:
  84                 raise subprocess.CalledProcessError(returncode, popenargs)
  85         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  86
  87     subprocess.run = _run
  88
  89
  90 class Context(object):
  91     """The global state associated with builds in a given directory."""
  92
  93     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  94                  action):
  95         """Initialize the context."""
  96         self.topdir = topdir
  97         self.parallelism = parallelism
  98         self.keep = keep
  99         self.replace_sources = replace_sources
 100         self.strip = strip
 101         self.srcdir = os.path.join(topdir, 'src')
 102         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 103         self.build_state_json = os.path.join(topdir, 'build-state.json')
 104         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 105         self.installdir = os.path.join(topdir, 'install')
 106         self.host_libraries_installdir = os.path.join(self.installdir,
 107                                                       'host-libraries')
 108         self.builddir = os.path.join(topdir, 'build')
 109         self.logsdir = os.path.join(topdir, 'logs')
 110         self.logsdir_old = os.path.join(topdir, 'logs-old')
 111         self.makefile = os.path.join(self.builddir, 'Makefile')
 112         self.wrapper = os.path.join(self.builddir, 'wrapper')
 113         self.save_logs = os.path.join(self.builddir, 'save-logs')
 114         self.script_text = self.get_script_text()
 115         if action != 'checkout':
 116             self.build_triplet = self.get_build_triplet()
 117             self.glibc_version = self.get_glibc_version()
 118         self.configs = {}
 119         self.glibc_configs = {}
 120         self.makefile_pieces = ['.PHONY: all\n']
 121         self.add_all_configs()
 122         self.load_versions_json()
 123         self.load_build_state_json()
 124         self.status_log_list = []
 125         self.email_warning = False
 126
 127     def get_script_text(self):
 128         """Return the text of this script."""
 129         with open(sys.argv[0], 'r') as f:
 130             return f.read()
 131
 132     def exec_self(self):
 133         """Re-execute this script with the same arguments."""
 134         sys.stdout.flush()
 135         os.execv(sys.executable, [sys.executable] + sys.argv)
 136
 137     def get_build_triplet(self):
 138         """Determine the build triplet with config.guess."""
 139         config_guess = os.path.join(self.component_srcdir('gcc'),
 140                                     'config.guess')
 141         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 142                                 check=True, universal_newlines=True).stdout
 143         return cg_out.rstrip()
 144
 145     def get_glibc_version(self):
 146         """Determine the glibc version number (major.minor)."""
 147         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 148         with open(version_h, 'r') as f:
 149             lines = f.readlines()
 150         starttext = '#define VERSION "'
 151         for l in lines:
 152             if l.startswith(starttext):
 153                 l = l[len(starttext):]
 154                 l = l.rstrip('"\n')
 155                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 156                 return '%s.%s' % m.group(1, 2)
 157         print('error: could not determine glibc version')
 158         exit(1)
 159
 160     def add_all_configs(self):
 161         """Add all known glibc build configurations."""
 162         self.add_config(arch='aarch64',
 163                         os_name='linux-gnu',
 164                         extra_glibcs=[{'variant': 'disable-multi-arch',
 165                                        'cfg': ['--disable-multi-arch']}])
 166         self.add_config(arch='aarch64_be',
 167                         os_name='linux-gnu')
 168         self.add_config(arch='alpha',
 169                         os_name='linux-gnu')
 170         self.add_config(arch='arm',
 171                         os_name='linux-gnueabi')
 172         self.add_config(arch='armeb',
 173                         os_name='linux-gnueabi')
 174         self.add_config(arch='armeb',
 175                         os_name='linux-gnueabi',
 176                         variant='be8',
 177                         gcc_cfg=['--with-arch=armv7-a'])
 178         self.add_config(arch='arm',
 179                         os_name='linux-gnueabihf',
 180                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 181                         extra_glibcs=[{'variant': 'v7a',
 182                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 183                                       {'variant': 'v7a-disable-multi-arch',
 184                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 185                                        'cfg': ['--disable-multi-arch']}])
 186         self.add_config(arch='armeb',
 187                         os_name='linux-gnueabihf',
 188                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 189         self.add_config(arch='armeb',
 190                         os_name='linux-gnueabihf',
 191                         variant='be8',
 192                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 193                                  '--with-fpu=vfpv3'])
 194         self.add_config(arch='hppa',
 195                         os_name='linux-gnu')
 196         self.add_config(arch='i686',
 197                         os_name='gnu')
 198         self.add_config(arch='ia64',
 199                         os_name='linux-gnu',
 200                         first_gcc_cfg=['--with-system-libunwind'])
 201         self.add_config(arch='m68k',
 202                         os_name='linux-gnu',
 203                         gcc_cfg=['--disable-multilib'])
 204         self.add_config(arch='m68k',
 205                         os_name='linux-gnu',
 206                         variant='coldfire',
 207                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 208         self.add_config(arch='m68k',
 209                         os_name='linux-gnu',
 210                         variant='coldfire-soft',
 211                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 212                                  '--disable-multilib'])
 213         self.add_config(arch='microblaze',
 214                         os_name='linux-gnu',
 215                         gcc_cfg=['--disable-multilib'])
 216         self.add_config(arch='microblazeel',
 217                         os_name='linux-gnu',
 218                         gcc_cfg=['--disable-multilib'])
 219         self.add_config(arch='mips64',
 220                         os_name='linux-gnu',
 221                         gcc_cfg=['--with-mips-plt'],
 222                         glibcs=[{'variant': 'n32'},
 223                                 {'arch': 'mips',
 224                                  'ccopts': '-mabi=32'},
 225                                 {'variant': 'n64',
 226                                  'ccopts': '-mabi=64'}])
 227         self.add_config(arch='mips64',
 228                         os_name='linux-gnu',
 229                         variant='soft',
 230                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 231                         glibcs=[{'variant': 'n32-soft'},
 232                                 {'variant': 'soft',
 233                                  'arch': 'mips',
 234                                  'ccopts': '-mabi=32'},
 235                                 {'variant': 'n64-soft',
 236                                  'ccopts': '-mabi=64'}])
 237         self.add_config(arch='mips64',
 238                         os_name='linux-gnu',
 239                         variant='nan2008',
 240                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 241                                  '--with-arch-64=mips64r2',
 242                                  '--with-arch-32=mips32r2'],
 243                         glibcs=[{'variant': 'n32-nan2008'},
 244                                 {'variant': 'nan2008',
 245                                  'arch': 'mips',
 246                                  'ccopts': '-mabi=32'},
 247                                 {'variant': 'n64-nan2008',
 248                                  'ccopts': '-mabi=64'}])
 249         self.add_config(arch='mips64',
 250                         os_name='linux-gnu',
 251                         variant='nan2008-soft',
 252                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 253                                  '--with-arch-64=mips64r2',
 254                                  '--with-arch-32=mips32r2',
 255                                  '--with-float=soft'],
 256                         glibcs=[{'variant': 'n32-nan2008-soft'},
 257                                 {'variant': 'nan2008-soft',
 258                                  'arch': 'mips',
 259                                  'ccopts': '-mabi=32'},
 260                                 {'variant': 'n64-nan2008-soft',
 261                                  'ccopts': '-mabi=64'}])
 262         self.add_config(arch='mips64el',
 263                         os_name='linux-gnu',
 264                         gcc_cfg=['--with-mips-plt'],
 265                         glibcs=[{'variant': 'n32'},
 266                                 {'arch': 'mipsel',
 267                                  'ccopts': '-mabi=32'},
 268                                 {'variant': 'n64',
 269                                  'ccopts': '-mabi=64'}])
 270         self.add_config(arch='mips64el',
 271                         os_name='linux-gnu',
 272                         variant='soft',
 273                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 274                         glibcs=[{'variant': 'n32-soft'},
 275                                 {'variant': 'soft',
 276                                  'arch': 'mipsel',
 277                                  'ccopts': '-mabi=32'},
 278                                 {'variant': 'n64-soft',
 279                                  'ccopts': '-mabi=64'}])
 280         self.add_config(arch='mips64el',
 281                         os_name='linux-gnu',
 282                         variant='nan2008',
 283                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 284                                  '--with-arch-64=mips64r2',
 285                                  '--with-arch-32=mips32r2'],
 286                         glibcs=[{'variant': 'n32-nan2008'},
 287                                 {'variant': 'nan2008',
 288                                  'arch': 'mipsel',
 289                                  'ccopts': '-mabi=32'},
 290                                 {'variant': 'n64-nan2008',
 291                                  'ccopts': '-mabi=64'}])
 292         self.add_config(arch='mips64el',
 293                         os_name='linux-gnu',
 294                         variant='nan2008-soft',
 295                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 296                                  '--with-arch-64=mips64r2',
 297                                  '--with-arch-32=mips32r2',
 298                                  '--with-float=soft'],
 299                         glibcs=[{'variant': 'n32-nan2008-soft'},
 300                                 {'variant': 'nan2008-soft',
 301                                  'arch': 'mipsel',
 302                                  'ccopts': '-mabi=32'},
 303                                 {'variant': 'n64-nan2008-soft',
 304                                  'ccopts': '-mabi=64'}])
 305         self.add_config(arch='nios2',
 306                         os_name='linux-gnu')
 307         self.add_config(arch='powerpc',
 308                         os_name='linux-gnu',
 309                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 310                         extra_glibcs=[{'variant': 'power4',
 311                                        'ccopts': '-mcpu=power4',
 312                                        'cfg': ['--with-cpu=power4']}])
 313         self.add_config(arch='powerpc',
 314                         os_name='linux-gnu',
 315                         variant='soft',
 316                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 317                                  '--enable-secureplt'])
 318         self.add_config(arch='powerpc64',
 319                         os_name='linux-gnu',
 320                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 321         self.add_config(arch='powerpc64le',
 322                         os_name='linux-gnu',
 323                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 324         self.add_config(arch='powerpc',
 325                         os_name='linux-gnuspe',
 326                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 327                                  '--enable-e500-double'])
 328         self.add_config(arch='powerpc',
 329                         os_name='linux-gnuspe',
 330                         variant='e500v1',
 331                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 332         self.add_config(arch='s390x',
 333                         os_name='linux-gnu',
 334                         glibcs=[{},
 335                                 {'arch': 's390', 'ccopts': '-m31'}])
 336         self.add_config(arch='sh3',
 337                         os_name='linux-gnu')
 338         self.add_config(arch='sh3eb',
 339                         os_name='linux-gnu')
 340         self.add_config(arch='sh4',
 341                         os_name='linux-gnu')
 342         self.add_config(arch='sh4eb',
 343                         os_name='linux-gnu')
 344         self.add_config(arch='sh4',
 345                         os_name='linux-gnu',
 346                         variant='soft',
 347                         gcc_cfg=['--without-fp'])
 348         self.add_config(arch='sh4eb',
 349                         os_name='linux-gnu',
 350                         variant='soft',
 351                         gcc_cfg=['--without-fp'])
 352         self.add_config(arch='sparc64',
 353                         os_name='linux-gnu',
 354                         glibcs=[{},
 355                                 {'arch': 'sparcv9',
 356                                  'ccopts': '-m32 -mlong-double-128'}],
 357                         extra_glibcs=[{'variant': 'disable-multi-arch',
 358                                        'cfg': ['--disable-multi-arch']},
 359                                       {'variant': 'disable-multi-arch',
 360                                        'arch': 'sparcv9',
 361                                        'ccopts': '-m32 -mlong-double-128',
 362                                        'cfg': ['--disable-multi-arch']}])
 363         self.add_config(arch='tilegx',
 364                         os_name='linux-gnu',
 365                         glibcs=[{},
 366                                 {'variant': '32', 'ccopts': '-m32'}])
 367         self.add_config(arch='tilegxbe',
 368                         os_name='linux-gnu',
 369                         glibcs=[{},
 370                                 {'variant': '32', 'ccopts': '-m32'}])
 371         self.add_config(arch='x86_64',
 372                         os_name='linux-gnu',
 373                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 374                         glibcs=[{},
 375                                 {'variant': 'x32', 'ccopts': '-mx32'},
 376                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 377                         extra_glibcs=[{'variant': 'disable-multi-arch',
 378                                        'cfg': ['--disable-multi-arch']},
 379                                       {'variant': 'static-pie',
 380                                        'cfg': ['--enable-static-pie']},
 381                                       {'variant': 'x32-static-pie',
 382                                        'ccopts': '-mx32',
 383                                        'cfg': ['--enable-static-pie']},
 384                                       {'variant': 'static-pie',
 385                                        'arch': 'i686',
 386                                        'ccopts': '-m32 -march=i686',
 387                                        'cfg': ['--enable-static-pie']},
 388                                       {'variant': 'disable-multi-arch',
 389                                        'arch': 'i686',
 390                                        'ccopts': '-m32 -march=i686',
 391                                        'cfg': ['--disable-multi-arch']},
 392                                       {'arch': 'i486',
 393                                        'ccopts': '-m32 -march=i486'},
 394                                       {'arch': 'i586',
 395                                        'ccopts': '-m32 -march=i586'}])
 396
 397     def add_config(self, **args):
 398         """Add an individual build configuration."""
 399         cfg = Config(self, **args)
 400         if cfg.name in self.configs:
 401             print('error: duplicate config %s' % cfg.name)
 402             exit(1)
 403         self.configs[cfg.name] = cfg
 404         for c in cfg.all_glibcs:
 405             if c.name in self.glibc_configs:
 406                 print('error: duplicate glibc config %s' % c.name)
 407                 exit(1)
 408             self.glibc_configs[c.name] = c
 409
 410     def component_srcdir(self, component):
 411         """Return the source directory for a given component, e.g. gcc."""
 412         return os.path.join(self.srcdir, component)
 413
 414     def component_builddir(self, action, config, component, subconfig=None):
 415         """Return the directory to use for a build."""
 416         if config is None:
 417             # Host libraries.
 418             assert subconfig is None
 419             return os.path.join(self.builddir, action, component)
 420         if subconfig is None:
 421             return os.path.join(self.builddir, action, config, component)
 422         else:
 423             # glibc build as part of compiler build.
 424             return os.path.join(self.builddir, action, config, component,
 425                                 subconfig)
 426
 427     def compiler_installdir(self, config):
 428         """Return the directory in which to install a compiler."""
 429         return os.path.join(self.installdir, 'compilers', config)
 430
 431     def compiler_bindir(self, config):
 432         """Return the directory in which to find compiler binaries."""
 433         return os.path.join(self.compiler_installdir(config), 'bin')
 434
 435     def compiler_sysroot(self, config):
 436         """Return the sysroot directory for a compiler."""
 437         return os.path.join(self.compiler_installdir(config), 'sysroot')
 438
 439     def glibc_installdir(self, config):
 440         """Return the directory in which to install glibc."""
 441         return os.path.join(self.installdir, 'glibcs', config)
 442
 443     def run_builds(self, action, configs):
 444         """Run the requested builds."""
 445         if action == 'checkout':
 446             self.checkout(configs)
 447             return
 448         if action == 'bot-cycle':
 449             if configs:
 450                 print('error: configurations specified for bot-cycle')
 451                 exit(1)
 452             self.bot_cycle()
 453             return
 454         if action == 'bot':
 455             if configs:
 456                 print('error: configurations specified for bot')
 457                 exit(1)
 458             self.bot()
 459             return
 460         if action == 'host-libraries' and configs:
 461             print('error: configurations specified for host-libraries')
 462             exit(1)
 463         self.clear_last_build_state(action)
 464         build_time = datetime.datetime.utcnow()
 465         if action == 'host-libraries':
 466             build_components = ('gmp', 'mpfr', 'mpc')
 467             old_components = ()
 468             old_versions = {}
 469             self.build_host_libraries()
 470         elif action == 'compilers':
 471             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 472                                 'gnumach', 'hurd')
 473             old_components = ('gmp', 'mpfr', 'mpc')
 474             old_versions = self.build_state['host-libraries']['build-versions']
 475             self.build_compilers(configs)
 476         else:
 477             build_components = ('glibc',)
 478             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 479                               'mig', 'gnumach', 'hurd')
 480             old_versions = self.build_state['compilers']['build-versions']
 481             self.build_glibcs(configs)
 482         self.write_files()
 483         self.do_build()
 484         if configs:
 485             # Partial build, do not update stored state.
 486             return
 487         build_versions = {}
 488         for k in build_components:
 489             if k in self.versions:
 490                 build_versions[k] = {'version': self.versions[k]['version'],
 491                                      'revision': self.versions[k]['revision']}
 492         for k in old_components:
 493             if k in old_versions:
 494                 build_versions[k] = {'version': old_versions[k]['version'],
 495                                      'revision': old_versions[k]['revision']}
 496         self.update_build_state(action, build_time, build_versions)
 497
 498     @staticmethod
 499     def remove_dirs(*args):
 500         """Remove directories and their contents if they exist."""
 501         for dir in args:
 502             shutil.rmtree(dir, ignore_errors=True)
 503
 504     @staticmethod
 505     def remove_recreate_dirs(*args):
 506         """Remove directories if they exist, and create them as empty."""
 507         Context.remove_dirs(*args)
 508         for dir in args:
 509             os.makedirs(dir, exist_ok=True)
 510
 511     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 512         """Add makefile text for a list of commands."""
 513         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 514         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 515                                     (target, target, target, commands))
 516         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 517
 518     def write_files(self):
 519         """Write out the Makefile and wrapper script."""
 520         mftext = ''.join(self.makefile_pieces)
 521         with open(self.makefile, 'w') as f:
 522             f.write(mftext)
 523         wrapper_text = (
 524             '#!/bin/sh\n'
 525             'prev_base=$1\n'
 526             'this_base=$2\n'
 527             'desc=$3\n'
 528             'dir=$4\n'
 529             'path=$5\n'
 530             'shift 5\n'
 531             'prev_status=$prev_base-status.txt\n'
 532             'this_status=$this_base-status.txt\n'
 533             'this_log=$this_base-log.txt\n'
 534             'date > "$this_log"\n'
 535             'echo >> "$this_log"\n'
 536             'echo "Description: $desc" >> "$this_log"\n'
 537             'printf "%s" "Command:" >> "$this_log"\n'
 538             'for word in "$@"; do\n'
 539             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 540             '    printf " %s" "$word"\n'
 541             '  else\n'
 542             '    printf " \'"\n'
 543             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 544             '    printf "\'"\n'
 545             '  fi\n'
 546             'done >> "$this_log"\n'
 547             'echo >> "$this_log"\n'
 548             'echo "Directory: $dir" >> "$this_log"\n'
 549             'echo "Path addition: $path" >> "$this_log"\n'
 550             'echo >> "$this_log"\n'
 551             'record_status ()\n'
 552             '{\n'
 553             '  echo >> "$this_log"\n'
 554             '  echo "$1: $desc" > "$this_status"\n'
 555             '  echo "$1: $desc" >> "$this_log"\n'
 556             '  echo >> "$this_log"\n'
 557             '  date >> "$this_log"\n'
 558             '  echo "$1: $desc"\n'
 559             '  exit 0\n'
 560             '}\n'
 561             'check_error ()\n'
 562             '{\n'
 563             '  if [ "$1" != "0" ]; then\n'
 564             '    record_status FAIL\n'
 565             '  fi\n'
 566             '}\n'
 567             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 568             '    record_status UNRESOLVED\n'
 569             'fi\n'
 570             'if [ "$dir" ]; then\n'
 571             '  cd "$dir"\n'
 572             '  check_error "$?"\n'
 573             'fi\n'
 574             'if [ "$path" ]; then\n'
 575             '  PATH=$path:$PATH\n'
 576             'fi\n'
 577             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 578             'check_error "$?"\n'
 579             'record_status PASS\n')
 580         with open(self.wrapper, 'w') as f:
 581             f.write(wrapper_text)
 582         # Mode 0o755.
 583         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 584                      stat.S_IROTH|stat.S_IXOTH)
 585         os.chmod(self.wrapper, mode_exec)
 586         save_logs_text = (
 587             '#!/bin/sh\n'
 588             'if ! [ -f tests.sum ]; then\n'
 589             '  echo "No test summary available."\n'
 590             '  exit 0\n'
 591             'fi\n'
 592             'save_file ()\n'
 593             '{\n'
 594             '  echo "Contents of $1:"\n'
 595             '  echo\n'
 596             '  cat "$1"\n'
 597             '  echo\n'
 598             '  echo "End of contents of $1."\n'
 599             '  echo\n'
 600             '}\n'
 601             'save_file tests.sum\n'
 602             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 603             'for t in $non_pass_tests; do\n'
 604             '  if [ -f "$t.out" ]; then\n'
 605             '    save_file "$t.out"\n'
 606             '  fi\n'
 607             'done\n')
 608         with open(self.save_logs, 'w') as f:
 609             f.write(save_logs_text)
 610         os.chmod(self.save_logs, mode_exec)
 611
 612     def do_build(self):
 613         """Do the actual build."""
 614         cmd = ['make', '-j%d' % self.parallelism]
 615         subprocess.run(cmd, cwd=self.builddir, check=True)
 616
 617     def build_host_libraries(self):
 618         """Build the host libraries."""
 619         installdir = self.host_libraries_installdir
 620         builddir = os.path.join(self.builddir, 'host-libraries')
 621         logsdir = os.path.join(self.logsdir, 'host-libraries')
 622         self.remove_recreate_dirs(installdir, builddir, logsdir)
 623         cmdlist = CommandList('host-libraries', self.keep)
 624         self.build_host_library(cmdlist, 'gmp')
 625         self.build_host_library(cmdlist, 'mpfr',
 626                                 ['--with-gmp=%s' % installdir])
 627         self.build_host_library(cmdlist, 'mpc',
 628                                 ['--with-gmp=%s' % installdir,
 629                                 '--with-mpfr=%s' % installdir])
 630         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 631         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 632
 633     def build_host_library(self, cmdlist, lib, extra_opts=None):
 634         """Build one host library."""
 635         srcdir = self.component_srcdir(lib)
 636         builddir = self.component_builddir('host-libraries', None, lib)
 637         installdir = self.host_libraries_installdir
 638         cmdlist.push_subdesc(lib)
 639         cmdlist.create_use_dir(builddir)
 640         cfg_cmd = [os.path.join(srcdir, 'configure'),
 641                    '--prefix=%s' % installdir,
 642                    '--disable-shared']
 643         if extra_opts:
 644             cfg_cmd.extend (extra_opts)
 645         cmdlist.add_command('configure', cfg_cmd)
 646         cmdlist.add_command('build', ['make'])
 647         cmdlist.add_command('check', ['make', 'check'])
 648         cmdlist.add_command('install', ['make', 'install'])
 649         cmdlist.cleanup_dir()
 650         cmdlist.pop_subdesc()
 651
 652     def build_compilers(self, configs):
 653         """Build the compilers."""
 654         if not configs:
 655             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 656             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 657             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 658             configs = sorted(self.configs.keys())
 659         for c in configs:
 660             self.configs[c].build()
 661
 662     def build_glibcs(self, configs):
 663         """Build the glibcs."""
 664         if not configs:
 665             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 666             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 667             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 668             configs = sorted(self.glibc_configs.keys())
 669         for c in configs:
 670             self.glibc_configs[c].build()
 671
 672     def load_versions_json(self):
 673         """Load information about source directory versions."""
 674         if not os.access(self.versions_json, os.F_OK):
 675             self.versions = {}
 676             return
 677         with open(self.versions_json, 'r') as f:
 678             self.versions = json.load(f)
 679
 680     def store_json(self, data, filename):
 681         """Store information in a JSON file."""
 682         filename_tmp = filename + '.tmp'
 683         with open(filename_tmp, 'w') as f:
 684             json.dump(data, f, indent=2, sort_keys=True)
 685         os.rename(filename_tmp, filename)
 686
 687     def store_versions_json(self):
 688         """Store information about source directory versions."""
 689         self.store_json(self.versions, self.versions_json)
 690
 691     def set_component_version(self, component, version, explicit, revision):
 692         """Set the version information for a component."""
 693         self.versions[component] = {'version': version,
 694                                     'explicit': explicit,
 695                                     'revision': revision}
 696         self.store_versions_json()
 697
 698     def checkout(self, versions):
 699         """Check out the desired component versions."""
 700         default_versions = {'binutils': 'vcs-2.30',
 701                             'gcc': 'vcs-7',
 702                             'glibc': 'vcs-mainline',
 703                             'gmp': '6.1.2',
 704                             'linux': '4.14',
 705                             'mpc': '1.1.0',
 706                             'mpfr': '4.0.0',
 707                             'mig': 'vcs-mainline',
 708                             'gnumach': '1.8',
 709                             'hurd': 'vcs-mainline'}
 710         use_versions = {}
 711         explicit_versions = {}
 712         for v in versions:
 713             found_v = False
 714             for k in default_versions.keys():
 715                 kx = k + '-'
 716                 if v.startswith(kx):
 717                     vx = v[len(kx):]
 718                     if k in use_versions:
 719                         print('error: multiple versions for %s' % k)
 720                         exit(1)
 721                     use_versions[k] = vx
 722                     explicit_versions[k] = True
 723                     found_v = True
 724                     break
 725             if not found_v:
 726                 print('error: unknown component in %s' % v)
 727                 exit(1)
 728         for k in default_versions.keys():
 729             if k not in use_versions:
 730                 if k in self.versions and self.versions[k]['explicit']:
 731                     use_versions[k] = self.versions[k]['version']
 732                     explicit_versions[k] = True
 733                 else:
 734                     use_versions[k] = default_versions[k]
 735                     explicit_versions[k] = False
 736         os.makedirs(self.srcdir, exist_ok=True)
 737         for k in sorted(default_versions.keys()):
 738             update = os.access(self.component_srcdir(k), os.F_OK)
 739             v = use_versions[k]
 740             if (update and
 741                 k in self.versions and
 742                 v != self.versions[k]['version']):
 743                 if not self.replace_sources:
 744                     print('error: version of %s has changed from %s to %s, '
 745                           'use --replace-sources to check out again' %
 746                           (k, self.versions[k]['version'], v))
 747                     exit(1)
 748                 shutil.rmtree(self.component_srcdir(k))
 749                 update = False
 750             if v.startswith('vcs-'):
 751                 revision = self.checkout_vcs(k, v[4:], update)
 752             else:
 753                 self.checkout_tar(k, v, update)
 754                 revision = v
 755             self.set_component_version(k, v, explicit_versions[k], revision)
 756         if self.get_script_text() != self.script_text:
 757             # Rerun the checkout process in case the updated script
 758             # uses different default versions or new components.
 759             self.exec_self()
 760
 761     def checkout_vcs(self, component, version, update):
 762         """Check out the given version of the given component from version
 763         control.  Return a revision identifier."""
 764         if component == 'binutils':
 765             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 766             if version == 'mainline':
 767                 git_branch = 'master'
 768             else:
 769                 trans = str.maketrans({'.': '_'})
 770                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 771             return self.git_checkout(component, git_url, git_branch, update)
 772         elif component == 'gcc':
 773             if version == 'mainline':
 774                 branch = 'trunk'
 775             else:
 776                 trans = str.maketrans({'.': '_'})
 777                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 778             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 779             return self.gcc_checkout(svn_url, update)
 780         elif component == 'glibc':
 781             git_url = 'git://sourceware.org/git/glibc.git'
 782             if version == 'mainline':
 783                 git_branch = 'master'
 784             else:
 785                 git_branch = 'release/%s/master' % version
 786             r = self.git_checkout(component, git_url, git_branch, update)
 787             self.fix_glibc_timestamps()
 788             return r
 789         elif component == 'mig':
 790             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 791             git_branch = 'master'
 792             r = self.git_checkout(component, git_url, git_branch, update)
 793             subprocess.run(['autoreconf', '-i'],
 794                            cwd=self.component_srcdir(component), check=True)
 795             return r
 796         elif component == 'hurd':
 797             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 798             git_branch = 'master'
 799             r = self.git_checkout(component, git_url, git_branch, update)
 800             subprocess.run(['autoconf'],
 801                            cwd=self.component_srcdir(component), check=True)
 802             return r
 803         else:
 804             print('error: component %s coming from VCS' % component)
 805             exit(1)
 806
 807     def git_checkout(self, component, git_url, git_branch, update):
 808         """Check out a component from git.  Return a commit identifier."""
 809         if update:
 810             subprocess.run(['git', 'remote', 'prune', 'origin'],
 811                            cwd=self.component_srcdir(component), check=True)
 812             subprocess.run(['git', 'pull', '-q'],
 813                            cwd=self.component_srcdir(component), check=True)
 814         else:
 815             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 816                             self.component_srcdir(component)], check=True)
 817         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 818                            cwd=self.component_srcdir(component),
 819                            stdout=subprocess.PIPE,
 820                            check=True, universal_newlines=True).stdout
 821         return r.rstrip()
 822
 823     def fix_glibc_timestamps(self):
 824         """Fix timestamps in a glibc checkout."""
 825         # Ensure that builds do not try to regenerate generated files
 826         # in the source tree.
 827         srcdir = self.component_srcdir('glibc')
 828         for dirpath, dirnames, filenames in os.walk(srcdir):
 829             for f in filenames:
 830                 if (f == 'configure' or
 831                     f == 'preconfigure' or
 832                     f.endswith('-kw.h')):
 833                     to_touch = os.path.join(dirpath, f)
 834                     subprocess.run(['touch', to_touch], check=True)
 835
 836     def gcc_checkout(self, svn_url, update):
 837         """Check out GCC from SVN.  Return the revision number."""
 838         if not update:
 839             subprocess.run(['svn', 'co', '-q', svn_url,
 840                             self.component_srcdir('gcc')], check=True)
 841         subprocess.run(['contrib/gcc_update', '--silent'],
 842                        cwd=self.component_srcdir('gcc'), check=True)
 843         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 844                            stdout=subprocess.PIPE,
 845                            check=True, universal_newlines=True).stdout
 846         return r.rstrip()
 847
 848     def checkout_tar(self, component, version, update):
 849         """Check out the given version of the given component from a
 850         tarball."""
 851         if update:
 852             return
 853         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 854                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 855                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 856                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 857                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 858                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 859                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 860                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 861                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 862         if component not in url_map:
 863             print('error: component %s coming from tarball' % component)
 864             exit(1)
 865         url = url_map[component] % {'version': version}
 866         filename = os.path.join(self.srcdir, url.split('/')[-1])
 867         response = urllib.request.urlopen(url)
 868         data = response.read()
 869         with open(filename, 'wb') as f:
 870             f.write(data)
 871         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 872                        check=True)
 873         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 874                   self.component_srcdir(component))
 875         os.remove(filename)
 876
 877     def load_build_state_json(self):
 878         """Load information about the state of previous builds."""
 879         if os.access(self.build_state_json, os.F_OK):
 880             with open(self.build_state_json, 'r') as f:
 881                 self.build_state = json.load(f)
 882         else:
 883             self.build_state = {}
 884         for k in ('host-libraries', 'compilers', 'glibcs'):
 885             if k not in self.build_state:
 886                 self.build_state[k] = {}
 887             if 'build-time' not in self.build_state[k]:
 888                 self.build_state[k]['build-time'] = ''
 889             if 'build-versions' not in self.build_state[k]:
 890                 self.build_state[k]['build-versions'] = {}
 891             if 'build-results' not in self.build_state[k]:
 892                 self.build_state[k]['build-results'] = {}
 893             if 'result-changes' not in self.build_state[k]:
 894                 self.build_state[k]['result-changes'] = {}
 895             if 'ever-passed' not in self.build_state[k]:
 896                 self.build_state[k]['ever-passed'] = []
 897
 898     def store_build_state_json(self):
 899         """Store information about the state of previous builds."""
 900         self.store_json(self.build_state, self.build_state_json)
 901
 902     def clear_last_build_state(self, action):
 903         """Clear information about the state of part of the build."""
 904         # We clear the last build time and versions when starting a
 905         # new build.  The results of the last build are kept around,
 906         # as comparison is still meaningful if this build is aborted
 907         # and a new one started.
 908         self.build_state[action]['build-time'] = ''
 909         self.build_state[action]['build-versions'] = {}
 910         self.store_build_state_json()
 911
 912     def update_build_state(self, action, build_time, build_versions):
 913         """Update the build state after a build."""
 914         build_time = build_time.replace(microsecond=0)
 915         self.build_state[action]['build-time'] = str(build_time)
 916         self.build_state[action]['build-versions'] = build_versions
 917         build_results = {}
 918         for log in self.status_log_list:
 919             with open(log, 'r') as f:
 920                 log_text = f.read()
 921             log_text = log_text.rstrip()
 922             m = re.fullmatch('([A-Z]+): (.*)', log_text)
 923             result = m.group(1)
 924             test_name = m.group(2)
 925             assert test_name not in build_results
 926             build_results[test_name] = result
 927         old_build_results = self.build_state[action]['build-results']
 928         self.build_state[action]['build-results'] = build_results
 929         result_changes = {}
 930         all_tests = set(old_build_results.keys()) | set(build_results.keys())
 931         for t in all_tests:
 932             if t in old_build_results:
 933                 old_res = old_build_results[t]
 934             else:
 935                 old_res = '(New test)'
 936             if t in build_results:
 937                 new_res = build_results[t]
 938             else:
 939                 new_res = '(Test removed)'
 940             if old_res != new_res:
 941                 result_changes[t] = '%s -> %s' % (old_res, new_res)
 942         self.build_state[action]['result-changes'] = result_changes
 943         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
 944                            if t in build_results}
 945         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
 946         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
 947                                                          new_passes)
 948         self.store_build_state_json()
 949
 950     def load_bot_config_json(self):
 951         """Load bot configuration."""
 952         with open(self.bot_config_json, 'r') as f:
 953             self.bot_config = json.load(f)
 954
 955     def part_build_old(self, action, delay):
 956         """Return whether the last build for a given action was at least a
 957         given number of seconds ago, or does not have a time recorded."""
 958         old_time_str = self.build_state[action]['build-time']
 959         if not old_time_str:
 960             return True
 961         old_time = datetime.datetime.strptime(old_time_str,
 962                                               '%Y-%m-%d %H:%M:%S')
 963         new_time = datetime.datetime.utcnow()
 964         delta = new_time - old_time
 965         return delta.total_seconds() >= delay
 966
 967     def bot_cycle(self):
 968         """Run a single round of checkout and builds."""
 969         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
 970         self.load_bot_config_json()
 971         actions = ('host-libraries', 'compilers', 'glibcs')
 972         self.bot_run_self(['--replace-sources'], 'checkout')
 973         self.load_versions_json()
 974         if self.get_script_text() != self.script_text:
 975             print('Script changed, re-execing.')
 976             # On script change, all parts of the build should be rerun.
 977             for a in actions:
 978                 self.clear_last_build_state(a)
 979             self.exec_self()
 980         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
 981                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
 982                                           'mig', 'gnumach', 'hurd'),
 983                             'glibcs': ('glibc',)}
 984         must_build = {}
 985         for a in actions:
 986             build_vers = self.build_state[a]['build-versions']
 987             must_build[a] = False
 988             if not self.build_state[a]['build-time']:
 989                 must_build[a] = True
 990             old_vers = {}
 991             new_vers = {}
 992             for c in check_components[a]:
 993                 if c in build_vers:
 994                     old_vers[c] = build_vers[c]
 995                 new_vers[c] = {'version': self.versions[c]['version'],
 996                                'revision': self.versions[c]['revision']}
 997             if new_vers == old_vers:
 998                 print('Versions for %s unchanged.' % a)
 999             else:
1000                 print('Versions changed or rebuild forced for %s.' % a)
1001                 if a == 'compilers' and not self.part_build_old(
1002                         a, self.bot_config['compilers-rebuild-delay']):
1003                     print('Not requiring rebuild of compilers this soon.')
1004                 else:
1005                     must_build[a] = True
1006         if must_build['host-libraries']:
1007             must_build['compilers'] = True
1008         if must_build['compilers']:
1009             must_build['glibcs'] = True
1010         for a in actions:
1011             if must_build[a]:
1012                 print('Must rebuild %s.' % a)
1013                 self.clear_last_build_state(a)
1014             else:
1015                 print('No need to rebuild %s.' % a)
1016         if os.access(self.logsdir, os.F_OK):
1017             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1018             shutil.copytree(self.logsdir, self.logsdir_old)
1019         for a in actions:
1020             if must_build[a]:
1021                 build_time = datetime.datetime.utcnow()
1022                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1023                 self.bot_run_self([], a)
1024                 self.load_build_state_json()
1025                 self.bot_build_mail(a, build_time)
1026         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1027
1028     def bot_build_mail(self, action, build_time):
1029         """Send email with the results of a build."""
1030         if not ('email-from' in self.bot_config and
1031                 'email-server' in self.bot_config and
1032                 'email-subject' in self.bot_config and
1033                 'email-to' in self.bot_config):
1034             if not self.email_warning:
1035                 print("Email not configured, not sending.")
1036                 self.email_warning = True
1037             return
1038
1039         build_time = build_time.replace(microsecond=0)
1040         subject = (self.bot_config['email-subject'] %
1041                    {'action': action,
1042                     'build-time': str(build_time)})
1043         results = self.build_state[action]['build-results']
1044         changes = self.build_state[action]['result-changes']
1045         ever_passed = set(self.build_state[action]['ever-passed'])
1046         versions = self.build_state[action]['build-versions']
1047         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1048         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1049         all_fails = {k for k in results if results[k] == 'FAIL'}
1050         if new_regressions:
1051             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1052             new_reg_text = ('New regressions:\n\n%s\n\n' %
1053                             '\n'.join(new_reg_list))
1054         else:
1055             new_reg_text = ''
1056         if all_regressions:
1057             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1058             all_reg_text = ('All regressions:\n\n%s\n\n' %
1059                             '\n'.join(all_reg_list))
1060         else:
1061             all_reg_text = ''
1062         if all_fails:
1063             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1064             all_fail_text = ('All failures:\n\n%s\n\n' %
1065                              '\n'.join(all_fail_list))
1066         else:
1067             all_fail_text = ''
1068         if changes:
1069             changes_list = sorted(changes.keys())
1070             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1071             changes_text = ('All changed results:\n\n%s\n\n' %
1072                             '\n'.join(changes_list))
1073         else:
1074             changes_text = ''
1075         results_text = (new_reg_text + all_reg_text + all_fail_text +
1076                         changes_text)
1077         if not results_text:
1078             results_text = 'Clean build with unchanged results.\n\n'
1079         versions_list = sorted(versions.keys())
1080         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1081                                           versions[k]['revision'])
1082                          for k in versions_list]
1083         versions_text = ('Component versions for this build:\n\n%s\n' %
1084                          '\n'.join(versions_list))
1085         body_text = results_text + versions_text
1086         msg = email.mime.text.MIMEText(body_text)
1087         msg['Subject'] = subject
1088         msg['From'] = self.bot_config['email-from']
1089         msg['To'] = self.bot_config['email-to']
1090         msg['Message-ID'] = email.utils.make_msgid()
1091         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1092         with smtplib.SMTP(self.bot_config['email-server']) as s:
1093             s.send_message(msg)
1094
1095     def bot_run_self(self, opts, action, check=True):
1096         """Run a copy of this script with given options."""
1097         cmd = [sys.executable, sys.argv[0], '--keep=none',
1098                '-j%d' % self.parallelism]
1099         cmd.extend(opts)
1100         cmd.extend([self.topdir, action])
1101         sys.stdout.flush()
1102         subprocess.run(cmd, check=check)
1103
1104     def bot(self):
1105         """Run repeated rounds of checkout and builds."""
1106         while True:
1107             self.load_bot_config_json()
1108             if not self.bot_config['run']:
1109                 print('Bot exiting by request.')
1110                 exit(0)
1111             self.bot_run_self([], 'bot-cycle', check=False)
1112             self.load_bot_config_json()
1113             if not self.bot_config['run']:
1114                 print('Bot exiting by request.')
1115                 exit(0)
1116             time.sleep(self.bot_config['delay'])
1117             if self.get_script_text() != self.script_text:
1118                 print('Script changed, bot re-execing.')
1119                 self.exec_self()
1120
1121
1122 class Config(object):
1123     """A configuration for building a compiler and associated libraries."""
1124
1125     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1126                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1127         """Initialize a Config object."""
1128         self.ctx = ctx
1129         self.arch = arch
1130         self.os = os_name
1131         self.variant = variant
1132         if variant is None:
1133             self.name = '%s-%s' % (arch, os_name)
1134         else:
1135             self.name = '%s-%s-%s' % (arch, os_name, variant)
1136         self.triplet = '%s-glibc-%s' % (arch, os_name)
1137         if gcc_cfg is None:
1138             self.gcc_cfg = []
1139         else:
1140             self.gcc_cfg = gcc_cfg
1141         if first_gcc_cfg is None:
1142             self.first_gcc_cfg = []
1143         else:
1144             self.first_gcc_cfg = first_gcc_cfg
1145         if glibcs is None:
1146             glibcs = [{'variant': variant}]
1147         if extra_glibcs is None:
1148             extra_glibcs = []
1149         glibcs = [Glibc(self, **g) for g in glibcs]
1150         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1151         self.all_glibcs = glibcs + extra_glibcs
1152         self.compiler_glibcs = glibcs
1153         self.installdir = ctx.compiler_installdir(self.name)
1154         self.bindir = ctx.compiler_bindir(self.name)
1155         self.sysroot = ctx.compiler_sysroot(self.name)
1156         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1157         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1158
1159     def component_builddir(self, component):
1160         """Return the directory to use for a (non-glibc) build."""
1161         return self.ctx.component_builddir('compilers', self.name, component)
1162
1163     def build(self):
1164         """Generate commands to build this compiler."""
1165         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1166                                       self.logsdir)
1167         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1168         cmdlist.add_command('check-host-libraries',
1169                             ['test', '-f',
1170                              os.path.join(self.ctx.host_libraries_installdir,
1171                                           'ok')])
1172         cmdlist.use_path(self.bindir)
1173         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1174                               ['--disable-gdb',
1175                                '--disable-libdecnumber',
1176                                '--disable-readline',
1177                                '--disable-sim'])
1178         if self.os.startswith('linux'):
1179             self.install_linux_headers(cmdlist)
1180         self.build_gcc(cmdlist, True)
1181         if self.os == 'gnu':
1182             self.install_gnumach_headers(cmdlist)
1183             self.build_cross_tool(cmdlist, 'mig', 'mig')
1184             self.install_hurd_headers(cmdlist)
1185         for g in self.compiler_glibcs:
1186             cmdlist.push_subdesc('glibc')
1187             cmdlist.push_subdesc(g.name)
1188             g.build_glibc(cmdlist, True)
1189             cmdlist.pop_subdesc()
1190             cmdlist.pop_subdesc()
1191         self.build_gcc(cmdlist, False)
1192         cmdlist.add_command('done', ['touch',
1193                                      os.path.join(self.installdir, 'ok')])
1194         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1195                                       self.logsdir)
1196
1197     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1198         """Build one cross tool."""
1199         srcdir = self.ctx.component_srcdir(tool_src)
1200         builddir = self.component_builddir(tool_build)
1201         cmdlist.push_subdesc(tool_build)
1202         cmdlist.create_use_dir(builddir)
1203         cfg_cmd = [os.path.join(srcdir, 'configure'),
1204                    '--prefix=%s' % self.installdir,
1205                    '--build=%s' % self.ctx.build_triplet,
1206                    '--host=%s' % self.ctx.build_triplet,
1207                    '--target=%s' % self.triplet,
1208                    '--with-sysroot=%s' % self.sysroot]
1209         if extra_opts:
1210             cfg_cmd.extend(extra_opts)
1211         cmdlist.add_command('configure', cfg_cmd)
1212         cmdlist.add_command('build', ['make'])
1213         # Parallel "make install" for GCC has race conditions that can
1214         # cause it to fail; see
1215         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1216         # problems are not known for binutils, but doing the
1217         # installation in parallel within a particular toolchain build
1218         # (as opposed to installation of one toolchain from
1219         # build-many-glibcs.py running in parallel to the installation
1220         # of other toolchains being built) is not known to be
1221         # significantly beneficial, so it is simplest just to disable
1222         # parallel install for cross tools here.
1223         cmdlist.add_command('install', ['make', '-j1', 'install'])
1224         cmdlist.cleanup_dir()
1225         cmdlist.pop_subdesc()
1226
1227     def install_linux_headers(self, cmdlist):
1228         """Install Linux kernel headers."""
1229         arch_map = {'aarch64': 'arm64',
1230                     'alpha': 'alpha',
1231                     'arm': 'arm',
1232                     'hppa': 'parisc',
1233                     'i486': 'x86',
1234                     'i586': 'x86',
1235                     'i686': 'x86',
1236                     'i786': 'x86',
1237                     'ia64': 'ia64',
1238                     'm68k': 'm68k',
1239                     'microblaze': 'microblaze',
1240                     'mips': 'mips',
1241                     'nios2': 'nios2',
1242                     'powerpc': 'powerpc',
1243                     's390': 's390',
1244                     'sh': 'sh',
1245                     'sparc': 'sparc',
1246                     'tile': 'tile',
1247                     'x86_64': 'x86'}
1248         linux_arch = None
1249         for k in arch_map:
1250             if self.arch.startswith(k):
1251                 linux_arch = arch_map[k]
1252                 break
1253         assert linux_arch is not None
1254         srcdir = self.ctx.component_srcdir('linux')
1255         builddir = self.component_builddir('linux')
1256         headers_dir = os.path.join(self.sysroot, 'usr')
1257         cmdlist.push_subdesc('linux')
1258         cmdlist.create_use_dir(builddir)
1259         cmdlist.add_command('install-headers',
1260                             ['make', '-C', srcdir, 'O=%s' % builddir,
1261                              'ARCH=%s' % linux_arch,
1262                              'INSTALL_HDR_PATH=%s' % headers_dir,
1263                              'headers_install'])
1264         cmdlist.cleanup_dir()
1265         cmdlist.pop_subdesc()
1266
1267     def install_gnumach_headers(self, cmdlist):
1268         """Install GNU Mach headers."""
1269         srcdir = self.ctx.component_srcdir('gnumach')
1270         builddir = self.component_builddir('gnumach')
1271         cmdlist.push_subdesc('gnumach')
1272         cmdlist.create_use_dir(builddir)
1273         cmdlist.add_command('configure',
1274                             [os.path.join(srcdir, 'configure'),
1275                              '--build=%s' % self.ctx.build_triplet,
1276                              '--host=%s' % self.triplet,
1277                              '--prefix=',
1278                              'CC=%s-gcc -nostdlib' % self.triplet])
1279         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1280                                         'install-data'])
1281         cmdlist.cleanup_dir()
1282         cmdlist.pop_subdesc()
1283
1284     def install_hurd_headers(self, cmdlist):
1285         """Install Hurd headers."""
1286         srcdir = self.ctx.component_srcdir('hurd')
1287         builddir = self.component_builddir('hurd')
1288         cmdlist.push_subdesc('hurd')
1289         cmdlist.create_use_dir(builddir)
1290         cmdlist.add_command('configure',
1291                             [os.path.join(srcdir, 'configure'),
1292                              '--build=%s' % self.ctx.build_triplet,
1293                              '--host=%s' % self.triplet,
1294                              '--prefix=',
1295                              '--disable-profile', '--without-parted',
1296                              'CC=%s-gcc -nostdlib' % self.triplet])
1297         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1298                                         'no_deps=t', 'install-headers'])
1299         cmdlist.cleanup_dir()
1300         cmdlist.pop_subdesc()
1301
1302     def build_gcc(self, cmdlist, bootstrap):
1303         """Build GCC."""
1304         # libsanitizer commonly breaks because of glibc header
1305         # changes, or on unusual targets.  libssp is of little
1306         # relevance with glibc's own stack checking support.
1307         cfg_opts = list(self.gcc_cfg)
1308         cfg_opts += ['--disable-libsanitizer', '--disable-libssp']
1309         host_libs = self.ctx.host_libraries_installdir
1310         cfg_opts += ['--with-gmp=%s' % host_libs,
1311                      '--with-mpfr=%s' % host_libs,
1312                      '--with-mpc=%s' % host_libs]
1313         if bootstrap:
1314             tool_build = 'gcc-first'
1315             # Building a static-only, C-only compiler that is
1316             # sufficient to build glibc.  Various libraries and
1317             # features that may require libc headers must be disabled.
1318             # When configuring with a sysroot, --with-newlib is
1319             # required to define inhibit_libc (to stop some parts of
1320             # libgcc including libc headers); --without-headers is not
1321             # sufficient.
1322             cfg_opts += ['--enable-languages=c', '--disable-shared',
1323                          '--disable-threads',
1324                          '--disable-libatomic',
1325                          '--disable-decimal-float',
1326                          '--disable-libffi',
1327                          '--disable-libgomp',
1328                          '--disable-libitm',
1329                          '--disable-libmpx',
1330                          '--disable-libquadmath',
1331                          '--without-headers', '--with-newlib',
1332                          '--with-glibc-version=%s' % self.ctx.glibc_version
1333                          ]
1334             cfg_opts += self.first_gcc_cfg
1335         else:
1336             tool_build = 'gcc'
1337             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1338                          '--enable-threads']
1339             if self.os == 'gnu':
1340                 cfg_opts += ['--disable-libcilkrts']
1341         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1342
1343
1344 class Glibc(object):
1345     """A configuration for building glibc."""
1346
1347     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1348                  cfg=None, ccopts=None):
1349         """Initialize a Glibc object."""
1350         self.ctx = compiler.ctx
1351         self.compiler = compiler
1352         if arch is None:
1353             self.arch = compiler.arch
1354         else:
1355             self.arch = arch
1356         if os_name is None:
1357             self.os = compiler.os
1358         else:
1359             self.os = os_name
1360         self.variant = variant
1361         if variant is None:
1362             self.name = '%s-%s' % (self.arch, self.os)
1363         else:
1364             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1365         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1366         if cfg is None:
1367             self.cfg = []
1368         else:
1369             self.cfg = cfg
1370         self.ccopts = ccopts
1371
1372     def tool_name(self, tool):
1373         """Return the name of a cross-compilation tool."""
1374         ctool = '%s-%s' % (self.compiler.triplet, tool)
1375         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1376             ctool = '%s %s' % (ctool, self.ccopts)
1377         return ctool
1378
1379     def build(self):
1380         """Generate commands to build this glibc."""
1381         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1382         installdir = self.ctx.glibc_installdir(self.name)
1383         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1384         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1385         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1386         cmdlist.add_command('check-compilers',
1387                             ['test', '-f',
1388                              os.path.join(self.compiler.installdir, 'ok')])
1389         cmdlist.use_path(self.compiler.bindir)
1390         self.build_glibc(cmdlist, False)
1391         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1392                                       logsdir)
1393
1394     def build_glibc(self, cmdlist, for_compiler):
1395         """Generate commands to build this glibc, either as part of a compiler
1396         build or with the bootstrapped compiler (and in the latter case, run
1397         tests as well)."""
1398         srcdir = self.ctx.component_srcdir('glibc')
1399         if for_compiler:
1400             builddir = self.ctx.component_builddir('compilers',
1401                                                    self.compiler.name, 'glibc',
1402                                                    self.name)
1403             installdir = self.compiler.sysroot
1404             srcdir_copy = self.ctx.component_builddir('compilers',
1405                                                       self.compiler.name,
1406                                                       'glibc-src',
1407                                                       self.name)
1408         else:
1409             builddir = self.ctx.component_builddir('glibcs', self.name,
1410                                                    'glibc')
1411             installdir = self.ctx.glibc_installdir(self.name)
1412             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1413                                                       'glibc-src')
1414         cmdlist.create_use_dir(builddir)
1415         # glibc builds write into the source directory, and even if
1416         # not intentionally there is a risk of bugs that involve
1417         # writing into the working directory.  To avoid possible
1418         # concurrency issues, copy the source directory.
1419         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1420         use_usr = self.os != 'gnu'
1421         prefix = '/usr' if use_usr else ''
1422         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1423                    '--prefix=%s' % prefix,
1424                    '--enable-profile',
1425                    '--build=%s' % self.ctx.build_triplet,
1426                    '--host=%s' % self.triplet,
1427                    'CC=%s' % self.tool_name('gcc'),
1428                    'CXX=%s' % self.tool_name('g++'),
1429                    'AR=%s' % self.tool_name('ar'),
1430                    'AS=%s' % self.tool_name('as'),
1431                    'LD=%s' % self.tool_name('ld'),
1432                    'NM=%s' % self.tool_name('nm'),
1433                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1434                    'OBJDUMP=%s' % self.tool_name('objdump'),
1435                    'RANLIB=%s' % self.tool_name('ranlib'),
1436                    'READELF=%s' % self.tool_name('readelf'),
1437                    'STRIP=%s' % self.tool_name('strip')]
1438         if self.os == 'gnu':
1439             cfg_cmd += ['MIG=%s' % self.tool_name('mig')]
1440         cfg_cmd += self.cfg
1441         cmdlist.add_command('configure', cfg_cmd)
1442         cmdlist.add_command('build', ['make'])
1443         cmdlist.add_command('install', ['make', 'install',
1444                                         'install_root=%s' % installdir])
1445         # GCC uses paths such as lib/../lib64, so make sure lib
1446         # directories always exist.
1447         mkdir_cmd = ['mkdir', '-p',
1448                      os.path.join(installdir, 'lib')]
1449         if use_usr:
1450             mkdir_cmd += [os.path.join(installdir, 'usr', 'lib')]
1451         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1452         if not for_compiler:
1453             if self.ctx.strip:
1454                 cmdlist.add_command('strip',
1455                                     ['sh', '-c',
1456                                      ('%s $(find %s/lib* -name "*.so")' %
1457                                       (self.tool_name('strip'), installdir))])
1458             cmdlist.add_command('check', ['make', 'check'])
1459             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1460                                 always_run=True)
1461         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1462         cmdlist.cleanup_dir()
1463
1464
1465 class Command(object):
1466     """A command run in the build process."""
1467
1468     def __init__(self, desc, num, dir, path, command, always_run=False):
1469         """Initialize a Command object."""
1470         self.dir = dir
1471         self.path = path
1472         self.desc = desc
1473         trans = str.maketrans({' ': '-'})
1474         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1475         self.command = command
1476         self.always_run = always_run
1477
1478     @staticmethod
1479     def shell_make_quote_string(s):
1480         """Given a string not containing a newline, quote it for use by the
1481         shell and make."""
1482         assert '\n' not in s
1483         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1484             return s
1485         strans = str.maketrans({"'": "'\\''"})
1486         s = "'%s'" % s.translate(strans)
1487         mtrans = str.maketrans({'$': '$$'})
1488         return s.translate(mtrans)
1489
1490     @staticmethod
1491     def shell_make_quote_list(l, translate_make):
1492         """Given a list of strings not containing newlines, quote them for use
1493         by the shell and make, returning a single string.  If translate_make
1494         is true and the first string is 'make', change it to $(MAKE)."""
1495         l = [Command.shell_make_quote_string(s) for s in l]
1496         if translate_make and l[0] == 'make':
1497             l[0] = '$(MAKE)'
1498         return ' '.join(l)
1499
1500     def shell_make_quote(self):
1501         """Return this command quoted for the shell and make."""
1502         return self.shell_make_quote_list(self.command, True)
1503
1504
1505 class CommandList(object):
1506     """A list of commands run in the build process."""
1507
1508     def __init__(self, desc, keep):
1509         """Initialize a CommandList object."""
1510         self.cmdlist = []
1511         self.dir = None
1512         self.path = None
1513         self.desc = [desc]
1514         self.keep = keep
1515
1516     def desc_txt(self, desc):
1517         """Return the description to use for a command."""
1518         return '%s %s' % (' '.join(self.desc), desc)
1519
1520     def use_dir(self, dir):
1521         """Set the default directory for subsequent commands."""
1522         self.dir = dir
1523
1524     def use_path(self, path):
1525         """Set a directory to be prepended to the PATH for subsequent
1526         commands."""
1527         self.path = path
1528
1529     def push_subdesc(self, subdesc):
1530         """Set the default subdescription for subsequent commands (e.g., the
1531         name of a component being built, within the series of commands
1532         building it)."""
1533         self.desc.append(subdesc)
1534
1535     def pop_subdesc(self):
1536         """Pop a subdescription from the list of descriptions."""
1537         self.desc.pop()
1538
1539     def create_use_dir(self, dir):
1540         """Remove and recreate a directory and use it for subsequent
1541         commands."""
1542         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1543         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1544         self.use_dir(dir)
1545
1546     def create_copy_dir(self, src, dest):
1547         """Remove a directory and recreate it as a copy from the given
1548         source."""
1549         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1550         parent = os.path.dirname(dest)
1551         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1552         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1553
1554     def add_command_dir(self, desc, dir, command, always_run=False):
1555         """Add a command to run in a given directory."""
1556         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1557                       command, always_run)
1558         self.cmdlist.append(cmd)
1559
1560     def add_command(self, desc, command, always_run=False):
1561         """Add a command to run in the default directory."""
1562         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1563                       self.path, command, always_run)
1564         self.cmdlist.append(cmd)
1565
1566     def cleanup_dir(self, desc='cleanup', dir=None):
1567         """Clean up a build directory.  If no directory is specified, the
1568         default directory is cleaned up and ceases to be the default
1569         directory."""
1570         if dir is None:
1571             dir = self.dir
1572             self.use_dir(None)
1573         if self.keep != 'all':
1574             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1575                                  always_run=(self.keep == 'none'))
1576
1577     def makefile_commands(self, wrapper, logsdir):
1578         """Return the sequence of commands in the form of text for a Makefile.
1579         The given wrapper script takes arguments: base of logs for
1580         previous command, or empty; base of logs for this command;
1581         description; directory; PATH addition; the command itself."""
1582         # prev_base is the base of the name for logs of the previous
1583         # command that is not always-run (that is, a build command,
1584         # whose failure should stop subsequent build commands from
1585         # being run, as opposed to a cleanup command, which is run
1586         # even if previous commands failed).
1587         prev_base = ''
1588         cmds = []
1589         for c in self.cmdlist:
1590             ctxt = c.shell_make_quote()
1591             if prev_base and not c.always_run:
1592                 prev_log = os.path.join(logsdir, prev_base)
1593             else:
1594                 prev_log = ''
1595             this_log = os.path.join(logsdir, c.logbase)
1596             if not c.always_run:
1597                 prev_base = c.logbase
1598             if c.dir is None:
1599                 dir = ''
1600             else:
1601                 dir = c.dir
1602             if c.path is None:
1603                 path = ''
1604             else:
1605                 path = c.path
1606             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1607             prelim_txt = Command.shell_make_quote_list(prelims, False)
1608             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1609         return '\n'.join(cmds)
1610
1611     def status_logs(self, logsdir):
1612         """Return the list of log files with command status."""
1613         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1614                 for c in self.cmdlist]
1615
1616
1617 def get_parser():
1618     """Return an argument parser for this module."""
1619     parser = argparse.ArgumentParser(description=__doc__)
1620     parser.add_argument('-j', dest='parallelism',
1621                         help='Run this number of jobs in parallel',
1622                         type=int, default=os.cpu_count())
1623     parser.add_argument('--keep', dest='keep',
1624                         help='Whether to keep all build directories, '
1625                         'none or only those from failed builds',
1626                         default='none', choices=('none', 'all', 'failed'))
1627     parser.add_argument('--replace-sources', action='store_true',
1628                         help='Remove and replace source directories '
1629                         'with the wrong version of a component')
1630     parser.add_argument('--strip', action='store_true',
1631                         help='Strip installed glibc libraries')
1632     parser.add_argument('topdir',
1633                         help='Toplevel working directory')
1634     parser.add_argument('action',
1635                         help='What to do',
1636                         choices=('checkout', 'bot-cycle', 'bot',
1637                                  'host-libraries', 'compilers', 'glibcs'))
1638     parser.add_argument('configs',
1639                         help='Versions to check out or configurations to build',
1640                         nargs='*')
1641     return parser
1642
1643
1644 def main(argv):
1645     """The main entry point."""
1646     parser = get_parser()
1647     opts = parser.parse_args(argv)
1648     topdir = os.path.abspath(opts.topdir)
1649     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1650                   opts.strip, opts.action)
1651     ctx.run_builds(opts.action, opts.configs)
1652
1653
1654 if __name__ == '__main__':
1655     main(sys.argv[1:])