scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2021 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <https://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 The 'list-compilers' command prints the name of each available
  36 compiler configuration, without building anything.  The 'list-glibcs'
  37 command prints the name of each glibc compiler configuration, followed
  38 by the space, followed by the name of the compiler configuration used
  39 for building this glibc variant.
  40
  41 """
  42
  43 import argparse
  44 import datetime
  45 import email.mime.text
  46 import email.utils
  47 import json
  48 import os
  49 import re
  50 import shutil
  51 import smtplib
  52 import stat
  53 import subprocess
  54 import sys
  55 import time
  56 import urllib.request
  57
  58 try:
  59     subprocess.run
  60 except:
  61     class _CompletedProcess:
  62         def __init__(self, args, returncode, stdout=None, stderr=None):
  63             self.args = args
  64             self.returncode = returncode
  65             self.stdout = stdout
  66             self.stderr = stderr
  67
  68     def _run(*popenargs, input=None, timeout=None, check=False, **kwargs):
  69         assert(timeout is None)
  70         with subprocess.Popen(*popenargs, **kwargs) as process:
  71             try:
  72                 stdout, stderr = process.communicate(input)
  73             except:
  74                 process.kill()
  75                 process.wait()
  76                 raise
  77             returncode = process.poll()
  78             if check and returncode:
  79                 raise subprocess.CalledProcessError(returncode, popenargs)
  80         return _CompletedProcess(popenargs, returncode, stdout, stderr)
  81
  82     subprocess.run = _run
  83
  84
  85 class Context(object):
  86     """The global state associated with builds in a given directory."""
  87
  88     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  89                  full_gcc, action, shallow=False):
  90         """Initialize the context."""
  91         self.topdir = topdir
  92         self.parallelism = parallelism
  93         self.keep = keep
  94         self.replace_sources = replace_sources
  95         self.strip = strip
  96         self.full_gcc = full_gcc
  97         self.shallow = shallow
  98         self.srcdir = os.path.join(topdir, 'src')
  99         self.versions_json = os.path.join(self.srcdir, 'versions.json')
 100         self.build_state_json = os.path.join(topdir, 'build-state.json')
 101         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
 102         self.installdir = os.path.join(topdir, 'install')
 103         self.host_libraries_installdir = os.path.join(self.installdir,
 104                                                       'host-libraries')
 105         self.builddir = os.path.join(topdir, 'build')
 106         self.logsdir = os.path.join(topdir, 'logs')
 107         self.logsdir_old = os.path.join(topdir, 'logs-old')
 108         self.makefile = os.path.join(self.builddir, 'Makefile')
 109         self.wrapper = os.path.join(self.builddir, 'wrapper')
 110         self.save_logs = os.path.join(self.builddir, 'save-logs')
 111         self.script_text = self.get_script_text()
 112         if action not in ('checkout', 'list-compilers', 'list-glibcs'):
 113             self.build_triplet = self.get_build_triplet()
 114             self.glibc_version = self.get_glibc_version()
 115         self.configs = {}
 116         self.glibc_configs = {}
 117         self.makefile_pieces = ['.PHONY: all\n']
 118         self.add_all_configs()
 119         self.load_versions_json()
 120         self.load_build_state_json()
 121         self.status_log_list = []
 122         self.email_warning = False
 123
 124     def get_script_text(self):
 125         """Return the text of this script."""
 126         with open(sys.argv[0], 'r') as f:
 127             return f.read()
 128
 129     def exec_self(self):
 130         """Re-execute this script with the same arguments."""
 131         sys.stdout.flush()
 132         os.execv(sys.executable, [sys.executable] + sys.argv)
 133
 134     def get_build_triplet(self):
 135         """Determine the build triplet with config.guess."""
 136         config_guess = os.path.join(self.component_srcdir('gcc'),
 137                                     'config.guess')
 138         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 139                                 check=True, universal_newlines=True).stdout
 140         return cg_out.rstrip()
 141
 142     def get_glibc_version(self):
 143         """Determine the glibc version number (major.minor)."""
 144         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 145         with open(version_h, 'r') as f:
 146             lines = f.readlines()
 147         starttext = '#define VERSION "'
 148         for l in lines:
 149             if l.startswith(starttext):
 150                 l = l[len(starttext):]
 151                 l = l.rstrip('"\n')
 152                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 153                 return '%s.%s' % m.group(1, 2)
 154         print('error: could not determine glibc version')
 155         exit(1)
 156
 157     def add_all_configs(self):
 158         """Add all known glibc build configurations."""
 159         self.add_config(arch='aarch64',
 160                         os_name='linux-gnu',
 161                         extra_glibcs=[{'variant': 'disable-multi-arch',
 162                                        'cfg': ['--disable-multi-arch']}])
 163         self.add_config(arch='aarch64_be',
 164                         os_name='linux-gnu')
 165         self.add_config(arch='arc',
 166                         os_name='linux-gnu',
 167                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 168         self.add_config(arch='arc',
 169                         os_name='linux-gnuhf',
 170                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38_linux'])
 171         self.add_config(arch='arceb',
 172                         os_name='linux-gnu',
 173                         gcc_cfg=['--disable-multilib', '--with-cpu=hs38'])
 174         self.add_config(arch='alpha',
 175                         os_name='linux-gnu')
 176         self.add_config(arch='arm',
 177                         os_name='linux-gnueabi',
 178                         extra_glibcs=[{'variant': 'v4t',
 179                                        'ccopts': '-march=armv4t'}])
 180         self.add_config(arch='armeb',
 181                         os_name='linux-gnueabi')
 182         self.add_config(arch='armeb',
 183                         os_name='linux-gnueabi',
 184                         variant='be8',
 185                         gcc_cfg=['--with-arch=armv7-a'])
 186         self.add_config(arch='arm',
 187                         os_name='linux-gnueabihf',
 188                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'],
 189                         extra_glibcs=[{'variant': 'v7a',
 190                                        'ccopts': '-march=armv7-a -mfpu=vfpv3'},
 191                                       {'variant': 'v7a-disable-multi-arch',
 192                                        'ccopts': '-march=armv7-a -mfpu=vfpv3',
 193                                        'cfg': ['--disable-multi-arch']}])
 194         self.add_config(arch='armeb',
 195                         os_name='linux-gnueabihf',
 196                         gcc_cfg=['--with-float=hard', '--with-cpu=arm926ej-s'])
 197         self.add_config(arch='armeb',
 198                         os_name='linux-gnueabihf',
 199                         variant='be8',
 200                         gcc_cfg=['--with-float=hard', '--with-arch=armv7-a',
 201                                  '--with-fpu=vfpv3'])
 202         self.add_config(arch='csky',
 203                         os_name='linux-gnuabiv2',
 204                         variant='soft',
 205                         gcc_cfg=['--disable-multilib'])
 206         self.add_config(arch='csky',
 207                         os_name='linux-gnuabiv2',
 208                         gcc_cfg=['--with-float=hard', '--disable-multilib'])
 209         self.add_config(arch='hppa',
 210                         os_name='linux-gnu')
 211         self.add_config(arch='i686',
 212                         os_name='gnu')
 213         self.add_config(arch='ia64',
 214                         os_name='linux-gnu',
 215                         first_gcc_cfg=['--with-system-libunwind'],
 216                         binutils_cfg=['--enable-obsolete'])
 217         self.add_config(arch='m68k',
 218                         os_name='linux-gnu',
 219                         gcc_cfg=['--disable-multilib'])
 220         self.add_config(arch='m68k',
 221                         os_name='linux-gnu',
 222                         variant='coldfire',
 223                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 224         self.add_config(arch='m68k',
 225                         os_name='linux-gnu',
 226                         variant='coldfire-soft',
 227                         gcc_cfg=['--with-arch=cf', '--with-cpu=54455',
 228                                  '--disable-multilib'])
 229         self.add_config(arch='microblaze',
 230                         os_name='linux-gnu',
 231                         gcc_cfg=['--disable-multilib'])
 232         self.add_config(arch='microblazeel',
 233                         os_name='linux-gnu',
 234                         gcc_cfg=['--disable-multilib'])
 235         self.add_config(arch='mips64',
 236                         os_name='linux-gnu',
 237                         gcc_cfg=['--with-mips-plt'],
 238                         glibcs=[{'variant': 'n32'},
 239                                 {'arch': 'mips',
 240                                  'ccopts': '-mabi=32'},
 241                                 {'variant': 'n64',
 242                                  'ccopts': '-mabi=64'}])
 243         self.add_config(arch='mips64',
 244                         os_name='linux-gnu',
 245                         variant='soft',
 246                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 247                         glibcs=[{'variant': 'n32-soft'},
 248                                 {'variant': 'soft',
 249                                  'arch': 'mips',
 250                                  'ccopts': '-mabi=32'},
 251                                 {'variant': 'n64-soft',
 252                                  'ccopts': '-mabi=64'}])
 253         self.add_config(arch='mips64',
 254                         os_name='linux-gnu',
 255                         variant='nan2008',
 256                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 257                                  '--with-arch-64=mips64r2',
 258                                  '--with-arch-32=mips32r2'],
 259                         glibcs=[{'variant': 'n32-nan2008'},
 260                                 {'variant': 'nan2008',
 261                                  'arch': 'mips',
 262                                  'ccopts': '-mabi=32'},
 263                                 {'variant': 'n64-nan2008',
 264                                  'ccopts': '-mabi=64'}])
 265         self.add_config(arch='mips64',
 266                         os_name='linux-gnu',
 267                         variant='nan2008-soft',
 268                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 269                                  '--with-arch-64=mips64r2',
 270                                  '--with-arch-32=mips32r2',
 271                                  '--with-float=soft'],
 272                         glibcs=[{'variant': 'n32-nan2008-soft'},
 273                                 {'variant': 'nan2008-soft',
 274                                  'arch': 'mips',
 275                                  'ccopts': '-mabi=32'},
 276                                 {'variant': 'n64-nan2008-soft',
 277                                  'ccopts': '-mabi=64'}])
 278         self.add_config(arch='mips64el',
 279                         os_name='linux-gnu',
 280                         gcc_cfg=['--with-mips-plt'],
 281                         glibcs=[{'variant': 'n32'},
 282                                 {'arch': 'mipsel',
 283                                  'ccopts': '-mabi=32'},
 284                                 {'variant': 'n64',
 285                                  'ccopts': '-mabi=64'}])
 286         self.add_config(arch='mips64el',
 287                         os_name='linux-gnu',
 288                         variant='soft',
 289                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 290                         glibcs=[{'variant': 'n32-soft'},
 291                                 {'variant': 'soft',
 292                                  'arch': 'mipsel',
 293                                  'ccopts': '-mabi=32'},
 294                                 {'variant': 'n64-soft',
 295                                  'ccopts': '-mabi=64'}])
 296         self.add_config(arch='mips64el',
 297                         os_name='linux-gnu',
 298                         variant='nan2008',
 299                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 300                                  '--with-arch-64=mips64r2',
 301                                  '--with-arch-32=mips32r2'],
 302                         glibcs=[{'variant': 'n32-nan2008'},
 303                                 {'variant': 'nan2008',
 304                                  'arch': 'mipsel',
 305                                  'ccopts': '-mabi=32'},
 306                                 {'variant': 'n64-nan2008',
 307                                  'ccopts': '-mabi=64'}])
 308         self.add_config(arch='mips64el',
 309                         os_name='linux-gnu',
 310                         variant='nan2008-soft',
 311                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 312                                  '--with-arch-64=mips64r2',
 313                                  '--with-arch-32=mips32r2',
 314                                  '--with-float=soft'],
 315                         glibcs=[{'variant': 'n32-nan2008-soft'},
 316                                 {'variant': 'nan2008-soft',
 317                                  'arch': 'mipsel',
 318                                  'ccopts': '-mabi=32'},
 319                                 {'variant': 'n64-nan2008-soft',
 320                                  'ccopts': '-mabi=64'}])
 321         self.add_config(arch='mipsisa64r6el',
 322                         os_name='linux-gnu',
 323                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 324                                  '--with-arch-64=mips64r6',
 325                                  '--with-arch-32=mips32r6',
 326                                  '--with-float=hard'],
 327                         glibcs=[{'variant': 'n32'},
 328                                 {'arch': 'mipsisa32r6el',
 329                                  'ccopts': '-mabi=32'},
 330                                 {'variant': 'n64',
 331                                  'ccopts': '-mabi=64'}])
 332         self.add_config(arch='nios2',
 333                         os_name='linux-gnu')
 334         self.add_config(arch='powerpc',
 335                         os_name='linux-gnu',
 336                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 337                         extra_glibcs=[{'variant': 'power4',
 338                                        'ccopts': '-mcpu=power4',
 339                                        'cfg': ['--with-cpu=power4']}])
 340         self.add_config(arch='powerpc',
 341                         os_name='linux-gnu',
 342                         variant='soft',
 343                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 344                                  '--enable-secureplt'])
 345         self.add_config(arch='powerpc64',
 346                         os_name='linux-gnu',
 347                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 348         self.add_config(arch='powerpc64le',
 349                         os_name='linux-gnu',
 350                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 351         self.add_config(arch='riscv32',
 352                         os_name='linux-gnu',
 353                         variant='rv32imac-ilp32',
 354                         gcc_cfg=['--with-arch=rv32imac', '--with-abi=ilp32',
 355                                  '--disable-multilib'])
 356         self.add_config(arch='riscv32',
 357                         os_name='linux-gnu',
 358                         variant='rv32imafdc-ilp32',
 359                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32',
 360                                  '--disable-multilib'])
 361         self.add_config(arch='riscv32',
 362                         os_name='linux-gnu',
 363                         variant='rv32imafdc-ilp32d',
 364                         gcc_cfg=['--with-arch=rv32imafdc', '--with-abi=ilp32d',
 365                                  '--disable-multilib'])
 366         self.add_config(arch='riscv64',
 367                         os_name='linux-gnu',
 368                         variant='rv64imac-lp64',
 369                         gcc_cfg=['--with-arch=rv64imac', '--with-abi=lp64',
 370                                  '--disable-multilib'])
 371         self.add_config(arch='riscv64',
 372                         os_name='linux-gnu',
 373                         variant='rv64imafdc-lp64',
 374                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64',
 375                                  '--disable-multilib'])
 376         self.add_config(arch='riscv64',
 377                         os_name='linux-gnu',
 378                         variant='rv64imafdc-lp64d',
 379                         gcc_cfg=['--with-arch=rv64imafdc', '--with-abi=lp64d',
 380                                  '--disable-multilib'])
 381         self.add_config(arch='s390x',
 382                         os_name='linux-gnu',
 383                         glibcs=[{},
 384                                 {'arch': 's390', 'ccopts': '-m31'}],
 385                         extra_glibcs=[{'variant': 'O3',
 386                                        'cflags': '-O3'}])
 387         self.add_config(arch='sh3',
 388                         os_name='linux-gnu')
 389         self.add_config(arch='sh3eb',
 390                         os_name='linux-gnu')
 391         self.add_config(arch='sh4',
 392                         os_name='linux-gnu')
 393         self.add_config(arch='sh4eb',
 394                         os_name='linux-gnu')
 395         self.add_config(arch='sh4',
 396                         os_name='linux-gnu',
 397                         variant='soft',
 398                         gcc_cfg=['--without-fp'])
 399         self.add_config(arch='sh4eb',
 400                         os_name='linux-gnu',
 401                         variant='soft',
 402                         gcc_cfg=['--without-fp'])
 403         self.add_config(arch='sparc64',
 404                         os_name='linux-gnu',
 405                         glibcs=[{},
 406                                 {'arch': 'sparcv9',
 407                                  'ccopts': '-m32 -mlong-double-128 -mcpu=v9'}],
 408                         extra_glibcs=[{'variant': 'leon3',
 409                                        'arch' : 'sparcv8',
 410                                        'ccopts' : '-m32 -mlong-double-128 -mcpu=leon3'},
 411                                       {'variant': 'disable-multi-arch',
 412                                        'cfg': ['--disable-multi-arch']},
 413                                       {'variant': 'disable-multi-arch',
 414                                        'arch': 'sparcv9',
 415                                        'ccopts': '-m32 -mlong-double-128 -mcpu=v9',
 416                                        'cfg': ['--disable-multi-arch']}])
 417         self.add_config(arch='x86_64',
 418                         os_name='linux-gnu',
 419                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 420                         glibcs=[{},
 421                                 {'variant': 'x32', 'ccopts': '-mx32'},
 422                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 423                         extra_glibcs=[{'variant': 'disable-multi-arch',
 424                                        'cfg': ['--disable-multi-arch']},
 425                                       {'variant': 'static-pie',
 426                                        'cfg': ['--enable-static-pie']},
 427                                       {'variant': 'x32-static-pie',
 428                                        'ccopts': '-mx32',
 429                                        'cfg': ['--enable-static-pie']},
 430                                       {'variant': 'static-pie',
 431                                        'arch': 'i686',
 432                                        'ccopts': '-m32 -march=i686',
 433                                        'cfg': ['--enable-static-pie']},
 434                                       {'variant': 'disable-multi-arch',
 435                                        'arch': 'i686',
 436                                        'ccopts': '-m32 -march=i686',
 437                                        'cfg': ['--disable-multi-arch']},
 438                                       {'arch': 'i486',
 439                                        'ccopts': '-m32 -march=i486'},
 440                                       {'arch': 'i586',
 441                                        'ccopts': '-m32 -march=i586'}])
 442
 443     def add_config(self, **args):
 444         """Add an individual build configuration."""
 445         cfg = Config(self, **args)
 446         if cfg.name in self.configs:
 447             print('error: duplicate config %s' % cfg.name)
 448             exit(1)
 449         self.configs[cfg.name] = cfg
 450         for c in cfg.all_glibcs:
 451             if c.name in self.glibc_configs:
 452                 print('error: duplicate glibc config %s' % c.name)
 453                 exit(1)
 454             self.glibc_configs[c.name] = c
 455
 456     def component_srcdir(self, component):
 457         """Return the source directory for a given component, e.g. gcc."""
 458         return os.path.join(self.srcdir, component)
 459
 460     def component_builddir(self, action, config, component, subconfig=None):
 461         """Return the directory to use for a build."""
 462         if config is None:
 463             # Host libraries.
 464             assert subconfig is None
 465             return os.path.join(self.builddir, action, component)
 466         if subconfig is None:
 467             return os.path.join(self.builddir, action, config, component)
 468         else:
 469             # glibc build as part of compiler build.
 470             return os.path.join(self.builddir, action, config, component,
 471                                 subconfig)
 472
 473     def compiler_installdir(self, config):
 474         """Return the directory in which to install a compiler."""
 475         return os.path.join(self.installdir, 'compilers', config)
 476
 477     def compiler_bindir(self, config):
 478         """Return the directory in which to find compiler binaries."""
 479         return os.path.join(self.compiler_installdir(config), 'bin')
 480
 481     def compiler_sysroot(self, config):
 482         """Return the sysroot directory for a compiler."""
 483         return os.path.join(self.compiler_installdir(config), 'sysroot')
 484
 485     def glibc_installdir(self, config):
 486         """Return the directory in which to install glibc."""
 487         return os.path.join(self.installdir, 'glibcs', config)
 488
 489     def run_builds(self, action, configs):
 490         """Run the requested builds."""
 491         if action == 'checkout':
 492             self.checkout(configs)
 493             return
 494         if action == 'bot-cycle':
 495             if configs:
 496                 print('error: configurations specified for bot-cycle')
 497                 exit(1)
 498             self.bot_cycle()
 499             return
 500         if action == 'bot':
 501             if configs:
 502                 print('error: configurations specified for bot')
 503                 exit(1)
 504             self.bot()
 505             return
 506         if action in ('host-libraries', 'list-compilers',
 507                       'list-glibcs') and configs:
 508             print('error: configurations specified for ' + action)
 509             exit(1)
 510         if action == 'list-compilers':
 511             for name in sorted(self.configs.keys()):
 512                 print(name)
 513             return
 514         if action == 'list-glibcs':
 515             for config in sorted(self.glibc_configs.values(),
 516                                  key=lambda c: c.name):
 517                 print(config.name, config.compiler.name)
 518             return
 519         self.clear_last_build_state(action)
 520         build_time = datetime.datetime.utcnow()
 521         if action == 'host-libraries':
 522             build_components = ('gmp', 'mpfr', 'mpc')
 523             old_components = ()
 524             old_versions = {}
 525             self.build_host_libraries()
 526         elif action == 'compilers':
 527             build_components = ('binutils', 'gcc', 'glibc', 'linux', 'mig',
 528                                 'gnumach', 'hurd')
 529             old_components = ('gmp', 'mpfr', 'mpc')
 530             old_versions = self.build_state['host-libraries']['build-versions']
 531             self.build_compilers(configs)
 532         else:
 533             build_components = ('glibc',)
 534             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux',
 535                               'mig', 'gnumach', 'hurd')
 536             old_versions = self.build_state['compilers']['build-versions']
 537             if action == 'update-syscalls':
 538                 self.update_syscalls(configs)
 539             else:
 540                 self.build_glibcs(configs)
 541         self.write_files()
 542         self.do_build()
 543         if configs:
 544             # Partial build, do not update stored state.
 545             return
 546         build_versions = {}
 547         for k in build_components:
 548             if k in self.versions:
 549                 build_versions[k] = {'version': self.versions[k]['version'],
 550                                      'revision': self.versions[k]['revision']}
 551         for k in old_components:
 552             if k in old_versions:
 553                 build_versions[k] = {'version': old_versions[k]['version'],
 554                                      'revision': old_versions[k]['revision']}
 555         self.update_build_state(action, build_time, build_versions)
 556
 557     @staticmethod
 558     def remove_dirs(*args):
 559         """Remove directories and their contents if they exist."""
 560         for dir in args:
 561             shutil.rmtree(dir, ignore_errors=True)
 562
 563     @staticmethod
 564     def remove_recreate_dirs(*args):
 565         """Remove directories if they exist, and create them as empty."""
 566         Context.remove_dirs(*args)
 567         for dir in args:
 568             os.makedirs(dir, exist_ok=True)
 569
 570     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 571         """Add makefile text for a list of commands."""
 572         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 573         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 574                                     (target, target, target, commands))
 575         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 576
 577     def write_files(self):
 578         """Write out the Makefile and wrapper script."""
 579         mftext = ''.join(self.makefile_pieces)
 580         with open(self.makefile, 'w') as f:
 581             f.write(mftext)
 582         wrapper_text = (
 583             '#!/bin/sh\n'
 584             'prev_base=$1\n'
 585             'this_base=$2\n'
 586             'desc=$3\n'
 587             'dir=$4\n'
 588             'path=$5\n'
 589             'shift 5\n'
 590             'prev_status=$prev_base-status.txt\n'
 591             'this_status=$this_base-status.txt\n'
 592             'this_log=$this_base-log.txt\n'
 593             'date > "$this_log"\n'
 594             'echo >> "$this_log"\n'
 595             'echo "Description: $desc" >> "$this_log"\n'
 596             'printf "%s" "Command:" >> "$this_log"\n'
 597             'for word in "$@"; do\n'
 598             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 599             '    printf " %s" "$word"\n'
 600             '  else\n'
 601             '    printf " \'"\n'
 602             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 603             '    printf "\'"\n'
 604             '  fi\n'
 605             'done >> "$this_log"\n'
 606             'echo >> "$this_log"\n'
 607             'echo "Directory: $dir" >> "$this_log"\n'
 608             'echo "Path addition: $path" >> "$this_log"\n'
 609             'echo >> "$this_log"\n'
 610             'record_status ()\n'
 611             '{\n'
 612             '  echo >> "$this_log"\n'
 613             '  echo "$1: $desc" > "$this_status"\n'
 614             '  echo "$1: $desc" >> "$this_log"\n'
 615             '  echo >> "$this_log"\n'
 616             '  date >> "$this_log"\n'
 617             '  echo "$1: $desc"\n'
 618             '  exit 0\n'
 619             '}\n'
 620             'check_error ()\n'
 621             '{\n'
 622             '  if [ "$1" != "0" ]; then\n'
 623             '    record_status FAIL\n'
 624             '  fi\n'
 625             '}\n'
 626             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 627             '    record_status UNRESOLVED\n'
 628             'fi\n'
 629             'if [ "$dir" ]; then\n'
 630             '  cd "$dir"\n'
 631             '  check_error "$?"\n'
 632             'fi\n'
 633             'if [ "$path" ]; then\n'
 634             '  PATH=$path:$PATH\n'
 635             'fi\n'
 636             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 637             'check_error "$?"\n'
 638             'record_status PASS\n')
 639         with open(self.wrapper, 'w') as f:
 640             f.write(wrapper_text)
 641         # Mode 0o755.
 642         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 643                      stat.S_IROTH|stat.S_IXOTH)
 644         os.chmod(self.wrapper, mode_exec)
 645         save_logs_text = (
 646             '#!/bin/sh\n'
 647             'if ! [ -f tests.sum ]; then\n'
 648             '  echo "No test summary available."\n'
 649             '  exit 0\n'
 650             'fi\n'
 651             'save_file ()\n'
 652             '{\n'
 653             '  echo "Contents of $1:"\n'
 654             '  echo\n'
 655             '  cat "$1"\n'
 656             '  echo\n'
 657             '  echo "End of contents of $1."\n'
 658             '  echo\n'
 659             '}\n'
 660             'save_file tests.sum\n'
 661             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 662             'for t in $non_pass_tests; do\n'
 663             '  if [ -f "$t.out" ]; then\n'
 664             '    save_file "$t.out"\n'
 665             '  fi\n'
 666             'done\n')
 667         with open(self.save_logs, 'w') as f:
 668             f.write(save_logs_text)
 669         os.chmod(self.save_logs, mode_exec)
 670
 671     def do_build(self):
 672         """Do the actual build."""
 673         cmd = ['make', '-j%d' % self.parallelism]
 674         subprocess.run(cmd, cwd=self.builddir, check=True)
 675
 676     def build_host_libraries(self):
 677         """Build the host libraries."""
 678         installdir = self.host_libraries_installdir
 679         builddir = os.path.join(self.builddir, 'host-libraries')
 680         logsdir = os.path.join(self.logsdir, 'host-libraries')
 681         self.remove_recreate_dirs(installdir, builddir, logsdir)
 682         cmdlist = CommandList('host-libraries', self.keep)
 683         self.build_host_library(cmdlist, 'gmp')
 684         self.build_host_library(cmdlist, 'mpfr',
 685                                 ['--with-gmp=%s' % installdir])
 686         self.build_host_library(cmdlist, 'mpc',
 687                                 ['--with-gmp=%s' % installdir,
 688                                 '--with-mpfr=%s' % installdir])
 689         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 690         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 691
 692     def build_host_library(self, cmdlist, lib, extra_opts=None):
 693         """Build one host library."""
 694         srcdir = self.component_srcdir(lib)
 695         builddir = self.component_builddir('host-libraries', None, lib)
 696         installdir = self.host_libraries_installdir
 697         cmdlist.push_subdesc(lib)
 698         cmdlist.create_use_dir(builddir)
 699         cfg_cmd = [os.path.join(srcdir, 'configure'),
 700                    '--prefix=%s' % installdir,
 701                    '--disable-shared']
 702         if extra_opts:
 703             cfg_cmd.extend (extra_opts)
 704         cmdlist.add_command('configure', cfg_cmd)
 705         cmdlist.add_command('build', ['make'])
 706         cmdlist.add_command('check', ['make', 'check'])
 707         cmdlist.add_command('install', ['make', 'install'])
 708         cmdlist.cleanup_dir()
 709         cmdlist.pop_subdesc()
 710
 711     def build_compilers(self, configs):
 712         """Build the compilers."""
 713         if not configs:
 714             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 715             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 716             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 717             configs = sorted(self.configs.keys())
 718         for c in configs:
 719             self.configs[c].build()
 720
 721     def build_glibcs(self, configs):
 722         """Build the glibcs."""
 723         if not configs:
 724             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 725             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 726             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 727             configs = sorted(self.glibc_configs.keys())
 728         for c in configs:
 729             self.glibc_configs[c].build()
 730
 731     def update_syscalls(self, configs):
 732         """Update the glibc syscall lists."""
 733         if not configs:
 734             self.remove_dirs(os.path.join(self.builddir, 'update-syscalls'))
 735             self.remove_dirs(os.path.join(self.logsdir, 'update-syscalls'))
 736             configs = sorted(self.glibc_configs.keys())
 737         for c in configs:
 738             self.glibc_configs[c].update_syscalls()
 739
 740     def load_versions_json(self):
 741         """Load information about source directory versions."""
 742         if not os.access(self.versions_json, os.F_OK):
 743             self.versions = {}
 744             return
 745         with open(self.versions_json, 'r') as f:
 746             self.versions = json.load(f)
 747
 748     def store_json(self, data, filename):
 749         """Store information in a JSON file."""
 750         filename_tmp = filename + '.tmp'
 751         with open(filename_tmp, 'w') as f:
 752             json.dump(data, f, indent=2, sort_keys=True)
 753         os.rename(filename_tmp, filename)
 754
 755     def store_versions_json(self):
 756         """Store information about source directory versions."""
 757         self.store_json(self.versions, self.versions_json)
 758
 759     def set_component_version(self, component, version, explicit, revision):
 760         """Set the version information for a component."""
 761         self.versions[component] = {'version': version,
 762                                     'explicit': explicit,
 763                                     'revision': revision}
 764         self.store_versions_json()
 765
 766     def checkout(self, versions):
 767         """Check out the desired component versions."""
 768         default_versions = {'binutils': 'vcs-2.35',
 769                             'gcc': 'vcs-10',
 770                             'glibc': 'vcs-mainline',
 771                             'gmp': '6.2.1',
 772                             'linux': '5.10',
 773                             'mpc': '1.2.1',
 774                             'mpfr': '4.1.0',
 775                             'mig': 'vcs-mainline',
 776                             'gnumach': 'vcs-mainline',
 777                             'hurd': 'vcs-mainline'}
 778         use_versions = {}
 779         explicit_versions = {}
 780         for v in versions:
 781             found_v = False
 782             for k in default_versions.keys():
 783                 kx = k + '-'
 784                 if v.startswith(kx):
 785                     vx = v[len(kx):]
 786                     if k in use_versions:
 787                         print('error: multiple versions for %s' % k)
 788                         exit(1)
 789                     use_versions[k] = vx
 790                     explicit_versions[k] = True
 791                     found_v = True
 792                     break
 793             if not found_v:
 794                 print('error: unknown component in %s' % v)
 795                 exit(1)
 796         for k in default_versions.keys():
 797             if k not in use_versions:
 798                 if k in self.versions and self.versions[k]['explicit']:
 799                     use_versions[k] = self.versions[k]['version']
 800                     explicit_versions[k] = True
 801                 else:
 802                     use_versions[k] = default_versions[k]
 803                     explicit_versions[k] = False
 804         os.makedirs(self.srcdir, exist_ok=True)
 805         for k in sorted(default_versions.keys()):
 806             update = os.access(self.component_srcdir(k), os.F_OK)
 807             v = use_versions[k]
 808             if (update and
 809                 k in self.versions and
 810                 v != self.versions[k]['version']):
 811                 if not self.replace_sources:
 812                     print('error: version of %s has changed from %s to %s, '
 813                           'use --replace-sources to check out again' %
 814                           (k, self.versions[k]['version'], v))
 815                     exit(1)
 816                 shutil.rmtree(self.component_srcdir(k))
 817                 update = False
 818             if v.startswith('vcs-'):
 819                 revision = self.checkout_vcs(k, v[4:], update)
 820             else:
 821                 self.checkout_tar(k, v, update)
 822                 revision = v
 823             self.set_component_version(k, v, explicit_versions[k], revision)
 824         if self.get_script_text() != self.script_text:
 825             # Rerun the checkout process in case the updated script
 826             # uses different default versions or new components.
 827             self.exec_self()
 828
 829     def checkout_vcs(self, component, version, update):
 830         """Check out the given version of the given component from version
 831         control.  Return a revision identifier."""
 832         if component == 'binutils':
 833             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 834             if version == 'mainline':
 835                 git_branch = 'master'
 836             else:
 837                 trans = str.maketrans({'.': '_'})
 838                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 839             return self.git_checkout(component, git_url, git_branch, update)
 840         elif component == 'gcc':
 841             if version == 'mainline':
 842                 branch = 'master'
 843             else:
 844                 branch = 'releases/gcc-%s' % version
 845             return self.gcc_checkout(branch, update)
 846         elif component == 'glibc':
 847             git_url = 'git://sourceware.org/git/glibc.git'
 848             if version == 'mainline':
 849                 git_branch = 'master'
 850             else:
 851                 git_branch = 'release/%s/master' % version
 852             r = self.git_checkout(component, git_url, git_branch, update)
 853             self.fix_glibc_timestamps()
 854             return r
 855         elif component == 'gnumach':
 856             git_url = 'git://git.savannah.gnu.org/hurd/gnumach.git'
 857             git_branch = 'master'
 858             r = self.git_checkout(component, git_url, git_branch, update)
 859             subprocess.run(['autoreconf', '-i'],
 860                            cwd=self.component_srcdir(component), check=True)
 861             return r
 862         elif component == 'mig':
 863             git_url = 'git://git.savannah.gnu.org/hurd/mig.git'
 864             git_branch = 'master'
 865             r = self.git_checkout(component, git_url, git_branch, update)
 866             subprocess.run(['autoreconf', '-i'],
 867                            cwd=self.component_srcdir(component), check=True)
 868             return r
 869         elif component == 'hurd':
 870             git_url = 'git://git.savannah.gnu.org/hurd/hurd.git'
 871             git_branch = 'master'
 872             r = self.git_checkout(component, git_url, git_branch, update)
 873             subprocess.run(['autoconf'],
 874                            cwd=self.component_srcdir(component), check=True)
 875             return r
 876         else:
 877             print('error: component %s coming from VCS' % component)
 878             exit(1)
 879
 880     def git_checkout(self, component, git_url, git_branch, update):
 881         """Check out a component from git.  Return a commit identifier."""
 882         if update:
 883             subprocess.run(['git', 'remote', 'prune', 'origin'],
 884                            cwd=self.component_srcdir(component), check=True)
 885             if self.replace_sources:
 886                 subprocess.run(['git', 'clean', '-dxfq'],
 887                                cwd=self.component_srcdir(component), check=True)
 888             subprocess.run(['git', 'pull', '-q'],
 889                            cwd=self.component_srcdir(component), check=True)
 890         else:
 891             if self.shallow:
 892                 depth_arg = ('--depth', '1')
 893             else:
 894                 depth_arg = ()
 895             subprocess.run(['git', 'clone', '-q', '-b', git_branch,
 896                             *depth_arg, git_url,
 897                             self.component_srcdir(component)], check=True)
 898         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 899                            cwd=self.component_srcdir(component),
 900                            stdout=subprocess.PIPE,
 901                            check=True, universal_newlines=True).stdout
 902         return r.rstrip()
 903
 904     def fix_glibc_timestamps(self):
 905         """Fix timestamps in a glibc checkout."""
 906         # Ensure that builds do not try to regenerate generated files
 907         # in the source tree.
 908         srcdir = self.component_srcdir('glibc')
 909         # These files have Makefile dependencies to regenerate them in
 910         # the source tree that may be active during a normal build.
 911         # Some other files have such dependencies but do not need to
 912         # be touched because nothing in a build depends on the files
 913         # in question.
 914         for f in ('sysdeps/mach/hurd/bits/errno.h',):
 915             to_touch = os.path.join(srcdir, f)
 916             subprocess.run(['touch', '-c', to_touch], check=True)
 917         for dirpath, dirnames, filenames in os.walk(srcdir):
 918             for f in filenames:
 919                 if (f == 'configure' or
 920                     f == 'preconfigure' or
 921                     f.endswith('-kw.h')):
 922                     to_touch = os.path.join(dirpath, f)
 923                     subprocess.run(['touch', to_touch], check=True)
 924
 925     def gcc_checkout(self, branch, update):
 926         """Check out GCC from git.  Return the commit identifier."""
 927         if os.access(os.path.join(self.component_srcdir('gcc'), '.svn'),
 928                      os.F_OK):
 929             if not self.replace_sources:
 930                 print('error: GCC has moved from SVN to git, use '
 931                       '--replace-sources to check out again')
 932                 exit(1)
 933             shutil.rmtree(self.component_srcdir('gcc'))
 934             update = False
 935         if not update:
 936             self.git_checkout('gcc', 'git://gcc.gnu.org/git/gcc.git',
 937                               branch, update)
 938         subprocess.run(['contrib/gcc_update', '--silent'],
 939                        cwd=self.component_srcdir('gcc'), check=True)
 940         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 941                            cwd=self.component_srcdir('gcc'),
 942                            stdout=subprocess.PIPE,
 943                            check=True, universal_newlines=True).stdout
 944         return r.rstrip()
 945
 946     def checkout_tar(self, component, version, update):
 947         """Check out the given version of the given component from a
 948         tarball."""
 949         if update:
 950             return
 951         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 952                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.gz',
 953                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 954                    'linux': 'https://www.kernel.org/pub/linux/kernel/v%(major)s.x/linux-%(version)s.tar.xz',
 955                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 956                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz',
 957                    'mig': 'https://ftp.gnu.org/gnu/mig/mig-%(version)s.tar.bz2',
 958                    'gnumach': 'https://ftp.gnu.org/gnu/gnumach/gnumach-%(version)s.tar.bz2',
 959                    'hurd': 'https://ftp.gnu.org/gnu/hurd/hurd-%(version)s.tar.bz2'}
 960         if component not in url_map:
 961             print('error: component %s coming from tarball' % component)
 962             exit(1)
 963         version_major = version.split('.')[0]
 964         url = url_map[component] % {'version': version, 'major': version_major}
 965         filename = os.path.join(self.srcdir, url.split('/')[-1])
 966         response = urllib.request.urlopen(url)
 967         data = response.read()
 968         with open(filename, 'wb') as f:
 969             f.write(data)
 970         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 971                        check=True)
 972         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 973                   self.component_srcdir(component))
 974         os.remove(filename)
 975
 976     def load_build_state_json(self):
 977         """Load information about the state of previous builds."""
 978         if os.access(self.build_state_json, os.F_OK):
 979             with open(self.build_state_json, 'r') as f:
 980                 self.build_state = json.load(f)
 981         else:
 982             self.build_state = {}
 983         for k in ('host-libraries', 'compilers', 'glibcs', 'update-syscalls'):
 984             if k not in self.build_state:
 985                 self.build_state[k] = {}
 986             if 'build-time' not in self.build_state[k]:
 987                 self.build_state[k]['build-time'] = ''
 988             if 'build-versions' not in self.build_state[k]:
 989                 self.build_state[k]['build-versions'] = {}
 990             if 'build-results' not in self.build_state[k]:
 991                 self.build_state[k]['build-results'] = {}
 992             if 'result-changes' not in self.build_state[k]:
 993                 self.build_state[k]['result-changes'] = {}
 994             if 'ever-passed' not in self.build_state[k]:
 995                 self.build_state[k]['ever-passed'] = []
 996
 997     def store_build_state_json(self):
 998         """Store information about the state of previous builds."""
 999         self.store_json(self.build_state, self.build_state_json)
1000
1001     def clear_last_build_state(self, action):
1002         """Clear information about the state of part of the build."""
1003         # We clear the last build time and versions when starting a
1004         # new build.  The results of the last build are kept around,
1005         # as comparison is still meaningful if this build is aborted
1006         # and a new one started.
1007         self.build_state[action]['build-time'] = ''
1008         self.build_state[action]['build-versions'] = {}
1009         self.store_build_state_json()
1010
1011     def update_build_state(self, action, build_time, build_versions):
1012         """Update the build state after a build."""
1013         build_time = build_time.replace(microsecond=0)
1014         self.build_state[action]['build-time'] = str(build_time)
1015         self.build_state[action]['build-versions'] = build_versions
1016         build_results = {}
1017         for log in self.status_log_list:
1018             with open(log, 'r') as f:
1019                 log_text = f.read()
1020             log_text = log_text.rstrip()
1021             m = re.fullmatch('([A-Z]+): (.*)', log_text)
1022             result = m.group(1)
1023             test_name = m.group(2)
1024             assert test_name not in build_results
1025             build_results[test_name] = result
1026         old_build_results = self.build_state[action]['build-results']
1027         self.build_state[action]['build-results'] = build_results
1028         result_changes = {}
1029         all_tests = set(old_build_results.keys()) | set(build_results.keys())
1030         for t in all_tests:
1031             if t in old_build_results:
1032                 old_res = old_build_results[t]
1033             else:
1034                 old_res = '(New test)'
1035             if t in build_results:
1036                 new_res = build_results[t]
1037             else:
1038                 new_res = '(Test removed)'
1039             if old_res != new_res:
1040                 result_changes[t] = '%s -> %s' % (old_res, new_res)
1041         self.build_state[action]['result-changes'] = result_changes
1042         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
1043                            if t in build_results}
1044         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
1045         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
1046                                                          new_passes)
1047         self.store_build_state_json()
1048
1049     def load_bot_config_json(self):
1050         """Load bot configuration."""
1051         with open(self.bot_config_json, 'r') as f:
1052             self.bot_config = json.load(f)
1053
1054     def part_build_old(self, action, delay):
1055         """Return whether the last build for a given action was at least a
1056         given number of seconds ago, or does not have a time recorded."""
1057         old_time_str = self.build_state[action]['build-time']
1058         if not old_time_str:
1059             return True
1060         old_time = datetime.datetime.strptime(old_time_str,
1061                                               '%Y-%m-%d %H:%M:%S')
1062         new_time = datetime.datetime.utcnow()
1063         delta = new_time - old_time
1064         return delta.total_seconds() >= delay
1065
1066     def bot_cycle(self):
1067         """Run a single round of checkout and builds."""
1068         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
1069         self.load_bot_config_json()
1070         actions = ('host-libraries', 'compilers', 'glibcs')
1071         self.bot_run_self(['--replace-sources'], 'checkout')
1072         self.load_versions_json()
1073         if self.get_script_text() != self.script_text:
1074             print('Script changed, re-execing.')
1075             # On script change, all parts of the build should be rerun.
1076             for a in actions:
1077                 self.clear_last_build_state(a)
1078             self.exec_self()
1079         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
1080                             'compilers': ('binutils', 'gcc', 'glibc', 'linux',
1081                                           'mig', 'gnumach', 'hurd'),
1082                             'glibcs': ('glibc',)}
1083         must_build = {}
1084         for a in actions:
1085             build_vers = self.build_state[a]['build-versions']
1086             must_build[a] = False
1087             if not self.build_state[a]['build-time']:
1088                 must_build[a] = True
1089             old_vers = {}
1090             new_vers = {}
1091             for c in check_components[a]:
1092                 if c in build_vers:
1093                     old_vers[c] = build_vers[c]
1094                 new_vers[c] = {'version': self.versions[c]['version'],
1095                                'revision': self.versions[c]['revision']}
1096             if new_vers == old_vers:
1097                 print('Versions for %s unchanged.' % a)
1098             else:
1099                 print('Versions changed or rebuild forced for %s.' % a)
1100                 if a == 'compilers' and not self.part_build_old(
1101                         a, self.bot_config['compilers-rebuild-delay']):
1102                     print('Not requiring rebuild of compilers this soon.')
1103                 else:
1104                     must_build[a] = True
1105         if must_build['host-libraries']:
1106             must_build['compilers'] = True
1107         if must_build['compilers']:
1108             must_build['glibcs'] = True
1109         for a in actions:
1110             if must_build[a]:
1111                 print('Must rebuild %s.' % a)
1112                 self.clear_last_build_state(a)
1113             else:
1114                 print('No need to rebuild %s.' % a)
1115         if os.access(self.logsdir, os.F_OK):
1116             shutil.rmtree(self.logsdir_old, ignore_errors=True)
1117             shutil.copytree(self.logsdir, self.logsdir_old)
1118         for a in actions:
1119             if must_build[a]:
1120                 build_time = datetime.datetime.utcnow()
1121                 print('Rebuilding %s at %s.' % (a, str(build_time)))
1122                 self.bot_run_self([], a)
1123                 self.load_build_state_json()
1124                 self.bot_build_mail(a, build_time)
1125         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
1126
1127     def bot_build_mail(self, action, build_time):
1128         """Send email with the results of a build."""
1129         if not ('email-from' in self.bot_config and
1130                 'email-server' in self.bot_config and
1131                 'email-subject' in self.bot_config and
1132                 'email-to' in self.bot_config):
1133             if not self.email_warning:
1134                 print("Email not configured, not sending.")
1135                 self.email_warning = True
1136             return
1137
1138         build_time = build_time.replace(microsecond=0)
1139         subject = (self.bot_config['email-subject'] %
1140                    {'action': action,
1141                     'build-time': str(build_time)})
1142         results = self.build_state[action]['build-results']
1143         changes = self.build_state[action]['result-changes']
1144         ever_passed = set(self.build_state[action]['ever-passed'])
1145         versions = self.build_state[action]['build-versions']
1146         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
1147         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
1148         all_fails = {k for k in results if results[k] == 'FAIL'}
1149         if new_regressions:
1150             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
1151             new_reg_text = ('New regressions:\n\n%s\n\n' %
1152                             '\n'.join(new_reg_list))
1153         else:
1154             new_reg_text = ''
1155         if all_regressions:
1156             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
1157             all_reg_text = ('All regressions:\n\n%s\n\n' %
1158                             '\n'.join(all_reg_list))
1159         else:
1160             all_reg_text = ''
1161         if all_fails:
1162             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
1163             all_fail_text = ('All failures:\n\n%s\n\n' %
1164                              '\n'.join(all_fail_list))
1165         else:
1166             all_fail_text = ''
1167         if changes:
1168             changes_list = sorted(changes.keys())
1169             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1170             changes_text = ('All changed results:\n\n%s\n\n' %
1171                             '\n'.join(changes_list))
1172         else:
1173             changes_text = ''
1174         results_text = (new_reg_text + all_reg_text + all_fail_text +
1175                         changes_text)
1176         if not results_text:
1177             results_text = 'Clean build with unchanged results.\n\n'
1178         versions_list = sorted(versions.keys())
1179         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1180                                           versions[k]['revision'])
1181                          for k in versions_list]
1182         versions_text = ('Component versions for this build:\n\n%s\n' %
1183                          '\n'.join(versions_list))
1184         body_text = results_text + versions_text
1185         msg = email.mime.text.MIMEText(body_text)
1186         msg['Subject'] = subject
1187         msg['From'] = self.bot_config['email-from']
1188         msg['To'] = self.bot_config['email-to']
1189         msg['Message-ID'] = email.utils.make_msgid()
1190         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1191         with smtplib.SMTP(self.bot_config['email-server']) as s:
1192             s.send_message(msg)
1193
1194     def bot_run_self(self, opts, action, check=True):
1195         """Run a copy of this script with given options."""
1196         cmd = [sys.executable, sys.argv[0], '--keep=none',
1197                '-j%d' % self.parallelism]
1198         if self.full_gcc:
1199             cmd.append('--full-gcc')
1200         cmd.extend(opts)
1201         cmd.extend([self.topdir, action])
1202         sys.stdout.flush()
1203         subprocess.run(cmd, check=check)
1204
1205     def bot(self):
1206         """Run repeated rounds of checkout and builds."""
1207         while True:
1208             self.load_bot_config_json()
1209             if not self.bot_config['run']:
1210                 print('Bot exiting by request.')
1211                 exit(0)
1212             self.bot_run_self([], 'bot-cycle', check=False)
1213             self.load_bot_config_json()
1214             if not self.bot_config['run']:
1215                 print('Bot exiting by request.')
1216                 exit(0)
1217             time.sleep(self.bot_config['delay'])
1218             if self.get_script_text() != self.script_text:
1219                 print('Script changed, bot re-execing.')
1220                 self.exec_self()
1221
1222 class LinuxHeadersPolicyForBuild(object):
1223     """Names and directories for installing Linux headers.  Build variant."""
1224
1225     def __init__(self, config):
1226         self.arch = config.arch
1227         self.srcdir = config.ctx.component_srcdir('linux')
1228         self.builddir = config.component_builddir('linux')
1229         self.headers_dir = os.path.join(config.sysroot, 'usr')
1230
1231 class LinuxHeadersPolicyForUpdateSyscalls(object):
1232     """Names and directories for Linux headers.  update-syscalls variant."""
1233
1234     def __init__(self, glibc, headers_dir):
1235         self.arch = glibc.compiler.arch
1236         self.srcdir = glibc.compiler.ctx.component_srcdir('linux')
1237         self.builddir = glibc.ctx.component_builddir(
1238             'update-syscalls', glibc.name, 'build-linux')
1239         self.headers_dir = headers_dir
1240
1241 def install_linux_headers(policy, cmdlist):
1242     """Install Linux kernel headers."""
1243     arch_map = {'aarch64': 'arm64',
1244                 'alpha': 'alpha',
1245                 'arc': 'arc',
1246                 'arm': 'arm',
1247                 'csky': 'csky',
1248                 'hppa': 'parisc',
1249                 'i486': 'x86',
1250                 'i586': 'x86',
1251                 'i686': 'x86',
1252                 'i786': 'x86',
1253                 'ia64': 'ia64',
1254                 'm68k': 'm68k',
1255                 'microblaze': 'microblaze',
1256                 'mips': 'mips',
1257                 'nios2': 'nios2',
1258                 'powerpc': 'powerpc',
1259                 's390': 's390',
1260                 'riscv32': 'riscv',
1261                 'riscv64': 'riscv',
1262                 'sh': 'sh',
1263                 'sparc': 'sparc',
1264                 'x86_64': 'x86'}
1265     linux_arch = None
1266     for k in arch_map:
1267         if policy.arch.startswith(k):
1268             linux_arch = arch_map[k]
1269             break
1270     assert linux_arch is not None
1271     cmdlist.push_subdesc('linux')
1272     cmdlist.create_use_dir(policy.builddir)
1273     cmdlist.add_command('install-headers',
1274                         ['make', '-C', policy.srcdir, 'O=%s' % policy.builddir,
1275                          'ARCH=%s' % linux_arch,
1276                          'INSTALL_HDR_PATH=%s' % policy.headers_dir,
1277                          'headers_install'])
1278     cmdlist.cleanup_dir()
1279     cmdlist.pop_subdesc()
1280
1281 class Config(object):
1282     """A configuration for building a compiler and associated libraries."""
1283
1284     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1285                  first_gcc_cfg=None, binutils_cfg=None, glibcs=None,
1286                  extra_glibcs=None):
1287         """Initialize a Config object."""
1288         self.ctx = ctx
1289         self.arch = arch
1290         self.os = os_name
1291         self.variant = variant
1292         if variant is None:
1293             self.name = '%s-%s' % (arch, os_name)
1294         else:
1295             self.name = '%s-%s-%s' % (arch, os_name, variant)
1296         self.triplet = '%s-glibc-%s' % (arch, os_name)
1297         if gcc_cfg is None:
1298             self.gcc_cfg = []
1299         else:
1300             self.gcc_cfg = gcc_cfg
1301         if first_gcc_cfg is None:
1302             self.first_gcc_cfg = []
1303         else:
1304             self.first_gcc_cfg = first_gcc_cfg
1305         if binutils_cfg is None:
1306             self.binutils_cfg = []
1307         else:
1308             self.binutils_cfg = binutils_cfg
1309         if glibcs is None:
1310             glibcs = [{'variant': variant}]
1311         if extra_glibcs is None:
1312             extra_glibcs = []
1313         glibcs = [Glibc(self, **g) for g in glibcs]
1314         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1315         self.all_glibcs = glibcs + extra_glibcs
1316         self.compiler_glibcs = glibcs
1317         self.installdir = ctx.compiler_installdir(self.name)
1318         self.bindir = ctx.compiler_bindir(self.name)
1319         self.sysroot = ctx.compiler_sysroot(self.name)
1320         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1321         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1322
1323     def component_builddir(self, component):
1324         """Return the directory to use for a (non-glibc) build."""
1325         return self.ctx.component_builddir('compilers', self.name, component)
1326
1327     def build(self):
1328         """Generate commands to build this compiler."""
1329         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1330                                       self.logsdir)
1331         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1332         cmdlist.add_command('check-host-libraries',
1333                             ['test', '-f',
1334                              os.path.join(self.ctx.host_libraries_installdir,
1335                                           'ok')])
1336         cmdlist.use_path(self.bindir)
1337         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1338                               ['--disable-gdb',
1339                                '--disable-gdbserver',
1340                                '--disable-libdecnumber',
1341                                '--disable-readline',
1342                                '--disable-sim'] + self.binutils_cfg)
1343         if self.os.startswith('linux'):
1344             install_linux_headers(LinuxHeadersPolicyForBuild(self), cmdlist)
1345         self.build_gcc(cmdlist, True)
1346         if self.os == 'gnu':
1347             self.install_gnumach_headers(cmdlist)
1348             self.build_cross_tool(cmdlist, 'mig', 'mig')
1349             self.install_hurd_headers(cmdlist)
1350         for g in self.compiler_glibcs:
1351             cmdlist.push_subdesc('glibc')
1352             cmdlist.push_subdesc(g.name)
1353             g.build_glibc(cmdlist, GlibcPolicyForCompiler(g))
1354             cmdlist.pop_subdesc()
1355             cmdlist.pop_subdesc()
1356         self.build_gcc(cmdlist, False)
1357         cmdlist.add_command('done', ['touch',
1358                                      os.path.join(self.installdir, 'ok')])
1359         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1360                                       self.logsdir)
1361
1362     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1363         """Build one cross tool."""
1364         srcdir = self.ctx.component_srcdir(tool_src)
1365         builddir = self.component_builddir(tool_build)
1366         cmdlist.push_subdesc(tool_build)
1367         cmdlist.create_use_dir(builddir)
1368         cfg_cmd = [os.path.join(srcdir, 'configure'),
1369                    '--prefix=%s' % self.installdir,
1370                    '--build=%s' % self.ctx.build_triplet,
1371                    '--host=%s' % self.ctx.build_triplet,
1372                    '--target=%s' % self.triplet,
1373                    '--with-sysroot=%s' % self.sysroot]
1374         if extra_opts:
1375             cfg_cmd.extend(extra_opts)
1376         cmdlist.add_command('configure', cfg_cmd)
1377         cmdlist.add_command('build', ['make'])
1378         # Parallel "make install" for GCC has race conditions that can
1379         # cause it to fail; see
1380         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=42980>.  Such
1381         # problems are not known for binutils, but doing the
1382         # installation in parallel within a particular toolchain build
1383         # (as opposed to installation of one toolchain from
1384         # build-many-glibcs.py running in parallel to the installation
1385         # of other toolchains being built) is not known to be
1386         # significantly beneficial, so it is simplest just to disable
1387         # parallel install for cross tools here.
1388         cmdlist.add_command('install', ['make', '-j1', 'install'])
1389         cmdlist.cleanup_dir()
1390         cmdlist.pop_subdesc()
1391
1392     def install_gnumach_headers(self, cmdlist):
1393         """Install GNU Mach headers."""
1394         srcdir = self.ctx.component_srcdir('gnumach')
1395         builddir = self.component_builddir('gnumach')
1396         cmdlist.push_subdesc('gnumach')
1397         cmdlist.create_use_dir(builddir)
1398         cmdlist.add_command('configure',
1399                             [os.path.join(srcdir, 'configure'),
1400                              '--build=%s' % self.ctx.build_triplet,
1401                              '--host=%s' % self.triplet,
1402                              '--prefix=',
1403                              'CC=%s-gcc -nostdlib' % self.triplet])
1404         cmdlist.add_command('install', ['make', 'DESTDIR=%s' % self.sysroot,
1405                                         'install-data'])
1406         cmdlist.cleanup_dir()
1407         cmdlist.pop_subdesc()
1408
1409     def install_hurd_headers(self, cmdlist):
1410         """Install Hurd headers."""
1411         srcdir = self.ctx.component_srcdir('hurd')
1412         builddir = self.component_builddir('hurd')
1413         cmdlist.push_subdesc('hurd')
1414         cmdlist.create_use_dir(builddir)
1415         cmdlist.add_command('configure',
1416                             [os.path.join(srcdir, 'configure'),
1417                              '--build=%s' % self.ctx.build_triplet,
1418                              '--host=%s' % self.triplet,
1419                              '--prefix=',
1420                              '--disable-profile', '--without-parted',
1421                              'CC=%s-gcc -nostdlib' % self.triplet])
1422         cmdlist.add_command('install', ['make', 'prefix=%s' % self.sysroot,
1423                                         'no_deps=t', 'install-headers'])
1424         cmdlist.cleanup_dir()
1425         cmdlist.pop_subdesc()
1426
1427     def build_gcc(self, cmdlist, bootstrap):
1428         """Build GCC."""
1429         # libssp is of little relevance with glibc's own stack
1430         # checking support.  libcilkrts does not support GNU/Hurd (and
1431         # has been removed in GCC 8, so --disable-libcilkrts can be
1432         # removed once glibc no longer supports building with older
1433         # GCC versions).
1434         cfg_opts = list(self.gcc_cfg)
1435         cfg_opts += ['--disable-libssp', '--disable-libcilkrts']
1436         host_libs = self.ctx.host_libraries_installdir
1437         cfg_opts += ['--with-gmp=%s' % host_libs,
1438                      '--with-mpfr=%s' % host_libs,
1439                      '--with-mpc=%s' % host_libs]
1440         if bootstrap:
1441             tool_build = 'gcc-first'
1442             # Building a static-only, C-only compiler that is
1443             # sufficient to build glibc.  Various libraries and
1444             # features that may require libc headers must be disabled.
1445             # When configuring with a sysroot, --with-newlib is
1446             # required to define inhibit_libc (to stop some parts of
1447             # libgcc including libc headers); --without-headers is not
1448             # sufficient.
1449             cfg_opts += ['--enable-languages=c', '--disable-shared',
1450                          '--disable-threads',
1451                          '--disable-libatomic',
1452                          '--disable-decimal-float',
1453                          '--disable-libffi',
1454                          '--disable-libgomp',
1455                          '--disable-libitm',
1456                          '--disable-libmpx',
1457                          '--disable-libquadmath',
1458                          '--disable-libsanitizer',
1459                          '--without-headers', '--with-newlib',
1460                          '--with-glibc-version=%s' % self.ctx.glibc_version
1461                          ]
1462             cfg_opts += self.first_gcc_cfg
1463         else:
1464             tool_build = 'gcc'
1465             # libsanitizer commonly breaks because of glibc header
1466             # changes, or on unusual targets.  C++ pre-compiled
1467             # headers are not used during the glibc build and are
1468             # expensive to create.
1469             if not self.ctx.full_gcc:
1470                 cfg_opts += ['--disable-libsanitizer',
1471                              '--disable-libstdcxx-pch']
1472             langs = 'all' if self.ctx.full_gcc else 'c,c++'
1473             cfg_opts += ['--enable-languages=%s' % langs,
1474                          '--enable-shared', '--enable-threads']
1475         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1476
1477 class GlibcPolicyDefault(object):
1478     """Build policy for glibc: common defaults."""
1479
1480     def __init__(self, glibc):
1481         self.srcdir = glibc.ctx.component_srcdir('glibc')
1482         self.use_usr = glibc.os != 'gnu'
1483         self.prefix = '/usr' if self.use_usr else ''
1484         self.configure_args = [
1485             '--prefix=%s' % self.prefix,
1486             '--enable-profile',
1487             '--build=%s' % glibc.ctx.build_triplet,
1488             '--host=%s' % glibc.triplet,
1489             'CC=%s' % glibc.tool_name('gcc'),
1490             'CXX=%s' % glibc.tool_name('g++'),
1491             'AR=%s' % glibc.tool_name('ar'),
1492             'AS=%s' % glibc.tool_name('as'),
1493             'LD=%s' % glibc.tool_name('ld'),
1494             'NM=%s' % glibc.tool_name('nm'),
1495             'OBJCOPY=%s' % glibc.tool_name('objcopy'),
1496             'OBJDUMP=%s' % glibc.tool_name('objdump'),
1497             'RANLIB=%s' % glibc.tool_name('ranlib'),
1498             'READELF=%s' % glibc.tool_name('readelf'),
1499             'STRIP=%s' % glibc.tool_name('strip'),
1500         ]
1501         if glibc.os == 'gnu':
1502             self.configure_args.append('MIG=%s' % glibc.tool_name('mig'))
1503         if glibc.cflags:
1504             self.configure_args.append('CFLAGS=%s' % glibc.cflags)
1505             self.configure_args.append('CXXFLAGS=%s' % glibc.cflags)
1506         self.configure_args += glibc.cfg
1507
1508     def configure(self, cmdlist):
1509         """Invoked to add the configure command to the command list."""
1510         cmdlist.add_command('configure',
1511                             [os.path.join(self.srcdir, 'configure'),
1512                              *self.configure_args])
1513
1514     def extra_commands(self, cmdlist):
1515         """Invoked to inject additional commands (make check) after build."""
1516         pass
1517
1518 class GlibcPolicyForCompiler(GlibcPolicyDefault):
1519     """Build policy for glibc during the compilers stage."""
1520
1521     def __init__(self, glibc):
1522         super().__init__(glibc)
1523         self.builddir = glibc.ctx.component_builddir(
1524             'compilers', glibc.compiler.name, 'glibc', glibc.name)
1525         self.installdir = glibc.compiler.sysroot
1526
1527 class GlibcPolicyForBuild(GlibcPolicyDefault):
1528     """Build policy for glibc during the glibcs stage."""
1529
1530     def __init__(self, glibc):
1531         super().__init__(glibc)
1532         self.builddir = glibc.ctx.component_builddir(
1533             'glibcs', glibc.name, 'glibc')
1534         self.installdir = glibc.ctx.glibc_installdir(glibc.name)
1535         if glibc.ctx.strip:
1536             self.strip = glibc.tool_name('strip')
1537         else:
1538             self.strip = None
1539         self.save_logs = glibc.ctx.save_logs
1540
1541     def extra_commands(self, cmdlist):
1542         if self.strip:
1543             # Avoid picking up libc.so and libpthread.so, which are
1544             # linker scripts stored in /lib on Hurd.  libc and
1545             # libpthread are still stripped via their libc-X.YY.so
1546             # implementation files.
1547             find_command = (('find %s/lib* -name "*.so"'
1548                              + r' \! -name libc.so \! -name libpthread.so')
1549                             % self.installdir)
1550             cmdlist.add_command('strip', ['sh', '-c', ('%s $(%s)' %
1551                                   (self.strip, find_command))])
1552         cmdlist.add_command('check', ['make', 'check'])
1553         cmdlist.add_command('save-logs', [self.save_logs], always_run=True)
1554
1555 class GlibcPolicyForUpdateSyscalls(GlibcPolicyDefault):
1556     """Build policy for glibc during update-syscalls."""
1557
1558     def __init__(self, glibc):
1559         super().__init__(glibc)
1560         self.builddir = glibc.ctx.component_builddir(
1561             'update-syscalls', glibc.name, 'glibc')
1562         self.linuxdir = glibc.ctx.component_builddir(
1563             'update-syscalls', glibc.name, 'linux')
1564         self.linux_policy = LinuxHeadersPolicyForUpdateSyscalls(
1565             glibc, self.linuxdir)
1566         self.configure_args.insert(
1567             0, '--with-headers=%s' % os.path.join(self.linuxdir, 'include'))
1568         # self.installdir not set because installation is not supported
1569
1570 class Glibc(object):
1571     """A configuration for building glibc."""
1572
1573     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1574                  cfg=None, ccopts=None, cflags=None):
1575         """Initialize a Glibc object."""
1576         self.ctx = compiler.ctx
1577         self.compiler = compiler
1578         if arch is None:
1579             self.arch = compiler.arch
1580         else:
1581             self.arch = arch
1582         if os_name is None:
1583             self.os = compiler.os
1584         else:
1585             self.os = os_name
1586         self.variant = variant
1587         if variant is None:
1588             self.name = '%s-%s' % (self.arch, self.os)
1589         else:
1590             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1591         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1592         if cfg is None:
1593             self.cfg = []
1594         else:
1595             self.cfg = cfg
1596         # ccopts contain ABI options and are passed to configure as CC / CXX.
1597         self.ccopts = ccopts
1598         # cflags contain non-ABI options like -g or -O and are passed to
1599         # configure as CFLAGS / CXXFLAGS.
1600         self.cflags = cflags
1601
1602     def tool_name(self, tool):
1603         """Return the name of a cross-compilation tool."""
1604         ctool = '%s-%s' % (self.compiler.triplet, tool)
1605         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1606             ctool = '%s %s' % (ctool, self.ccopts)
1607         return ctool
1608
1609     def build(self):
1610         """Generate commands to build this glibc."""
1611         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1612         installdir = self.ctx.glibc_installdir(self.name)
1613         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1614         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1615         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1616         cmdlist.add_command('check-compilers',
1617                             ['test', '-f',
1618                              os.path.join(self.compiler.installdir, 'ok')])
1619         cmdlist.use_path(self.compiler.bindir)
1620         self.build_glibc(cmdlist, GlibcPolicyForBuild(self))
1621         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1622                                       logsdir)
1623
1624     def build_glibc(self, cmdlist, policy):
1625         """Generate commands to build this glibc, either as part of a compiler
1626         build or with the bootstrapped compiler (and in the latter case, run
1627         tests as well)."""
1628         cmdlist.create_use_dir(policy.builddir)
1629         policy.configure(cmdlist)
1630         cmdlist.add_command('build', ['make'])
1631         cmdlist.add_command('install', ['make', 'install',
1632                                         'install_root=%s' % policy.installdir])
1633         # GCC uses paths such as lib/../lib64, so make sure lib
1634         # directories always exist.
1635         mkdir_cmd = ['mkdir', '-p',
1636                      os.path.join(policy.installdir, 'lib')]
1637         if policy.use_usr:
1638             mkdir_cmd += [os.path.join(policy.installdir, 'usr', 'lib')]
1639         cmdlist.add_command('mkdir-lib', mkdir_cmd)
1640         policy.extra_commands(cmdlist)
1641         cmdlist.cleanup_dir()
1642
1643     def update_syscalls(self):
1644         if self.os == 'gnu':
1645             # Hurd does not have system call tables that need updating.
1646             return
1647
1648         policy = GlibcPolicyForUpdateSyscalls(self)
1649         logsdir = os.path.join(self.ctx.logsdir, 'update-syscalls', self.name)
1650         self.ctx.remove_recreate_dirs(policy.builddir, logsdir)
1651         cmdlist = CommandList('update-syscalls-%s' % self.name, self.ctx.keep)
1652         cmdlist.add_command('check-compilers',
1653                             ['test', '-f',
1654                              os.path.join(self.compiler.installdir, 'ok')])
1655         cmdlist.use_path(self.compiler.bindir)
1656
1657         install_linux_headers(policy.linux_policy, cmdlist)
1658
1659         cmdlist.create_use_dir(policy.builddir)
1660         policy.configure(cmdlist)
1661         cmdlist.add_command('build', ['make', 'update-syscall-lists'])
1662         cmdlist.cleanup_dir()
1663         self.ctx.add_makefile_cmdlist('update-syscalls-%s' % self.name,
1664                                       cmdlist, logsdir)
1665
1666 class Command(object):
1667     """A command run in the build process."""
1668
1669     def __init__(self, desc, num, dir, path, command, always_run=False):
1670         """Initialize a Command object."""
1671         self.dir = dir
1672         self.path = path
1673         self.desc = desc
1674         trans = str.maketrans({' ': '-'})
1675         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1676         self.command = command
1677         self.always_run = always_run
1678
1679     @staticmethod
1680     def shell_make_quote_string(s):
1681         """Given a string not containing a newline, quote it for use by the
1682         shell and make."""
1683         assert '\n' not in s
1684         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1685             return s
1686         strans = str.maketrans({"'": "'\\''"})
1687         s = "'%s'" % s.translate(strans)
1688         mtrans = str.maketrans({'$': '$$'})
1689         return s.translate(mtrans)
1690
1691     @staticmethod
1692     def shell_make_quote_list(l, translate_make):
1693         """Given a list of strings not containing newlines, quote them for use
1694         by the shell and make, returning a single string.  If translate_make
1695         is true and the first string is 'make', change it to $(MAKE)."""
1696         l = [Command.shell_make_quote_string(s) for s in l]
1697         if translate_make and l[0] == 'make':
1698             l[0] = '$(MAKE)'
1699         return ' '.join(l)
1700
1701     def shell_make_quote(self):
1702         """Return this command quoted for the shell and make."""
1703         return self.shell_make_quote_list(self.command, True)
1704
1705
1706 class CommandList(object):
1707     """A list of commands run in the build process."""
1708
1709     def __init__(self, desc, keep):
1710         """Initialize a CommandList object."""
1711         self.cmdlist = []
1712         self.dir = None
1713         self.path = None
1714         self.desc = [desc]
1715         self.keep = keep
1716
1717     def desc_txt(self, desc):
1718         """Return the description to use for a command."""
1719         return '%s %s' % (' '.join(self.desc), desc)
1720
1721     def use_dir(self, dir):
1722         """Set the default directory for subsequent commands."""
1723         self.dir = dir
1724
1725     def use_path(self, path):
1726         """Set a directory to be prepended to the PATH for subsequent
1727         commands."""
1728         self.path = path
1729
1730     def push_subdesc(self, subdesc):
1731         """Set the default subdescription for subsequent commands (e.g., the
1732         name of a component being built, within the series of commands
1733         building it)."""
1734         self.desc.append(subdesc)
1735
1736     def pop_subdesc(self):
1737         """Pop a subdescription from the list of descriptions."""
1738         self.desc.pop()
1739
1740     def create_use_dir(self, dir):
1741         """Remove and recreate a directory and use it for subsequent
1742         commands."""
1743         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1744         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1745         self.use_dir(dir)
1746
1747     def add_command_dir(self, desc, dir, command, always_run=False):
1748         """Add a command to run in a given directory."""
1749         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1750                       command, always_run)
1751         self.cmdlist.append(cmd)
1752
1753     def add_command(self, desc, command, always_run=False):
1754         """Add a command to run in the default directory."""
1755         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1756                       self.path, command, always_run)
1757         self.cmdlist.append(cmd)
1758
1759     def cleanup_dir(self, desc='cleanup', dir=None):
1760         """Clean up a build directory.  If no directory is specified, the
1761         default directory is cleaned up and ceases to be the default
1762         directory."""
1763         if dir is None:
1764             dir = self.dir
1765             self.use_dir(None)
1766         if self.keep != 'all':
1767             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1768                                  always_run=(self.keep == 'none'))
1769
1770     def makefile_commands(self, wrapper, logsdir):
1771         """Return the sequence of commands in the form of text for a Makefile.
1772         The given wrapper script takes arguments: base of logs for
1773         previous command, or empty; base of logs for this command;
1774         description; directory; PATH addition; the command itself."""
1775         # prev_base is the base of the name for logs of the previous
1776         # command that is not always-run (that is, a build command,
1777         # whose failure should stop subsequent build commands from
1778         # being run, as opposed to a cleanup command, which is run
1779         # even if previous commands failed).
1780         prev_base = ''
1781         cmds = []
1782         for c in self.cmdlist:
1783             ctxt = c.shell_make_quote()
1784             if prev_base and not c.always_run:
1785                 prev_log = os.path.join(logsdir, prev_base)
1786             else:
1787                 prev_log = ''
1788             this_log = os.path.join(logsdir, c.logbase)
1789             if not c.always_run:
1790                 prev_base = c.logbase
1791             if c.dir is None:
1792                 dir = ''
1793             else:
1794                 dir = c.dir
1795             if c.path is None:
1796                 path = ''
1797             else:
1798                 path = c.path
1799             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1800             prelim_txt = Command.shell_make_quote_list(prelims, False)
1801             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1802         return '\n'.join(cmds)
1803
1804     def status_logs(self, logsdir):
1805         """Return the list of log files with command status."""
1806         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1807                 for c in self.cmdlist]
1808
1809
1810 def get_parser():
1811     """Return an argument parser for this module."""
1812     parser = argparse.ArgumentParser(description=__doc__)
1813     parser.add_argument('-j', dest='parallelism',
1814                         help='Run this number of jobs in parallel',
1815                         type=int, default=os.cpu_count())
1816     parser.add_argument('--keep', dest='keep',
1817                         help='Whether to keep all build directories, '
1818                         'none or only those from failed builds',
1819                         default='none', choices=('none', 'all', 'failed'))
1820     parser.add_argument('--replace-sources', action='store_true',
1821                         help='Remove and replace source directories '
1822                         'with the wrong version of a component')
1823     parser.add_argument('--strip', action='store_true',
1824                         help='Strip installed glibc libraries')
1825     parser.add_argument('--full-gcc', action='store_true',
1826                         help='Build GCC with all languages and libsanitizer')
1827     parser.add_argument('--shallow', action='store_true',
1828                         help='Do not download Git history during checkout')
1829     parser.add_argument('topdir',
1830                         help='Toplevel working directory')
1831     parser.add_argument('action',
1832                         help='What to do',
1833                         choices=('checkout', 'bot-cycle', 'bot',
1834                                  'host-libraries', 'compilers', 'glibcs',
1835                                  'update-syscalls', 'list-compilers',
1836                                  'list-glibcs'))
1837     parser.add_argument('configs',
1838                         help='Versions to check out or configurations to build',
1839                         nargs='*')
1840     return parser
1841
1842
1843 def main(argv):
1844     """The main entry point."""
1845     parser = get_parser()
1846     opts = parser.parse_args(argv)
1847     topdir = os.path.abspath(opts.topdir)
1848     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1849                   opts.strip, opts.full_gcc, opts.action,
1850                   shallow=opts.shallow)
1851     ctx.run_builds(opts.action, opts.configs)
1852
1853
1854 if __name__ == '__main__':
1855     main(sys.argv[1:])