scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016-2017 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'bot-cycle', to run a series of checkout and build steps,
  26 'bot', to run 'bot-cycle' repeatedly, 'host-libraries', to build
  27 libraries required by the toolchain, 'compilers', to build
  28 cross-compilers for various configurations, or 'glibcs', to build
  29 glibc for various configurations and run the compilation parts of the
  30 testsuite.  Subsequent arguments name the versions of components to
  31 check out (<component>-<version), for 'checkout', or, for actions
  32 other than 'checkout' and 'bot-cycle', name configurations for which
  33 compilers or glibc are to be built.
  34
  35 """
  36
  37 import argparse
  38 import datetime
  39 import email.mime.text
  40 import email.utils
  41 import json
  42 import os
  43 import re
  44 import shutil
  45 import smtplib
  46 import stat
  47 import subprocess
  48 import sys
  49 import time
  50 import urllib.request
  51
  52
  53 class Context(object):
  54     """The global state associated with builds in a given directory."""
  55
  56     def __init__(self, topdir, parallelism, keep, replace_sources, strip,
  57                  action):
  58         """Initialize the context."""
  59         self.topdir = topdir
  60         self.parallelism = parallelism
  61         self.keep = keep
  62         self.replace_sources = replace_sources
  63         self.strip = strip
  64         self.srcdir = os.path.join(topdir, 'src')
  65         self.versions_json = os.path.join(self.srcdir, 'versions.json')
  66         self.build_state_json = os.path.join(topdir, 'build-state.json')
  67         self.bot_config_json = os.path.join(topdir, 'bot-config.json')
  68         self.installdir = os.path.join(topdir, 'install')
  69         self.host_libraries_installdir = os.path.join(self.installdir,
  70                                                       'host-libraries')
  71         self.builddir = os.path.join(topdir, 'build')
  72         self.logsdir = os.path.join(topdir, 'logs')
  73         self.logsdir_old = os.path.join(topdir, 'logs-old')
  74         self.makefile = os.path.join(self.builddir, 'Makefile')
  75         self.wrapper = os.path.join(self.builddir, 'wrapper')
  76         self.save_logs = os.path.join(self.builddir, 'save-logs')
  77         self.script_text = self.get_script_text()
  78         if action != 'checkout':
  79             self.build_triplet = self.get_build_triplet()
  80             self.glibc_version = self.get_glibc_version()
  81         self.configs = {}
  82         self.glibc_configs = {}
  83         self.makefile_pieces = ['.PHONY: all\n']
  84         self.add_all_configs()
  85         self.load_versions_json()
  86         self.load_build_state_json()
  87         self.status_log_list = []
  88
  89     def get_script_text(self):
  90         """Return the text of this script."""
  91         with open(sys.argv[0], 'r') as f:
  92             return f.read()
  93
  94     def exec_self(self):
  95         """Re-execute this script with the same arguments."""
  96         sys.stdout.flush()
  97         os.execv(sys.executable, [sys.executable] + sys.argv)
  98
  99     def get_build_triplet(self):
 100         """Determine the build triplet with config.guess."""
 101         config_guess = os.path.join(self.component_srcdir('gcc'),
 102                                     'config.guess')
 103         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
 104                                 check=True, universal_newlines=True).stdout
 105         return cg_out.rstrip()
 106
 107     def get_glibc_version(self):
 108         """Determine the glibc version number (major.minor)."""
 109         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
 110         with open(version_h, 'r') as f:
 111             lines = f.readlines()
 112         starttext = '#define VERSION "'
 113         for l in lines:
 114             if l.startswith(starttext):
 115                 l = l[len(starttext):]
 116                 l = l.rstrip('"\n')
 117                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 118                 return '%s.%s' % m.group(1, 2)
 119         print('error: could not determine glibc version')
 120         exit(1)
 121
 122     def add_all_configs(self):
 123         """Add all known glibc build configurations."""
 124         # On architectures missing __builtin_trap support, these
 125         # options may be needed as a workaround; see
 126         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70216> for SH.
 127         no_isolate = ('-fno-isolate-erroneous-paths-dereference'
 128                       ' -fno-isolate-erroneous-paths-attribute')
 129         self.add_config(arch='aarch64',
 130                         os_name='linux-gnu')
 131         self.add_config(arch='aarch64_be',
 132                         os_name='linux-gnu')
 133         self.add_config(arch='alpha',
 134                         os_name='linux-gnu')
 135         self.add_config(arch='arm',
 136                         os_name='linux-gnueabi')
 137         self.add_config(arch='armeb',
 138                         os_name='linux-gnueabi')
 139         self.add_config(arch='armeb',
 140                         os_name='linux-gnueabi',
 141                         variant='be8',
 142                         gcc_cfg=['--with-arch=armv7-a'])
 143         self.add_config(arch='arm',
 144                         os_name='linux-gnueabihf')
 145         self.add_config(arch='armeb',
 146                         os_name='linux-gnueabihf')
 147         self.add_config(arch='armeb',
 148                         os_name='linux-gnueabihf',
 149                         variant='be8',
 150                         gcc_cfg=['--with-arch=armv7-a'])
 151         self.add_config(arch='hppa',
 152                         os_name='linux-gnu')
 153         self.add_config(arch='ia64',
 154                         os_name='linux-gnu',
 155                         first_gcc_cfg=['--with-system-libunwind'])
 156         self.add_config(arch='m68k',
 157                         os_name='linux-gnu',
 158                         gcc_cfg=['--disable-multilib'])
 159         self.add_config(arch='m68k',
 160                         os_name='linux-gnu',
 161                         variant='coldfire',
 162                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 163         self.add_config(arch='microblaze',
 164                         os_name='linux-gnu',
 165                         gcc_cfg=['--disable-multilib'])
 166         self.add_config(arch='microblazeel',
 167                         os_name='linux-gnu',
 168                         gcc_cfg=['--disable-multilib'])
 169         self.add_config(arch='mips64',
 170                         os_name='linux-gnu',
 171                         gcc_cfg=['--with-mips-plt'],
 172                         glibcs=[{'variant': 'n32'},
 173                                 {'arch': 'mips',
 174                                  'ccopts': '-mabi=32'},
 175                                 {'variant': 'n64',
 176                                  'ccopts': '-mabi=64'}])
 177         self.add_config(arch='mips64',
 178                         os_name='linux-gnu',
 179                         variant='soft',
 180                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 181                         glibcs=[{'variant': 'n32-soft',
 182                                  'cfg': ['--without-fp']},
 183                                 {'variant': 'soft',
 184                                  'arch': 'mips',
 185                                  'ccopts': '-mabi=32',
 186                                  'cfg': ['--without-fp']},
 187                                 {'variant': 'n64-soft',
 188                                  'ccopts': '-mabi=64',
 189                                  'cfg': ['--without-fp']}])
 190         self.add_config(arch='mips64',
 191                         os_name='linux-gnu',
 192                         variant='nan2008',
 193                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 194                                  '--with-arch-64=mips64r2',
 195                                  '--with-arch-32=mips32r2'],
 196                         glibcs=[{'variant': 'n32-nan2008'},
 197                                 {'variant': 'nan2008',
 198                                  'arch': 'mips',
 199                                  'ccopts': '-mabi=32'},
 200                                 {'variant': 'n64-nan2008',
 201                                  'ccopts': '-mabi=64'}])
 202         self.add_config(arch='mips64',
 203                         os_name='linux-gnu',
 204                         variant='nan2008-soft',
 205                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 206                                  '--with-arch-64=mips64r2',
 207                                  '--with-arch-32=mips32r2',
 208                                  '--with-float=soft'],
 209                         glibcs=[{'variant': 'n32-nan2008-soft',
 210                                  'cfg': ['--without-fp']},
 211                                 {'variant': 'nan2008-soft',
 212                                  'arch': 'mips',
 213                                  'ccopts': '-mabi=32',
 214                                  'cfg': ['--without-fp']},
 215                                 {'variant': 'n64-nan2008-soft',
 216                                  'ccopts': '-mabi=64',
 217                                  'cfg': ['--without-fp']}])
 218         self.add_config(arch='mips64el',
 219                         os_name='linux-gnu',
 220                         gcc_cfg=['--with-mips-plt'],
 221                         glibcs=[{'variant': 'n32'},
 222                                 {'arch': 'mipsel',
 223                                  'ccopts': '-mabi=32'},
 224                                 {'variant': 'n64',
 225                                  'ccopts': '-mabi=64'}])
 226         self.add_config(arch='mips64el',
 227                         os_name='linux-gnu',
 228                         variant='soft',
 229                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 230                         glibcs=[{'variant': 'n32-soft',
 231                                  'cfg': ['--without-fp']},
 232                                 {'variant': 'soft',
 233                                  'arch': 'mipsel',
 234                                  'ccopts': '-mabi=32',
 235                                  'cfg': ['--without-fp']},
 236                                 {'variant': 'n64-soft',
 237                                  'ccopts': '-mabi=64',
 238                                  'cfg': ['--without-fp']}])
 239         self.add_config(arch='mips64el',
 240                         os_name='linux-gnu',
 241                         variant='nan2008',
 242                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 243                                  '--with-arch-64=mips64r2',
 244                                  '--with-arch-32=mips32r2'],
 245                         glibcs=[{'variant': 'n32-nan2008'},
 246                                 {'variant': 'nan2008',
 247                                  'arch': 'mipsel',
 248                                  'ccopts': '-mabi=32'},
 249                                 {'variant': 'n64-nan2008',
 250                                  'ccopts': '-mabi=64'}])
 251         self.add_config(arch='mips64el',
 252                         os_name='linux-gnu',
 253                         variant='nan2008-soft',
 254                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 255                                  '--with-arch-64=mips64r2',
 256                                  '--with-arch-32=mips32r2',
 257                                  '--with-float=soft'],
 258                         glibcs=[{'variant': 'n32-nan2008-soft',
 259                                  'cfg': ['--without-fp']},
 260                                 {'variant': 'nan2008-soft',
 261                                  'arch': 'mipsel',
 262                                  'ccopts': '-mabi=32',
 263                                  'cfg': ['--without-fp']},
 264                                 {'variant': 'n64-nan2008-soft',
 265                                  'ccopts': '-mabi=64',
 266                                  'cfg': ['--without-fp']}])
 267         self.add_config(arch='nios2',
 268                         os_name='linux-gnu')
 269         self.add_config(arch='powerpc',
 270                         os_name='linux-gnu',
 271                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 272                         extra_glibcs=[{'variant': 'power4',
 273                                        'ccopts': '-mcpu=power4',
 274                                        'cfg': ['--with-cpu=power4']}])
 275         self.add_config(arch='powerpc',
 276                         os_name='linux-gnu',
 277                         variant='soft',
 278                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 279                                  '--enable-secureplt'],
 280                         glibcs=[{'variant': 'soft', 'cfg': ['--without-fp']}])
 281         self.add_config(arch='powerpc64',
 282                         os_name='linux-gnu',
 283                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 284         self.add_config(arch='powerpc64le',
 285                         os_name='linux-gnu',
 286                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 287         self.add_config(arch='powerpc',
 288                         os_name='linux-gnuspe',
 289                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 290                                  '--enable-e500-double'],
 291                         glibcs=[{'cfg': ['--without-fp']}])
 292         self.add_config(arch='powerpc',
 293                         os_name='linux-gnuspe',
 294                         variant='e500v1',
 295                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 296                         glibcs=[{'variant': 'e500v1', 'cfg': ['--without-fp']}])
 297         self.add_config(arch='s390x',
 298                         os_name='linux-gnu',
 299                         glibcs=[{},
 300                                 {'arch': 's390', 'ccopts': '-m31'}])
 301         self.add_config(arch='sh3',
 302                         os_name='linux-gnu',
 303                         glibcs=[{'ccopts': no_isolate}])
 304         self.add_config(arch='sh3eb',
 305                         os_name='linux-gnu',
 306                         glibcs=[{'ccopts': no_isolate}])
 307         self.add_config(arch='sh4',
 308                         os_name='linux-gnu',
 309                         glibcs=[{'ccopts': no_isolate}])
 310         self.add_config(arch='sh4eb',
 311                         os_name='linux-gnu',
 312                         glibcs=[{'ccopts': no_isolate}])
 313         self.add_config(arch='sh4',
 314                         os_name='linux-gnu',
 315                         variant='soft',
 316                         gcc_cfg=['--without-fp'],
 317                         glibcs=[{'variant': 'soft',
 318                                  'cfg': ['--without-fp'],
 319                                  'ccopts': no_isolate}])
 320         self.add_config(arch='sh4eb',
 321                         os_name='linux-gnu',
 322                         variant='soft',
 323                         gcc_cfg=['--without-fp'],
 324                         glibcs=[{'variant': 'soft',
 325                                  'cfg': ['--without-fp'],
 326                                  'ccopts': no_isolate}])
 327         self.add_config(arch='sparc64',
 328                         os_name='linux-gnu',
 329                         glibcs=[{},
 330                                 {'arch': 'sparcv9',
 331                                  'ccopts': '-m32 -mlong-double-128'}])
 332         self.add_config(arch='tilegx',
 333                         os_name='linux-gnu',
 334                         glibcs=[{},
 335                                 {'variant': '32', 'ccopts': '-m32'}])
 336         self.add_config(arch='tilegxbe',
 337                         os_name='linux-gnu',
 338                         glibcs=[{},
 339                                 {'variant': '32', 'ccopts': '-m32'}])
 340         self.add_config(arch='tilepro',
 341                         os_name='linux-gnu')
 342         self.add_config(arch='x86_64',
 343                         os_name='linux-gnu',
 344                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 345                         glibcs=[{},
 346                                 {'variant': 'x32', 'ccopts': '-mx32'},
 347                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 348                         extra_glibcs=[{'variant': 'disable-multi-arch',
 349                                        'cfg': ['--disable-multi-arch']},
 350                                       {'variant': 'disable-multi-arch',
 351                                        'arch': 'i686',
 352                                        'ccopts': '-m32 -march=i686',
 353                                        'cfg': ['--disable-multi-arch']},
 354                                       {'arch': 'i486',
 355                                        'ccopts': '-m32 -march=i486'},
 356                                       {'arch': 'i586',
 357                                        'ccopts': '-m32 -march=i586'}])
 358
 359     def add_config(self, **args):
 360         """Add an individual build configuration."""
 361         cfg = Config(self, **args)
 362         if cfg.name in self.configs:
 363             print('error: duplicate config %s' % cfg.name)
 364             exit(1)
 365         self.configs[cfg.name] = cfg
 366         for c in cfg.all_glibcs:
 367             if c.name in self.glibc_configs:
 368                 print('error: duplicate glibc config %s' % c.name)
 369                 exit(1)
 370             self.glibc_configs[c.name] = c
 371
 372     def component_srcdir(self, component):
 373         """Return the source directory for a given component, e.g. gcc."""
 374         return os.path.join(self.srcdir, component)
 375
 376     def component_builddir(self, action, config, component, subconfig=None):
 377         """Return the directory to use for a build."""
 378         if config is None:
 379             # Host libraries.
 380             assert subconfig is None
 381             return os.path.join(self.builddir, action, component)
 382         if subconfig is None:
 383             return os.path.join(self.builddir, action, config, component)
 384         else:
 385             # glibc build as part of compiler build.
 386             return os.path.join(self.builddir, action, config, component,
 387                                 subconfig)
 388
 389     def compiler_installdir(self, config):
 390         """Return the directory in which to install a compiler."""
 391         return os.path.join(self.installdir, 'compilers', config)
 392
 393     def compiler_bindir(self, config):
 394         """Return the directory in which to find compiler binaries."""
 395         return os.path.join(self.compiler_installdir(config), 'bin')
 396
 397     def compiler_sysroot(self, config):
 398         """Return the sysroot directory for a compiler."""
 399         return os.path.join(self.compiler_installdir(config), 'sysroot')
 400
 401     def glibc_installdir(self, config):
 402         """Return the directory in which to install glibc."""
 403         return os.path.join(self.installdir, 'glibcs', config)
 404
 405     def run_builds(self, action, configs):
 406         """Run the requested builds."""
 407         if action == 'checkout':
 408             self.checkout(configs)
 409             return
 410         if action == 'bot-cycle':
 411             if configs:
 412                 print('error: configurations specified for bot-cycle')
 413                 exit(1)
 414             self.bot_cycle()
 415             return
 416         if action == 'bot':
 417             if configs:
 418                 print('error: configurations specified for bot')
 419                 exit(1)
 420             self.bot()
 421             return
 422         if action == 'host-libraries' and configs:
 423             print('error: configurations specified for host-libraries')
 424             exit(1)
 425         self.clear_last_build_state(action)
 426         build_time = datetime.datetime.utcnow()
 427         if action == 'host-libraries':
 428             build_components = ('gmp', 'mpfr', 'mpc')
 429             old_components = ()
 430             old_versions = {}
 431             self.build_host_libraries()
 432         elif action == 'compilers':
 433             build_components = ('binutils', 'gcc', 'glibc', 'linux')
 434             old_components = ('gmp', 'mpfr', 'mpc')
 435             old_versions = self.build_state['host-libraries']['build-versions']
 436             self.build_compilers(configs)
 437         else:
 438             build_components = ('glibc',)
 439             old_components = ('gmp', 'mpfr', 'mpc', 'binutils', 'gcc', 'linux')
 440             old_versions = self.build_state['compilers']['build-versions']
 441             self.build_glibcs(configs)
 442         self.write_files()
 443         self.do_build()
 444         if configs:
 445             # Partial build, do not update stored state.
 446             return
 447         build_versions = {}
 448         for k in build_components:
 449             if k in self.versions:
 450                 build_versions[k] = {'version': self.versions[k]['version'],
 451                                      'revision': self.versions[k]['revision']}
 452         for k in old_components:
 453             if k in old_versions:
 454                 build_versions[k] = {'version': old_versions[k]['version'],
 455                                      'revision': old_versions[k]['revision']}
 456         self.update_build_state(action, build_time, build_versions)
 457
 458     @staticmethod
 459     def remove_dirs(*args):
 460         """Remove directories and their contents if they exist."""
 461         for dir in args:
 462             shutil.rmtree(dir, ignore_errors=True)
 463
 464     @staticmethod
 465     def remove_recreate_dirs(*args):
 466         """Remove directories if they exist, and create them as empty."""
 467         Context.remove_dirs(*args)
 468         for dir in args:
 469             os.makedirs(dir, exist_ok=True)
 470
 471     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 472         """Add makefile text for a list of commands."""
 473         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 474         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 475                                     (target, target, target, commands))
 476         self.status_log_list.extend(cmdlist.status_logs(logsdir))
 477
 478     def write_files(self):
 479         """Write out the Makefile and wrapper script."""
 480         mftext = ''.join(self.makefile_pieces)
 481         with open(self.makefile, 'w') as f:
 482             f.write(mftext)
 483         wrapper_text = (
 484             '#!/bin/sh\n'
 485             'prev_base=$1\n'
 486             'this_base=$2\n'
 487             'desc=$3\n'
 488             'dir=$4\n'
 489             'path=$5\n'
 490             'shift 5\n'
 491             'prev_status=$prev_base-status.txt\n'
 492             'this_status=$this_base-status.txt\n'
 493             'this_log=$this_base-log.txt\n'
 494             'date > "$this_log"\n'
 495             'echo >> "$this_log"\n'
 496             'echo "Description: $desc" >> "$this_log"\n'
 497             'printf "%s" "Command:" >> "$this_log"\n'
 498             'for word in "$@"; do\n'
 499             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 500             '    printf " %s" "$word"\n'
 501             '  else\n'
 502             '    printf " \'"\n'
 503             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 504             '    printf "\'"\n'
 505             '  fi\n'
 506             'done >> "$this_log"\n'
 507             'echo >> "$this_log"\n'
 508             'echo "Directory: $dir" >> "$this_log"\n'
 509             'echo "Path addition: $path" >> "$this_log"\n'
 510             'echo >> "$this_log"\n'
 511             'record_status ()\n'
 512             '{\n'
 513             '  echo >> "$this_log"\n'
 514             '  echo "$1: $desc" > "$this_status"\n'
 515             '  echo "$1: $desc" >> "$this_log"\n'
 516             '  echo >> "$this_log"\n'
 517             '  date >> "$this_log"\n'
 518             '  echo "$1: $desc"\n'
 519             '  exit 0\n'
 520             '}\n'
 521             'check_error ()\n'
 522             '{\n'
 523             '  if [ "$1" != "0" ]; then\n'
 524             '    record_status FAIL\n'
 525             '  fi\n'
 526             '}\n'
 527             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 528             '    record_status UNRESOLVED\n'
 529             'fi\n'
 530             'if [ "$dir" ]; then\n'
 531             '  cd "$dir"\n'
 532             '  check_error "$?"\n'
 533             'fi\n'
 534             'if [ "$path" ]; then\n'
 535             '  PATH=$path:$PATH\n'
 536             'fi\n'
 537             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 538             'check_error "$?"\n'
 539             'record_status PASS\n')
 540         with open(self.wrapper, 'w') as f:
 541             f.write(wrapper_text)
 542         # Mode 0o755.
 543         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 544                      stat.S_IROTH|stat.S_IXOTH)
 545         os.chmod(self.wrapper, mode_exec)
 546         save_logs_text = (
 547             '#!/bin/sh\n'
 548             'if ! [ -f tests.sum ]; then\n'
 549             '  echo "No test summary available."\n'
 550             '  exit 0\n'
 551             'fi\n'
 552             'save_file ()\n'
 553             '{\n'
 554             '  echo "Contents of $1:"\n'
 555             '  echo\n'
 556             '  cat "$1"\n'
 557             '  echo\n'
 558             '  echo "End of contents of $1."\n'
 559             '  echo\n'
 560             '}\n'
 561             'save_file tests.sum\n'
 562             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 563             'for t in $non_pass_tests; do\n'
 564             '  if [ -f "$t.out" ]; then\n'
 565             '    save_file "$t.out"\n'
 566             '  fi\n'
 567             'done\n')
 568         with open(self.save_logs, 'w') as f:
 569             f.write(save_logs_text)
 570         os.chmod(self.save_logs, mode_exec)
 571
 572     def do_build(self):
 573         """Do the actual build."""
 574         cmd = ['make', '-j%d' % self.parallelism]
 575         subprocess.run(cmd, cwd=self.builddir, check=True)
 576
 577     def build_host_libraries(self):
 578         """Build the host libraries."""
 579         installdir = self.host_libraries_installdir
 580         builddir = os.path.join(self.builddir, 'host-libraries')
 581         logsdir = os.path.join(self.logsdir, 'host-libraries')
 582         self.remove_recreate_dirs(installdir, builddir, logsdir)
 583         cmdlist = CommandList('host-libraries', self.keep)
 584         self.build_host_library(cmdlist, 'gmp')
 585         self.build_host_library(cmdlist, 'mpfr',
 586                                 ['--with-gmp=%s' % installdir])
 587         self.build_host_library(cmdlist, 'mpc',
 588                                 ['--with-gmp=%s' % installdir,
 589                                 '--with-mpfr=%s' % installdir])
 590         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 591         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 592
 593     def build_host_library(self, cmdlist, lib, extra_opts=None):
 594         """Build one host library."""
 595         srcdir = self.component_srcdir(lib)
 596         builddir = self.component_builddir('host-libraries', None, lib)
 597         installdir = self.host_libraries_installdir
 598         cmdlist.push_subdesc(lib)
 599         cmdlist.create_use_dir(builddir)
 600         cfg_cmd = [os.path.join(srcdir, 'configure'),
 601                    '--prefix=%s' % installdir,
 602                    '--disable-shared']
 603         if extra_opts:
 604             cfg_cmd.extend (extra_opts)
 605         cmdlist.add_command('configure', cfg_cmd)
 606         cmdlist.add_command('build', ['make'])
 607         cmdlist.add_command('check', ['make', 'check'])
 608         cmdlist.add_command('install', ['make', 'install'])
 609         cmdlist.cleanup_dir()
 610         cmdlist.pop_subdesc()
 611
 612     def build_compilers(self, configs):
 613         """Build the compilers."""
 614         if not configs:
 615             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 616             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 617             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 618             configs = sorted(self.configs.keys())
 619         for c in configs:
 620             self.configs[c].build()
 621
 622     def build_glibcs(self, configs):
 623         """Build the glibcs."""
 624         if not configs:
 625             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 626             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 627             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 628             configs = sorted(self.glibc_configs.keys())
 629         for c in configs:
 630             self.glibc_configs[c].build()
 631
 632     def load_versions_json(self):
 633         """Load information about source directory versions."""
 634         if not os.access(self.versions_json, os.F_OK):
 635             self.versions = {}
 636             return
 637         with open(self.versions_json, 'r') as f:
 638             self.versions = json.load(f)
 639
 640     def store_json(self, data, filename):
 641         """Store information in a JSON file."""
 642         filename_tmp = filename + '.tmp'
 643         with open(filename_tmp, 'w') as f:
 644             json.dump(data, f, indent=2, sort_keys=True)
 645         os.rename(filename_tmp, filename)
 646
 647     def store_versions_json(self):
 648         """Store information about source directory versions."""
 649         self.store_json(self.versions, self.versions_json)
 650
 651     def set_component_version(self, component, version, explicit, revision):
 652         """Set the version information for a component."""
 653         self.versions[component] = {'version': version,
 654                                     'explicit': explicit,
 655                                     'revision': revision}
 656         self.store_versions_json()
 657
 658     def checkout(self, versions):
 659         """Check out the desired component versions."""
 660         default_versions = {'binutils': 'vcs-2.28',
 661                             'gcc': 'vcs-6',
 662                             'glibc': 'vcs-mainline',
 663                             'gmp': '6.1.1',
 664                             'linux': '4.9',
 665                             'mpc': '1.0.3',
 666                             'mpfr': '3.1.5'}
 667         use_versions = {}
 668         explicit_versions = {}
 669         for v in versions:
 670             found_v = False
 671             for k in default_versions.keys():
 672                 kx = k + '-'
 673                 if v.startswith(kx):
 674                     vx = v[len(kx):]
 675                     if k in use_versions:
 676                         print('error: multiple versions for %s' % k)
 677                         exit(1)
 678                     use_versions[k] = vx
 679                     explicit_versions[k] = True
 680                     found_v = True
 681                     break
 682             if not found_v:
 683                 print('error: unknown component in %s' % v)
 684                 exit(1)
 685         for k in default_versions.keys():
 686             if k not in use_versions:
 687                 if k in self.versions and self.versions[k]['explicit']:
 688                     use_versions[k] = self.versions[k]['version']
 689                     explicit_versions[k] = True
 690                 else:
 691                     use_versions[k] = default_versions[k]
 692                     explicit_versions[k] = False
 693         os.makedirs(self.srcdir, exist_ok=True)
 694         for k in sorted(default_versions.keys()):
 695             update = os.access(self.component_srcdir(k), os.F_OK)
 696             v = use_versions[k]
 697             if (update and
 698                 k in self.versions and
 699                 v != self.versions[k]['version']):
 700                 if not self.replace_sources:
 701                     print('error: version of %s has changed from %s to %s, '
 702                           'use --replace-sources to check out again' %
 703                           (k, self.versions[k]['version'], v))
 704                     exit(1)
 705                 shutil.rmtree(self.component_srcdir(k))
 706                 update = False
 707             if v.startswith('vcs-'):
 708                 revision = self.checkout_vcs(k, v[4:], update)
 709             else:
 710                 self.checkout_tar(k, v, update)
 711                 revision = v
 712             self.set_component_version(k, v, explicit_versions[k], revision)
 713         if self.get_script_text() != self.script_text:
 714             # Rerun the checkout process in case the updated script
 715             # uses different default versions or new components.
 716             self.exec_self()
 717
 718     def checkout_vcs(self, component, version, update):
 719         """Check out the given version of the given component from version
 720         control.  Return a revision identifier."""
 721         if component == 'binutils':
 722             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 723             if version == 'mainline':
 724                 git_branch = 'master'
 725             else:
 726                 trans = str.maketrans({'.': '_'})
 727                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 728             return self.git_checkout(component, git_url, git_branch, update)
 729         elif component == 'gcc':
 730             if version == 'mainline':
 731                 branch = 'trunk'
 732             else:
 733                 trans = str.maketrans({'.': '_'})
 734                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 735             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 736             return self.gcc_checkout(svn_url, update)
 737         elif component == 'glibc':
 738             git_url = 'git://sourceware.org/git/glibc.git'
 739             if version == 'mainline':
 740                 git_branch = 'master'
 741             else:
 742                 git_branch = 'release/%s/master' % version
 743             r = self.git_checkout(component, git_url, git_branch, update)
 744             self.fix_glibc_timestamps()
 745             return r
 746         else:
 747             print('error: component %s coming from VCS' % component)
 748             exit(1)
 749
 750     def git_checkout(self, component, git_url, git_branch, update):
 751         """Check out a component from git.  Return a commit identifier."""
 752         if update:
 753             subprocess.run(['git', 'remote', 'prune', 'origin'],
 754                            cwd=self.component_srcdir(component), check=True)
 755             subprocess.run(['git', 'pull', '-q'],
 756                            cwd=self.component_srcdir(component), check=True)
 757         else:
 758             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 759                             self.component_srcdir(component)], check=True)
 760         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 761                            cwd=self.component_srcdir(component),
 762                            stdout=subprocess.PIPE,
 763                            check=True, universal_newlines=True).stdout
 764         return r.rstrip()
 765
 766     def fix_glibc_timestamps(self):
 767         """Fix timestamps in a glibc checkout."""
 768         # Ensure that builds do not try to regenerate generated files
 769         # in the source tree.
 770         srcdir = self.component_srcdir('glibc')
 771         for dirpath, dirnames, filenames in os.walk(srcdir):
 772             for f in filenames:
 773                 if (f == 'configure' or
 774                     f == 'preconfigure' or
 775                     f.endswith('-kw.h')):
 776                     to_touch = os.path.join(dirpath, f)
 777                     subprocess.run(['touch', to_touch], check=True)
 778
 779     def gcc_checkout(self, svn_url, update):
 780         """Check out GCC from SVN.  Return the revision number."""
 781         if not update:
 782             subprocess.run(['svn', 'co', '-q', svn_url,
 783                             self.component_srcdir('gcc')], check=True)
 784         subprocess.run(['contrib/gcc_update', '--silent'],
 785                        cwd=self.component_srcdir('gcc'), check=True)
 786         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 787                            stdout=subprocess.PIPE,
 788                            check=True, universal_newlines=True).stdout
 789         return r.rstrip()
 790
 791     def checkout_tar(self, component, version, update):
 792         """Check out the given version of the given component from a
 793         tarball."""
 794         if update:
 795             return
 796         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 797                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 798                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 799                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 800                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 801                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'}
 802         if component not in url_map:
 803             print('error: component %s coming from tarball' % component)
 804             exit(1)
 805         url = url_map[component] % {'version': version}
 806         filename = os.path.join(self.srcdir, url.split('/')[-1])
 807         response = urllib.request.urlopen(url)
 808         data = response.read()
 809         with open(filename, 'wb') as f:
 810             f.write(data)
 811         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 812                        check=True)
 813         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 814                   self.component_srcdir(component))
 815         os.remove(filename)
 816
 817     def load_build_state_json(self):
 818         """Load information about the state of previous builds."""
 819         if os.access(self.build_state_json, os.F_OK):
 820             with open(self.build_state_json, 'r') as f:
 821                 self.build_state = json.load(f)
 822         else:
 823             self.build_state = {}
 824         for k in ('host-libraries', 'compilers', 'glibcs'):
 825             if k not in self.build_state:
 826                 self.build_state[k] = {}
 827             if 'build-time' not in self.build_state[k]:
 828                 self.build_state[k]['build-time'] = ''
 829             if 'build-versions' not in self.build_state[k]:
 830                 self.build_state[k]['build-versions'] = {}
 831             if 'build-results' not in self.build_state[k]:
 832                 self.build_state[k]['build-results'] = {}
 833             if 'result-changes' not in self.build_state[k]:
 834                 self.build_state[k]['result-changes'] = {}
 835             if 'ever-passed' not in self.build_state[k]:
 836                 self.build_state[k]['ever-passed'] = []
 837
 838     def store_build_state_json(self):
 839         """Store information about the state of previous builds."""
 840         self.store_json(self.build_state, self.build_state_json)
 841
 842     def clear_last_build_state(self, action):
 843         """Clear information about the state of part of the build."""
 844         # We clear the last build time and versions when starting a
 845         # new build.  The results of the last build are kept around,
 846         # as comparison is still meaningful if this build is aborted
 847         # and a new one started.
 848         self.build_state[action]['build-time'] = ''
 849         self.build_state[action]['build-versions'] = {}
 850         self.store_build_state_json()
 851
 852     def update_build_state(self, action, build_time, build_versions):
 853         """Update the build state after a build."""
 854         build_time = build_time.replace(microsecond=0)
 855         self.build_state[action]['build-time'] = str(build_time)
 856         self.build_state[action]['build-versions'] = build_versions
 857         build_results = {}
 858         for log in self.status_log_list:
 859             with open(log, 'r') as f:
 860                 log_text = f.read()
 861             log_text = log_text.rstrip()
 862             m = re.fullmatch('([A-Z]+): (.*)', log_text)
 863             result = m.group(1)
 864             test_name = m.group(2)
 865             assert test_name not in build_results
 866             build_results[test_name] = result
 867         old_build_results = self.build_state[action]['build-results']
 868         self.build_state[action]['build-results'] = build_results
 869         result_changes = {}
 870         all_tests = set(old_build_results.keys()) | set(build_results.keys())
 871         for t in all_tests:
 872             if t in old_build_results:
 873                 old_res = old_build_results[t]
 874             else:
 875                 old_res = '(New test)'
 876             if t in build_results:
 877                 new_res = build_results[t]
 878             else:
 879                 new_res = '(Test removed)'
 880             if old_res != new_res:
 881                 result_changes[t] = '%s -> %s' % (old_res, new_res)
 882         self.build_state[action]['result-changes'] = result_changes
 883         old_ever_passed = {t for t in self.build_state[action]['ever-passed']
 884                            if t in build_results}
 885         new_passes = {t for t in build_results if build_results[t] == 'PASS'}
 886         self.build_state[action]['ever-passed'] = sorted(old_ever_passed |
 887                                                          new_passes)
 888         self.store_build_state_json()
 889
 890     def load_bot_config_json(self):
 891         """Load bot configuration."""
 892         with open(self.bot_config_json, 'r') as f:
 893             self.bot_config = json.load(f)
 894
 895     def part_build_old(self, action, delay):
 896         """Return whether the last build for a given action was at least a
 897         given number of seconds ago, or does not have a time recorded."""
 898         old_time_str = self.build_state[action]['build-time']
 899         if not old_time_str:
 900             return True
 901         old_time = datetime.datetime.strptime(old_time_str,
 902                                               '%Y-%m-%d %H:%M:%S')
 903         new_time = datetime.datetime.utcnow()
 904         delta = new_time - old_time
 905         return delta.total_seconds() >= delay
 906
 907     def bot_cycle(self):
 908         """Run a single round of checkout and builds."""
 909         print('Bot cycle starting %s.' % str(datetime.datetime.utcnow()))
 910         self.load_bot_config_json()
 911         actions = ('host-libraries', 'compilers', 'glibcs')
 912         self.bot_run_self(['--replace-sources'], 'checkout')
 913         self.load_versions_json()
 914         if self.get_script_text() != self.script_text:
 915             print('Script changed, re-execing.')
 916             # On script change, all parts of the build should be rerun.
 917             for a in actions:
 918                 self.clear_last_build_state(a)
 919             self.exec_self()
 920         check_components = {'host-libraries': ('gmp', 'mpfr', 'mpc'),
 921                             'compilers': ('binutils', 'gcc', 'glibc', 'linux'),
 922                             'glibcs': ('glibc',)}
 923         must_build = {}
 924         for a in actions:
 925             build_vers = self.build_state[a]['build-versions']
 926             must_build[a] = False
 927             if not self.build_state[a]['build-time']:
 928                 must_build[a] = True
 929             old_vers = {}
 930             new_vers = {}
 931             for c in check_components[a]:
 932                 if c in build_vers:
 933                     old_vers[c] = build_vers[c]
 934                 new_vers[c] = {'version': self.versions[c]['version'],
 935                                'revision': self.versions[c]['revision']}
 936             if new_vers == old_vers:
 937                 print('Versions for %s unchanged.' % a)
 938             else:
 939                 print('Versions changed or rebuild forced for %s.' % a)
 940                 if a == 'compilers' and not self.part_build_old(
 941                         a, self.bot_config['compilers-rebuild-delay']):
 942                     print('Not requiring rebuild of compilers this soon.')
 943                 else:
 944                     must_build[a] = True
 945         if must_build['host-libraries']:
 946             must_build['compilers'] = True
 947         if must_build['compilers']:
 948             must_build['glibcs'] = True
 949         for a in actions:
 950             if must_build[a]:
 951                 print('Must rebuild %s.' % a)
 952                 self.clear_last_build_state(a)
 953             else:
 954                 print('No need to rebuild %s.' % a)
 955         if os.access(self.logsdir, os.F_OK):
 956             shutil.rmtree(self.logsdir_old, ignore_errors=True)
 957             shutil.copytree(self.logsdir, self.logsdir_old)
 958         for a in actions:
 959             if must_build[a]:
 960                 build_time = datetime.datetime.utcnow()
 961                 print('Rebuilding %s at %s.' % (a, str(build_time)))
 962                 self.bot_run_self([], a)
 963                 self.load_build_state_json()
 964                 self.bot_build_mail(a, build_time)
 965         print('Bot cycle done at %s.' % str(datetime.datetime.utcnow()))
 966
 967     def bot_build_mail(self, action, build_time):
 968         """Send email with the results of a build."""
 969         build_time = build_time.replace(microsecond=0)
 970         subject = (self.bot_config['email-subject'] %
 971                    {'action': action,
 972                     'build-time': str(build_time)})
 973         results = self.build_state[action]['build-results']
 974         changes = self.build_state[action]['result-changes']
 975         ever_passed = set(self.build_state[action]['ever-passed'])
 976         versions = self.build_state[action]['build-versions']
 977         new_regressions = {k for k in changes if changes[k] == 'PASS -> FAIL'}
 978         all_regressions = {k for k in ever_passed if results[k] == 'FAIL'}
 979         all_fails = {k for k in results if results[k] == 'FAIL'}
 980         if new_regressions:
 981             new_reg_list = sorted(['FAIL: %s' % k for k in new_regressions])
 982             new_reg_text = ('New regressions:\n\n%s\n\n' %
 983                             '\n'.join(new_reg_list))
 984         else:
 985             new_reg_text = ''
 986         if all_regressions:
 987             all_reg_list = sorted(['FAIL: %s' % k for k in all_regressions])
 988             all_reg_text = ('All regressions:\n\n%s\n\n' %
 989                             '\n'.join(all_reg_list))
 990         else:
 991             all_reg_text = ''
 992         if all_fails:
 993             all_fail_list = sorted(['FAIL: %s' % k for k in all_fails])
 994             all_fail_text = ('All failures:\n\n%s\n\n' %
 995                              '\n'.join(all_fail_list))
 996         else:
 997             all_fail_text = ''
 998         if changes:
 999             changes_list = sorted(changes.keys())
1000             changes_list = ['%s: %s' % (changes[k], k) for k in changes_list]
1001             changes_text = ('All changed results:\n\n%s\n\n' %
1002                             '\n'.join(changes_list))
1003         else:
1004             changes_text = ''
1005         results_text = (new_reg_text + all_reg_text + all_fail_text +
1006                         changes_text)
1007         if not results_text:
1008             results_text = 'Clean build with unchanged results.\n\n'
1009         versions_list = sorted(versions.keys())
1010         versions_list = ['%s: %s (%s)' % (k, versions[k]['version'],
1011                                           versions[k]['revision'])
1012                          for k in versions_list]
1013         versions_text = ('Component versions for this build:\n\n%s\n' %
1014                          '\n'.join(versions_list))
1015         body_text = results_text + versions_text
1016         msg = email.mime.text.MIMEText(body_text)
1017         msg['Subject'] = subject
1018         msg['From'] = self.bot_config['email-from']
1019         msg['To'] = self.bot_config['email-to']
1020         msg['Message-ID'] = email.utils.make_msgid()
1021         msg['Date'] = email.utils.format_datetime(datetime.datetime.utcnow())
1022         with smtplib.SMTP(self.bot_config['email-server']) as s:
1023             s.send_message(msg)
1024
1025     def bot_run_self(self, opts, action, check=True):
1026         """Run a copy of this script with given options."""
1027         cmd = [sys.executable, sys.argv[0], '--keep=none',
1028                '-j%d' % self.parallelism]
1029         cmd.extend(opts)
1030         cmd.extend([self.topdir, action])
1031         sys.stdout.flush()
1032         subprocess.run(cmd, check=check)
1033
1034     def bot(self):
1035         """Run repeated rounds of checkout and builds."""
1036         while True:
1037             self.load_bot_config_json()
1038             if not self.bot_config['run']:
1039                 print('Bot exiting by request.')
1040                 exit(0)
1041             self.bot_run_self([], 'bot-cycle', check=False)
1042             self.load_bot_config_json()
1043             if not self.bot_config['run']:
1044                 print('Bot exiting by request.')
1045                 exit(0)
1046             time.sleep(self.bot_config['delay'])
1047             if self.get_script_text() != self.script_text:
1048                 print('Script changed, bot re-execing.')
1049                 self.exec_self()
1050
1051
1052 class Config(object):
1053     """A configuration for building a compiler and associated libraries."""
1054
1055     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
1056                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
1057         """Initialize a Config object."""
1058         self.ctx = ctx
1059         self.arch = arch
1060         self.os = os_name
1061         self.variant = variant
1062         if variant is None:
1063             self.name = '%s-%s' % (arch, os_name)
1064         else:
1065             self.name = '%s-%s-%s' % (arch, os_name, variant)
1066         self.triplet = '%s-glibc-%s' % (arch, os_name)
1067         if gcc_cfg is None:
1068             self.gcc_cfg = []
1069         else:
1070             self.gcc_cfg = gcc_cfg
1071         if first_gcc_cfg is None:
1072             self.first_gcc_cfg = []
1073         else:
1074             self.first_gcc_cfg = first_gcc_cfg
1075         if glibcs is None:
1076             glibcs = [{'variant': variant}]
1077         if extra_glibcs is None:
1078             extra_glibcs = []
1079         glibcs = [Glibc(self, **g) for g in glibcs]
1080         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
1081         self.all_glibcs = glibcs + extra_glibcs
1082         self.compiler_glibcs = glibcs
1083         self.installdir = ctx.compiler_installdir(self.name)
1084         self.bindir = ctx.compiler_bindir(self.name)
1085         self.sysroot = ctx.compiler_sysroot(self.name)
1086         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
1087         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
1088
1089     def component_builddir(self, component):
1090         """Return the directory to use for a (non-glibc) build."""
1091         return self.ctx.component_builddir('compilers', self.name, component)
1092
1093     def build(self):
1094         """Generate commands to build this compiler."""
1095         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
1096                                       self.logsdir)
1097         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
1098         cmdlist.add_command('check-host-libraries',
1099                             ['test', '-f',
1100                              os.path.join(self.ctx.host_libraries_installdir,
1101                                           'ok')])
1102         cmdlist.use_path(self.bindir)
1103         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
1104                               ['--disable-gdb',
1105                                '--disable-libdecnumber',
1106                                '--disable-readline',
1107                                '--disable-sim'])
1108         if self.os.startswith('linux'):
1109             self.install_linux_headers(cmdlist)
1110         self.build_gcc(cmdlist, True)
1111         for g in self.compiler_glibcs:
1112             cmdlist.push_subdesc('glibc')
1113             cmdlist.push_subdesc(g.name)
1114             g.build_glibc(cmdlist, True)
1115             cmdlist.pop_subdesc()
1116             cmdlist.pop_subdesc()
1117         self.build_gcc(cmdlist, False)
1118         cmdlist.add_command('done', ['touch',
1119                                      os.path.join(self.installdir, 'ok')])
1120         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
1121                                       self.logsdir)
1122
1123     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
1124         """Build one cross tool."""
1125         srcdir = self.ctx.component_srcdir(tool_src)
1126         builddir = self.component_builddir(tool_build)
1127         cmdlist.push_subdesc(tool_build)
1128         cmdlist.create_use_dir(builddir)
1129         cfg_cmd = [os.path.join(srcdir, 'configure'),
1130                    '--prefix=%s' % self.installdir,
1131                    '--build=%s' % self.ctx.build_triplet,
1132                    '--host=%s' % self.ctx.build_triplet,
1133                    '--target=%s' % self.triplet,
1134                    '--with-sysroot=%s' % self.sysroot]
1135         if extra_opts:
1136             cfg_cmd.extend(extra_opts)
1137         cmdlist.add_command('configure', cfg_cmd)
1138         cmdlist.add_command('build', ['make'])
1139         cmdlist.add_command('install', ['make', 'install'])
1140         cmdlist.cleanup_dir()
1141         cmdlist.pop_subdesc()
1142
1143     def install_linux_headers(self, cmdlist):
1144         """Install Linux kernel headers."""
1145         arch_map = {'aarch64': 'arm64',
1146                     'alpha': 'alpha',
1147                     'arm': 'arm',
1148                     'hppa': 'parisc',
1149                     'i486': 'x86',
1150                     'i586': 'x86',
1151                     'i686': 'x86',
1152                     'i786': 'x86',
1153                     'ia64': 'ia64',
1154                     'm68k': 'm68k',
1155                     'microblaze': 'microblaze',
1156                     'mips': 'mips',
1157                     'nios2': 'nios2',
1158                     'powerpc': 'powerpc',
1159                     's390': 's390',
1160                     'sh': 'sh',
1161                     'sparc': 'sparc',
1162                     'tile': 'tile',
1163                     'x86_64': 'x86'}
1164         linux_arch = None
1165         for k in arch_map:
1166             if self.arch.startswith(k):
1167                 linux_arch = arch_map[k]
1168                 break
1169         assert linux_arch is not None
1170         srcdir = self.ctx.component_srcdir('linux')
1171         builddir = self.component_builddir('linux')
1172         headers_dir = os.path.join(self.sysroot, 'usr')
1173         cmdlist.push_subdesc('linux')
1174         cmdlist.create_use_dir(builddir)
1175         cmdlist.add_command('install-headers',
1176                             ['make', '-C', srcdir, 'O=%s' % builddir,
1177                              'ARCH=%s' % linux_arch,
1178                              'INSTALL_HDR_PATH=%s' % headers_dir,
1179                              'headers_install'])
1180         cmdlist.cleanup_dir()
1181         cmdlist.pop_subdesc()
1182
1183     def build_gcc(self, cmdlist, bootstrap):
1184         """Build GCC."""
1185         # libsanitizer commonly breaks because of glibc header
1186         # changes, or on unusual targets.  libssp is of little
1187         # relevance with glibc's own stack checking support.
1188         cfg_opts = list(self.gcc_cfg)
1189         cfg_opts += ['--disable-libsanitizer', '--disable-libssp']
1190         host_libs = self.ctx.host_libraries_installdir
1191         cfg_opts += ['--with-gmp=%s' % host_libs,
1192                      '--with-mpfr=%s' % host_libs,
1193                      '--with-mpc=%s' % host_libs]
1194         if bootstrap:
1195             tool_build = 'gcc-first'
1196             # Building a static-only, C-only compiler that is
1197             # sufficient to build glibc.  Various libraries and
1198             # features that may require libc headers must be disabled.
1199             # When configuring with a sysroot, --with-newlib is
1200             # required to define inhibit_libc (to stop some parts of
1201             # libgcc including libc headers); --without-headers is not
1202             # sufficient.
1203             cfg_opts += ['--enable-languages=c', '--disable-shared',
1204                          '--disable-threads',
1205                          '--disable-libatomic',
1206                          '--disable-decimal-float',
1207                          '--disable-libffi',
1208                          '--disable-libgomp',
1209                          '--disable-libitm',
1210                          '--disable-libmpx',
1211                          '--disable-libquadmath',
1212                          '--without-headers', '--with-newlib',
1213                          '--with-glibc-version=%s' % self.ctx.glibc_version
1214                          ]
1215             cfg_opts += self.first_gcc_cfg
1216         else:
1217             tool_build = 'gcc'
1218             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
1219                          '--enable-threads']
1220         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
1221
1222
1223 class Glibc(object):
1224     """A configuration for building glibc."""
1225
1226     def __init__(self, compiler, arch=None, os_name=None, variant=None,
1227                  cfg=None, ccopts=None):
1228         """Initialize a Glibc object."""
1229         self.ctx = compiler.ctx
1230         self.compiler = compiler
1231         if arch is None:
1232             self.arch = compiler.arch
1233         else:
1234             self.arch = arch
1235         if os_name is None:
1236             self.os = compiler.os
1237         else:
1238             self.os = os_name
1239         self.variant = variant
1240         if variant is None:
1241             self.name = '%s-%s' % (self.arch, self.os)
1242         else:
1243             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
1244         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
1245         if cfg is None:
1246             self.cfg = []
1247         else:
1248             self.cfg = cfg
1249         self.ccopts = ccopts
1250
1251     def tool_name(self, tool):
1252         """Return the name of a cross-compilation tool."""
1253         ctool = '%s-%s' % (self.compiler.triplet, tool)
1254         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
1255             ctool = '%s %s' % (ctool, self.ccopts)
1256         return ctool
1257
1258     def build(self):
1259         """Generate commands to build this glibc."""
1260         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
1261         installdir = self.ctx.glibc_installdir(self.name)
1262         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
1263         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
1264         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
1265         cmdlist.add_command('check-compilers',
1266                             ['test', '-f',
1267                              os.path.join(self.compiler.installdir, 'ok')])
1268         cmdlist.use_path(self.compiler.bindir)
1269         self.build_glibc(cmdlist, False)
1270         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
1271                                       logsdir)
1272
1273     def build_glibc(self, cmdlist, for_compiler):
1274         """Generate commands to build this glibc, either as part of a compiler
1275         build or with the bootstrapped compiler (and in the latter case, run
1276         tests as well)."""
1277         srcdir = self.ctx.component_srcdir('glibc')
1278         if for_compiler:
1279             builddir = self.ctx.component_builddir('compilers',
1280                                                    self.compiler.name, 'glibc',
1281                                                    self.name)
1282             installdir = self.compiler.sysroot
1283             srcdir_copy = self.ctx.component_builddir('compilers',
1284                                                       self.compiler.name,
1285                                                       'glibc-src',
1286                                                       self.name)
1287         else:
1288             builddir = self.ctx.component_builddir('glibcs', self.name,
1289                                                    'glibc')
1290             installdir = self.ctx.glibc_installdir(self.name)
1291             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1292                                                       'glibc-src')
1293         cmdlist.create_use_dir(builddir)
1294         # glibc builds write into the source directory, and even if
1295         # not intentionally there is a risk of bugs that involve
1296         # writing into the working directory.  To avoid possible
1297         # concurrency issues, copy the source directory.
1298         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1299         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1300                    '--prefix=/usr',
1301                    '--enable-add-ons',
1302                    '--build=%s' % self.ctx.build_triplet,
1303                    '--host=%s' % self.triplet,
1304                    'CC=%s' % self.tool_name('gcc'),
1305                    'CXX=%s' % self.tool_name('g++'),
1306                    'AR=%s' % self.tool_name('ar'),
1307                    'AS=%s' % self.tool_name('as'),
1308                    'LD=%s' % self.tool_name('ld'),
1309                    'NM=%s' % self.tool_name('nm'),
1310                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1311                    'OBJDUMP=%s' % self.tool_name('objdump'),
1312                    'RANLIB=%s' % self.tool_name('ranlib'),
1313                    'READELF=%s' % self.tool_name('readelf'),
1314                    'STRIP=%s' % self.tool_name('strip')]
1315         cfg_cmd += self.cfg
1316         cmdlist.add_command('configure', cfg_cmd)
1317         cmdlist.add_command('build', ['make'])
1318         cmdlist.add_command('install', ['make', 'install',
1319                                         'install_root=%s' % installdir])
1320         # GCC uses paths such as lib/../lib64, so make sure lib
1321         # directories always exist.
1322         cmdlist.add_command('mkdir-lib', ['mkdir', '-p',
1323                                           os.path.join(installdir, 'lib'),
1324                                           os.path.join(installdir,
1325                                                        'usr', 'lib')])
1326         if not for_compiler:
1327             if self.ctx.strip:
1328                 cmdlist.add_command('strip',
1329                                     ['sh', '-c',
1330                                      ('%s %s/lib*/*.so' %
1331                                       (self.tool_name('strip'), installdir))])
1332             cmdlist.add_command('check', ['make', 'check'])
1333             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1334                                 always_run=True)
1335         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1336         cmdlist.cleanup_dir()
1337
1338
1339 class Command(object):
1340     """A command run in the build process."""
1341
1342     def __init__(self, desc, num, dir, path, command, always_run=False):
1343         """Initialize a Command object."""
1344         self.dir = dir
1345         self.path = path
1346         self.desc = desc
1347         trans = str.maketrans({' ': '-'})
1348         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1349         self.command = command
1350         self.always_run = always_run
1351
1352     @staticmethod
1353     def shell_make_quote_string(s):
1354         """Given a string not containing a newline, quote it for use by the
1355         shell and make."""
1356         assert '\n' not in s
1357         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1358             return s
1359         strans = str.maketrans({"'": "'\\''"})
1360         s = "'%s'" % s.translate(strans)
1361         mtrans = str.maketrans({'$': '$$'})
1362         return s.translate(mtrans)
1363
1364     @staticmethod
1365     def shell_make_quote_list(l, translate_make):
1366         """Given a list of strings not containing newlines, quote them for use
1367         by the shell and make, returning a single string.  If translate_make
1368         is true and the first string is 'make', change it to $(MAKE)."""
1369         l = [Command.shell_make_quote_string(s) for s in l]
1370         if translate_make and l[0] == 'make':
1371             l[0] = '$(MAKE)'
1372         return ' '.join(l)
1373
1374     def shell_make_quote(self):
1375         """Return this command quoted for the shell and make."""
1376         return self.shell_make_quote_list(self.command, True)
1377
1378
1379 class CommandList(object):
1380     """A list of commands run in the build process."""
1381
1382     def __init__(self, desc, keep):
1383         """Initialize a CommandList object."""
1384         self.cmdlist = []
1385         self.dir = None
1386         self.path = None
1387         self.desc = [desc]
1388         self.keep = keep
1389
1390     def desc_txt(self, desc):
1391         """Return the description to use for a command."""
1392         return '%s %s' % (' '.join(self.desc), desc)
1393
1394     def use_dir(self, dir):
1395         """Set the default directory for subsequent commands."""
1396         self.dir = dir
1397
1398     def use_path(self, path):
1399         """Set a directory to be prepended to the PATH for subsequent
1400         commands."""
1401         self.path = path
1402
1403     def push_subdesc(self, subdesc):
1404         """Set the default subdescription for subsequent commands (e.g., the
1405         name of a component being built, within the series of commands
1406         building it)."""
1407         self.desc.append(subdesc)
1408
1409     def pop_subdesc(self):
1410         """Pop a subdescription from the list of descriptions."""
1411         self.desc.pop()
1412
1413     def create_use_dir(self, dir):
1414         """Remove and recreate a directory and use it for subsequent
1415         commands."""
1416         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1417         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1418         self.use_dir(dir)
1419
1420     def create_copy_dir(self, src, dest):
1421         """Remove a directory and recreate it as a copy from the given
1422         source."""
1423         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1424         parent = os.path.dirname(dest)
1425         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1426         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1427
1428     def add_command_dir(self, desc, dir, command, always_run=False):
1429         """Add a command to run in a given directory."""
1430         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1431                       command, always_run)
1432         self.cmdlist.append(cmd)
1433
1434     def add_command(self, desc, command, always_run=False):
1435         """Add a command to run in the default directory."""
1436         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1437                       self.path, command, always_run)
1438         self.cmdlist.append(cmd)
1439
1440     def cleanup_dir(self, desc='cleanup', dir=None):
1441         """Clean up a build directory.  If no directory is specified, the
1442         default directory is cleaned up and ceases to be the default
1443         directory."""
1444         if dir is None:
1445             dir = self.dir
1446             self.use_dir(None)
1447         if self.keep != 'all':
1448             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1449                                  always_run=(self.keep == 'none'))
1450
1451     def makefile_commands(self, wrapper, logsdir):
1452         """Return the sequence of commands in the form of text for a Makefile.
1453         The given wrapper script takes arguments: base of logs for
1454         previous command, or empty; base of logs for this command;
1455         description; directory; PATH addition; the command itself."""
1456         # prev_base is the base of the name for logs of the previous
1457         # command that is not always-run (that is, a build command,
1458         # whose failure should stop subsequent build commands from
1459         # being run, as opposed to a cleanup command, which is run
1460         # even if previous commands failed).
1461         prev_base = ''
1462         cmds = []
1463         for c in self.cmdlist:
1464             ctxt = c.shell_make_quote()
1465             if prev_base and not c.always_run:
1466                 prev_log = os.path.join(logsdir, prev_base)
1467             else:
1468                 prev_log = ''
1469             this_log = os.path.join(logsdir, c.logbase)
1470             if not c.always_run:
1471                 prev_base = c.logbase
1472             if c.dir is None:
1473                 dir = ''
1474             else:
1475                 dir = c.dir
1476             if c.path is None:
1477                 path = ''
1478             else:
1479                 path = c.path
1480             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1481             prelim_txt = Command.shell_make_quote_list(prelims, False)
1482             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1483         return '\n'.join(cmds)
1484
1485     def status_logs(self, logsdir):
1486         """Return the list of log files with command status."""
1487         return [os.path.join(logsdir, '%s-status.txt' % c.logbase)
1488                 for c in self.cmdlist]
1489
1490
1491 def get_parser():
1492     """Return an argument parser for this module."""
1493     parser = argparse.ArgumentParser(description=__doc__)
1494     parser.add_argument('-j', dest='parallelism',
1495                         help='Run this number of jobs in parallel',
1496                         type=int, default=os.cpu_count())
1497     parser.add_argument('--keep', dest='keep',
1498                         help='Whether to keep all build directories, '
1499                         'none or only those from failed builds',
1500                         default='none', choices=('none', 'all', 'failed'))
1501     parser.add_argument('--replace-sources', action='store_true',
1502                         help='Remove and replace source directories '
1503                         'with the wrong version of a component')
1504     parser.add_argument('--strip', action='store_true',
1505                         help='Strip installed glibc libraries')
1506     parser.add_argument('topdir',
1507                         help='Toplevel working directory')
1508     parser.add_argument('action',
1509                         help='What to do',
1510                         choices=('checkout', 'bot-cycle', 'bot',
1511                                  'host-libraries', 'compilers', 'glibcs'))
1512     parser.add_argument('configs',
1513                         help='Versions to check out or configurations to build',
1514                         nargs='*')
1515     return parser
1516
1517
1518 def main(argv):
1519     """The main entry point."""
1520     parser = get_parser()
1521     opts = parser.parse_args(argv)
1522     topdir = os.path.abspath(opts.topdir)
1523     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1524                   opts.strip, opts.action)
1525     ctx.run_builds(opts.action, opts.configs)
1526
1527
1528 if __name__ == '__main__':
1529     main(sys.argv[1:])