scripts/build-many-glibcs.py

   1 #!/usr/bin/python3
   2 # Build many configurations of glibc.
   3 # Copyright (C) 2016 Free Software Foundation, Inc.
   4 # This file is part of the GNU C Library.
   5 #
   6 # The GNU C Library is free software; you can redistribute it and/or
   7 # modify it under the terms of the GNU Lesser General Public
   8 # License as published by the Free Software Foundation; either
   9 # version 2.1 of the License, or (at your option) any later version.
  10 #
  11 # The GNU C Library is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14 # Lesser General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU Lesser General Public
  17 # License along with the GNU C Library; if not, see
  18 # <http://www.gnu.org/licenses/>.
  19
  20 """Build many configurations of glibc.
  21
  22 This script takes as arguments a directory name (containing a src
  23 subdirectory with sources of the relevant toolchain components) and a
  24 description of what to do: 'checkout', to check out sources into that
  25 directory, 'host-libraries', to build libraries required by the
  26 toolchain, 'compilers', to build cross-compilers for various
  27 configurations, or 'glibcs', to build glibc for various configurations
  28 and run the compilation parts of the testsuite.  Subsequent arguments
  29 name the versions of components to check out (<component>-<version),
  30 for 'checkout', or, for actions other than 'checkout', name
  31 configurations for which compilers or glibc are to be built.
  32 """
  33
  34 import argparse
  35 import json
  36 import os
  37 import re
  38 import shutil
  39 import stat
  40 import subprocess
  41 import sys
  42 import urllib.request
  43
  44
  45 class Context(object):
  46     """The global state associated with builds in a given directory."""
  47
  48     def __init__(self, topdir, parallelism, keep, replace_sources, action):
  49         """Initialize the context."""
  50         self.topdir = topdir
  51         self.parallelism = parallelism
  52         self.keep = keep
  53         self.replace_sources = replace_sources
  54         self.srcdir = os.path.join(topdir, 'src')
  55         self.versions_json = os.path.join(self.srcdir, 'versions.json')
  56         self.installdir = os.path.join(topdir, 'install')
  57         self.host_libraries_installdir = os.path.join(self.installdir,
  58                                                       'host-libraries')
  59         self.builddir = os.path.join(topdir, 'build')
  60         self.logsdir = os.path.join(topdir, 'logs')
  61         self.makefile = os.path.join(self.builddir, 'Makefile')
  62         self.wrapper = os.path.join(self.builddir, 'wrapper')
  63         self.save_logs = os.path.join(self.builddir, 'save-logs')
  64         self.script_text = self.get_script_text()
  65         if action != 'checkout':
  66             self.build_triplet = self.get_build_triplet()
  67             self.glibc_version = self.get_glibc_version()
  68         self.configs = {}
  69         self.glibc_configs = {}
  70         self.makefile_pieces = ['.PHONY: all\n']
  71         self.add_all_configs()
  72         self.load_versions_json()
  73
  74     def get_script_text(self):
  75         """Return the text of this script."""
  76         with open(sys.argv[0], 'r') as f:
  77             return f.read()
  78
  79     def exec_self(self):
  80         """Re-execute this script with the same arguments."""
  81         os.execv(sys.executable, [sys.executable] + sys.argv)
  82
  83     def get_build_triplet(self):
  84         """Determine the build triplet with config.guess."""
  85         config_guess = os.path.join(self.component_srcdir('gcc'),
  86                                     'config.guess')
  87         cg_out = subprocess.run([config_guess], stdout=subprocess.PIPE,
  88                                 check=True, universal_newlines=True).stdout
  89         return cg_out.rstrip()
  90
  91     def get_glibc_version(self):
  92         """Determine the glibc version number (major.minor)."""
  93         version_h = os.path.join(self.component_srcdir('glibc'), 'version.h')
  94         with open(version_h, 'r') as f:
  95             lines = f.readlines()
  96         starttext = '#define VERSION "'
  97         for l in lines:
  98             if l.startswith(starttext):
  99                 l = l[len(starttext):]
 100                 l = l.rstrip('"\n')
 101                 m = re.fullmatch('([0-9]+)\.([0-9]+)[.0-9]*', l)
 102                 return '%s.%s' % m.group(1, 2)
 103         print('error: could not determine glibc version')
 104         exit(1)
 105
 106     def add_all_configs(self):
 107         """Add all known glibc build configurations."""
 108         # On architectures missing __builtin_trap support, these
 109         # options may be needed as a workaround; see
 110         # <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70216> for SH.
 111         no_isolate = ('-fno-isolate-erroneous-paths-dereference'
 112                       ' -fno-isolate-erroneous-paths-attribute')
 113         self.add_config(arch='aarch64',
 114                         os_name='linux-gnu')
 115         self.add_config(arch='aarch64_be',
 116                         os_name='linux-gnu')
 117         self.add_config(arch='alpha',
 118                         os_name='linux-gnu')
 119         self.add_config(arch='arm',
 120                         os_name='linux-gnueabi')
 121         self.add_config(arch='armeb',
 122                         os_name='linux-gnueabi')
 123         self.add_config(arch='armeb',
 124                         os_name='linux-gnueabi',
 125                         variant='be8',
 126                         gcc_cfg=['--with-arch=armv7-a'])
 127         self.add_config(arch='arm',
 128                         os_name='linux-gnueabihf')
 129         self.add_config(arch='armeb',
 130                         os_name='linux-gnueabihf')
 131         self.add_config(arch='armeb',
 132                         os_name='linux-gnueabihf',
 133                         variant='be8',
 134                         gcc_cfg=['--with-arch=armv7-a'])
 135         self.add_config(arch='hppa',
 136                         os_name='linux-gnu')
 137         self.add_config(arch='ia64',
 138                         os_name='linux-gnu',
 139                         first_gcc_cfg=['--with-system-libunwind'])
 140         self.add_config(arch='m68k',
 141                         os_name='linux-gnu',
 142                         gcc_cfg=['--disable-multilib'])
 143         self.add_config(arch='m68k',
 144                         os_name='linux-gnu',
 145                         variant='coldfire',
 146                         gcc_cfg=['--with-arch=cf', '--disable-multilib'])
 147         self.add_config(arch='microblaze',
 148                         os_name='linux-gnu',
 149                         gcc_cfg=['--disable-multilib'])
 150         self.add_config(arch='microblazeel',
 151                         os_name='linux-gnu',
 152                         gcc_cfg=['--disable-multilib'])
 153         self.add_config(arch='mips64',
 154                         os_name='linux-gnu',
 155                         gcc_cfg=['--with-mips-plt'],
 156                         glibcs=[{'variant': 'n32'},
 157                                 {'arch': 'mips',
 158                                  'ccopts': '-mabi=32'},
 159                                 {'variant': 'n64',
 160                                  'ccopts': '-mabi=64'}])
 161         self.add_config(arch='mips64',
 162                         os_name='linux-gnu',
 163                         variant='soft',
 164                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 165                         glibcs=[{'variant': 'n32-soft',
 166                                  'cfg': ['--without-fp']},
 167                                 {'variant': 'soft',
 168                                  'arch': 'mips',
 169                                  'ccopts': '-mabi=32',
 170                                  'cfg': ['--without-fp']},
 171                                 {'variant': 'n64-soft',
 172                                  'ccopts': '-mabi=64',
 173                                  'cfg': ['--without-fp']}])
 174         self.add_config(arch='mips64',
 175                         os_name='linux-gnu',
 176                         variant='nan2008',
 177                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 178                                  '--with-arch-64=mips64r2',
 179                                  '--with-arch-32=mips32r2'],
 180                         glibcs=[{'variant': 'n32-nan2008'},
 181                                 {'variant': 'nan2008',
 182                                  'arch': 'mips',
 183                                  'ccopts': '-mabi=32'},
 184                                 {'variant': 'n64-nan2008',
 185                                  'ccopts': '-mabi=64'}])
 186         self.add_config(arch='mips64',
 187                         os_name='linux-gnu',
 188                         variant='nan2008-soft',
 189                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 190                                  '--with-arch-64=mips64r2',
 191                                  '--with-arch-32=mips32r2',
 192                                  '--with-float=soft'],
 193                         glibcs=[{'variant': 'n32-nan2008-soft',
 194                                  'cfg': ['--without-fp']},
 195                                 {'variant': 'nan2008-soft',
 196                                  'arch': 'mips',
 197                                  'ccopts': '-mabi=32',
 198                                  'cfg': ['--without-fp']},
 199                                 {'variant': 'n64-nan2008-soft',
 200                                  'ccopts': '-mabi=64',
 201                                  'cfg': ['--without-fp']}])
 202         self.add_config(arch='mips64el',
 203                         os_name='linux-gnu',
 204                         gcc_cfg=['--with-mips-plt'],
 205                         glibcs=[{'variant': 'n32'},
 206                                 {'arch': 'mipsel',
 207                                  'ccopts': '-mabi=32'},
 208                                 {'variant': 'n64',
 209                                  'ccopts': '-mabi=64'}])
 210         self.add_config(arch='mips64el',
 211                         os_name='linux-gnu',
 212                         variant='soft',
 213                         gcc_cfg=['--with-mips-plt', '--with-float=soft'],
 214                         glibcs=[{'variant': 'n32-soft',
 215                                  'cfg': ['--without-fp']},
 216                                 {'variant': 'soft',
 217                                  'arch': 'mipsel',
 218                                  'ccopts': '-mabi=32',
 219                                  'cfg': ['--without-fp']},
 220                                 {'variant': 'n64-soft',
 221                                  'ccopts': '-mabi=64',
 222                                  'cfg': ['--without-fp']}])
 223         self.add_config(arch='mips64el',
 224                         os_name='linux-gnu',
 225                         variant='nan2008',
 226                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 227                                  '--with-arch-64=mips64r2',
 228                                  '--with-arch-32=mips32r2'],
 229                         glibcs=[{'variant': 'n32-nan2008'},
 230                                 {'variant': 'nan2008',
 231                                  'arch': 'mipsel',
 232                                  'ccopts': '-mabi=32'},
 233                                 {'variant': 'n64-nan2008',
 234                                  'ccopts': '-mabi=64'}])
 235         self.add_config(arch='mips64el',
 236                         os_name='linux-gnu',
 237                         variant='nan2008-soft',
 238                         gcc_cfg=['--with-mips-plt', '--with-nan=2008',
 239                                  '--with-arch-64=mips64r2',
 240                                  '--with-arch-32=mips32r2',
 241                                  '--with-float=soft'],
 242                         glibcs=[{'variant': 'n32-nan2008-soft',
 243                                  'cfg': ['--without-fp']},
 244                                 {'variant': 'nan2008-soft',
 245                                  'arch': 'mipsel',
 246                                  'ccopts': '-mabi=32',
 247                                  'cfg': ['--without-fp']},
 248                                 {'variant': 'n64-nan2008-soft',
 249                                  'ccopts': '-mabi=64',
 250                                  'cfg': ['--without-fp']}])
 251         self.add_config(arch='nios2',
 252                         os_name='linux-gnu')
 253         self.add_config(arch='powerpc',
 254                         os_name='linux-gnu',
 255                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 256         self.add_config(arch='powerpc',
 257                         os_name='linux-gnu',
 258                         variant='soft',
 259                         gcc_cfg=['--disable-multilib', '--with-float=soft',
 260                                  '--enable-secureplt'],
 261                         glibcs=[{'variant': 'soft', 'cfg': ['--without-fp']}])
 262         self.add_config(arch='powerpc64',
 263                         os_name='linux-gnu',
 264                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 265         self.add_config(arch='powerpc64le',
 266                         os_name='linux-gnu',
 267                         gcc_cfg=['--disable-multilib', '--enable-secureplt'])
 268         self.add_config(arch='powerpc',
 269                         os_name='linux-gnuspe',
 270                         gcc_cfg=['--disable-multilib', '--enable-secureplt',
 271                                  '--enable-e500-double'],
 272                         glibcs=[{'cfg': ['--without-fp']}])
 273         self.add_config(arch='powerpc',
 274                         os_name='linux-gnuspe',
 275                         variant='e500v1',
 276                         gcc_cfg=['--disable-multilib', '--enable-secureplt'],
 277                         glibcs=[{'variant': 'e500v1', 'cfg': ['--without-fp']}])
 278         self.add_config(arch='s390x',
 279                         os_name='linux-gnu',
 280                         glibcs=[{},
 281                                 {'arch': 's390', 'ccopts': '-m31'}])
 282         self.add_config(arch='sh3',
 283                         os_name='linux-gnu',
 284                         glibcs=[{'ccopts': no_isolate}])
 285         self.add_config(arch='sh3eb',
 286                         os_name='linux-gnu',
 287                         glibcs=[{'ccopts': no_isolate}])
 288         self.add_config(arch='sh4',
 289                         os_name='linux-gnu',
 290                         glibcs=[{'ccopts': no_isolate}])
 291         self.add_config(arch='sh4eb',
 292                         os_name='linux-gnu',
 293                         glibcs=[{'ccopts': no_isolate}])
 294         self.add_config(arch='sh4',
 295                         os_name='linux-gnu',
 296                         variant='soft',
 297                         gcc_cfg=['--without-fp'],
 298                         glibcs=[{'variant': 'soft',
 299                                  'cfg': ['--without-fp'],
 300                                  'ccopts': no_isolate}])
 301         self.add_config(arch='sh4eb',
 302                         os_name='linux-gnu',
 303                         variant='soft',
 304                         gcc_cfg=['--without-fp'],
 305                         glibcs=[{'variant': 'soft',
 306                                  'cfg': ['--without-fp'],
 307                                  'ccopts': no_isolate}])
 308         self.add_config(arch='sparc64',
 309                         os_name='linux-gnu',
 310                         glibcs=[{},
 311                                 {'arch': 'sparcv9',
 312                                  'ccopts': '-m32 -mlong-double-128'}])
 313         self.add_config(arch='tilegx',
 314                         os_name='linux-gnu',
 315                         glibcs=[{},
 316                                 {'variant': '32', 'ccopts': '-m32'}])
 317         self.add_config(arch='tilegxbe',
 318                         os_name='linux-gnu',
 319                         glibcs=[{},
 320                                 {'variant': '32', 'ccopts': '-m32'}])
 321         self.add_config(arch='tilepro',
 322                         os_name='linux-gnu')
 323         self.add_config(arch='x86_64',
 324                         os_name='linux-gnu',
 325                         gcc_cfg=['--with-multilib-list=m64,m32,mx32'],
 326                         glibcs=[{},
 327                                 {'variant': 'x32', 'ccopts': '-mx32'},
 328                                 {'arch': 'i686', 'ccopts': '-m32 -march=i686'}],
 329                         extra_glibcs=[{'variant': 'disable-multi-arch',
 330                                        'cfg': ['--disable-multi-arch']},
 331                                       {'variant': 'disable-multi-arch',
 332                                        'arch': 'i686',
 333                                        'ccopts': '-m32 -march=i686',
 334                                        'cfg': ['--disable-multi-arch']},
 335                                       {'arch': 'i486',
 336                                        'ccopts': '-m32 -march=i486'},
 337                                       {'arch': 'i586',
 338                                        'ccopts': '-m32 -march=i586'}])
 339
 340     def add_config(self, **args):
 341         """Add an individual build configuration."""
 342         cfg = Config(self, **args)
 343         if cfg.name in self.configs:
 344             print('error: duplicate config %s' % cfg.name)
 345             exit(1)
 346         self.configs[cfg.name] = cfg
 347         for c in cfg.all_glibcs:
 348             if c.name in self.glibc_configs:
 349                 print('error: duplicate glibc config %s' % c.name)
 350                 exit(1)
 351             self.glibc_configs[c.name] = c
 352
 353     def component_srcdir(self, component):
 354         """Return the source directory for a given component, e.g. gcc."""
 355         return os.path.join(self.srcdir, component)
 356
 357     def component_builddir(self, action, config, component, subconfig=None):
 358         """Return the directory to use for a build."""
 359         if config is None:
 360             # Host libraries.
 361             assert subconfig is None
 362             return os.path.join(self.builddir, action, component)
 363         if subconfig is None:
 364             return os.path.join(self.builddir, action, config, component)
 365         else:
 366             # glibc build as part of compiler build.
 367             return os.path.join(self.builddir, action, config, component,
 368                                 subconfig)
 369
 370     def compiler_installdir(self, config):
 371         """Return the directory in which to install a compiler."""
 372         return os.path.join(self.installdir, 'compilers', config)
 373
 374     def compiler_bindir(self, config):
 375         """Return the directory in which to find compiler binaries."""
 376         return os.path.join(self.compiler_installdir(config), 'bin')
 377
 378     def compiler_sysroot(self, config):
 379         """Return the sysroot directory for a compiler."""
 380         return os.path.join(self.compiler_installdir(config), 'sysroot')
 381
 382     def glibc_installdir(self, config):
 383         """Return the directory in which to install glibc."""
 384         return os.path.join(self.installdir, 'glibcs', config)
 385
 386     def run_builds(self, action, configs):
 387         """Run the requested builds."""
 388         if action == 'checkout':
 389             self.checkout(configs)
 390             return
 391         elif action == 'host-libraries':
 392             if configs:
 393                 print('error: configurations specified for host-libraries')
 394                 exit(1)
 395             self.build_host_libraries()
 396         elif action == 'compilers':
 397             self.build_compilers(configs)
 398         else:
 399             self.build_glibcs(configs)
 400         self.write_files()
 401         self.do_build()
 402
 403     @staticmethod
 404     def remove_dirs(*args):
 405         """Remove directories and their contents if they exist."""
 406         for dir in args:
 407             shutil.rmtree(dir, ignore_errors=True)
 408
 409     @staticmethod
 410     def remove_recreate_dirs(*args):
 411         """Remove directories if they exist, and create them as empty."""
 412         Context.remove_dirs(*args)
 413         for dir in args:
 414             os.makedirs(dir, exist_ok=True)
 415
 416     def add_makefile_cmdlist(self, target, cmdlist, logsdir):
 417         """Add makefile text for a list of commands."""
 418         commands = cmdlist.makefile_commands(self.wrapper, logsdir)
 419         self.makefile_pieces.append('all: %s\n.PHONY: %s\n%s:\n%s\n' %
 420                                     (target, target, target, commands))
 421
 422     def write_files(self):
 423         """Write out the Makefile and wrapper script."""
 424         mftext = ''.join(self.makefile_pieces)
 425         with open(self.makefile, 'w') as f:
 426             f.write(mftext)
 427         wrapper_text = (
 428             '#!/bin/sh\n'
 429             'prev_base=$1\n'
 430             'this_base=$2\n'
 431             'desc=$3\n'
 432             'dir=$4\n'
 433             'path=$5\n'
 434             'shift 5\n'
 435             'prev_status=$prev_base-status.txt\n'
 436             'this_status=$this_base-status.txt\n'
 437             'this_log=$this_base-log.txt\n'
 438             'date > "$this_log"\n'
 439             'echo >> "$this_log"\n'
 440             'echo "Description: $desc" >> "$this_log"\n'
 441             'printf "%s" "Command:" >> "$this_log"\n'
 442             'for word in "$@"; do\n'
 443             '  if expr "$word" : "[]+,./0-9@A-Z_a-z-]\\\\{1,\\\\}\\$" > /dev/null; then\n'
 444             '    printf " %s" "$word"\n'
 445             '  else\n'
 446             '    printf " \'"\n'
 447             '    printf "%s" "$word" | sed -e "s/\'/\'\\\\\\\\\'\'/"\n'
 448             '    printf "\'"\n'
 449             '  fi\n'
 450             'done >> "$this_log"\n'
 451             'echo >> "$this_log"\n'
 452             'echo "Directory: $dir" >> "$this_log"\n'
 453             'echo "Path addition: $path" >> "$this_log"\n'
 454             'echo >> "$this_log"\n'
 455             'record_status ()\n'
 456             '{\n'
 457             '  echo >> "$this_log"\n'
 458             '  echo "$1: $desc" > "$this_status"\n'
 459             '  echo "$1: $desc" >> "$this_log"\n'
 460             '  echo >> "$this_log"\n'
 461             '  date >> "$this_log"\n'
 462             '  echo "$1: $desc"\n'
 463             '  exit 0\n'
 464             '}\n'
 465             'check_error ()\n'
 466             '{\n'
 467             '  if [ "$1" != "0" ]; then\n'
 468             '    record_status FAIL\n'
 469             '  fi\n'
 470             '}\n'
 471             'if [ "$prev_base" ] && ! grep -q "^PASS" "$prev_status"; then\n'
 472             '    record_status UNRESOLVED\n'
 473             'fi\n'
 474             'if [ "$dir" ]; then\n'
 475             '  cd "$dir"\n'
 476             '  check_error "$?"\n'
 477             'fi\n'
 478             'if [ "$path" ]; then\n'
 479             '  PATH=$path:$PATH\n'
 480             'fi\n'
 481             '"$@" < /dev/null >> "$this_log" 2>&1\n'
 482             'check_error "$?"\n'
 483             'record_status PASS\n')
 484         with open(self.wrapper, 'w') as f:
 485             f.write(wrapper_text)
 486         # Mode 0o755.
 487         mode_exec = (stat.S_IRWXU|stat.S_IRGRP|stat.S_IXGRP|
 488                      stat.S_IROTH|stat.S_IXOTH)
 489         os.chmod(self.wrapper, mode_exec)
 490         save_logs_text = (
 491             '#!/bin/sh\n'
 492             'if ! [ -f tests.sum ]; then\n'
 493             '  echo "No test summary available."\n'
 494             '  exit 0\n'
 495             'fi\n'
 496             'save_file ()\n'
 497             '{\n'
 498             '  echo "Contents of $1:"\n'
 499             '  echo\n'
 500             '  cat "$1"\n'
 501             '  echo\n'
 502             '  echo "End of contents of $1."\n'
 503             '  echo\n'
 504             '}\n'
 505             'save_file tests.sum\n'
 506             'non_pass_tests=$(grep -v "^PASS: " tests.sum | sed -e "s/^PASS: //")\n'
 507             'for t in $non_pass_tests; do\n'
 508             '  if [ -f "$t.out" ]; then\n'
 509             '    save_file "$t.out"\n'
 510             '  fi\n'
 511             'done\n')
 512         with open(self.save_logs, 'w') as f:
 513             f.write(save_logs_text)
 514         os.chmod(self.save_logs, mode_exec)
 515
 516     def do_build(self):
 517         """Do the actual build."""
 518         cmd = ['make', '-j%d' % self.parallelism]
 519         subprocess.run(cmd, cwd=self.builddir, check=True)
 520
 521     def build_host_libraries(self):
 522         """Build the host libraries."""
 523         installdir = self.host_libraries_installdir
 524         builddir = os.path.join(self.builddir, 'host-libraries')
 525         logsdir = os.path.join(self.logsdir, 'host-libraries')
 526         self.remove_recreate_dirs(installdir, builddir, logsdir)
 527         cmdlist = CommandList('host-libraries', self.keep)
 528         self.build_host_library(cmdlist, 'gmp')
 529         self.build_host_library(cmdlist, 'mpfr',
 530                                 ['--with-gmp=%s' % installdir])
 531         self.build_host_library(cmdlist, 'mpc',
 532                                 ['--with-gmp=%s' % installdir,
 533                                 '--with-mpfr=%s' % installdir])
 534         cmdlist.add_command('done', ['touch', os.path.join(installdir, 'ok')])
 535         self.add_makefile_cmdlist('host-libraries', cmdlist, logsdir)
 536
 537     def build_host_library(self, cmdlist, lib, extra_opts=None):
 538         """Build one host library."""
 539         srcdir = self.component_srcdir(lib)
 540         builddir = self.component_builddir('host-libraries', None, lib)
 541         installdir = self.host_libraries_installdir
 542         cmdlist.push_subdesc(lib)
 543         cmdlist.create_use_dir(builddir)
 544         cfg_cmd = [os.path.join(srcdir, 'configure'),
 545                    '--prefix=%s' % installdir,
 546                    '--disable-shared']
 547         if extra_opts:
 548             cfg_cmd.extend (extra_opts)
 549         cmdlist.add_command('configure', cfg_cmd)
 550         cmdlist.add_command('build', ['make'])
 551         cmdlist.add_command('check', ['make', 'check'])
 552         cmdlist.add_command('install', ['make', 'install'])
 553         cmdlist.cleanup_dir()
 554         cmdlist.pop_subdesc()
 555
 556     def build_compilers(self, configs):
 557         """Build the compilers."""
 558         if not configs:
 559             self.remove_dirs(os.path.join(self.builddir, 'compilers'))
 560             self.remove_dirs(os.path.join(self.installdir, 'compilers'))
 561             self.remove_dirs(os.path.join(self.logsdir, 'compilers'))
 562             configs = sorted(self.configs.keys())
 563         for c in configs:
 564             self.configs[c].build()
 565
 566     def build_glibcs(self, configs):
 567         """Build the glibcs."""
 568         if not configs:
 569             self.remove_dirs(os.path.join(self.builddir, 'glibcs'))
 570             self.remove_dirs(os.path.join(self.installdir, 'glibcs'))
 571             self.remove_dirs(os.path.join(self.logsdir, 'glibcs'))
 572             configs = sorted(self.glibc_configs.keys())
 573         for c in configs:
 574             self.glibc_configs[c].build()
 575
 576     def load_versions_json(self):
 577         """Load information about source directory versions."""
 578         if not os.access(self.versions_json, os.F_OK):
 579             self.versions = {}
 580             return
 581         with open(self.versions_json, 'r') as f:
 582             self.versions = json.load(f)
 583
 584     def store_json(self, data, filename):
 585         """Store information in a JSON file."""
 586         filename_tmp = filename + '.tmp'
 587         with open(filename_tmp, 'w') as f:
 588             json.dump(data, f, indent=2, sort_keys=True)
 589         os.rename(filename_tmp, filename)
 590
 591     def store_versions_json(self):
 592         """Store information about source directory versions."""
 593         self.store_json(self.versions, self.versions_json)
 594
 595     def set_component_version(self, component, version, explicit, revision):
 596         """Set the version information for a component."""
 597         self.versions[component] = {'version': version,
 598                                     'explicit': explicit,
 599                                     'revision': revision}
 600         self.store_versions_json()
 601
 602     def checkout(self, versions):
 603         """Check out the desired component versions."""
 604         default_versions = {'binutils': 'vcs-2.27',
 605                             'gcc': 'vcs-6',
 606                             'glibc': 'vcs-mainline',
 607                             'gmp': '6.1.1',
 608                             'linux': '4.8.6',
 609                             'mpc': '1.0.3',
 610                             'mpfr': '3.1.5'}
 611         use_versions = {}
 612         explicit_versions = {}
 613         for v in versions:
 614             found_v = False
 615             for k in default_versions.keys():
 616                 kx = k + '-'
 617                 if v.startswith(kx):
 618                     vx = v[len(kx):]
 619                     if k in use_versions:
 620                         print('error: multiple versions for %s' % k)
 621                         exit(1)
 622                     use_versions[k] = vx
 623                     explicit_versions[k] = True
 624                     found_v = True
 625                     break
 626             if not found_v:
 627                 print('error: unknown component in %s' % v)
 628                 exit(1)
 629         for k in default_versions.keys():
 630             if k not in use_versions:
 631                 if k in self.versions and self.versions[k]['explicit']:
 632                     use_versions[k] = self.versions[k]['version']
 633                     explicit_versions[k] = True
 634                 else:
 635                     use_versions[k] = default_versions[k]
 636                     explicit_versions[k] = False
 637         os.makedirs(self.srcdir, exist_ok=True)
 638         for k in sorted(default_versions.keys()):
 639             update = os.access(self.component_srcdir(k), os.F_OK)
 640             v = use_versions[k]
 641             if (update and
 642                 k in self.versions and
 643                 v != self.versions[k]['version']):
 644                 if not self.replace_sources:
 645                     print('error: version of %s has changed from %s to %s, '
 646                           'use --replace-sources to check out again' %
 647                           (k, self.versions[k]['version'], v))
 648                     exit(1)
 649                 shutil.rmtree(self.component_srcdir(k))
 650                 update = False
 651             if v.startswith('vcs-'):
 652                 revision = self.checkout_vcs(k, v[4:], update)
 653             else:
 654                 self.checkout_tar(k, v, update)
 655                 revision = v
 656             self.set_component_version(k, v, explicit_versions[k], revision)
 657         if self.get_script_text() != self.script_text:
 658             # Rerun the checkout process in case the updated script
 659             # uses different default versions or new components.
 660             self.exec_self()
 661
 662     def checkout_vcs(self, component, version, update):
 663         """Check out the given version of the given component from version
 664         control.  Return a revision identifier."""
 665         if component == 'binutils':
 666             git_url = 'git://sourceware.org/git/binutils-gdb.git'
 667             if version == 'mainline':
 668                 git_branch = 'master'
 669             else:
 670                 trans = str.maketrans({'.': '_'})
 671                 git_branch = 'binutils-%s-branch' % version.translate(trans)
 672             return self.git_checkout(component, git_url, git_branch, update)
 673         elif component == 'gcc':
 674             if version == 'mainline':
 675                 branch = 'trunk'
 676             else:
 677                 trans = str.maketrans({'.': '_'})
 678                 branch = 'branches/gcc-%s-branch' % version.translate(trans)
 679             svn_url = 'svn://gcc.gnu.org/svn/gcc/%s' % branch
 680             return self.gcc_checkout(svn_url, update)
 681         elif component == 'glibc':
 682             git_url = 'git://sourceware.org/git/glibc.git'
 683             if version == 'mainline':
 684                 git_branch = 'master'
 685             else:
 686                 git_branch = 'release/%s/master' % version
 687             r = self.git_checkout(component, git_url, git_branch, update)
 688             self.fix_glibc_timestamps()
 689             return r
 690         else:
 691             print('error: component %s coming from VCS' % component)
 692             exit(1)
 693
 694     def git_checkout(self, component, git_url, git_branch, update):
 695         """Check out a component from git.  Return a commit identifier."""
 696         if update:
 697             subprocess.run(['git', 'remote', 'prune', 'origin'],
 698                            cwd=self.component_srcdir(component), check=True)
 699             subprocess.run(['git', 'pull', '-q'],
 700                            cwd=self.component_srcdir(component), check=True)
 701         else:
 702             subprocess.run(['git', 'clone', '-q', '-b', git_branch, git_url,
 703                             self.component_srcdir(component)], check=True)
 704         r = subprocess.run(['git', 'rev-parse', 'HEAD'],
 705                            cwd=self.component_srcdir(component),
 706                            stdout=subprocess.PIPE,
 707                            check=True, universal_newlines=True).stdout
 708         return r.rstrip()
 709
 710     def fix_glibc_timestamps(self):
 711         """Fix timestamps in a glibc checkout."""
 712         # Ensure that builds do not try to regenerate generated files
 713         # in the source tree.
 714         srcdir = self.component_srcdir('glibc')
 715         for dirpath, dirnames, filenames in os.walk(srcdir):
 716             for f in filenames:
 717                 if (f == 'configure' or
 718                     f == 'preconfigure' or
 719                     f.endswith('-kw.h')):
 720                     to_touch = os.path.join(dirpath, f)
 721                     subprocess.run(['touch', to_touch], check=True)
 722
 723     def gcc_checkout(self, svn_url, update):
 724         """Check out GCC from SVN.  Return the revision number."""
 725         if not update:
 726             subprocess.run(['svn', 'co', '-q', svn_url,
 727                             self.component_srcdir('gcc')], check=True)
 728         subprocess.run(['contrib/gcc_update', '--silent'],
 729                        cwd=self.component_srcdir('gcc'), check=True)
 730         r = subprocess.run(['svnversion', self.component_srcdir('gcc')],
 731                            stdout=subprocess.PIPE,
 732                            check=True, universal_newlines=True).stdout
 733         return r.rstrip()
 734
 735     def checkout_tar(self, component, version, update):
 736         """Check out the given version of the given component from a
 737         tarball."""
 738         if update:
 739             return
 740         url_map = {'binutils': 'https://ftp.gnu.org/gnu/binutils/binutils-%(version)s.tar.bz2',
 741                    'gcc': 'https://ftp.gnu.org/gnu/gcc/gcc-%(version)s/gcc-%(version)s.tar.bz2',
 742                    'gmp': 'https://ftp.gnu.org/gnu/gmp/gmp-%(version)s.tar.xz',
 743                    'linux': 'https://www.kernel.org/pub/linux/kernel/v4.x/linux-%(version)s.tar.xz',
 744                    'mpc': 'https://ftp.gnu.org/gnu/mpc/mpc-%(version)s.tar.gz',
 745                    'mpfr': 'https://ftp.gnu.org/gnu/mpfr/mpfr-%(version)s.tar.xz'}
 746         if component not in url_map:
 747             print('error: component %s coming from tarball' % component)
 748             exit(1)
 749         url = url_map[component] % {'version': version}
 750         filename = os.path.join(self.srcdir, url.split('/')[-1])
 751         response = urllib.request.urlopen(url)
 752         data = response.read()
 753         with open(filename, 'wb') as f:
 754             f.write(data)
 755         subprocess.run(['tar', '-C', self.srcdir, '-x', '-f', filename],
 756                        check=True)
 757         os.rename(os.path.join(self.srcdir, '%s-%s' % (component, version)),
 758                   self.component_srcdir(component))
 759         os.remove(filename)
 760
 761
 762 class Config(object):
 763     """A configuration for building a compiler and associated libraries."""
 764
 765     def __init__(self, ctx, arch, os_name, variant=None, gcc_cfg=None,
 766                  first_gcc_cfg=None, glibcs=None, extra_glibcs=None):
 767         """Initialize a Config object."""
 768         self.ctx = ctx
 769         self.arch = arch
 770         self.os = os_name
 771         self.variant = variant
 772         if variant is None:
 773             self.name = '%s-%s' % (arch, os_name)
 774         else:
 775             self.name = '%s-%s-%s' % (arch, os_name, variant)
 776         self.triplet = '%s-glibc-%s' % (arch, os_name)
 777         if gcc_cfg is None:
 778             self.gcc_cfg = []
 779         else:
 780             self.gcc_cfg = gcc_cfg
 781         if first_gcc_cfg is None:
 782             self.first_gcc_cfg = []
 783         else:
 784             self.first_gcc_cfg = first_gcc_cfg
 785         if glibcs is None:
 786             glibcs = [{'variant': variant}]
 787         if extra_glibcs is None:
 788             extra_glibcs = []
 789         glibcs = [Glibc(self, **g) for g in glibcs]
 790         extra_glibcs = [Glibc(self, **g) for g in extra_glibcs]
 791         self.all_glibcs = glibcs + extra_glibcs
 792         self.compiler_glibcs = glibcs
 793         self.installdir = ctx.compiler_installdir(self.name)
 794         self.bindir = ctx.compiler_bindir(self.name)
 795         self.sysroot = ctx.compiler_sysroot(self.name)
 796         self.builddir = os.path.join(ctx.builddir, 'compilers', self.name)
 797         self.logsdir = os.path.join(ctx.logsdir, 'compilers', self.name)
 798
 799     def component_builddir(self, component):
 800         """Return the directory to use for a (non-glibc) build."""
 801         return self.ctx.component_builddir('compilers', self.name, component)
 802
 803     def build(self):
 804         """Generate commands to build this compiler."""
 805         self.ctx.remove_recreate_dirs(self.installdir, self.builddir,
 806                                       self.logsdir)
 807         cmdlist = CommandList('compilers-%s' % self.name, self.ctx.keep)
 808         cmdlist.add_command('check-host-libraries',
 809                             ['test', '-f',
 810                              os.path.join(self.ctx.host_libraries_installdir,
 811                                           'ok')])
 812         cmdlist.use_path(self.bindir)
 813         self.build_cross_tool(cmdlist, 'binutils', 'binutils',
 814                               ['--disable-gdb',
 815                                '--disable-libdecnumber',
 816                                '--disable-readline',
 817                                '--disable-sim'])
 818         if self.os.startswith('linux'):
 819             self.install_linux_headers(cmdlist)
 820         self.build_gcc(cmdlist, True)
 821         for g in self.compiler_glibcs:
 822             cmdlist.push_subdesc('glibc')
 823             cmdlist.push_subdesc(g.name)
 824             g.build_glibc(cmdlist, True)
 825             cmdlist.pop_subdesc()
 826             cmdlist.pop_subdesc()
 827         self.build_gcc(cmdlist, False)
 828         cmdlist.add_command('done', ['touch',
 829                                      os.path.join(self.installdir, 'ok')])
 830         self.ctx.add_makefile_cmdlist('compilers-%s' % self.name, cmdlist,
 831                                       self.logsdir)
 832
 833     def build_cross_tool(self, cmdlist, tool_src, tool_build, extra_opts=None):
 834         """Build one cross tool."""
 835         srcdir = self.ctx.component_srcdir(tool_src)
 836         builddir = self.component_builddir(tool_build)
 837         cmdlist.push_subdesc(tool_build)
 838         cmdlist.create_use_dir(builddir)
 839         cfg_cmd = [os.path.join(srcdir, 'configure'),
 840                    '--prefix=%s' % self.installdir,
 841                    '--build=%s' % self.ctx.build_triplet,
 842                    '--host=%s' % self.ctx.build_triplet,
 843                    '--target=%s' % self.triplet,
 844                    '--with-sysroot=%s' % self.sysroot]
 845         if extra_opts:
 846             cfg_cmd.extend(extra_opts)
 847         cmdlist.add_command('configure', cfg_cmd)
 848         cmdlist.add_command('build', ['make'])
 849         cmdlist.add_command('install', ['make', 'install'])
 850         cmdlist.cleanup_dir()
 851         cmdlist.pop_subdesc()
 852
 853     def install_linux_headers(self, cmdlist):
 854         """Install Linux kernel headers."""
 855         arch_map = {'aarch64': 'arm64',
 856                     'alpha': 'alpha',
 857                     'arm': 'arm',
 858                     'hppa': 'parisc',
 859                     'i486': 'x86',
 860                     'i586': 'x86',
 861                     'i686': 'x86',
 862                     'i786': 'x86',
 863                     'ia64': 'ia64',
 864                     'm68k': 'm68k',
 865                     'microblaze': 'microblaze',
 866                     'mips': 'mips',
 867                     'nios2': 'nios2',
 868                     'powerpc': 'powerpc',
 869                     's390': 's390',
 870                     'sh': 'sh',
 871                     'sparc': 'sparc',
 872                     'tile': 'tile',
 873                     'x86_64': 'x86'}
 874         linux_arch = None
 875         for k in arch_map:
 876             if self.arch.startswith(k):
 877                 linux_arch = arch_map[k]
 878                 break
 879         assert linux_arch is not None
 880         srcdir = self.ctx.component_srcdir('linux')
 881         builddir = self.component_builddir('linux')
 882         headers_dir = os.path.join(self.sysroot, 'usr')
 883         cmdlist.push_subdesc('linux')
 884         cmdlist.create_use_dir(builddir)
 885         cmdlist.add_command('install-headers',
 886                             ['make', '-C', srcdir, 'O=%s' % builddir,
 887                              'ARCH=%s' % linux_arch,
 888                              'INSTALL_HDR_PATH=%s' % headers_dir,
 889                              'headers_install'])
 890         cmdlist.cleanup_dir()
 891         cmdlist.pop_subdesc()
 892
 893     def build_gcc(self, cmdlist, bootstrap):
 894         """Build GCC."""
 895         # libsanitizer commonly breaks because of glibc header
 896         # changes, or on unusual targets.  libssp is of little
 897         # relevance with glibc's own stack checking support.
 898         cfg_opts = list(self.gcc_cfg)
 899         cfg_opts += ['--disable-libsanitizer', '--disable-libssp']
 900         host_libs = self.ctx.host_libraries_installdir
 901         cfg_opts += ['--with-gmp=%s' % host_libs,
 902                      '--with-mpfr=%s' % host_libs,
 903                      '--with-mpc=%s' % host_libs]
 904         if bootstrap:
 905             tool_build = 'gcc-first'
 906             # Building a static-only, C-only compiler that is
 907             # sufficient to build glibc.  Various libraries and
 908             # features that may require libc headers must be disabled.
 909             # When configuring with a sysroot, --with-newlib is
 910             # required to define inhibit_libc (to stop some parts of
 911             # libgcc including libc headers); --without-headers is not
 912             # sufficient.
 913             cfg_opts += ['--enable-languages=c', '--disable-shared',
 914                          '--disable-threads',
 915                          '--disable-libatomic',
 916                          '--disable-decimal-float',
 917                          '--disable-libffi',
 918                          '--disable-libgomp',
 919                          '--disable-libitm',
 920                          '--disable-libmpx',
 921                          '--disable-libquadmath',
 922                          '--without-headers', '--with-newlib',
 923                          '--with-glibc-version=%s' % self.ctx.glibc_version
 924                          ]
 925             cfg_opts += self.first_gcc_cfg
 926         else:
 927             tool_build = 'gcc'
 928             cfg_opts += ['--enable-languages=c,c++', '--enable-shared',
 929                          '--enable-threads']
 930         self.build_cross_tool(cmdlist, 'gcc', tool_build, cfg_opts)
 931
 932
 933 class Glibc(object):
 934     """A configuration for building glibc."""
 935
 936     def __init__(self, compiler, arch=None, os_name=None, variant=None,
 937                  cfg=None, ccopts=None):
 938         """Initialize a Glibc object."""
 939         self.ctx = compiler.ctx
 940         self.compiler = compiler
 941         if arch is None:
 942             self.arch = compiler.arch
 943         else:
 944             self.arch = arch
 945         if os_name is None:
 946             self.os = compiler.os
 947         else:
 948             self.os = os_name
 949         self.variant = variant
 950         if variant is None:
 951             self.name = '%s-%s' % (self.arch, self.os)
 952         else:
 953             self.name = '%s-%s-%s' % (self.arch, self.os, variant)
 954         self.triplet = '%s-glibc-%s' % (self.arch, self.os)
 955         if cfg is None:
 956             self.cfg = []
 957         else:
 958             self.cfg = cfg
 959         self.ccopts = ccopts
 960
 961     def tool_name(self, tool):
 962         """Return the name of a cross-compilation tool."""
 963         ctool = '%s-%s' % (self.compiler.triplet, tool)
 964         if self.ccopts and (tool == 'gcc' or tool == 'g++'):
 965             ctool = '%s %s' % (ctool, self.ccopts)
 966         return ctool
 967
 968     def build(self):
 969         """Generate commands to build this glibc."""
 970         builddir = self.ctx.component_builddir('glibcs', self.name, 'glibc')
 971         installdir = self.ctx.glibc_installdir(self.name)
 972         logsdir = os.path.join(self.ctx.logsdir, 'glibcs', self.name)
 973         self.ctx.remove_recreate_dirs(installdir, builddir, logsdir)
 974         cmdlist = CommandList('glibcs-%s' % self.name, self.ctx.keep)
 975         cmdlist.add_command('check-compilers',
 976                             ['test', '-f',
 977                              os.path.join(self.compiler.installdir, 'ok')])
 978         cmdlist.use_path(self.compiler.bindir)
 979         self.build_glibc(cmdlist, False)
 980         self.ctx.add_makefile_cmdlist('glibcs-%s' % self.name, cmdlist,
 981                                       logsdir)
 982
 983     def build_glibc(self, cmdlist, for_compiler):
 984         """Generate commands to build this glibc, either as part of a compiler
 985         build or with the bootstrapped compiler (and in the latter case, run
 986         tests as well)."""
 987         srcdir = self.ctx.component_srcdir('glibc')
 988         if for_compiler:
 989             builddir = self.ctx.component_builddir('compilers',
 990                                                    self.compiler.name, 'glibc',
 991                                                    self.name)
 992             installdir = self.compiler.sysroot
 993             srcdir_copy = self.ctx.component_builddir('compilers',
 994                                                       self.compiler.name,
 995                                                       'glibc-src',
 996                                                       self.name)
 997         else:
 998             builddir = self.ctx.component_builddir('glibcs', self.name,
 999                                                    'glibc')
1000             installdir = self.ctx.glibc_installdir(self.name)
1001             srcdir_copy = self.ctx.component_builddir('glibcs', self.name,
1002                                                       'glibc-src')
1003         cmdlist.create_use_dir(builddir)
1004         # glibc builds write into the source directory, and even if
1005         # not intentionally there is a risk of bugs that involve
1006         # writing into the working directory.  To avoid possible
1007         # concurrency issues, copy the source directory.
1008         cmdlist.create_copy_dir(srcdir, srcdir_copy)
1009         cfg_cmd = [os.path.join(srcdir_copy, 'configure'),
1010                    '--prefix=/usr',
1011                    '--enable-add-ons',
1012                    '--build=%s' % self.ctx.build_triplet,
1013                    '--host=%s' % self.triplet,
1014                    'CC=%s' % self.tool_name('gcc'),
1015                    'CXX=%s' % self.tool_name('g++'),
1016                    'AR=%s' % self.tool_name('ar'),
1017                    'AS=%s' % self.tool_name('as'),
1018                    'LD=%s' % self.tool_name('ld'),
1019                    'NM=%s' % self.tool_name('nm'),
1020                    'OBJCOPY=%s' % self.tool_name('objcopy'),
1021                    'OBJDUMP=%s' % self.tool_name('objdump'),
1022                    'RANLIB=%s' % self.tool_name('ranlib'),
1023                    'READELF=%s' % self.tool_name('readelf'),
1024                    'STRIP=%s' % self.tool_name('strip')]
1025         cfg_cmd += self.cfg
1026         cmdlist.add_command('configure', cfg_cmd)
1027         cmdlist.add_command('build', ['make'])
1028         cmdlist.add_command('install', ['make', 'install',
1029                                         'install_root=%s' % installdir])
1030         # GCC uses paths such as lib/../lib64, so make sure lib
1031         # directories always exist.
1032         cmdlist.add_command('mkdir-lib', ['mkdir', '-p',
1033                                           os.path.join(installdir, 'lib'),
1034                                           os.path.join(installdir,
1035                                                        'usr', 'lib')])
1036         if not for_compiler:
1037             cmdlist.add_command('check', ['make', 'check'])
1038             cmdlist.add_command('save-logs', [self.ctx.save_logs],
1039                                 always_run=True)
1040         cmdlist.cleanup_dir('cleanup-src', srcdir_copy)
1041         cmdlist.cleanup_dir()
1042
1043
1044 class Command(object):
1045     """A command run in the build process."""
1046
1047     def __init__(self, desc, num, dir, path, command, always_run=False):
1048         """Initialize a Command object."""
1049         self.dir = dir
1050         self.path = path
1051         self.desc = desc
1052         trans = str.maketrans({' ': '-'})
1053         self.logbase = '%03d-%s' % (num, desc.translate(trans))
1054         self.command = command
1055         self.always_run = always_run
1056
1057     @staticmethod
1058     def shell_make_quote_string(s):
1059         """Given a string not containing a newline, quote it for use by the
1060         shell and make."""
1061         assert '\n' not in s
1062         if re.fullmatch('[]+,./0-9@A-Z_a-z-]+', s):
1063             return s
1064         strans = str.maketrans({"'": "'\\''"})
1065         s = "'%s'" % s.translate(strans)
1066         mtrans = str.maketrans({'$': '$$'})
1067         return s.translate(mtrans)
1068
1069     @staticmethod
1070     def shell_make_quote_list(l, translate_make):
1071         """Given a list of strings not containing newlines, quote them for use
1072         by the shell and make, returning a single string.  If translate_make
1073         is true and the first string is 'make', change it to $(MAKE)."""
1074         l = [Command.shell_make_quote_string(s) for s in l]
1075         if translate_make and l[0] == 'make':
1076             l[0] = '$(MAKE)'
1077         return ' '.join(l)
1078
1079     def shell_make_quote(self):
1080         """Return this command quoted for the shell and make."""
1081         return self.shell_make_quote_list(self.command, True)
1082
1083
1084 class CommandList(object):
1085     """A list of commands run in the build process."""
1086
1087     def __init__(self, desc, keep):
1088         """Initialize a CommandList object."""
1089         self.cmdlist = []
1090         self.dir = None
1091         self.path = None
1092         self.desc = [desc]
1093         self.keep = keep
1094
1095     def desc_txt(self, desc):
1096         """Return the description to use for a command."""
1097         return '%s %s' % (' '.join(self.desc), desc)
1098
1099     def use_dir(self, dir):
1100         """Set the default directory for subsequent commands."""
1101         self.dir = dir
1102
1103     def use_path(self, path):
1104         """Set a directory to be prepended to the PATH for subsequent
1105         commands."""
1106         self.path = path
1107
1108     def push_subdesc(self, subdesc):
1109         """Set the default subdescription for subsequent commands (e.g., the
1110         name of a component being built, within the series of commands
1111         building it)."""
1112         self.desc.append(subdesc)
1113
1114     def pop_subdesc(self):
1115         """Pop a subdescription from the list of descriptions."""
1116         self.desc.pop()
1117
1118     def create_use_dir(self, dir):
1119         """Remove and recreate a directory and use it for subsequent
1120         commands."""
1121         self.add_command_dir('rm', None, ['rm', '-rf', dir])
1122         self.add_command_dir('mkdir', None, ['mkdir', '-p', dir])
1123         self.use_dir(dir)
1124
1125     def create_copy_dir(self, src, dest):
1126         """Remove a directory and recreate it as a copy from the given
1127         source."""
1128         self.add_command_dir('copy-rm', None, ['rm', '-rf', dest])
1129         parent = os.path.dirname(dest)
1130         self.add_command_dir('copy-mkdir', None, ['mkdir', '-p', parent])
1131         self.add_command_dir('copy', None, ['cp', '-a', src, dest])
1132
1133     def add_command_dir(self, desc, dir, command, always_run=False):
1134         """Add a command to run in a given directory."""
1135         cmd = Command(self.desc_txt(desc), len(self.cmdlist), dir, self.path,
1136                       command, always_run)
1137         self.cmdlist.append(cmd)
1138
1139     def add_command(self, desc, command, always_run=False):
1140         """Add a command to run in the default directory."""
1141         cmd = Command(self.desc_txt(desc), len(self.cmdlist), self.dir,
1142                       self.path, command, always_run)
1143         self.cmdlist.append(cmd)
1144
1145     def cleanup_dir(self, desc='cleanup', dir=None):
1146         """Clean up a build directory.  If no directory is specified, the
1147         default directory is cleaned up and ceases to be the default
1148         directory."""
1149         if dir is None:
1150             dir = self.dir
1151             self.use_dir(None)
1152         if self.keep != 'all':
1153             self.add_command_dir(desc, None, ['rm', '-rf', dir],
1154                                  always_run=(self.keep == 'none'))
1155
1156     def makefile_commands(self, wrapper, logsdir):
1157         """Return the sequence of commands in the form of text for a Makefile.
1158         The given wrapper script takes arguments: base of logs for
1159         previous command, or empty; base of logs for this command;
1160         description; directory; PATH addition; the command itself."""
1161         # prev_base is the base of the name for logs of the previous
1162         # command that is not always-run (that is, a build command,
1163         # whose failure should stop subsequent build commands from
1164         # being run, as opposed to a cleanup command, which is run
1165         # even if previous commands failed).
1166         prev_base = ''
1167         cmds = []
1168         for c in self.cmdlist:
1169             ctxt = c.shell_make_quote()
1170             if prev_base and not c.always_run:
1171                 prev_log = os.path.join(logsdir, prev_base)
1172             else:
1173                 prev_log = ''
1174             this_log = os.path.join(logsdir, c.logbase)
1175             if not c.always_run:
1176                 prev_base = c.logbase
1177             if c.dir is None:
1178                 dir = ''
1179             else:
1180                 dir = c.dir
1181             if c.path is None:
1182                 path = ''
1183             else:
1184                 path = c.path
1185             prelims = [wrapper, prev_log, this_log, c.desc, dir, path]
1186             prelim_txt = Command.shell_make_quote_list(prelims, False)
1187             cmds.append('\t@%s %s' % (prelim_txt, ctxt))
1188         return '\n'.join(cmds)
1189
1190
1191 def get_parser():
1192     """Return an argument parser for this module."""
1193     parser = argparse.ArgumentParser(description=__doc__)
1194     parser.add_argument('-j', dest='parallelism',
1195                         help='Run this number of jobs in parallel',
1196                         type=int, default=os.cpu_count())
1197     parser.add_argument('--keep', dest='keep',
1198                         help='Whether to keep all build directories, '
1199                         'none or only those from failed builds',
1200                         default='none', choices=('none', 'all', 'failed'))
1201     parser.add_argument('--replace-sources', action='store_true',
1202                         help='Remove and replace source directories '
1203                         'with the wrong version of a component')
1204     parser.add_argument('topdir',
1205                         help='Toplevel working directory')
1206     parser.add_argument('action',
1207                         help='What to do',
1208                         choices=('checkout', 'host-libraries', 'compilers',
1209                                  'glibcs'))
1210     parser.add_argument('configs',
1211                         help='Versions to check out or configurations to build',
1212                         nargs='*')
1213     return parser
1214
1215
1216 def main(argv):
1217     """The main entry point."""
1218     parser = get_parser()
1219     opts = parser.parse_args(argv)
1220     topdir = os.path.abspath(opts.topdir)
1221     ctx = Context(topdir, opts.parallelism, opts.keep, opts.replace_sources,
1222                   opts.action)
1223     ctx.run_builds(opts.action, opts.configs)
1224
1225
1226 if __name__ == '__main__':
1227     main(sys.argv[1:])